diff --git a/.changeset/delegated-release-client.md b/.changeset/delegated-release-client.md index e94812a092..93788ea9c7 100644 --- a/.changeset/delegated-release-client.md +++ b/.changeset/delegated-release-client.md @@ -3,7 +3,7 @@ "@emdash-cms/plugin-cli": minor --- -Adds typed clients for the experimental delegated release service. `ReleaseServiceClient` submits, polls, and cancels GitHub OpenID Connect release intents, and manages publisher workload policies and retained delegation through a publisher session. `ReleaseServiceOperatorClient` exposes the Cloudflare Access status, pause, suspension, revocation, cancellation, and reconciliation operations. +Adds typed clients for the experimental delegated release service. `ReleaseServiceClient` submits, polls, and cancels GitHub OpenID Connect release intents, and manages publisher workload policies and retained delegation through a publisher session. `ReleaseServiceOperatorClient` exposes the Cloudflare Access status, sharded publisher and approver inventory, pause, suspension, revocation, cancellation, reconciliation, resumable encryption-key rotation, Workflow-backed encrypted R2 archive, and fail-safe publisher restore and abort operations. Both clients validate response envelopes and return stable `ReleaseServiceError` codes with retry metadata. Mutation helpers require idempotency keys, and workload polling requests a fresh token from the configured provider for each call. diff --git a/apps/release-service/src/api/errors.ts b/apps/release-service/src/api/errors.ts index 654211e985..0a851c783b 100644 --- a/apps/release-service/src/api/errors.ts +++ b/apps/release-service/src/api/errors.ts @@ -5,6 +5,7 @@ export type ApiErrorCode = | "APPROVAL_INVALID" | "APPROVER_SESSION_INVALID" | "APPROVER_SUSPENDED" + | "ARCHIVE_OPERATION_FAILED" | "AUTH_INVALID" | "CONFIGURATION_ERROR" | "CREDENTIAL_LIMIT_REACHED" @@ -12,6 +13,7 @@ export type ApiErrorCode = | "CREDENTIAL_REVOKED" | "CSRF_INVALID" | "DELEGATION_REQUIRED" + | "ENCRYPTION_OPERATION_FAILED" | "IDEMPOTENCY_KEY_INVALID" | "IDEMPOTENCY_CONFLICT" | "INTERNAL_ERROR" @@ -27,11 +29,13 @@ export type ApiErrorCode = | "PROFILE_CHANGED" | "PROFILE_FETCH_FAILED" | "RELEASE_EXISTS" + | "RESTORE_OPERATION_FAILED" | "SERVICE_PAUSED" | "SERVICE_UNAVAILABLE" | "VERSION_RESERVED" | "WORKFLOW_UNAVAILABLE" - | "WORKLOAD_NOT_ALLOWED"; + | "WORKLOAD_NOT_ALLOWED" + | "WORKLOAD_RATE_LIMITED"; export interface SerializedApiError { code: ApiErrorCode; diff --git a/apps/release-service/src/approver-do/approver-do.ts b/apps/release-service/src/approver-do/approver-do.ts index b1a3d46728..21dea1e74c 100644 --- a/apps/release-service/src/approver-do/approver-do.ts +++ b/apps/release-service/src/approver-do/approver-do.ts @@ -1,5 +1,9 @@ import { DurableObject } from "cloudflare:workers"; +import type { + EncryptionRecordPage, + EncryptionRecordReplacement, +} from "../operations/encryption-records.js"; import { initializeApproverSchema } from "./schema.js"; import { ApproverStore, @@ -253,6 +257,21 @@ export class ApproverDurableObject extends DurableObject { return this.#store.listAuditEvents(approverDid, afterSequence, limit); } + listEncryptionRecords( + approverDid: string, + afterCursor: string | null, + limit: number, + now = Date.now(), + ): EncryptionRecordPage { + this.#assertApproverDid(approverDid); + return this.#store.listEncryptionRecords(approverDid, afterCursor, limit, now); + } + + replaceEncryptionRecord(input: EncryptionRecordReplacement & { approverDid: string }): boolean { + this.#assertApproverDid(input.approverDid); + return this.#store.replaceEncryptionRecord(input); + } + async cleanupExpired(approverDid: string, now = Date.now(), limit = 100): Promise { this.#assertApproverDid(approverDid); const result = this.#store.cleanupExpired(now, limit); diff --git a/apps/release-service/src/approver-do/schema.ts b/apps/release-service/src/approver-do/schema.ts index 1d725539d9..642e107266 100644 --- a/apps/release-service/src/approver-do/schema.ts +++ b/apps/release-service/src/approver-do/schema.ts @@ -82,7 +82,7 @@ export function initializeApproverSchema(storage: DurableObjectStorage): void { CREATE TABLE IF NOT EXISTS audit_events ( sequence INTEGER PRIMARY KEY AUTOINCREMENT, event_type TEXT NOT NULL, - actor_realm TEXT NOT NULL CHECK (actor_realm IN ('approver', 'system')), + actor_realm TEXT NOT NULL CHECK (actor_realm IN ('access', 'approver', 'system')), actor_identity TEXT NOT NULL, subject TEXT NOT NULL, reason_code TEXT, diff --git a/apps/release-service/src/approver-do/store.ts b/apps/release-service/src/approver-do/store.ts index bbffd020fe..310db3ae22 100644 --- a/apps/release-service/src/approver-do/store.ts +++ b/apps/release-service/src/approver-do/store.ts @@ -1,5 +1,11 @@ import type { AuthenticatorTransport } from "@emdash-cms/auth"; +import type { + EncryptionRecordPage, + EncryptionRecordReplacement, +} from "../operations/encryption-records.js"; +import { MAX_ENCRYPTION_RECORD_PAGE } from "../operations/encryption-records.js"; + const DID_PATTERN = /^did:[a-z0-9]+:[A-Za-z0-9._:%-]+$/; const BASE64URL_PATTERN = /^[A-Za-z0-9_-]+$/; const HASH_PATTERN = /^[A-Za-z0-9_-]{43}$/; @@ -18,6 +24,8 @@ const MAX_SESSION_MS = 24 * 60 * 60_000; const MAX_CHALLENGE_MS = 5 * 60_000; const MAX_CHALLENGE_CLOCK_SKEW_MS = 5_000; const COMPLETED_IDENTITY_RETENTION_MS = 60 * 60_000; +const ACTOR_IDENTITY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,255}$/; +const ENCRYPTION_CURSOR_PATTERN = /^identity-transaction:[A-Za-z0-9_-]{43}$/; export type ApproverStoreErrorCode = | "APPROVER_DID_INVALID" @@ -210,7 +218,7 @@ export type FindDecisionResult = export interface ApproverAuditEvent { sequence: number; eventType: string; - actorRealm: "approver" | "system"; + actorRealm: "access" | "approver" | "system"; actorIdentity: string; subject: string; reasonCode: string | null; @@ -240,6 +248,13 @@ interface IdentityTransactionRow { completed_at: number | null; } +interface EncryptionRecordRow { + [key: string]: string | number | ArrayBuffer | null; + cursor: string; + envelope: string; + key_version: number; +} + interface ApproverSessionRow { [key: string]: string | number | ArrayBuffer | null; csrf_hash: string; @@ -296,7 +311,7 @@ interface AuditRow { [key: string]: string | number | ArrayBuffer | null; sequence: number; event_type: string; - actor_realm: "approver" | "system"; + actor_realm: "access" | "approver" | "system"; actor_identity: string; subject: string; reason_code: string | null; @@ -1068,6 +1083,96 @@ export class ApproverStore { .map(auditView); } + listEncryptionRecords( + approverDid: string, + afterCursor: string | null, + limit: number, + now = Date.now(), + ): EncryptionRecordPage { + this.#assertOwner(approverDid); + if ( + (afterCursor !== null && !ENCRYPTION_CURSOR_PATTERN.test(afterCursor)) || + !validInteger(limit) || + limit < 1 || + limit > MAX_ENCRYPTION_RECORD_PAGE || + !validInteger(now) || + now < 0 + ) { + throw new ApproverStoreError("APPROVER_INPUT_INVALID"); + } + const rows = this.storage.sql + .exec( + `SELECT 'identity-transaction:' || state_hash AS cursor, + encrypted_state AS envelope, encryption_key_version AS key_version + FROM identity_transactions + WHERE completed_at IS NULL AND expires_at > ? AND encrypted_state != '' + AND ('identity-transaction:' || state_hash) > ? + ORDER BY state_hash LIMIT ?`, + now, + afterCursor ?? "", + limit + 1, + ) + .toArray(); + const hasMore = rows.length > limit; + const visible = hasMore ? rows.slice(0, limit) : rows; + const items = visible.map((row) => ({ + cursor: row.cursor, + envelope: row.envelope, + keyVersion: row.key_version, + context: { + purpose: "oauth-approver-transaction" as const, + objectClass: "ApproverDurableObject", + table: "identity_transactions", + primaryKey: row.cursor.slice("identity-transaction:".length), + ownerDid: approverDid, + }, + })); + return { + items, + nextCursor: hasMore ? (items.at(-1)?.cursor ?? null) : null, + }; + } + + replaceEncryptionRecord(input: EncryptionRecordReplacement & { approverDid: string }): boolean { + this.#assertOwner(input.approverDid); + const now = input.now ?? Date.now(); + if ( + !ENCRYPTION_CURSOR_PATTERN.test(input.cursor) || + !validBoundedString(input.expectedEnvelope, MAX_CIPHERTEXT_CHARS) || + !validBoundedString(input.replacementEnvelope, MAX_CIPHERTEXT_CHARS) || + !validPositiveInteger(input.replacementKeyVersion) || + !ACTOR_IDENTITY_PATTERN.test(input.actorIdentity) || + !validInteger(now) || + now < 0 + ) { + throw new ApproverStoreError("APPROVER_INPUT_INVALID"); + } + return this.storage.transactionSync(() => { + const result = this.storage.sql.exec( + `UPDATE identity_transactions + SET encrypted_state = ?, encryption_key_version = ? + WHERE state_hash = ? AND encrypted_state = ? + AND completed_at IS NULL AND expires_at > ?`, + input.replacementEnvelope, + input.replacementKeyVersion, + input.cursor.slice("identity-transaction:".length), + input.expectedEnvelope, + now, + ); + if (result.rowsWritten !== 1) return false; + this.storage.sql.exec( + `INSERT INTO audit_events ( + event_type, actor_realm, actor_identity, subject, + reason_code, public_payload, created_at + ) VALUES ('encryption-rotated', 'access', ?, ?, NULL, '{}', ?)`, + input.actorIdentity, + input.cursor, + now, + ); + return true; + }); + } + cleanupExpired(now = Date.now(), limit = 100): CleanupResult { if (!validInteger(now) || !validInteger(limit) || limit < 1 || limit > 1000) { throw new ApproverStoreError("APPROVER_INPUT_INVALID"); diff --git a/apps/release-service/src/backup/routes.ts b/apps/release-service/src/backup/routes.ts new file mode 100644 index 0000000000..d1e38127f0 --- /dev/null +++ b/apps/release-service/src/backup/routes.ts @@ -0,0 +1,759 @@ +import { isDid } from "@atcute/lexicons/syntax"; +import { env } from "cloudflare:workers"; +import { base64url } from "jose"; + +import type { AccessActor } from "../access/auth.js"; +import { readJsonObject } from "../api/body.js"; +import { ApiError } from "../api/errors.js"; +import { apiFailure, apiSuccess } from "../api/response.js"; +import type { ServiceConfiguration } from "../config.js"; +import { SERVICE_CONTROL_OBJECT_NAME } from "../control-do/service-control-do.js"; +import type { EncryptionContext } from "../crypto/encryption.js"; +import { writeOperationsMetric } from "../observability/metrics.js"; + +const ARCHIVE_PATH_PATTERN = /^\/admin\/api\/publishers\/([^/]+)\/archive$/; +const RESTORE_PATH_PATTERN = /^\/admin\/api\/publishers\/([^/]+)\/restore$/; +const RESTORE_PREPARE_PATH_PATTERN = /^\/admin\/api\/publishers\/([^/]+)\/restore\/prepare$/; +const RESTORE_ABORT_PATH_PATTERN = /^\/admin\/api\/publishers\/([^/]+)\/restore\/abort$/; +const ARCHIVE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]{15,63}$/; +const IDEMPOTENCY_KEY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$/; +const CURSOR_PATTERN = + /^(?:workloads:[A-Za-z0-9_-]{0,64}|intents:[0-9A-HJKMNP-TV-Z]{0,26}|audit:[0-9]+)$/; +const WORKLOAD_PAGE_SIZE = 20; +const INTENT_PAGE_SIZE = 1; +const AUDIT_PAGE_SIZE = 100; +const MAX_ARCHIVE_PAGE = 999_999; +const MAX_ARCHIVE_OBJECT_BYTES = 1_500_000; +const SNAPSHOT_VERSION = 1; +const encoder = new TextEncoder(); +const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: false }); + +type SnapshotKind = "audit-events" | "intents" | "metadata" | "workload-policies"; + +interface SnapshotPage { + version: typeof SNAPSHOT_VERSION; + archiveId: string; + publisherDid: string; + page: number; + kind: SnapshotKind; + data: unknown; +} + +interface PageResult { + kind: SnapshotKind; + data: unknown; + nextCursor: string | null; + auditEvents?: readonly unknown[]; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function hasExactKeys(value: Record, expected: readonly string[]): boolean { + const keys = Object.keys(value); + return keys.length === expected.length && keys.every((key) => expected.includes(key)); +} + +function requireActor(actor: AccessActor | null): AccessActor { + if (!actor) throw new ApiError("ACCESS_AUTH_REQUIRED", 401, "Access authentication required"); + return actor; +} + +function requireIdempotencyKey(request: Request): void { + const value = request.headers.get("idempotency-key"); + if (!value || !IDEMPOTENCY_KEY_PATTERN.test(value)) { + throw new ApiError("IDEMPOTENCY_KEY_INVALID", 400, "Valid idempotency key required"); + } +} + +async function archiveInput( + request: Request, +): Promise<{ archiveId: string; cursor: string | null; page: number }> { + const body = await readJsonObject(request); + if ( + !hasExactKeys(body, ["archiveId", "cursor", "page"]) || + typeof body["archiveId"] !== "string" || + !ARCHIVE_ID_PATTERN.test(body["archiveId"]) || + (body["cursor"] !== null && + (typeof body["cursor"] !== "string" || !CURSOR_PATTERN.test(body["cursor"]))) || + !Number.isSafeInteger(body["page"]) || + Number(body["page"]) < 0 || + Number(body["page"]) > MAX_ARCHIVE_PAGE + ) { + throw new ApiError("INVALID_REQUEST", 400, "Invalid publisher archive request"); + } + return { archiveId: body["archiveId"], cursor: body["cursor"], page: Number(body["page"]) }; +} + +async function restoreInput(request: Request): Promise<{ archiveId: string; page: number }> { + const body = await readJsonObject(request); + if ( + !hasExactKeys(body, ["archiveId", "page"]) || + typeof body["archiveId"] !== "string" || + !ARCHIVE_ID_PATTERN.test(body["archiveId"]) || + !Number.isSafeInteger(body["page"]) || + Number(body["page"]) < 0 || + Number(body["page"]) > MAX_ARCHIVE_PAGE + ) { + throw new ApiError("INVALID_REQUEST", 400, "Invalid publisher restore request"); + } + return { archiveId: body["archiveId"], page: Number(body["page"]) }; +} + +async function hashOwner(publisherDid: string): Promise { + return base64url.encode( + new Uint8Array(await crypto.subtle.digest("SHA-256", encoder.encode(publisherDid))), + ); +} + +function snapshotContext( + publisherDid: string, + archiveId: string, + primaryKey: string, +): EncryptionContext { + return { + purpose: "publisher-snapshot", + objectClass: "PublisherDurableObject", + table: "operations_archive", + primaryKey: `${archiveId}:${primaryKey}`, + ownerDid: publisherDid, + }; +} + +async function writeEncryptedObject( + key: string, + plaintext: string, + context: EncryptionContext, + configuration: ServiceConfiguration, + metadata: Record, +): Promise { + const encrypted = await configuration.encryption.encrypt(encoder.encode(plaintext), context); + const created = await env.OPERATIONS_ARCHIVE.put(key, encrypted.envelope, { + onlyIf: { etagDoesNotMatch: "*" }, + httpMetadata: { contentType: "application/jose" }, + customMetadata: metadata, + }); + if (created) return false; + const existing = await env.OPERATIONS_ARCHIVE.get(key); + if (!existing) throw new ApiError("ARCHIVE_OPERATION_FAILED", 503, "Archive write failed"); + let existingPlaintext: string; + try { + existingPlaintext = decoder.decode( + await configuration.encryption.decrypt(await existing.text(), context), + ); + } catch { + throw new ApiError("ARCHIVE_OPERATION_FAILED", 409, "Archive page conflicts with prior write"); + } + if (existingPlaintext !== plaintext) { + throw new ApiError("ARCHIVE_OPERATION_FAILED", 409, "Archive page conflicts with prior write"); + } + return true; +} + +async function readEncryptedObject( + key: string, + context: EncryptionContext, + configuration: ServiceConfiguration, +): Promise { + const object = await env.OPERATIONS_ARCHIVE.get(key); + if (!object) throw new ApiError("NOT_FOUND", 404, "Publisher archive not found"); + if (object.size > MAX_ARCHIVE_OBJECT_BYTES) { + throw new ApiError("RESTORE_OPERATION_FAILED", 409, "Publisher archive is invalid"); + } + try { + return decoder.decode(await configuration.encryption.decrypt(await object.text(), context)); + } catch { + throw new ApiError("RESTORE_OPERATION_FAILED", 409, "Publisher archive is invalid"); + } +} + +async function writeAuditObject(ownerHash: string, events: readonly unknown[]): Promise { + if (events.length === 0) return; + const first = events[0]; + const last = events.at(-1); + if ( + first === null || + typeof first !== "object" || + last === null || + typeof last !== "object" || + !("sequence" in first) || + !("sequence" in last) || + !Number.isSafeInteger(first.sequence) || + !Number.isSafeInteger(last.sequence) + ) { + throw new ApiError("ARCHIVE_OPERATION_FAILED", 500, "Audit export failed"); + } + const publicEvents = events.map((event) => { + if (!isRecord(event) || typeof event["publicPayloadJson"] !== "string") { + throw new ApiError("ARCHIVE_OPERATION_FAILED", 500, "Audit export failed"); + } + let payload: unknown; + try { + payload = JSON.parse(event["publicPayloadJson"]); + } catch { + throw new ApiError("ARCHIVE_OPERATION_FAILED", 500, "Audit export failed"); + } + if (!isRecord(payload) || JSON.stringify(payload) !== event["publicPayloadJson"]) { + throw new ApiError("ARCHIVE_OPERATION_FAILED", 500, "Audit export failed"); + } + return payload; + }); + const firstSequence = String(first.sequence).padStart(20, "0"); + const lastSequence = String(last.sequence).padStart(20, "0"); + const content = JSON.stringify({ version: SNAPSHOT_VERSION, events: publicEvents }); + const contentDigest = base64url.encode( + new Uint8Array(await crypto.subtle.digest("SHA-256", encoder.encode(content))), + ); + const key = `audit/${ownerHash}/${firstSequence}-${lastSequence}-${contentDigest}.json`; + const created = await env.OPERATIONS_ARCHIVE.put(key, content, { + onlyIf: { etagDoesNotMatch: "*" }, + httpMetadata: { contentType: "application/json" }, + }); + if (created) return; + const existing = await env.OPERATIONS_ARCHIVE.get(key); + if (!existing || (await existing.text()) !== content) { + throw new ApiError("ARCHIVE_OPERATION_FAILED", 409, "Audit export conflicts with prior write"); + } +} + +async function buildPage(publisherDid: string, cursor: string | null): Promise { + const publisher = env.PUBLISHER_DO.getByName(publisherDid); + if (cursor === null) { + return { + kind: "metadata", + data: await publisher.getOperationsMetadata(publisherDid), + nextCursor: "workloads:", + }; + } + if (cursor.startsWith("workloads:")) { + const after = cursor.slice("workloads:".length) || null; + const items = await publisher.listWorkloadPolicies(publisherDid, after, WORKLOAD_PAGE_SIZE); + return { + kind: "workload-policies", + data: { items }, + nextCursor: + items.length === WORKLOAD_PAGE_SIZE ? `workloads:${items.at(-1)!.packageSlug}` : "intents:", + }; + } + if (cursor.startsWith("intents:")) { + const after = cursor.slice("intents:".length) || null; + const intents = await publisher.listIntents(publisherDid, after, INTENT_PAGE_SIZE + 1); + const items = intents.slice(0, INTENT_PAGE_SIZE); + return { + kind: "intents", + data: { items }, + nextCursor: intents.length > INTENT_PAGE_SIZE ? `intents:${items.at(-1)!.id}` : "audit:0", + }; + } + const afterSequence = Number(cursor.slice("audit:".length)); + const items = await publisher.listAuditEvents(publisherDid, afterSequence, AUDIT_PAGE_SIZE); + return { + kind: "audit-events", + data: { items }, + nextCursor: items.length === AUDIT_PAGE_SIZE ? `audit:${items.at(-1)!.sequence}` : null, + auditEvents: items, + }; +} + +export function matchPublisherArchivePath( + pathname: string, +): Readonly> | null { + const match = ARCHIVE_PATH_PATTERN.exec(pathname); + if (!match?.[1]) return null; + let publisherDid: string; + try { + publisherDid = decodeURIComponent(match[1]); + } catch { + return null; + } + return isDid(publisherDid) ? { publisherDid } : null; +} + +export function matchPublisherRestorePath( + pathname: string, +): Readonly> | null { + const match = RESTORE_PATH_PATTERN.exec(pathname); + if (!match?.[1]) return null; + let publisherDid: string; + try { + publisherDid = decodeURIComponent(match[1]); + } catch { + return null; + } + return isDid(publisherDid) ? { publisherDid } : null; +} + +export function matchPublisherRestorePreparePath( + pathname: string, +): Readonly> | null { + const match = RESTORE_PREPARE_PATH_PATTERN.exec(pathname); + if (!match?.[1]) return null; + let publisherDid: string; + try { + publisherDid = decodeURIComponent(match[1]); + } catch { + return null; + } + return isDid(publisherDid) ? { publisherDid } : null; +} + +export function matchPublisherRestoreAbortPath( + pathname: string, +): Readonly> | null { + const match = RESTORE_ABORT_PATH_PATTERN.exec(pathname); + if (!match?.[1]) return null; + let publisherDid: string; + try { + publisherDid = decodeURIComponent(match[1]); + } catch { + return null; + } + return isDid(publisherDid) ? { publisherDid } : null; +} + +export async function handleArchivePublisher( + request: Request, + requestId: string, + configuration: ServiceConfiguration, + params: Readonly>, + accessActor: AccessActor | null, +): Promise { + try { + requireActor(accessActor); + requireIdempotencyKey(request); + const publisherDid = params["publisherDid"]; + if (!publisherDid || !isDid(publisherDid)) { + throw new ApiError("NOT_FOUND", 404, "Publisher not found"); + } + const input = await archiveInput(request); + const ownerHash = await hashOwner(publisherDid); + const result = await buildPage(publisherDid, input.cursor); + const snapshot: SnapshotPage = { + version: SNAPSHOT_VERSION, + archiveId: input.archiveId, + publisherDid, + page: input.page, + kind: result.kind, + data: result.data, + }; + const key = `snapshots/${ownerHash}/${input.archiveId}/${String(input.page).padStart(6, "0")}.json.jwe`; + const replayed = await writeEncryptedObject( + key, + JSON.stringify(snapshot), + snapshotContext(publisherDid, input.archiveId, String(input.page)), + configuration, + { + archive: input.archiveId, + kind: result.kind, + owner: ownerHash, + page: String(input.page), + }, + ); + if (result.auditEvents) { + await writeAuditObject(ownerHash, result.auditEvents); + } + let manifestWritten = false; + if (result.nextCursor === null) { + const manifest = JSON.stringify({ + version: SNAPSHOT_VERSION, + archiveId: input.archiveId, + publisherDid, + pages: input.page + 1, + complete: true, + }); + await writeEncryptedObject( + `snapshots/${ownerHash}/${input.archiveId}/manifest.json.jwe`, + manifest, + snapshotContext(publisherDid, input.archiveId, "manifest"), + configuration, + { archive: input.archiveId, owner: ownerHash, pages: String(input.page + 1) }, + ); + manifestWritten = true; + } + console.log( + JSON.stringify({ + event: "publisher_archive_page", + ownerHash, + archiveId: input.archiveId, + page: input.page, + kind: result.kind, + replayed, + complete: result.nextCursor === null, + }), + ); + return apiSuccess( + { + archiveId: input.archiveId, + ownerHash, + page: input.page, + kind: result.kind, + nextCursor: result.nextCursor, + nextPage: input.page + 1, + replayed, + complete: result.nextCursor === null, + manifestWritten, + }, + requestId, + ); + } catch (error) { + writeOperationsMetric({ + event: "archive_gap", + outcome: error instanceof ApiError ? error.code : "internal", + requestId, + }); + if (error instanceof ApiError) return apiFailure(error, requestId); + console.error( + JSON.stringify({ + event: "publisher_archive_failed", + requestId, + name: error instanceof Error ? error.name : "UnknownError", + }), + ); + return apiFailure( + new ApiError("ARCHIVE_OPERATION_FAILED", 503, "Publisher archive failed"), + requestId, + ); + } +} + +function isSnapshotKind(value: unknown): value is SnapshotKind { + return ( + value === "audit-events" || + value === "intents" || + value === "metadata" || + value === "workload-policies" + ); +} + +function parseManifest(value: string, publisherDid: string, archiveId: string): { pages: number } { + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + throw new ApiError("RESTORE_OPERATION_FAILED", 409, "Publisher archive is invalid"); + } + if ( + !isRecord(parsed) || + !hasExactKeys(parsed, ["version", "archiveId", "publisherDid", "pages", "complete"]) || + parsed["version"] !== SNAPSHOT_VERSION || + parsed["archiveId"] !== archiveId || + parsed["publisherDid"] !== publisherDid || + !Number.isSafeInteger(parsed["pages"]) || + Number(parsed["pages"]) < 1 || + Number(parsed["pages"]) > MAX_ARCHIVE_PAGE + 1 || + parsed["complete"] !== true || + JSON.stringify(parsed) !== value + ) { + throw new ApiError("RESTORE_OPERATION_FAILED", 409, "Publisher archive is invalid"); + } + return { pages: Number(parsed["pages"]) }; +} + +function parseSnapshot( + value: string, + publisherDid: string, + archiveId: string, + page: number, +): SnapshotPage { + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + throw new ApiError("RESTORE_OPERATION_FAILED", 409, "Publisher archive is invalid"); + } + if ( + !isRecord(parsed) || + !hasExactKeys(parsed, ["version", "archiveId", "publisherDid", "page", "kind", "data"]) || + parsed["version"] !== SNAPSHOT_VERSION || + parsed["archiveId"] !== archiveId || + parsed["publisherDid"] !== publisherDid || + parsed["page"] !== page || + !isSnapshotKind(parsed["kind"]) || + !isRecord(parsed["data"]) || + JSON.stringify(parsed) !== value + ) { + throw new ApiError("RESTORE_OPERATION_FAILED", 409, "Publisher archive is invalid"); + } + return { + version: SNAPSHOT_VERSION, + archiveId, + publisherDid, + page, + kind: parsed["kind"], + data: parsed["data"], + }; +} + +async function digestPage(value: string): Promise { + return base64url.encode( + new Uint8Array(await crypto.subtle.digest("SHA-256", encoder.encode(value))), + ); +} + +export async function handleRestorePublisher( + request: Request, + requestId: string, + configuration: ServiceConfiguration, + params: Readonly>, + accessActor: AccessActor | null, +): Promise { + try { + const actor = requireActor(accessActor); + requireIdempotencyKey(request); + const publisherDid = params["publisherDid"]; + if (!publisherDid || !isDid(publisherDid)) { + throw new ApiError("NOT_FOUND", 404, "Publisher not found"); + } + const input = await restoreInput(request); + const control = await env.SERVICE_CONTROL_DO.getByName( + SERVICE_CONTROL_OBJECT_NAME, + ).readPublisherControl(actor, publisherDid); + if (control.status !== "suspended") { + throw new ApiError( + "RESTORE_OPERATION_FAILED", + 409, + "Suspend the publisher before restoring a shard", + ); + } + const ownerHash = await hashOwner(publisherDid); + const prefix = `snapshots/${ownerHash}/${input.archiveId}`; + const manifestPlaintext = await readEncryptedObject( + `${prefix}/manifest.json.jwe`, + snapshotContext(publisherDid, input.archiveId, "manifest"), + configuration, + ); + const manifest = parseManifest(manifestPlaintext, publisherDid, input.archiveId); + if (input.page >= manifest.pages) { + throw new ApiError("INVALID_REQUEST", 400, "Publisher restore page is out of range"); + } + const pagePlaintext = await readEncryptedObject( + `${prefix}/${String(input.page).padStart(6, "0")}.json.jwe`, + snapshotContext(publisherDid, input.archiveId, String(input.page)), + configuration, + ); + const snapshot = parseSnapshot(pagePlaintext, publisherDid, input.archiveId, input.page); + const result = await env.PUBLISHER_DO.getByName(publisherDid).applyOperationsRestorePage({ + publisherDid, + archiveId: input.archiveId, + page: input.page, + totalPages: manifest.pages, + kind: snapshot.kind, + dataJson: JSON.stringify(snapshot.data), + pageDigest: await digestPage(pagePlaintext), + actorIdentity: actor.identity, + }); + if (!result.ok) { + const message = + result.code === "RESTORE_NOT_EMPTY" + ? "Publisher shard is not empty" + : result.code === "RESTORE_OUT_OF_ORDER" + ? "Publisher restore page is out of order" + : "Publisher restore conflicts with prior state"; + throw new ApiError("RESTORE_OPERATION_FAILED", 409, message); + } + console.log( + JSON.stringify({ + event: "publisher_restore_page", + ownerHash, + archiveId: input.archiveId, + page: input.page, + kind: snapshot.kind, + replayed: result.replayed, + complete: result.complete, + }), + ); + return apiSuccess( + { + archiveId: input.archiveId, + ownerHash, + page: input.page, + kind: snapshot.kind, + nextPage: result.nextPage, + totalPages: manifest.pages, + replayed: result.replayed, + complete: result.complete, + authorityStatus: "reauthorization_required", + }, + requestId, + ); + } catch (error) { + writeOperationsMetric({ + event: "restore_failure", + outcome: error instanceof ApiError ? error.code : "internal", + requestId, + }); + if (error instanceof ApiError) return apiFailure(error, requestId); + console.error( + JSON.stringify({ + event: "publisher_restore_failed", + requestId, + name: error instanceof Error ? error.name : "UnknownError", + }), + ); + return apiFailure( + new ApiError("RESTORE_OPERATION_FAILED", 503, "Publisher restore failed"), + requestId, + ); + } +} + +export async function handlePreparePublisherRestore( + request: Request, + requestId: string, + configuration: ServiceConfiguration, + params: Readonly>, + accessActor: AccessActor | null, +): Promise { + try { + const actor = requireActor(accessActor); + requireIdempotencyKey(request); + const publisherDid = params["publisherDid"]; + if (!publisherDid || !isDid(publisherDid)) { + throw new ApiError("NOT_FOUND", 404, "Publisher not found"); + } + const body = await readJsonObject(request); + if ( + !hasExactKeys(body, ["archiveId", "confirmPublisherDid"]) || + typeof body["archiveId"] !== "string" || + !ARCHIVE_ID_PATTERN.test(body["archiveId"]) || + body["confirmPublisherDid"] !== publisherDid + ) { + throw new ApiError("INVALID_REQUEST", 400, "Publisher restore confirmation is invalid"); + } + const control = await env.SERVICE_CONTROL_DO.getByName( + SERVICE_CONTROL_OBJECT_NAME, + ).readPublisherControl(actor, publisherDid); + if (control.status !== "suspended") { + throw new ApiError( + "RESTORE_OPERATION_FAILED", + 409, + "Suspend the publisher before preparing a restore", + ); + } + const ownerHash = await hashOwner(publisherDid); + const manifestPlaintext = await readEncryptedObject( + `snapshots/${ownerHash}/${body["archiveId"]}/manifest.json.jwe`, + snapshotContext(publisherDid, body["archiveId"], "manifest"), + configuration, + ); + const manifest = parseManifest(manifestPlaintext, publisherDid, body["archiveId"]); + const publisher = env.PUBLISHER_DO.getByName(publisherDid); + await publisher.setPublisherSuspended(publisherDid, true, actor.identity); + const result = await publisher.prepareOperationsRestore( + publisherDid, + body["archiveId"], + manifest.pages, + actor.identity, + ); + if (!result.ok) { + throw new ApiError( + "RESTORE_OPERATION_FAILED", + 409, + result.code === "PUBLISHER_NOT_SUSPENDED" + ? "Publisher is not suspended" + : "Another publisher restore is already in progress", + ); + } + return apiSuccess( + { + archiveId: body["archiveId"], + publisherDid, + prepared: true, + deletedIntents: result.deletedIntents, + deletedWorkloads: result.deletedWorkloads, + replayed: result.replayed, + }, + requestId, + ); + } catch (error) { + writeOperationsMetric({ + event: "restore_failure", + outcome: error instanceof ApiError ? error.code : "internal", + requestId, + }); + if (error instanceof ApiError) return apiFailure(error, requestId); + return apiFailure( + new ApiError("RESTORE_OPERATION_FAILED", 503, "Publisher restore preparation failed"), + requestId, + ); + } +} + +export async function handleAbortPublisherRestore( + request: Request, + requestId: string, + _configuration: ServiceConfiguration, + params: Readonly>, + accessActor: AccessActor | null, +): Promise { + try { + const actor = requireActor(accessActor); + requireIdempotencyKey(request); + const publisherDid = params["publisherDid"]; + if (!publisherDid || !isDid(publisherDid)) { + throw new ApiError("NOT_FOUND", 404, "Publisher not found"); + } + const body = await readJsonObject(request); + if ( + !hasExactKeys(body, ["archiveId", "confirmPublisherDid"]) || + typeof body["archiveId"] !== "string" || + !ARCHIVE_ID_PATTERN.test(body["archiveId"]) || + body["confirmPublisherDid"] !== publisherDid + ) { + throw new ApiError("INVALID_REQUEST", 400, "Publisher restore confirmation is invalid"); + } + const control = await env.SERVICE_CONTROL_DO.getByName( + SERVICE_CONTROL_OBJECT_NAME, + ).readPublisherControl(actor, publisherDid); + if (control.status !== "suspended") { + throw new ApiError( + "RESTORE_OPERATION_FAILED", + 409, + "Suspend the publisher before aborting a restore", + ); + } + const result = await env.PUBLISHER_DO.getByName(publisherDid).abortOperationsRestore( + publisherDid, + body["archiveId"], + actor.identity, + ); + if (!result.ok) { + throw new ApiError( + "RESTORE_OPERATION_FAILED", + 409, + result.code === "PUBLISHER_NOT_SUSPENDED" + ? "Publisher is not suspended" + : "Publisher restore cannot be aborted", + ); + } + console.log( + JSON.stringify({ + event: "publisher_restore_aborted", + archiveId: body["archiveId"], + publisherHash: await hashOwner(publisherDid), + replayed: result.replayed, + }), + ); + return apiSuccess( + { + archiveId: body["archiveId"], + publisherDid, + aborted: true, + replayed: result.replayed, + }, + requestId, + ); + } catch (error) { + writeOperationsMetric({ + event: "restore_failure", + outcome: error instanceof ApiError ? error.code : "internal", + requestId, + }); + if (error instanceof ApiError) return apiFailure(error, requestId); + return apiFailure( + new ApiError("RESTORE_OPERATION_FAILED", 503, "Publisher restore abort failed"), + requestId, + ); + } +} diff --git a/apps/release-service/src/backup/workflow-route.ts b/apps/release-service/src/backup/workflow-route.ts new file mode 100644 index 0000000000..46149a482e --- /dev/null +++ b/apps/release-service/src/backup/workflow-route.ts @@ -0,0 +1,85 @@ +import { isDid } from "@atcute/lexicons/syntax"; +import { env } from "cloudflare:workers"; + +import type { AccessActor } from "../access/auth.js"; +import { readJsonObject } from "../api/body.js"; +import { ApiError } from "../api/errors.js"; +import { apiFailure, apiSuccess } from "../api/response.js"; +import type { ServiceConfiguration } from "../config.js"; +import { startPublisherArchiveWorkflow } from "../workflows/publisher-archive.js"; + +const ARCHIVE_START_PATH_PATTERN = /^\/admin\/api\/publishers\/([^/]+)\/archive\/start$/; +const ARCHIVE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]{15,63}$/; +const IDEMPOTENCY_KEY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$/; + +export interface ArchiveWorkflowRouteDependencies { + startWorkflow?: typeof startPublisherArchiveWorkflow; +} + +function hasExactKeys(value: Record, expected: readonly string[]): boolean { + const keys = Object.keys(value); + return keys.length === expected.length && keys.every((key) => expected.includes(key)); +} + +export function matchPublisherArchiveStartPath( + pathname: string, +): Readonly> | null { + const match = ARCHIVE_START_PATH_PATTERN.exec(pathname); + if (!match?.[1]) return null; + let publisherDid: string; + try { + publisherDid = decodeURIComponent(match[1]); + } catch { + return null; + } + return isDid(publisherDid) ? { publisherDid } : null; +} + +export async function handleStartPublisherArchive( + request: Request, + requestId: string, + _configuration: ServiceConfiguration, + params: Readonly>, + accessActor: AccessActor | null, + dependencies: ArchiveWorkflowRouteDependencies = {}, +): Promise { + try { + if (!accessActor) { + throw new ApiError("ACCESS_AUTH_REQUIRED", 401, "Access authentication required"); + } + const idempotencyKey = request.headers.get("idempotency-key"); + if (!idempotencyKey || !IDEMPOTENCY_KEY_PATTERN.test(idempotencyKey)) { + throw new ApiError("IDEMPOTENCY_KEY_INVALID", 400, "Valid idempotency key required"); + } + const publisherDid = params["publisherDid"]; + if (!publisherDid || !isDid(publisherDid)) { + throw new ApiError("NOT_FOUND", 404, "Publisher not found"); + } + const body = await readJsonObject(request); + if ( + !hasExactKeys(body, ["archiveId"]) || + typeof body["archiveId"] !== "string" || + !ARCHIVE_ID_PATTERN.test(body["archiveId"]) + ) { + throw new ApiError("INVALID_REQUEST", 400, "Invalid publisher archive request"); + } + const result = await (dependencies.startWorkflow ?? startPublisherArchiveWorkflow)( + env.PUBLISHER_ARCHIVE_WORKFLOW, + { + publisherDid, + archiveId: body["archiveId"], + actorIdentity: accessActor.identity, + }, + ); + if (!result.ok) { + throw new ApiError("ARCHIVE_OPERATION_FAILED", 503, "Publisher archive could not start"); + } + return apiSuccess( + { archiveId: body["archiveId"], workflowId: result.workflowId, created: result.created }, + requestId, + result.created ? 202 : 200, + ); + } catch (error) { + return apiFailure(error, requestId); + } +} diff --git a/apps/release-service/src/config.ts b/apps/release-service/src/config.ts index 19f597b77d..9fb09fd717 100644 --- a/apps/release-service/src/config.ts +++ b/apps/release-service/src/config.ts @@ -8,19 +8,21 @@ import { getDelegatedReleasePermission } from "@emdash-cms/registry-lexicons"; import type { AccessConfiguration } from "./access/auth.js"; import { createEnvelopeEncryption, type EnvelopeEncryption } from "./crypto/encryption.js"; -export type ConfigurationBindings = Record< - keyof Pick< - Env, - | "PUBLIC_ORIGIN" - | "DEPLOYMENT_ID" - | "OAUTH_REDIRECT_URIS" - | "OAUTH_ASSERTION_KEYSET" - | "ENCRYPTION_KEYRING" - | "ACCESS_TEAM_DOMAIN" - | "ACCESS_VIEWER_AUD" - | "ACCESS_REVIEWER_AUD" - | "ACCESS_ADMIN_AUD" - >, +type ConfigurationStringBinding = + | "PUBLIC_ORIGIN" + | "DEPLOYMENT_ID" + | "OAUTH_REDIRECT_URIS" + | "ACCESS_TEAM_DOMAIN" + | "ACCESS_VIEWER_AUD" + | "ACCESS_REVIEWER_AUD" + | "ACCESS_ADMIN_AUD"; +type ConfigurationSecretBinding = "OAUTH_ASSERTION_KEYSET" | "ENCRYPTION_KEYRING"; +type ConfigurationSecretSource = string | SecretsStoreSecret; + +export type ConfigurationBindings = Record & + Record; +type ResolvedConfigurationBindings = Record< + ConfigurationStringBinding | ConfigurationSecretBinding, string >; @@ -35,7 +37,6 @@ const CONFIGURATION_BINDING_KEYS = [ "DEPLOYMENT_ID", "OAUTH_REDIRECT_URIS", "OAUTH_ASSERTION_KEYSET", - "ENCRYPTION_KEYRING", "ACCESS_TEAM_DOMAIN", "ACCESS_VIEWER_AUD", "ACCESS_REVIEWER_AUD", @@ -44,7 +45,8 @@ const CONFIGURATION_BINDING_KEYS = [ interface ConfigurationCacheEntry { snapshot: readonly string[]; - promise: Promise; + promise: Promise; + encryption?: { keyring: string; value: EnvelopeEncryption }; } export interface ServiceConfiguration { @@ -55,6 +57,8 @@ export interface ServiceConfiguration { encryption: EnvelopeEncryption; } +type CachedServiceConfiguration = Omit; + export type P256AssertionPrivateJwk = ClientAssertionPrivateJwk & { kty: "EC"; crv: "P-256"; @@ -104,7 +108,7 @@ function parseAccessTeamDomain(value: unknown): string | null { } function parseAccessAudiences( - bindings: ConfigurationBindings, + bindings: ResolvedConfigurationBindings, ): AccessConfiguration["audiences"] | null { const audiences = { viewer: bindings.ACCESS_VIEWER_AUD, @@ -242,7 +246,9 @@ async function parseAssertionKeyset(value: string): Promise<{ } } -async function parseConfiguration(bindings: ConfigurationBindings): Promise { +async function parseConfiguration( + bindings: ResolvedConfigurationBindings, +): Promise { const issues: string[] = []; const publicOrigin = parseOrigin(bindings.PUBLIC_ORIGIN); if (!publicOrigin) issues.push("PUBLIC_ORIGIN_INVALID"); @@ -271,12 +277,6 @@ async function parseConfiguration(bindings: ConfigurationBindings): Promise { return (target[CONFIGURATION_CACHE_SYMBOL] ??= new WeakMap()); } +async function resolveSecret(source: ConfigurationSecretSource): Promise { + if (typeof source === "string") return source; + const value = await source.get(); + if (typeof value !== "string") throw new TypeError("Secret value is invalid"); + return value; +} + export async function loadConfiguration( bindings: ConfigurationBindings, ): Promise { - const snapshot = CONFIGURATION_BINDING_KEYS.map((key) => bindings[key]); + let assertionKeyset: string; + let encryptionKeyring: string; + try { + [assertionKeyset, encryptionKeyring] = await Promise.all([ + resolveSecret(bindings.OAUTH_ASSERTION_KEYSET), + resolveSecret(bindings.ENCRYPTION_KEYRING), + ]); + } catch { + throw new ConfigurationError(["SECRET_STORE_UNAVAILABLE"]); + } + const resolved: ResolvedConfigurationBindings = { + ...bindings, + OAUTH_ASSERTION_KEYSET: assertionKeyset, + ENCRYPTION_KEYRING: encryptionKeyring, + }; + const snapshot = CONFIGURATION_BINDING_KEYS.map((key) => resolved[key]); const cache = getConfigurationCache(); const cached = cache.get(bindings); let promise = cached?.snapshot.every((value, index) => value === snapshot[index]) ? cached.promise : null; if (!promise) { - promise = parseConfiguration(bindings); + promise = parseConfiguration(resolved); cache.set(bindings, { snapshot, promise }); } - return promise; + const configuration = await promise; + const entry = cache.get(bindings); + let encryption = entry?.encryption?.keyring === encryptionKeyring ? entry.encryption.value : null; + if (!encryption) { + try { + encryption = createEnvelopeEncryption(encryptionKeyring, configuration.deploymentId); + } catch { + throw new ConfigurationError(["ENCRYPTION_KEYRING_INVALID"]); + } + if (entry) entry.encryption = { keyring: encryptionKeyring, value: encryption }; + } + return { ...configuration, encryption }; } diff --git a/apps/release-service/src/control-do/routes.ts b/apps/release-service/src/control-do/routes.ts index 1f8730e4d3..2c38344f3d 100644 --- a/apps/release-service/src/control-do/routes.ts +++ b/apps/release-service/src/control-do/routes.ts @@ -6,6 +6,7 @@ import { readJsonObject } from "../api/body.js"; import { ApiError } from "../api/errors.js"; import { apiSuccess } from "../api/response.js"; import type { ServiceConfiguration } from "../config.js"; +import { writeOperationsMetric } from "../observability/metrics.js"; import { SERVICE_CONTROL_OBJECT_NAME, type ServiceMode } from "./service-control-do.js"; const REASON_CODE_PATTERN = /^[A-Z][A-Z0-9_]{0,63}$/; @@ -121,6 +122,14 @@ export async function handleSetServiceMode( if (!result.ok) { throw new ApiError("IDEMPOTENCY_CONFLICT", 409, "Idempotency key conflicts with prior use"); } + if (mode === "publication-paused") { + writeOperationsMetric({ + event: "publication_paused", + outcome: reasonCode ?? "unspecified", + scope: "service", + requestId, + }); + } return apiSuccess({ state: result.value, replayed: result.replayed }, requestId); } catch (error) { mapControlError(error); diff --git a/apps/release-service/src/crypto/encryption.ts b/apps/release-service/src/crypto/encryption.ts index 895ba8ceda..c785b89459 100644 --- a/apps/release-service/src/crypto/encryption.ts +++ b/apps/release-service/src/crypto/encryption.ts @@ -41,6 +41,7 @@ const OWNED_PURPOSES: ReadonlySet = new Set([ "webhook-destination", "webhook-secret", "csrf-secret", + "publisher-snapshot", ]); const UNOWNED_PURPOSES: ReadonlySet = new Set([ "confidential-client-private-key", @@ -58,7 +59,8 @@ export type OwnedEncryptionPurpose = | "email-address" | "webhook-destination" | "webhook-secret" - | "csrf-secret"; + | "csrf-secret" + | "publisher-snapshot"; export type OptionalOwnerEncryptionPurpose = | "oauth-transaction" diff --git a/apps/release-service/src/directory/identity-directory-do.ts b/apps/release-service/src/directory/identity-directory-do.ts new file mode 100644 index 0000000000..dfe22208fe --- /dev/null +++ b/apps/release-service/src/directory/identity-directory-do.ts @@ -0,0 +1,149 @@ +import { DurableObject } from "cloudflare:workers"; + +const DID_PATTERN = /^did:[a-z0-9]+:[A-Za-z0-9._:%-]+$/; +const SHARD_PATTERN = /^[0-9a-f]{2}$/; + +export type DirectoryIdentityKind = "approver" | "publisher"; + +export interface DirectoryIdentity { + kind: DirectoryIdentityKind; + did: string; + registeredAt: number; + lastSeenAt: number; +} + +export type DirectoryErrorCode = + | "DIRECTORY_INPUT_INVALID" + | "DIRECTORY_SHARD_INVALID" + | "DIRECTORY_SHARD_MISMATCH"; + +export class DirectoryError extends Error { + constructor(readonly code: DirectoryErrorCode) { + super(code); + this.name = "DirectoryError"; + } +} + +interface DirectoryRow { + [key: string]: string | number | ArrayBuffer | null; + kind: DirectoryIdentityKind; + did: string; + registered_at: number; + last_seen_at: number; +} + +async function expectedShard(did: string): Promise { + const digest = new Uint8Array( + await crypto.subtle.digest("SHA-256", new TextEncoder().encode(did)), + ); + return digest[0]!.toString(16).padStart(2, "0"); +} + +function validKind(value: unknown): value is DirectoryIdentityKind { + return value === "approver" || value === "publisher"; +} + +export class IdentityDirectoryDurableObject extends DurableObject { + readonly #shard: string; + + constructor(ctx: DurableObjectState, env: Env) { + super(ctx, env); + if (ctx.id.name === undefined || !SHARD_PATTERN.test(ctx.id.name)) { + throw new DirectoryError("DIRECTORY_SHARD_INVALID"); + } + this.#shard = ctx.id.name; + void ctx.blockConcurrencyWhile(async () => { + ctx.storage.sql.exec(` + CREATE TABLE IF NOT EXISTS identities ( + kind TEXT NOT NULL CHECK (kind IN ('approver', 'publisher')), + did TEXT NOT NULL, + registered_at INTEGER NOT NULL, + last_seen_at INTEGER NOT NULL, + PRIMARY KEY (kind, did) + ); + CREATE INDEX IF NOT EXISTS idx_directory_last_seen + ON identities(kind, last_seen_at, did); + `); + }); + } + + async register( + kind: DirectoryIdentityKind, + did: string, + now = Date.now(), + ): Promise<{ created: boolean; identity: DirectoryIdentity }> { + if (!validKind(kind) || !DID_PATTERN.test(did) || !Number.isSafeInteger(now) || now < 0) { + throw new DirectoryError("DIRECTORY_INPUT_INVALID"); + } + if ((await expectedShard(did)) !== this.#shard) { + throw new DirectoryError("DIRECTORY_SHARD_MISMATCH"); + } + return this.ctx.storage.transactionSync(() => { + const existing = this.ctx.storage.sql + .exec( + `SELECT kind, did, registered_at, last_seen_at FROM identities + WHERE kind = ? AND did = ?`, + kind, + did, + ) + .toArray()[0]; + if (existing) { + this.ctx.storage.sql.exec( + "UPDATE identities SET last_seen_at = MAX(last_seen_at, ?) WHERE kind = ? AND did = ?", + now, + kind, + did, + ); + return { + created: false, + identity: { + kind, + did, + registeredAt: existing.registered_at, + lastSeenAt: Math.max(existing.last_seen_at, now), + }, + }; + } + this.ctx.storage.sql.exec( + "INSERT INTO identities (kind, did, registered_at, last_seen_at) VALUES (?, ?, ?, ?)", + kind, + did, + now, + now, + ); + return { created: true, identity: { kind, did, registeredAt: now, lastSeenAt: now } }; + }); + } + + list( + kind: DirectoryIdentityKind, + afterDid: string | null, + limit: number, + ): readonly DirectoryIdentity[] { + if ( + !validKind(kind) || + (afterDid !== null && !DID_PATTERN.test(afterDid)) || + !Number.isSafeInteger(limit) || + limit < 1 || + limit > 100 + ) { + throw new DirectoryError("DIRECTORY_INPUT_INVALID"); + } + return this.ctx.storage.sql + .exec( + `SELECT kind, did, registered_at, last_seen_at FROM identities + WHERE kind = ? AND (? IS NULL OR did > ?) ORDER BY did LIMIT ?`, + kind, + afterDid, + afterDid, + limit, + ) + .toArray() + .map((row) => ({ + kind: row.kind, + did: row.did, + registeredAt: row.registered_at, + lastSeenAt: row.last_seen_at, + })); + } +} diff --git a/apps/release-service/src/directory/routes.ts b/apps/release-service/src/directory/routes.ts new file mode 100644 index 0000000000..bec9e9b58e --- /dev/null +++ b/apps/release-service/src/directory/routes.ts @@ -0,0 +1,127 @@ +import { env } from "cloudflare:workers"; +import { base64url } from "jose"; + +import type { AccessActor } from "../access/auth.js"; +import { ApiError } from "../api/errors.js"; +import { apiFailure, apiSuccess } from "../api/response.js"; +import type { ServiceConfiguration } from "../config.js"; +import type { DirectoryIdentityKind } from "./identity-directory-do.js"; + +const DID_PATTERN = /^did:[a-z0-9]+:[A-Za-z0-9._:%-]+$/; +const POSITIVE_INTEGER_PATTERN = /^[1-9][0-9]*$/; +const MAX_CURSOR_CHARS = 4096; + +export interface DirectoryCursor { + shard: number; + afterDid: string | null; +} + +function validCursor(value: DirectoryCursor): boolean { + return ( + Number.isSafeInteger(value.shard) && + value.shard >= 0 && + value.shard <= 255 && + (value.afterDid === null || DID_PATTERN.test(value.afterDid)) + ); +} + +export function encodeDirectoryCursor(value: DirectoryCursor): string { + if (!validCursor(value)) throw new TypeError("Invalid directory cursor"); + return base64url.encode(new TextEncoder().encode(JSON.stringify(value))); +} + +function decodeDirectoryCursor(value: string | null): DirectoryCursor { + if (value === null) return { shard: 0, afterDid: null }; + if (value.length === 0 || value.length > MAX_CURSOR_CHARS) { + throw new ApiError("INVALID_REQUEST", 400, "Invalid directory cursor"); + } + let text: string; + let parsed: unknown; + try { + const bytes = base64url.decode(value); + if (base64url.encode(bytes) !== value) throw new Error(); + text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: false }).decode(bytes); + parsed = JSON.parse(text); + } catch { + throw new ApiError("INVALID_REQUEST", 400, "Invalid directory cursor"); + } + if ( + parsed === null || + typeof parsed !== "object" || + Array.isArray(parsed) || + Object.keys(parsed).length !== 2 || + !("shard" in parsed) || + !("afterDid" in parsed) + ) { + throw new ApiError("INVALID_REQUEST", 400, "Invalid directory cursor"); + } + const rawShard = parsed.shard; + const rawAfterDid = parsed.afterDid; + if (typeof rawShard !== "number" || (rawAfterDid !== null && typeof rawAfterDid !== "string")) { + throw new ApiError("INVALID_REQUEST", 400, "Invalid directory cursor"); + } + const cursor: DirectoryCursor = { shard: rawShard, afterDid: rawAfterDid }; + if (!validCursor(cursor) || JSON.stringify(cursor) !== text) { + throw new ApiError("INVALID_REQUEST", 400, "Invalid directory cursor"); + } + return cursor; +} + +function directoryKind(value: string | null): DirectoryIdentityKind { + if (value !== "publisher" && value !== "approver") { + throw new ApiError("INVALID_REQUEST", 400, "Directory kind is required"); + } + return value; +} + +function directoryLimit(value: string | null): number { + if (value === null) return 50; + if (!POSITIVE_INTEGER_PATTERN.test(value)) { + throw new ApiError("INVALID_REQUEST", 400, "Invalid directory limit"); + } + const limit = Number(value); + if (!Number.isSafeInteger(limit) || limit > 100) { + throw new ApiError("INVALID_REQUEST", 400, "Invalid directory limit"); + } + return limit; +} + +export async function handleListDirectory( + request: Request, + requestId: string, + _configuration: ServiceConfiguration, + _params: Readonly>, + accessActor: AccessActor | null, +): Promise { + try { + if (!accessActor) { + throw new ApiError("ACCESS_AUTH_REQUIRED", 401, "Access authentication required"); + } + const url = new URL(request.url); + const kind = directoryKind(url.searchParams.get("kind")); + const limit = directoryLimit(url.searchParams.get("limit")); + const cursor = decodeDirectoryCursor(url.searchParams.get("cursor")); + if ( + url.searchParams.getAll("kind").length !== 1 || + url.searchParams.getAll("limit").length > 1 || + url.searchParams.getAll("cursor").length > 1 + ) { + throw new ApiError("INVALID_REQUEST", 400, "Invalid directory query"); + } + const shard = cursor.shard.toString(16).padStart(2, "0"); + const items = await env.IDENTITY_DIRECTORY_DO.getByName(shard).list( + kind, + cursor.afterDid, + limit, + ); + const nextCursor = + items.length === limit + ? encodeDirectoryCursor({ shard: cursor.shard, afterDid: items.at(-1)!.did }) + : cursor.shard < 255 + ? encodeDirectoryCursor({ shard: cursor.shard + 1, afterDid: null }) + : null; + return apiSuccess({ items: items.map((item) => ({ ...item, shard })), nextCursor }, requestId); + } catch (error) { + return apiFailure(error, requestId); + } +} diff --git a/apps/release-service/src/directory/sharding.ts b/apps/release-service/src/directory/sharding.ts new file mode 100644 index 0000000000..8d5aff752a --- /dev/null +++ b/apps/release-service/src/directory/sharding.ts @@ -0,0 +1,24 @@ +import { env } from "cloudflare:workers"; + +import type { + DirectoryIdentityKind, + IdentityDirectoryDurableObject, +} from "./identity-directory-do.js"; + +export async function identityDirectoryShard(did: string): Promise { + const digest = new Uint8Array( + await crypto.subtle.digest("SHA-256", new TextEncoder().encode(did)), + ); + return digest[0]!.toString(16).padStart(2, "0"); +} + +export async function registerDirectoryIdentity( + kind: DirectoryIdentityKind, + did: string, + now = Date.now(), + directory: DurableObjectNamespace = env.IDENTITY_DIRECTORY_DO, +): Promise<{ shard: string; created: boolean }> { + const shard = await identityDirectoryShard(did); + const result = await directory.getByName(shard).register(kind, did, now); + return { shard, created: result.created }; +} diff --git a/apps/release-service/src/index.ts b/apps/release-service/src/index.ts index c155ea257f..5decea1e69 100644 --- a/apps/release-service/src/index.ts +++ b/apps/release-service/src/index.ts @@ -16,13 +16,16 @@ import { type ConfigurationBindings, type ServiceConfiguration, } from "./config.js"; +import { writeOperationsMetric } from "./observability/metrics.js"; import { ROUTES, type RouteDefinition } from "./routes.js"; export { PublisherDurableObject } from "./publisher-do/publisher-do.js"; export { ApproverDurableObject } from "./approver-do/approver-do.js"; export { OAuthStateDurableObject } from "./oauth/state-do.js"; export { ReleaseIntentWorkflow } from "./workflows/release-intent.js"; +export { PublisherArchiveWorkflow } from "./workflows/publisher-archive.js"; export { ServiceControlDurableObject } from "./control-do/service-control-do.js"; +export { IdentityDirectoryDurableObject } from "./directory/identity-directory-do.js"; const DYNAMIC_PATH_PREFIXES = ["/.well-known/", "/admin/api/", "/oauth/", "/v1/"] as const; @@ -66,6 +69,13 @@ export async function handleUiRequest( try { await authenticateOperatorUi(request, await loadConfiguration(bindings), accessKeyResolver); } catch (error) { + if (error instanceof ApiError) { + writeOperationsMetric({ + event: "access_denied", + outcome: error.code, + requestId: getRequestId(request), + }); + } return apiFailure(error, getRequestId(request)); } } @@ -143,12 +153,29 @@ export async function handleRequest( return apiFailure(new ApiError("NOT_FOUND", 404, "Not found"), requestId); } catch (error) { if (error instanceof ConfigurationError) { + writeOperationsMetric({ + event: "configuration_failure", + outcome: "invalid", + requestId, + }); console.error(JSON.stringify({ event: "configuration_error", issues: error.issues })); return apiFailure( new ApiError("CONFIGURATION_ERROR", 503, "Service is not configured"), requestId, ); } + if ( + error instanceof ApiError && + (error.code === "ACCESS_AUTH_INVALID" || + error.code === "ACCESS_AUTH_REQUIRED" || + error.code === "ACCESS_DENIED") + ) { + writeOperationsMetric({ + event: "access_denied", + outcome: error.code, + requestId, + }); + } console.error( JSON.stringify({ event: "request_error", diff --git a/apps/release-service/src/intents/routes.ts b/apps/release-service/src/intents/routes.ts index d328e6e263..d65d59dc37 100644 --- a/apps/release-service/src/intents/routes.ts +++ b/apps/release-service/src/intents/routes.ts @@ -11,6 +11,7 @@ import { decodeAwaitingApprovalState } from "../approvals/digest.js"; import { invalidateApprovalChallenges } from "../approvals/invalidation.js"; import type { ServiceConfiguration } from "../config.js"; import { SERVICE_CONTROL_OBJECT_NAME } from "../control-do/service-control-do.js"; +import { writeOperationsMetric } from "../observability/metrics.js"; import type { IntentState, StoredIntent } from "../publisher-do/publisher-do.js"; import { PublisherSessionError, @@ -326,6 +327,46 @@ export async function handleSubmitReleaseIntent( if (!policy || !evaluateWorkloadPolicy(identity, policy).ok) { throw new ApiError("WORKLOAD_NOT_ALLOWED", 403, "Workload is not authorized"); } + const workloadRateKey = await digest([ + "intent-rate-limit", + 1, + identity.repository.id, + identity.workflow.ref, + release.package, + ]); + const rateLimit = await publisher.consumeIntentRateLimit({ + publisherDid, + repositoryId: identity.repository.id, + workloadKey: workloadRateKey, + idempotencyKey, + expiresAt: now + INTENT_LIFETIME_MS, + now, + }); + if (!rateLimit.ok) { + writeOperationsMetric({ + event: "intent_rate_limited", + ownerHash: workloadRateKey, + outcome: "denied", + scope: rateLimit.scope, + requestId, + }); + console.warn( + JSON.stringify({ + event: "release_intent_rate_limited", + requestId, + scope: rateLimit.scope, + workloadKey: workloadRateKey, + retryAt: rateLimit.retryAt, + }), + ); + const response = apiFailure( + new ApiError("WORKLOAD_RATE_LIMITED", 429, "Release intent rate limit exceeded"), + requestId, + ); + const headers = new Headers(response.headers); + headers.set("retry-after", String(Math.max(1, Math.ceil((rateLimit.retryAt - now) / 1000)))); + return new Response(response.body, { status: response.status, headers }); + } const workloadIdentityDigest = await digestWorkloadIdentity(identity); const created = await publisher.createIntent({ publisherDid, diff --git a/apps/release-service/src/oauth/custody.ts b/apps/release-service/src/oauth/custody.ts index dea6f4709c..a32e4cc75d 100644 --- a/apps/release-service/src/oauth/custody.ts +++ b/apps/release-service/src/oauth/custody.ts @@ -358,6 +358,7 @@ interface PutDurableOAuthStateInput { stateHash: string; encryptedState: string; encryptionKeyVersion: number; + encryptionPurpose: ReturnType; clientKeyId: string; redirectTarget: string; expiresAt: number; @@ -440,6 +441,7 @@ class DurableOAuthStateStore implements Store { stateHash, encryptedState: encrypted.envelope, encryptionKeyVersion: encrypted.keyVersion, + encryptionPurpose: transactionEncryptionPurpose(this.#options.purpose), clientKeyId: keyId, redirectTarget: userState.redirectTarget, expiresAt: state.expiresAt, diff --git a/apps/release-service/src/oauth/routes.ts b/apps/release-service/src/oauth/routes.ts index 9f0b93e81f..e079ba0b7c 100644 --- a/apps/release-service/src/oauth/routes.ts +++ b/apps/release-service/src/oauth/routes.ts @@ -6,6 +6,8 @@ import { ApiError } from "../api/errors.js"; import { apiFailure, apiSuccess } from "../api/response.js"; import { createApproverApplicationSession } from "../approver-session/session.js"; import type { ServiceConfiguration } from "../config.js"; +import { registerDirectoryIdentity } from "../directory/sharding.js"; +import { writeOperationsMetric } from "../observability/metrics.js"; import { PublisherSessionError, clearOAuthRouteCookie, @@ -338,6 +340,25 @@ export async function handleOAuthCallback( fetch: callbackFetch, }); await client.callback(params); + try { + await registerDirectoryIdentity( + route.purpose === "approver_identity" ? "approver" : "publisher", + route.expectedDid, + ); + } catch (error) { + writeOperationsMetric({ + event: "directory_failure", + outcome: route.purpose === "approver_identity" ? "approver" : "publisher", + requestId, + }); + console.error( + JSON.stringify({ + event: "identity_directory_registration_failed", + requestId, + name: error instanceof Error ? error.name : "UnknownError", + }), + ); + } const headers = new Headers({ "cache-control": "no-store", location: new URL(route.redirectTarget, configuration.publicOrigin).toString(), diff --git a/apps/release-service/src/observability/metrics.ts b/apps/release-service/src/observability/metrics.ts new file mode 100644 index 0000000000..eea23be566 --- /dev/null +++ b/apps/release-service/src/observability/metrics.ts @@ -0,0 +1,75 @@ +import { env } from "cloudflare:workers"; + +const HASH_PATTERN = /^[A-Za-z0-9_-]{43}$/; +const DIMENSION_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +const EVENTS = new Set([ + "access_denied", + "archive_gap", + "configuration_failure", + "directory_failure", + "intent_rate_limited", + "publication_paused", + "reconciliation_required", + "refresh_failure", + "restore_failure", + "verifier_failure", +]); + +export type OperationsMetricEvent = + | "access_denied" + | "archive_gap" + | "configuration_failure" + | "directory_failure" + | "intent_rate_limited" + | "publication_paused" + | "reconciliation_required" + | "refresh_failure" + | "restore_failure" + | "verifier_failure"; + +export interface OperationsMetricInput { + event: OperationsMetricEvent; + ownerHash?: string; + outcome?: string; + scope?: string; + requestId?: string; + value?: number; + timestamp?: number; +} + +function optionalDimension(value: string | undefined): string | null { + if (value === undefined) return null; + if (!DIMENSION_PATTERN.test(value)) throw new TypeError("Invalid operations metric"); + return value; +} + +export function writeOperationsMetric( + input: OperationsMetricInput, + dataset: AnalyticsEngineDataset = env.OPERATIONS_METRICS, +): void { + const timestamp = input.timestamp ?? Date.now(); + const value = input.value ?? 1; + if ( + !EVENTS.has(input.event) || + (input.ownerHash !== undefined && !HASH_PATTERN.test(input.ownerHash)) || + !Number.isFinite(value) || + !Number.isSafeInteger(timestamp) || + timestamp < 0 + ) { + throw new TypeError("Invalid operations metric"); + } + try { + dataset.writeDataPoint({ + indexes: [input.ownerHash ?? "global"], + blobs: [ + input.event, + optionalDimension(input.outcome), + optionalDimension(input.scope), + optionalDimension(input.requestId), + ], + doubles: [value, timestamp], + }); + } catch { + console.error(JSON.stringify({ event: "operations_metric_write_failed" })); + } +} diff --git a/apps/release-service/src/operations/encryption-records.ts b/apps/release-service/src/operations/encryption-records.ts new file mode 100644 index 0000000000..bb6370795d --- /dev/null +++ b/apps/release-service/src/operations/encryption-records.ts @@ -0,0 +1,24 @@ +import type { EncryptionContext } from "../crypto/encryption.js"; + +export const MAX_ENCRYPTION_RECORD_PAGE = 100; + +export interface EncryptionRecord { + cursor: string; + envelope: string; + keyVersion: number; + context: EncryptionContext; +} + +export interface EncryptionRecordPage { + items: readonly EncryptionRecord[]; + nextCursor: string | null; +} + +export interface EncryptionRecordReplacement { + cursor: string; + expectedEnvelope: string; + replacementEnvelope: string; + replacementKeyVersion: number; + actorIdentity: string; + now?: number; +} diff --git a/apps/release-service/src/operations/encryption-routes.ts b/apps/release-service/src/operations/encryption-routes.ts new file mode 100644 index 0000000000..71a450745d --- /dev/null +++ b/apps/release-service/src/operations/encryption-routes.ts @@ -0,0 +1,232 @@ +import { isDid } from "@atcute/lexicons/syntax"; +import { env } from "cloudflare:workers"; + +import type { AccessActor } from "../access/auth.js"; +import { readJsonObject } from "../api/body.js"; +import { ApiError } from "../api/errors.js"; +import { apiFailure, apiSuccess } from "../api/response.js"; +import type { ServiceConfiguration } from "../config.js"; +import { EncryptionError } from "../crypto/encryption.js"; +import type { EncryptionRecordPage, EncryptionRecordReplacement } from "./encryption-records.js"; +import { MAX_ENCRYPTION_RECORD_PAGE } from "./encryption-records.js"; + +const PUBLISHER_ROTATION_PATH_PATTERN = /^\/admin\/api\/publishers\/([^/]+)\/encryption\/rotate$/; +const APPROVER_ROTATION_PATH_PATTERN = /^\/admin\/api\/approvers\/([^/]+)\/encryption\/rotate$/; +const IDEMPOTENCY_KEY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$/; +const PUBLISHER_CURSOR_PATTERN = /^(?:delegation:1|oauth-state:[A-Za-z0-9_-]{32,128})$/; +const APPROVER_CURSOR_PATTERN = /^identity-transaction:[A-Za-z0-9_-]{43}$/; +const RACED_CURSOR_PREFIX = "raced:"; +const RESCAN_CURSOR = "rescan"; + +interface EncryptionShard { + list(afterCursor: string | null, limit: number): Promise; + replace(input: EncryptionRecordReplacement): Promise; +} + +function hasExactKeys(value: Record, expected: readonly string[]): boolean { + const keys = Object.keys(value); + return keys.length === expected.length && keys.every((key) => expected.includes(key)); +} + +function requireActor(actor: AccessActor | null): AccessActor { + if (!actor) throw new ApiError("ACCESS_AUTH_REQUIRED", 401, "Access authentication required"); + return actor; +} + +function requireIdempotencyKey(request: Request): void { + const value = request.headers.get("idempotency-key"); + if (!value || !IDEMPOTENCY_KEY_PATTERN.test(value)) { + throw new ApiError("IDEMPOTENCY_KEY_INVALID", 400, "Valid idempotency key required"); + } +} + +async function rotationPage( + request: Request, +): Promise<{ afterCursor: string | null; limit: number }> { + const body = await readJsonObject(request); + if ( + !hasExactKeys(body, ["afterCursor", "limit"]) || + (body["afterCursor"] !== null && typeof body["afterCursor"] !== "string") || + !Number.isSafeInteger(body["limit"]) || + Number(body["limit"]) < 1 || + Number(body["limit"]) > MAX_ENCRYPTION_RECORD_PAGE + ) { + throw new ApiError("INVALID_REQUEST", 400, "Invalid encryption rotation request"); + } + return { afterCursor: body["afterCursor"], limit: Number(body["limit"]) }; +} + +function matchOwner( + pathname: string, + pattern: RegExp, + key: "approverDid" | "publisherDid", +): Readonly> | null { + const match = pattern.exec(pathname); + if (!match?.[1]) return null; + let did: string; + try { + did = decodeURIComponent(match[1]); + } catch { + return null; + } + return isDid(did) ? { [key]: did } : null; +} + +function decodeRotationCursor( + value: string | null, + cursorPattern: RegExp, +): { afterCursor: string | null; raced: boolean } { + if (value === null || value === RESCAN_CURSOR) return { afterCursor: null, raced: false }; + const raced = value.startsWith(RACED_CURSOR_PREFIX); + const afterCursor = raced ? value.slice(RACED_CURSOR_PREFIX.length) : value; + if (!cursorPattern.test(afterCursor)) { + throw new ApiError("INVALID_REQUEST", 400, "Invalid encryption rotation cursor"); + } + return { afterCursor, raced }; +} + +export function matchPublisherEncryptionRotationPath( + pathname: string, +): Readonly> | null { + return matchOwner(pathname, PUBLISHER_ROTATION_PATH_PATTERN, "publisherDid"); +} + +export function matchApproverEncryptionRotationPath( + pathname: string, +): Readonly> | null { + return matchOwner(pathname, APPROVER_ROTATION_PATH_PATTERN, "approverDid"); +} + +async function rotateEncryptionRecords( + request: Request, + requestId: string, + configuration: ServiceConfiguration, + ownerDid: string, + actor: AccessActor, + shard: EncryptionShard, + cursorPattern: RegExp, +): Promise { + requireIdempotencyKey(request); + const pageInput = await rotationPage(request); + const cursor = decodeRotationCursor(pageInput.afterCursor, cursorPattern); + const page = await shard.list(cursor.afterCursor, pageInput.limit); + let rotated = 0; + let raced = 0; + for (const record of page.items) { + let replacement; + try { + replacement = await configuration.encryption.rotate(record.envelope, record.context); + } catch (error) { + if (error instanceof EncryptionError) { + throw new ApiError( + "ENCRYPTION_OPERATION_FAILED", + 409, + "Retained ciphertext could not be verified", + ); + } + throw error; + } + if (replacement.envelope === record.envelope && replacement.keyVersion === record.keyVersion) { + continue; + } + const replaced = await shard.replace({ + cursor: record.cursor, + expectedEnvelope: record.envelope, + replacementEnvelope: replacement.envelope, + replacementKeyVersion: replacement.keyVersion, + actorIdentity: actor.identity, + }); + if (replaced) rotated += 1; + else raced += 1; + } + const raceSeen = cursor.raced || raced > 0; + const nextCursor = + page.nextCursor === null + ? raceSeen + ? RESCAN_CURSOR + : null + : raceSeen + ? `${RACED_CURSOR_PREFIX}${page.nextCursor}` + : page.nextCursor; + return apiSuccess( + { + ownerDid, + targetKeyVersion: configuration.encryption.currentKeyVersion, + scanned: page.items.length, + rotated, + raced, + nextCursor, + complete: nextCursor === null, + }, + requestId, + ); +} + +function routeFailure(error: unknown, requestId: string): Response { + if (error instanceof ApiError) return apiFailure(error, requestId); + throw error; +} + +export async function handleRotatePublisherEncryption( + request: Request, + requestId: string, + configuration: ServiceConfiguration, + params: Readonly>, + accessActor: AccessActor | null, +): Promise { + try { + const actor = requireActor(accessActor); + const publisherDid = params["publisherDid"]; + if (!publisherDid || !isDid(publisherDid)) { + throw new ApiError("NOT_FOUND", 404, "Publisher not found"); + } + const publisher = env.PUBLISHER_DO.getByName(publisherDid); + return await rotateEncryptionRecords( + request, + requestId, + configuration, + publisherDid, + actor, + { + list: (afterCursor, limit) => + publisher.listEncryptionRecords(publisherDid, afterCursor, limit), + replace: (input) => publisher.replaceEncryptionRecord({ publisherDid, ...input }), + }, + PUBLISHER_CURSOR_PATTERN, + ); + } catch (error) { + return routeFailure(error, requestId); + } +} + +export async function handleRotateApproverEncryption( + request: Request, + requestId: string, + configuration: ServiceConfiguration, + params: Readonly>, + accessActor: AccessActor | null, +): Promise { + try { + const actor = requireActor(accessActor); + const approverDid = params["approverDid"]; + if (!approverDid || !isDid(approverDid)) { + throw new ApiError("NOT_FOUND", 404, "Approver not found"); + } + const approver = env.APPROVER_DO.getByName(approverDid); + return await rotateEncryptionRecords( + request, + requestId, + configuration, + approverDid, + actor, + { + list: (afterCursor, limit) => + approver.listEncryptionRecords(approverDid, afterCursor, limit), + replace: (input) => approver.replaceEncryptionRecord({ approverDid, ...input }), + }, + APPROVER_CURSOR_PATTERN, + ); + } catch (error) { + return routeFailure(error, requestId); + } +} diff --git a/apps/release-service/src/publisher-do/intent-state.ts b/apps/release-service/src/publisher-do/intent-state.ts index 0c0c5e5a49..27cbe5de93 100644 --- a/apps/release-service/src/publisher-do/intent-state.ts +++ b/apps/release-service/src/publisher-do/intent-state.ts @@ -654,6 +654,33 @@ export class IntentStateStore { .map(rowToIntent); } + listExpirable(now: number, limit: number): readonly StoredIntent[] { + if ( + !Number.isSafeInteger(now) || + now < 0 || + !Number.isSafeInteger(limit) || + limit < 1 || + limit > 100 + ) { + throw new IntentStateError(); + } + return this.#storage.sql + .exec( + `SELECT id, package_slug, version, state, state_generation, + workload_policy_version, workload_identity_digest, workload_idempotency_digest, + request_digest, workload_identity_json, release_input_json, state_data_json, + workflow_id, expires_at, created_at, updated_at + FROM intents + WHERE expires_at <= ? + AND state IN ('received', 'verifying', 'verified', 'awaiting_approval', 'ready') + ORDER BY expires_at, id LIMIT ?`, + now, + limit, + ) + .toArray() + .map(rowToIntent); + } + listTransitions(intentId: string): readonly IntentTransition[] { if (!ULID_PATTERN.test(intentId)) throw new IntentStateError(); return this.#storage.sql diff --git a/apps/release-service/src/publisher-do/operations-restore.ts b/apps/release-service/src/publisher-do/operations-restore.ts new file mode 100644 index 0000000000..9aad478432 --- /dev/null +++ b/apps/release-service/src/publisher-do/operations-restore.ts @@ -0,0 +1,505 @@ +const DID_PATTERN = /^did:[a-z0-9]+:[A-Za-z0-9._:%-]+$/; +const PACKAGE_SLUG_PATTERN = /^[A-Za-z][A-Za-z0-9_-]{0,63}$/; +const VERSION_PATTERN = /^[0-9A-Za-z][0-9A-Za-z.-]{0,127}$/; +const ULID_PATTERN = /^[0-9A-HJKMNP-TV-Z]{26}$/; +const DIGEST_PATTERN = /^[A-Za-z0-9_-]{43}$/; +const ARCHIVE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]{15,63}$/; +const ACTOR_IDENTITY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,255}$/; +const MAX_JSON_CHARS = 1024 * 1024; + +export type PublisherRestoreKind = "audit-events" | "intents" | "metadata" | "workload-policies"; + +export interface ApplyPublisherRestorePageInput { + publisherDid: string; + archiveId: string; + page: number; + totalPages: number; + kind: PublisherRestoreKind; + dataJson: string; + pageDigest: string; + actorIdentity: string; + now?: number; +} + +export type ApplyPublisherRestorePageResult = + | { ok: true; replayed: boolean; complete: boolean; nextPage: number } + | { ok: false; code: "RESTORE_CONFLICT" | "RESTORE_NOT_EMPTY" | "RESTORE_OUT_OF_ORDER" }; + +export class OperationsRestoreError extends Error { + constructor() { + super("OPERATIONS_RESTORE_INVALID"); + this.name = "OperationsRestoreError"; + } +} + +interface RestoreStateRow { + [key: string]: string | number | ArrayBuffer | null; + archive_id: string; + total_pages: number; + next_page: number; + last_kind: PublisherRestoreKind; + status: "aborted" | "complete" | "prepared" | "restoring"; + deleted_intents: number; + deleted_workloads: number; +} + +interface RestorePageRow { + [key: string]: string | number | ArrayBuffer | null; + page_digest: string; +} + +const KIND_ORDER: Readonly> = { + metadata: 0, + "workload-policies": 1, + intents: 2, + "audit-events": 3, +}; + +const TERMINAL_STATES = new Set([ + "published", + "invalid", + "rejected", + "cancelled", + "expired", + "failed", + "conflict", +]); + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function stringField(value: Record, key: string): string { + const item = value[key]; + if (typeof item !== "string") throw new OperationsRestoreError(); + return item; +} + +function nullableStringField(value: Record, key: string): string | null { + const item = value[key]; + if (item !== null && typeof item !== "string") throw new OperationsRestoreError(); + return item; +} + +function integerField(value: Record, key: string): number { + const item = value[key]; + if (!Number.isSafeInteger(item)) throw new OperationsRestoreError(); + return Number(item); +} + +function stringArray(value: unknown): string[] { + if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) { + throw new OperationsRestoreError(); + } + return value; +} + +function parseData(value: string): Record { + if (typeof value !== "string" || value.length === 0 || value.length > MAX_JSON_CHARS) { + throw new OperationsRestoreError(); + } + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + throw new OperationsRestoreError(); + } + if (!isRecord(parsed) || JSON.stringify(parsed) !== value) throw new OperationsRestoreError(); + return parsed; +} + +export function initializeOperationsRestoreSchema(storage: DurableObjectStorage): void { + storage.sql.exec(` + CREATE TABLE IF NOT EXISTS operations_restore ( + id INTEGER PRIMARY KEY CHECK (id = 1), + archive_id TEXT NOT NULL, + total_pages INTEGER NOT NULL CHECK (total_pages >= 1), + next_page INTEGER NOT NULL CHECK (next_page >= 0), + last_kind TEXT NOT NULL CHECK ( + last_kind IN ('metadata', 'workload-policies', 'intents', 'audit-events') + ), + status TEXT NOT NULL CHECK (status IN ('prepared', 'restoring', 'complete', 'aborted')), + deleted_intents INTEGER NOT NULL CHECK (deleted_intents >= 0), + deleted_workloads INTEGER NOT NULL CHECK (deleted_workloads >= 0), + actor_identity TEXT NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS operations_restore_pages ( + archive_id TEXT NOT NULL, + page INTEGER NOT NULL CHECK (page >= 0), + page_digest TEXT NOT NULL, + kind TEXT NOT NULL CHECK ( + kind IN ('metadata', 'workload-policies', 'intents', 'audit-events') + ), + applied_at INTEGER NOT NULL, + PRIMARY KEY (archive_id, page) + ); + `); +} + +export class OperationsRestoreStore { + constructor(private readonly storage: DurableObjectStorage) {} + + apply(input: ApplyPublisherRestorePageInput): ApplyPublisherRestorePageResult { + const now = input.now ?? Date.now(); + if ( + !DID_PATTERN.test(input.publisherDid) || + !ARCHIVE_ID_PATTERN.test(input.archiveId) || + !Number.isSafeInteger(input.page) || + input.page < 0 || + !Number.isSafeInteger(input.totalPages) || + input.totalPages < 1 || + input.page >= input.totalPages || + !Object.hasOwn(KIND_ORDER, input.kind) || + !DIGEST_PATTERN.test(input.pageDigest) || + !ACTOR_IDENTITY_PATTERN.test(input.actorIdentity) || + !Number.isSafeInteger(now) || + now < 0 + ) { + throw new OperationsRestoreError(); + } + const data = parseData(input.dataJson); + return this.storage.transactionSync(() => { + const applied = this.storage.sql + .exec( + "SELECT page_digest FROM operations_restore_pages WHERE archive_id = ? AND page = ?", + input.archiveId, + input.page, + ) + .toArray()[0]; + if (applied) { + if (applied.page_digest !== input.pageDigest) { + return { ok: false, code: "RESTORE_CONFLICT" } as const; + } + const state = this.#state(); + return { + ok: true, + replayed: true, + complete: state?.status === "complete", + nextPage: state?.next_page ?? input.page + 1, + } as const; + } + const state = this.#state(); + if (input.page === 0) { + if ( + input.kind !== "metadata" || + !state || + state.archive_id !== input.archiveId || + state.total_pages !== input.totalPages || + state.next_page !== 0 || + state.status !== "prepared" + ) { + return { ok: false, code: "RESTORE_OUT_OF_ORDER" } as const; + } + if (!this.#emptyShard()) { + return { ok: false, code: "RESTORE_NOT_EMPTY" } as const; + } + } else if ( + !state || + state.archive_id !== input.archiveId || + state.total_pages !== input.totalPages || + state.next_page !== input.page || + state.status !== "restoring" || + KIND_ORDER[input.kind] < KIND_ORDER[state.last_kind] + ) { + return { ok: false, code: "RESTORE_OUT_OF_ORDER" } as const; + } + + this.#applyData(input.publisherDid, input.kind, data, input.actorIdentity, now); + const nextPage = input.page + 1; + const complete = nextPage === input.totalPages; + this.storage.sql.exec( + `UPDATE operations_restore SET + next_page = ?, last_kind = ?, status = ?, actor_identity = ?, updated_at = ? + WHERE id = 1 AND archive_id = ? AND total_pages = ?`, + nextPage, + input.kind, + complete ? "complete" : "restoring", + input.actorIdentity, + now, + input.archiveId, + input.totalPages, + ); + this.storage.sql.exec( + `INSERT INTO operations_restore_pages ( + archive_id, page, page_digest, kind, applied_at + ) VALUES (?, ?, ?, ?, ?)`, + input.archiveId, + input.page, + input.pageDigest, + input.kind, + now, + ); + if (complete) { + this.#audit( + "publisher-restore-completed", + input.actorIdentity, + input.archiveId, + now, + "REAUTHORIZATION_REQUIRED", + ); + } + return { ok: true, replayed: false, complete, nextPage } as const; + }); + } + + #state(): RestoreStateRow | null { + return ( + this.storage.sql + .exec( + `SELECT archive_id, total_pages, next_page, last_kind, status, + deleted_intents, deleted_workloads + FROM operations_restore WHERE id = 1`, + ) + .toArray()[0] ?? null + ); + } + + #emptyShard(): boolean { + const counts = this.storage.sql + .exec<{ count: number }>( + `SELECT ( + (SELECT COUNT(*) FROM workload_policies) + + (SELECT COUNT(*) FROM intents) + + (SELECT COUNT(*) FROM delegation) + ) AS count`, + ) + .one(); + return counts.count === 0; + } + + #applyData( + publisherDid: string, + kind: PublisherRestoreKind, + data: Record, + actorIdentity: string, + now: number, + ): void { + if (kind === "metadata") { + this.#restoreMetadata(publisherDid, data, actorIdentity, now); + return; + } + const items = data["items"]; + if (!Array.isArray(items)) throw new OperationsRestoreError(); + if (kind === "workload-policies") { + for (const item of items) this.#restoreWorkload(publisherDid, item, now); + return; + } + if (kind === "intents") { + for (const item of items) this.#restoreIntent(item, now); + return; + } + for (const item of items) { + if (!isRecord(item) || !Number.isSafeInteger(item["sequence"])) { + throw new OperationsRestoreError(); + } + } + } + + #restoreMetadata( + publisherDid: string, + data: Record, + actorIdentity: string, + now: number, + ): void { + const publisher = data["publisher"]; + if (!isRecord(publisher) || stringField(publisher, "did") !== publisherDid) { + throw new OperationsRestoreError(); + } + const createdAt = integerField(publisher, "createdAt"); + this.storage.sql.exec( + `UPDATE publisher SET status = 'suspended', session_epoch = session_epoch + 1, + created_at = ? WHERE id = 1 AND did = ?`, + createdAt, + publisherDid, + ); + this.storage.sql.exec("DELETE FROM publisher_sessions"); + this.storage.sql.exec("DELETE FROM oauth_states"); + this.storage.sql.exec("DELETE FROM delegation"); + const delegation = data["delegation"]; + if (delegation !== null) { + if (!isRecord(delegation)) throw new OperationsRestoreError(); + const originalStatus = stringField(delegation, "status"); + if ( + originalStatus !== "active" && + originalStatus !== "revoked" && + originalStatus !== "reauthorization_required" + ) { + throw new OperationsRestoreError(); + } + this.storage.sql.exec( + `INSERT INTO delegation ( + id, release_nsid, scope, client_key_id, encrypted_session, + encryption_key_version, issuer, pds_url, expires_at, refresh_before, + status, state_version, updated_at + ) VALUES (1, ?, ?, 'restore-required', '', NULL, ?, ?, ?, ?, ?, ?, ?)`, + stringField(delegation, "releaseNsid"), + stringField(delegation, "scope"), + nullableStringField(delegation, "issuer"), + nullableStringField(delegation, "pdsUrl"), + delegation["expiresAt"] === null ? null : integerField(delegation, "expiresAt"), + delegation["refreshBefore"] === null ? null : integerField(delegation, "refreshBefore"), + originalStatus === "revoked" ? "revoked" : "reauthorization_required", + integerField(delegation, "stateVersion") + 1, + now, + ); + } + this.storage.sql.exec( + `UPDATE delegation_operations SET generation = generation + 1, + token_hash = NULL, delegation_version = NULL, expires_at = NULL, updated_at = ? + WHERE kind = 'refresh'`, + now, + ); + this.#audit( + "publisher-restore-started", + actorIdentity, + publisherDid, + now, + "PUBLISHER_SUSPENDED", + ); + } + + #restoreWorkload(publisherDid: string, value: unknown, now: number): void { + if (!isRecord(value)) throw new OperationsRestoreError(); + const packageSlug = stringField(value, "packageSlug"); + const repository = stringField(value, "repository"); + const repositoryId = stringField(value, "repositoryId"); + const repositoryOwnerId = stringField(value, "repositoryOwnerId"); + const workflowRef = stringField(value, "workflowRef"); + if ( + !PACKAGE_SLUG_PATTERN.test(packageSlug) || + repository.length === 0 || + repository.length > 256 || + repositoryId.length === 0 || + repositoryOwnerId.length === 0 || + workflowRef.length === 0 || + workflowRef.length > 1024 + ) { + throw new OperationsRestoreError(); + } + this.storage.sql.exec( + `INSERT INTO workload_policies ( + package_slug, repository, repository_id, repository_owner_id, + workflow_ref, allowed_refs, allowed_environments, active, + state_version, authorized_by, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?)`, + packageSlug, + repository, + repositoryId, + repositoryOwnerId, + workflowRef, + JSON.stringify(stringArray(value["allowedRefs"])), + JSON.stringify(stringArray(value["allowedEnvironments"])), + Math.max(1, integerField(value, "stateVersion")), + publisherDid, + integerField(value, "createdAt"), + now, + ); + } + + #restoreIntent(value: unknown, now: number): void { + if (!isRecord(value)) throw new OperationsRestoreError(); + const intent = value; + const id = stringField(intent, "id"); + const packageSlug = stringField(intent, "packageSlug"); + const version = stringField(intent, "version"); + const state = stringField(intent, "state"); + const stateGeneration = integerField(intent, "stateGeneration"); + const requestDigest = stringField(intent, "requestDigest"); + const workloadIdentityDigest = stringField(intent, "workloadIdentityDigest"); + const workloadIdempotencyDigest = stringField(intent, "workloadIdempotencyDigest"); + const workloadIdentityJson = stringField(intent, "workloadIdentityJson"); + const releaseInputJson = stringField(intent, "releaseInputJson"); + if ( + !ULID_PATTERN.test(id) || + !PACKAGE_SLUG_PATTERN.test(packageSlug) || + !VERSION_PATTERN.test(version) || + !DIGEST_PATTERN.test(requestDigest) || + !DIGEST_PATTERN.test(workloadIdentityDigest) || + !DIGEST_PATTERN.test(workloadIdempotencyDigest) || + workloadIdentityJson.length > 64 * 1024 || + releaseInputJson.length > 128 * 1024 || + stateGeneration < 1 + ) { + throw new OperationsRestoreError(); + } + const restoredState = TERMINAL_STATES.has(state) ? state : "failed"; + const restoredGeneration = TERMINAL_STATES.has(state) ? stateGeneration : stateGeneration + 1; + const stateDataJson = TERMINAL_STATES.has(state) + ? stringField(intent, "stateDataJson") + : '{"reasonCode":"SHARD_RESTORED_REVIEW_REQUIRED"}'; + this.storage.sql.exec( + `INSERT INTO intents ( + id, package_slug, version, state, state_generation, + workload_policy_version, workload_identity_digest, workload_idempotency_digest, + request_digest, workload_identity_json, release_input_json, state_data_json, + workflow_id, expires_at, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?)`, + id, + packageSlug, + version, + restoredState, + restoredGeneration, + Math.max(1, integerField(intent, "workloadPolicyVersion")), + workloadIdentityDigest, + workloadIdempotencyDigest, + requestDigest, + workloadIdentityJson, + releaseInputJson, + stateDataJson, + integerField(intent, "expiresAt"), + integerField(intent, "createdAt"), + now, + ); + this.storage.sql.exec( + `INSERT INTO release_reservations (package_slug, version, intent_id, created_at) + VALUES (?, ?, ?, ?)`, + packageSlug, + version, + id, + integerField(intent, "createdAt"), + ); + this.storage.sql.exec( + `INSERT INTO intent_transitions ( + intent_id, sequence, from_state, to_state, state_generation, + transition_digest, actor_realm, actor_identity, reason_code, + state_data_json, created_at + ) VALUES (?, 1, NULL, ?, ?, ?, 'system', 'release-service', ?, ?, ?)`, + id, + restoredState, + restoredGeneration, + requestDigest, + TERMINAL_STATES.has(state) ? "SHARD_RESTORED" : "SHARD_RESTORED_REVIEW_REQUIRED", + stateDataJson, + now, + ); + this.#audit( + "intent-restored", + "release-service", + id, + now, + TERMINAL_STATES.has(state) ? "SHARD_RESTORED" : "SHARD_RESTORED_REVIEW_REQUIRED", + ); + } + + #audit( + eventType: string, + actorIdentity: string, + subject: string, + createdAt: number, + reasonCode: string, + ): void { + this.storage.sql.exec( + `INSERT INTO audit_events ( + event_type, actor_realm, actor_identity, subject, + reason_code, public_payload, created_at + ) VALUES (?, ?, ?, ?, ?, '{}', ?)`, + eventType, + actorIdentity === "release-service" ? "system" : "access", + actorIdentity, + subject, + reasonCode, + createdAt, + ); + } +} diff --git a/apps/release-service/src/publisher-do/publisher-do.ts b/apps/release-service/src/publisher-do/publisher-do.ts index 024df4ffad..83d8193e36 100644 --- a/apps/release-service/src/publisher-do/publisher-do.ts +++ b/apps/release-service/src/publisher-do/publisher-do.ts @@ -1,5 +1,10 @@ import { DurableObject } from "cloudflare:workers"; +import type { + EncryptionRecordPage, + EncryptionRecordReplacement, +} from "../operations/encryption-records.js"; +import { MAX_ENCRYPTION_RECORD_PAGE } from "../operations/encryption-records.js"; import { initializeIntentStateSchema, IntentStateStore, @@ -11,6 +16,12 @@ import { type TransitionIntentInput, type TransitionIntentResult, } from "./intent-state.js"; +import { + initializeOperationsRestoreSchema, + OperationsRestoreStore, + type ApplyPublisherRestorePageInput, + type ApplyPublisherRestorePageResult, +} from "./operations-restore.js"; import { initializePublicationMaterializationSchema, PublicationMaterializationStore, @@ -29,6 +40,12 @@ import { type CompletePublicationOperationInput, type CompletePublicationOperationResult, } from "./publication-operation.js"; +import { + initializeIntentRateLimitSchema, + IntentRateLimitStore, + type ConsumeIntentRateLimitInput, + type ConsumeIntentRateLimitResult, +} from "./rate-limit.js"; import { initializeVerificationStepSchema, VerificationStepStore, @@ -86,6 +103,12 @@ export type { StoredVerificationStep, VerificationStepName, } from "./verification-step.js"; +export type { + ApplyPublisherRestorePageInput, + ApplyPublisherRestorePageResult, + PublisherRestoreKind, +} from "./operations-restore.js"; +export type { ConsumeIntentRateLimitInput, ConsumeIntentRateLimitResult } from "./rate-limit.js"; const DID_PATTERN = /^did:[a-z][a-z0-9]*:[A-Za-z0-9._:%-]+$/; const HASH_PATTERN = /^[A-Za-z0-9_-]{32,128}$/; @@ -94,8 +117,16 @@ const ACTOR_IDENTITY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,255}$/; const MAX_CIPHERTEXT_CHARS = 1_500_000; const MAX_REFRESH_LEASE_MS = 5 * 60_000; const MAX_PUBLISHER_SESSION_MS = 24 * 60 * 60_000; +const MAINTENANCE_BATCH_SIZE = 100; const REFRESH_TOKEN_BYTES = 32; const BASE64_PADDING_PATTERN = /=+$/; +const ENCRYPTION_CURSOR_PATTERN = /^(?:delegation:1|oauth-state:[A-Za-z0-9_-]{32,128})$/; +const ARCHIVE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]{15,63}$/; +const MAX_RESTORE_PAGES = 1_000_000; + +export type PublisherOAuthEncryptionPurpose = + | "oauth-console-transaction" + | "oauth-delegation-transaction"; export type PublisherStateErrorCode = | "PUBLISHER_DID_INVALID" @@ -105,7 +136,10 @@ export type PublisherStateErrorCode = | "DELEGATION_INVALID" | "DELEGATION_CAS_REQUIRED" | "DELEGATION_UNAVAILABLE" - | "PUBLISHER_SESSION_INVALID"; + | "ENCRYPTION_OPERATION_INVALID" + | "OPERATIONS_EXPORT_INVALID" + | "PUBLISHER_SESSION_INVALID" + | "PUBLISHER_STATE_CORRUPT"; export class PublisherStateError extends Error { readonly code: PublisherStateErrorCode; @@ -122,6 +156,7 @@ export interface PutOAuthStateInput { stateHash: string; encryptedState: string; encryptionKeyVersion: number; + encryptionPurpose: PublisherOAuthEncryptionPurpose; clientKeyId: string; redirectTarget: string; expiresAt: number; @@ -165,6 +200,37 @@ export interface StoredDelegation { stateVersion: number; } +export interface PublisherOperationsMetadata { + publisher: { + did: string; + status: "active" | "suspended"; + createdAt: number; + }; + delegation: Omit< + StoredDelegation, + "clientKeyId" | "encryptedSession" | "encryptionKeyVersion" + > | null; +} + +export interface PublisherAuditEvent { + sequence: number; + eventType: string; + actorRealm: "access" | "approver" | "oidc" | "publisher" | "system"; + actorIdentity: string; + subject: string; + reasonCode: string | null; + publicPayloadJson: string; + createdAt: number; +} + +export type PreparePublisherRestoreResult = + | { ok: true; deletedIntents: number; deletedWorkloads: number; replayed: boolean } + | { ok: false; code: "PUBLISHER_NOT_SUSPENDED" | "RESTORE_CONFLICT" }; + +export type AbortPublisherRestoreResult = + | { ok: true; replayed: boolean } + | { ok: false; code: "PUBLISHER_NOT_SUSPENDED" | "RESTORE_CONFLICT" }; + export type PutDelegationResult = | { ok: true; delegation: StoredDelegation } | { ok: false; code: "DELEGATION_CAS_REQUIRED" }; @@ -225,6 +291,13 @@ interface PublisherSessionOwnerRow { session_epoch: number; } +interface PublisherOperationsMetadataRow { + [key: string]: string | number | ArrayBuffer | null; + did: string; + status: "active" | "suspended"; + created_at: number; +} + interface PublisherSessionRow { [key: string]: string | number | ArrayBuffer | null; token_hash: string; @@ -290,6 +363,26 @@ interface OperationRow { expires_at: number | null; } +interface EncryptionRecordRow { + [key: string]: string | number | ArrayBuffer | null; + cursor: string; + envelope: string; + key_version: number; + purpose: "oauth-session" | PublisherOAuthEncryptionPurpose; +} + +interface AuditRow { + [key: string]: string | number | ArrayBuffer | null; + sequence: number; + event_type: string; + actor_realm: PublisherAuditEvent["actorRealm"]; + actor_identity: string; + subject: string; + reason_code: string | null; + public_payload: string; + created_at: number; +} + function validBoundedString(value: unknown, maxLength: number): value is string { return typeof value === "string" && value.length > 0 && value.length <= maxLength; } @@ -308,6 +401,12 @@ function validPositiveInteger(value: unknown): value is number { return Number.isSafeInteger(value) && Number(value) >= 1; } +function validPublisherOAuthEncryptionPurpose( + value: unknown, +): value is PublisherOAuthEncryptionPurpose { + return value === "oauth-console-transaction" || value === "oauth-delegation-transaction"; +} + function validOptionalTimestamp(value: unknown): value is number | null { return value === null || Number.isSafeInteger(value); } @@ -341,6 +440,14 @@ async function hashRefreshToken(token: string): Promise { return encodeBase64Url(new Uint8Array(digest)); } +async function expirationDigest(intentId: string, expiresAt: number): Promise { + const digest = await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(JSON.stringify(["intent-expired", intentId, expiresAt])), + ); + return encodeBase64Url(new Uint8Array(digest)); +} + export class PublisherDurableObject extends DurableObject { readonly #objectName: string | undefined; readonly #workloadPolicies: WorkloadPolicyStore; @@ -348,6 +455,8 @@ export class PublisherDurableObject extends DurableObject { readonly #publicationMaterializations: PublicationMaterializationStore; readonly #publicationOperations: PublicationOperationStore; readonly #verificationSteps: VerificationStepStore; + readonly #operationsRestore: OperationsRestoreStore; + readonly #intentRateLimits: IntentRateLimitStore; constructor(ctx: DurableObjectState, env: Env) { super(ctx, env); @@ -357,6 +466,8 @@ export class PublisherDurableObject extends DurableObject { this.#publicationMaterializations = new PublicationMaterializationStore(ctx.storage); this.#publicationOperations = new PublicationOperationStore(ctx.storage); this.#verificationSteps = new VerificationStepStore(ctx.storage); + this.#operationsRestore = new OperationsRestoreStore(ctx.storage); + this.#intentRateLimits = new IntentRateLimitStore(ctx.storage); void ctx.blockConcurrencyWhile(() => { this.#initializeSchema(); return Promise.resolve(); @@ -375,7 +486,10 @@ export class PublisherDurableObject extends DurableObject { CREATE TABLE IF NOT EXISTS oauth_states ( state_hash TEXT PRIMARY KEY, encrypted_state TEXT NOT NULL, - encryption_key_version INTEGER, + encryption_key_version INTEGER NOT NULL CHECK (encryption_key_version >= 1), + encryption_purpose TEXT NOT NULL CHECK ( + encryption_purpose IN ('oauth-console-transaction', 'oauth-delegation-transaction') + ), client_key_id TEXT NOT NULL, redirect_target TEXT NOT NULL, expires_at INTEGER NOT NULL, @@ -438,6 +552,8 @@ export class PublisherDurableObject extends DurableObject { initializePublicationMaterializationSchema(this.ctx.storage); initializePublicationOperationSchema(this.ctx.storage); initializeVerificationStepSchema(this.ctx.storage); + initializeOperationsRestoreSchema(this.ctx.storage); + initializeIntentRateLimitSchema(this.ctx.storage); } #assertPublisherObjectName(publisherDid: string): void { @@ -555,7 +671,16 @@ export class PublisherDurableObject extends DurableObject { async createIntent(input: CreateIntentInput): Promise { this.#assertPublisherDid(input.publisherDid); const result = this.#intents.create(input); - await this.#scheduleNextAlarm(input.now ?? Date.now()); + if (result.ok) await this.#scheduleNextAlarm(input.now ?? Date.now()); + return result; + } + + async consumeIntentRateLimit( + input: ConsumeIntentRateLimitInput, + ): Promise { + this.#assertPublisherDid(input.publisherDid); + const result = this.#intentRateLimits.consume(input); + if (result.ok) await this.#scheduleNextAlarm(input.now ?? Date.now()); return result; } @@ -852,6 +977,7 @@ export class PublisherDurableObject extends DurableObject { !HASH_PATTERN.test(input.stateHash) || !validBoundedString(input.encryptedState, MAX_CIPHERTEXT_CHARS) || !validPositiveInteger(input.encryptionKeyVersion) || + !validPublisherOAuthEncryptionPurpose(input.encryptionPurpose) || !validBoundedString(input.clientKeyId, 128) || !validRelativeRedirectPath(input.redirectTarget) || !Number.isSafeInteger(input.expiresAt) || @@ -869,12 +995,13 @@ export class PublisherDurableObject extends DurableObject { if (existing) return { ok: false, code: "OAUTH_STATE_EXISTS" } as const; this.ctx.storage.sql.exec( `INSERT INTO oauth_states ( - state_hash, encrypted_state, encryption_key_version, client_key_id, + state_hash, encrypted_state, encryption_key_version, encryption_purpose, client_key_id, redirect_target, expires_at, created_at - ) VALUES (?, ?, ?, ?, ?, ?, ?)`, + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, input.stateHash, input.encryptedState, input.encryptionKeyVersion, + input.encryptionPurpose, input.clientKeyId, input.redirectTarget, input.expiresAt, @@ -1009,6 +1136,364 @@ export class PublisherDurableObject extends DurableObject { return this.#readDelegation(); } + getOperationsMetadata(publisherDid: string): PublisherOperationsMetadata { + this.#assertPublisherDid(publisherDid); + const publisher = this.ctx.storage.sql + .exec( + "SELECT did, status, created_at FROM publisher WHERE id = 1", + ) + .one(); + const delegation = this.#readDelegation(); + return { + publisher: { + did: publisher.did, + status: publisher.status, + createdAt: publisher.created_at, + }, + delegation: delegation + ? { + releaseNsid: delegation.releaseNsid, + scope: delegation.scope, + issuer: delegation.issuer, + pdsUrl: delegation.pdsUrl, + expiresAt: delegation.expiresAt, + refreshBefore: delegation.refreshBefore, + status: delegation.status, + stateVersion: delegation.stateVersion, + } + : null, + }; + } + + applyOperationsRestorePage( + input: ApplyPublisherRestorePageInput, + ): ApplyPublisherRestorePageResult { + this.#assertPublisherDid(input.publisherDid); + return this.#operationsRestore.apply(input); + } + + prepareOperationsRestore( + publisherDid: string, + archiveId: string, + totalPages: number, + actorIdentity: string, + now = Date.now(), + ): PreparePublisherRestoreResult { + this.#assertPublisherDid(publisherDid); + if ( + !ARCHIVE_ID_PATTERN.test(archiveId) || + !Number.isSafeInteger(totalPages) || + totalPages < 1 || + totalPages > MAX_RESTORE_PAGES || + !ACTOR_IDENTITY_PATTERN.test(actorIdentity) || + !Number.isSafeInteger(now) || + now < 0 + ) { + throw new PublisherStateError("OPERATIONS_EXPORT_INVALID"); + } + return this.ctx.storage.transactionSync(() => { + const existing = this.ctx.storage.sql + .exec<{ + archive_id: string; + total_pages: number; + status: "aborted" | "complete" | "prepared" | "restoring"; + deleted_intents: number; + deleted_workloads: number; + }>( + `SELECT archive_id, total_pages, status, deleted_intents, deleted_workloads + FROM operations_restore WHERE id = 1`, + ) + .toArray()[0]; + if ( + existing?.archive_id === archiveId && + existing.total_pages === totalPages && + existing.status !== "aborted" + ) { + return { + ok: true, + deletedIntents: existing.deleted_intents, + deletedWorkloads: existing.deleted_workloads, + replayed: true, + } as const; + } + if (existing && existing.status !== "aborted" && existing.status !== "complete") { + return { ok: false, code: "RESTORE_CONFLICT" } as const; + } + const publisher = this.ctx.storage.sql + .exec<{ status: string }>("SELECT status FROM publisher WHERE id = 1") + .one(); + if (publisher.status !== "suspended") { + return { ok: false, code: "PUBLISHER_NOT_SUSPENDED" } as const; + } + const deletedIntents = this.ctx.storage.sql + .exec<{ count: number }>("SELECT COUNT(*) AS count FROM intents") + .one().count; + const deletedWorkloads = this.ctx.storage.sql + .exec<{ count: number }>("SELECT COUNT(*) AS count FROM workload_policies") + .one().count; + this.ctx.storage.sql.exec("DELETE FROM intent_verification_steps"); + this.ctx.storage.sql.exec("DELETE FROM intent_transitions"); + this.ctx.storage.sql.exec("DELETE FROM release_reservations"); + this.ctx.storage.sql.exec("DELETE FROM intent_idempotency"); + this.ctx.storage.sql.exec("DELETE FROM publication_operations"); + this.ctx.storage.sql.exec("DELETE FROM deadlines"); + this.ctx.storage.sql.exec("DELETE FROM intents"); + this.ctx.storage.sql.exec("DELETE FROM workload_policies"); + this.ctx.storage.sql.exec("DELETE FROM publisher_sessions"); + this.ctx.storage.sql.exec("DELETE FROM oauth_states"); + this.ctx.storage.sql.exec("DELETE FROM delegation"); + this.ctx.storage.sql.exec("DELETE FROM intent_rate_windows"); + this.ctx.storage.sql.exec("DELETE FROM intent_rate_idempotency"); + this.ctx.storage.sql.exec("DELETE FROM operations_restore_pages"); + this.ctx.storage.sql.exec("DELETE FROM operations_restore"); + this.ctx.storage.sql.exec("DELETE FROM audit_events"); + this.ctx.storage.sql.exec( + `UPDATE delegation_operations SET generation = generation + 1, + token_hash = NULL, delegation_version = NULL, expires_at = NULL, updated_at = ? + WHERE kind = 'refresh'`, + now, + ); + this.ctx.storage.sql.exec( + "UPDATE publisher SET session_epoch = session_epoch + 1 WHERE id = 1", + ); + this.ctx.storage.sql.exec( + `INSERT INTO operations_restore ( + id, archive_id, total_pages, next_page, last_kind, status, + deleted_intents, deleted_workloads, actor_identity, updated_at + ) VALUES (1, ?, ?, 0, 'metadata', 'prepared', ?, ?, ?, ?)`, + archiveId, + totalPages, + deletedIntents, + deletedWorkloads, + actorIdentity, + now, + ); + this.#appendAudit( + "publisher-restore-prepared", + "access", + actorIdentity, + archiveId, + now, + "PUBLISHER_SUSPENDED", + ); + return { ok: true, deletedIntents, deletedWorkloads, replayed: false } as const; + }); + } + + abortOperationsRestore( + publisherDid: string, + archiveId: string, + actorIdentity: string, + now = Date.now(), + ): AbortPublisherRestoreResult { + this.#assertPublisherDid(publisherDid); + if ( + !ARCHIVE_ID_PATTERN.test(archiveId) || + !ACTOR_IDENTITY_PATTERN.test(actorIdentity) || + !Number.isSafeInteger(now) || + now < 0 + ) { + throw new PublisherStateError("OPERATIONS_EXPORT_INVALID"); + } + return this.ctx.storage.transactionSync(() => { + const publisher = this.ctx.storage.sql + .exec<{ status: string }>("SELECT status FROM publisher WHERE id = 1") + .one(); + if (publisher.status !== "suspended") { + return { ok: false, code: "PUBLISHER_NOT_SUSPENDED" } as const; + } + const existing = this.ctx.storage.sql + .exec<{ + archive_id: string; + status: "aborted" | "complete" | "prepared" | "restoring"; + }>("SELECT archive_id, status FROM operations_restore WHERE id = 1") + .toArray()[0]; + if (existing?.archive_id === archiveId && existing.status === "aborted") { + return { ok: true, replayed: true } as const; + } + if (!existing || existing.archive_id !== archiveId || existing.status === "complete") { + return { ok: false, code: "RESTORE_CONFLICT" } as const; + } + this.ctx.storage.sql.exec( + `UPDATE operations_restore SET status = 'aborted', actor_identity = ?, updated_at = ? + WHERE id = 1 AND archive_id = ?`, + actorIdentity, + now, + archiveId, + ); + this.#appendAudit( + "publisher-restore-aborted", + "access", + actorIdentity, + archiveId, + now, + "RESTORE_ABORTED", + ); + return { ok: true, replayed: false } as const; + }); + } + + listAuditEvents( + publisherDid: string, + afterSequence: number, + limit: number, + ): readonly PublisherAuditEvent[] { + this.#assertPublisherDid(publisherDid); + if ( + !Number.isSafeInteger(afterSequence) || + afterSequence < 0 || + !Number.isSafeInteger(limit) || + limit < 1 || + limit > 100 + ) { + throw new PublisherStateError("OPERATIONS_EXPORT_INVALID"); + } + return this.ctx.storage.sql + .exec( + `SELECT sequence, event_type, actor_realm, actor_identity, + subject, reason_code, public_payload, created_at + FROM audit_events WHERE sequence > ? ORDER BY sequence LIMIT ?`, + afterSequence, + limit, + ) + .toArray() + .map((row) => { + let payload: unknown; + try { + payload = JSON.parse(row.public_payload); + } catch { + throw new PublisherStateError("PUBLISHER_STATE_CORRUPT"); + } + if ( + payload === null || + typeof payload !== "object" || + Array.isArray(payload) || + JSON.stringify(payload) !== row.public_payload + ) { + throw new PublisherStateError("PUBLISHER_STATE_CORRUPT"); + } + return { + sequence: row.sequence, + eventType: row.event_type, + actorRealm: row.actor_realm, + actorIdentity: row.actor_identity, + subject: row.subject, + reasonCode: row.reason_code, + publicPayloadJson: row.public_payload, + createdAt: row.created_at, + }; + }); + } + + listEncryptionRecords( + publisherDid: string, + afterCursor: string | null, + limit: number, + now = Date.now(), + ): EncryptionRecordPage { + this.#assertPublisherDid(publisherDid); + if ( + (afterCursor !== null && !ENCRYPTION_CURSOR_PATTERN.test(afterCursor)) || + !Number.isSafeInteger(limit) || + limit < 1 || + limit > MAX_ENCRYPTION_RECORD_PAGE || + !Number.isSafeInteger(now) || + now < 0 + ) { + throw new PublisherStateError("ENCRYPTION_OPERATION_INVALID"); + } + const rows = this.ctx.storage.sql + .exec( + `SELECT cursor, envelope, key_version, purpose FROM ( + SELECT 'delegation:1' AS cursor, encrypted_session AS envelope, + encryption_key_version AS key_version, 'oauth-session' AS purpose + FROM delegation + WHERE status != 'revoked' AND encrypted_session != '' + AND encryption_key_version IS NOT NULL + UNION ALL + SELECT 'oauth-state:' || state_hash AS cursor, encrypted_state AS envelope, + encryption_key_version AS key_version, encryption_purpose AS purpose + FROM oauth_states + WHERE expires_at > ? AND encrypted_state != '' + ) WHERE cursor > ? ORDER BY cursor LIMIT ?`, + now, + afterCursor ?? "", + limit + 1, + ) + .toArray(); + const hasMore = rows.length > limit; + const visible = hasMore ? rows.slice(0, limit) : rows; + const items = visible.map((row) => { + if (row.purpose !== "oauth-session" && !validPublisherOAuthEncryptionPurpose(row.purpose)) { + throw new PublisherStateError("ENCRYPTION_OPERATION_INVALID"); + } + return { + cursor: row.cursor, + envelope: row.envelope, + keyVersion: row.key_version, + context: + row.cursor === "delegation:1" + ? { + purpose: "oauth-session" as const, + objectClass: "PublisherDurableObject", + table: "delegation", + primaryKey: "1", + ownerDid: publisherDid, + } + : { + purpose: row.purpose, + objectClass: "PublisherDurableObject", + table: "oauth_states", + primaryKey: row.cursor.slice("oauth-state:".length), + ownerDid: publisherDid, + }, + }; + }); + return { + items, + nextCursor: hasMore ? (items.at(-1)?.cursor ?? null) : null, + }; + } + + replaceEncryptionRecord(input: EncryptionRecordReplacement & { publisherDid: string }): boolean { + this.#assertPublisherDid(input.publisherDid); + const now = input.now ?? Date.now(); + if ( + !ENCRYPTION_CURSOR_PATTERN.test(input.cursor) || + !validBoundedString(input.expectedEnvelope, MAX_CIPHERTEXT_CHARS) || + !validBoundedString(input.replacementEnvelope, MAX_CIPHERTEXT_CHARS) || + !validPositiveInteger(input.replacementKeyVersion) || + !ACTOR_IDENTITY_PATTERN.test(input.actorIdentity) || + !Number.isSafeInteger(now) || + now < 0 + ) { + throw new PublisherStateError("ENCRYPTION_OPERATION_INVALID"); + } + return this.ctx.storage.transactionSync(() => { + const result = + input.cursor === "delegation:1" + ? this.ctx.storage.sql.exec( + `UPDATE delegation SET encrypted_session = ?, encryption_key_version = ? + WHERE id = 1 AND status != 'revoked' AND encrypted_session = ?`, + input.replacementEnvelope, + input.replacementKeyVersion, + input.expectedEnvelope, + ) + : this.ctx.storage.sql.exec( + `UPDATE oauth_states SET encrypted_state = ?, encryption_key_version = ? + WHERE state_hash = ? AND encrypted_state = ? AND expires_at > ?`, + input.replacementEnvelope, + input.replacementKeyVersion, + input.cursor.slice("oauth-state:".length), + input.expectedEnvelope, + now, + ); + if (result.rowsWritten !== 1) return false; + this.#appendAudit("encryption-rotated", "access", input.actorIdentity, input.cursor, now); + return true; + }); + } + async beginDelegationRefresh( publisherDid: string, leaseDurationMs: number, @@ -1316,22 +1801,31 @@ export class PublisherDurableObject extends DurableObject { } async #scheduleNextAlarm(now: number): Promise { - const stateDeadline = this.ctx.storage.sql - .exec<{ deadline: number | null }>( - `SELECT MIN(expires_at) AS deadline FROM ( - SELECT expires_at FROM oauth_states - UNION ALL SELECT expires_at FROM publisher_sessions - UNION ALL SELECT expires_at FROM intent_idempotency - )`, + const candidates = this.ctx.storage.sql + .exec<{ + operation_deadline: number | null; + oauth_expiry: number | null; + session_expiry: number | null; + idempotency_expiry: number | null; + rate_expiry: number | null; + intent_expiry: number | null; + }>( + `SELECT + (SELECT MIN(scheduled_at) FROM deadlines) AS operation_deadline, + (SELECT MIN(expires_at) FROM oauth_states) AS oauth_expiry, + (SELECT MIN(expires_at) FROM publisher_sessions) AS session_expiry, + (SELECT MIN(expires_at) FROM intent_idempotency) AS idempotency_expiry, + (SELECT MIN(expires_at) FROM intent_rate_idempotency) AS rate_expiry, + (SELECT MIN(expires_at) FROM intents + WHERE state IN ( + 'received', 'verifying', 'verified', 'awaiting_approval', 'ready' + )) AS intent_expiry`, ) - .one().deadline; - const operationDeadline = this.#publicationOperations.nextDeadline(); - const deadline = - stateDeadline === null - ? operationDeadline - : operationDeadline === null - ? stateDeadline - : Math.min(stateDeadline, operationDeadline); + .one(); + const deadlines = Object.values(candidates).filter( + (value): value is number => typeof value === "number", + ); + const deadline = deadlines.length === 0 ? null : Math.min(...deadlines); if (deadline === null) { await this.ctx.storage.deleteAlarm(); return; @@ -1342,10 +1836,29 @@ export class PublisherDurableObject extends DurableObject { override async alarm(): Promise { const now = Date.now(); this.#publicationOperations.recoverExpired(now); + const publisherDid = this.#objectName; + if (publisherDid !== undefined) { + for (const intent of this.#intents.listExpirable(now, MAINTENANCE_BATCH_SIZE)) { + this.#intents.transition({ + publisherDid, + intentId: intent.id, + expectedState: intent.state, + expectedGeneration: intent.stateGeneration, + toState: "expired", + transitionDigest: await expirationDigest(intent.id, intent.expiresAt), + actorRealm: "system", + actorIdentity: "release-service", + reasonCode: "INTENT_EXPIRED", + stateDataJson: '{"reasonCode":"INTENT_EXPIRED"}', + now, + }); + } + } this.ctx.storage.transactionSync(() => { this.ctx.storage.sql.exec("DELETE FROM oauth_states WHERE expires_at <= ?", now); this.ctx.storage.sql.exec("DELETE FROM publisher_sessions WHERE expires_at <= ?", now); this.ctx.storage.sql.exec("DELETE FROM intent_idempotency WHERE expires_at <= ?", now); + this.ctx.storage.sql.exec("DELETE FROM intent_rate_idempotency WHERE expires_at <= ?", now); }); await this.#scheduleNextAlarm(now); } diff --git a/apps/release-service/src/publisher-do/rate-limit.ts b/apps/release-service/src/publisher-do/rate-limit.ts new file mode 100644 index 0000000000..4fe37439bd --- /dev/null +++ b/apps/release-service/src/publisher-do/rate-limit.ts @@ -0,0 +1,157 @@ +const DID_PATTERN = /^did:[a-z0-9]+:[A-Za-z0-9._:%-]+$/; +const DECIMAL_ID_PATTERN = /^[1-9][0-9]*$/; +const DIGEST_PATTERN = /^[A-Za-z0-9_-]{43}$/; +const IDEMPOTENCY_KEY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$/; +const WINDOW_MS = 60_000; +const MAX_IDEMPOTENCY_MS = 24 * 60 * 60_000; +const LIMITS = { + publisher: 120, + repository: 60, + workload: 30, +} as const; + +type RateLimitScope = keyof typeof LIMITS; + +export interface ConsumeIntentRateLimitInput { + publisherDid: string; + repositoryId: string; + workloadKey: string; + idempotencyKey: string; + expiresAt: number; + now?: number; +} + +export type ConsumeIntentRateLimitResult = + | { ok: true; replayed: boolean; retryAt: number } + | { ok: false; code: "RATE_LIMITED"; scope: RateLimitScope; retryAt: number }; + +export class IntentRateLimitError extends Error { + constructor() { + super("INTENT_RATE_LIMIT_INVALID"); + this.name = "IntentRateLimitError"; + } +} + +interface RateWindowRow { + [key: string]: string | number | ArrayBuffer | null; + window_start: number; + count: number; +} + +interface IdempotencyRow { + [key: string]: string | number | ArrayBuffer | null; + expires_at: number; +} + +export function initializeIntentRateLimitSchema(storage: DurableObjectStorage): void { + storage.sql.exec(` + CREATE TABLE IF NOT EXISTS intent_rate_windows ( + scope TEXT NOT NULL CHECK (scope IN ('publisher', 'repository', 'workload')), + subject_key TEXT NOT NULL, + window_start INTEGER NOT NULL, + count INTEGER NOT NULL CHECK (count >= 1), + updated_at INTEGER NOT NULL, + PRIMARY KEY (scope, subject_key) + ); + CREATE TABLE IF NOT EXISTS intent_rate_idempotency ( + workload_key TEXT NOT NULL, + mutation_key TEXT NOT NULL, + expires_at INTEGER NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (workload_key, mutation_key) + ); + CREATE INDEX IF NOT EXISTS idx_intent_rate_idempotency_expiry + ON intent_rate_idempotency(expires_at); + `); +} + +export class IntentRateLimitStore { + constructor(private readonly storage: DurableObjectStorage) {} + + consume(input: ConsumeIntentRateLimitInput): ConsumeIntentRateLimitResult { + const now = input.now ?? Date.now(); + if ( + !DID_PATTERN.test(input.publisherDid) || + !DECIMAL_ID_PATTERN.test(input.repositoryId) || + !DIGEST_PATTERN.test(input.workloadKey) || + !IDEMPOTENCY_KEY_PATTERN.test(input.idempotencyKey) || + !Number.isSafeInteger(now) || + now < 0 || + !Number.isSafeInteger(input.expiresAt) || + input.expiresAt <= now || + input.expiresAt - now > MAX_IDEMPOTENCY_MS + ) { + throw new IntentRateLimitError(); + } + return this.storage.transactionSync(() => { + const idempotency = this.storage.sql + .exec( + `SELECT expires_at FROM intent_rate_idempotency + WHERE workload_key = ? AND mutation_key = ?`, + input.workloadKey, + input.idempotencyKey, + ) + .toArray()[0]; + const windowStart = Math.floor(now / WINDOW_MS) * WINDOW_MS; + const retryAt = windowStart + WINDOW_MS; + if (idempotency && idempotency.expires_at > now) { + return { ok: true, replayed: true, retryAt } as const; + } + if (idempotency) { + this.storage.sql.exec( + `DELETE FROM intent_rate_idempotency + WHERE workload_key = ? AND mutation_key = ?`, + input.workloadKey, + input.idempotencyKey, + ); + } + const subjects: ReadonlyArray = [ + ["workload", input.workloadKey], + ["repository", input.repositoryId], + ["publisher", input.publisherDid], + ]; + for (const [scope, subject] of subjects) { + const current = this.storage.sql + .exec( + `SELECT window_start, count FROM intent_rate_windows + WHERE scope = ? AND subject_key = ?`, + scope, + subject, + ) + .toArray()[0]; + const count = current?.window_start === windowStart ? current.count : 0; + if (count >= LIMITS[scope]) { + return { ok: false, code: "RATE_LIMITED", scope, retryAt } as const; + } + } + for (const [scope, subject] of subjects) { + this.storage.sql.exec( + `INSERT INTO intent_rate_windows ( + scope, subject_key, window_start, count, updated_at + ) VALUES (?, ?, ?, 1, ?) + ON CONFLICT(scope, subject_key) DO UPDATE SET + window_start = excluded.window_start, + count = CASE + WHEN intent_rate_windows.window_start = excluded.window_start + THEN intent_rate_windows.count + 1 ELSE 1 END, + updated_at = excluded.updated_at`, + scope, + subject, + windowStart, + now, + ); + } + this.storage.sql.exec( + `INSERT INTO intent_rate_idempotency ( + workload_key, mutation_key, expires_at, created_at + ) VALUES (?, ?, ?, ?)`, + input.workloadKey, + input.idempotencyKey, + input.expiresAt, + now, + ); + this.storage.sql.exec("DELETE FROM intent_rate_idempotency WHERE expires_at <= ?", now); + return { ok: true, replayed: false, retryAt } as const; + }); + } +} diff --git a/apps/release-service/src/publishing/workflow.ts b/apps/release-service/src/publishing/workflow.ts index c6420e37e7..bea70ee8b6 100644 --- a/apps/release-service/src/publishing/workflow.ts +++ b/apps/release-service/src/publishing/workflow.ts @@ -16,6 +16,7 @@ import { type ServiceControlDurableObject, } from "../control-do/service-control-do.js"; import { createPublisherOAuthClient, OAuthCustodyError } from "../oauth/custody.js"; +import { writeOperationsMetric } from "../observability/metrics.js"; import type { IntentState, PublicationArtifactSlot, @@ -909,7 +910,26 @@ export async function publishVerifiedIntent( const errorCode = writeStarted ? "PUBLICATION_AMBIGUOUS" : publicationErrorCode(error, "PUBLICATION_PRECONDITION_FAILED"); + if (error instanceof OAuthCustodyError) { + writeOperationsMetric( + { + event: "refresh_failure", + outcome: error.code, + scope: "publisher", + }, + env.OPERATIONS_METRICS, + ); + } if (!writeStarted) return failBeforeWrite(errorCode); + writeOperationsMetric( + { + event: "reconciliation_required", + outcome: errorCode, + scope: "publication", + value: attempt, + }, + env.OPERATIONS_METRICS, + ); console.error( JSON.stringify({ event: "publication_attempt_ambiguous", diff --git a/apps/release-service/src/routes.ts b/apps/release-service/src/routes.ts index 2ad17ce894..b17666eacd 100644 --- a/apps/release-service/src/routes.ts +++ b/apps/release-service/src/routes.ts @@ -13,6 +13,20 @@ import { handleRevokeApproverCredential, matchApproverCredentialPath, } from "./approvals/routes.js"; +import { + handleAbortPublisherRestore, + handleArchivePublisher, + handlePreparePublisherRestore, + handleRestorePublisher, + matchPublisherArchivePath, + matchPublisherRestoreAbortPath, + matchPublisherRestorePreparePath, + matchPublisherRestorePath, +} from "./backup/routes.js"; +import { + handleStartPublisherArchive, + matchPublisherArchiveStartPath, +} from "./backup/workflow-route.js"; import type { ServiceConfiguration } from "./config.js"; import { handleControlAudit, @@ -20,6 +34,7 @@ import { handleServiceStatus, handleSetServiceMode, } from "./control-do/routes.js"; +import { handleListDirectory } from "./directory/routes.js"; import { handleCancelReleaseIntent, handleGetReleaseIntent, @@ -34,6 +49,12 @@ import { handlePublisherDelegationAuthorize, handlePublisherIdentityAuthorize, } from "./oauth/routes.js"; +import { + handleRotateApproverEncryption, + handleRotatePublisherEncryption, + matchApproverEncryptionRotationPath, + matchPublisherEncryptionRotationPath, +} from "./operations/encryption-routes.js"; import { handleCancelOperatorIntent, handleGetOperatorPublisher, @@ -205,6 +226,12 @@ export const ROUTES = Object.freeze([ accessRole: "viewer", handler: handleServiceStatus, }, + { + method: "GET", + path: "/admin/api/directory", + accessRole: "viewer", + handler: handleListDirectory, + }, { method: "POST", path: "/admin/api/pause", @@ -232,6 +259,55 @@ export const ROUTES = Object.freeze([ accessRole: "admin", handler: handleRevokeOperatorPublisher, }, + { + method: "POST", + path: "/admin/api/publishers/{publisherDid}/encryption/rotate", + match: matchPublisherEncryptionRotationPath, + accessRole: "admin", + handler: handleRotatePublisherEncryption, + }, + { + method: "POST", + path: "/admin/api/publishers/{publisherDid}/archive", + match: matchPublisherArchivePath, + accessRole: "admin", + handler: handleArchivePublisher, + }, + { + method: "POST", + path: "/admin/api/publishers/{publisherDid}/archive/start", + match: matchPublisherArchiveStartPath, + accessRole: "admin", + handler: handleStartPublisherArchive, + }, + { + method: "POST", + path: "/admin/api/publishers/{publisherDid}/restore", + match: matchPublisherRestorePath, + accessRole: "admin", + handler: handleRestorePublisher, + }, + { + method: "POST", + path: "/admin/api/publishers/{publisherDid}/restore/prepare", + match: matchPublisherRestorePreparePath, + accessRole: "admin", + handler: handlePreparePublisherRestore, + }, + { + method: "POST", + path: "/admin/api/publishers/{publisherDid}/restore/abort", + match: matchPublisherRestoreAbortPath, + accessRole: "admin", + handler: handleAbortPublisherRestore, + }, + { + method: "POST", + path: "/admin/api/approvers/{approverDid}/encryption/rotate", + match: matchApproverEncryptionRotationPath, + accessRole: "admin", + handler: handleRotateApproverEncryption, + }, { method: "POST", path: "/admin/api/intents/{intentId}/cancel", diff --git a/apps/release-service/src/ui/App.test.tsx b/apps/release-service/src/ui/App.test.tsx index a1414925b6..071c386676 100644 --- a/apps/release-service/src/ui/App.test.tsx +++ b/apps/release-service/src/ui/App.test.tsx @@ -218,6 +218,11 @@ describe("release-service web surfaces", () => { expect(await screen.findByRole("heading", { name: "Service control" })).toBeTruthy(); expect(screen.getByRole("button", { name: "Pause admission" })).toBeTruthy(); + expect(screen.getByRole("heading", { name: "Operations directory" })).toBeTruthy(); + expect(screen.getByRole("button", { name: "List publishers" })).toBeTruthy(); + expect(screen.getByRole("heading", { name: "Publisher archive" })).toBeTruthy(); + expect(screen.getByRole("button", { name: "Start archive workflow" })).toBeTruthy(); + expect(screen.getByRole("heading", { name: "Encryption maintenance" })).toBeTruthy(); expect(screen.getByRole("heading", { name: "Publisher lookup" })).toBeTruthy(); }); diff --git a/apps/release-service/src/ui/OperatorPage.tsx b/apps/release-service/src/ui/OperatorPage.tsx index ef58d76973..891c3d39ec 100644 --- a/apps/release-service/src/ui/OperatorPage.tsx +++ b/apps/release-service/src/ui/OperatorPage.tsx @@ -1,9 +1,14 @@ -import { Badge, Button, Input, Surface } from "@cloudflare/kumo"; +import { Badge, Button, Input, Surface, Table } from "@cloudflare/kumo"; import { ReleaseServiceOperatorClient, createReleaseIdempotencyKey, + type DirectoryIdentityKind, + type DirectoryIdentityResource, + type EncryptionRotationResult, type OperatorPublisherResource, + type PublisherArchivePageResult, type ServiceControlState, + type StartPublisherArchiveResult, } from "@emdash-cms/registry-client/release-service"; import { type FormEvent, useCallback, useEffect, useMemo, useState } from "react"; @@ -24,6 +29,17 @@ function operatorStatus(t: ReturnType, status: string): string { return t("operator.status.unknown", "Unknown"); } +function archiveKindLabel( + t: ReturnType, + kind: PublisherArchivePageResult["kind"], +): string { + if (kind === "metadata") return t("operator.archive.kind.metadata", "metadata"); + if (kind === "workload-policies") + return t("operator.archive.kind.workloads", "workload policies"); + if (kind === "intents") return t("operator.archive.kind.intents", "release intents"); + return t("operator.archive.kind.audit", "audit events"); +} + export function OperatorPage() { const t = useT(); const client = useMemo( @@ -33,7 +49,19 @@ export function OperatorPage() { const [state, setState] = useState(null); const [publisher, setPublisher] = useState(null); const [publisherDid, setPublisherDid] = useState(""); + const [approverDid, setApproverDid] = useState(""); const [intentId, setIntentId] = useState(""); + const [publisherRotationCursor, setPublisherRotationCursor] = useState(""); + const [approverRotationCursor, setApproverRotationCursor] = useState(""); + const [rotation, setRotation] = useState(null); + const [archiveId, setArchiveId] = useState(() => `archive-${crypto.randomUUID()}`); + const [archiveCursor, setArchiveCursor] = useState(""); + const [archivePage, setArchivePage] = useState("0"); + const [archive, setArchive] = useState(null); + const [archiveWorkflow, setArchiveWorkflow] = useState(null); + const [directoryKind, setDirectoryKind] = useState("publisher"); + const [directoryCursor, setDirectoryCursor] = useState(""); + const [directoryItems, setDirectoryItems] = useState([]); const [error, setError] = useState(null); const [busy, setBusy] = useState(false); @@ -110,6 +138,90 @@ export function OperatorPage() { } } + async function rotateEncryption(owner: "approver" | "publisher") { + setBusy(true); + setError(null); + try { + const result = + owner === "publisher" + ? await client.rotatePublisherEncryption( + publisherDid, + { afterCursor: publisherRotationCursor || null, limit: 50 }, + { idempotencyKey: createReleaseIdempotencyKey("web-publisher-rotation") }, + ) + : await client.rotateApproverEncryption( + approverDid, + { afterCursor: approverRotationCursor || null, limit: 50 }, + { idempotencyKey: createReleaseIdempotencyKey("web-approver-rotation") }, + ); + setRotation(result); + if (owner === "publisher") setPublisherRotationCursor(result.nextCursor ?? ""); + else setApproverRotationCursor(result.nextCursor ?? ""); + } catch (cause) { + setError(cause); + } finally { + setBusy(false); + } + } + + async function archivePublisher() { + setBusy(true); + setError(null); + try { + const result = await client.archivePublisher( + publisherDid, + { archiveId, cursor: archiveCursor || null, page: Number(archivePage) }, + { idempotencyKey: createReleaseIdempotencyKey("web-publisher-archive") }, + ); + setArchive(result); + setArchiveCursor(result.nextCursor ?? ""); + setArchivePage(String(result.nextPage)); + } catch (cause) { + setError(cause); + } finally { + setBusy(false); + } + } + + async function startPublisherArchive() { + setBusy(true); + setError(null); + try { + setArchiveWorkflow( + await client.startPublisherArchive(publisherDid, archiveId, { + idempotencyKey: createReleaseIdempotencyKey("web-publisher-archive-start"), + }), + ); + } catch (cause) { + setError(cause); + } finally { + setBusy(false); + } + } + + async function listDirectory(kind: DirectoryIdentityKind) { + setBusy(true); + setError(null); + try { + let cursor = kind === directoryKind ? directoryCursor || undefined : undefined; + for (let shard = 0; shard < 256; shard += 1) { + const result = await client.listDirectory(kind, { cursor, limit: 50 }); + cursor = result.nextCursor; + if (result.items.length > 0 || !cursor) { + setDirectoryKind(kind); + setDirectoryItems(result.items); + setDirectoryCursor(cursor ?? ""); + return; + } + } + throw new Error("Directory traversal did not terminate"); + } catch (cause) { + setError(cause); + } finally { + setBusy(false); + } + } + async function operateIntent(action: "cancel" | "reconcile") { setBusy(true); setError(null); @@ -165,6 +277,220 @@ export function OperatorPage() { + +
+
+

+ {t("operator.directory.title", "Operations directory")} +

+

+ {t( + "operator.directory.description", + "List the next populated identity shard for fleet maintenance. Directory entries do not grant authority.", + )} +

+
+ + {directoryKind === "publisher" + ? t("operator.directory.publishers", "Publishers") + : t("operator.directory.approvers", "Approvers")} + +
+
+ + +
+ {directoryItems.length > 0 ? ( +
+ + + + {t("operator.directory.did", "DID")} + {t("operator.directory.shard", "Shard")} + {t("operator.directory.lastSeen", "Last seen")} + + + + {directoryItems.map((item) => ( + + {item.did} + {item.shard} + + {new Intl.DateTimeFormat(document.documentElement.lang, { + dateStyle: "medium", + timeStyle: "short", + }).format(item.lastSeenAt)} + + + ))} + +
+
+ ) : null} + + + +
+
+

+ {t("operator.archive.title", "Publisher archive")} +

+

+ {t( + "operator.archive.description", + "Write one encrypted snapshot page and resume until the completion manifest is stored.", + )} +

+
+ {archive ? ( + + {archive.complete + ? t("operator.archive.complete", "Archive complete") + : t("operator.archive.incomplete", "Resume required")} + + ) : null} +
+
+ setPublisherDid(event.currentTarget.value)} + /> + setArchiveId(event.currentTarget.value)} + /> + setArchiveCursor(event.currentTarget.value)} + /> + setArchivePage(event.currentTarget.value)} + /> +
+
+ + + {archiveWorkflow ? ( +

+ {t("operator.archive.workflow", "Workflow: {workflowId}", { + workflowId: archiveWorkflow.workflowId, + })} +

+ ) : null} + {archive ? ( +

+ {t("operator.archive.result", "Stored {kind} page {page}.", { + kind: archiveKindLabel(t, archive.kind), + page: archive.page, + })} +

+ ) : null} +
+ + + +
+
+

+ {t("operator.encryption.title", "Encryption maintenance")} +

+

+ {t( + "operator.encryption.description", + "Rotate one bounded shard page, then resume from the returned cursor until verification completes.", + )} +

+
+ {rotation ? ( + + {rotation.complete + ? t("operator.encryption.complete", "Verified") + : t("operator.encryption.incomplete", "Resume required")} + + ) : null} +
+
+
+ setPublisherDid(event.currentTarget.value)} + /> + setPublisherRotationCursor(event.currentTarget.value)} + /> + +
+
+ setApproverDid(event.currentTarget.value)} + /> + setApproverRotationCursor(event.currentTarget.value)} + /> + +
+
+ {rotation ? ( +

+ {t( + "operator.encryption.result", + "Key {keyVersion}: scanned {scanned}, rotated {rotated}, raced {raced}.", + { + keyVersion: rotation.targetKeyVersion, + scanned: rotation.scanned, + rotated: rotation.rotated, + raced: rotation.raced, + }, + )} +

+ ) : null} + +

{t("operator.publisher.title", "Publisher lookup")} diff --git a/apps/release-service/src/workflows/publisher-archive.ts b/apps/release-service/src/workflows/publisher-archive.ts new file mode 100644 index 0000000000..c82c298dd7 --- /dev/null +++ b/apps/release-service/src/workflows/publisher-archive.ts @@ -0,0 +1,184 @@ +import { WorkflowEntrypoint, type WorkflowEvent, type WorkflowStep } from "cloudflare:workers"; +import { NonRetryableError } from "cloudflare:workflows"; +import { base64url } from "jose"; + +import type { AccessActor } from "../access/auth.js"; +import { handleArchivePublisher } from "../backup/routes.js"; +import { loadConfiguration } from "../config.js"; + +const DID_PATTERN = /^did:[a-z0-9]+:[A-Za-z0-9._:%-]+$/; +const ARCHIVE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]{15,63}$/; +const ACTOR_IDENTITY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:@/-]{0,255}$/; +const MAX_ARCHIVE_PAGES = 10_000; +const STEP_CONFIG = { + retries: { limit: 5, delay: "2 seconds", backoff: "exponential" }, + timeout: "5 minutes", +} as const; + +export interface PublisherArchiveWorkflowParams { + publisherDid: string; + archiveId: string; + actorIdentity: string; +} + +export interface PublisherArchiveWorkflowOutput { + publisherDid: string; + archiveId: string; + ownerHash: string; + pages: number; +} + +export type StartPublisherArchiveWorkflowResult = + | { ok: true; workflowId: string; created: boolean } + | { ok: false; code: "ARCHIVE_WORKFLOW_UNAVAILABLE" }; + +interface ArchivePageOutput { + ownerHash: string; + nextCursor: string | null; + nextPage: number; + complete: boolean; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function validParams(value: unknown): value is PublisherArchiveWorkflowParams { + return ( + isRecord(value) && + typeof value["publisherDid"] === "string" && + DID_PATTERN.test(value["publisherDid"]) && + typeof value["archiveId"] === "string" && + ARCHIVE_ID_PATTERN.test(value["archiveId"]) && + typeof value["actorIdentity"] === "string" && + ACTOR_IDENTITY_PATTERN.test(value["actorIdentity"]) + ); +} + +async function workflowId(publisherDid: string, archiveId: string): Promise { + return base64url.encode( + new Uint8Array( + await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode( + JSON.stringify(["publisher-archive-workflow", 1, publisherDid, archiveId]), + ), + ), + ), + ); +} + +export async function startPublisherArchiveWorkflow( + workflow: Workflow, + params: PublisherArchiveWorkflowParams, +): Promise { + if (!validParams(params)) return { ok: false, code: "ARCHIVE_WORKFLOW_UNAVAILABLE" }; + const id = await workflowId(params.publisherDid, params.archiveId); + try { + await workflow.create({ id, params }); + return { ok: true, workflowId: id, created: true }; + } catch { + try { + const existing = await workflow.get(id); + const status = await existing.status(); + return status.status === "unknown" + ? { ok: false, code: "ARCHIVE_WORKFLOW_UNAVAILABLE" } + : { ok: true, workflowId: id, created: false }; + } catch { + return { ok: false, code: "ARCHIVE_WORKFLOW_UNAVAILABLE" }; + } + } +} + +function parseArchivePage(value: unknown, expectedPage: number): ArchivePageOutput { + if (!isRecord(value) || !isRecord(value["data"])) { + throw new Error("Publisher archive page response is invalid"); + } + const data = value["data"]; + if ( + typeof data["ownerHash"] !== "string" || + data["ownerHash"].length !== 43 || + (data["nextCursor"] !== null && typeof data["nextCursor"] !== "string") || + data["nextPage"] !== expectedPage + 1 || + typeof data["complete"] !== "boolean" || + data["complete"] !== (data["nextCursor"] === null) + ) { + throw new Error("Publisher archive page response is invalid"); + } + return { + ownerHash: data["ownerHash"], + nextCursor: data["nextCursor"], + nextPage: data["nextPage"], + complete: data["complete"], + }; +} + +export class PublisherArchiveWorkflow extends WorkflowEntrypoint< + Env, + PublisherArchiveWorkflowParams +> { + override async run( + event: Readonly>, + step: WorkflowStep, + ): Promise { + if (!validParams(event.payload)) { + throw new NonRetryableError("Invalid publisher-archive Workflow parameters"); + } + const params = event.payload; + const actor: AccessActor = { + realm: "access", + identity: params.actorIdentity, + email: "archive-workflow@emdash.invalid", + role: "admin", + }; + let cursor: string | null = null; + let page = 0; + let ownerHash: string | null = null; + while (page < MAX_ARCHIVE_PAGES) { + const pageCursor = cursor; + const pageNumber = page; + const result = await step.do( + `publisher-archive-${pageNumber}`, + STEP_CONFIG, + async () => { + const configuration = await loadConfiguration(this.env); + const response = await handleArchivePublisher( + new Request(`${configuration.publicOrigin}/admin/api/publishers/archive`, { + method: "POST", + headers: { + "content-type": "application/json", + "idempotency-key": `archive-${params.archiveId}-${pageNumber}`, + }, + body: JSON.stringify({ + archiveId: params.archiveId, + cursor: pageCursor, + page: pageNumber, + }), + }), + `archive-${pageNumber}`, + configuration, + { publisherDid: params.publisherDid }, + actor, + ); + if (!response.ok) throw new Error("Publisher archive page failed"); + return parseArchivePage(await response.json(), pageNumber); + }, + ); + ownerHash ??= result.ownerHash; + if (ownerHash !== result.ownerHash) { + throw new NonRetryableError("Publisher archive owner changed"); + } + page = result.nextPage; + cursor = result.nextCursor; + if (result.complete) { + return { + publisherDid: params.publisherDid, + archiveId: params.archiveId, + ownerHash, + pages: page, + }; + } + } + throw new NonRetryableError("Publisher archive exceeded the page limit"); + } +} diff --git a/apps/release-service/src/workflows/release-intent.ts b/apps/release-service/src/workflows/release-intent.ts index a8a8cb369a..a181e684fc 100644 --- a/apps/release-service/src/workflows/release-intent.ts +++ b/apps/release-service/src/workflows/release-intent.ts @@ -5,6 +5,7 @@ import { base64url } from "jose"; import type ReleaseVerifier from "../../../release-verifier/src/index.js"; import { encodeAwaitingApprovalState, type ApprovalEvidence } from "../approvals/digest.js"; import { invalidateApprovalChallenges } from "../approvals/invalidation.js"; +import { writeOperationsMetric } from "../observability/metrics.js"; import type { IntentState, PublisherDurableObject, @@ -459,6 +460,16 @@ export class ReleaseIntentWorkflow extends WorkflowEntrypoint< return await failVerifyingIntent(publisher, params, intent, "VERIFIER_INPUT_INVALID"); } const report = normalizeVerifierReport(await this.env.RELEASE_VERIFIER.verifyRelease(input)); + if (!report.success) { + writeOperationsMetric( + { + event: "verifier_failure", + outcome: report.error.code, + scope: "isolated", + }, + this.env.OPERATIONS_METRICS, + ); + } const resultJson = JSON.stringify(report); const stored = await publisher.putVerificationStep({ publisherDid: params.publisherDid, diff --git a/apps/release-service/test/approver-do.test.ts b/apps/release-service/test/approver-do.test.ts index 8afa80c0b2..6a6067468a 100644 --- a/apps/release-service/test/approver-do.test.ts +++ b/apps/release-service/test/approver-do.test.ts @@ -131,6 +131,73 @@ describe("ApproverDurableObject", () => { ); }); + it("pages live identity ciphertexts and rotates them by compare-and-set", async () => { + const stub = approver(); + const now = 1_800_000_000_000; + await stub.putIdentityTransaction({ + approverDid: APPROVER_DID, + stateHash: STATE_HASH, + encryptedState: "approver-ciphertext-v1", + encryptionKeyVersion: 1, + clientKeyId: "assertion-1", + redirectTarget: `/approvals/${INTENT_ID}`, + expiresAt: now + 60_000, + now, + }); + + await expect(stub.listEncryptionRecords(APPROVER_DID, null, 10, now)).resolves.toEqual({ + items: [ + { + cursor: `identity-transaction:${STATE_HASH}`, + envelope: "approver-ciphertext-v1", + keyVersion: 1, + context: { + purpose: "oauth-approver-transaction", + objectClass: "ApproverDurableObject", + table: "identity_transactions", + primaryKey: STATE_HASH, + ownerDid: APPROVER_DID, + }, + }, + ], + nextCursor: null, + }); + await expect( + stub.replaceEncryptionRecord({ + approverDid: APPROVER_DID, + cursor: `identity-transaction:${STATE_HASH}`, + expectedEnvelope: "approver-ciphertext-v1", + replacementEnvelope: "approver-ciphertext-v2", + replacementKeyVersion: 2, + actorIdentity: "operator@example.com", + now, + }), + ).resolves.toBe(true); + await expect( + stub.replaceEncryptionRecord({ + approverDid: APPROVER_DID, + cursor: `identity-transaction:${STATE_HASH}`, + expectedEnvelope: "approver-ciphertext-v1", + replacementEnvelope: "approver-ciphertext-v3", + replacementKeyVersion: 3, + actorIdentity: "operator@example.com", + now, + }), + ).resolves.toBe(false); + await expect(stub.listEncryptionRecords(APPROVER_DID, null, 10, now)).resolves.toMatchObject({ + items: [{ envelope: "approver-ciphertext-v2", keyVersion: 2 }], + }); + expect( + await runInDurableObject(stub, (_instance, state) => + state.storage.sql + .exec<{ event_type: string; public_payload: string }>( + "SELECT event_type, public_payload FROM audit_events WHERE event_type = 'encryption-rotated'", + ) + .toArray(), + ), + ).toEqual([{ event_type: "encryption-rotated", public_payload: "{}" }]); + }); + it("creates, validates, expires, revokes, and epoch-invalidates approver sessions", async () => { const stub = approver(); const now = 1_800_000_000_000; diff --git a/apps/release-service/test/config.test.ts b/apps/release-service/test/config.test.ts index 8f0409fab6..f65452f78d 100644 --- a/apps/release-service/test/config.test.ts +++ b/apps/release-service/test/config.test.ts @@ -107,4 +107,46 @@ describe("release-service OAuth configuration", () => { issues: ["ENCRYPTION_KEYRING_INVALID"], }); }); + + it("resolves assertion and encryption values from Secrets Store bindings", async () => { + let reads = 0; + const configuration = await loadConfiguration({ + ...TEST_BINDINGS, + OAUTH_ASSERTION_KEYSET: { + async get() { + reads += 1; + return TEST_BINDINGS.OAUTH_ASSERTION_KEYSET; + }, + }, + ENCRYPTION_KEYRING: { + async get() { + reads += 1; + return TEST_BINDINGS.ENCRYPTION_KEYRING; + }, + }, + }); + + expect(configuration.oauth.activeAssertionKeyId).toBe(ASSERTION_KEY_2.kid); + expect(configuration.encryption.currentKeyVersion).toBe(1); + expect(reads).toBe(2); + }); + + it("fails closed when Secrets Store retrieval fails without exposing the cause", async () => { + const sensitive = "secret store returned sensitive provider detail"; + try { + await loadConfiguration({ + ...TEST_BINDINGS, + ENCRYPTION_KEYRING: { + async get() { + throw new Error(sensitive); + }, + }, + }); + expect.fail("expected configuration failure"); + } catch (error) { + expect(error).toBeInstanceOf(ConfigurationError); + expect(error).toMatchObject({ issues: ["SECRET_STORE_UNAVAILABLE"] }); + expect(JSON.stringify(error)).not.toContain(sensitive); + } + }); }); diff --git a/apps/release-service/test/encryption-operations-routes.test.ts b/apps/release-service/test/encryption-operations-routes.test.ts new file mode 100644 index 0000000000..9a44ad80f5 --- /dev/null +++ b/apps/release-service/test/encryption-operations-routes.test.ts @@ -0,0 +1,346 @@ +import { reset } from "cloudflare:test"; +import { env } from "cloudflare:workers"; +import { afterEach, describe, expect, it } from "vitest"; + +import type { AccessActor } from "../src/access/auth.js"; +import { loadConfiguration } from "../src/config.js"; +import { + handleRotateApproverEncryption, + handleRotatePublisherEncryption, +} from "../src/operations/encryption-routes.js"; +import { TEST_BINDINGS } from "./fixtures/oauth.js"; + +const PUBLISHER_DID = "did:web:publisher.example.com"; +const APPROVER_DID = "did:plc:approver"; +const STATE_HASH = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFG"; +const KEYRING_V2 = + '{"current":2,"keys":[{"version":1,"key":"AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8"},{"version":2,"key":"ICEiIyQlJicoKSorLC0uLzAxMjM0NTY3ODk6Ozw9Pj8"}]}'; +const KEYRING_V2_RETIRED = + '{"current":2,"keys":[{"version":2,"key":"ICEiIyQlJicoKSorLC0uLzAxMjM0NTY3ODk6Ozw9Pj8"}]}'; +const ADMIN: AccessActor = { + realm: "access", + identity: "admin@example.com", + email: "admin@example.com", + role: "admin", +}; + +function request(body: unknown): Request { + return new Request(`${TEST_BINDINGS.PUBLIC_ORIGIN}/admin/api/encryption/rotate`, { + method: "POST", + headers: { + "content-type": "application/json", + "idempotency-key": "rotate-encryption-test", + }, + body: JSON.stringify(body), + }); +} + +function bindings(keyring: string) { + return { ...TEST_BINDINGS, ENCRYPTION_KEYRING: keyring }; +} + +afterEach(async () => { + await reset(); +}); + +describe("Access encryption operations", () => { + it("rotates a resumable publisher page and proves retirement readability", async () => { + const initial = await loadConfiguration(TEST_BINDINGS); + const now = Date.now(); + const delegationContext = { + purpose: "oauth-session", + objectClass: "PublisherDurableObject", + table: "delegation", + primaryKey: "1", + ownerDid: PUBLISHER_DID, + } as const; + const stateContext = { + purpose: "oauth-console-transaction", + objectClass: "PublisherDurableObject", + table: "oauth_states", + primaryKey: STATE_HASH, + ownerDid: PUBLISHER_DID, + } as const; + const delegation = await initial.encryption.encrypt( + new TextEncoder().encode("delegation-plaintext"), + delegationContext, + ); + const state = await initial.encryption.encrypt( + new TextEncoder().encode("state-plaintext"), + stateContext, + ); + const publisher = env.PUBLISHER_DO.getByName(PUBLISHER_DID); + await publisher.putDelegation({ + publisherDid: PUBLISHER_DID, + releaseNsid: initial.oauth.releaseNsid, + scope: initial.oauth.releaseScope, + clientKeyId: initial.oauth.activeAssertionKeyId, + encryptedSession: delegation.envelope, + encryptionKeyVersion: delegation.keyVersion, + issuer: "https://authorization.example.com", + pdsUrl: "https://pds.example.com", + expiresAt: null, + refreshBefore: null, + expectedVersion: null, + }); + await publisher.putOAuthState({ + publisherDid: PUBLISHER_DID, + stateHash: STATE_HASH, + encryptedState: state.envelope, + encryptionKeyVersion: state.keyVersion, + encryptionPurpose: "oauth-console-transaction", + clientKeyId: initial.oauth.activeAssertionKeyId, + redirectTarget: "/publisher", + expiresAt: now + 60_000, + }); + const rotating = await loadConfiguration(bindings(KEYRING_V2)); + + const first = await handleRotatePublisherEncryption( + request({ afterCursor: null, limit: 1 }), + "request-1", + rotating, + { publisherDid: PUBLISHER_DID }, + ADMIN, + ); + expect(first.status).toBe(200); + const firstText = await first.text(); + expect(firstText).not.toContain("plaintext"); + expect(firstText).not.toContain(delegation.envelope); + expect(JSON.parse(firstText)).toMatchObject({ + data: { + ownerDid: PUBLISHER_DID, + targetKeyVersion: 2, + scanned: 1, + rotated: 1, + raced: 0, + nextCursor: "delegation:1", + }, + }); + const second = await handleRotatePublisherEncryption( + request({ afterCursor: "delegation:1", limit: 1 }), + "request-2", + rotating, + { publisherDid: PUBLISHER_DID }, + ADMIN, + ); + expect(second.status).toBe(200); + await expect(second.json()).resolves.toMatchObject({ + data: { scanned: 1, rotated: 1, raced: 0, nextCursor: null }, + }); + + const records = await publisher.listEncryptionRecords(PUBLISHER_DID, null, 10, now); + expect(records.items.map((record) => record.keyVersion)).toEqual([2, 2]); + const retired = await loadConfiguration(bindings(KEYRING_V2_RETIRED)); + for (const record of records.items) { + await expect( + retired.encryption.decrypt(record.envelope, record.context), + ).resolves.toBeInstanceOf(Uint8Array); + } + }); + + it("requires a clean rescan when an earlier publisher page races", async () => { + const initial = await loadConfiguration(TEST_BINDINGS); + const now = Date.now(); + const delegationContext = { + purpose: "oauth-session", + objectClass: "PublisherDurableObject", + table: "delegation", + primaryKey: "1", + ownerDid: PUBLISHER_DID, + } as const; + const stateContext = { + purpose: "oauth-console-transaction", + objectClass: "PublisherDurableObject", + table: "oauth_states", + primaryKey: STATE_HASH, + ownerDid: PUBLISHER_DID, + } as const; + const [delegation, racedDelegation, state] = await Promise.all([ + initial.encryption.encrypt(new TextEncoder().encode("delegation-before"), delegationContext), + initial.encryption.encrypt(new TextEncoder().encode("delegation-raced"), delegationContext), + initial.encryption.encrypt(new TextEncoder().encode("state"), stateContext), + ]); + const publisher = env.PUBLISHER_DO.getByName(PUBLISHER_DID); + await publisher.putDelegation({ + publisherDid: PUBLISHER_DID, + releaseNsid: initial.oauth.releaseNsid, + scope: initial.oauth.releaseScope, + clientKeyId: initial.oauth.activeAssertionKeyId, + encryptedSession: delegation.envelope, + encryptionKeyVersion: delegation.keyVersion, + issuer: "https://authorization.example.com", + pdsUrl: "https://pds.example.com", + expiresAt: null, + refreshBefore: null, + expectedVersion: null, + }); + await publisher.putOAuthState({ + publisherDid: PUBLISHER_DID, + stateHash: STATE_HASH, + encryptedState: state.envelope, + encryptionKeyVersion: state.keyVersion, + encryptionPurpose: "oauth-console-transaction", + clientKeyId: initial.oauth.activeAssertionKeyId, + redirectTarget: "/publisher", + expiresAt: now + 60_000, + }); + const rotating = await loadConfiguration(bindings(KEYRING_V2)); + let injectRace = true; + const racedConfiguration = { + ...rotating, + encryption: { + ...rotating.encryption, + rotate: async (...args: Parameters) => { + const replacement = await rotating.encryption.rotate(...args); + if (injectRace) { + injectRace = false; + await publisher.putDelegation({ + publisherDid: PUBLISHER_DID, + releaseNsid: initial.oauth.releaseNsid, + scope: initial.oauth.releaseScope, + clientKeyId: initial.oauth.activeAssertionKeyId, + encryptedSession: racedDelegation.envelope, + encryptionKeyVersion: racedDelegation.keyVersion, + issuer: "https://authorization.example.com", + pdsUrl: "https://pds.example.com", + expiresAt: null, + refreshBefore: null, + expectedVersion: 1, + }); + } + return replacement; + }, + }, + }; + + const first = await handleRotatePublisherEncryption( + request({ afterCursor: null, limit: 1 }), + "race-page-1", + racedConfiguration, + { publisherDid: PUBLISHER_DID }, + ADMIN, + ); + expect(first.status).toBe(200); + const firstBody = await first.json<{ data: { nextCursor: string } }>(); + expect(firstBody.data.nextCursor).toContain("delegation:1"); + + const second = await handleRotatePublisherEncryption( + request({ afterCursor: firstBody.data.nextCursor, limit: 1 }), + "race-page-2", + rotating, + { publisherDid: PUBLISHER_DID }, + ADMIN, + ); + expect(second.status).toBe(200); + const secondBody = await second.json<{ + data: { complete: boolean; nextCursor: string | null }; + }>(); + expect(secondBody.data).toMatchObject({ complete: false, nextCursor: "rescan" }); + + const rescan = await handleRotatePublisherEncryption( + request({ afterCursor: secondBody.data.nextCursor, limit: 10 }), + "race-rescan", + rotating, + { publisherDid: PUBLISHER_DID }, + ADMIN, + ); + expect(rescan.status).toBe(200); + await expect(rescan.json()).resolves.toMatchObject({ + data: { complete: true, nextCursor: null, rotated: 1, raced: 0 }, + }); + }); + + it("rotates an approver identity transaction without returning ciphertext", async () => { + const initial = await loadConfiguration(TEST_BINDINGS); + const now = Date.now(); + const context = { + purpose: "oauth-approver-transaction", + objectClass: "ApproverDurableObject", + table: "identity_transactions", + primaryKey: STATE_HASH, + ownerDid: APPROVER_DID, + } as const; + const encrypted = await initial.encryption.encrypt( + new TextEncoder().encode("approver-plaintext"), + context, + ); + const approver = env.APPROVER_DO.getByName(APPROVER_DID); + await approver.putIdentityTransaction({ + approverDid: APPROVER_DID, + stateHash: STATE_HASH, + encryptedState: encrypted.envelope, + encryptionKeyVersion: encrypted.keyVersion, + clientKeyId: initial.oauth.activeAssertionKeyId, + redirectTarget: "/approvals/example", + expiresAt: now + 60_000, + now, + }); + + const response = await handleRotateApproverEncryption( + request({ afterCursor: null, limit: 10 }), + "request-approver", + await loadConfiguration(bindings(KEYRING_V2)), + { approverDid: APPROVER_DID }, + ADMIN, + ); + expect(response.status).toBe(200); + const text = await response.text(); + expect(text).not.toContain(encrypted.envelope); + expect(text).not.toContain("approver-plaintext"); + expect(JSON.parse(text)).toMatchObject({ + data: { ownerDid: APPROVER_DID, targetKeyVersion: 2, rotated: 1, nextCursor: null }, + }); + }); + + it("fails closed when a retained key is missing", async () => { + const initial = await loadConfiguration(TEST_BINDINGS); + const context = { + purpose: "oauth-session", + objectClass: "PublisherDurableObject", + table: "delegation", + primaryKey: "1", + ownerDid: PUBLISHER_DID, + } as const; + const encrypted = await initial.encryption.encrypt( + new TextEncoder().encode("retained-authority"), + context, + ); + await env.PUBLISHER_DO.getByName(PUBLISHER_DID).putDelegation({ + publisherDid: PUBLISHER_DID, + releaseNsid: initial.oauth.releaseNsid, + scope: initial.oauth.releaseScope, + clientKeyId: initial.oauth.activeAssertionKeyId, + encryptedSession: encrypted.envelope, + encryptionKeyVersion: encrypted.keyVersion, + issuer: "https://authorization.example.com", + pdsUrl: "https://pds.example.com", + expiresAt: null, + refreshBefore: null, + expectedVersion: null, + }); + + const response = await handleRotatePublisherEncryption( + request({ afterCursor: null, limit: 10 }), + "request-missing-key", + await loadConfiguration(bindings(KEYRING_V2_RETIRED)), + { publisherDid: PUBLISHER_DID }, + ADMIN, + ); + expect(response.status).toBe(409); + await expect(response.json()).resolves.toMatchObject({ + error: { code: "ENCRYPTION_OPERATION_FAILED" }, + }); + }); + + it("rejects a resume cursor from another shard type", async () => { + const response = await handleRotatePublisherEncryption( + request({ afterCursor: `identity-transaction:${STATE_HASH}`, limit: 10 }), + "request-invalid-cursor", + await loadConfiguration(bindings(KEYRING_V2)), + { publisherDid: PUBLISHER_DID }, + ADMIN, + ); + expect(response.status).toBe(400); + await expect(response.json()).resolves.toMatchObject({ error: { code: "INVALID_REQUEST" } }); + }); +}); diff --git a/apps/release-service/test/identity-directory.test.ts b/apps/release-service/test/identity-directory.test.ts new file mode 100644 index 0000000000..6792d190cc --- /dev/null +++ b/apps/release-service/test/identity-directory.test.ts @@ -0,0 +1,88 @@ +import { reset, runInDurableObject } from "cloudflare:test"; +import { env } from "cloudflare:workers"; +import { afterEach, describe, expect, it } from "vitest"; + +import type { AccessActor } from "../src/access/auth.js"; +import { loadConfiguration } from "../src/config.js"; +import { encodeDirectoryCursor, handleListDirectory } from "../src/directory/routes.js"; +import { identityDirectoryShard, registerDirectoryIdentity } from "../src/directory/sharding.js"; +import { TEST_BINDINGS } from "./fixtures/oauth.js"; + +const PUBLISHER_DID = "did:web:publisher.example.com"; +const SECOND_PUBLISHER_DID = "did:plc:second-publisher"; +const APPROVER_DID = "did:plc:approver"; +const VIEWER: AccessActor = { + realm: "access", + identity: "viewer@example.com", + email: "viewer@example.com", + role: "viewer", +}; + +afterEach(async () => { + await reset(); +}); + +describe("IdentityDirectoryDurableObject", () => { + it("routes identities to deterministic shards and lists each kind independently", async () => { + const publisherShard = await identityDirectoryShard(PUBLISHER_DID); + expect(publisherShard).toMatch(/^[0-9a-f]{2}$/); + await expect(registerDirectoryIdentity("publisher", PUBLISHER_DID, 100)).resolves.toMatchObject( + { + created: true, + shard: publisherShard, + }, + ); + await expect(registerDirectoryIdentity("publisher", PUBLISHER_DID, 200)).resolves.toMatchObject( + { + created: false, + shard: publisherShard, + }, + ); + await registerDirectoryIdentity("publisher", SECOND_PUBLISHER_DID, 150); + await registerDirectoryIdentity("approver", APPROVER_DID, 175); + + const publisher = env.IDENTITY_DIRECTORY_DO.getByName(publisherShard); + await expect(publisher.list("publisher", null, 10)).resolves.toEqual([ + { + kind: "publisher", + did: PUBLISHER_DID, + registeredAt: 100, + lastSeenAt: 200, + }, + ]); + await expect(publisher.list("approver", null, 10)).resolves.toEqual([]); + }); + + it("rejects a DID routed to a different shard", async () => { + const expected = await identityDirectoryShard(PUBLISHER_DID); + const wrong = expected === "00" ? "01" : "00"; + const stub = env.IDENTITY_DIRECTORY_DO.getByName(wrong); + await runInDurableObject(stub, async (instance) => { + await expect(instance.register("publisher", PUBLISHER_DID, 100)).rejects.toMatchObject({ + code: "DIRECTORY_SHARD_MISMATCH", + }); + }); + }); + + it("lists one bounded directory shard through Access", async () => { + const shard = await identityDirectoryShard(PUBLISHER_DID); + await registerDirectoryIdentity("publisher", PUBLISHER_DID, 100); + const cursor = encodeDirectoryCursor({ shard: Number.parseInt(shard, 16), afterDid: null }); + const response = await handleListDirectory( + new Request( + `${TEST_BINDINGS.PUBLIC_ORIGIN}/admin/api/directory?kind=publisher&limit=10&cursor=${encodeURIComponent(cursor)}`, + ), + "request-directory", + await loadConfiguration(TEST_BINDINGS), + {}, + VIEWER, + ); + + expect(response.status).toBe(200); + await expect(response.json()).resolves.toMatchObject({ + data: { + items: [{ did: PUBLISHER_DID, kind: "publisher", shard }], + }, + }); + }); +}); diff --git a/apps/release-service/test/intent-routes.test.ts b/apps/release-service/test/intent-routes.test.ts index 61c7ecaa6e..d2f1d3e6d6 100644 --- a/apps/release-service/test/intent-routes.test.ts +++ b/apps/release-service/test/intent-routes.test.ts @@ -2,6 +2,7 @@ import { NSID, type PackageRelease } from "@emdash-cms/registry-lexicons"; import { reset, runInDurableObject } from "cloudflare:test"; import { env } from "cloudflare:workers"; import { createLocalJWKSet, exportJWK, generateKeyPair, SignJWT, type JWTVerifyGetKey } from "jose"; +import { ulid } from "ulidx"; import { afterEach, beforeAll, describe, expect, it } from "vitest"; import releaseFixture from "../../../packages/registry-verification/fixtures/records/release.json"; @@ -383,4 +384,64 @@ describe("release intent API", () => { expect(response.status).toBe(503); expect(await response.json()).toMatchObject({ error: { code: "SERVICE_PAUSED" } }); }); + + it("rate limits one workload without consuming another publisher shard", async () => { + await putPolicy(); + const configuration = await loadConfiguration(TEST_BINDINGS); + const workloadToken = await token(); + for (let index = 0; index < 30; index += 1) { + const version = `1.2.${index}`; + const value = release(); + value.version = version; + const response = await handleSubmitReleaseIntent( + request("/v1/release-intents", workloadToken, { + method: "POST", + body: { + publisherDid: PUBLISHER_DID, + packageSlug: "gallery", + version, + release: value, + }, + idempotencyKey: `github-rate-limit-${String(index).padStart(4, "0")}`, + }), + `request-${index}`, + configuration, + { ...submitDependencies, intentId: () => ulid(NOW + index) }, + ); + expect(response.status).toBe(202); + } + const blockedRelease = release(); + blockedRelease.version = "1.2.30"; + const blocked = await handleSubmitReleaseIntent( + request("/v1/release-intents", workloadToken, { + method: "POST", + body: { + publisherDid: PUBLISHER_DID, + packageSlug: "gallery", + version: "1.2.30", + release: blockedRelease, + }, + idempotencyKey: "github-rate-limit-over-limit", + }), + "request-blocked", + configuration, + { ...submitDependencies, intentId: () => ulid(NOW + 31) }, + ); + expect(blocked.status).toBe(429); + expect(blocked.headers.get("retry-after")).toBe("60"); + await expect(blocked.json()).resolves.toMatchObject({ + error: { code: "WORKLOAD_RATE_LIMITED" }, + }); + + await expect( + env.PUBLISHER_DO.getByName("did:plc:other").consumeIntentRateLimit({ + publisherDid: "did:plc:other", + repositoryId: "123456789", + workloadKey: "Z".repeat(43), + idempotencyKey: "other-publisher-rate-limit", + expiresAt: NOW + 24 * 60 * 60_000, + now: NOW, + }), + ).resolves.toMatchObject({ ok: true }); + }); }); diff --git a/apps/release-service/test/oauth-routes.test.ts b/apps/release-service/test/oauth-routes.test.ts index 10da23359a..a07698a0bd 100644 --- a/apps/release-service/test/oauth-routes.test.ts +++ b/apps/release-service/test/oauth-routes.test.ts @@ -3,6 +3,7 @@ import { env } from "cloudflare:workers"; import { afterEach, describe, expect, it, vi } from "vitest"; import { loadConfiguration } from "../src/config.js"; +import { identityDirectoryShard } from "../src/directory/sharding.js"; import { handleApproverIdentityAuthorize, handleOAuthCallback, @@ -179,6 +180,13 @@ describe("publisher OAuth routes", () => { expect(setCookie).toContain("__Host-emdash_oauth_route="); expect(network.requests.some((request) => request.path === "/revoke")).toBe(true); await expect(env.PUBLISHER_DO.getByName(DID).getDelegation(DID)).resolves.toBeNull(); + await expect( + env.IDENTITY_DIRECTORY_DO.getByName(await identityDirectoryShard(DID)).list( + "publisher", + null, + 10, + ), + ).resolves.toEqual([expect.objectContaining({ did: DID, kind: "publisher" })]); }); it("keeps approver identity state and cookies in the approver realm", async () => { @@ -221,6 +229,13 @@ describe("publisher OAuth routes", () => { await expect(env.APPROVER_DO.getByName(DID).listCredentials(DID, null, 10)).resolves.toEqual( [], ); + await expect( + env.IDENTITY_DIRECTORY_DO.getByName(await identityDirectoryShard(DID)).list( + "approver", + null, + 10, + ), + ).resolves.toEqual([expect.objectContaining({ did: DID, kind: "approver" })]); }); it("keeps attacker-triggerable publisher authorization state out of the publisher shard", async () => { diff --git a/apps/release-service/test/operations-metrics.test.ts b/apps/release-service/test/operations-metrics.test.ts new file mode 100644 index 0000000000..f214fa93d3 --- /dev/null +++ b/apps/release-service/test/operations-metrics.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from "vitest"; + +import { writeOperationsMetric } from "../src/observability/metrics.js"; + +describe("release-service operations metrics", () => { + it("writes a bounded privacy-safe Analytics Engine point", () => { + const points: AnalyticsEngineDataPoint[] = []; + const dataset = { + writeDataPoint(point?: AnalyticsEngineDataPoint) { + if (point) points.push(point); + }, + } satisfies AnalyticsEngineDataset; + + writeOperationsMetric( + { + event: "intent_rate_limited", + ownerHash: "A".repeat(43), + outcome: "denied", + scope: "workload", + requestId: "request-1", + value: 1, + timestamp: 1_800_000_000_000, + }, + dataset, + ); + + expect(points).toEqual([ + { + indexes: ["A".repeat(43)], + blobs: ["intent_rate_limited", "denied", "workload", "request-1"], + doubles: [1, 1_800_000_000_000], + }, + ]); + expect(JSON.stringify(points)).not.toContain("did:"); + }); + + it("rejects unbounded or identifying dimensions", () => { + const dataset = { writeDataPoint() {} } satisfies AnalyticsEngineDataset; + expect(() => + writeOperationsMetric( + { event: "access_denied", ownerHash: "did:plc:publisher", value: 1 }, + dataset, + ), + ).toThrow("Invalid operations metric"); + }); +}); diff --git a/apps/release-service/test/publisher-archive-routes.test.ts b/apps/release-service/test/publisher-archive-routes.test.ts new file mode 100644 index 0000000000..441deab696 --- /dev/null +++ b/apps/release-service/test/publisher-archive-routes.test.ts @@ -0,0 +1,514 @@ +import { abortAllDurableObjects, reset, runInDurableObject } from "cloudflare:test"; +import { env } from "cloudflare:workers"; +import { afterEach, describe, expect, it } from "vitest"; + +import type { AccessActor } from "../src/access/auth.js"; +import { + handleAbortPublisherRestore, + handleArchivePublisher, + handlePreparePublisherRestore, + handleRestorePublisher, +} from "../src/backup/routes.js"; +import { loadConfiguration } from "../src/config.js"; +import { SERVICE_CONTROL_OBJECT_NAME } from "../src/control-do/service-control-do.js"; +import { TEST_BINDINGS } from "./fixtures/oauth.js"; + +const PUBLISHER_DID = "did:web:publisher.example.com"; +const INTENT_ID = "01JABCDEFGHJKMNPQRSTVWXYZ0"; +const ARCHIVE_ID = "publisher-archive-0001"; +const NOW = 1_800_000_000_000; +const ADMIN: AccessActor = { + realm: "access", + identity: "admin@example.com", + email: "admin@example.com", + role: "admin", +}; + +function request(cursor: string | null, page: number, archiveId = ARCHIVE_ID): Request { + return new Request(`${TEST_BINDINGS.PUBLIC_ORIGIN}/admin/api/publishers/archive`, { + method: "POST", + headers: { + "content-type": "application/json", + "idempotency-key": `publisher-archive-page-${page}`, + }, + body: JSON.stringify({ archiveId, cursor, page }), + }); +} + +function restoreRequest(page: number): Request { + return new Request(`${TEST_BINDINGS.PUBLIC_ORIGIN}/admin/api/publishers/restore`, { + method: "POST", + headers: { + "content-type": "application/json", + "idempotency-key": `publisher-restore-page-${page}`, + }, + body: JSON.stringify({ archiveId: ARCHIVE_ID, page }), + }); +} + +function prepareRestoreRequest(): Request { + return new Request(`${TEST_BINDINGS.PUBLIC_ORIGIN}/admin/api/publishers/restore/prepare`, { + method: "POST", + headers: { + "content-type": "application/json", + "idempotency-key": "publisher-restore-prepare-0001", + }, + body: JSON.stringify({ archiveId: ARCHIVE_ID, confirmPublisherDid: PUBLISHER_DID }), + }); +} + +function abortRestoreRequest(): Request { + return new Request(`${TEST_BINDINGS.PUBLIC_ORIGIN}/admin/api/publishers/restore/abort`, { + method: "POST", + headers: { + "content-type": "application/json", + "idempotency-key": "publisher-restore-abort-0001", + }, + body: JSON.stringify({ archiveId: ARCHIVE_ID, confirmPublisherDid: PUBLISHER_DID }), + }); +} + +function maximalCanonicalObject(maxChars: number): string { + const empty = JSON.stringify({ value: "" }); + return JSON.stringify({ value: "\\".repeat(Math.floor((maxChars - empty.length) / 2)) }); +} + +afterEach(async () => { + await reset(); +}); + +describe("publisher operations archive", () => { + it("writes resumable encrypted snapshots and append-only sanitized audit pages", async () => { + const configuration = await loadConfiguration(TEST_BINDINGS); + const publisher = env.PUBLISHER_DO.getByName(PUBLISHER_DID); + await publisher.putWorkloadPolicy({ + publisherDid: PUBLISHER_DID, + packageSlug: "gallery", + repository: "example/gallery", + repositoryId: "123", + repositoryOwnerId: "456", + workflowRef: "example/gallery/.github/workflows/release.yml@refs/heads/main", + allowedRefs: ["refs/heads/main"], + allowedEnvironments: [], + active: true, + expectedVersion: null, + now: NOW, + }); + await publisher.createIntent({ + publisherDid: PUBLISHER_DID, + intentId: INTENT_ID, + packageSlug: "gallery", + version: "1.2.3", + workloadPolicyVersion: 1, + workloadIdentityDigest: "A".repeat(43), + workloadIdempotencyDigest: "I".repeat(43), + idempotencyKey: "github-run-100-attempt-1", + requestDigest: "B".repeat(43), + workloadIdentityJson: '{"issuer":"github-actions","private":"repository-metadata"}', + releaseInputJson: '{"release":{"package":"gallery","version":"1.2.3"}}', + expiresAt: NOW + 60_000, + now: NOW + 1, + }); + await publisher.putDelegation({ + publisherDid: PUBLISHER_DID, + releaseNsid: configuration.oauth.releaseNsid, + scope: configuration.oauth.releaseScope, + clientKeyId: configuration.oauth.activeAssertionKeyId, + encryptedSession: "retained-authority-ciphertext", + encryptionKeyVersion: 1, + issuer: "https://authorization.example.com", + pdsUrl: "https://pds.example.com", + expiresAt: null, + refreshBefore: null, + expectedVersion: null, + }); + + let cursor: string | null = null; + let page = 0; + const responses: Array> = []; + do { + const response = await handleArchivePublisher( + request(cursor, page), + `request-${page}`, + configuration, + { publisherDid: PUBLISHER_DID }, + ADMIN, + ); + expect(response.status).toBe(200); + const body = await response.json<{ data: Record }>(); + responses.push(body.data); + cursor = typeof body.data["nextCursor"] === "string" ? body.data["nextCursor"] : null; + page = Number(body.data["nextPage"]); + } while (cursor !== null); + + expect(responses.map((item) => item["kind"])).toEqual([ + "metadata", + "workload-policies", + "intents", + "audit-events", + ]); + expect(responses.at(-1)).toMatchObject({ complete: true, manifestWritten: true }); + const snapshots = await env.OPERATIONS_ARCHIVE.list({ prefix: "snapshots/" }); + expect(snapshots.objects).toHaveLength(5); + for (const object of snapshots.objects) { + const stored = await env.OPERATIONS_ARCHIVE.get(object.key); + const text = await stored!.text(); + expect(text.split(".")).toHaveLength(5); + expect(text).not.toContain(PUBLISHER_DID); + expect(text).not.toContain("retained-authority-ciphertext"); + expect(text).not.toContain("repository-metadata"); + } + + const ownerHash = String(responses[0]?.["ownerHash"]); + const firstSnapshot = await env.OPERATIONS_ARCHIVE.get( + `snapshots/${ownerHash}/${ARCHIVE_ID}/000000.json.jwe`, + ); + const decrypted = await configuration.encryption.decrypt(await firstSnapshot!.text(), { + purpose: "publisher-snapshot", + objectClass: "PublisherDurableObject", + table: "operations_archive", + primaryKey: `${ARCHIVE_ID}:0`, + ownerDid: PUBLISHER_DID, + }); + const metadata = JSON.parse(new TextDecoder().decode(decrypted)); + expect(metadata).toMatchObject({ + kind: "metadata", + publisherDid: PUBLISHER_DID, + data: { delegation: { status: "active" } }, + }); + expect(JSON.stringify(metadata)).not.toContain("retained-authority-ciphertext"); + + const audit = await env.OPERATIONS_ARCHIVE.list({ prefix: `audit/${ownerHash}/` }); + expect(audit.objects.length).toBeGreaterThan(0); + for (const object of audit.objects) { + const text = await (await env.OPERATIONS_ARCHIVE.get(object.key))!.text(); + expect(text).not.toContain(PUBLISHER_DID); + expect(text).not.toContain(ADMIN.identity); + expect(text).not.toContain("retained-authority-ciphertext"); + expect(text).not.toContain("repository-metadata"); + const parsed = JSON.parse(text) as Record; + expect(Object.keys(parsed).toSorted()).toEqual(["events", "version"]); + expect(parsed["events"]).toEqual(expect.arrayContaining([{}])); + } + + const replay = await handleArchivePublisher( + request(null, 0), + "request-replay", + configuration, + { publisherDid: PUBLISHER_DID }, + ADMIN, + ); + expect(replay.status).toBe(200); + await expect(replay.json()).resolves.toMatchObject({ data: { replayed: true } }); + expect((await env.OPERATIONS_ARCHIVE.list({ prefix: "snapshots/" })).objects).toHaveLength(5); + + const notSuspended = await handleRestorePublisher( + restoreRequest(0), + "restore-not-suspended", + configuration, + { publisherDid: PUBLISHER_DID }, + ADMIN, + ); + expect(notSuspended.status).toBe(409); + await env.SERVICE_CONTROL_DO.getByName(SERVICE_CONTROL_OBJECT_NAME).setPublisherControl({ + actor: ADMIN, + idempotencyKey: "suspend-before-restore-0001", + requestDigest: "R".repeat(43), + publisherDid: PUBLISHER_DID, + status: "suspended", + reasonCode: "SHARD_RESTORE", + now: NOW + 2, + }); + const prepared = await handlePreparePublisherRestore( + prepareRestoreRequest(), + "restore-prepare", + configuration, + { publisherDid: PUBLISHER_DID }, + ADMIN, + ); + expect(prepared.status).toBe(200); + await expect(prepared.json()).resolves.toMatchObject({ + data: { archiveId: ARCHIVE_ID, publisherDid: PUBLISHER_DID, prepared: true }, + }); + await abortAllDurableObjects(); + + for (let restorePage = 0; restorePage < page; restorePage += 1) { + const response = await handleRestorePublisher( + restoreRequest(restorePage), + `restore-${restorePage}`, + configuration, + { publisherDid: PUBLISHER_DID }, + ADMIN, + ); + expect(response.status, await response.clone().text()).toBe(200); + await expect(response.json()).resolves.toMatchObject({ + data: { + page: restorePage, + nextPage: restorePage + 1, + complete: restorePage === page - 1, + authorityStatus: "reauthorization_required", + }, + }); + if (restorePage === 0) { + const prepareReplay = await handlePreparePublisherRestore( + prepareRestoreRequest(), + "restore-prepare-replay", + configuration, + { publisherDid: PUBLISHER_DID }, + ADMIN, + ); + expect(prepareReplay.status).toBe(200); + await expect(prepareReplay.json()).resolves.toMatchObject({ data: { replayed: true } }); + } + } + const restored = env.PUBLISHER_DO.getByName(PUBLISHER_DID); + await expect(restored.getOperationsMetadata(PUBLISHER_DID)).resolves.toMatchObject({ + publisher: { status: "suspended" }, + delegation: { status: "reauthorization_required" }, + }); + await expect(restored.getDelegation(PUBLISHER_DID)).resolves.toMatchObject({ + encryptedSession: "", + encryptionKeyVersion: null, + status: "reauthorization_required", + }); + await expect(restored.getWorkloadPolicy(PUBLISHER_DID, "gallery")).resolves.toMatchObject({ + active: false, + }); + await expect(restored.getIntent(PUBLISHER_DID, INTENT_ID)).resolves.toMatchObject({ + state: "failed", + stateDataJson: '{"reasonCode":"SHARD_RESTORED_REVIEW_REQUIRED"}', + }); + await expect(restored.listAuditEvents(PUBLISHER_DID, 0, 100)).resolves.toEqual( + expect.arrayContaining([ + expect.objectContaining({ eventType: "publisher-restore-completed" }), + ]), + ); + }); + + it("keeps sanitized audit exports append-only when a restored history resets sequences", async () => { + const configuration = await loadConfiguration(TEST_BINDINGS); + const publisher = env.PUBLISHER_DO.getByName(PUBLISHER_DID); + await publisher.getOperationsMetadata(PUBLISHER_DID); + await runInDurableObject(publisher, (_instance, state) => { + state.storage.sql.exec( + `INSERT INTO audit_events ( + event_type, actor_realm, actor_identity, subject, + reason_code, public_payload, created_at + ) VALUES + ('before-1', 'access', ?, ?, NULL, '{"history":"before"}', ?), + ('before-2', 'access', ?, ?, NULL, '{"history":"before"}', ?)`, + ADMIN.identity, + PUBLISHER_DID, + NOW, + ADMIN.identity, + PUBLISHER_DID, + NOW + 1, + ); + }); + + const before = await handleArchivePublisher( + request("audit:0", 0, "publisher-archive-before"), + "audit-before", + configuration, + { publisherDid: PUBLISHER_DID }, + ADMIN, + ); + expect(before.status).toBe(200); + const ownerHash = String((await before.json<{ data: { ownerHash: string } }>()).data.ownerHash); + + await runInDurableObject(publisher, (_instance, state) => { + state.storage.sql.exec("DELETE FROM audit_events"); + state.storage.sql.exec("DELETE FROM sqlite_sequence WHERE name = 'audit_events'"); + state.storage.sql.exec( + `INSERT INTO audit_events ( + event_type, actor_realm, actor_identity, subject, + reason_code, public_payload, created_at + ) VALUES + ('after-1', 'access', ?, ?, NULL, '{"history":"after"}', ?), + ('after-2', 'access', ?, ?, NULL, '{"history":"after"}', ?)`, + ADMIN.identity, + PUBLISHER_DID, + NOW + 2, + ADMIN.identity, + PUBLISHER_DID, + NOW + 3, + ); + }); + + const after = await handleArchivePublisher( + request("audit:0", 0, "publisher-archive-after"), + "audit-after", + configuration, + { publisherDid: PUBLISHER_DID }, + ADMIN, + ); + expect(after.status, await after.clone().text()).toBe(200); + const objects = await env.OPERATIONS_ARCHIVE.list({ prefix: `audit/${ownerHash}/` }); + expect(objects.objects).toHaveLength(2); + for (const object of objects.objects) { + const text = await (await env.OPERATIONS_ARCHIVE.get(object.key))!.text(); + expect(text).not.toContain(PUBLISHER_DID); + expect(text).not.toContain(ADMIN.identity); + } + }); + + it("bounds intent archive pages below the encryption plaintext limit", async () => { + const configuration = await loadConfiguration(TEST_BINDINGS); + const publisher = env.PUBLISHER_DO.getByName(PUBLISHER_DID); + await publisher.putWorkloadPolicy({ + publisherDid: PUBLISHER_DID, + packageSlug: "gallery", + repository: "example/gallery", + repositoryId: "123", + repositoryOwnerId: "456", + workflowRef: "example/gallery/.github/workflows/release.yml@refs/heads/main", + allowedRefs: ["refs/heads/main"], + allowedEnvironments: [], + active: true, + expectedVersion: null, + now: NOW, + }); + const workloadIdentityJson = maximalCanonicalObject(16 * 1024); + const releaseInputJson = maximalCanonicalObject(64 * 1024); + const stateDataJson = maximalCanonicalObject(64 * 1024); + for (let index = 0; index < 4; index += 1) { + const intentId = `${INTENT_ID.slice(0, -1)}${String(index)}`; + await publisher.createIntent({ + publisherDid: PUBLISHER_DID, + intentId, + packageSlug: "gallery", + version: `1.2.${index}`, + workloadPolicyVersion: 1, + workloadIdentityDigest: String(index).repeat(43), + workloadIdempotencyDigest: String(index + 4).repeat(43), + idempotencyKey: `github-run-${index}-attempt-1`, + requestDigest: String(index + 5).repeat(43), + workloadIdentityJson, + releaseInputJson, + expiresAt: NOW + 60_000, + now: NOW + index + 1, + }); + await runInDurableObject(publisher, (_instance, state) => { + state.storage.sql.exec( + "UPDATE intents SET state_data_json = ? WHERE id = ?", + stateDataJson, + intentId, + ); + }); + } + + const response = await handleArchivePublisher( + request("intents:", 0, "publisher-archive-large"), + "archive-large-intents", + configuration, + { publisherDid: PUBLISHER_DID }, + ADMIN, + ); + expect(response.status, await response.clone().text()).toBe(200); + }); + + it("can abort a restore whose next archive page is missing and prepare it again", async () => { + const configuration = await loadConfiguration(TEST_BINDINGS); + let cursor: string | null = null; + let page = 0; + let ownerHash = ""; + do { + const response = await handleArchivePublisher( + request(cursor, page), + `archive-for-abort-${page}`, + configuration, + { publisherDid: PUBLISHER_DID }, + ADMIN, + ); + expect(response.status).toBe(200); + const body = await response.json<{ + data: { nextCursor: string | null; nextPage: number; ownerHash: string }; + }>(); + ownerHash = body.data.ownerHash; + cursor = body.data.nextCursor; + page = body.data.nextPage; + } while (cursor !== null); + await env.SERVICE_CONTROL_DO.getByName(SERVICE_CONTROL_OBJECT_NAME).setPublisherControl({ + actor: ADMIN, + idempotencyKey: "suspend-before-restore-abort", + requestDigest: "R".repeat(43), + publisherDid: PUBLISHER_DID, + status: "suspended", + reasonCode: "SHARD_RESTORE", + now: NOW, + }); + expect( + ( + await handlePreparePublisherRestore( + prepareRestoreRequest(), + "prepare-for-abort", + configuration, + { publisherDid: PUBLISHER_DID }, + ADMIN, + ) + ).status, + ).toBe(200); + expect( + ( + await handleRestorePublisher( + restoreRequest(0), + "restore-before-abort", + configuration, + { publisherDid: PUBLISHER_DID }, + ADMIN, + ) + ).status, + ).toBe(200); + const missingPageKey = `snapshots/${ownerHash}/${ARCHIVE_ID}/000001.json.jwe`; + const missingPage = await (await env.OPERATIONS_ARCHIVE.get(missingPageKey))!.text(); + await env.OPERATIONS_ARCHIVE.delete(missingPageKey); + const missing = await handleRestorePublisher( + restoreRequest(1), + "restore-missing-page", + configuration, + { publisherDid: PUBLISHER_DID }, + ADMIN, + ); + expect(missing.status).toBe(404); + + const aborted = await handleAbortPublisherRestore( + abortRestoreRequest(), + "abort-wedged-restore", + configuration, + { publisherDid: PUBLISHER_DID }, + ADMIN, + ); + expect(aborted.status).toBe(200); + await expect(aborted.json()).resolves.toMatchObject({ + data: { archiveId: ARCHIVE_ID, publisherDid: PUBLISHER_DID, aborted: true }, + }); + const abortReplay = await handleAbortPublisherRestore( + abortRestoreRequest(), + "abort-wedged-restore-replay", + configuration, + { publisherDid: PUBLISHER_DID }, + ADMIN, + ); + expect(abortReplay.status).toBe(200); + await expect(abortReplay.json()).resolves.toMatchObject({ data: { replayed: true } }); + await env.OPERATIONS_ARCHIVE.put(missingPageKey, missingPage); + const stalePage = await handleRestorePublisher( + restoreRequest(1), + "restore-after-abort", + configuration, + { publisherDid: PUBLISHER_DID }, + ADMIN, + ); + expect(stalePage.status).toBe(409); + + const preparedAgain = await handlePreparePublisherRestore( + prepareRestoreRequest(), + "prepare-after-abort", + configuration, + { publisherDid: PUBLISHER_DID }, + ADMIN, + ); + expect(preparedAgain.status, await preparedAgain.clone().text()).toBe(200); + await expect(preparedAgain.json()).resolves.toMatchObject({ data: { replayed: false } }); + }); +}); diff --git a/apps/release-service/test/publisher-archive-workflow.test.ts b/apps/release-service/test/publisher-archive-workflow.test.ts new file mode 100644 index 0000000000..fd127f8303 --- /dev/null +++ b/apps/release-service/test/publisher-archive-workflow.test.ts @@ -0,0 +1,84 @@ +import { reset } from "cloudflare:test"; +import { env } from "cloudflare:workers"; +import { afterEach, describe, expect, it } from "vitest"; + +import type { AccessActor } from "../src/access/auth.js"; +import { handleStartPublisherArchive } from "../src/backup/workflow-route.js"; +import { loadConfiguration } from "../src/config.js"; +import { startPublisherArchiveWorkflow } from "../src/workflows/publisher-archive.js"; +import { TEST_BINDINGS } from "./fixtures/oauth.js"; + +const PUBLISHER_DID = "did:web:publisher.example.com"; +const ARCHIVE_ID = "workflow-archive-0001"; +const ADMIN: AccessActor = { + realm: "access", + identity: "admin@example.com", + email: "admin@example.com", + role: "admin", +}; + +afterEach(async () => { + await reset(); +}); + +describe("PublisherArchiveWorkflow", () => { + it("starts from the Access operator route", async () => { + const response = await handleStartPublisherArchive( + new Request(`${TEST_BINDINGS.PUBLIC_ORIGIN}/admin/api/publishers/archive/start`, { + method: "POST", + headers: { + "content-type": "application/json", + "idempotency-key": "start-publisher-archive-0001", + }, + body: JSON.stringify({ archiveId: ARCHIVE_ID }), + }), + "request-start", + await loadConfiguration(TEST_BINDINGS), + { publisherDid: PUBLISHER_DID }, + ADMIN, + { + startWorkflow: async (_workflow, params) => ({ + ok: true, + workflowId: `${params.archiveId}-workflow`, + created: true, + }), + }, + ); + + expect(response.status).toBe(202); + await expect(response.json()).resolves.toMatchObject({ + data: { + archiveId: ARCHIVE_ID, + workflowId: `${ARCHIVE_ID}-workflow`, + created: true, + }, + }); + }); + + it("resumes bounded pages to an encrypted completion manifest", async () => { + await env.PUBLISHER_DO.getByName(PUBLISHER_DID).initializePublisher(PUBLISHER_DID); + const started = await startPublisherArchiveWorkflow(env.PUBLISHER_ARCHIVE_WORKFLOW, { + publisherDid: PUBLISHER_DID, + archiveId: ARCHIVE_ID, + actorIdentity: "admin@example.com", + }); + expect(started).toMatchObject({ ok: true, created: true }); + if (!started.ok) return; + const instance = await env.PUBLISHER_ARCHIVE_WORKFLOW.get(started.workflowId); + let status = await instance.status(); + for (let attempt = 0; attempt < 100 && status.status !== "complete"; attempt += 1) { + if (status.status === "errored" || status.status === "terminated") break; + await new Promise((resolve) => setTimeout(resolve, 10)); + status = await instance.status(); + } + + expect(status.status, JSON.stringify(status.error)).toBe("complete"); + expect(status.output).toMatchObject({ + publisherDid: PUBLISHER_DID, + archiveId: ARCHIVE_ID, + pages: 4, + }); + const objects = await env.OPERATIONS_ARCHIVE.list({ prefix: "snapshots/" }); + expect(objects.objects.some((object) => object.key.endsWith("/manifest.json.jwe"))).toBe(true); + }); +}); diff --git a/apps/release-service/test/publisher-do.test.ts b/apps/release-service/test/publisher-do.test.ts index 1ba51ebcfa..a756065e82 100644 --- a/apps/release-service/test/publisher-do.test.ts +++ b/apps/release-service/test/publisher-do.test.ts @@ -104,6 +104,134 @@ describe("PublisherDurableObject", () => { ).resolves.toEqual({ ok: false, code: "PUBLISHER_SUSPENDED" }); }); + it("isolates publisher, repository, and workload admission budgets", async () => { + const stub = publisher(); + const now = 1_800_000_000_000; + const workloadKey = "W".repeat(43); + for (let index = 0; index < 30; index += 1) { + await expect( + stub.consumeIntentRateLimit({ + publisherDid: DID, + repositoryId: "123", + workloadKey, + idempotencyKey: `rate-workload-a-${String(index).padStart(4, "0")}`, + expiresAt: now + 24 * 60 * 60_000, + now, + }), + ).resolves.toMatchObject({ ok: true, replayed: false }); + } + await expect( + stub.consumeIntentRateLimit({ + publisherDid: DID, + repositoryId: "123", + workloadKey, + idempotencyKey: "rate-workload-a-over-limit", + expiresAt: now + 24 * 60 * 60_000, + now, + }), + ).resolves.toMatchObject({ ok: false, code: "RATE_LIMITED", scope: "workload" }); + await expect( + stub.consumeIntentRateLimit({ + publisherDid: DID, + repositoryId: "123", + workloadKey, + idempotencyKey: "rate-workload-a-0000", + expiresAt: now + 24 * 60 * 60_000, + now, + }), + ).resolves.toMatchObject({ ok: true, replayed: true }); + await expect( + stub.consumeIntentRateLimit({ + publisherDid: DID, + repositoryId: "123", + workloadKey: "X".repeat(43), + idempotencyKey: "rate-workload-b-0000", + expiresAt: now + 24 * 60 * 60_000, + now, + }), + ).resolves.toMatchObject({ ok: true }); + for (let index = 1; index <= 29; index += 1) { + await stub.consumeIntentRateLimit({ + publisherDid: DID, + repositoryId: "123", + workloadKey: `Y${String(index).padStart(42, "0")}`, + idempotencyKey: `rate-repository-${String(index).padStart(4, "0")}`, + expiresAt: now + 24 * 60 * 60_000, + now, + }); + } + await expect( + stub.consumeIntentRateLimit({ + publisherDid: DID, + repositoryId: "123", + workloadKey: "Q".repeat(43), + idempotencyKey: "rate-repository-over-limit", + expiresAt: now + 24 * 60 * 60_000, + now, + }), + ).resolves.toMatchObject({ ok: false, code: "RATE_LIMITED", scope: "repository" }); + for (let index = 0; index < 60; index += 1) { + await stub.consumeIntentRateLimit({ + publisherDid: DID, + repositoryId: "456", + workloadKey: `P${String(index).padStart(42, "0")}`, + idempotencyKey: `rate-publisher-${String(index).padStart(4, "0")}`, + expiresAt: now + 24 * 60 * 60_000, + now, + }); + } + await expect( + stub.consumeIntentRateLimit({ + publisherDid: DID, + repositoryId: "789", + workloadKey: "V".repeat(43), + idempotencyKey: "rate-publisher-over-limit", + expiresAt: now + 24 * 60 * 60_000, + now, + }), + ).resolves.toMatchObject({ ok: false, code: "RATE_LIMITED", scope: "publisher" }); + await expect( + env.PUBLISHER_DO.getByName(OTHER_DID).consumeIntentRateLimit({ + publisherDid: OTHER_DID, + repositoryId: "123", + workloadKey, + idempotencyKey: "rate-other-publisher-0000", + expiresAt: now + 24 * 60 * 60_000, + now, + }), + ).resolves.toMatchObject({ ok: true }); + await expect( + stub.consumeIntentRateLimit({ + publisherDid: DID, + repositoryId: "123", + workloadKey, + idempotencyKey: "rate-next-window-0000", + expiresAt: now + 24 * 60 * 60_000, + now: now + 60_000, + }), + ).resolves.toMatchObject({ ok: true, replayed: false }); + }); + + it("arms cleanup for rate-limit idempotency without another publisher operation", async () => { + const stub = publisher(); + const now = 1_800_000_000_000; + const expiresAt = now + 24 * 60 * 60_000; + await expect( + stub.consumeIntentRateLimit({ + publisherDid: DID, + repositoryId: "123", + workloadKey: "W".repeat(43), + idempotencyKey: "rate-alarm-idempotency-0001", + expiresAt, + now, + }), + ).resolves.toMatchObject({ ok: true, replayed: false }); + + await expect( + runInDurableObject(stub, (_instance, state) => state.storage.getAlarm()), + ).resolves.toBe(expiresAt); + }); + it("routes and binds one object to one publisher DID", async () => { const stub = publisher(); await stub.initializePublisher(DID); @@ -138,6 +266,7 @@ describe("PublisherDurableObject", () => { stateHash: STATE_HASH, encryptedState: "encrypted-oauth-state", encryptionKeyVersion: 2, + encryptionPurpose: "oauth-console-transaction", clientKeyId: "assertion-1", redirectTarget: "/publisher/delegation", expiresAt, @@ -204,6 +333,7 @@ describe("PublisherDurableObject", () => { stateHash: STATE_HASH, encryptedState: "encrypted-oauth-state", encryptionKeyVersion: 2, + encryptionPurpose: "oauth-console-transaction", clientKeyId: "assertion-1", redirectTarget, expiresAt: Date.now() + 60_000, @@ -222,6 +352,7 @@ describe("PublisherDurableObject", () => { stateHash, encryptedState: "encrypted", encryptionKeyVersion: 2, + encryptionPurpose: "oauth-delegation-transaction" as const, clientKeyId: "assertion-1", redirectTarget: "/callback", expiresAt, @@ -247,6 +378,95 @@ describe("PublisherDurableObject", () => { }); }); + it("pages live ciphertexts and replaces them only by compare-and-set", async () => { + const stub = publisher(); + const now = Date.now(); + await stub.putOAuthState({ + publisherDid: DID, + stateHash: STATE_HASH, + encryptedState: "oauth-ciphertext-v2", + encryptionKeyVersion: 2, + encryptionPurpose: "oauth-console-transaction", + clientKeyId: "assertion-1", + redirectTarget: "/publisher", + expiresAt: now + 60_000, + }); + await stub.putDelegation({ + publisherDid: DID, + releaseNsid: "com.emdashcms.experimental.package.release", + scope: "atproto repo:com.emdashcms.experimental.package.release?action=create", + clientKeyId: "assertion-1", + encryptedSession: "delegation-ciphertext-v2", + ...DELEGATION_METADATA, + refreshBefore: now + 60_000, + expectedVersion: null, + }); + + const first = await stub.listEncryptionRecords(DID, null, 1, now); + expect(first).toMatchObject({ + items: [ + { + cursor: "delegation:1", + envelope: "delegation-ciphertext-v2", + keyVersion: 2, + context: { purpose: "oauth-session", ownerDid: DID }, + }, + ], + nextCursor: "delegation:1", + }); + const second = await stub.listEncryptionRecords(DID, first.nextCursor, 1, now); + expect(second).toMatchObject({ + items: [ + { + cursor: `oauth-state:${STATE_HASH}`, + envelope: "oauth-ciphertext-v2", + context: { purpose: "oauth-console-transaction", ownerDid: DID }, + }, + ], + nextCursor: null, + }); + + await expect( + stub.replaceEncryptionRecord({ + publisherDid: DID, + cursor: `oauth-state:${STATE_HASH}`, + expectedEnvelope: "wrong-ciphertext", + replacementEnvelope: "oauth-ciphertext-v3", + replacementKeyVersion: 3, + actorIdentity: "operator@example.com", + now, + }), + ).resolves.toBe(false); + await expect( + stub.replaceEncryptionRecord({ + publisherDid: DID, + cursor: `oauth-state:${STATE_HASH}`, + expectedEnvelope: "oauth-ciphertext-v2", + replacementEnvelope: "oauth-ciphertext-v3", + replacementKeyVersion: 3, + actorIdentity: "operator@example.com", + now, + }), + ).resolves.toBe(true); + await runInDurableObject(stub, (instance) => { + expect(() => instance.listEncryptionRecords(DID, "not-a-cursor", 10, now)).toThrowError( + expect.objectContaining({ code: "ENCRYPTION_OPERATION_INVALID" }), + ); + }); + + const records = await stub.listEncryptionRecords(DID, null, 10, now); + expect(records.items[1]).toMatchObject({ envelope: "oauth-ciphertext-v3", keyVersion: 3 }); + const audit = await runInDurableObject(stub, (_instance, state) => + state.storage.sql + .exec<{ event_type: string; public_payload: string }>( + "SELECT event_type, public_payload FROM audit_events WHERE event_type = 'encryption-rotated'", + ) + .toArray(), + ); + expect(audit).toEqual([{ event_type: "encryption-rotated", public_payload: "{}" }]); + expect(JSON.stringify(audit)).not.toContain("ciphertext"); + }); + it("applies compare-and-set delegation updates and revocation", async () => { const stub = publisher(); const firstResult = await stub.putDelegation({ diff --git a/apps/release-service/test/publisher-intent-state.test.ts b/apps/release-service/test/publisher-intent-state.test.ts index c985bbe0eb..28e08a5480 100644 --- a/apps/release-service/test/publisher-intent-state.test.ts +++ b/apps/release-service/test/publisher-intent-state.test.ts @@ -1,4 +1,4 @@ -import { reset, runInDurableObject } from "cloudflare:test"; +import { reset, runDurableObjectAlarm, runInDurableObject } from "cloudflare:test"; import { env } from "cloudflare:workers"; import { afterEach, describe, expect, it } from "vitest"; @@ -319,6 +319,28 @@ describe("publisher release intents", () => { expect(await stub.listIntentTransitions(DID, INTENT_1)).toHaveLength(8); }); + it("expires an approval wait from the publisher alarm", async () => { + const stub = publisher(); + const now = Date.now(); + await stub.putWorkloadPolicy(policy()); + await stub.createIntent(intent({ now, expiresAt: now + 60_000 })); + await stub.transitionIntent(transition("received", 1, "verifying", { now: now + 1 })); + await stub.transitionIntent(transition("verifying", 2, "verified", { now: now + 2 })); + await stub.transitionIntent( + transition("verified", 3, "awaiting_approval", { + reasonCode: "APPROVAL_REQUIRED", + now: now + 3, + }), + ); + await runInDurableObject(stub, (_instance, state) => { + state.storage.sql.exec("UPDATE intents SET expires_at = ? WHERE id = ?", now - 1, INTENT_1); + }); + + await runDurableObjectAlarm(stub); + + await expect(stub.getIntent(DID, INTENT_1)).resolves.toMatchObject({ state: "expired" }); + }); + it("blocks suspended publishers and rejects noncanonical private input", async () => { const stub = publisher(); await stub.putWorkloadPolicy(policy()); @@ -332,7 +354,7 @@ describe("publisher release intents", () => { await runInDurableObject(stub, async (instance) => { await expect( instance.createIntent(intent({ workloadIdentityJson: '{ "runId": "100" }' })), - ).rejects.toEqual(expect.objectContaining({ code: "INTENT_INPUT_INVALID" })); + ).rejects.toMatchObject({ code: "INTENT_INPUT_INVALID" }); }); }); }); diff --git a/apps/release-service/worker-configuration.d.ts b/apps/release-service/worker-configuration.d.ts index 9240b99ecd..2152a0bc29 100644 --- a/apps/release-service/worker-configuration.d.ts +++ b/apps/release-service/worker-configuration.d.ts @@ -1,8 +1,10 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types` (hash: f7554f803601bb371e79597019eba45d) +// Generated by Wrangler by running `wrangler types` (hash: 6cedf1555243b18d28cdf49bc5763980) // Runtime types generated with workerd@1.20260815.1 2026-05-14 nodejs_compat interface __BaseEnv_Env { PUBLICATION_STAGING: R2Bucket; + OPERATIONS_ARCHIVE: R2Bucket; + OPERATIONS_METRICS: AnalyticsEngineDataset; ASSETS: Fetcher; PUBLIC_ORIGIN: ""; DEPLOYMENT_ID: ""; @@ -13,17 +15,19 @@ interface __BaseEnv_Env { OAUTH_REDIRECT_URIS: "[]"; OAUTH_ASSERTION_KEYSET: string; ENCRYPTION_KEYRING: string; + IDENTITY_DIRECTORY_DO: DurableObjectNamespace; APPROVER_DO: DurableObjectNamespace; OAUTH_STATE_DO: DurableObjectNamespace; SERVICE_CONTROL_DO: DurableObjectNamespace; PUBLISHER_DO: DurableObjectNamespace; RELEASE_VERIFIER: Fetcher /* emdash-release-verifier */; RELEASE_INTENT_WORKFLOW: Workflow[0]['payload']>; + PUBLISHER_ARCHIVE_WORKFLOW: Workflow[0]['payload']>; } declare namespace Cloudflare { interface GlobalProps { mainModule: typeof import("./src/index"); - durableNamespaces: "ApproverDurableObject" | "OAuthStateDurableObject" | "PublisherDurableObject" | "ServiceControlDurableObject"; + durableNamespaces: "ApproverDurableObject" | "IdentityDirectoryDurableObject" | "OAuthStateDurableObject" | "PublisherDurableObject" | "ServiceControlDurableObject"; } interface Env extends __BaseEnv_Env {} } diff --git a/apps/release-service/wrangler.jsonc b/apps/release-service/wrangler.jsonc index ac1ac703a8..61bc684f63 100644 --- a/apps/release-service/wrangler.jsonc +++ b/apps/release-service/wrangler.jsonc @@ -6,8 +6,18 @@ // runtime. Request handling uses stable Web Platform APIs. "compatibility_date": "2026-05-14", "compatibility_flags": ["nodejs_compat"], + "analytics_engine_datasets": [ + { + "binding": "OPERATIONS_METRICS", + "dataset": "emdash_release_service_operations", + }, + ], "durable_objects": { "bindings": [ + { + "name": "IDENTITY_DIRECTORY_DO", + "class_name": "IdentityDirectoryDurableObject", + }, { "name": "APPROVER_DO", "class_name": "ApproverDurableObject", @@ -31,6 +41,7 @@ "tag": "v1", "new_sqlite_classes": [ "ApproverDurableObject", + "IdentityDirectoryDurableObject", "OAuthStateDurableObject", "PublisherDurableObject", "ServiceControlDurableObject", @@ -48,6 +59,10 @@ "binding": "PUBLICATION_STAGING", "bucket_name": "emdash-release-service-publication-staging", }, + { + "binding": "OPERATIONS_ARCHIVE", + "bucket_name": "emdash-release-service-operations", + }, ], "workflows": [ { @@ -55,6 +70,11 @@ "name": "emdash-release-intent", "class_name": "ReleaseIntentWorkflow", }, + { + "binding": "PUBLISHER_ARCHIVE_WORKFLOW", + "name": "emdash-publisher-archive", + "class_name": "PublisherArchiveWorkflow", + }, ], "assets": { "directory": "./dist/client", diff --git a/docs/technical-specs/delegated-release-service-operations.md b/docs/technical-specs/delegated-release-service-operations.md new file mode 100644 index 0000000000..a797001e94 --- /dev/null +++ b/docs/technical-specs/delegated-release-service-operations.md @@ -0,0 +1,277 @@ +# Delegated release service operations + +This runbook covers self-host deployment, routine maintenance, and incident recovery for the delegated release service. The service is experimental and must not be deployed until the complete implementation stack and conformance gates have been accepted. + +## Operational invariants + +- Publisher and approver Durable Objects are authoritative for retained authority and decisions. +- The identity directory is a non-authoritative projection split across 256 Durable Objects. Deleting it does not change authority, release state, or approval state. +- The initial deployment does not use D1. The operator console performs direct DID lookup and uses the sharded identity directory for fleet maintenance. +- R2 snapshot pages are encrypted before storage. Audit export objects contain only the sanitized `audit_events.public_payload` contract. +- Restore requires a suspended publisher and a complete, decryptable archive manifest. +- Restore clears retained OAuth authority, disables workload policies, and converts nonterminal intents to `failed`. A publisher must reauthorize before publication resumes. +- Operators use supported Access routes and clients. Runbooks never require direct Durable Object SQLite edits. + +## Cloudflare resources + +The release-service Worker expects the following resources. + +| Binding | Resource | Purpose | +| ---------------------------- | -------------------------------- | --------------------------------------------------------------------------------------------- | +| `PUBLISHER_DO` | `PublisherDurableObject` | Per-publisher delegation, workload, intent, publication, audit, restore, and rate-limit state | +| `APPROVER_DO` | `ApproverDurableObject` | Per-approver sessions, passkeys, decisions, audit, and encrypted OAuth transactions | +| `SERVICE_CONTROL_DO` | `ServiceControlDurableObject` | Global pause mode, publisher suspension, publication permits, and operator audit | +| `IDENTITY_DIRECTORY_DO` | `IdentityDirectoryDurableObject` | Non-authoritative publisher and approver inventory, sharded by DID hash | +| `RELEASE_INTENT_WORKFLOW` | Workflow | Verification, approval wait, publication, and reconciliation | +| `PUBLISHER_ARCHIVE_WORKFLOW` | Workflow | Bounded, retryable publisher snapshot and audit export | +| `RELEASE_VERIFIER` | Service binding | Isolated artifact and provenance verification | +| `OPERATIONS_ARCHIVE` | R2 bucket | Encrypted publisher snapshot pages and append-only sanitized audit pages | +| `OPERATIONS_METRICS` | Analytics Engine dataset | Privacy-safe operational alert events | +| `ASSETS` | Worker static assets | Publisher, approver, and Access operator web surfaces | + +The initial Durable Object migration tag is `v1`. It contains every class and table required before the first deployment. + +## Configure a self-hosted deployment + +### Public origin and OAuth + +Set the following non-secret variables in `apps/release-service/wrangler.jsonc`: + +- `PUBLIC_ORIGIN`: the canonical HTTPS custom origin, without a trailing path; +- `DEPLOYMENT_ID`: a stable identifier that remains unchanged across deployments and key rotations; +- `OAUTH_REDIRECT_URIS`: a JSON array containing `${PUBLIC_ORIGIN}/oauth/callback`; +- `ACCESS_TEAM_DOMAIN`: the exact HTTPS issuer origin for Cloudflare Access, including a custom Access hostname when configured, without a port or path; +- `ACCESS_VIEWER_AUD`, `ACCESS_REVIEWER_AUD`, and `ACCESS_ADMIN_AUD`: the distinct Access application audiences described below. + +Changing `DEPLOYMENT_ID` makes existing encryption envelopes unreadable because the deployment identifier is part of the authenticated encryption context. + +### Cloudflare Access + +Create Access applications whose path specificity supplies the audience required by each route family. + +| Audience | Paths | Capability | +| -------- | -------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------- | +| Viewer | `/admin*`, `/admin/api/status`, `/admin/api/directory`, read-only publisher and audit routes | Load the operator console and inspect state | +| Reviewer | `/admin/api/intents/*` | Cancel or reconcile release intents | +| Admin | `/admin/api/pause`, `/admin/api/publishers/*`, `/admin/api/approvers/*` | Pause publication, suspend or revoke publishers, rotate keys, archive, and restore | + +The Worker verifies the Access JWT issuer and the route-specific audience. Access group claims do not grant a role inside the Worker. + +### Secrets + +Set the assertion key set and encryption keyring as Worker secrets. Do not pass their values as command arguments. + +```sh +cd apps/release-service +pnpm exec wrangler secret put OAUTH_ASSERTION_KEYSET +pnpm exec wrangler secret put ENCRYPTION_KEYRING +``` + +`OAUTH_ASSERTION_KEYSET` contains the active confidential-client assertion key and any previous public keys still needed by an authorization server. `ENCRYPTION_KEYRING` contains the active encryption key and retained decryption keys. + +The base configuration uses Worker secret bindings. `loadConfiguration()` also accepts `SecretsStoreSecret` bindings and resolves each value with `get()` on every load, so key rotation is visible to the configuration cache. Define Secrets Store bindings in a deployment-specific Wrangler environment. Do not commit a Secrets Store ID to the reusable base configuration. + +### R2 and verifier + +Create the private operations bucket before deploying the service. + +```sh +pnpm exec wrangler r2 bucket create emdash-release-service-operations +``` + +Deploy the release verifier under the service name `emdash-release-verifier` before the release-service Worker. The release-service Worker calls it through the `RELEASE_VERIFIER` service binding rather than public HTTP. + +### Validate the deployment artifact + +Generate binding types, run the Worker and UI tests, and build the production artifact before deployment. + +```sh +cd apps/release-service +pnpm exec wrangler types --check +pnpm test +pnpm build +pnpm exec wrangler deploy --dry-run +``` + +After deployment, `GET /health` must return `200` without loading configuration. `GET /ready` must return `200` only after configuration and the service-control Durable Object initialize successfully. + +## Operations directory + +Successful publisher and approver OAuth callbacks register the DID in one of 256 directory Durable Objects. Directory registration failure emits `directory_failure` but does not block OAuth or create authority. + +The operator console skips empty partitions when listing publishers or approvers. API clients can resume one partition at a time with `ReleaseServiceOperatorClient.listDirectory()`. Fleet operations must retain the returned cursor until it becomes absent. + +Rebuild a missing directory as publishers and approvers complete OAuth again. Directory rows are not evidence of active delegation or approver eligibility; query each authoritative shard before acting. + +## Rotate encryption keys + +### Routine rotation + +1. Pause publication in the operator console. +2. Add the new key version to `ENCRYPTION_KEYRING`, retain every old key, and set `current` to the new version. +3. Deploy the keyring change without removing old keys. +4. Enumerate every publisher and approver from the operations directory. +5. For each DID, run the relevant rotation operation from an empty cursor until it reports `Verified`. A compare-and-set race changes the resume cursor to a required rescan, so completion cannot discard a race reported by an earlier page. +6. Repeat a full scan from an empty cursor. Every page must report the new target version, zero races, and completion. +7. Confirm that no `refresh_failure`, `archive_gap`, or `restore_failure` event appeared during the scan. +8. Remove the retired key version from `ENCRYPTION_KEYRING` and deploy the reduced keyring. +9. Run another full verification scan. Missing retained key material must fail with `ENCRYPTION_OPERATION_FAILED`. +10. Restore the previous service mode. + +Rotation decrypts and re-encrypts outside the Durable Object storage transaction. Each replacement uses a ciphertext compare-and-set, so concurrent refresh or OAuth completion wins safely and appears as a race that requires another scan. + +### Compromised encryption key + +1. Pause publication immediately. +2. Revoke affected publisher delegation when retained state cannot be trusted. +3. Activate a new encryption key while retaining the compromised key only for the bounded rotation window. +4. Rotate and verify every directory entry. +5. Remove the compromised key after a complete zero-race verification pass. +6. Require reauthorization for every publisher whose ciphertext could not be proved readable and authentic. + +## Archive publisher shards + +Use `Start archive workflow` in the operator console or `ReleaseServiceOperatorClient.startPublisherArchive()`. The Workflow writes bounded pages in this order: + +1. sanitized publisher and delegation metadata; +2. disabled-capable workload policy records; +3. canonical intent rows; +4. sanitized audit events; +5. an encrypted completion manifest. + +Snapshot objects use the following prefix: + +```text +snapshots/{publisherHash}/{archiveId}/ +``` + +Sanitized audit pages use this prefix and never overwrite an existing sequence range: + +```text +audit/{publisherHash}/{firstSequence}-{lastSequence}-{contentDigest}.json +``` + +The content digest keeps audit histories append-only when recovery resets a shard's local audit sequence. Snapshot writes use create-only R2 conditions. A retried page decrypts and compares the existing object; different content at the same key fails with `ARCHIVE_OPERATION_FAILED`. + +## Restore a publisher shard + +Preparing a restore deletes publisher state. Confirm the DID and archive ID before continuing. + +1. Pause publication or suspend the publisher through the operator console. +2. Confirm that the selected archive has an encrypted completion manifest. +3. Call `ReleaseServiceOperatorClient.preparePublisherRestore()`. The API checks global suspension, local suspension, exact DID confirmation, and manifest decryption before clearing the shard. +4. Apply pages in ascending order with `ReleaseServiceOperatorClient.restorePublisher()`. +5. Continue until `complete` is true. +6. Confirm the restored publisher remains suspended. +7. Confirm delegation is `reauthorization_required` with no retained ciphertext. +8. Review every restored nonterminal intent. Restore changes it to `failed` with `SHARD_RESTORED_REVIEW_REQUIRED`. +9. Ask the publisher to reauthorize and re-enable each workload policy explicitly. +10. Reconcile any release that may have reached the PDS before the shard was lost. +11. Remove publisher suspension only after reauthorization and reconciliation complete. + +If a missing or corrupt page prevents completion, call `ReleaseServiceOperatorClient.abortPublisherRestore()` with the same publisher DID and archive ID. The operation marks the active restore as aborted and leaves the publisher suspended. A later `preparePublisherRestore()` call clears the abandoned partial state before starting another restore attempt. A completed restore cannot be aborted. + +Archive audit pages remain in R2. Restore begins a new shard audit history with `publisher-restore-prepared`, `publisher-restore-started`, per-intent restore events, and `publisher-restore-completed`. + +## Incident runbooks + +### Compromised Access operator identity + +1. Remove the identity from every Access policy. +2. Rotate the affected Access application credentials and review Access logs. +3. Pause publication when the identity could reach an admin audience. +4. Review service-control and publisher audit events for the operator subject. +5. Revoke publisher authority changed by the identity unless each action can be independently validated. + +### Publisher-requested revocation + +1. Revoke publisher authority from the operator console. +2. Confirm delegation status is `revoked` and encrypted session state is empty. +3. Confirm publisher application sessions were invalidated. +4. Keep listing and moderation state unchanged; revocation controls future publication authority only. + +### Authorization-server or PDS outage + +1. Pause admission when new authorization or refresh requests fail broadly. +2. Pause publication when refresh or PDS writes cannot be distinguished from partial completion. +3. Do not replace exact create-only scope with a broader permission. +4. Resume after the provider succeeds and retained sessions pass refresh verification. + +### Ambiguous PDS write + +1. Keep the intent in `reconciling`. +2. Query the deterministic release record key directly from the publisher PDS. +3. Accept the exact expected record as published. +4. Retry only after confirmed absence and a fresh publication permit. +5. Mark a different record at the deterministic key as `conflict`. + +### Workflow loss or prolonged retry + +1. Inspect the authoritative publisher intent state. +2. Restart only `ready` or `reconciling` intents through the operator reconciliation operation. +3. Do not reconstruct authority from Workflow state. +4. Archive the publisher shard before destructive recovery. + +### Verifier failure + +1. Pause admission when verifier failures affect unrelated publishers. +2. Keep failed verification terminal for the supplied input. +3. Verify the service binding and verifier egress policy. +4. Resume only after substituted artifact, provenance, source, builder, and commit cases still fail closed. + +### Durable Object schema initialization failure + +1. Pause admission and publication. +2. Preserve the failing Worker version and structured error evidence. +3. Do not edit Durable Object SQLite directly. +4. Roll back the Worker when the previous version accepts the existing initial schema. +5. For a lost or corrupt publisher shard, use encrypted archive preparation and restore. + +### Passkey compromise or counter anomaly + +1. Revoke the credential from the approver account. +2. Invalidate outstanding approval challenges. +3. Review decisions made with the credential ID. +4. Require a new user-verified credential. Never reset a decreasing signature counter. + +### Hosted-service rollback + +1. Pause admission and publication. +2. Roll back to an application version compatible with the current Durable Object schema and encryption profile. +3. Keep new encryption keys available while any ciphertext uses them. +4. Run readiness, key verification, archive, and reconciliation checks before resuming. + +## Alerts + +`OPERATIONS_METRICS` writes privacy-safe Analytics Engine points with this layout: + +| Position | Value | +| --------- | ----------------------------------- | +| `index1` | Publisher/workload hash or `global` | +| `blob1` | Event name | +| `blob2` | Outcome or error code | +| `blob3` | Scope | +| `blob4` | Request ID | +| `double1` | Event value, normally `1` | +| `double2` | Unix time in milliseconds | + +Configure alert queries for these event names: + +| Event | Required response | +| ------------------------- | --------------------------------------------------------------- | +| `publication_paused` | Confirm the incident owner and reason immediately | +| `refresh_failure` | Check authorization-server health and retained key availability | +| `reconciliation_required` | Monitor backlog age and deterministic PDS outcomes | +| `verifier_failure` | Check verifier availability and failure-code distribution | +| `access_denied` | Investigate spikes by audience and request path logs | +| `archive_gap` | Resume the failed archive page and verify the manifest | +| `restore_failure` | Keep the publisher suspended and inspect archive/page ordering | +| `configuration_failure` | Keep readiness failed and correct variables or secrets | +| `directory_failure` | Repair projection registration without changing authority | +| `intent_rate_limited` | Review workload, repository, and publisher abuse patterns | + +Alert delivery is deployment-specific. A production launch requires tested notification routing and an on-call owner for every event above. + +## Conformance after deployment + +Run the same G0 create-only, refresh, and revocation probes against npmX and Cirrus. Then run the service conformance suite against the hosted and self-hosted origins. Deployment is not complete until both origins publish the same fixture and a clean installer independently verifies and installs it. diff --git a/packages/registry-client/src/release-service/index.ts b/packages/registry-client/src/release-service/index.ts index 07c6374f00..1a43876cc1 100644 --- a/packages/registry-client/src/release-service/index.ts +++ b/packages/registry-client/src/release-service/index.ts @@ -3,12 +3,24 @@ import type { PackageRelease } from "@emdash-cms/registry-lexicons"; import { parseDelegatedReleaseSourceRecord } from "./source-record.js"; import { TERMINAL_RELEASE_INTENT_STATES, + type AbortPublisherRestoreResult, type CursorPage, type DelegationResource, + type DirectoryIdentityKind, + type DirectoryIdentityResource, + type DirectoryListOptions, + type EncryptionRotationPageInput, + type EncryptionRotationResult, type MutationResult, type OperatorPublisherResource, + type PreparePublisherRestoreResult, + type PublisherArchiveKind, + type PublisherArchivePageInput, + type PublisherArchivePageResult, type PublisherControlResource, type PublisherResource, + type PublisherRestorePageInput, + type PublisherRestorePageResult, type PutWorkloadPolicyInput, type ReleaseIntentResource, type ReleaseIntentResult, @@ -16,6 +28,7 @@ import { type ReleaseServiceApiErrorCode, type ReleaseServiceClientErrorCode, type ServiceControlState, + type StartPublisherArchiveResult, type SubmitReleaseIntentInput, type SubmitReleaseIntentResult, type WorkloadPolicyResource, @@ -32,12 +45,24 @@ export type { export { parseDelegatedReleaseSourceRecord } from "./source-record.js"; export type { + AbortPublisherRestoreResult, CursorPage, DelegationResource, + DirectoryIdentityKind, + DirectoryIdentityResource, + DirectoryListOptions, + EncryptionRotationPageInput, + EncryptionRotationResult, MutationResult, OperatorPublisherResource, + PreparePublisherRestoreResult, + PublisherArchiveKind, + PublisherArchivePageInput, + PublisherArchivePageResult, PublisherControlResource, PublisherResource, + PublisherRestorePageInput, + PublisherRestorePageResult, PutWorkloadPolicyInput, ReleaseIntentResource, ReleaseIntentResult, @@ -45,6 +70,7 @@ export type { ReleaseServiceApiErrorCode, ReleaseServiceClientErrorCode, ServiceControlState, + StartPublisherArchiveResult, SubmitReleaseIntentInput, SubmitReleaseIntentResult, WorkloadPolicyResource, @@ -60,6 +86,8 @@ const IDEMPOTENCY_KEY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{15,127}$/; const IDEMPOTENCY_PREFIX_PATTERN = /[^A-Za-z0-9._:-]/g; const DIGITS_PATTERN = /^[0-9]+$/; const CSRF_TOKEN_PATTERN = /^[A-Za-z0-9_-]{43}$/; +const ARCHIVE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]{15,63}$/; +const DIRECTORY_SHARD_PATTERN = /^[0-9a-f]{2}$/; const API_ERROR_CODES: Readonly> = { ACCESS_DENIED: true, ACCESS_AUTH_INVALID: true, @@ -67,6 +95,7 @@ const API_ERROR_CODES: Readonly> = { APPROVAL_INVALID: true, APPROVER_SESSION_INVALID: true, APPROVER_SUSPENDED: true, + ARCHIVE_OPERATION_FAILED: true, AUTH_INVALID: true, CONFIGURATION_ERROR: true, CREDENTIAL_LIMIT_REACHED: true, @@ -74,6 +103,7 @@ const API_ERROR_CODES: Readonly> = { CREDENTIAL_REVOKED: true, CSRF_INVALID: true, DELEGATION_REQUIRED: true, + ENCRYPTION_OPERATION_FAILED: true, IDEMPOTENCY_KEY_INVALID: true, IDEMPOTENCY_CONFLICT: true, INTERNAL_ERROR: true, @@ -89,11 +119,13 @@ const API_ERROR_CODES: Readonly> = { PUBLISHER_SESSION_INVALID: true, PUBLISHER_SUSPENDED: true, RELEASE_EXISTS: true, + RESTORE_OPERATION_FAILED: true, SERVICE_PAUSED: true, SERVICE_UNAVAILABLE: true, VERSION_RESERVED: true, WORKFLOW_UNAVAILABLE: true, WORKLOAD_NOT_ALLOWED: true, + WORKLOAD_RATE_LIMITED: true, }; const RETRYABLE_ERROR_CODES: ReadonlySet = new Set([ "CONFIGURATION_ERROR", @@ -104,6 +136,7 @@ const RETRYABLE_ERROR_CODES: ReadonlySet = new Se "SERVICE_PAUSED", "SERVICE_UNAVAILABLE", "WORKFLOW_UNAVAILABLE", + "WORKLOAD_RATE_LIMITED", ]); const INTENT_STATES: Readonly> = { received: true, @@ -829,6 +862,267 @@ function parsePage(value: unknown, parseItem: (item: unknown) => T): CursorPa }; } +function parseDirectoryIdentity(value: unknown): DirectoryIdentityResource { + if (!isRecord(value)) throw invalidResponse(); + const did = stringValue(value, "did"); + const shard = stringValue(value, "shard"); + const registeredAt = safeInteger(value, "registeredAt"); + const lastSeenAt = safeInteger(value, "lastSeenAt"); + if ( + (value["kind"] !== "approver" && value["kind"] !== "publisher") || + !did || + !DID_PATTERN.test(did) || + !shard || + !DIRECTORY_SHARD_PATTERN.test(shard) || + registeredAt === null || + registeredAt < 0 || + lastSeenAt === null || + lastSeenAt < registeredAt + ) { + throw invalidResponse(); + } + return { kind: value["kind"], did, shard, registeredAt, lastSeenAt }; +} + +function rotationPageInput(value: EncryptionRotationPageInput): EncryptionRotationPageInput { + if ( + (value.afterCursor !== null && + (typeof value.afterCursor !== "string" || value.afterCursor.length === 0)) || + !Number.isSafeInteger(value.limit) || + value.limit < 1 || + value.limit > 100 + ) { + throw new ReleaseServiceError({ + code: "CLIENT_RESPONSE_INVALID", + message: "Encryption rotation page is invalid", + }); + } + return value; +} + +function parseEncryptionRotation(value: unknown): EncryptionRotationResult { + if (!isRecord(value)) throw invalidResponse(); + const ownerDid = stringValue(value, "ownerDid"); + const targetKeyVersion = safeInteger(value, "targetKeyVersion"); + const scanned = safeInteger(value, "scanned"); + const rotated = safeInteger(value, "rotated"); + const raced = safeInteger(value, "raced"); + const nextCursor = value["nextCursor"]; + if ( + !ownerDid || + !DID_PATTERN.test(ownerDid) || + targetKeyVersion === null || + targetKeyVersion < 1 || + scanned === null || + scanned < 0 || + rotated === null || + rotated < 0 || + raced === null || + raced < 0 || + rotated + raced > scanned || + (nextCursor !== null && typeof nextCursor !== "string") || + typeof value["complete"] !== "boolean" + ) { + throw invalidResponse(); + } + return { + ownerDid, + targetKeyVersion, + scanned, + rotated, + raced, + nextCursor, + complete: value["complete"], + }; +} + +function archivePageInput(value: PublisherArchivePageInput): PublisherArchivePageInput { + if ( + !ARCHIVE_ID_PATTERN.test(value.archiveId) || + (value.cursor !== null && (typeof value.cursor !== "string" || value.cursor.length === 0)) || + !Number.isSafeInteger(value.page) || + value.page < 0 || + value.page > 999_999 + ) { + throw new ReleaseServiceError({ + code: "CLIENT_RESPONSE_INVALID", + message: "Publisher archive page is invalid", + }); + } + return value; +} + +function isPublisherArchiveKind(value: unknown): value is PublisherArchiveKind { + return ( + value === "audit-events" || + value === "intents" || + value === "metadata" || + value === "workload-policies" + ); +} + +function parsePublisherArchivePage(value: unknown): PublisherArchivePageResult { + if (!isRecord(value)) throw invalidResponse(); + const archiveId = stringValue(value, "archiveId"); + const ownerHash = stringValue(value, "ownerHash"); + const page = safeInteger(value, "page"); + const nextPage = safeInteger(value, "nextPage"); + const nextCursor = value["nextCursor"]; + if ( + !archiveId || + !ARCHIVE_ID_PATTERN.test(archiveId) || + !ownerHash || + !CSRF_TOKEN_PATTERN.test(ownerHash) || + page === null || + page < 0 || + nextPage === null || + nextPage !== page + 1 || + !isPublisherArchiveKind(value["kind"]) || + (nextCursor !== null && typeof nextCursor !== "string") || + typeof value["replayed"] !== "boolean" || + typeof value["complete"] !== "boolean" || + typeof value["manifestWritten"] !== "boolean" || + value["complete"] !== (nextCursor === null) || + (value["manifestWritten"] && !value["complete"]) + ) { + throw invalidResponse(); + } + return { + archiveId, + ownerHash, + page, + kind: value["kind"], + nextCursor, + nextPage, + replayed: value["replayed"], + complete: value["complete"], + manifestWritten: value["manifestWritten"], + }; +} + +function parseStartedPublisherArchive(value: unknown): StartPublisherArchiveResult { + if (!isRecord(value)) throw invalidResponse(); + const archiveId = stringValue(value, "archiveId"); + const workflowId = stringValue(value, "workflowId"); + if ( + !archiveId || + !ARCHIVE_ID_PATTERN.test(archiveId) || + !workflowId || + !CSRF_TOKEN_PATTERN.test(workflowId) || + typeof value["created"] !== "boolean" + ) { + throw invalidResponse(); + } + return { archiveId, workflowId, created: value["created"] }; +} + +function restorePageInput(value: PublisherRestorePageInput): PublisherRestorePageInput { + if ( + !ARCHIVE_ID_PATTERN.test(value.archiveId) || + !Number.isSafeInteger(value.page) || + value.page < 0 || + value.page > 999_999 + ) { + throw new ReleaseServiceError({ + code: "CLIENT_RESPONSE_INVALID", + message: "Publisher restore page is invalid", + }); + } + return value; +} + +function parsePublisherRestorePage(value: unknown): PublisherRestorePageResult { + if (!isRecord(value)) throw invalidResponse(); + const archiveId = stringValue(value, "archiveId"); + const ownerHash = stringValue(value, "ownerHash"); + const page = safeInteger(value, "page"); + const nextPage = safeInteger(value, "nextPage"); + const totalPages = safeInteger(value, "totalPages"); + if ( + !archiveId || + !ARCHIVE_ID_PATTERN.test(archiveId) || + !ownerHash || + !CSRF_TOKEN_PATTERN.test(ownerHash) || + page === null || + page < 0 || + nextPage === null || + nextPage < page + 1 || + totalPages === null || + totalPages < 1 || + nextPage > totalPages || + !isPublisherArchiveKind(value["kind"]) || + typeof value["replayed"] !== "boolean" || + typeof value["complete"] !== "boolean" || + value["authorityStatus"] !== "reauthorization_required" || + value["complete"] !== (nextPage === totalPages) + ) { + throw invalidResponse(); + } + return { + archiveId, + ownerHash, + page, + kind: value["kind"], + nextPage, + totalPages, + replayed: value["replayed"], + complete: value["complete"], + authorityStatus: "reauthorization_required", + }; +} + +function parsePreparedPublisherRestore(value: unknown): PreparePublisherRestoreResult { + if (!isRecord(value)) throw invalidResponse(); + const archiveId = stringValue(value, "archiveId"); + const publisherDid = stringValue(value, "publisherDid"); + const deletedIntents = safeInteger(value, "deletedIntents"); + const deletedWorkloads = safeInteger(value, "deletedWorkloads"); + if ( + !archiveId || + !ARCHIVE_ID_PATTERN.test(archiveId) || + !publisherDid || + !DID_PATTERN.test(publisherDid) || + value["prepared"] !== true || + typeof value["replayed"] !== "boolean" || + deletedIntents === null || + deletedIntents < 0 || + deletedWorkloads === null || + deletedWorkloads < 0 + ) { + throw invalidResponse(); + } + return { + archiveId, + publisherDid, + prepared: true, + deletedIntents, + deletedWorkloads, + replayed: value["replayed"], + }; +} + +function parseAbortedPublisherRestore(value: unknown): AbortPublisherRestoreResult { + if (!isRecord(value)) throw invalidResponse(); + const archiveId = stringValue(value, "archiveId"); + const publisherDid = stringValue(value, "publisherDid"); + if ( + !archiveId || + !ARCHIVE_ID_PATTERN.test(archiveId) || + !publisherDid || + !DID_PATTERN.test(publisherDid) || + value["aborted"] !== true || + typeof value["replayed"] !== "boolean" + ) { + throw invalidResponse(); + } + return { + archiveId, + publisherDid, + aborted: true, + replayed: value["replayed"], + }; +} + export class ReleaseServiceOperatorClient extends BaseReleaseServiceClient { #mutationHeaders(idempotencyKey: string): Headers { return new Headers({ @@ -849,6 +1143,32 @@ export class ReleaseServiceOperatorClient extends BaseReleaseServiceClient { ); } + async listDirectory( + kind: DirectoryIdentityKind, + options: DirectoryListOptions & RequestOptions = {}, + ): Promise> { + if ( + (kind !== "approver" && kind !== "publisher") || + (options.cursor !== undefined && options.cursor.length === 0) || + (options.limit !== undefined && + (!Number.isSafeInteger(options.limit) || options.limit < 1 || options.limit > 100)) + ) { + throw new ReleaseServiceError({ + code: "CLIENT_RESPONSE_INVALID", + message: "Directory list request is invalid", + }); + } + const url = new URL("/admin/api/directory", this.serviceUrl); + url.searchParams.set("kind", kind); + if (options.cursor !== undefined) url.searchParams.set("cursor", options.cursor); + if (options.limit !== undefined) url.searchParams.set("limit", String(options.limit)); + return await this.call( + `${url.pathname}${url.search}`, + { method: "GET", credentials: "include", signal: options.signal }, + (value) => parsePage(value, parseDirectoryIdentity), + ); + } + async setMode( mode: ServiceControlState["mode"], reasonCode: string | null, @@ -929,6 +1249,150 @@ export class ReleaseServiceOperatorClient extends BaseReleaseServiceClient { ); } + async archivePublisher( + publisherDid: string, + page: PublisherArchivePageInput, + options: MutationOptions, + ): Promise { + return await this.call( + `/admin/api/publishers/${encodeURIComponent(publisherDid)}/archive`, + { + method: "POST", + credentials: "include", + headers: this.#mutationHeaders(options.idempotencyKey), + body: JSON.stringify(archivePageInput(page)), + signal: options.signal, + }, + parsePublisherArchivePage, + ); + } + + async startPublisherArchive( + publisherDid: string, + archiveId: string, + options: MutationOptions, + ): Promise { + if (!ARCHIVE_ID_PATTERN.test(archiveId)) { + throw new ReleaseServiceError({ + code: "CLIENT_RESPONSE_INVALID", + message: "Publisher archive ID is invalid", + }); + } + return await this.call( + `/admin/api/publishers/${encodeURIComponent(publisherDid)}/archive/start`, + { + method: "POST", + credentials: "include", + headers: this.#mutationHeaders(options.idempotencyKey), + body: JSON.stringify({ archiveId }), + signal: options.signal, + }, + parseStartedPublisherArchive, + ); + } + + async restorePublisher( + publisherDid: string, + page: PublisherRestorePageInput, + options: MutationOptions, + ): Promise { + return await this.call( + `/admin/api/publishers/${encodeURIComponent(publisherDid)}/restore`, + { + method: "POST", + credentials: "include", + headers: this.#mutationHeaders(options.idempotencyKey), + body: JSON.stringify(restorePageInput(page)), + signal: options.signal, + }, + parsePublisherRestorePage, + ); + } + + async preparePublisherRestore( + publisherDid: string, + archiveId: string, + options: MutationOptions, + ): Promise { + if (!ARCHIVE_ID_PATTERN.test(archiveId)) { + throw new ReleaseServiceError({ + code: "CLIENT_RESPONSE_INVALID", + message: "Publisher archive ID is invalid", + }); + } + return await this.call( + `/admin/api/publishers/${encodeURIComponent(publisherDid)}/restore/prepare`, + { + method: "POST", + credentials: "include", + headers: this.#mutationHeaders(options.idempotencyKey), + body: JSON.stringify({ archiveId, confirmPublisherDid: publisherDid }), + signal: options.signal, + }, + parsePreparedPublisherRestore, + ); + } + + async abortPublisherRestore( + publisherDid: string, + archiveId: string, + options: MutationOptions, + ): Promise { + if (!ARCHIVE_ID_PATTERN.test(archiveId)) { + throw new ReleaseServiceError({ + code: "CLIENT_RESPONSE_INVALID", + message: "Publisher archive ID is invalid", + }); + } + return await this.call( + `/admin/api/publishers/${encodeURIComponent(publisherDid)}/restore/abort`, + { + method: "POST", + credentials: "include", + headers: this.#mutationHeaders(options.idempotencyKey), + body: JSON.stringify({ archiveId, confirmPublisherDid: publisherDid }), + signal: options.signal, + }, + parseAbortedPublisherRestore, + ); + } + + async rotatePublisherEncryption( + publisherDid: string, + page: EncryptionRotationPageInput, + options: MutationOptions, + ): Promise { + return await this.call( + `/admin/api/publishers/${encodeURIComponent(publisherDid)}/encryption/rotate`, + { + method: "POST", + credentials: "include", + headers: this.#mutationHeaders(options.idempotencyKey), + body: JSON.stringify(rotationPageInput(page)), + signal: options.signal, + }, + parseEncryptionRotation, + ); + } + + async rotateApproverEncryption( + approverDid: string, + page: EncryptionRotationPageInput, + options: MutationOptions, + ): Promise { + return await this.call( + `/admin/api/approvers/${encodeURIComponent(approverDid)}/encryption/rotate`, + { + method: "POST", + credentials: "include", + headers: this.#mutationHeaders(options.idempotencyKey), + body: JSON.stringify(rotationPageInput(page)), + signal: options.signal, + }, + parseEncryptionRotation, + ); + } + async cancelIntent( publisherDid: string, intentId: string, diff --git a/packages/registry-client/src/release-service/types.ts b/packages/registry-client/src/release-service/types.ts index 6e26dcd910..48c9e199f6 100644 --- a/packages/registry-client/src/release-service/types.ts +++ b/packages/registry-client/src/release-service/types.ts @@ -23,6 +23,7 @@ export type ReleaseServiceApiErrorCode = | "APPROVAL_INVALID" | "APPROVER_SESSION_INVALID" | "APPROVER_SUSPENDED" + | "ARCHIVE_OPERATION_FAILED" | "AUTH_INVALID" | "CONFIGURATION_ERROR" | "CREDENTIAL_LIMIT_REACHED" @@ -30,6 +31,7 @@ export type ReleaseServiceApiErrorCode = | "CREDENTIAL_REVOKED" | "CSRF_INVALID" | "DELEGATION_REQUIRED" + | "ENCRYPTION_OPERATION_FAILED" | "IDEMPOTENCY_KEY_INVALID" | "IDEMPOTENCY_CONFLICT" | "INTERNAL_ERROR" @@ -45,11 +47,13 @@ export type ReleaseServiceApiErrorCode = | "PUBLISHER_SESSION_INVALID" | "PUBLISHER_SUSPENDED" | "RELEASE_EXISTS" + | "RESTORE_OPERATION_FAILED" | "SERVICE_PAUSED" | "SERVICE_UNAVAILABLE" | "VERSION_RESERVED" | "WORKFLOW_UNAVAILABLE" - | "WORKLOAD_NOT_ALLOWED"; + | "WORKLOAD_NOT_ALLOWED" + | "WORKLOAD_RATE_LIMITED"; export type ReleaseServiceClientErrorCode = | ReleaseServiceApiErrorCode @@ -153,6 +157,95 @@ export interface OperatorPublisherResource extends PublisherResource { control: PublisherControlResource; } +export type DirectoryIdentityKind = "approver" | "publisher"; + +export interface DirectoryIdentityResource { + kind: DirectoryIdentityKind; + did: string; + shard: string; + registeredAt: number; + lastSeenAt: number; +} + +export interface DirectoryListOptions { + cursor?: string; + limit?: number; +} + +export interface EncryptionRotationPageInput { + afterCursor: string | null; + limit: number; +} + +export interface EncryptionRotationResult { + ownerDid: string; + targetKeyVersion: number; + scanned: number; + rotated: number; + raced: number; + nextCursor: string | null; + complete: boolean; +} + +export type PublisherArchiveKind = "audit-events" | "intents" | "metadata" | "workload-policies"; + +export interface PublisherArchivePageInput { + archiveId: string; + cursor: string | null; + page: number; +} + +export interface PublisherArchivePageResult { + archiveId: string; + ownerHash: string; + page: number; + kind: PublisherArchiveKind; + nextCursor: string | null; + nextPage: number; + replayed: boolean; + complete: boolean; + manifestWritten: boolean; +} + +export interface StartPublisherArchiveResult { + archiveId: string; + workflowId: string; + created: boolean; +} + +export interface PublisherRestorePageInput { + archiveId: string; + page: number; +} + +export interface PublisherRestorePageResult { + archiveId: string; + ownerHash: string; + page: number; + kind: PublisherArchiveKind; + nextPage: number; + totalPages: number; + replayed: boolean; + complete: boolean; + authorityStatus: "reauthorization_required"; +} + +export interface PreparePublisherRestoreResult { + archiveId: string; + publisherDid: string; + prepared: true; + deletedIntents: number; + deletedWorkloads: number; + replayed: boolean; +} + +export interface AbortPublisherRestoreResult { + archiveId: string; + publisherDid: string; + aborted: true; + replayed: boolean; +} + export interface CursorPage { items: T[]; nextCursor?: string; diff --git a/packages/registry-client/tests/release-service.test.ts b/packages/registry-client/tests/release-service.test.ts index 5d2f173b35..40a6af4347 100644 --- a/packages/registry-client/tests/release-service.test.ts +++ b/packages/registry-client/tests/release-service.test.ts @@ -311,6 +311,37 @@ describe("ReleaseServiceClient", () => { }); describe("ReleaseServiceOperatorClient", () => { + it("lists one bounded operations-directory shard", async () => { + let captured = ""; + const fetch: typeof globalThis.fetch = async (input) => { + captured = input instanceof Request ? input.url : input.toString(); + return success({ + items: [ + { + kind: "publisher", + did: PUBLISHER_DID, + shard: "7f", + registeredAt: 1_800_000_000_000, + lastSeenAt: 1_800_000_000_001, + }, + ], + nextCursor: "cursor-next", + }); + }; + const client = new ReleaseServiceOperatorClient({ serviceUrl: SERVICE, fetch }); + await expect( + client.listDirectory("publisher", { cursor: "cursor-current", limit: 25 }), + ).resolves.toMatchObject({ + items: [{ did: PUBLISHER_DID, kind: "publisher", shard: "7f" }], + nextCursor: "cursor-next", + }); + const url = new URL(captured); + expect(url.pathname).toBe("/admin/api/directory"); + expect(url.searchParams.get("kind")).toBe("publisher"); + expect(url.searchParams.get("cursor")).toBe("cursor-current"); + expect(url.searchParams.get("limit")).toBe("25"); + }); + it("uses Access cookie credentials and roleless operator paths", async () => { const calls: Array<{ init: RequestInit | undefined; url: string }> = []; const fetch: typeof globalThis.fetch = async (input, init) => { @@ -357,4 +388,189 @@ describe("ReleaseServiceOperatorClient", () => { expect(headers.get("x-emdash-request")).toBe("1"); expect(captured!.init?.credentials).toBe("include"); }); + + it("pages publisher and approver encryption rotation through Access", async () => { + const calls: Array<{ body: string | null; path: string }> = []; + const fetch: typeof globalThis.fetch = async (input, init) => { + const url = new URL(input instanceof Request ? input.url : input.toString()); + calls.push({ path: url.pathname, body: typeof init?.body === "string" ? init.body : null }); + return success({ + ownerDid: url.pathname.includes("/approvers/") ? "did:plc:approver" : PUBLISHER_DID, + targetKeyVersion: 2, + scanned: 1, + rotated: 1, + raced: 0, + nextCursor: null, + complete: true, + }); + }; + const client = new ReleaseServiceOperatorClient({ serviceUrl: SERVICE, fetch }); + await expect( + client.rotatePublisherEncryption( + PUBLISHER_DID, + { afterCursor: null, limit: 25 }, + { idempotencyKey: "operator-publisher-rotation-0001" }, + ), + ).resolves.toMatchObject({ ownerDid: PUBLISHER_DID, targetKeyVersion: 2, complete: true }); + await expect( + client.rotateApproverEncryption( + "did:plc:approver", + { + afterCursor: "identity-transaction:abcdefghijklmnopqrstuvwxyz0123456789ABCDEFG", + limit: 10, + }, + { idempotencyKey: "operator-approver-rotation-0001" }, + ), + ).resolves.toMatchObject({ ownerDid: "did:plc:approver", rotated: 1 }); + + expect(calls).toEqual([ + { + path: `/admin/api/publishers/${encodeURIComponent(PUBLISHER_DID)}/encryption/rotate`, + body: '{"afterCursor":null,"limit":25}', + }, + { + path: "/admin/api/approvers/did%3Aplc%3Aapprover/encryption/rotate", + body: '{"afterCursor":"identity-transaction:abcdefghijklmnopqrstuvwxyz0123456789ABCDEFG","limit":10}', + }, + ]); + }); + + it("resumes encrypted publisher archive pages through Access", async () => { + let captured: { init: RequestInit | undefined; url: string } | null = null; + const fetch: typeof globalThis.fetch = async (input, init) => { + captured = { url: input instanceof Request ? input.url : input.toString(), init }; + return success({ + archiveId: "publisher-archive-0001", + ownerHash: "A".repeat(43), + page: 2, + kind: "intents", + nextCursor: "audit:0", + nextPage: 3, + replayed: false, + complete: false, + manifestWritten: false, + }); + }; + const client = new ReleaseServiceOperatorClient({ serviceUrl: SERVICE, fetch }); + await expect( + client.archivePublisher( + PUBLISHER_DID, + { archiveId: "publisher-archive-0001", cursor: "intents:", page: 2 }, + { idempotencyKey: "operator-publisher-archive-0001" }, + ), + ).resolves.toMatchObject({ kind: "intents", nextCursor: "audit:0", nextPage: 3 }); + expect(new URL(captured!.url).pathname).toBe( + `/admin/api/publishers/${encodeURIComponent(PUBLISHER_DID)}/archive`, + ); + expect(captured!.init?.body).toBe( + '{"archiveId":"publisher-archive-0001","cursor":"intents:","page":2}', + ); + }); + + it("starts a durable publisher archive Workflow through Access", async () => { + let captured: { init: RequestInit | undefined; url: string } | null = null; + const fetch: typeof globalThis.fetch = async (input, init) => { + captured = { url: input instanceof Request ? input.url : input.toString(), init }; + return success( + { + archiveId: "publisher-archive-0001", + workflowId: "W".repeat(43), + created: true, + }, + 202, + ); + }; + const client = new ReleaseServiceOperatorClient({ serviceUrl: SERVICE, fetch }); + await expect( + client.startPublisherArchive(PUBLISHER_DID, "publisher-archive-0001", { + idempotencyKey: "operator-publisher-archive-start-0001", + }), + ).resolves.toMatchObject({ workflowId: "W".repeat(43), created: true }); + expect(new URL(captured!.url).pathname).toBe( + `/admin/api/publishers/${encodeURIComponent(PUBLISHER_DID)}/archive/start`, + ); + expect(captured!.init?.body).toBe('{"archiveId":"publisher-archive-0001"}'); + }); + + it("applies suspended publisher restore pages through Access", async () => { + let captured: { init: RequestInit | undefined; url: string } | null = null; + const fetch: typeof globalThis.fetch = async (input, init) => { + captured = { url: input instanceof Request ? input.url : input.toString(), init }; + return success({ + archiveId: "publisher-archive-0001", + ownerHash: "A".repeat(43), + page: 3, + kind: "audit-events", + nextPage: 4, + totalPages: 4, + replayed: false, + complete: true, + authorityStatus: "reauthorization_required", + }); + }; + const client = new ReleaseServiceOperatorClient({ serviceUrl: SERVICE, fetch }); + await expect( + client.restorePublisher( + PUBLISHER_DID, + { archiveId: "publisher-archive-0001", page: 3 }, + { idempotencyKey: "operator-publisher-restore-0001" }, + ), + ).resolves.toMatchObject({ complete: true, authorityStatus: "reauthorization_required" }); + expect(new URL(captured!.url).pathname).toBe( + `/admin/api/publishers/${encodeURIComponent(PUBLISHER_DID)}/restore`, + ); + expect(captured!.init?.body).toBe('{"archiveId":"publisher-archive-0001","page":3}'); + }); + + it("prepares a suspended shard for restore with exact DID confirmation", async () => { + let captured: { init: RequestInit | undefined; url: string } | null = null; + const fetch: typeof globalThis.fetch = async (input, init) => { + captured = { url: input instanceof Request ? input.url : input.toString(), init }; + return success({ + archiveId: "publisher-archive-0001", + publisherDid: PUBLISHER_DID, + prepared: true, + deletedIntents: 3, + deletedWorkloads: 1, + replayed: false, + }); + }; + const client = new ReleaseServiceOperatorClient({ serviceUrl: SERVICE, fetch }); + await expect( + client.preparePublisherRestore(PUBLISHER_DID, "publisher-archive-0001", { + idempotencyKey: "operator-publisher-restore-prepare-0001", + }), + ).resolves.toMatchObject({ prepared: true, deletedIntents: 3, replayed: false }); + expect(new URL(captured!.url).pathname).toBe( + `/admin/api/publishers/${encodeURIComponent(PUBLISHER_DID)}/restore/prepare`, + ); + expect(captured!.init?.body).toBe( + `{"archiveId":"publisher-archive-0001","confirmPublisherDid":"${PUBLISHER_DID}"}`, + ); + }); + + it("aborts a suspended shard restore with exact DID confirmation", async () => { + let captured: { init: RequestInit | undefined; url: string } | null = null; + const fetch: typeof globalThis.fetch = async (input, init) => { + captured = { url: input instanceof Request ? input.url : input.toString(), init }; + return success({ + archiveId: "publisher-archive-0001", + publisherDid: PUBLISHER_DID, + aborted: true, + replayed: false, + }); + }; + const client = new ReleaseServiceOperatorClient({ serviceUrl: SERVICE, fetch }); + await expect( + client.abortPublisherRestore(PUBLISHER_DID, "publisher-archive-0001", { + idempotencyKey: "operator-publisher-restore-abort-0001", + }), + ).resolves.toMatchObject({ aborted: true, replayed: false }); + expect(new URL(captured!.url).pathname).toBe( + `/admin/api/publishers/${encodeURIComponent(PUBLISHER_DID)}/restore/abort`, + ); + expect(captured!.init?.body).toBe( + `{"archiveId":"publisher-archive-0001","confirmPublisherDid":"${PUBLISHER_DID}"}`, + ); + }); });