From d3793d4a040e3d2f6ead757277526cc15661ecff Mon Sep 17 00:00:00 2001 From: Lucas Patron <86905052+lucas-barake@users.noreply.github.com> Date: Sat, 25 Jul 2026 19:06:15 -0500 Subject: [PATCH 01/29] Implement durable peer store and forward --- .../local-rpc/bench/PeerRelayStore.bench.ts | 193 ++ packages/local-rpc/package.json | 1 + .../local-rpc/src/PeerRelayAuthorization.ts | 90 + packages/local-rpc/src/PeerRelayIngress.ts | 774 ++++++++ packages/local-rpc/src/PeerRelayLimits.ts | 393 ++++ packages/local-rpc/src/PeerRelayRpc.ts | 149 ++ packages/local-rpc/src/PeerRelayStore.ts | 1673 +++++++++++++++++ packages/local-rpc/src/PeerRpcServer.ts | 1425 ++++++++++++++ packages/local-rpc/src/RpcPeerTransport.ts | 473 +++++ packages/local-rpc/src/index.ts | 5 + .../src/internal/peerRelayMigrations.ts | 204 ++ .../internal/peerRelaySqliteTransaction.ts | 169 ++ .../test/PeerRelayAuthorization.test.ts | 248 +++ .../local-rpc/test/PeerRelayIngress.test.ts | 590 ++++++ .../local-rpc/test/PeerRelayLimits.test.ts | 168 ++ packages/local-rpc/test/PeerRelayRpc.test.ts | 191 ++ .../test/PeerRelaySqliteTransaction.test.ts | 165 ++ .../local-rpc/test/PeerRelayStore.test.ts | 284 +++ packages/local-rpc/test/PeerRpcServer.test.ts | 316 ++++ .../local-rpc/test/PublicApi.types.test.ts | 53 + .../local-rpc/test/RpcPeerTransport.test.ts | 584 ++++++ packages/local-sql/src/BackupStore.ts | 6 + packages/local-sql/src/DocumentEntity.ts | 51 +- packages/local-sql/src/DurableRuntime.ts | 20 +- packages/local-sql/src/Migrations.ts | 207 +- .../local-sql/src/PeerRelayClientRuntime.ts | 203 ++ packages/local-sql/src/PeerRelayOutbox.ts | 935 +++++++++ .../local-sql/src/PeerRelayOutboxLimits.ts | 53 + .../local-sql/src/PeerRelayReceiptLimits.ts | 53 + packages/local-sql/src/PeerSession.ts | 207 +- packages/local-sql/src/PeerSync.ts | 460 ++++- packages/local-sql/src/PeerSyncEnvelope.ts | 235 +++ packages/local-sql/src/SqlReplica.ts | 79 +- packages/local-sql/src/index.ts | 5 + .../internal/peerRelayOutboxMaintenance.ts | 29 + .../internal/peerRelayReceiptMaintenance.ts | 25 + packages/local-sql/test/BackupStore.test.ts | 98 +- .../local-sql/test/DocumentEntity.test.ts | 97 +- .../local-sql/test/MigrationFixtures.test.ts | 15 +- packages/local-sql/test/Migrations.test.ts | 236 ++- .../test/PeerRelayClientRuntime.test.ts | 241 +++ .../local-sql/test/PeerRelayOutbox.test.ts | 334 ++++ .../test/PeerRelayOutboxLimits.test.ts | 40 + .../test/PeerRelayReceiptLimits.test.ts | 40 + packages/local-sql/test/PeerSession.test.ts | 191 +- packages/local-sql/test/PeerSync.test.ts | 153 ++ .../local-sql/test/PeerSyncEnvelope.test.ts | 268 +++ .../local-sql/test/PublicApi.types.test.ts | 20 + packages/local/src/Identity.ts | 6 + packages/local/src/PeerTransport.ts | 25 + packages/local/test/Identity.test.ts | 4 + packages/local/test/PublicApi.types.test.ts | 67 +- pnpm-lock.yaml | 3 + 53 files changed, 12418 insertions(+), 136 deletions(-) create mode 100644 packages/local-rpc/bench/PeerRelayStore.bench.ts create mode 100644 packages/local-rpc/src/PeerRelayAuthorization.ts create mode 100644 packages/local-rpc/src/PeerRelayIngress.ts create mode 100644 packages/local-rpc/src/PeerRelayLimits.ts create mode 100644 packages/local-rpc/src/PeerRelayRpc.ts create mode 100644 packages/local-rpc/src/PeerRelayStore.ts create mode 100644 packages/local-rpc/src/internal/peerRelayMigrations.ts create mode 100644 packages/local-rpc/src/internal/peerRelaySqliteTransaction.ts create mode 100644 packages/local-rpc/test/PeerRelayAuthorization.test.ts create mode 100644 packages/local-rpc/test/PeerRelayIngress.test.ts create mode 100644 packages/local-rpc/test/PeerRelayLimits.test.ts create mode 100644 packages/local-rpc/test/PeerRelayRpc.test.ts create mode 100644 packages/local-rpc/test/PeerRelaySqliteTransaction.test.ts create mode 100644 packages/local-rpc/test/PeerRelayStore.test.ts create mode 100644 packages/local-rpc/test/PublicApi.types.test.ts create mode 100644 packages/local-sql/src/PeerRelayClientRuntime.ts create mode 100644 packages/local-sql/src/PeerRelayOutbox.ts create mode 100644 packages/local-sql/src/PeerRelayOutboxLimits.ts create mode 100644 packages/local-sql/src/PeerRelayReceiptLimits.ts create mode 100644 packages/local-sql/src/PeerSyncEnvelope.ts create mode 100644 packages/local-sql/src/internal/peerRelayOutboxMaintenance.ts create mode 100644 packages/local-sql/src/internal/peerRelayReceiptMaintenance.ts create mode 100644 packages/local-sql/test/PeerRelayClientRuntime.test.ts create mode 100644 packages/local-sql/test/PeerRelayOutbox.test.ts create mode 100644 packages/local-sql/test/PeerRelayOutboxLimits.test.ts create mode 100644 packages/local-sql/test/PeerRelayReceiptLimits.test.ts create mode 100644 packages/local-sql/test/PeerSyncEnvelope.test.ts create mode 100644 packages/local-sql/test/PublicApi.types.test.ts diff --git a/packages/local-rpc/bench/PeerRelayStore.bench.ts b/packages/local-rpc/bench/PeerRelayStore.bench.ts new file mode 100644 index 0000000..5c28ff4 --- /dev/null +++ b/packages/local-rpc/bench/PeerRelayStore.bench.ts @@ -0,0 +1,193 @@ +import { NodeCrypto } from "@effect/platform-node" +import { SqliteClient } from "@effect/sql-sqlite-node" +import * as Identity from "@lucas-barake/effect-local/Identity" +import * as Context from "effect/Context" +import * as Effect from "effect/Effect" +import * as Exit from "effect/Exit" +import * as Layer from "effect/Layer" +import * as Option from "effect/Option" +import * as Scope from "effect/Scope" +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { afterAll, bench } from "vitest" +import * as PeerRelayLimits from "../src/PeerRelayLimits.js" +import * as PeerRelayRpc from "../src/PeerRelayRpc.js" +import * as PeerRelayStore from "../src/PeerRelayStore.js" + +const backlogSizes = [128, 1_024] as const +const drainBatchSize = 8 +const payloadBytes = 1_024 +const benchmarkOptions = { + iterations: 8, + time: 0, + warmupIterations: 2, + warmupTime: 0 +} as const + +const suffix = (value: number) => String(value).padStart(12, "0") +const peerId = (value: number) => Identity.PeerId.make(`peer_00000000-0000-4000-8000-${suffix(value)}`) +const relayMessageId = (value: number) => Identity.RelayMessageId.make(`rly_00000000-0000-4000-8000-${suffix(value)}`) +const documentId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000001") +const relayPeerId = peerId(1) +const recipientPeerId = peerId(2) +const payload = Uint8Array.from({ length: payloadBytes }, (_, index) => index % 251) + +let nextMessage = 1 +let nextSenderPeer = 10 + +interface WorkItem { + readonly admission: PeerRelayStore.Admission + readonly claim: PeerRelayStore.ClaimRequest +} + +const makeWorkItem = ( + workload: string, + index: number, + distribution: "Hot" | "Distributed" +): WorkItem => { + const messageNumber = nextMessage++ + const senderPeerId = distribution === "Hot" ? peerId(3) : peerId(nextSenderPeer++) + const channel = PeerRelayStore.ChannelKey.make({ + tenantId: `benchmark-${workload}`, + senderSubjectId: "sender", + senderPeerId, + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + recipientSubjectId: "recipient", + recipientPeerId + }) + return { + admission: PeerRelayStore.Admission.make({ + channel, + relayMessageId: relayMessageId(messageNumber), + relayPeerId, + documentIds: [documentId], + senderConnectionEpoch: `epoch-${workload}`, + senderSequence: distribution === "Hot" ? index : 0, + payloadVersion: 1, + messageHash: `message-${messageNumber}`, + outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make(messageNumber.toString(16).padStart(64, "0")), + payload, + messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, + senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, + minimumTerminalRetentionMillis: PeerRelayLimits.defaults.minimumTerminalRetentionMillis + }), + claim: PeerRelayStore.ClaimRequest.make({ + recipient: { + tenantId: channel.tenantId, + subjectId: channel.recipientSubjectId, + peerId: channel.recipientPeerId + }, + sender: { + subjectId: channel.senderSubjectId, + peerId: channel.senderPeerId + }, + sessionGeneration: 1, + authorizedDocumentIds: [documentId] + }) + } +} + +const temporaryDirectory = mkdtempSync(join(tmpdir(), "effect-local-peer-relay-bench-")) +const filename = join(temporaryDirectory, "relay.sqlite") +const scope = await Effect.runPromise(Scope.make()) +const baseLayer = Layer.mergeAll( + SqliteClient.layer({ filename }), + NodeCrypto.layer, + PeerRelayLimits.layerDefaults +) +const benchmarkContext = await Effect.runPromise( + Layer.build( + Layer.merge( + baseLayer, + PeerRelayStore.layerSqlite.pipe(Layer.provide(baseLayer)) + ) + ).pipe(Effect.provideService(Scope.Scope, scope)) +) +const store = Context.get(benchmarkContext, PeerRelayStore.PeerRelayStore) + +const workloads = new Map>() + +for (const backlogSize of backlogSizes) { + for (const distribution of ["Hot", "Distributed"] as const) { + const workload = `${distribution.toLowerCase()}-${backlogSize}` + const items = Array.from( + { length: backlogSize }, + (_, index) => makeWorkItem(workload, index, distribution) + ) + workloads.set(workload, items) + } +} + +await Effect.runPromise( + Effect.forEach( + Array.from(workloads.values()).flat(), + ({ admission }) => store.admit(admission), + { concurrency: 1, discard: true } + ) +) + +const drain = ( + items: ReadonlyArray, + cursor: { value: number } +) => + Effect.gen(function*() { + for (let offset = 0; offset < drainBatchSize; offset++) { + const item = items[cursor.value++] + if (item === undefined) { + return yield* Effect.die(new Error("PeerRelayStore benchmark exhausted its prepared backlog")) + } + const claimed = yield* store.claim(item.claim) + if (Option.isNone(claimed.message)) { + return yield* Effect.die(new Error("PeerRelayStore benchmark could not claim prepared work")) + } + const message = claimed.message.value + const acknowledged = yield* store.acknowledge({ + channel: message.channel, + relayMessageId: message.relayMessageId, + claimToken: message.claimToken, + messageHash: message.messageHash, + sessionGeneration: message.sessionGeneration, + recipient: item.claim.recipient + }) + if (acknowledged.status !== "Changed") { + return yield* Effect.die(new Error("PeerRelayStore benchmark did not acknowledge its active claim")) + } + } + }) + +for (const backlogSize of backlogSizes) { + for (const distribution of ["Hot", "Distributed"] as const) { + const workload = `${distribution.toLowerCase()}-${backlogSize}` + const items = workloads.get(workload)! + const cursor = { value: 0 } + + bench( + `SQLite drain batch=${drainBatchSize}, backlog=${backlogSize}, channels=${distribution.toLowerCase()}, payload=${payloadBytes} bytes`, + async () => { + await Effect.runPromise(drain(items, cursor)) + }, + benchmarkOptions + ) + } +} + +for (const distribution of ["Hot", "Distributed"] as const) { + let sequence = 0 + const workload = `admission-${distribution.toLowerCase()}` + + bench( + `SQLite admission, channels=${distribution.toLowerCase()}, payload=${payloadBytes} bytes`, + async () => { + await Effect.runPromise( + store.admit(makeWorkItem(workload, sequence++, distribution).admission) + ) + }, + benchmarkOptions + ) +} + +afterAll(async () => { + await Effect.runPromise(Scope.close(scope, Exit.void)) + rmSync(temporaryDirectory, { recursive: true, force: true }) +}) diff --git a/packages/local-rpc/package.json b/packages/local-rpc/package.json index ef8eec0..cec63ec 100644 --- a/packages/local-rpc/package.json +++ b/packages/local-rpc/package.json @@ -56,6 +56,7 @@ }, "devDependencies": { "@effect/platform-node": "4.0.0-beta.99", + "@effect/sql-sqlite-node": "4.0.0-beta.99", "@lucas-barake/effect-local-test": "workspace:^", "effect": "4.0.0-beta.99" } diff --git a/packages/local-rpc/src/PeerRelayAuthorization.ts b/packages/local-rpc/src/PeerRelayAuthorization.ts new file mode 100644 index 0000000..3b836de --- /dev/null +++ b/packages/local-rpc/src/PeerRelayAuthorization.ts @@ -0,0 +1,90 @@ +import type * as PeerSession from "@lucas-barake/effect-local-sql/PeerSession" +import * as Identity from "@lucas-barake/effect-local/Identity" +import * as Context from "effect/Context" +import * as Effect from "effect/Effect" +import * as Layer from "effect/Layer" +import * as Schema from "effect/Schema" +import { validate, validateRequest } from "./internal/peerAuthorization.js" +import * as PeerAuthentication from "./PeerAuthentication.js" +import * as PeerRpcError from "./PeerRpcError.js" + +export const Direction = Schema.Literals(["Send", "Receive"]) +export type Direction = typeof Direction.Type + +export const RemotePeer = Schema.Struct({ + subjectId: Schema.NonEmptyString, + peerId: Identity.PeerId +}) +export type RemotePeer = typeof RemotePeer.Type + +export interface Request { + readonly direction: Direction + readonly principal: PeerAuthentication.PeerPrincipal + readonly remote: RemotePeer + readonly documents: ReadonlyArray<{ + readonly documentType: string + readonly documentId: PeerSession.SelectedDocument["documentId"] + }> +} + +export interface Result { + readonly remote: PeerAuthentication.PeerPrincipal + readonly documents: ReadonlyArray + readonly validUntil: number + readonly invalidated: Effect.Effect +} + +export type Authorize = ( + request: Request +) => Effect.Effect + +export class PeerRelayAuthorization extends Context.Service()("@lucas-barake/effect-local-rpc/PeerRelayAuthorization") {} + +const accessDeniedOnSchemaError = (effect: Effect.Effect) => + effect.pipe( + Effect.catchIf( + (error): error is E & { readonly _tag: "SchemaError" } => + typeof error === "object" && error !== null && "_tag" in error && error._tag === "SchemaError", + () => new PeerRpcError.AccessDenied() + ) + ) + +const validateRequestShape = (request: Request) => + accessDeniedOnSchemaError( + Effect.all([ + Direction.makeEffect(request.direction), + PeerAuthentication.PeerPrincipal.makeEffect(request.principal), + RemotePeer.makeEffect(request.remote) + ]) + ).pipe( + Effect.flatMap(() => validateRequest(request.documents)), + Effect.flatMap((requested) => + new Set(request.documents.map((document) => document.documentId)).size === + request.documents.length + ? Effect.succeed(requested) + : Effect.fail(new PeerRpcError.AccessDenied()) + ) + ) + +const validateResult = (request: Request, result: Result) => + accessDeniedOnSchemaError(PeerAuthentication.PeerPrincipal.makeEffect(result.remote)).pipe( + Effect.flatMap((remote) => + remote.tenantId === request.principal.tenantId && + remote.subjectId === request.remote.subjectId && + remote.peerId === request.remote.peerId + ? validate(request, result) + : Effect.fail(new PeerRpcError.AccessDenied()) + ), + Effect.as(result) + ) + +export const layer = (authorize: Authorize) => + Layer.succeed(PeerRelayAuthorization)({ + authorize: (request) => + validateRequestShape(request).pipe( + Effect.flatMap(() => authorize(request)), + Effect.flatMap((result) => validateResult(request, result)) + ) + }) diff --git a/packages/local-rpc/src/PeerRelayIngress.ts b/packages/local-rpc/src/PeerRelayIngress.ts new file mode 100644 index 0000000..db019ee --- /dev/null +++ b/packages/local-rpc/src/PeerRelayIngress.ts @@ -0,0 +1,774 @@ +import { Cause, Context, Deferred, Effect, Fiber, Layer, Queue, Scope } from "effect" +import * as RpcClient from "effect/unstable/rpc/RpcClient" +import * as RpcClientError from "effect/unstable/rpc/RpcClientError" +import type * as RpcMessage from "effect/unstable/rpc/RpcMessage" +import * as RpcServer from "effect/unstable/rpc/RpcServer" +import { Socket, SocketServer } from "effect/unstable/socket" +import { PeerRelayLimits } from "./PeerRelayLimits.ts" +import * as PeerRpcError from "./PeerRpcError.ts" + +export interface Usage { + readonly connections: number + readonly reservedBytes: number + readonly byteReservationWaiters: number +} + +export interface Reservation { + readonly bytes: number + readonly release: Effect.Effect + readonly transferToCurrentRequest: Effect.Effect +} + +export class PeerRelayIngress extends Context.Service< + PeerRelayIngress, + { + readonly address: SocketServer.Address + readonly reserveOutbound: ( + bytes: number + ) => Effect.Effect< + Reservation, + PeerRpcError.RequestLimitExceeded | PeerRpcError.RequestCapacityExceeded + > + readonly usage: Effect.Effect + readonly await: Effect.Effect + } +>()("@lucas-barake/effect-local-rpc/PeerRelayIngress") {} + +interface RequestKey { + readonly clientId: number + readonly requestId: string | number +} + +const CurrentRequestKey = Context.Reference( + "@lucas-barake/effect-local-rpc/PeerRelayIngress/CurrentRequestKey", + { defaultValue: () => undefined } +) + +interface InternalReservation extends Reservation { + readonly shrinkTo: (bytes: number) => Effect.Effect + readonly transfer: (key: RequestKey) => Effect.Effect +} + +interface ByteWaiter { + readonly bytes: number + readonly deferred: Deferred.Deferred + state: "Waiting" | "Reserved" | "Delivered" | "Cancelled" +} + +const makeByteBudget = ( + capacity: number, + maximumWaiters: number, + outbound: Map>, + isClientActive: (clientId: number) => boolean = () => true +) => { + let reservedBytes = 0 + let waiterId = 0 + const waiters = new Map() + + const releaseBytes = (bytes: number) => + Effect.sync(() => { + reservedBytes -= bytes + drain() + }) + + const makeReservation = (bytes: number): InternalReservation => { + let reserved = bytes + let released = false + let transferred = false + const release = Effect.suspend(() => { + if (released) return Effect.void + released = true + return releaseBytes(reserved) + }) + const shrinkTo = (target: number) => + Effect.suspend(() => { + if ( + released || + transferred || + !Number.isSafeInteger(target) || + target <= 0 || + target > reserved + ) { + return Effect.die(new Error("Invalid relay byte reservation shrink")) + } + if (target === reserved) return Effect.void + const releasedBytes = reserved - target + reserved = target + return releaseBytes(releasedBytes) + }) + const transfer = (key: RequestKey) => + Effect.suspend(() => { + if (released || transferred || !isClientActive(key.clientId)) { + return Effect.fail(new PeerRpcError.SessionUnavailable()) + } + let client = outbound.get(key.clientId) + if (!client) { + client = new Map() + outbound.set(key.clientId, client) + } + if (client.has(key.requestId)) { + return Effect.fail(new PeerRpcError.SessionUnavailable()) + } + client.set(key.requestId, reservation) + transferred = true + return Effect.void + }) + const reservation: InternalReservation = { + get bytes() { + return reserved + }, + release, + shrinkTo, + transfer, + transferToCurrentRequest: Effect.flatMap(CurrentRequestKey, (key) => + key === undefined + ? Effect.fail(new PeerRpcError.SessionUnavailable()) + : transfer(key)) + } + return reservation + } + + const drain = () => { + for (const [id, waiter] of waiters) { + if (waiter.state !== "Waiting") { + waiters.delete(id) + continue + } + if (reservedBytes + waiter.bytes > capacity) return + waiters.delete(id) + waiter.state = "Reserved" + reservedBytes += waiter.bytes + Deferred.doneUnsafe(waiter.deferred, Effect.succeed(makeReservation(waiter.bytes))) + } + } + + const cancelWaiter = (id: number, waiter: ByteWaiter) => + Effect.sync(() => { + if (waiter.state === "Waiting") { + waiter.state = "Cancelled" + waiters.delete(id) + drain() + } else if (waiter.state === "Reserved") { + waiter.state = "Cancelled" + reservedBytes -= waiter.bytes + drain() + } + }) + + const reserve = ( + bytes: number + ): Effect.Effect< + InternalReservation, + PeerRpcError.RequestLimitExceeded | PeerRpcError.RequestCapacityExceeded + > => + Effect.uninterruptibleMask((restore) => { + const acquire = (): Effect.Effect< + InternalReservation, + PeerRpcError.RequestLimitExceeded | PeerRpcError.RequestCapacityExceeded + > => { + if (!Number.isSafeInteger(bytes) || bytes <= 0 || bytes > capacity) { + return Effect.fail(new PeerRpcError.RequestLimitExceeded()) + } + if (waiters.size === 0 && reservedBytes + bytes <= capacity) { + reservedBytes += bytes + return Effect.succeed(makeReservation(bytes)) + } + if (waiters.size >= maximumWaiters) { + return Effect.fail(new PeerRpcError.RequestCapacityExceeded()) + } + const id = waiterId++ + const waiter: ByteWaiter = { + bytes, + deferred: Deferred.makeUnsafe(), + state: "Waiting" + } + waiters.set(id, waiter) + return restore(Deferred.await(waiter.deferred)).pipe( + Effect.onInterrupt(() => cancelWaiter(id, waiter)), + Effect.tap(() => + Effect.sync(() => { + waiter.state = "Delivered" + }) + ) + ) + } + return Effect.suspend(acquire) + }) + + return { + reserve, + usage: () => ({ + reservedBytes, + byteReservationWaiters: waiters.size + }) + } +} + +const invalidFrame = () => + new Socket.SocketError({ + reason: new Socket.SocketCloseError({ code: 1009 }) + }) + +const utf8Length = (value: string): number => { + let bytes = 0 + for (let index = 0; index < value.length; index++) { + const code = value.charCodeAt(index) + if (code < 0x80) { + bytes++ + } else if (code < 0x800) { + bytes += 2 + } else if (code >= 0xd800 && code <= 0xdbff && index + 1 < value.length) { + const next = value.charCodeAt(index + 1) + if (next >= 0xdc00 && next <= 0xdfff) { + bytes += 4 + index++ + } else { + bytes += 3 + } + } else { + bytes += 3 + } + } + return bytes +} + +const encodeFrame = (value: unknown, maximumFrameBytes: number) => { + const body = new TextEncoder().encode(JSON.stringify(value)) + if (body.byteLength === 0 || body.byteLength > maximumFrameBytes) { + throw invalidFrame() + } + const frame = new Uint8Array(4 + body.byteLength) + new DataView(frame.buffer).setUint32(0, body.byteLength, false) + frame.set(body, 4) + return { bodyBytes: body.byteLength, frame } +} + +const makeFrameReader = ( + socket: Socket.Socket, + limits: { + readonly maximumRawChunkBytes: number + readonly maximumDeclaredFrameBytes: number + readonly maximumIncompleteFrameBytes: number + readonly incompleteFrameTimeoutMillis: number + }, + reserve: ( + bytes: number + ) => Effect.Effect< + InternalReservation, + PeerRpcError.RequestLimitExceeded | PeerRpcError.RequestCapacityExceeded + >, + onFrame: ( + value: unknown, + reservation: InternalReservation + ) => Effect.Effect +) => + Effect.gen(function*() { + const scope = yield* Effect.scope + const timeout = Deferred.makeUnsafe() + const header = new Uint8Array(4) + const decoder = new TextDecoder("utf-8", { fatal: true }) + let headerBytes = 0 + let body: Uint8Array | undefined + let bodyBytes = 0 + let reservation: InternalReservation | undefined + let timer: Fiber.Fiber | undefined + let chunkActive = false + let chunkRejected = false + + const stopTimer = Effect.suspend(() => { + if (timer === undefined) return Effect.void + const current = timer + timer = undefined + return Fiber.interrupt(current) + }) + + const startTimer = Effect.suspend(() => { + if (timer !== undefined) return Effect.void + return Effect.forkIn( + Effect.sleep(limits.incompleteFrameTimeoutMillis).pipe( + Effect.andThen(Effect.sync(() => { + Deferred.doneUnsafe(timeout, Effect.fail(invalidFrame())) + })) + ), + scope + ).pipe( + Effect.tap((fiber) => + Effect.sync(() => { + timer = fiber + }) + ), + Effect.asVoid + ) + }) + + const resetFrame = () => { + headerBytes = 0 + body = undefined + bodyBytes = 0 + reservation = undefined + } + + const consume = (raw: string | Uint8Array) => + Effect.gen(function*() { + const rawBytes = typeof raw === "string" ? utf8Length(raw) : raw.byteLength + if (rawBytes === 0) return + if (rawBytes > limits.maximumRawChunkBytes) { + return yield* Effect.fail(invalidFrame()) + } + const chunk = typeof raw === "string" ? new TextEncoder().encode(raw) : raw + if (chunk.byteLength !== rawBytes) { + return yield* Effect.fail(invalidFrame()) + } + let offset = 0 + while (offset < chunk.byteLength) { + if (headerBytes === 0 && body === undefined) { + yield* startTimer + } + if (headerBytes < 4) { + const count = Math.min(4 - headerBytes, chunk.byteLength - offset) + header.set(chunk.subarray(offset, offset + count), headerBytes) + headerBytes += count + offset += count + if (headerBytes < 4) continue + + const declared = new DataView(header.buffer).getUint32(0, false) + if ( + declared === 0 || + declared > limits.maximumDeclaredFrameBytes || + declared + 4 > limits.maximumIncompleteFrameBytes + ) { + return yield* Effect.fail(invalidFrame()) + } + reservation = yield* reserve(declared).pipe( + Effect.mapError(() => invalidFrame()) + ) + body = new Uint8Array(declared) + } + + const count = Math.min(body!.byteLength - bodyBytes, chunk.byteLength - offset) + body!.set(chunk.subarray(offset, offset + count), bodyBytes) + bodyBytes += count + offset += count + if (bodyBytes < body!.byteLength) continue + + yield* stopTimer + const retained = reservation! + let value: unknown + try { + value = JSON.parse(decoder.decode(body!)) + } catch { + yield* retained.release + return yield* Effect.fail(invalidFrame()) + } + resetFrame() + yield* onFrame(value, retained).pipe( + Effect.onExit((exit) => exit._tag === "Failure" ? retained.release : Effect.void) + ) + } + }) + + return yield* socket.runRaw((chunk) => { + if (chunkRejected) return + if (chunkActive) { + chunkRejected = true + return Effect.fail(invalidFrame()) + } + chunkActive = true + return consume(chunk).pipe( + Effect.ensuring( + Effect.sync(() => { + chunkActive = false + }) + ) + ) + }).pipe( + Effect.raceFirst(Deferred.await(timeout)), + Effect.ensuring( + Effect.andThen(stopTimer, Effect.suspend(() => reservation === undefined ? Effect.void : reservation.release)) + ) + ) + }) + +const removeReservation = ( + reservations: Map>, + key: RequestKey +) => { + const client = reservations.get(key.clientId) + const reservation = client?.get(key.requestId) + if (reservation === undefined) return undefined + client!.delete(key.requestId) + if (client!.size === 0) reservations.delete(key.clientId) + return reservation +} + +const releaseClientReservations = ( + reservations: Map>, + clientId: number +) => + Effect.suspend(() => { + const client = reservations.get(clientId) + if (client === undefined) return Effect.void + reservations.delete(clientId) + return Effect.forEach(client.values(), (reservation) => reservation.release, { + concurrency: 1, + discard: true + }) + }) + +const makeServerProtocol = ( + server: SocketServer.SocketServer["Service"], + limits: Context.Service.Shape, + childScope: Scope.Closeable +) => + Effect.gen(function*() { + const disconnects = yield* Queue.bounded(limits.maxRelayConnections) + const clients = new Map Effect.Effect + }>() + const clientIds = new Set() + const inbound = new Map>() + const outbound = new Map>() + const budget = makeByteBudget( + limits.maximumSharedPayloadBytes, + limits.maximumByteReservationWaiters, + outbound, + (clientId) => clientIds.has(clientId) + ) + let connections = 0 + let nextClientId = 0 + let protocolReady = false + let writeRequest!: ( + clientId: number, + message: RpcMessage.FromClientEncoded + ) => Effect.Effect + + const cleanupClient = (clientId: number) => + Effect.gen(function*() { + clients.delete(clientId) + const wasRegistered = clientIds.delete(clientId) + yield* releaseClientReservations(inbound, clientId) + yield* releaseClientReservations(outbound, clientId) + if (wasRegistered) { + yield* Queue.offer(disconnects, clientId) + } + }) + + const baseProtocol = yield* RpcServer.Protocol.make((writeRequest_) => { + writeRequest = writeRequest_ + return Effect.succeed({ + disconnects, + send: (clientId, response) => + Effect.suspend(() => { + const key = "requestId" in response + ? { clientId, requestId: response.requestId } + : undefined + const transferred = key === undefined + ? undefined + : removeReservation(outbound, key) + return Effect.gen(function*() { + const client = clients.get(clientId) + if (client === undefined) { + return + } + + const maximumAdditional = transferred === undefined + ? limits.maximumDeclaredFrameBytes + : Math.max(0, limits.maximumDeclaredFrameBytes - transferred.bytes) + const extra = maximumAdditional === 0 + ? undefined + : yield* budget.reserve(maximumAdditional).pipe( + Effect.match({ + onFailure: () => undefined, + onSuccess: (reservation) => reservation + }) + ) + if (maximumAdditional > 0 && extra === undefined) { + yield* client.write(new Socket.CloseEvent(1013)).pipe(Effect.ignore) + return + } + + yield* Effect.gen(function*() { + let encoded: ReturnType + try { + encoded = encodeFrame(response, limits.maximumDeclaredFrameBytes) + } catch { + yield* client.write(new Socket.CloseEvent(1009)).pipe(Effect.ignore) + return + } + + const additional = transferred === undefined + ? encoded.bodyBytes + : Math.max(0, encoded.bodyBytes - transferred.bytes) + if (extra !== undefined && additional > 0) { + yield* extra.shrinkTo(additional) + } else if (extra !== undefined) { + yield* extra.release + } + + yield* client.write(encoded.frame).pipe(Effect.ignore) + + if (response._tag === "Exit" && key !== undefined) { + const retained = removeReservation(inbound, key) + if (retained !== undefined) yield* retained.release + } else if (response._tag === "Defect") { + yield* releaseClientReservations(inbound, clientId) + } + }).pipe(Effect.ensuring(extra?.release ?? Effect.void)) + }).pipe(Effect.ensuring(transferred?.release ?? Effect.void)) + }), + end: (clientId) => + Effect.gen(function*() { + const client = clients.get(clientId) + if (client !== undefined) { + yield* client.write(new Socket.CloseEvent()).pipe(Effect.ignore) + } + yield* cleanupClient(clientId) + }), + clientIds: Effect.sync(() => clientIds), + initialMessage: Effect.succeedNone, + supportsAck: true, + supportsTransferables: false, + supportsSpanPropagation: true + }) + }) + const protocol: RpcServer.Protocol["Service"] = { + ...baseProtocol, + run: (handler) => + Effect.suspend(() => { + protocolReady = true + return baseProtocol.run(handler).pipe( + Effect.onExit(() => + Effect.sync(() => { + protocolReady = false + }) + ) + ) + }) + } + + const onSocket = (socket: Socket.Socket) => + Effect.scoped( + Effect.suspend(() => { + if (!protocolReady || connections >= limits.maxRelayConnections) { + return Effect.gen(function*() { + const write = yield* socket.writer + yield* socket.runRaw( + () => Effect.void, + { onOpen: write(new Socket.CloseEvent(1013)).pipe(Effect.ignore) } + ).pipe(Effect.catch(() => Effect.void)) + }) + } + connections++ + const clientId = nextClientId++ + return Effect.gen(function*() { + const write = yield* socket.writer + clients.set(clientId, { write }) + clientIds.add(clientId) + + yield* makeFrameReader( + socket, + limits, + budget.reserve, + (value, reservation): Effect.Effect => { + if ( + typeof value !== "object" || + value === null || + Array.isArray(value) || + !("_tag" in value) + ) { + return Effect.andThen(reservation.release, Effect.fail(invalidFrame())) + } + const message = value as RpcMessage.FromClientEncoded + if (message._tag !== "Request") { + return Effect.provideService( + writeRequest(clientId, message), + CurrentRequestKey, + undefined + ).pipe(Effect.ensuring(reservation.release)) + } + if (typeof message.id !== "string" && typeof message.id !== "number") { + return Effect.andThen(reservation.release, Effect.fail(invalidFrame())) + } + let client = inbound.get(clientId) + if (client === undefined) { + client = new Map() + inbound.set(clientId, client) + } + if (client.has(message.id)) { + return Effect.andThen(reservation.release, Effect.fail(invalidFrame())) + } + client.set(message.id, reservation) + const key = { clientId, requestId: message.id } + return Effect.provideService( + writeRequest(clientId, message), + CurrentRequestKey, + key + ).pipe( + Effect.onExit((exit) => { + if (exit._tag === "Success") return Effect.void + removeReservation(inbound, key) + return reservation.release + }) + ) + } + ).pipe(Effect.catch(() => Effect.void)) + }).pipe( + Effect.ensuring( + Effect.gen(function*() { + yield* cleanupClient(clientId) + connections-- + }) + ) + ) + }) + ) + + const acceptFiber = yield* Effect.forkIn(server.run(onSocket), childScope) + const service = PeerRelayIngress.of({ + address: server.address, + reserveOutbound: budget.reserve, + usage: Effect.sync(() => ({ + connections, + ...budget.usage() + })), + await: Fiber.join(acceptFiber) + }) + return { protocol, service } + }) + +export const layerProtocolSocketServer = ( + socketLayer: Layer.Layer +): Layer.Layer< + PeerRelayIngress | RpcServer.Protocol, + E, + R | PeerRelayLimits +> => + Layer.effectContext( + Effect.gen(function*() { + const limits = yield* PeerRelayLimits + const childScope = yield* Effect.acquireRelease( + Scope.make("sequential"), + (scope, exit) => Scope.close(scope, exit) + ) + const socketContext = yield* Layer.buildWithScope(socketLayer, childScope) + const server = Context.get(socketContext, SocketServer.SocketServer) + const { protocol, service } = yield* makeServerProtocol(server, limits, childScope) + return Context.make(PeerRelayIngress, service).pipe( + Context.add(RpcServer.Protocol, protocol) + ) + }) + ) as Layer.Layer + +const toClientError = (message: string, cause: unknown) => + new RpcClientError.RpcClientError({ + reason: new RpcClientError.RpcClientDefect({ message, cause }) + }) + +export const makeProtocolSocket = Effect.gen(function*() { + const socket = yield* Socket.Socket + const limits = yield* PeerRelayLimits + const outbound = new Map>() + const budget = makeByteBudget( + limits.maximumSharedPayloadBytes, + limits.maximumByteReservationWaiters, + outbound + ) + + return yield* RpcClient.Protocol.make((writeResponse, clientIds) => + Effect.gen(function*() { + const write = yield* socket.writer + const requestClients = new Map() + let currentError: RpcClientError.RpcClientError | undefined + + const read = makeFrameReader( + socket, + limits, + budget.reserve, + (value, reservation) => { + if ( + typeof value !== "object" || + value === null || + Array.isArray(value) || + !("_tag" in value) + ) { + return Effect.andThen(reservation.release, Effect.fail(invalidFrame())) + } + const response = value as RpcMessage.FromServerEncoded + const requestClient = "requestId" in response + ? requestClients.get(response.requestId) + : undefined + const effect = requestClient === undefined + ? Effect.forEach(clientIds, (clientId) => writeResponse(clientId, response), { + discard: true + }) + : Effect.suspend(() => { + if (response._tag === "Exit" && "requestId" in response) { + requestClients.delete(response.requestId) + } + return writeResponse(requestClient, response) + }) + return Effect.ensuring(effect, reservation.release) + } + ).pipe( + Effect.flatMap(() => + Effect.fail( + new Socket.SocketError({ + reason: new Socket.SocketCloseError({ code: 1000 }) + }) + ) + ), + Effect.tapCause((cause) => { + if (Cause.hasInterruptsOnly(cause)) return Effect.void + currentError = toClientError("Relay socket protocol failed", Cause.squash(cause)) + return Effect.forEach( + clientIds, + (clientId) => + writeResponse(clientId, { + _tag: "ClientProtocolError", + error: currentError! + }), + { discard: true } + ) + }), + Effect.catchCause((cause) => Cause.hasInterruptsOnly(cause) ? Effect.failCause(cause) : Effect.void), + Effect.forkScoped + ) + yield* read + + return { + send: (clientId: number, request: RpcMessage.FromClientEncoded) => + Effect.gen(function*() { + if (currentError !== undefined) return yield* Effect.fail(currentError) + const reservation = yield* budget.reserve(limits.maximumDeclaredFrameBytes).pipe( + Effect.mapError((cause) => toClientError("Relay request capacity is exhausted", cause)) + ) + yield* Effect.gen(function*() { + let encoded: ReturnType + try { + encoded = encodeFrame(request, limits.maximumDeclaredFrameBytes) + } catch (cause) { + return yield* Effect.fail(toClientError("Relay request frame is invalid", cause)) + } + yield* reservation.shrinkTo(encoded.bodyBytes) + if (request._tag === "Request") requestClients.set(request.id, clientId) + yield* write(encoded.frame).pipe( + Effect.mapError((cause) => toClientError("Relay socket write failed", cause)) + ) + }).pipe( + Effect.ensuring(reservation.release) + ) + }), + supportsAck: true, + supportsTransferables: false + } + }) + ) +}) + +export const layerProtocolSocket: Layer.Layer< + RpcClient.Protocol, + never, + Socket.Socket | PeerRelayLimits +> = Layer.effect(RpcClient.Protocol, makeProtocolSocket) diff --git a/packages/local-rpc/src/PeerRelayLimits.ts b/packages/local-rpc/src/PeerRelayLimits.ts new file mode 100644 index 0000000..6bcba92 --- /dev/null +++ b/packages/local-rpc/src/PeerRelayLimits.ts @@ -0,0 +1,393 @@ +import * as Context from "effect/Context" +import * as Effect from "effect/Effect" +import * as Layer from "effect/Layer" +import * as Schema from "effect/Schema" +import type * as SchemaIssue from "effect/SchemaIssue" +import * as PeerRelayRpc from "./PeerRelayRpc.js" + +const PositiveInt = Schema.Int.check(Schema.isGreaterThan(0)) +const PositiveNumber = Schema.Number.check(Schema.isFinite(), Schema.isGreaterThan(0)) +const NegotiatedDuration = PositiveInt.check( + Schema.isLessThanOrEqualTo(PeerRelayRpc.maximumNegotiatedDurationMillis) +) +const Percentage = PositiveNumber.check(Schema.isLessThanOrEqualTo(100)) + +export const Values = Schema.Struct({ + maxActiveMessagesPerSenderPeer: PositiveInt, + maxActiveBytesPerSenderPeer: PositiveInt, + maxActiveMessagesPerRecipientPeer: PositiveInt, + maxActiveBytesPerRecipientPeer: PositiveInt, + maxActiveMessagesPerRecipientSubject: PositiveInt, + maxActiveBytesPerRecipientSubject: PositiveInt, + maxActiveMessagesPerTenant: PositiveInt, + maxActiveBytesPerTenant: PositiveInt, + maxActiveMessagesPerShard: PositiveInt, + maxActiveBytesPerShard: PositiveInt, + + maxRetainedRowsPerSenderPeer: PositiveInt, + maxRetainedBytesPerSenderPeer: PositiveInt, + maxRetainedRowsPerRecipientPeer: PositiveInt, + maxRetainedBytesPerRecipientPeer: PositiveInt, + maxRetainedRowsPerRecipientSubject: PositiveInt, + maxRetainedBytesPerRecipientSubject: PositiveInt, + maxRetainedRowsPerTenant: PositiveInt, + maxRetainedBytesPerTenant: PositiveInt, + maxRetainedRowsPerShard: PositiveInt, + maxRetainedBytesPerShard: PositiveInt, + + messageTtlMillis: NegotiatedDuration, + maximumSenderRetryHorizonMillis: NegotiatedDuration, + minimumTerminalRetentionMillis: NegotiatedDuration, + maximumReceiptRetentionMillis: NegotiatedDuration, + claimLeaseMillis: PositiveInt, + maximumRecipientProcessingMillis: PositiveInt, + acknowledgementTimeoutMillis: PositiveInt, + claimSafetyMarginMillis: PositiveInt, + retryBaseDelayMillis: PositiveInt, + retryMaximumDelayMillis: PositiveInt, + sqliteLockRetryBaseDelayMillis: PositiveInt, + sqliteLockRetryMaximumDelayMillis: PositiveInt, + sqliteLockRetryMaxAttempts: PositiveInt, + + maxRelayConnections: PositiveInt, + maximumRawChunkBytes: PositiveInt, + maximumDeclaredFrameBytes: PositiveInt, + maximumIncompleteFrameBytes: PositiveInt, + incompleteFrameTimeoutMillis: PositiveInt, + maximumSharedPayloadBytes: PositiveInt, + maximumByteReservationWaiters: PositiveInt, + + maxSessionsPerSubject: PositiveInt, + maxInFlightOpen: PositiveInt, + maxInFlightOpenPerSubject: PositiveInt, + openRatePerSecond: PositiveNumber, + openBurst: PositiveInt, + maxInFlightPush: PositiveInt, + maxInFlightPushPerSubject: PositiveInt, + admissionRatePerSecond: PositiveNumber, + admissionBurst: PositiveInt, + maxRetainedRateLimitedConnections: PositiveInt, + maxRetainedRateLimitedSubjects: PositiveInt, + rateLimitIdleRetentionMillis: PositiveInt, + + terminalResponseQueueCapacity: PositiveInt, + maxInFlightTerminalResponses: PositiveInt, + maxInFlightTerminalResponsesPerSubject: PositiveInt, + terminalResponseRatePerSecond: PositiveNumber, + terminalResponseBurst: PositiveInt, + maxRetainedTerminalResponseSubjects: PositiveInt, + terminalResponseSubjectIdleRetentionMillis: PositiveInt, + + newWorkQueueCapacity: PositiveInt, + retryQueueCapacity: PositiveInt, + relayWorkerConcurrency: PositiveInt, + newWorkWeight: PositiveInt, + retryWorkWeight: PositiveInt, + maxActiveChannels: PositiveInt, + compensationIntervalMillis: PositiveInt, + compensationBatchSize: PositiveInt, + + sqlAdmissionQueueCapacity: PositiveInt, + sqlTerminalQueueCapacity: PositiveInt, + sqlDeliveryQueueCapacity: PositiveInt, + sqlMaintenanceQueueCapacity: PositiveInt, + maxInFlightSqlTransactions: PositiveInt, + maxInFlightSqlAdmission: PositiveInt, + maxInFlightSqlTerminal: PositiveInt, + maxInFlightSqlDelivery: PositiveInt, + maxInFlightSqlMaintenance: PositiveInt, + + maintenanceIntervalMillis: PositiveInt, + claimRecoveryBatchSize: PositiveInt, + claimRecoveryRowsPerSecond: PositiveNumber, + expiryBatchSize: PositiveInt, + expiryRowsPerSecond: PositiveNumber, + integrityBatchSize: PositiveInt, + integrityRowsPerSecond: PositiveNumber, + reconciliationBatchSize: PositiveInt, + reconciliationRowsPerSecond: PositiveNumber, + terminalCollectionBatchSize: PositiveInt, + terminalCollectionRowsPerSecond: PositiveNumber, + orphanChannelCleanupBatchSize: PositiveInt, + orphanChannelCleanupRowsPerSecond: PositiveNumber, + sqliteTransactionCapacityPerSecond: PositiveNumber, + sqliteCapacityHeadroomPercent: Percentage, + + shutdownReleaseConcurrency: PositiveInt, + shutdownReleaseTimeoutMillis: PositiveInt +}) +export type Values = typeof Values.Type + +const kibibyte = 1_024 +const mebibyte = 1_024 * kibibyte +const gibibyte = 1_024 * mebibyte +const day = 24 * 60 * 60 * 1_000 + +export const defaults: Values = Values.make({ + maxActiveMessagesPerSenderPeer: 10_000, + maxActiveBytesPerSenderPeer: 256 * mebibyte, + maxActiveMessagesPerRecipientPeer: 10_000, + maxActiveBytesPerRecipientPeer: 256 * mebibyte, + maxActiveMessagesPerRecipientSubject: 40_000, + maxActiveBytesPerRecipientSubject: gibibyte, + maxActiveMessagesPerTenant: 100_000, + maxActiveBytesPerTenant: 4 * gibibyte, + maxActiveMessagesPerShard: 1_000_000, + maxActiveBytesPerShard: 16 * gibibyte, + + maxRetainedRowsPerSenderPeer: 10_000, + maxRetainedBytesPerSenderPeer: 64 * mebibyte, + maxRetainedRowsPerRecipientPeer: 10_000, + maxRetainedBytesPerRecipientPeer: 64 * mebibyte, + maxRetainedRowsPerRecipientSubject: 40_000, + maxRetainedBytesPerRecipientSubject: 256 * mebibyte, + maxRetainedRowsPerTenant: 100_000, + maxRetainedBytesPerTenant: gibibyte, + maxRetainedRowsPerShard: 1_000_000, + maxRetainedBytesPerShard: 8 * gibibyte, + + messageTtlMillis: 7 * day, + maximumSenderRetryHorizonMillis: 7 * day, + minimumTerminalRetentionMillis: day, + maximumReceiptRetentionMillis: 8 * day, + claimLeaseMillis: 60_000, + maximumRecipientProcessingMillis: 40_000, + acknowledgementTimeoutMillis: 10_000, + claimSafetyMarginMillis: 5_000, + retryBaseDelayMillis: 250, + retryMaximumDelayMillis: 30_000, + sqliteLockRetryBaseDelayMillis: 10, + sqliteLockRetryMaximumDelayMillis: 250, + sqliteLockRetryMaxAttempts: 5, + + maxRelayConnections: 1_024, + maximumRawChunkBytes: 8 * mebibyte + 4, + maximumDeclaredFrameBytes: 8 * mebibyte, + maximumIncompleteFrameBytes: 8 * mebibyte + 4, + incompleteFrameTimeoutMillis: 10_000, + maximumSharedPayloadBytes: 64 * mebibyte, + maximumByteReservationWaiters: 1_024, + + maxSessionsPerSubject: 4, + maxInFlightOpen: 32, + maxInFlightOpenPerSubject: 2, + openRatePerSecond: 16, + openBurst: 32, + maxInFlightPush: 128, + maxInFlightPushPerSubject: 8, + admissionRatePerSecond: 100, + admissionBurst: 128, + maxRetainedRateLimitedConnections: 10_000, + maxRetainedRateLimitedSubjects: 10_000, + rateLimitIdleRetentionMillis: 10 * 60_000, + + terminalResponseQueueCapacity: 256, + maxInFlightTerminalResponses: 64, + maxInFlightTerminalResponsesPerSubject: 4, + terminalResponseRatePerSecond: 100, + terminalResponseBurst: 128, + maxRetainedTerminalResponseSubjects: 10_000, + terminalResponseSubjectIdleRetentionMillis: 10 * 60_000, + + newWorkQueueCapacity: 1_024, + retryQueueCapacity: 1_024, + relayWorkerConcurrency: 8, + newWorkWeight: 3, + retryWorkWeight: 1, + maxActiveChannels: 10_000, + compensationIntervalMillis: 1_000, + compensationBatchSize: 256, + + sqlAdmissionQueueCapacity: 256, + sqlTerminalQueueCapacity: 256, + sqlDeliveryQueueCapacity: 256, + sqlMaintenanceQueueCapacity: 256, + maxInFlightSqlTransactions: 16, + maxInFlightSqlAdmission: 4, + maxInFlightSqlTerminal: 4, + maxInFlightSqlDelivery: 4, + maxInFlightSqlMaintenance: 4, + + maintenanceIntervalMillis: 1_000, + claimRecoveryBatchSize: 100, + claimRecoveryRowsPerSecond: 100, + expiryBatchSize: 100, + expiryRowsPerSecond: 100, + integrityBatchSize: 100, + integrityRowsPerSecond: 100, + reconciliationBatchSize: 100, + reconciliationRowsPerSecond: 100, + terminalCollectionBatchSize: 100, + terminalCollectionRowsPerSecond: 100, + orphanChannelCleanupBatchSize: 100, + orphanChannelCleanupRowsPerSecond: 100, + sqliteTransactionCapacityPerSecond: 200, + sqliteCapacityHeadroomPercent: 20, + + shutdownReleaseConcurrency: 16, + shutdownReleaseTimeoutMillis: 30_000 +}) + +export class InvalidPeerRelayLimits extends Schema.TaggedErrorClass( + "@lucas-barake/effect-local-rpc/PeerRelayLimits/InvalidPeerRelayLimits" +)("InvalidPeerRelayLimits", { field: Schema.String }) {} + +export class PeerRelayLimits extends Context.Service()( + "@lucas-barake/effect-local-rpc/PeerRelayLimits" +) {} + +const invalid = (field: keyof Values) => Effect.fail(new InvalidPeerRelayLimits({ field })) + +const firstField = (issue: SchemaIssue.Issue): string | undefined => { + if (issue._tag === "Pointer") { + const field = issue.path[0] + return typeof field === "string" ? field : firstField(issue.issue) + } + if (issue._tag === "Composite" || issue._tag === "AnyOf") { + for (const nested of issue.issues) { + const field = firstField(nested) + if (field !== undefined) return field + } + } + return undefined +} + +const validate = (values: Values) => { + const relations: ReadonlyArray = [ + [ + "maxActiveMessagesPerRecipientSubject", + values.maxActiveMessagesPerRecipientSubject >= values.maxActiveMessagesPerRecipientPeer + ], + [ + "maxActiveBytesPerRecipientSubject", + values.maxActiveBytesPerRecipientSubject >= values.maxActiveBytesPerRecipientPeer + ], + [ + "maxActiveMessagesPerTenant", + values.maxActiveMessagesPerTenant >= values.maxActiveMessagesPerRecipientSubject && + values.maxActiveMessagesPerTenant >= values.maxActiveMessagesPerSenderPeer + ], + [ + "maxActiveBytesPerTenant", + values.maxActiveBytesPerTenant >= values.maxActiveBytesPerRecipientSubject && + values.maxActiveBytesPerTenant >= values.maxActiveBytesPerSenderPeer + ], + ["maxActiveMessagesPerShard", values.maxActiveMessagesPerShard >= values.maxActiveMessagesPerTenant], + ["maxActiveBytesPerShard", values.maxActiveBytesPerShard >= values.maxActiveBytesPerTenant], + [ + "maxRetainedRowsPerRecipientSubject", + values.maxRetainedRowsPerRecipientSubject >= values.maxRetainedRowsPerRecipientPeer + ], + [ + "maxRetainedBytesPerRecipientSubject", + values.maxRetainedBytesPerRecipientSubject >= values.maxRetainedBytesPerRecipientPeer + ], + [ + "maxRetainedRowsPerTenant", + values.maxRetainedRowsPerTenant >= values.maxRetainedRowsPerRecipientSubject && + values.maxRetainedRowsPerTenant >= values.maxRetainedRowsPerSenderPeer + ], + [ + "maxRetainedBytesPerTenant", + values.maxRetainedBytesPerTenant >= values.maxRetainedBytesPerRecipientSubject && + values.maxRetainedBytesPerTenant >= values.maxRetainedBytesPerSenderPeer + ], + ["maxRetainedRowsPerShard", values.maxRetainedRowsPerShard >= values.maxRetainedRowsPerTenant], + ["maxRetainedBytesPerShard", values.maxRetainedBytesPerShard >= values.maxRetainedBytesPerTenant], + [ + "maximumReceiptRetentionMillis", + values.maximumReceiptRetentionMillis >= + Math.max(values.messageTtlMillis, values.maximumSenderRetryHorizonMillis) + + values.minimumTerminalRetentionMillis + ], + [ + "claimLeaseMillis", + values.maximumRecipientProcessingMillis + + values.acknowledgementTimeoutMillis + + values.claimSafetyMarginMillis < + values.claimLeaseMillis + ], + ["claimLeaseMillis", values.claimLeaseMillis < values.messageTtlMillis], + ["retryMaximumDelayMillis", values.retryMaximumDelayMillis >= values.retryBaseDelayMillis], + ["retryMaximumDelayMillis", values.retryMaximumDelayMillis < values.claimLeaseMillis], + [ + "sqliteLockRetryMaximumDelayMillis", + values.sqliteLockRetryMaximumDelayMillis >= values.sqliteLockRetryBaseDelayMillis + ], + ["maximumRawChunkBytes", values.maximumRawChunkBytes <= values.maximumIncompleteFrameBytes], + ["maximumDeclaredFrameBytes", values.maximumDeclaredFrameBytes + 4 <= values.maximumIncompleteFrameBytes], + ["maximumDeclaredFrameBytes", values.maximumDeclaredFrameBytes >= PeerRelayRpc.maximumRelayPayloadBytes], + ["maximumSharedPayloadBytes", values.maximumSharedPayloadBytes >= values.maximumIncompleteFrameBytes], + [ + "maximumSharedPayloadBytes", + values.relayWorkerConcurrency * PeerRelayRpc.maximumRelayPayloadBytes <= + values.maximumSharedPayloadBytes + ], + ["maximumByteReservationWaiters", values.maximumByteReservationWaiters >= values.maxRelayConnections], + ["maxSessionsPerSubject", values.maxSessionsPerSubject <= values.maxRelayConnections], + ["maxInFlightOpen", values.maxInFlightOpen <= values.maxRelayConnections], + ["maxInFlightOpenPerSubject", values.maxInFlightOpenPerSubject <= values.maxInFlightOpen], + ["openBurst", values.openBurst >= values.maxInFlightOpen], + ["maxInFlightPushPerSubject", values.maxInFlightPushPerSubject <= values.maxInFlightPush], + ["admissionBurst", values.admissionBurst >= values.maxInFlightPush], + ["maxRetainedRateLimitedConnections", values.maxRetainedRateLimitedConnections >= values.maxRelayConnections], + ["maxRetainedRateLimitedSubjects", values.maxRetainedRateLimitedSubjects >= values.maxInFlightOpen], + [ + "maxInFlightTerminalResponsesPerSubject", + values.maxInFlightTerminalResponsesPerSubject <= values.maxInFlightTerminalResponses + ], + ["terminalResponseQueueCapacity", values.terminalResponseQueueCapacity >= values.maxInFlightTerminalResponses], + ["terminalResponseBurst", values.terminalResponseBurst >= values.maxInFlightTerminalResponses], + ["terminalResponseRatePerSecond", values.terminalResponseRatePerSecond >= values.admissionRatePerSecond], + [ + "maxRetainedTerminalResponseSubjects", + values.maxRetainedTerminalResponseSubjects >= values.maxInFlightTerminalResponses + ], + ["compensationBatchSize", values.compensationBatchSize <= values.maxActiveChannels], + ["sqlAdmissionQueueCapacity", values.sqlAdmissionQueueCapacity >= values.maxInFlightSqlAdmission], + ["sqlTerminalQueueCapacity", values.sqlTerminalQueueCapacity >= values.maxInFlightSqlTerminal], + ["sqlDeliveryQueueCapacity", values.sqlDeliveryQueueCapacity >= values.maxInFlightSqlDelivery], + ["sqlMaintenanceQueueCapacity", values.sqlMaintenanceQueueCapacity >= values.maxInFlightSqlMaintenance], + [ + "maxInFlightSqlTransactions", + values.maxInFlightSqlAdmission + + values.maxInFlightSqlTerminal + + values.maxInFlightSqlDelivery + + values.maxInFlightSqlMaintenance <= + values.maxInFlightSqlTransactions + ], + ["claimRecoveryRowsPerSecond", values.claimRecoveryRowsPerSecond >= values.admissionRatePerSecond], + ["expiryRowsPerSecond", values.expiryRowsPerSecond >= values.admissionRatePerSecond], + ["integrityRowsPerSecond", values.integrityRowsPerSecond >= values.admissionRatePerSecond], + ["reconciliationRowsPerSecond", values.reconciliationRowsPerSecond >= values.admissionRatePerSecond], + ["terminalCollectionRowsPerSecond", values.terminalCollectionRowsPerSecond >= values.admissionRatePerSecond], + ["orphanChannelCleanupRowsPerSecond", values.orphanChannelCleanupRowsPerSecond >= values.admissionRatePerSecond], + ["shutdownReleaseConcurrency", values.shutdownReleaseConcurrency <= values.maxInFlightSqlTransactions], + ["shutdownReleaseTimeoutMillis", values.shutdownReleaseTimeoutMillis < values.claimLeaseMillis] + ] + const firstInvalid = relations.find(([, valid]) => !valid) + if (firstInvalid !== undefined) return invalid(firstInvalid[0]) + + const requiredTransactionsPerSecond = values.admissionRatePerSecond + + values.claimRecoveryRowsPerSecond / values.claimRecoveryBatchSize + + values.expiryRowsPerSecond / values.expiryBatchSize + + values.integrityRowsPerSecond / values.integrityBatchSize + + values.reconciliationRowsPerSecond / values.reconciliationBatchSize + + values.terminalCollectionRowsPerSecond / values.terminalCollectionBatchSize + + values.orphanChannelCleanupRowsPerSecond / values.orphanChannelCleanupBatchSize + return values.sqliteTransactionCapacityPerSecond >= + requiredTransactionsPerSecond * (1 + values.sqliteCapacityHeadroomPercent / 100) + ? Effect.succeed(values) + : invalid("sqliteTransactionCapacityPerSecond") +} + +export const make = (values: Values) => + Values.makeEffect(values).pipe( + Effect.catchTag("SchemaError", (error) => + Effect.fail(new InvalidPeerRelayLimits({ field: firstField(error.issue) ?? "values" }))), + Effect.flatMap(validate) + ) + +export const layer = (values: Values) => Layer.effect(PeerRelayLimits, make(values)) + +export const layerDefaults = layer(defaults) diff --git a/packages/local-rpc/src/PeerRelayRpc.ts b/packages/local-rpc/src/PeerRelayRpc.ts new file mode 100644 index 0000000..b429f0f --- /dev/null +++ b/packages/local-rpc/src/PeerRelayRpc.ts @@ -0,0 +1,149 @@ +import * as Identity from "@lucas-barake/effect-local/Identity" +import * as PeerSyncEnvelope from "@lucas-barake/effect-local-sql/PeerSyncEnvelope" +import type * as Effect from "effect/Effect" +import * as Schema from "effect/Schema" +import type * as Scope from "effect/Scope" +import * as Rpc from "effect/unstable/rpc/Rpc" +import { make as makeClient } from "effect/unstable/rpc/RpcClient" +import type * as RpcClient_ from "effect/unstable/rpc/RpcClient" +import type { RpcClientError } from "effect/unstable/rpc/RpcClientError" +import * as RpcGroup from "effect/unstable/rpc/RpcGroup" +import type * as RpcMiddleware from "effect/unstable/rpc/RpcMiddleware" +import * as PeerAuthentication from "./PeerAuthentication.js" +import * as PeerRpc from "./PeerRpc.js" +import * as PeerRpcError from "./PeerRpcError.js" + +export const protocolVersion = 3 + +export const maximumNegotiatedDurationMillis = 90 * 24 * 60 * 60 * 1_000 +export const maximumRelayPayloadBytes = PeerSyncEnvelope.maximumRelayPayloadBytes +export const maximumRequestedDocuments = 1_000 + +const DurationMillis = Schema.Int.check( + Schema.isGreaterThan(0), + Schema.isLessThanOrEqualTo(maximumNegotiatedDurationMillis) +) + +const Credential = Schema.optionalKey(Schema.RedactedFromValue(Schema.String)) +const BoundedName = Schema.NonEmptyString.check(Schema.isMaxLength(256)) +const BoundedPeerPrincipal = PeerSyncEnvelope.RelayPeerPrincipal +const MessageHash = PeerSyncEnvelope.RelayOuterEnvelope.fields.messageHash +const Payload = PeerSyncEnvelope.RelayOuterEnvelope.fields.payload + +export const ClaimToken = Schema.String.check( + Schema.isPattern(/^clm_[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/) +).pipe(Schema.brand("@lucas-barake/effect-local-rpc/ClaimToken")) +export type ClaimToken = typeof ClaimToken.Type + +export const RelayDigest = Schema.String.check(Schema.isPattern(/^[0-9a-f]{64}$/)) +export type RelayDigest = typeof RelayDigest.Type + +export const RelayOpened = Schema.TaggedStruct("RelayOpened", { + version: Schema.Literal(protocolVersion), + sessionId: Identity.SessionId, + remotePeerId: Identity.PeerId, + authenticatedLocal: BoundedPeerPrincipal, + capabilities: Schema.Struct({ storeAndForward: Schema.Literal(true) }) +}) +export type RelayOpened = typeof RelayOpened.Type + +export const StoredMessage = Schema.TaggedStruct("StoredMessage", { + relayMessageId: Identity.RelayMessageId, + claimToken: ClaimToken, + relayPeerId: Identity.PeerId, + sender: Schema.Struct({ + tenantId: PeerSyncEnvelope.RelayPeerPrincipal.fields.tenantId, + subjectId: PeerSyncEnvelope.RelayPeerPrincipal.fields.subjectId, + peerId: PeerSyncEnvelope.RelayPeerPrincipal.fields.peerId, + replicaIncarnation: Identity.ReplicaIncarnation, + connectionEpoch: PeerSyncEnvelope.RelayOuterEnvelope.fields.senderConnectionEpoch, + sequence: PeerSyncEnvelope.RelayOuterEnvelope.fields.senderSequence + }), + recipient: BoundedPeerPrincipal, + payloadVersion: PeerSyncEnvelope.RelayOuterEnvelope.fields.payloadVersion, + document: PeerSyncEnvelope.RelayOuterEnvelope.fields.document, + writerProvenance: PeerSyncEnvelope.RelayOuterEnvelope.fields.writerProvenance, + messageHash: MessageHash, + outerEnvelopeDigest: RelayDigest, + payload: Payload +}) +export type StoredMessage = typeof StoredMessage.Type + +export const OpenRelayEvent = Schema.Union([RelayOpened, StoredMessage]) +export type OpenRelayEvent = typeof OpenRelayEvent.Type + +export const RejectReason = Schema.Literals(["ProtocolInvalid", "ApplicationRejected"]) +export type RejectReason = typeof RejectReason.Type + +export class OpenRelayRpc extends Rpc.make("OpenRelay", { + payload: { + version: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + expectedRelayPeerId: Identity.PeerId, + expectedLocal: BoundedPeerPrincipal, + senderReplicaIncarnation: Identity.ReplicaIncarnation, + remote: Schema.Struct({ + subjectId: BoundedName, + peerId: Identity.PeerId + }), + documents: Schema.Array(PeerRpc.RequestedDocument).check( + Schema.isMinLength(1), + Schema.isMaxLength(maximumRequestedDocuments) + ), + receiptRetentionMillis: DurationMillis, + senderRetryHorizonMillis: DurationMillis, + credential: Credential + }, + success: OpenRelayEvent, + error: PeerRpcError.PeerRpcError, + defect: PeerRpcError.Defect, + stream: true +}) {} + +export class PushRelayRpc extends Rpc.make("PushRelay", { + payload: { + sessionId: Identity.SessionId, + relayMessageId: Identity.RelayMessageId, + payload: Payload, + credential: Credential + }, + error: PeerRpcError.PeerRpcError, + defect: PeerRpcError.Defect +}) {} + +const TerminalPayload = { + sessionId: Identity.SessionId, + relayMessageId: Identity.RelayMessageId, + claimToken: ClaimToken, + messageHash: MessageHash, + credential: Credential +} + +export class AcknowledgeRelayRpc extends Rpc.make("AcknowledgeRelay", { + payload: TerminalPayload, + error: PeerRpcError.PeerRpcError, + defect: PeerRpcError.Defect +}) {} + +export class RejectRelayRpc extends Rpc.make("RejectRelay", { + payload: { + ...TerminalPayload, + reason: RejectReason + }, + error: PeerRpcError.PeerRpcError, + defect: PeerRpcError.Defect +}) {} + +export class Rpcs extends RpcGroup.make( + OpenRelayRpc, + PushRelayRpc, + AcknowledgeRelayRpc, + RejectRelayRpc +).middleware(PeerAuthentication.PeerAuthentication) {} + +export interface RpcClient extends RpcClient_.FromGroup {} + +export const makeRpcClient: Effect.Effect< + RpcClient, + never, + RpcClient_.Protocol | RpcMiddleware.ForClient | Scope.Scope +> = makeClient(Rpcs) diff --git a/packages/local-rpc/src/PeerRelayStore.ts b/packages/local-rpc/src/PeerRelayStore.ts new file mode 100644 index 0000000..3f881d6 --- /dev/null +++ b/packages/local-rpc/src/PeerRelayStore.ts @@ -0,0 +1,1673 @@ +import * as Identity from "@lucas-barake/effect-local/Identity" +import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" +import type * as Cause from "effect/Cause" +import * as Context from "effect/Context" +import * as Crypto from "effect/Crypto" +import * as Effect from "effect/Effect" +import * as Layer from "effect/Layer" +import * as Option from "effect/Option" +import type * as PlatformError from "effect/PlatformError" +import * as Random from "effect/Random" +import * as Schema from "effect/Schema" +import type * as Migrator from "effect/unstable/sql/Migrator" +import * as SqlClient from "effect/unstable/sql/SqlClient" +import type * as SqlError from "effect/unstable/sql/SqlError" +import * as SqlSchema from "effect/unstable/sql/SqlSchema" +import { PeerPrincipal } from "./internal/peerPrincipal.js" +import * as PeerRelayMigrations from "./internal/peerRelayMigrations.js" +import { + make as makeImmediateTransaction, + type NestedPeerRelayTransactionError +} from "./internal/peerRelaySqliteTransaction.js" +import * as PeerRelayLimits from "./PeerRelayLimits.js" +import * as PeerRelayRpc from "./PeerRelayRpc.js" + +const PositiveInt = Schema.Int.check(Schema.isGreaterThan(0)) +const NonNegativeInt = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)) +const DocumentIds = Schema.Array(Identity.DocumentId).check(Schema.isMinLength(1)) + +export const ChannelKey = Schema.Struct({ + tenantId: Schema.NonEmptyString, + senderSubjectId: Schema.NonEmptyString, + senderPeerId: Identity.PeerId, + senderReplicaIncarnation: Identity.ReplicaIncarnation, + recipientSubjectId: Schema.NonEmptyString, + recipientPeerId: Identity.PeerId +}) +export type ChannelKey = typeof ChannelKey.Type + +export const Admission = Schema.Struct({ + channel: ChannelKey, + relayMessageId: Identity.RelayMessageId, + relayPeerId: Identity.PeerId, + documentIds: DocumentIds, + senderConnectionEpoch: Schema.NonEmptyString.check(Schema.isMaxLength(256)), + senderSequence: NonNegativeInt, + payloadVersion: Schema.Literal(1), + messageHash: Schema.NonEmptyString.check(Schema.isMaxLength(256)), + outerEnvelopeDigest: PeerRelayRpc.RelayDigest, + payload: Schema.Uint8Array, + messageTtlMillis: PositiveInt, + senderRetryHorizonMillis: PositiveInt, + minimumTerminalRetentionMillis: PositiveInt +}) +export type Admission = typeof Admission.Type + +export const AdmissionResult = Schema.Struct({ + status: Schema.Literals(["Accepted", "Duplicate"]), + channel: ChannelKey, + ready: Schema.Boolean, + nextEligibleAt: NonNegativeInt, + lane: Schema.Literal("New") +}) +export type AdmissionResult = typeof AdmissionResult.Type + +export const ClaimRequest = Schema.Struct({ + recipient: PeerPrincipal, + sender: Schema.Struct({ + subjectId: Schema.NonEmptyString, + peerId: Identity.PeerId + }), + sessionGeneration: NonNegativeInt, + authorizedDocumentIds: DocumentIds +}) +export type ClaimRequest = typeof ClaimRequest.Type + +export const ClaimedMessage = Schema.Struct({ + rowId: PositiveInt, + channel: ChannelKey, + relayMessageId: Identity.RelayMessageId, + relayPeerId: Identity.PeerId, + senderConnectionEpoch: Schema.NonEmptyString, + senderSequence: NonNegativeInt, + documentIds: DocumentIds, + payloadVersion: Schema.Literal(1), + messageHash: Schema.NonEmptyString, + outerEnvelopeDigest: PeerRelayRpc.RelayDigest, + payloadBytes: NonNegativeInt, + claimToken: PeerRelayRpc.ClaimToken, + claimDeadline: NonNegativeInt, + sessionGeneration: NonNegativeInt +}) +export type ClaimedMessage = typeof ClaimedMessage.Type + +export interface ClaimResult { + readonly message: Option.Option + readonly ready: boolean + readonly nextEligibleAt: Option.Option + readonly lane: "New" | "Retry" +} + +export const LoadClaimedPayloadRequest = Schema.Struct({ + rowId: PositiveInt, + channel: ChannelKey, + relayMessageId: Identity.RelayMessageId, + claimToken: PeerRelayRpc.ClaimToken, + sessionGeneration: NonNegativeInt, + payloadBytes: NonNegativeInt +}) +export type LoadClaimedPayloadRequest = typeof LoadClaimedPayloadRequest.Type + +export const TerminalRequest = Schema.Struct({ + channel: ChannelKey, + relayMessageId: Identity.RelayMessageId, + claimToken: PeerRelayRpc.ClaimToken, + messageHash: Schema.NonEmptyString.check(Schema.isMaxLength(256)), + sessionGeneration: NonNegativeInt, + recipient: PeerPrincipal +}) +export type TerminalRequest = typeof TerminalRequest.Type + +export const RejectRequest = Schema.Struct({ + ...TerminalRequest.fields, + reason: PeerRelayRpc.RejectReason +}) +export type RejectRequest = typeof RejectRequest.Type + +export const ReleaseRequest = Schema.Struct({ + channel: ChannelKey, + relayMessageId: Identity.RelayMessageId, + claimToken: PeerRelayRpc.ClaimToken, + sessionGeneration: NonNegativeInt +}) +export type ReleaseRequest = typeof ReleaseRequest.Type + +export interface TransitionResult { + readonly status: "Changed" | "Duplicate" | "Stale" + readonly ready: boolean + readonly nextEligibleAt: Option.Option + readonly lane: "New" | "Retry" +} + +export const MaintenanceRequest = Schema.Struct({ + cursor: Schema.optionalKey(NonNegativeInt), + batchSize: PositiveInt +}) +export type MaintenanceRequest = typeof MaintenanceRequest.Type + +export interface MaintenanceResult { + readonly cursor?: number + readonly processed: number + readonly hasMore: boolean +} + +export const UsageRequest = Schema.Struct({ + scopeKind: Schema.Literals(["SenderPeer", "RecipientPeer", "RecipientSubject", "Tenant", "Shard"]), + scopeKey: Schema.String +}) +export type UsageRequest = typeof UsageRequest.Type + +export interface Usage { + readonly activeCount: number + readonly activeBytes: number + readonly retainedCount: number + readonly retainedBytes: number +} + +export type StoreError = + | ReplicaError.ReplicaError + | NestedPeerRelayTransactionError + +export interface Service { + readonly admit: (input: Admission) => Effect.Effect + readonly claim: (input: ClaimRequest) => Effect.Effect + readonly loadClaimedPayload: ( + input: LoadClaimedPayloadRequest + ) => Effect.Effect + readonly acknowledge: (input: TerminalRequest) => Effect.Effect + readonly reject: (input: RejectRequest) => Effect.Effect + readonly release: (input: ReleaseRequest) => Effect.Effect + readonly recover: (input: MaintenanceRequest) => Effect.Effect + readonly expire: (input: MaintenanceRequest) => Effect.Effect + readonly repair: (input: MaintenanceRequest) => Effect.Effect + readonly reconcile: (input: MaintenanceRequest) => Effect.Effect + readonly collect: (input: MaintenanceRequest) => Effect.Effect + readonly usage: (input?: UsageRequest) => Effect.Effect +} + +export class PeerRelayStore extends Context.Service()( + "@lucas-barake/effect-local-rpc/PeerRelayStore" +) {} + +type UsageKind = UsageRequest["scopeKind"] + +interface UsageScope { + readonly kind: UsageKind + readonly key: string + readonly activeCountLimit: number + readonly activeBytesLimit: number + readonly retainedCountLimit: number + readonly retainedBytesLimit: number +} + +const storageUnavailable = (cause: unknown) => + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + +const storageCorrupt = (cause: unknown) => + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + +const protocolMismatch = (expected: string, observed: string) => + new ReplicaError.ReplicaError({ + reason: new ReplicaError.ProtocolMismatch({ expected, observed }) + }) + +const quotaExceeded = (resource: string, limit: number) => + new ReplicaError.ReplicaError({ + reason: new ReplicaError.QuotaExceeded({ resource, limit }) + }) + +type BoundaryError = + | StoreError + | SqlError.SqlError + | Schema.SchemaError + | PlatformError.PlatformError + | Cause.NoSuchElementError + +const mapStorageErrors = (effect: Effect.Effect) => + effect.pipe( + Effect.catchTag("SqlError", (cause) => Effect.fail(storageUnavailable(cause))), + Effect.catchTag("SchemaError", (cause) => Effect.fail(storageCorrupt(cause))), + Effect.catchTag("PlatformError", (cause) => Effect.fail(storageUnavailable(cause))), + Effect.catchTag("NoSuchElementError", (cause) => Effect.fail(storageCorrupt(cause))) + ) + +const encodeKey = (...parts: ReadonlyArray) => JSON.stringify(parts) + +const channelScopes = ( + channel: ChannelKey, + limits: PeerRelayLimits.Values +): ReadonlyArray => [ + { + kind: "SenderPeer", + key: encodeKey(channel.tenantId, channel.senderSubjectId, channel.senderPeerId), + activeCountLimit: limits.maxActiveMessagesPerSenderPeer, + activeBytesLimit: limits.maxActiveBytesPerSenderPeer, + retainedCountLimit: limits.maxRetainedRowsPerSenderPeer, + retainedBytesLimit: limits.maxRetainedBytesPerSenderPeer + }, + { + kind: "RecipientPeer", + key: encodeKey(channel.tenantId, channel.recipientSubjectId, channel.recipientPeerId), + activeCountLimit: limits.maxActiveMessagesPerRecipientPeer, + activeBytesLimit: limits.maxActiveBytesPerRecipientPeer, + retainedCountLimit: limits.maxRetainedRowsPerRecipientPeer, + retainedBytesLimit: limits.maxRetainedBytesPerRecipientPeer + }, + { + kind: "RecipientSubject", + key: encodeKey(channel.tenantId, channel.recipientSubjectId), + activeCountLimit: limits.maxActiveMessagesPerRecipientSubject, + activeBytesLimit: limits.maxActiveBytesPerRecipientSubject, + retainedCountLimit: limits.maxRetainedRowsPerRecipientSubject, + retainedBytesLimit: limits.maxRetainedBytesPerRecipientSubject + }, + { + kind: "Tenant", + key: encodeKey(channel.tenantId), + activeCountLimit: limits.maxActiveMessagesPerTenant, + activeBytesLimit: limits.maxActiveBytesPerTenant, + retainedCountLimit: limits.maxRetainedRowsPerTenant, + retainedBytesLimit: limits.maxRetainedBytesPerTenant + }, + { + kind: "Shard", + key: encodeKey("local"), + activeCountLimit: limits.maxActiveMessagesPerShard, + activeBytesLimit: limits.maxActiveBytesPerShard, + retainedCountLimit: limits.maxRetainedRowsPerShard, + retainedBytesLimit: limits.maxRetainedBytesPerShard + } +] + +const TimeRow = Schema.Struct({ now: NonNegativeInt }) +const UnitRow = Schema.Struct({ value: Schema.Int }) + +const nowQuery = (sql: SqlClient.SqlClient) => + SqlSchema.findOne({ + Request: Schema.Void, + Result: TimeRow, + execute: () => sql`SELECT CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) AS now` + })(undefined) + +const decodeDocuments = Schema.decodeUnknownEffect( + Schema.fromJsonString(DocumentIds) +) + +const parseDocuments = (value: string) => + decodeDocuments(value).pipe( + Effect.mapError((cause) => storageCorrupt(cause)) + ) + +const validateInput = (schema: S, input: unknown) => + Schema.decodeUnknownEffect(schema)(input).pipe( + Effect.mapError((cause) => protocolMismatch("valid relay store request", String(cause))) + ) + +const checkPragmas = Effect.gen(function*() { + const sql = yield* SqlClient.SqlClient + const journal = SqlSchema.findOne({ + Request: Schema.Void, + Result: Schema.Struct({ journal_mode: Schema.String }), + execute: () => sql`PRAGMA journal_mode = WAL` + }) + const synchronous = SqlSchema.findOne({ + Request: Schema.Void, + Result: Schema.Struct({ synchronous: Schema.Int }), + execute: () => sql`PRAGMA synchronous` + }) + const mode = yield* journal(undefined) + if (mode.journal_mode.toLowerCase() !== "wal") { + return yield* storageCorrupt(new Error("Relay custody requires SQLite WAL mode")) + } + yield* sql`PRAGMA synchronous = FULL` + const setting = yield* synchronous(undefined) + if (setting.synchronous !== 2) { + return yield* storageCorrupt(new Error("Relay custody requires SQLite FULL synchronous mode")) + } +}) + +const DuplicateRow = Schema.Struct({ + channelId: PositiveInt, + tenantId: Schema.NonEmptyString, + senderSubjectId: Schema.NonEmptyString, + senderPeerId: Identity.PeerId, + senderReplicaIncarnation: Identity.ReplicaIncarnation, + recipientSubjectId: Schema.NonEmptyString, + recipientPeerId: Identity.PeerId, + outerEnvelopeDigest: PeerRelayRpc.RelayDigest, + state: Schema.Literals(["Pending", "Claimed", "Acknowledged", "DeadLettered", "Expired"]), + nextEligibleAt: NonNegativeInt +}) + +const ChannelRow = Schema.Struct({ + channelId: PositiveInt, + nextSequence: NonNegativeInt +}) + +const MessageIdRow = Schema.Struct({ messageId: PositiveInt }) + +const CandidateRow = Schema.Struct({ + messageId: PositiveInt, + channelId: PositiveInt, + tenantId: Schema.NonEmptyString, + senderSubjectId: Schema.NonEmptyString, + senderPeerId: Identity.PeerId, + senderReplicaIncarnation: Identity.ReplicaIncarnation, + recipientSubjectId: Schema.NonEmptyString, + recipientPeerId: Identity.PeerId, + relayMessageId: Identity.RelayMessageId, + relayPeerId: Identity.PeerId, + senderConnectionEpoch: Schema.NonEmptyString, + senderSequence: NonNegativeInt, + documentIds: Schema.String, + payloadVersion: Schema.Literal(1), + messageHash: Schema.NonEmptyString, + outerEnvelopeDigest: PeerRelayRpc.RelayDigest, + payloadBytes: NonNegativeInt, + createdAt: NonNegativeInt, + nextEligibleAt: NonNegativeInt, + retryCount: NonNegativeInt +}) + +const ReservationRow = Schema.Struct({ + senderPeerUsageKey: Schema.String, + recipientPeerUsageKey: Schema.String, + recipientSubjectUsageKey: Schema.String, + tenantUsageKey: Schema.String, + shardUsageKey: Schema.String, + activeCountDelta: Schema.Literal(1), + activeBytesDelta: NonNegativeInt, + retainedCountDelta: Schema.Literal(1), + retainedBytesDelta: NonNegativeInt, + activeConsumed: Schema.Literals([0, 1]), + retainedConsumed: Schema.Literals([0, 1]) +}) + +const UsageRow = Schema.Struct({ + activeCount: NonNegativeInt, + activeBytes: NonNegativeInt, + retainedCount: NonNegativeInt, + retainedBytes: NonNegativeInt +}) + +const KeyRow = Schema.Struct({ messageId: PositiveInt }) + +const makeService = Effect.gen(function*() { + const sql = yield* SqlClient.SqlClient + const crypto = yield* Crypto.Crypto + const limits = yield* PeerRelayLimits.PeerRelayLimits + yield* PeerRelayMigrations.run + yield* checkPragmas.pipe( + Effect.catchTag("NoSuchElementError", (cause) => Effect.fail(storageCorrupt(cause))) + ) + + const write = makeImmediateTransaction(sql, { + maxAcquireAttempts: limits.sqliteLockRetryMaxAttempts, + acquireRetryBaseDelayMillis: limits.sqliteLockRetryBaseDelayMillis, + acquireRetryMaximumDelayMillis: limits.sqliteLockRetryMaximumDelayMillis + }) + + const findDuplicate = SqlSchema.findOneOption({ + Request: Schema.Struct({ + tenantId: Schema.String, + senderSubjectId: Schema.String, + senderPeerId: Schema.String, + relayMessageId: Schema.String + }), + Result: DuplicateRow, + execute: (request) => + sql`SELECT + m.channel_id AS channelId, + c.tenant_id AS tenantId, + c.sender_subject_id AS senderSubjectId, + c.sender_peer_id AS senderPeerId, + c.sender_replica_incarnation AS senderReplicaIncarnation, + c.recipient_subject_id AS recipientSubjectId, + c.recipient_peer_id AS recipientPeerId, + m.outer_envelope_digest AS outerEnvelopeDigest, + m.state, + m.next_eligible_at AS nextEligibleAt + FROM effect_local_relay_messages m + JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id + JOIN effect_local_relay_reservations r ON r.message_id = m.message_id + WHERE m.tenant_id = ${request.tenantId} + AND m.sender_subject_id = ${request.senderSubjectId} + AND m.sender_peer_id = ${request.senderPeerId} + AND m.relay_message_id = ${request.relayMessageId}` + }) + + const findChannel = SqlSchema.findOne({ + Request: ChannelKey, + Result: ChannelRow, + execute: (channel) => + sql`SELECT + channel_id AS channelId, + next_sequence AS nextSequence + FROM effect_local_relay_channels + WHERE tenant_id = ${channel.tenantId} + AND sender_subject_id = ${channel.senderSubjectId} + AND sender_peer_id = ${channel.senderPeerId} + AND sender_replica_incarnation = ${channel.senderReplicaIncarnation} + AND recipient_subject_id = ${channel.recipientSubjectId} + AND recipient_peer_id = ${channel.recipientPeerId}` + }) + + const reserveUsage = ( + scope: UsageScope, + payloadBytes: number + ): Effect.Effect => + Effect.gen(function*() { + yield* sql`INSERT INTO effect_local_relay_usage ( + scope_kind, + scope_key, + active_count, + active_bytes, + retained_count, + retained_bytes + ) VALUES (${scope.kind}, ${scope.key}, 0, 0, 0, 0) + ON CONFLICT(scope_kind, scope_key) DO NOTHING` + const changed = yield* SqlSchema.findAll({ + Request: Schema.Void, + Result: UnitRow, + execute: () => + sql`UPDATE effect_local_relay_usage + SET active_count = active_count + 1, + active_bytes = active_bytes + ${payloadBytes}, + retained_count = retained_count + 1, + retained_bytes = retained_bytes + ${payloadBytes} + WHERE scope_kind = ${scope.kind} + AND scope_key = ${scope.key} + AND active_count + 1 <= ${scope.activeCountLimit} + AND active_bytes + ${payloadBytes} <= ${scope.activeBytesLimit} + AND retained_count + 1 <= ${scope.retainedCountLimit} + AND retained_bytes + ${payloadBytes} <= ${scope.retainedBytesLimit} + RETURNING 1 AS value` + })(undefined) + if (changed.length !== 1) { + return yield* quotaExceeded( + `${scope.kind} relay custody`, + Math.min(scope.activeCountLimit, scope.retainedCountLimit) + ) + } + }) + + const findReservation = SqlSchema.findOneOption({ + Request: PositiveInt, + Result: ReservationRow, + execute: (messageId) => + sql`SELECT + sender_peer_usage_key AS senderPeerUsageKey, + recipient_peer_usage_key AS recipientPeerUsageKey, + recipient_subject_usage_key AS recipientSubjectUsageKey, + tenant_usage_key AS tenantUsageKey, + shard_usage_key AS shardUsageKey, + active_count_delta AS activeCountDelta, + active_bytes_delta AS activeBytesDelta, + retained_count_delta AS retainedCountDelta, + retained_bytes_delta AS retainedBytesDelta, + active_consumed AS activeConsumed, + retained_consumed AS retainedConsumed + FROM effect_local_relay_reservations + WHERE message_id = ${messageId}` + }) + + const usageKeys = ( + reservation: typeof ReservationRow.Type + ): ReadonlyArray => [ + ["SenderPeer", reservation.senderPeerUsageKey], + ["RecipientPeer", reservation.recipientPeerUsageKey], + ["RecipientSubject", reservation.recipientSubjectUsageKey], + ["Tenant", reservation.tenantUsageKey], + ["Shard", reservation.shardUsageKey] + ] + + const releaseActiveUsage = ( + messageId: number + ): Effect.Effect => + Effect.gen(function*() { + const reservationOption = yield* findReservation(messageId) + if (Option.isNone(reservationOption)) { + return yield* storageCorrupt(new Error("Missing relay quota reservation")) + } + const reservation = reservationOption.value + if (reservation.activeConsumed === 1) { + return + } + for (const [kind, key] of usageKeys(reservation)) { + const changed = yield* SqlSchema.findAll({ + Request: Schema.Void, + Result: UnitRow, + execute: () => + sql`UPDATE effect_local_relay_usage + SET active_count = active_count - ${reservation.activeCountDelta}, + active_bytes = active_bytes - ${reservation.activeBytesDelta} + WHERE scope_kind = ${kind} + AND scope_key = ${key} + AND active_count >= ${reservation.activeCountDelta} + AND active_bytes >= ${reservation.activeBytesDelta} + RETURNING 1 AS value` + })(undefined) + if (changed.length !== 1) { + return yield* storageCorrupt(new Error("Invalid active relay quota reservation")) + } + yield* sql`DELETE FROM effect_local_relay_usage + WHERE scope_kind = ${kind} + AND scope_key = ${key} + AND active_count = 0 + AND active_bytes = 0 + AND retained_count = 0 + AND retained_bytes = 0` + } + yield* sql`UPDATE effect_local_relay_reservations + SET active_consumed = 1 + WHERE message_id = ${messageId} AND active_consumed = 0` + }) + + const releaseRetainedUsage = ( + messageId: number + ): Effect.Effect => + Effect.gen(function*() { + const reservationOption = yield* findReservation(messageId) + if (Option.isNone(reservationOption)) { + return yield* storageCorrupt(new Error("Missing relay quota reservation")) + } + const reservation = reservationOption.value + if (reservation.retainedConsumed === 1) { + return + } + for (const [kind, key] of usageKeys(reservation)) { + const changed = yield* SqlSchema.findAll({ + Request: Schema.Void, + Result: UnitRow, + execute: () => + sql`UPDATE effect_local_relay_usage + SET retained_count = retained_count - ${reservation.retainedCountDelta}, + retained_bytes = retained_bytes - ${reservation.retainedBytesDelta} + WHERE scope_kind = ${kind} + AND scope_key = ${key} + AND retained_count >= ${reservation.retainedCountDelta} + AND retained_bytes >= ${reservation.retainedBytesDelta} + RETURNING 1 AS value` + })(undefined) + if (changed.length !== 1) { + return yield* storageCorrupt(new Error("Invalid retained relay quota reservation")) + } + } + yield* sql`UPDATE effect_local_relay_reservations + SET retained_consumed = 1 + WHERE message_id = ${messageId} AND retained_consumed = 0` + }) + + const terminalize = ( + messageId: number, + state: "Acknowledged" | "DeadLettered" | "Expired", + now: number, + terminal: { + readonly token?: PeerRelayRpc.ClaimToken + readonly sessionGeneration?: number + readonly reason: string + } + ) => + Effect.gen(function*() { + yield* releaseActiveUsage(messageId) + yield* sql`UPDATE effect_local_relay_messages + SET state = ${state}, + payload = NULL, + payload_length = 0, + claim_token = NULL, + claim_session_generation = NULL, + claim_deadline = NULL, + terminal_at = ${now}, + terminal_claim_token = ${terminal.token ?? null}, + terminal_session_generation = ${terminal.sessionGeneration ?? null}, + terminal_reason = ${terminal.reason}, + deduplicate_until = MAX( + deduplicate_until, + ${now + limits.minimumTerminalRetentionMillis} + ) + WHERE message_id = ${messageId} + AND state IN ('Pending', 'Claimed')` + yield* sql`UPDATE effect_local_relay_channels + SET claimed_message_id = NULL, + claim_session_generation = NULL, + claim_token = NULL, + claim_deadline = NULL + WHERE claimed_message_id = ${messageId}` + }) + + const admit: Service["admit"] = (unsafeInput) => + mapStorageErrors(Effect.gen(function*() { + const input = yield* validateInput(Admission, unsafeInput) + const payloadBytes = input.payload.byteLength + if ( + input.messageTtlMillis > limits.messageTtlMillis || + input.senderRetryHorizonMillis > limits.maximumSenderRetryHorizonMillis || + input.minimumTerminalRetentionMillis < limits.minimumTerminalRetentionMillis + ) { + return yield* protocolMismatch("configured relay retention horizon", "unsupported horizon") + } + if ( + payloadBytes > limits.maxActiveBytesPerSenderPeer || + payloadBytes > limits.maxActiveBytesPerRecipientPeer || + payloadBytes > limits.maxActiveBytesPerRecipientSubject || + payloadBytes > limits.maxActiveBytesPerTenant || + payloadBytes > limits.maxActiveBytesPerShard + ) { + return yield* quotaExceeded("relay payload bytes", limits.maxActiveBytesPerShard) + } + return yield* write(Effect.gen(function*() { + const now = (yield* nowQuery(sql)).now + const existing = yield* findDuplicate({ + tenantId: input.channel.tenantId, + senderSubjectId: input.channel.senderSubjectId, + senderPeerId: input.channel.senderPeerId, + relayMessageId: input.relayMessageId + }) + if (Option.isSome(existing)) { + if (existing.value.outerEnvelopeDigest !== input.outerEnvelopeDigest) { + return yield* protocolMismatch( + existing.value.outerEnvelopeDigest, + input.outerEnvelopeDigest + ) + } + return AdmissionResult.make({ + status: "Duplicate", + channel: ChannelKey.make({ + tenantId: existing.value.tenantId, + senderSubjectId: existing.value.senderSubjectId, + senderPeerId: existing.value.senderPeerId, + senderReplicaIncarnation: existing.value.senderReplicaIncarnation, + recipientSubjectId: existing.value.recipientSubjectId, + recipientPeerId: existing.value.recipientPeerId + }), + ready: existing.value.state === "Pending" && existing.value.nextEligibleAt <= now, + nextEligibleAt: existing.value.nextEligibleAt, + lane: "New" + }) + } + yield* sql`INSERT INTO effect_local_relay_channels ( + tenant_id, + sender_subject_id, + sender_peer_id, + sender_replica_incarnation, + recipient_subject_id, + recipient_peer_id + ) VALUES ( + ${input.channel.tenantId}, + ${input.channel.senderSubjectId}, + ${input.channel.senderPeerId}, + ${input.channel.senderReplicaIncarnation}, + ${input.channel.recipientSubjectId}, + ${input.channel.recipientPeerId} + ) + ON CONFLICT ( + tenant_id, + sender_subject_id, + sender_peer_id, + sender_replica_incarnation, + recipient_subject_id, + recipient_peer_id + ) DO NOTHING` + const channel = yield* findChannel(input.channel) + const scopes = channelScopes(input.channel, limits) + for (const scope of scopes) { + yield* reserveUsage(scope, payloadBytes) + } + const duplicateHorizon = Math.max( + input.messageTtlMillis, + input.senderRetryHorizonMillis + ) + const inserted = yield* SqlSchema.findOne({ + Request: Schema.Void, + Result: MessageIdRow, + execute: () => + sql`INSERT INTO effect_local_relay_messages ( + channel_id, + channel_sequence, + tenant_id, + sender_subject_id, + sender_peer_id, + relay_message_id, + relay_peer_id, + sender_connection_epoch, + sender_sequence, + document_ids, + payload_version, + message_hash, + outer_envelope_digest, + payload, + payload_length, + state, + created_at, + expires_at, + deduplicate_until, + next_eligible_at + ) VALUES ( + ${channel.channelId}, + ${channel.nextSequence}, + ${input.channel.tenantId}, + ${input.channel.senderSubjectId}, + ${input.channel.senderPeerId}, + ${input.relayMessageId}, + ${input.relayPeerId}, + ${input.senderConnectionEpoch}, + ${input.senderSequence}, + ${JSON.stringify(input.documentIds)}, + ${input.payloadVersion}, + ${input.messageHash}, + ${input.outerEnvelopeDigest}, + ${input.payload}, + ${payloadBytes}, + 'Pending', + ${now}, + ${now + input.messageTtlMillis}, + ${now + duplicateHorizon}, + ${now} + ) + RETURNING message_id AS messageId` + })(undefined) + yield* sql`INSERT INTO effect_local_relay_reservations ( + message_id, + sender_peer_usage_key, + recipient_peer_usage_key, + recipient_subject_usage_key, + tenant_usage_key, + shard_usage_key, + active_count_delta, + active_bytes_delta, + retained_count_delta, + retained_bytes_delta + ) VALUES ( + ${inserted.messageId}, + ${scopes[0]!.key}, + ${scopes[1]!.key}, + ${scopes[2]!.key}, + ${scopes[3]!.key}, + ${scopes[4]!.key}, + 1, + ${payloadBytes}, + 1, + ${payloadBytes} + )` + yield* sql`UPDATE effect_local_relay_channels + SET next_sequence = next_sequence + 1 + WHERE channel_id = ${channel.channelId} + AND next_sequence = ${channel.nextSequence}` + return AdmissionResult.make({ + status: "Accepted", + channel: input.channel, + ready: true, + nextEligibleAt: now, + lane: "New" + }) + })) + })).pipe(Effect.withSpan("PeerRelayStore.admit")) + + const findCandidate = SqlSchema.findOneOption({ + Request: Schema.Struct({ + tenantId: Schema.String, + recipientSubjectId: Schema.String, + recipientPeerId: Schema.String, + senderSubjectId: Schema.String, + senderPeerId: Schema.String, + authorizedDocumentIds: Schema.String, + now: NonNegativeInt + }), + Result: CandidateRow, + execute: (request) => + sql`SELECT + m.message_id AS messageId, + m.channel_id AS channelId, + c.tenant_id AS tenantId, + c.sender_subject_id AS senderSubjectId, + c.sender_peer_id AS senderPeerId, + c.sender_replica_incarnation AS senderReplicaIncarnation, + c.recipient_subject_id AS recipientSubjectId, + c.recipient_peer_id AS recipientPeerId, + m.relay_message_id AS relayMessageId, + m.relay_peer_id AS relayPeerId, + m.sender_connection_epoch AS senderConnectionEpoch, + m.sender_sequence AS senderSequence, + m.document_ids AS documentIds, + m.payload_version AS payloadVersion, + m.message_hash AS messageHash, + m.outer_envelope_digest AS outerEnvelopeDigest, + m.payload_length AS payloadBytes, + m.created_at AS createdAt, + m.next_eligible_at AS nextEligibleAt, + m.retry_count AS retryCount + FROM effect_local_relay_messages m + JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id + JOIN effect_local_relay_reservations r ON r.message_id = m.message_id + WHERE c.tenant_id = ${request.tenantId} + AND c.recipient_subject_id = ${request.recipientSubjectId} + AND c.recipient_peer_id = ${request.recipientPeerId} + AND c.sender_subject_id = ${request.senderSubjectId} + AND c.sender_peer_id = ${request.senderPeerId} + AND m.tenant_id = c.tenant_id + AND m.sender_subject_id = c.sender_subject_id + AND m.sender_peer_id = c.sender_peer_id + AND c.claimed_message_id IS NULL + AND m.state = 'Pending' + AND m.next_eligible_at <= ${request.now} + AND m.expires_at > ${request.now} + AND m.payload IS NOT NULL + AND m.payload_length = length(m.payload) + AND r.active_consumed = 0 + AND NOT EXISTS ( + SELECT 1 + FROM effect_local_relay_messages earlier + WHERE earlier.channel_id = m.channel_id + AND earlier.channel_sequence < m.channel_sequence + AND earlier.state IN ('Pending', 'Claimed') + ) + AND NOT EXISTS ( + SELECT 1 + FROM json_each(m.document_ids) document + WHERE document.value NOT IN ( + SELECT value FROM json_each(${request.authorizedDocumentIds}) + ) + ) + ORDER BY m.created_at, m.message_id + LIMIT 1` + }) + + const claim: Service["claim"] = (unsafeInput) => + mapStorageErrors(Effect.gen(function*() { + const input = yield* validateInput(ClaimRequest, unsafeInput) + return yield* write(Effect.gen(function*() { + const now = (yield* nowQuery(sql)).now + const candidateOption = yield* findCandidate({ + tenantId: input.recipient.tenantId, + recipientSubjectId: input.recipient.subjectId, + recipientPeerId: input.recipient.peerId, + senderSubjectId: input.sender.subjectId, + senderPeerId: input.sender.peerId, + authorizedDocumentIds: JSON.stringify(input.authorizedDocumentIds), + now + }) + if (Option.isNone(candidateOption)) { + return { + message: Option.none(), + ready: false, + nextEligibleAt: Option.none(), + lane: "New" as const + } + } + const candidate = candidateOption.value + const documentIds = yield* parseDocuments(candidate.documentIds) + const uuid = yield* crypto.randomUUIDv4 + const token = PeerRelayRpc.ClaimToken.make(`clm_${uuid}`) + const deadline = now + limits.claimLeaseMillis + const claimed = yield* SqlSchema.findAll({ + Request: Schema.Void, + Result: UnitRow, + execute: () => + sql`UPDATE effect_local_relay_messages + SET state = 'Claimed', + claim_token = ${token}, + claim_session_generation = ${input.sessionGeneration}, + claim_deadline = ${deadline} + WHERE message_id = ${candidate.messageId} + AND state = 'Pending' + RETURNING 1 AS value` + })(undefined) + if (claimed.length !== 1) { + return yield* storageCorrupt(new Error("Relay claim compare and swap failed")) + } + const channelClaimed = yield* SqlSchema.findAll({ + Request: Schema.Void, + Result: UnitRow, + execute: () => + sql`UPDATE effect_local_relay_channels + SET claimed_message_id = ${candidate.messageId}, + claim_session_generation = ${input.sessionGeneration}, + claim_token = ${token}, + claim_deadline = ${deadline} + WHERE channel_id = ${candidate.channelId} + AND claimed_message_id IS NULL + RETURNING 1 AS value` + })(undefined) + if (channelClaimed.length !== 1) { + return yield* storageCorrupt(new Error("Relay channel claim compare and swap failed")) + } + return { + message: Option.some(ClaimedMessage.make({ + rowId: candidate.messageId, + channel: ChannelKey.make({ + tenantId: candidate.tenantId, + senderSubjectId: candidate.senderSubjectId, + senderPeerId: candidate.senderPeerId, + senderReplicaIncarnation: candidate.senderReplicaIncarnation, + recipientSubjectId: candidate.recipientSubjectId, + recipientPeerId: candidate.recipientPeerId + }), + relayMessageId: candidate.relayMessageId, + relayPeerId: candidate.relayPeerId, + senderConnectionEpoch: candidate.senderConnectionEpoch, + senderSequence: candidate.senderSequence, + documentIds, + payloadVersion: candidate.payloadVersion, + messageHash: candidate.messageHash, + outerEnvelopeDigest: candidate.outerEnvelopeDigest, + payloadBytes: candidate.payloadBytes, + claimToken: token, + claimDeadline: deadline, + sessionGeneration: input.sessionGeneration + })), + ready: false, + nextEligibleAt: Option.none(), + lane: candidate.retryCount === 0 ? "New" as const : "Retry" as const + } + })) + })).pipe(Effect.withSpan("PeerRelayStore.claim")) + + const loadClaimedPayload: Service["loadClaimedPayload"] = (unsafeInput) => + mapStorageErrors(Effect.gen(function*() { + const input = yield* validateInput(LoadClaimedPayloadRequest, unsafeInput) + const rows = yield* SqlSchema.findAll({ + Request: Schema.Void, + Result: Schema.Struct({ payload: Schema.Uint8Array }), + execute: () => + sql`SELECT m.payload + FROM effect_local_relay_messages m + JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id + WHERE m.message_id = ${input.rowId} + AND m.channel_id = c.channel_id + AND c.tenant_id = ${input.channel.tenantId} + AND c.sender_subject_id = ${input.channel.senderSubjectId} + AND c.sender_peer_id = ${input.channel.senderPeerId} + AND c.sender_replica_incarnation = ${input.channel.senderReplicaIncarnation} + AND c.recipient_subject_id = ${input.channel.recipientSubjectId} + AND c.recipient_peer_id = ${input.channel.recipientPeerId} + AND m.tenant_id = c.tenant_id + AND m.sender_subject_id = c.sender_subject_id + AND m.sender_peer_id = c.sender_peer_id + AND m.relay_message_id = ${input.relayMessageId} + AND m.state = 'Claimed' + AND m.claim_token = ${input.claimToken} + AND m.claim_session_generation = ${input.sessionGeneration} + AND c.claimed_message_id = m.message_id + AND c.claim_token = m.claim_token + AND c.claim_session_generation = m.claim_session_generation + AND c.claim_deadline = m.claim_deadline + AND m.claim_deadline > CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + AND m.expires_at > CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + AND m.payload_length = ${input.payloadBytes} + AND length(m.payload) = ${input.payloadBytes} + AND m.payload_length <= ${limits.maxActiveBytesPerShard}` + })(undefined) + if (rows.length !== 1) { + return yield* protocolMismatch("active relay claim", "stale relay claim") + } + return rows[0]!.payload + })).pipe(Effect.withSpan("PeerRelayStore.loadClaimedPayload")) + + const TerminalRow = Schema.Struct({ + messageId: PositiveInt, + channelId: PositiveInt, + state: Schema.Literals(["Pending", "Claimed", "Acknowledged", "DeadLettered", "Expired"]), + messageHash: Schema.NonEmptyString, + claimToken: Schema.NullOr(PeerRelayRpc.ClaimToken), + claimSessionGeneration: Schema.NullOr(NonNegativeInt), + claimDeadline: Schema.NullOr(NonNegativeInt), + expiresAt: NonNegativeInt, + channelClaimedMessageId: Schema.NullOr(PositiveInt), + channelClaimToken: Schema.NullOr(PeerRelayRpc.ClaimToken), + channelClaimSessionGeneration: Schema.NullOr(NonNegativeInt), + channelClaimDeadline: Schema.NullOr(NonNegativeInt), + terminalClaimToken: Schema.NullOr(PeerRelayRpc.ClaimToken), + terminalSessionGeneration: Schema.NullOr(NonNegativeInt), + terminalReason: Schema.NullOr(Schema.String) + }) + + const findTerminal = SqlSchema.findOneOption({ + Request: Schema.Struct({ + channel: ChannelKey, + relayMessageId: Identity.RelayMessageId + }), + Result: TerminalRow, + execute: ({ channel, relayMessageId }) => + sql`SELECT + m.message_id AS messageId, + m.channel_id AS channelId, + m.state, + m.message_hash AS messageHash, + m.claim_token AS claimToken, + m.claim_session_generation AS claimSessionGeneration, + m.claim_deadline AS claimDeadline, + m.expires_at AS expiresAt, + c.claimed_message_id AS channelClaimedMessageId, + c.claim_token AS channelClaimToken, + c.claim_session_generation AS channelClaimSessionGeneration, + c.claim_deadline AS channelClaimDeadline, + m.terminal_claim_token AS terminalClaimToken, + m.terminal_session_generation AS terminalSessionGeneration, + m.terminal_reason AS terminalReason + FROM effect_local_relay_messages m + JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id + WHERE c.tenant_id = ${channel.tenantId} + AND c.sender_subject_id = ${channel.senderSubjectId} + AND c.sender_peer_id = ${channel.senderPeerId} + AND c.sender_replica_incarnation = ${channel.senderReplicaIncarnation} + AND c.recipient_subject_id = ${channel.recipientSubjectId} + AND c.recipient_peer_id = ${channel.recipientPeerId} + AND m.relay_message_id = ${relayMessageId}` + }) + + const ReadyRow = Schema.Struct({ + nextEligibleAt: NonNegativeInt, + retryCount: NonNegativeInt + }) + + const nextReady = ( + channel: ChannelKey, + now: number + ): Effect.Effect< + { readonly ready: boolean; readonly nextEligibleAt: Option.Option; readonly lane: "New" | "Retry" }, + SqlError.SqlError | Schema.SchemaError + > => + SqlSchema.findOneOption({ + Request: Schema.Void, + Result: ReadyRow, + execute: () => + sql`SELECT + m.next_eligible_at AS nextEligibleAt, + m.retry_count AS retryCount + FROM effect_local_relay_messages m + JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id + WHERE c.tenant_id = ${channel.tenantId} + AND c.sender_subject_id = ${channel.senderSubjectId} + AND c.sender_peer_id = ${channel.senderPeerId} + AND c.recipient_subject_id = ${channel.recipientSubjectId} + AND c.recipient_peer_id = ${channel.recipientPeerId} + AND c.claimed_message_id IS NULL + AND m.state = 'Pending' + AND m.expires_at > ${now} + AND NOT EXISTS ( + SELECT 1 + FROM effect_local_relay_messages earlier + WHERE earlier.channel_id = m.channel_id + AND earlier.channel_sequence < m.channel_sequence + AND earlier.state IN ('Pending', 'Claimed') + ) + ORDER BY m.next_eligible_at, m.created_at, m.message_id + LIMIT 1` + })(undefined).pipe( + Effect.map((row) => + Option.isNone(row) + ? { + ready: false, + nextEligibleAt: Option.none(), + lane: "New" as const + } + : { + ready: row.value.nextEligibleAt <= now, + nextEligibleAt: Option.some(row.value.nextEligibleAt), + lane: row.value.retryCount === 0 ? "New" as const : "Retry" as const + } + ) + ) + + const terminalTransition = ( + unsafeInput: TerminalRequest, + state: "Acknowledged" | "DeadLettered", + reason: string + ): Effect.Effect => + mapStorageErrors(Effect.gen(function*() { + const input = yield* validateInput(TerminalRequest, unsafeInput) + if ( + input.recipient.tenantId !== input.channel.tenantId || + input.recipient.subjectId !== input.channel.recipientSubjectId || + input.recipient.peerId !== input.channel.recipientPeerId + ) { + return { + status: "Stale", + ready: false, + nextEligibleAt: Option.none(), + lane: "New" + } as const + } + return yield* write(Effect.gen(function*() { + const now = (yield* nowQuery(sql)).now + const rowOption = yield* findTerminal({ + channel: input.channel, + relayMessageId: input.relayMessageId + }) + if (Option.isNone(rowOption)) { + return { + status: "Stale", + ready: false, + nextEligibleAt: Option.none(), + lane: "New" + } as const + } + const row = rowOption.value + const active = row.state === "Claimed" && + row.messageHash === input.messageHash && + row.claimToken === input.claimToken && + row.claimSessionGeneration === input.sessionGeneration && + row.claimDeadline !== null && + row.claimDeadline > now && + row.expiresAt > now && + row.channelClaimedMessageId === row.messageId && + row.channelClaimToken === row.claimToken && + row.channelClaimSessionGeneration === row.claimSessionGeneration && + row.channelClaimDeadline === row.claimDeadline + if (active) { + yield* terminalize(row.messageId, state, now, { + token: input.claimToken, + sessionGeneration: input.sessionGeneration, + reason + }) + const hint = yield* nextReady(input.channel, now) + return { status: "Changed", ...hint } as const + } + const duplicate = row.state === state && + row.messageHash === input.messageHash && + row.terminalClaimToken === input.claimToken && + row.terminalSessionGeneration === input.sessionGeneration && + row.terminalReason === reason + if (duplicate) { + const hint = yield* nextReady(input.channel, now) + return { status: "Duplicate", ...hint } as const + } + return { + status: "Stale", + ready: false, + nextEligibleAt: Option.none(), + lane: "New" + } as const + })) + })) + + const acknowledge: Service["acknowledge"] = (input) => + terminalTransition(input, "Acknowledged", "Acknowledged").pipe( + Effect.withSpan("PeerRelayStore.acknowledge") + ) + + const reject: Service["reject"] = (unsafeInput) => + mapStorageErrors(Effect.gen(function*() { + const input = yield* validateInput(RejectRequest, unsafeInput) + return yield* terminalTransition( + TerminalRequest.make({ + channel: input.channel, + relayMessageId: input.relayMessageId, + claimToken: input.claimToken, + messageHash: input.messageHash, + sessionGeneration: input.sessionGeneration, + recipient: input.recipient + }), + "DeadLettered", + input.reason + ) + })).pipe(Effect.withSpan("PeerRelayStore.reject")) + + const release: Service["release"] = (unsafeInput) => + mapStorageErrors(Effect.gen(function*() { + const input = yield* validateInput(ReleaseRequest, unsafeInput) + return yield* write(Effect.gen(function*() { + const now = (yield* nowQuery(sql)).now + const rowOption = yield* findTerminal({ + channel: input.channel, + relayMessageId: input.relayMessageId + }) + if (Option.isNone(rowOption)) { + return { + status: "Stale", + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" + } as const + } + const row = rowOption.value + if ( + row.state !== "Claimed" || + row.claimToken !== input.claimToken || + row.claimSessionGeneration !== input.sessionGeneration + ) { + return { + status: "Stale", + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" + } as const + } + const random = yield* Random.next + const retry = yield* SqlSchema.findOne({ + Request: Schema.Void, + Result: Schema.Struct({ retryCount: PositiveInt }), + execute: () => + sql`SELECT retry_count + 1 AS retryCount + FROM effect_local_relay_messages + WHERE message_id = ${row.messageId}` + })(undefined) + const maximum = Math.min( + limits.retryMaximumDelayMillis, + limits.retryBaseDelayMillis * 2 ** Math.min(retry.retryCount - 1, 30) + ) + const delay = Math.max(1, Math.floor(maximum / 2 + random * maximum / 2)) + const nextEligibleAt = now + delay + yield* sql`UPDATE effect_local_relay_messages + SET state = 'Pending', + retry_count = ${retry.retryCount}, + next_eligible_at = ${nextEligibleAt}, + claim_token = NULL, + claim_session_generation = NULL, + claim_deadline = NULL + WHERE message_id = ${row.messageId} + AND state = 'Claimed' + AND claim_token = ${input.claimToken} + AND claim_session_generation = ${input.sessionGeneration}` + yield* sql`UPDATE effect_local_relay_channels + SET claimed_message_id = NULL, + claim_session_generation = NULL, + claim_token = NULL, + claim_deadline = NULL + WHERE channel_id = ${row.channelId} + AND claimed_message_id = ${row.messageId} + AND claim_token = ${input.claimToken} + AND claim_session_generation = ${input.sessionGeneration}` + return { + status: "Changed", + ready: false, + nextEligibleAt: Option.some(nextEligibleAt), + lane: "Retry" + } as const + })) + })).pipe(Effect.withSpan("PeerRelayStore.release")) + + const maintenanceResult = ( + ids: ReadonlyArray, + requestedBatchSize: number + ): MaintenanceResult => { + const processedIds = ids.slice(0, requestedBatchSize) + const cursor = processedIds.at(-1) + return cursor === undefined + ? { processed: 0, hasMore: false } + : { + cursor, + processed: processedIds.length, + hasMore: ids.length > requestedBatchSize + } + } + + const recover: Service["recover"] = (unsafeInput) => + mapStorageErrors(Effect.gen(function*() { + const input = yield* validateInput(MaintenanceRequest, unsafeInput) + const effectiveBatch = Math.min(input.batchSize, limits.claimRecoveryBatchSize) + return yield* write(Effect.gen(function*() { + const now = (yield* nowQuery(sql)).now + const rows = yield* SqlSchema.findAll({ + Request: Schema.Void, + Result: Schema.Struct({ + messageId: PositiveInt, + retryCount: NonNegativeInt + }), + execute: () => + sql`SELECT + message_id AS messageId, + retry_count AS retryCount + FROM effect_local_relay_messages + WHERE message_id > ${input.cursor ?? 0} + AND state = 'Claimed' + AND claim_deadline <= ${now} + ORDER BY message_id + LIMIT ${effectiveBatch + 1}` + })(undefined) + for (const row of rows.slice(0, effectiveBatch)) { + const random = yield* Random.next + const retryCount = row.retryCount + 1 + const maximum = Math.min( + limits.retryMaximumDelayMillis, + limits.retryBaseDelayMillis * 2 ** Math.min(retryCount - 1, 30) + ) + const delay = Math.max(1, Math.floor(maximum / 2 + random * maximum / 2)) + yield* sql`UPDATE effect_local_relay_messages + SET state = 'Pending', + retry_count = ${retryCount}, + next_eligible_at = ${now + delay}, + claim_token = NULL, + claim_session_generation = NULL, + claim_deadline = NULL + WHERE message_id = ${row.messageId} + AND state = 'Claimed' + AND claim_deadline <= ${now}` + yield* sql`UPDATE effect_local_relay_channels + SET claimed_message_id = NULL, + claim_session_generation = NULL, + claim_token = NULL, + claim_deadline = NULL + WHERE claimed_message_id = ${row.messageId} + AND claim_deadline <= ${now}` + } + return maintenanceResult(rows.map((row) => row.messageId), effectiveBatch) + })) + })).pipe(Effect.withSpan("PeerRelayStore.recover")) + + const expire: Service["expire"] = (unsafeInput) => + mapStorageErrors(Effect.gen(function*() { + const input = yield* validateInput(MaintenanceRequest, unsafeInput) + const effectiveBatch = Math.min(input.batchSize, limits.expiryBatchSize) + return yield* write(Effect.gen(function*() { + const now = (yield* nowQuery(sql)).now + const rows = yield* SqlSchema.findAll({ + Request: Schema.Void, + Result: KeyRow, + execute: () => + sql`SELECT message_id AS messageId + FROM effect_local_relay_messages + WHERE message_id > ${input.cursor ?? 0} + AND state IN ('Pending', 'Claimed') + AND expires_at <= ${now} + ORDER BY message_id + LIMIT ${effectiveBatch + 1}` + })(undefined) + for (const row of rows.slice(0, effectiveBatch)) { + yield* terminalize(row.messageId, "Expired", now, { reason: "Expired" }) + } + return maintenanceResult(rows.map((row) => row.messageId), effectiveBatch) + })) + })).pipe(Effect.withSpan("PeerRelayStore.expire")) + + const IntegrityRow = Schema.Struct({ + messageId: PositiveInt, + state: Schema.String, + messageTenantId: Schema.NonEmptyString, + channelTenantId: Schema.NonEmptyString, + messageSenderSubjectId: Schema.NonEmptyString, + channelSenderSubjectId: Schema.NonEmptyString, + messageSenderPeerId: Schema.String, + channelSenderPeerId: Schema.String, + payloadLength: NonNegativeInt, + actualLength: Schema.NullOr(NonNegativeInt), + reservationPresent: Schema.Literals([0, 1]), + activeConsumed: Schema.NullOr(Schema.Literals([0, 1])), + retainedConsumed: Schema.NullOr(Schema.Literals([0, 1])) + }) + + const repair: Service["repair"] = (unsafeInput) => + mapStorageErrors(Effect.gen(function*() { + const input = yield* validateInput(MaintenanceRequest, unsafeInput) + const effectiveBatch = Math.min(input.batchSize, limits.integrityBatchSize) + return yield* write(Effect.gen(function*() { + const now = (yield* nowQuery(sql)).now + const rows = yield* SqlSchema.findAll({ + Request: Schema.Void, + Result: IntegrityRow, + execute: () => + sql`SELECT + m.message_id AS messageId, + m.state, + m.tenant_id AS messageTenantId, + c.tenant_id AS channelTenantId, + m.sender_subject_id AS messageSenderSubjectId, + c.sender_subject_id AS channelSenderSubjectId, + m.sender_peer_id AS messageSenderPeerId, + c.sender_peer_id AS channelSenderPeerId, + m.payload_length AS payloadLength, + length(m.payload) AS actualLength, + CASE WHEN r.message_id IS NULL THEN 0 ELSE 1 END AS reservationPresent, + r.active_consumed AS activeConsumed, + r.retained_consumed AS retainedConsumed + FROM effect_local_relay_messages m + JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id + LEFT JOIN effect_local_relay_reservations r ON r.message_id = m.message_id + WHERE m.message_id > ${input.cursor ?? 0} + ORDER BY m.message_id + LIMIT ${effectiveBatch + 1}` + })(undefined) + for (const row of rows.slice(0, effectiveBatch)) { + if ( + row.reservationPresent === 0 || + row.activeConsumed === null || + row.retainedConsumed === null + ) { + return yield* storageCorrupt(new Error("Relay reservation reconciliation required")) + } + const active = row.state === "Pending" || row.state === "Claimed" + const terminal = row.state === "Acknowledged" || + row.state === "DeadLettered" || + row.state === "Expired" + if ( + (active && row.activeConsumed !== 0) || + (terminal && row.activeConsumed !== 1) + ) { + return yield* storageCorrupt(new Error("Corrupt relay reservation entitlement")) + } + const corrupt = (!active && !terminal) || + row.messageTenantId !== row.channelTenantId || + row.messageSenderSubjectId !== row.channelSenderSubjectId || + row.messageSenderPeerId !== row.channelSenderPeerId || + (active && ( + row.actualLength === null || + row.actualLength !== row.payloadLength + )) || + (terminal && ( + row.actualLength !== null || + row.payloadLength !== 0 + )) + if (corrupt && active) { + yield* terminalize(row.messageId, "DeadLettered", now, { reason: "Corrupt" }) + } else if (corrupt) { + return yield* storageCorrupt(new Error("Corrupt terminal relay row")) + } + } + return maintenanceResult(rows.map((row) => row.messageId), effectiveBatch) + })) + })).pipe(Effect.withSpan("PeerRelayStore.repair")) + + const reconcile: Service["reconcile"] = (unsafeInput) => + mapStorageErrors(Effect.gen(function*() { + const input = yield* validateInput(MaintenanceRequest, unsafeInput) + const effectiveBatch = Math.min(input.batchSize, limits.reconciliationBatchSize) + return yield* write(Effect.gen(function*() { + const rows = yield* SqlSchema.findAll({ + Request: Schema.Void, + Result: KeyRow, + execute: () => + sql`SELECT m.message_id AS messageId + FROM effect_local_relay_messages m + LEFT JOIN effect_local_relay_reservations r ON r.message_id = m.message_id + WHERE m.message_id > ${input.cursor ?? 0} + AND r.message_id IS NULL + ORDER BY m.message_id + LIMIT ${effectiveBatch + 1}` + })(undefined) + if (rows.length > 0) { + return yield* storageCorrupt(new Error("Missing immutable relay quota reservation")) + } + if (input.cursor === undefined || input.cursor === 0) { + const orphanReservations = yield* SqlSchema.findAll({ + Request: Schema.Void, + Result: KeyRow, + execute: () => + sql`SELECT r.message_id AS messageId + FROM effect_local_relay_reservations r + LEFT JOIN effect_local_relay_messages m ON m.message_id = r.message_id + WHERE m.message_id IS NULL + ORDER BY r.message_id + LIMIT ${effectiveBatch + 1}` + })(undefined) + if (orphanReservations.length > 0) { + return yield* storageCorrupt(new Error("Orphan relay quota reservation")) + } + } + yield* sql`DELETE FROM effect_local_relay_usage` + yield* sql`INSERT INTO effect_local_relay_usage ( + scope_kind, + scope_key, + active_count, + active_bytes, + retained_count, + retained_bytes + ) + SELECT + scope_kind, + scope_key, + SUM(CASE WHEN active_consumed = 0 THEN active_count_delta ELSE 0 END), + SUM(CASE WHEN active_consumed = 0 THEN active_bytes_delta ELSE 0 END), + SUM(CASE WHEN retained_consumed = 0 THEN retained_count_delta ELSE 0 END), + SUM(CASE WHEN retained_consumed = 0 THEN retained_bytes_delta ELSE 0 END) + FROM ( + SELECT + 'SenderPeer' AS scope_kind, + sender_peer_usage_key AS scope_key, + active_count_delta, + active_bytes_delta, + retained_count_delta, + retained_bytes_delta, + active_consumed, + retained_consumed + FROM effect_local_relay_reservations + UNION ALL + SELECT + 'RecipientPeer', + recipient_peer_usage_key, + active_count_delta, + active_bytes_delta, + retained_count_delta, + retained_bytes_delta, + active_consumed, + retained_consumed + FROM effect_local_relay_reservations + UNION ALL + SELECT + 'RecipientSubject', + recipient_subject_usage_key, + active_count_delta, + active_bytes_delta, + retained_count_delta, + retained_bytes_delta, + active_consumed, + retained_consumed + FROM effect_local_relay_reservations + UNION ALL + SELECT + 'Tenant', + tenant_usage_key, + active_count_delta, + active_bytes_delta, + retained_count_delta, + retained_bytes_delta, + active_consumed, + retained_consumed + FROM effect_local_relay_reservations + UNION ALL + SELECT + 'Shard', + shard_usage_key, + active_count_delta, + active_bytes_delta, + retained_count_delta, + retained_bytes_delta, + active_consumed, + retained_consumed + FROM effect_local_relay_reservations + ) + GROUP BY scope_kind, scope_key + HAVING + SUM(CASE WHEN active_consumed = 0 THEN active_count_delta ELSE 0 END) > 0 + OR SUM(CASE WHEN retained_consumed = 0 THEN retained_count_delta ELSE 0 END) > 0` + return { processed: 0, hasMore: false } + })) + })).pipe(Effect.withSpan("PeerRelayStore.reconcile")) + + const collect: Service["collect"] = (unsafeInput) => + mapStorageErrors(Effect.gen(function*() { + const input = yield* validateInput(MaintenanceRequest, unsafeInput) + const effectiveBatch = Math.min(input.batchSize, limits.terminalCollectionBatchSize) + return yield* write(Effect.gen(function*() { + const now = (yield* nowQuery(sql)).now + const rows = yield* SqlSchema.findAll({ + Request: Schema.Void, + Result: KeyRow, + execute: () => + sql`SELECT message_id AS messageId + FROM effect_local_relay_messages + WHERE message_id > ${input.cursor ?? 0} + AND state IN ('Acknowledged', 'DeadLettered', 'Expired') + AND deduplicate_until <= ${now} + ORDER BY message_id + LIMIT ${effectiveBatch + 1}` + })(undefined) + for (const row of rows.slice(0, effectiveBatch)) { + yield* releaseRetainedUsage(row.messageId) + yield* sql`DELETE FROM effect_local_relay_messages + WHERE message_id = ${row.messageId} + AND state IN ('Acknowledged', 'DeadLettered', 'Expired') + AND deduplicate_until <= ${now}` + } + yield* sql`DELETE FROM effect_local_relay_usage + WHERE active_count = 0 + AND active_bytes = 0 + AND retained_count = 0 + AND retained_bytes = 0` + yield* sql`DELETE FROM effect_local_relay_channels + WHERE claimed_message_id IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM effect_local_relay_messages m + WHERE m.channel_id = effect_local_relay_channels.channel_id + ) + LIMIT ${limits.orphanChannelCleanupBatchSize}` + return maintenanceResult(rows.map((row) => row.messageId), effectiveBatch) + })) + })).pipe(Effect.withSpan("PeerRelayStore.collect")) + + const usage: Service["usage"] = (unsafeInput) => + mapStorageErrors(Effect.gen(function*() { + const input = unsafeInput === undefined + ? UsageRequest.make({ scopeKind: "Shard", scopeKey: encodeKey("local") }) + : yield* validateInput(UsageRequest, unsafeInput) + const row = yield* SqlSchema.findOneOption({ + Request: Schema.Void, + Result: UsageRow, + execute: () => + sql`SELECT + active_count AS activeCount, + active_bytes AS activeBytes, + retained_count AS retainedCount, + retained_bytes AS retainedBytes + FROM effect_local_relay_usage + WHERE scope_kind = ${input.scopeKind} + AND scope_key = ${input.scopeKey}` + })(undefined) + return Option.getOrElse(row, (): Usage => ({ + activeCount: 0, + activeBytes: 0, + retainedCount: 0, + retainedBytes: 0 + })) + })).pipe(Effect.withSpan("PeerRelayStore.usage")) + + return PeerRelayStore.of({ + admit, + claim, + loadClaimedPayload, + acknowledge, + reject, + release, + recover, + expire, + repair, + reconcile, + collect, + usage + }) +}) + +export const make = makeService + +export const layerSqlite: Layer.Layer< + PeerRelayStore, + | Migrator.MigrationError + | SqlError.SqlError + | Schema.SchemaError + | ReplicaError.ReplicaError, + SqlClient.SqlClient | Crypto.Crypto | PeerRelayLimits.PeerRelayLimits +> = Layer.effect(PeerRelayStore, makeService) diff --git a/packages/local-rpc/src/PeerRpcServer.ts b/packages/local-rpc/src/PeerRpcServer.ts index 8d5080a..6c2b858 100644 --- a/packages/local-rpc/src/PeerRpcServer.ts +++ b/packages/local-rpc/src/PeerRpcServer.ts @@ -1,5 +1,7 @@ import * as CommitPublisher from "@lucas-barake/effect-local-sql/CommitPublisher" import * as PeerSession from "@lucas-barake/effect-local-sql/PeerSession" +import * as PeerSyncEnvelope from "@lucas-barake/effect-local-sql/PeerSyncEnvelope" +import * as Canonical from "@lucas-barake/effect-local/Canonical" import * as Identity from "@lucas-barake/effect-local/Identity" import * as PeerTransport from "@lucas-barake/effect-local/PeerTransport" import type * as ReplicaDefinition from "@lucas-barake/effect-local/ReplicaDefinition" @@ -8,20 +10,31 @@ import * as ReplicaLimits from "@lucas-barake/effect-local/ReplicaLimits" import * as Arr from "effect/Array" import * as Cause from "effect/Cause" import * as Clock from "effect/Clock" +import * as Context from "effect/Context" +import * as Crypto from "effect/Crypto" import * as Deferred from "effect/Deferred" import * as Duration from "effect/Duration" import * as Effect from "effect/Effect" import * as Exit from "effect/Exit" import * as Fiber from "effect/Fiber" +import * as Layer from "effect/Layer" import * as Option from "effect/Option" import * as Queue from "effect/Queue" +import * as Schema from "effect/Schema" import * as Scope from "effect/Scope" import * as Semaphore from "effect/Semaphore" import * as Stream from "effect/Stream" +import * as RpcServer from "effect/unstable/rpc/RpcServer" +import type * as SocketServer from "effect/unstable/socket/SocketServer" import * as PeerAuthorizationValidation from "./internal/peerAuthorization.js" import * as PeerRpcObservability from "./internal/peerRpcObservability.js" import * as PeerAuthentication from "./PeerAuthentication.js" import * as PeerAuthorization from "./PeerAuthorization.js" +import * as PeerRelayAuthorization from "./PeerRelayAuthorization.js" +import * as PeerRelayIngress from "./PeerRelayIngress.js" +import * as PeerRelayLimits from "./PeerRelayLimits.js" +import * as PeerRelayRpc from "./PeerRelayRpc.js" +import * as PeerRelayStore from "./PeerRelayStore.js" import * as PeerRpc from "./PeerRpc.js" import * as PeerRpcError from "./PeerRpcError.js" import * as PeerRpcLimits from "./PeerRpcLimits.js" @@ -1216,3 +1229,1415 @@ export const layerHandlers = ( Push: push }) })) + +type RelaySqlLane = "Admission" | "Terminal" | "Delivery" | "Maintenance" + +interface RelaySqlTask { + state: "Queued" | "Acquired" | "Cancelled" | "Completed" + readonly cancel: Deferred.Deferred + readonly execute: Effect.Effect +} + +interface RelaySqlScheduler { + readonly submit: ( + lane: RelaySqlLane, + effect: Effect.Effect + ) => Effect.Effect + readonly shutdown: Effect.Effect +} + +const drainRelayQueue = ( + queue: Queue.Dequeue +): Effect.Effect> => + Effect.gen(function*() { + const items: Array = [] + while (true) { + const item = yield* Queue.poll(queue) + if (Option.isNone(item)) return items + items.push(item.value) + } + }) + +const makeRelaySqlScheduler = ( + limits: PeerRelayLimits.Values, + runtimeScope: Scope.Scope, + onFatal: (cause: Cause.Cause) => Effect.Effect +): Effect.Effect => + Effect.gen(function*() { + const capacities: Record = { + Admission: limits.sqlAdmissionQueueCapacity, + Terminal: limits.sqlTerminalQueueCapacity, + Delivery: limits.sqlDeliveryQueueCapacity, + Maintenance: limits.sqlMaintenanceQueueCapacity + } + const queues = { + Admission: yield* Queue.dropping(capacities.Admission), + Terminal: yield* Queue.dropping(capacities.Terminal), + Delivery: yield* Queue.dropping(capacities.Delivery), + Maintenance: yield* Queue.dropping(capacities.Maintenance) + } satisfies Record> + const wake = yield* Queue.dropping( + capacities.Admission + capacities.Terminal + capacities.Delivery + capacities.Maintenance + ) + const lock = yield* Semaphore.make(1) + const globalPermits = yield* Semaphore.make(limits.maxInFlightSqlTransactions) + const lanePermits = { + Admission: yield* Semaphore.make(limits.maxInFlightSqlAdmission), + Terminal: yield* Semaphore.make(limits.maxInFlightSqlTerminal), + Delivery: yield* Semaphore.make(limits.maxInFlightSqlDelivery), + Maintenance: yield* Semaphore.make(limits.maxInFlightSqlMaintenance) + } satisfies Record + let accepting = true + + const cancelTask = (task: RelaySqlTask) => + lock.withPermit(Effect.sync(() => { + if (task.state === "Completed" || task.state === "Cancelled") return + task.state = "Cancelled" + Deferred.doneUnsafe(task.cancel, Effect.void) + })) + + const submit: RelaySqlScheduler["submit"] = (lane, effect) => + Effect.uninterruptibleMask((restore) => + Effect.gen(function*() { + const result = yield* Deferred.make, Effect.Error>() + const cancel = yield* Deferred.make() + const task: RelaySqlTask = { + state: "Queued", + cancel, + execute: Effect.uninterruptibleMask((restoreTask) => + lock.withPermit(Effect.sync(() => { + if (task.state !== "Queued") return false + task.state = "Acquired" + return true + })).pipe( + Effect.flatMap((acquired) => { + if (!acquired) return Effect.void + const cancelled = Deferred.await(cancel).pipe(Effect.flatMap(() => Effect.interrupt)) + return Effect.raceFirst(restoreTask(effect), cancelled).pipe( + Effect.exit, + Effect.flatMap((exit) => Deferred.done(result, exit)), + Effect.ensuring(lock.withPermit(Effect.sync(() => { + task.state = "Completed" + }))) + ) + }) + ) + ) + } + const offered = yield* lock.withPermit( + Effect.suspend(() => { + if (!accepting) return Effect.succeed(false) + return Queue.offer(queues[lane], task) + }) + ) + if (!offered) return yield* new PeerRpcError.RequestCapacityExceeded() + yield* Queue.offer(wake, undefined) + return yield* restore(Deferred.await(result)).pipe( + Effect.onInterrupt(() => cancelTask(task)) + ) + }) + ) + + const lanes = [ + "Terminal", + "Admission", + "Delivery", + "Maintenance" + ] as const satisfies ReadonlyArray + + const worker = Effect.suspend(() => { + let cursor = 0 + const take = (): Effect.Effect => + Effect.gen(function*() { + for (let offset = 0; offset < lanes.length; offset++) { + const index = (cursor + offset) % lanes.length + const lane = lanes[index]! + const task = yield* Queue.poll(queues[lane]) + if (Option.isSome(task)) { + cursor = (index + 1) % lanes.length + return [lane, task.value] as const + } + } + yield* Queue.take(wake) + return yield* take() + }) + return Effect.forever( + take().pipe( + Effect.flatMap(([lane, task]) => + task.execute.pipe( + lanePermits[lane].withPermits(1), + globalPermits.withPermits(1) + ) + ) + ) + ) + }) + + const workerFibers: Array> = [] + for (let index = 0; index < limits.maxInFlightSqlTransactions; index++) { + const fiber = yield* Effect.forkIn(worker, runtimeScope) + workerFibers.push(fiber) + yield* Effect.forkIn( + Fiber.await(fiber).pipe( + Effect.flatMap((exit) => + lock.withPermit(Effect.sync(() => accepting)).pipe( + Effect.flatMap((wasAccepting) => + wasAccepting && Exit.isFailure(exit) + ? onFatal(exit.cause) + : Effect.void + ) + ) + ) + ), + runtimeScope + ) + } + + const shutdown = Effect.uninterruptible( + lock.withPermit(Effect.sync(() => { + accepting = false + })).pipe( + Effect.andThen(Effect.forEach(Object.values(queues), (queue) => + drainRelayQueue(queue).pipe( + Effect.flatMap((tasks) => Effect.forEach(tasks, cancelTask, { discard: true })), + Effect.andThen(Queue.shutdown(queue)) + ), { discard: true })), + Effect.andThen(Queue.shutdown(wake)), + Effect.andThen(Effect.forEach(workerFibers, Fiber.interrupt, { discard: true })) + ) + ).pipe(Effect.ignore) + + return { submit, shutdown } + }) + +interface RelaySubjectState { + tokens: number + updatedAt: number + lastUsedAt: number + inFlight: number +} + +interface RelayAdmissionLane { + readonly run: ( + subjectId: string, + effect: Effect.Effect + ) => Effect.Effect + readonly clear: Effect.Effect +} + +const makeRelayAdmissionLane = (options: { + readonly maximumInFlight: number + readonly maximumInFlightPerSubject: number + readonly ratePerSecond: number + readonly burst: number + readonly maximumSubjects: number + readonly idleRetentionMillis: number +}): Effect.Effect => + Effect.gen(function*() { + const lock = yield* Semaphore.make(1) + const permits = yield* Semaphore.make(options.maximumInFlight) + const subjects = new Map() + const inactive = new Map() + + const removeExpired = (now: number) => { + while (inactive.size > 0) { + const oldest = inactive.entries().next().value! + if (now - oldest[1].lastUsedAt < options.idleRetentionMillis) return + inactive.delete(oldest[0]) + subjects.delete(oldest[0]) + } + } + + const admit = (subjectId: string, now: number) => + lock.withPermit(Effect.sync(() => { + removeExpired(now) + let state = subjects.get(subjectId) + if (state?.inFlight === 0) inactive.delete(subjectId) + if (state === undefined) { + while (subjects.size >= options.maximumSubjects) { + const evictable = inactive.entries().next().value + if (evictable === undefined) return false + inactive.delete(evictable[0]) + subjects.delete(evictable[0]) + } + state = { + tokens: options.burst, + updatedAt: now, + lastUsedAt: now, + inFlight: 0 + } + subjects.set(subjectId, state) + } + const effectiveNow = Math.max(now, state.updatedAt, state.lastUsedAt) + state.tokens = Math.min( + options.burst, + state.tokens + ((effectiveNow - state.updatedAt) / 1_000) * options.ratePerSecond + ) + state.updatedAt = effectiveNow + state.lastUsedAt = effectiveNow + if (state.inFlight >= options.maximumInFlightPerSubject || state.tokens < 1) { + if (state.inFlight === 0) inactive.set(subjectId, state) + return false + } + state.tokens -= 1 + state.inFlight += 1 + return true + })) + + const release = (subjectId: string) => + Clock.currentTimeMillis.pipe( + Effect.flatMap((now) => + lock.withPermit(Effect.sync(() => { + const state = subjects.get(subjectId) + if (state === undefined) return + state.inFlight -= 1 + state.lastUsedAt = Math.max(now, state.lastUsedAt) + if (state.inFlight === 0) { + inactive.delete(subjectId) + inactive.set(subjectId, state) + } + })) + ) + ) + + const run: RelayAdmissionLane["run"] = (subjectId, effect) => + Effect.gen(function*() { + const admittedAt = yield* Clock.currentTimeMillis + if (!(yield* admit(subjectId, admittedAt))) { + return yield* new PeerRpcError.RequestCapacityExceeded() + } + const result = yield* effect.pipe(permits.withPermitsIfAvailable(1)) + if (Option.isNone(result)) return yield* new PeerRpcError.RequestCapacityExceeded() + return result.value + }).pipe(Effect.ensuring(release(subjectId))) + + return { + run, + clear: lock.withPermit(Effect.sync(() => { + subjects.clear() + inactive.clear() + })) + } + }) + +export interface PeerRelayServerUsage { + readonly accepting: boolean + readonly sessions: number + readonly subjects: number + readonly activeClaims: number + readonly queuedChannels: number +} + +export class PeerRelayServerRuntime extends Context.Service + readonly owner: Effect.Effect + readonly shutdown: Effect.Effect + readonly usage: Effect.Effect +}>()("@lucas-barake/effect-local-rpc/PeerRelayServerRuntime") {} + +class PeerRelayFatalSignal extends Context.Service) => Effect.Effect +}>()("@lucas-barake/effect-local-rpc/PeerRelayFatalSignal") {} + +interface RelayWorkSelector { + readonly recipient: { + readonly tenantId: string + readonly subjectId: string + readonly peerId: Identity.PeerId + } + readonly sender: { + readonly subjectId: string + readonly peerId: Identity.PeerId + } +} + +interface RelayOutboundItem { + readonly event: PeerRelayRpc.StoredMessage + readonly reservation: PeerRelayIngress.Reservation + transferred: boolean +} + +interface RelayEntry { + readonly sessionId: Identity.SessionId + readonly generation: number + readonly principal: PeerAuthentication.PeerPrincipal + readonly senderReplicaIncarnation: Identity.ReplicaIncarnation + readonly remote: PeerRelayAuthorization.RemotePeer + readonly documents: ReadonlyArray + readonly receiptRetentionMillis: number + readonly senderRetryHorizonMillis: number + readonly outbound: Queue.Queue + readonly claims: Map + readonly watcherFibers: Array> + active: boolean +} + +type RelayWorkOwner = + | { + readonly _tag: "Worker" + readonly lane: "New" | "Retry" + pending: boolean + } + | { + readonly _tag: "Entry" + readonly generation: number + pending: boolean + } + | { + readonly _tag: "Terminal" + } + +const relaySelectorKey = (selector: RelayWorkSelector) => + JSON.stringify([ + selector.recipient.tenantId, + selector.recipient.subjectId, + selector.recipient.peerId, + selector.sender.subjectId, + selector.sender.peerId + ]) + +const relayEntryKey = ( + principal: PeerAuthentication.PeerPrincipal, + remote: PeerRelayAuthorization.RemotePeer +) => + JSON.stringify([ + principal.tenantId, + principal.subjectId, + principal.peerId, + remote.subjectId, + remote.peerId + ]) + +const relayIncarnationKey = ( + principal: PeerAuthentication.PeerPrincipal, + incarnation: Identity.ReplicaIncarnation, + remote: PeerRelayAuthorization.RemotePeer +) => + JSON.stringify([ + principal.tenantId, + principal.subjectId, + principal.peerId, + incarnation, + remote.subjectId, + remote.peerId + ]) + +const relaySelectorForEntry = (entry: RelayEntry): RelayWorkSelector => ({ + recipient: entry.principal, + sender: entry.remote +}) + +const sameRelayPrincipal = ( + left: PeerAuthentication.PeerPrincipal, + right: PeerAuthentication.PeerPrincipal +) => + left.tenantId === right.tenantId && + left.subjectId === right.subjectId && + left.peerId === right.peerId + +const relayStoreFailure = (error: PeerRelayStore.StoreError): PeerRpcError.PeerRpcError => { + if (error._tag !== "ReplicaError") { + return new PeerRpcError.ServerUnavailable() + } + switch (error.reason._tag) { + case "QuotaExceeded": + return new PeerRpcError.RequestCapacityExceeded() + case "ProtocolMismatch": + return new PeerRpcError.InvalidRequest() + default: + return new PeerRpcError.ServerUnavailable() + } +} + +const SyncEnvelopeJson = Schema.fromJsonString(Schema.toCodecJson(PeerSession.SyncEnvelope)) + +const decodeRelayEnvelope = (payload: Uint8Array) => + Schema.decodeUnknownEffect(SyncEnvelopeJson)(new TextDecoder().decode(payload)).pipe( + Effect.mapError(() => new PeerRpcError.InvalidRequest()) + ) + +export const layerRelayHandlers = ( + options: { + readonly tenantId: string + readonly peerId: Identity.PeerId + } +) => + Layer.effectContext(Effect.gen(function*() { + const serverScope = yield* Scope.Scope + const runtimeScope = yield* Scope.fork(serverScope, "sequential") + const authorization = yield* PeerRelayAuthorization.PeerRelayAuthorization + const store = yield* PeerRelayStore.PeerRelayStore + const limits = yield* PeerRelayLimits.PeerRelayLimits + const ingress = yield* PeerRelayIngress.PeerRelayIngress + yield* Crypto.Crypto + const lock = yield* Semaphore.make(1) + const fatal = yield* Deferred.make() + const shutdownStarted = yield* Deferred.make() + const shutdownFinished = yield* Deferred.make() + const sessions = new Map() + const endpoints = new Map() + const incarnations = new Map() + const subjectSessions = new Map() + const workOwners = new Map() + const selectors = new Map() + const newWork = yield* Queue.dropping(limits.newWorkQueueCapacity) + const retryWork = yield* Queue.dropping(limits.retryQueueCapacity) + const workWake = yield* Queue.dropping( + limits.newWorkQueueCapacity + limits.retryQueueCapacity + ) + let accepting = true + let generation = 0 + let compensationCursor = 0 + + const signalFatal = (cause: Cause.Cause) => Deferred.done(fatal, Exit.failCause(cause)).pipe(Effect.asVoid) + + const sql = yield* makeRelaySqlScheduler(limits, runtimeScope, signalFatal) + const openLane = yield* makeRelayAdmissionLane({ + maximumInFlight: limits.maxInFlightOpen, + maximumInFlightPerSubject: limits.maxInFlightOpenPerSubject, + ratePerSecond: limits.openRatePerSecond, + burst: limits.openBurst, + maximumSubjects: limits.maxRetainedRateLimitedSubjects, + idleRetentionMillis: limits.rateLimitIdleRetentionMillis + }) + const pushLane = yield* makeRelayAdmissionLane({ + maximumInFlight: limits.maxInFlightPush, + maximumInFlightPerSubject: limits.maxInFlightPushPerSubject, + ratePerSecond: limits.admissionRatePerSecond, + burst: limits.admissionBurst, + maximumSubjects: limits.maxRetainedRateLimitedSubjects, + idleRetentionMillis: limits.rateLimitIdleRetentionMillis + }) + const terminalLane = yield* makeRelayAdmissionLane({ + maximumInFlight: limits.maxInFlightTerminalResponses, + maximumInFlightPerSubject: limits.maxInFlightTerminalResponsesPerSubject, + ratePerSecond: limits.terminalResponseRatePerSecond, + burst: limits.terminalResponseBurst, + maximumSubjects: limits.maxRetainedTerminalResponseSubjects, + idleRetentionMillis: limits.terminalResponseSubjectIdleRetentionMillis + }) + + const storeEffect = (effect: Effect.Effect) => + effect.pipe(Effect.mapError(relayStoreFailure)) + + const releaseClaim = ( + entry: RelayEntry, + claim: PeerRelayStore.ClaimedMessage + ) => + store.release({ + channel: claim.channel, + relayMessageId: claim.relayMessageId, + claimToken: claim.claimToken, + sessionGeneration: entry.generation + }).pipe( + Effect.timeoutOrElse({ + duration: limits.shutdownReleaseTimeoutMillis, + orElse: () => Effect.succeed(undefined) + }), + Effect.ignore + ) + + const detachEntry = ( + entry: RelayEntry, + fromWatcher: boolean + ): Effect.Effect => + Effect.uninterruptible(Effect.gen(function*() { + const cleanup = yield* lock.withPermit(Effect.sync(() => { + if (!entry.active) return undefined + entry.active = false + sessions.delete(entry.sessionId) + const endpoint = relayEntryKey(entry.principal, entry.remote) + if (endpoints.get(endpoint) === entry.sessionId) endpoints.delete(endpoint) + const incarnation = relayIncarnationKey( + entry.principal, + entry.senderReplicaIncarnation, + entry.remote + ) + if (incarnations.get(incarnation) === entry.sessionId) incarnations.delete(incarnation) + const current = subjectSessions.get(entry.principal.subjectId) ?? 0 + if (current <= 1) subjectSessions.delete(entry.principal.subjectId) + else subjectSessions.set(entry.principal.subjectId, current - 1) + const selector = relaySelectorForEntry(entry) + const key = relaySelectorKey(selector) + const owner = workOwners.get(key) + if (owner?._tag === "Entry" && owner.generation === entry.generation) { + workOwners.delete(key) + } + if (endpoints.get(endpoint) === undefined) { + selectors.delete(key) + } + const claims = [...entry.claims.values()] + entry.claims.clear() + return { claims } + })) + if (cleanup === undefined) return + const buffered = yield* drainRelayQueue(entry.outbound) + yield* Effect.forEach(buffered, (item) => item.reservation.release, { + concurrency: 1, + discard: true + }) + yield* Queue.fail(entry.outbound, new PeerRpcError.SessionUnavailable()) + yield* Queue.shutdown(entry.outbound) + yield* Effect.forEach( + cleanup.claims, + (claim) => releaseClaim(entry, claim), + { concurrency: limits.shutdownReleaseConcurrency, discard: true } + ) + if (!fromWatcher) { + yield* Effect.forEach(entry.watcherFibers, Fiber.interrupt, { discard: true }) + } + })) + + const notify = ( + selector: RelayWorkSelector, + lane: "New" | "Retry" + ): Effect.Effect => + Effect.gen(function*() { + const key = relaySelectorKey(selector) + const shouldOffer = yield* lock.withPermit(Effect.sync(() => { + if (!accepting) return false + if ( + endpoints.get(relayEntryKey(selector.recipient, selector.sender)) === undefined + ) { + return false + } + selectors.set(key, selector) + const current = workOwners.get(key) + if (current === undefined || current._tag === "Terminal") { + workOwners.set(key, { _tag: "Worker", lane, pending: false }) + return true + } + current.pending = true + return false + })) + if (!shouldOffer) return + const queue = lane === "New" ? newWork : retryWork + const offered = yield* Queue.offer(queue, selector) + if (!offered) { + yield* lock.withPermit(Effect.sync(() => { + const current = workOwners.get(key) + if (current?._tag === "Worker" && current.lane === lane) { + workOwners.delete(key) + } + })) + return + } + yield* Queue.offer(workWake, undefined) + }) + + const freshEntry = ( + sessionId: Identity.SessionId, + authenticated: Context.Service.Shape + ) => + Effect.gen(function*() { + const now = yield* Clock.currentTimeMillis + if (authenticated.validUntil <= now) { + return yield* new PeerRpcError.SessionUnavailable() + } + const entry = yield* lock.withPermit(Effect.sync(() => sessions.get(sessionId))) + if ( + entry === undefined || + !entry.active || + !sameRelayPrincipal(entry.principal, authenticated.principal) || + endpoints.get(relayEntryKey(entry.principal, entry.remote)) !== entry.sessionId || + incarnations.get( + relayIncarnationKey( + entry.principal, + entry.senderReplicaIncarnation, + entry.remote + ) + ) !== entry.sessionId + ) { + return yield* new PeerRpcError.SessionUnavailable() + } + return entry + }) + + const authorizeEntry = ( + entry: RelayEntry, + direction: PeerRelayAuthorization.Direction, + documents: ReadonlyArray + ) => + authorization.authorize({ + direction, + principal: entry.principal, + remote: entry.remote, + documents + }) + + const validateClaimPayload = ( + claim: PeerRelayStore.ClaimedMessage, + payload: Uint8Array + ) => + Effect.gen(function*() { + if (payload.byteLength !== claim.payloadBytes) { + return yield* new PeerRpcError.ServerUnavailable() + } + const envelope = yield* decodeRelayEnvelope(payload) + if ( + claim.relayPeerId !== options.peerId || + envelope.connectionEpoch !== claim.senderConnectionEpoch || + envelope.sequence !== claim.senderSequence || + envelope.messageHash !== claim.messageHash || + claim.documentIds.length !== 1 || + envelope.documentId !== claim.documentIds[0] + ) { + return yield* new PeerRpcError.ServerUnavailable() + } + const digest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope({ + domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, + version: PeerSyncEnvelope.relayOuterEnvelopeVersion, + expectedLocal: { + tenantId: claim.channel.tenantId, + subjectId: claim.channel.senderSubjectId, + peerId: claim.channel.senderPeerId + }, + remote: { + tenantId: claim.channel.tenantId, + subjectId: claim.channel.recipientSubjectId, + peerId: claim.channel.recipientPeerId + }, + relayPeerId: claim.relayPeerId, + relayMessageId: claim.relayMessageId, + protocolVersion: PeerRelayRpc.protocolVersion, + payloadVersion: claim.payloadVersion, + senderReplicaIncarnation: claim.channel.senderReplicaIncarnation, + senderConnectionEpoch: claim.senderConnectionEpoch, + senderSequence: claim.senderSequence, + document: { + documentId: envelope.documentId, + documentType: envelope.documentType + }, + writerProvenance: envelope.writerProvenance, + messageHash: envelope.messageHash, + payload + }).pipe(Effect.mapError(relayStoreFailure)) + if (digest !== claim.outerEnvelopeDigest) { + return yield* new PeerRpcError.ServerUnavailable() + } + return envelope + }) + + const abandonClaim = ( + entry: RelayEntry, + claim: PeerRelayStore.ClaimedMessage + ) => + lock.withPermit(Effect.sync(() => { + if (entry.claims.get(claim.relayMessageId) === claim) { + entry.claims.delete(claim.relayMessageId) + const key = relaySelectorKey(relaySelectorForEntry(entry)) + const owner = workOwners.get(key) + if ( + owner?._tag === "Worker" || + owner?._tag === "Entry" && owner.generation === entry.generation + ) { + workOwners.delete(key) + } + return true + } + return false + })).pipe( + Effect.flatMap((removed) => removed ? releaseClaim(entry, claim) : Effect.void) + ) + + const deliver = (selector: RelayWorkSelector) => + Effect.gen(function*() { + const key = relaySelectorKey(selector) + const entry = yield* lock.withPermit(Effect.sync(() => { + const sessionId = endpoints.get(relayEntryKey(selector.recipient, selector.sender)) + return sessionId === undefined ? undefined : sessions.get(sessionId) + })) + if (entry === undefined || !entry.active) { + yield* lock.withPermit(Effect.sync(() => { + workOwners.delete(key) + })) + return + } + const allowed = yield* authorizeEntry(entry, "Receive", entry.documents) + const authorizedDocumentIds = allowed.documents.map((document) => document.documentId) + type ClaimAcquisition = { + readonly claimed: PeerRelayStore.ClaimResult + readonly claim: PeerRelayStore.ClaimedMessage | undefined + readonly recorded: boolean + } + const acquisition = yield* Effect.uninterruptibleMask((restore) => + restore(sql.submit( + "Delivery", + storeEffect(store.claim({ + recipient: selector.recipient, + sender: selector.sender, + sessionGeneration: entry.generation, + authorizedDocumentIds + })) + )).pipe( + Effect.flatMap((claimed) => { + if (Option.isNone(claimed.message)) { + return Effect.succeed({ + claimed, + claim: undefined, + recorded: true + } as ClaimAcquisition) + } + const claim = claimed.message.value + return lock.withPermit(Effect.sync(() => { + if ( + sessions.get(entry.sessionId) !== entry || + !entry.active || + endpoints.get(relayEntryKey(entry.principal, entry.remote)) !== entry.sessionId + ) { + return { claimed, claim, recorded: false } as ClaimAcquisition + } + entry.claims.set(claim.relayMessageId, claim) + return { claimed, claim, recorded: true } as ClaimAcquisition + })) + }) + ) + ) + if (acquisition.claim === undefined) { + const pending = yield* lock.withPermit(Effect.sync(() => { + const current = workOwners.get(key) + workOwners.delete(key) + return current?._tag === "Worker" && current.pending + })) + if (pending) yield* notify(selector, acquisition.claimed.lane) + return + } + const claim = acquisition.claim + if (!acquisition.recorded) { + yield* releaseClaim(entry, claim) + return + } + let pendingReservation: PeerRelayIngress.Reservation | undefined + yield* Effect.gen(function*() { + const reservation = yield* ingress.reserveOutbound(claim.payloadBytes).pipe( + Effect.onError(() => abandonClaim(entry, claim)) + ) + pendingReservation = reservation + const payload = yield* sql.submit( + "Delivery", + storeEffect(store.loadClaimedPayload({ + channel: claim.channel, + rowId: claim.rowId, + relayMessageId: claim.relayMessageId, + claimToken: claim.claimToken, + sessionGeneration: entry.generation, + payloadBytes: claim.payloadBytes + })) + ).pipe(Effect.onError(() => Effect.andThen(reservation.release, abandonClaim(entry, claim)))) + const envelope = yield* validateClaimPayload(claim, payload).pipe( + Effect.onError(() => Effect.andThen(reservation.release, abandonClaim(entry, claim))) + ) + const event = PeerRelayRpc.StoredMessage.make({ + _tag: "StoredMessage", + relayMessageId: claim.relayMessageId, + claimToken: claim.claimToken, + relayPeerId: claim.relayPeerId, + sender: { + tenantId: claim.channel.tenantId, + subjectId: claim.channel.senderSubjectId, + peerId: claim.channel.senderPeerId, + replicaIncarnation: claim.channel.senderReplicaIncarnation, + connectionEpoch: claim.senderConnectionEpoch, + sequence: claim.senderSequence + }, + recipient: selector.recipient, + payloadVersion: 1, + document: { + documentType: envelope.documentType, + documentId: envelope.documentId + }, + writerProvenance: envelope.writerProvenance, + messageHash: envelope.messageHash, + outerEnvelopeDigest: claim.outerEnvelopeDigest, + payload + }) + const transferred = yield* lock.withPermit(Effect.sync(() => { + const currentSession = sessions.get(entry.sessionId) + const owner = workOwners.get(key) + if ( + currentSession !== entry || + !entry.active || + owner?._tag !== "Worker" + ) { + return false + } + workOwners.set(key, { + _tag: "Entry", + generation: entry.generation, + pending: owner.pending + }) + return true + })) + if (!transferred) { + yield* reservation.release + yield* abandonClaim(entry, claim) + return + } + const offered = yield* Queue.offer(entry.outbound, { + event, + reservation, + transferred: false + }) + if (!offered) { + yield* lock.withPermit(Effect.sync(() => { + const owner = workOwners.get(key) + if (owner?._tag === "Entry" && owner.generation === entry.generation) { + workOwners.delete(key) + } + })) + yield* reservation.release + yield* abandonClaim(entry, claim) + } + }).pipe( + Effect.onInterrupt(() => + Effect.andThen( + pendingReservation?.release ?? Effect.void, + abandonClaim(entry, claim) + ) + ) + ) + }).pipe( + Effect.catchTags({ + AccessDenied: () => + lock.withPermit(Effect.sync(() => { + workOwners.delete(relaySelectorKey(selector)) + })), + RequestCapacityExceeded: () => + lock.withPermit(Effect.sync(() => { + workOwners.delete(relaySelectorKey(selector)) + })) + }), + Effect.catchCause((cause) => signalFatal(cause)) + ) + + const workOrder = [ + ...Array.from({ length: limits.newWorkWeight }, () => "New" as const), + ...Array.from({ length: limits.retryWorkWeight }, () => "Retry" as const) + ] + const makeWorker = Effect.suspend(() => { + let cursor = 0 + const take = (): Effect.Effect => + Effect.gen(function*() { + for (let offset = 0; offset < workOrder.length; offset++) { + const index = (cursor + offset) % workOrder.length + const lane = workOrder[index]! + const item = yield* Queue.poll(lane === "New" ? newWork : retryWork) + if (Option.isSome(item)) { + cursor = (index + 1) % workOrder.length + return item.value + } + } + yield* Queue.take(workWake) + return yield* take() + }) + return Effect.forever(take().pipe(Effect.flatMap(deliver))) + }) + + const workerFibers: Array> = [] + for (let index = 0; index < limits.relayWorkerConcurrency; index++) { + const fiber = yield* Effect.forkIn(makeWorker, runtimeScope) + workerFibers.push(fiber) + yield* Effect.forkIn( + Fiber.await(fiber).pipe( + Effect.flatMap((exit) => + lock.withPermit(Effect.sync(() => accepting)).pipe( + Effect.flatMap((wasAccepting) => + wasAccepting && Exit.isFailure(exit) + ? signalFatal(exit.cause) + : Effect.void + ) + ) + ) + ), + runtimeScope + ) + } + + const compensationFiber = yield* Effect.forkIn( + Effect.forever( + Effect.sleep(limits.compensationIntervalMillis).pipe( + Effect.andThen(lock.withPermit(Effect.sync(() => { + const active = [...selectors.values()] + if (active.length === 0) return [] + const batch: Array = [] + for ( + let index = 0; + index < Math.min(limits.compensationBatchSize, active.length); + index++ + ) { + batch.push(active[(compensationCursor + index) % active.length]!) + } + compensationCursor = (compensationCursor + batch.length) % active.length + return batch + }))), + Effect.flatMap((batch) => + Effect.forEach(batch, (selector) => notify(selector, "Retry"), { + discard: true + }) + ) + ) + ), + runtimeScope + ) + + type MaintenanceStage = { + cursor: number | undefined + readonly batchSize: number + readonly run: ( + request: PeerRelayStore.MaintenanceRequest + ) => Effect.Effect + } + const maintenanceStages: Array = [ + { cursor: undefined, batchSize: limits.claimRecoveryBatchSize, run: store.recover }, + { cursor: undefined, batchSize: limits.expiryBatchSize, run: store.expire }, + { cursor: undefined, batchSize: limits.integrityBatchSize, run: store.repair }, + { cursor: undefined, batchSize: limits.reconciliationBatchSize, run: store.reconcile }, + { cursor: undefined, batchSize: limits.terminalCollectionBatchSize, run: store.collect } + ] + const maintenanceFiber = yield* Effect.forkIn( + Effect.forever( + Effect.sleep(limits.maintenanceIntervalMillis).pipe( + Effect.andThen(Effect.forEach(maintenanceStages, (stage) => + sql.submit( + "Maintenance", + storeEffect(stage.run({ + ...(stage.cursor === undefined ? {} : { cursor: stage.cursor }), + batchSize: stage.batchSize + })) + ).pipe( + Effect.tap((result) => + Effect.sync(() => { + stage.cursor = result.hasMore ? result.cursor : undefined + }) + ) + ), { discard: true })) + ) + ).pipe(Effect.catchCause(signalFatal)), + runtimeScope + ) + + const open = ( + request: typeof PeerRelayRpc.OpenRelayRpc.payloadSchema.Type + ) => + Effect.gen(function*() { + const authenticated = yield* PeerAuthentication.AuthenticatedPeer + return yield* openLane.run( + authenticated.principal.subjectId, + Effect.gen(function*() { + const principal = authenticated.principal + const now = yield* Clock.currentTimeMillis + if (authenticated.validUntil <= now) { + return yield* new PeerRpcError.AuthenticationFailure() + } + if (request.version !== PeerRelayRpc.protocolVersion) { + return yield* new PeerRpcError.UnsupportedVersion() + } + if ( + request.expectedRelayPeerId !== options.peerId || + request.expectedLocal.tenantId !== principal.tenantId || + request.expectedLocal.subjectId !== principal.subjectId || + request.expectedLocal.peerId !== principal.peerId + ) { + return yield* new PeerRpcError.PeerMismatch() + } + if (principal.tenantId !== options.tenantId) { + return yield* new PeerRpcError.AccessDenied() + } + if ( + request.senderRetryHorizonMillis > limits.maximumSenderRetryHorizonMillis || + request.receiptRetentionMillis > limits.maximumReceiptRetentionMillis || + request.receiptRetentionMillis < + Math.max(limits.messageTtlMillis, request.senderRetryHorizonMillis) + + limits.minimumTerminalRetentionMillis + ) { + return yield* new PeerRpcError.InvalidRequest() + } + const send = yield* authorization.authorize({ + direction: "Send", + principal, + remote: request.remote, + documents: request.documents + }) + const receive = yield* authorization.authorize({ + direction: "Receive", + principal, + remote: request.remote, + documents: request.documents + }) + const outbound = yield* Queue.dropping< + RelayOutboundItem, + PeerRpcError.PeerRpcError | Cause.Done + >(1) + const sessionId = yield* Identity.makeSessionId.pipe( + Effect.mapError(() => new PeerRpcError.ServerUnavailable()) + ) + const entry = yield* lock.withPermit(Effect.gen(function*() { + if (!accepting) return yield* new PeerRpcError.ServerUnavailable() + const current = subjectSessions.get(principal.subjectId) ?? 0 + const endpoint = relayEntryKey(principal, request.remote) + const replacedId = endpoints.get(endpoint) + const replaced = replacedId === undefined ? undefined : sessions.get(replacedId) + if (replaced === undefined && current >= limits.maxSessionsPerSubject) { + return yield* new PeerRpcError.RequestCapacityExceeded() + } + if (replaced === undefined && endpoints.size >= limits.maxActiveChannels) { + return yield* new PeerRpcError.RequestCapacityExceeded() + } + const entry: RelayEntry = { + sessionId, + generation: generation++, + principal, + senderReplicaIncarnation: request.senderReplicaIncarnation, + remote: request.remote, + documents: request.documents, + receiptRetentionMillis: request.receiptRetentionMillis, + senderRetryHorizonMillis: request.senderRetryHorizonMillis, + outbound, + claims: new Map(), + watcherFibers: [], + active: true + } + sessions.set(sessionId, entry) + endpoints.set(endpoint, sessionId) + incarnations.set( + relayIncarnationKey(principal, request.senderReplicaIncarnation, request.remote), + sessionId + ) + subjectSessions.set(principal.subjectId, current + 1) + selectors.set( + relaySelectorKey(relaySelectorForEntry(entry)), + relaySelectorForEntry(entry) + ) + return { entry, replaced } + })) + if (entry.replaced !== undefined) yield* detachEntry(entry.replaced, false) + const validUntil = Math.min( + authenticated.validUntil, + send.validUntil, + receive.validUntil + ) + const watcher = Effect.raceFirst( + Effect.raceFirst(authenticated.invalidated, send.invalidated), + Effect.raceFirst( + receive.invalidated, + Effect.sleep(Math.max(0, validUntil - now)) + ) + ).pipe( + Effect.andThen(detachEntry(entry.entry, true)), + Effect.forkIn(runtimeScope) + ) + entry.entry.watcherFibers.push(yield* watcher) + yield* notify(relaySelectorForEntry(entry.entry), "Retry") + const opened = PeerRelayRpc.RelayOpened.make({ + _tag: "RelayOpened", + version: PeerRelayRpc.protocolVersion, + sessionId, + remotePeerId: entry.entry.remote.peerId, + authenticatedLocal: principal, + capabilities: { storeAndForward: true } + }) + const deliveries = Stream.fromQueue(entry.entry.outbound).pipe( + Stream.mapEffect((item) => + item.reservation.transferToCurrentRequest.pipe( + Effect.tap(() => + Effect.sync(() => { + item.transferred = true + }) + ), + Effect.as(item.event), + Effect.onError(() => item.reservation.release) + ) + ) + ) + return Stream.concat(Stream.make(opened), deliveries).pipe( + Stream.ensuring(detachEntry(entry.entry, false)) + ) + }) + ) + }) + + const push = (request: typeof PeerRelayRpc.PushRelayRpc.payloadSchema.Type) => + Effect.gen(function*() { + const authenticated = yield* PeerAuthentication.AuthenticatedPeer + return yield* pushLane.run( + authenticated.principal.subjectId, + Effect.gen(function*() { + const entry = yield* freshEntry(request.sessionId, authenticated) + const envelope = yield* decodeRelayEnvelope(request.payload) + const document = entry.documents.find((candidate) => + candidate.documentId === envelope.documentId && + candidate.documentType === envelope.documentType + ) + if (document === undefined || !/^[0-9a-f]{64}$/.test(envelope.messageHash)) { + return yield* new PeerRpcError.InvalidRequest() + } + const messageHash = yield* Canonical.digest(envelope.message).pipe( + Effect.mapError(relayStoreFailure) + ) + if (messageHash !== envelope.messageHash) { + return yield* new PeerRpcError.InvalidRequest() + } + yield* authorizeEntry(entry, "Send", [document]) + const outerEnvelopeDigest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope({ + domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, + version: PeerSyncEnvelope.relayOuterEnvelopeVersion, + expectedLocal: entry.principal, + remote: { + tenantId: entry.principal.tenantId, + subjectId: entry.remote.subjectId, + peerId: entry.remote.peerId + }, + relayPeerId: options.peerId, + relayMessageId: request.relayMessageId, + protocolVersion: PeerRelayRpc.protocolVersion, + payloadVersion: 1, + senderReplicaIncarnation: entry.senderReplicaIncarnation, + senderConnectionEpoch: envelope.connectionEpoch, + senderSequence: envelope.sequence, + document: { + documentId: envelope.documentId, + documentType: envelope.documentType + }, + writerProvenance: envelope.writerProvenance, + messageHash: envelope.messageHash, + payload: request.payload + }).pipe(Effect.mapError(relayStoreFailure)) + const channel: PeerRelayStore.ChannelKey = { + tenantId: entry.principal.tenantId, + senderSubjectId: entry.principal.subjectId, + senderPeerId: entry.principal.peerId, + senderReplicaIncarnation: entry.senderReplicaIncarnation, + recipientSubjectId: entry.remote.subjectId, + recipientPeerId: entry.remote.peerId + } + const result = yield* sql.submit( + "Admission", + storeEffect(store.admit({ + channel, + relayMessageId: request.relayMessageId, + relayPeerId: options.peerId, + documentIds: [envelope.documentId], + senderConnectionEpoch: envelope.connectionEpoch, + senderSequence: envelope.sequence, + payloadVersion: 1, + messageHash: envelope.messageHash, + outerEnvelopeDigest, + payload: request.payload, + messageTtlMillis: limits.messageTtlMillis, + senderRetryHorizonMillis: entry.senderRetryHorizonMillis, + minimumTerminalRetentionMillis: limits.minimumTerminalRetentionMillis + })) + ) + if (result.ready) { + yield* notify({ + recipient: { + tenantId: channel.tenantId, + subjectId: channel.recipientSubjectId, + peerId: channel.recipientPeerId + }, + sender: { + subjectId: channel.senderSubjectId, + peerId: channel.senderPeerId + } + }, "New") + } + }) + ) + }) + + const terminal = ( + request: + | typeof PeerRelayRpc.AcknowledgeRelayRpc.payloadSchema.Type + | typeof PeerRelayRpc.RejectRelayRpc.payloadSchema.Type, + reason: PeerRelayRpc.RejectReason | undefined + ) => + Effect.gen(function*() { + const authenticated = yield* PeerAuthentication.AuthenticatedPeer + return yield* terminalLane.run( + authenticated.principal.subjectId, + Effect.gen(function*() { + const entry = yield* freshEntry(request.sessionId, authenticated) + const claim = yield* lock.withPermit(Effect.sync(() => entry.claims.get(request.relayMessageId))) + if ( + claim === undefined || + claim.claimToken !== request.claimToken || + claim.messageHash !== request.messageHash + ) { + return yield* new PeerRpcError.SessionUnavailable() + } + const document = entry.documents.filter((candidate) => claim.documentIds.includes(candidate.documentId)) + yield* authorizeEntry(entry, "Receive", document) + const transition = yield* sql.submit( + "Terminal", + storeEffect( + reason === undefined + ? store.acknowledge({ + channel: claim.channel, + relayMessageId: claim.relayMessageId, + claimToken: claim.claimToken, + messageHash: claim.messageHash, + sessionGeneration: entry.generation, + recipient: entry.principal + }) + : store.reject({ + channel: claim.channel, + relayMessageId: claim.relayMessageId, + claimToken: claim.claimToken, + messageHash: claim.messageHash, + sessionGeneration: entry.generation, + recipient: entry.principal, + reason + }) + ) + ) + if (transition.status === "Stale") { + return yield* new PeerRpcError.SessionUnavailable() + } + const selector = relaySelectorForEntry(entry) + const shouldNotify = yield* lock.withPermit(Effect.sync(() => { + entry.claims.delete(claim.relayMessageId) + const key = relaySelectorKey(selector) + const owner = workOwners.get(key) + if (owner?._tag === "Entry" && owner.generation === entry.generation) { + workOwners.delete(key) + } + return transition.ready || owner?._tag === "Entry" && owner.pending + })) + if (shouldNotify) yield* notify(selector, transition.lane) + }) + ) + }) + + const shutdown = Effect.uninterruptibleMask(() => + Effect.gen(function*() { + const first = yield* Deferred.succeed(shutdownStarted, undefined) + if (!first) return yield* Deferred.await(shutdownFinished) + yield* lock.withPermit(Effect.sync(() => { + accepting = false + })) + yield* Fiber.interrupt(compensationFiber) + yield* Fiber.interrupt(maintenanceFiber) + yield* Effect.forEach(workerFibers, Fiber.interrupt, { discard: true }) + yield* Queue.shutdown(newWork) + yield* Queue.shutdown(retryWork) + yield* Queue.shutdown(workWake) + const active = yield* lock.withPermit(Effect.sync(() => [...sessions.values()])) + yield* Effect.forEach(active, (entry) => detachEntry(entry, false), { + concurrency: limits.shutdownReleaseConcurrency, + discard: true + }) + yield* lock.withPermit(Effect.sync(() => { + workOwners.clear() + selectors.clear() + })) + yield* sql.shutdown + yield* openLane.clear + yield* pushLane.clear + yield* terminalLane.clear + yield* Scope.close(runtimeScope, Exit.void) + yield* Deferred.succeed(shutdownFinished, undefined) + }) + ) + + yield* Effect.addFinalizer(() => shutdown) + yield* Effect.forkIn( + Deferred.await(fatal).pipe( + Effect.ensuring(shutdown) + ), + serverScope + ) + + const runtime = PeerRelayServerRuntime.of({ + health: Deferred.poll(fatal).pipe( + Effect.flatMap((exit) => + Option.isNone(exit) + ? Effect.void + : Effect.fail(new PeerRpcError.ServerUnavailable()) + ) + ), + owner: Deferred.await(fatal), + shutdown, + usage: lock.withPermit(Effect.sync(() => ({ + accepting, + sessions: sessions.size, + subjects: subjectSessions.size, + activeClaims: [...sessions.values()].reduce((sum, entry) => sum + entry.claims.size, 0), + queuedChannels: workOwners.size + }))) + }) + const handlerContext = yield* PeerRelayRpc.Rpcs.toHandlers(PeerRelayRpc.Rpcs.of({ + OpenRelay: (request) => Stream.unwrap(open(request)), + PushRelay: push, + AcknowledgeRelay: (request) => terminal(request, undefined), + RejectRelay: (request) => terminal(request, request.reason) + })) + return Context.add(handlerContext, PeerRelayServerRuntime, runtime).pipe( + Context.add(PeerRelayFatalSignal, PeerRelayFatalSignal.of({ signal: signalFatal })) + ) + })) + +export const layerRelayServer = Layer.effectDiscard(Effect.gen(function*() { + const scope = yield* Scope.Scope + const runtime = yield* PeerRelayServerRuntime + const fatalSignal = yield* PeerRelayFatalSignal + const ingress = yield* PeerRelayIngress.PeerRelayIngress + let stopping = false + const serverFiber = yield* Effect.forkIn( + RpcServer.make(PeerRelayRpc.Rpcs), + scope + ) + const ingressFiber = yield* Effect.forkIn(ingress.await, scope) + const stop = Effect.uninterruptible( + Effect.sync(() => { + stopping = true + }).pipe( + Effect.andThen(runtime.shutdown), + Effect.andThen(Fiber.interrupt(serverFiber)), + Effect.andThen(Fiber.interrupt(ingressFiber)) + ) + ) + const observe = (fiber: Fiber.Fiber) => + Fiber.await(fiber).pipe( + Effect.flatMap((exit) => + stopping || Exit.isSuccess(exit) + ? Effect.void + : fatalSignal.signal(exit.cause) + ) + ) + yield* Effect.forkIn(observe(serverFiber), scope) + yield* Effect.forkIn(observe(ingressFiber), scope) + yield* Effect.forkIn( + runtime.owner.pipe( + Effect.catchCause(() => stop) + ), + scope + ) + yield* Effect.addFinalizer(() => stop) +})) + +export const layerStoreAndForwardDeployment = < + DirectOut, + DirectError, + DirectRequirements, + RelaySocketError, + RelaySocketRequirements, +>(options: { + readonly directDeployment: Layer.Layer + readonly relaySocketLayer: Layer.Layer< + SocketServer.SocketServer, + RelaySocketError, + RelaySocketRequirements + > + readonly relay: { + readonly tenantId: string + readonly peerId: Identity.PeerId + } +}) => { + const ingress = PeerRelayIngress.layerProtocolSocketServer(options.relaySocketLayer) + const handlers = layerRelayHandlers(options.relay) + const relay = layerRelayServer.pipe( + Layer.provideMerge(handlers), + Layer.provideMerge(ingress) + ) + return Layer.merge(options.directDeployment, relay) +} diff --git a/packages/local-rpc/src/RpcPeerTransport.ts b/packages/local-rpc/src/RpcPeerTransport.ts index 44060de..d243423 100644 --- a/packages/local-rpc/src/RpcPeerTransport.ts +++ b/packages/local-rpc/src/RpcPeerTransport.ts @@ -1,8 +1,12 @@ +import * as PeerRelayClientRuntime from "@lucas-barake/effect-local-sql/PeerRelayClientRuntime" import * as PeerSession from "@lucas-barake/effect-local-sql/PeerSession" +import * as PeerSyncEnvelope from "@lucas-barake/effect-local-sql/PeerSyncEnvelope" import type * as Identity from "@lucas-barake/effect-local/Identity" import * as PeerTransport from "@lucas-barake/effect-local/PeerTransport" import type * as ReplicaDefinition from "@lucas-barake/effect-local/ReplicaDefinition" import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" +import * as Cause from "effect/Cause" +import * as Crypto from "effect/Crypto" import * as Deferred from "effect/Deferred" import * as Effect from "effect/Effect" import * as Exit from "effect/Exit" @@ -14,6 +18,7 @@ import * as Sink from "effect/Sink" import * as Stream from "effect/Stream" import type { RpcClientError } from "effect/unstable/rpc/RpcClientError" import * as PeerRpcObservability from "./internal/peerRpcObservability.js" +import * as PeerRelayRpc from "./PeerRelayRpc.js" import * as PeerRpc from "./PeerRpc.js" import type * as PeerRpcError from "./PeerRpcError.js" @@ -251,3 +256,471 @@ export const makeSession = ( PeerSession.makeLive(options).pipe( Effect.provide(layer(client, { documents: options.documents, definition: options.definition })) ) + +export interface StoreAndForwardOptions { + readonly expectedLocal: PeerSyncEnvelope.RelayPeerPrincipal + readonly senderReplicaIncarnation: Identity.ReplicaIncarnation + readonly expectedRelayPeerId: Identity.PeerId + readonly remote: { + readonly subjectId: string + readonly peerId: Identity.PeerId + } + readonly documents: ReadonlyArray + readonly definition: ReplicaDefinition.Any + readonly receiptRetentionMillis: number + readonly senderRetryHorizonMillis: number + readonly replayBatchSize: number +} + +const samePrincipal = ( + left: PeerSyncEnvelope.RelayPeerPrincipal, + right: PeerSyncEnvelope.RelayPeerPrincipal +) => + left.tenantId === right.tenantId && + left.subjectId === right.subjectId && + left.peerId === right.peerId + +const validateRelayOptions = (options: StoreAndForwardOptions) => + Effect.suspend(() => { + for ( + const [name, value] of [ + ["receipt retention", options.receiptRetentionMillis], + ["sender retry horizon", options.senderRetryHorizonMillis] + ] as const + ) { + if ( + !Number.isSafeInteger(value) || + value <= 0 || + value > PeerRelayRpc.maximumNegotiatedDurationMillis + ) { + return Effect.fail(protocolFailure(`valid ${name}`)) + } + } + if (!Number.isSafeInteger(options.replayBatchSize) || options.replayBatchSize <= 0) { + return Effect.fail(protocolFailure("valid replay batch size")) + } + return Effect.void + }) + +const validateStoredMessage = ( + event: PeerRelayRpc.StoredMessage, + options: StoreAndForwardOptions, + crypto: Crypto.Crypto +) => + Effect.gen(function*() { + const expectedRecipient: PeerSyncEnvelope.RelayPeerPrincipal = { + ...options.expectedLocal + } + const expectedSender: PeerSyncEnvelope.RelayPeerPrincipal = { + tenantId: options.expectedLocal.tenantId, + subjectId: options.remote.subjectId, + peerId: options.remote.peerId + } + if ( + event.relayPeerId !== options.expectedRelayPeerId || + !samePrincipal(event.sender, expectedSender) || + !samePrincipal(event.recipient, expectedRecipient) + ) { + return yield* protocolFailure("relay delivery endpoint") + } + const selected = options.documents.some((entry) => + entry.document.name === event.document.documentType && + entry.documentId === event.document.documentId + ) + if (!selected) return yield* protocolFailure("selected relay document") + const digest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope({ + domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, + version: PeerSyncEnvelope.relayOuterEnvelopeVersion, + expectedLocal: expectedSender, + remote: expectedRecipient, + relayPeerId: event.relayPeerId, + relayMessageId: event.relayMessageId, + protocolVersion: PeerRelayRpc.protocolVersion, + payloadVersion: event.payloadVersion, + senderReplicaIncarnation: event.sender.replicaIncarnation, + senderConnectionEpoch: event.sender.connectionEpoch, + senderSequence: event.sender.sequence, + document: event.document, + writerProvenance: event.writerProvenance, + messageHash: event.messageHash, + payload: event.payload + }).pipe(Effect.provideService(Crypto.Crypto, crypto)) + if (digest !== event.outerEnvelopeDigest) { + return yield* protocolFailure("relay outer envelope digest") + } + }) + +export const layerStoreAndForward = ( + client: PeerRelayRpc.RpcClient, + options: StoreAndForwardOptions +) => + Layer.effect( + PeerTransport.PeerTransport, + Effect.gen(function*() { + const runtime = yield* PeerRelayClientRuntime.PeerRelayClientRuntime + const crypto = yield* Crypto.Crypto + const endpoint = { + expectedLocal: options.expectedLocal, + remote: { + tenantId: options.expectedLocal.tenantId, + subjectId: options.remote.subjectId, + peerId: options.remote.peerId + }, + relayPeerId: options.expectedRelayPeerId + } as const + + return { + capabilities: { storeAndForward: true }, + connect: (connectOptions) => + PeerRpcObservability.observe({ + effect: Effect.gen(function*() { + yield* validateDocuments(options.documents, options.definition) + yield* validateRelayOptions(options) + if (connectOptions.peerId !== options.remote.peerId) { + return yield* protocolFailure("configured remote peer") + } + yield* runtime.health + yield* runtime.validateConnectionConfiguration({ + replicaIncarnation: options.senderReplicaIncarnation, + retryHorizonMillis: options.senderRetryHorizonMillis, + replayBatchSize: options.replayBatchSize + }) + const pendingHorizon = yield* runtime.maximumPendingHorizon(endpoint) + if ( + pendingHorizon !== null && + options.senderRetryHorizonMillis < pendingHorizon + ) { + return yield* protocolFailure("sender retry horizon covering pending relay outbox") + } + const advertisedRetryHorizon = Math.max( + options.senderRetryHorizonMillis, + pendingHorizon ?? 0 + ) + const parentScope = yield* Scope.Scope + return yield* Effect.uninterruptibleMask((restore) => + Effect.gen(function*() { + const lifetimeScope = yield* Scope.fork(parentScope, "sequential") + const connectionScope = yield* Scope.make("parallel") + const stateLock = yield* Semaphore.make(1) + const closeCompleted = yield* Deferred.make() + const fatalCause = yield* Deferred.make< + Cause.Cause + >() + const activeDrained = yield* Deferred.make() + const interruptOnClose = Deferred.await(closeCompleted).pipe( + Effect.andThen(Effect.interrupt) + ) + let closing = false + let activeUses = 0 + const awaitFatal = Deferred.await(fatalCause).pipe( + Effect.flatMap(Effect.failCause) + ) + const closeConnection = (exit: Exit.Exit) => + Effect.sync(() => { + if (closing) return false + closing = true + return true + }).pipe( + stateLock.withPermit, + Effect.flatMap((owner) => + owner + ? Scope.close(connectionScope, exit).pipe( + Effect.ensuring(Deferred.succeed(closeCompleted, undefined)) + ) + : Deferred.await(closeCompleted) + ), + Effect.uninterruptible + ) + const closeWithExit = (exit: Exit.Exit) => + closeConnection(exit).pipe( + Effect.ensuring(Scope.close(lifetimeScope, exit)) + ) + const releaseUse = stateLock.withPermit( + Effect.gen(function*() { + activeUses -= 1 + if ( + activeUses === 0 && + (yield* Deferred.isDone(fatalCause)) + ) { + yield* Deferred.succeed(activeDrained, undefined) + } + }) + ) + const beginUse = stateLock.withPermit( + Effect.gen(function*() { + if (yield* Deferred.isDone(fatalCause)) { + return yield* awaitFatal + } + if (closing) return yield* unavailable() + activeUses += 1 + return releaseUse + }) + ) + yield* Scope.addFinalizerExit(lifetimeScope, closeConnection) + yield* runtime.awaitFatal.pipe( + Effect.exit, + Effect.flatMap((exit) => + Exit.isFailure(exit) && + !Cause.hasInterruptsOnly(exit.cause) + ? Deferred.succeed(fatalCause, exit.cause) + : Effect.void + ), + Effect.forkIn(lifetimeScope, { startImmediately: true }) + ) + + return yield* restore(Effect.gen(function*() { + const openCompleted = yield* Deferred.make< + Exit.Exit< + readonly [ + ReadonlyArray, + Stream.Stream< + PeerRelayRpc.OpenRelayEvent, + ReplicaError.ReplicaError + > + ], + ReplicaError.ReplicaError + > + >() + const openRequest = client.OpenRelay({ + version: PeerRelayRpc.protocolVersion, + expectedRelayPeerId: options.expectedRelayPeerId, + expectedLocal: options.expectedLocal, + senderReplicaIncarnation: options.senderReplicaIncarnation, + remote: options.remote, + documents: options.documents.map((entry) => ({ + documentType: entry.document.name, + documentId: entry.documentId + })), + receiptRetentionMillis: options.receiptRetentionMillis, + senderRetryHorizonMillis: advertisedRetryHorizon + }, { streamBufferSize: 1 }).pipe( + Stream.mapError(mapError), + Stream.peel(Sink.take(1)), + Effect.provideService(Scope.Scope, connectionScope), + Effect.onExit((exit) => Deferred.succeed(openCompleted, exit).pipe(Effect.asVoid)) + ) + const openFiber = yield* stateLock.withPermit( + Effect.suspend(() => + closing + ? Effect.fail(unavailable()) + : Effect.forkIn(openRequest, connectionScope) + ) + ) + const [first, remainder] = yield* Effect.raceFirst( + awaitFatal, + Deferred.await(openCompleted).pipe( + Effect.onInterrupt(() => Fiber.interrupt(openFiber)), + Effect.flatten + ) + ) + const handshake = first[0] + if ( + handshake === undefined || + handshake._tag !== "RelayOpened" || + handshake.version !== PeerRelayRpc.protocolVersion || + handshake.remotePeerId !== options.remote.peerId || + handshake.capabilities.storeAndForward !== true || + !samePrincipal(handshake.authenticatedLocal, options.expectedLocal) + ) { + return yield* protocolFailure("valid relay handshake") + } + yield* stateLock.withPermit( + Effect.gen(function*() { + if (closing) return yield* unavailable() + if (yield* Deferred.isDone(fatalCause)) { + return yield* awaitFatal + } + }) + ) + + const sendLock = yield* Semaphore.make(1) + const terminalLock = yield* Semaphore.make(1) + const callWithinConnection = ( + effect: Effect.Effect, + lock: Semaphore.Semaphore + ) => + Effect.uninterruptibleMask((restoreCall) => + Effect.gen(function*() { + const release = yield* beginUse + const completed = yield* Deferred.make< + Exit.Exit + >() + const fiber = yield* Effect.raceFirst( + awaitFatal, + lock.withPermit(effect) + ).pipe( + Effect.onExit((exit) => Deferred.succeed(completed, exit).pipe(Effect.asVoid)), + Effect.ensuring(release), + Effect.forkIn(connectionScope, { startImmediately: true }) + ) + return [fiber, completed] as const + }).pipe( + Effect.flatMap(([fiber, completed]) => + Deferred.await(completed).pipe( + restoreCall, + Effect.flatten, + Effect.onInterrupt(() => Fiber.interrupt(fiber)) + ) + ) + ) + ) + + const pushEntry = ( + entry: Effect.Success> + ) => + client.PushRelay({ + sessionId: handshake.sessionId, + relayMessageId: entry.relayMessageId, + payload: entry.payload + }).pipe( + Effect.mapError(mapError), + Effect.andThen(runtime.markCustody({ + relayMessageId: entry.relayMessageId, + outerEnvelopeDigest: entry.outerEnvelopeDigest + })) + ) + + const replay = Effect.gen(function*() { + while (true) { + const entries = yield* runtime.dueForEndpoint({ + ...endpoint, + maximum: options.replayBatchSize + }) + if (entries.length === 0) return + for (const entry of entries) yield* pushEntry(entry) + } + }) + yield* callWithinConnection(replay, sendLock) + + const send = (message: Uint8Array) => + PeerRpcObservability.observe({ + effect: callWithinConnection( + runtime.admit({ + ...endpoint, + payload: message, + retryHorizonMillis: options.senderRetryHorizonMillis + }).pipe(Effect.flatMap(pushEntry)), + sendLock + ), + operation: "AdapterPush", + spanName: "effect_local_rpc.adapter.relay_push", + attributes: { "rpc.payload_bytes": message.byteLength }, + result: adapterResult + }) + + const terminalCall = ( + effect: Effect.Effect + ) => + callWithinConnection( + effect, + terminalLock + ).pipe( + Effect.onExitIf(Exit.isFailure, closeWithExit) + ) + + const acknowledged = Stream.scoped( + Stream.fromEffect( + Effect.acquireRelease(beginUse, (release) => release) + ).pipe( + Stream.flatMap(() => + remainder.pipe( + Stream.mapEffect((event) => + event._tag !== "StoredMessage" + ? Effect.fail(protocolFailure(event._tag)) + : validateStoredMessage(event, options, crypto).pipe( + Effect.as( + { + message: event.payload, + identity: { + relayMessageId: event.relayMessageId, + relayPeerId: event.relayPeerId, + senderTenantId: event.sender.tenantId, + senderSubjectId: event.sender.subjectId, + senderPeerId: event.sender.peerId, + senderReplicaIncarnation: event.sender.replicaIncarnation, + messageHash: event.messageHash, + outerEnvelopeDigest: event.outerEnvelopeDigest + }, + receiptRetentionMillis: options.receiptRetentionMillis, + acknowledge: terminalCall( + client.AcknowledgeRelay({ + sessionId: handshake.sessionId, + relayMessageId: event.relayMessageId, + claimToken: event.claimToken, + messageHash: event.messageHash + }).pipe( + Effect.mapError(mapError), + Effect.andThen(runtime.signalReceiptPrune) + ) + ), + reject: (reason: PeerTransport.PermanentRejectReason) => + terminalCall( + client.RejectRelay({ + sessionId: handshake.sessionId, + relayMessageId: event.relayMessageId, + claimToken: event.claimToken, + messageHash: event.messageHash, + reason + }).pipe(Effect.mapError(mapError)) + ) + } satisfies PeerTransport.AcknowledgedDelivery + ) + ) + ), + Stream.interruptWhen(awaitFatal), + Stream.interruptWhen(interruptOnClose) + ) + ) + ) + ) + + yield* awaitFatal.pipe( + Effect.catchCause((cause) => + stateLock.withPermit( + Effect.gen(function*() { + if (activeUses === 0) { + yield* Deferred.succeed(activeDrained, undefined) + } + }) + ).pipe( + Effect.andThen(Deferred.await(activeDrained)), + Effect.andThen(closeConnection(Exit.failCause(cause))) + ) + ), + Effect.forkIn(lifetimeScope, { startImmediately: true }) + ) + + return { + peerId: handshake.remotePeerId, + relayPeerId: options.expectedRelayPeerId, + capabilities: handshake.capabilities, + receive: acknowledged.pipe( + Stream.map((delivery) => delivery.message) + ), + receiveWithAcknowledgement: acknowledged, + send, + close: closeWithExit(Exit.void) + } + })).pipe(Effect.onExitIf(Exit.isFailure, closeWithExit)) + }) + ) + }), + operation: "AdapterOpen", + spanName: "effect_local_rpc.adapter.relay_open", + attributes: { "rpc.selected_documents": options.documents.length }, + result: adapterResult + }) + } + }) + ) + +export const makeStoreAndForwardSession = ( + client: PeerRelayRpc.RpcClient, + options: StoreAndForwardOptions +) => + PeerSession.makeLive({ + peerId: options.remote.peerId, + documents: options.documents + }).pipe( + Effect.provide(layerStoreAndForward(client, options)) + ) diff --git a/packages/local-rpc/src/index.ts b/packages/local-rpc/src/index.ts index 873b1c4..2515a66 100644 --- a/packages/local-rpc/src/index.ts +++ b/packages/local-rpc/src/index.ts @@ -2,6 +2,11 @@ export * as PeerAuthentication from "./PeerAuthentication.js" export * as PeerAuthenticator from "./PeerAuthenticator.js" export * as PeerAuthorization from "./PeerAuthorization.js" export * as PeerCredentials from "./PeerCredentials.js" +export * as PeerRelayAuthorization from "./PeerRelayAuthorization.js" +export * as PeerRelayIngress from "./PeerRelayIngress.js" +export * as PeerRelayLimits from "./PeerRelayLimits.js" +export * as PeerRelayRpc from "./PeerRelayRpc.js" +export * as PeerRelayStore from "./PeerRelayStore.js" export * as PeerRpc from "./PeerRpc.js" export * as PeerRpcError from "./PeerRpcError.js" export * as PeerRpcLimits from "./PeerRpcLimits.js" diff --git a/packages/local-rpc/src/internal/peerRelayMigrations.ts b/packages/local-rpc/src/internal/peerRelayMigrations.ts new file mode 100644 index 0000000..0f4f36a --- /dev/null +++ b/packages/local-rpc/src/internal/peerRelayMigrations.ts @@ -0,0 +1,204 @@ +import * as Effect from "effect/Effect" +import * as Layer from "effect/Layer" +import * as Schema from "effect/Schema" +import * as Migrator from "effect/unstable/sql/Migrator" +import * as SqlClient from "effect/unstable/sql/SqlClient" +import type * as SqlError from "effect/unstable/sql/SqlError" +import * as SqlSchema from "effect/unstable/sql/SqlSchema" + +export const relayCustodyChecksum = "sha256:effect-local-relay-custody-v1" +export const relayMaintenanceIndexesChecksum = "sha256:effect-local-relay-maintenance-indexes-v1" + +const custody = Effect.gen(function*() { + const sql = yield* SqlClient.SqlClient + yield* sql`CREATE TABLE effect_local_relay_migration_catalog ( + migration_id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + checksum TEXT NOT NULL + )` + yield* sql`CREATE TABLE effect_local_relay_channels ( + channel_id INTEGER PRIMARY KEY AUTOINCREMENT, + tenant_id TEXT NOT NULL, + sender_subject_id TEXT NOT NULL, + sender_peer_id TEXT NOT NULL, + sender_replica_incarnation INTEGER NOT NULL, + recipient_subject_id TEXT NOT NULL, + recipient_peer_id TEXT NOT NULL, + next_sequence INTEGER NOT NULL DEFAULT 0 CHECK (next_sequence >= 0), + claimed_message_id INTEGER, + claim_session_generation INTEGER, + claim_token TEXT, + claim_deadline INTEGER, + UNIQUE ( + tenant_id, + sender_subject_id, + sender_peer_id, + sender_replica_incarnation, + recipient_subject_id, + recipient_peer_id + ) + )` + yield* sql`CREATE TABLE effect_local_relay_messages ( + message_id INTEGER PRIMARY KEY AUTOINCREMENT, + channel_id INTEGER NOT NULL REFERENCES effect_local_relay_channels(channel_id) ON DELETE CASCADE, + channel_sequence INTEGER NOT NULL CHECK (channel_sequence >= 0), + tenant_id TEXT NOT NULL, + sender_subject_id TEXT NOT NULL, + sender_peer_id TEXT NOT NULL, + relay_message_id TEXT NOT NULL, + relay_peer_id TEXT NOT NULL, + sender_connection_epoch TEXT NOT NULL, + sender_sequence INTEGER NOT NULL CHECK (sender_sequence >= 0), + document_ids TEXT NOT NULL, + payload_version INTEGER NOT NULL, + message_hash TEXT NOT NULL, + outer_envelope_digest TEXT NOT NULL, + payload BLOB, + payload_length INTEGER NOT NULL CHECK (payload_length >= 0), + state TEXT NOT NULL CHECK ( + state IN ('Pending', 'Claimed', 'Acknowledged', 'DeadLettered', 'Expired') + ), + created_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + deduplicate_until INTEGER NOT NULL, + next_eligible_at INTEGER NOT NULL, + retry_count INTEGER NOT NULL DEFAULT 0 CHECK (retry_count >= 0), + claim_token TEXT, + claim_session_generation INTEGER, + claim_deadline INTEGER, + terminal_at INTEGER, + terminal_claim_token TEXT, + terminal_session_generation INTEGER, + terminal_reason TEXT, + UNIQUE(channel_id, channel_sequence), + UNIQUE(tenant_id, sender_subject_id, sender_peer_id, relay_message_id) + )` + yield* sql`CREATE TABLE effect_local_relay_usage ( + scope_kind TEXT NOT NULL CHECK ( + scope_kind IN ('SenderPeer', 'RecipientPeer', 'RecipientSubject', 'Tenant', 'Shard') + ), + scope_key TEXT NOT NULL, + active_count INTEGER NOT NULL CHECK (active_count >= 0), + active_bytes INTEGER NOT NULL CHECK (active_bytes >= 0), + retained_count INTEGER NOT NULL CHECK (retained_count >= 0), + retained_bytes INTEGER NOT NULL CHECK (retained_bytes >= 0), + PRIMARY KEY(scope_kind, scope_key) + )` + yield* sql`CREATE TABLE effect_local_relay_reservations ( + message_id INTEGER PRIMARY KEY REFERENCES effect_local_relay_messages(message_id) ON DELETE CASCADE, + sender_peer_usage_key TEXT NOT NULL, + recipient_peer_usage_key TEXT NOT NULL, + recipient_subject_usage_key TEXT NOT NULL, + tenant_usage_key TEXT NOT NULL, + shard_usage_key TEXT NOT NULL, + active_count_delta INTEGER NOT NULL CHECK (active_count_delta = 1), + active_bytes_delta INTEGER NOT NULL CHECK (active_bytes_delta >= 0), + retained_count_delta INTEGER NOT NULL CHECK (retained_count_delta = 1), + retained_bytes_delta INTEGER NOT NULL CHECK (retained_bytes_delta >= 0), + active_consumed INTEGER NOT NULL DEFAULT 0 CHECK (active_consumed IN (0, 1)), + retained_consumed INTEGER NOT NULL DEFAULT 0 CHECK (retained_consumed IN (0, 1)) + )` + yield* sql`INSERT INTO effect_local_relay_migration_catalog (migration_id, name, checksum) + VALUES (1, 'relay_custody', ${relayCustodyChecksum})` +}) + +const maintenanceIndexes = Effect.gen(function*() { + const sql = yield* SqlClient.SqlClient + yield* sql`CREATE INDEX effect_local_relay_messages_channel_head + ON effect_local_relay_messages(channel_id, channel_sequence, state, next_eligible_at)` + yield* sql`CREATE INDEX effect_local_relay_channels_discovery + ON effect_local_relay_channels( + tenant_id, + recipient_subject_id, + recipient_peer_id, + sender_subject_id, + sender_peer_id, + channel_id + )` + yield* sql`CREATE INDEX effect_local_relay_messages_admission_order + ON effect_local_relay_messages(created_at, message_id, channel_id, channel_sequence)` + yield* sql`CREATE INDEX effect_local_relay_messages_recovery + ON effect_local_relay_messages(state, claim_deadline, message_id)` + yield* sql`CREATE INDEX effect_local_relay_messages_expiry + ON effect_local_relay_messages(expires_at, message_id) + WHERE state IN ('Pending', 'Claimed')` + yield* sql`CREATE INDEX effect_local_relay_messages_collection + ON effect_local_relay_messages(deduplicate_until, message_id) + WHERE state IN ('Acknowledged', 'DeadLettered', 'Expired')` + yield* sql`CREATE INDEX effect_local_relay_channels_claim + ON effect_local_relay_channels(claimed_message_id, channel_id)` + yield* sql`INSERT INTO effect_local_relay_migration_catalog (migration_id, name, checksum) + VALUES (2, 'relay_maintenance_indexes', ${relayMaintenanceIndexesChecksum})` +}) + +export const loader = Migrator.fromRecord({ + "1_relay_custody": custody, + "2_relay_maintenance_indexes": maintenanceIndexes +}) + +const migrate = Migrator.make({})({ + loader, + table: "effect_local_relay_migrations" +}) + +const expectedCatalog = [ + { + id: 1, + name: "relay_custody", + checksum: relayCustodyChecksum, + label: "Relay custody" + }, + { + id: 2, + name: "relay_maintenance_indexes", + checksum: relayMaintenanceIndexesChecksum, + label: "Relay maintenance indexes" + } +] as const + +export const run = Effect.gen(function*() { + const sql = yield* SqlClient.SqlClient + const findCatalog = SqlSchema.findAll({ + Request: Schema.Int, + Result: Schema.Struct({ + name: Schema.String, + checksum: Schema.String + }), + execute: (migrationId) => + sql`SELECT name, checksum + FROM effect_local_relay_migration_catalog + WHERE migration_id = ${migrationId}` + }) + return yield* sql.withTransaction(Effect.gen(function*() { + const applied = yield* migrate + for (const expected of expectedCatalog) { + const rows = yield* findCatalog(expected.id) + const row = rows[0] + if ( + rows.length !== 1 || + row?.name !== expected.name || + row.checksum !== expected.checksum + ) { + return yield* new Migrator.MigrationError({ + kind: "BadState", + message: `${expected.label} migration checksum mismatch` + }) + } + } + return applied + })) +}).pipe( + Effect.catchTag("SchemaError", (cause) => + Effect.fail( + new Migrator.MigrationError({ + kind: "BadState", + message: `Invalid relay migration catalog: ${cause}` + }) + )) +) + +export const layer: Layer.Layer< + never, + Migrator.MigrationError | SqlError.SqlError, + SqlClient.SqlClient +> = Layer.effectDiscard(run) diff --git a/packages/local-rpc/src/internal/peerRelaySqliteTransaction.ts b/packages/local-rpc/src/internal/peerRelaySqliteTransaction.ts new file mode 100644 index 0000000..2d89fbf --- /dev/null +++ b/packages/local-rpc/src/internal/peerRelaySqliteTransaction.ts @@ -0,0 +1,169 @@ +import * as Cause from "effect/Cause" +import * as Effect from "effect/Effect" +import * as Exit from "effect/Exit" +import * as Option from "effect/Option" +import * as Schema from "effect/Schema" +import * as Scope from "effect/Scope" +import * as SqlClient from "effect/unstable/sql/SqlClient" +import type * as SqlConnection from "effect/unstable/sql/SqlConnection" +import type * as SqlError from "effect/unstable/sql/SqlError" + +export class NestedPeerRelayTransactionError extends Schema.TaggedErrorClass( + "@lucas-barake/effect-local-rpc/internal/peerRelaySqliteTransaction/NestedPeerRelayTransactionError" +)("NestedPeerRelayTransactionError", {}) {} + +export interface Options { + readonly maxAcquireAttempts: number + readonly acquireRetryBaseDelayMillis: number + readonly acquireRetryMaximumDelayMillis: number +} + +const execute = (connection: SqlConnection.Connection, statement: string) => + connection.executeUnprepared(statement, [], undefined).pipe(Effect.asVoid) + +export const cleanupAcquisition = ( + original: Cause.Cause, + scope: Scope.Closeable | undefined, + connection: SqlConnection.Connection | undefined, + began: boolean +): Effect.Effect => + Effect.uninterruptible( + Effect.gen(function*() { + let combined: Cause.Cause = original + if (began && connection !== undefined) { + const rollback = yield* execute(connection, "ROLLBACK").pipe(Effect.exit) + if (Exit.isFailure(rollback)) { + combined = Cause.combine(combined, rollback.cause) + } + } + if (scope !== undefined) { + const closed = yield* Scope.close(scope, Exit.failCause(combined)).pipe(Effect.exit) + if (Exit.isFailure(closed)) { + combined = Cause.combine(combined, closed.cause) + } + } + return yield* Effect.failCause(combined) + }) + ) + +const acquireImmediate = ( + sql: SqlClient.SqlClient, + options: Options +): Effect.Effect< + readonly [Scope.Closeable, SqlConnection.Connection], + SqlError.SqlError +> => + Effect.interruptibleMask((restore) => { + const attempt = ( + remaining: number, + attemptNumber: number + ): Effect.Effect< + readonly [Scope.Closeable, SqlConnection.Connection], + SqlError.SqlError + > => + Effect.suspend(() => { + let began = false + let scope: Scope.Closeable | undefined + let connection: SqlConnection.Connection | undefined + const acquire = Effect.gen(function*() { + scope = yield* Scope.make("sequential") + const reserved = yield* sql.reserve.pipe( + Effect.provideService(Scope.Scope, scope) + ) + connection = reserved + return yield* restore( + execute(reserved, "BEGIN IMMEDIATE").pipe( + Effect.tap(() => + Effect.sync(() => { + began = true + }) + ), + Effect.as([scope, reserved] as const) + ) + ) + }).pipe( + Effect.catchCause((cause) => cleanupAcquisition(cause, scope, connection, began)) + ) + return acquire.pipe( + Effect.catchCause((cause) => { + const only = cause.reasons.length === 1 ? cause.reasons[0] : undefined + const lockTimeout = only !== undefined && + Cause.isFailReason(only) && + only.error._tag === "SqlError" && + only.error.reason._tag === "LockTimeoutError" + if (!lockTimeout || remaining <= 1) { + return Effect.failCause(cause) + } + return Effect.sleep(Math.min( + options.acquireRetryMaximumDelayMillis, + options.acquireRetryBaseDelayMillis * 2 ** attemptNumber + )).pipe( + Effect.andThen(attempt(remaining - 1, attemptNumber + 1)) + ) + }) + ) + }) + return attempt(options.maxAcquireAttempts, 0) + }) + +export const make = ( + sql: SqlClient.SqlClient, + options: Options +) => { + return ( + effect: Effect.Effect + ): Effect.Effect< + A, + E | SqlError.SqlError | NestedPeerRelayTransactionError, + R + > => + Effect.gen(function*() { + const active = yield* Effect.serviceOption(sql.transactionService) + if (Option.isSome(active)) { + return yield* new NestedPeerRelayTransactionError() + } + return yield* Effect.suspend(() => { + let bodyCause: Cause.Cause | undefined + let rollbackCause: Cause.Cause | undefined + const withTransaction = SqlClient.makeWithTransaction({ + transactionService: sql.transactionService, + spanAttributes: [["db.system", "sqlite"]], + acquireConnection: acquireImmediate(sql, options), + begin: () => Effect.void, + savepoint: () => Effect.die(new Error("Nested relay transactions are forbidden")), + commit: (connection) => execute(connection, "COMMIT"), + rollback: (connection) => + execute(connection, "ROLLBACK").pipe( + Effect.exit, + Effect.tap((exit) => + Effect.sync(() => { + if (Exit.isFailure(exit)) { + rollbackCause = exit.cause + } + }) + ), + Effect.asVoid + ), + rollbackSavepoint: () => Effect.die(new Error("Nested relay transactions are forbidden")) + }) + const observed = effect.pipe( + Effect.catchCause((cause) => + Effect.sync(() => { + bodyCause = cause + }).pipe(Effect.andThen(Effect.failCause(cause))) + ) + ) + return withTransaction(observed).pipe( + Effect.catchCause((cause) => { + let combined = bodyCause === undefined + ? cause + : Cause.combine(bodyCause, cause) + if (rollbackCause !== undefined) { + combined = Cause.combine(combined, rollbackCause) + } + return Effect.failCause(combined) + }) + ) + }) + }) +} diff --git a/packages/local-rpc/test/PeerRelayAuthorization.test.ts b/packages/local-rpc/test/PeerRelayAuthorization.test.ts new file mode 100644 index 0000000..7a9fdf9 --- /dev/null +++ b/packages/local-rpc/test/PeerRelayAuthorization.test.ts @@ -0,0 +1,248 @@ +import { assert, describe, it } from "@effect/vitest" +import * as Document from "@lucas-barake/effect-local/Document" +import * as Identity from "@lucas-barake/effect-local/Identity" +import * as Cause from "effect/Cause" +import * as Deferred from "effect/Deferred" +import * as Effect from "effect/Effect" +import * as Schema from "effect/Schema" +import * as TestClock from "effect/testing/TestClock" +import * as PeerAuthentication from "../src/PeerAuthentication.js" +import * as PeerRelayAuthorization from "../src/PeerRelayAuthorization.js" +import * as PeerRpcError from "../src/PeerRpcError.js" + +const task = Document.make("Task", { + schema: Schema.Struct({ title: Schema.String }), + version: 1 +}) +const note = Document.make("Note", { + schema: Schema.Struct({ body: Schema.String }), + version: 1 +}) +const taskId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000001") +const noteId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000002") +const localPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000001") +const remotePeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000002") +const otherPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000003") +const principal = PeerAuthentication.PeerPrincipal.make({ + tenantId: "tenant", + subjectId: "local", + peerId: localPeerId +}) +const remote = PeerRelayAuthorization.RemotePeer.make({ + subjectId: "remote", + peerId: remotePeerId +}) +const resolvedRemote = PeerAuthentication.PeerPrincipal.make({ + tenantId: principal.tenantId, + subjectId: remote.subjectId, + peerId: remote.peerId +}) +const documents = [ + { documentType: task.name, documentId: taskId }, + { documentType: note.name, documentId: noteId } +] as const + +const request = (direction: PeerRelayAuthorization.Direction): PeerRelayAuthorization.Request => ({ + direction, + principal, + remote, + documents +}) + +const result = ( + selected: PeerRelayAuthorization.Result["documents"] = [ + { document: task, documentId: taskId }, + { document: note, documentId: noteId } + ], + endpoint: PeerAuthentication.PeerPrincipal = resolvedRemote, + validUntil = Number.MAX_SAFE_INTEGER, + invalidated: Effect.Effect = Effect.void +): PeerRelayAuthorization.Result => ({ + remote: endpoint, + documents: selected, + validUntil, + invalidated +}) + +describe("PeerRelayAuthorization", () => { + it.effect.each(["Send", "Receive"] as const)( + "resolves the exact duplex endpoint and document set for %s", + (direction) => { + let observed: PeerRelayAuthorization.Request | undefined + return Effect.gen(function*() { + const authorization = yield* PeerRelayAuthorization.PeerRelayAuthorization + const authorized = yield* authorization.authorize(request(direction)) + assert.deepStrictEqual(observed, request(direction)) + assert.deepStrictEqual(authorized.remote, resolvedRemote) + assert.deepStrictEqual(authorized.documents, [ + { document: note, documentId: noteId }, + { document: task, documentId: taskId } + ]) + }).pipe( + Effect.provide( + PeerRelayAuthorization.layer((input) => { + observed = input + return Effect.succeed(result([ + { document: note, documentId: noteId }, + { document: task, documentId: taskId } + ])) + }) + ) + ) + } + ) + + it.effect("rejects invalid direction endpoint and duplicate request shapes before policy evaluation", () => { + let calls = 0 + const cases = [ + { ...request("Send"), direction: "Forward" }, + { ...request("Send"), remote: { ...remote, subjectId: "" } }, + { ...request("Send"), remote: { ...remote, peerId: "peer_invalid" } }, + { + ...request("Send"), + documents: [documents[0], documents[0]] + }, + { + ...request("Send"), + documents: [ + documents[0], + { documentType: note.name, documentId: taskId } + ] + } + ] as const + + return Effect.gen(function*() { + const authorization = yield* PeerRelayAuthorization.PeerRelayAuthorization + for (const invalid of cases) { + const error = yield* authorization.authorize( + invalid as unknown as PeerRelayAuthorization.Request + ).pipe(Effect.flip) + assert.deepStrictEqual(error, new PeerRpcError.AccessDenied()) + } + assert.strictEqual(calls, 0) + }).pipe( + Effect.provide( + PeerRelayAuthorization.layer(() => { + calls++ + return Effect.never + }) + ) + ) + }) + + it.effect("rejects missing extra duplicate and substituted document grants", () => + Effect.gen(function*() { + const cases: ReadonlyArray = [ + [], + [ + { document: task, documentId: taskId }, + { document: note, documentId: noteId }, + { document: task, documentId: noteId } + ], + [ + { document: task, documentId: taskId }, + { document: task, documentId: taskId } + ], + [ + { document: note, documentId: taskId }, + { document: note, documentId: noteId } + ], + [ + { document: task, documentId: noteId }, + { document: note, documentId: noteId } + ] + ] + for (const selected of cases) { + const authorization = yield* PeerRelayAuthorization.PeerRelayAuthorization.pipe( + Effect.provide(PeerRelayAuthorization.layer(() => Effect.succeed(result(selected)))) + ) + const error = yield* authorization.authorize(request("Send")).pipe(Effect.flip) + assert.deepStrictEqual(error, new PeerRpcError.AccessDenied()) + } + })) + + it.effect("rejects every substituted resolved endpoint with the same fieldless error", () => + Effect.gen(function*() { + const endpoints = [ + { ...resolvedRemote, tenantId: "other-tenant" }, + { ...resolvedRemote, subjectId: "other-subject" }, + { ...resolvedRemote, peerId: otherPeerId } + ] + for (const endpoint of endpoints) { + const authorization = yield* PeerRelayAuthorization.PeerRelayAuthorization.pipe( + Effect.provide( + PeerRelayAuthorization.layer(() => Effect.succeed(result(undefined, endpoint))) + ) + ) + const error = yield* authorization.authorize(request("Receive")).pipe(Effect.flip) + assert.deepStrictEqual(error, new PeerRpcError.AccessDenied()) + assert.deepStrictEqual(Object.keys(error), ["_tag"]) + } + })) + + it.effect.each( + [ + ["expired", 0], + ["NaN", Number.NaN], + ["positive infinity", Number.POSITIVE_INFINITY], + ["negative infinity", Number.NEGATIVE_INFINITY] + ] as const + )("rejects a %s authorization lease", ([, validUntil]) => + Effect.gen(function*() { + yield* TestClock.setTime(1_000) + const authorization = yield* PeerRelayAuthorization.PeerRelayAuthorization + const error = yield* authorization.authorize(request("Send")).pipe(Effect.flip) + assert.deepStrictEqual(error, new PeerRpcError.AccessDenied()) + }).pipe( + Effect.provide( + PeerRelayAuthorization.layer(() => Effect.succeed(result(undefined, undefined, validUntil))) + ) + )) + + it.effect("exposes the finite lease and live invalidation effect without caching authority", () => + Effect.gen(function*() { + const invalidated = yield* Deferred.make() + let allowed = true + const authorization = yield* PeerRelayAuthorization.PeerRelayAuthorization.pipe( + Effect.provide( + PeerRelayAuthorization.layer(() => + allowed + ? Effect.succeed(result(undefined, undefined, 2_000, Deferred.await(invalidated))) + : Effect.fail(new PeerRpcError.AccessDenied()) + ) + ) + ) + yield* TestClock.setTime(1_000) + const grant = yield* authorization.authorize(request("Receive")) + assert.strictEqual(grant.validUntil, 2_000) + allowed = false + yield* Deferred.succeed(invalidated, undefined) + yield* grant.invalidated + const error = yield* authorization.authorize(request("Receive")).pipe(Effect.flip) + assert.deepStrictEqual(error, new PeerRpcError.AccessDenied()) + })) + + it.effect("preserves target independent policy failure and defects", () => + Effect.gen(function*() { + const unavailable = yield* PeerRelayAuthorization.PeerRelayAuthorization.pipe( + Effect.provide( + PeerRelayAuthorization.layer(() => Effect.fail(new PeerRpcError.ServerUnavailable())) + ) + ) + assert.deepStrictEqual( + yield* unavailable.authorize(request("Send")).pipe(Effect.flip), + new PeerRpcError.ServerUnavailable() + ) + + const defect = new Error("policy defect") + const defective = yield* PeerRelayAuthorization.PeerRelayAuthorization.pipe( + Effect.provide(PeerRelayAuthorization.layer(() => Effect.die(defect))) + ) + const cause = yield* defective.authorize(request("Send")).pipe( + Effect.sandbox, + Effect.flip + ) + assert.isTrue(Cause.hasDies(cause)) + assert.strictEqual(Cause.squash(cause), defect) + })) +}) diff --git a/packages/local-rpc/test/PeerRelayIngress.test.ts b/packages/local-rpc/test/PeerRelayIngress.test.ts new file mode 100644 index 0000000..3f372f9 --- /dev/null +++ b/packages/local-rpc/test/PeerRelayIngress.test.ts @@ -0,0 +1,590 @@ +import { NodeSocket, NodeSocketServer } from "@effect/platform-node" +import { assert, describe, it } from "@effect/vitest" +import { Context, Deferred, Effect, Exit, Fiber, Layer, Scope } from "effect" +import { TestClock } from "effect/testing" +import type * as RpcMessage from "effect/unstable/rpc/RpcMessage" +import * as RpcServer from "effect/unstable/rpc/RpcServer" +import { Socket } from "effect/unstable/socket" +import type { SocketServer } from "effect/unstable/socket" +import * as PeerRelayIngress from "../src/PeerRelayIngress.ts" +import * as PeerRelayLimits from "../src/PeerRelayLimits.ts" + +const frame = (value: unknown) => { + const body = new TextEncoder().encode(JSON.stringify(value)) + const result = new Uint8Array(body.byteLength + 4) + new DataView(result.buffer).setUint32(0, body.byteLength, false) + result.set(body, 4) + return result +} + +const request = (id: number, padding = ""): RpcMessage.RequestEncoded => ({ + _tag: "Request", + id, + tag: "Test", + payload: { padding }, + headers: [] +}) + +const testLimits = ( + overrides: Partial = {} +): PeerRelayLimits.Values => ({ + ...PeerRelayLimits.defaults, + maxRelayConnections: 4, + maximumRawChunkBytes: 512, + maximumDeclaredFrameBytes: 4 * 1_024 * 1_024, + maximumIncompleteFrameBytes: 4 * 1_024 * 1_024 + 4, + incompleteFrameTimeoutMillis: 1_000, + maximumByteReservationWaiters: 4, + maxSessionsPerSubject: 4, + maxInFlightOpen: 4, + ...overrides +}) + +const buildServer = Effect.fnUntraced(function*( + values: PeerRelayLimits.Values, + port = 0 +) { + const scope = yield* Scope.make("sequential") + const context = yield* Layer.buildWithScope( + PeerRelayIngress.layerProtocolSocketServer(NodeSocketServer.layer({ port })).pipe( + Layer.provide(PeerRelayLimits.layer(values)) + ), + scope + ) + const ingress = Context.get(context, PeerRelayIngress.PeerRelayIngress) + const protocol = Context.get(context, RpcServer.Protocol) + return { ingress, protocol, scope } +}) + +const connect = Effect.fnUntraced(function*( + port: number, + scope: Scope.Closeable, + read = true +) { + const context = yield* Layer.buildWithScope(NodeSocket.layerNet({ port }), scope) + const socket = Context.get(context, Socket.Socket) + const readFiber = read + ? yield* Effect.forkIn(socket.runRaw(() => Effect.void), scope) + : undefined + const write = yield* Scope.provide(socket.writer, scope) + return { readFiber, socket, write } +}) + +const tcpPort = (address: SocketServer.Address) => { + assert.strictEqual(address._tag, "TcpAddress") + return (address as SocketServer.TcpAddress).port +} + +const awaitConnections = (ingress: PeerRelayIngress.PeerRelayIngress["Service"], expected: number) => + Effect.gen(function*() { + for (let attempt = 0; attempt < 10_000; attempt++) { + const usage = yield* ingress.usage + if (usage.connections === expected) return usage + yield* Effect.yieldNow + } + return yield* Effect.die(new Error("Connection count did not converge")) + }) + +const awaitReservedBytes = (ingress: PeerRelayIngress.PeerRelayIngress["Service"]) => + Effect.gen(function*() { + for (let attempt = 0; attempt < 10_000; attempt++) { + const usage = yield* ingress.usage + if (usage.reservedBytes > 0) return usage + yield* Effect.yieldNow + } + return yield* Effect.die(new Error("Byte reservation did not become observable")) + }) + +const awaitByteReservationWaiters = ( + ingress: PeerRelayIngress.PeerRelayIngress["Service"], + expected: number +) => + Effect.gen(function*() { + for (let attempt = 0; attempt < 10_000; attempt++) { + const usage = yield* ingress.usage + if (usage.byteReservationWaiters === expected) return usage + yield* Effect.yieldNow + } + return yield* Effect.die(new Error("Byte reservation waiter count did not converge")) + }) + +describe("PeerRelayIngress", () => { + it.effect("rejects an oversized declared frame before dispatch", () => + Effect.scoped(Effect.gen(function*() { + const received = Deferred.makeUnsafe() + const values = testLimits() + const server = yield* buildServer(values) + yield* Effect.forkIn( + server.protocol.run(() => Deferred.succeed(received, undefined)), + server.scope + ) + const clientScope = yield* Scope.make() + const client = yield* connect(tcpPort(server.ingress.address), clientScope) + + const header = new Uint8Array(4) + new DataView(header.buffer).setUint32(0, values.maximumDeclaredFrameBytes + 1, false) + yield* client.write(header) + const clientExit = yield* Fiber.await(client.readFiber!) + + assert.match(clientExit._tag, /^(Failure|Success)$/) + assert.strictEqual(Deferred.isDoneUnsafe(received), false) + assert.deepStrictEqual(yield* server.ingress.usage, { + connections: 0, + reservedBytes: 0, + byteReservationWaiters: 0 + }) + yield* Scope.close(clientScope, Exit.void) + yield* Scope.close(server.scope, Exit.void) + }))) + + it.effect("rejects an oversized raw chunk before JSON decode", () => + Effect.scoped(Effect.gen(function*() { + const received = Deferred.makeUnsafe() + const server = yield* buildServer(testLimits({ + maximumRawChunkBytes: 8 + })) + yield* Effect.forkIn( + server.protocol.run(() => Deferred.succeed(received, undefined)), + server.scope + ) + const clientScope = yield* Scope.make() + const client = yield* connect(tcpPort(server.ingress.address), clientScope) + + yield* client.write(frame(request(1))) + const clientExit = yield* Fiber.await(client.readFiber!) + + assert.match(clientExit._tag, /^(Failure|Success)$/) + assert.strictEqual(Deferred.isDoneUnsafe(received), false) + assert.strictEqual((yield* server.ingress.usage).reservedBytes, 0) + yield* Scope.close(clientScope, Exit.void) + yield* Scope.close(server.scope, Exit.void) + }))) + + it.effect("accepts a frame fragmented across every header and body byte", () => + Effect.scoped(Effect.gen(function*() { + const received = Deferred.makeUnsafe() + const server = yield* buildServer(testLimits()) + yield* Effect.forkIn( + server.protocol.run((clientId, message) => Deferred.succeed(received, [clientId, message] as const)), + server.scope + ) + const clientScope = yield* Scope.make() + const client = yield* connect(tcpPort(server.ingress.address), clientScope) + const encoded = frame(request(7, "fragmented")) + + for (const byte of encoded) { + yield* client.write(Uint8Array.of(byte)) + } + const [clientId, message] = yield* Deferred.await(received) + + assert.strictEqual(clientId, 0) + assert.strictEqual(message._tag, "Request") + assert.strictEqual( + message._tag === "Request" ? message.id : undefined, + 7 + ) + yield* Scope.close(clientScope, Exit.void) + yield* Scope.close(server.scope, Exit.void) + assert.strictEqual((yield* server.ingress.usage).reservedBytes, 0) + }))) + + it.effect("times out a slow incomplete frame and releases its connection", () => + Effect.scoped(Effect.gen(function*() { + const server = yield* buildServer(testLimits({ incompleteFrameTimeoutMillis: 500 })) + yield* Effect.forkIn(server.protocol.run(() => Effect.void), server.scope) + const clientScope = yield* Scope.make() + const client = yield* connect(tcpPort(server.ingress.address), clientScope) + + const partial = new Uint8Array(5) + new DataView(partial.buffer).setUint32(0, 10, false) + partial[4] = 123 + yield* client.write(partial) + yield* awaitConnections(server.ingress, 1) + yield* awaitReservedBytes(server.ingress) + yield* TestClock.adjust(500) + const clientExit = yield* Fiber.await(client.readFiber!) + + assert.match(clientExit._tag, /^(Failure|Success)$/) + assert.deepStrictEqual(yield* server.ingress.usage, { + connections: 0, + reservedBytes: 0, + byteReservationWaiters: 0 + }) + yield* Scope.close(clientScope, Exit.void) + yield* Scope.close(server.scope, Exit.void) + }))) + + it.effect("decodes many maximum sized fragmented frames without exceeding the shared budget", () => + Effect.scoped(Effect.gen(function*() { + const count = 8 + const maximumFrameBytes = 4 * 1_024 * 1_024 + const received = yield* Deferred.make() + let seen = 0 + const server = yield* buildServer(testLimits({ + maximumDeclaredFrameBytes: maximumFrameBytes, + maximumRawChunkBytes: maximumFrameBytes + 4, + maximumIncompleteFrameBytes: maximumFrameBytes + 4 + })) + yield* Effect.forkIn( + server.protocol.run(() => + Effect.sync(() => { + seen++ + if (seen === count) Deferred.doneUnsafe(received, Effect.succeed(seen)) + }) + ), + server.scope + ) + const clientScope = yield* Scope.make() + const client = yield* connect(tcpPort(server.ingress.address), clientScope) + + for (let id = 0; id < count; id++) { + const empty = frame(request(id)) + const encoded = frame(request(id, "x".repeat(maximumFrameBytes + 4 - empty.byteLength))) + assert.strictEqual(encoded.byteLength, maximumFrameBytes + 4) + yield* client.write(encoded.subarray(0, 4)) + yield* client.write(encoded.subarray(4)) + } + assert.strictEqual(yield* Deferred.await(received), count) + const usage = yield* server.ingress.usage + assert.isAtMost(usage.reservedBytes, PeerRelayLimits.defaults.maximumSharedPayloadBytes) + assert.strictEqual(usage.byteReservationWaiters, 0) + + yield* Scope.close(clientScope, Exit.void) + yield* Scope.close(server.scope, Exit.void) + assert.strictEqual((yield* server.ingress.usage).reservedBytes, 0) + }))) + + it.effect("caps connections and closes saturated clients", () => + Effect.scoped(Effect.gen(function*() { + const values = testLimits({ + maxRelayConnections: 1, + maximumByteReservationWaiters: 1, + maxSessionsPerSubject: 1, + maxInFlightOpen: 1, + maxInFlightOpenPerSubject: 1 + }) + const server = yield* buildServer(values) + yield* Effect.forkIn(server.protocol.run(() => Effect.void), server.scope) + const firstScope = yield* Scope.make() + const first = yield* connect(tcpPort(server.ingress.address), firstScope) + yield* first.write(Uint8Array.of(0)) + yield* awaitConnections(server.ingress, 1) + + const secondScope = yield* Scope.make() + const second = yield* connect(tcpPort(server.ingress.address), secondScope) + const secondExit = yield* Fiber.await(second.readFiber!) + + assert.match(secondExit._tag, /^(Failure|Success)$/) + assert.strictEqual((yield* server.ingress.usage).connections, 1) + yield* Scope.close(secondScope, Exit.void) + yield* Scope.close(firstScope, Exit.void) + yield* Scope.close(server.scope, Exit.void) + }))) + + it.effect("bounds stalled outbound reservations and releases every waiter on cleanup", () => + Effect.scoped(Effect.gen(function*() { + const server = yield* buildServer(testLimits()) + const reservation = yield* server.ingress.reserveOutbound( + PeerRelayLimits.defaults.maximumSharedPayloadBytes + ) + const waiting = yield* Effect.forkChild( + server.ingress.reserveOutbound(1) + ) + yield* Effect.yieldNow + + assert.deepStrictEqual(yield* server.ingress.usage, { + connections: 0, + reservedBytes: PeerRelayLimits.defaults.maximumSharedPayloadBytes, + byteReservationWaiters: 1 + }) + yield* Fiber.interrupt(waiting) + yield* reservation.release + assert.deepStrictEqual(yield* server.ingress.usage, { + connections: 0, + reservedBytes: 0, + byteReservationWaiters: 0 + }) + yield* Scope.close(server.scope, Exit.void) + }))) + + it.effect("reserves outbound capacity before serializing a response", () => + Effect.scoped(Effect.gen(function*() { + const values = testLimits() + const server = yield* buildServer(values) + yield* Effect.forkIn(server.protocol.run(() => Effect.void), server.scope) + const clientScope = yield* Scope.make() + yield* connect(tcpPort(server.ingress.address), clientScope) + yield* awaitConnections(server.ingress, 1) + const [clientId] = yield* server.protocol.clientIds + assert.isDefined(clientId) + const blocker = yield* server.ingress.reserveOutbound( + values.maximumSharedPayloadBytes + ) + const serialized = Deferred.makeUnsafe() + const response: RpcMessage.FromServerEncoded = { + _tag: "Defect", + defect: { + toJSON() { + Deferred.doneUnsafe(serialized, Effect.void) + return "serialized" + } + } + } + + const send = yield* Effect.forkChild(server.protocol.send(clientId!, response)) + yield* awaitByteReservationWaiters(server.ingress, 1) + + assert.strictEqual(Deferred.isDoneUnsafe(serialized), false) + yield* Fiber.interrupt(send) + yield* blocker.release + assert.deepStrictEqual(yield* server.ingress.usage, { + connections: 1, + reservedBytes: 0, + byteReservationWaiters: 0 + }) + yield* Scope.close(clientScope, Exit.void) + yield* Scope.close(server.scope, Exit.void) + }))) + + it.effect("rejects overlapping raw chunks instead of accumulating parser fibers", () => + Effect.scoped(Effect.gen(function*() { + const values = testLimits() + const server = yield* buildServer(values) + yield* Effect.forkIn(server.protocol.run(() => Effect.void), server.scope) + const blocker = yield* server.ingress.reserveOutbound( + values.maximumSharedPayloadBytes + ) + const clientScope = yield* Scope.make() + const client = yield* connect(tcpPort(server.ingress.address), clientScope) + const header = new Uint8Array(4) + new DataView(header.buffer).setUint32(0, 10, false) + + yield* client.write(header) + yield* awaitByteReservationWaiters(server.ingress, 1) + yield* client.write(Uint8Array.of(123)).pipe(Effect.ignore) + yield* Fiber.await(client.readFiber!) + yield* awaitConnections(server.ingress, 0) + + assert.deepStrictEqual(yield* server.ingress.usage, { + connections: 0, + reservedBytes: values.maximumSharedPayloadBytes, + byteReservationWaiters: 0 + }) + yield* blocker.release + assert.deepStrictEqual(yield* server.ingress.usage, { + connections: 0, + reservedBytes: 0, + byteReservationWaiters: 0 + }) + yield* Scope.close(clientScope, Exit.void) + yield* Scope.close(server.scope, Exit.void) + }))) + + it.effect("releases a transferred reservation when an outbound write wait is interrupted", () => + Effect.scoped(Effect.gen(function*() { + const values = testLimits() + const server = yield* buildServer(values) + const ready = Deferred.makeUnsafe<{ + readonly blocker: PeerRelayIngress.Reservation + readonly clientId: number + }>() + const input = request(1) + const inboundBytes = frame(input).byteLength - 4 + yield* Effect.forkIn( + server.protocol.run((clientId) => + Effect.gen(function*() { + const transferred = yield* server.ingress.reserveOutbound(1) + yield* transferred.transferToCurrentRequest + const blocker = yield* server.ingress.reserveOutbound( + values.maximumSharedPayloadBytes - inboundBytes - 1 + ) + yield* Deferred.succeed(ready, { blocker, clientId }) + }) + ), + server.scope + ) + const clientScope = yield* Scope.make() + const client = yield* connect(tcpPort(server.ingress.address), clientScope) + yield* client.write(frame(input)) + const { blocker, clientId } = yield* Deferred.await(ready) + + const send = yield* Effect.forkChild( + server.protocol.send(clientId, { + _tag: "Chunk", + requestId: 1, + values: ["requires-more-than-one-byte"] + }) + ) + yield* Effect.yieldNow + assert.strictEqual( + (yield* server.ingress.usage).reservedBytes, + values.maximumSharedPayloadBytes + ) + yield* Fiber.interrupt(send) + yield* blocker.release + yield* Scope.close(clientScope, Exit.void) + yield* awaitConnections(server.ingress, 0) + assert.deepStrictEqual(yield* server.ingress.usage, { + connections: 0, + reservedBytes: 0, + byteReservationWaiters: 0 + }) + yield* Scope.close(server.scope, Exit.void) + }))) + + it.effect("rejects reservation transfer after client disconnect cleanup", () => + Effect.scoped(Effect.gen(function*() { + const server = yield* buildServer(testLimits()) + const ready = Deferred.makeUnsafe() + const transfer = Deferred.makeUnsafe() + const transferSucceeded = Deferred.makeUnsafe() + yield* Effect.forkIn( + server.protocol.run((clientId) => + Effect.gen(function*() { + const reservation = yield* server.ingress.reserveOutbound(1) + yield* Deferred.succeed(ready, clientId) + yield* Effect.uninterruptible( + Effect.gen(function*() { + yield* Deferred.await(transfer) + const succeeded = yield* reservation.transferToCurrentRequest.pipe( + Effect.match({ + onFailure: () => false, + onSuccess: () => true + }) + ) + if (!succeeded) yield* reservation.release + yield* Deferred.succeed(transferSucceeded, succeeded) + }) + ) + }) + ), + server.scope + ) + const clientScope = yield* Scope.make() + const client = yield* connect(tcpPort(server.ingress.address), clientScope) + yield* client.write(frame(request(1))) + const clientId = yield* Deferred.await(ready) + + yield* server.protocol.end(clientId) + yield* Deferred.succeed(transfer, undefined) + assert.strictEqual(yield* Deferred.await(transferSucceeded), false) + yield* Fiber.await(client.readFiber!) + yield* awaitConnections(server.ingress, 0) + + assert.deepStrictEqual(yield* server.ingress.usage, { + connections: 0, + reservedBytes: 0, + byteReservationWaiters: 0 + }) + yield* Scope.close(clientScope, Exit.void) + yield* Scope.close(server.scope, Exit.void) + }))) + + it.effect("rejects pre-run connection churn without filling the disconnect queue", () => + Effect.scoped(Effect.gen(function*() { + const values = testLimits({ + maxRelayConnections: 1, + maximumByteReservationWaiters: 1, + maxSessionsPerSubject: 1, + maxInFlightOpen: 1, + maxInFlightOpenPerSubject: 1 + }) + const server = yield* buildServer(values) + for (let attempt = 0; attempt < 8; attempt++) { + const rejectedScope = yield* Scope.make() + const rejected = yield* connect(tcpPort(server.ingress.address), rejectedScope) + yield* Fiber.await(rejected.readFiber!) + yield* Scope.close(rejectedScope, Exit.void) + } + assert.strictEqual((yield* server.ingress.usage).connections, 0) + + yield* Effect.forkIn(server.protocol.run(() => Effect.void), server.scope) + const acceptedScope = yield* Scope.make() + const accepted = yield* connect(tcpPort(server.ingress.address), acceptedScope) + yield* accepted.write(Uint8Array.of(0)) + yield* awaitConnections(server.ingress, 1) + yield* Scope.close(acceptedScope, Exit.void) + yield* Scope.close(server.scope, Exit.void) + }))) + + it.effect("reports normal client EOF and fails later sends promptly", () => + Effect.scoped(Effect.gen(function*() { + const values = testLimits() + const server = yield* buildServer(values) + yield* Effect.forkIn(server.protocol.run(() => Effect.void), server.scope) + const clientScope = yield* Scope.make() + const socketContext = yield* Layer.buildWithScope( + NodeSocket.layerNet({ port: tcpPort(server.ingress.address) }), + clientScope + ) + const socket = Context.get(socketContext, Socket.Socket) + const clientProtocol = yield* Scope.provide( + PeerRelayIngress.makeProtocolSocket.pipe( + Effect.provideService(Socket.Socket, socket), + Effect.provideService(PeerRelayLimits.PeerRelayLimits, values) + ), + clientScope + ) + const protocolFailure = Deferred.makeUnsafe() + yield* Effect.forkIn( + clientProtocol.run(0, (message) => + message._tag === "ClientProtocolError" + ? Deferred.succeed(protocolFailure, message) + : Effect.void), + clientScope + ) + yield* awaitConnections(server.ingress, 1) + yield* Scope.close(server.scope, Exit.void) + + const failure = yield* Deferred.await(protocolFailure) + assert.strictEqual(failure._tag, "ClientProtocolError") + const sendExit = yield* Effect.exit( + clientProtocol.send(0, { _tag: "Ping" }) + ) + assert.strictEqual(sendExit._tag, "Failure") + yield* Scope.close(clientScope, Exit.void) + }))) + + it.effect("closes a partially built socket layer and can bind the same address again", () => + Effect.scoped(Effect.gen(function*() { + const probeScope = yield* Scope.make() + const probe = yield* Scope.provide(NodeSocketServer.make({ port: 0 }), probeScope) + const port = tcpPort(probe.address) + yield* Scope.close(probeScope, Exit.void) + + const partial = Layer.effectContext( + Effect.gen(function*() { + yield* NodeSocketServer.make({ port }) + return yield* Effect.fail("partial-build") + }) + ) as Layer.Layer + const failedScope = yield* Scope.make() + const failed = yield* Effect.exit( + Layer.buildWithScope( + PeerRelayIngress.layerProtocolSocketServer(partial).pipe( + Layer.provide(PeerRelayLimits.layer(testLimits())) + ), + failedScope + ) + ) + assert.strictEqual(failed._tag, "Failure") + yield* Scope.close(failedScope, failed) + + const restarted = yield* buildServer(testLimits(), port) + assert.strictEqual(tcpPort(restarted.ingress.address), port) + yield* Scope.close(restarted.scope, Exit.void) + }))) + + it.effect("closes the child listener scope and restarts on the retained port", () => + Effect.scoped(Effect.gen(function*() { + const first = yield* buildServer(testLimits()) + const port = tcpPort(first.ingress.address) + yield* Scope.close(first.scope, Exit.void) + const firstExit = yield* Effect.exit(first.ingress.await) + assert.strictEqual(firstExit._tag, "Failure") + + const second = yield* buildServer(testLimits(), port) + assert.strictEqual(tcpPort(second.ingress.address), port) + yield* Scope.close(second.scope, Exit.void) + }))) +}) diff --git a/packages/local-rpc/test/PeerRelayLimits.test.ts b/packages/local-rpc/test/PeerRelayLimits.test.ts new file mode 100644 index 0000000..acb7a39 --- /dev/null +++ b/packages/local-rpc/test/PeerRelayLimits.test.ts @@ -0,0 +1,168 @@ +import { assert, describe, it } from "@effect/vitest" +import * as Effect from "effect/Effect" +import * as Layer from "effect/Layer" +import * as PeerRelayLimits from "../src/PeerRelayLimits.js" +import * as PeerRelayRpc from "../src/PeerRelayRpc.js" + +describe("PeerRelayLimits", () => { + it.effect("publishes the complete validated production defaults through its Layer", () => + Effect.gen(function*() { + const limits = yield* PeerRelayLimits.PeerRelayLimits + assert.deepStrictEqual(limits, PeerRelayLimits.defaults) + assert.strictEqual(Object.keys(limits).length, 93) + }).pipe(Effect.provide(PeerRelayLimits.layerDefaults))) + + it.effect.each( + [ + ["maxActiveMessagesPerShard", 0, "Expected a value greater than 0, got 0"], + ["maxRetainedBytesPerTenant", -1, "Expected a value greater than 0, got -1"], + ["claimLeaseMillis", 1.5, "Expected an integer, got 1.5"], + ["openRatePerSecond", Number.NaN, "Expected a finite number, got NaN"], + [ + "terminalResponseRatePerSecond", + Number.POSITIVE_INFINITY, + "Expected a finite number, got Infinity" + ], + [ + "maximumReceiptRetentionMillis", + PeerRelayRpc.maximumNegotiatedDurationMillis + 1, + `Expected a value less than or equal to ${PeerRelayRpc.maximumNegotiatedDurationMillis}, got ${ + PeerRelayRpc.maximumNegotiatedDurationMillis + 1 + }` + ], + ["sqliteCapacityHeadroomPercent", 101, "Expected a value less than or equal to 100, got 101"], + ["shutdownReleaseConcurrency", Number.MAX_SAFE_INTEGER + 1, "Expected an integer"] + ] as const + )("rejects scalar %s with the stable configuration error", ([field, value]) => + Effect.gen(function*() { + const error = yield* PeerRelayLimits.make({ + ...PeerRelayLimits.defaults, + [field]: value + }).pipe(Effect.flip) + assert.strictEqual(error._tag, "InvalidPeerRelayLimits") + if (error._tag === "InvalidPeerRelayLimits") assert.strictEqual(error.field, field) + })) + + const crossFieldCases = [ + ["maxActiveMessagesPerRecipientSubject", { maxActiveMessagesPerRecipientSubject: 9_999 }], + ["maxActiveBytesPerRecipientSubject", { maxActiveBytesPerRecipientSubject: 1 }], + ["maxActiveMessagesPerTenant", { maxActiveMessagesPerTenant: 9_999 }], + ["maxActiveBytesPerTenant", { maxActiveBytesPerTenant: 1 }], + ["maxActiveMessagesPerShard", { maxActiveMessagesPerShard: 99_999 }], + ["maxActiveBytesPerShard", { maxActiveBytesPerShard: 1 }], + ["maxRetainedRowsPerRecipientSubject", { maxRetainedRowsPerRecipientSubject: 9_999 }], + ["maxRetainedBytesPerRecipientSubject", { maxRetainedBytesPerRecipientSubject: 1 }], + ["maxRetainedRowsPerTenant", { maxRetainedRowsPerTenant: 9_999 }], + ["maxRetainedBytesPerTenant", { maxRetainedBytesPerTenant: 1 }], + ["maxRetainedRowsPerShard", { maxRetainedRowsPerShard: 99_999 }], + ["maxRetainedBytesPerShard", { maxRetainedBytesPerShard: 1 }], + [ + "maximumReceiptRetentionMillis", + { maximumReceiptRetentionMillis: PeerRelayLimits.defaults.maximumReceiptRetentionMillis - 1 } + ], + ["claimLeaseMillis", { maximumRecipientProcessingMillis: 45_000 }], + ["claimLeaseMillis", { claimLeaseMillis: PeerRelayLimits.defaults.messageTtlMillis }], + ["retryMaximumDelayMillis", { retryMaximumDelayMillis: 249 }], + ["retryMaximumDelayMillis", { retryMaximumDelayMillis: 60_000 }], + ["sqliteLockRetryMaximumDelayMillis", { sqliteLockRetryMaximumDelayMillis: 9 }], + [ + "maximumRawChunkBytes", + { + maximumRawChunkBytes: PeerRelayLimits.defaults.maximumIncompleteFrameBytes + 1 + } + ], + [ + "maximumDeclaredFrameBytes", + { + maximumDeclaredFrameBytes: PeerRelayLimits.defaults.maximumIncompleteFrameBytes - 3 + } + ], + [ + "maximumDeclaredFrameBytes", + { maximumDeclaredFrameBytes: PeerRelayRpc.maximumRelayPayloadBytes - 1 } + ], + [ + "maximumSharedPayloadBytes", + { + maximumSharedPayloadBytes: PeerRelayLimits.defaults.maximumIncompleteFrameBytes - 1 + } + ], + ["maximumSharedPayloadBytes", { relayWorkerConcurrency: 17 }], + ["maximumByteReservationWaiters", { maximumByteReservationWaiters: 1_023 }], + ["maxSessionsPerSubject", { maxSessionsPerSubject: 1_025 }], + ["maxInFlightOpen", { maxInFlightOpen: 1_025, openBurst: 1_025 }], + ["maxInFlightOpenPerSubject", { maxInFlightOpenPerSubject: 33 }], + ["openBurst", { openBurst: 31 }], + ["maxInFlightPushPerSubject", { maxInFlightPushPerSubject: 129 }], + ["admissionBurst", { admissionBurst: 127 }], + ["maxRetainedRateLimitedConnections", { maxRetainedRateLimitedConnections: 1_023 }], + ["maxRetainedRateLimitedSubjects", { maxRetainedRateLimitedSubjects: 31 }], + [ + "maxInFlightTerminalResponsesPerSubject", + { maxInFlightTerminalResponsesPerSubject: 65 } + ], + ["terminalResponseQueueCapacity", { terminalResponseQueueCapacity: 63 }], + ["terminalResponseBurst", { terminalResponseBurst: 63 }], + ["terminalResponseRatePerSecond", { terminalResponseRatePerSecond: 99 }], + ["maxRetainedTerminalResponseSubjects", { maxRetainedTerminalResponseSubjects: 63 }], + ["compensationBatchSize", { compensationBatchSize: 10_001 }], + ["sqlAdmissionQueueCapacity", { sqlAdmissionQueueCapacity: 3 }], + ["sqlTerminalQueueCapacity", { sqlTerminalQueueCapacity: 3 }], + ["sqlDeliveryQueueCapacity", { sqlDeliveryQueueCapacity: 3 }], + ["sqlMaintenanceQueueCapacity", { sqlMaintenanceQueueCapacity: 3 }], + ["maxInFlightSqlTransactions", { maxInFlightSqlTransactions: 15 }], + ["claimRecoveryRowsPerSecond", { claimRecoveryRowsPerSecond: 99 }], + ["expiryRowsPerSecond", { expiryRowsPerSecond: 99 }], + ["integrityRowsPerSecond", { integrityRowsPerSecond: 99 }], + ["reconciliationRowsPerSecond", { reconciliationRowsPerSecond: 99 }], + ["terminalCollectionRowsPerSecond", { terminalCollectionRowsPerSecond: 99 }], + ["orphanChannelCleanupRowsPerSecond", { orphanChannelCleanupRowsPerSecond: 99 }], + ["shutdownReleaseConcurrency", { shutdownReleaseConcurrency: 17 }], + ["shutdownReleaseTimeoutMillis", { shutdownReleaseTimeoutMillis: 60_000 }], + ["sqliteTransactionCapacityPerSecond", { sqliteTransactionCapacityPerSecond: 127 }] + ] as const satisfies ReadonlyArray< + readonly [keyof PeerRelayLimits.Values, Partial] + > + + it.effect.each(crossFieldCases)( + "rejects the %s cross field constraint through the production Layer", + ([field, patch]) => + Effect.gen(function*() { + const error = yield* Layer.build( + PeerRelayLimits.layer({ + ...PeerRelayLimits.defaults, + ...patch + }) + ).pipe(Effect.scoped, Effect.flip) + assert.strictEqual(error._tag, "InvalidPeerRelayLimits") + if (error._tag === "InvalidPeerRelayLimits") assert.strictEqual(error.field, field) + }) + ) + + it.effect("accepts an exact aggregate SQLite capacity boundary with configured headroom", () => + Effect.gen(function*() { + const required = PeerRelayLimits.defaults.admissionRatePerSecond + + PeerRelayLimits.defaults.claimRecoveryRowsPerSecond / + PeerRelayLimits.defaults.claimRecoveryBatchSize + + PeerRelayLimits.defaults.expiryRowsPerSecond / + PeerRelayLimits.defaults.expiryBatchSize + + PeerRelayLimits.defaults.integrityRowsPerSecond / + PeerRelayLimits.defaults.integrityBatchSize + + PeerRelayLimits.defaults.reconciliationRowsPerSecond / + PeerRelayLimits.defaults.reconciliationBatchSize + + PeerRelayLimits.defaults.terminalCollectionRowsPerSecond / + PeerRelayLimits.defaults.terminalCollectionBatchSize + + PeerRelayLimits.defaults.orphanChannelCleanupRowsPerSecond / + PeerRelayLimits.defaults.orphanChannelCleanupBatchSize + const exact = required * (1 + PeerRelayLimits.defaults.sqliteCapacityHeadroomPercent / 100) + const limits = yield* PeerRelayLimits.PeerRelayLimits + assert.isAtLeast(limits.sqliteTransactionCapacityPerSecond, exact) + }).pipe( + Effect.provide( + PeerRelayLimits.layer({ + ...PeerRelayLimits.defaults, + sqliteTransactionCapacityPerSecond: 127.2 + }) + ) + )) +}) diff --git a/packages/local-rpc/test/PeerRelayRpc.test.ts b/packages/local-rpc/test/PeerRelayRpc.test.ts new file mode 100644 index 0000000..dc41bd7 --- /dev/null +++ b/packages/local-rpc/test/PeerRelayRpc.test.ts @@ -0,0 +1,191 @@ +import { assert, describe, it } from "@effect/vitest" +import * as Identity from "@lucas-barake/effect-local/Identity" +import * as Schema from "effect/Schema" +import * as PeerRelayRpc from "../src/PeerRelayRpc.js" +import * as PeerRpc from "../src/PeerRpc.js" + +const localPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000001") +const remotePeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000002") +const relayPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000003") +const sessionId = Identity.SessionId.make("ses_00000000-0000-4000-8000-000000000001") +const relayMessageId = Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000001") +const documentId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000001") +const claimToken = PeerRelayRpc.ClaimToken.make("clm_00000000-0000-4000-8000-000000000001") +const hash = "0".repeat(64) + +describe("PeerRelayRpc", () => { + it("roundtrips the strict version 3 handshake", () => { + const opened = PeerRelayRpc.RelayOpened.make({ + _tag: "RelayOpened", + version: PeerRelayRpc.protocolVersion, + sessionId, + remotePeerId, + authenticatedLocal: { + tenantId: "tenant", + subjectId: "subject", + peerId: localPeerId + }, + capabilities: { storeAndForward: true } + }) + + assert.deepStrictEqual(Schema.decodeUnknownSync(PeerRelayRpc.OpenRelayEvent)(opened), opened) + assert.throws(() => Schema.decodeUnknownSync(PeerRpc.OpenEvent)(opened)) + }) + + it("roundtrips every relay request and stored message field", () => { + const open = PeerRelayRpc.OpenRelayRpc.payloadSchema.make({ + version: PeerRelayRpc.protocolVersion, + expectedRelayPeerId: relayPeerId, + expectedLocal: { + tenantId: "tenant", + subjectId: "subject", + peerId: localPeerId + }, + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + remote: { + subjectId: "remote", + peerId: remotePeerId + }, + documents: [{ documentType: "Task", documentId }], + receiptRetentionMillis: 8 * 24 * 60 * 60 * 1_000, + senderRetryHorizonMillis: 7 * 24 * 60 * 60 * 1_000 + }) + const stored = PeerRelayRpc.StoredMessage.make({ + _tag: "StoredMessage", + relayMessageId, + claimToken, + relayPeerId, + sender: { + tenantId: "tenant", + subjectId: "subject", + peerId: localPeerId, + replicaIncarnation: Identity.ReplicaIncarnation.make(1), + connectionEpoch: "epoch", + sequence: 0 + }, + recipient: { + tenantId: "tenant", + subjectId: "remote", + peerId: remotePeerId + }, + payloadVersion: 1, + document: { documentType: "Task", documentId }, + writerProvenance: [], + messageHash: hash, + outerEnvelopeDigest: hash, + payload: Uint8Array.of(1, 2, 3) + }) + const push = PeerRelayRpc.PushRelayRpc.payloadSchema.make({ + sessionId, + relayMessageId, + payload: Uint8Array.of(1, 2, 3) + }) + const acknowledge = PeerRelayRpc.AcknowledgeRelayRpc.payloadSchema.make({ + sessionId, + relayMessageId, + claimToken, + messageHash: hash + }) + const reject = PeerRelayRpc.RejectRelayRpc.payloadSchema.make({ + ...acknowledge, + reason: "ProtocolInvalid" + }) + + assert.deepStrictEqual(Schema.decodeUnknownSync(PeerRelayRpc.OpenRelayRpc.payloadSchema)(open), open) + assert.deepStrictEqual(Schema.decodeUnknownSync(PeerRelayRpc.OpenRelayEvent)(stored), stored) + assert.deepStrictEqual(Schema.decodeUnknownSync(PeerRelayRpc.PushRelayRpc.payloadSchema)(push), push) + assert.deepStrictEqual( + Schema.decodeUnknownSync(PeerRelayRpc.AcknowledgeRelayRpc.payloadSchema)(acknowledge), + acknowledge + ) + assert.deepStrictEqual(Schema.decodeUnknownSync(PeerRelayRpc.RejectRelayRpc.payloadSchema)(reject), reject) + }) + + it("rejects invalid durations, empty document sets, identifiers, hashes, and oversized payloads", () => { + const baseOpen = { + version: PeerRelayRpc.protocolVersion, + expectedRelayPeerId: relayPeerId, + expectedLocal: { + tenantId: "tenant", + subjectId: "subject", + peerId: localPeerId + }, + senderReplicaIncarnation: 1, + remote: { + subjectId: "remote", + peerId: remotePeerId + }, + documents: [{ documentType: "Task", documentId }], + receiptRetentionMillis: 1, + senderRetryHorizonMillis: 1 + } + + for (const duration of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) { + assert.throws(() => + Schema.decodeUnknownSync(PeerRelayRpc.OpenRelayRpc.payloadSchema)({ + ...baseOpen, + receiptRetentionMillis: duration + }) + ) + } + assert.throws(() => + Schema.decodeUnknownSync(PeerRelayRpc.OpenRelayRpc.payloadSchema)({ ...baseOpen, documents: [] }) + ) + assert.throws(() => + Schema.decodeUnknownSync(PeerRelayRpc.AcknowledgeRelayRpc.payloadSchema)({ + sessionId, + relayMessageId: "rly_invalid", + claimToken, + messageHash: hash + }) + ) + assert.throws(() => + Schema.decodeUnknownSync(PeerRelayRpc.AcknowledgeRelayRpc.payloadSchema)({ + sessionId, + relayMessageId, + claimToken, + messageHash: "not-a-hash" + }) + ) + assert.throws(() => + Schema.decodeUnknownSync(PeerRelayRpc.PushRelayRpc.payloadSchema)({ + sessionId, + relayMessageId, + payload: new Uint8Array(PeerRelayRpc.maximumRelayPayloadBytes + 1) + }) + ) + assert.throws(() => + Schema.decodeUnknownSync(PeerRelayRpc.OpenRelayEvent)( + PeerRelayRpc.StoredMessage.make({ + _tag: "StoredMessage", + relayMessageId, + claimToken, + relayPeerId, + sender: { + tenantId: "tenant", + subjectId: "subject", + peerId: localPeerId, + replicaIncarnation: Identity.ReplicaIncarnation.make(1), + connectionEpoch: "epoch", + sequence: 0 + }, + recipient: { + tenantId: "tenant", + subjectId: "remote", + peerId: remotePeerId + }, + payloadVersion: 1, + document: { documentType: "Task", documentId }, + writerProvenance: [{ + changeHash: hash, + writerSchemaVersion: 1, + writerDefinitionHash: "invalid\"definition" + }], + messageHash: hash, + outerEnvelopeDigest: hash, + payload: Uint8Array.of(1) + }) + ) + ) + }) +}) diff --git a/packages/local-rpc/test/PeerRelaySqliteTransaction.test.ts b/packages/local-rpc/test/PeerRelaySqliteTransaction.test.ts new file mode 100644 index 0000000..441d194 --- /dev/null +++ b/packages/local-rpc/test/PeerRelaySqliteTransaction.test.ts @@ -0,0 +1,165 @@ +import { assert, describe, it } from "@effect/vitest" +import * as Cause from "effect/Cause" +import * as Effect from "effect/Effect" +import * as Exit from "effect/Exit" +import * as Result from "effect/Result" +import * as Scope from "effect/Scope" +import * as SqlClient from "effect/unstable/sql/SqlClient" +import type * as SqlConnection from "effect/unstable/sql/SqlConnection" +import * as SqlError from "effect/unstable/sql/SqlError" +import { cleanupAcquisition, make } from "../src/internal/peerRelaySqliteTransaction.js" + +describe("peerRelaySqliteTransaction", () => { + it.effect("preserves interruption together with rollback failure during acquisition", () => + Effect.gen(function*() { + const rollbackDefect = new Error("rollback failed") + const connection = { + executeUnprepared: (statement: string) => { + if (statement === "ROLLBACK") { + return Effect.die(rollbackDefect) + } + return Effect.succeed([]) + } + } as unknown as SqlConnection.Connection + const scope = yield* Scope.make("sequential") + const exit = yield* cleanupAcquisition( + Cause.interrupt(99_023), + scope, + connection, + true + ).pipe(Effect.exit) + assert.strictEqual(Exit.isFailure(exit), true) + if (Exit.isFailure(exit)) { + assert.strictEqual(Cause.hasInterrupts(exit.cause), true) + assert.strictEqual(Result.getOrThrow(Cause.findDefect(exit.cause)), rollbackDefect) + } + })) + + it.effect("does not retry LockTimeoutError when Scope release also defects", () => + Effect.gen(function*() { + const closeDefect = new Error("scope close failed") + const lock = new SqlError.SqlError({ + reason: new SqlError.LockTimeoutError({ cause: new Error("busy") }) + }) + let begins = 0 + const connection = { + executeUnprepared: (statement: string) => { + if (statement === "BEGIN IMMEDIATE") { + begins++ + return Effect.fail(lock) + } + return Effect.succeed([]) + } + } as unknown as SqlConnection.Connection + const reserve = Effect.gen(function*() { + const scope = yield* Scope.Scope + yield* Scope.addFinalizer(scope, Effect.die(closeDefect)) + return connection + }) + const sql = { + reserve, + transactionService: SqlClient.TransactionConnection(999_024) + } as unknown as SqlClient.SqlClient + const exit = yield* make(sql, { + maxAcquireAttempts: 2, + acquireRetryBaseDelayMillis: 0, + acquireRetryMaximumDelayMillis: 0 + })(Effect.void).pipe(Effect.exit) + assert.strictEqual(begins, 1) + assert.strictEqual(Exit.isFailure(exit), true) + if (Exit.isFailure(exit)) { + assert.strictEqual(Result.getOrThrow(Cause.findDefect(exit.cause)), closeDefect) + const failure = Result.getOrThrow(Cause.findError(exit.cause)) + assert.strictEqual(failure, lock) + } + })) + + it.effect("preserves a body failure together with rollback failure", () => + Effect.gen(function*() { + const bodyFailure = new Error("body failed") + const rollbackDefect = new Error("rollback failed") + const connection = { + executeUnprepared: (statement: string) => + statement === "ROLLBACK" + ? Effect.die(rollbackDefect) + : Effect.succeed([]) + } as unknown as SqlConnection.Connection + const sql = { + reserve: Effect.succeed(connection), + transactionService: SqlClient.TransactionConnection(999_025) + } as unknown as SqlClient.SqlClient + const exit = yield* make(sql, { + maxAcquireAttempts: 1, + acquireRetryBaseDelayMillis: 0, + acquireRetryMaximumDelayMillis: 0 + })(Effect.fail(bodyFailure)).pipe(Effect.exit) + assert.strictEqual(Exit.isFailure(exit), true) + if (Exit.isFailure(exit)) { + assert.strictEqual(Result.getOrThrow(Cause.findError(exit.cause)), bodyFailure) + assert.strictEqual(Result.getOrThrow(Cause.findDefect(exit.cause)), rollbackDefect) + } + })) + + it.effect("preserves a body interruption together with rollback failure", () => + Effect.gen(function*() { + const rollbackDefect = new Error("rollback failed") + const connection = { + executeUnprepared: (statement: string) => + statement === "ROLLBACK" + ? Effect.die(rollbackDefect) + : Effect.succeed([]) + } as unknown as SqlConnection.Connection + const sql = { + reserve: Effect.succeed(connection), + transactionService: SqlClient.TransactionConnection(999_026) + } as unknown as SqlClient.SqlClient + const exit = yield* make(sql, { + maxAcquireAttempts: 1, + acquireRetryBaseDelayMillis: 0, + acquireRetryMaximumDelayMillis: 0 + })(Effect.interrupt).pipe(Effect.exit) + assert.strictEqual(Exit.isFailure(exit), true) + if (Exit.isFailure(exit)) { + assert.strictEqual(Cause.hasInterrupts(exit.cause), true) + assert.strictEqual(Result.getOrThrow(Cause.findDefect(exit.cause)), rollbackDefect) + } + })) + + it.effect("preserves body, rollback, and transaction scope close failures", () => + Effect.gen(function*() { + const bodyFailure = new Error("body failed") + const rollbackDefect = new Error("rollback failed") + const closeDefect = new Error("scope close failed") + const connection = { + executeUnprepared: (statement: string) => + statement === "ROLLBACK" + ? Effect.die(rollbackDefect) + : Effect.succeed([]) + } as unknown as SqlConnection.Connection + const reserve = Effect.gen(function*() { + const scope = yield* Scope.Scope + yield* Scope.addFinalizer(scope, Effect.die(closeDefect)) + return connection + }) + const sql = { + reserve, + transactionService: SqlClient.TransactionConnection(999_027) + } as unknown as SqlClient.SqlClient + const exit = yield* make(sql, { + maxAcquireAttempts: 1, + acquireRetryBaseDelayMillis: 0, + acquireRetryMaximumDelayMillis: 0 + })(Effect.fail(bodyFailure)).pipe(Effect.exit) + assert.strictEqual(Exit.isFailure(exit), true) + if (Exit.isFailure(exit)) { + assert.strictEqual(Result.getOrThrow(Cause.findError(exit.cause)), bodyFailure) + const defects = new Set( + exit.cause.reasons + .filter(Cause.isDieReason) + .map((reason) => reason.defect) + ) + assert.strictEqual(defects.has(rollbackDefect), true) + assert.strictEqual(defects.has(closeDefect), true) + } + })) +}) diff --git a/packages/local-rpc/test/PeerRelayStore.test.ts b/packages/local-rpc/test/PeerRelayStore.test.ts new file mode 100644 index 0000000..6eae6ec --- /dev/null +++ b/packages/local-rpc/test/PeerRelayStore.test.ts @@ -0,0 +1,284 @@ +import { NodeCrypto } from "@effect/platform-node" +import { SqliteClient } from "@effect/sql-sqlite-node" +import { assert, describe, it } from "@effect/vitest" +import * as Identity from "@lucas-barake/effect-local/Identity" +import * as Effect from "effect/Effect" +import * as Layer from "effect/Layer" +import * as Option from "effect/Option" +import * as SqlClient from "effect/unstable/sql/SqlClient" +import { rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import * as PeerRelayLimits from "../src/PeerRelayLimits.js" +import * as PeerRelayRpc from "../src/PeerRelayRpc.js" +import * as PeerRelayStore from "../src/PeerRelayStore.js" + +const peer = (value: string) => Identity.PeerId.make(`peer_00000000-0000-4000-8000-${value}`) +const relayId = (value: string) => Identity.RelayMessageId.make(`rly_00000000-0000-4000-8000-${value}`) +const documentId = (value: string) => Identity.DocumentId.make(`doc_00000000-0000-4000-8000-${value}`) + +const makeLayer = (filename: string) => { + const base = Layer.mergeAll( + SqliteClient.layer({ filename }), + NodeCrypto.layer, + PeerRelayLimits.layerDefaults + ) + const store = PeerRelayStore.layerSqlite.pipe(Layer.provide(base)) + return Layer.merge(base, store) +} + +describe("PeerRelayStore", () => { + it.effect("migrates a real WAL database and fences claim payload loading and terminal duplicates", () => + Effect.gen(function*() { + const filename = join(tmpdir(), `effect-local-relay-${globalThis.crypto.randomUUID()}.sqlite`) + yield* Effect.addFinalizer(() => + Effect.sync(() => { + rmSync(filename, { force: true }) + rmSync(`${filename}-shm`, { force: true }) + rmSync(`${filename}-wal`, { force: true }) + }) + ) + yield* Effect.scoped( + Effect.gen(function*() { + const store = yield* PeerRelayStore.PeerRelayStore + const sql = yield* SqlClient.SqlClient + const channel = PeerRelayStore.ChannelKey.make({ + tenantId: "tenant", + senderSubjectId: "sender", + senderPeerId: peer("000000000001"), + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + recipientSubjectId: "recipient", + recipientPeerId: peer("000000000002") + }) + const payload = new Uint8Array([1, 2, 3, 4]) + const admission = PeerRelayStore.Admission.make({ + channel, + relayMessageId: relayId("000000000001"), + relayPeerId: peer("000000000003"), + documentIds: [documentId("000000000001")], + senderConnectionEpoch: "epoch-1", + senderSequence: 0, + payloadVersion: 1, + messageHash: "message-hash", + outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make("a".repeat(64)), + payload, + messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, + senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, + minimumTerminalRetentionMillis: PeerRelayLimits.defaults.minimumTerminalRetentionMillis + }) + const accepted = yield* store.admit(admission) + assert.strictEqual(accepted.status, "Accepted") + assert.deepStrictEqual(yield* store.usage(), { + activeCount: 1, + activeBytes: payload.byteLength, + retainedCount: 1, + retainedBytes: payload.byteLength + }) + const duplicate = yield* store.admit(admission) + assert.strictEqual(duplicate.status, "Duplicate") + assert.strictEqual((yield* store.usage()).activeCount, 1) + + const claimed = yield* store.claim({ + recipient: { + tenantId: "tenant", + subjectId: "recipient", + peerId: channel.recipientPeerId + }, + sender: { + subjectId: "sender", + peerId: channel.senderPeerId + }, + sessionGeneration: 7, + authorizedDocumentIds: admission.documentIds + }) + assert.strictEqual(Option.isSome(claimed.message), true) + if (Option.isNone(claimed.message)) return + const message = claimed.message.value + assert.strictEqual(Object.hasOwn(message, "payload"), false) + assert.strictEqual(message.payloadBytes, payload.byteLength) + const routeColumns = [ + ["tenant_id", "corrupt-tenant", channel.tenantId], + ["sender_subject_id", "corrupt-sender", channel.senderSubjectId], + ["sender_peer_id", "corrupt-peer", channel.senderPeerId] + ] as const + for (const [column, corrupt, original] of routeColumns) { + yield* sql.unsafe( + `UPDATE effect_local_relay_messages SET ${column} = ? WHERE message_id = ?`, + [corrupt, message.rowId] + ) + assert.strictEqual( + (yield* Effect.exit(store.loadClaimedPayload({ + rowId: message.rowId, + channel, + relayMessageId: message.relayMessageId, + claimToken: message.claimToken, + sessionGeneration: message.sessionGeneration, + payloadBytes: message.payloadBytes + })))._tag, + "Failure" + ) + yield* sql.unsafe( + `UPDATE effect_local_relay_messages SET ${column} = ? WHERE message_id = ?`, + [original, message.rowId] + ) + } + assert.deepStrictEqual( + yield* store.loadClaimedPayload({ + rowId: message.rowId, + channel, + relayMessageId: message.relayMessageId, + claimToken: message.claimToken, + sessionGeneration: message.sessionGeneration, + payloadBytes: message.payloadBytes + }), + payload + ) + const stalePayload = yield* Effect.exit(store.loadClaimedPayload({ + rowId: message.rowId, + channel, + relayMessageId: message.relayMessageId, + claimToken: message.claimToken, + sessionGeneration: message.sessionGeneration + 1, + payloadBytes: message.payloadBytes + })) + assert.strictEqual(stalePayload._tag, "Failure") + const terminal = { + channel, + relayMessageId: message.relayMessageId, + claimToken: message.claimToken, + messageHash: message.messageHash, + sessionGeneration: message.sessionGeneration, + recipient: { + tenantId: "tenant", + subjectId: "recipient", + peerId: channel.recipientPeerId + } + } as const + assert.strictEqual((yield* store.acknowledge(terminal)).status, "Changed") + assert.strictEqual((yield* store.acknowledge(terminal)).status, "Duplicate") + assert.strictEqual((yield* store.usage()).activeCount, 0) + assert.strictEqual((yield* store.usage()).retainedCount, 1) + const stored = yield* sql<{ readonly payload: Uint8Array | null }>` + SELECT payload FROM effect_local_relay_messages + ` + assert.strictEqual(stored[0]?.payload, null) + + const expiringAdmission = PeerRelayStore.Admission.make({ + ...admission, + relayMessageId: relayId("000000000002"), + outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make("b".repeat(64)) + }) + yield* store.admit(expiringAdmission) + const expiringClaim = yield* store.claim({ + recipient: terminal.recipient, + sender: { + subjectId: channel.senderSubjectId, + peerId: channel.senderPeerId + }, + sessionGeneration: 8, + authorizedDocumentIds: expiringAdmission.documentIds + }) + assert.strictEqual(Option.isSome(expiringClaim.message), true) + if (Option.isNone(expiringClaim.message)) return + const expiring = expiringClaim.message.value + const loadExpiring = () => + store.loadClaimedPayload({ + rowId: expiring.rowId, + channel, + relayMessageId: expiring.relayMessageId, + claimToken: expiring.claimToken, + sessionGeneration: expiring.sessionGeneration, + payloadBytes: expiring.payloadBytes + }) + const claimColumns = [ + ["claimed_message_id", String(expiring.rowId + 1)], + ["claim_token", "'clm_00000000-0000-4000-8000-000000000009'"], + ["claim_session_generation", String(expiring.sessionGeneration + 1)], + ["claim_deadline", String(expiring.claimDeadline + 1)] + ] as const + for (const [column, corrupt] of claimColumns) { + yield* sql.unsafe( + `UPDATE effect_local_relay_channels SET ${column} = ${corrupt} WHERE channel_id = ?`, + [1] + ) + assert.strictEqual((yield* Effect.exit(loadExpiring()))._tag, "Failure") + yield* sql`UPDATE effect_local_relay_channels + SET claimed_message_id = ${expiring.rowId}, + claim_token = ${expiring.claimToken}, + claim_session_generation = ${expiring.sessionGeneration}, + claim_deadline = ${expiring.claimDeadline} + WHERE channel_id = 1` + } + yield* sql`UPDATE effect_local_relay_messages + SET expires_at = CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) - 1 + WHERE message_id = ${expiring.rowId}` + assert.strictEqual((yield* Effect.exit(loadExpiring()))._tag, "Failure") + assert.strictEqual( + (yield* store.acknowledge({ + ...terminal, + relayMessageId: expiring.relayMessageId, + claimToken: expiring.claimToken, + sessionGeneration: expiring.sessionGeneration + })).status, + "Stale" + ) + yield* store.expire({ batchSize: 10 }) + const expired = yield* sql<{ readonly state: string }>` + SELECT state FROM effect_local_relay_messages WHERE message_id = ${expiring.rowId} + ` + assert.strictEqual(expired[0]?.state, "Expired") + + const corruptAdmission = PeerRelayStore.Admission.make({ + ...admission, + relayMessageId: relayId("000000000003"), + outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make("c".repeat(64)) + }) + yield* store.admit(corruptAdmission) + yield* sql`UPDATE effect_local_relay_messages + SET tenant_id = 'other' + WHERE relay_message_id = ${corruptAdmission.relayMessageId}` + const undisclosed = yield* store.claim({ + recipient: terminal.recipient, + sender: { + subjectId: channel.senderSubjectId, + peerId: channel.senderPeerId + }, + sessionGeneration: 9, + authorizedDocumentIds: corruptAdmission.documentIds + }) + assert.strictEqual(Option.isNone(undisclosed.message), true) + yield* store.repair({ batchSize: 10 }) + const repaired = yield* sql<{ readonly state: string }>` + SELECT state FROM effect_local_relay_messages + WHERE relay_message_id = ${corruptAdmission.relayMessageId} + ` + assert.strictEqual(repaired[0]?.state, "DeadLettered") + + const restartedChannel = PeerRelayStore.ChannelKey.make({ + ...channel, + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(2) + }) + const restartedAdmission = PeerRelayStore.Admission.make({ + ...admission, + channel: restartedChannel, + relayMessageId: relayId("000000000004"), + outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make("d".repeat(64)) + }) + yield* store.admit(restartedAdmission) + const discovered = yield* store.claim({ + recipient: terminal.recipient, + sender: { + subjectId: restartedChannel.senderSubjectId, + peerId: restartedChannel.senderPeerId + }, + sessionGeneration: 10, + authorizedDocumentIds: restartedAdmission.documentIds + }) + assert.strictEqual(Option.isSome(discovered.message), true) + if (Option.isSome(discovered.message)) { + assert.strictEqual(discovered.message.value.channel.senderReplicaIncarnation, 2) + } + }).pipe(Effect.provide(makeLayer(filename))) + ) + })) +}) diff --git a/packages/local-rpc/test/PeerRpcServer.test.ts b/packages/local-rpc/test/PeerRpcServer.test.ts index bf4990f..ea3d530 100644 --- a/packages/local-rpc/test/PeerRpcServer.test.ts +++ b/packages/local-rpc/test/PeerRpcServer.test.ts @@ -3,6 +3,7 @@ import * as CommitPublisher from "@lucas-barake/effect-local-sql/CommitPublisher import type * as DocumentEntity from "@lucas-barake/effect-local-sql/DocumentEntity" import * as PeerSession from "@lucas-barake/effect-local-sql/PeerSession" import * as PeerSync from "@lucas-barake/effect-local-sql/PeerSync" +import * as PeerSyncEnvelope from "@lucas-barake/effect-local-sql/PeerSyncEnvelope" import * as ReplicaGate from "@lucas-barake/effect-local-sql/ReplicaGate" import * as Canonical from "@lucas-barake/effect-local/Canonical" import * as Document from "@lucas-barake/effect-local/Document" @@ -33,6 +34,7 @@ import * as TestClock from "effect/testing/TestClock" import * as Tracer from "effect/Tracer" import * as Sharding from "effect/unstable/cluster/Sharding" import type * as Rpc from "effect/unstable/rpc/Rpc" +import * as RpcServer from "effect/unstable/rpc/RpcServer" import * as RpcTest from "effect/unstable/rpc/RpcTest" import { createHash, randomBytes } from "node:crypto" import * as PeerRpcObservability from "../src/internal/peerRpcObservability.js" @@ -40,6 +42,11 @@ import * as PeerAuthentication from "../src/PeerAuthentication.js" import * as PeerAuthenticator from "../src/PeerAuthenticator.js" import * as PeerAuthorization from "../src/PeerAuthorization.js" import * as PeerCredentials from "../src/PeerCredentials.js" +import * as PeerRelayAuthorization from "../src/PeerRelayAuthorization.js" +import * as PeerRelayIngress from "../src/PeerRelayIngress.js" +import * as PeerRelayLimits from "../src/PeerRelayLimits.js" +import * as PeerRelayRpc from "../src/PeerRelayRpc.js" +import * as PeerRelayStore from "../src/PeerRelayStore.js" import * as PeerRpc from "../src/PeerRpc.js" import * as PeerRpcError from "../src/PeerRpcError.js" import * as PeerRpcLimits from "../src/PeerRpcLimits.js" @@ -2754,3 +2761,312 @@ describe("PeerRpcServer", () => { yield* Fiber.interrupt(note.fiber) }))) }) + +describe("PeerRpcServer relay", () => { + it.effect("reports the logical remote recipient separately from the relay endpoint", () => + Effect.scoped(Effect.gen(function*() { + const localPrincipal: PeerAuthentication.PeerPrincipal = { + tenantId: "tenant", + subjectId: "sender", + peerId: remotePeerA + } + let authorizationDefect: unknown | undefined + const relayAuthorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ + authorize: (request) => + authorizationDefect === undefined + ? Effect.succeed({ + remote: { + tenantId: request.principal.tenantId, + subjectId: request.remote.subjectId, + peerId: request.remote.peerId + }, + documents: request.documents.map((document) => ({ + document: Task, + documentId: document.documentId + })), + validUntil: Number.MAX_SAFE_INTEGER, + invalidated: Effect.never + }) + : Effect.die(authorizationDefect) + }) + const transition: PeerRelayStore.TransitionResult = { + status: "Changed", + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" + } + let maintenanceDefect: unknown | undefined + const admitted = yield* Deferred.make() + const acknowledgeCalls = yield* Ref.make(0) + const relayStore = PeerRelayStore.PeerRelayStore.of({ + admit: (input) => + Deferred.succeed(admitted, input).pipe( + Effect.as({ + status: "Accepted", + channel: input.channel, + ready: false, + nextEligibleAt: 0, + lane: "New" + }) + ), + claim: () => + Effect.succeed({ + message: Option.none(), + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" + }), + loadClaimedPayload: () => Effect.succeed(new Uint8Array()), + acknowledge: () => Ref.update(acknowledgeCalls, (count) => count + 1).pipe(Effect.as(transition)), + reject: () => Effect.succeed(transition), + release: () => Effect.succeed(transition), + recover: ({ cursor }) => + maintenanceDefect === undefined + ? Effect.succeed({ + ...(cursor === undefined ? {} : { cursor }), + processed: 0, + hasMore: false + }) + : Effect.die(maintenanceDefect), + expire: ({ cursor }) => + Effect.succeed({ ...(cursor === undefined ? {} : { cursor }), processed: 0, hasMore: false }), + repair: ({ cursor }) => + Effect.succeed({ ...(cursor === undefined ? {} : { cursor }), processed: 0, hasMore: false }), + reconcile: ({ cursor }) => + Effect.succeed({ ...(cursor === undefined ? {} : { cursor }), processed: 0, hasMore: false }), + collect: ({ cursor }) => + Effect.succeed({ ...(cursor === undefined ? {} : { cursor }), processed: 0, hasMore: false }), + usage: () => + Effect.succeed({ + activeCount: 0, + activeBytes: 0, + retainedCount: 0, + retainedBytes: 0 + }) + }) + const relayIngress = PeerRelayIngress.PeerRelayIngress.of({ + address: { _tag: "UnixAddress", path: "relay-test" }, + reserveOutbound: () => Effect.fail(new PeerRpcError.RequestCapacityExceeded()), + usage: Effect.succeed({ connections: 0, reservedBytes: 0, byteReservationWaiters: 0 }), + await: Effect.never + }) + const handlers = PeerRpcServer.layerRelayHandlers({ + tenantId: "tenant", + peerId: serverPeerId + }).pipe( + Layer.provide(Layer.mergeAll( + Layer.succeed(Crypto.Crypto, crypto), + Layer.succeed(PeerRelayLimits.PeerRelayLimits, PeerRelayLimits.defaults), + Layer.succeed(PeerRelayAuthorization.PeerRelayAuthorization, relayAuthorization), + Layer.succeed(PeerRelayStore.PeerRelayStore, relayStore), + Layer.succeed(PeerRelayIngress.PeerRelayIngress, relayIngress) + )) + ) + const context = yield* Layer.build(handlers) + const openHandler = context.mapUnsafe.get(PeerRelayRpc.OpenRelayRpc.key) as Rpc.Handler<"OpenRelay"> + const pushHandler = context.mapUnsafe.get(PeerRelayRpc.PushRelayRpc.key) as Rpc.Handler<"PushRelay"> + const acknowledgeHandler = context.mapUnsafe.get( + PeerRelayRpc.AcknowledgeRelayRpc.key + ) as Rpc.Handler<"AcknowledgeRelay"> + const runtime = Context.get(context, PeerRpcServer.PeerRelayServerRuntime) + const protocolStopped = yield* Deferred.make() + const ingressStopped = yield* Deferred.make() + const baseProtocol = yield* RpcServer.Protocol.make(() => + Effect.succeed({ + disconnects: Queue.unbounded().pipe(Effect.runSync), + send: () => Effect.void, + end: () => Effect.void, + clientIds: Effect.succeed(new Set()), + initialMessage: Effect.succeed(Option.none()), + supportsAck: true, + supportsTransferables: false, + supportsSpanPropagation: true + }) + ) + const protocol: RpcServer.Protocol["Service"] = { + ...baseProtocol, + run: (handler) => + baseProtocol.run(handler).pipe( + Effect.ensuring(Deferred.succeed(protocolStopped, undefined)) + ) + } + const openRequest = { + version: PeerRelayRpc.protocolVersion, + expectedRelayPeerId: serverPeerId, + expectedLocal: localPrincipal, + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + remote: { + subjectId: "recipient", + peerId: remotePeerB + }, + documents: [{ documentType: Task.name, documentId: taskId }], + receiptRetentionMillis: PeerRelayLimits.defaults.maximumReceiptRetentionMillis, + senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis + } satisfies typeof PeerRelayRpc.OpenRelayRpc.payloadSchema.Type + const authenticated = { + principal: localPrincipal, + validUntil: Number.MAX_SAFE_INTEGER, + invalidated: Effect.never + } + const stream = openHandler.handler( + openRequest, + {} as never + ) as Stream.Stream + const events = yield* Queue.unbounded() + const openFiber = yield* Stream.runForEach( + stream.pipe( + Stream.provideContext(Context.add( + openHandler.context, + PeerAuthentication.AuthenticatedPeer, + authenticated + )) + ), + (event) => Queue.offer(events, event) + ).pipe(Effect.forkScoped) + const opened = yield* Queue.take(events) + assert.strictEqual(opened._tag, "RelayOpened") + assert.strictEqual( + (opened as PeerRelayRpc.RelayOpened).remotePeerId, + remotePeerB + ) + const relayMessageId = Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000001") + const message = Uint8Array.of(1, 2, 3) + const messageHash = yield* Canonical.digest(message).pipe( + Effect.provideService(Crypto.Crypto, crypto) + ) + const payload = new TextEncoder().encode( + yield* Schema.encodeEffect(SyncEnvelopeJson)({ + connectionEpoch: "epoch", + sequence: 0, + documentId: taskId, + documentType: Task.name, + messageHash, + message, + writerProvenance: [] + }) + ) + const expectedDigest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope({ + domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, + version: PeerSyncEnvelope.relayOuterEnvelopeVersion, + expectedLocal: localPrincipal, + remote: { + tenantId: "tenant", + subjectId: "recipient", + peerId: remotePeerB + }, + relayPeerId: serverPeerId, + relayMessageId, + protocolVersion: PeerRelayRpc.protocolVersion, + payloadVersion: 1, + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + senderConnectionEpoch: "epoch", + senderSequence: 0, + document: { + documentId: taskId, + documentType: Task.name + }, + writerProvenance: [], + messageHash, + payload + }).pipe(Effect.provideService(Crypto.Crypto, crypto)) + yield* (pushHandler.handler({ + sessionId: (opened as PeerRelayRpc.RelayOpened).sessionId, + relayMessageId, + payload + }, {} as never) as Effect.Effect).pipe( + Effect.provideContext(Context.add( + pushHandler.context, + PeerAuthentication.AuthenticatedPeer, + authenticated + )) + ) + assert.strictEqual((yield* Deferred.await(admitted)).outerEnvelopeDigest, expectedDigest) + const defect = new Error("relay authorization defect") + authorizationDefect = defect + const defectExit = yield* Stream.runHead( + (openHandler.handler( + openRequest, + {} as never + ) as Stream.Stream).pipe( + Stream.provideContext(Context.add( + openHandler.context, + PeerAuthentication.AuthenticatedPeer, + authenticated + )) + ) + ).pipe(Effect.exit) + assert.isTrue(Exit.isFailure(defectExit)) + if (Exit.isFailure(defectExit)) { + assert.strictEqual(Cause.squash(defectExit.cause), defect) + } + authorizationDefect = undefined + const ownerContext = yield* Layer.build(Layer.fresh(handlers)) + const ownerRuntime = Context.get(ownerContext, PeerRpcServer.PeerRelayServerRuntime) + const serverContext = Context.add( + Context.add( + Context.add( + ownerContext, + RpcServer.Protocol, + protocol + ), + PeerRelayIngress.PeerRelayIngress, + { + ...relayIngress, + await: Effect.never.pipe( + Effect.ensuring(Deferred.succeed(ingressStopped, undefined)) + ) + } + ), + PeerAuthentication.PeerAuthentication, + PeerAuthentication.PeerAuthentication.of((effect) => + Effect.provideService(effect, PeerAuthentication.AuthenticatedPeer, { + principal: localPrincipal, + validUntil: Number.MAX_SAFE_INTEGER, + invalidated: Effect.never + }) + ) + ) + yield* Layer.build( + PeerRpcServer.layerRelayServer.pipe( + Layer.provide(Layer.succeedContext(serverContext)) + ) + ) + const ownerDefect = new Error("relay maintenance defect") + maintenanceDefect = ownerDefect + yield* TestClock.adjust(PeerRelayLimits.defaults.maintenanceIntervalMillis) + const ownerExit = yield* ownerRuntime.owner.pipe(Effect.exit) + assert.isTrue(Exit.isFailure(ownerExit)) + if (Exit.isFailure(ownerExit)) { + assert.strictEqual(Cause.squash(ownerExit.cause), ownerDefect) + } + yield* Effect.yieldNow + yield* Effect.yieldNow + assert.isTrue(Option.isSome(yield* Deferred.poll(protocolStopped))) + assert.isTrue(Option.isSome(yield* Deferred.poll(ingressStopped))) + yield* runtime.shutdown + yield* runtime.shutdown + const stale = yield* (acknowledgeHandler.handler({ + sessionId: (opened as PeerRelayRpc.RelayOpened).sessionId, + relayMessageId, + claimToken: PeerRelayRpc.ClaimToken.make("clm_00000000-0000-4000-8000-000000000001"), + messageHash + }, {} as never) as Effect.Effect).pipe( + Effect.provideContext(Context.add( + acknowledgeHandler.context, + PeerAuthentication.AuthenticatedPeer, + authenticated + )), + Effect.flip + ) + assert.strictEqual(stale._tag, "SessionUnavailable") + assert.strictEqual(yield* Ref.get(acknowledgeCalls), 0) + assert.deepStrictEqual(yield* runtime.usage, { + accepting: false, + sessions: 0, + subjects: 0, + activeClaims: 0, + queuedChannels: 0 + }) + yield* Fiber.interrupt(openFiber) + }))) +}) diff --git a/packages/local-rpc/test/PublicApi.types.test.ts b/packages/local-rpc/test/PublicApi.types.test.ts new file mode 100644 index 0000000..471202f --- /dev/null +++ b/packages/local-rpc/test/PublicApi.types.test.ts @@ -0,0 +1,53 @@ +import { assert, describe, it } from "@effect/vitest" +import * as Schema from "effect/Schema" +import * as PublicApi from "../src/index.js" + +describe("public API", () => { + it("exports the version 3 relay protocol without widening version 2", () => { + const publicApi: Record = PublicApi + + for ( + const name of [ + "PeerRelayAuthorization", + "PeerRelayIngress", + "PeerRelayLimits", + "PeerRelayRpc", + "PeerRelayStore" + ] + ) { + assert.property(publicApi, name) + } + + const peerRelayRpc = publicApi.PeerRelayRpc as Record + const relayOpened = peerRelayRpc.RelayOpened as Schema.Schema + const decoded = Schema.decodeUnknownSync(relayOpened)({ + _tag: "RelayOpened", + version: 3, + sessionId: "ses_00000000-0000-4000-8000-000000000001", + remotePeerId: "peer_00000000-0000-4000-8000-000000000002", + authenticatedLocal: { + tenantId: "tenant", + subjectId: "subject", + peerId: "peer_00000000-0000-4000-8000-000000000001" + }, + capabilities: { + storeAndForward: true + } + }) + + assert.deepStrictEqual(decoded, { + _tag: "RelayOpened", + version: 3, + sessionId: "ses_00000000-0000-4000-8000-000000000001", + remotePeerId: "peer_00000000-0000-4000-8000-000000000002", + authenticatedLocal: { + tenantId: "tenant", + subjectId: "subject", + peerId: "peer_00000000-0000-4000-8000-000000000001" + }, + capabilities: { + storeAndForward: true + } + }) + }) +}) diff --git a/packages/local-rpc/test/RpcPeerTransport.test.ts b/packages/local-rpc/test/RpcPeerTransport.test.ts index 477849f..aeef705 100644 --- a/packages/local-rpc/test/RpcPeerTransport.test.ts +++ b/packages/local-rpc/test/RpcPeerTransport.test.ts @@ -1,4 +1,8 @@ +import { NodeCrypto } from "@effect/platform-node" import { assert, describe, it } from "@effect/vitest" +import * as PeerRelayClientRuntime from "@lucas-barake/effect-local-sql/PeerRelayClientRuntime" +import type * as PeerRelayOutbox from "@lucas-barake/effect-local-sql/PeerRelayOutbox" +import * as PeerSyncEnvelope from "@lucas-barake/effect-local-sql/PeerSyncEnvelope" import * as Document from "@lucas-barake/effect-local/Document" import * as DocumentSet from "@lucas-barake/effect-local/DocumentSet" import * as Identity from "@lucas-barake/effect-local/Identity" @@ -22,6 +26,7 @@ import * as Stream from "effect/Stream" import * as Tracer from "effect/Tracer" import { RpcClientDefect, RpcClientError } from "effect/unstable/rpc/RpcClientError" import * as PeerRpcObservability from "../src/internal/peerRpcObservability.js" +import * as PeerRelayRpc from "../src/PeerRelayRpc.js" import * as PeerRpc from "../src/PeerRpc.js" import * as PeerRpcError from "../src/PeerRpcError.js" import * as RpcPeerTransport from "../src/RpcPeerTransport.js" @@ -34,6 +39,13 @@ const serverPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-00000000 const otherPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000002") const sessionId = Identity.SessionId.make("ses_00000000-0000-4000-8000-000000000001") const otherSessionId = Identity.SessionId.make("ses_00000000-0000-4000-8000-000000000002") +const relayPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000003") +const localPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000004") +const remotePeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000005") +const relayMessageId = Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000001") +const senderReplicaIncarnation = Identity.ReplicaIncarnation.make(1) +const claimToken = PeerRelayRpc.ClaimToken.make("clm_00000000-0000-4000-8000-000000000001") +const relayMessageHash = "1".repeat(64) const documents = [{ document: Task, documentId }] const definition = ReplicaDefinition.make({ name: "rpc-peer-transport-test", @@ -72,6 +84,189 @@ const liveOpen = (event: PeerRpc.OpenEvent) => Stream.concat(Stream.make(event), const openWithMessage = (payload: Uint8Array) => Stream.fromIterable([serverOpened, PeerRpc.Message.make({ _tag: "Message", payload })]).pipe(Stream.rechunk(1)) +const relayOptions: RpcPeerTransport.StoreAndForwardOptions = { + expectedLocal: { + tenantId: "tenant", + subjectId: "local-subject", + peerId: localPeerId + }, + senderReplicaIncarnation, + expectedRelayPeerId: relayPeerId, + remote: { + subjectId: "remote-subject", + peerId: remotePeerId + }, + documents, + definition, + receiptRetentionMillis: 8 * 24 * 60 * 60 * 1_000, + senderRetryHorizonMillis: 7 * 24 * 60 * 60 * 1_000, + replayBatchSize: 16 +} + +const relayOpened = PeerRelayRpc.RelayOpened.make({ + _tag: "RelayOpened", + version: PeerRelayRpc.protocolVersion, + sessionId, + remotePeerId, + authenticatedLocal: relayOptions.expectedLocal, + capabilities: { storeAndForward: true } +}) + +const makeRelayClient = ( + open: ( + request: typeof PeerRelayRpc.OpenRelayRpc.payloadSchema.Type, + options: { readonly streamBufferSize?: number | undefined } + ) => Stream.Stream, + push: ( + request: typeof PeerRelayRpc.PushRelayRpc.payloadSchema.Type + ) => Effect.Effect = () => Effect.void, + acknowledge: ( + request: typeof PeerRelayRpc.AcknowledgeRelayRpc.payloadSchema.Type + ) => Effect.Effect = () => Effect.void, + reject: ( + request: typeof PeerRelayRpc.RejectRelayRpc.payloadSchema.Type + ) => Effect.Effect = () => Effect.void +): PeerRelayRpc.RpcClient => + ({ + OpenRelay: open, + PushRelay: push, + AcknowledgeRelay: acknowledge, + RejectRelay: reject + }) as never + +const relayEntry = (payload: Uint8Array): PeerRelayOutbox.Entry => ({ + rowId: 1, + replicaId, + replicaIncarnation: senderReplicaIncarnation, + writerGeneration: Identity.WriterGeneration.make(1), + expectedLocal: relayOptions.expectedLocal, + remote: { + tenantId: relayOptions.expectedLocal.tenantId, + subjectId: relayOptions.remote.subjectId, + peerId: relayOptions.remote.peerId + }, + relayPeerId, + relayMessageId, + outerEnvelopeDigest: "2".repeat(64), + protocolVersion: 3, + payloadVersion: 1, + senderConnectionEpoch: "sender-epoch", + senderSequence: 0, + document: { documentType: Task.name, documentId }, + writerProvenance: [], + messageHash: relayMessageHash, + payload, + encodedSize: payload.byteLength, + createdAt: "2026-07-25T00:00:00.000Z", + retryDeadline: "2026-08-01T00:00:00.000Z", + nextAttemptAt: "2026-07-25T00:00:00.000Z" +}) + +const makeStoredMessage = (payload: Uint8Array) => + Effect.gen(function*() { + const envelope: PeerSyncEnvelope.RelayOuterEnvelope = { + domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, + version: PeerSyncEnvelope.relayOuterEnvelopeVersion, + expectedLocal: { + tenantId: relayOptions.expectedLocal.tenantId, + subjectId: relayOptions.remote.subjectId, + peerId: relayOptions.remote.peerId + }, + remote: relayOptions.expectedLocal, + relayPeerId, + relayMessageId, + protocolVersion: PeerRelayRpc.protocolVersion, + payloadVersion: PeerSyncEnvelope.syncEnvelopeVersion, + senderReplicaIncarnation, + senderConnectionEpoch: "remote-epoch", + senderSequence: 3, + document: { documentType: Task.name, documentId }, + writerProvenance: [], + messageHash: relayMessageHash, + payload + } + const outerEnvelopeDigest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope(envelope) + return PeerRelayRpc.StoredMessage.make({ + _tag: "StoredMessage", + relayMessageId, + claimToken, + relayPeerId, + sender: { + tenantId: envelope.expectedLocal.tenantId, + subjectId: envelope.expectedLocal.subjectId, + peerId: envelope.expectedLocal.peerId, + replicaIncarnation: envelope.senderReplicaIncarnation, + connectionEpoch: envelope.senderConnectionEpoch, + sequence: envelope.senderSequence + }, + recipient: envelope.remote, + payloadVersion: envelope.payloadVersion, + document: envelope.document, + writerProvenance: envelope.writerProvenance, + messageHash: envelope.messageHash, + outerEnvelopeDigest, + payload + }) + }) + +interface RuntimeOverrides { + readonly admit?: ( + input: PeerRelayOutbox.AdmitInput + ) => Effect.Effect + readonly dueForEndpoint?: ( + input: PeerRelayOutbox.ReplayInput + ) => Effect.Effect, ReplicaError.ReplicaError> + readonly maximumPendingHorizon?: ( + endpoint: PeerRelayOutbox.Endpoint + ) => Effect.Effect + readonly markCustody?: ( + input: PeerRelayOutbox.CustodyInput + ) => Effect.Effect + readonly validateConnectionConfiguration?: ( + input: { + readonly replicaIncarnation: Identity.ReplicaIncarnation + readonly retryHorizonMillis: number + readonly replayBatchSize: number + } + ) => Effect.Effect + readonly signalReceiptPrune?: Effect.Effect + readonly health?: Effect.Effect + readonly awaitFatal?: Effect.Effect +} + +const makeRuntime = (overrides: RuntimeOverrides = {}) => + PeerRelayClientRuntime.PeerRelayClientRuntime.of({ + admit: overrides.admit ?? ((input) => Effect.succeed(relayEntry(input.payload))), + dueForEndpoint: overrides.dueForEndpoint ?? (() => Effect.succeed([])), + maximumPendingHorizon: overrides.maximumPendingHorizon ?? (() => Effect.succeed(null)), + markCustody: overrides.markCustody ?? (() => Effect.void), + validateReplicaIncarnation: () => Effect.void, + validateConnectionConfiguration: overrides.validateConnectionConfiguration ?? (() => Effect.void), + signalReceiptPrune: overrides.signalReceiptPrune ?? Effect.void, + health: overrides.health ?? Effect.void, + awaitFatal: overrides.awaitFatal ?? Effect.never + }) + +const connectRelay = ( + client: PeerRelayRpc.RpcClient, + runtime: ReturnType, + peerId: Identity.PeerId = remotePeerId +) => + Effect.gen(function*() { + const context = yield* Layer.build( + RpcPeerTransport.layerStoreAndForward(client, relayOptions).pipe( + Layer.provide(Layer.merge( + Layer.succeed(PeerRelayClientRuntime.PeerRelayClientRuntime, runtime), + NodeCrypto.layer + )) + ) + ) + return yield* Context.get(context, PeerTransport.PeerTransport).connect({ + replicaId, + peerId + }) + }) + describe("RpcPeerTransport", () => { it.effect("validates the first streamed event as the handshake and uses response capacity one", () => Effect.scoped(Effect.gen(function*() { @@ -950,3 +1145,392 @@ describe("RpcPeerTransport", () => { )) }) }) + +describe("RpcPeerTransport store and forward", () => { + it.effect("opens strict version 3 relay mode with distinct logical and relay identities", () => + Effect.scoped(Effect.gen(function*() { + const configurations = yield* Ref.make(0) + const client = makeRelayClient((request, options) => { + assert.strictEqual(options.streamBufferSize, 1) + assert.strictEqual(request.version, PeerRelayRpc.protocolVersion) + assert.strictEqual(request.expectedRelayPeerId, relayPeerId) + assert.deepStrictEqual(request.expectedLocal, relayOptions.expectedLocal) + assert.strictEqual(request.senderReplicaIncarnation, senderReplicaIncarnation) + assert.deepStrictEqual(request.remote, relayOptions.remote) + assert.strictEqual( + request.senderRetryHorizonMillis, + relayOptions.senderRetryHorizonMillis + ) + return Stream.concat(Stream.make(relayOpened), Stream.never) + }) + const runtime = makeRuntime({ + validateConnectionConfiguration: (input) => { + assert.deepStrictEqual(input, { + replicaIncarnation: senderReplicaIncarnation, + retryHorizonMillis: relayOptions.senderRetryHorizonMillis, + replayBatchSize: relayOptions.replayBatchSize + }) + return Ref.update(configurations, (count) => count + 1) + } + }) + const connection = yield* connectRelay(client, runtime) + assert.strictEqual(connection.peerId, remotePeerId) + assert.strictEqual(connection.relayPeerId, relayPeerId) + assert.isTrue(connection.capabilities.storeAndForward) + assert.strictEqual(yield* Ref.get(configurations), 1) + yield* connection.close + }))) + + it.effect("replays stable outbox rows before new Push and retires only after custody", () => + Effect.scoped(Effect.gen(function*() { + const replayPayload = Uint8Array.of(7) + const newPayload = Uint8Array.of(8) + const dueCalls = yield* Ref.make(0) + const admitted = yield* Ref.make(false) + const pushed = yield* Queue.unbounded() + const custody = yield* Queue.unbounded() + const runtime = makeRuntime({ + maximumPendingHorizon: () => Effect.succeed(relayOptions.senderRetryHorizonMillis), + dueForEndpoint: () => + Ref.updateAndGet(dueCalls, (count) => count + 1).pipe( + Effect.map((count) => count === 1 ? [relayEntry(replayPayload)] : []) + ), + admit: (input) => + Ref.set(admitted, true).pipe( + Effect.as(relayEntry(input.payload)) + ), + markCustody: (input) => Queue.offer(custody, input.relayMessageId).pipe(Effect.asVoid) + }) + const client = makeRelayClient( + () => Stream.concat(Stream.make(relayOpened), Stream.never), + (request) => + Effect.gen(function*() { + if (request.payload === newPayload) assert.isTrue(yield* Ref.get(admitted)) + yield* Queue.offer(pushed, request.payload) + }) + ) + const connection = yield* connectRelay(client, runtime) + assert.deepStrictEqual(yield* Queue.take(pushed), replayPayload) + assert.strictEqual(yield* Queue.take(custody), relayMessageId) + yield* connection.send(newPayload) + assert.deepStrictEqual(yield* Queue.take(pushed), newPayload) + assert.strictEqual(yield* Queue.take(custody), relayMessageId) + yield* connection.close + }))) + + it.effect("keeps one stable relay identity across a failed Push and reconnect replay", () => + Effect.scoped(Effect.gen(function*() { + const payload = Uint8Array.of(10) + const entry = relayEntry(payload) + const pushAttempts = yield* Ref.make(0) + const replayed = yield* Ref.make(false) + const pushedIds = yield* Queue.unbounded() + const firstRuntime = makeRuntime({ + admit: () => Effect.succeed(entry), + dueForEndpoint: () => Effect.succeed([]) + }) + const restartedRuntime = makeRuntime({ + admit: () => Effect.succeed(entry), + dueForEndpoint: () => + Ref.getAndSet(replayed, true).pipe( + Effect.map((alreadyReplayed) => alreadyReplayed ? [] : [entry]) + ) + }) + const client = makeRelayClient( + () => Stream.concat(Stream.make(relayOpened), Stream.never), + (request) => + Queue.offer(pushedIds, request.relayMessageId).pipe( + Effect.andThen(Ref.updateAndGet(pushAttempts, (count) => count + 1)), + Effect.flatMap((attempt) => + attempt === 1 + ? Effect.fail(new PeerRpcError.ServerUnavailable()) + : Effect.void + ) + ) + ) + const first = yield* connectRelay(client, firstRuntime) + const firstError = yield* first.send(payload).pipe(Effect.flip) + assert.strictEqual(firstError.reason._tag, "StorageUnavailable") + yield* first.close + + const second = yield* connectRelay(client, restartedRuntime) + yield* second.close + assert.deepStrictEqual(yield* Queue.takeAll(pushedIds), [ + relayMessageId, + relayMessageId + ]) + }))) + + it.effect("rejects a retry horizon shrink before opening the relay stream", () => + Effect.scoped(Effect.gen(function*() { + const opens = yield* Ref.make(0) + const client = makeRelayClient( + () => + Ref.update(opens, (count) => count + 1).pipe( + Effect.as(Stream.concat(Stream.make(relayOpened), Stream.never)), + Stream.unwrap + ) + ) + const error = yield* connectRelay( + client, + makeRuntime({ + maximumPendingHorizon: () => Effect.succeed(relayOptions.senderRetryHorizonMillis + 1) + }) + ).pipe(Effect.flip) + assert.strictEqual(error.reason._tag, "ProtocolMismatch") + assert.strictEqual(yield* Ref.get(opens), 0) + }))) + + it.effect("rejects a mismatched RelayOpened before exposing a connection", () => + Effect.scoped(Effect.gen(function*() { + const finalized = yield* Ref.make(0) + const mismatched = { + ...relayOpened, + authenticatedLocal: { + ...relayOpened.authenticatedLocal, + subjectId: "other-subject" + } + } as PeerRelayRpc.RelayOpened + const client = makeRelayClient( + () => + Stream.concat(Stream.make(mismatched), Stream.never).pipe( + Stream.ensuring(Ref.update(finalized, (count) => count + 1)) + ) + ) + const error = yield* connectRelay(client, makeRuntime()).pipe(Effect.flip) + assert.strictEqual(error.reason._tag, "ProtocolMismatch") + assert.strictEqual(yield* Ref.get(finalized), 1) + }))) + + it.effect("validates the complete outer digest before exposing acknowledged delivery", () => + Effect.scoped( + Effect.gen(function*() { + const valid = yield* makeStoredMessage(Uint8Array.of(1, 2, 3)) + const invalid = PeerRelayRpc.StoredMessage.make({ + ...valid, + outerEnvelopeDigest: "0".repeat(64) + }) + const acknowledgements = yield* Ref.make(0) + const client = makeRelayClient( + () => Stream.fromIterable([relayOpened, invalid]).pipe(Stream.rechunk(1)), + undefined, + () => Ref.update(acknowledgements, (count) => count + 1) + ) + const connection = yield* connectRelay(client, makeRuntime()) + const error = yield* Stream.runHead( + connection.receiveWithAcknowledgement! + ).pipe(Effect.flip) + assert.strictEqual(error.reason._tag, "ProtocolMismatch") + assert.strictEqual(yield* Ref.get(acknowledgements), 0) + yield* connection.close + }).pipe(Effect.provide(NodeCrypto.layer)) + )) + + it.effect("defers Ack until the delivery consumer commits and closes on Ack failure", () => + Effect.scoped( + Effect.gen(function*() { + const stored = yield* makeStoredMessage(Uint8Array.of(4, 5, 6)) + const acknowledgements = yield* Ref.make(0) + const finalized = yield* Ref.make(0) + const client = makeRelayClient( + () => + Stream.fromIterable([relayOpened, stored]).pipe( + Stream.rechunk(1), + Stream.concat(Stream.never), + Stream.ensuring(Ref.update(finalized, (count) => count + 1)) + ), + undefined, + (request) => { + assert.strictEqual(request.sessionId, sessionId) + assert.strictEqual(request.relayMessageId, relayMessageId) + assert.strictEqual(request.claimToken, claimToken) + assert.strictEqual(request.messageHash, relayMessageHash) + return Ref.update(acknowledgements, (count) => count + 1).pipe( + Effect.andThen(Effect.fail(new PeerRpcError.ServerUnavailable())) + ) + } + ) + const connection = yield* connectRelay(client, makeRuntime()) + const deliveryOption = yield* Stream.runHead(connection.receiveWithAcknowledgement!) + assert.isTrue(Option.isSome(deliveryOption)) + if (Option.isNone(deliveryOption)) return + const delivery = deliveryOption.value + assert.strictEqual(yield* Ref.get(acknowledgements), 0) + assert.strictEqual( + delivery.receiptRetentionMillis, + relayOptions.receiptRetentionMillis + ) + assert.strictEqual(delivery.identity.relayPeerId, relayPeerId) + assert.strictEqual(delivery.identity.senderPeerId, remotePeerId) + const error = yield* delivery.acknowledge.pipe(Effect.flip) + assert.strictEqual(error.reason._tag, "StorageUnavailable") + assert.strictEqual(yield* Ref.get(acknowledgements), 1) + assert.strictEqual(yield* Ref.get(finalized), 1) + }).pipe(Effect.provide(NodeCrypto.layer)) + )) + + it.effect("signals receipt maintenance only after Ack succeeds", () => + Effect.scoped( + Effect.gen(function*() { + const stored = yield* makeStoredMessage(Uint8Array.of(5, 6, 7)) + const acknowledgements = yield* Ref.make(0) + const pruneSignals = yield* Ref.make(0) + const client = makeRelayClient( + () => Stream.fromIterable([relayOpened, stored]).pipe(Stream.rechunk(1)), + undefined, + () => Ref.update(acknowledgements, (count) => count + 1) + ) + const connection = yield* connectRelay( + client, + makeRuntime({ + signalReceiptPrune: Ref.update(pruneSignals, (count) => count + 1) + }) + ) + const delivery = yield* Stream.runHead( + connection.receiveWithAcknowledgement! + ).pipe(Effect.map(Option.getOrThrow)) + assert.strictEqual(yield* Ref.get(acknowledgements), 0) + assert.strictEqual(yield* Ref.get(pruneSignals), 0) + yield* delivery.acknowledge + assert.strictEqual(yield* Ref.get(acknowledgements), 1) + assert.strictEqual(yield* Ref.get(pruneSignals), 1) + yield* connection.close + }).pipe(Effect.provide(NodeCrypto.layer)) + )) + + it.effect("binds permanent rejection to the current relay session and claim", () => + Effect.scoped( + Effect.gen(function*() { + const stored = yield* makeStoredMessage(Uint8Array.of(6, 7, 8)) + const rejected = yield* Deferred.make< + typeof PeerRelayRpc.RejectRelayRpc.payloadSchema.Type + >() + const client = makeRelayClient( + () => Stream.fromIterable([relayOpened, stored]).pipe(Stream.rechunk(1)), + undefined, + undefined, + (request) => Deferred.succeed(rejected, request).pipe(Effect.asVoid) + ) + const connection = yield* connectRelay(client, makeRuntime()) + const delivery = yield* Stream.runHead( + connection.receiveWithAcknowledgement! + ).pipe(Effect.map(Option.getOrThrow)) + yield* delivery.reject("ApplicationRejected") + assert.deepStrictEqual(yield* Deferred.await(rejected), { + sessionId, + relayMessageId, + claimToken, + messageHash: relayMessageHash, + reason: "ApplicationRejected" + }) + yield* connection.close + }).pipe(Effect.provide(NodeCrypto.layer)) + )) + + it.effect("terminates a dependent relay connection when runtime health fails", () => + Effect.scoped(Effect.gen(function*() { + const fatal = yield* Deferred.make() + const finalized = yield* Deferred.make() + const pushStarted = yield* Deferred.make() + const client = makeRelayClient( + () => + Stream.concat(Stream.make(relayOpened), Stream.never).pipe( + Stream.ensuring(Deferred.succeed(finalized, undefined)) + ), + () => + Deferred.succeed(pushStarted, undefined).pipe( + Effect.andThen(Effect.never) + ) + ) + const connection = yield* connectRelay( + client, + makeRuntime({ + awaitFatal: Deferred.await(fatal) + }) + ) + const receiving = yield* Stream.runDrain( + connection.receiveWithAcknowledgement! + ).pipe(Effect.forkChild) + const sending = yield* connection.send(Uint8Array.of(9)).pipe( + Effect.forkChild + ) + yield* Deferred.await(pushStarted) + const fatalError = new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ + cause: new Error("runtime maintenance failed") + }) + }) + yield* Deferred.fail( + fatal, + fatalError + ) + const receiveExit = yield* Fiber.await(receiving) + assert.isTrue(Exit.isFailure(receiveExit)) + if (Exit.isFailure(receiveExit)) { + const error = Cause.findErrorOption(receiveExit.cause) + assert.isTrue(Option.isSome(error)) + if (Option.isSome(error)) assert.strictEqual(error.value, fatalError) + } + const sendExit = yield* Fiber.await(sending) + assert.isTrue(Exit.isFailure(sendExit)) + if (Exit.isFailure(sendExit)) { + const error = Cause.findErrorOption(sendExit.cause) + assert.isTrue(Option.isSome(error)) + if (Option.isSome(error)) assert.strictEqual(error.value, fatalError) + } + yield* Deferred.await(finalized) + const lateSendExit = yield* connection.send(Uint8Array.of(10)).pipe(Effect.exit) + assert.isTrue(Exit.isFailure(lateSendExit)) + if (Exit.isFailure(lateSendExit)) { + const error = Cause.findErrorOption(lateSendExit.cause) + assert.isTrue(Option.isSome(error)) + if (Option.isSome(error)) assert.strictEqual(error.value, fatalError) + } + }))) + + it.effect("records relay boundaries without endpoint payload or claim values", () => { + const spans: Array = [] + const tracer = Tracer.make({ + span: (options) => { + const span = new Tracer.NativeSpan(options) + spans.push(span) + return span + } + }) + return Effect.scoped(Effect.gen(function*() { + const client = makeRelayClient( + () => Stream.concat(Stream.make(relayOpened), Stream.never) + ) + const connection = yield* connectRelay(client, makeRuntime()) + yield* connection.send(Uint8Array.of(201, 202, 203)) + yield* connection.close + const telemetry = JSON.stringify( + spans + .filter((span) => span.name.startsWith("effect_local_rpc.adapter.relay_")) + .map((span) => ({ + name: span.name, + attributes: [...span.attributes] + })) + ) + (yield* Metric.dump) + for ( + const forbidden of [ + relayPeerId, + localPeerId, + remotePeerId, + sessionId, + relayMessageId, + claimToken, + documentId, + "local-subject", + "remote-subject", + "201,202,203" + ] + ) { + assert.notInclude(telemetry, forbidden) + } + })).pipe( + Effect.provideService(Metric.MetricRegistry, new Map()), + Effect.provideService(Tracer.Tracer, tracer) + ) + }) +}) diff --git a/packages/local-sql/src/BackupStore.ts b/packages/local-sql/src/BackupStore.ts index 6f88023..ae8a96f 100644 --- a/packages/local-sql/src/BackupStore.ts +++ b/packages/local-sql/src/BackupStore.ts @@ -781,6 +781,12 @@ export const layer = (definition: ReplicaDefinition.Any): Layer.Layer< ${options.installationId}, ${options.mode}, ${manifestEnvelope.checksum}, ${DateTime.formatIso(yield* DateTime.now)}, ${restoredPermit.incarnation} )` + yield* sql`DELETE FROM effect_local_peer_relay_outbox` + yield* sql`DELETE FROM effect_local_peer_relay_outbox_remote_usage` + yield* sql`DELETE FROM effect_local_peer_relay_outbox_replica_usage` + yield* sql`DELETE FROM effect_local_peer_receipts + WHERE relay_message_id IS NOT NULL` + yield* sql`DELETE FROM effect_local_peer_relay_receipt_usage` const clusterTables = yield* findClusterTables(undefined) if (clusterTables.some((table) => table.name === `${ClusterStorage.messagePrefix}_replies`)) { yield* sql`DELETE FROM ${sql(`${ClusterStorage.messagePrefix}_replies`)}` diff --git a/packages/local-sql/src/DocumentEntity.ts b/packages/local-sql/src/DocumentEntity.ts index dde13b2..8177766 100644 --- a/packages/local-sql/src/DocumentEntity.ts +++ b/packages/local-sql/src/DocumentEntity.ts @@ -45,16 +45,43 @@ const syncPrimaryKey = (payload: { readonly messageHash: string readonly lineage?: Identity.DocumentLineage readonly writerProvenance: ReadonlyArray + readonly relay?: PeerSync.RelayReceipt | undefined }) => - JSON.stringify([ - payload.replicaIncarnation, - payload.peerId, - payload.connectionEpoch, - payload.receiveSequence, - payload.messageHash, - WriterProvenance.canonicalize(payload.writerProvenance), - payload.lineage ?? Identity.genesisLineage - ]) + payload.relay === undefined + ? JSON.stringify([ + payload.replicaIncarnation, + payload.peerId, + payload.connectionEpoch, + payload.receiveSequence, + payload.messageHash, + WriterProvenance.canonicalize(payload.writerProvenance), + payload.lineage ?? Identity.genesisLineage + ]) + : JSON.stringify([ + "Relay", + payload.replicaIncarnation, + payload.relay.relayPeerId, + payload.relay.senderTenantId, + payload.relay.senderSubjectId, + payload.relay.senderPeerId, + payload.relay.senderReplicaIncarnation, + payload.relay.relayMessageId, + payload.relay.messageHash, + payload.relay.outerEnvelopeDigest + ]) + +const RelayReceipt = Schema.Struct({ + relayMessageId: Identity.RelayMessageId, + relayPeerId: Identity.PeerId, + senderTenantId: Schema.NonEmptyString, + senderSubjectId: Schema.NonEmptyString, + senderPeerId: Identity.PeerId, + senderReplicaIncarnation: Identity.ReplicaIncarnation, + messageHash: Schema.String.check(Schema.isPattern(/^[0-9a-f]{64}$/)), + outerEnvelopeDigest: Schema.String.check(Schema.isPattern(/^[0-9a-f]{64}$/)), + receiptExpiresAt: Schema.NonEmptyString, + encodedSize: Schema.Int.check(Schema.isGreaterThan(0)) +}) export const Create = Rpc.make("Create", { payload: { ...commandFields, payload: Schema.Uint8ArrayFromBase64 }, @@ -115,7 +142,8 @@ export const ApplySync = Rpc.make("ApplySync", { // previous build has no lineage key, and a required field would make it fail to decode on // replay instead of replaying as the genesis lineage it was written under. lineage: Schema.optionalKey(Identity.DocumentLineage), - writerProvenance: WriterProvenance.ChangeProvenances + writerProvenance: WriterProvenance.ChangeProvenances, + relay: Schema.optional(RelayReceipt) }, success: ApplySyncResult, error: ReplicaError.ReplicaError, @@ -311,7 +339,8 @@ export const layer = (definition: ReplicaDefinition.Any): Layer.Layer< receiveSequence: request.payload.receiveSequence, lineage: request.payload.lineage ?? Identity.genesisLineage, message: request.payload.message, - writerProvenance: request.payload.writerProvenance + writerProvenance: request.payload.writerProvenance, + ...(request.payload.relay === undefined ? {} : { relay: request.payload.relay }) } ) }) diff --git a/packages/local-sql/src/DurableRuntime.ts b/packages/local-sql/src/DurableRuntime.ts index 520f4ab..4f05b76 100644 --- a/packages/local-sql/src/DurableRuntime.ts +++ b/packages/local-sql/src/DurableRuntime.ts @@ -9,9 +9,10 @@ import * as PeerSync from "./PeerSync.js" import * as ReplicaBootstrap from "./ReplicaBootstrap.js" import * as ReplicaWorkflow from "./ReplicaWorkflow.js" -export const layerWith = ( +const layerWithPeerSync = ( definition: ReplicaDefinition.Any, - workflowRegistrations: Layer.Layer + workflowRegistrations: Layer.Layer, + peerSync: Layer.Layer ) => Layer.unwrap(Effect.gen(function*() { yield* ReplicaBootstrap.ReplicaBootstrap @@ -29,13 +30,22 @@ export const layerWith = ( ) // `PeerSync` is provided to both branches, not just the entity. A workflow that changes what a // document's history is has to be able to reach the in-memory sync state keyed on that document, - // and it is reachable only from whatever branch the layer is provided into. `PeerSync.layer` - // requires no cluster service, so hoisting it above `cluster` introduces no cycle. + // and it is reachable only from whatever branch the layer is provided into. The selected + // `peerSync` layer requires no cluster service, so hoisting it above `cluster` introduces no + // cycle and preserves the relay-specific implementation. return Layer.merge(DocumentEntity.layer(definition), workflows) .pipe( - Layer.provideMerge(PeerSync.layer), + Layer.provideMerge(peerSync), Layer.provideMerge(cluster) ) })) +export const layerWith = ( + definition: ReplicaDefinition.Any, + workflowRegistrations: Layer.Layer +) => layerWithPeerSync(definition, workflowRegistrations, PeerSync.layer) + export const layer = (definition: ReplicaDefinition.Any) => layerWith(definition, Layer.empty) + +export const layerRelay = (definition: ReplicaDefinition.Any) => + layerWithPeerSync(definition, Layer.empty, PeerSync.layerRelay) diff --git a/packages/local-sql/src/Migrations.ts b/packages/local-sql/src/Migrations.ts index 5acdb3b..b674ec0 100644 --- a/packages/local-sql/src/Migrations.ts +++ b/packages/local-sql/src/Migrations.ts @@ -18,6 +18,7 @@ export const peerWriterProvenanceChecksum = "sha256:effect-local-peer-writer-pro export const replicaHealthIndexesChecksum = "sha256:effect-local-replica-health-indexes-v1" export const documentLineageChecksum = "sha256:effect-local-document-lineage-v1" export const historyRewriteMarkersChecksum = "sha256:effect-local-history-rewrite-markers-v1" +export const peerRelayStateChecksum = "sha256:effect-local-peer-relay-state-v2" const migration = Effect.gen(function*() { const sql = yield* SqlClient.SqlClient @@ -422,6 +423,203 @@ const historyRewriteMarkersMigration = Effect.gen(function*() { VALUES (9, 'history_rewrite_markers', ${historyRewriteMarkersChecksum})` }) +const peerRelayStateMigration = Effect.gen(function*() { + const sql = yield* SqlClient.SqlClient + yield* sql`CREATE TABLE effect_local_peer_receipts_relay ( + row_id INTEGER PRIMARY KEY AUTOINCREMENT, + replica_incarnation INTEGER NOT NULL, + peer_id TEXT NOT NULL, + connection_epoch TEXT NOT NULL, + receive_sequence INTEGER NOT NULL, + document_id TEXT NOT NULL REFERENCES effect_local_documents(document_id) ON DELETE CASCADE, + message_hash TEXT NOT NULL, + reply BLOB, + reply_hash TEXT, + pending_message BLOB, + heads TEXT NOT NULL, + accepted_heads TEXT NOT NULL, + commit_sequence INTEGER NOT NULL, + accepted_at TEXT NOT NULL, + writer_provenance TEXT NOT NULL DEFAULT '[]', + relay_sender_tenant_id TEXT, + relay_sender_subject_id TEXT, + relay_sender_peer_id TEXT, + relay_message_id TEXT, + relay_outer_envelope_digest TEXT, + relay_receipt_expires_at TEXT, + relay_encoded_size INTEGER, + CHECK ( + ( + relay_sender_tenant_id IS NULL + AND relay_sender_subject_id IS NULL + AND relay_sender_peer_id IS NULL + AND relay_message_id IS NULL + AND relay_outer_envelope_digest IS NULL + AND relay_receipt_expires_at IS NULL + AND relay_encoded_size IS NULL + ) + OR ( + relay_sender_tenant_id IS NOT NULL + AND length(relay_sender_tenant_id) > 0 + AND relay_sender_subject_id IS NOT NULL + AND length(relay_sender_subject_id) > 0 + AND relay_sender_peer_id IS NOT NULL + AND length(relay_sender_peer_id) > 0 + AND relay_message_id IS NOT NULL + AND length(relay_message_id) > 0 + AND relay_outer_envelope_digest IS NOT NULL + AND length(relay_outer_envelope_digest) = 64 + AND relay_receipt_expires_at IS NOT NULL + AND length(relay_receipt_expires_at) > 0 + AND relay_encoded_size > 0 + ) + ) + )` + yield* sql`INSERT INTO effect_local_peer_receipts_relay ( + replica_incarnation, peer_id, connection_epoch, receive_sequence, document_id, + message_hash, reply, reply_hash, pending_message, heads, accepted_heads, + commit_sequence, accepted_at, writer_provenance + ) + SELECT + replica_incarnation, peer_id, connection_epoch, receive_sequence, document_id, + message_hash, reply, reply_hash, pending_message, heads, accepted_heads, + commit_sequence, accepted_at, writer_provenance + FROM effect_local_peer_receipts` + yield* sql`DROP TABLE effect_local_peer_receipts` + yield* sql`ALTER TABLE effect_local_peer_receipts_relay + RENAME TO effect_local_peer_receipts` + yield* sql`CREATE UNIQUE INDEX effect_local_peer_receipts_direct_identity + ON effect_local_peer_receipts( + replica_incarnation, + peer_id, + connection_epoch, + receive_sequence + ) + WHERE relay_message_id IS NULL` + yield* sql`CREATE UNIQUE INDEX effect_local_peer_receipts_relay_identity + ON effect_local_peer_receipts( + replica_incarnation, + relay_sender_tenant_id, + relay_sender_subject_id, + relay_sender_peer_id, + relay_message_id + ) + WHERE relay_message_id IS NOT NULL` + yield* sql`CREATE INDEX effect_local_peer_receipts_document_sequence + ON effect_local_peer_receipts(document_id, commit_sequence)` + yield* sql`CREATE INDEX effect_local_peer_receipts_incarnation_accepted + ON effect_local_peer_receipts(replica_incarnation, accepted_at)` + yield* sql`CREATE INDEX effect_local_peer_receipts_pending_document + ON effect_local_peer_receipts(replica_incarnation, document_id) + WHERE pending_message IS NOT NULL` + yield* sql`CREATE INDEX effect_local_peer_receipts_pending_peer + ON effect_local_peer_receipts(replica_incarnation, peer_id) + WHERE pending_message IS NOT NULL` + yield* sql`CREATE INDEX effect_local_peer_receipts_relay_expiry + ON effect_local_peer_receipts( + replica_incarnation, + relay_receipt_expires_at, + relay_sender_tenant_id, + relay_sender_subject_id, + relay_sender_peer_id, + relay_message_id, + row_id + ) + WHERE relay_message_id IS NOT NULL` + yield* sql`CREATE TABLE effect_local_peer_relay_receipt_usage ( + replica_incarnation INTEGER NOT NULL CHECK (replica_incarnation >= 0), + sender_tenant_id TEXT NOT NULL CHECK (length(sender_tenant_id) > 0), + sender_subject_id TEXT NOT NULL CHECK (length(sender_subject_id) > 0), + sender_peer_id TEXT NOT NULL CHECK (length(sender_peer_id) > 0), + receipt_count INTEGER NOT NULL CHECK (receipt_count >= 0), + encoded_bytes INTEGER NOT NULL CHECK (encoded_bytes >= 0), + PRIMARY KEY(replica_incarnation, sender_tenant_id, sender_subject_id, sender_peer_id), + CHECK ( + (receipt_count = 0 AND encoded_bytes = 0) + OR (receipt_count > 0 AND encoded_bytes > 0) + ) + )` + yield* sql`CREATE TABLE effect_local_peer_relay_outbox ( + row_id INTEGER PRIMARY KEY AUTOINCREMENT, + replica_id TEXT NOT NULL CHECK (length(replica_id) > 0), + replica_incarnation INTEGER NOT NULL CHECK (replica_incarnation >= 0), + writer_generation INTEGER NOT NULL CHECK (writer_generation >= 0), + expected_local_tenant_id TEXT NOT NULL CHECK (length(expected_local_tenant_id) > 0), + expected_local_subject_id TEXT NOT NULL CHECK (length(expected_local_subject_id) > 0), + expected_local_peer_id TEXT NOT NULL CHECK (length(expected_local_peer_id) > 0), + remote_tenant_id TEXT NOT NULL CHECK (length(remote_tenant_id) > 0), + remote_subject_id TEXT NOT NULL CHECK (length(remote_subject_id) > 0), + remote_peer_id TEXT NOT NULL CHECK (length(remote_peer_id) > 0), + relay_peer_id TEXT NOT NULL CHECK (length(relay_peer_id) > 0), + relay_message_id TEXT NOT NULL CHECK (length(relay_message_id) > 0), + outer_envelope_digest TEXT NOT NULL CHECK (length(outer_envelope_digest) = 64), + protocol_version INTEGER NOT NULL CHECK (protocol_version = 3), + payload_version INTEGER NOT NULL CHECK (payload_version = 1), + sender_connection_epoch TEXT NOT NULL CHECK (length(sender_connection_epoch) > 0), + sender_sequence INTEGER NOT NULL CHECK (sender_sequence >= 0), + document_id TEXT NOT NULL REFERENCES effect_local_documents(document_id) ON DELETE RESTRICT, + document_type TEXT NOT NULL CHECK (length(document_type) > 0), + writer_provenance TEXT NOT NULL, + message_hash TEXT NOT NULL CHECK (length(message_hash) = 64), + payload BLOB NOT NULL, + encoded_size INTEGER NOT NULL CHECK (encoded_size > 0 AND encoded_size = length(payload)), + created_at TEXT NOT NULL CHECK (length(created_at) > 0), + retry_deadline TEXT NOT NULL CHECK (length(retry_deadline) > 0), + next_attempt_at TEXT NOT NULL CHECK (length(next_attempt_at) > 0), + custody_state TEXT NOT NULL CHECK (custody_state IN ('Pending', 'InFlight')), + CHECK (expected_local_tenant_id = remote_tenant_id), + UNIQUE(replica_id, replica_incarnation, relay_message_id), + UNIQUE( + replica_id, + replica_incarnation, + relay_peer_id, + remote_tenant_id, + remote_subject_id, + remote_peer_id, + sender_connection_epoch, + sender_sequence + ) + )` + yield* sql`CREATE INDEX effect_local_peer_relay_outbox_due_endpoint + ON effect_local_peer_relay_outbox( + replica_id, + replica_incarnation, + relay_peer_id, + remote_tenant_id, + remote_subject_id, + remote_peer_id, + custody_state, + next_attempt_at, + row_id + )` + yield* sql`CREATE INDEX effect_local_peer_relay_outbox_retry_deadline + ON effect_local_peer_relay_outbox(replica_id, replica_incarnation, retry_deadline, row_id)` + yield* sql`CREATE TABLE effect_local_peer_relay_outbox_remote_usage ( + replica_incarnation INTEGER NOT NULL CHECK (replica_incarnation >= 0), + remote_tenant_id TEXT NOT NULL CHECK (length(remote_tenant_id) > 0), + remote_subject_id TEXT NOT NULL CHECK (length(remote_subject_id) > 0), + remote_peer_id TEXT NOT NULL CHECK (length(remote_peer_id) > 0), + message_count INTEGER NOT NULL CHECK (message_count >= 0), + encoded_bytes INTEGER NOT NULL CHECK (encoded_bytes >= 0), + PRIMARY KEY(replica_incarnation, remote_tenant_id, remote_subject_id, remote_peer_id), + CHECK ( + (message_count = 0 AND encoded_bytes = 0) + OR (message_count > 0 AND encoded_bytes > 0) + ) + )` + yield* sql`CREATE TABLE effect_local_peer_relay_outbox_replica_usage ( + replica_incarnation INTEGER PRIMARY KEY CHECK (replica_incarnation >= 0), + message_count INTEGER NOT NULL CHECK (message_count >= 0), + encoded_bytes INTEGER NOT NULL CHECK (encoded_bytes >= 0), + CHECK ( + (message_count = 0 AND encoded_bytes = 0) + OR (message_count > 0 AND encoded_bytes > 0) + ) + )` + yield* sql`INSERT INTO effect_local_migration_catalog (migration_id, name, checksum) + VALUES (10, 'peer_relay_state', ${peerRelayStateChecksum})` +}) + export const loader = Migrator.fromRecord({ "1_canonical_store": migration, "2_peer_sync": peerSyncMigration, @@ -431,7 +629,8 @@ export const loader = Migrator.fromRecord({ "6_peer_writer_provenance": peerWriterProvenanceMigration, "7_replica_health_indexes": replicaHealthIndexesMigration, "8_document_lineage": documentLineageMigration, - "9_history_rewrite_markers": historyRewriteMarkersMigration + "9_history_rewrite_markers": historyRewriteMarkersMigration, + "10_peer_relay_state": peerRelayStateMigration }) const migrate = Migrator.make({})({ loader, table: "effect_local_migrations" }) @@ -473,6 +672,12 @@ export const run = Effect.gen(function*() { name: "history_rewrite_markers", checksum: historyRewriteMarkersChecksum, label: "History rewrite markers" + }, + { + id: 10, + name: "peer_relay_state", + checksum: peerRelayStateChecksum, + label: "Peer relay state" } ] as const // One transaction over migrate + validation so a rejected catalog rolls back diff --git a/packages/local-sql/src/PeerRelayClientRuntime.ts b/packages/local-sql/src/PeerRelayClientRuntime.ts new file mode 100644 index 0000000..bfb3a68 --- /dev/null +++ b/packages/local-sql/src/PeerRelayClientRuntime.ts @@ -0,0 +1,203 @@ +import type * as Identity from "@lucas-barake/effect-local/Identity" +import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" +import * as Cause from "effect/Cause" +import * as Context from "effect/Context" +import * as Deferred from "effect/Deferred" +import * as Effect from "effect/Effect" +import * as Exit from "effect/Exit" +import * as Fiber from "effect/Fiber" +import * as Layer from "effect/Layer" +import * as Queue from "effect/Queue" +import * as Ref from "effect/Ref" +import * as OutboxMaintenance from "./internal/peerRelayOutboxMaintenance.js" +import * as ReceiptMaintenance from "./internal/peerRelayReceiptMaintenance.js" +import * as PeerRelayOutbox from "./PeerRelayOutbox.js" +import * as PeerRelayOutboxLimits from "./PeerRelayOutboxLimits.js" +import * as PeerRelayReceiptLimits from "./PeerRelayReceiptLimits.js" +import * as PeerSync from "./PeerSync.js" +import * as ReplicaGate from "./ReplicaGate.js" + +export interface ConnectionConfiguration { + readonly replicaIncarnation: Identity.ReplicaIncarnation + readonly retryHorizonMillis: number + readonly replayBatchSize: number +} + +type RuntimeState = + | { readonly _tag: "Running" } + | { readonly _tag: "Closing" } + | { + readonly _tag: "Failed" + readonly cause: Cause.Cause + } + +export class PeerRelayClientRuntime extends Context.Service Effect.Effect + readonly signalReceiptPrune: Effect.Effect + readonly health: Effect.Effect + readonly awaitFatal: Effect.Effect +}>()("@lucas-barake/effect-local-sql/PeerRelayClientRuntime") {} + +const protocolMismatch = (expected: string, observed: string) => + new ReplicaError.ReplicaError({ + reason: new ReplicaError.ProtocolMismatch({ expected, observed }) + }) + +const unexpectedExit = ( + name: string, + exit: Exit.Exit +): Cause.Cause => + Exit.isFailure(exit) + ? exit.cause + : Cause.die(new Error(`${name} stopped unexpectedly`)) + +export const makeScoped = Effect.gen(function*() { + const outbox = yield* PeerRelayOutbox.PeerRelayOutbox + const sync = yield* PeerSync.PeerSync + const gate = yield* ReplicaGate.ReplicaGate + const receiptLimits = yield* PeerRelayReceiptLimits.PeerRelayReceiptLimits + const outboxLimits = yield* PeerRelayOutboxLimits.PeerRelayOutboxLimits + const pruneRelayReceipts = sync.pruneRelayReceipts + if (pruneRelayReceipts === undefined) { + return yield* Effect.fail( + protocolMismatch("relay enabled PeerSync", "direct PeerSync") + ) + } + const state = yield* Ref.make({ _tag: "Running" }) + const fatal = yield* Deferred.make>() + const receiptWakeup = yield* Effect.acquireRelease( + Queue.dropping(1), + Queue.shutdown + ) + + const awaitFatal: Effect.Effect = Deferred.await(fatal).pipe( + Effect.flatMap(Effect.failCause) + ) + + const health = Ref.get(state).pipe( + Effect.flatMap((current) => { + switch (current._tag) { + case "Running": + return Effect.void + case "Failed": + return Effect.failCause(current.cause) + case "Closing": + return Effect.interrupt + } + }) + ) + + const guarded = ( + effect: Effect.Effect + ): Effect.Effect => + health.pipe( + Effect.andThen(effect), + Effect.raceFirst(awaitFatal) + ) + + const outboxFiber = yield* OutboxMaintenance.run({ + prune: outbox.pruneExpired, + intervalMillis: outboxLimits.maintenanceIntervalMillis, + pruneRowsPerSecond: outboxLimits.pruneRowsPerSecond + }).pipe(Effect.forkScoped({ startImmediately: true })) + + const receiptFiber = yield* ReceiptMaintenance.run({ + prune: pruneRelayReceipts, + intervalMillis: receiptLimits.maintenanceIntervalMillis, + pruneRowsPerSecond: receiptLimits.pruneRowsPerSecond, + wakeup: receiptWakeup + }).pipe(Effect.forkScoped({ startImmediately: true })) + + const failRuntime = ( + name: string, + exit: Exit.Exit, + sibling: Fiber.Fiber + ) => + Effect.gen(function*() { + const cause = unexpectedExit(name, exit) + const first = yield* Ref.modify(state, (current) => + current._tag === "Running" + ? [true, { _tag: "Failed", cause } as const] + : [false, current]) + if (!first) return + yield* Deferred.succeed(fatal, cause) + yield* Fiber.interrupt(sibling) + }) + + yield* Effect.raceFirst( + Fiber.await(outboxFiber).pipe( + Effect.flatMap((exit) => failRuntime("relay outbox maintenance", exit, receiptFiber)) + ), + Fiber.await(receiptFiber).pipe( + Effect.flatMap((exit) => failRuntime("relay receipt maintenance", exit, outboxFiber)) + ) + ).pipe(Effect.forkScoped({ startImmediately: true })) + + yield* Effect.addFinalizer(() => + Ref.update(state, (current) => current._tag === "Running" ? { _tag: "Closing" } as const : current).pipe( + Effect.andThen(Deferred.interrupt(fatal)), + Effect.asVoid + ) + ) + + const validateConnectionConfiguration = (input: ConnectionConfiguration) => + guarded(Effect.gen(function*() { + yield* outbox.validateReplicaIncarnation(input.replicaIncarnation) + if ( + !Number.isSafeInteger(input.retryHorizonMillis) || + input.retryHorizonMillis <= 0 || + input.retryHorizonMillis > outboxLimits.maxRetryHorizonMillis + ) { + return yield* Effect.fail( + protocolMismatch( + `retry horizon from 1 through ${outboxLimits.maxRetryHorizonMillis}`, + "invalid retry horizon" + ) + ) + } + if ( + !Number.isSafeInteger(input.replayBatchSize) || + input.replayBatchSize <= 0 || + input.replayBatchSize > outboxLimits.maxMessagesPerRemote + ) { + return yield* Effect.fail( + protocolMismatch( + `replay batch from 1 through ${outboxLimits.maxMessagesPerRemote}`, + "invalid replay batch" + ) + ) + } + const permit = yield* gate.current + if (permit.incarnation !== input.replicaIncarnation) { + return yield* Effect.fail( + protocolMismatch("current replica incarnation", "stale replica incarnation") + ) + } + })) + + return { + admit: (input: PeerRelayOutbox.AdmitInput) => guarded(outbox.admit(input)), + dueForEndpoint: (input: PeerRelayOutbox.ReplayInput) => guarded(outbox.dueForEndpoint(input)), + maximumPendingHorizon: (input: PeerRelayOutbox.Endpoint) => guarded(outbox.maximumPendingHorizon(input)), + markCustody: (input: PeerRelayOutbox.CustodyInput) => guarded(outbox.markCustody(input)), + validateReplicaIncarnation: (expected: Identity.ReplicaIncarnation) => + guarded(outbox.validateReplicaIncarnation(expected)), + validateConnectionConfiguration, + signalReceiptPrune: health.pipe( + Effect.andThen(Queue.offer(receiptWakeup, undefined)), + Effect.asVoid, + Effect.raceFirst(awaitFatal) + ), + health, + awaitFatal + } +}) + +export const layer = Layer.effect(PeerRelayClientRuntime, makeScoped) diff --git a/packages/local-sql/src/PeerRelayOutbox.ts b/packages/local-sql/src/PeerRelayOutbox.ts new file mode 100644 index 0000000..bbfdf45 --- /dev/null +++ b/packages/local-sql/src/PeerRelayOutbox.ts @@ -0,0 +1,935 @@ +import * as Identity from "@lucas-barake/effect-local/Identity" +import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" +import * as ReplicaLimits from "@lucas-barake/effect-local/ReplicaLimits" +import * as Clock from "effect/Clock" +import * as Context from "effect/Context" +import * as Crypto from "effect/Crypto" +import * as Effect from "effect/Effect" +import * as Layer from "effect/Layer" +import * as Schema from "effect/Schema" +import type * as SchemaError from "effect/SchemaError" +import * as SqlClient from "effect/unstable/sql/SqlClient" +import type * as SqlError from "effect/unstable/sql/SqlError" +import * as SqlSchema from "effect/unstable/sql/SqlSchema" +import * as WriterProvenance from "./internal/writerProvenance.js" +import * as PeerRelayOutboxLimits from "./PeerRelayOutboxLimits.js" +import * as PeerSyncEnvelope from "./PeerSyncEnvelope.js" +import * as ReplicaGate from "./ReplicaGate.js" + +export interface Endpoint { + readonly expectedLocal: PeerSyncEnvelope.RelayPeerPrincipal + readonly remote: PeerSyncEnvelope.RelayPeerPrincipal + readonly relayPeerId: Identity.PeerId +} + +export interface AdmitInput extends Endpoint { + readonly payload: Uint8Array + readonly retryHorizonMillis: number +} + +export interface ReplayInput extends Endpoint { + readonly maximum: number +} + +export interface CustodyInput { + readonly relayMessageId: Identity.RelayMessageId + readonly outerEnvelopeDigest: string +} + +export interface Entry extends Endpoint { + readonly rowId: number + readonly replicaId: Identity.ReplicaId + readonly replicaIncarnation: Identity.ReplicaIncarnation + readonly writerGeneration: Identity.WriterGeneration + readonly relayMessageId: Identity.RelayMessageId + readonly outerEnvelopeDigest: string + readonly protocolVersion: 3 + readonly payloadVersion: 1 + readonly senderConnectionEpoch: string + readonly senderSequence: number + readonly document: { + readonly documentId: Identity.DocumentId + readonly documentType: string + } + readonly writerProvenance: PeerSyncEnvelope.RelayOuterEnvelope["writerProvenance"] + readonly messageHash: string + readonly payload: Uint8Array + readonly encodedSize: number + readonly createdAt: string + readonly retryDeadline: string + readonly nextAttemptAt: string +} + +export interface Usage { + readonly remote: { + readonly messageCount: number + readonly encodedBytes: number + } + readonly replica: { + readonly messageCount: number + readonly encodedBytes: number + } +} + +const RelayDigest = Schema.String.check(Schema.isPattern(/^[0-9a-f]{64}$/)) +const IsoDate = Schema.String +const PositiveInt = Schema.Int.check(Schema.isGreaterThan(0)) +const NonNegativeInt = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)) + +const EndpointSchema = Schema.Struct({ + expectedLocal: PeerSyncEnvelope.RelayPeerPrincipal, + remote: PeerSyncEnvelope.RelayPeerPrincipal, + relayPeerId: Identity.PeerId +}) + +const Row = Schema.Struct({ + row_id: PositiveInt, + replica_id: Identity.ReplicaId, + replica_incarnation: Identity.ReplicaIncarnation, + writer_generation: Identity.WriterGeneration, + expected_local_tenant_id: Schema.String, + expected_local_subject_id: Schema.String, + expected_local_peer_id: Identity.PeerId, + remote_tenant_id: Schema.String, + remote_subject_id: Schema.String, + remote_peer_id: Identity.PeerId, + relay_peer_id: Identity.PeerId, + relay_message_id: Identity.RelayMessageId, + outer_envelope_digest: RelayDigest, + protocol_version: Schema.Literal(3), + payload_version: Schema.Literal(1), + sender_connection_epoch: Schema.String, + sender_sequence: NonNegativeInt, + document_id: Identity.DocumentId, + document_type: Schema.String, + writer_provenance: WriterProvenance.StoredChangeProvenances, + message_hash: RelayDigest, + payload: Schema.Uint8Array, + encoded_size: PositiveInt, + created_at: IsoDate, + retry_deadline: IsoDate, + next_attempt_at: IsoDate, + custody_state: Schema.Literals(["Pending", "InFlight"]) +}) + +const RowMetadata = Schema.Struct({ + row_id: PositiveInt, + encoded_size: PositiveInt, + actual_size: PositiveInt +}) + +const InsertedRow = Schema.Struct({ row_id: PositiveInt }) +const UsageRow = Schema.Struct({ + message_count: NonNegativeInt, + encoded_bytes: NonNegativeInt +}) +const HorizonRow = Schema.Struct({ horizon_millis: Schema.NullOr(Schema.Number) }) + +const replicaFailure = (reason: ReplicaError.Reason): ReplicaError.ReplicaError => + new ReplicaError.ReplicaError({ reason }) + +const storageUnavailable = (cause: unknown) => + Effect.fail(replicaFailure(new ReplicaError.StorageUnavailable({ cause }))) + +const storageCorrupt = (cause: unknown) => Effect.fail(replicaFailure(new ReplicaError.StorageCorrupt({ cause }))) + +const protocolMismatch = (expected: string, observed: string) => + replicaFailure(new ReplicaError.ProtocolMismatch({ expected, observed })) + +const quotaExceeded = (resource: string, limit: number) => + replicaFailure(new ReplicaError.QuotaExceeded({ resource, limit })) + +const parseIso = (value: string): number | null => { + const millis = Date.parse(value) + return Number.isFinite(millis) && new Date(millis).toISOString() === value ? millis : null +} + +export class PeerRelayOutbox extends Context.Service Effect.Effect + readonly dueForEndpoint: ( + input: ReplayInput + ) => Effect.Effect, ReplicaError.ReplicaError> + readonly maximumPendingHorizon: ( + endpoint: Endpoint + ) => Effect.Effect + readonly markCustody: (input: CustodyInput) => Effect.Effect + readonly pruneExpired: Effect.Effect + readonly usage: (endpoint: Endpoint) => Effect.Effect + readonly validateReplicaIncarnation: ( + expected: Identity.ReplicaIncarnation + ) => Effect.Effect +}>()("@lucas-barake/effect-local-sql/PeerRelayOutbox") {} + +type Requirements = + | SqlClient.SqlClient + | Crypto.Crypto + | ReplicaLimits.ReplicaLimits + | ReplicaGate.ReplicaGate + | PeerRelayOutboxLimits.PeerRelayOutboxLimits + +const make = Effect.gen(function*() { + const sql = yield* SqlClient.SqlClient + const crypto = yield* Crypto.Crypto + const replicaLimits = yield* ReplicaLimits.ReplicaLimits + const gate = yield* ReplicaGate.ReplicaGate + const limits = yield* PeerRelayOutboxLimits.PeerRelayOutboxLimits + + const decodeEndpoint = (input: Endpoint) => + Schema.decodeUnknownEffect(EndpointSchema)(input).pipe( + Effect.mapError(() => protocolMismatch("valid relay endpoint", "invalid relay endpoint")) + ) + + const findSource = SqlSchema.findAll({ + Request: Schema.Struct({ + replicaId: Identity.ReplicaId, + replicaIncarnation: Identity.ReplicaIncarnation, + relayPeerId: Identity.PeerId, + remoteTenantId: Schema.String, + remoteSubjectId: Schema.String, + remotePeerId: Identity.PeerId, + senderConnectionEpoch: Schema.String, + senderSequence: NonNegativeInt + }), + Result: Row, + execute: (request) => + sql`SELECT * + FROM effect_local_peer_relay_outbox + WHERE replica_id = ${request.replicaId} + AND replica_incarnation = ${request.replicaIncarnation} + AND relay_peer_id = ${request.relayPeerId} + AND remote_tenant_id = ${request.remoteTenantId} + AND remote_subject_id = ${request.remoteSubjectId} + AND remote_peer_id = ${request.remotePeerId} + AND sender_connection_epoch = ${request.senderConnectionEpoch} + AND sender_sequence = ${request.senderSequence}` + }) + + const findRow = SqlSchema.findAll({ + Request: Schema.Struct({ + replicaId: Identity.ReplicaId, + replicaIncarnation: Identity.ReplicaIncarnation, + relayMessageId: Identity.RelayMessageId + }), + Result: Row, + execute: (request) => + sql`SELECT * + FROM effect_local_peer_relay_outbox + WHERE replica_id = ${request.replicaId} + AND replica_incarnation = ${request.replicaIncarnation} + AND relay_message_id = ${request.relayMessageId}` + }) + + const findDueMetadata = SqlSchema.findAll({ + Request: Schema.Struct({ + replicaId: Identity.ReplicaId, + replicaIncarnation: Identity.ReplicaIncarnation, + relayPeerId: Identity.PeerId, + expectedLocalTenantId: Schema.String, + expectedLocalSubjectId: Schema.String, + expectedLocalPeerId: Identity.PeerId, + remoteTenantId: Schema.String, + remoteSubjectId: Schema.String, + remotePeerId: Identity.PeerId, + now: Schema.String, + maximum: PositiveInt + }), + Result: RowMetadata, + execute: (request) => + sql`SELECT row_id, encoded_size, length(payload) AS actual_size + FROM effect_local_peer_relay_outbox + WHERE replica_id = ${request.replicaId} + AND replica_incarnation = ${request.replicaIncarnation} + AND relay_peer_id = ${request.relayPeerId} + AND expected_local_tenant_id = ${request.expectedLocalTenantId} + AND expected_local_subject_id = ${request.expectedLocalSubjectId} + AND expected_local_peer_id = ${request.expectedLocalPeerId} + AND remote_tenant_id = ${request.remoteTenantId} + AND remote_subject_id = ${request.remoteSubjectId} + AND remote_peer_id = ${request.remotePeerId} + AND custody_state = 'Pending' + AND next_attempt_at <= ${request.now} + AND retry_deadline > ${request.now} + ORDER BY next_attempt_at, row_id + LIMIT ${request.maximum}` + }) + + const findByRowId = SqlSchema.findAll({ + Request: Schema.Struct({ + replicaId: Identity.ReplicaId, + replicaIncarnation: Identity.ReplicaIncarnation, + rowId: PositiveInt + }), + Result: Row, + execute: (request) => + sql`SELECT * + FROM effect_local_peer_relay_outbox + WHERE replica_id = ${request.replicaId} + AND replica_incarnation = ${request.replicaIncarnation} + AND row_id = ${request.rowId}` + }) + + const insertRow = SqlSchema.findAll({ + Request: Schema.Struct({ + replicaId: Identity.ReplicaId, + replicaIncarnation: Identity.ReplicaIncarnation, + writerGeneration: Identity.WriterGeneration, + expectedLocalTenantId: Schema.String, + expectedLocalSubjectId: Schema.String, + expectedLocalPeerId: Identity.PeerId, + remoteTenantId: Schema.String, + remoteSubjectId: Schema.String, + remotePeerId: Identity.PeerId, + relayPeerId: Identity.PeerId, + relayMessageId: Identity.RelayMessageId, + outerEnvelopeDigest: RelayDigest, + senderConnectionEpoch: Schema.String, + senderSequence: NonNegativeInt, + documentId: Identity.DocumentId, + documentType: Schema.String, + writerProvenance: WriterProvenance.StoredChangeProvenances, + messageHash: RelayDigest, + payload: Schema.Uint8Array, + encodedSize: PositiveInt, + createdAt: Schema.String, + retryDeadline: Schema.String, + nextAttemptAt: Schema.String + }), + Result: InsertedRow, + execute: (request) => + sql`INSERT INTO effect_local_peer_relay_outbox ( + replica_id, replica_incarnation, writer_generation, + expected_local_tenant_id, expected_local_subject_id, expected_local_peer_id, + remote_tenant_id, remote_subject_id, remote_peer_id, relay_peer_id, + relay_message_id, outer_envelope_digest, protocol_version, payload_version, + sender_connection_epoch, sender_sequence, document_id, document_type, + writer_provenance, message_hash, payload, encoded_size, + created_at, retry_deadline, next_attempt_at, custody_state + ) VALUES ( + ${request.replicaId}, ${request.replicaIncarnation}, ${request.writerGeneration}, + ${request.expectedLocalTenantId}, ${request.expectedLocalSubjectId}, ${request.expectedLocalPeerId}, + ${request.remoteTenantId}, ${request.remoteSubjectId}, ${request.remotePeerId}, ${request.relayPeerId}, + ${request.relayMessageId}, ${request.outerEnvelopeDigest}, 3, 1, + ${request.senderConnectionEpoch}, ${request.senderSequence}, ${request.documentId}, + ${request.documentType}, ${request.writerProvenance}, ${request.messageHash}, + ${request.payload}, ${request.encodedSize}, ${request.createdAt}, ${request.retryDeadline}, + ${request.nextAttemptAt}, 'Pending' + ) RETURNING row_id` + }) + + const reserveRemote = SqlSchema.findAll({ + Request: Schema.Struct({ + replicaIncarnation: Identity.ReplicaIncarnation, + remoteTenantId: Schema.String, + remoteSubjectId: Schema.String, + remotePeerId: Identity.PeerId, + encodedSize: PositiveInt, + maxMessages: PositiveInt, + maxBytes: PositiveInt + }), + Result: UsageRow, + execute: (request) => + sql`INSERT INTO effect_local_peer_relay_outbox_remote_usage ( + replica_incarnation, remote_tenant_id, remote_subject_id, remote_peer_id, + message_count, encoded_bytes + ) VALUES ( + ${request.replicaIncarnation}, ${request.remoteTenantId}, ${request.remoteSubjectId}, + ${request.remotePeerId}, 1, ${request.encodedSize} + ) + ON CONFLICT (replica_incarnation, remote_tenant_id, remote_subject_id, remote_peer_id) + DO UPDATE SET + message_count = message_count + 1, + encoded_bytes = encoded_bytes + excluded.encoded_bytes + WHERE message_count + 1 <= ${request.maxMessages} + AND encoded_bytes + excluded.encoded_bytes <= ${request.maxBytes} + RETURNING message_count, encoded_bytes` + }) + + const reserveReplica = SqlSchema.findAll({ + Request: Schema.Struct({ + replicaIncarnation: Identity.ReplicaIncarnation, + encodedSize: PositiveInt, + maxMessages: PositiveInt, + maxBytes: PositiveInt + }), + Result: UsageRow, + execute: (request) => + sql`INSERT INTO effect_local_peer_relay_outbox_replica_usage ( + replica_incarnation, message_count, encoded_bytes + ) VALUES (${request.replicaIncarnation}, 1, ${request.encodedSize}) + ON CONFLICT (replica_incarnation) + DO UPDATE SET + message_count = message_count + 1, + encoded_bytes = encoded_bytes + excluded.encoded_bytes + WHERE message_count + 1 <= ${request.maxMessages} + AND encoded_bytes + excluded.encoded_bytes <= ${request.maxBytes} + RETURNING message_count, encoded_bytes` + }) + + const decrementRemote = SqlSchema.findAll({ + Request: Schema.Struct({ + replicaIncarnation: Identity.ReplicaIncarnation, + remoteTenantId: Schema.String, + remoteSubjectId: Schema.String, + remotePeerId: Identity.PeerId, + encodedSize: PositiveInt + }), + Result: UsageRow, + execute: (request) => + sql`UPDATE effect_local_peer_relay_outbox_remote_usage + SET message_count = message_count - 1, + encoded_bytes = encoded_bytes - ${request.encodedSize} + WHERE replica_incarnation = ${request.replicaIncarnation} + AND remote_tenant_id = ${request.remoteTenantId} + AND remote_subject_id = ${request.remoteSubjectId} + AND remote_peer_id = ${request.remotePeerId} + AND message_count >= 1 + AND encoded_bytes >= ${request.encodedSize} + RETURNING message_count, encoded_bytes` + }) + + const decrementReplica = SqlSchema.findAll({ + Request: Schema.Struct({ + replicaIncarnation: Identity.ReplicaIncarnation, + encodedSize: PositiveInt + }), + Result: UsageRow, + execute: (request) => + sql`UPDATE effect_local_peer_relay_outbox_replica_usage + SET message_count = message_count - 1, + encoded_bytes = encoded_bytes - ${request.encodedSize} + WHERE replica_incarnation = ${request.replicaIncarnation} + AND message_count >= 1 + AND encoded_bytes >= ${request.encodedSize} + RETURNING message_count, encoded_bytes` + }) + + const decrementUsage = (row: typeof Row.Type) => + Effect.gen(function*() { + const remote = yield* decrementRemote({ + replicaIncarnation: row.replica_incarnation, + remoteTenantId: row.remote_tenant_id, + remoteSubjectId: row.remote_subject_id, + remotePeerId: row.remote_peer_id, + encodedSize: row.encoded_size + }) + const replica = yield* decrementReplica({ + replicaIncarnation: row.replica_incarnation, + encodedSize: row.encoded_size + }) + if (remote.length !== 1 || replica.length !== 1) { + return yield* storageCorrupt(new Error("Relay outbox usage reservation mismatch")) + } + yield* sql`DELETE FROM effect_local_peer_relay_outbox_remote_usage + WHERE replica_incarnation = ${row.replica_incarnation} + AND remote_tenant_id = ${row.remote_tenant_id} + AND remote_subject_id = ${row.remote_subject_id} + AND remote_peer_id = ${row.remote_peer_id} + AND message_count = 0 + AND encoded_bytes = 0` + yield* sql`DELETE FROM effect_local_peer_relay_outbox_replica_usage + WHERE replica_incarnation = ${row.replica_incarnation} + AND message_count = 0 + AND encoded_bytes = 0` + }) + + const validateRow = ( + row: typeof Row.Type, + permit: ReplicaGate.Permit + ): Effect.Effect => + Effect.gen(function*() { + if ( + row.replica_incarnation !== permit.incarnation || + row.replica_id !== permit.replicaId || + row.encoded_size !== row.payload.byteLength + ) { + return yield* storageCorrupt(new Error("Relay outbox metadata mismatch")) + } + const createdAt = parseIso(row.created_at) + const retryDeadline = parseIso(row.retry_deadline) + const nextAttemptAt = parseIso(row.next_attempt_at) + if ( + createdAt === null || + retryDeadline === null || + nextAttemptAt === null || + retryDeadline <= createdAt || + retryDeadline - createdAt > limits.maxRetryHorizonMillis || + nextAttemptAt < createdAt || + nextAttemptAt >= retryDeadline + ) { + return yield* storageCorrupt(new Error("Relay outbox deadline mismatch")) + } + const syncEnvelope = yield* PeerSyncEnvelope.decodeSyncEnvelope(row.payload, replicaLimits).pipe( + Effect.provideService(Crypto.Crypto, crypto) + ) + if ( + syncEnvelope.connectionEpoch !== row.sender_connection_epoch || + syncEnvelope.sequence !== row.sender_sequence || + syncEnvelope.documentId !== row.document_id || + syncEnvelope.documentType !== row.document_type || + syncEnvelope.messageHash !== row.message_hash + ) { + return yield* storageCorrupt(new Error("Relay outbox payload metadata mismatch")) + } + const outerEnvelope: PeerSyncEnvelope.RelayOuterEnvelope = { + domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, + version: PeerSyncEnvelope.relayOuterEnvelopeVersion, + expectedLocal: { + tenantId: row.expected_local_tenant_id, + subjectId: row.expected_local_subject_id, + peerId: row.expected_local_peer_id + }, + remote: { + tenantId: row.remote_tenant_id, + subjectId: row.remote_subject_id, + peerId: row.remote_peer_id + }, + relayPeerId: row.relay_peer_id, + relayMessageId: row.relay_message_id, + protocolVersion: row.protocol_version, + payloadVersion: row.payload_version, + senderReplicaIncarnation: row.replica_incarnation, + senderConnectionEpoch: row.sender_connection_epoch, + senderSequence: row.sender_sequence, + document: { + documentId: row.document_id, + documentType: row.document_type + }, + writerProvenance: row.writer_provenance, + messageHash: row.message_hash, + payload: row.payload + } + const digest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope(outerEnvelope).pipe( + Effect.provideService(Crypto.Crypto, crypto) + ) + if (digest !== row.outer_envelope_digest) { + return yield* storageCorrupt(new Error("Relay outbox digest mismatch")) + } + return { + rowId: row.row_id, + replicaId: row.replica_id, + replicaIncarnation: row.replica_incarnation, + writerGeneration: row.writer_generation, + expectedLocal: outerEnvelope.expectedLocal, + remote: outerEnvelope.remote, + relayPeerId: row.relay_peer_id, + relayMessageId: row.relay_message_id, + outerEnvelopeDigest: row.outer_envelope_digest, + protocolVersion: row.protocol_version, + payloadVersion: row.payload_version, + senderConnectionEpoch: row.sender_connection_epoch, + senderSequence: row.sender_sequence, + document: outerEnvelope.document, + writerProvenance: outerEnvelope.writerProvenance, + messageHash: row.message_hash, + payload: row.payload, + encodedSize: row.encoded_size, + createdAt: row.created_at, + retryDeadline: row.retry_deadline, + nextAttemptAt: row.next_attempt_at + } + }) + + const mapStorageFailures = ( + effect: Effect.Effect< + A, + ReplicaError.ReplicaError | SchemaError.SchemaError | SqlError.SqlError, + R + > + ): Effect.Effect => + effect.pipe( + Effect.catchTags({ + SqlError: storageUnavailable, + SchemaError: storageCorrupt + }) + ) + + const validateReplicaIncarnation = (expected: Identity.ReplicaIncarnation) => + gate.current.pipe( + Effect.flatMap((permit) => + permit.incarnation === expected + ? Effect.void + : Effect.fail(protocolMismatch("current replica incarnation", "stale replica incarnation")) + ) + ) + + const admit = (input: AdmitInput): Effect.Effect => + Effect.scoped(Effect.gen(function*() { + const endpoint = yield* decodeEndpoint(input) + if ( + !Number.isSafeInteger(input.retryHorizonMillis) || + input.retryHorizonMillis <= 0 || + input.retryHorizonMillis > limits.maxRetryHorizonMillis + ) { + return yield* Effect.fail( + protocolMismatch( + `retry horizon from 1 through ${limits.maxRetryHorizonMillis}`, + "invalid retry horizon" + ) + ) + } + const permit = yield* gate.shared + const syncEnvelope = yield* PeerSyncEnvelope.decodeSyncEnvelope(input.payload, replicaLimits).pipe( + Effect.provideService(Crypto.Crypto, crypto) + ) + const relayMessageId = yield* Identity.makeRelayMessageId.pipe( + Effect.provideService(Crypto.Crypto, crypto), + Effect.mapError((cause) => replicaFailure(new ReplicaError.StorageUnavailable({ cause }))) + ) + const outerEnvelope: PeerSyncEnvelope.RelayOuterEnvelope = { + domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, + version: PeerSyncEnvelope.relayOuterEnvelopeVersion, + expectedLocal: endpoint.expectedLocal, + remote: endpoint.remote, + relayPeerId: endpoint.relayPeerId, + relayMessageId, + protocolVersion: 3, + payloadVersion: PeerSyncEnvelope.syncEnvelopeVersion, + senderReplicaIncarnation: permit.incarnation, + senderConnectionEpoch: syncEnvelope.connectionEpoch, + senderSequence: syncEnvelope.sequence, + document: PeerSyncEnvelope.syncEnvelopeDocument(syncEnvelope), + writerProvenance: syncEnvelope.writerProvenance, + messageHash: syncEnvelope.messageHash, + payload: input.payload + } + const outerEnvelopeDigest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope(outerEnvelope).pipe( + Effect.provideService(Crypto.Crypto, crypto) + ) + const nowMillis = yield* Clock.currentTimeMillis + const createdAt = new Date(nowMillis).toISOString() + const retryDeadline = new Date(nowMillis + input.retryHorizonMillis).toISOString() + const request = { + replicaId: permit.replicaId, + replicaIncarnation: permit.incarnation, + relayPeerId: endpoint.relayPeerId, + remoteTenantId: endpoint.remote.tenantId, + remoteSubjectId: endpoint.remote.subjectId, + remotePeerId: endpoint.remote.peerId, + senderConnectionEpoch: syncEnvelope.connectionEpoch, + senderSequence: syncEnvelope.sequence + } + return yield* mapStorageFailures(sql.withTransaction(Effect.gen(function*() { + const existing = yield* findSource(request) + if (existing.length > 1) { + return yield* storageCorrupt(new Error("Duplicate relay source operation")) + } + if (existing.length === 1) { + const entry = yield* validateRow(existing[0]!, permit) + const existingEnvelope: PeerSyncEnvelope.RelayOuterEnvelope = { + ...outerEnvelope, + relayMessageId: entry.relayMessageId + } + const existingDigest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope(existingEnvelope).pipe( + Effect.provideService(Crypto.Crypto, crypto) + ) + const existingCreatedAt = parseIso(entry.createdAt) + const existingRetryDeadline = parseIso(entry.retryDeadline) + if ( + entry.outerEnvelopeDigest !== existingDigest || + existingCreatedAt === null || + existingRetryDeadline === null || + existingRetryDeadline - existingCreatedAt !== input.retryHorizonMillis + ) { + return yield* Effect.fail( + protocolMismatch("stable relay source operation", "conflicting relay source operation") + ) + } + yield* gate.validate(permit) + return entry + } + const remoteUsage = yield* reserveRemote({ + replicaIncarnation: permit.incarnation, + remoteTenantId: endpoint.remote.tenantId, + remoteSubjectId: endpoint.remote.subjectId, + remotePeerId: endpoint.remote.peerId, + encodedSize: input.payload.byteLength, + maxMessages: limits.maxMessagesPerRemote, + maxBytes: limits.maxEncodedBytesPerRemote + }) + if (remoteUsage.length !== 1) { + return yield* Effect.fail( + quotaExceeded("relay outbox remote quota", limits.maxMessagesPerRemote) + ) + } + const replicaUsage = yield* reserveReplica({ + replicaIncarnation: permit.incarnation, + encodedSize: input.payload.byteLength, + maxMessages: limits.maxMessagesPerReplica, + maxBytes: limits.maxEncodedBytesPerReplica + }) + if (replicaUsage.length !== 1) { + return yield* Effect.fail( + quotaExceeded("relay outbox replica quota", limits.maxMessagesPerReplica) + ) + } + const inserted = yield* insertRow({ + replicaIncarnation: permit.incarnation, + replicaId: permit.replicaId, + writerGeneration: permit.writerGeneration, + expectedLocalTenantId: endpoint.expectedLocal.tenantId, + expectedLocalSubjectId: endpoint.expectedLocal.subjectId, + expectedLocalPeerId: endpoint.expectedLocal.peerId, + remoteTenantId: endpoint.remote.tenantId, + remoteSubjectId: endpoint.remote.subjectId, + remotePeerId: endpoint.remote.peerId, + relayPeerId: endpoint.relayPeerId, + relayMessageId, + outerEnvelopeDigest, + senderConnectionEpoch: syncEnvelope.connectionEpoch, + senderSequence: syncEnvelope.sequence, + documentId: syncEnvelope.documentId, + documentType: syncEnvelope.documentType, + writerProvenance: syncEnvelope.writerProvenance, + messageHash: syncEnvelope.messageHash, + payload: input.payload, + encodedSize: input.payload.byteLength, + createdAt, + retryDeadline, + nextAttemptAt: createdAt + }) + if (inserted.length !== 1) { + return yield* storageCorrupt(new Error("Relay outbox insert did not return one row")) + } + yield* gate.validate(permit) + return { + rowId: inserted[0]!.row_id, + replicaId: permit.replicaId, + replicaIncarnation: permit.incarnation, + writerGeneration: permit.writerGeneration, + expectedLocal: endpoint.expectedLocal, + remote: endpoint.remote, + relayPeerId: endpoint.relayPeerId, + relayMessageId, + outerEnvelopeDigest, + protocolVersion: 3 as const, + payloadVersion: 1 as const, + senderConnectionEpoch: syncEnvelope.connectionEpoch, + senderSequence: syncEnvelope.sequence, + document: PeerSyncEnvelope.syncEnvelopeDocument(syncEnvelope), + writerProvenance: syncEnvelope.writerProvenance, + messageHash: syncEnvelope.messageHash, + payload: input.payload, + encodedSize: input.payload.byteLength, + createdAt, + retryDeadline, + nextAttemptAt: createdAt + } + }))) + })) + + const dueForEndpoint = (input: ReplayInput) => + Effect.scoped(Effect.gen(function*() { + const endpoint = yield* decodeEndpoint(input) + if ( + !Number.isSafeInteger(input.maximum) || + input.maximum <= 0 || + input.maximum > limits.maxMessagesPerRemote + ) { + return yield* Effect.fail( + protocolMismatch( + `replay maximum from 1 through ${limits.maxMessagesPerRemote}`, + "invalid replay maximum" + ) + ) + } + const permit = yield* gate.shared + const now = new Date(yield* Clock.currentTimeMillis).toISOString() + return yield* mapStorageFailures(sql.withTransaction(Effect.gen(function*() { + const metadata = yield* findDueMetadata({ + replicaId: permit.replicaId, + replicaIncarnation: permit.incarnation, + relayPeerId: endpoint.relayPeerId, + expectedLocalTenantId: endpoint.expectedLocal.tenantId, + expectedLocalSubjectId: endpoint.expectedLocal.subjectId, + expectedLocalPeerId: endpoint.expectedLocal.peerId, + remoteTenantId: endpoint.remote.tenantId, + remoteSubjectId: endpoint.remote.subjectId, + remotePeerId: endpoint.remote.peerId, + now, + maximum: input.maximum + }) + const entries: Array = [] + for (const item of metadata) { + if ( + item.actual_size !== item.encoded_size || + item.actual_size > PeerSyncEnvelope.maximumSyncEnvelopeBytes( + replicaLimits.maxSyncMessageBytes, + replicaLimits.maxSyncChangesPerMessage + ) + ) { + return yield* storageCorrupt(new Error("Relay outbox payload length mismatch")) + } + const rows = yield* findByRowId({ + replicaId: permit.replicaId, + replicaIncarnation: permit.incarnation, + rowId: item.row_id + }) + if (rows.length !== 1) { + return yield* storageCorrupt(new Error("Relay outbox row disappeared during replay")) + } + const entry = yield* validateRow(rows[0]!, permit) + if ( + entry.expectedLocal.tenantId !== endpoint.expectedLocal.tenantId || + entry.expectedLocal.subjectId !== endpoint.expectedLocal.subjectId || + entry.expectedLocal.peerId !== endpoint.expectedLocal.peerId + ) { + return yield* storageCorrupt(new Error("Relay outbox local endpoint mismatch")) + } + entries.push(entry) + } + return entries + }))) + })) + + const maximumPendingHorizon = (input: Endpoint) => + Effect.scoped(Effect.gen(function*() { + const endpoint = yield* decodeEndpoint(input) + const permit = yield* gate.shared + const query = SqlSchema.findAll({ + Request: Schema.Void, + Result: HorizonRow, + execute: () => + sql`SELECT MAX( + (julianday(retry_deadline) - julianday(created_at)) * 86400000.0 + ) AS horizon_millis + FROM effect_local_peer_relay_outbox + WHERE replica_id = ${permit.replicaId} + AND replica_incarnation = ${permit.incarnation} + AND expected_local_tenant_id = ${endpoint.expectedLocal.tenantId} + AND expected_local_subject_id = ${endpoint.expectedLocal.subjectId} + AND expected_local_peer_id = ${endpoint.expectedLocal.peerId} + AND relay_peer_id = ${endpoint.relayPeerId} + AND remote_tenant_id = ${endpoint.remote.tenantId} + AND remote_subject_id = ${endpoint.remote.subjectId} + AND remote_peer_id = ${endpoint.remote.peerId}` + }) + const rows = yield* mapStorageFailures(query(undefined)) + const horizon = rows[0]?.horizon_millis ?? null + if (horizon === null) return null + const rounded = Math.round(horizon) + if (rounded <= 0 || rounded > limits.maxRetryHorizonMillis) { + return yield* storageCorrupt(new Error("Relay outbox horizon mismatch")) + } + return rounded + })) + + const markCustody = (input: CustodyInput) => + Effect.scoped(Effect.gen(function*() { + const permit = yield* gate.shared + yield* mapStorageFailures(sql.withTransaction(Effect.gen(function*() { + const rows = yield* findRow({ + replicaId: permit.replicaId, + replicaIncarnation: permit.incarnation, + relayMessageId: input.relayMessageId + }) + if (rows.length === 0) { + yield* gate.validate(permit) + return + } + if (rows.length !== 1) { + return yield* storageCorrupt(new Error("Duplicate relay message identity")) + } + const row = rows[0]! + yield* validateRow(row, permit) + if (row.outer_envelope_digest !== input.outerEnvelopeDigest) { + return yield* Effect.fail( + protocolMismatch("matching relay outer envelope digest", "conflicting custody digest") + ) + } + yield* decrementUsage(row) + yield* sql`DELETE FROM effect_local_peer_relay_outbox + WHERE row_id = ${row.row_id} + AND replica_id = ${permit.replicaId} + AND replica_incarnation = ${permit.incarnation} + AND relay_message_id = ${input.relayMessageId} + AND outer_envelope_digest = ${input.outerEnvelopeDigest}` + yield* gate.validate(permit) + }))) + })) + + const pruneExpired = Effect.scoped(Effect.gen(function*() { + const permit = yield* gate.shared + const now = new Date(yield* Clock.currentTimeMillis).toISOString() + return yield* mapStorageFailures(sql.withTransaction(Effect.gen(function*() { + const query = SqlSchema.findAll({ + Request: Schema.Void, + Result: Row, + execute: () => + sql`SELECT * + FROM effect_local_peer_relay_outbox + WHERE replica_id = ${permit.replicaId} + AND replica_incarnation = ${permit.incarnation} + AND retry_deadline <= ${now} + ORDER BY row_id + LIMIT ${limits.pruneBatchSize}` + }) + const rows = yield* query(undefined) + for (const row of rows) { + yield* validateRow(row, permit) + yield* decrementUsage(row) + yield* sql`DELETE FROM effect_local_peer_relay_outbox + WHERE row_id = ${row.row_id} + AND replica_id = ${permit.replicaId} + AND replica_incarnation = ${permit.incarnation}` + } + yield* gate.validate(permit) + return rows.length + }))) + })) + + const usage = (input: Endpoint) => + Effect.scoped(Effect.gen(function*() { + const endpoint = yield* decodeEndpoint(input) + const permit = yield* gate.shared + const remoteQuery = SqlSchema.findAll({ + Request: Schema.Void, + Result: UsageRow, + execute: () => + sql`SELECT message_count, encoded_bytes + FROM effect_local_peer_relay_outbox_remote_usage + WHERE replica_incarnation = ${permit.incarnation} + AND remote_tenant_id = ${endpoint.remote.tenantId} + AND remote_subject_id = ${endpoint.remote.subjectId} + AND remote_peer_id = ${endpoint.remote.peerId}` + }) + const replicaQuery = SqlSchema.findAll({ + Request: Schema.Void, + Result: UsageRow, + execute: () => + sql`SELECT message_count, encoded_bytes + FROM effect_local_peer_relay_outbox_replica_usage + WHERE replica_incarnation = ${permit.incarnation}` + }) + const [remoteRows, replicaRows] = yield* mapStorageFailures( + sql.withTransaction(Effect.all([remoteQuery(undefined), replicaQuery(undefined)])) + ) + const remote = remoteRows[0] ?? { message_count: 0, encoded_bytes: 0 } + const replica = replicaRows[0] ?? { message_count: 0, encoded_bytes: 0 } + return { + remote: { + messageCount: remote.message_count, + encodedBytes: remote.encoded_bytes + }, + replica: { + messageCount: replica.message_count, + encodedBytes: replica.encoded_bytes + } + } + })) + + return { + admit, + dueForEndpoint, + maximumPendingHorizon, + markCustody, + pruneExpired, + usage, + validateReplicaIncarnation + } +}) + +export const layerSql: Layer.Layer< + PeerRelayOutbox, + never, + Requirements +> = Layer.effect(PeerRelayOutbox, make) diff --git a/packages/local-sql/src/PeerRelayOutboxLimits.ts b/packages/local-sql/src/PeerRelayOutboxLimits.ts new file mode 100644 index 0000000..436b8bb --- /dev/null +++ b/packages/local-sql/src/PeerRelayOutboxLimits.ts @@ -0,0 +1,53 @@ +import * as Context from "effect/Context" +import * as Layer from "effect/Layer" +import * as Schema from "effect/Schema" + +const PositiveInt = Schema.Int.check(Schema.isGreaterThan(0)) + +export const maximumRetryHorizonMillis = 90 * 24 * 60 * 60 * 1_000 + +const RetryHorizonMillis = PositiveInt.check( + Schema.isLessThanOrEqualTo(maximumRetryHorizonMillis) +) + +export const Values = Schema.Struct({ + maxMessagesPerRemote: PositiveInt, + maxEncodedBytesPerRemote: PositiveInt, + maxMessagesPerReplica: PositiveInt, + maxEncodedBytesPerReplica: PositiveInt, + maxRetryHorizonMillis: RetryHorizonMillis, + pruneBatchSize: PositiveInt, + pruneRowsPerSecond: PositiveInt, + maintenanceIntervalMillis: PositiveInt +}).check( + Schema.makeFilter( + (values) => values.maxMessagesPerRemote <= values.maxMessagesPerReplica, + { expected: "maxMessagesPerRemote being at most maxMessagesPerReplica" } + ), + Schema.makeFilter( + (values) => values.maxEncodedBytesPerRemote <= values.maxEncodedBytesPerReplica, + { expected: "maxEncodedBytesPerRemote being at most maxEncodedBytesPerReplica" } + ) +) +export type Values = typeof Values.Type + +export const defaults: Values = Values.make({ + maxMessagesPerRemote: 10_000, + maxEncodedBytesPerRemote: 64 * 1_024 * 1_024, + maxMessagesPerReplica: 100_000, + maxEncodedBytesPerReplica: 512 * 1_024 * 1_024, + maxRetryHorizonMillis: 7 * 24 * 60 * 60 * 1_000, + pruneBatchSize: 1_000, + pruneRowsPerSecond: 10_000, + maintenanceIntervalMillis: 1_000 +}) + +export class PeerRelayOutboxLimits extends Context.Service()( + "@lucas-barake/effect-local-sql/PeerRelayOutboxLimits" +) {} + +export const make = (values: Values) => Values.makeEffect(values) + +export const layer = (values: Values) => Layer.effect(PeerRelayOutboxLimits, make(values)) + +export const layerDefaults = layer(defaults) diff --git a/packages/local-sql/src/PeerRelayReceiptLimits.ts b/packages/local-sql/src/PeerRelayReceiptLimits.ts new file mode 100644 index 0000000..ffde763 --- /dev/null +++ b/packages/local-sql/src/PeerRelayReceiptLimits.ts @@ -0,0 +1,53 @@ +import * as Context from "effect/Context" +import * as Layer from "effect/Layer" +import * as Schema from "effect/Schema" + +const PositiveInt = Schema.Int.check(Schema.isGreaterThan(0)) + +export const maximumReceiptRetentionMillis = 90 * 24 * 60 * 60 * 1_000 + +const ReceiptRetentionMillis = PositiveInt.check( + Schema.isLessThanOrEqualTo(maximumReceiptRetentionMillis) +) + +export const Values = Schema.Struct({ + maxReceiptsPerRemote: PositiveInt, + maxEncodedBytesPerRemote: PositiveInt, + maxReceiptsPerReplica: PositiveInt, + maxEncodedBytesPerReplica: PositiveInt, + receiptRetentionMillis: ReceiptRetentionMillis, + pruneBatchSize: PositiveInt, + pruneRowsPerSecond: PositiveInt, + maintenanceIntervalMillis: PositiveInt +}).check( + Schema.makeFilter( + (values) => values.maxReceiptsPerRemote <= values.maxReceiptsPerReplica, + { expected: "maxReceiptsPerRemote being at most maxReceiptsPerReplica" } + ), + Schema.makeFilter( + (values) => values.maxEncodedBytesPerRemote <= values.maxEncodedBytesPerReplica, + { expected: "maxEncodedBytesPerRemote being at most maxEncodedBytesPerReplica" } + ) +) +export type Values = typeof Values.Type + +export const defaults: Values = Values.make({ + maxReceiptsPerRemote: 10_000, + maxEncodedBytesPerRemote: 64 * 1_024 * 1_024, + maxReceiptsPerReplica: 100_000, + maxEncodedBytesPerReplica: 512 * 1_024 * 1_024, + receiptRetentionMillis: 8 * 24 * 60 * 60 * 1_000, + pruneBatchSize: 1_000, + pruneRowsPerSecond: 10_000, + maintenanceIntervalMillis: 1_000 +}) + +export class PeerRelayReceiptLimits extends Context.Service()( + "@lucas-barake/effect-local-sql/PeerRelayReceiptLimits" +) {} + +export const make = (values: Values) => Values.makeEffect(values) + +export const layer = (values: Values) => Layer.effect(PeerRelayReceiptLimits, make(values)) + +export const layerDefaults = layer(defaults) diff --git a/packages/local-sql/src/PeerSession.ts b/packages/local-sql/src/PeerSession.ts index 2911a7b..a8adabd 100644 --- a/packages/local-sql/src/PeerSession.ts +++ b/packages/local-sql/src/PeerSession.ts @@ -5,6 +5,7 @@ import * as PeerTransport from "@lucas-barake/effect-local/PeerTransport" import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" import * as ReplicaLimits from "@lucas-barake/effect-local/ReplicaLimits" import * as Cause from "effect/Cause" +import * as Clock from "effect/Clock" import * as Crypto from "effect/Crypto" import * as Deferred from "effect/Deferred" import * as Effect from "effect/Effect" @@ -18,7 +19,9 @@ import type * as Sharding from "effect/unstable/cluster/Sharding" import * as CommitPublisher from "./CommitPublisher.js" import * as DocumentEntity from "./DocumentEntity.js" import * as WriterProvenance from "./internal/writerProvenance.js" +import * as PeerRelayReceiptLimits from "./PeerRelayReceiptLimits.js" import * as PeerSync from "./PeerSync.js" +import * as PeerSyncEnvelope from "./PeerSyncEnvelope.js" import * as ReplicaGate from "./ReplicaGate.js" export interface SelectedDocument { @@ -62,7 +65,7 @@ const SyncEnvelopeJson = Schema.fromJsonString(Schema.toCodecJson(SyncEnvelope)) const key = (documentType: string, documentId: Identity.DocumentId) => `${documentType}:${documentId}` -const encode = (envelope: typeof SyncEnvelope.Type) => +const encodeDirect = (envelope: typeof SyncEnvelope.Type) => Schema.encodeEffect(SyncEnvelopeJson)(envelope).pipe( Effect.map((value) => new TextEncoder().encode(value)), Effect.mapError((cause) => @@ -75,18 +78,6 @@ const encode = (envelope: typeof SyncEnvelope.Type) => ) ) -const decode = (bytes: Uint8Array) => - Schema.decodeUnknownEffect(SyncEnvelopeJson)(new TextDecoder().decode(bytes)).pipe( - Effect.mapError((cause) => - new ReplicaError.ReplicaError({ - reason: new ReplicaError.ProtocolMismatch({ - expected: "sync envelope", - observed: String(cause) - }) - }) - ) - ) - const supervise = ( terminalFailure: Deferred.Deferred, effect: Effect.Effect @@ -132,6 +123,37 @@ const makeWithTerminal = ( const transport = yield* PeerTransport.PeerTransport const sync = yield* PeerSync.PeerSync const crypto = yield* Crypto.Crypto + const relayReceiptLimits = yield* Effect.serviceOption(PeerRelayReceiptLimits.PeerRelayReceiptLimits) + const decodeDirect = (bytes: Uint8Array) => { + const maximumBytes = maximumSyncEnvelopeBytes( + limits.maxSyncMessageBytes, + limits.maxSyncChangesPerMessage + ) + if (bytes.byteLength > maximumBytes) { + return Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.ProtocolMismatch({ + expected: `sync envelope at most ${maximumBytes} bytes`, + observed: String(bytes.byteLength) + }) + }) + ) + } + return Schema.decodeUnknownEffect(SyncEnvelopeJson)(new TextDecoder().decode(bytes)).pipe( + Effect.mapError(() => + new ReplicaError.ReplicaError({ + reason: new ReplicaError.ProtocolMismatch({ + expected: "sync envelope", + observed: "invalid sync envelope" + }) + }) + ) + ) + } + const decodeRelay = (bytes: Uint8Array) => + PeerSyncEnvelope.decodeSyncEnvelope(bytes, limits).pipe( + Effect.provideService(Crypto.Crypto, crypto) + ) const selected = new Set(options.documents.map((entry) => key(entry.document.name, entry.documentId))) const selectedDocumentIds = new Set(options.documents.map((entry) => entry.documentId)) if ( @@ -236,7 +258,11 @@ const makeWithTerminal = ( Effect.gen(function*() { if (!(yield* Ref.get(active))) return const entry = yield* selectedById(outbound.documentId) - const bytes = yield* encode({ + const bytes = yield* ( + connection.receiveWithAcknowledgement === undefined + ? encodeDirect + : PeerSyncEnvelope.encodeSyncEnvelope + )({ connectionEpoch: session.connectionEpoch, sequence: outbound.sendSequence, documentId: outbound.documentId, @@ -397,33 +423,56 @@ const makeWithTerminal = ( ) const guardedFlush = guardTerminal(flush) - const receive = (bytes: Uint8Array) => + const bindRemoteEpoch = (connectionEpoch: string, rotate: boolean) => Effect.gen(function*() { - const maximumBytes = maximumSyncEnvelopeBytes( - limits.maxSyncMessageBytes, - limits.maxSyncChangesPerMessage + const transition = yield* Ref.modify( + remoteEpoch, + (current): [{ readonly bound: string; readonly previous: string | null }, string] => { + if (current === null) return [{ bound: connectionEpoch, previous: null }, connectionEpoch] + if (current === connectionEpoch) return [{ bound: current, previous: null }, current] + if (rotate) return [{ bound: connectionEpoch, previous: current }, connectionEpoch] + return [{ bound: current, previous: null }, current] + } ) - if (bytes.byteLength > maximumBytes) { + if (transition.bound !== connectionEpoch) { return yield* new ReplicaError.ReplicaError({ reason: new ReplicaError.ProtocolMismatch({ - expected: `sync envelope at most ${maximumBytes} bytes`, - observed: String(bytes.byteLength) + expected: transition.bound, + observed: connectionEpoch }) }) } - const envelope = yield* decode(bytes) - const boundEpoch = yield* Ref.modify( - remoteEpoch, - (current) => current === null ? [envelope.connectionEpoch, envelope.connectionEpoch] : [current, current] - ) - if (boundEpoch !== envelope.connectionEpoch) { - return yield* new ReplicaError.ReplicaError({ - reason: new ReplicaError.ProtocolMismatch({ - expected: boundEpoch, - observed: envelope.connectionEpoch - }) + if (transition.previous !== null) { + yield* sync.reset({ + peerId: connection.peerId, + connectionEpoch: transition.previous, + replicaIncarnation: session.replicaIncarnation }) } + return transition.bound + }) + + const processReceive = ( + bytes: Uint8Array, + delivery?: PeerTransport.AcknowledgedDelivery + ) => + Effect.gen(function*() { + const envelope = yield* ( + delivery === undefined + ? decodeDirect(bytes).pipe( + Effect.map((envelope) => ({ + ...envelope, + lineage: envelope.lineage ?? Identity.genesisLineage + })) + ) + : decodeRelay(bytes).pipe( + Effect.map((envelope) => ({ + ...envelope, + lineage: Identity.genesisLineage + })) + ) + ) + const boundEpoch = yield* bindRemoteEpoch(envelope.connectionEpoch, delivery !== undefined) const selectedDocument = selected.has(key(envelope.documentType, envelope.documentId)) // Dropped before the digest and entity dispatch, not after. Nothing can restore the refused // lineage inside this session, so hashing and dispatching every further message the peer @@ -465,6 +514,63 @@ const makeWithTerminal = ( })) const observationRevision = (yield* Ref.get(observed)).get(envelope.documentId)?.revision ?? 0 const client = yield* entity(envelope.documentId) + const relay = delivery === undefined + ? undefined + : yield* Effect.gen(function*() { + if (relayReceiptLimits._tag === "None") { + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.ProtocolMismatch({ + expected: "relay-enabled peer session", + observed: "direct peer session" + }) + }) + } + if ( + !Number.isSafeInteger(delivery.receiptRetentionMillis) || + delivery.receiptRetentionMillis <= 0 || + delivery.receiptRetentionMillis > relayReceiptLimits.value.receiptRetentionMillis + ) { + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.ProtocolMismatch({ + expected: "bounded relay receipt retention", + observed: String(delivery.receiptRetentionMillis) + }) + }) + } + if ( + connection.relayPeerId === undefined || + delivery.identity.relayPeerId !== connection.relayPeerId + ) { + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.ProtocolMismatch({ + expected: connection.relayPeerId ?? "relay peer identity", + observed: delivery.identity.relayPeerId + }) + }) + } + if (delivery.identity.senderPeerId !== connection.peerId) { + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.ProtocolMismatch({ + expected: connection.peerId, + observed: delivery.identity.senderPeerId + }) + }) + } + if (delivery.identity.messageHash !== envelope.messageHash) { + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.ProtocolMismatch({ + expected: envelope.messageHash, + observed: delivery.identity.messageHash + }) + }) + } + const nowMillis = yield* Clock.currentTimeMillis + return { + ...delivery.identity, + receiptExpiresAt: new Date(nowMillis + delivery.receiptRetentionMillis).toISOString(), + encodedSize: bytes.byteLength + } satisfies PeerSync.RelayReceipt + }) const result = yield* client.ApplySync({ replicaIncarnation: incarnation, peerId: connection.peerId, @@ -476,8 +582,9 @@ const makeWithTerminal = ( message: envelope.message, // The single point an envelope's lineage enters the system, so the single point the // absent key of a pre lineage peer becomes the genesis lineage. - lineage: envelope.lineage ?? Identity.genesisLineage, - writerProvenance: envelope.writerProvenance + lineage: envelope.lineage, + writerProvenance: envelope.writerProvenance, + ...(relay === undefined ? {} : { relay }) }).pipe( Effect.catchTag( ["MailboxFull", "AlreadyProcessingMessage", "PersistenceError"], @@ -519,6 +626,32 @@ const makeWithTerminal = ( } }) + const relayCall = ( + operation: string, + effect: Effect.Effect + ) => + effect.pipe( + Effect.timeout(limits.maxPeerSendMillis), + Effect.catchTag("TimeoutError", () => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.OperationTimeout({ + operation, + timeoutMillis: limits.maxPeerSendMillis + }) + }) + )) + ) + + const receiveAcknowledged = (delivery: PeerTransport.AcknowledgedDelivery) => + processReceive(delivery.message, delivery).pipe( + Effect.andThen(relayCall("relay acknowledge", delivery.acknowledge)), + Effect.catchIf( + (error) => error.reason._tag === "ProtocolMismatch", + () => relayCall("relay reject", delivery.reject("ProtocolInvalid")) + ) + ) + yield* Effect.addFinalizer(() => Effect.gen(function*() { const boundEpoch = yield* Ref.get(remoteEpoch) @@ -540,7 +673,11 @@ const makeWithTerminal = ( ) yield* supervise( terminalFailure, - Stream.runForEach(connection.receive, receive).pipe( + ( + connection.receiveWithAcknowledgement === undefined + ? Stream.runForEach(connection.receive, processReceive) + : Stream.runForEach(connection.receiveWithAcknowledgement, receiveAcknowledged) + ).pipe( Effect.andThen( Effect.fail( new ReplicaError.ReplicaError({ diff --git a/packages/local-sql/src/PeerSync.ts b/packages/local-sql/src/PeerSync.ts index a73bbfd..a5104a6 100644 --- a/packages/local-sql/src/PeerSync.ts +++ b/packages/local-sql/src/PeerSync.ts @@ -2,6 +2,7 @@ import * as Automerge from "@automerge/automerge" import * as Canonical from "@lucas-barake/effect-local/Canonical" import type * as Document from "@lucas-barake/effect-local/Document" import * as Identity from "@lucas-barake/effect-local/Identity" +import type * as PeerTransport from "@lucas-barake/effect-local/PeerTransport" import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" import * as ReplicaLimits from "@lucas-barake/effect-local/ReplicaLimits" import * as Clock from "effect/Clock" @@ -19,6 +20,7 @@ import * as SqlSchema from "effect/unstable/sql/SqlSchema" import * as DocumentStore from "./DocumentStore.js" import * as InternalAutomerge from "./internal/automerge.js" import * as WriterProvenance from "./internal/writerProvenance.js" +import * as PeerRelayReceiptLimits from "./PeerRelayReceiptLimits.js" import * as ReplicaBootstrap from "./ReplicaBootstrap.js" import * as ReplicaGate from "./ReplicaGate.js" @@ -68,6 +70,11 @@ export interface Received { readonly duplicate: boolean } +export interface RelayReceipt extends PeerTransport.RelayDeliveryIdentity { + readonly receiptExpiresAt: string + readonly encodedSize: number +} + const Heads = Schema.fromJsonString(Schema.Array(Schema.String)) class ConcurrentDocumentWrite extends Schema.TaggedErrorClass( @@ -85,6 +92,27 @@ const ReceiptRow = Schema.Struct({ writer_provenance: WriterProvenance.StoredChangeProvenances }) +const RelayReceiptRow = Schema.Struct({ + ...ReceiptRow.fields, + relay_encoded_size: Schema.Int, + relay_outer_envelope_digest: Schema.String, + relay_receipt_expires_at: Schema.String +}) + +const RelayReceiptPruneRow = Schema.Struct({ + encoded_size: Schema.Int, + relay_message_id: Identity.RelayMessageId, + row_id: Schema.Int, + sender_peer_id: Identity.PeerId, + sender_subject_id: Schema.String, + sender_tenant_id: Schema.String +}) + +const RelayReceiptUsageRow = Schema.Struct({ + encoded_bytes: Schema.Int, + receipt_count: Schema.Int +}) + const PendingRow = Schema.Struct({ actor: Schema.String, bytes: Schema.Uint8Array, @@ -97,9 +125,7 @@ const PendingRow = Schema.Struct({ const PendingReceiptRow = Schema.Struct({ accepted_heads: Heads, - connection_epoch: Schema.String, - peer_id: Schema.String, - receive_sequence: Schema.Number, + row_id: Schema.Int, writer_provenance: WriterProvenance.StoredChangeProvenances }) @@ -257,9 +283,14 @@ export class PeerSync extends Context.Service + readonly relay?: RelayReceipt } ) => Effect.Effect readonly enqueue: (session: Session, reply: Reply) => Effect.Effect @@ -291,20 +322,22 @@ export class PeerSync extends Context.Service Effect.Effect + readonly pruneRelayReceipts?: Effect.Effect }>()("@lucas-barake/effect-local-sql/PeerSync") {} -export const layer: Layer.Layer< - PeerSync, - ReplicaError.ReplicaError, +type Requirements = | DocumentStore.DocumentStore | ReplicaBootstrap.ReplicaBootstrap | ReplicaGate.ReplicaGate | ReplicaLimits.ReplicaLimits | Crypto.Crypto | SqlClient.SqlClient -> = Layer.effect( - PeerSync, + +const make = ( + relayReceiptLimits: PeerRelayReceiptLimits.Values | null +) => Effect.gen(function*() { + void relayReceiptLimits const sql = yield* SqlClient.SqlClient const store = yield* DocumentStore.DocumentStore const bootstrap = yield* ReplicaBootstrap.ReplicaBootstrap @@ -340,7 +373,106 @@ export const layer: Layer.Layer< WHERE replica_incarnation = ${request.replicaIncarnation} AND peer_id = ${request.peerId} AND connection_epoch = ${request.connectionEpoch} - AND receive_sequence = ${request.receiveSequence}` + AND receive_sequence = ${request.receiveSequence} + AND relay_message_id IS NULL` + }) + const findRelayReceipts = SqlSchema.findAll({ + Request: Schema.Struct({ + relayMessageId: Identity.RelayMessageId, + replicaIncarnation: Identity.ReplicaIncarnation, + senderPeerId: Identity.PeerId, + senderSubjectId: Schema.String, + senderTenantId: Schema.String + }), + Result: RelayReceiptRow, + execute: (request) => + sql`SELECT accepted_heads, commit_sequence, document_id, heads, message_hash, reply, reply_hash, + writer_provenance, relay_encoded_size, relay_outer_envelope_digest, relay_receipt_expires_at + FROM effect_local_peer_receipts + WHERE replica_incarnation = ${request.replicaIncarnation} + AND relay_sender_tenant_id = ${request.senderTenantId} + AND relay_sender_subject_id = ${request.senderSubjectId} + AND relay_sender_peer_id = ${request.senderPeerId} + AND relay_message_id = ${request.relayMessageId} + LIMIT 1` + }) + const findRelayReceiptsToPrune = SqlSchema.findAll({ + Request: Schema.Struct({ + expiresAt: Schema.String, + limit: Schema.Int.check(Schema.isGreaterThan(0)), + replicaIncarnation: Identity.ReplicaIncarnation + }), + Result: RelayReceiptPruneRow, + execute: (request) => + sql`SELECT relay_encoded_size AS encoded_size, relay_message_id, row_id, + relay_sender_peer_id AS sender_peer_id, + relay_sender_subject_id AS sender_subject_id, + relay_sender_tenant_id AS sender_tenant_id + FROM effect_local_peer_receipts + WHERE replica_incarnation = ${request.replicaIncarnation} + AND relay_message_id IS NOT NULL + AND relay_receipt_expires_at <= ${request.expiresAt} + ORDER BY relay_receipt_expires_at, relay_sender_tenant_id, relay_sender_subject_id, + relay_sender_peer_id, relay_message_id, row_id + LIMIT ${request.limit}` + }) + const decrementRelayReceiptUsage = SqlSchema.findAll({ + Request: Schema.Struct({ + encodedBytes: Schema.Int.check(Schema.isGreaterThan(0)), + receiptCount: Schema.Int.check(Schema.isGreaterThan(0)), + replicaIncarnation: Identity.ReplicaIncarnation, + senderPeerId: Identity.PeerId, + senderSubjectId: Schema.String, + senderTenantId: Schema.String + }), + Result: RelayReceiptUsageRow, + execute: (request) => + sql`UPDATE effect_local_peer_relay_receipt_usage + SET receipt_count = receipt_count - ${request.receiptCount}, + encoded_bytes = encoded_bytes - ${request.encodedBytes} + WHERE replica_incarnation = ${request.replicaIncarnation} + AND sender_tenant_id = ${request.senderTenantId} + AND sender_subject_id = ${request.senderSubjectId} + AND sender_peer_id = ${request.senderPeerId} + AND receipt_count >= ${request.receiptCount} + AND encoded_bytes >= ${request.encodedBytes} + RETURNING receipt_count, encoded_bytes` + }) + const deleteRelayReceipt = SqlSchema.findAll({ + Request: Schema.Struct({ + rowId: Schema.Int + }), + Result: Schema.Struct({ row_id: Schema.Int }), + execute: (request) => + sql`DELETE FROM effect_local_peer_receipts + WHERE row_id = ${request.rowId} + AND relay_message_id IS NOT NULL + RETURNING row_id` + }) + const findRelayReceiptUsage = SqlSchema.findAll({ + Request: Schema.Struct({ + replicaIncarnation: Identity.ReplicaIncarnation, + senderPeerId: Identity.PeerId, + senderSubjectId: Schema.String, + senderTenantId: Schema.String + }), + Result: RelayReceiptUsageRow, + execute: (request) => + sql`SELECT receipt_count, encoded_bytes + FROM effect_local_peer_relay_receipt_usage + WHERE replica_incarnation = ${request.replicaIncarnation} + AND sender_tenant_id = ${request.senderTenantId} + AND sender_subject_id = ${request.senderSubjectId} + AND sender_peer_id = ${request.senderPeerId}` + }) + const findRelayReplicaReceiptUsage = SqlSchema.findAll({ + Request: Identity.ReplicaIncarnation, + Result: TotalsRow, + execute: (replicaIncarnation) => + sql`SELECT COALESCE(SUM(receipt_count), 0) AS count, + COALESCE(SUM(encoded_bytes), 0) AS bytes + FROM effect_local_peer_relay_receipt_usage + WHERE replica_incarnation = ${replicaIncarnation}` }) const findExistingChanges = SqlSchema.findAll({ Request: Schema.Struct({ @@ -376,7 +508,7 @@ export const layer: Layer.Layer< }), Result: PendingReceiptRow, execute: (request) => - sql`SELECT accepted_heads, connection_epoch, peer_id, receive_sequence, writer_provenance + sql`SELECT accepted_heads, row_id, writer_provenance FROM effect_local_peer_receipts WHERE replica_incarnation = ${request.replicaIncarnation} AND document_id = ${request.documentId} @@ -671,6 +803,57 @@ export const layer: Layer.Layer< AND status = 'Pending' RETURNING send_sequence` }) + const pruneRelayReceiptsInTransaction = ( + replicaIncarnation: Identity.ReplicaIncarnation, + expiresAt: string + ) => + relayReceiptLimits === null + ? Effect.succeed(0) + : Effect.gen(function*() { + const rows = yield* findRelayReceiptsToPrune({ + expiresAt, + limit: relayReceiptLimits.pruneBatchSize, + replicaIncarnation + }) + const usage = new Map() + for (const row of rows) { + const key = JSON.stringify([row.sender_tenant_id, row.sender_subject_id, row.sender_peer_id]) + const current = usage.get(key) + usage.set(key, { + encodedBytes: (current?.encodedBytes ?? 0) + row.encoded_size, + receiptCount: (current?.receiptCount ?? 0) + 1, + senderPeerId: row.sender_peer_id, + senderSubjectId: row.sender_subject_id, + senderTenantId: row.sender_tenant_id + }) + } + for (const entry of usage.values()) { + const updated = yield* decrementRelayReceiptUsage({ + ...entry, + replicaIncarnation + }) + if (updated.length !== 1) { + return yield* failStorageCorrupt(new Error("Relay receipt usage is inconsistent")) + } + } + for (const row of rows) { + const deleted = yield* deleteRelayReceipt({ rowId: row.row_id }) + if (deleted.length !== 1) { + return yield* failStorageCorrupt(new Error("Relay receipt disappeared during pruning")) + } + } + yield* sql`DELETE FROM effect_local_peer_relay_receipt_usage + WHERE replica_incarnation = ${replicaIncarnation} + AND receipt_count = 0 + AND encoded_bytes = 0` + return rows.length + }) yield* sql.withTransaction(Effect.gen(function*() { yield* sql`INSERT INTO effect_local_quarantine (document_id, peer_id, reason, bytes, created_at) SELECT document_id, peer_id, 'Expired pending sync change', bytes, ${startupAt} @@ -682,6 +865,7 @@ export const layer: Layer.Layer< SELECT document_id, peer_id, 'Expired pending sync message', pending_message, ${startupAt} FROM effect_local_peer_receipts WHERE replica_incarnation = ${bootstrap.incarnation} + AND relay_message_id IS NULL AND pending_message IS NOT NULL AND accepted_at < ${startupCutoff}` yield* sql`INSERT INTO effect_local_quarantine (document_id, peer_id, reason, bytes, created_at) @@ -691,10 +875,19 @@ export const layer: Layer.Layer< AND status = 'Pending' AND created_at < ${startupCutoff}` yield* sql`DELETE FROM effect_local_peer_receipts - WHERE replica_incarnation != ${bootstrap.incarnation} OR accepted_at < ${startupCutoff}` + WHERE relay_message_id IS NULL + AND (replica_incarnation != ${bootstrap.incarnation} OR accepted_at < ${startupCutoff})` yield* sql`DELETE FROM effect_local_peer_outbox WHERE replica_incarnation != ${bootstrap.incarnation} OR created_at < ${startupCutoff}` - })).pipe(Effect.catchTag("SqlError", failStorageUnavailable)) + if (relayReceiptLimits !== null) { + yield* pruneRelayReceiptsInTransaction(bootstrap.incarnation, startupAt) + } + })).pipe( + Effect.catchTags({ + SchemaError: failStorageCorrupt, + SqlError: failStorageUnavailable + }) + ) const readState = (session: Session, documentId: Identity.DocumentId) => Ref.get(states).pipe( @@ -799,11 +992,13 @@ export const layer: Layer.Layer< FROM effect_local_peer_receipts WHERE replica_incarnation = ${session.replicaIncarnation} AND document_id = ${documentId} + AND relay_message_id IS NULL AND pending_message IS NOT NULL AND accepted_at < ${cutoff}` yield* sql`DELETE FROM effect_local_peer_receipts WHERE replica_incarnation = ${session.replicaIncarnation} AND document_id = ${documentId} + AND relay_message_id IS NULL AND pending_message IS NOT NULL AND accepted_at < ${cutoff}` })) @@ -1083,9 +1278,10 @@ export const layer: Layer.Layer< input: { readonly remoteConnectionEpoch: string readonly receiveSequence: number - readonly lineage: Identity.DocumentLineage + readonly lineage?: Identity.DocumentLineage readonly message: Uint8Array readonly writerProvenance: ReadonlyArray + readonly relay?: RelayReceipt } ) => withSessionGeneration(session, (generation) => @@ -1096,6 +1292,15 @@ export const layer: Layer.Layer< Effect.scoped(Effect.gen(function*() { const receiptSession = { ...session, connectionEpoch: input.remoteConnectionEpoch } const { message, receiveSequence } = input + const relay = input.relay + if (relay !== undefined && relayReceiptLimits === null) { + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.ProtocolMismatch({ + expected: "direct peer receipt", + observed: "relay peer receipt" + }) + }) + } const writerProvenance = yield* Schema.decodeUnknownEffect( WriterProvenance.ChangeProvenances )(input.writerProvenance).pipe( @@ -1113,7 +1318,7 @@ export const layer: Layer.Layer< // peer controlled, and a direct caller of `receive` has not necessarily passed it // through the wire schema that already checks it. const remoteLineage = yield* Schema.decodeUnknownEffect(Identity.DocumentLineage)( - input.lineage + input.lineage ?? Identity.genesisLineage ).pipe( Effect.mapError(() => new ReplicaError.ReplicaError({ @@ -1177,6 +1382,40 @@ export const layer: Layer.Layer< yield* validateSession(permit, session) const nowMillis = yield* Clock.currentTimeMillis const acceptedAt = new Date(nowMillis).toISOString() + if (relay !== undefined) { + if (relay.senderPeerId !== session.peerId) { + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.ProtocolMismatch({ + expected: session.peerId, + observed: relay.senderPeerId + }) + }) + } + if ( + !Number.isSafeInteger(relay.encodedSize) || + relay.encodedSize <= 0 + ) { + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.ProtocolMismatch({ + expected: "positive safe relay receipt encoded size", + observed: String(relay.encodedSize) + }) + }) + } + const receiptExpiresAtMillis = Date.parse(relay.receiptExpiresAt) + if ( + !Number.isFinite(receiptExpiresAtMillis) || + receiptExpiresAtMillis <= nowMillis || + receiptExpiresAtMillis - nowMillis > relayReceiptLimits!.receiptRetentionMillis + ) { + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.ProtocolMismatch({ + expected: "bounded future relay receipt expiry", + observed: "invalid relay receipt expiry" + }) + }) + } + } yield* quotaLock.withPermit(Effect.gen(function*() { yield* validateSessionGeneration(generation, sessionGeneration) yield* expirePending( @@ -1185,8 +1424,27 @@ export const layer: Layer.Layer< acceptedAt, new Date(nowMillis - limits.maxPendingAgeMillis).toISOString() ).pipe(Effect.catchTag("SqlError", failStorageUnavailable)) + if (relayReceiptLimits !== null) { + yield* sql.withTransaction(Effect.gen(function*() { + yield* pruneRelayReceiptsInTransaction(permit.incarnation, acceptedAt) + yield* gate.validate(permit) + })).pipe( + Effect.catchTags({ + SqlError: failStorageUnavailable, + SchemaError: failStorageCorrupt + }) + ) + } })) const messageHash = yield* digest(message) + if (relay !== undefined && relay.messageHash !== messageHash) { + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.ProtocolMismatch({ + expected: messageHash, + observed: relay.messageHash + }) + }) + } const validateReceipt = (receipt: typeof ReceiptRow.Type) => Effect.gen(function*() { if (receipt.document_id !== documentId) { @@ -1214,12 +1472,43 @@ export const layer: Layer.Layer< }) } }) - const receiptRows = yield* findReceipts({ - replicaIncarnation: receiptSession.replicaIncarnation, - peerId: receiptSession.peerId, - connectionEpoch: receiptSession.connectionEpoch, - receiveSequence - }).pipe( + const loadReceipt = () => + relay === undefined + ? findReceipts({ + replicaIncarnation: receiptSession.replicaIncarnation, + peerId: receiptSession.peerId, + connectionEpoch: receiptSession.connectionEpoch, + receiveSequence + }) + : findRelayReceipts({ + relayMessageId: relay.relayMessageId, + replicaIncarnation: receiptSession.replicaIncarnation, + senderPeerId: relay.senderPeerId, + senderSubjectId: relay.senderSubjectId, + senderTenantId: relay.senderTenantId + }) + const validateStoredReceipt = ( + receipt: typeof ReceiptRow.Type | typeof RelayReceiptRow.Type + ) => + Effect.gen(function*() { + yield* validateReceipt(receipt) + if ( + relay !== undefined && + ( + !("relay_outer_envelope_digest" in receipt) || + receipt.relay_outer_envelope_digest !== relay.outerEnvelopeDigest || + receipt.relay_encoded_size !== relay.encodedSize + ) + ) { + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.ProtocolMismatch({ + expected: "matching relay receipt identity", + observed: "conflicting relay receipt identity" + }) + }) + } + }) + const receiptRows = yield* loadReceipt().pipe( Effect.catchTags({ SqlError: failStorageUnavailable, SchemaError: failStorageCorrupt @@ -1227,7 +1516,7 @@ export const layer: Layer.Layer< ) const receipt = receiptRows[0] if (receipt !== undefined) { - yield* validateReceipt(receipt) + yield* validateStoredReceipt(receipt) yield* quotaLock.withPermit(validateSessionGeneration(generation, sessionGeneration)) return receivedFromReceipt(documentId, receipt) } @@ -1268,6 +1557,51 @@ export const layer: Layer.Layer< }) } }) + const validateRelayReceiptQuota = relay === undefined + ? Effect.void + : Effect.gen(function*() { + const remote = (yield* findRelayReceiptUsage({ + replicaIncarnation: receiptSession.replicaIncarnation, + senderPeerId: relay.senderPeerId, + senderSubjectId: relay.senderSubjectId, + senderTenantId: relay.senderTenantId + }))[0] + const replica = (yield* findRelayReplicaReceiptUsage( + receiptSession.replicaIncarnation + ))[0] + if ((remote?.receipt_count ?? 0) > relayReceiptLimits!.maxReceiptsPerRemote) { + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.QuotaExceeded({ + resource: "relay receipts per remote", + limit: relayReceiptLimits!.maxReceiptsPerRemote + }) + }) + } + if ((remote?.encoded_bytes ?? 0) > relayReceiptLimits!.maxEncodedBytesPerRemote) { + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.QuotaExceeded({ + resource: "relay receipt bytes per remote", + limit: relayReceiptLimits!.maxEncodedBytesPerRemote + }) + }) + } + if ((replica?.count ?? 0) > relayReceiptLimits!.maxReceiptsPerReplica) { + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.QuotaExceeded({ + resource: "relay receipts per replica", + limit: relayReceiptLimits!.maxReceiptsPerReplica + }) + }) + } + if ((replica?.bytes ?? 0) > relayReceiptLimits!.maxEncodedBytesPerReplica) { + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.QuotaExceeded({ + resource: "relay receipt bytes per replica", + limit: relayReceiptLimits!.maxEncodedBytesPerReplica + }) + }) + } + }) const decoded = yield* Effect.try({ try: () => Automerge.decodeSyncMessage(message), catch: (cause) => @@ -1785,15 +2119,13 @@ export const layer: Layer.Layer< const result = yield* quotaLock.withPermit(Effect.gen(function*() { const result = yield* sql.withTransaction(Effect.gen(function*() { yield* validateSessionGeneration(generation, sessionGeneration) - const receiptRows = yield* findReceipts({ - replicaIncarnation: receiptSession.replicaIncarnation, - peerId: receiptSession.peerId, - connectionEpoch: receiptSession.connectionEpoch, - receiveSequence - }) + if (relayReceiptLimits !== null) { + yield* pruneRelayReceiptsInTransaction(permit.incarnation, acceptedAt) + } + const receiptRows = yield* loadReceipt() const receipt = receiptRows[0] if (receipt !== undefined) { - yield* validateReceipt(receipt) + yield* validateStoredReceipt(receipt) return { _tag: "Duplicate" as const, received: receivedFromReceipt(documentId, receipt) } } const committedChanges = validationChanges.length === 0 ? [] : yield* findExistingChanges({ @@ -1855,10 +2187,7 @@ export const layer: Layer.Layer< for (const row of pendingReceiptRows) { if (Automerge.hasHeads(staged, [...row.accepted_heads])) { yield* sql`UPDATE effect_local_peer_receipts SET pending_message = NULL - WHERE replica_incarnation = ${receiptSession.replicaIncarnation} - AND peer_id = ${row.peer_id} - AND connection_epoch = ${row.connection_epoch} - AND receive_sequence = ${row.receive_sequence}` + WHERE row_id = ${row.row_id}` } } if (checkpoint !== null) { @@ -1925,15 +2254,35 @@ export const layer: Layer.Layer< yield* sql`INSERT INTO effect_local_peer_receipts ( replica_incarnation, peer_id, connection_epoch, receive_sequence, document_id, message_hash, reply, reply_hash, pending_message, - heads, accepted_heads, commit_sequence, accepted_at, writer_provenance + heads, accepted_heads, commit_sequence, accepted_at, writer_provenance, + relay_sender_tenant_id, relay_sender_subject_id, relay_sender_peer_id, + relay_message_id, relay_outer_envelope_digest, relay_receipt_expires_at, + relay_encoded_size ) VALUES ( ${receiptSession.replicaIncarnation}, ${receiptSession.peerId}, ${receiptSession.connectionEpoch}, ${receiveSequence}, ${documentId}, ${messageHash}, ${reply?.message ?? null}, ${reply?.messageHash ?? null}, ${unresolvedBytes === 0 ? null : message}, ${Schema.encodeSync(Heads)(materializedHeads)}, ${Schema.encodeSync(Heads)(acceptedHeads)}, ${commitSequence}, ${acceptedAt}, - ${Schema.encodeSync(WriterProvenance.StoredChangeProvenances)(writerProvenance)} + ${Schema.encodeSync(WriterProvenance.StoredChangeProvenances)(writerProvenance)}, + ${relay?.senderTenantId ?? null}, ${relay?.senderSubjectId ?? null}, + ${relay?.senderPeerId ?? null}, ${relay?.relayMessageId ?? null}, + ${relay?.outerEnvelopeDigest ?? null}, ${relay?.receiptExpiresAt ?? null}, + ${relay?.encodedSize ?? null} )` + if (relay !== undefined) { + yield* sql`INSERT INTO effect_local_peer_relay_receipt_usage ( + replica_incarnation, sender_tenant_id, sender_subject_id, sender_peer_id, + receipt_count, encoded_bytes + ) VALUES ( + ${receiptSession.replicaIncarnation}, ${relay.senderTenantId}, ${relay.senderSubjectId}, + ${relay.senderPeerId}, 1, ${relay.encodedSize} + ) ON CONFLICT(replica_incarnation, sender_tenant_id, sender_subject_id, sender_peer_id) + DO UPDATE SET + receipt_count = effect_local_peer_relay_receipt_usage.receipt_count + 1, + encoded_bytes = effect_local_peer_relay_receipt_usage.encoded_bytes + excluded.encoded_bytes` + yield* validateRelayReceiptQuota + } if (unresolvedBytes !== 0) { yield* validateReceiptQuota yield* validatePendingQuota @@ -1981,6 +2330,26 @@ export const layer: Layer.Layer< ) )) + const pruneRelayReceipts = relayReceiptLimits === null + ? {} + : { + pruneRelayReceipts: Effect.scoped(Effect.gen(function*() { + const permit = yield* gate.shared + const expiresAt = new Date(yield* Clock.currentTimeMillis).toISOString() + return yield* quotaLock.withPermit( + sql.withTransaction(Effect.gen(function*() { + const pruned = yield* pruneRelayReceiptsInTransaction(permit.incarnation, expiresAt) + yield* gate.validate(permit) + return pruned + })).pipe( + Effect.catchTags({ + SqlError: failStorageUnavailable, + SchemaError: failStorageCorrupt + }) + ) + ) + })) + } return PeerSync.of({ open: (peerId) => Effect.scoped(Effect.gen(function*() { @@ -2009,7 +2378,8 @@ export const layer: Layer.Layer< yield* sql`DELETE FROM effect_local_peer_receipts WHERE replica_incarnation = ${session.replicaIncarnation} AND peer_id = ${session.peerId} - AND connection_epoch = ${session.connectionEpoch}` + AND connection_epoch = ${session.connectionEpoch} + AND relay_message_id IS NULL` })).pipe(Effect.catchTag("SqlError", failStorageUnavailable)) yield* Ref.update(generation, (current) => current + 1) yield* removeState(session) @@ -2056,6 +2426,7 @@ export const layer: Layer.Layer< }) ) })), + ...pruneRelayReceipts, markSent: (session, sendSequence, messageHash) => Effect.scoped(Effect.gen(function*() { const permit = yield* gate.shared @@ -2087,4 +2458,21 @@ export const layer: Layer.Layer< })) }) }) + +export const layer: Layer.Layer< + PeerSync, + ReplicaError.ReplicaError, + Requirements +> = Layer.effect(PeerSync, make(null)) + +export const layerRelay: Layer.Layer< + PeerSync, + ReplicaError.ReplicaError, + Requirements | PeerRelayReceiptLimits.PeerRelayReceiptLimits +> = Layer.effect( + PeerSync, + Effect.gen(function*() { + const relayReceiptLimits = yield* PeerRelayReceiptLimits.PeerRelayReceiptLimits + return yield* make(relayReceiptLimits) + }) ) diff --git a/packages/local-sql/src/PeerSyncEnvelope.ts b/packages/local-sql/src/PeerSyncEnvelope.ts new file mode 100644 index 0000000..6d28735 --- /dev/null +++ b/packages/local-sql/src/PeerSyncEnvelope.ts @@ -0,0 +1,235 @@ +import * as Automerge from "@automerge/automerge" +import * as Canonical from "@lucas-barake/effect-local/Canonical" +import * as Identity from "@lucas-barake/effect-local/Identity" +import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" +import type * as ReplicaLimits from "@lucas-barake/effect-local/ReplicaLimits" +import * as Crypto from "effect/Crypto" +import * as Effect from "effect/Effect" +import * as Schema from "effect/Schema" +import * as WriterProvenance from "./internal/writerProvenance.js" + +export const syncEnvelopeVersion = 1 +export const relayOuterEnvelopeVersion = 1 +export const relayOuterEnvelopeDomain = "effect-local/relay-outer-envelope" +export const relayProtocolVersion = 3 +export const maximumWriterProvenanceEntries = 1_000 +export const maximumRelayPayloadBytes = 4 * 1_024 * 1_024 + +const BoundedName = Schema.NonEmptyString.check(Schema.isMaxLength(256)) +const MessageHash = Schema.String.check(Schema.isPattern(/^[0-9a-f]{64}$/)) +const Sequence = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)) + +export const SyncEnvelope = Schema.Struct({ + connectionEpoch: BoundedName, + sequence: Sequence, + documentId: Identity.DocumentId, + documentType: BoundedName, + messageHash: MessageHash, + message: Schema.Uint8ArrayFromBase64, + writerProvenance: WriterProvenance.ChangeProvenances +}) +export type SyncEnvelope = typeof SyncEnvelope.Type + +export interface SyncEnvelopeLimits + extends Pick< + ReplicaLimits.Values, + | "maxSyncMessageBytes" + | "maxSyncChangesPerMessage" + | "maxSyncDependencyEdgesPerMessage" + | "maxSyncOperationsPerMessage" + > +{} + +export const maximumSyncEnvelopeBytes = ( + maxSyncMessageBytes: number, + maxSyncChangesPerMessage: number +): number => maxSyncMessageBytes * 2 + maxSyncChangesPerMessage * 512 + 4_096 + +const SyncEnvelopeJson = Schema.fromJsonString(Schema.toCodecJson(SyncEnvelope)) + +const protocolMismatch = (expected: string, observed: string): ReplicaError.ReplicaError => + new ReplicaError.ReplicaError({ + reason: new ReplicaError.ProtocolMismatch({ expected, observed }) + }) + +export const encodeSyncEnvelope = ( + envelope: SyncEnvelope +): Effect.Effect => + Schema.encodeEffect(SyncEnvelopeJson)({ + ...envelope, + writerProvenance: WriterProvenance.canonicalize(envelope.writerProvenance) + }).pipe( + Effect.map((value) => new TextEncoder().encode(value)), + Effect.mapError(() => protocolMismatch("encodable sync envelope", "invalid sync envelope")) + ) + +const decodeSyncChanges = (message: Uint8Array): ReadonlyArray => { + const changes = new Map() + for (const chunk of Automerge.decodeSyncMessage(message).changes) { + try { + const change = Automerge.decodeChange(chunk) + changes.set(change.hash, change) + } catch { + const document = Automerge.load(chunk) + try { + for (const bytes of Automerge.getAllChanges(document)) { + const change = Automerge.decodeChange(bytes) + changes.set(change.hash, change) + } + } finally { + Automerge.free(document) + } + } + } + return [...changes.values()] +} + +export const validateSyncEnvelope = ( + envelope: SyncEnvelope, + limits: SyncEnvelopeLimits +): Effect.Effect => + Effect.gen(function*() { + if (envelope.message.byteLength > limits.maxSyncMessageBytes) { + return yield* protocolMismatch( + `sync message at most ${limits.maxSyncMessageBytes} bytes`, + "oversized sync message" + ) + } + if (envelope.writerProvenance.length > limits.maxSyncChangesPerMessage) { + return yield* protocolMismatch( + `at most ${limits.maxSyncChangesPerMessage} writer provenance entries`, + "excess writer provenance" + ) + } + const changes = yield* Effect.try({ + try: () => decodeSyncChanges(envelope.message), + catch: () => protocolMismatch("valid Automerge sync message", "invalid Automerge sync message") + }) + if (changes.length > limits.maxSyncChangesPerMessage) { + return yield* protocolMismatch( + `at most ${limits.maxSyncChangesPerMessage} sync changes`, + "excess sync changes" + ) + } + const dependencyEdges = changes.reduce((total, change) => total + change.deps.length, 0) + if (dependencyEdges > limits.maxSyncDependencyEdgesPerMessage) { + return yield* protocolMismatch( + `at most ${limits.maxSyncDependencyEdgesPerMessage} sync dependency edges`, + "excess sync dependency edges" + ) + } + const operations = changes.reduce((total, change) => total + change.ops.length, 0) + if (operations > limits.maxSyncOperationsPerMessage) { + return yield* protocolMismatch( + `at most ${limits.maxSyncOperationsPerMessage} sync operations`, + "excess sync operations" + ) + } + yield* Effect.try({ + try: () => WriterProvenance.validateExact( + changes.map((change) => change.hash), + envelope.writerProvenance + ), + catch: () => protocolMismatch( + "one canonical writer provenance entry per sync change", + "invalid writer provenance" + ) + }) + const messageHash = yield* Canonical.digest(envelope.message) + if (messageHash !== envelope.messageHash) { + return yield* protocolMismatch("matching sync message hash", "conflicting sync message hash") + } + return { + ...envelope, + writerProvenance: WriterProvenance.canonicalize(envelope.writerProvenance) + } + }) + +export const decodeSyncEnvelope = ( + bytes: Uint8Array, + limits: SyncEnvelopeLimits +): Effect.Effect => { + const maximumBytes = maximumSyncEnvelopeBytes( + limits.maxSyncMessageBytes, + limits.maxSyncChangesPerMessage + ) + if (bytes.byteLength > maximumBytes) { + return Effect.fail( + protocolMismatch(`sync envelope at most ${maximumBytes} bytes`, "oversized sync envelope") + ) + } + return Schema.decodeUnknownEffect(SyncEnvelopeJson)(new TextDecoder().decode(bytes)).pipe( + Effect.mapError(() => protocolMismatch("sync envelope", "invalid sync envelope")), + Effect.flatMap((envelope) => validateSyncEnvelope(envelope, limits)) + ) +} + +export const syncEnvelopeDocument = ( + envelope: Pick +): { readonly documentId: Identity.DocumentId; readonly documentType: string } => ({ + documentId: envelope.documentId, + documentType: envelope.documentType +}) + +export const RelayPeerPrincipal = Schema.Struct({ + tenantId: BoundedName, + subjectId: BoundedName, + peerId: Identity.PeerId +}) +export type RelayPeerPrincipal = typeof RelayPeerPrincipal.Type + +export const RelayOuterEnvelope = Schema.Struct({ + domain: Schema.Literal(relayOuterEnvelopeDomain), + version: Schema.Literal(relayOuterEnvelopeVersion), + expectedLocal: RelayPeerPrincipal, + remote: RelayPeerPrincipal, + relayPeerId: Identity.PeerId, + relayMessageId: Identity.RelayMessageId, + protocolVersion: Schema.Literal(relayProtocolVersion), + payloadVersion: Schema.Literal(syncEnvelopeVersion), + senderReplicaIncarnation: Identity.ReplicaIncarnation, + senderConnectionEpoch: BoundedName, + senderSequence: Sequence, + document: Schema.Struct({ + documentId: Identity.DocumentId, + documentType: BoundedName + }), + writerProvenance: WriterProvenance.ChangeProvenances.check( + Schema.isMaxLength(maximumWriterProvenanceEntries) + ), + messageHash: MessageHash, + payload: Schema.Uint8Array.check( + Schema.makeFilter( + (payload) => payload.byteLength <= maximumRelayPayloadBytes, + { expected: `Uint8Array with at most ${maximumRelayPayloadBytes} bytes` } + ) + ) +}) +export type RelayOuterEnvelope = typeof RelayOuterEnvelope.Type + +export const canonicalizeRelayOuterEnvelope = ( + envelope: RelayOuterEnvelope +): RelayOuterEnvelope => ({ + ...envelope, + writerProvenance: WriterProvenance.canonicalize(envelope.writerProvenance) +}) + +export const encodeRelayOuterEnvelope = ( + envelope: RelayOuterEnvelope +): Effect.Effect => + Schema.encodeEffect(RelayOuterEnvelope)(canonicalizeRelayOuterEnvelope(envelope)).pipe( + Effect.map((encoded) => new TextEncoder().encode(Canonical.stringify(encoded))), + Effect.mapError(() => protocolMismatch("encodable relay outer envelope", "invalid relay outer envelope")) + ) + +export const digestRelayOuterEnvelope = ( + envelope: RelayOuterEnvelope +): Effect.Effect => + Schema.encodeEffect(RelayOuterEnvelope)(canonicalizeRelayOuterEnvelope(envelope)).pipe( + Effect.mapError(() => protocolMismatch("encodable relay outer envelope", "invalid relay outer envelope")), + Effect.flatMap(Canonical.digest) + ) + +export const relayOuterEnvelopeDocument = ( + envelope: Pick +): RelayOuterEnvelope["document"] => envelope.document diff --git a/packages/local-sql/src/SqlReplica.ts b/packages/local-sql/src/SqlReplica.ts index 60a1ffd..825c39c 100644 --- a/packages/local-sql/src/SqlReplica.ts +++ b/packages/local-sql/src/SqlReplica.ts @@ -23,6 +23,7 @@ import * as DocumentStore from "./DocumentStore.js" import * as DurableRuntime from "./DurableRuntime.js" import * as EntityReplica from "./EntityReplica.js" import * as InternalAutomerge from "./internal/automerge.js" +import type * as PeerRelayReceiptLimits from "./PeerRelayReceiptLimits.js" import type * as PeerSync from "./PeerSync.js" import * as ProjectionStore from "./ProjectionStore.js" import * as QueryExecutor from "./QueryExecutor.js" @@ -174,26 +175,13 @@ export const layerFromServices = (definition: ReplicaDefinition.Any): Layer.Laye }) ) -export const layer = ,>( +const makeBase = < + D extends ReplicaDefinition.Any, + const Bindings extends ReadonlyArray, +>( definition: D, options: { readonly health?: ReplicaHealth.Options; readonly projections: Bindings } -): Layer.Layer< - | CommitPublisher.CommitPublisher - | PeerSync.PeerSync - | Replica.Replica - | ReplicaEvolution.ReplicaEvolution - | ReplicaGate.ReplicaGate - | ReplicaWorkflow.CompactionWorkflow - | ReplicaWorkflow.HistoryRewriteWorkflow - | Sharding.Sharding, - ConfigError | Migrator.MigrationError | ReplicaError.ReplicaError | SqlError.SqlError, - | CommandExecutor.MutationHandlers - | ProjectionStore.BindingServices - | QueryExecutor.QueryHandlers - | ReplicaLimits.ReplicaLimits - | Crypto.Crypto - | SqlClient.SqlClient -> => { +) => { const expected = new Set(definition.projections) const actual = new Set(options.projections.map((binding) => binding.projection)) if ( @@ -218,12 +206,67 @@ export const layer = ,>( + definition: D, + options: { readonly health?: ReplicaHealth.Options; readonly projections: Bindings } +): Layer.Layer< + | CommitPublisher.CommitPublisher + | PeerSync.PeerSync + | Replica.Replica + | ReplicaEvolution.ReplicaEvolution + | ReplicaGate.ReplicaGate + | ReplicaWorkflow.CompactionWorkflow + | ReplicaWorkflow.HistoryRewriteWorkflow + | Sharding.Sharding, + ConfigError | Migrator.MigrationError | ReplicaError.ReplicaError | SqlError.SqlError, + | CommandExecutor.MutationHandlers + | ProjectionStore.BindingServices + | QueryExecutor.QueryHandlers + | ReplicaLimits.ReplicaLimits + | Crypto.Crypto + | SqlClient.SqlClient +> => { + const { backups, compaction } = makeBase(definition, options) const durable = DurableRuntime.layer(definition).pipe( Layer.provideMerge(Layer.merge(backups, compaction)) ) return EntityReplica.layer(definition).pipe(Layer.provideMerge(durable)) } +export const layerRelay = < + D extends ReplicaDefinition.Any, + const Bindings extends ReadonlyArray, +>( + definition: D, + options: { readonly health?: ReplicaHealth.Options; readonly projections: Bindings } +): Layer.Layer< + | CommitPublisher.CommitPublisher + | PeerSync.PeerSync + | Replica.Replica + | ReplicaEvolution.ReplicaEvolution + | ReplicaGate.ReplicaGate + | ReplicaWorkflow.CompactionWorkflow + | ReplicaWorkflow.HistoryRewriteWorkflow + | Sharding.Sharding, + ConfigError | Migrator.MigrationError | ReplicaError.ReplicaError | SqlError.SqlError, + | CommandExecutor.MutationHandlers + | ProjectionStore.BindingServices + | QueryExecutor.QueryHandlers + | ReplicaLimits.ReplicaLimits + | PeerRelayReceiptLimits.PeerRelayReceiptLimits + | Crypto.Crypto + | SqlClient.SqlClient +> => { + const { backups, compaction } = makeBase(definition, options) + const durable = DurableRuntime.layerRelay(definition).pipe( + Layer.provideMerge(Layer.merge(backups, compaction)) + ) + return EntityReplica.layer(definition).pipe(Layer.provideMerge(durable)) +} + export const layerWithBindings = < D extends ReplicaDefinition.Any, const Bindings extends ReadonlyArray, diff --git a/packages/local-sql/src/index.ts b/packages/local-sql/src/index.ts index b572e2e..a0d1eca 100644 --- a/packages/local-sql/src/index.ts +++ b/packages/local-sql/src/index.ts @@ -7,8 +7,13 @@ export * as DocumentStore from "./DocumentStore.js" export * as DurableRuntime from "./DurableRuntime.js" export * as EntityReplica from "./EntityReplica.js" export * as Migrations from "./Migrations.js" +export * as PeerRelayClientRuntime from "./PeerRelayClientRuntime.js" +export * as PeerRelayOutbox from "./PeerRelayOutbox.js" +export * as PeerRelayOutboxLimits from "./PeerRelayOutboxLimits.js" +export * as PeerRelayReceiptLimits from "./PeerRelayReceiptLimits.js" export * as PeerSession from "./PeerSession.js" export * as PeerSync from "./PeerSync.js" +export * as PeerSyncEnvelope from "./PeerSyncEnvelope.js" export * as ProjectionStore from "./ProjectionStore.js" export * as QueryExecutor from "./QueryExecutor.js" export * as Recovery from "./Recovery.js" diff --git a/packages/local-sql/src/internal/peerRelayOutboxMaintenance.ts b/packages/local-sql/src/internal/peerRelayOutboxMaintenance.ts new file mode 100644 index 0000000..a921131 --- /dev/null +++ b/packages/local-sql/src/internal/peerRelayOutboxMaintenance.ts @@ -0,0 +1,29 @@ +import * as Effect from "effect/Effect" + +export interface Options { + readonly prune: Effect.Effect + readonly intervalMillis: number + readonly pruneRowsPerSecond: number +} + +const delayAfter = ( + rows: number, + intervalMillis: number, + pruneRowsPerSecond: number +): number => + Math.max( + intervalMillis, + Math.ceil((rows / pruneRowsPerSecond) * 1_000) + ) + +export const run = ( + options: Options +): Effect.Effect => + Effect.gen(function*() { + while (true) { + const rows: number = yield* options.prune + yield* Effect.sleep( + delayAfter(rows, options.intervalMillis, options.pruneRowsPerSecond) + ) + } + }) diff --git a/packages/local-sql/src/internal/peerRelayReceiptMaintenance.ts b/packages/local-sql/src/internal/peerRelayReceiptMaintenance.ts new file mode 100644 index 0000000..c11e401 --- /dev/null +++ b/packages/local-sql/src/internal/peerRelayReceiptMaintenance.ts @@ -0,0 +1,25 @@ +import * as Effect from "effect/Effect" +import * as Queue from "effect/Queue" + +export interface Options { + readonly prune: Effect.Effect + readonly intervalMillis: number + readonly pruneRowsPerSecond: number + readonly wakeup: Queue.Dequeue +} + +const rateDelay = (rows: number, pruneRowsPerSecond: number): number => Math.ceil((rows / pruneRowsPerSecond) * 1_000) + +export const run = ( + options: Options +): Effect.Effect => + Effect.gen(function*() { + while (true) { + yield* Queue.take(options.wakeup).pipe( + Effect.raceFirst(Effect.sleep(options.intervalMillis)) + ) + const rows: number = yield* options.prune + const delay = rateDelay(rows, options.pruneRowsPerSecond) + if (delay > 0) yield* Effect.sleep(delay) + } + }) diff --git a/packages/local-sql/test/BackupStore.test.ts b/packages/local-sql/test/BackupStore.test.ts index 27a7ece..4fe5393 100644 --- a/packages/local-sql/test/BackupStore.test.ts +++ b/packages/local-sql/test/BackupStore.test.ts @@ -152,6 +152,52 @@ describe("BackupStore", () => { Limits, ProjectedBackup ) + const seedRelayState = (documentId: Identity.DocumentId) => + Effect.gen(function*() { + const sql = yield* SqlClient.SqlClient + const current = yield* ReplicaGate.ReplicaGate.pipe(Effect.flatMap((gate) => gate.current)) + const relayMessageId = "rly_00000000-0000-4000-8000-000000000001" + const senderPeerId = "peer_00000000-0000-4000-8000-000000000001" + const remotePeerId = "peer_00000000-0000-4000-8000-000000000002" + yield* sql`INSERT INTO effect_local_peer_receipts ( + replica_incarnation, peer_id, connection_epoch, receive_sequence, document_id, + message_hash, reply, reply_hash, pending_message, heads, accepted_heads, + commit_sequence, accepted_at, writer_provenance, relay_sender_tenant_id, + relay_sender_subject_id, relay_sender_peer_id, relay_message_id, + relay_outer_envelope_digest, relay_receipt_expires_at, relay_encoded_size + ) VALUES ( + ${current.incarnation}, ${remotePeerId}, 'sender-epoch', 0, ${documentId}, + ${"a".repeat(64)}, NULL, NULL, NULL, '[]', '[]', 0, + '2026-07-25T00:00:00.000Z', '[]', 'tenant', 'sender', ${senderPeerId}, + ${relayMessageId}, ${"b".repeat(64)}, '2026-08-02T00:00:00.000Z', 128 + )` + yield* sql`INSERT INTO effect_local_peer_relay_receipt_usage ( + replica_incarnation, sender_tenant_id, sender_subject_id, sender_peer_id, + receipt_count, encoded_bytes + ) VALUES (${current.incarnation}, 'tenant', 'sender', ${senderPeerId}, 1, 128)` + yield* sql`INSERT INTO effect_local_peer_relay_outbox ( + replica_id, replica_incarnation, writer_generation, expected_local_tenant_id, + expected_local_subject_id, expected_local_peer_id, remote_tenant_id, + remote_subject_id, remote_peer_id, relay_peer_id, relay_message_id, + outer_envelope_digest, protocol_version, payload_version, sender_connection_epoch, + sender_sequence, document_id, document_type, writer_provenance, message_hash, + payload, encoded_size, created_at, retry_deadline, next_attempt_at, custody_state + ) VALUES ( + ${current.replicaId}, ${current.incarnation}, ${current.writerGeneration}, + 'tenant', 'local', ${senderPeerId}, + 'tenant', 'remote', ${remotePeerId}, ${senderPeerId}, ${relayMessageId}, + ${"b".repeat(64)}, 3, 1, 'sender-epoch', 0, ${documentId}, ${Task.name}, '[]', + ${"a".repeat(64)}, ${Uint8Array.of(1)}, 1, '2026-07-25T00:00:00.000Z', + '2026-08-01T00:00:00.000Z', '2026-07-25T00:00:00.000Z', 'Pending' + )` + yield* sql`INSERT INTO effect_local_peer_relay_outbox_remote_usage ( + replica_incarnation, remote_tenant_id, remote_subject_id, remote_peer_id, + message_count, encoded_bytes + ) VALUES (${current.incarnation}, 'tenant', 'remote', ${remotePeerId}, 1, 1)` + yield* sql`INSERT INTO effect_local_peer_relay_outbox_replica_usage ( + replica_incarnation, message_count, encoded_bytes + ) VALUES (${current.incarnation}, 1, 1)` + }) const ProjectedRecovery = Recovery.layer.pipe(Layer.provide(ProjectedLive)) const ProjectedCompaction = Compaction.layer.pipe( Layer.provide(Layer.merge(ProjectedLive, ProjectedRecovery)) @@ -591,7 +637,12 @@ describe("BackupStore", () => { it.effect("retires cluster request and reply state during restore", () => Effect.gen(function*() { const backups = yield* BackupStore.BackupStore + const store = yield* DocumentStore.DocumentStore const sql = yield* SqlClient.SqlClient + const documentId = yield* Identity.makeDocumentId + const created = yield* store.create(Task, documentId, { title: "relay state" }) + InternalAutomerge.free(created.automerge) + yield* seedRelayState(documentId) yield* sql`CREATE TABLE ${sql(`${ClusterStorage.messagePrefix}_messages`)} (id INTEGER PRIMARY KEY)` yield* sql`CREATE TABLE ${sql(`${ClusterStorage.messagePrefix}_replies`)} (id INTEGER PRIMARY KEY)` yield* sql`INSERT INTO ${sql(`${ClusterStorage.messagePrefix}_messages`)} (id) VALUES (1)` @@ -606,10 +657,31 @@ describe("BackupStore", () => { expectedDefinitionHash: definition.hash }) - const rows = yield* sql<{ readonly messages: number; readonly replies: number }>`SELECT + const rows = yield* sql<{ + readonly messages: number + readonly outbox: number + readonly outboxRemoteUsage: number + readonly outboxReplicaUsage: number + readonly receipts: number + readonly receiptUsage: number + readonly replies: number + }>`SELECT (SELECT COUNT(*) FROM ${sql(`${ClusterStorage.messagePrefix}_messages`)}) AS messages, - (SELECT COUNT(*) FROM ${sql(`${ClusterStorage.messagePrefix}_replies`)}) AS replies` - assert.deepStrictEqual(rows[0], { messages: 0, replies: 0 }) + (SELECT COUNT(*) FROM ${sql(`${ClusterStorage.messagePrefix}_replies`)}) AS replies, + (SELECT COUNT(*) FROM effect_local_peer_relay_outbox) AS outbox, + (SELECT COUNT(*) FROM effect_local_peer_relay_outbox_remote_usage) AS outboxRemoteUsage, + (SELECT COUNT(*) FROM effect_local_peer_relay_outbox_replica_usage) AS outboxReplicaUsage, + (SELECT COUNT(*) FROM effect_local_peer_receipts WHERE relay_message_id IS NOT NULL) AS receipts, + (SELECT COUNT(*) FROM effect_local_peer_relay_receipt_usage) AS receiptUsage` + assert.deepStrictEqual(rows[0], { + messages: 0, + outbox: 0, + outboxRemoteUsage: 0, + outboxReplicaUsage: 0, + receipts: 0, + receiptUsage: 0, + replies: 0 + }) }).pipe(Effect.provide(Live))) it.effect("reports invalid local rows as storage corruption during export", () => @@ -646,6 +718,7 @@ describe("BackupStore", () => { draft.title = "preserved" }) const current = yield* store.persist(Task, documentId, created, staged) + yield* seedRelayState(documentId) yield* sql`CREATE TRIGGER fence_restore AFTER DELETE ON effect_local_documents BEGIN @@ -666,6 +739,25 @@ describe("BackupStore", () => { SELECT installation_id FROM effect_local_backup_installations ` assert.strictEqual(installations.length, 0) + const relayRows = yield* sql<{ + readonly outbox: number + readonly outboxRemoteUsage: number + readonly outboxReplicaUsage: number + readonly receipts: number + readonly receiptUsage: number + }>`SELECT + (SELECT COUNT(*) FROM effect_local_peer_relay_outbox) AS outbox, + (SELECT COUNT(*) FROM effect_local_peer_relay_outbox_remote_usage) AS outboxRemoteUsage, + (SELECT COUNT(*) FROM effect_local_peer_relay_outbox_replica_usage) AS outboxReplicaUsage, + (SELECT COUNT(*) FROM effect_local_peer_receipts WHERE relay_message_id IS NOT NULL) AS receipts, + (SELECT COUNT(*) FROM effect_local_peer_relay_receipt_usage) AS receiptUsage` + assert.deepStrictEqual(relayRows, [{ + outbox: 1, + outboxRemoteUsage: 1, + outboxReplicaUsage: 1, + receipts: 1, + receiptUsage: 1 + }]) InternalAutomerge.free(preserved.automerge) InternalAutomerge.free(current.automerge) InternalAutomerge.free(staged) diff --git a/packages/local-sql/test/DocumentEntity.test.ts b/packages/local-sql/test/DocumentEntity.test.ts index d738216..0a875e7 100644 --- a/packages/local-sql/test/DocumentEntity.test.ts +++ b/packages/local-sql/test/DocumentEntity.test.ts @@ -108,7 +108,8 @@ describe("DocumentEntity", () => { enqueue: (_session, reply) => Effect.succeed({ ...reply, sendSequence: 0, lineage: Identity.genesisLineage, writerProvenance: [] }), pending: () => Effect.succeed([]), - markSent: () => Effect.succeed(false) + markSent: () => Effect.succeed(false), + pruneRelayReceipts: Effect.succeed(0) }) const replicaGate = (permit: ReplicaGate.Permit) => ReplicaGate.ReplicaGate.of({ @@ -303,6 +304,63 @@ describe("DocumentEntity", () => { ) }) + it("scopes relay sync primary keys by the complete sender identity", () => { + const relayPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000001") + const senderPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000002") + const relayMessageId = Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000001") + const hash = "a".repeat(64) + const base = { + replicaIncarnation: Identity.ReplicaIncarnation.make(1), + peerId: relayPeerId, + connectionEpoch: "sender-epoch", + localConnectionEpoch: "local-epoch", + receiveSequence: 2, + documentType: Task.name, + messageHash: hash, + message: new Uint8Array([1]), + writerProvenance: [], + relay: { + relayMessageId, + relayPeerId, + senderTenantId: "tenant", + senderSubjectId: "sender-a", + senderPeerId, + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(3), + messageHash: hash, + outerEnvelopeDigest: "b".repeat(64), + receiptExpiresAt: "2026-08-02T00:00:00.000Z", + encodedSize: 10 + } + } + const keyOf = (payload: unknown) => { + if (!PrimaryKey.isPrimaryKey(payload)) throw new TypeError("Expected a primary key payload") + return PrimaryKey.value(payload) + } + const key = keyOf(DocumentEntity.ApplySync.payloadSchema.make(base)) + assert.strictEqual( + key, + keyOf(DocumentEntity.ApplySync.payloadSchema.make({ + ...base, + connectionEpoch: "another-epoch", + receiveSequence: 99 + })) + ) + assert.notStrictEqual( + key, + keyOf(DocumentEntity.ApplySync.payloadSchema.make({ + ...base, + relay: { ...base.relay, senderSubjectId: "sender-b" } + })) + ) + assert.notStrictEqual( + key, + keyOf(DocumentEntity.ApplySync.payloadSchema.make({ + ...base, + relay: { ...base.relay, outerEnvelopeDigest: "c".repeat(64) } + })) + ) + }) + it("persists RPCs in the shared SQL transaction without server interruption", () => { for (const rpc of [DocumentEntity.Create, DocumentEntity.Mutate, DocumentEntity.Delete]) { assert.strictEqual(Context.get(rpc.annotations, ClusterSchema.Persisted), true) @@ -342,10 +400,12 @@ describe("DocumentEntity", () => { }> >([]) const receivedLineages = yield* Ref.make>([]) + const receivedRelay = yield* Ref.make(undefined) const sync = peerSync((_document, _documentId, _session, input) => Effect.all([ Ref.set(receivedProvenance, input.writerProvenance), - Ref.update(receivedLineages, (lineages) => [...lineages, input.lineage]) + Ref.update(receivedLineages, (lineages) => [...lineages, input.lineage]), + Ref.set(receivedRelay, input.relay) ]).pipe(Effect.as(syncResult)) ) const makeClient = yield* Entity.makeTestClient( @@ -440,6 +500,39 @@ describe("DocumentEntity", () => { }) assert.deepStrictEqual(rewritten, syncResult) assert.deepStrictEqual(yield* Ref.get(receivedLineages), [Identity.genesisLineage, rewrittenLineage]) + const messageHash = yield* Canonical.digest(message) + const relay = { + relayMessageId: Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000001"), + relayPeerId: Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000001"), + senderTenantId: "tenant", + senderSubjectId: "sender", + senderPeerId: Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000002"), + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(4), + messageHash, + outerEnvelopeDigest: "b".repeat(64), + receiptExpiresAt: "2026-08-02T00:00:00.000Z", + encodedSize: 128 + } satisfies PeerSync.RelayReceipt + assert.deepStrictEqual( + yield* client.ApplySync({ + replicaIncarnation: permit.incarnation, + peerId: relay.relayPeerId, + connectionEpoch: "relay-connection", + localConnectionEpoch: "local-connection", + receiveSequence: 1, + documentType: Task.name, + messageHash, + message, + writerProvenance, + relay + }), + syncResult + ) + assert.deepStrictEqual(yield* Ref.get(receivedRelay), relay) + assert.deepStrictEqual( + yield* Ref.get(receivedLineages), + [Identity.genesisLineage, rewrittenLineage, Identity.genesisLineage] + ) const stale = yield* Effect.exit(client.ApplySync({ replicaIncarnation: Identity.ReplicaIncarnation.make(permit.incarnation - 1), peerId: (yield* Identity.makePeerId), diff --git a/packages/local-sql/test/MigrationFixtures.test.ts b/packages/local-sql/test/MigrationFixtures.test.ts index bac87a1..375ec30 100644 --- a/packages/local-sql/test/MigrationFixtures.test.ts +++ b/packages/local-sql/test/MigrationFixtures.test.ts @@ -21,7 +21,8 @@ const expectations = { [6, "peer_writer_provenance"], [7, "replica_health_indexes"], [8, "document_lineage"], - [9, "history_rewrite_markers"] + [9, "history_rewrite_markers"], + [10, "peer_relay_state"] ], outbox: "none" }, @@ -33,7 +34,8 @@ const expectations = { [6, "peer_writer_provenance"], [7, "replica_health_indexes"], [8, "document_lineage"], - [9, "history_rewrite_markers"] + [9, "history_rewrite_markers"], + [10, "peer_relay_state"] ], outbox: "backfilled" }, @@ -44,7 +46,8 @@ const expectations = { [6, "peer_writer_provenance"], [7, "replica_health_indexes"], [8, "document_lineage"], - [9, "history_rewrite_markers"] + [9, "history_rewrite_markers"], + [10, "peer_relay_state"] ], outbox: { frozen: "2020-01-01T00:00:00.000Z" } } @@ -131,7 +134,8 @@ const assertMigrationHistory = Effect.gen(function*() { { migration_id: 6, name: "peer_writer_provenance" }, { migration_id: 7, name: "replica_health_indexes" }, { migration_id: 8, name: "document_lineage" }, - { migration_id: 9, name: "history_rewrite_markers" } + { migration_id: 9, name: "history_rewrite_markers" }, + { migration_id: 10, name: "peer_relay_state" } ]) const catalog = yield* SqlSchema.findAll({ @@ -152,7 +156,8 @@ const assertMigrationHistory = Effect.gen(function*() { migration_id: 9, name: "history_rewrite_markers", checksum: Migrations.historyRewriteMarkersChecksum - } + }, + { migration_id: 10, name: "peer_relay_state", checksum: Migrations.peerRelayStateChecksum } ]) }) diff --git a/packages/local-sql/test/Migrations.test.ts b/packages/local-sql/test/Migrations.test.ts index 7003307..c116bc6 100644 --- a/packages/local-sql/test/Migrations.test.ts +++ b/packages/local-sql/test/Migrations.test.ts @@ -23,10 +23,15 @@ describe("Migrations", () => { assert.deepStrictEqual(indexes.map((row) => row.name).toSorted(), [ "effect_local_peer_outbox_connection_status", "effect_local_peer_outbox_incarnation_created", + "effect_local_peer_receipts_direct_identity", "effect_local_peer_receipts_document_sequence", "effect_local_peer_receipts_incarnation_accepted", "effect_local_peer_receipts_pending_document", - "effect_local_peer_receipts_pending_peer" + "effect_local_peer_receipts_pending_peer", + "effect_local_peer_receipts_relay_expiry", + "effect_local_peer_receipts_relay_identity", + "effect_local_peer_relay_outbox_due_endpoint", + "effect_local_peer_relay_outbox_retry_deadline" ]) const readinessIndexes = yield* sql<{ readonly name: string }>` SELECT name FROM sqlite_master WHERE type = 'index' AND name IN ( @@ -128,13 +133,28 @@ describe("Migrations", () => { ` assert.strictEqual(indexes.length, 0) const recorded = yield* sql<{ readonly migration_id: number }>` - SELECT migration_id FROM effect_local_migrations WHERE migration_id IN (4, 5, 6, 7, 8, 9) + SELECT migration_id FROM effect_local_migrations WHERE migration_id IN (4, 5, 6, 7, 8, 9, 10) ` assert.strictEqual(recorded.length, 0) const catalog = yield* sql<{ readonly migration_id: number }>` - SELECT migration_id FROM effect_local_migration_catalog WHERE migration_id IN (4, 5, 6, 7, 8, 9) + SELECT migration_id FROM effect_local_migration_catalog WHERE migration_id IN (4, 5, 6, 7, 8, 9, 10) ` assert.strictEqual(catalog.length, 0) + const receiptPrimaryKey = yield* sql<{ readonly name: string; readonly pk: number }>` + SELECT name, pk FROM pragma_table_info('effect_local_peer_receipts') + WHERE pk != 0 ORDER BY pk + ` + assert.deepStrictEqual(receiptPrimaryKey, [ + { name: "replica_incarnation", pk: 1 }, + { name: "peer_id", pk: 2 }, + { name: "connection_epoch", pk: 3 }, + { name: "receive_sequence", pk: 4 } + ]) + assert.deepStrictEqual( + yield* sql`SELECT name FROM sqlite_master + WHERE type = 'table' AND name = 'effect_local_peer_relay_outbox'`, + [] + ) }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:", disableWAL: true })))) it.effect("upgrades populated version two storage without losing durability state", () => @@ -219,7 +239,8 @@ describe("Migrations", () => { [6, "peer_writer_provenance"], [7, "replica_health_indexes"], [8, "document_lineage"], - [9, "history_rewrite_markers"] + [9, "history_rewrite_markers"], + [10, "peer_relay_state"] ]) const outbox = yield* sql<{ @@ -244,6 +265,24 @@ describe("Migrations", () => { SELECT name FROM pragma_table_info('effect_local_peer_receipts') ` assert.isTrue(receiptColumns.some((column) => column.name === "writer_provenance")) + for ( + const name of [ + "relay_sender_tenant_id", + "relay_sender_subject_id", + "relay_sender_peer_id", + "relay_message_id", + "relay_outer_envelope_digest", + "relay_receipt_expires_at", + "relay_encoded_size" + ] + ) { + assert.isTrue(receiptColumns.some((column) => column.name === name)) + } + const receiptPrimaryKey = yield* sql<{ readonly name: string; readonly pk: number }>` + SELECT name, pk FROM pragma_table_info('effect_local_peer_receipts') + WHERE pk != 0 + ` + assert.deepStrictEqual(receiptPrimaryKey, [{ name: "row_id", pk: 1 }]) const checkpoints = yield* sql<{ readonly checkpoint_hash: string @@ -294,6 +333,27 @@ describe("Migrations", () => { WHERE message_hash = 'malformed-incoming'`, [{ writer_provenance: "[]" }] ) + assert.deepStrictEqual( + yield* sql`SELECT + relay_sender_tenant_id, + relay_sender_subject_id, + relay_sender_peer_id, + relay_message_id, + relay_outer_envelope_digest, + relay_receipt_expires_at, + relay_encoded_size + FROM effect_local_peer_receipts + WHERE message_hash = 'malformed-incoming'`, + [{ + relay_sender_tenant_id: null, + relay_sender_subject_id: null, + relay_sender_peer_id: null, + relay_message_id: null, + relay_outer_envelope_digest: null, + relay_receipt_expires_at: null, + relay_encoded_size: null + }] + ) Automerge.free(legacyCheckpoint) Automerge.free(emptyPeer) @@ -315,7 +375,8 @@ describe("Migrations", () => { migration_id: 9, name: "history_rewrite_markers", checksum: Migrations.historyRewriteMarkersChecksum - } + }, + { migration_id: 10, name: "peer_relay_state", checksum: Migrations.peerRelayStateChecksum } ]) const indexes = yield* sql<{ readonly name: string }>` @@ -345,4 +406,169 @@ describe("Migrations", () => { "effect_local_projection_registry_name_status" ]) }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:", disableWAL: true })))) + + it.effect("enforces relay identity, payload, and usage invariants", () => + Effect.gen(function*() { + yield* Migrations.run + const sql = yield* SqlClient.SqlClient + yield* sql`INSERT INTO effect_local_documents ( + document_id, document_type, schema_version, observed_versions, materialized_heads, + accepted_heads, tombstone, projection_status + ) VALUES ( + 'doc_00000000-0000-4000-8000-000000000001', + 'Task', + 1, + '[]', + '[]', + '[]', + 0, + 'Ready' + )` + + const partialReceipt = yield* Effect.exit(sql`INSERT INTO effect_local_peer_receipts ( + replica_incarnation, peer_id, connection_epoch, receive_sequence, document_id, + message_hash, heads, accepted_heads, commit_sequence, accepted_at, writer_provenance, + relay_message_id + ) VALUES ( + 0, 'peer-direct', 'epoch-partial', 0, + 'doc_00000000-0000-4000-8000-000000000001', + ${"a".repeat(64)}, '[]', '[]', 0, '2026-01-01T00:00:00.000Z', '[]', 'relay-partial' + )`) + assert.strictEqual(partialReceipt._tag, "Failure") + + const insertRelayReceipt = ( + peerId: string, + epoch: string, + senderSubjectId: string + ) => + sql`INSERT INTO effect_local_peer_receipts ( + replica_incarnation, peer_id, connection_epoch, receive_sequence, document_id, + message_hash, heads, accepted_heads, commit_sequence, accepted_at, writer_provenance, + relay_sender_tenant_id, relay_sender_subject_id, relay_sender_peer_id, + relay_message_id, relay_outer_envelope_digest, relay_receipt_expires_at, relay_encoded_size + ) VALUES ( + 0, ${peerId}, ${epoch}, 0, + 'doc_00000000-0000-4000-8000-000000000001', + ${"b".repeat(64)}, '[]', '[]', 0, '2026-01-01T00:00:00.000Z', '[]', + 'tenant-1', ${senderSubjectId}, 'peer-sender', 'relay-id-1', + ${"c".repeat(64)}, '2026-01-09T00:00:00.000Z', 32 + )` + yield* insertRelayReceipt("peer-direct", "epoch-1", "subject-1") + assert.strictEqual( + (yield* Effect.exit(insertRelayReceipt("peer-direct", "epoch-2", "subject-1")))._tag, + "Failure" + ) + yield* insertRelayReceipt("peer-direct", "epoch-1", "subject-2") + + const insertDirectReceipt = (messageHash: string) => + sql`INSERT INTO effect_local_peer_receipts ( + replica_incarnation, peer_id, connection_epoch, receive_sequence, document_id, + message_hash, heads, accepted_heads, commit_sequence, accepted_at, writer_provenance + ) VALUES ( + 0, 'peer-direct', 'epoch-1', 0, + 'doc_00000000-0000-4000-8000-000000000001', + ${messageHash}, '[]', '[]', 0, '2026-01-01T00:00:00.000Z', '[]' + )` + yield* insertDirectReceipt("1".repeat(64)) + assert.strictEqual( + (yield* Effect.exit(insertDirectReceipt("2".repeat(64))))._tag, + "Failure" + ) + + const payload = Uint8Array.of(1, 2, 3) + const insertRelayOutbox = ( + relayMessageId: string, + senderSequence: number, + writerGeneration = 1 + ) => + sql`INSERT INTO effect_local_peer_relay_outbox ( + replica_id, replica_incarnation, writer_generation, + expected_local_tenant_id, expected_local_subject_id, expected_local_peer_id, + remote_tenant_id, remote_subject_id, remote_peer_id, relay_peer_id, + relay_message_id, outer_envelope_digest, protocol_version, payload_version, + sender_connection_epoch, sender_sequence, document_id, document_type, + writer_provenance, message_hash, payload, encoded_size, + created_at, retry_deadline, next_attempt_at, custody_state + ) VALUES ( + 'rep_00000000-0000-4000-8000-000000000011', 0, ${writerGeneration}, + 'tenant-1', 'subject-local', 'peer-local', + 'tenant-1', 'subject-remote', 'peer-remote', 'peer-relay', + ${relayMessageId}, ${"d".repeat(64)}, 3, 1, + 'epoch-outbound', ${senderSequence}, + 'doc_00000000-0000-4000-8000-000000000001', 'Task', + '[]', ${"e".repeat(64)}, ${payload}, ${payload.byteLength}, + '2026-01-01T00:00:00.000Z', '2026-01-08T00:00:00.000Z', + '2026-01-01T00:00:00.000Z', 'Pending' + )` + yield* insertRelayOutbox("relay-outbox-1", 1) + assert.strictEqual( + (yield* Effect.exit(insertRelayOutbox("relay-outbox-2", 1, 2)))._tag, + "Failure" + ) + assert.strictEqual( + (yield* Effect.exit(insertRelayOutbox("relay-outbox-1", 2)))._tag, + "Failure" + ) + assert.strictEqual( + (yield* Effect.exit(sql`UPDATE effect_local_peer_relay_outbox + SET encoded_size = encoded_size + 1 WHERE relay_message_id = 'relay-outbox-1'`))._tag, + "Failure" + ) + + assert.strictEqual( + (yield* Effect.exit(sql`INSERT INTO effect_local_peer_relay_receipt_usage ( + replica_incarnation, sender_tenant_id, sender_subject_id, sender_peer_id, + receipt_count, encoded_bytes + ) VALUES (0, 'tenant-1', 'subject-1', 'peer-sender', 1, 0)`))._tag, + "Failure" + ) + yield* sql`INSERT INTO effect_local_peer_relay_outbox_remote_usage ( + replica_incarnation, remote_tenant_id, remote_subject_id, remote_peer_id, + message_count, encoded_bytes + ) VALUES (0, 'tenant-1', 'subject-remote', 'peer-remote', 1, 3)` + yield* sql`INSERT INTO effect_local_peer_relay_outbox_replica_usage ( + replica_incarnation, message_count, encoded_bytes + ) VALUES (0, 1, 3)` + + assert.strictEqual( + (yield* Effect.exit(sql`DELETE FROM effect_local_documents + WHERE document_id = 'doc_00000000-0000-4000-8000-000000000001'`))._tag, + "Failure" + ) + const [outboxCount] = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count + FROM effect_local_peer_relay_outbox + WHERE relay_message_id = 'relay-outbox-1' + ` + const [remoteUsage] = yield* sql<{ + readonly message_count: number + readonly encoded_bytes: number + }>`SELECT message_count, encoded_bytes + FROM effect_local_peer_relay_outbox_remote_usage + WHERE replica_incarnation = 0 + AND remote_tenant_id = 'tenant-1' + AND remote_subject_id = 'subject-remote' + AND remote_peer_id = 'peer-remote'` + const [replicaUsage] = yield* sql<{ + readonly message_count: number + readonly encoded_bytes: number + }>`SELECT message_count, encoded_bytes + FROM effect_local_peer_relay_outbox_replica_usage + WHERE replica_incarnation = 0` + assert.strictEqual(outboxCount?.count, 1) + assert.deepStrictEqual(remoteUsage, { message_count: 1, encoded_bytes: 3 }) + assert.deepStrictEqual(replicaUsage, { message_count: 1, encoded_bytes: 3 }) + }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:", disableWAL: true })))) + + it.effect("rejects a changed relay migration checksum", () => + Effect.gen(function*() { + yield* Migrations.run + const sql = yield* SqlClient.SqlClient + yield* sql`UPDATE effect_local_migration_catalog + SET checksum = 'changed' + WHERE migration_id = 10` + const error = yield* Effect.flip(Migrations.run) + assert.strictEqual(error._tag, "MigrationError") + assert.include(error.message, "Peer relay state") + }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:", disableWAL: true })))) }) diff --git a/packages/local-sql/test/PeerRelayClientRuntime.test.ts b/packages/local-sql/test/PeerRelayClientRuntime.test.ts new file mode 100644 index 0000000..9fd3e0d --- /dev/null +++ b/packages/local-sql/test/PeerRelayClientRuntime.test.ts @@ -0,0 +1,241 @@ +import * as Automerge from "@automerge/automerge" +import { NodeCrypto } from "@effect/platform-node" +import { SqliteClient } from "@effect/sql-sqlite-node" +import { assert, describe, it } from "@effect/vitest" +import * as Canonical from "@lucas-barake/effect-local/Canonical" +import * as Document from "@lucas-barake/effect-local/Document" +import * as DocumentSet from "@lucas-barake/effect-local/DocumentSet" +import * as Identity from "@lucas-barake/effect-local/Identity" +import * as ReplicaDefinition from "@lucas-barake/effect-local/ReplicaDefinition" +import * as ReplicaLimits from "@lucas-barake/effect-local/ReplicaLimits" +import * as Cause from "effect/Cause" +import * as Context from "effect/Context" +import * as Effect from "effect/Effect" +import * as Exit from "effect/Exit" +import * as Fiber from "effect/Fiber" +import * as Layer from "effect/Layer" +import * as Schema from "effect/Schema" +import * as Scope from "effect/Scope" +import { TestClock } from "effect/testing" +import * as SqlClient from "effect/unstable/sql/SqlClient" +import * as PeerRelayClientRuntime from "../src/PeerRelayClientRuntime.js" +import * as PeerRelayOutbox from "../src/PeerRelayOutbox.js" +import * as PeerRelayOutboxLimits from "../src/PeerRelayOutboxLimits.js" +import * as PeerRelayReceiptLimits from "../src/PeerRelayReceiptLimits.js" +import * as PeerSync from "../src/PeerSync.js" +import * as PeerSyncEnvelope from "../src/PeerSyncEnvelope.js" +import * as ReplicaBootstrap from "../src/ReplicaBootstrap.js" +import * as ReplicaGate from "../src/ReplicaGate.js" + +describe("PeerRelayClientRuntime", () => { + const Task = Document.make("Task", { + schema: Schema.Struct({ title: Schema.String }), + version: 1 + }) + const definition = ReplicaDefinition.make({ + name: "relay-runtime", + documents: DocumentSet.make(Task), + mutations: [], + projections: [], + queries: [] + }) + const replicaLimits: ReplicaLimits.Values = { + maxBackupBytes: 1_000_000, + maxChunkBytes: 64_000, + maxArchiveRecords: 1_000, + maxJsonDepth: 64, + maxSyncMessageBytes: 64_000, + maxPeerSendMillis: 1_000, + maxSyncChangesPerMessage: 100, + maxSyncDependencyEdgesPerMessage: 1_000, + maxSyncOperationsPerMessage: 10_000, + maxPendingBytesPerDocument: 1_000_000, + maxPendingBytesPerPeer: 1_000_000, + maxPendingBytesPerReplica: 1_000_000, + maxPendingAgeMillis: 60_000, + maxPendingChangesPerDocument: 1_000, + maxPendingChangesPerPeer: 1_000, + maxPendingChangesPerReplica: 1_000, + maxPendingDependencyEdgesPerDocument: 10_000, + maxPendingDependencyEdgesPerPeer: 10_000, + maxPendingDependencyEdgesPerReplica: 10_000, + maxSessions: 10, + maxStreamsPerSession: 10, + maxInFlightPerSession: 1, + maxQueuedRpc: 100, + maxQueuedPermits: 100, + maxActiveRestores: 10, + maxRestoresPerSession: 1, + maxRestoreMillis: 30_000, + maxRestorePullMillis: 10_000, + maxRestoreCoalesceMillis: 25, + maxRestoreErrorBytes: 4_096 + } + const outboxLimits: PeerRelayOutboxLimits.Values = { + ...PeerRelayOutboxLimits.defaults, + maxMessagesPerRemote: 10, + maxMessagesPerReplica: 10, + maxEncodedBytesPerRemote: 1_000_000, + maxEncodedBytesPerReplica: 1_000_000, + maxRetryHorizonMillis: 60_000, + maintenanceIntervalMillis: 1_000, + pruneBatchSize: 10 + } + const receiptLimits: PeerRelayReceiptLimits.Values = { + ...PeerRelayReceiptLimits.defaults, + maintenanceIntervalMillis: 1_000 + } + const documentId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000011") + const endpoint: PeerRelayOutbox.Endpoint = { + expectedLocal: { + tenantId: "tenant-a", + subjectId: "sender-a", + peerId: Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000012") + }, + remote: { + tenantId: "tenant-a", + subjectId: "recipient-a", + peerId: Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000013") + }, + relayPeerId: Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000014") + } + + const fakePeerSync = ( + pruneRelayReceipts: PeerSync.PeerSync["Service"]["pruneRelayReceipts"] + ): PeerSync.PeerSync["Service"] => ({ + open: () => Effect.die(new Error("unused")), + reset: () => Effect.die(new Error("unused")), + generate: () => Effect.die(new Error("unused")), + receive: () => Effect.die(new Error("unused")), + enqueue: () => Effect.die(new Error("unused")), + pending: () => Effect.die(new Error("unused")), + markSent: () => Effect.die(new Error("unused")), + ...(pruneRelayReceipts === undefined ? {} : { pruneRelayReceipts }) + }) + + const layers = ( + pruneRelayReceipts: PeerSync.PeerSync["Service"]["pruneRelayReceipts"] | null = Effect.succeed(0) + ) => { + const Database = Layer.merge( + SqliteClient.layer({ filename: ":memory:", disableWAL: true }), + NodeCrypto.layer + ) + const Bootstrap = ReplicaBootstrap.layer(definition).pipe(Layer.provide(Database)) + const Base = Layer.merge(Database, Bootstrap) + const ReplicaLimitLayer = ReplicaLimits.layer(replicaLimits) + const Gate = ReplicaGate.layer.pipe( + Layer.provide(ReplicaLimitLayer), + Layer.provide(Base) + ) + const Dependencies = Layer.mergeAll( + Base, + Gate, + ReplicaLimitLayer, + PeerRelayOutboxLimits.layer(outboxLimits), + PeerRelayReceiptLimits.layer(receiptLimits), + Layer.succeed( + PeerSync.PeerSync, + fakePeerSync(pruneRelayReceipts === null ? undefined : pruneRelayReceipts) + ) + ) + const Outbox = PeerRelayOutbox.layerSql.pipe(Layer.provide(Dependencies)) + const RuntimeDependencies = Layer.merge(Dependencies, Outbox) + const Runtime = PeerRelayClientRuntime.layer.pipe(Layer.provide(RuntimeDependencies)) + return Layer.merge(RuntimeDependencies, Runtime) + } + + const insertDocument = Effect.gen(function*() { + const sql = yield* SqlClient.SqlClient + yield* sql`INSERT INTO effect_local_documents ( + document_id, document_type, schema_version, observed_versions, + materialized_heads, accepted_heads, tombstone, projection_status, checkpoint_hash + ) VALUES (${documentId}, 'Task', 1, '[1]', '[]', '[]', 0, 'Ready', NULL)` + }) + + const makePayload = Effect.gen(function*() { + let source = Automerge.from( + { value: { title: "one" }, tombstone: false }, + { actor: "b".repeat(32) } + ) + const remote = Automerge.init() + const handshake = Automerge.generateSyncMessage(remote, Automerge.initSyncState())[1]! + const received = Automerge.receiveSyncMessage(source, Automerge.initSyncState(), handshake) + source = received[0] + const message = Automerge.generateSyncMessage(source, received[1])[1]! + const changes = Automerge.getAllChanges(source).map(Automerge.decodeChange) + const payload = yield* PeerSyncEnvelope.encodeSyncEnvelope(PeerSyncEnvelope.SyncEnvelope.make({ + connectionEpoch: "epoch-runtime", + sequence: 1, + documentId, + documentType: "Task", + messageHash: yield* Canonical.digest(message), + message, + writerProvenance: changes.map((change) => ({ + changeHash: change.hash, + writerSchemaVersion: 1, + writerDefinitionHash: definition.hash + })) + })) + Automerge.free(source) + Automerge.free(remote) + return payload + }) + + it.effect("propagates a real SQLite maintenance failure and fails later methods", () => + Effect.scoped( + Effect.gen(function*() { + const runtime = yield* PeerRelayClientRuntime.PeerRelayClientRuntime + const outbox = yield* PeerRelayOutbox.PeerRelayOutbox + const sql = yield* SqlClient.SqlClient + const gate = yield* ReplicaGate.ReplicaGate + yield* insertDocument + const payload = yield* makePayload + yield* outbox.admit({ ...endpoint, payload, retryHorizonMillis: 1_000 }) + yield* sql`CREATE TRIGGER fail_relay_outbox_delete + BEFORE DELETE ON effect_local_peer_relay_outbox + BEGIN + SELECT RAISE(FAIL, 'forced relay maintenance failure'); + END` + yield* runtime.validateConnectionConfiguration({ + replicaIncarnation: (yield* gate.current).incarnation, + retryHorizonMillis: 1_000, + replayBatchSize: 1 + }) + yield* TestClock.adjust("1 second") + const fatal = yield* Effect.exit(runtime.awaitFatal) + assert.strictEqual(fatal._tag, "Failure") + if (Exit.isFailure(fatal)) assert.isFalse(Cause.hasInterruptsOnly(fatal.cause)) + assert.strictEqual((yield* Effect.exit(runtime.health))._tag, "Failure") + assert.strictEqual( + (yield* Effect.exit(runtime.maximumPendingHorizon(endpoint)))._tag, + "Failure" + ) + }).pipe(Effect.provide(layers())) + )) + + it.effect("closes both scoped maintenance fibers before scope release completes", () => + Effect.gen(function*() { + const scope = yield* Scope.make() + const context = yield* Scope.provide(Layer.build(layers()), scope) + const runtime = Context.get(context, PeerRelayClientRuntime.PeerRelayClientRuntime) + yield* runtime.health + const fatalWaiter = yield* Effect.exit(runtime.awaitFatal).pipe(Effect.forkChild) + yield* Scope.close(scope, Exit.void) + const fatalExit = yield* Fiber.join(fatalWaiter) + assert.strictEqual(fatalExit._tag, "Failure") + if (Exit.isFailure(fatalExit)) assert.isTrue(Cause.hasInterruptsOnly(fatalExit.cause)) + const health = yield* Effect.exit(runtime.health) + assert.strictEqual(health._tag, "Failure") + if (Exit.isFailure(health)) assert.isTrue(Cause.hasInterruptsOnly(health.cause)) + })) + + it.effect("rejects a direct PeerSync before starting runtime maintenance", () => + Effect.gen(function*() { + const scope = yield* Scope.make() + const exit = yield* Effect.exit( + Scope.provide(Layer.build(layers(null)), scope) + ) + assert.strictEqual(exit._tag, "Failure") + yield* Scope.close(scope, Exit.void) + })) +}) diff --git a/packages/local-sql/test/PeerRelayOutbox.test.ts b/packages/local-sql/test/PeerRelayOutbox.test.ts new file mode 100644 index 0000000..a3c4422 --- /dev/null +++ b/packages/local-sql/test/PeerRelayOutbox.test.ts @@ -0,0 +1,334 @@ +import * as Automerge from "@automerge/automerge" +import { NodeCrypto } from "@effect/platform-node" +import { SqliteClient } from "@effect/sql-sqlite-node" +import { assert, describe, it } from "@effect/vitest" +import * as Canonical from "@lucas-barake/effect-local/Canonical" +import * as Document from "@lucas-barake/effect-local/Document" +import * as DocumentSet from "@lucas-barake/effect-local/DocumentSet" +import * as Identity from "@lucas-barake/effect-local/Identity" +import * as ReplicaDefinition from "@lucas-barake/effect-local/ReplicaDefinition" +import * as ReplicaLimits from "@lucas-barake/effect-local/ReplicaLimits" +import * as Effect from "effect/Effect" +import * as Layer from "effect/Layer" +import * as Schema from "effect/Schema" +import { TestClock } from "effect/testing" +import * as SqlClient from "effect/unstable/sql/SqlClient" +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import * as PeerRelayOutbox from "../src/PeerRelayOutbox.js" +import * as PeerRelayOutboxLimits from "../src/PeerRelayOutboxLimits.js" +import * as PeerSyncEnvelope from "../src/PeerSyncEnvelope.js" +import * as ReplicaBootstrap from "../src/ReplicaBootstrap.js" +import * as ReplicaGate from "../src/ReplicaGate.js" + +describe("PeerRelayOutbox", () => { + const Task = Document.make("Task", { + schema: Schema.Struct({ title: Schema.String }), + version: 1 + }) + const definition = ReplicaDefinition.make({ + name: "relay-outbox", + documents: DocumentSet.make(Task), + mutations: [], + projections: [], + queries: [] + }) + const replicaLimits: ReplicaLimits.Values = { + maxBackupBytes: 1_000_000, + maxChunkBytes: 64_000, + maxArchiveRecords: 1_000, + maxJsonDepth: 64, + maxSyncMessageBytes: 64_000, + maxPeerSendMillis: 1_000, + maxSyncChangesPerMessage: 100, + maxSyncDependencyEdgesPerMessage: 1_000, + maxSyncOperationsPerMessage: 10_000, + maxPendingBytesPerDocument: 1_000_000, + maxPendingBytesPerPeer: 1_000_000, + maxPendingBytesPerReplica: 1_000_000, + maxPendingAgeMillis: 60_000, + maxPendingChangesPerDocument: 1_000, + maxPendingChangesPerPeer: 1_000, + maxPendingChangesPerReplica: 1_000, + maxPendingDependencyEdgesPerDocument: 10_000, + maxPendingDependencyEdgesPerPeer: 10_000, + maxPendingDependencyEdgesPerReplica: 10_000, + maxSessions: 10, + maxStreamsPerSession: 10, + maxInFlightPerSession: 1, + maxQueuedRpc: 100, + maxQueuedPermits: 100, + maxActiveRestores: 10, + maxRestoresPerSession: 1, + maxRestoreMillis: 30_000, + maxRestorePullMillis: 10_000, + maxRestoreCoalesceMillis: 25, + maxRestoreErrorBytes: 4_096 + } + const outboxLimits: PeerRelayOutboxLimits.Values = { + ...PeerRelayOutboxLimits.defaults, + maxMessagesPerRemote: 2, + maxMessagesPerReplica: 3, + maxEncodedBytesPerRemote: 1_000_000, + maxEncodedBytesPerReplica: 2_000_000, + maxRetryHorizonMillis: 60_000, + pruneBatchSize: 10 + } + const documentId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000001") + const endpoint: PeerRelayOutbox.Endpoint = { + expectedLocal: { + tenantId: "tenant-a", + subjectId: "sender-a", + peerId: Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000002") + }, + remote: { + tenantId: "tenant-a", + subjectId: "recipient-a", + peerId: Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000003") + }, + relayPeerId: Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000004") + } + + const layer = ( + filename: string, + limits: PeerRelayOutboxLimits.Values = outboxLimits + ) => { + const Database = Layer.merge( + SqliteClient.layer({ filename, disableWAL: true }), + NodeCrypto.layer + ) + const Bootstrap = ReplicaBootstrap.layer(definition).pipe(Layer.provide(Database)) + const Base = Layer.merge(Database, Bootstrap) + const ReplicaLimitLayer = ReplicaLimits.layer(replicaLimits) + const Gate = ReplicaGate.layer.pipe( + Layer.provide(ReplicaLimitLayer), + Layer.provide(Base) + ) + const Infrastructure = Layer.mergeAll( + Base, + Gate, + ReplicaLimitLayer, + PeerRelayOutboxLimits.layer(limits) + ) + const Outbox = PeerRelayOutbox.layerSql.pipe(Layer.provide(Infrastructure)) + return Layer.merge(Infrastructure, Outbox) + } + + const insertDocument = Effect.gen(function*() { + const sql = yield* SqlClient.SqlClient + yield* sql`INSERT OR IGNORE INTO effect_local_documents ( + document_id, document_type, schema_version, observed_versions, + materialized_heads, accepted_heads, tombstone, projection_status, checkpoint_hash + ) VALUES (${documentId}, 'Task', 1, '[1]', '[]', '[]', 0, 'Ready', NULL)` + }) + + const makePayload = (sequence: number) => + Effect.gen(function*() { + let source = Automerge.from( + { value: { title: "one" }, tombstone: false }, + { actor: "a".repeat(32) } + ) + const remote = Automerge.init() + const handshake = Automerge.generateSyncMessage(remote, Automerge.initSyncState())[1]! + const received = Automerge.receiveSyncMessage(source, Automerge.initSyncState(), handshake) + source = received[0] + const message = Automerge.generateSyncMessage(source, received[1])[1]! + const changes = Automerge.getAllChanges(source).map(Automerge.decodeChange) + const envelope = PeerSyncEnvelope.SyncEnvelope.make({ + connectionEpoch: "epoch-1", + sequence, + documentId, + documentType: "Task", + messageHash: yield* Canonical.digest(message), + message, + writerProvenance: changes.map((change) => ({ + changeHash: change.hash, + writerSchemaVersion: 1, + writerDefinitionHash: definition.hash + })) + }) + Automerge.free(source) + Automerge.free(remote) + return yield* PeerSyncEnvelope.encodeSyncEnvelope(envelope) + }) + + it.effect("reuses one stable admission after time advances without incrementing quota", () => + Effect.gen(function*() { + yield* insertDocument + const outbox = yield* PeerRelayOutbox.PeerRelayOutbox + const payload = yield* makePayload(1) + const first = yield* outbox.admit({ ...endpoint, payload, retryHorizonMillis: 30_000 }) + yield* TestClock.adjust("10 seconds") + const duplicate = yield* outbox.admit({ + ...endpoint, + payload, + retryHorizonMillis: 30_000 + }) + assert.strictEqual(duplicate.relayMessageId, first.relayMessageId) + assert.strictEqual(duplicate.createdAt, first.createdAt) + assert.strictEqual(duplicate.retryDeadline, first.retryDeadline) + assert.deepStrictEqual(yield* outbox.usage(endpoint), { + remote: { messageCount: 1, encodedBytes: payload.byteLength }, + replica: { messageCount: 1, encodedBytes: payload.byteLength } + }) + }).pipe(Effect.provide(layer(":memory:")))) + + it.effect("rejects source collisions and concurrent quota growth without changing usage", () => + Effect.gen(function*() { + yield* insertDocument + const outbox = yield* PeerRelayOutbox.PeerRelayOutbox + const firstPayload = yield* makePayload(1) + yield* outbox.admit({ ...endpoint, payload: firstPayload, retryHorizonMillis: 30_000 }) + const collision = yield* Effect.exit(outbox.admit({ + ...endpoint, + expectedLocal: { ...endpoint.expectedLocal, subjectId: "other-sender" }, + payload: firstPayload, + retryHorizonMillis: 30_000 + })) + assert.strictEqual(collision._tag, "Failure") + const secondPayload = yield* makePayload(2) + yield* outbox.admit({ ...endpoint, payload: secondPayload, retryHorizonMillis: 30_000 }) + const thirdPayload = yield* makePayload(3) + const quota = yield* Effect.exit(outbox.admit({ + ...endpoint, + payload: thirdPayload, + retryHorizonMillis: 30_000 + })) + assert.strictEqual(quota._tag, "Failure") + const usage = yield* outbox.usage(endpoint) + assert.strictEqual(usage.remote.messageCount, 2) + assert.strictEqual(usage.replica.messageCount, 2) + }).pipe(Effect.provide(layer(":memory:")))) + + it.effect("prunes expired rows and deletes zero usage rows", () => + Effect.gen(function*() { + yield* insertDocument + const outbox = yield* PeerRelayOutbox.PeerRelayOutbox + const sql = yield* SqlClient.SqlClient + const payload = yield* makePayload(1) + yield* outbox.admit({ ...endpoint, payload, retryHorizonMillis: 1_000 }) + yield* TestClock.adjust("1 second") + assert.strictEqual(yield* outbox.pruneExpired, 1) + assert.deepStrictEqual(yield* outbox.usage(endpoint), { + remote: { messageCount: 0, encodedBytes: 0 }, + replica: { messageCount: 0, encodedBytes: 0 } + }) + const usageRows = yield* sql`SELECT * FROM effect_local_peer_relay_outbox_remote_usage` + assert.strictEqual(usageRows.length, 0) + }).pipe(Effect.provide(layer(":memory:")))) + + it.effect("fails closed when a durable retry is scheduled at its deadline", () => + Effect.gen(function*() { + yield* insertDocument + const outbox = yield* PeerRelayOutbox.PeerRelayOutbox + const sql = yield* SqlClient.SqlClient + const payload = yield* makePayload(1) + yield* outbox.admit({ ...endpoint, payload, retryHorizonMillis: 1_000 }) + yield* sql`UPDATE effect_local_peer_relay_outbox + SET next_attempt_at = retry_deadline` + yield* TestClock.adjust("1 second") + const exit = yield* Effect.exit(outbox.pruneExpired) + assert.strictEqual(exit._tag, "Failure") + assert.deepStrictEqual(yield* outbox.usage(endpoint), { + remote: { messageCount: 1, encodedBytes: payload.byteLength }, + replica: { messageCount: 1, encodedBytes: payload.byteLength } + }) + }).pipe(Effect.provide(layer(":memory:")))) + + it.effect("rejects document deletion while pending custody owns the payload and quota", () => + Effect.gen(function*() { + yield* insertDocument + const outbox = yield* PeerRelayOutbox.PeerRelayOutbox + const sql = yield* SqlClient.SqlClient + const payload = yield* makePayload(1) + const entry = yield* outbox.admit({ + ...endpoint, + payload, + retryHorizonMillis: 30_000 + }) + assert.strictEqual( + (yield* Effect.exit(sql`DELETE FROM effect_local_documents + WHERE document_id = ${documentId}`))._tag, + "Failure" + ) + const replay = yield* outbox.dueForEndpoint({ ...endpoint, maximum: 2 }) + assert.strictEqual(replay.length, 1) + assert.strictEqual(replay[0]!.relayMessageId, entry.relayMessageId) + assert.deepStrictEqual(yield* outbox.usage(endpoint), { + remote: { messageCount: 1, encodedBytes: payload.byteLength }, + replica: { messageCount: 1, encodedBytes: payload.byteLength } + }) + }).pipe(Effect.provide(layer(":memory:")))) + + it.effect("fences every replay operation after the replica incarnation changes", () => + Effect.gen(function*() { + yield* insertDocument + const outbox = yield* PeerRelayOutbox.PeerRelayOutbox + const gate = yield* ReplicaGate.ReplicaGate + const sql = yield* SqlClient.SqlClient + const payload = yield* makePayload(1) + const before = yield* gate.current + yield* outbox.admit({ ...endpoint, payload, retryHorizonMillis: 30_000 }) + yield* gate.claim(() => Effect.void) + assert.strictEqual( + (yield* Effect.exit(outbox.validateReplicaIncarnation(before.incarnation)))._tag, + "Failure" + ) + assert.deepStrictEqual(yield* outbox.dueForEndpoint({ ...endpoint, maximum: 2 }), []) + assert.strictEqual(yield* outbox.maximumPendingHorizon(endpoint), null) + const rows = yield* sql`SELECT replica_incarnation + FROM effect_local_peer_relay_outbox` + assert.strictEqual(rows.length, 1) + assert.strictEqual(rows[0]!.replica_incarnation, before.incarnation) + }).pipe(Effect.provide(layer(":memory:")))) + + it.effect("replays and retires a pending row after writer generation changes on restart", () => + Effect.acquireUseRelease( + Effect.sync(() => mkdtempSync(join(tmpdir(), "effect-local-relay-outbox-"))), + (directory) => + Effect.gen(function*() { + const filename = join(directory, "replica.sqlite") + const first = yield* Effect.scoped( + Effect.gen(function*() { + yield* insertDocument + const outbox = yield* PeerRelayOutbox.PeerRelayOutbox + const gate = yield* ReplicaGate.ReplicaGate + const payload = yield* makePayload(1) + const entry = yield* outbox.admit({ + ...endpoint, + payload, + retryHorizonMillis: 30_000 + }) + return { entry, permit: yield* gate.current } + }).pipe(Effect.provide(layer(filename))) + ) + + yield* Effect.scoped( + Effect.gen(function*() { + const outbox = yield* PeerRelayOutbox.PeerRelayOutbox + const gate = yield* ReplicaGate.ReplicaGate + const permit = yield* gate.current + assert.strictEqual(permit.replicaId, first.permit.replicaId) + assert.strictEqual(permit.incarnation, first.permit.incarnation) + assert.isAbove(permit.writerGeneration, first.permit.writerGeneration) + const replay = yield* outbox.dueForEndpoint({ ...endpoint, maximum: 2 }) + assert.strictEqual(replay.length, 1) + assert.strictEqual(replay[0]!.relayMessageId, first.entry.relayMessageId) + yield* outbox.markCustody({ + relayMessageId: replay[0]!.relayMessageId, + outerEnvelopeDigest: replay[0]!.outerEnvelopeDigest + }) + assert.deepStrictEqual( + yield* outbox.dueForEndpoint({ + ...endpoint, + maximum: 2 + }), + [] + ) + }).pipe(Effect.provide(layer(filename))) + ) + }), + (directory) => Effect.sync(() => rmSync(directory, { recursive: true, force: true })) + )) +}) diff --git a/packages/local-sql/test/PeerRelayOutboxLimits.test.ts b/packages/local-sql/test/PeerRelayOutboxLimits.test.ts new file mode 100644 index 0000000..4dbf870 --- /dev/null +++ b/packages/local-sql/test/PeerRelayOutboxLimits.test.ts @@ -0,0 +1,40 @@ +import { assert, describe, it } from "@effect/vitest" +import * as Effect from "effect/Effect" +import * as PeerRelayOutboxLimits from "../src/PeerRelayOutboxLimits.js" + +describe("PeerRelayOutboxLimits", () => { + it.effect("accepts the bounded defaults", () => + Effect.gen(function*() { + assert.deepStrictEqual( + yield* PeerRelayOutboxLimits.make(PeerRelayOutboxLimits.defaults), + PeerRelayOutboxLimits.defaults + ) + })) + + it.effect("rejects invalid scalar and aggregate relationships", () => + Effect.gen(function*() { + for (const values of [ + { ...PeerRelayOutboxLimits.defaults, maxMessagesPerRemote: 0 }, + { ...PeerRelayOutboxLimits.defaults, pruneRowsPerSecond: Number.NaN }, + { + ...PeerRelayOutboxLimits.defaults, + maxRetryHorizonMillis: PeerRelayOutboxLimits.maximumRetryHorizonMillis + 1 + }, + { + ...PeerRelayOutboxLimits.defaults, + maxMessagesPerRemote: PeerRelayOutboxLimits.defaults.maxMessagesPerReplica + 1 + }, + { + ...PeerRelayOutboxLimits.defaults, + maxEncodedBytesPerRemote: PeerRelayOutboxLimits.defaults.maxEncodedBytesPerReplica + 1 + } + ]) { + assert.strictEqual( + (yield* Effect.exit(PeerRelayOutboxLimits.make( + values as PeerRelayOutboxLimits.Values + )))._tag, + "Failure" + ) + } + })) +}) diff --git a/packages/local-sql/test/PeerRelayReceiptLimits.test.ts b/packages/local-sql/test/PeerRelayReceiptLimits.test.ts new file mode 100644 index 0000000..2353c11 --- /dev/null +++ b/packages/local-sql/test/PeerRelayReceiptLimits.test.ts @@ -0,0 +1,40 @@ +import { assert, describe, it } from "@effect/vitest" +import * as Effect from "effect/Effect" +import * as PeerRelayReceiptLimits from "../src/PeerRelayReceiptLimits.js" + +describe("PeerRelayReceiptLimits", () => { + it.effect("accepts the bounded defaults", () => + Effect.gen(function*() { + assert.deepStrictEqual( + yield* PeerRelayReceiptLimits.make(PeerRelayReceiptLimits.defaults), + PeerRelayReceiptLimits.defaults + ) + })) + + it.effect("rejects invalid scalar and aggregate relationships", () => + Effect.gen(function*() { + for (const values of [ + { ...PeerRelayReceiptLimits.defaults, maxReceiptsPerRemote: 0 }, + { ...PeerRelayReceiptLimits.defaults, maxEncodedBytesPerRemote: Number.POSITIVE_INFINITY }, + { + ...PeerRelayReceiptLimits.defaults, + receiptRetentionMillis: PeerRelayReceiptLimits.maximumReceiptRetentionMillis + 1 + }, + { + ...PeerRelayReceiptLimits.defaults, + maxReceiptsPerRemote: PeerRelayReceiptLimits.defaults.maxReceiptsPerReplica + 1 + }, + { + ...PeerRelayReceiptLimits.defaults, + maxEncodedBytesPerRemote: PeerRelayReceiptLimits.defaults.maxEncodedBytesPerReplica + 1 + } + ]) { + assert.strictEqual( + (yield* Effect.exit(PeerRelayReceiptLimits.make( + values as PeerRelayReceiptLimits.Values + )))._tag, + "Failure" + ) + } + })) +}) diff --git a/packages/local-sql/test/PeerSession.test.ts b/packages/local-sql/test/PeerSession.test.ts index 98ffde5..148b683 100644 --- a/packages/local-sql/test/PeerSession.test.ts +++ b/packages/local-sql/test/PeerSession.test.ts @@ -30,6 +30,7 @@ import * as ShardingConfig from "effect/unstable/cluster/ShardingConfig" import * as CommandExecutor from "../src/CommandExecutor.js" import * as CommitPublisher from "../src/CommitPublisher.js" import * as DocumentEntity from "../src/DocumentEntity.js" +import * as PeerRelayReceiptLimits from "../src/PeerRelayReceiptLimits.js" import * as PeerSession from "../src/PeerSession.js" import * as PeerSync from "../src/PeerSync.js" import * as ReplicaBootstrap from "../src/ReplicaBootstrap.js" @@ -239,7 +240,7 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { sequence: Number.MAX_SAFE_INTEGER, documentId: yield* Identity.makeDocumentId, documentType: "x".repeat(256), - messageHash: "x".repeat(256), + messageHash: "a".repeat(64), message: new Uint8Array(limits.maxSyncMessageBytes), lineage: maximalLineage, writerProvenance: Array.from( @@ -693,7 +694,7 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { const reply = { documentId, message: new Uint8Array([4, 5, 6]), - messageHash: "reply-hash", + messageHash: "b".repeat(64), heads: [] } const replyProvenance = [{ @@ -856,7 +857,7 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { sendSequence: 0, documentId, message: Uint8Array.of(2), - messageHash: "initial", + messageHash: "1".repeat(64), heads: [], lineage: Identity.genesisLineage, writerProvenance: [] @@ -864,14 +865,14 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { const reply = { documentId, message: Uint8Array.of(3), - messageHash: "reply", + messageHash: "2".repeat(64), heads: [] } const generated = { sendSequence: 2, documentId, message: Uint8Array.of(4), - messageHash: "generated", + messageHash: "3".repeat(64), heads: [], lineage: Identity.genesisLineage, writerProvenance: [] @@ -1038,7 +1039,7 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { sendSequence, documentId, message: Uint8Array.of(sendSequence), - messageHash: `hash-${sendSequence}`, + messageHash: sendSequence.toString(16).padStart(64, "0"), heads: [], lineage: Identity.genesisLineage, writerProvenance: [] @@ -1126,7 +1127,7 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { sendSequence: 9, documentId, message: Uint8Array.of(9), - messageHash: "hash-9", + messageHash: "9".repeat(64), heads: [], lineage: Identity.genesisLineage, writerProvenance: [] @@ -1201,7 +1202,7 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { const reply = { documentId, message: Uint8Array.of(2), - messageHash: "reply", + messageHash: "2".repeat(64), heads: [] } const sync = PeerSync.PeerSync.of({ @@ -1304,7 +1305,7 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { sequence: 0, documentId, documentType: Task.name, - messageHash: "incorrect-hash", + messageHash: "b".repeat(64), message, writerProvenance: [{ changeHash: "a".repeat(64), @@ -1819,7 +1820,7 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { reply: { documentId, message, - messageHash: "reply-hash", + messageHash: "b".repeat(64), heads: [] } }) @@ -1909,7 +1910,7 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { sendSequence: 0, documentId, message: Uint8Array.of(1), - messageHash: "message-hash", + messageHash: "a".repeat(64), heads: [], lineage: Identity.genesisLineage, writerProvenance: [] @@ -1995,7 +1996,7 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { sendSequence: call, documentId: id, message: new Uint8Array([call]), - messageHash: `hash-${call}`, + messageHash: call.toString(16).padStart(64, "0"), heads: [], lineage: Identity.genesisLineage, writerProvenance: [] @@ -2122,7 +2123,7 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { sendSequence: 1, documentId, message: Uint8Array.of(1), - messageHash: "message-hash", + messageHash: "a".repeat(64), heads: [], lineage: Identity.genesisLineage, writerProvenance: [] @@ -2434,6 +2435,167 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { ) }).pipe(Effect.provide(NodeCrypto.layer))) + it.effect("rotates relay sender epochs and acknowledges only after the durable receive workflow", () => + Effect.gen(function*() { + const deliveries = yield* Queue.unbounded() + const events = yield* Queue.unbounded() + const resets = yield* Ref.make>([]) + const received = yield* Ref.make>([]) + const documentId = yield* Identity.makeDocumentId + const senderPeerId = yield* Identity.makePeerId + const relayPeerId = yield* Identity.makePeerId + const message = yield* Effect.acquireUseRelease( + Effect.sync(() => Automerge.init()), + (document) => + Effect.sync(() => { + const encoded = Automerge.generateSyncMessage(document, Automerge.initSyncState())[1] + if (encoded === null) throw new TypeError("Expected an initial sync message") + return encoded + }), + (document) => Effect.sync(() => Automerge.free(document)) + ) + const messageHash = yield* Canonical.digest(message) + const reply = { documentId, message, messageHash, heads: [] } + const sync = PeerSync.PeerSync.of({ + open: (peerId) => + Effect.succeed({ + peerId, + connectionEpoch: "local-epoch", + replicaIncarnation: permit.incarnation + }), + reset: (session) => + Ref.update(resets, (current) => [...current, session.connectionEpoch]).pipe( + Effect.andThen(Queue.offer(events, `reset:${session.connectionEpoch}`)), + Effect.asVoid + ), + generate: () => Effect.succeed({ outbound: null, observedByPeer: false, dirty: false }), + receive: () => Effect.succeed(result), + enqueue: (_session, value) => + Queue.offer(events, "enqueue").pipe( + Effect.as({ ...value, sendSequence: 0, writerProvenance: [] }) + ), + pending: () => Effect.succeed([]), + markSent: () => Effect.succeed(true), + pruneRelayReceipts: Effect.succeed(0) + }) + const transport = PeerTransport.PeerTransport.of({ + capabilities: { storeAndForward: true }, + connect: () => + Effect.succeed({ + peerId: senderPeerId, + relayPeerId, + capabilities: { storeAndForward: true }, + receive: Stream.never, + receiveWithAcknowledgement: Stream.fromQueue(deliveries), + send: () => Effect.void, + close: Effect.void + }) + }) + const publisher = CommitPublisher.CommitPublisher.of({ + publishPending: Queue.offer(events, "publish").pipe(Effect.as(0)), + invalidate: () => Effect.void, + subscribe: Effect.succeed({ + watermark: Identity.CommitSequence.make(0), + refreshGeneration: 0, + events: Stream.never + }) + }) + const envelope = (connectionEpoch: string, sequence: number) => + encode({ + connectionEpoch, + sequence, + documentId, + documentType: Task.name, + messageHash, + message, + writerProvenance: [] + }) + const delivery = ( + connectionEpoch: string, + sequence: number, + relayMessageId: Identity.RelayMessageId + ): Effect.Effect => + envelope(connectionEpoch, sequence).pipe( + Effect.map((bytes) => ({ + message: bytes, + identity: { + relayMessageId, + relayPeerId, + senderTenantId: "tenant", + senderSubjectId: "sender", + senderPeerId, + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(9), + messageHash, + outerEnvelopeDigest: "a".repeat(64) + }, + receiptRetentionMillis: PeerRelayReceiptLimits.defaults.receiptRetentionMillis, + acknowledge: Queue.offer(events, "ack").pipe(Effect.asVoid), + reject: () => Queue.offer(events, "reject").pipe(Effect.asVoid) + })) + ) + yield* Effect.scoped( + Effect.gen(function*() { + yield* PeerSession.makeTestClient( + { peerId: senderPeerId, documents: [{ document: Task, documentId }] }, + () => + Effect.succeed({ + ApplySync: (request: typeof DocumentEntity.ApplySync.payloadSchema.Type) => + Queue.offer(events, "apply").pipe( + Effect.andThen( + request.relay === undefined + ? Effect.die("Expected relay receipt") + : Ref.update(received, (current) => [...current, request.relay!]) + ), + Effect.as({ ...result, reply }) + ) + } as never) + ) + yield* Queue.offer( + deliveries, + yield* delivery( + "sender-epoch-a", + 0, + Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000001") + ) + ) + assert.deepStrictEqual( + yield* Effect.forEach([0, 1, 2, 3], () => Queue.take(events)), + ["apply", "publish", "enqueue", "ack"] + ) + yield* Queue.offer( + deliveries, + yield* delivery( + "sender-epoch-b", + 0, + Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000002") + ) + ) + assert.deepStrictEqual( + yield* Effect.forEach([0, 1, 2, 3, 4], () => Queue.take(events)), + ["reset:sender-epoch-a", "apply", "publish", "enqueue", "ack"] + ) + assert.deepStrictEqual( + (yield* Ref.get(received)).map((entry) => entry.senderPeerId), + [senderPeerId, senderPeerId] + ) + }).pipe( + Effect.provideService(PeerTransport.PeerTransport, transport), + Effect.provideService(PeerSync.PeerSync, sync), + Effect.provideService(ReplicaGate.ReplicaGate, gate), + Effect.provideService(CommitPublisher.CommitPublisher, publisher), + Effect.provideService(ReplicaLimits.ReplicaLimits, limits), + Effect.provideService( + PeerRelayReceiptLimits.PeerRelayReceiptLimits, + PeerRelayReceiptLimits.defaults + ) + ) + ) + assert.deepStrictEqual( + (yield* Ref.get(resets)).toSorted(), + ["local-epoch", "sender-epoch-a", "sender-epoch-b"] + ) + }).pipe(Effect.provide(NodeCrypto.layer))) + it.effect("retries every unsent dirty output after a send fails", () => Effect.scoped(Effect.gen(function*() { const firstDocumentId = yield* Identity.makeDocumentId @@ -2462,7 +2624,7 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { sendSequence: current.length, documentId, message: new Uint8Array([current.length]), - messageHash: `hash-${current.length}`, + messageHash: current.length.toString(16).padStart(64, "0"), heads: [], lineage: Identity.genesisLineage, writerProvenance: [] @@ -3083,3 +3245,4 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { }).pipe(Effect.provide(TestGate)) }, 20_000) }) +import * as Automerge from "@automerge/automerge" diff --git a/packages/local-sql/test/PeerSync.test.ts b/packages/local-sql/test/PeerSync.test.ts index 7a7475a..b027fb7 100644 --- a/packages/local-sql/test/PeerSync.test.ts +++ b/packages/local-sql/test/PeerSync.test.ts @@ -10,6 +10,7 @@ import * as ReplicaDefinition from "@lucas-barake/effect-local/ReplicaDefinition import type * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" import * as ReplicaLimits from "@lucas-barake/effect-local/ReplicaLimits" import * as Cause from "effect/Cause" +import * as Clock from "effect/Clock" import * as Deferred from "effect/Deferred" import * as Effect from "effect/Effect" import type * as Exit from "effect/Exit" @@ -26,6 +27,7 @@ import { join } from "node:path" import * as Compaction from "../src/Compaction.js" import * as DocumentStore from "../src/DocumentStore.js" import * as InternalAutomerge from "../src/internal/automerge.js" +import * as PeerRelayReceiptLimits from "../src/PeerRelayReceiptLimits.js" import * as PeerSync from "../src/PeerSync.js" import * as Recovery from "../src/Recovery.js" import * as ReplicaBootstrap from "../src/ReplicaBootstrap.js" @@ -88,6 +90,15 @@ describe("PeerSync", () => { const Services = Layer.merge(Infrastructure, StoreService) const SyncService = PeerSync.layer.pipe(Layer.provide(Services)) const TestLayer = Layer.merge(Services, SyncService) + const RelayServices = Layer.merge( + Services, + PeerRelayReceiptLimits.layer({ + ...PeerRelayReceiptLimits.defaults, + pruneBatchSize: 1 + }) + ) + const RelaySyncService = PeerSync.layerRelay.pipe(Layer.provide(RelayServices)) + const RelayTestLayer = Layer.merge(RelayServices, RelaySyncService) const RecoveryService = Recovery.layer.pipe(Layer.provide(Services)) const CompactionService = Compaction.layer.pipe( Layer.provide(Layer.merge(Services, RecoveryService)) @@ -203,6 +214,148 @@ describe("PeerSync", () => { assert.strictEqual(session.replicaIncarnation, permit.incarnation) })).pipe(Effect.provide(TestLayer))) + it.effect("exposes relay receipt maintenance only from the relay layer", () => + Effect.gen(function*() { + assert.isUndefined((yield* PeerSync.PeerSync).pruneRelayReceipts) + }).pipe(Effect.provide(TestLayer)).pipe( + Effect.andThen( + Effect.gen(function*() { + assert.isDefined((yield* PeerSync.PeerSync).pruneRelayReceipts) + }).pipe(Effect.provide(RelayTestLayer)) + ) + )) + + it.effect("scopes relay deduplication by sender without colliding with direct receipts and prunes usage", () => + Effect.gen(function*() { + const store = yield* DocumentStore.DocumentStore + const sync = yield* PeerSync.PeerSync + const sql = yield* SqlClient.SqlClient + const documentId = yield* Identity.makeDocumentId + const senderOne = yield* Identity.makePeerId + const senderTwo = yield* Identity.makePeerId + const relayPeerId = yield* Identity.makePeerId + const relayMessageId = yield* Identity.makeRelayMessageId + const sessionOne = yield* sync.open(senderOne) + const sessionTwo = yield* sync.open(senderTwo) + const created = yield* store.create(Task, documentId, { title: "one", labels: [] }) + const remote = Automerge.change( + Automerge.clone(created.automerge, { actor: "1".repeat(32) }), + (draft) => { + ;(draft.value as { title: string }).title = "relayed" + } + ) + const generated = Automerge.generateSyncMessage(remote, Automerge.initSyncState()) + assert.isNotNull(generated[1]) + const message = generated[1]! + const messageHash = yield* Canonical.digest(message) + const receiptExpiresAt = new Date( + (yield* Clock.currentTimeMillis) + PeerRelayReceiptLimits.defaults.receiptRetentionMillis + ).toISOString() + const relay = ( + senderPeerId: Identity.PeerId, + senderSubjectId: string, + outerEnvelopeDigest: string + ): PeerSync.RelayReceipt => ({ + relayMessageId, + relayPeerId, + senderTenantId: "tenant", + senderSubjectId, + senderPeerId, + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(7), + messageHash, + outerEnvelopeDigest, + receiptExpiresAt, + encodedSize: message.byteLength + }) + const input = { + remoteConnectionEpoch: "shared-epoch", + receiveSequence: 0, + message, + writerProvenance: provenanceFor(message, Task.version, definition.hash) + } + + const firstRelay = yield* sync.receive(Task, documentId, sessionOne, { + ...input, + relay: relay(senderOne, "sender-one", "a".repeat(64)) + }) + assert.isFalse(firstRelay.duplicate) + + const direct = yield* sync.receive(Task, documentId, sessionOne, input) + assert.isFalse(direct.duplicate) + + const secondRelay = yield* sync.receive(Task, documentId, sessionTwo, { + ...input, + relay: relay(senderTwo, "sender-two", "b".repeat(64)) + }) + assert.isFalse(secondRelay.duplicate) + + const duplicate = yield* sync.receive(Task, documentId, sessionOne, { + ...input, + relay: relay(senderOne, "sender-one", "a".repeat(64)) + }) + assert.isTrue(duplicate.duplicate) + + const beforePrune = yield* sql<{ + readonly directReceipts: number + readonly relayBytes: number + readonly relayReceipts: number + readonly usageRows: number + }>`SELECT + (SELECT COUNT(*) FROM effect_local_peer_receipts + WHERE relay_message_id IS NULL) AS directReceipts, + (SELECT COUNT(*) FROM effect_local_peer_receipts + WHERE relay_message_id IS NOT NULL) AS relayReceipts, + (SELECT COUNT(*) FROM effect_local_peer_relay_receipt_usage) AS usageRows, + (SELECT COALESCE(SUM(encoded_bytes), 0) + FROM effect_local_peer_relay_receipt_usage) AS relayBytes` + assert.deepStrictEqual(beforePrune, [{ + directReceipts: 1, + relayBytes: message.byteLength * 2, + relayReceipts: 2, + usageRows: 2 + }]) + + yield* sql`UPDATE effect_local_peer_receipts + SET relay_receipt_expires_at = '1970-01-01T00:00:00.000Z' + WHERE relay_message_id IS NOT NULL` + assert.strictEqual(yield* sync.pruneRelayReceipts!, 1) + + const afterFirstBatch = yield* sql<{ + readonly directReceipts: number + readonly relayReceipts: number + readonly usageRows: number + }>`SELECT + (SELECT COUNT(*) FROM effect_local_peer_receipts + WHERE relay_message_id IS NULL) AS directReceipts, + (SELECT COUNT(*) FROM effect_local_peer_receipts + WHERE relay_message_id IS NOT NULL) AS relayReceipts, + (SELECT COUNT(*) FROM effect_local_peer_relay_receipt_usage) AS usageRows` + assert.deepStrictEqual(afterFirstBatch, [{ + directReceipts: 1, + relayReceipts: 1, + usageRows: 1 + }]) + assert.strictEqual(yield* sync.pruneRelayReceipts!, 1) + + const afterPrune = yield* sql<{ + readonly directReceipts: number + readonly relayReceipts: number + readonly usageRows: number + }>`SELECT + (SELECT COUNT(*) FROM effect_local_peer_receipts + WHERE relay_message_id IS NULL) AS directReceipts, + (SELECT COUNT(*) FROM effect_local_peer_receipts + WHERE relay_message_id IS NOT NULL) AS relayReceipts, + (SELECT COUNT(*) FROM effect_local_peer_relay_receipt_usage) AS usageRows` + assert.deepStrictEqual(afterPrune, [{ + directReceipts: 1, + relayReceipts: 0, + usageRows: 0 + }]) + InternalAutomerge.free(remote) + InternalAutomerge.free(created.automerge) + }).pipe(Effect.provide(RelayTestLayer))) + it.effect("persists inbound application and exact retransmission replies", () => Effect.gen(function*() { const store = yield* DocumentStore.DocumentStore diff --git a/packages/local-sql/test/PeerSyncEnvelope.test.ts b/packages/local-sql/test/PeerSyncEnvelope.test.ts new file mode 100644 index 0000000..36f1643 --- /dev/null +++ b/packages/local-sql/test/PeerSyncEnvelope.test.ts @@ -0,0 +1,268 @@ +import * as Automerge from "@automerge/automerge" +import * as Canonical from "@lucas-barake/effect-local/Canonical" +import * as Identity from "@lucas-barake/effect-local/Identity" +import * as NodeCrypto from "@effect/platform-node/NodeCrypto" +import { assert, describe, it } from "@effect/vitest" +import * as Effect from "effect/Effect" +import * as Schema from "effect/Schema" +import * as PeerSyncEnvelope from "../src/PeerSyncEnvelope.js" + +const limits: PeerSyncEnvelope.SyncEnvelopeLimits = { + maxSyncMessageBytes: 64 * 1_024, + maxSyncChangesPerMessage: 100, + maxSyncDependencyEdgesPerMessage: 1_000, + maxSyncOperationsPerMessage: 10_000 +} + +const makeSyncEnvelope = Effect.gen(function*() { + let source = Automerge.from( + { value: { title: "one" }, tombstone: false }, + { actor: "a".repeat(32) } + ) + const remote = Automerge.init() + const handshake = Automerge.generateSyncMessage(remote, Automerge.initSyncState())[1]! + const received = Automerge.receiveSyncMessage(source, Automerge.initSyncState(), handshake) + source = received[0] + const message = Automerge.generateSyncMessage(source, received[1])[1]! + const changes = Automerge.getAllChanges(source).map((bytes) => Automerge.decodeChange(bytes)) + const writerProvenance = changes.map((change) => ({ + changeHash: change.hash, + writerSchemaVersion: 1, + writerDefinitionHash: "definition-1" + })) + const envelope = PeerSyncEnvelope.SyncEnvelope.make({ + connectionEpoch: "epoch-1", + sequence: 1, + documentId: Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000001"), + documentType: "Task", + messageHash: yield* Canonical.digest(message), + message, + writerProvenance + }) + Automerge.free(source) + Automerge.free(remote) + return envelope +}) + +describe("PeerSyncEnvelope", () => { + it.effect("round trips and validates a bounded Automerge sync envelope", () => + Effect.gen(function*() { + const envelope = yield* makeSyncEnvelope + const bytes = yield* PeerSyncEnvelope.encodeSyncEnvelope(envelope) + const decoded = yield* PeerSyncEnvelope.decodeSyncEnvelope(bytes, limits) + assert.deepStrictEqual(decoded, envelope) + assert.deepStrictEqual(PeerSyncEnvelope.syncEnvelopeDocument(decoded), { + documentId: envelope.documentId, + documentType: "Task" + }) + }).pipe(Effect.provide(NodeCrypto.layer))) + + it.effect("rejects oversized, malformed, conflicting, and unprovenanced payloads", () => + Effect.gen(function*() { + const envelope = yield* makeSyncEnvelope + const bytes = yield* PeerSyncEnvelope.encodeSyncEnvelope(envelope) + assert.strictEqual( + (yield* Effect.exit(PeerSyncEnvelope.decodeSyncEnvelope(bytes, { + ...limits, + maxSyncMessageBytes: 1 + })))._tag, + "Failure" + ) + assert.strictEqual( + (yield* Effect.exit(PeerSyncEnvelope.decodeSyncEnvelope(Uint8Array.of(1, 2, 3), limits)))._tag, + "Failure" + ) + assert.strictEqual( + (yield* Effect.exit(PeerSyncEnvelope.validateSyncEnvelope({ + ...envelope, + messageHash: "f".repeat(64) + }, limits)))._tag, + "Failure" + ) + assert.strictEqual( + (yield* Effect.exit(PeerSyncEnvelope.validateSyncEnvelope({ + ...envelope, + writerProvenance: [] + }, limits)))._tag, + "Failure" + ) + }).pipe(Effect.provide(NodeCrypto.layer))) +}) + +describe("RelayOuterEnvelope", () => { + const base = PeerSyncEnvelope.RelayOuterEnvelope.make({ + domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, + version: PeerSyncEnvelope.relayOuterEnvelopeVersion, + expectedLocal: { + tenantId: "tenant-a", + subjectId: "subject-a", + peerId: Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000001") + }, + remote: { + tenantId: "tenant-a", + subjectId: "subject-b", + peerId: Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000002") + }, + relayPeerId: Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000003"), + relayMessageId: Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000004"), + protocolVersion: 3, + payloadVersion: 1, + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(2), + senderConnectionEpoch: "epoch-7", + senderSequence: 11, + document: { + documentId: Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000005"), + documentType: "Task" + }, + writerProvenance: [ + { + changeHash: "b".repeat(64), + writerSchemaVersion: 2, + writerDefinitionHash: "definition-b" + }, + { + changeHash: "a".repeat(64), + writerSchemaVersion: 1, + writerDefinitionHash: "definition-a" + } + ], + messageHash: "c".repeat(64), + payload: Uint8Array.of(1, 2, 3, 4) + }) + + it.effect("has one stable canonical byte and digest vector", () => + Effect.gen(function*() { + const relayAdmission = PeerSyncEnvelope.RelayOuterEnvelope.make({ + ...base, + expectedLocal: { ...base.expectedLocal }, + remote: { ...base.remote }, + document: { ...base.document }, + writerProvenance: [...base.writerProvenance], + payload: base.payload.slice() + }) + const recipientReplay = PeerSyncEnvelope.RelayOuterEnvelope.make({ + ...base, + expectedLocal: { ...base.expectedLocal }, + remote: { ...base.remote }, + document: { ...base.document }, + writerProvenance: [...base.writerProvenance].reverse() + }) + assert.deepStrictEqual( + yield* PeerSyncEnvelope.encodeRelayOuterEnvelope(base), + yield* PeerSyncEnvelope.encodeRelayOuterEnvelope(relayAdmission) + ) + assert.deepStrictEqual( + yield* PeerSyncEnvelope.encodeRelayOuterEnvelope(base), + yield* PeerSyncEnvelope.encodeRelayOuterEnvelope(recipientReplay) + ) + const digest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope(base) + assert.strictEqual(digest, "8e52edf067d2fdbc9be0c5f266f53cb9f12c4557c976e47bdc567c4d4f4efa1b") + assert.strictEqual( + yield* PeerSyncEnvelope.digestRelayOuterEnvelope(relayAdmission), + digest + ) + assert.strictEqual( + yield* PeerSyncEnvelope.digestRelayOuterEnvelope(recipientReplay), + digest + ) + assert.deepStrictEqual(PeerSyncEnvelope.relayOuterEnvelopeDocument(base), base.document) + assert.strictEqual( + (yield* Effect.exit(Schema.decodeUnknownEffect(PeerSyncEnvelope.RelayOuterEnvelope)({ + ...base, + domain: "unrelated-domain" + })))._tag, + "Failure" + ) + assert.strictEqual( + (yield* Effect.exit(Schema.decodeUnknownEffect(PeerSyncEnvelope.RelayOuterEnvelope)({ + ...base, + version: 2 + })))._tag, + "Failure" + ) + assert.strictEqual( + (yield* Effect.exit(Schema.decodeUnknownEffect(PeerSyncEnvelope.RelayOuterEnvelope)({ + ...base, + protocolVersion: 4 + })))._tag, + "Failure" + ) + assert.strictEqual( + (yield* Effect.exit(Schema.decodeUnknownEffect(PeerSyncEnvelope.RelayOuterEnvelope)({ + ...base, + payloadVersion: 2 + })))._tag, + "Failure" + ) + }).pipe(Effect.provide(NodeCrypto.layer))) + + it.effect("binds every mutable outer envelope field into the digest", () => + Effect.gen(function*() { + const expected = yield* PeerSyncEnvelope.digestRelayOuterEnvelope(base) + const mutations: ReadonlyArray = [ + { ...base, expectedLocal: { ...base.expectedLocal, tenantId: "tenant-b" } }, + { ...base, expectedLocal: { ...base.expectedLocal, subjectId: "subject-z" } }, + { + ...base, + expectedLocal: { + ...base.expectedLocal, + peerId: Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000006") + } + }, + { ...base, remote: { ...base.remote, tenantId: "tenant-b" } }, + { ...base, remote: { ...base.remote, subjectId: "subject-c" } }, + { + ...base, + remote: { + ...base.remote, + peerId: Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000007") + } + }, + { + ...base, + relayPeerId: Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000008") + }, + { + ...base, + relayMessageId: Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000009") + }, + { ...base, senderReplicaIncarnation: Identity.ReplicaIncarnation.make(3) }, + { ...base, senderConnectionEpoch: "epoch-8" }, + { ...base, senderSequence: 12 }, + { ...base, document: { ...base.document, documentType: "Note" } }, + { + ...base, + document: { + ...base.document, + documentId: Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000010") + } + }, + { + ...base, + writerProvenance: [{ + ...base.writerProvenance[0]!, + writerSchemaVersion: 3 + }, base.writerProvenance[1]!] + }, + { + ...base, + writerProvenance: [{ + ...base.writerProvenance[0]!, + changeHash: "e".repeat(64) + }, base.writerProvenance[1]!] + }, + { + ...base, + writerProvenance: [{ + ...base.writerProvenance[0]!, + writerDefinitionHash: "definition-c" + }, base.writerProvenance[1]!] + }, + { ...base, messageHash: "d".repeat(64) }, + { ...base, payload: Uint8Array.of(1, 2, 3, 5) } + ] + for (const mutation of mutations) { + assert.notStrictEqual(yield* PeerSyncEnvelope.digestRelayOuterEnvelope(mutation), expected) + } + }).pipe(Effect.provide(NodeCrypto.layer))) +}) diff --git a/packages/local-sql/test/PublicApi.types.test.ts b/packages/local-sql/test/PublicApi.types.test.ts new file mode 100644 index 0000000..f91a2b4 --- /dev/null +++ b/packages/local-sql/test/PublicApi.types.test.ts @@ -0,0 +1,20 @@ +import { assert, describe, it } from "@effect/vitest" +import * as PublicApi from "../src/index.js" + +describe("public API", () => { + it("exports the opt in relay persistence services", () => { + const publicApi: Record = PublicApi + + for ( + const name of [ + "PeerRelayClientRuntime", + "PeerRelayOutbox", + "PeerRelayOutboxLimits", + "PeerRelayReceiptLimits", + "PeerSyncEnvelope" + ] + ) { + assert.property(publicApi, name) + } + }) +}) diff --git a/packages/local/src/Identity.ts b/packages/local/src/Identity.ts index 9f3c171..fad11e6 100644 --- a/packages/local/src/Identity.ts +++ b/packages/local/src/Identity.ts @@ -39,6 +39,9 @@ export type PeerId = typeof PeerId.Type export const BackupInstallationId = identifier("BackupInstallationId", "bak") export type BackupInstallationId = typeof BackupInstallationId.Type +export const RelayMessageId = identifier("RelayMessageId", "rly") +export type RelayMessageId = typeof RelayMessageId.Type + export const ProjectionVersion = Schema.Int.check(Schema.isGreaterThan(0)).pipe( Schema.brand("@lucas-barake/effect-local/ProjectionVersion") ) @@ -71,6 +74,9 @@ export const makePeerId = Crypto.Crypto.use((crypto) => export const makeBackupInstallationId = Crypto.Crypto.use((crypto) => crypto.randomUUIDv4.pipe(Effect.map((uuid) => BackupInstallationId.make(`bak_${uuid}`))) ) +export const makeRelayMessageId = Crypto.Crypto.use((crypto) => + crypto.randomUUIDv4.pipe(Effect.map((uuid) => RelayMessageId.make(`rly_${uuid}`))) +) export const makeDocumentLineage = Crypto.Crypto.use((crypto) => crypto.randomUUIDv4.pipe(Effect.map((uuid) => DocumentLineage.make(`lin_${uuid}`))) diff --git a/packages/local/src/PeerTransport.ts b/packages/local/src/PeerTransport.ts index f5db5df..54ddafe 100644 --- a/packages/local/src/PeerTransport.ts +++ b/packages/local/src/PeerTransport.ts @@ -18,10 +18,35 @@ export interface Capabilities { readonly lineageAware?: boolean } +export type PermanentRejectReason = "ProtocolInvalid" | "ApplicationRejected" + +export interface RelayDeliveryIdentity { + readonly relayMessageId: Identity.RelayMessageId + readonly relayPeerId: Identity.PeerId + readonly senderTenantId: string + readonly senderSubjectId: string + readonly senderPeerId: Identity.PeerId + readonly senderReplicaIncarnation: Identity.ReplicaIncarnation + readonly messageHash: string + readonly outerEnvelopeDigest: string +} + +export interface AcknowledgedDelivery { + readonly message: Uint8Array + readonly identity: RelayDeliveryIdentity + readonly receiptRetentionMillis: number + readonly acknowledge: Effect.Effect + readonly reject: ( + reason: PermanentRejectReason + ) => Effect.Effect +} + export interface Connection { readonly peerId: Identity.PeerId + readonly relayPeerId?: Identity.PeerId readonly capabilities: Capabilities readonly receive: Stream.Stream + readonly receiveWithAcknowledgement?: Stream.Stream readonly send: (message: Uint8Array) => Effect.Effect /** * Terminates `receive`. A fiber parked consuming it observes an interrupt only `Exit`, never a normal end and diff --git a/packages/local/test/Identity.test.ts b/packages/local/test/Identity.test.ts index 1afe3a1..7b80fd6 100644 --- a/packages/local/test/Identity.test.ts +++ b/packages/local/test/Identity.test.ts @@ -12,12 +12,16 @@ describe("Identity", () => { const commandId = yield* Identity.makeCommandId const documentId = yield* Identity.makeDocumentId const peerId = yield* Identity.makePeerId + const relayMessageId = yield* Identity.makeRelayMessageId assert.strictEqual(Schema.decodeUnknownSync(Identity.ReplicaId)(replicaId), replicaId) assert.strictEqual(Schema.decodeUnknownSync(Identity.SessionId)(sessionId), sessionId) assert.strictEqual(Schema.decodeUnknownSync(Identity.CommandId)(commandId), commandId) assert.strictEqual(Schema.decodeUnknownSync(Identity.DocumentId)(documentId), documentId) assert.strictEqual(Schema.decodeUnknownSync(Identity.PeerId)(peerId), peerId) + assert.strictEqual(Schema.decodeUnknownSync(Identity.RelayMessageId)(relayMessageId), relayMessageId) + assert.isTrue(relayMessageId.startsWith("rly_")) assert.throws(() => Schema.decodeUnknownSync(Identity.DocumentId)(commandId as unknown as Identity.DocumentId)) + assert.throws(() => Schema.decodeUnknownSync(Identity.PeerId)(relayMessageId)) }).pipe(Effect.provide(NodeCrypto.layer))) it("rejects empty and malformed identifiers", () => { diff --git a/packages/local/test/PublicApi.types.test.ts b/packages/local/test/PublicApi.types.test.ts index 9f19907..d09c794 100644 --- a/packages/local/test/PublicApi.types.test.ts +++ b/packages/local/test/PublicApi.types.test.ts @@ -1,6 +1,9 @@ import { assert, describe, it } from "@effect/vitest" +import * as Effect from "effect/Effect" import * as Schema from "effect/Schema" -import { Document, DocumentSet, Mutation, Query, SchemaDescriptor } from "../src/index.js" +import * as Stream from "effect/Stream" +import { Document, DocumentSet, Identity, Mutation, PeerTransport, Query, SchemaDescriptor } from "../src/index.js" +import type * as ReplicaError from "../src/ReplicaError.js" type Equal = (() => T extends A ? 1 : 2) extends () => T extends B ? 1 : 2 ? true : false @@ -87,4 +90,66 @@ describe("public API types", () => { const descriptor: SchemaDescriptor.Descriptor = SchemaDescriptor.make(Schema.String) assert.isDefined(descriptor) }) + + it("adds acknowledged relay delivery without changing the direct transport shape", () => { + const peerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000002") + const relayMessageId = Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000003") + const relayPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000004") + const directConnection = { + peerId, + capabilities: { storeAndForward: false }, + receive: Stream.empty, + send: () => Effect.void, + close: Effect.void + } satisfies PeerTransport.Connection + const directConnectionApi: PeerTransport.Connection = directConnection + const directTransport = PeerTransport.PeerTransport.of({ + capabilities: { storeAndForward: false }, + connect: () => Effect.succeed(directConnection) + }) + const identity: PeerTransport.RelayDeliveryIdentity = { + relayMessageId, + relayPeerId: peerId, + senderTenantId: "tenant", + senderSubjectId: "subject", + senderPeerId: peerId, + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + messageHash: "a".repeat(64), + outerEnvelopeDigest: "b".repeat(64) + } + const delivery: PeerTransport.AcknowledgedDelivery = { + message: new Uint8Array([1]), + identity, + receiptRetentionMillis: 1, + acknowledge: Effect.void, + reject: (_reason) => Effect.void + } + const relayConnection = { + ...directConnection, + relayPeerId, + capabilities: { storeAndForward: true }, + receiveWithAcknowledgement: Stream.make(delivery) + } satisfies PeerTransport.Connection + const optionalAcknowledgedReceive: Equal< + PeerTransport.Connection["receiveWithAcknowledgement"], + Stream.Stream | undefined + > = true + const optionalRelayPeerId: Equal< + PeerTransport.Connection["relayPeerId"], + Identity.PeerId | undefined + > = true + const protocolRejection = delivery.reject("ProtocolInvalid") + const applicationRejection = delivery.reject("ApplicationRejected") + assert.isDefined(directTransport) + assert.isUndefined(directConnectionApi.relayPeerId) + assert.isUndefined(directConnectionApi.receiveWithAcknowledgement) + assert.strictEqual(relayConnection.relayPeerId, relayPeerId) + assert.isDefined(relayConnection.receiveWithAcknowledgement) + assert.isTrue(optionalAcknowledgedReceive) + assert.isTrue(optionalRelayPeerId) + assert.isFalse(directConnection.capabilities.storeAndForward) + assert.isTrue(relayConnection.capabilities.storeAndForward) + assert.isDefined(protocolRejection) + assert.isDefined(applicationRejection) + }) }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f84130a..6fb44be 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -139,6 +139,9 @@ importers: '@effect/platform-node': specifier: 4.0.0-beta.99 version: 4.0.0-beta.99(effect@4.0.0-beta.99)(ioredis@5.11.1) + '@effect/sql-sqlite-node': + specifier: 4.0.0-beta.99 + version: 4.0.0-beta.99(effect@4.0.0-beta.99) '@lucas-barake/effect-local-test': specifier: workspace:^ version: link:../local-test From 65efd02bc80132392b3158be2c594d10f4bc8feb Mon Sep 17 00:00:00 2001 From: Lucas Patron <86905052+lucas-barake@users.noreply.github.com> Date: Sun, 26 Jul 2026 00:41:01 -0500 Subject: [PATCH 02/29] Extend durable peer store and forward Working-tree snapshot pushed on behalf of the issue-23 session so the work is not stranded locally. Adds relay store error typing and observability coverage (`peerRelayStoreErrors.ts`, `PeerRelayStoreErrors.test.ts`, `PeerRpcObservability.test.ts`), and broadens the relay, outbox, sync-envelope and documentation surface across `local-rpc` and `local-sql`. Known incomplete: `packages/local-rpc/test/PeerRelayStore.test.ts` has two `Metric.MetricRegistry` type errors (value used as a type) that still need fixing. --- README.md | 347 ++- docs/architecture.md | 43 + docs/durability.md | 38 + docs/store-and-forward.md | 301 +++ docs/sync.md | 41 +- packages/local-rpc/README.md | 15 +- .../local-rpc/src/PeerRelayAuthorization.ts | 120 +- packages/local-rpc/src/PeerRelayIngress.ts | 341 ++- packages/local-rpc/src/PeerRelayLimits.ts | 2 + packages/local-rpc/src/PeerRelayStore.ts | 626 +++-- packages/local-rpc/src/PeerRpcServer.ts | 1083 +++++--- packages/local-rpc/src/RpcPeerTransport.ts | 84 +- .../src/internal/peerRelayMigrations.ts | 82 +- .../internal/peerRelaySqliteTransaction.ts | 45 +- .../src/internal/peerRelayStoreErrors.ts | 45 + .../src/internal/peerRpcObservability.ts | 568 +++- .../test/PeerRelayAuthorization.test.ts | 281 +- .../local-rpc/test/PeerRelayIngress.test.ts | 153 +- .../local-rpc/test/PeerRelayLimits.test.ts | 4 +- .../test/PeerRelaySqliteTransaction.test.ts | 113 + .../local-rpc/test/PeerRelayStore.test.ts | 985 ++++++- .../test/PeerRelayStoreErrors.test.ts | 41 + .../test/PeerRpcObservability.test.ts | 399 +++ packages/local-rpc/test/PeerRpcServer.test.ts | 2329 ++++++++++++++++- .../local-rpc/test/PublicApi.types.test.ts | 8 +- .../local-rpc/test/RpcPeerTransport.test.ts | 191 +- packages/local-sql/README.md | 11 +- packages/local-sql/src/BackupStore.ts | 4 + packages/local-sql/src/Compaction.ts | 205 +- packages/local-sql/src/Migrations.ts | 23 +- packages/local-sql/src/PeerRelayOutbox.ts | 2 + packages/local-sql/src/PeerSession.ts | 142 +- packages/local-sql/src/PeerSync.ts | 23 +- packages/local-sql/src/PeerSyncEnvelope.ts | 64 +- packages/local-sql/test/BackupStore.test.ts | 10 +- packages/local-sql/test/Compaction.test.ts | 192 ++ .../local-sql/test/DocumentEntity.test.ts | 5 +- packages/local-sql/test/Migrations.test.ts | 71 + .../test/PeerRelayClientRuntime.test.ts | 4 + .../local-sql/test/PeerRelayOutbox.test.ts | 25 + packages/local-sql/test/PeerSession.test.ts | 275 +- packages/local-sql/test/PeerSync.test.ts | 157 +- .../local-sql/test/PeerSyncEnvelope.test.ts | 94 +- 43 files changed, 8403 insertions(+), 1189 deletions(-) create mode 100644 docs/store-and-forward.md create mode 100644 packages/local-rpc/src/internal/peerRelayStoreErrors.ts create mode 100644 packages/local-rpc/test/PeerRelayStoreErrors.test.ts create mode 100644 packages/local-rpc/test/PeerRpcObservability.test.ts diff --git a/README.md b/README.md index 6de1841..37ae692 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,8 @@ WebSocket, serialization, routing, and server ownership to Effect and the applic > **Beta:** The library targets Effect `4.0.0-beta.99` and Automerge `3.3.2`. Durable formats, > worker protocols, and public APIs can still change. Read [Limits and security](#limits-and-security) before adopting -> it for user data. The external RPC transport is alpha and live only. It does not claim durable remote custody. +> it for user data. Direct external RPC remains live only. Optional store and forward provides single authority +> SQLite custody with an at least once delivery contract. It does not claim a globally replicated relay service. ## Retrieval contract @@ -54,14 +55,14 @@ sessions, peer sessions, and presence connect processes without becoming another This separates concerns that are often collapsed into one client state library: -| Concern | Responsibility | What it does not own | -| ------------------------ | ------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | -| Canonical recovery state | Automerge changes, heads, checkpoints, tombstones, and command receipts | Physical storage bytes or UI cache state | -| Physical persistence | SQLite transactions and database bytes, stored in OPFS in the browser | Merge semantics or permanent browser retention | -| Local execution | Serialized commands, durable replies, maintenance workflows, and recovery | Replicated convergence | -| Derived durable views | SQL projection tables optimized for application queries | Canonical history | -| Ephemeral views | Atom caches, RPC leases, and presence | Durable facts | -| Connectivity | Worker RPC, peer sessions, and application supplied transports | Identity issuance, application routing policy, platform socket or server ownership, or durable relay custody | +| Concern | Responsibility | What it does not own | +| ------------------------ | -------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------- | +| Canonical recovery state | Automerge changes, heads, checkpoints, tombstones, and command receipts | Physical storage bytes or UI cache state | +| Physical persistence | SQLite transactions and database bytes, stored in OPFS in the browser | Merge semantics or permanent browser retention | +| Local execution | Serialized commands, durable replies, maintenance workflows, and recovery | Replicated convergence | +| Derived durable views | SQL projection tables optimized for application queries | Canonical history | +| Ephemeral views | Atom caches, RPC leases, and presence | Durable facts | +| Connectivity | Worker RPC, peer sessions, application supplied transports, and optional relay custody | Identity issuance, application routing policy, platform socket or server ownership | The distinction follows Automerge's separation between a [CRDT document and its storage adapter](https://automerge.org/docs/reference/repositories/storage/) and SQLite's @@ -93,7 +94,9 @@ SQLite does not decide how concurrent document changes merge. Automerge `save` e | **Invalidation key** | Notification metadata for refreshing a document or projection read. It is not canonical data or a durability acknowledgement. | | **Presence** | Expiring best effort metadata about connected peers or tabs. Presence is never durable state and must never authorize an operation. | | **Hosted canonical replica** | One ordinary `SqlReplica` running in a server process. It participates as a CRDT peer. It is not a transaction authority for other replicas. | -| **Live relay** | The bounded in memory `Open` stream and `Push` path owned by `PeerRpcServer`. It has no durable custody or replay log. | +| **Direct RPC** | The version `2` `Open` stream and `Push` path owned by `PeerRpcServer.layerHandlers`. It has no durable custody or replay log. | +| **Store and forward relay** | The optional version `3` protocol on a separate bounded listener. A stable sender outbox transfers custody to one SQLite relay authority for at least once reconnect delivery. | +| **Relay acknowledgement** | A fenced recipient response sent only after the production SQL sync workflow and sender scoped replay receipt commit. It can retire only the exact live claim. | | **Principal** | A request scoped `tenantId`, `subjectId`, and stable `peerId` produced by `PeerAuthenticator`. It is derived from a credential on every RPC operation. | | **Authorization lease** | A bounded grant for exactly the requested whole document set. Expiry or invalidation terminates the session. It is not a distributed lock. | | **RPC session** | One scoped live mapping between an authenticated peer and a SQL `PeerSession`. A replacement connection closes the previous incarnation. | @@ -172,6 +175,8 @@ Transport neutral synchronization has three layers. The optional RPC package add | `PeerSession` | Scoped orchestration that binds one transport connection to one replica incarnation and a selected set of whole document instances. | | `PeerRpc` | Versioned `Open` and `Push` contract plus the generated Effect RPC client. It does not own an Effect `RpcClient.Protocol`, `RpcServer.Protocol`, serializer, platform socket, or server. | | `RpcPeerTransport` | Adapter from the generated RPC client to `PeerTransport`, then to the existing SQL `PeerSession`. | +| `PeerRelayRpc` | Optional version `3` `OpenRelay`, `PushRelay`, `AcknowledgeRelay`, and `RejectRelay` contract. It runs on a separate bounded protocol and listener. | +| Relay SQL state | Stable sender outbox, single authority relay custody, and sender scoped recipient receipts for at least once reconnect delivery. | `observedByPeer` means Automerge's per peer sync state indicates that the peer has all local changes. It does not prove remote storage durability. Effect Local therefore keeps durable confirmation separate and currently reports it as @@ -220,20 +225,21 @@ always recover by reading the replica again because Atom is not part of the pers ### Choosing the right abstraction -| Need | Use | -| ----------------------------------------- | ---------------------------------------------------- | -| Change one aggregate atomically | One document mutation | -| Read one aggregate with causal metadata | `Replica.get` | -| Filter, sort, or join local data | Projection plus query | -| Keep React views current | `ReplicaAtom` builders | -| Retry a command after losing the response | The same command ID, then the matching lookup method | -| Coordinate several durable steps | Effect Workflow | -| Exchange changes with another replica | `PeerSession` and an application supplied transport | -| Host a live authenticated replica peer | `PeerRpcServer` plus application owned Effect RPC | -| Connect through the generated RPC client | `RpcPeerTransport.makeSession` | -| Bound a high churn document's checkpoint | `ReplicaWorkflow.HistoryRewriteWorkflow` | -| Show cursors or online state | Presence | -| Move or recover all local data | Backup and restore | +| Need | Use | +| ----------------------------------------- | --------------------------------------------------------------------- | +| Change one aggregate atomically | One document mutation | +| Read one aggregate with causal metadata | `Replica.get` | +| Filter, sort, or join local data | Projection plus query | +| Keep React views current | `ReplicaAtom` builders | +| Retry a command after losing the response | The same command ID, then the matching lookup method | +| Coordinate several durable steps | Effect Workflow | +| Exchange changes with another replica | `PeerSession` and an application supplied transport | +| Host a live authenticated replica peer | `PeerRpcServer` plus application owned Effect RPC | +| Connect through the generated RPC client | `RpcPeerTransport.makeSession` | +| Bound a high churn document's checkpoint | `ReplicaWorkflow.HistoryRewriteWorkflow` | +| Add asynchronous relay custody | The opt in [store and forward](docs/store-and-forward.md) composition | +| Show cursors or online state | Presence | +| Move or recover all local data | Backup and restore | The main design rule is simple: durable facts flow outward from canonical recovery state into projections and reactive views. They never flow back from a cache, presence record, or transport connection into canonical history without a @@ -298,13 +304,13 @@ pnpm add -D @effect/platform-node@4.0.0-beta.99 @effect/platform-node-shared@4.0 Package roles: -| Package | Purpose | -| ------------------------------------ | ----------------------------------------------------------------------------------------------------------------- | -| `@lucas-barake/effect-local` | Documents, mutations, projections, queries, backups, sync transport, and `Replica` | -| `@lucas-barake/effect-local-sql` | SQLite persistence, durable Cluster execution, Workflow, recovery, compaction, and peer sync | -| `@lucas-barake/effect-local-browser` | Effect Worker and RPC composition, OPFS ports, sessions, presence, and Atom builders | -| `@lucas-barake/effect-local-test` | In memory production shaped replicas and deterministic bounded peer faults | -| `@lucas-barake/effect-local-rpc` | Effect RPC contract, authentication middleware, authorization policy, bounded server sessions, and client adapter | +| Package | Purpose | +| ------------------------------------ | --------------------------------------------------------------------------------------------------------------- | +| `@lucas-barake/effect-local` | Documents, mutations, projections, queries, backups, sync transport, and `Replica` | +| `@lucas-barake/effect-local-sql` | SQLite persistence, durable execution, recovery, compaction, peer sync, sender relay outbox, and relay receipts | +| `@lucas-barake/effect-local-browser` | Effect Worker and RPC composition, OPFS ports, sessions, presence, and Atom builders | +| `@lucas-barake/effect-local-test` | In memory production shaped replicas and deterministic bounded peer faults | +| `@lucas-barake/effect-local-rpc` | Direct RPC plus optional bounded store and forward protocol, custody, policies, server, and client adapter | Package dependency direction is: @@ -336,12 +342,16 @@ The browser suites currently exercise Chromium. Other engines are not claimed as worker, locking, reload, and browser test suite passes there. Browser storage starts as best effort. Request `navigator.storage.persist()`, report the result, and provide backup and restore controls. -The RPC package itself is platform neutral. A network deployment requires an Effect `RpcServer.Protocol` and +The RPC package itself is platform neutral. A direct network deployment requires an Effect `RpcServer.Protocol` and `RpcClient.Protocol`, the same `RpcSerialization` on both ends, a platform `Socket`, an HTTP server, TLS, WebSocket upgrade routing, Origin policy, ingress byte and connection limits, credential issuance, secret rotation, tenant routing, process supervision, and graceful shutdown. None of those responsibilities are inferred from `PeerRpcServer.layerHandlers`. +The optional relay uses its own `PeerRelayIngress` protocol and distinct socket listener. It must not reuse the direct +WebSocket MessagePack protocol or listener. The application still owns TLS, routing, identity, policy, process +supervision, and the single SQLite process and durable volume that hold one relay shard. + ## Composition recipes The following sections build one task replica from schema to React. Imports use public subpath exports so every @@ -1187,6 +1197,30 @@ synchronization, a Push reply, or commit driven flush is also terminal. The trig stream fail with `SessionOverloaded`. Later pushes for that session receive `SessionUnavailable`, so recovery creates a fresh connection scope and session. +### Optional store and forward + +Direct protocol version `2` stays source compatible and live only. Store and forward is a separate opt in composition +using relay protocol version `3`, a distinct bounded socket listener, stable sender outbox rows, one SQLite custody +authority per shard, and sender scoped recipient receipts. + +`PeerRpcServer.layerStoreAndForwardDeployment` merges an existing direct deployment with the separate relay listener. +The client uses `SqlReplica.layerRelay`, `PeerRelayOutbox.layerSql`, `PeerRelayClientRuntime.layer`, and +`RpcPeerTransport.makeStoreAndForwardSession`. Relay acceptance proves durable custody. Recipient acknowledgement is +attempted only after the production SQL sync workflow and replay receipt commit. Delivery remains at least once. + +Automerge `3.3.2` does not expose an allocation bounded semantic decode API. Relay use therefore requires ordinary +endpoint and document authorization plus a separate explicit unsafe resource trust grant for the exact principal, +remote, direction, and documents. The recommended default is +`PeerRelayAuthorization.denyUnsafeUnboundedAutomerge3Decode`. Authentication and ordinary document access do not +imply that grant. + +Recipient attempts are durable and bounded by `PeerRelayLimits.maximumDeliveryAttempts`, which defaults to `16`. +Reaching the cap dead letters the message and erases its payload. + +See [Store and forward](docs/store-and-forward.md) for the exact Layer composition, protocol, acknowledgement +boundary, ordering scope, expiry, retention, quotas, security, single authority deployment boundary, failure limits, +and exposed usage values. + ### 14. Run compaction and recovery workflows `SqlReplica.layer` provides the registered compaction workflow runtime. Handles are scoped to the replica incarnation. @@ -1392,30 +1426,37 @@ validate their bounds in the Effect error channel with the tagged `InvalidOption | Atom values | Atom registry | No | No | Yes, from `Replica` reads and queries | | Presence and tab sessions | Browser process | No | Best effort transport only | Not applicable | | RPC principals, leases, sessions, queues | RPC process scope | No | No | Recreated by authentication and synchronization | +| Sender relay outbox | Sender SQLite replica | Yes | No | Removed after relay custody or retry horizon | +| Relay custody and terminal evidence | Relay SQLite authority | Yes | No | Active payload is not reconstructable after expiry | +| Recipient relay receipts | Recipient SQLite replica | Yes | No | Bounded replay evidence, not canonical CRDT history | ### Boundary guarantees Consistency guarantees: -| Boundary | Guarantee | -| ------------------------ | -------------------------------------------------------------------------------------------------------------------------- | -| One document command | Serialized through its Cluster entity and committed with canonical state, projections, receipt, sequence, and stored reply | -| Command retry | Within one replica incarnation, the same command ID and canonical request returns the durable result | -| Different command input | Within one replica incarnation, reusing a command ID for different input fails | -| Query | Reads local projection state under the replica operation gate | -| Multi document invariant | Not transactional. Model one aggregate document or an explicit Workflow | -| Peer convergence | Replicas converge after receiving the same valid Automerge change set | -| Cross lineage sync | Refused, never merged. A rewritten document stops synchronizing with every peer that holds the superseded lineage | -| Restore | Exclusive, fenced, staged, schema checked, and projection rebuilding | -| Atom invalidation | Reactive cache refresh, not a durability acknowledgement | -| Presence | Expiring best effort state with no durability guarantee | -| Direct peer send | Local SQL outbox exists before send. A successful send permits local `markSent`. It does not prove remote application | -| Automerge observation | The peer's sync state reports all current local changes observed. It does not prove remote storage durability | -| RPC `Open` handshake | One authenticated, authorized, bounded live session exists for exactly the selected whole documents | -| RPC `Push` success | Bytes were accepted into the current bounded in memory session. No custody, replay, or remote durability is implied | -| Effect RPC stream ack | The response chunk was acknowledged by the RPC protocol. It is neither a byte credit nor an application receipt | -| WebSocket frame order | Frames are ordered within one live RFC 6455 connection. Reconnect, replay, authorization, and persistence are separate | -| `durableConfirmation` | Always `false` in the shipped peer transports | +| Boundary | Guarantee | +| ------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| One document command | Serialized through its Cluster entity and committed with canonical state, projections, receipt, sequence, and stored reply | +| Command retry | Within one replica incarnation, the same command ID and canonical request returns the durable result | +| Different command input | Within one replica incarnation, reusing a command ID for different input fails | +| Query | Reads local projection state under the replica operation gate | +| Multi document invariant | Not transactional. Model one aggregate document or an explicit Workflow | +| Peer convergence | Replicas converge after receiving the same valid Automerge change set | +| Cross lineage sync | Refused, never merged. A rewritten document stops synchronizing with every peer that holds the superseded lineage | +| Restore | Exclusive, fenced, staged, schema checked, and projection rebuilding | +| Atom invalidation | Reactive cache refresh, not a durability acknowledgement | +| Presence | Expiring best effort state with no durability guarantee | +| Direct peer send | Local SQL outbox exists before send. A successful send permits local `markSent`. It does not prove remote application | +| Automerge observation | The peer's sync state reports all current local changes observed. It does not prove remote storage durability | +| RPC `Open` handshake | One authenticated, authorized, bounded live session exists for exactly the selected whole documents | +| RPC `Push` success | Bytes were accepted into the current bounded in memory session. No custody, replay, or remote durability is implied | +| Relay `PushRelay` success | The single SQLite relay authority committed the complete envelope and quota reservation before replying | +| Relay recipient ack | The recipient SQL sync workflow and sender scoped receipt committed before the fenced acknowledgement was attempted | +| Relay delivery | At least once within configured retry, expiry, retention, authorization, and capacity boundaries | +| Relay poison bound | Delivery attempts are durable. Reaching `maximumDeliveryAttempts`, default `16`, dead letters the row and erases its payload | +| Effect RPC stream ack | The response chunk was acknowledged by the RPC protocol. It is neither a byte credit nor an application receipt | +| WebSocket frame order | Frames are ordered within one live RFC 6455 connection. Reconnect, replay, authorization, and persistence are separate | +| `durableConfirmation` | Always `false` in the shipped peer transports | ### Retry and ambiguity @@ -1433,6 +1474,12 @@ Consistency guarantees: safe. This is endpoint deduplication, not exactly once transport delivery. - Retry `RpcPeerTransport` only for `StorageUnavailable`. Rebuild the complete connection scope. Policy and protocol mismatches are not transient. +- Retry relay admission with the same stable `RelayMessageId` and exact outer envelope. A lost custody response, + recipient acknowledgement, reconnect, or expired claim can duplicate delivery. Sender scoped recipient receipts + suppress repeated application during the negotiated retention window. +- Failed, interrupted, disconnected, and expired relay claims advance the durable delivery attempt count. Reaching + `PeerRelayLimits.maximumDeliveryAttempts` dead letters the message and erases its payload. Restart does not reset + the count. - Durable commit events are retried until publication into the bounded process local stream. Subscriber delivery is best effort because the stream uses sliding capacity. A later sequence gap, `FullRefreshRequired`, or refresh generation change instructs the subscriber to reread canonical state. If no later event arrives, no notification @@ -1462,6 +1509,18 @@ Consistency guarantees: 10. All long lived connections, subscriptions, streams, queues, fibers, and server sessions must be owned by an Effect `Scope`. Native documents may instead use a deterministic acquisition bracket. Scope close is cleanup. It is not persistence. +11. Relay ordering is FIFO only within one exact tenant, sender subject, sender peer, sender replica incarnation, + recipient subject, and recipient peer channel. +12. One SQLite process and durable volume must own each relay shard. The application must not run a second authority + or open that database through a network file system. +13. Relay authentication and ordinary document authorization must not imply unsafe Automerge resource trust. The + explicit grant must match the principal, resolved remote, direction, complete document set, finite lease, and + revocation Effect. +14. Relay infrastructure validates bounded opaque envelope structure, routing, hashes, digest, and provenance shape. + Relay enabled `PeerSync` performs the one semantic Automerge decode. +15. After fresh ordinary and unsafe grants, relay policy revocation and the bounded operation contend on a local gate. + Revocation admitted first prevents SQL mutation or payload emission. An operation admitted first may finish and + returns its real result. Revocation drains that in flight operation and blocks later work. ### Invalid compositions @@ -1477,6 +1536,10 @@ Consistency guarantees: as authorization evidence. - Do not share one RPC handler Layer across tenants when the underlying SQL replica has no tenant column. - Do not treat `Push`, WebSocket, stream acknowledgement, `markSent`, or `observedByPeer` as remote durable custody. +- Do not treat store and forward as exactly once delivery. A stale claim may duplicate delivery, but its old token + cannot retire a newer claim. +- Do not run two relay custody authorities for one shard or infer automatic failover, quorum, or split brain recovery. +- Do not grant `UnsafeUnboundedAutomerge3DecodeGrant` merely because a peer is authenticated or document authorized. - Do not resume an `Open` stream or Automerge connection state after reconnect. Create a fresh scope and session. - Do not keep the browser SQLite connection in page code or open the same OPFS database outside the elected owner. - Do not add a second Automerge repository or backend materialization and call it canonical. Projections remain @@ -1486,8 +1549,8 @@ Consistency guarantees: This beta deliberately provides building blocks rather than a complete collaboration product. -- The RPC package provides a live server replica protocol and policy extension points. It does not provide a managed - backend, durable relay, peer discovery, asynchronous store and forward, account system, credential issuer, or tenant +- The RPC package provides a live direct protocol and an optional single authority SQLite store and forward relay. It + does not provide a managed backend, peer discovery, replicated relay, account system, credential issuer, or tenant registry. - The application owns credential issuance and rotation, authenticator and authorization implementations, stable peer identity assignment, tenant routing, TLS, Origin validation, ingress controls, logging policy, and end to end @@ -1500,6 +1563,15 @@ This beta deliberately provides building blocks rather than a complete collabora identity, and payload content must not be attached to spans or metrics. Application logs must preserve the same rule. - Authentication and authorization leases are upper bounds on reuse. Their invalidation Effects may terminate them earlier. They do not revoke data already synchronized to an authorized replica. +- Automerge `3.3.2` does not expose allocation bounded semantic decode. Relay policy must deny + `authorizeUnsafeUnboundedAutomerge3Decode` unless the application controls and resource trusts the producer bytes. + Ordinary authentication and document authorization are insufficient. The exact grant binds principal, remote, + direction, documents, finite expiry, and revocation. Direct protocol version `2` retains the preexisting authorized + peer allocation risk. A future allocation bounded Automerge API should replace this exception and remove the unsafe + grant. +- Relay revocation is not atomic with SQLite. It does not retroactively cancel an operation that already won the local + gate. That bounded operation may finish its durable commit or delivery and returns its real result. Revocation drains + it and prevents later operations. If revocation wins the gate first, no SQL mutation or payload emission occurs. - `AuthenticationFailure` and `AccessDenied` intentionally disclose no reason. `UnsupportedVersion`, `PeerMismatch`, `InvalidRequest`, and limit errors are typed protocol failures. Capacity and session availability errors are live resource failures. @@ -1517,12 +1589,14 @@ This beta deliberately provides building blocks rather than a complete collabora - An old backup can require a matching application build until versioned migration support exists. - One mutation targets one document. There is no replicated transaction across documents. - Whole document sync is the only sync granularity. Subtree sync is not implemented. -- Store and forward capability is a transport declaration. The shipped direct and RPC transports report it as false. - Document lineage is an unauthenticated peer assertion. It is carried on the sync envelope, it is not covered by the envelope's message digest, and no authorization check restricts what a peer may claim. It exists to stop an honest but stale peer from silently resurrecting discarded history. It is not evidence about the peer and must not be used as one. A refusal is not proof that the local document is stale. Confirm locally that a rewrite ran first. A forged lineage costs the forger that one document in that one session and nothing else. +- Direct `RpcPeerTransport` reports store and forward as false. The separate + `RpcPeerTransport.layerStoreAndForward` and `makeStoreAndForwardSession` constructors report it as true only after + the version `3` relay handshake. - Conflict inspection, history browsing, sharing policy, and resolution UI belong to the application. - Presence is not durable awareness and must not carry authorization decisions. - Limits must be selected for the product. They bound backup bytes, archive records, JSON depth, sync messages, @@ -1537,11 +1611,11 @@ This beta deliberately provides building blocks rather than a complete collabora | Owner | Required responsibility | | ---------------------- | ----------------------------------------------------------------------------------------------------------------- | -| Application | `PeerCredentials`, `PeerAuthenticator`, `PeerAuthorization`, tenant mapping, stable peer IDs, reconnect policy | +| Application | Credentials, authentication, direct and relay authorization, tenant mapping, stable peer IDs, reconnect policy | | Effect RPC composition | `RpcClient.Protocol`, `RpcServer.Protocol`, serialization, request and stream lifecycle | | Platform | Socket and WebSocket implementation, HTTP server, TLS, DNS, process signals | | Ingress | Origin policy, connection limits, upgrade timeout, maximum frame and request bytes, load shedding | -| Effect Local RPC | Versioned `Open` and `Push`, required auth middleware, exact authorization, bounded live registry and adapter | +| Effect Local RPC | Direct version `2`, relay version `3`, required auth middleware, bounded sessions, custody adapter, and limits | | SQL replica | Canonical Automerge state, durable peer outbox and receipts, fencing, recovery, projections | | Operations | Secret rotation, telemetry redaction, capacity configuration, backup policy, incident response, graceful shutdown | @@ -1551,12 +1625,13 @@ bounded number of concurrent session cleanups to complete before terminating the not bounded. Operations that require a hard process deadline must enforce it outside the Layer and decide when to force termination. -Read [architecture](docs/architecture.md), [durability](docs/durability.md), [sync](docs/sync.md), and -[schema evolution](docs/schema-evolution.md) for the detailed contracts. +Read [architecture](docs/architecture.md), [durability](docs/durability.md), [sync](docs/sync.md), +[store and forward](docs/store-and-forward.md), and [schema evolution](docs/schema-evolution.md) for the detailed +contracts. ## Non goals -- Durable opaque envelope relay, custody receipts, remote durable cursors, and asynchronous replay. +- Globally replicated relay custody, automatic failover, quorum operation, shard rebalance, and cross region routing. - Backend owned application state outside the canonical Automerge SQL replica. - NATS, Kafka, broker, Cluster, or Workflow orchestration for live network sessions. - Automatic token issuance, user storage, tenant registry, policy language, peer discovery, or sharing UI. @@ -1566,7 +1641,7 @@ Read [architecture](docs/architecture.md), [durability](docs/durability.md), [sy - Automatic, scheduled, or threshold triggered history truncation, and any path that lets a rewritten document resume synchronizing with a peer that still holds the superseded lineage. - Remote durability confirmation. The current `durableConfirmation` API returns only `false`. -- Protocol compatibility with versions other than `PeerRpc.protocolVersion`. +- Protocol compatibility with versions other than the declared direct and relay protocol versions. - React APIs in the RPC package. React consumers continue to use `ReplicaAtom` over their local replica. ## Repository scripts @@ -1606,15 +1681,15 @@ Every root package exports module namespaces. Every module is also available thr The export surface supports several assembly levels. Most applications start with everyday domain and composition modules. Feature and advanced services remain public for products that need direct control. -| Level | Modules | -| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| Everyday domain | `Document`, `DocumentSet`, `Mutation`, `Projection`, `Query`, `ReplicaDefinition`, `Replica`, `CommandOutcome`, `Snapshot`, `Identity`, `ReplicaStatus`, `Backup` | -| Durable runtime | `SqlProjection`, `SqlReplica`, `BrowserReplica`, `BrowserSqlite` | -| Reactive and test adapters | `ReplicaAtom`, `TestReplica` | -| Optional features | `PeerTransport`, `PeerSession`, `Presence`, `ReplicaWorkflow`, `TestPeer`, `FaultInjection`, `PeerRpc`, `RpcPeerTransport` | -| RPC policy and server | `PeerCredentials`, `PeerAuthenticator`, `PeerAuthentication`, `PeerAuthorization`, `PeerRpcLimits`, `PeerRpcServer`, `PeerRpcError` | -| Advanced assembly | `CommitPublisher`, `Compaction`, `Recovery`, `PeerSync`, `DurableRuntime`, `ReplicaClient`, `ReplicaOwner`, `SessionManager` | -| Advanced framework assembly | `CommandExecutor`, `DocumentEntity`, `DocumentStore`, `EntityReplica`, `Migrations`, `ProjectionStore`, `QueryExecutor`, `ReplicaBootstrap`, `ReplicaGate`, `ReplicaRpc` | +| Level | Modules | +| --------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Everyday domain | `Document`, `DocumentSet`, `Mutation`, `Projection`, `Query`, `ReplicaDefinition`, `Replica`, `CommandOutcome`, `Snapshot`, `Identity`, `ReplicaStatus`, `Backup` | +| Durable runtime | `SqlProjection`, `SqlReplica`, `BrowserReplica`, `BrowserSqlite` | +| Reactive and test adapters | `ReplicaAtom`, `TestReplica` | +| Optional features | `PeerTransport`, `PeerSession`, `Presence`, `ReplicaWorkflow`, `TestPeer`, `FaultInjection`, `PeerRpc`, `PeerRelayRpc`, `RpcPeerTransport` | +| RPC policy and server | `PeerCredentials`, `PeerAuthenticator`, `PeerAuthentication`, `PeerAuthorization`, `PeerRelayAuthorization`, `PeerRpcLimits`, `PeerRelayLimits`, `PeerRelayIngress`, `PeerRelayStore`, `PeerRpcServer`, `PeerRpcError` | +| Advanced assembly | `CommitPublisher`, `Compaction`, `Recovery`, `PeerSync`, `DurableRuntime`, `ReplicaClient`, `ReplicaOwner`, `SessionManager` | +| Advanced framework assembly | `CommandExecutor`, `DocumentEntity`, `DocumentStore`, `EntityReplica`, `Migrations`, `ProjectionStore`, `QueryExecutor`, `ReplicaBootstrap`, `ReplicaGate`, `ReplicaRpc` | These public framework assembly modules support custom runtimes and diagnostics. Paths under `internal/*` remain private and unsupported. The public assembly modules are not required for the normal application path shown in the @@ -1630,9 +1705,9 @@ composition recipes. | `Commit` | `Heads`, `Commit` and their inferred types | | `Document` | `WireSchema`, `AutomergeEncoded`, `DocumentSchema`, `Document`, `Any`, `make`, `isAutomergeValue`, `decode`, `encode` | | `DocumentSet` | `DocumentSet`, `make`, `get` | -| `Identity` | Schemas and types for `ReplicaId`, `ReplicaIncarnation`, `SessionId`, `DocumentId`, `CommandId`, `WriterGeneration`, `CommitSequence`, `PeerId`, `ProjectionVersion`, and `DocumentLineage`. `genesisLineage`, `makeReplicaId`, `makeSessionId`, `makeDocumentId`, `makeCommandId`, `makePeerId`, `makeDocumentLineage`, `documentIdFromCommandId` | +| `Identity` | Schemas and types for `ReplicaId`, `ReplicaIncarnation`, `SessionId`, `DocumentId`, `CommandId`, `RelayMessageId`, `WriterGeneration`, `CommitSequence`, `PeerId`, `ProjectionVersion`, and `DocumentLineage`. `genesisLineage`, identity generators including `makeRelayMessageId`, `makeDocumentLineage`, `documentIdFromCommandId` | | `Mutation` | `DraftValue`, `Draft`, `SuccessResult`, `HandlerResult`, `HandlerOptions`, `Handler`, `HandlerService`, `Mutation`, `Any`, `make`; definitions expose `payloadSchema`, `successSchema`, `errorSchema`, `of`, and `toLayer`; `toLayer` accepts a handler or an Effect that builds one | -| `PeerTransport` | `Capabilities`, `Connection`, `ConnectOptions`, `PeerTransport` service | +| `PeerTransport` | `Capabilities`, `PermanentRejectReason`, `RelayDeliveryIdentity`, `AcknowledgedDelivery`, `Connection`, `ConnectOptions`, `PeerTransport` service | | `Projection` | `Projection`, `Any`, `make`, `assertUniqueKeys`, `evaluate` | | `Query` | `Handler`, `HandlerService`, `Query`, `Any`, `make`; definitions expose `payloadSchema`, `successSchema`, `errorSchema`, `of`, and `toLayer`; `toLayer` accepts a handler or an Effect that builds one | | `Replica` | `Replica` service | @@ -1716,27 +1791,32 @@ descriptor does not install its handler. ### `@lucas-barake/effect-local-sql` -| Namespace | Public API | -| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `BackupStore` | `BackupStore` service, `layer` | -| `CommandExecutor` | `createRequestHash`, `mutationRequestHash`, `deleteRequestHash`, `CommandExecutor` service, `MutationHandlers`, `layer` | -| `CommitPublisher` | `CommitEvent`, `CommitSubscription`, `CommitPublisher` service, `layer` | -| `Compaction` | `OperationId`, `PreparedCheckpoint`, `CompactResult`, `Compaction` service, `layer` | -| `DocumentEntity` | Cluster RPC definitions `Create`, `Mutate`, `Delete`, `ApplySync`, plus `ApplySyncResult`, `DocumentEntity`, `layer` | -| `DocumentStore` | `Stored`, `DocumentStore` service, `layer` | -| `DurableRuntime` | `layer`, `layerWith` | -| `EntityReplica` | `layer` | -| `Migrations` | `canonicalStoreChecksum`, `peerSyncChecksum`, `durabilityIndexesChecksum`, `projectionReadinessChecksum`, `pendingReceiptIndexesChecksum`, `peerWriterProvenanceChecksum`, `replicaHealthIndexesChecksum`, `documentLineageChecksum`, `historyRewriteMarkersChecksum`, `loader`, `run`, `layer` | -| `PeerSession` | `SelectedDocument`, `PeerSession`, `SupervisedPeerSession`, `SyncEnvelope`, `maximumSyncEnvelopeBytes`, `makeTestClient`, `makeSupervised`, `make`, `makeLive` | -| `PeerSync` | `Session`, `Outbound`, `Reply`, `Generated`, `Received`, `PeerSync` service, `layer` | -| `ProjectionStore` | `ProjectionStore` service, `BindingServices`, `layer` | -| `QueryExecutor` | `QueryExecutor` service, `QueryHandlers`, `layer` | -| `Recovery` | `RawRecoveryExport`, `Recovery` service, `make`, `layer` | -| `ReplicaBootstrap` | `State`, `ReplicaBootstrap` service, `make`, `layer` | -| `ReplicaGate` | `Permit`, `ReplicaGate` service, `layer` | -| `ReplicaWorkflow` | `OperationId` re-exported from `Compaction`, `CompactReplica`, `RewriteDocumentHistory`, `Execution`, `DocumentExecution`, `CompactionWorkflow`, `HistoryRewriteWorkflow`, `layerRegistration`, `layerRuntime`, `layerHistoryRewriteRegistration`, `layerHistoryRewriteRuntime` | -| `SqlProjection` | `Migration`, `SqlProjection`, `BindingService`, `make`, `Any` | -| `SqlReplica` | `layerFromServices`, `layer`, `layerWithBindings` | +| Namespace | Public API | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `BackupStore` | `BackupStore` service, `layer` | +| `CommandExecutor` | `createRequestHash`, `mutationRequestHash`, `deleteRequestHash`, `CommandExecutor` service, `MutationHandlers`, `layer` | +| `CommitPublisher` | `CommitEvent`, `CommitSubscription`, `CommitPublisher` service, `layer` | +| `Compaction` | `OperationId`, `PreparedCheckpoint`, `CompactResult`, `Compaction` service, `layer` | +| `DocumentEntity` | Cluster RPC definitions `Create`, `Mutate`, `Delete`, `ApplySync`, plus `ApplySyncResult`, `DocumentEntity`, `layer` | +| `DocumentStore` | `Stored`, `DocumentStore` service, `layer` | +| `DurableRuntime` | `layer`, `layerWith`, `layerRelay` | +| `EntityReplica` | `layer` | +| `Migrations` | `canonicalStoreChecksum`, `peerSyncChecksum`, `durabilityIndexesChecksum`, `projectionReadinessChecksum`, `pendingReceiptIndexesChecksum`, `peerWriterProvenanceChecksum`, `replicaHealthIndexesChecksum`, `documentLineageChecksum`, `historyRewriteMarkersChecksum`, `peerRelayStateChecksum`, `loader`, `run`, `layer` | +| `PeerRelayClientRuntime` | `ConnectionConfiguration`, `PeerRelayClientRuntime` service, `makeScoped`, `layer` | +| `PeerRelayOutbox` | `Endpoint`, `AdmitInput`, `ReplayInput`, `CustodyInput`, `Entry`, `Usage`, `PeerRelayOutbox` service, `layerSql` | +| `PeerRelayOutboxLimits` | `Values`, `defaults`, `PeerRelayOutboxLimits` service, `make`, `layer`, `layerDefaults` | +| `PeerRelayReceiptLimits` | `Values`, `defaults`, `PeerRelayReceiptLimits` service, `make`, `layer`, `layerDefaults` | +| `PeerSession` | `SelectedDocument`, `PeerSession`, `SupervisedPeerSession`, `SyncEnvelope`, `maximumSyncEnvelopeBytes`, `makeTestClient`, `makeSupervised`, `make`, `makeLive` | +| `PeerSync` | `Session`, `Outbound`, `Reply`, `Generated`, `Received`, `RelayReceipt`, `PeerSync` service, `layer`, `layerRelay` | +| `PeerSyncEnvelope` | Direct and relay envelope schemas, validation, encoding, canonical outer digest, and declared size bounds | +| `ProjectionStore` | `ProjectionStore` service, `BindingServices`, `layer` | +| `QueryExecutor` | `QueryExecutor` service, `QueryHandlers`, `layer` | +| `Recovery` | `RawRecoveryExport`, `Recovery` service, `make`, `layer` | +| `ReplicaBootstrap` | `State`, `ReplicaBootstrap` service, `make`, `layer` | +| `ReplicaGate` | `Permit`, `ReplicaGate` service, `layer` | +| `ReplicaWorkflow` | `OperationId` reexported from `Compaction`, `CompactReplica`, `RewriteDocumentHistory`, `Execution`, `DocumentExecution`, `CompactionWorkflow`, `HistoryRewriteWorkflow`, `layerRegistration`, `layerRuntime`, `layerHistoryRewriteRegistration`, `layerHistoryRewriteRuntime` | +| `SqlProjection` | `Migration`, `SqlProjection`, `BindingService`, `make`, `Any` | +| `SqlReplica` | `layerFromServices`, `layer`, `layerRelay`, `layerWithBindings` | Public SQL service methods: @@ -1747,6 +1827,8 @@ Public SQL service methods: | `CommitPublisher` | `publishPending`, `invalidate(keys)`, scoped `subscribe` | | `Compaction` | `prepare(document, documentId)`, `publish(checkpoint)`, `compact(document, documentId)`, `prune(documentId)`, `pruneCommandReceipts`, `rewriteHistory(document, documentId, operationId)` | | `DocumentStore` | `create`, `load`, `stage`, `tombstone`, `persist`; callers own the native document returned by `load` | +| `PeerRelayClientRuntime` | Stable admission, exact endpoint replay, custody retirement, receipt pruning signal, health, and fatal failure observation | +| `PeerRelayOutbox` | `admit`, `dueForEndpoint`, `maximumPendingHorizon`, `markCustody`, `pruneExpired`, `usage`, `validateReplicaIncarnation` | | `PeerSync` | `open(peerId)`, `reset(session)`, `generate(document, documentId, session, peer)`, `receive(document, documentId, session, input)`, `enqueue(session, reply)`, `pending(session)`, `markSent(session, sendSequence, messageHash)`, `invalidateDocument(documentId)` | | `ProjectionStore` | `clear`, `replace(binding, snapshot, destinationTable)`, `replaceDocument(document, snapshot, commitSequence)` | | `QueryExecutor` | `execute(query, payload)` and reactive `reactive(query, payload)` | @@ -1905,17 +1987,22 @@ planning: ### `@lucas-barake/effect-local-rpc` -| Namespace | Public API | -| -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `PeerAuthentication` | `PeerPrincipal` schema and type, request service `AuthenticatedPeer`, required RPC middleware `PeerAuthentication`, argument free `layerServer`, argument free `layerClient` | -| `PeerAuthenticator` | `PeerAuthenticator` service. `authenticate(Redacted)` returns principal, finite `validUntil`, and `invalidated`, or `AuthenticationFailure` | -| `PeerAuthorization` | `PeerAuthorization` service and validating constructor `layer`. `authorize` returns exactly resolved `SelectedDocument` values, finite `validUntil`, and `invalidated`, or `AccessDenied` or `ServerUnavailable` | -| `PeerCredentials` | `PeerCredentials` service. `get` is an Effect that returns a rotating `Redacted` or `AuthenticationFailure` | -| `PeerRpc` | `protocolVersion`, `DefinitionHash`, `RequestedDocument`, `Opened`, `Message`, `OpenEvent`, `OpenRpc`, `PushRpc`, `Rpcs`, generated `RpcClient`, `makeRpcClient` | -| `PeerRpcError` | `AuthenticationFailure`, `AccessDenied`, `UnsupportedVersion`, `PeerMismatch`, `DefinitionMismatch`, `InvalidRequest`, `RequestLimitExceeded`, `RequestCapacityExceeded`, `SessionUnavailable`, `SessionOverloaded`, `ServerUnavailable`, `DocumentLineageChanged`, union schema and type `PeerRpcError`, fixed `Defect` schema | -| `PeerRpcLimits` | `Values` schema and type, `defaults`, `InvalidPeerRpcLimits`, `PeerRpcLimits` service, `make`, `layer`, `layerDefaults` | -| `PeerRpcServer` | `layerHandlers({ tenantId, peerId, definition })` | -| `RpcPeerTransport` | `isRetryable`, advanced `layer(client, { documents, definition })`, preferred `makeSession(client, { peerId, documents, definition })` | +| Namespace | Public API | +| ------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `PeerAuthentication` | `PeerPrincipal` schema and type, request service `AuthenticatedPeer`, required RPC middleware `PeerAuthentication`, argument free `layerServer`, argument free `layerClient` | +| `PeerAuthenticator` | `PeerAuthenticator` service. `authenticate(Redacted)` returns principal, finite `validUntil`, and `invalidated`, or `AuthenticationFailure` | +| `PeerAuthorization` | `PeerAuthorization` service and validating constructor `layer`. `authorize` returns exactly resolved `SelectedDocument` values, finite `validUntil`, and `invalidated`, or `AccessDenied` or `ServerUnavailable` | +| `PeerCredentials` | `PeerCredentials` service. `get` is an Effect that returns a rotating `Redacted` or `AuthenticationFailure` | +| `PeerRelayAuthorization` | `Direction`, `RemotePeer`, ordinary request and result contracts, `UnsafeUnboundedAutomerge3DecodeRequest`, `UnsafeUnboundedAutomerge3DecodeGrant`, `unsafeUnboundedAutomerge3DecodeRisk`, `AuthorizeUnsafeUnboundedAutomerge3Decode`, `denyUnsafeUnboundedAutomerge3Decode`, `PeerRelayAuthorization` service, validating two callback `layer` | +| `PeerRelayIngress` | `Usage`, `Reservation`, `PeerRelayIngress` service, server `layerProtocolSocketServer`, client `makeProtocolSocket` and `layerProtocolSocket` | +| `PeerRelayLimits` | `Values`, `defaults`, `InvalidPeerRelayLimits`, `PeerRelayLimits` service, `make`, `layer`, `layerDefaults` | +| `PeerRelayRpc` | Protocol version `3`, relay event schemas, four RPC definitions, `Rpcs`, generated `RpcClient`, `makeRpcClient` | +| `PeerRelayStore` | Relay channel, admission, claim, terminal, maintenance, and usage contracts, `PeerRelayStore` service, `make`, `layerSqlite` | +| `PeerRpc` | `protocolVersion`, `DefinitionHash`, `RequestedDocument`, `Opened`, `Message`, `OpenEvent`, `OpenRpc`, `PushRpc`, `Rpcs`, generated `RpcClient`, `makeRpcClient` | +| `PeerRpcError` | `AuthenticationFailure`, `AccessDenied`, `UnsupportedVersion`, `PeerMismatch`, `DefinitionMismatch`, `InvalidRequest`, `RequestLimitExceeded`, `RequestCapacityExceeded`, `SessionUnavailable`, `SessionOverloaded`, `ServerUnavailable`, `DocumentLineageChanged`, union schema and type `PeerRpcError`, fixed `Defect` schema | +| `PeerRpcLimits` | `Values` schema and type, `defaults`, `InvalidPeerRpcLimits`, `PeerRpcLimits` service, `make`, `layer`, `layerDefaults` | +| `PeerRpcServer` | Direct `layerHandlers`, relay `layerRelayHandlers` and `layerRelayServer`, `layerStoreAndForwardDeployment`, `PeerRelayServerRuntime` | +| `RpcPeerTransport` | Direct `layer` and `makeSession`, relay `StoreAndForwardOptions`, `layerStoreAndForward`, `makeStoreAndForwardSession`, `isRetryable` | #### RPC procedure contract @@ -1934,18 +2021,44 @@ optional client `capabilities: { lineageAware }`, which is how the server learns required `PeerAuthentication` middleware. `makeRpcClient` requires `RpcClient.Protocol`, client authentication middleware, and `Scope`. It deliberately does not return a package owned connection service. +Relay procedure contract: + +| Procedure | Request | Success | Boundary | +| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `OpenRelay` | Version `3`, expected relay and local principals, sender incarnation, exact remote subject and peer, whole documents, receipt retention, sender retry horizon, middleware owned credential | Stream beginning with `RelayOpened`, followed by `StoredMessage` values | Authentication and ordinary send or receive authorization bind the exact endpoint and documents. Stream interruption releases live claims for bounded retry | +| `PushRelay` | Current `sessionId`, stable `relayMessageId`, bounded payload, middleware owned credential | `void` after SQLite custody commits | Admission also requires the exact unsafe Automerge resource trust grant. Duplicate matching admission is safe. Conflicting identity reuse and failures are typed | +| `AcknowledgeRelay` | Current `sessionId`, `relayMessageId`, `claimToken`, `messageHash`, middleware owned credential | `void` after a conditional terminal transition | Only the exact live recipient claim can acknowledge. Duplicate matching acknowledgement is safe | +| `RejectRelay` | The acknowledgement identity plus `ProtocolInvalid` or `ApplicationRejected` | `void` after a conditional dead letter transition | Permanent rejection erases the active payload and retains bounded terminal evidence | + +Every relay procedure can fail with a fieldless `PeerRpcError`. Delivery of a stored message also requires the exact +unsafe Automerge resource trust grant for `Receive`. `RelayOpened` reports the authenticated local +principal, exact remote peer, a new `SessionId`, version `3`, and +`capabilities: { storeAndForward: true }`. `StoredMessage` carries the complete delivery identity and an opaque claim +token. `PeerRelayRpc.makeRpcClient` requires the custom `PeerRelayIngress.layerProtocolSocket` client protocol, +client authentication middleware, and `Scope`. + RPC environment requirements: -| Constructor or Layer | Required Effect services | -| ---------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `PeerAuthentication.layerClient` | `PeerCredentials` | -| `PeerAuthentication.layerServer` | `PeerAuthenticator`, `PeerRpcLimits` | -| `PeerAuthorization.layer(authorize)` | None. The callback must return an environment free Effect. Build `PeerAuthorization` with an application `Layer.effect` instead when authorization needs other services | -| `PeerRpcLimits.make`, `layer`, `layerDefaults` | Core `ReplicaLimits` | -| `PeerRpcServer.layerHandlers` | `CommitPublisher`, `PeerRpcLimits`, core `ReplicaLimits`, `PeerAuthorization`, `Crypto`, `PeerSync`, `ReplicaGate`, `Sharding` | -| `PeerRpc.makeRpcClient` | `RpcClient.Protocol`, required client `PeerAuthentication` middleware, `Scope` | -| `RpcPeerTransport.layer` | `Scope` for each connection acquisition | -| `RpcPeerTransport.makeSession` | `Scope`, `CommitPublisher`, `Crypto`, `PeerSync`, `ReplicaGate`, core `ReplicaLimits`, `Sharding`; the adapter supplies `PeerTransport` | +| Constructor or Layer | Required Effect services | +| ----------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `PeerAuthentication.layerClient` | `PeerCredentials` | +| `PeerAuthentication.layerServer` | `PeerAuthenticator`, `PeerRpcLimits` | +| `PeerAuthorization.layer(authorize)` | None. The callback must return an environment free Effect. Build `PeerAuthorization` with an application `Layer.effect` instead when authorization needs other services | +| `PeerRpcLimits.make`, `layer`, `layerDefaults` | Core `ReplicaLimits` | +| `PeerRpcServer.layerHandlers` | `CommitPublisher`, `PeerRpcLimits`, core `ReplicaLimits`, `PeerAuthorization`, `Crypto`, `PeerSync`, `ReplicaGate`, `Sharding` | +| `PeerRpc.makeRpcClient` | `RpcClient.Protocol`, required client `PeerAuthentication` middleware, `Scope` | +| `RpcPeerTransport.layer` | `Scope` for each connection acquisition | +| `RpcPeerTransport.makeSession` | `Scope`, `CommitPublisher`, `Crypto`, `PeerSync`, `ReplicaGate`, core `ReplicaLimits`, `Sharding`; the adapter supplies `PeerTransport` | +| `PeerRelayAuthorization.layer(authorize, authorizeUnsafeUnboundedAutomerge3Decode)` | None. Use `denyUnsafeUnboundedAutomerge3Decode` by default. Use an application `Layer.effect` when either policy needs other services | +| `PeerRelayLimits.make`, `layer`, `layerDefaults` | None | +| `PeerRelayStore.layerSqlite` | `SqlClient`, `Crypto`, `PeerRelayLimits` | +| `PeerRelayIngress.layerProtocolSocketServer` | `PeerRelayLimits` plus the requirements of the supplied `SocketServer` Layer | +| `PeerRpcServer.layerRelayHandlers` | `Crypto`, `PeerRelayAuthorization`, `PeerRelayIngress`, `PeerRelayLimits`, `PeerRelayStore` | +| `PeerRpcServer.layerRelayServer` | Use with `layerRelayHandlers` and `PeerRelayIngress`. Together they require `PeerAuthentication` and the relay `RpcServer.Protocol` | +| `PeerRpcServer.layerStoreAndForwardDeployment` | Requirements of the supplied direct deployment and relay socket Layer, plus `PeerAuthentication`, `Crypto`, `PeerRelayAuthorization`, `PeerRelayLimits`, `PeerRelayStore` | +| `PeerRelayRpc.makeRpcClient` | Custom relay `RpcClient.Protocol`, required client `PeerAuthentication` middleware, `Scope` | +| `RpcPeerTransport.layerStoreAndForward` | `Crypto`, `PeerRelayClientRuntime`; `Scope` for each connection acquisition | +| `RpcPeerTransport.makeStoreAndForwardSession` | `Scope`, `CommitPublisher`, `Crypto`, relay enabled `PeerSync`, `ReplicaGate`, core `ReplicaLimits`, `Sharding`, `PeerRelayClientRuntime` | `PeerPrincipal` is `{ tenantId, subjectId, peerId }`. `RequestedDocument` is `{ documentType, documentId }`. Every `PeerRpcError` class is a fieldless tagged error, including `DocumentLineageChanged`. A peer therefore learns that @@ -1986,6 +2099,14 @@ tree. Relational failures use `InvalidPeerRpcLimits` with the failing field. The | Retained rate state | Connections `10_000`, subjects `10_000`, idle retention `600_000 ms` | | Policy and workers | Maximum reauthorization interval `300_000 ms`, commit flush concurrency `8`, shutdown cleanup concurrency `16` | +#### Relay limits contract + +`PeerRelayLimits.Values` separately bounds relay storage, framing, sessions, byte reservations, rate state, terminal +responses, channel queues, workers, SQL work classes, maintenance, claim leases, retries, and shutdown. Its +`maximumDeliveryAttempts` is a positive integer and defaults to `16`. Each failed, interrupted, disconnected, or +expired claim advances the durable count. Reaching the cap moves the row to `DeadLettered`, erases its payload, and +retains only bounded terminal deduplication evidence. + #### RPC observability contract RPC spans are `effect_local_rpc.authentication`, `effect_local_rpc.server.open`, diff --git a/docs/architecture.md b/docs/architecture.md index d8679dc..e751c83 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -39,10 +39,53 @@ for reconstructing the session in place. 4. Workflow journals are durable orchestration records. 5. Atom values are reactive caches. 6. Presence and tab sessions are ephemeral. +7. Optional sender relay outbox rows are temporary durable transfer state. +8. Optional relay custody rows are temporary durable delivery state owned by one SQLite authority per shard. Cluster and Workflow do not replace Automerge. They serialize local effects, resolve retry ambiguity, and resume operations after process loss. Automerge remains responsible for causal history, conflicts, and convergence. +## Relay topology + +The direct RPC deployment and optional store and forward deployment have separate protocols and listeners. Direct +version `2` continues through the application owned protocol and `PeerRpcServer.layerHandlers`. Relay version `3` +uses `PeerRelayRpc.Rpcs`, the bounded `PeerRelayIngress` socket protocol, and a distinct socket Layer through +`PeerRpcServer.layerStoreAndForwardDeployment`. + +```mermaid +flowchart LR + Sender["Sender SQL replica"] --> Outbox["Stable sender relay outbox"] + Outbox --> RelayListener["Relay protocol v3 listener"] + RelayListener --> Custody["Single authority relay SQLite"] + Custody --> RecipientListener["Recipient relay session"] + RecipientListener --> Receipt["Recipient SQL sync and receipt"] + Receipt --> Ack["Fenced relay acknowledgement"] + Ack --> Custody +``` + +One relay SQLite database is authoritative for one shard. The application must route every custody write and claim +for that shard to one process and one durable volume. This topology does not include leader election, quorum +replication, automatic failover, shard rebalance, network file system sharing, or split brain recovery. + +Relay ordering is FIFO within one exact directed peer channel. Distinct channels can progress concurrently. Claims +are finite fences. A stale claim may duplicate delivery, but its old token cannot retire a newer claim. See +[Store and forward](store-and-forward.md) for delivery, capacity, security, and lifecycle details. + +Relay infrastructure treats the Automerge message as opaque. It validates bounded framing, envelope Schema, endpoint +and document routing, hashes, outer digest, and writer provenance shape. Relay enabled `PeerSync` performs the one +semantic Automerge decode. + +That boundary is explicit because Automerge `3.3.2` has no allocation bounded decode API. Ordinary authentication and +document authorization do not imply resource trust. `PeerRelayAuthorization` requires a second exact +`UnsafeUnboundedAutomerge3DecodeGrant` for relay admission and delivery. Its principal, remote, direction, documents, +finite lease, and revocation Effect are all scoped and validated. Default deny is +`denyUnsafeUnboundedAutomerge3Decode`. + +Fresh ordinary and unsafe grants feed a local operation gate. Revocation that reaches the gate first prevents SQL +mutation or payload emission. An operation admitted first may finish and returns its real result. Revocation drains +that bounded in flight work and blocks later work. This is a local ordering boundary, not an atomic transaction +between SQLite and an external policy authority. + ## Domain API Applications define schema coded `Document`, `Mutation`, `Projection`, and `Query` values, then collect them in one diff --git a/docs/durability.md b/docs/durability.md index f1eb5bd..0d10a6f 100644 --- a/docs/durability.md +++ b/docs/durability.md @@ -134,3 +134,41 @@ Projection failure rolls back the replacement. Browser persistence does not make backup optional. OPFS may be evicted while its origin storage bucket remains best effort. A complete product must expose export, restore, duplicate, and deletion flows to the person who owns the data. + +## Store and forward custody + +Store and forward adds three different durable boundaries. + +1. Sender admission commits a stable relay message identity and complete outer envelope to the local relay outbox + before a network attempt. +2. `PushRelay` succeeds only after the relay SQLite authority commits the envelope and immutable quota reservation. +3. `AcknowledgeRelay` is attempted only after the recipient commits the Automerge application result and sender + scoped replay receipt through the production sync path. + +Relay admission and delivery require the ordinary authorization lease and a separate exact unsafe Automerge resource +trust lease. After fresh grants, a bounded server operation and policy expiry or revocation contend on a local gate. +Revocation that wins before admission prevents the SQL mutation or payload emission. An operation that wins is in +flight, may finish its durable commit or delivery, and returns its real result. Revocation drains that operation and +prevents later work. It does not retroactively cancel the winner. SQLite and the external policy authority do not +share an atomic transaction. The extra grant exists because Automerge `3.3.2` does not expose allocation bounded +semantic decode. Authentication and document access alone do not satisfy it. + +These boundaries provide at least once transfer. They do not provide exactly once delivery. A response can be lost +after any commit. A claim can expire while an old recipient worker still runs. The durable identities and claim +tokens make retries and stale acknowledgements safe, while retained recipient receipts make duplicate application +idempotent during the negotiated window. + +Relay FIFO ordering is scoped to one exact directed peer channel. One claimed head blocks later rows in that channel. +Unrelated channels do not share that ordering fence. + +Recipient delivery attempts are durable and bounded by `PeerRelayLimits.maximumDeliveryAttempts`, which defaults to +`16`. Failed, interrupted, disconnected, and expired claims advance the count. Reaching the cap changes the message +to `DeadLettered` and erases the payload. Restart cannot reset this poison bound. + +Active payloads expire at `PeerRelayLimits.messageTtlMillis`. Acknowledged, rejected, and expired rows erase their +payload and retain bounded deduplication evidence until their retention deadline. Sender outbox rows expire at their +configured retry horizon. Recipient receipts expire according to `PeerRelayReceiptLimits`. + +Relay custody rows and client relay outbox or receipt rows are not canonical document history and are not included in +canonical backup archives. Restore fences the previous incarnation, removes stale relay client state, and reconciles +usage before installing the new incarnation. See [Store and forward](store-and-forward.md) for the complete contract. diff --git a/docs/store-and-forward.md b/docs/store-and-forward.md new file mode 100644 index 0000000..cead894 --- /dev/null +++ b/docs/store-and-forward.md @@ -0,0 +1,301 @@ +# Store and forward + +## Contract + +Store and forward is an optional relay mode for SQL replicas. It adds durable sender admission, durable relay +custody, reconnect delivery, and durable recipient receipts. Direct RPC remains protocol version `2` with +`capabilities: { storeAndForward: false }`. Relay RPC uses protocol version `3` on a distinct listener and reports +`capabilities: { storeAndForward: true }`. + +The delivery guarantee is at least once. A successful relay `PushRelay` means the relay SQLite authority committed +the complete envelope before replying. A recipient can therefore see a duplicate after a lost response, expired +claim, interrupted acknowledgement, or reconnect. The recipient SQL path makes that duplicate safe within the +negotiated receipt retention window. Effect Local does not claim exactly once delivery. + +Store and forward does not make the relay authoritative for document contents. Each replica remains authoritative for +its local writes. Automerge remains responsible for convergence after valid changes reach a replica. + +## Opt in composition + +The application must opt in on both sides. It must also keep the direct and relay transports separate. + +On the server, `PeerRpcServer.layerHandlers` remains the direct version `2` handler Layer. The application continues +to own its direct `RpcServer.Protocol`, serializer, socket, and listener. The optional +`PeerRpcServer.layerStoreAndForwardDeployment` merges that unchanged direct deployment with a relay deployment built +from `PeerRelayRpc.Rpcs`, `PeerRelayIngress`, and a different `SocketServer` Layer. + +On the client, `SqlReplica.layerRelay` installs relay receipt support. `PeerRelayOutbox.layerSql` stores stable sender +admissions. `PeerRelayClientRuntime.layer` supervises sender outbox and receipt maintenance. +`RpcPeerTransport.makeStoreAndForwardSession` binds the generated relay client to the ordinary `PeerSession`. + +This example shows the relay specific composition. The named application values provide the existing SQL, Crypto, +authentication, direct RPC, socket, definition, projection, and mutation or query Layers. + +```ts +import * as PeerRelayAuthorization from "@lucas-barake/effect-local-rpc/PeerRelayAuthorization" +import * as PeerRelayLimits from "@lucas-barake/effect-local-rpc/PeerRelayLimits" +import * as PeerRelayStore from "@lucas-barake/effect-local-rpc/PeerRelayStore" +import * as PeerRpcServer from "@lucas-barake/effect-local-rpc/PeerRpcServer" +import * as RpcPeerTransport from "@lucas-barake/effect-local-rpc/RpcPeerTransport" +import * as PeerRelayClientRuntime from "@lucas-barake/effect-local-sql/PeerRelayClientRuntime" +import * as PeerRelayOutbox from "@lucas-barake/effect-local-sql/PeerRelayOutbox" +import * as PeerRelayOutboxLimits from "@lucas-barake/effect-local-sql/PeerRelayOutboxLimits" +import * as PeerRelayReceiptLimits from "@lucas-barake/effect-local-sql/PeerRelayReceiptLimits" +import * as SqlReplica from "@lucas-barake/effect-local-sql/SqlReplica" +import { Effect, Layer } from "effect" + +const RelayLimitsLive = PeerRelayLimits.layer(relayLimits) +const RelayStoreLive = PeerRelayStore.layerSqlite.pipe( + Layer.provideMerge(RelayLimitsLive) +) +const RelayAuthorizationLive = PeerRelayAuthorization.layer( + authorizeRelay, + PeerRelayAuthorization.denyUnsafeUnboundedAutomerge3Decode +) +const RelayServerLive = PeerRpcServer.layerStoreAndForwardDeployment({ + directDeployment, + relaySocketLayer, + relay: { tenantId, peerId: relayPeerId } +}).pipe( + Layer.provideMerge(RelayLimitsLive), + Layer.provideMerge(RelayStoreLive), + Layer.provideMerge(RelayAuthorizationLive) +) + +const ReceiptLimitsLive = PeerRelayReceiptLimits.layer(receiptLimits) +const RelayReplicaLive = SqlReplica.layerRelay(definition, { projections }).pipe( + Layer.provideMerge(ReceiptLimitsLive) +) +const RelayOutboxLive = PeerRelayOutbox.layerSql.pipe( + Layer.provideMerge(RelayReplicaLive), + Layer.provideMerge(PeerRelayOutboxLimits.layer(outboxLimits)) +) +const RelayClientLive = PeerRelayClientRuntime.layer.pipe( + Layer.provideMerge(RelayOutboxLive) +) + +const session = RpcPeerTransport.makeStoreAndForwardSession(relayRpcClient, { + expectedLocal, + senderReplicaIncarnation, + expectedRelayPeerId: relayPeerId, + remote: { subjectId: remoteSubjectId, peerId: remotePeerId }, + documents: selectedDocuments, + definition, + receiptRetentionMillis, + senderRetryHorizonMillis, + replayBatchSize +}).pipe(Effect.provide(RelayClientLive)) +``` + +`relayRpcClient` must be a `PeerRelayRpc.RpcClient` created with `PeerRelayRpc.makeRpcClient` over +`PeerRelayIngress.layerProtocolSocket`. It must not use the direct WebSocket MessagePack protocol. The server +composition still requires the application SQL and Crypto Layers, `PeerAuthentication.layerServer`, the relay +authorization Layer, and a `PeerRelayLimits` Layer. The client composition still requires the application SQL, +Crypto, core replica limits, generated client authentication middleware, and a scoped relay socket. + +The example denies the separate unsafe Automerge decode grant. That is the required default. Ordinary authentication +and `authorizeRelay` document authorization do not promote a caller into resource trust. + +The `expectedRelayPeerId`, `expectedLocal`, exact remote subject and peer, replica incarnation, selected documents, +receipt retention, and sender retry horizon are part of the version `3` handshake. A mismatch fails the connection. + +## Protocol and durable identity + +`PeerRelayRpc.Rpcs` contains `OpenRelay`, `PushRelay`, `AcknowledgeRelay`, and `RejectRelay`. Every request uses the +same required authentication middleware as direct RPC. The first `OpenRelay` stream value is `RelayOpened`. Later +values are `StoredMessage` deliveries. + +Each sender admission has one stable `RelayMessageId`. The sender outbox persists that identity, the exact relay and +recipient endpoint, the sender replica incarnation and connection sequence, document identity, writer provenance, +message hash, payload, and canonical outer envelope digest before the network call. Retrying a pending row sends the +same identity and envelope to the same relay endpoint. + +The relay admits an envelope only after Schema decoding, authentication, send authorization, digest verification, +payload validation, document selection validation, and quota reservation. Duplicate admission with the same identity +and digest is safe. Conflicting reuse is rejected. + +## Acknowledgement boundary + +A relay claim carries an opaque claim token and a finite deadline. A recipient acknowledgement can terminalize only +the exact message, recipient principal, live session generation, token, and message hash. + +`PeerSession` sends `AcknowledgeRelay` only after all of these operations succeed: + +1. The relay outer envelope and inner sync envelope are validated. +2. The Automerge message is applied through the production `DocumentEntity.ApplySync` path. +3. The sender scoped relay receipt and any resulting canonical state commit in recipient SQLite. +4. Pending commit invalidations are published. +5. Any generated reply is durably enqueued for the local peer session. + +This boundary proves durable recipient processing for replay suppression. It does not change +`PeerSession.durableConfirmation()`, which remains `false`. It also does not prove that another peer observed the +change. + +A stable protocol violation uses `RejectRelay` with `ProtocolInvalid`. Application code can use +`ApplicationRejected` through the acknowledged delivery contract. A permanent rejection becomes a retained dead +letter row. Infrastructure failure, interruption, timeout, disconnect, and claim expiry release or recover the +message for retry. + +## Ordering, retry, expiry, and retention + +Ordering is FIFO only within one exact directed channel. The channel key is the tenant, sender subject, sender peer, +sender replica incarnation, recipient subject, and recipient peer. The relay permits one claimed head for that +channel. Earlier pending or claimed rows block later rows in the same channel. Different channels can progress +independently. + +Claims are finite fences. A stale worker can duplicate delivery after its claim expires, but its old token cannot +acknowledge or reject the new claim. Retry uses bounded exponential delay with jitter. The sender outbox keeps pending +admissions until custody succeeds or their configured retry horizon expires. + +`PeerRelayLimits.maximumDeliveryAttempts` caps recipient delivery. The default is `16`. A failed, interrupted, +disconnected, or expired claim durably advances the attempt count. When that count reaches the configured cap, the +relay moves the message to `DeadLettered`, erases its payload, and retains only bounded terminal deduplication +evidence. Process restart does not reset the count. Message expiry can terminalize the row before the attempt cap. + +The relay message time to live is configured by `PeerRelayLimits.messageTtlMillis`. Expired active rows erase their +payload and become terminal. Acknowledged, rejected, and expired rows retain only bounded deduplication evidence until +their retention deadline. `PeerRelayLimits.maximumReceiptRetentionMillis` must cover the message time to live, the +maximum sender retry horizon, and minimum terminal retention. + +The recipient keeps sender scoped receipts under its current replica incarnation. Receipt identity includes sender +tenant, sender subject, sender peer, and relay message ID. `PeerRelayReceiptLimits` bounds their count, encoded bytes, +retention, pruning batch, rate, and interval. Replace or clone restore fences old relay state and reconciles receipt +and outbox usage for the installed incarnation. + +Relay custody rows and client relay outbox or receipt rows are not part of the canonical backup archive. + +## Capacity and overload + +`PeerRelayLimits` validates active and retained count and byte quotas for sender peers, recipient peers, recipient +subjects, tenants, and the shard. Admission reserves immutable quota entitlements in the same SQLite transaction as +the message. Accepted work is not evicted to admit new work. + +The same limits bound relay connections, raw chunks, declared frames, incomplete frames, shared payload bytes, byte +reservation waiters, per subject sessions, Open and Push concurrency and rates, terminal response work, channel +queues, relay workers, maximum delivery attempts, SQL work classes, maintenance batches and rates, and shutdown claim +release. + +`PeerRelayOutboxLimits` independently bounds pending sender rows and encoded bytes per remote and per replica. It also +bounds retry horizon, pruning batch, pruning rate, and maintenance interval. + +Capacity failures are explicit. They do not imply custody. An application may retry +`RequestCapacityExceeded` only with its own bounded backoff and the same stable sender admission. + +## Security + +The application remains responsible for credentials, identity issuance, TLS, endpoint routing, and authorization +policy. `PeerAuthenticator` derives a fresh request principal for every RPC operation. +`PeerRelayAuthorization.authorize` receives the direction, authenticated principal, exact remote subject and peer, +and selected whole documents. The server checks send authorization before custody and receive authorization before +disclosure. + +### Unsafe Automerge resource trust + +[Automerge `3.3.2`](https://github.com/automerge/automerge/releases/tag/js%2Fautomerge-3.3.2) does not expose an +allocation bounded semantic decode API. A small authenticated byte string can therefore cause allocation beyond its +wire size when Automerge parses it. Authentication proves a principal. Ordinary relay authorization grants access to +an endpoint and document set. Neither fact proves that the producer's Automerge bytes are safe to allocate and +decode. This contract is pinned to the installed `3.3.2` release. It makes no claim about a later Automerge release. + +`PeerRelayAuthorization` consequently has a second, explicit policy callback: +`authorizeUnsafeUnboundedAutomerge3Decode`. It is default deny when the application supplies +`denyUnsafeUnboundedAutomerge3Decode`. Relay send admission and recipient delivery require a grant from this callback +in addition to ordinary authentication and document authorization. + +The unsafe grant is exact. It carries the literal +`risk: "Automerge3.3.2DecodeIsNotAllocationBounded"`, the authenticated local principal, the resolved remote principal +in the same tenant, `Send` or `Receive`, the complete unique document set, a finite future `validUntil`, and an +`invalidated` Effect. The constructor rejects substitutions, missing or extra documents, duplicate document +identities, malformed grants, and expired grants. A grant for one direction, endpoint, principal, or document set +cannot authorize another. + +After fresh ordinary and unsafe grants, the bounded server operation and policy expiry or revocation contend on a +local gate. If revocation wins before operation admission, no relay SQL mutation or payload emission occurs. If the +operation wins, it is in flight. It may finish its durable commit or delivery and returns its real result. Revocation +drains that bounded in flight operation and prevents later operations. It does not retroactively cancel the winner. +SQLite and an external authentication or authorization authority do not share an atomic transaction. + +Grant this capability only when the application controls the producer and treats its bytes as resource trusted. Do +not grant it merely because the producer is authenticated, is allowed to edit the documents, is another tenant +member, or supplied a valid envelope. + +> **Warning:** The following callback accepts the known allocation risk. Use this shape only behind an application +> policy that has independently established producer byte resource trust. + +```ts +import * as PeerRelayAuthorization from "@lucas-barake/effect-local-rpc/PeerRelayAuthorization" +import { Clock, Effect } from "effect" + +const ResourceTrustedRelayAuthorizationLive = PeerRelayAuthorization.layer( + authorizeRelay, + (request) => + Effect.gen(function*() { + yield* assertResourceTrustedProducer(request) + const now = yield* Clock.currentTimeMillis + return { + _tag: "UnsafeUnboundedAutomerge3DecodeGrant", + risk: PeerRelayAuthorization.unsafeUnboundedAutomerge3DecodeRisk, + principal: request.principal, + remote: { + tenantId: request.principal.tenantId, + subjectId: request.remote.subjectId, + peerId: request.remote.peerId + }, + direction: request.direction, + documents: request.documents, + validUntil: now + unsafeGrantLeaseMillis, + invalidated: unsafeGrantInvalidated + } + }) +) +``` + +Relay infrastructure remains opaque to Automerge semantics. Before custody or forwarding it validates bounded frame +and envelope size, Effect Schema shape, endpoint and document routing, message hash, canonical outer digest, and +writer provenance shape. It does not interpret Automerge changes, dependency graphs, or operations. The relay enabled +`PeerSync` receive path performs the one semantic Automerge decode and applies the existing change, dependency, and +operation limits. + +Direct protocol version `2` retains its preexisting authorized peer allocation risk because it has no separate unsafe +resource trust callback. Only connect direct peers whose produced Automerge bytes the application resource trusts. +When Automerge exposes an allocation bounded decode API, Effect Local should use it and remove this unsafe grant +instead of normalizing the exception as permanent policy. + +The relay outer digest binds the complete versioned envelope. It provides integrity and conflict detection. It is not +encryption or a signature. The relay stores and can read the payload supplied by the client. Applications that need +relay blind content must encrypt before Effect Local and must bind their own authenticated encryption context to the +relay identities. + +Wire and SQL values are Schema decoded. Payload, credential, claim token, and full principals are not added to relay +span attributes. Public wire errors are fieldless so authorization and storage failures do not disclose message +existence. + +## Deployment boundary and limitations + +One SQLite database is the custody authority for one relay shard. The application must route every write and claim +for that shard to one process and one durable volume. A second process must not open the same database through a +network file system. There is no built in leader election, quorum, replicated log, automatic failover, shard +rebalance, cross region routing, or split brain recovery. + +When that SQLite authority is unavailable, new custody stops and delivery pauses. Durable pending rows remain +discoverable after restart on the same volume. Claims recover after their deadlines. The server also runs bounded +compensation and maintenance so a missed process notification does not permanently strand committed work. + +This package supplies a bounded single authority relay building block. It does not claim WhatsApp scale, global +availability, managed operations, peer discovery, end to end encryption, account management, or tenant routing. + +## Observability + +The relay exposes Effect services and spans. It does not install a metrics exporter. + +| Source | Exposed values | +| ------------------------------ | ----------------------------------------------------------------------------------- | +| `PeerRelayServerRuntime.usage` | `accepting`, `sessions`, `subjects`, `activeClaims`, `queuedChannels` | +| `PeerRelayIngress.usage` | `connections`, `reservedBytes`, `byteReservationWaiters` | +| `PeerRelayStore.usage` | `activeCount`, `activeBytes`, `retainedCount`, `retainedBytes` for a selected scope | +| `PeerRelayOutbox.usage` | Remote and replica `messageCount` and `encodedBytes` | + +Store operations use spans named `PeerRelayStore.admit`, `claim`, `loadClaimedPayload`, `acknowledge`, `reject`, +`release`, `recover`, `expire`, `repair`, `reconcile`, `collect`, and `usage`. Client relay spans are +`effect_local_rpc.adapter.relay_open` with `rpc.selected_documents` and +`effect_local_rpc.adapter.relay_push` with `rpc.payload_bytes`. diff --git a/docs/sync.md b/docs/sync.md index bcf0f7c..55c4a4c 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -38,13 +38,42 @@ Effect Local does not ship a backend or a server transport. An application suppl identity, authentication, authorization, encryption, and routing. The test package supplies a bounded duplex transport with deterministic drop, delay, duplicate, reorder, partition, and reconnect behavior. -The current protocol proves direct peer exchange while both peers participate. It does not yet implement asynchronous -store and forward collaboration. That requires relay storage, peer discovery, durable head receipts, retry policy, and -protocol handling for a peer that reconnects long after the sender has gone away. Setting a transport capability flag -does not supply those semantics. +Direct Effect RPC uses protocol version `2` and remains a live exchange while both peers participate. Its application +owned listener, protocol, serializer, and `PeerRpcServer.layerHandlers` composition remain unchanged. -Authentication, end to end encryption, subtree sync, and relay deployment are outside the first release. The current -engine is suitable as a local first persistence and client sync core, not a complete collaboration product by itself. +Optional store and forward uses protocol version `3` on a distinct bounded listener. The sender first writes a stable +envelope to its local SQL outbox. Relay acceptance means that the single SQLite relay authority committed custody. +The relay then delivers the oldest eligible message for one exact directed channel when the recipient reconnects. +The recipient acknowledges only after its production sync workflow and sender scoped receipt commit. Lost +acknowledgements and expired claims can duplicate delivery, so the guarantee is at least once. + +Ordering is FIFO within the tenant, sender subject, sender peer, sender replica incarnation, recipient subject, and +recipient peer channel. Other channels can progress independently. Message expiry, terminal retention, receipt +retention, sender retry horizon, storage quotas, connection bounds, worker bounds, maximum delivery attempts, and +maintenance rates are explicit configuration. The default maximum is `16` delivery attempts. Reaching it durably +dead letters the message and erases its payload. + +Authentication, authorization, TLS, endpoint routing, and optional payload encryption remain application +responsibilities. The relay outer digest provides integrity and conflict detection. It does not encrypt the payload. +Automerge `3.3.2` does not expose allocation bounded semantic decode. Relay infrastructure therefore validates only +the bounded opaque frame, envelope Schema, routing, hashes, outer digest, and provenance shape. Relay enabled +`PeerSync` performs the one semantic decode. Relay admission and delivery also require an explicit +`UnsafeUnboundedAutomerge3DecodeGrant` that exactly binds principal, remote, direction, documents, finite lease, and +revocation. Ordinary authentication and document authorization do not imply this resource trust. Default deny with +`PeerRelayAuthorization.denyUnsafeUnboundedAutomerge3Decode`. + +After fresh ordinary and unsafe grants, revocation and the bounded relay operation contend on a local gate. A +revocation admitted first prevents SQL mutation or payload emission. An operation admitted first may finish and +returns its real result. Revocation drains that in flight operation and prevents later work. SQLite and the external +policy authority do not share an atomic transaction. + +Direct version `2` retains the preexisting authorized peer allocation risk because it has no separate unsafe grant. +Applications must resource trust the Automerge bytes produced by direct peers. A future allocation bounded Automerge +decode API should remove the relay unsafe grant. + +The relay is a single SQLite custody authority per shard. It does not provide quorum replication, automatic failover, +cross region routing, or peer discovery. See [Store and forward](store-and-forward.md) for the exact contract and +composition. Automerge provides merge infrastructure, not collaboration UX. The public snapshot exposes the decoded value and head frontier. History traversal, conflict inspection, review, sharing, and conflict resolution interfaces remain diff --git a/packages/local-rpc/README.md b/packages/local-rpc/README.md index 5edd288..eb095d8 100644 --- a/packages/local-rpc/README.md +++ b/packages/local-rpc/README.md @@ -1,5 +1,18 @@ # @lucas-barake/effect-local-rpc Platform neutral Effect RPC building blocks for authenticated peer synchronization with an Effect Local SQL replica. +Direct protocol version `2` remains application owned and live only. Optional protocol version `3` adds a separate +bounded listener, authenticated relay authorization, and single authority SQLite store and forward custody. +Automerge `3.3.2` has no allocation bounded semantic decode API, so safe relay composition explicitly passes +`denyUnsafeUnboundedAutomerge3Decode`. A separate exact unsafe resource trust grant is required to proceed. Ordinary +authentication and document authorization do not provide that grant. +Delivery attempts are durable and bounded. `PeerRelayLimits.maximumDeliveryAttempts` defaults to `16`, then dead +letters the message and erases its payload. +Policy revocation and each bounded relay operation contend on a local gate. Revocation admitted first prevents SQL +mutation or payload emission. An operation admitted first may finish. Revocation drains it and blocks later work. +External policy and SQLite do not share an atomic transaction. -See the [Effect Local documentation](https://github.com/lucas-barake/effect-local#readme) for synchronization architecture, deployment boundaries, and API reference. +See the [Effect Local documentation](https://github.com/lucas-barake/effect-local#readme) for synchronization +architecture, deployment boundaries, and API reference. The optional relay contract, composition, limits, security, +and failure model are documented in +[Store and forward](https://github.com/lucas-barake/effect-local/blob/main/docs/store-and-forward.md). diff --git a/packages/local-rpc/src/PeerRelayAuthorization.ts b/packages/local-rpc/src/PeerRelayAuthorization.ts index 3b836de..3bb42e9 100644 --- a/packages/local-rpc/src/PeerRelayAuthorization.ts +++ b/packages/local-rpc/src/PeerRelayAuthorization.ts @@ -1,5 +1,6 @@ import type * as PeerSession from "@lucas-barake/effect-local-sql/PeerSession" import * as Identity from "@lucas-barake/effect-local/Identity" +import * as Clock from "effect/Clock" import * as Context from "effect/Context" import * as Effect from "effect/Effect" import * as Layer from "effect/Layer" @@ -17,6 +18,42 @@ export const RemotePeer = Schema.Struct({ }) export type RemotePeer = typeof RemotePeer.Type +const RequestedDocument = Schema.Struct({ + documentType: Schema.NonEmptyString, + documentId: Identity.DocumentId +}) +const RequestedDocuments = Schema.Array(RequestedDocument).check(Schema.isMinLength(1)) + +export const unsafeUnboundedAutomerge3DecodeRisk = "Automerge3.3.2DecodeIsNotAllocationBounded" as const + +export const UnsafeUnboundedAutomerge3DecodeRequest = Schema.Struct({ + risk: Schema.Literal(unsafeUnboundedAutomerge3DecodeRisk), + direction: Direction, + principal: PeerAuthentication.PeerPrincipal, + remote: RemotePeer, + documents: RequestedDocuments +}) +export type UnsafeUnboundedAutomerge3DecodeRequest = typeof UnsafeUnboundedAutomerge3DecodeRequest.Type + +const Invalidation = Schema.declare>( + (value): value is Effect.Effect => Effect.isEffect(value), + { expected: "Effect invalidation" } +) + +export const UnsafeUnboundedAutomerge3DecodeGrant = Schema.TaggedStruct( + "UnsafeUnboundedAutomerge3DecodeGrant", + { + risk: Schema.Literal(unsafeUnboundedAutomerge3DecodeRisk), + principal: PeerAuthentication.PeerPrincipal, + remote: PeerAuthentication.PeerPrincipal, + direction: Direction, + documents: RequestedDocuments, + validUntil: Schema.Number.check(Schema.isFinite()), + invalidated: Invalidation + } +) +export type UnsafeUnboundedAutomerge3DecodeGrant = typeof UnsafeUnboundedAutomerge3DecodeGrant.Type + export interface Request { readonly direction: Direction readonly principal: PeerAuthentication.PeerPrincipal @@ -38,8 +75,16 @@ export type Authorize = ( request: Request ) => Effect.Effect +export type AuthorizeUnsafeUnboundedAutomerge3Decode = ( + request: UnsafeUnboundedAutomerge3DecodeRequest +) => Effect.Effect< + UnsafeUnboundedAutomerge3DecodeGrant, + PeerRpcError.AccessDenied | PeerRpcError.ServerUnavailable +> + export class PeerRelayAuthorization extends Context.Service()("@lucas-barake/effect-local-rpc/PeerRelayAuthorization") {} const accessDeniedOnSchemaError = (effect: Effect.Effect) => @@ -80,11 +125,84 @@ const validateResult = (request: Request, result: Result) => Effect.as(result) ) -export const layer = (authorize: Authorize) => +const documentKey = ( + document: UnsafeUnboundedAutomerge3DecodeRequest["documents"][number] +) => JSON.stringify([document.documentType, document.documentId]) + +const validateUnsafeDocuments = ( + documents: UnsafeUnboundedAutomerge3DecodeRequest["documents"] +) => { + const keys = new Set(documents.map(documentKey)) + const documentIds = new Set(documents.map((document) => document.documentId)) + return keys.size === documents.length && documentIds.size === documents.length + ? Effect.succeed(keys) + : Effect.fail(new PeerRpcError.AccessDenied()) +} + +const canonicalDocuments = ( + documents: UnsafeUnboundedAutomerge3DecodeRequest["documents"] +) => + documents.toSorted((left, right) => { + const leftKey = documentKey(left) + const rightKey = documentKey(right) + return leftKey < rightKey ? -1 : leftKey > rightKey ? 1 : 0 + }) + +const validateUnsafeRequest = (request: UnsafeUnboundedAutomerge3DecodeRequest) => + accessDeniedOnSchemaError(UnsafeUnboundedAutomerge3DecodeRequest.makeEffect(request)).pipe( + Effect.flatMap((validated) => validateUnsafeDocuments(validated.documents).pipe(Effect.as(validated))) + ) + +const validateUnsafeGrant = ( + request: UnsafeUnboundedAutomerge3DecodeRequest, + result: UnsafeUnboundedAutomerge3DecodeGrant +) => + accessDeniedOnSchemaError( + Schema.decodeUnknownEffect(UnsafeUnboundedAutomerge3DecodeGrant)(result) + ).pipe( + Effect.flatMap((grant) => + Effect.gen(function*() { + const requested = yield* validateUnsafeDocuments(request.documents) + const granted = yield* validateUnsafeDocuments(grant.documents) + const now = yield* Clock.currentTimeMillis + if ( + grant.principal.tenantId !== request.principal.tenantId || + grant.principal.subjectId !== request.principal.subjectId || + grant.principal.peerId !== request.principal.peerId || + grant.remote.tenantId !== request.principal.tenantId || + grant.remote.subjectId !== request.remote.subjectId || + grant.remote.peerId !== request.remote.peerId || + grant.direction !== request.direction || + granted.size !== requested.size || + [...granted].some((entry) => !requested.has(entry)) || + grant.validUntil <= now + ) { + return yield* new PeerRpcError.AccessDenied() + } + return { + ...grant, + documents: canonicalDocuments(grant.documents) + } + }) + ) + ) + +export const denyUnsafeUnboundedAutomerge3Decode: AuthorizeUnsafeUnboundedAutomerge3Decode = () => + Effect.fail(new PeerRpcError.AccessDenied()) + +export const layer = ( + authorize: Authorize, + authorizeUnsafeUnboundedAutomerge3Decode: AuthorizeUnsafeUnboundedAutomerge3Decode +) => Layer.succeed(PeerRelayAuthorization)({ authorize: (request) => validateRequestShape(request).pipe( Effect.flatMap(() => authorize(request)), Effect.flatMap((result) => validateResult(request, result)) + ), + authorizeUnsafeUnboundedAutomerge3Decode: (request) => + validateUnsafeRequest(request).pipe( + Effect.flatMap((validated) => authorizeUnsafeUnboundedAutomerge3Decode(validated)), + Effect.flatMap((grant) => validateUnsafeGrant(request, grant)) ) }) diff --git a/packages/local-rpc/src/PeerRelayIngress.ts b/packages/local-rpc/src/PeerRelayIngress.ts index db019ee..d82daf3 100644 --- a/packages/local-rpc/src/PeerRelayIngress.ts +++ b/packages/local-rpc/src/PeerRelayIngress.ts @@ -45,6 +45,7 @@ const CurrentRequestKey = Context.Reference( ) interface InternalReservation extends Reservation { + readonly releaseUnsafe: () => void readonly shrinkTo: (bytes: number) => Effect.Effect readonly transfer: (key: RequestKey) => Effect.Effect } @@ -65,23 +66,19 @@ const makeByteBudget = ( let waiterId = 0 const waiters = new Map() - const releaseBytes = (bytes: number) => - Effect.sync(() => { - reservedBytes -= bytes - drain() - }) - const makeReservation = (bytes: number): InternalReservation => { let reserved = bytes let released = false let transferred = false - const release = Effect.suspend(() => { - if (released) return Effect.void + const releaseUnsafe = () => { + if (released) return released = true - return releaseBytes(reserved) - }) + reservedBytes -= reserved + drain() + } + const release = Effect.sync(releaseUnsafe) const shrinkTo = (target: number) => - Effect.suspend(() => { + Effect.sync(() => { if ( released || transferred || @@ -89,12 +86,13 @@ const makeByteBudget = ( target <= 0 || target > reserved ) { - return Effect.die(new Error("Invalid relay byte reservation shrink")) + throw new Error("Invalid relay byte reservation shrink") } - if (target === reserved) return Effect.void + if (target === reserved) return const releasedBytes = reserved - target reserved = target - return releaseBytes(releasedBytes) + reservedBytes -= releasedBytes + drain() }) const transfer = (key: RequestKey) => Effect.suspend(() => { @@ -118,6 +116,7 @@ const makeByteBudget = ( return reserved }, release, + releaseUnsafe, shrinkTo, transfer, transferToCurrentRequest: Effect.flatMap(CurrentRequestKey, (key) => @@ -155,48 +154,86 @@ const makeByteBudget = ( } }) + const acquire = ( + bytes: number + ): Effect.Effect< + InternalReservation, + PeerRpcError.RequestLimitExceeded | PeerRpcError.RequestCapacityExceeded + > => + Effect.suspend< + InternalReservation, + PeerRpcError.RequestLimitExceeded | PeerRpcError.RequestCapacityExceeded, + never + >(() => { + if (!Number.isSafeInteger(bytes) || bytes <= 0 || bytes > capacity) { + return Effect.fail(new PeerRpcError.RequestLimitExceeded()) + } + if (waiters.size === 0 && reservedBytes + bytes <= capacity) { + reservedBytes += bytes + return Effect.succeed(makeReservation(bytes)) + } + if (waiters.size >= maximumWaiters) { + return Effect.fail(new PeerRpcError.RequestCapacityExceeded()) + } + const id = waiterId++ + const waiter: ByteWaiter = { + bytes, + deferred: Deferred.makeUnsafe(), + state: "Waiting" + } + waiters.set(id, waiter) + return Effect.interruptible(Deferred.await(waiter.deferred)).pipe( + Effect.onInterrupt(() => cancelWaiter(id, waiter)), + Effect.tap(() => + Effect.sync(() => { + waiter.state = "Delivered" + }) + ) + ) + }) + const reserve = ( bytes: number ): Effect.Effect< InternalReservation, PeerRpcError.RequestLimitExceeded | PeerRpcError.RequestCapacityExceeded > => - Effect.uninterruptibleMask((restore) => { - const acquire = (): Effect.Effect< - InternalReservation, - PeerRpcError.RequestLimitExceeded | PeerRpcError.RequestCapacityExceeded - > => { - if (!Number.isSafeInteger(bytes) || bytes <= 0 || bytes > capacity) { - return Effect.fail(new PeerRpcError.RequestLimitExceeded()) - } - if (waiters.size === 0 && reservedBytes + bytes <= capacity) { - reservedBytes += bytes - return Effect.succeed(makeReservation(bytes)) - } - if (waiters.size >= maximumWaiters) { - return Effect.fail(new PeerRpcError.RequestCapacityExceeded()) - } - const id = waiterId++ - const waiter: ByteWaiter = { - bytes, - deferred: Deferred.makeUnsafe(), - state: "Waiting" - } - waiters.set(id, waiter) - return restore(Deferred.await(waiter.deferred)).pipe( - Effect.onInterrupt(() => cancelWaiter(id, waiter)), - Effect.tap(() => + Effect.suspend(() => { + let acquired: InternalReservation | undefined + return Effect.uninterruptibleMask(() => + acquire(bytes).pipe( + Effect.tap((reservation) => Effect.sync(() => { - waiter.state = "Delivered" + acquired = reservation }) ) ) - } - return Effect.suspend(acquire) + ).pipe( + Effect.onInterrupt(() => acquired?.release ?? Effect.void) + ) }) + const use = ( + bytes: number, + f: (reservation: InternalReservation) => Effect.Effect + ): Effect.Effect< + A, + E | PeerRpcError.RequestLimitExceeded | PeerRpcError.RequestCapacityExceeded, + R + > => + Effect.uninterruptibleMask((restore) => + acquire(bytes).pipe( + Effect.flatMap((reservation) => + restore(f(reservation)).pipe( + Effect.ensuring(reservation.release) + ) + ) + ) + ) + return { reserve, + use, usage: () => ({ reservedBytes, byteReservationWaiters: waiters.size @@ -401,18 +438,25 @@ const removeReservation = ( return reservation } +const removeAndReleaseReservation = ( + reservations: Map>, + key: RequestKey +) => + Effect.sync(() => { + removeReservation(reservations, key)?.releaseUnsafe() + }) + const releaseClientReservations = ( reservations: Map>, clientId: number ) => - Effect.suspend(() => { + Effect.sync(() => { const client = reservations.get(clientId) - if (client === undefined) return Effect.void + if (client === undefined) return reservations.delete(clientId) - return Effect.forEach(client.values(), (reservation) => reservation.release, { - concurrency: 1, - discard: true - }) + for (const reservation of client.values()) { + reservation.releaseUnsafe() + } }) const makeServerProtocol = ( @@ -438,7 +482,7 @@ const makeServerProtocol = ( ) let connections = 0 let nextClientId = 0 - let protocolReady = false + let protocolRunners = 0 let writeRequest!: ( clientId: number, message: RpcMessage.FromClientEncoded @@ -460,64 +504,71 @@ const makeServerProtocol = ( return Effect.succeed({ disconnects, send: (clientId, response) => - Effect.suspend(() => { - const key = "requestId" in response - ? { clientId, requestId: response.requestId } - : undefined - const transferred = key === undefined - ? undefined - : removeReservation(outbound, key) - return Effect.gen(function*() { - const client = clients.get(clientId) - if (client === undefined) { - return - } - - const maximumAdditional = transferred === undefined - ? limits.maximumDeclaredFrameBytes - : Math.max(0, limits.maximumDeclaredFrameBytes - transferred.bytes) - const extra = maximumAdditional === 0 - ? undefined - : yield* budget.reserve(maximumAdditional).pipe( - Effect.match({ - onFailure: () => undefined, - onSuccess: (reservation) => reservation - }) + Effect.uninterruptibleMask((restore) => + Effect.flatMap(CurrentRequestKey, (currentRequest) => + Effect.sync(() => { + const key = "requestId" in response + ? { clientId, requestId: response.requestId } + : response._tag === "Defect" + ? currentRequest + : undefined + const transferred = key === undefined + ? undefined + : removeReservation(outbound, key) + return { key, transferred } + })).pipe( + Effect.flatMap(({ key, transferred }) => + restore(Effect.gen(function*() { + const client = clients.get(clientId) + if (client === undefined) { + return + } + + const maximumAdditional = transferred === undefined + ? limits.maximumDeclaredFrameBytes + : Math.max(0, limits.maximumDeclaredFrameBytes - transferred.bytes) + const sendWithExtra = (extra: InternalReservation | undefined) => + Effect.gen(function*() { + let encoded: ReturnType + try { + encoded = encodeFrame(response, limits.maximumDeclaredFrameBytes) + } catch { + yield* client.write(new Socket.CloseEvent(1009)).pipe(Effect.ignore) + return + } + + const additional = transferred === undefined + ? encoded.bodyBytes + : Math.max(0, encoded.bodyBytes - transferred.bytes) + if (extra !== undefined && additional > 0) { + yield* extra.shrinkTo(additional) + } else if (extra !== undefined) { + yield* extra.release + } + + yield* client.write(encoded.frame).pipe(Effect.ignore) + + if ( + (response._tag === "Exit" || response._tag === "Defect") && + key !== undefined + ) { + yield* removeAndReleaseReservation(inbound, key) + } + }) + if (maximumAdditional === 0) { + yield* sendWithExtra(undefined) + } else { + yield* budget.use( + maximumAdditional, + sendWithExtra + ).pipe( + Effect.catch(() => client.write(new Socket.CloseEvent(1013)).pipe(Effect.ignore)) + ) + } + })).pipe(Effect.ensuring(transferred?.release ?? Effect.void)) ) - if (maximumAdditional > 0 && extra === undefined) { - yield* client.write(new Socket.CloseEvent(1013)).pipe(Effect.ignore) - return - } - - yield* Effect.gen(function*() { - let encoded: ReturnType - try { - encoded = encodeFrame(response, limits.maximumDeclaredFrameBytes) - } catch { - yield* client.write(new Socket.CloseEvent(1009)).pipe(Effect.ignore) - return - } - - const additional = transferred === undefined - ? encoded.bodyBytes - : Math.max(0, encoded.bodyBytes - transferred.bytes) - if (extra !== undefined && additional > 0) { - yield* extra.shrinkTo(additional) - } else if (extra !== undefined) { - yield* extra.release - } - - yield* client.write(encoded.frame).pipe(Effect.ignore) - - if (response._tag === "Exit" && key !== undefined) { - const retained = removeReservation(inbound, key) - if (retained !== undefined) yield* retained.release - } else if (response._tag === "Defect") { - yield* releaseClientReservations(inbound, clientId) - } - }).pipe(Effect.ensuring(extra?.release ?? Effect.void)) - }).pipe(Effect.ensuring(transferred?.release ?? Effect.void)) - }), + ) + ), end: (clientId) => Effect.gen(function*() { const client = clients.get(clientId) @@ -536,22 +587,25 @@ const makeServerProtocol = ( const protocol: RpcServer.Protocol["Service"] = { ...baseProtocol, run: (handler) => - Effect.suspend(() => { - protocolReady = true - return baseProtocol.run(handler).pipe( - Effect.onExit(() => - Effect.sync(() => { - protocolReady = false - }) - ) - ) - }) + Effect.acquireUseRelease( + Effect.sync(() => { + protocolRunners++ + }), + () => baseProtocol.run(handler), + () => + Effect.sync(() => { + protocolRunners-- + }) + ) } const onSocket = (socket: Socket.Socket) => Effect.scoped( Effect.suspend(() => { - if (!protocolReady || connections >= limits.maxRelayConnections) { + if ( + protocolRunners === 0 || + connections + Queue.sizeUnsafe(disconnects) >= limits.maxRelayConnections + ) { return Effect.gen(function*() { const write = yield* socket.writer yield* socket.runRaw( @@ -580,6 +634,16 @@ const makeServerProtocol = ( ) { return Effect.andThen(reservation.release, Effect.fail(invalidFrame())) } + const tag = value._tag + if ( + tag !== "Request" && + tag !== "Ack" && + tag !== "Interrupt" && + tag !== "Ping" && + tag !== "Eof" + ) { + return Effect.andThen(reservation.release, Effect.fail(invalidFrame())) + } const message = value as RpcMessage.FromClientEncoded if (message._tag !== "Request") { return Effect.provideService( @@ -608,8 +672,7 @@ const makeServerProtocol = ( ).pipe( Effect.onExit((exit) => { if (exit._tag === "Success") return Effect.void - removeReservation(inbound, key) - return reservation.release + return removeAndReleaseReservation(inbound, key) }) ) } @@ -741,23 +804,29 @@ export const makeProtocolSocket = Effect.gen(function*() { send: (clientId: number, request: RpcMessage.FromClientEncoded) => Effect.gen(function*() { if (currentError !== undefined) return yield* Effect.fail(currentError) - const reservation = yield* budget.reserve(limits.maximumDeclaredFrameBytes).pipe( - Effect.mapError((cause) => toClientError("Relay request capacity is exhausted", cause)) - ) - yield* Effect.gen(function*() { - let encoded: ReturnType - try { - encoded = encodeFrame(request, limits.maximumDeclaredFrameBytes) - } catch (cause) { - return yield* Effect.fail(toClientError("Relay request frame is invalid", cause)) - } - yield* reservation.shrinkTo(encoded.bodyBytes) - if (request._tag === "Request") requestClients.set(request.id, clientId) - yield* write(encoded.frame).pipe( - Effect.mapError((cause) => toClientError("Relay socket write failed", cause)) - ) - }).pipe( - Effect.ensuring(reservation.release) + yield* budget.use( + limits.maximumDeclaredFrameBytes, + (reservation) => + Effect.gen(function*() { + let encoded: ReturnType + try { + encoded = encodeFrame(request, limits.maximumDeclaredFrameBytes) + } catch (cause) { + return yield* Effect.fail(toClientError("Relay request frame is invalid", cause)) + } + yield* reservation.shrinkTo(encoded.bodyBytes) + if (request._tag === "Request") requestClients.set(request.id, clientId) + yield* write(encoded.frame).pipe( + Effect.mapError((cause) => toClientError("Relay socket write failed", cause)) + ) + }) + ).pipe( + Effect.catchTags({ + RequestLimitExceeded: (cause) => + Effect.fail(toClientError("Relay request capacity is exhausted", cause)), + RequestCapacityExceeded: (cause) => + Effect.fail(toClientError("Relay request capacity is exhausted", cause)) + }) ) }), supportsAck: true, diff --git a/packages/local-rpc/src/PeerRelayLimits.ts b/packages/local-rpc/src/PeerRelayLimits.ts index 6bcba92..e8a8127 100644 --- a/packages/local-rpc/src/PeerRelayLimits.ts +++ b/packages/local-rpc/src/PeerRelayLimits.ts @@ -45,6 +45,7 @@ export const Values = Schema.Struct({ claimSafetyMarginMillis: PositiveInt, retryBaseDelayMillis: PositiveInt, retryMaximumDelayMillis: PositiveInt, + maximumDeliveryAttempts: PositiveInt, sqliteLockRetryBaseDelayMillis: PositiveInt, sqliteLockRetryMaximumDelayMillis: PositiveInt, sqliteLockRetryMaxAttempts: PositiveInt, @@ -156,6 +157,7 @@ export const defaults: Values = Values.make({ claimSafetyMarginMillis: 5_000, retryBaseDelayMillis: 250, retryMaximumDelayMillis: 30_000, + maximumDeliveryAttempts: 16, sqliteLockRetryBaseDelayMillis: 10, sqliteLockRetryMaximumDelayMillis: 250, sqliteLockRetryMaxAttempts: 5, diff --git a/packages/local-rpc/src/PeerRelayStore.ts b/packages/local-rpc/src/PeerRelayStore.ts index 3f881d6..27d1c7b 100644 --- a/packages/local-rpc/src/PeerRelayStore.ts +++ b/packages/local-rpc/src/PeerRelayStore.ts @@ -1,12 +1,11 @@ import * as Identity from "@lucas-barake/effect-local/Identity" import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" -import type * as Cause from "effect/Cause" import * as Context from "effect/Context" import * as Crypto from "effect/Crypto" import * as Effect from "effect/Effect" +import * as Exit from "effect/Exit" import * as Layer from "effect/Layer" import * as Option from "effect/Option" -import type * as PlatformError from "effect/PlatformError" import * as Random from "effect/Random" import * as Schema from "effect/Schema" import type * as Migrator from "effect/unstable/sql/Migrator" @@ -19,6 +18,8 @@ import { make as makeImmediateTransaction, type NestedPeerRelayTransactionError } from "./internal/peerRelaySqliteTransaction.js" +import { mapStoreErrors } from "./internal/peerRelayStoreErrors.js" +import * as PeerRpcObservability from "./internal/peerRpcObservability.js" import * as PeerRelayLimits from "./PeerRelayLimits.js" import * as PeerRelayRpc from "./PeerRelayRpc.js" @@ -200,11 +201,6 @@ interface UsageScope { readonly retainedBytesLimit: number } -const storageUnavailable = (cause: unknown) => - new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageUnavailable({ cause }) - }) - const storageCorrupt = (cause: unknown) => new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageCorrupt({ cause }) @@ -220,20 +216,51 @@ const quotaExceeded = (resource: string, limit: number) => reason: new ReplicaError.QuotaExceeded({ resource, limit }) }) -type BoundaryError = - | StoreError - | SqlError.SqlError - | Schema.SchemaError - | PlatformError.PlatformError - | Cause.NoSuchElementError - -const mapStorageErrors = (effect: Effect.Effect) => - effect.pipe( - Effect.catchTag("SqlError", (cause) => Effect.fail(storageUnavailable(cause))), - Effect.catchTag("SchemaError", (cause) => Effect.fail(storageCorrupt(cause))), - Effect.catchTag("PlatformError", (cause) => Effect.fail(storageUnavailable(cause))), - Effect.catchTag("NoSuchElementError", (cause) => Effect.fail(storageCorrupt(cause))) - ) +const relayFailureResult = (exit: Exit.Exit) => { + const error = PeerRpcObservability.failure(exit) + if (error?._tag !== "ReplicaError") return "Failure" as const + switch (error.reason._tag) { + case "ProtocolMismatch": + return "ProtocolRejected" as const + case "QuotaExceeded": + return "CapacityRejected" as const + case "StorageUnavailable": + return "Unavailable" as const + default: + return "Failure" as const + } +} + +const relayQuotaDomain = ( + error: StoreError +): Option.Option => { + if (error._tag !== "ReplicaError" || error.reason._tag !== "QuotaExceeded") { + return Option.none() + } + const resource = error.reason.resource + if (resource === "relay payload bytes") return Option.some("Payload") + for ( + const domain of [ + "SenderPeer", + "RecipientPeer", + "RecipientSubject", + "Tenant", + "Shard" + ] as const + ) { + if (resource === `${domain} relay custody`) return Option.some(domain) + } + return Option.none() +} + +const recordQuotaRejection = (error: StoreError) => + Option.match(relayQuotaDomain(error), { + onNone: () => Effect.void, + onSome: (domain) => + PeerRpcObservability.recordRelayQuotaRejection(domain).pipe( + Effect.catchCause(() => Effect.void) + ) + }) const encodeKey = (...parts: ReadonlyArray) => JSON.stringify(parts) @@ -596,6 +623,13 @@ const makeService = Effect.gen(function*() { if (changed.length !== 1) { return yield* storageCorrupt(new Error("Invalid retained relay quota reservation")) } + yield* sql`DELETE FROM effect_local_relay_usage + WHERE scope_kind = ${kind} + AND scope_key = ${key} + AND active_count = 0 + AND active_bytes = 0 + AND retained_count = 0 + AND retained_bytes = 0` } yield* sql`UPDATE effect_local_relay_reservations SET retained_consumed = 1 @@ -639,10 +673,14 @@ const makeService = Effect.gen(function*() { WHERE claimed_message_id = ${messageId}` }) - const admit: Service["admit"] = (unsafeInput) => - mapStorageErrors(Effect.gen(function*() { + const admit: Service["admit"] = (unsafeInput) => { + let observedBytes: number | undefined + let observedVersion: number | undefined + const effect = mapStoreErrors(Effect.gen(function*() { const input = yield* validateInput(Admission, unsafeInput) const payloadBytes = input.payload.byteLength + observedBytes = payloadBytes + observedVersion = input.payloadVersion if ( input.messageTtlMillis > limits.messageTtlMillis || input.senderRetryHorizonMillis > limits.maximumSenderRetryHorizonMillis || @@ -731,6 +769,8 @@ const makeService = Effect.gen(function*() { tenant_id, sender_subject_id, sender_peer_id, + recipient_subject_id, + recipient_peer_id, relay_message_id, relay_peer_id, sender_connection_epoch, @@ -752,6 +792,8 @@ const makeService = Effect.gen(function*() { ${input.channel.tenantId}, ${input.channel.senderSubjectId}, ${input.channel.senderPeerId}, + ${input.channel.recipientSubjectId}, + ${input.channel.recipientPeerId}, ${input.relayMessageId}, ${input.relayPeerId}, ${input.senderConnectionEpoch}, @@ -805,7 +847,22 @@ const makeService = Effect.gen(function*() { lane: "New" }) })) - })).pipe(Effect.withSpan("PeerRelayStore.admit")) + })).pipe(Effect.tapError(recordQuotaRejection)) + return PeerRpcObservability.observeRelay({ + effect, + operation: "RelayAdmit", + direction: "Send", + facts: () => ({ + ...(observedBytes === undefined ? {} : { bytes: observedBytes }), + ...(observedVersion === undefined ? {} : { version: observedVersion }), + items: observedBytes === undefined ? 0 : 1 + }), + result: (exit) => + Exit.isSuccess(exit) + ? exit.value.status + : relayFailureResult(exit) + }) + } const findCandidate = SqlSchema.findOneOption({ Request: Schema.Struct({ @@ -843,7 +900,12 @@ const makeService = Effect.gen(function*() { FROM effect_local_relay_messages m JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id JOIN effect_local_relay_reservations r ON r.message_id = m.message_id - WHERE c.tenant_id = ${request.tenantId} + WHERE m.tenant_id = ${request.tenantId} + AND m.sender_subject_id = ${request.senderSubjectId} + AND m.sender_peer_id = ${request.senderPeerId} + AND m.recipient_subject_id = ${request.recipientSubjectId} + AND m.recipient_peer_id = ${request.recipientPeerId} + AND c.tenant_id = ${request.tenantId} AND c.recipient_subject_id = ${request.recipientSubjectId} AND c.recipient_peer_id = ${request.recipientPeerId} AND c.sender_subject_id = ${request.senderSubjectId} @@ -851,6 +913,8 @@ const makeService = Effect.gen(function*() { AND m.tenant_id = c.tenant_id AND m.sender_subject_id = c.sender_subject_id AND m.sender_peer_id = c.sender_peer_id + AND m.recipient_subject_id = c.recipient_subject_id + AND m.recipient_peer_id = c.recipient_peer_id AND c.claimed_message_id IS NULL AND m.state = 'Pending' AND m.next_eligible_at <= ${request.now} @@ -876,8 +940,9 @@ const makeService = Effect.gen(function*() { LIMIT 1` }) - const claim: Service["claim"] = (unsafeInput) => - mapStorageErrors(Effect.gen(function*() { + const claim: Service["claim"] = (unsafeInput) => { + let observedAttempt: number | undefined + const effect = mapStoreErrors(Effect.gen(function*() { const input = yield* validateInput(ClaimRequest, unsafeInput) return yield* write(Effect.gen(function*() { const now = (yield* nowQuery(sql)).now @@ -899,6 +964,7 @@ const makeService = Effect.gen(function*() { } } const candidate = candidateOption.value + observedAttempt = candidate.retryCount + 1 const documentIds = yield* parseDocuments(candidate.documentIds) const uuid = yield* crypto.randomUUIDv4 const token = PeerRelayRpc.ClaimToken.make(`clm_${uuid}`) @@ -964,10 +1030,32 @@ const makeService = Effect.gen(function*() { lane: candidate.retryCount === 0 ? "New" as const : "Retry" as const } })) - })).pipe(Effect.withSpan("PeerRelayStore.claim")) + })) + return PeerRpcObservability.observeRelay({ + effect, + operation: "RelayClaim", + direction: "Receive", + facts: (exit) => { + if (Exit.isFailure(exit) || Option.isNone(exit.value.message)) { + return { items: 0 } + } + const message = exit.value.message.value + return { + bytes: message.payloadBytes, + items: 1, + ...(observedAttempt === undefined ? {} : { attempt: observedAttempt }), + version: message.payloadVersion + } + }, + result: (exit) => + Exit.isSuccess(exit) + ? Option.isSome(exit.value.message) ? "Claimed" : "Empty" + : relayFailureResult(exit) + }) + } const loadClaimedPayload: Service["loadClaimedPayload"] = (unsafeInput) => - mapStorageErrors(Effect.gen(function*() { + mapStoreErrors(Effect.gen(function*() { const input = yield* validateInput(LoadClaimedPayloadRequest, unsafeInput) const rows = yield* SqlSchema.findAll({ Request: Schema.Void, @@ -1005,7 +1093,7 @@ const makeService = Effect.gen(function*() { return yield* protocolMismatch("active relay claim", "stale relay claim") } return rows[0]!.payload - })).pipe(Effect.withSpan("PeerRelayStore.loadClaimedPayload")) + })) const TerminalRow = Schema.Struct({ messageId: PositiveInt, @@ -1015,6 +1103,7 @@ const makeService = Effect.gen(function*() { claimToken: Schema.NullOr(PeerRelayRpc.ClaimToken), claimSessionGeneration: Schema.NullOr(NonNegativeInt), claimDeadline: Schema.NullOr(NonNegativeInt), + createdAt: NonNegativeInt, expiresAt: NonNegativeInt, channelClaimedMessageId: Schema.NullOr(PositiveInt), channelClaimToken: Schema.NullOr(PeerRelayRpc.ClaimToken), @@ -1040,6 +1129,7 @@ const makeService = Effect.gen(function*() { m.claim_token AS claimToken, m.claim_session_generation AS claimSessionGeneration, m.claim_deadline AS claimDeadline, + m.created_at AS createdAt, m.expires_at AS expiresAt, c.claimed_message_id AS channelClaimedMessageId, c.claim_token AS channelClaimToken, @@ -1116,9 +1206,10 @@ const makeService = Effect.gen(function*() { const terminalTransition = ( unsafeInput: TerminalRequest, state: "Acknowledged" | "DeadLettered", - reason: string + reason: string, + onChanged?: (latencyMillis: number) => void ): Effect.Effect => - mapStorageErrors(Effect.gen(function*() { + mapStoreErrors(Effect.gen(function*() { const input = yield* validateInput(TerminalRequest, unsafeInput) if ( input.recipient.tenantId !== input.channel.tenantId || @@ -1164,6 +1255,9 @@ const makeService = Effect.gen(function*() { sessionGeneration: input.sessionGeneration, reason }) + if (onChanged !== undefined) { + yield* Effect.sync(() => onChanged(now - row.createdAt)) + } const hint = yield* nextReady(input.channel, now) return { status: "Changed", ...hint } as const } @@ -1185,13 +1279,39 @@ const makeService = Effect.gen(function*() { })) })) - const acknowledge: Service["acknowledge"] = (input) => - terminalTransition(input, "Acknowledged", "Acknowledged").pipe( - Effect.withSpan("PeerRelayStore.acknowledge") + const acknowledge: Service["acknowledge"] = (input) => { + let latencyMillis: number | undefined + const effect = terminalTransition( + input, + "Acknowledged", + "Acknowledged", + (latency) => { + latencyMillis = latency + } ) + return PeerRpcObservability.observeRelay({ + effect, + operation: "RelayAcknowledge", + direction: "Receive", + facts: (exit) => ({ + items: Exit.isSuccess(exit) && exit.value.status !== "Stale" ? 1 : 0, + ...(Exit.isSuccess(exit) && + exit.value.status === "Changed" && + latencyMillis !== undefined + ? { latencyMillis } + : {}) + }), + result: (exit) => + Exit.isSuccess(exit) + ? exit.value.status === "Changed" + ? "Acknowledged" + : exit.value.status + : relayFailureResult(exit) + }) + } - const reject: Service["reject"] = (unsafeInput) => - mapStorageErrors(Effect.gen(function*() { + const reject: Service["reject"] = (unsafeInput) => { + const effect = mapStoreErrors(Effect.gen(function*() { const input = yield* validateInput(RejectRequest, unsafeInput) return yield* terminalTransition( TerminalRequest.make({ @@ -1205,10 +1325,25 @@ const makeService = Effect.gen(function*() { "DeadLettered", input.reason ) - })).pipe(Effect.withSpan("PeerRelayStore.reject")) + })) + return PeerRpcObservability.observeRelay({ + effect, + operation: "RelayAcknowledge", + direction: "Receive", + facts: (exit) => ({ + items: Exit.isSuccess(exit) && exit.value.status !== "Stale" ? 1 : 0 + }), + result: (exit) => + Exit.isSuccess(exit) + ? exit.value.status === "Changed" + ? "DeadLettered" + : exit.value.status + : relayFailureResult(exit) + }) + } - const release: Service["release"] = (unsafeInput) => - mapStorageErrors(Effect.gen(function*() { + const release: Service["release"] = (unsafeInput) => { + const effect = mapStoreErrors(Effect.gen(function*() { const input = yield* validateInput(ReleaseRequest, unsafeInput) return yield* write(Effect.gen(function*() { const now = (yield* nowQuery(sql)).now @@ -1237,7 +1372,6 @@ const makeService = Effect.gen(function*() { lane: "Retry" } as const } - const random = yield* Random.next const retry = yield* SqlSchema.findOne({ Request: Schema.Void, Result: Schema.Struct({ retryCount: PositiveInt }), @@ -1246,6 +1380,24 @@ const makeService = Effect.gen(function*() { FROM effect_local_relay_messages WHERE message_id = ${row.messageId}` })(undefined) + if (retry.retryCount >= limits.maximumDeliveryAttempts) { + yield* sql`UPDATE effect_local_relay_messages + SET retry_count = ${retry.retryCount} + WHERE message_id = ${row.messageId} + AND state = 'Claimed' + AND claim_token = ${input.claimToken} + AND claim_session_generation = ${input.sessionGeneration}` + yield* terminalize(row.messageId, "DeadLettered", now, { + reason: "MaximumDeliveryAttempts" + }) + return { + status: "Changed", + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" + } as const + } + const random = yield* Random.next const maximum = Math.min( limits.retryMaximumDelayMillis, limits.retryBaseDelayMillis * 2 ** Math.min(retry.retryCount - 1, 30) @@ -1279,7 +1431,20 @@ const makeService = Effect.gen(function*() { lane: "Retry" } as const })) - })).pipe(Effect.withSpan("PeerRelayStore.release")) + })) + return PeerRpcObservability.observeRelay({ + effect, + operation: "RelayRelease", + direction: "Receive", + facts: (exit) => ({ + items: Exit.isSuccess(exit) && exit.value.status === "Changed" ? 1 : 0 + }), + result: (exit) => + Exit.isSuccess(exit) + ? exit.value.status === "Changed" ? "Released" : exit.value.status + : relayFailureResult(exit) + }) + } const maintenanceResult = ( ids: ReadonlyArray, @@ -1297,37 +1462,50 @@ const makeService = Effect.gen(function*() { } const recover: Service["recover"] = (unsafeInput) => - mapStorageErrors(Effect.gen(function*() { - const input = yield* validateInput(MaintenanceRequest, unsafeInput) - const effectiveBatch = Math.min(input.batchSize, limits.claimRecoveryBatchSize) - return yield* write(Effect.gen(function*() { - const now = (yield* nowQuery(sql)).now - const rows = yield* SqlSchema.findAll({ - Request: Schema.Void, - Result: Schema.Struct({ - messageId: PositiveInt, - retryCount: NonNegativeInt - }), - execute: () => - sql`SELECT + Effect.suspend(() => { + let deadLettered = 0 + const effect = mapStoreErrors(Effect.gen(function*() { + const input = yield* validateInput(MaintenanceRequest, unsafeInput) + const effectiveBatch = Math.min(input.batchSize, limits.claimRecoveryBatchSize) + return yield* write(Effect.gen(function*() { + const now = (yield* nowQuery(sql)).now + const rows = yield* SqlSchema.findAll({ + Request: Schema.Void, + Result: Schema.Struct({ + messageId: PositiveInt, + retryCount: NonNegativeInt + }), + execute: () => + sql`SELECT message_id AS messageId, retry_count AS retryCount FROM effect_local_relay_messages - WHERE message_id > ${input.cursor ?? 0} - AND state = 'Claimed' + WHERE state = 'Claimed' AND claim_deadline <= ${now} - ORDER BY message_id + ORDER BY claim_deadline, message_id LIMIT ${effectiveBatch + 1}` - })(undefined) - for (const row of rows.slice(0, effectiveBatch)) { - const random = yield* Random.next - const retryCount = row.retryCount + 1 - const maximum = Math.min( - limits.retryMaximumDelayMillis, - limits.retryBaseDelayMillis * 2 ** Math.min(retryCount - 1, 30) - ) - const delay = Math.max(1, Math.floor(maximum / 2 + random * maximum / 2)) - yield* sql`UPDATE effect_local_relay_messages + })(undefined) + for (const row of rows.slice(0, effectiveBatch)) { + const retryCount = row.retryCount + 1 + if (retryCount >= limits.maximumDeliveryAttempts) { + deadLettered += 1 + yield* sql`UPDATE effect_local_relay_messages + SET retry_count = ${retryCount} + WHERE message_id = ${row.messageId} + AND state = 'Claimed' + AND claim_deadline <= ${now}` + yield* terminalize(row.messageId, "DeadLettered", now, { + reason: "MaximumDeliveryAttempts" + }) + continue + } + const random = yield* Random.next + const maximum = Math.min( + limits.retryMaximumDelayMillis, + limits.retryBaseDelayMillis * 2 ** Math.min(retryCount - 1, 30) + ) + const delay = Math.max(1, Math.floor(maximum / 2 + random * maximum / 2)) + yield* sql`UPDATE effect_local_relay_messages SET state = 'Pending', retry_count = ${retryCount}, next_eligible_at = ${now + delay}, @@ -1337,20 +1515,34 @@ const makeService = Effect.gen(function*() { WHERE message_id = ${row.messageId} AND state = 'Claimed' AND claim_deadline <= ${now}` - yield* sql`UPDATE effect_local_relay_channels + yield* sql`UPDATE effect_local_relay_channels SET claimed_message_id = NULL, claim_session_generation = NULL, claim_token = NULL, claim_deadline = NULL WHERE claimed_message_id = ${row.messageId} AND claim_deadline <= ${now}` - } - return maintenanceResult(rows.map((row) => row.messageId), effectiveBatch) + } + return maintenanceResult(rows.map((row) => row.messageId), effectiveBatch) + })) })) - })).pipe(Effect.withSpan("PeerRelayStore.recover")) + return PeerRpcObservability.observeRelay({ + effect, + operation: "RelayMaintenance", + direction: "Receive", + stage: "Recover", + facts: (exit) => ({ + items: Exit.isSuccess(exit) ? exit.value.processed : 0 + }), + result: (exit) => + Exit.isSuccess(exit) + ? deadLettered > 0 ? "DeadLettered" : "Released" + : relayFailureResult(exit) + }) + }) - const expire: Service["expire"] = (unsafeInput) => - mapStorageErrors(Effect.gen(function*() { + const expire: Service["expire"] = (unsafeInput) => { + const effect = mapStoreErrors(Effect.gen(function*() { const input = yield* validateInput(MaintenanceRequest, unsafeInput) const effectiveBatch = Math.min(input.batchSize, limits.expiryBatchSize) return yield* write(Effect.gen(function*() { @@ -1361,10 +1553,9 @@ const makeService = Effect.gen(function*() { execute: () => sql`SELECT message_id AS messageId FROM effect_local_relay_messages - WHERE message_id > ${input.cursor ?? 0} - AND state IN ('Pending', 'Claimed') + WHERE state IN ('Pending', 'Claimed') AND expires_at <= ${now} - ORDER BY message_id + ORDER BY expires_at, message_id LIMIT ${effectiveBatch + 1}` })(undefined) for (const row of rows.slice(0, effectiveBatch)) { @@ -1372,17 +1563,32 @@ const makeService = Effect.gen(function*() { } return maintenanceResult(rows.map((row) => row.messageId), effectiveBatch) })) - })).pipe(Effect.withSpan("PeerRelayStore.expire")) + })) + return PeerRpcObservability.observeRelay({ + effect, + operation: "RelayMaintenance", + direction: "Receive", + stage: "Expire", + facts: (exit) => ({ + items: Exit.isSuccess(exit) ? exit.value.processed : 0 + }), + result: (exit) => Exit.isSuccess(exit) ? "Expired" : relayFailureResult(exit) + }) + } const IntegrityRow = Schema.Struct({ messageId: PositiveInt, state: Schema.String, messageTenantId: Schema.NonEmptyString, - channelTenantId: Schema.NonEmptyString, + channelTenantId: Schema.NullOr(Schema.NonEmptyString), messageSenderSubjectId: Schema.NonEmptyString, - channelSenderSubjectId: Schema.NonEmptyString, + channelSenderSubjectId: Schema.NullOr(Schema.NonEmptyString), messageSenderPeerId: Schema.String, - channelSenderPeerId: Schema.String, + channelSenderPeerId: Schema.NullOr(Schema.String), + messageRecipientSubjectId: Schema.NullOr(Schema.String), + channelRecipientSubjectId: Schema.NullOr(Schema.String), + messageRecipientPeerId: Schema.NullOr(Schema.String), + channelRecipientPeerId: Schema.NullOr(Schema.String), payloadLength: NonNegativeInt, actualLength: Schema.NullOr(NonNegativeInt), reservationPresent: Schema.Literals([0, 1]), @@ -1390,8 +1596,8 @@ const makeService = Effect.gen(function*() { retainedConsumed: Schema.NullOr(Schema.Literals([0, 1])) }) - const repair: Service["repair"] = (unsafeInput) => - mapStorageErrors(Effect.gen(function*() { + const repair: Service["repair"] = (unsafeInput) => { + const effect = mapStoreErrors(Effect.gen(function*() { const input = yield* validateInput(MaintenanceRequest, unsafeInput) const effectiveBatch = Math.min(input.batchSize, limits.integrityBatchSize) return yield* write(Effect.gen(function*() { @@ -1409,13 +1615,17 @@ const makeService = Effect.gen(function*() { c.sender_subject_id AS channelSenderSubjectId, m.sender_peer_id AS messageSenderPeerId, c.sender_peer_id AS channelSenderPeerId, + m.recipient_subject_id AS messageRecipientSubjectId, + c.recipient_subject_id AS channelRecipientSubjectId, + m.recipient_peer_id AS messageRecipientPeerId, + c.recipient_peer_id AS channelRecipientPeerId, m.payload_length AS payloadLength, length(m.payload) AS actualLength, CASE WHEN r.message_id IS NULL THEN 0 ELSE 1 END AS reservationPresent, r.active_consumed AS activeConsumed, r.retained_consumed AS retainedConsumed FROM effect_local_relay_messages m - JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id + LEFT JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id LEFT JOIN effect_local_relay_reservations r ON r.message_id = m.message_id WHERE m.message_id > ${input.cursor ?? 0} ORDER BY m.message_id @@ -1443,6 +1653,8 @@ const makeService = Effect.gen(function*() { row.messageTenantId !== row.channelTenantId || row.messageSenderSubjectId !== row.channelSenderSubjectId || row.messageSenderPeerId !== row.channelSenderPeerId || + row.messageRecipientSubjectId !== row.channelRecipientSubjectId || + row.messageRecipientPeerId !== row.channelRecipientPeerId || (active && ( row.actualLength === null || row.actualLength !== row.payloadLength @@ -1459,168 +1671,133 @@ const makeService = Effect.gen(function*() { } return maintenanceResult(rows.map((row) => row.messageId), effectiveBatch) })) - })).pipe(Effect.withSpan("PeerRelayStore.repair")) + })) + return PeerRpcObservability.observeRelay({ + effect, + operation: "RelayMaintenance", + direction: "Receive", + stage: "Repair", + facts: (exit) => ({ + items: Exit.isSuccess(exit) ? exit.value.processed : 0 + }), + result: (exit) => Exit.isSuccess(exit) ? "DeadLettered" : relayFailureResult(exit) + }) + } - const reconcile: Service["reconcile"] = (unsafeInput) => - mapStorageErrors(Effect.gen(function*() { + const reconcile: Service["reconcile"] = (unsafeInput) => { + const effect = mapStoreErrors(Effect.gen(function*() { const input = yield* validateInput(MaintenanceRequest, unsafeInput) const effectiveBatch = Math.min(input.batchSize, limits.reconciliationBatchSize) return yield* write(Effect.gen(function*() { - const rows = yield* SqlSchema.findAll({ + const messageRows = yield* SqlSchema.findAll({ Request: Schema.Void, Result: KeyRow, execute: () => - sql`SELECT m.message_id AS messageId - FROM effect_local_relay_messages m - LEFT JOIN effect_local_relay_reservations r ON r.message_id = m.message_id - WHERE m.message_id > ${input.cursor ?? 0} - AND r.message_id IS NULL - ORDER BY m.message_id + sql`SELECT message_id AS messageId + FROM effect_local_relay_messages + WHERE message_id > ${input.cursor ?? 0} + ORDER BY message_id LIMIT ${effectiveBatch + 1}` })(undefined) - if (rows.length > 0) { - return yield* storageCorrupt(new Error("Missing immutable relay quota reservation")) - } - if (input.cursor === undefined || input.cursor === 0) { - const orphanReservations = yield* SqlSchema.findAll({ - Request: Schema.Void, - Result: KeyRow, - execute: () => - sql`SELECT r.message_id AS messageId - FROM effect_local_relay_reservations r - LEFT JOIN effect_local_relay_messages m ON m.message_id = r.message_id - WHERE m.message_id IS NULL - ORDER BY r.message_id - LIMIT ${effectiveBatch + 1}` - })(undefined) - if (orphanReservations.length > 0) { - return yield* storageCorrupt(new Error("Orphan relay quota reservation")) - } + const reservationRows = yield* SqlSchema.findAll({ + Request: Schema.Void, + Result: KeyRow, + execute: () => + sql`SELECT message_id AS messageId + FROM effect_local_relay_reservations + WHERE message_id > ${input.cursor ?? 0} + ORDER BY message_id + LIMIT ${effectiveBatch + 1}` + })(undefined) + if ( + messageRows.length !== reservationRows.length || + messageRows.some((row, index) => row.messageId !== reservationRows[index]?.messageId) + ) { + return yield* storageCorrupt(new Error("Relay reservation bijection is corrupt")) } - yield* sql`DELETE FROM effect_local_relay_usage` - yield* sql`INSERT INTO effect_local_relay_usage ( - scope_kind, - scope_key, - active_count, - active_bytes, - retained_count, - retained_bytes - ) - SELECT - scope_kind, - scope_key, - SUM(CASE WHEN active_consumed = 0 THEN active_count_delta ELSE 0 END), - SUM(CASE WHEN active_consumed = 0 THEN active_bytes_delta ELSE 0 END), - SUM(CASE WHEN retained_consumed = 0 THEN retained_count_delta ELSE 0 END), - SUM(CASE WHEN retained_consumed = 0 THEN retained_bytes_delta ELSE 0 END) - FROM ( - SELECT - 'SenderPeer' AS scope_kind, - sender_peer_usage_key AS scope_key, - active_count_delta, - active_bytes_delta, - retained_count_delta, - retained_bytes_delta, - active_consumed, - retained_consumed - FROM effect_local_relay_reservations - UNION ALL - SELECT - 'RecipientPeer', - recipient_peer_usage_key, - active_count_delta, - active_bytes_delta, - retained_count_delta, - retained_bytes_delta, - active_consumed, - retained_consumed - FROM effect_local_relay_reservations - UNION ALL - SELECT - 'RecipientSubject', - recipient_subject_usage_key, - active_count_delta, - active_bytes_delta, - retained_count_delta, - retained_bytes_delta, - active_consumed, - retained_consumed - FROM effect_local_relay_reservations - UNION ALL - SELECT - 'Tenant', - tenant_usage_key, - active_count_delta, - active_bytes_delta, - retained_count_delta, - retained_bytes_delta, - active_consumed, - retained_consumed - FROM effect_local_relay_reservations - UNION ALL - SELECT - 'Shard', - shard_usage_key, - active_count_delta, - active_bytes_delta, - retained_count_delta, - retained_bytes_delta, - active_consumed, - retained_consumed - FROM effect_local_relay_reservations - ) - GROUP BY scope_kind, scope_key - HAVING - SUM(CASE WHEN active_consumed = 0 THEN active_count_delta ELSE 0 END) > 0 - OR SUM(CASE WHEN retained_consumed = 0 THEN retained_count_delta ELSE 0 END) > 0` - return { processed: 0, hasMore: false } + return maintenanceResult(messageRows.map((row) => row.messageId), effectiveBatch) })) - })).pipe(Effect.withSpan("PeerRelayStore.reconcile")) + })) + return PeerRpcObservability.observeRelay({ + effect, + operation: "RelayMaintenance", + direction: "Receive", + stage: "Reconcile", + facts: (exit) => ({ + items: Exit.isSuccess(exit) ? exit.value.processed : 0 + }), + result: (exit) => Exit.isSuccess(exit) ? "Success" : relayFailureResult(exit) + }) + } - const collect: Service["collect"] = (unsafeInput) => - mapStorageErrors(Effect.gen(function*() { + const collect: Service["collect"] = (unsafeInput) => { + const effect = mapStoreErrors(Effect.gen(function*() { const input = yield* validateInput(MaintenanceRequest, unsafeInput) const effectiveBatch = Math.min(input.batchSize, limits.terminalCollectionBatchSize) return yield* write(Effect.gen(function*() { const now = (yield* nowQuery(sql)).now const rows = yield* SqlSchema.findAll({ Request: Schema.Void, - Result: KeyRow, + Result: Schema.Struct({ + messageId: PositiveInt, + channelId: PositiveInt + }), execute: () => - sql`SELECT message_id AS messageId + sql`SELECT + message_id AS messageId, + channel_id AS channelId FROM effect_local_relay_messages - WHERE message_id > ${input.cursor ?? 0} - AND state IN ('Acknowledged', 'DeadLettered', 'Expired') + WHERE state IN ('Acknowledged', 'DeadLettered', 'Expired') AND deduplicate_until <= ${now} - ORDER BY message_id + ORDER BY deduplicate_until, message_id LIMIT ${effectiveBatch + 1}` })(undefined) for (const row of rows.slice(0, effectiveBatch)) { yield* releaseRetainedUsage(row.messageId) + const reservationDeleted = yield* SqlSchema.findAll({ + Request: Schema.Void, + Result: UnitRow, + execute: () => + sql`DELETE FROM effect_local_relay_reservations + WHERE message_id = ${row.messageId} + AND active_consumed = 1 + AND retained_consumed = 1 + RETURNING 1 AS value` + })(undefined) + if (reservationDeleted.length !== 1) { + return yield* storageCorrupt(new Error("Invalid collected relay quota reservation")) + } yield* sql`DELETE FROM effect_local_relay_messages WHERE message_id = ${row.messageId} AND state IN ('Acknowledged', 'DeadLettered', 'Expired') AND deduplicate_until <= ${now}` + yield* sql`DELETE FROM effect_local_relay_channels + WHERE channel_id = ${row.channelId} + AND claimed_message_id IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM effect_local_relay_messages m + WHERE m.channel_id = ${row.channelId} + )` } - yield* sql`DELETE FROM effect_local_relay_usage - WHERE active_count = 0 - AND active_bytes = 0 - AND retained_count = 0 - AND retained_bytes = 0` - yield* sql`DELETE FROM effect_local_relay_channels - WHERE claimed_message_id IS NULL - AND NOT EXISTS ( - SELECT 1 - FROM effect_local_relay_messages m - WHERE m.channel_id = effect_local_relay_channels.channel_id - ) - LIMIT ${limits.orphanChannelCleanupBatchSize}` return maintenanceResult(rows.map((row) => row.messageId), effectiveBatch) })) - })).pipe(Effect.withSpan("PeerRelayStore.collect")) + })) + return PeerRpcObservability.observeRelay({ + effect, + operation: "RelayMaintenance", + direction: "Receive", + stage: "Collect", + facts: (exit) => ({ + items: Exit.isSuccess(exit) ? exit.value.processed : 0 + }), + result: (exit) => Exit.isSuccess(exit) ? "Success" : relayFailureResult(exit) + }) + } - const usage: Service["usage"] = (unsafeInput) => - mapStorageErrors(Effect.gen(function*() { + const usage: Service["usage"] = (unsafeInput) => { + const exactShard = unsafeInput === undefined + const effect = mapStoreErrors(Effect.gen(function*() { const input = unsafeInput === undefined ? UsageRequest.make({ scopeKind: "Shard", scopeKey: encodeKey("local") }) : yield* validateInput(UsageRequest, unsafeInput) @@ -1643,7 +1820,28 @@ const makeService = Effect.gen(function*() { retainedCount: 0, retainedBytes: 0 })) - })).pipe(Effect.withSpan("PeerRelayStore.usage")) + })) + return PeerRpcObservability.observeRelay({ + effect, + operation: "RelayMaintenance", + direction: "Receive", + stage: "Usage", + facts: (exit) => ({ + items: Exit.isSuccess(exit) ? exit.value.activeCount : 0, + ...(Exit.isSuccess(exit) ? { bytes: exit.value.activeBytes, version: 1 } : {}) + }), + result: (exit) => Exit.isSuccess(exit) ? "Success" : relayFailureResult(exit) + }).pipe( + Effect.tap((value) => + exactShard + ? PeerRpcObservability.setRelayPending( + value.activeCount, + value.activeBytes + ).pipe(Effect.catchCause(() => Effect.void)) + : Effect.void + ) + ) + } return PeerRelayStore.of({ admit, diff --git a/packages/local-rpc/src/PeerRpcServer.ts b/packages/local-rpc/src/PeerRpcServer.ts index 6c2b858..cb9c8c7 100644 --- a/packages/local-rpc/src/PeerRpcServer.ts +++ b/packages/local-rpc/src/PeerRpcServer.ts @@ -1235,6 +1235,7 @@ type RelaySqlLane = "Admission" | "Terminal" | "Delivery" | "Maintenance" interface RelaySqlTask { state: "Queued" | "Acquired" | "Cancelled" | "Completed" readonly cancel: Deferred.Deferred + readonly cancelResult: () => void readonly execute: Effect.Effect } @@ -1276,17 +1277,15 @@ const makeRelaySqlScheduler = ( Delivery: yield* Queue.dropping(capacities.Delivery), Maintenance: yield* Queue.dropping(capacities.Maintenance) } satisfies Record> - const wake = yield* Queue.dropping( - capacities.Admission + capacities.Terminal + capacities.Delivery + capacities.Maintenance - ) const lock = yield* Semaphore.make(1) const globalPermits = yield* Semaphore.make(limits.maxInFlightSqlTransactions) - const lanePermits = { - Admission: yield* Semaphore.make(limits.maxInFlightSqlAdmission), - Terminal: yield* Semaphore.make(limits.maxInFlightSqlTerminal), - Delivery: yield* Semaphore.make(limits.maxInFlightSqlDelivery), - Maintenance: yield* Semaphore.make(limits.maxInFlightSqlMaintenance) - } satisfies Record + const laneConcurrency = { + Admission: limits.maxInFlightSqlAdmission, + Terminal: limits.maxInFlightSqlTerminal, + Delivery: limits.maxInFlightSqlDelivery, + Maintenance: limits.maxInFlightSqlMaintenance + } satisfies Record + const activeTasks = new Set() let accepting = true const cancelTask = (task: RelaySqlTask) => @@ -1294,6 +1293,8 @@ const makeRelaySqlScheduler = ( if (task.state === "Completed" || task.state === "Cancelled") return task.state = "Cancelled" Deferred.doneUnsafe(task.cancel, Effect.void) + task.cancelResult() + activeTasks.delete(task) })) const submit: RelaySqlScheduler["submit"] = (lane, effect) => @@ -1304,6 +1305,9 @@ const makeRelaySqlScheduler = ( const task: RelaySqlTask = { state: "Queued", cancel, + cancelResult: () => { + Deferred.doneUnsafe(result, Effect.interrupt) + }, execute: Effect.uninterruptibleMask((restoreTask) => lock.withPermit(Effect.sync(() => { if (task.state !== "Queued") return false @@ -1318,6 +1322,7 @@ const makeRelaySqlScheduler = ( Effect.flatMap((exit) => Deferred.done(result, exit)), Effect.ensuring(lock.withPermit(Effect.sync(() => { task.state = "Completed" + activeTasks.delete(task) }))) ) }) @@ -1327,82 +1332,60 @@ const makeRelaySqlScheduler = ( const offered = yield* lock.withPermit( Effect.suspend(() => { if (!accepting) return Effect.succeed(false) - return Queue.offer(queues[lane], task) + return Queue.offer(queues[lane], task).pipe( + Effect.tap((offered) => + Effect.sync(() => { + if (offered) activeTasks.add(task) + }) + ) + ) }) ) if (!offered) return yield* new PeerRpcError.RequestCapacityExceeded() - yield* Queue.offer(wake, undefined) return yield* restore(Deferred.await(result)).pipe( Effect.onInterrupt(() => cancelTask(task)) ) }) ) - const lanes = [ - "Terminal", - "Admission", - "Delivery", - "Maintenance" - ] as const satisfies ReadonlyArray - - const worker = Effect.suspend(() => { - let cursor = 0 - const take = (): Effect.Effect => - Effect.gen(function*() { - for (let offset = 0; offset < lanes.length; offset++) { - const index = (cursor + offset) % lanes.length - const lane = lanes[index]! - const task = yield* Queue.poll(queues[lane]) - if (Option.isSome(task)) { - cursor = (index + 1) % lanes.length - return [lane, task.value] as const - } - } - yield* Queue.take(wake) - return yield* take() - }) - return Effect.forever( - take().pipe( - Effect.flatMap(([lane, task]) => - task.execute.pipe( - lanePermits[lane].withPermits(1), - globalPermits.withPermits(1) - ) - ) + const workerFibers: Array> = [] + for (const lane of Object.keys(queues) as Array) { + const worker = Effect.forever( + Queue.take(queues[lane]).pipe( + Effect.flatMap((task) => task.execute.pipe(globalPermits.withPermits(1))) ) ) - }) - - const workerFibers: Array> = [] - for (let index = 0; index < limits.maxInFlightSqlTransactions; index++) { - const fiber = yield* Effect.forkIn(worker, runtimeScope) - workerFibers.push(fiber) - yield* Effect.forkIn( - Fiber.await(fiber).pipe( - Effect.flatMap((exit) => - lock.withPermit(Effect.sync(() => accepting)).pipe( - Effect.flatMap((wasAccepting) => - wasAccepting && Exit.isFailure(exit) - ? onFatal(exit.cause) - : Effect.void + for (let index = 0; index < laneConcurrency[lane]; index++) { + const fiber = yield* Effect.forkIn(worker, runtimeScope) + workerFibers.push(fiber) + yield* Effect.forkIn( + Fiber.await(fiber).pipe( + Effect.flatMap((exit) => + lock.withPermit(Effect.sync(() => accepting)).pipe( + Effect.flatMap((wasAccepting) => + wasAccepting && Exit.isFailure(exit) + ? onFatal(exit.cause) + : Effect.void + ) ) ) - ) - ), - runtimeScope - ) + ), + runtimeScope + ) + } } const shutdown = Effect.uninterruptible( lock.withPermit(Effect.sync(() => { accepting = false + return [...activeTasks] })).pipe( - Effect.andThen(Effect.forEach(Object.values(queues), (queue) => - drainRelayQueue(queue).pipe( - Effect.flatMap((tasks) => Effect.forEach(tasks, cancelTask, { discard: true })), - Effect.andThen(Queue.shutdown(queue)) - ), { discard: true })), - Effect.andThen(Queue.shutdown(wake)), + Effect.flatMap((tasks) => Effect.forEach(tasks, cancelTask, { discard: true })), + Effect.andThen(Effect.forEach( + Object.values(queues), + (queue) => Queue.shutdown(queue), + { discard: true } + )), Effect.andThen(Effect.forEach(workerFibers, Fiber.interrupt, { discard: true })) ) ).pipe(Effect.ignore) @@ -1501,15 +1484,20 @@ const makeRelayAdmissionLane = (options: { ) const run: RelayAdmissionLane["run"] = (subjectId, effect) => - Effect.gen(function*() { - const admittedAt = yield* Clock.currentTimeMillis - if (!(yield* admit(subjectId, admittedAt))) { - return yield* new PeerRpcError.RequestCapacityExceeded() - } - const result = yield* effect.pipe(permits.withPermitsIfAvailable(1)) - if (Option.isNone(result)) return yield* new PeerRpcError.RequestCapacityExceeded() - return result.value - }).pipe(Effect.ensuring(release(subjectId))) + Effect.uninterruptibleMask((restore) => + Effect.gen(function*() { + const admittedAt = yield* Clock.currentTimeMillis + if (!(yield* admit(subjectId, admittedAt))) { + return yield* new PeerRpcError.RequestCapacityExceeded() + } + return yield* restore( + effect.pipe( + permits.withPermitsIfAvailable(1), + Effect.flatMap(Effect.fromOption(() => new PeerRpcError.RequestCapacityExceeded())) + ) + ).pipe(Effect.ensuring(release(subjectId))) + }) + ) return { run, @@ -1568,7 +1556,9 @@ interface RelayEntry { readonly senderRetryHorizonMillis: number readonly outbound: Queue.Queue readonly claims: Map - readonly watcherFibers: Array> + readonly revoked: Deferred.Deferred + readonly monitorFibers: Array> + watcher: Fiber.Fiber | undefined active: boolean } @@ -1649,10 +1639,12 @@ const relayStoreFailure = (error: PeerRelayStore.StoreError): PeerRpcError.PeerR } } -const SyncEnvelopeJson = Schema.fromJsonString(Schema.toCodecJson(PeerSession.SyncEnvelope)) +const RelaySyncEnvelopeJson = Schema.fromJsonString( + Schema.toCodecJson(PeerSyncEnvelope.SyncEnvelope) +) const decodeRelayEnvelope = (payload: Uint8Array) => - Schema.decodeUnknownEffect(SyncEnvelopeJson)(new TextDecoder().decode(payload)).pipe( + Schema.decodeUnknownEffect(RelaySyncEnvelopeJson)(new TextDecoder().decode(payload)).pipe( Effect.mapError(() => new PeerRpcError.InvalidRequest()) ) @@ -1682,6 +1674,8 @@ export const layerRelayHandlers = ( const selectors = new Map() const newWork = yield* Queue.dropping(limits.newWorkQueueCapacity) const retryWork = yield* Queue.dropping(limits.retryQueueCapacity) + const newWorkLock = yield* Semaphore.make(1) + const retryWorkLock = yield* Semaphore.make(1) const workWake = yield* Queue.dropping( limits.newWorkQueueCapacity + limits.retryQueueCapacity ) @@ -1720,16 +1714,34 @@ export const layerRelayHandlers = ( const storeEffect = (effect: Effect.Effect) => effect.pipe(Effect.mapError(relayStoreFailure)) + const activeClaimCount = () => [...sessions.values()].reduce((sum, entry) => sum + entry.claims.size, 0) + + const refreshRelayPending = sql.submit( + "Maintenance", + storeEffect(store.usage()) + ).pipe( + Effect.flatMap((usage) => PeerRpcObservability.setRelayPending(usage.activeCount, usage.activeBytes)) + ) + + yield* refreshRelayPending + yield* PeerRpcObservability.setRelayActiveClaims(0) + yield* PeerRpcObservability.setRelayWorkers(0) + yield* PeerRpcObservability.setRelayReadyQueueItems("New", 0) + yield* PeerRpcObservability.setRelayReadyQueueItems("Retry", 0) + const releaseClaim = ( entry: RelayEntry, claim: PeerRelayStore.ClaimedMessage ) => - store.release({ - channel: claim.channel, - relayMessageId: claim.relayMessageId, - claimToken: claim.claimToken, - sessionGeneration: entry.generation - }).pipe( + sql.submit( + "Terminal", + storeEffect(store.release({ + channel: claim.channel, + relayMessageId: claim.relayMessageId, + claimToken: claim.claimToken, + sessionGeneration: entry.generation + })) + ).pipe( Effect.timeoutOrElse({ duration: limits.shutdownReleaseTimeoutMillis, orElse: () => Effect.succeed(undefined) @@ -1742,7 +1754,8 @@ export const layerRelayHandlers = ( fromWatcher: boolean ): Effect.Effect => Effect.uninterruptible(Effect.gen(function*() { - const cleanup = yield* lock.withPermit(Effect.sync(() => { + yield* Deferred.succeed(entry.revoked, undefined) + const cleanup = yield* lock.withPermit(Effect.gen(function*() { if (!entry.active) return undefined entry.active = false sessions.delete(entry.sessionId) @@ -1768,6 +1781,7 @@ export const layerRelayHandlers = ( } const claims = [...entry.claims.values()] entry.claims.clear() + yield* PeerRpcObservability.setRelayActiveClaims(activeClaimCount()) return { claims } })) if (cleanup === undefined) return @@ -1783,9 +1797,14 @@ export const layerRelayHandlers = ( (claim) => releaseClaim(entry, claim), { concurrency: limits.shutdownReleaseConcurrency, discard: true } ) - if (!fromWatcher) { - yield* Effect.forEach(entry.watcherFibers, Fiber.interrupt, { discard: true }) - } + const watcherFibers = fromWatcher || entry.watcher === undefined + ? entry.monitorFibers + : [...entry.monitorFibers, entry.watcher] + yield* Effect.forEach( + watcherFibers, + (fiber) => Effect.forkIn(Fiber.interrupt(fiber), runtimeScope), + { discard: true } + ) })) const notify = ( @@ -1812,7 +1831,15 @@ export const layerRelayHandlers = ( })) if (!shouldOffer) return const queue = lane === "New" ? newWork : retryWork - const offered = yield* Queue.offer(queue, selector) + const queueLock = lane === "New" ? newWorkLock : retryWorkLock + const offered = yield* queueLock.withPermit(Effect.gen(function*() { + const offered = yield* Queue.offer(queue, selector) + if (offered) { + const size = yield* Queue.size(queue) + yield* PeerRpcObservability.setRelayReadyQueueItems(lane, size) + } + return offered + })) if (!offered) { yield* lock.withPermit(Effect.sync(() => { const current = workOwners.get(key) @@ -1825,6 +1852,27 @@ export const layerRelayHandlers = ( yield* Queue.offer(workWake, undefined) }) + const ensureCurrentEntry = (entry: RelayEntry) => + lock.withPermit(Effect.gen(function*() { + const revoked = yield* Deferred.poll(entry.revoked) + if ( + Option.isSome(revoked) || + !entry.active || + sessions.get(entry.sessionId) !== entry || + endpoints.get(relayEntryKey(entry.principal, entry.remote)) !== entry.sessionId || + incarnations.get( + relayIncarnationKey( + entry.principal, + entry.senderReplicaIncarnation, + entry.remote + ) + ) !== entry.sessionId + ) { + return yield* new PeerRpcError.SessionUnavailable() + } + return entry + })) + const freshEntry = ( sessionId: Identity.SessionId, authenticated: Context.Service.Shape @@ -1837,22 +1885,24 @@ export const layerRelayHandlers = ( const entry = yield* lock.withPermit(Effect.sync(() => sessions.get(sessionId))) if ( entry === undefined || - !entry.active || - !sameRelayPrincipal(entry.principal, authenticated.principal) || - endpoints.get(relayEntryKey(entry.principal, entry.remote)) !== entry.sessionId || - incarnations.get( - relayIncarnationKey( - entry.principal, - entry.senderReplicaIncarnation, - entry.remote - ) - ) !== entry.sessionId + !sameRelayPrincipal(entry.principal, authenticated.principal) ) { return yield* new PeerRpcError.SessionUnavailable() } - return entry + return yield* ensureCurrentEntry(entry) }) + const raceRevocation = ( + entry: RelayEntry, + effect: Effect.Effect + ) => + Effect.raceFirst( + effect, + Deferred.await(entry.revoked).pipe( + Effect.andThen(Effect.fail(new PeerRpcError.SessionUnavailable())) + ) + ) + const authorizeEntry = ( entry: RelayEntry, direction: PeerRelayAuthorization.Direction, @@ -1865,6 +1915,94 @@ export const layerRelayHandlers = ( documents }) + const withRelayGrantAdmission = ( + entry: RelayEntry, + grants: ReadonlyArray<{ + readonly validUntil: number + readonly invalidated: Effect.Effect + }>, + effect: Effect.Effect + ) => + Effect.gen(function*() { + const now = yield* Clock.currentTimeMillis + const validUntil = Math.min(...grants.map((grant) => grant.validUntil)) + if (validUntil <= now) { + return yield* new PeerRpcError.AccessDenied() + } + const admissionGate = yield* Semaphore.make(1) + let admitted = false + let invalidated = false + const invalidate = admissionGate.withPermit( + Effect.sync(() => { + if (!admitted) invalidated = true + }) + ) + const enter = admissionGate.withPermit( + Effect.gen(function*() { + const currentTime = yield* Clock.currentTimeMillis + if (invalidated || validUntil <= currentTime) return false + admitted = true + return true + }) + ).pipe( + Effect.flatMap((entered) => + entered + ? Effect.void + : Effect.fail(new PeerRpcError.AccessDenied()) + ) + ) + return yield* Effect.acquireUseRelease( + Effect.forEach( + [ + ...grants.map((grant) => grant.invalidated), + Deferred.await(entry.revoked), + Effect.sleep(Math.max(0, validUntil - now)) + ], + (monitor) => + monitor.pipe( + Effect.exit, + Effect.andThen(invalidate), + Effect.forkIn(runtimeScope) + ) + ), + () => + Effect.gen(function*() { + yield* Effect.yieldNow + yield* ensureCurrentEntry(entry) + yield* enter + return yield* effect + }), + (fibers) => + Effect.forEach( + fibers, + (fiber) => Effect.forkIn(Fiber.interrupt(fiber), runtimeScope), + { discard: true } + ) + ) + }) + + const withRelayAuthorizationGrants = ( + entry: RelayEntry, + direction: PeerRelayAuthorization.Direction, + normalGrant: PeerRelayAuthorization.Result, + documents: PeerRelayAuthorization.UnsafeUnboundedAutomerge3DecodeRequest["documents"], + effect: Effect.Effect + ) => + Effect.gen(function*() { + const grant = yield* authorization.authorizeUnsafeUnboundedAutomerge3Decode({ + risk: PeerRelayAuthorization.unsafeUnboundedAutomerge3DecodeRisk, + direction, + principal: entry.principal, + remote: entry.remote, + documents + }) + return yield* withRelayGrantAdmission( + entry, + [normalGrant, grant], + effect + ) + }) + const validateClaimPayload = ( claim: PeerRelayStore.ClaimedMessage, payload: Uint8Array @@ -1908,6 +2046,7 @@ export const layerRelayHandlers = ( documentId: envelope.documentId, documentType: envelope.documentType }, + lineage: envelope.lineage, writerProvenance: envelope.writerProvenance, messageHash: envelope.messageHash, payload @@ -1922,7 +2061,7 @@ export const layerRelayHandlers = ( entry: RelayEntry, claim: PeerRelayStore.ClaimedMessage ) => - lock.withPermit(Effect.sync(() => { + lock.withPermit(Effect.gen(function*() { if (entry.claims.get(claim.relayMessageId) === claim) { entry.claims.delete(claim.relayMessageId) const key = relaySelectorKey(relaySelectorForEntry(entry)) @@ -1933,6 +2072,7 @@ export const layerRelayHandlers = ( ) { workOwners.delete(key) } + yield* PeerRpcObservability.setRelayActiveClaims(activeClaimCount()) return true } return false @@ -1955,144 +2095,201 @@ export const layerRelayHandlers = ( } const allowed = yield* authorizeEntry(entry, "Receive", entry.documents) const authorizedDocumentIds = allowed.documents.map((document) => document.documentId) - type ClaimAcquisition = { - readonly claimed: PeerRelayStore.ClaimResult - readonly claim: PeerRelayStore.ClaimedMessage | undefined - readonly recorded: boolean - } - const acquisition = yield* Effect.uninterruptibleMask((restore) => - restore(sql.submit( - "Delivery", - storeEffect(store.claim({ - recipient: selector.recipient, - sender: selector.sender, - sessionGeneration: entry.generation, - authorizedDocumentIds - })) - )).pipe( - Effect.flatMap((claimed) => { - if (Option.isNone(claimed.message)) { - return Effect.succeed({ - claimed, - claim: undefined, - recorded: true - } as ClaimAcquisition) + const unsafeDocuments = allowed.documents.map((document) => ({ + documentType: document.document.name, + documentId: document.documentId + })) + let ownedClaim: PeerRelayStore.ClaimedMessage | undefined + yield* withRelayAuthorizationGrants( + entry, + "Receive", + allowed, + unsafeDocuments, + Effect.gen(function*() { + type ClaimAcquisition = { + readonly claimed: PeerRelayStore.ClaimResult + readonly claim: PeerRelayStore.ClaimedMessage | undefined + readonly recorded: boolean + } + const acquisition = yield* Effect.uninterruptibleMask((restore) => + restore(sql.submit( + "Delivery", + storeEffect(store.claim({ + recipient: selector.recipient, + sender: selector.sender, + sessionGeneration: entry.generation, + authorizedDocumentIds + })) + )).pipe( + Effect.flatMap((claimed) => { + if (Option.isNone(claimed.message)) { + return Effect.succeed({ + claimed, + claim: undefined, + recorded: true + } as ClaimAcquisition) + } + const claim = claimed.message.value + return lock.withPermit(Effect.gen(function*() { + if ( + sessions.get(entry.sessionId) !== entry || + !entry.active || + endpoints.get(relayEntryKey(entry.principal, entry.remote)) !== entry.sessionId + ) { + return { claimed, claim, recorded: false } as ClaimAcquisition + } + entry.claims.set(claim.relayMessageId, claim) + yield* PeerRpcObservability.setRelayActiveClaims(activeClaimCount()) + return { claimed, claim, recorded: true } as ClaimAcquisition + })) + }) + ) + ) + if (acquisition.claim === undefined) { + const pending = yield* lock.withPermit(Effect.sync(() => { + const current = workOwners.get(key) + workOwners.delete(key) + return current?._tag === "Worker" && current.pending + })) + if (pending) yield* notify(selector, acquisition.claimed.lane) + return + } + const claim = acquisition.claim + if (!acquisition.recorded) { + yield* releaseClaim(entry, claim) + return + } + ownedClaim = claim + let pendingReservation: PeerRelayIngress.Reservation | undefined + yield* Effect.gen(function*() { + const reservation = yield* ingress.reserveOutbound(claim.payloadBytes).pipe( + Effect.onError(() => abandonClaim(entry, claim)) + ) + pendingReservation = reservation + const payload = yield* sql.submit( + "Delivery", + storeEffect(store.loadClaimedPayload({ + channel: claim.channel, + rowId: claim.rowId, + relayMessageId: claim.relayMessageId, + claimToken: claim.claimToken, + sessionGeneration: entry.generation, + payloadBytes: claim.payloadBytes + })) + ).pipe( + Effect.catchTag( + "InvalidRequest", + () => Effect.fail(new PeerRpcError.SessionUnavailable()) + ), + Effect.onError(() => Effect.andThen(reservation.release, abandonClaim(entry, claim))) + ) + const envelope = yield* validateClaimPayload(claim, payload).pipe( + Effect.onError(() => Effect.andThen(reservation.release, abandonClaim(entry, claim))) + ) + const authorized = allowed.documents.some((document) => + document.documentId === envelope.documentId && + document.document.name === envelope.documentType + ) + if (!authorized) { + yield* Effect.uninterruptible( + Effect.sync(() => { + pendingReservation = undefined + }).pipe( + Effect.andThen(reservation.release), + Effect.ensuring(abandonClaim(entry, claim)) + ) + ) + return } - const claim = claimed.message.value - return lock.withPermit(Effect.sync(() => { + const event = PeerRelayRpc.StoredMessage.make({ + _tag: "StoredMessage", + relayMessageId: claim.relayMessageId, + claimToken: claim.claimToken, + relayPeerId: claim.relayPeerId, + sender: { + tenantId: claim.channel.tenantId, + subjectId: claim.channel.senderSubjectId, + peerId: claim.channel.senderPeerId, + replicaIncarnation: claim.channel.senderReplicaIncarnation, + connectionEpoch: claim.senderConnectionEpoch, + sequence: claim.senderSequence + }, + recipient: selector.recipient, + payloadVersion: 1, + document: { + documentType: envelope.documentType, + documentId: envelope.documentId + }, + writerProvenance: envelope.writerProvenance, + messageHash: envelope.messageHash, + outerEnvelopeDigest: claim.outerEnvelopeDigest, + payload + }) + const transferred = yield* lock.withPermit(Effect.sync(() => { + const currentSession = sessions.get(entry.sessionId) + const owner = workOwners.get(key) if ( - sessions.get(entry.sessionId) !== entry || + currentSession !== entry || !entry.active || - endpoints.get(relayEntryKey(entry.principal, entry.remote)) !== entry.sessionId + owner?._tag !== "Worker" ) { - return { claimed, claim, recorded: false } as ClaimAcquisition + return false } - entry.claims.set(claim.relayMessageId, claim) - return { claimed, claim, recorded: true } as ClaimAcquisition + workOwners.set(key, { + _tag: "Entry", + generation: entry.generation, + pending: owner.pending + }) + return true })) - }) - ) - ) - if (acquisition.claim === undefined) { - const pending = yield* lock.withPermit(Effect.sync(() => { - const current = workOwners.get(key) - workOwners.delete(key) - return current?._tag === "Worker" && current.pending - })) - if (pending) yield* notify(selector, acquisition.claimed.lane) - return - } - const claim = acquisition.claim - if (!acquisition.recorded) { - yield* releaseClaim(entry, claim) - return - } - let pendingReservation: PeerRelayIngress.Reservation | undefined - yield* Effect.gen(function*() { - const reservation = yield* ingress.reserveOutbound(claim.payloadBytes).pipe( - Effect.onError(() => abandonClaim(entry, claim)) - ) - pendingReservation = reservation - const payload = yield* sql.submit( - "Delivery", - storeEffect(store.loadClaimedPayload({ - channel: claim.channel, - rowId: claim.rowId, - relayMessageId: claim.relayMessageId, - claimToken: claim.claimToken, - sessionGeneration: entry.generation, - payloadBytes: claim.payloadBytes - })) - ).pipe(Effect.onError(() => Effect.andThen(reservation.release, abandonClaim(entry, claim)))) - const envelope = yield* validateClaimPayload(claim, payload).pipe( - Effect.onError(() => Effect.andThen(reservation.release, abandonClaim(entry, claim))) - ) - const event = PeerRelayRpc.StoredMessage.make({ - _tag: "StoredMessage", - relayMessageId: claim.relayMessageId, - claimToken: claim.claimToken, - relayPeerId: claim.relayPeerId, - sender: { - tenantId: claim.channel.tenantId, - subjectId: claim.channel.senderSubjectId, - peerId: claim.channel.senderPeerId, - replicaIncarnation: claim.channel.senderReplicaIncarnation, - connectionEpoch: claim.senderConnectionEpoch, - sequence: claim.senderSequence - }, - recipient: selector.recipient, - payloadVersion: 1, - document: { - documentType: envelope.documentType, - documentId: envelope.documentId - }, - writerProvenance: envelope.writerProvenance, - messageHash: envelope.messageHash, - outerEnvelopeDigest: claim.outerEnvelopeDigest, - payload - }) - const transferred = yield* lock.withPermit(Effect.sync(() => { - const currentSession = sessions.get(entry.sessionId) - const owner = workOwners.get(key) - if ( - currentSession !== entry || - !entry.active || - owner?._tag !== "Worker" - ) { - return false - } - workOwners.set(key, { - _tag: "Entry", - generation: entry.generation, - pending: owner.pending - }) - return true - })) - if (!transferred) { - yield* reservation.release - yield* abandonClaim(entry, claim) - return - } - const offered = yield* Queue.offer(entry.outbound, { - event, - reservation, - transferred: false - }) - if (!offered) { - yield* lock.withPermit(Effect.sync(() => { - const owner = workOwners.get(key) - if (owner?._tag === "Entry" && owner.generation === entry.generation) { - workOwners.delete(key) + if (!transferred) { + yield* reservation.release + yield* abandonClaim(entry, claim) + return } - })) - yield* reservation.release - yield* abandonClaim(entry, claim) - } - }).pipe( - Effect.onInterrupt(() => - Effect.andThen( - pendingReservation?.release ?? Effect.void, - abandonClaim(entry, claim) + const offered = yield* Queue.offer(entry.outbound, { + event, + reservation, + transferred: false + }) + if (!offered) { + yield* lock.withPermit(Effect.sync(() => { + const owner = workOwners.get(key) + if (owner?._tag === "Entry" && owner.generation === entry.generation) { + workOwners.delete(key) + } + })) + yield* reservation.release + yield* abandonClaim(entry, claim) + return + } + pendingReservation = undefined + ownedClaim = undefined + yield* PeerRpcObservability.recordRelayOutcome({ + operation: "RelayClaim", + direction: "Receive", + result: "Delivered", + facts: { + bytes: payload.byteLength, + items: 1, + version: claim.payloadVersion + } + }) + }).pipe( + Effect.onInterrupt(() => + pendingReservation === undefined + ? Effect.void + : Effect.andThen( + pendingReservation.release, + abandonClaim(entry, claim) + ) + ) + ) + ownedClaim = undefined + }).pipe( + Effect.onExitIf( + Exit.isFailure, + () => ownedClaim === undefined ? Effect.void : abandonClaim(entry, ownedClaim) ) ) ) @@ -2103,6 +2300,10 @@ export const layerRelayHandlers = ( workOwners.delete(relaySelectorKey(selector)) })), RequestCapacityExceeded: () => + lock.withPermit(Effect.sync(() => { + workOwners.delete(relaySelectorKey(selector)) + })), + SessionUnavailable: () => lock.withPermit(Effect.sync(() => { workOwners.delete(relaySelectorKey(selector)) })) @@ -2121,7 +2322,16 @@ export const layerRelayHandlers = ( for (let offset = 0; offset < workOrder.length; offset++) { const index = (cursor + offset) % workOrder.length const lane = workOrder[index]! - const item = yield* Queue.poll(lane === "New" ? newWork : retryWork) + const queue = lane === "New" ? newWork : retryWork + const queueLock = lane === "New" ? newWorkLock : retryWorkLock + const item = yield* queueLock.withPermit(Effect.gen(function*() { + const item = yield* Queue.poll(queue) + if (Option.isSome(item)) { + const size = yield* Queue.size(queue) + yield* PeerRpcObservability.setRelayReadyQueueItems(lane, size) + } + return item + })) if (Option.isSome(item)) { cursor = (index + 1) % workOrder.length return item.value @@ -2133,10 +2343,29 @@ export const layerRelayHandlers = ( return Effect.forever(take().pipe(Effect.flatMap(deliver))) }) - const workerFibers: Array> = [] + const workerFibers = new Set>() + let workerCount = 0 for (let index = 0; index < limits.relayWorkerConcurrency; index++) { - const fiber = yield* Effect.forkIn(makeWorker, runtimeScope) - workerFibers.push(fiber) + const countedWorker = Effect.acquireUseRelease( + lock.withPermit(Effect.gen(function*() { + workerCount += 1 + yield* PeerRpcObservability.setRelayWorkers(workerCount) + })), + () => makeWorker, + () => + lock.withPermit(Effect.gen(function*() { + workerCount -= 1 + yield* PeerRpcObservability.setRelayWorkers(workerCount) + })) + ) + const fiber = yield* Effect.forkIn(countedWorker, runtimeScope).pipe( + Effect.tap((fiber) => + lock.withPermit(Effect.sync(() => { + workerFibers.add(fiber) + })) + ), + Effect.uninterruptible + ) yield* Effect.forkIn( Fiber.await(fiber).pipe( Effect.flatMap((exit) => @@ -2210,7 +2439,8 @@ export const layerRelayHandlers = ( stage.cursor = result.hasMore ? result.cursor : undefined }) ) - ), { discard: true })) + ), { discard: true })), + Effect.andThen(refreshRelayPending) ) ).pipe(Effect.catchCause(signalFatal)), runtimeScope @@ -2264,93 +2494,165 @@ export const layerRelayHandlers = ( remote: request.remote, documents: request.documents }) - const outbound = yield* Queue.dropping< - RelayOutboundItem, - PeerRpcError.PeerRpcError | Cause.Done - >(1) - const sessionId = yield* Identity.makeSessionId.pipe( - Effect.mapError(() => new PeerRpcError.ServerUnavailable()) - ) - const entry = yield* lock.withPermit(Effect.gen(function*() { - if (!accepting) return yield* new PeerRpcError.ServerUnavailable() - const current = subjectSessions.get(principal.subjectId) ?? 0 - const endpoint = relayEntryKey(principal, request.remote) - const replacedId = endpoints.get(endpoint) - const replaced = replacedId === undefined ? undefined : sessions.get(replacedId) - if (replaced === undefined && current >= limits.maxSessionsPerSubject) { - return yield* new PeerRpcError.RequestCapacityExceeded() - } - if (replaced === undefined && endpoints.size >= limits.maxActiveChannels) { - return yield* new PeerRpcError.RequestCapacityExceeded() - } - const entry: RelayEntry = { - sessionId, - generation: generation++, - principal, - senderReplicaIncarnation: request.senderReplicaIncarnation, - remote: request.remote, - documents: request.documents, - receiptRetentionMillis: request.receiptRetentionMillis, - senderRetryHorizonMillis: request.senderRetryHorizonMillis, - outbound, - claims: new Map(), - watcherFibers: [], - active: true - } - sessions.set(sessionId, entry) - endpoints.set(endpoint, sessionId) - incarnations.set( - relayIncarnationKey(principal, request.senderReplicaIncarnation, request.remote), - sessionId - ) - subjectSessions.set(principal.subjectId, current + 1) - selectors.set( - relaySelectorKey(relaySelectorForEntry(entry)), - relaySelectorForEntry(entry) - ) - return { entry, replaced } - })) - if (entry.replaced !== undefined) yield* detachEntry(entry.replaced, false) - const validUntil = Math.min( - authenticated.validUntil, - send.validUntil, - receive.validUntil - ) - const watcher = Effect.raceFirst( - Effect.raceFirst(authenticated.invalidated, send.invalidated), - Effect.raceFirst( - receive.invalidated, - Effect.sleep(Math.max(0, validUntil - now)) - ) - ).pipe( - Effect.andThen(detachEntry(entry.entry, true)), - Effect.forkIn(runtimeScope) - ) - entry.entry.watcherFibers.push(yield* watcher) - yield* notify(relaySelectorForEntry(entry.entry), "Retry") - const opened = PeerRelayRpc.RelayOpened.make({ - _tag: "RelayOpened", - version: PeerRelayRpc.protocolVersion, - sessionId, - remotePeerId: entry.entry.remote.peerId, - authenticatedLocal: principal, - capabilities: { storeAndForward: true } - }) - const deliveries = Stream.fromQueue(entry.entry.outbound).pipe( - Stream.mapEffect((item) => - item.reservation.transferToCurrentRequest.pipe( - Effect.tap(() => - Effect.sync(() => { - item.transferred = true - }) - ), - Effect.as(item.event), - Effect.onError(() => item.reservation.release) + const authorizationCompletedAt = yield* Clock.currentTimeMillis + if (authenticated.validUntil <= authorizationCompletedAt) { + return yield* new PeerRpcError.AuthenticationFailure() + } + if ( + send.validUntil <= authorizationCompletedAt || + receive.validUntil <= authorizationCompletedAt + ) { + return yield* new PeerRpcError.AccessDenied() + } + return yield* Effect.uninterruptibleMask((restore) => + Effect.gen(function*() { + const outbound = yield* Queue.dropping< + RelayOutboundItem, + PeerRpcError.PeerRpcError | Cause.Done + >(1) + const sessionId = yield* Identity.makeSessionId.pipe( + Effect.mapError(() => new PeerRpcError.ServerUnavailable()) ) - ) - ) - return Stream.concat(Stream.make(opened), deliveries).pipe( - Stream.ensuring(detachEntry(entry.entry, false)) + const revoked = yield* Deferred.make() + const entry = yield* lock.withPermit(Effect.gen(function*() { + if (!accepting) return yield* new PeerRpcError.ServerUnavailable() + const current = subjectSessions.get(principal.subjectId) ?? 0 + const endpoint = relayEntryKey(principal, request.remote) + const replacedId = endpoints.get(endpoint) + const replaced = replacedId === undefined ? undefined : sessions.get(replacedId) + if (replaced === undefined && current >= limits.maxSessionsPerSubject) { + return yield* new PeerRpcError.RequestCapacityExceeded() + } + if (replaced === undefined && endpoints.size >= limits.maxActiveChannels) { + return yield* new PeerRpcError.RequestCapacityExceeded() + } + const entry: RelayEntry = { + sessionId, + generation: generation++, + principal, + senderReplicaIncarnation: request.senderReplicaIncarnation, + remote: request.remote, + documents: request.documents, + receiptRetentionMillis: request.receiptRetentionMillis, + senderRetryHorizonMillis: request.senderRetryHorizonMillis, + outbound, + claims: new Map(), + revoked, + monitorFibers: [], + watcher: undefined, + active: true + } + sessions.set(sessionId, entry) + endpoints.set(endpoint, sessionId) + incarnations.set( + relayIncarnationKey(principal, request.senderReplicaIncarnation, request.remote), + sessionId + ) + subjectSessions.set(principal.subjectId, current + 1) + selectors.set( + relaySelectorKey(relaySelectorForEntry(entry)), + relaySelectorForEntry(entry) + ) + return { entry, replaced } + })) + return yield* restore(Effect.gen(function*() { + if (entry.replaced !== undefined) yield* detachEntry(entry.replaced, false) + const monitorStartedAt = yield* Clock.currentTimeMillis + if (authenticated.validUntil <= monitorStartedAt) { + return yield* new PeerRpcError.AuthenticationFailure() + } + if ( + send.validUntil <= monitorStartedAt || + receive.validUntil <= monitorStartedAt + ) { + return yield* new PeerRpcError.AccessDenied() + } + const validUntil = Math.min( + authenticated.validUntil, + send.validUntil, + receive.validUntil + ) + const monitors = [ + authenticated.invalidated, + send.invalidated, + receive.invalidated, + Effect.sleep(Math.max(0, validUntil - monitorStartedAt)) + ] + for (const monitor of monitors) { + yield* monitor.pipe( + Effect.exit, + Effect.andThen(Deferred.succeed(entry.entry.revoked, undefined)), + Effect.asVoid, + Effect.forkIn(runtimeScope), + Effect.tap((fiber) => + lock.withPermit(Effect.sync(() => { + if ( + entry.entry.active && + sessions.get(entry.entry.sessionId) === entry.entry + ) { + entry.entry.monitorFibers.push(fiber) + return true + } + return false + })).pipe( + Effect.flatMap((owned) => + owned + ? Effect.void + : Effect.forkIn(Fiber.interrupt(fiber), runtimeScope).pipe( + Effect.asVoid + ) + ) + ) + ), + Effect.uninterruptible + ) + } + yield* Deferred.await(entry.entry.revoked).pipe( + Effect.andThen(detachEntry(entry.entry, true)), + Effect.forkIn(runtimeScope), + Effect.tap((fiber) => + Effect.sync(() => { + entry.entry.watcher = fiber + }) + ), + Effect.uninterruptible + ) + yield* notify(relaySelectorForEntry(entry.entry), "Retry") + const opened = PeerRelayRpc.RelayOpened.make({ + _tag: "RelayOpened", + version: PeerRelayRpc.protocolVersion, + sessionId, + remotePeerId: entry.entry.remote.peerId, + authenticatedLocal: principal, + capabilities: { storeAndForward: true } + }) + const deliveries = Stream.fromQueue(entry.entry.outbound).pipe( + Stream.mapEffect((item) => + raceRevocation( + entry.entry, + item.reservation.transferToCurrentRequest + ).pipe( + Effect.tap(() => + Effect.sync(() => { + item.transferred = true + }) + ), + Effect.andThen(ensureCurrentEntry(entry.entry)), + Effect.as(item.event), + Effect.onError(() => item.reservation.release) + ) + ) + ) + return Stream.concat(Stream.make(opened), deliveries).pipe( + Stream.ensuring(detachEntry(entry.entry, false)) + ) + })).pipe( + Effect.onExitIf( + Exit.isFailure, + () => detachEntry(entry.entry, false) + ) + ) + }) ) }) ) @@ -2377,7 +2679,15 @@ export const layerRelayHandlers = ( if (messageHash !== envelope.messageHash) { return yield* new PeerRpcError.InvalidRequest() } - yield* authorizeEntry(entry, "Send", [document]) + const send = yield* authorizeEntry(entry, "Send", [document]) + const sendDocument = send.documents.find((candidate) => + candidate.documentId === document.documentId && + candidate.document.name === document.documentType + ) + if (sendDocument === undefined) { + return yield* new PeerRpcError.AccessDenied() + } + yield* ensureCurrentEntry(entry) const outerEnvelopeDigest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope({ domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, version: PeerSyncEnvelope.relayOuterEnvelopeVersion, @@ -2398,6 +2708,7 @@ export const layerRelayHandlers = ( documentId: envelope.documentId, documentType: envelope.documentType }, + lineage: envelope.lineage, writerProvenance: envelope.writerProvenance, messageHash: envelope.messageHash, payload: request.payload @@ -2410,23 +2721,32 @@ export const layerRelayHandlers = ( recipientSubjectId: entry.remote.subjectId, recipientPeerId: entry.remote.peerId } - const result = yield* sql.submit( - "Admission", - storeEffect(store.admit({ - channel, - relayMessageId: request.relayMessageId, - relayPeerId: options.peerId, - documentIds: [envelope.documentId], - senderConnectionEpoch: envelope.connectionEpoch, - senderSequence: envelope.sequence, - payloadVersion: 1, - messageHash: envelope.messageHash, - outerEnvelopeDigest, - payload: request.payload, - messageTtlMillis: limits.messageTtlMillis, - senderRetryHorizonMillis: entry.senderRetryHorizonMillis, - minimumTerminalRetentionMillis: limits.minimumTerminalRetentionMillis - })) + const result = yield* withRelayAuthorizationGrants( + entry, + "Send", + send, + [{ + documentType: sendDocument.document.name, + documentId: sendDocument.documentId + }], + sql.submit( + "Admission", + storeEffect(store.admit({ + channel, + relayMessageId: request.relayMessageId, + relayPeerId: options.peerId, + documentIds: [envelope.documentId], + senderConnectionEpoch: envelope.connectionEpoch, + senderSequence: envelope.sequence, + payloadVersion: 1, + messageHash: envelope.messageHash, + outerEnvelopeDigest, + payload: request.payload, + messageTtlMillis: limits.messageTtlMillis, + senderRetryHorizonMillis: entry.senderRetryHorizonMillis, + minimumTerminalRetentionMillis: limits.minimumTerminalRetentionMillis + })) + ) ) if (result.ready) { yield* notify({ @@ -2466,43 +2786,48 @@ export const layerRelayHandlers = ( return yield* new PeerRpcError.SessionUnavailable() } const document = entry.documents.filter((candidate) => claim.documentIds.includes(candidate.documentId)) - yield* authorizeEntry(entry, "Receive", document) - const transition = yield* sql.submit( - "Terminal", - storeEffect( - reason === undefined - ? store.acknowledge({ - channel: claim.channel, - relayMessageId: claim.relayMessageId, - claimToken: claim.claimToken, - messageHash: claim.messageHash, - sessionGeneration: entry.generation, - recipient: entry.principal - }) - : store.reject({ - channel: claim.channel, - relayMessageId: claim.relayMessageId, - claimToken: claim.claimToken, - messageHash: claim.messageHash, - sessionGeneration: entry.generation, - recipient: entry.principal, - reason - }) + const receive = yield* authorizeEntry(entry, "Receive", document) + const transition = yield* withRelayGrantAdmission( + entry, + [receive], + sql.submit( + "Terminal", + storeEffect( + reason === undefined + ? store.acknowledge({ + channel: claim.channel, + relayMessageId: claim.relayMessageId, + claimToken: claim.claimToken, + messageHash: claim.messageHash, + sessionGeneration: entry.generation, + recipient: entry.principal + }) + : store.reject({ + channel: claim.channel, + relayMessageId: claim.relayMessageId, + claimToken: claim.claimToken, + messageHash: claim.messageHash, + sessionGeneration: entry.generation, + recipient: entry.principal, + reason + }) + ) ) ) - if (transition.status === "Stale") { - return yield* new PeerRpcError.SessionUnavailable() - } const selector = relaySelectorForEntry(entry) - const shouldNotify = yield* lock.withPermit(Effect.sync(() => { + const shouldNotify = yield* lock.withPermit(Effect.gen(function*() { entry.claims.delete(claim.relayMessageId) const key = relaySelectorKey(selector) const owner = workOwners.get(key) if (owner?._tag === "Entry" && owner.generation === entry.generation) { workOwners.delete(key) } + yield* PeerRpcObservability.setRelayActiveClaims(activeClaimCount()) return transition.ready || owner?._tag === "Entry" && owner.pending })) + if (transition.status === "Stale") { + return yield* new PeerRpcError.SessionUnavailable() + } if (shouldNotify) yield* notify(selector, transition.lane) }) ) @@ -2517,9 +2842,24 @@ export const layerRelayHandlers = ( })) yield* Fiber.interrupt(compensationFiber) yield* Fiber.interrupt(maintenanceFiber) - yield* Effect.forEach(workerFibers, Fiber.interrupt, { discard: true }) - yield* Queue.shutdown(newWork) - yield* Queue.shutdown(retryWork) + const activeWorkers = yield* lock.withPermit(Effect.sync(() => [...workerFibers])) + yield* Effect.forEach(activeWorkers, Fiber.interrupt, { discard: true }) + yield* lock.withPermit(Effect.gen(function*() { + workerFibers.clear() + yield* PeerRpcObservability.setRelayWorkers(workerCount) + })) + yield* newWorkLock.withPermit( + Queue.shutdown(newWork).pipe( + Effect.andThen(Queue.size(newWork)), + Effect.flatMap((size) => PeerRpcObservability.setRelayReadyQueueItems("New", size)) + ) + ) + yield* retryWorkLock.withPermit( + Queue.shutdown(retryWork).pipe( + Effect.andThen(Queue.size(retryWork)), + Effect.flatMap((size) => PeerRpcObservability.setRelayReadyQueueItems("Retry", size)) + ) + ) yield* Queue.shutdown(workWake) const active = yield* lock.withPermit(Effect.sync(() => [...sessions.values()])) yield* Effect.forEach(active, (entry) => detachEntry(entry, false), { @@ -2530,6 +2870,7 @@ export const layerRelayHandlers = ( workOwners.clear() selectors.clear() })) + yield* PeerRpcObservability.setRelayActiveClaims(0) yield* sql.shutdown yield* openLane.clear yield* pushLane.clear @@ -2583,7 +2924,7 @@ export const layerRelayServer = Layer.effectDiscard(Effect.gen(function*() { const ingress = yield* PeerRelayIngress.PeerRelayIngress let stopping = false const serverFiber = yield* Effect.forkIn( - RpcServer.make(PeerRelayRpc.Rpcs), + RpcServer.make(PeerRelayRpc.Rpcs, { disableFatalDefects: true }), scope ) const ingressFiber = yield* Effect.forkIn(ingress.await, scope) diff --git a/packages/local-rpc/src/RpcPeerTransport.ts b/packages/local-rpc/src/RpcPeerTransport.ts index d243423..2eba82d 100644 --- a/packages/local-rpc/src/RpcPeerTransport.ts +++ b/packages/local-rpc/src/RpcPeerTransport.ts @@ -5,6 +5,7 @@ import type * as Identity from "@lucas-barake/effect-local/Identity" import * as PeerTransport from "@lucas-barake/effect-local/PeerTransport" import type * as ReplicaDefinition from "@lucas-barake/effect-local/ReplicaDefinition" import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" +import * as ReplicaLimits from "@lucas-barake/effect-local/ReplicaLimits" import * as Cause from "effect/Cause" import * as Crypto from "effect/Crypto" import * as Deferred from "effect/Deferred" @@ -87,6 +88,26 @@ const adapterResult = (exit: Exit.Exit) => { : "Failure" as const } +const adapterAcknowledgeResult = ( + success: "Acknowledged" | "DeadLettered" +) => +(exit: Exit.Exit) => { + if (Exit.isSuccess(exit)) return success + const error = PeerRpcObservability.failure(exit) + if (error === undefined) return "Failure" as const + switch (error.reason._tag) { + case "ProtocolMismatch": + case "DocumentLineageChanged": + return "ProtocolRejected" as const + case "QuotaExceeded": + return "CapacityRejected" as const + case "StorageUnavailable": + return "Unavailable" as const + default: + return "Failure" as const + } +} + export const layer = ( client: PeerRpc.RpcClient, options: { @@ -305,7 +326,8 @@ const validateRelayOptions = (options: StoreAndForwardOptions) => const validateStoredMessage = ( event: PeerRelayRpc.StoredMessage, options: StoreAndForwardOptions, - crypto: Crypto.Crypto + crypto: Crypto.Crypto, + limits: ReplicaLimits.Values ) => Effect.gen(function*() { const expectedRecipient: PeerSyncEnvelope.RelayPeerPrincipal = { @@ -328,6 +350,10 @@ const validateStoredMessage = ( entry.documentId === event.document.documentId ) if (!selected) return yield* protocolFailure("selected relay document") + const decoded = yield* PeerSyncEnvelope.decodeSyncEnvelope( + event.payload, + limits + ).pipe(Effect.provideService(Crypto.Crypto, crypto)) const digest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope({ domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, version: PeerSyncEnvelope.relayOuterEnvelopeVersion, @@ -341,6 +367,7 @@ const validateStoredMessage = ( senderConnectionEpoch: event.sender.connectionEpoch, senderSequence: event.sender.sequence, document: event.document, + lineage: decoded.lineage, writerProvenance: event.writerProvenance, messageHash: event.messageHash, payload: event.payload @@ -359,6 +386,7 @@ export const layerStoreAndForward = ( Effect.gen(function*() { const runtime = yield* PeerRelayClientRuntime.PeerRelayClientRuntime const crypto = yield* Crypto.Crypto + const limits = yield* ReplicaLimits.ReplicaLimits const endpoint = { expectedLocal: options.expectedLocal, remote: { @@ -627,7 +655,7 @@ export const layerStoreAndForward = ( Stream.mapEffect((event) => event._tag !== "StoredMessage" ? Effect.fail(protocolFailure(event._tag)) - : validateStoredMessage(event, options, crypto).pipe( + : validateStoredMessage(event, options, crypto, limits).pipe( Effect.as( { message: event.payload, @@ -643,25 +671,45 @@ export const layerStoreAndForward = ( }, receiptRetentionMillis: options.receiptRetentionMillis, acknowledge: terminalCall( - client.AcknowledgeRelay({ - sessionId: handshake.sessionId, - relayMessageId: event.relayMessageId, - claimToken: event.claimToken, - messageHash: event.messageHash - }).pipe( - Effect.mapError(mapError), - Effect.andThen(runtime.signalReceiptPrune) - ) - ), - reject: (reason: PeerTransport.PermanentRejectReason) => - terminalCall( - client.RejectRelay({ + PeerRpcObservability.observeRelay({ + effect: client.AcknowledgeRelay({ sessionId: handshake.sessionId, relayMessageId: event.relayMessageId, claimToken: event.claimToken, - messageHash: event.messageHash, - reason - }).pipe(Effect.mapError(mapError)) + messageHash: event.messageHash + }).pipe( + Effect.mapError(mapError), + Effect.andThen(runtime.signalReceiptPrune) + ), + operation: "AdapterAcknowledge", + direction: "Receive", + facts: () => ({ + bytes: event.payload.byteLength, + items: 1, + version: event.payloadVersion + }), + result: adapterAcknowledgeResult("Acknowledged") + }) + ), + reject: (reason: PeerTransport.PermanentRejectReason) => + terminalCall( + PeerRpcObservability.observeRelay({ + effect: client.RejectRelay({ + sessionId: handshake.sessionId, + relayMessageId: event.relayMessageId, + claimToken: event.claimToken, + messageHash: event.messageHash, + reason + }).pipe(Effect.mapError(mapError)), + operation: "AdapterAcknowledge", + direction: "Receive", + facts: () => ({ + bytes: event.payload.byteLength, + items: 1, + version: event.payloadVersion + }), + result: adapterAcknowledgeResult("DeadLettered") + }) ) } satisfies PeerTransport.AcknowledgedDelivery ) diff --git a/packages/local-rpc/src/internal/peerRelayMigrations.ts b/packages/local-rpc/src/internal/peerRelayMigrations.ts index 0f4f36a..5a68521 100644 --- a/packages/local-rpc/src/internal/peerRelayMigrations.ts +++ b/packages/local-rpc/src/internal/peerRelayMigrations.ts @@ -8,6 +8,10 @@ import * as SqlSchema from "effect/unstable/sql/SqlSchema" export const relayCustodyChecksum = "sha256:effect-local-relay-custody-v1" export const relayMaintenanceIndexesChecksum = "sha256:effect-local-relay-maintenance-indexes-v1" +export const relayClaimAdmissionIndexChecksum = "sha256:effect-local-relay-claim-admission-index-v1" +export const relayClaimRecipientRouteChecksum = "sha256:effect-local-relay-claim-recipient-route-v1" + +const NonNegativeInt = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)) const custody = Effect.gen(function*() { const sql = yield* SqlClient.SqlClient @@ -131,9 +135,73 @@ const maintenanceIndexes = Effect.gen(function*() { VALUES (2, 'relay_maintenance_indexes', ${relayMaintenanceIndexesChecksum})` }) +const claimAdmissionIndex = Effect.gen(function*() { + const sql = yield* SqlClient.SqlClient + yield* sql`CREATE INDEX effect_local_relay_messages_claim_admission + ON effect_local_relay_messages( + tenant_id, + sender_subject_id, + sender_peer_id, + created_at, + message_id + ) + WHERE state = 'Pending' AND payload IS NOT NULL` + yield* sql`INSERT INTO effect_local_relay_migration_catalog (migration_id, name, checksum) + VALUES (3, 'relay_claim_admission_index', ${relayClaimAdmissionIndexChecksum})` +}) + +const claimRecipientRoute = Effect.gen(function*() { + const sql = yield* SqlClient.SqlClient + yield* sql`DROP INDEX effect_local_relay_messages_claim_admission` + yield* sql`ALTER TABLE effect_local_relay_messages + ADD COLUMN recipient_subject_id TEXT` + yield* sql`ALTER TABLE effect_local_relay_messages + ADD COLUMN recipient_peer_id TEXT` + yield* sql`UPDATE effect_local_relay_messages + SET recipient_subject_id = ( + SELECT c.recipient_subject_id + FROM effect_local_relay_channels c + WHERE c.channel_id = effect_local_relay_messages.channel_id + ), + recipient_peer_id = ( + SELECT c.recipient_peer_id + FROM effect_local_relay_channels c + WHERE c.channel_id = effect_local_relay_messages.channel_id + )` + const invalid = yield* SqlSchema.findOne({ + Request: Schema.Void, + Result: Schema.Struct({ count: NonNegativeInt }), + execute: () => + sql`SELECT COUNT(*) AS count + FROM effect_local_relay_messages + WHERE recipient_subject_id IS NULL OR recipient_peer_id IS NULL` + })(undefined) + if (invalid.count !== 0) { + return yield* new Migrator.MigrationError({ + kind: "BadState", + message: "Relay recipient route backfill is incomplete" + }) + } + yield* sql`CREATE INDEX effect_local_relay_messages_claim_admission + ON effect_local_relay_messages( + tenant_id, + sender_subject_id, + sender_peer_id, + recipient_subject_id, + recipient_peer_id, + created_at, + message_id + ) + WHERE state = 'Pending' AND payload IS NOT NULL` + yield* sql`INSERT INTO effect_local_relay_migration_catalog (migration_id, name, checksum) + VALUES (4, 'relay_claim_recipient_route', ${relayClaimRecipientRouteChecksum})` +}) + export const loader = Migrator.fromRecord({ "1_relay_custody": custody, - "2_relay_maintenance_indexes": maintenanceIndexes + "2_relay_maintenance_indexes": maintenanceIndexes, + "3_relay_claim_admission_index": claimAdmissionIndex, + "4_relay_claim_recipient_route": claimRecipientRoute }) const migrate = Migrator.make({})({ @@ -153,6 +221,18 @@ const expectedCatalog = [ name: "relay_maintenance_indexes", checksum: relayMaintenanceIndexesChecksum, label: "Relay maintenance indexes" + }, + { + id: 3, + name: "relay_claim_admission_index", + checksum: relayClaimAdmissionIndexChecksum, + label: "Relay claim admission index" + }, + { + id: 4, + name: "relay_claim_recipient_route", + checksum: relayClaimRecipientRouteChecksum, + label: "Relay claim recipient route" } ] as const diff --git a/packages/local-rpc/src/internal/peerRelaySqliteTransaction.ts b/packages/local-rpc/src/internal/peerRelaySqliteTransaction.ts index 2d89fbf..019d9c4 100644 --- a/packages/local-rpc/src/internal/peerRelaySqliteTransaction.ts +++ b/packages/local-rpc/src/internal/peerRelaySqliteTransaction.ts @@ -124,14 +124,35 @@ export const make = ( } return yield* Effect.suspend(() => { let bodyCause: Cause.Cause | undefined + let commitCause: Cause.Cause | undefined let rollbackCause: Cause.Cause | undefined + const commitTelemetryMarker = new Error("Peer relay transaction commit failed") const withTransaction = SqlClient.makeWithTransaction({ transactionService: sql.transactionService, spanAttributes: [["db.system", "sqlite"]], acquireConnection: acquireImmediate(sql, options), begin: () => Effect.void, savepoint: () => Effect.die(new Error("Nested relay transactions are forbidden")), - commit: (connection) => execute(connection, "COMMIT"), + commit: (connection) => + execute(connection, "COMMIT").pipe( + Effect.exit, + Effect.flatMap((commitExit) => { + if (Exit.isSuccess(commitExit)) { + return Effect.void + } + return execute(connection, "ROLLBACK").pipe( + Effect.exit, + Effect.flatMap((rollbackExit) => { + const combined = Exit.isFailure(rollbackExit) + ? Cause.combine(commitExit.cause, rollbackExit.cause) + : commitExit.cause + return Effect.sync(() => { + commitCause = combined + }).pipe(Effect.andThen(Effect.die(commitTelemetryMarker))) + }) + ) + }) + ), rollback: (connection) => execute(connection, "ROLLBACK").pipe( Effect.exit, @@ -154,10 +175,26 @@ export const make = ( ) ) return withTransaction(observed).pipe( - Effect.catchCause((cause) => { + Effect.exit, + Effect.flatMap((exit) => { + if (Exit.isSuccess(exit)) { + return commitCause === undefined + ? Effect.succeed(exit.value) + : Effect.failCause(commitCause) + } + const returnedCause = commitCause === undefined + ? exit.cause + : Cause.fromReasons( + exit.cause.reasons.filter((reason) => + !(Cause.isDieReason(reason) && reason.defect === commitTelemetryMarker) + ) + ) let combined = bodyCause === undefined - ? cause - : Cause.combine(bodyCause, cause) + ? returnedCause + : Cause.combine(bodyCause, returnedCause) + if (commitCause !== undefined) { + combined = Cause.combine(combined, commitCause) + } if (rollbackCause !== undefined) { combined = Cause.combine(combined, rollbackCause) } diff --git a/packages/local-rpc/src/internal/peerRelayStoreErrors.ts b/packages/local-rpc/src/internal/peerRelayStoreErrors.ts new file mode 100644 index 0000000..db8f8c4 --- /dev/null +++ b/packages/local-rpc/src/internal/peerRelayStoreErrors.ts @@ -0,0 +1,45 @@ +import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" +import * as Cause from "effect/Cause" +import * as Effect from "effect/Effect" +import type * as PlatformError from "effect/PlatformError" +import type * as Schema from "effect/Schema" +import type * as SqlError from "effect/unstable/sql/SqlError" +import type { NestedPeerRelayTransactionError } from "./peerRelaySqliteTransaction.js" + +export type StoreBoundaryError = + | ReplicaError.ReplicaError + | NestedPeerRelayTransactionError + | SqlError.SqlError + | Schema.SchemaError + | PlatformError.PlatformError + | Cause.NoSuchElementError + +const storageUnavailable = (cause: unknown) => + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + +const storageCorrupt = (cause: unknown) => + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + +export const mapStoreErrors = ( + effect: Effect.Effect +) => + Effect.catchCause( + effect, + (cause) => + Effect.failCause(Cause.map(cause, (error) => { + switch (error._tag) { + case "SqlError": + case "PlatformError": + return storageUnavailable(error) + case "SchemaError": + case "NoSuchElementError": + return storageCorrupt(error) + default: + return error + } + })) + ) diff --git a/packages/local-rpc/src/internal/peerRpcObservability.ts b/packages/local-rpc/src/internal/peerRpcObservability.ts index b18c69e..b729633 100644 --- a/packages/local-rpc/src/internal/peerRpcObservability.ts +++ b/packages/local-rpc/src/internal/peerRpcObservability.ts @@ -1,7 +1,11 @@ +import * as Clock from "effect/Clock" +import * as Duration from "effect/Duration" import * as Effect from "effect/Effect" import * as Exit from "effect/Exit" import * as Metric from "effect/Metric" import * as Option from "effect/Option" +import * as References from "effect/References" +import * as Tracer from "effect/Tracer" export type Operation = | "Authentication" @@ -12,6 +16,12 @@ export type Operation = | "Inbound" | "Outbound" | "Server" + | "RelayAdmit" + | "RelayClaim" + | "RelayAcknowledge" + | "RelayRelease" + | "RelayMaintenance" + | "AdapterAcknowledge" export type Result = | "Attempt" @@ -24,6 +34,194 @@ export type Result = | "Failure" | "Replaced" | "ShutdownClosed" + | "Accepted" + | "Duplicate" + | "Claimed" + | "Empty" + | "Delivered" + | "Acknowledged" + | "Released" + | "Expired" + | "DeadLettered" + | "Stale" + | "Unavailable" + +export type RelayResult = Extract< + Result, + | "Attempt" + | "Success" + | "AuthorizationDenied" + | "ProtocolRejected" + | "CapacityRejected" + | "Failure" + | "Accepted" + | "Duplicate" + | "Claimed" + | "Empty" + | "Delivered" + | "Acknowledged" + | "Released" + | "Expired" + | "DeadLettered" + | "Stale" + | "Unavailable" +> + +export type RelayOperation = Extract< + Operation, + | "RelayAdmit" + | "RelayClaim" + | "RelayAcknowledge" + | "RelayRelease" + | "RelayMaintenance" + | "AdapterAcknowledge" +> + +export type RelayDirection = "Send" | "Receive" + +export type RelayMaintenanceStage = + | "Recover" + | "Expire" + | "Repair" + | "Reconcile" + | "Collect" + | "Usage" + +export type RelayQuotaDomain = + | "Payload" + | "SenderPeer" + | "RecipientPeer" + | "RecipientSubject" + | "Tenant" + | "Shard" + +export interface RelayFacts { + readonly bytes?: number + readonly items?: number + readonly attempt?: number + readonly version?: number + readonly latencyMillis?: number +} + +const relayVersion = (version: number): "1" | "3" | "Unsupported" => + version === 1 ? "1" : version === 3 ? "3" : "Unsupported" + +const relaySpanName = (operation: RelayOperation) => { + switch (operation) { + case "RelayAdmit": + return "effect_local_rpc.relay.admit" + case "RelayClaim": + return "effect_local_rpc.relay.claim" + case "RelayAcknowledge": + return "effect_local_rpc.relay.acknowledge" + case "RelayRelease": + return "effect_local_rpc.relay.release" + case "RelayMaintenance": + return "effect_local_rpc.relay.maintenance" + case "AdapterAcknowledge": + return "effect_local_rpc.adapter.relay_acknowledge" + } +} + +const relayAttributes = ( + operation: RelayOperation, + direction: RelayDirection, + result: RelayResult, + stage?: RelayMaintenanceStage +) => ({ + operation, + direction, + result, + ...(stage === undefined ? {} : { stage }) +}) + +export const relayOutcomes = ( + operation: RelayOperation, + direction: RelayDirection, + result: RelayResult, + stage?: RelayMaintenanceStage +) => + Metric.counter("effect_local_rpc_relay_outcome_total", { + incremental: true, + attributes: relayAttributes(operation, direction, result, stage) + }) + +export const relayBytes = ( + operation: RelayOperation, + direction: RelayDirection, + result: RelayResult, + version: number, + stage?: RelayMaintenanceStage +) => + Metric.histogram("effect_local_rpc_relay_bytes", { + attributes: { + ...relayAttributes(operation, direction, result, stage), + version: relayVersion(version) + }, + boundaries: [0, 64, 256, 1_024, 4_096, 16_384, 65_536, 262_144, 1_048_576] + }) + +export const relayItems = ( + operation: RelayOperation, + direction: RelayDirection, + result: RelayResult, + stage?: RelayMaintenanceStage +) => + Metric.histogram("effect_local_rpc_relay_items", { + attributes: relayAttributes(operation, direction, result, stage), + boundaries: [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1_024] + }) + +export const relayAttempts = ( + operation: RelayOperation, + direction: RelayDirection, + result: RelayResult, + stage?: RelayMaintenanceStage +) => + Metric.histogram("effect_local_rpc_relay_attempt", { + attributes: relayAttributes(operation, direction, result, stage), + boundaries: [0, 1, 2, 3, 4, 8, 16, 32, 64, 128, 256, 512, 1_024] + }) + +export const relayDurationMillis = ( + operation: RelayOperation, + direction: RelayDirection, + result: RelayResult, + stage?: RelayMaintenanceStage +) => + Metric.histogram("effect_local_rpc_relay_duration_millis", { + attributes: relayAttributes(operation, direction, result, stage), + boundaries: [0, 1, 5, 10, 25, 50, 100, 250, 500, 1_000, 5_000, 30_000, 300_000] + }) + +export const relayLatencyMillis = ( + operation: "RelayAcknowledge" | "AdapterAcknowledge", + direction: RelayDirection, + result: RelayResult +) => + Metric.histogram("effect_local_rpc_relay_latency_millis", { + attributes: relayAttributes(operation, direction, result), + boundaries: [0, 1, 5, 10, 25, 50, 100, 250, 500, 1_000, 5_000, 30_000, 300_000, 3_600_000, 86_400_000] + }) + +export const relayQuotaRejections = (domain: RelayQuotaDomain) => + Metric.counter("effect_local_rpc_relay_quota_rejection_total", { + incremental: true, + attributes: { domain } + }) + +export const relayPendingItems = () => Metric.gauge("effect_local_rpc_relay_pending_items") + +export const relayPendingBytes = () => Metric.gauge("effect_local_rpc_relay_pending_bytes") + +export const relayActiveClaims = () => Metric.gauge("effect_local_rpc_relay_active_claims") + +export const relayWorkers = () => Metric.gauge("effect_local_rpc_relay_workers") + +export const relayReadyQueueItems = (lane: "New" | "Retry") => + Metric.gauge("effect_local_rpc_relay_ready_queue_items", { + attributes: { lane } + }) export const boundary = (operation: Operation, result: Result) => Metric.counter("effect_local_rpc_boundary_total", { @@ -73,6 +271,221 @@ export const recordSelectedDocuments = (amount: number) => Effect.provideService(Metric.CurrentMetricAttributes, {}) ) +const withoutMetricAttributes = (effect: Effect.Effect) => + effect.pipe(Effect.provideService(Metric.CurrentMetricAttributes, {})) + +const bestEffort = (effect: Effect.Effect) => effect.pipe(Effect.catchCause(() => Effect.void)) + +const safeSpan = (span: Tracer.Span): Tracer.Span => { + let status = span.status + return { + _tag: "Span", + name: span.name, + spanId: span.spanId, + traceId: span.traceId, + parent: span.parent, + annotations: span.annotations, + get status() { + return status + }, + attributes: span.attributes, + links: span.links, + sampled: span.sampled, + kind: span.kind, + end(endTime, exit) { + if (status._tag === "Ended") return + status = { + _tag: "Ended", + startTime: status.startTime, + endTime, + exit + } + try { + span.end(endTime, exit) + } catch { + // Telemetry must never replace the business outcome. + } + }, + attribute(key, value) { + try { + span.attribute(key, value) + } catch { + // Telemetry must never replace the business outcome. + } + }, + event(name, startTime, attributes) { + try { + span.event(name, startTime, attributes) + } catch { + // Telemetry must never replace the business outcome. + } + }, + addLinks(links) { + try { + span.addLinks(links) + } catch { + // Telemetry must never replace the business outcome. + } + } + } +} + +const safeTracer = (tracer: Tracer.Tracer): Tracer.Tracer => + Tracer.make({ + span: (options) => { + try { + return safeSpan(tracer.span(options)) + } catch { + return new Tracer.NativeSpan(options) + } + } + }) + +const safeClock = (clock: Clock.Clock): Clock.Clock => { + let lastMillis = 0 + let lastNanos = BigInt(0) + const currentTimeMillisUnsafe = () => { + try { + return lastMillis = clock.currentTimeMillisUnsafe() + } catch { + return lastMillis + } + } + const currentTimeNanosUnsafe = () => { + try { + return lastNanos = clock.currentTimeNanosUnsafe() + } catch { + return lastNanos + } + } + return { + currentTimeMillisUnsafe, + currentTimeMillis: Effect.sync(currentTimeMillisUnsafe), + currentTimeNanosUnsafe, + currentTimeNanos: Effect.sync(currentTimeNanosUnsafe), + sleep: (duration) => clock.sleep(duration) + } +} + +const useSafeSpan = ( + name: string, + attributes: Readonly>, + clock: Clock.Clock, + evaluate: (span: Tracer.Span) => Effect.Effect +) => + Effect.flatMap(Tracer.Tracer, (tracer) => + Effect.useSpan( + name, + { attributes }, + evaluate + ).pipe( + Effect.provideService(Tracer.Tracer, safeTracer(tracer)), + Effect.provideService(Clock.Clock, safeClock(clock)), + Effect.provideService(References.TracerSpanAnnotations, {}), + Effect.provideService(References.TracerSpanLinks, []) + )) + +export const recordRelayOutcome = (options: { + readonly operation: RelayOperation + readonly direction: RelayDirection + readonly result: RelayResult + readonly facts?: RelayFacts + readonly durationMillis?: number + readonly stage?: RelayMaintenanceStage +}) => { + const facts = options.facts ?? {} + const updates: Array> = [ + Metric.update( + relayOutcomes( + options.operation, + options.direction, + options.result, + options.stage + ), + 1 + ) + ] + if (facts.bytes !== undefined && facts.version !== undefined) { + updates.push(Metric.update( + relayBytes( + options.operation, + options.direction, + options.result, + facts.version, + options.stage + ), + facts.bytes + )) + } + if (facts.items !== undefined) { + updates.push(Metric.update( + relayItems( + options.operation, + options.direction, + options.result, + options.stage + ), + facts.items + )) + } + if (facts.attempt !== undefined) { + updates.push(Metric.update( + relayAttempts( + options.operation, + options.direction, + options.result, + options.stage + ), + facts.attempt + )) + } + if ( + facts.latencyMillis !== undefined && + (options.operation === "RelayAcknowledge" || + options.operation === "AdapterAcknowledge") + ) { + updates.push(Metric.update( + relayLatencyMillis( + options.operation, + options.direction, + options.result + ), + facts.latencyMillis + )) + } + if (options.durationMillis !== undefined) { + updates.push(Metric.update( + relayDurationMillis( + options.operation, + options.direction, + options.result, + options.stage + ), + options.durationMillis + )) + } + return Effect.all(updates, { discard: true }).pipe(withoutMetricAttributes) +} + +export const recordRelayQuotaRejection = (domain: RelayQuotaDomain) => + Metric.update(relayQuotaRejections(domain), 1).pipe(withoutMetricAttributes) + +export const setRelayPending = (items: number, amountBytes: number) => + Effect.all([ + Metric.update(relayPendingItems(), items), + Metric.update(relayPendingBytes(), amountBytes) + ], { discard: true }).pipe(withoutMetricAttributes) + +export const setRelayActiveClaims = (amount: number) => + Metric.update(relayActiveClaims(), amount).pipe(withoutMetricAttributes) + +export const setRelayWorkers = (amount: number) => Metric.update(relayWorkers(), amount).pipe(withoutMetricAttributes) + +export const setRelayReadyQueueItems = ( + lane: "New" | "Retry", + amount: number +) => Metric.update(relayReadyQueueItems(lane), amount).pipe(withoutMetricAttributes) + export const observe = (options: { readonly effect: Effect.Effect readonly operation: Operation @@ -80,27 +493,142 @@ export const observe = (options: { readonly attributes: Readonly> readonly result: (exit: Exit.Exit) => Result }) => - Effect.uninterruptibleMask((restore) => - Effect.useSpan( - options.spanName, - { - attributes: { - "rpc.operation": options.operation, - ...options.attributes - } - }, - (span) => - record(options.operation, "Attempt", 1).pipe( - Effect.andThen(restore(options.effect).pipe(Effect.exit)), - Effect.tap((exit) => { - const result = options.result(exit) - return Effect.sync(() => span.attribute("rpc.result", result)).pipe( - Effect.andThen(record(options.operation, result, 1)) + Effect.suspend(() => { + let captured: Exit.Exit | undefined + return Effect.uninterruptibleMask((restore) => + Effect.flatMap(Clock.Clock, (clock) => + useSafeSpan( + options.spanName, + { + "rpc.operation": options.operation, + ...options.attributes + }, + clock, + (span) => + Effect.suspend(() => record(options.operation, "Attempt", 1)).pipe( + bestEffort, + Effect.andThen( + restore(options.effect).pipe( + Effect.provideService(Clock.Clock, clock), + Effect.exit + ) + ), + Effect.tap((exit) => + Effect.sync(() => { + captured = exit + }) + ), + Effect.tap((exit) => + Effect.sync(() => options.result(exit)).pipe( + Effect.catchCause(() => Effect.succeed("Failure" as const)), + Effect.flatMap((result) => + Effect.sync(() => span.attribute("rpc.result", result)).pipe( + Effect.andThen(record(options.operation, result, 1)) + ) + ), + bestEffort + ) + ), + Effect.asVoid ) - }) - ) - ).pipe(Effect.flatten) - ) + ).pipe( + Effect.exit, + Effect.flatMap(() => + captured === undefined + ? restore(options.effect).pipe(Effect.provideService(Clock.Clock, clock)) + : captured + ) + )) + ) + }) + +export const observeRelay = (options: { + readonly effect: Effect.Effect + readonly operation: RelayOperation + readonly direction: RelayDirection + readonly stage?: RelayMaintenanceStage + readonly facts: (exit: Exit.Exit) => RelayFacts + readonly result: (exit: Exit.Exit) => RelayResult +}) => + Effect.suspend(() => { + let captured: Exit.Exit | undefined + return Effect.uninterruptibleMask((restore) => + Effect.flatMap(Clock.Clock, (clock) => + useSafeSpan( + relaySpanName(options.operation), + { + "rpc.operation": options.operation, + "rpc.direction": options.direction + }, + clock, + (span) => + Effect.suspend(() => + recordRelayOutcome({ + operation: options.operation, + direction: options.direction, + result: "Attempt", + ...(options.stage === undefined ? {} : { stage: options.stage }) + }) + ).pipe( + bestEffort, + Effect.andThen( + restore(options.effect).pipe( + Effect.provideService(Clock.Clock, clock), + Effect.exit, + Effect.tap((exit) => + Effect.sync(() => { + captured = exit + }) + ), + Effect.timed + ) + ), + Effect.tap(([duration, exit]) => { + const durationMillis = Duration.toMillis(duration) + return Effect.gen(function*() { + const result = yield* Effect.sync(() => options.result(exit)).pipe( + Effect.catchCause(() => Effect.succeed("Failure" as const)) + ) + const facts = yield* Effect.sync(() => options.facts(exit)).pipe( + Effect.catchCause(() => Effect.succeed({})) + ) + yield* Effect.sync(() => { + span.attribute("rpc.result", result) + if (facts.bytes !== undefined) span.attribute("rpc.bytes", facts.bytes) + if (facts.items !== undefined) span.attribute("rpc.items", facts.items) + if (facts.attempt !== undefined) span.attribute("rpc.attempt", facts.attempt) + if (facts.version !== undefined) { + span.attribute("rpc.version", relayVersion(facts.version)) + } + if (facts.latencyMillis !== undefined) { + span.attribute("rpc.latency_millis", facts.latencyMillis) + } + span.attribute("rpc.duration_millis", durationMillis) + }) + yield* recordRelayOutcome({ + operation: options.operation, + direction: options.direction, + result, + facts, + durationMillis, + ...(options.stage === undefined ? {} : { stage: options.stage }) + }) + }).pipe( + bestEffort + ) + }), + Effect.asVoid + ) + ).pipe( + Effect.exit, + Effect.flatMap(() => + captured === undefined + ? restore(options.effect).pipe(Effect.provideService(Clock.Clock, clock)) + : captured + ) + )) + ) + }) export const failure = (exit: Exit.Exit): E | undefined => Exit.findErrorOption(exit).pipe(Option.getOrUndefined) diff --git a/packages/local-rpc/test/PeerRelayAuthorization.test.ts b/packages/local-rpc/test/PeerRelayAuthorization.test.ts index 7a9fdf9..a7a0d70 100644 --- a/packages/local-rpc/test/PeerRelayAuthorization.test.ts +++ b/packages/local-rpc/test/PeerRelayAuthorization.test.ts @@ -64,7 +64,240 @@ const result = ( invalidated }) +const authorizationLayer = ( + authorize: PeerRelayAuthorization.Authorize, + authorizeUnsafeUnboundedAutomerge3Decode: PeerRelayAuthorization.AuthorizeUnsafeUnboundedAutomerge3Decode = + PeerRelayAuthorization.denyUnsafeUnboundedAutomerge3Decode +) => PeerRelayAuthorization.layer(authorize, authorizeUnsafeUnboundedAutomerge3Decode) + +const unsafeRequest = ( + direction: PeerRelayAuthorization.Direction, + selectedDocuments: PeerRelayAuthorization.UnsafeUnboundedAutomerge3DecodeRequest["documents"] = documents +): PeerRelayAuthorization.UnsafeUnboundedAutomerge3DecodeRequest => ({ + risk: PeerRelayAuthorization.unsafeUnboundedAutomerge3DecodeRisk, + direction, + principal, + remote, + documents: selectedDocuments +}) + +const unsafeGrant = ( + overrides: Partial = {} +): PeerRelayAuthorization.UnsafeUnboundedAutomerge3DecodeGrant => ({ + _tag: "UnsafeUnboundedAutomerge3DecodeGrant", + risk: PeerRelayAuthorization.unsafeUnboundedAutomerge3DecodeRisk, + principal, + remote: resolvedRemote, + direction: "Send", + documents: [documents[0], documents[1]], + validUntil: Number.MAX_SAFE_INTEGER, + invalidated: Effect.void, + ...overrides +}) + describe("PeerRelayAuthorization", () => { + it.effect("does not promote ordinary authorization into unsafe Automerge decode trust", () => { + let unsafeCalls = 0 + return Effect.gen(function*() { + const authorization = yield* PeerRelayAuthorization.PeerRelayAuthorization + yield* authorization.authorize(request("Send")) + assert.strictEqual(unsafeCalls, 0) + const error = yield* authorization.authorizeUnsafeUnboundedAutomerge3Decode( + unsafeRequest("Send") + ).pipe(Effect.flip) + assert.deepStrictEqual(error, new PeerRpcError.AccessDenied()) + assert.strictEqual(unsafeCalls, 1) + }).pipe( + Effect.provide( + authorizationLayer( + () => Effect.succeed(result()), + (input) => { + unsafeCalls++ + return PeerRelayAuthorization.denyUnsafeUnboundedAutomerge3Decode(input) + } + ) + ) + ) + }) + + it.effect.each(["Send", "Receive"] as const)( + "validates and canonicalizes the explicit unsafe Automerge decode grant for %s", + (direction) => + Effect.gen(function*() { + const invalidated = yield* Deferred.make() + let observed: + | PeerRelayAuthorization.UnsafeUnboundedAutomerge3DecodeRequest + | undefined + const authorization = yield* PeerRelayAuthorization.PeerRelayAuthorization.pipe( + Effect.provide( + authorizationLayer( + () => Effect.succeed(result()), + (input) => { + observed = input + return Effect.succeed(unsafeGrant({ + direction, + documents: [documents[0], documents[1]], + validUntil: 2_000, + invalidated: Deferred.await(invalidated) + })) + } + ) + ) + ) + yield* TestClock.setTime(1_000) + const grant = yield* authorization.authorizeUnsafeUnboundedAutomerge3Decode( + unsafeRequest(direction, [documents[1], documents[0]]) + ) + assert.deepStrictEqual(observed, unsafeRequest(direction, [documents[1], documents[0]])) + assert.deepStrictEqual(grant, { + _tag: "UnsafeUnboundedAutomerge3DecodeGrant", + risk: "Automerge3.3.2DecodeIsNotAllocationBounded", + principal, + remote: resolvedRemote, + direction, + documents: [documents[1], documents[0]], + validUntil: 2_000, + invalidated: grant.invalidated + }) + yield* Deferred.succeed(invalidated, undefined) + yield* grant.invalidated + }) + ) + + it.effect("rejects malformed and duplicate unsafe requests before policy evaluation", () => { + let calls = 0 + const cases = [ + { ...unsafeRequest("Send"), risk: "AutomergeDecodeIsSafe" }, + { ...unsafeRequest("Send"), direction: "Forward" }, + { ...unsafeRequest("Send"), principal: { ...principal, tenantId: "" } }, + { ...unsafeRequest("Send"), remote: { ...remote, subjectId: "" } }, + { ...unsafeRequest("Send"), documents: [] }, + { + ...unsafeRequest("Send"), + documents: [documents[0], documents[0]] + }, + { + ...unsafeRequest("Send"), + documents: [ + documents[0], + { documentType: note.name, documentId: taskId } + ] + }, + { + ...unsafeRequest("Send"), + documents: [{ ...documents[0], documentType: "" }] + }, + { + ...unsafeRequest("Send"), + documents: [{ ...documents[0], documentId: "doc_invalid" }] + } + ] as const + return Effect.gen(function*() { + const authorization = yield* PeerRelayAuthorization.PeerRelayAuthorization + for (const invalid of cases) { + const error = yield* authorization.authorizeUnsafeUnboundedAutomerge3Decode( + invalid as unknown as PeerRelayAuthorization.UnsafeUnboundedAutomerge3DecodeRequest + ).pipe(Effect.flip) + assert.deepStrictEqual(error, new PeerRpcError.AccessDenied()) + } + assert.strictEqual(calls, 0) + }).pipe( + Effect.provide( + authorizationLayer( + () => Effect.succeed(result()), + () => { + calls++ + return Effect.never + } + ) + ) + ) + }) + + it.effect("rejects principal remote direction and document substitutions in unsafe grants", () => + Effect.gen(function*() { + const cases = [ + unsafeGrant({ principal: { ...principal, tenantId: "other-tenant" } }), + unsafeGrant({ principal: { ...principal, subjectId: "other-subject" } }), + unsafeGrant({ principal: { ...principal, peerId: otherPeerId } }), + unsafeGrant({ remote: { ...resolvedRemote, tenantId: "other-tenant" } }), + unsafeGrant({ remote: { ...resolvedRemote, subjectId: "other-subject" } }), + unsafeGrant({ remote: { ...resolvedRemote, peerId: otherPeerId } }), + unsafeGrant({ direction: "Receive" }), + unsafeGrant({ documents: [documents[0]] }), + unsafeGrant({ documents: [documents[0], documents[0]] }), + unsafeGrant({ + documents: [ + documents[0], + { documentType: note.name, documentId: taskId } + ] + }), + unsafeGrant({ + documents: [ + documents[0], + { + documentType: note.name, + documentId: Identity.DocumentId.make( + "doc_00000000-0000-4000-8000-000000000003" + ) + } + ] + }) + ] + for (const invalid of cases) { + const authorization = yield* PeerRelayAuthorization.PeerRelayAuthorization.pipe( + Effect.provide( + authorizationLayer( + () => Effect.succeed(result()), + () => Effect.succeed(invalid) + ) + ) + ) + const error = yield* authorization.authorizeUnsafeUnboundedAutomerge3Decode( + unsafeRequest("Send") + ).pipe(Effect.flip) + assert.deepStrictEqual(error, new PeerRpcError.AccessDenied()) + assert.deepStrictEqual(Object.keys(error), ["_tag"]) + } + })) + + const missingTagGrant = Object.fromEntries( + Object.entries(unsafeGrant()).filter(([key]) => key !== "_tag") + ) + + it.effect.each( + [ + ["expired", unsafeGrant({ validUntil: 1_000 })], + ["past", unsafeGrant({ validUntil: 999 })], + ["NaN", unsafeGrant({ validUntil: Number.NaN })], + ["positive infinity", unsafeGrant({ validUntil: Number.POSITIVE_INFINITY })], + ["negative infinity", unsafeGrant({ validUntil: Number.NEGATIVE_INFINITY })], + ["missing tag", missingTagGrant], + ["wrong tag", { ...unsafeGrant(), _tag: "UnsafeAutomergeGrant" }], + ["wrong risk", { ...unsafeGrant(), risk: "AutomergeDecodeIsSafe" }], + ["missing invalidation", { ...unsafeGrant(), invalidated: undefined }], + ["malformed invalidation", { ...unsafeGrant(), invalidated: "not-an-effect" }] + ] as const + )("rejects an unsafe grant with %s", ([, grant]) => + Effect.gen(function*() { + yield* TestClock.setTime(1_000) + const authorization = yield* PeerRelayAuthorization.PeerRelayAuthorization + const error = yield* authorization.authorizeUnsafeUnboundedAutomerge3Decode( + unsafeRequest("Send") + ).pipe(Effect.flip) + assert.deepStrictEqual(error, new PeerRpcError.AccessDenied()) + }).pipe( + Effect.provide( + authorizationLayer( + () => Effect.succeed(result()), + () => + Effect.succeed( + grant as unknown as PeerRelayAuthorization.UnsafeUnboundedAutomerge3DecodeGrant + ) + ) + ) + )) + it.effect.each(["Send", "Receive"] as const)( "resolves the exact duplex endpoint and document set for %s", (direction) => { @@ -80,7 +313,7 @@ describe("PeerRelayAuthorization", () => { ]) }).pipe( Effect.provide( - PeerRelayAuthorization.layer((input) => { + authorizationLayer((input) => { observed = input return Effect.succeed(result([ { document: note, documentId: noteId }, @@ -122,7 +355,7 @@ describe("PeerRelayAuthorization", () => { assert.strictEqual(calls, 0) }).pipe( Effect.provide( - PeerRelayAuthorization.layer(() => { + authorizationLayer(() => { calls++ return Effect.never }) @@ -154,7 +387,7 @@ describe("PeerRelayAuthorization", () => { ] for (const selected of cases) { const authorization = yield* PeerRelayAuthorization.PeerRelayAuthorization.pipe( - Effect.provide(PeerRelayAuthorization.layer(() => Effect.succeed(result(selected)))) + Effect.provide(authorizationLayer(() => Effect.succeed(result(selected)))) ) const error = yield* authorization.authorize(request("Send")).pipe(Effect.flip) assert.deepStrictEqual(error, new PeerRpcError.AccessDenied()) @@ -171,7 +404,7 @@ describe("PeerRelayAuthorization", () => { for (const endpoint of endpoints) { const authorization = yield* PeerRelayAuthorization.PeerRelayAuthorization.pipe( Effect.provide( - PeerRelayAuthorization.layer(() => Effect.succeed(result(undefined, endpoint))) + authorizationLayer(() => Effect.succeed(result(undefined, endpoint))) ) ) const error = yield* authorization.authorize(request("Receive")).pipe(Effect.flip) @@ -195,7 +428,7 @@ describe("PeerRelayAuthorization", () => { assert.deepStrictEqual(error, new PeerRpcError.AccessDenied()) }).pipe( Effect.provide( - PeerRelayAuthorization.layer(() => Effect.succeed(result(undefined, undefined, validUntil))) + authorizationLayer(() => Effect.succeed(result(undefined, undefined, validUntil))) ) )) @@ -205,7 +438,7 @@ describe("PeerRelayAuthorization", () => { let allowed = true const authorization = yield* PeerRelayAuthorization.PeerRelayAuthorization.pipe( Effect.provide( - PeerRelayAuthorization.layer(() => + authorizationLayer(() => allowed ? Effect.succeed(result(undefined, undefined, 2_000, Deferred.await(invalidated))) : Effect.fail(new PeerRpcError.AccessDenied()) @@ -226,7 +459,7 @@ describe("PeerRelayAuthorization", () => { Effect.gen(function*() { const unavailable = yield* PeerRelayAuthorization.PeerRelayAuthorization.pipe( Effect.provide( - PeerRelayAuthorization.layer(() => Effect.fail(new PeerRpcError.ServerUnavailable())) + authorizationLayer(() => Effect.fail(new PeerRpcError.ServerUnavailable())) ) ) assert.deepStrictEqual( @@ -236,7 +469,7 @@ describe("PeerRelayAuthorization", () => { const defect = new Error("policy defect") const defective = yield* PeerRelayAuthorization.PeerRelayAuthorization.pipe( - Effect.provide(PeerRelayAuthorization.layer(() => Effect.die(defect))) + Effect.provide(authorizationLayer(() => Effect.die(defect))) ) const cause = yield* defective.authorize(request("Send")).pipe( Effect.sandbox, @@ -244,5 +477,37 @@ describe("PeerRelayAuthorization", () => { ) assert.isTrue(Cause.hasDies(cause)) assert.strictEqual(Cause.squash(cause), defect) + + const unsafeUnavailable = yield* PeerRelayAuthorization.PeerRelayAuthorization.pipe( + Effect.provide( + authorizationLayer( + () => Effect.succeed(result()), + () => Effect.fail(new PeerRpcError.ServerUnavailable()) + ) + ) + ) + assert.deepStrictEqual( + yield* unsafeUnavailable.authorizeUnsafeUnboundedAutomerge3Decode( + unsafeRequest("Send") + ).pipe(Effect.flip), + new PeerRpcError.ServerUnavailable() + ) + + const unsafeDefective = yield* PeerRelayAuthorization.PeerRelayAuthorization.pipe( + Effect.provide( + authorizationLayer( + () => Effect.succeed(result()), + () => Effect.die(defect) + ) + ) + ) + const unsafeCause = yield* unsafeDefective.authorizeUnsafeUnboundedAutomerge3Decode( + unsafeRequest("Send") + ).pipe( + Effect.sandbox, + Effect.flip + ) + assert.isTrue(Cause.hasDies(unsafeCause)) + assert.strictEqual(Cause.squash(unsafeCause), defect) })) }) diff --git a/packages/local-rpc/test/PeerRelayIngress.test.ts b/packages/local-rpc/test/PeerRelayIngress.test.ts index 3f372f9..b00fa8e 100644 --- a/packages/local-rpc/test/PeerRelayIngress.test.ts +++ b/packages/local-rpc/test/PeerRelayIngress.test.ts @@ -1,6 +1,6 @@ import { NodeSocket, NodeSocketServer } from "@effect/platform-node" import { assert, describe, it } from "@effect/vitest" -import { Context, Deferred, Effect, Exit, Fiber, Layer, Scope } from "effect" +import { Context, Deferred, Effect, Exit, Fiber, Layer, Scheduler, Scope } from "effect" import { TestClock } from "effect/testing" import type * as RpcMessage from "effect/unstable/rpc/RpcMessage" import * as RpcServer from "effect/unstable/rpc/RpcServer" @@ -307,6 +307,94 @@ describe("PeerRelayIngress", () => { yield* Scope.close(server.scope, Exit.void) }))) + it.effect("releases an interrupted reservation during ownership handoff", () => + Effect.scoped(Effect.gen(function*() { + const server = yield* buildServer(testLimits()) + const reservation = yield* Effect.forkDetach( + server.ingress.reserveOutbound(1).pipe( + Effect.provideService(Scheduler.MaxOpsBeforeYield, 8) + ) + ) + let observedHandoff = false + for (let attempt = 0; attempt < 10_000; attempt++) { + const usage = yield* server.ingress.usage + if (usage.reservedBytes === 1 && reservation.pollUnsafe() === undefined) { + observedHandoff = true + break + } + yield* Effect.yieldNow + } + assert.strictEqual(observedHandoff, true) + + yield* Fiber.interrupt(reservation) + assert.deepStrictEqual(yield* server.ingress.usage, { + connections: 0, + reservedBytes: 0, + byteReservationWaiters: 0 + }) + yield* Scope.close(server.scope, Exit.void) + }))) + + it.effect("retains active request capacity through a defect and rejects unknown client tags", () => + Effect.scoped(Effect.gen(function*() { + const received = Deferred.makeUnsafe() + const defectSent = Deferred.makeUnsafe() + const messages: Array = [] + const server = yield* buildServer(testLimits({ + maximumRawChunkBytes: 2 * 1_024 + })) + yield* Effect.forkIn( + server.protocol.run((clientId, message) => + Effect.gen(function*() { + messages.push(message) + if (message._tag !== "Request") return + if (message.id === 1) { + Deferred.doneUnsafe(received, Effect.void) + return + } + if (message.id === 2) { + yield* server.protocol.send(clientId, { + _tag: "Defect", + defect: "request-defect" + }) + Deferred.doneUnsafe(defectSent, Effect.void) + } + }) + ), + server.scope + ) + const clientScope = yield* Scope.make() + const client = yield* connect(tcpPort(server.ingress.address), clientScope) + const encodedRequest = frame(request(1, "x".repeat(1_024))) + + yield* client.write(encodedRequest) + yield* Deferred.await(received) + assert.strictEqual( + (yield* server.ingress.usage).reservedBytes, + encodedRequest.byteLength - 4 + ) + + yield* client.write(frame(request(2))) + yield* Deferred.await(defectSent) + assert.strictEqual( + (yield* server.ingress.usage).reservedBytes, + encodedRequest.byteLength - 4 + ) + + yield* client.write(frame({ _tag: "Bogus" })) + const clientExit = yield* Fiber.await(client.readFiber!) + + assert.match(clientExit._tag, /^(Failure|Success)$/) + assert.deepStrictEqual(messages.map((message) => message._tag), ["Request", "Request"]) + assert.deepStrictEqual(yield* server.ingress.usage, { + connections: 0, + reservedBytes: 0, + byteReservationWaiters: 0 + }) + yield* Scope.close(clientScope, Exit.void) + yield* Scope.close(server.scope, Exit.void) + }))) + it.effect("reserves outbound capacity before serializing a response", () => Effect.scoped(Effect.gen(function*() { const values = testLimits() @@ -399,7 +487,7 @@ describe("PeerRelayIngress", () => { values.maximumSharedPayloadBytes - inboundBytes - 1 ) yield* Deferred.succeed(ready, { blocker, clientId }) - }) + }).pipe(Effect.orDie) ), server.scope ) @@ -456,7 +544,7 @@ describe("PeerRelayIngress", () => { yield* Deferred.succeed(transferSucceeded, succeeded) }) ) - }) + }).pipe(Effect.orDie) ), server.scope ) @@ -480,6 +568,65 @@ describe("PeerRelayIngress", () => { yield* Scope.close(server.scope, Exit.void) }))) + it.effect("keeps accepting connections when a waiting protocol runner takes ownership", () => + Effect.scoped(Effect.gen(function*() { + const firstReceived = Deferred.makeUnsafe() + const secondReceived = Deferred.makeUnsafe() + const server = yield* buildServer(testLimits()) + const firstRunner = yield* Effect.forkIn( + server.protocol.run(() => Deferred.succeed(firstReceived, undefined)), + server.scope + ) + const firstClientScope = yield* Scope.make() + const firstClient = yield* connect(tcpPort(server.ingress.address), firstClientScope) + yield* firstClient.write(frame({ _tag: "Ping" })) + yield* Deferred.await(firstReceived) + + yield* Effect.forkIn( + server.protocol.run(() => Deferred.succeed(secondReceived, undefined)), + server.scope + ) + yield* Effect.yieldNow + yield* Fiber.interrupt(firstRunner) + + const secondClientScope = yield* Scope.make() + const secondClient = yield* connect(tcpPort(server.ingress.address), secondClientScope) + yield* secondClient.write(frame({ _tag: "Ping" })) + yield* Deferred.await(secondReceived) + + assert.strictEqual((yield* server.ingress.usage).connections, 2) + yield* Scope.close(secondClientScope, Exit.void) + yield* Scope.close(firstClientScope, Exit.void) + yield* Scope.close(server.scope, Exit.void) + }))) + + it.effect("reserves disconnect queue capacity before accepting a connection", () => + Effect.scoped(Effect.gen(function*() { + const server = yield* buildServer(testLimits({ + maxRelayConnections: 1, + maximumByteReservationWaiters: 1, + maxSessionsPerSubject: 1, + maxInFlightOpen: 1, + maxInFlightOpenPerSubject: 1 + })) + yield* Effect.forkIn(server.protocol.run(() => Effect.void), server.scope) + + const firstScope = yield* Scope.make() + yield* connect(tcpPort(server.ingress.address), firstScope) + yield* awaitConnections(server.ingress, 1) + yield* Scope.close(firstScope, Exit.void) + yield* awaitConnections(server.ingress, 0) + + const secondScope = yield* Scope.make() + const second = yield* connect(tcpPort(server.ingress.address), secondScope) + const secondExit = yield* Fiber.await(second.readFiber!) + + assert.match(secondExit._tag, /^(Failure|Success)$/) + assert.strictEqual((yield* server.ingress.usage).connections, 0) + yield* Scope.close(secondScope, Exit.void) + yield* Scope.close(server.scope, Exit.void) + }))) + it.effect("rejects pre-run connection churn without filling the disconnect queue", () => Effect.scoped(Effect.gen(function*() { const values = testLimits({ diff --git a/packages/local-rpc/test/PeerRelayLimits.test.ts b/packages/local-rpc/test/PeerRelayLimits.test.ts index acb7a39..9551019 100644 --- a/packages/local-rpc/test/PeerRelayLimits.test.ts +++ b/packages/local-rpc/test/PeerRelayLimits.test.ts @@ -9,7 +9,8 @@ describe("PeerRelayLimits", () => { Effect.gen(function*() { const limits = yield* PeerRelayLimits.PeerRelayLimits assert.deepStrictEqual(limits, PeerRelayLimits.defaults) - assert.strictEqual(Object.keys(limits).length, 93) + assert.strictEqual(Object.keys(limits).length, 94) + assert.strictEqual(limits.maximumDeliveryAttempts, 16) }).pipe(Effect.provide(PeerRelayLimits.layerDefaults))) it.effect.each( @@ -17,6 +18,7 @@ describe("PeerRelayLimits", () => { ["maxActiveMessagesPerShard", 0, "Expected a value greater than 0, got 0"], ["maxRetainedBytesPerTenant", -1, "Expected a value greater than 0, got -1"], ["claimLeaseMillis", 1.5, "Expected an integer, got 1.5"], + ["maximumDeliveryAttempts", 0, "Expected a value greater than 0, got 0"], ["openRatePerSecond", Number.NaN, "Expected a finite number, got NaN"], [ "terminalResponseRatePerSecond", diff --git a/packages/local-rpc/test/PeerRelaySqliteTransaction.test.ts b/packages/local-rpc/test/PeerRelaySqliteTransaction.test.ts index 441d194..340e2c2 100644 --- a/packages/local-rpc/test/PeerRelaySqliteTransaction.test.ts +++ b/packages/local-rpc/test/PeerRelaySqliteTransaction.test.ts @@ -1,3 +1,4 @@ +import { SqliteClient } from "@effect/sql-sqlite-node" import { assert, describe, it } from "@effect/vitest" import * as Cause from "effect/Cause" import * as Effect from "effect/Effect" @@ -162,4 +163,116 @@ describe("peerRelaySqliteTransaction", () => { assert.strictEqual(defects.has(closeDefect), true) } })) + + it.effect("rolls back a deferred constraint commit failure before reusing the connection", () => + Effect.gen(function*() { + const sql = yield* SqlClient.SqlClient + const transaction = make(sql, { + maxAcquireAttempts: 1, + acquireRetryBaseDelayMillis: 0, + acquireRetryMaximumDelayMillis: 0 + }) + yield* sql`PRAGMA foreign_keys = ON` + yield* sql`CREATE TABLE parents ( + parent_id INTEGER PRIMARY KEY + )` + yield* sql`CREATE TABLE children ( + child_id INTEGER PRIMARY KEY, + parent_id INTEGER NOT NULL, + FOREIGN KEY (parent_id) REFERENCES parents(parent_id) + DEFERRABLE INITIALLY DEFERRED + )` + + const failed = yield* transaction( + sql`INSERT INTO children (child_id, parent_id) VALUES (1, 99)` + ).pipe(Effect.exit) + assert.strictEqual(Exit.isFailure(failed), true) + + const succeeded = yield* transaction( + sql`INSERT INTO parents (parent_id) VALUES (1)` + ).pipe(Effect.exit) + assert.strictEqual(Exit.isSuccess(succeeded), true) + + const children = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count FROM children + ` + const parents = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count FROM parents + ` + assert.strictEqual(children[0]?.count, 0) + assert.strictEqual(parents[0]?.count, 1) + }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:", disableWAL: true })))) + + it.effect("preserves commit, rollback, and transaction scope close failures", () => + Effect.gen(function*() { + const commitDefect = new Error("commit failed") + const rollbackDefect = new Error("rollback failed") + const closeDefect = new Error("scope close failed") + const connection = { + executeUnprepared: (statement: string) => { + if (statement === "COMMIT") { + return Effect.die(commitDefect) + } + if (statement === "ROLLBACK") { + return Effect.die(rollbackDefect) + } + return Effect.succeed([]) + } + } as unknown as SqlConnection.Connection + const reserve = Effect.gen(function*() { + const scope = yield* Scope.Scope + yield* Scope.addFinalizer(scope, Effect.die(closeDefect)) + return connection + }) + const sql = { + reserve, + transactionService: SqlClient.TransactionConnection(999_028) + } as unknown as SqlClient.SqlClient + const exit = yield* make(sql, { + maxAcquireAttempts: 1, + acquireRetryBaseDelayMillis: 0, + acquireRetryMaximumDelayMillis: 0 + })(Effect.void).pipe(Effect.exit) + assert.strictEqual(Exit.isFailure(exit), true) + if (Exit.isFailure(exit)) { + const defects = new Set( + exit.cause.reasons + .filter(Cause.isDieReason) + .map((reason) => reason.defect) + ) + assert.strictEqual(defects.has(commitDefect), true) + assert.strictEqual(defects.has(rollbackDefect), true) + assert.strictEqual(defects.has(closeDefect), true) + } + })) + + it.effect("does not duplicate a typed commit failure as a defect", () => + Effect.gen(function*() { + const commitFailure = new SqlError.SqlError({ + reason: new SqlError.UnknownError({ cause: new Error("commit failed") }) + }) + const connection = { + executeUnprepared: (statement: string) => + statement === "COMMIT" + ? Effect.fail(commitFailure) + : Effect.succeed([]) + } as unknown as SqlConnection.Connection + const sql = { + reserve: Effect.succeed(connection), + transactionService: SqlClient.TransactionConnection(999_029) + } as unknown as SqlClient.SqlClient + const exit = yield* make(sql, { + maxAcquireAttempts: 1, + acquireRetryBaseDelayMillis: 0, + acquireRetryMaximumDelayMillis: 0 + })(Effect.void).pipe(Effect.exit) + assert.strictEqual(Exit.isFailure(exit), true) + if (Exit.isFailure(exit)) { + assert.strictEqual(Result.getOrThrow(Cause.findError(exit.cause)), commitFailure) + assert.strictEqual( + exit.cause.reasons.some((reason) => Cause.isDieReason(reason) && reason.defect === commitFailure), + false + ) + } + })) }) diff --git a/packages/local-rpc/test/PeerRelayStore.test.ts b/packages/local-rpc/test/PeerRelayStore.test.ts index 6eae6ec..85a75b1 100644 --- a/packages/local-rpc/test/PeerRelayStore.test.ts +++ b/packages/local-rpc/test/PeerRelayStore.test.ts @@ -3,12 +3,15 @@ import { SqliteClient } from "@effect/sql-sqlite-node" import { assert, describe, it } from "@effect/vitest" import * as Identity from "@lucas-barake/effect-local/Identity" import * as Effect from "effect/Effect" +import * as Exit from "effect/Exit" import * as Layer from "effect/Layer" +import * as Metric from "effect/Metric" import * as Option from "effect/Option" import * as SqlClient from "effect/unstable/sql/SqlClient" import { rmSync } from "node:fs" import { tmpdir } from "node:os" import { join } from "node:path" +import * as PeerRpcObservability from "../src/internal/peerRpcObservability.js" import * as PeerRelayLimits from "../src/PeerRelayLimits.js" import * as PeerRelayRpc from "../src/PeerRelayRpc.js" import * as PeerRelayStore from "../src/PeerRelayStore.js" @@ -17,17 +20,246 @@ const peer = (value: string) => Identity.PeerId.make(`peer_00000000-0000-4000-80 const relayId = (value: string) => Identity.RelayMessageId.make(`rly_00000000-0000-4000-8000-${value}`) const documentId = (value: string) => Identity.DocumentId.make(`doc_00000000-0000-4000-8000-${value}`) -const makeLayer = (filename: string) => { +const makeLayer = ( + filename: string, + limits: PeerRelayLimits.Values = PeerRelayLimits.defaults +) => { const base = Layer.mergeAll( SqliteClient.layer({ filename }), NodeCrypto.layer, - PeerRelayLimits.layerDefaults + PeerRelayLimits.layer(limits) ) const store = PeerRelayStore.layerSqlite.pipe(Layer.provide(base)) return Layer.merge(base, store) } +const withStore = ( + effect: Effect.Effect< + A, + E, + PeerRelayStore.PeerRelayStore | SqlClient.SqlClient + >, + limits: PeerRelayLimits.Values = PeerRelayLimits.defaults +) => + Effect.gen(function*() { + const filename = join(tmpdir(), `effect-local-relay-${globalThis.crypto.randomUUID()}.sqlite`) + yield* Effect.addFinalizer(() => + Effect.sync(() => { + rmSync(filename, { force: true }) + rmSync(`${filename}-shm`, { force: true }) + rmSync(`${filename}-wal`, { force: true }) + }) + ) + return yield* Effect.scoped(effect.pipe(Effect.provide(makeLayer(filename, limits)))) + }) + describe("PeerRelayStore", () => { + it.effect("records fixed relay outcomes, acknowledgement latency, and exact pending gauges", () => { + const registry: Metric.MetricRegistry = new Map() + const limits = PeerRelayLimits.Values.make({ + ...PeerRelayLimits.defaults, + maxActiveMessagesPerSenderPeer: 1, + maxRetainedRowsPerSenderPeer: 1 + }) + const metricValue = (metric: Metric.Metric) => + Metric.value(metric).pipe( + Effect.provideService(Metric.CurrentMetricAttributes, {}) + ) + return withStore( + Effect.gen(function*() { + const store = yield* PeerRelayStore.PeerRelayStore + const sql = yield* SqlClient.SqlClient + const channel = PeerRelayStore.ChannelKey.make({ + tenantId: "tenant-observe", + senderSubjectId: "sender-observe", + senderPeerId: peer("000000000051"), + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + recipientSubjectId: "recipient-observe", + recipientPeerId: peer("000000000052") + }) + const admission = PeerRelayStore.Admission.make({ + channel, + relayMessageId: relayId("000000000051"), + relayPeerId: peer("000000000053"), + documentIds: [documentId("000000000051")], + senderConnectionEpoch: "epoch-observe", + senderSequence: 0, + payloadVersion: 1, + messageHash: "message-hash-observe", + outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make("5".repeat(64)), + payload: new Uint8Array([5]), + messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, + senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, + minimumTerminalRetentionMillis: PeerRelayLimits.defaults.minimumTerminalRetentionMillis + }) + yield* store.admit(admission) + const claimed = yield* store.claim({ + recipient: { + tenantId: channel.tenantId, + subjectId: channel.recipientSubjectId, + peerId: channel.recipientPeerId + }, + sender: { + subjectId: channel.senderSubjectId, + peerId: channel.senderPeerId + }, + sessionGeneration: 1, + authorizedDocumentIds: admission.documentIds + }) + assert.strictEqual(Option.isSome(claimed.message), true) + if (Option.isNone(claimed.message)) return + const message = claimed.message.value + yield* sql`UPDATE effect_local_relay_messages + SET created_at = 0 + WHERE message_id = ${message.rowId}` + yield* store.acknowledge({ + channel, + relayMessageId: message.relayMessageId, + claimToken: message.claimToken, + messageHash: message.messageHash, + sessionGeneration: message.sessionGeneration, + recipient: { + tenantId: channel.tenantId, + subjectId: channel.recipientSubjectId, + peerId: channel.recipientPeerId + } + }) + yield* store.usage() + const rejected = yield* store.admit(PeerRelayStore.Admission.make({ + ...admission, + relayMessageId: relayId("000000000054"), + outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make("4".repeat(64)) + })).pipe(Effect.exit) + assert.strictEqual(Exit.isFailure(rejected), true) + assert.strictEqual( + (yield* metricValue( + PeerRpcObservability.relayOutcomes("RelayAdmit", "Send", "Accepted") + )).count, + 1 + ) + assert.strictEqual( + (yield* metricValue( + PeerRpcObservability.relayOutcomes("RelayClaim", "Receive", "Claimed") + )).count, + 1 + ) + assert.strictEqual( + (yield* metricValue( + PeerRpcObservability.relayOutcomes( + "RelayAcknowledge", + "Receive", + "Acknowledged" + ) + )).count, + 1 + ) + assert.strictEqual( + (yield* metricValue( + PeerRpcObservability.relayLatencyMillis( + "RelayAcknowledge", + "Receive", + "Acknowledged" + ) + )).count, + 1 + ) + assert.strictEqual( + (yield* metricValue(PeerRpcObservability.relayPendingItems())).value, + 0 + ) + assert.strictEqual( + (yield* metricValue(PeerRpcObservability.relayPendingBytes())).value, + 0 + ) + assert.strictEqual( + (yield* metricValue( + PeerRpcObservability.relayQuotaRejections("SenderPeer") + )).count, + 1 + ) + }), + limits + ).pipe(Effect.provideService(Metric.MetricRegistry, registry)) + }) + + it.effect("upgrades the exact version 3 claim index and backfills recipient routes", () => + Effect.gen(function*() { + const filename = join(tmpdir(), `effect-local-relay-${globalThis.crypto.randomUUID()}.sqlite`) + yield* Effect.addFinalizer(() => + Effect.sync(() => { + rmSync(filename, { force: true }) + rmSync(`${filename}-shm`, { force: true }) + rmSync(`${filename}-wal`, { force: true }) + }) + ) + const channel = PeerRelayStore.ChannelKey.make({ + tenantId: "tenant-upgrade", + senderSubjectId: "sender-upgrade", + senderPeerId: peer("000000000091"), + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + recipientSubjectId: "recipient-upgrade", + recipientPeerId: peer("000000000092") + }) + yield* Effect.scoped( + Effect.gen(function*() { + const store = yield* PeerRelayStore.PeerRelayStore + const sql = yield* SqlClient.SqlClient + yield* store.admit(PeerRelayStore.Admission.make({ + channel, + relayMessageId: relayId("000000000091"), + relayPeerId: peer("000000000093"), + documentIds: [documentId("000000000091")], + senderConnectionEpoch: "epoch-upgrade", + senderSequence: 0, + payloadVersion: 1, + messageHash: "message-hash-upgrade", + outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make("9".repeat(64)), + payload: new Uint8Array([9]), + messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, + senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, + minimumTerminalRetentionMillis: PeerRelayLimits.defaults.minimumTerminalRetentionMillis + })) + yield* sql`DROP INDEX effect_local_relay_messages_claim_admission` + yield* sql`ALTER TABLE effect_local_relay_messages DROP COLUMN recipient_peer_id` + yield* sql`ALTER TABLE effect_local_relay_messages DROP COLUMN recipient_subject_id` + yield* sql`CREATE INDEX effect_local_relay_messages_claim_admission + ON effect_local_relay_messages( + tenant_id, + sender_subject_id, + sender_peer_id, + created_at, + message_id + ) + WHERE state = 'Pending' AND payload IS NOT NULL` + yield* sql`DELETE FROM effect_local_relay_migration_catalog WHERE migration_id = 4` + yield* sql`DELETE FROM effect_local_relay_migrations WHERE migration_id = 4` + }).pipe(Effect.provide(makeLayer(filename))) + ) + yield* Effect.scoped( + Effect.gen(function*() { + const sql = yield* SqlClient.SqlClient + const routes = yield* sql<{ + readonly recipientSubjectId: string + readonly recipientPeerId: string + }>`SELECT + recipient_subject_id AS recipientSubjectId, + recipient_peer_id AS recipientPeerId + FROM effect_local_relay_messages` + assert.deepStrictEqual(routes, [{ + recipientSubjectId: channel.recipientSubjectId, + recipientPeerId: channel.recipientPeerId + }]) + const catalog = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count + FROM effect_local_relay_migration_catalog + WHERE migration_id = 4 + AND name = 'relay_claim_recipient_route' + ` + assert.strictEqual(catalog[0]?.count, 1) + }).pipe(Effect.provide(makeLayer(filename))) + ) + })) + it.effect("migrates a real WAL database and fences claim payload loading and terminal duplicates", () => Effect.gen(function*() { const filename = join(tmpdir(), `effect-local-relay-${globalThis.crypto.randomUUID()}.sqlite`) @@ -281,4 +513,753 @@ describe("PeerRelayStore", () => { }).pipe(Effect.provide(makeLayer(filename))) ) })) + + it.effect("collects reservations when SQLite foreign keys are disabled", () => + Effect.gen(function*() { + const filename = join(tmpdir(), `effect-local-relay-${globalThis.crypto.randomUUID()}.sqlite`) + yield* Effect.addFinalizer(() => + Effect.sync(() => { + rmSync(filename, { force: true }) + rmSync(`${filename}-shm`, { force: true }) + rmSync(`${filename}-wal`, { force: true }) + }) + ) + yield* Effect.scoped( + Effect.gen(function*() { + const store = yield* PeerRelayStore.PeerRelayStore + const sql = yield* SqlClient.SqlClient + yield* sql`PRAGMA foreign_keys = OFF` + const foreignKeys = yield* sql<{ readonly foreign_keys: number }>`PRAGMA foreign_keys` + assert.strictEqual(foreignKeys[0]?.foreign_keys, 0) + const channel = PeerRelayStore.ChannelKey.make({ + tenantId: "tenant", + senderSubjectId: "sender", + senderPeerId: peer("000000000011"), + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + recipientSubjectId: "recipient", + recipientPeerId: peer("000000000012") + }) + const admission = PeerRelayStore.Admission.make({ + channel, + relayMessageId: relayId("000000000011"), + relayPeerId: peer("000000000013"), + documentIds: [documentId("000000000011")], + senderConnectionEpoch: "epoch-1", + senderSequence: 0, + payloadVersion: 1, + messageHash: "message-hash", + outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make("e".repeat(64)), + payload: new Uint8Array([1]), + messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, + senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, + minimumTerminalRetentionMillis: PeerRelayLimits.defaults.minimumTerminalRetentionMillis + }) + yield* store.admit(admission) + const claimed = yield* store.claim({ + recipient: { + tenantId: channel.tenantId, + subjectId: channel.recipientSubjectId, + peerId: channel.recipientPeerId + }, + sender: { + subjectId: channel.senderSubjectId, + peerId: channel.senderPeerId + }, + sessionGeneration: 1, + authorizedDocumentIds: admission.documentIds + }) + assert.strictEqual(Option.isSome(claimed.message), true) + if (Option.isNone(claimed.message)) return + const message = claimed.message.value + yield* store.acknowledge({ + channel, + relayMessageId: message.relayMessageId, + claimToken: message.claimToken, + messageHash: message.messageHash, + sessionGeneration: message.sessionGeneration, + recipient: { + tenantId: channel.tenantId, + subjectId: channel.recipientSubjectId, + peerId: channel.recipientPeerId + } + }) + yield* sql`UPDATE effect_local_relay_messages + SET deduplicate_until = 0 + WHERE message_id = ${message.rowId}` + yield* sql`INSERT INTO effect_local_relay_usage ( + scope_kind, + scope_key, + active_count, + active_bytes, + retained_count, + retained_bytes + ) VALUES ('Tenant', 'unrelated-zero', 0, 0, 0, 0)` + yield* store.collect({ batchSize: 10 }) + const reservations = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count + FROM effect_local_relay_reservations + WHERE message_id = ${message.rowId} + ` + assert.strictEqual(reservations[0]?.count, 0) + const unrelated = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count + FROM effect_local_relay_usage + WHERE scope_kind = 'Tenant' + AND scope_key = 'unrelated-zero' + ` + assert.strictEqual(unrelated[0]?.count, 1) + }).pipe(Effect.provide(makeLayer(filename))) + ) + })) + + it.effect("repairs an active message whose channel is missing", () => + withStore(Effect.gen(function*() { + const store = yield* PeerRelayStore.PeerRelayStore + const sql = yield* SqlClient.SqlClient + const channel = PeerRelayStore.ChannelKey.make({ + tenantId: "tenant-orphan", + senderSubjectId: "sender-orphan", + senderPeerId: peer("000000000081"), + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + recipientSubjectId: "recipient-orphan", + recipientPeerId: peer("000000000082") + }) + yield* store.admit(PeerRelayStore.Admission.make({ + channel, + relayMessageId: relayId("000000000081"), + relayPeerId: peer("000000000083"), + documentIds: [documentId("000000000081")], + senderConnectionEpoch: "epoch-orphan", + senderSequence: 0, + payloadVersion: 1, + messageHash: "message-hash-orphan", + outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make("8".repeat(64)), + payload: new Uint8Array([8]), + messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, + senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, + minimumTerminalRetentionMillis: PeerRelayLimits.defaults.minimumTerminalRetentionMillis + })) + yield* sql`PRAGMA foreign_keys = OFF` + yield* sql`DELETE FROM effect_local_relay_channels` + yield* store.repair({ batchSize: 1 }) + const rows = yield* sql<{ + readonly state: string + readonly payload: Uint8Array | null + }>`SELECT state, payload FROM effect_local_relay_messages` + assert.deepStrictEqual(rows, [{ + state: "DeadLettered", + payload: null + }]) + assert.deepStrictEqual(yield* store.usage(), { + activeCount: 0, + activeBytes: 0, + retainedCount: 1, + retainedBytes: 1 + }) + }))) + + it.effect("dead letters a released claim at the delivery attempt cap and preserves it across restart", () => + Effect.gen(function*() { + const filename = join(tmpdir(), `effect-local-relay-${globalThis.crypto.randomUUID()}.sqlite`) + const limits = PeerRelayLimits.Values.make({ + ...PeerRelayLimits.defaults, + maximumDeliveryAttempts: 1 + }) + yield* Effect.addFinalizer(() => + Effect.sync(() => { + rmSync(filename, { force: true }) + rmSync(`${filename}-shm`, { force: true }) + rmSync(`${filename}-wal`, { force: true }) + }) + ) + const channel = PeerRelayStore.ChannelKey.make({ + tenantId: "tenant-release-cap", + senderSubjectId: "sender-release-cap", + senderPeerId: peer("000000000071"), + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + recipientSubjectId: "recipient-release-cap", + recipientPeerId: peer("000000000072") + }) + const admission = PeerRelayStore.Admission.make({ + channel, + relayMessageId: relayId("000000000071"), + relayPeerId: peer("000000000073"), + documentIds: [documentId("000000000071")], + senderConnectionEpoch: "epoch-release-cap", + senderSequence: 0, + payloadVersion: 1, + messageHash: "message-hash-release-cap", + outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make("7".repeat(64)), + payload: new Uint8Array([7]), + messageTtlMillis: limits.messageTtlMillis, + senderRetryHorizonMillis: limits.maximumSenderRetryHorizonMillis, + minimumTerminalRetentionMillis: limits.minimumTerminalRetentionMillis + }) + const claimRequest = { + recipient: { + tenantId: channel.tenantId, + subjectId: channel.recipientSubjectId, + peerId: channel.recipientPeerId + }, + sender: { + subjectId: channel.senderSubjectId, + peerId: channel.senderPeerId + }, + sessionGeneration: 1, + authorizedDocumentIds: admission.documentIds + } as const + yield* Effect.scoped( + Effect.gen(function*() { + const store = yield* PeerRelayStore.PeerRelayStore + const sql = yield* SqlClient.SqlClient + yield* store.admit(admission) + const claimed = yield* store.claim(claimRequest) + assert.strictEqual(Option.isSome(claimed.message), true) + if (Option.isNone(claimed.message)) return + const message = claimed.message.value + const releaseRequest = PeerRelayStore.ReleaseRequest.make({ + channel, + relayMessageId: message.relayMessageId, + claimToken: message.claimToken, + sessionGeneration: message.sessionGeneration + }) + assert.deepStrictEqual(yield* store.release(releaseRequest), { + status: "Changed", + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" + }) + const rows = yield* sql<{ + readonly state: string + readonly retryCount: number + readonly payload: Uint8Array | null + readonly payloadLength: number + readonly claimToken: string | null + readonly terminalReason: string | null + }>`SELECT + state, + retry_count AS retryCount, + payload, + payload_length AS payloadLength, + claim_token AS claimToken, + terminal_reason AS terminalReason + FROM effect_local_relay_messages` + assert.deepStrictEqual(rows, [{ + state: "DeadLettered", + retryCount: 1, + payload: null, + payloadLength: 0, + claimToken: null, + terminalReason: "MaximumDeliveryAttempts" + }]) + const fences = yield* sql<{ + readonly claimedMessageId: number | null + readonly claimToken: string | null + }>`SELECT + claimed_message_id AS claimedMessageId, + claim_token AS claimToken + FROM effect_local_relay_channels` + assert.deepStrictEqual(fences, [{ + claimedMessageId: null, + claimToken: null + }]) + const reservations = yield* sql<{ + readonly activeConsumed: number + readonly retainedConsumed: number + }>`SELECT + active_consumed AS activeConsumed, + retained_consumed AS retainedConsumed + FROM effect_local_relay_reservations` + assert.deepStrictEqual(reservations, [{ + activeConsumed: 1, + retainedConsumed: 0 + }]) + assert.deepStrictEqual(yield* store.usage(), { + activeCount: 0, + activeBytes: 0, + retainedCount: 1, + retainedBytes: 1 + }) + assert.strictEqual(Option.isNone((yield* store.claim(claimRequest)).message), true) + assert.strictEqual((yield* store.release(releaseRequest)).status, "Stale") + }).pipe(Effect.provide(makeLayer(filename, limits))) + ) + yield* Effect.scoped( + Effect.gen(function*() { + const store = yield* PeerRelayStore.PeerRelayStore + const sql = yield* SqlClient.SqlClient + const rows = yield* sql<{ + readonly state: string + readonly retryCount: number + readonly payload: Uint8Array | null + }>`SELECT + state, + retry_count AS retryCount, + payload + FROM effect_local_relay_messages` + assert.deepStrictEqual(rows, [{ + state: "DeadLettered", + retryCount: 1, + payload: null + }]) + assert.deepStrictEqual(yield* store.usage(), { + activeCount: 0, + activeBytes: 0, + retainedCount: 1, + retainedBytes: 1 + }) + assert.strictEqual(Option.isNone((yield* store.claim(claimRequest)).message), true) + }).pipe(Effect.provide(makeLayer(filename, limits))) + ) + })) + + it.effect("dead letters an abandoned claim at the delivery attempt cap exactly once", () => { + const registry: Metric.MetricRegistry = new Map() + const metricValue = (metric: Metric.Metric) => + Metric.value(metric).pipe( + Effect.provideService(Metric.CurrentMetricAttributes, {}) + ) + return withStore( + Effect.gen(function*() { + const store = yield* PeerRelayStore.PeerRelayStore + const sql = yield* SqlClient.SqlClient + const channel = PeerRelayStore.ChannelKey.make({ + tenantId: "tenant-recover-cap", + senderSubjectId: "sender-recover-cap", + senderPeerId: peer("000000000061"), + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + recipientSubjectId: "recipient-recover-cap", + recipientPeerId: peer("000000000062") + }) + const admission = PeerRelayStore.Admission.make({ + channel, + relayMessageId: relayId("000000000061"), + relayPeerId: peer("000000000063"), + documentIds: [documentId("000000000061")], + senderConnectionEpoch: "epoch-recover-cap", + senderSequence: 0, + payloadVersion: 1, + messageHash: "message-hash-recover-cap", + outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make("6".repeat(64)), + payload: new Uint8Array([6]), + messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, + senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, + minimumTerminalRetentionMillis: PeerRelayLimits.defaults.minimumTerminalRetentionMillis + }) + yield* store.admit(admission) + const claimRequest = { + recipient: { + tenantId: channel.tenantId, + subjectId: channel.recipientSubjectId, + peerId: channel.recipientPeerId + }, + sender: { + subjectId: channel.senderSubjectId, + peerId: channel.senderPeerId + }, + sessionGeneration: 1, + authorizedDocumentIds: admission.documentIds + } as const + const claimed = yield* store.claim(claimRequest) + assert.strictEqual(Option.isSome(claimed.message), true) + if (Option.isNone(claimed.message)) return + yield* sql`UPDATE effect_local_relay_messages + SET claim_deadline = 0 + WHERE message_id = ${claimed.message.value.rowId}` + yield* sql`UPDATE effect_local_relay_channels + SET claim_deadline = 0 + WHERE claimed_message_id = ${claimed.message.value.rowId}` + const recovery = store.recover({ batchSize: 1 }) + assert.strictEqual((yield* recovery).processed, 1) + const rows = yield* sql<{ + readonly state: string + readonly retryCount: number + readonly payload: Uint8Array | null + readonly terminalReason: string | null + }>`SELECT + state, + retry_count AS retryCount, + payload, + terminal_reason AS terminalReason + FROM effect_local_relay_messages` + assert.deepStrictEqual(rows, [{ + state: "DeadLettered", + retryCount: 1, + payload: null, + terminalReason: "MaximumDeliveryAttempts" + }]) + assert.deepStrictEqual(yield* store.usage(), { + activeCount: 0, + activeBytes: 0, + retainedCount: 1, + retainedBytes: 1 + }) + assert.strictEqual((yield* recovery).processed, 0) + assert.strictEqual( + (yield* metricValue( + PeerRpcObservability.relayOutcomes( + "RelayMaintenance", + "Receive", + "DeadLettered", + "Recover" + ) + )).count, + 1 + ) + assert.strictEqual( + (yield* metricValue( + PeerRpcObservability.relayOutcomes( + "RelayMaintenance", + "Receive", + "Released", + "Recover" + ) + )).count, + 1 + ) + assert.strictEqual(Option.isNone((yield* store.claim(claimRequest)).message), true) + }), + PeerRelayLimits.Values.make({ + ...PeerRelayLimits.defaults, + maximumDeliveryAttempts: 1 + }) + ).pipe(Effect.provideService(Metric.MetricRegistry, registry)) + }) + + it.effect("drains maintenance by deadline even when the row cursor is ahead", () => + withStore(Effect.gen(function*() { + const store = yield* PeerRelayStore.PeerRelayStore + const sql = yield* SqlClient.SqlClient + const channel = PeerRelayStore.ChannelKey.make({ + tenantId: "tenant", + senderSubjectId: "sender", + senderPeerId: peer("000000000021"), + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + recipientSubjectId: "recipient", + recipientPeerId: peer("000000000022") + }) + const admission = PeerRelayStore.Admission.make({ + channel, + relayMessageId: relayId("000000000021"), + relayPeerId: peer("000000000023"), + documentIds: [documentId("000000000021")], + senderConnectionEpoch: "epoch-1", + senderSequence: 0, + payloadVersion: 1, + messageHash: "message-hash", + outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make("f".repeat(64)), + payload: new Uint8Array([1]), + messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, + senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, + minimumTerminalRetentionMillis: PeerRelayLimits.defaults.minimumTerminalRetentionMillis + }) + yield* store.admit(admission) + const claim = yield* store.claim({ + recipient: { + tenantId: channel.tenantId, + subjectId: channel.recipientSubjectId, + peerId: channel.recipientPeerId + }, + sender: { + subjectId: channel.senderSubjectId, + peerId: channel.senderPeerId + }, + sessionGeneration: 1, + authorizedDocumentIds: admission.documentIds + }) + assert.strictEqual(Option.isSome(claim.message), true) + if (Option.isNone(claim.message)) return + const message = claim.message.value + yield* sql`UPDATE effect_local_relay_messages + SET claim_deadline = 0 + WHERE message_id = ${message.rowId}` + yield* sql`UPDATE effect_local_relay_channels + SET claim_deadline = 0 + WHERE claimed_message_id = ${message.rowId}` + const recovered = yield* store.recover({ + cursor: message.rowId, + batchSize: 1 + }) + assert.strictEqual(recovered.processed, 1) + + yield* sql`UPDATE effect_local_relay_messages + SET expires_at = 0 + WHERE message_id = ${message.rowId}` + const expired = yield* store.expire({ + cursor: message.rowId, + batchSize: 1 + }) + assert.strictEqual(expired.processed, 1) + + yield* sql`UPDATE effect_local_relay_messages + SET deduplicate_until = 0 + WHERE message_id = ${message.rowId}` + const collected = yield* store.collect({ + cursor: message.rowId, + batchSize: 1 + }) + assert.strictEqual(collected.processed, 1) + const remaining = yield* sql<{ readonly count: number }>` + SELECT COUNT(*) AS count + FROM effect_local_relay_messages + WHERE message_id = ${message.rowId} + ` + assert.strictEqual(remaining[0]?.count, 0) + }))) + + it.effect("reconciles one bounded structural page without rebuilding usage", () => + withStore(Effect.gen(function*() { + const store = yield* PeerRelayStore.PeerRelayStore + const sql = yield* SqlClient.SqlClient + const channel = PeerRelayStore.ChannelKey.make({ + tenantId: "tenant", + senderSubjectId: "sender", + senderPeerId: peer("000000000031"), + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + recipientSubjectId: "recipient", + recipientPeerId: peer("000000000032") + }) + for (let index = 1; index <= 3; index++) { + yield* store.admit(PeerRelayStore.Admission.make({ + channel, + relayMessageId: relayId(`00000000003${index}`), + relayPeerId: peer("000000000033"), + documentIds: [documentId("000000000031")], + senderConnectionEpoch: "epoch-1", + senderSequence: index, + payloadVersion: 1, + messageHash: `message-hash-${index}`, + outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make(String(index).repeat(64)), + payload: new Uint8Array([index]), + messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, + senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, + minimumTerminalRetentionMillis: PeerRelayLimits.defaults.minimumTerminalRetentionMillis + })) + } + const ids = yield* sql<{ readonly messageId: number }>` + SELECT message_id AS messageId + FROM effect_local_relay_messages + ORDER BY message_id + ` + const first = yield* store.reconcile({ batchSize: 1 }) + assert.deepStrictEqual(first, { + cursor: ids[0]!.messageId, + processed: 1, + hasMore: true + }) + const firstCursor = first.cursor + assert.isDefined(firstCursor) + const second = yield* store.reconcile({ + cursor: firstCursor, + batchSize: 1 + }) + assert.deepStrictEqual(second, { + cursor: ids[1]!.messageId, + processed: 1, + hasMore: true + }) + const secondCursor = second.cursor + assert.isDefined(secondCursor) + const third = yield* store.reconcile({ + cursor: secondCursor, + batchSize: 1 + }) + assert.deepStrictEqual(third, { + cursor: ids[2]!.messageId, + processed: 1, + hasMore: false + }) + }))) + + it.effect("plans cross incarnation claims without a temporary ordering tree", () => + withStore(Effect.gen(function*() { + const sql = yield* SqlClient.SqlClient + yield* sql`WITH RECURSIVE incarnations(value) AS ( + VALUES(1) + UNION ALL + SELECT value + 1 FROM incarnations WHERE value < 10000 + ) + INSERT INTO effect_local_relay_channels ( + tenant_id, + sender_subject_id, + sender_peer_id, + sender_replica_incarnation, + recipient_subject_id, + recipient_peer_id, + next_sequence + ) + SELECT + 'tenant-plan', + 'sender-plan', + 'peer_00000000-0000-4000-8000-000000000041', + value, + 'recipient-plan', + 'peer_00000000-0000-4000-8000-000000000042', + 1 + FROM incarnations` + yield* sql`INSERT INTO effect_local_relay_messages ( + channel_id, + channel_sequence, + tenant_id, + sender_subject_id, + sender_peer_id, + recipient_subject_id, + recipient_peer_id, + relay_message_id, + relay_peer_id, + sender_connection_epoch, + sender_sequence, + document_ids, + payload_version, + message_hash, + outer_envelope_digest, + payload, + payload_length, + state, + created_at, + expires_at, + deduplicate_until, + next_eligible_at + ) + SELECT + channel_id, + 0, + tenant_id, + sender_subject_id, + sender_peer_id, + recipient_subject_id, + recipient_peer_id, + 'relay-' || channel_id, + 'peer_00000000-0000-4000-8000-000000000043', + 'epoch-1', + 0, + '["doc_00000000-0000-4000-8000-000000000041"]', + 1, + 'hash-' || channel_id, + ${"a".repeat(64)}, + x'01', + 1, + 'Pending', + channel_id, + 9999999999999, + 9999999999999, + 0 + FROM effect_local_relay_channels` + yield* sql`INSERT INTO effect_local_relay_reservations ( + message_id, + sender_peer_usage_key, + recipient_peer_usage_key, + recipient_subject_usage_key, + tenant_usage_key, + shard_usage_key, + active_count_delta, + active_bytes_delta, + retained_count_delta, + retained_bytes_delta + ) + SELECT + message_id, + 'sender', + 'recipient-peer', + 'recipient-subject', + 'tenant', + 'shard', + 1, + 1, + 1, + 1 + FROM effect_local_relay_messages` + yield* sql`ANALYZE` + const plan = yield* sql<{ readonly detail: string }>` + EXPLAIN QUERY PLAN + SELECT m.message_id + FROM effect_local_relay_messages m + JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id + JOIN effect_local_relay_reservations r ON r.message_id = m.message_id + WHERE m.tenant_id = 'tenant-plan' + AND m.sender_subject_id = 'sender-plan' + AND m.sender_peer_id = 'peer_00000000-0000-4000-8000-000000000041' + AND m.recipient_subject_id = 'recipient-plan' + AND m.recipient_peer_id = 'peer_00000000-0000-4000-8000-000000000042' + AND c.tenant_id = 'tenant-plan' + AND c.recipient_subject_id = 'recipient-plan' + AND c.recipient_peer_id = 'peer_00000000-0000-4000-8000-000000000042' + AND c.sender_subject_id = 'sender-plan' + AND c.sender_peer_id = 'peer_00000000-0000-4000-8000-000000000041' + AND m.tenant_id = c.tenant_id + AND m.sender_subject_id = c.sender_subject_id + AND m.sender_peer_id = c.sender_peer_id + AND c.claimed_message_id IS NULL + AND m.state = 'Pending' + AND m.next_eligible_at <= 1 + AND m.expires_at > 1 + AND m.payload IS NOT NULL + AND m.payload_length = length(m.payload) + AND r.active_consumed = 0 + AND NOT EXISTS ( + SELECT 1 + FROM effect_local_relay_messages earlier + WHERE earlier.channel_id = m.channel_id + AND earlier.channel_sequence < m.channel_sequence + AND earlier.state IN ('Pending', 'Claimed') + ) + AND NOT EXISTS ( + SELECT 1 + FROM json_each(m.document_ids) document + WHERE document.value NOT IN ( + SELECT value + FROM json_each('["doc_00000000-0000-4000-8000-000000000041"]') + ) + ) + ORDER BY m.created_at, m.message_id + LIMIT 1` + assert.strictEqual( + plan.some((row) => row.detail.includes("effect_local_relay_messages_claim_admission")), + true + ) + assert.strictEqual( + plan.some((row) => row.detail.includes("USE TEMP B-TREE")), + false + ) + const recoveryPlan = yield* sql<{ readonly detail: string }>` + EXPLAIN QUERY PLAN + SELECT message_id + FROM effect_local_relay_messages + WHERE state = 'Claimed' + AND claim_deadline <= 1 + ORDER BY claim_deadline, message_id + LIMIT 101` + const expiryPlan = yield* sql<{ readonly detail: string }>` + EXPLAIN QUERY PLAN + SELECT message_id + FROM effect_local_relay_messages + WHERE state IN ('Pending', 'Claimed') + AND expires_at <= 1 + ORDER BY expires_at, message_id + LIMIT 101` + const collectionPlan = yield* sql<{ readonly detail: string }>` + EXPLAIN QUERY PLAN + SELECT message_id + FROM effect_local_relay_messages + WHERE state IN ('Acknowledged', 'DeadLettered', 'Expired') + AND deduplicate_until <= 1 + ORDER BY deduplicate_until, message_id + LIMIT 101` + for ( + const [maintenancePlan, index] of [ + [recoveryPlan, "effect_local_relay_messages_recovery"], + [expiryPlan, "effect_local_relay_messages_expiry"], + [collectionPlan, "effect_local_relay_messages_collection"] + ] as const + ) { + assert.strictEqual( + maintenancePlan.some((row) => row.detail.includes(index)), + true + ) + assert.strictEqual( + maintenancePlan.some((row) => row.detail.includes("USE TEMP B-TREE")), + false + ) + } + }))) }) diff --git a/packages/local-rpc/test/PeerRelayStoreErrors.test.ts b/packages/local-rpc/test/PeerRelayStoreErrors.test.ts new file mode 100644 index 0000000..6e1dcd7 --- /dev/null +++ b/packages/local-rpc/test/PeerRelayStoreErrors.test.ts @@ -0,0 +1,41 @@ +import { assert, describe, it } from "@effect/vitest" +import * as Cause from "effect/Cause" +import * as Effect from "effect/Effect" +import * as Exit from "effect/Exit" +import * as SqlError from "effect/unstable/sql/SqlError" +import { mapStoreErrors } from "../src/internal/peerRelayStoreErrors.js" + +describe("PeerRelayStore errors", () => { + it.effect("maps typed SQL failures while preserving rollback defects", () => + Effect.gen(function*() { + const sqlFailure = new SqlError.SqlError({ + reason: new SqlError.UnknownError({ cause: new Error("commit failed") }) + }) + const rollbackDefect = new Error("rollback failed") + const exit = yield* mapStoreErrors( + Effect.failCause( + Cause.combine( + Cause.fail(sqlFailure), + Cause.die(rollbackDefect) + ) + ) + ).pipe(Effect.exit) + assert.strictEqual(Exit.isFailure(exit), true) + if (Exit.isFailure(exit)) { + const failures = exit.cause.reasons.filter(Cause.isFailReason) + assert.strictEqual(failures.length, 1) + const mapped = failures[0]!.error + assert.strictEqual(mapped._tag, "ReplicaError") + if (mapped._tag === "ReplicaError") { + assert.strictEqual(mapped.reason._tag, "StorageUnavailable") + if (mapped.reason._tag === "StorageUnavailable") { + assert.strictEqual(mapped.reason.cause, sqlFailure) + } + } + assert.strictEqual( + exit.cause.reasons.some((reason) => Cause.isDieReason(reason) && reason.defect === rollbackDefect), + true + ) + } + })) +}) diff --git a/packages/local-rpc/test/PeerRpcObservability.test.ts b/packages/local-rpc/test/PeerRpcObservability.test.ts new file mode 100644 index 0000000..00c24bd --- /dev/null +++ b/packages/local-rpc/test/PeerRpcObservability.test.ts @@ -0,0 +1,399 @@ +import { assert, describe, it } from "@effect/vitest" +import * as Cause from "effect/Cause" +import * as Clock from "effect/Clock" +import * as Effect from "effect/Effect" +import * as Exit from "effect/Exit" +import * as Metric from "effect/Metric" +import * as References from "effect/References" +import * as Tracer from "effect/Tracer" +import * as PeerRpcObservability from "../src/internal/peerRpcObservability.js" + +describe("PeerRpcObservability", () => { + it.effect("does not let telemetry Clock defects replace the business Cause", () => + Effect.gen(function*() { + const clock = yield* Clock.Clock + const original = Cause.fail({ _tag: "OriginalFailure" }) + const defectiveClock = (defectAtRead: number): Clock.Clock => { + let reads = 0 + const currentTimeNanosUnsafe = () => { + reads += 1 + if (reads === defectAtRead) throw new Error("telemetry clock defect") + return clock.currentTimeNanosUnsafe() + } + return { + currentTimeMillisUnsafe: () => clock.currentTimeMillisUnsafe(), + currentTimeMillis: clock.currentTimeMillis, + currentTimeNanosUnsafe, + currentTimeNanos: Effect.sync(currentTimeNanosUnsafe), + sleep: (duration) => clock.sleep(duration) + } + } + let observeRuns = 0 + let relayRuns = 0 + + const observeExit = yield* PeerRpcObservability.observe({ + effect: Effect.sync(() => { + observeRuns += 1 + }).pipe(Effect.andThen(Effect.failCause(original))), + operation: "Open", + spanName: "effect_local_rpc.server.open", + attributes: {}, + result: () => "Failure" + }).pipe( + Effect.provideService(Clock.Clock, defectiveClock(2)), + Effect.exit + ) + const relayExit = yield* PeerRpcObservability.observeRelay({ + effect: Effect.sync(() => { + relayRuns += 1 + }).pipe(Effect.andThen(Effect.failCause(original))), + operation: "RelayClaim", + direction: "Receive", + facts: () => ({ items: 0 }), + result: () => "Failure" + }).pipe( + Effect.provideService(Clock.Clock, defectiveClock(3)), + Effect.exit + ) + + assert.strictEqual(observeRuns, 1) + assert.strictEqual(relayRuns, 1) + for (const exit of [observeExit, relayExit]) { + assert.isTrue(Exit.isFailure(exit)) + if (Exit.isFailure(exit)) assert.strictEqual(exit.cause, original) + } + })) + + it.effect("does not let telemetry defects replace a captured business Cause", () => { + const spans: Array = [] + const tracer = Tracer.make({ + span: (options) => { + const span = new Tracer.NativeSpan(options) + spans.push(span) + return span + } + }) + const original = Cause.fail({ _tag: "OriginalFailure" }) + const throwTelemetry = () => { + throw new Error("telemetry defect") + } + + return Effect.gen(function*() { + const exits = [ + yield* PeerRpcObservability.observe({ + effect: Effect.failCause(original), + operation: "Open", + spanName: "effect_local_rpc.server.open", + attributes: {}, + result: throwTelemetry + }).pipe(Effect.exit), + yield* PeerRpcObservability.observeRelay({ + effect: Effect.failCause(original), + operation: "RelayClaim", + direction: "Receive", + facts: () => ({ items: 0 }), + result: throwTelemetry + }).pipe(Effect.exit), + yield* PeerRpcObservability.observeRelay({ + effect: Effect.failCause(original), + operation: "RelayClaim", + direction: "Receive", + facts: throwTelemetry, + result: () => "Failure" + }).pipe(Effect.exit) + ] + for (const exit of exits) { + assert.isTrue(Exit.isFailure(exit)) + if (Exit.isFailure(exit)) assert.strictEqual(exit.cause, original) + } + assert.strictEqual(spans.length, exits.length) + for (const span of spans) { + assert.strictEqual(span.status._tag, "Ended") + if (span.status._tag === "Ended") { + assert.isTrue(Exit.isSuccess(span.status.exit)) + if (Exit.isSuccess(span.status.exit)) { + assert.isUndefined(span.status.exit.value) + } + } + } + }).pipe( + Effect.provideService(Metric.MetricRegistry, new Map()), + Effect.provideService(Tracer.Tracer, tracer) + ) + }) + + it.effect("does not let tracer setup or finalization replace the business Cause", () => { + const original = Cause.fail({ _tag: "OriginalFailure" }) + let createRan = false + const createTracer = Tracer.make({ + span: () => { + throw new Error("span create defect") + } + }) + let endRan = false + const endTracer = Tracer.make({ + span: (options) => { + const span = new Tracer.NativeSpan(options) + span.end = () => { + throw new Error("span end defect") + } + return span + } + }) + + return Effect.gen(function*() { + const createExit = yield* PeerRpcObservability.observe({ + effect: Effect.sync(() => { + createRan = true + }).pipe(Effect.andThen(Effect.failCause(original))), + operation: "Open", + spanName: "effect_local_rpc.server.open", + attributes: {}, + result: () => "Failure" + }).pipe( + Effect.provideService(Tracer.Tracer, createTracer), + Effect.exit + ) + assert.isTrue(createRan) + assert.isTrue(Exit.isFailure(createExit)) + if (Exit.isFailure(createExit)) assert.strictEqual(createExit.cause, original) + + const endExit = yield* PeerRpcObservability.observeRelay({ + effect: Effect.sync(() => { + endRan = true + }).pipe(Effect.andThen(Effect.failCause(original))), + operation: "RelayRelease", + direction: "Receive", + facts: () => ({ items: 1 }), + result: () => "Failure" + }).pipe( + Effect.provideService(Tracer.Tracer, endTracer), + Effect.exit + ) + assert.isTrue(endRan) + assert.isTrue(Exit.isFailure(endExit)) + if (Exit.isFailure(endExit)) assert.strictEqual(endExit.cause, original) + }).pipe( + Effect.provideService(Metric.MetricRegistry, new Map()) + ) + }) + + it.effect("keeps relay spans and metrics safe while preserving every original Cause", () => { + const spans: Array = [] + const tracer = Tracer.make({ + span: (options) => { + const span = new Tracer.NativeSpan(options) + spans.push(span) + return span + } + }) + const forbidden = "tenant-subject-peer-session-document-relay-token-payload" + const ambientLink: Tracer.SpanLink = { + span: Tracer.externalSpan({ + spanId: "forbidden-span-id", + traceId: "forbidden-trace-id" + }), + attributes: { forbidden } + } + const causes = [ + Cause.fail({ _tag: "ExpectedFailure", forbidden }), + Cause.die(new Error(forbidden)), + Cause.interrupt(123) + ] as const + const metricValue = ( + metric: Metric.Metric + ) => + Metric.value(metric).pipe( + Effect.provideService(Metric.CurrentMetricAttributes, {}) + ) + + return Effect.gen(function*() { + for (const cause of causes) { + const exit = yield* PeerRpcObservability.observeRelay({ + effect: Effect.failCause(cause), + operation: "RelayAdmit", + direction: "Send", + facts: () => ({ + bytes: 4, + items: 1, + attempt: 1, + version: 1 + }), + result: () => "Failure" + }).pipe(Effect.exit) + assert.isTrue(Exit.isFailure(exit)) + if (Exit.isFailure(exit)) assert.strictEqual(exit.cause, cause) + } + yield* PeerRpcObservability.recordRelayQuotaRejection("Shard") + yield* PeerRpcObservability.setRelayPending(2, 8) + yield* PeerRpcObservability.setRelayActiveClaims(1) + yield* PeerRpcObservability.setRelayWorkers(3) + yield* PeerRpcObservability.setRelayReadyQueueItems("New", 2) + yield* PeerRpcObservability.setRelayReadyQueueItems("Retry", 1) + yield* PeerRpcObservability.recordRelayOutcome({ + operation: "RelayAcknowledge", + direction: "Receive", + result: "Acknowledged", + facts: { + bytes: 4, + items: 1, + version: 1, + latencyMillis: 25 + } + }) + yield* PeerRpcObservability.recordRelayOutcome({ + operation: "RelayMaintenance", + direction: "Receive", + result: "Expired", + facts: { items: 2 }, + stage: "Expire" + }) + for (const version of [1_000_001, 1_000_002]) { + yield* PeerRpcObservability.recordRelayOutcome({ + operation: "RelayAdmit", + direction: "Send", + result: "Success", + facts: { bytes: 4, version } + }) + } + + assert.strictEqual( + (yield* metricValue( + PeerRpcObservability.relayOutcomes("RelayAdmit", "Send", "Attempt") + )).count, + causes.length + ) + assert.strictEqual( + (yield* metricValue( + PeerRpcObservability.relayOutcomes("RelayAdmit", "Send", "Failure") + )).count, + causes.length + ) + assert.strictEqual( + (yield* metricValue( + PeerRpcObservability.relayBytes("RelayAdmit", "Send", "Failure", 1) + )).count, + causes.length + ) + assert.strictEqual( + (yield* metricValue( + PeerRpcObservability.relayItems("RelayAdmit", "Send", "Failure") + )).count, + causes.length + ) + assert.strictEqual( + (yield* metricValue( + PeerRpcObservability.relayAttempts("RelayAdmit", "Send", "Failure") + )).count, + causes.length + ) + assert.strictEqual( + (yield* metricValue( + PeerRpcObservability.relayDurationMillis("RelayAdmit", "Send", "Failure") + )).count, + causes.length + ) + assert.strictEqual( + (yield* metricValue(PeerRpcObservability.relayQuotaRejections("Shard"))).count, + 1 + ) + assert.strictEqual( + (yield* metricValue(PeerRpcObservability.relayPendingItems())).value, + 2 + ) + assert.strictEqual( + (yield* metricValue(PeerRpcObservability.relayPendingBytes())).value, + 8 + ) + assert.strictEqual( + (yield* metricValue(PeerRpcObservability.relayActiveClaims())).value, + 1 + ) + assert.strictEqual( + (yield* metricValue(PeerRpcObservability.relayWorkers())).value, + 3 + ) + assert.strictEqual( + (yield* metricValue(PeerRpcObservability.relayReadyQueueItems("New"))).value, + 2 + ) + assert.strictEqual( + (yield* metricValue(PeerRpcObservability.relayReadyQueueItems("Retry"))).value, + 1 + ) + assert.strictEqual( + (yield* metricValue( + PeerRpcObservability.relayLatencyMillis( + "RelayAcknowledge", + "Receive", + "Acknowledged" + ) + )).count, + 1 + ) + assert.strictEqual( + (yield* metricValue( + PeerRpcObservability.relayOutcomes( + "RelayMaintenance", + "Receive", + "Expired", + "Expire" + ) + )).count, + 1 + ) + assert.strictEqual( + (yield* metricValue( + PeerRpcObservability.relayBytes( + "RelayAdmit", + "Send", + "Success", + 1_000_001 + ) + )).count, + 2 + ) + + assert.strictEqual(spans.length, causes.length) + for (const span of spans) { + assert.strictEqual(span.name, "effect_local_rpc.relay.admit") + assert.strictEqual(span.status._tag, "Ended") + if (span.status._tag === "Ended") { + assert.isTrue(Exit.isSuccess(span.status.exit)) + if (Exit.isSuccess(span.status.exit)) { + assert.isUndefined(span.status.exit.value) + } + } + assert.deepStrictEqual( + [...span.attributes], + [ + ["rpc.operation", "RelayAdmit"], + ["rpc.direction", "Send"], + ["rpc.result", "Failure"], + ["rpc.bytes", 4], + ["rpc.items", 1], + ["rpc.attempt", 1], + ["rpc.version", "1"], + ["rpc.duration_millis", 0] + ] + ) + assert.deepStrictEqual(span.links, []) + assert.deepStrictEqual(span.events, []) + } + + assert.notInclude( + JSON.stringify(spans.map((span) => ({ + attributes: [...span.attributes], + status: span.status._tag + }))) + (yield* Metric.dump), + forbidden + ) + }).pipe( + Effect.provideService(Metric.MetricRegistry, new Map()), + Effect.provideService(Metric.CurrentMetricAttributes, { forbidden }), + Effect.provideService(References.TracerSpanAnnotations, { forbidden }), + Effect.provideService(References.TracerSpanLinks, [ambientLink]), + Effect.provideService(Tracer.Tracer, tracer) + ) + }) +}) diff --git a/packages/local-rpc/test/PeerRpcServer.test.ts b/packages/local-rpc/test/PeerRpcServer.test.ts index ea3d530..b17608d 100644 --- a/packages/local-rpc/test/PeerRpcServer.test.ts +++ b/packages/local-rpc/test/PeerRpcServer.test.ts @@ -2762,7 +2762,1415 @@ describe("PeerRpcServer", () => { }))) }) +const relayTransition: PeerRelayStore.TransitionResult = { + status: "Changed", + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" +} + +const makeRelayStore = ( + overrides: Partial = {} +): PeerRelayStore.Service => ({ + admit: (input) => + Effect.succeed({ + status: "Accepted", + channel: input.channel, + ready: false, + nextEligibleAt: 0, + lane: "New" + }), + claim: () => + Effect.succeed({ + message: Option.none(), + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" + }), + loadClaimedPayload: () => Effect.succeed(new Uint8Array()), + acknowledge: () => Effect.succeed(relayTransition), + reject: () => Effect.succeed(relayTransition), + release: () => Effect.succeed(relayTransition), + recover: ({ cursor }) => + Effect.succeed({ + ...(cursor === undefined ? {} : { cursor }), + processed: 0, + hasMore: false + }), + expire: ({ cursor }) => + Effect.succeed({ + ...(cursor === undefined ? {} : { cursor }), + processed: 0, + hasMore: false + }), + repair: ({ cursor }) => + Effect.succeed({ + ...(cursor === undefined ? {} : { cursor }), + processed: 0, + hasMore: false + }), + reconcile: ({ cursor }) => + Effect.succeed({ + ...(cursor === undefined ? {} : { cursor }), + processed: 0, + hasMore: false + }), + collect: ({ cursor }) => + Effect.succeed({ + ...(cursor === undefined ? {} : { cursor }), + processed: 0, + hasMore: false + }), + usage: () => + Effect.succeed({ + activeCount: 0, + activeBytes: 0, + retainedCount: 0, + retainedBytes: 0 + }), + ...overrides +}) + +const relayPrincipal: PeerAuthentication.PeerPrincipal = { + tenantId: "tenant", + subjectId: "sender", + peerId: remotePeerA +} + +const relayOpenRequest = { + version: PeerRelayRpc.protocolVersion, + expectedRelayPeerId: serverPeerId, + expectedLocal: relayPrincipal, + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + remote: { + subjectId: "recipient", + peerId: remotePeerB + }, + documents: [{ documentType: Task.name, documentId: taskId }], + receiptRetentionMillis: PeerRelayLimits.defaults.maximumReceiptRetentionMillis, + senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis +} satisfies typeof PeerRelayRpc.OpenRelayRpc.payloadSchema.Type + +const relayAuthenticated = { + principal: relayPrincipal, + validUntil: Number.MAX_SAFE_INTEGER, + invalidated: Effect.never +} + +const authorizeUnsafeRelayDecode: PeerRelayAuthorization.AuthorizeUnsafeUnboundedAutomerge3Decode = ( + request +) => + Effect.succeed({ + _tag: "UnsafeUnboundedAutomerge3DecodeGrant", + risk: PeerRelayAuthorization.unsafeUnboundedAutomerge3DecodeRisk, + principal: request.principal, + remote: { + tenantId: request.principal.tenantId, + subjectId: request.remote.subjectId, + peerId: request.remote.peerId + }, + direction: request.direction, + documents: request.documents, + validUntil: Number.MAX_SAFE_INTEGER, + invalidated: Effect.never + }) + +const authorizeRelay: PeerRelayAuthorization.Authorize = (request) => + Effect.succeed({ + remote: { + tenantId: request.principal.tenantId, + subjectId: request.remote.subjectId, + peerId: request.remote.peerId + }, + documents: request.documents.map((document) => ({ + document: Task, + documentId: document.documentId + })), + validUntil: Number.MAX_SAFE_INTEGER, + invalidated: Effect.never + }) + +const RelaySyncEnvelopeJson = Schema.fromJsonString( + Schema.toCodecJson(PeerSyncEnvelope.SyncEnvelope) +) + +const makeRelayPayload = Effect.gen(function*() { + const message = Uint8Array.of(1, 2, 3) + const messageHash = yield* Canonical.digest(message).pipe( + Effect.provideService(Crypto.Crypto, crypto) + ) + const payload = new TextEncoder().encode( + yield* Schema.encodeEffect(RelaySyncEnvelopeJson)({ + connectionEpoch: "epoch", + sequence: 0, + documentId: taskId, + documentType: Task.name, + messageHash, + message, + lineage: Identity.genesisLineage, + writerProvenance: [] + }) + ) + return { messageHash, payload } +}) + +const makeRelayClaim = ( + relayMessageId: Identity.RelayMessageId, + claimToken: PeerRelayRpc.ClaimToken, + messageHash: string, + outerEnvelopeDigest: PeerRelayRpc.RelayDigest, + payloadBytes: number, + sessionGeneration: number +) => + PeerRelayStore.ClaimedMessage.make({ + rowId: 1, + channel: { + tenantId: "tenant", + senderSubjectId: relayPrincipal.subjectId, + senderPeerId: relayPrincipal.peerId, + senderReplicaIncarnation: relayOpenRequest.senderReplicaIncarnation, + recipientSubjectId: relayOpenRequest.remote.subjectId, + recipientPeerId: relayOpenRequest.remote.peerId + }, + relayMessageId, + relayPeerId: serverPeerId, + senderConnectionEpoch: "epoch", + senderSequence: 0, + documentIds: [taskId], + payloadVersion: 1, + messageHash, + outerEnvelopeDigest, + payloadBytes, + claimToken, + claimDeadline: Number.MAX_SAFE_INTEGER, + sessionGeneration + }) + +const makeRelayOuterEnvelopeDigest = ( + relayMessageId: Identity.RelayMessageId, + messageHash: string, + payload: Uint8Array +) => + PeerSyncEnvelope.digestRelayOuterEnvelope({ + domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, + version: PeerSyncEnvelope.relayOuterEnvelopeVersion, + expectedLocal: relayPrincipal, + remote: { + tenantId: relayPrincipal.tenantId, + subjectId: relayOpenRequest.remote.subjectId, + peerId: relayOpenRequest.remote.peerId + }, + relayPeerId: serverPeerId, + relayMessageId, + protocolVersion: PeerRelayRpc.protocolVersion, + payloadVersion: 1, + senderReplicaIncarnation: relayOpenRequest.senderReplicaIncarnation, + senderConnectionEpoch: "epoch", + senderSequence: 0, + document: { + documentId: taskId, + documentType: Task.name + }, + lineage: Identity.genesisLineage, + writerProvenance: [], + messageHash, + payload + }).pipe(Effect.provideService(Crypto.Crypto, crypto)) + +const makeRelayHandlerContext = ( + authorization: Context.Service.Shape, + store: PeerRelayStore.Service, + limits: PeerRelayLimits.Values = PeerRelayLimits.defaults, + ingress: Context.Service.Shape = { + address: { _tag: "UnixAddress", path: "relay-test" }, + reserveOutbound: () => Effect.fail(new PeerRpcError.RequestCapacityExceeded()), + usage: Effect.succeed({ + connections: 0, + reservedBytes: 0, + byteReservationWaiters: 0 + }), + await: Effect.never + } +) => + Layer.build( + PeerRpcServer.layerRelayHandlers({ + tenantId: "tenant", + peerId: serverPeerId + }).pipe( + Layer.provide(Layer.mergeAll( + Layer.succeed(Crypto.Crypto, crypto), + Layer.succeed(PeerRelayLimits.PeerRelayLimits, limits), + Layer.succeed(PeerRelayAuthorization.PeerRelayAuthorization, authorization), + Layer.succeed(PeerRelayStore.PeerRelayStore, store), + Layer.succeed(PeerRelayIngress.PeerRelayIngress, ingress) + )) + ) + ) + +const relayHandlerEffect = ( + handler: Rpc.Handler, + effect: Effect.Effect +) => + effect.pipe( + Effect.provideContext(Context.add( + handler.context, + PeerAuthentication.AuthenticatedPeer, + relayAuthenticated + )) + ) + +const makeTerminalRelayContext = ( + relayMessageId: Identity.RelayMessageId, + claimToken: PeerRelayRpc.ClaimToken, + authorization: Context.Service.Shape, + acknowledge: PeerRelayStore.Service["acknowledge"] +) => + Effect.gen(function*() { + const { messageHash, payload } = yield* makeRelayPayload + const outerEnvelopeDigest = yield* makeRelayOuterEnvelopeDigest( + relayMessageId, + messageHash, + payload + ) + let claimed = false + const store = makeRelayStore({ + claim: (request) => + Effect.sync(() => { + if (claimed) { + return { + message: Option.none(), + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" as const + } + } + claimed = true + return { + message: Option.some(makeRelayClaim( + relayMessageId, + claimToken, + messageHash, + outerEnvelopeDigest, + payload.byteLength, + request.sessionGeneration + )), + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" as const + } + }), + loadClaimedPayload: () => Effect.succeed(payload), + acknowledge + }) + const ingress = PeerRelayIngress.PeerRelayIngress.of({ + address: { _tag: "UnixAddress", path: "relay-test" }, + reserveOutbound: (bytes) => + Effect.succeed({ + bytes, + release: Effect.void, + transferToCurrentRequest: Effect.void + }), + usage: Effect.succeed({ + connections: 0, + reservedBytes: 0, + byteReservationWaiters: 0 + }), + await: Effect.never + }) + return yield* makeRelayHandlerContext( + authorization, + store, + PeerRelayLimits.defaults, + ingress + ) + }) + describe("PeerRpcServer relay", () => { + it.effect("denies operation admission after an absolute grant deadline", () => + Effect.scoped(Effect.gen(function*() { + let now = 0 + const clock: Clock.Clock = { + currentTimeMillisUnsafe: () => now, + currentTimeMillis: Effect.sync(() => now), + currentTimeNanosUnsafe: () => BigInt(now) * 1_000_000n, + currentTimeNanos: Effect.sync(() => BigInt(now) * 1_000_000n), + sleep: () => Effect.never + } + let sendAuthorizations = 0 + const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ + authorize: (request) => + authorizeRelay(request).pipe( + Effect.map((result) => { + if (request.direction === "Send") sendAuthorizations += 1 + return request.direction === "Send" && sendAuthorizations === 2 + ? { + ...result, + validUntil: 1_000, + invalidated: Effect.sync(() => { + now = 1_001 + }).pipe(Effect.andThen(Effect.never)) + } + : result + }) + ), + authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode + }) + const admitCalls = yield* Ref.make(0) + const store = makeRelayStore({ + admit: (input) => + Ref.update(admitCalls, (count) => count + 1).pipe( + Effect.as({ + status: "Accepted", + channel: input.channel, + ready: false, + nextEligibleAt: 0, + lane: "New" + }) + ) + }) + const context = yield* makeRelayHandlerContext(authorization, store) + const openHandler = context.mapUnsafe.get( + PeerRelayRpc.OpenRelayRpc.key + ) as Rpc.Handler<"OpenRelay"> + const pushHandler = context.mapUnsafe.get( + PeerRelayRpc.PushRelayRpc.key + ) as Rpc.Handler<"PushRelay"> + const provide = ( + handler: Rpc.Handler, + effect: Effect.Effect + ) => + effect.pipe( + Effect.provideContext( + Context.add( + Context.add( + handler.context, + PeerAuthentication.AuthenticatedPeer, + relayAuthenticated + ), + Clock.Clock, + clock + ) + ) + ) + const events = yield* Queue.unbounded() + const openFiber = yield* Stream.runForEach( + openHandler.handler( + relayOpenRequest, + {} as never + ) as Stream.Stream, + (event) => Queue.offer(events, event) + ).pipe( + (effect) => provide(openHandler, effect), + Effect.forkScoped + ) + const opened = yield* Queue.take(events) + assert.strictEqual(opened._tag, "RelayOpened") + const { payload } = yield* makeRelayPayload + const denied = yield* ( + pushHandler.handler({ + sessionId: (opened as PeerRelayRpc.RelayOpened).sessionId, + relayMessageId: Identity.RelayMessageId.make( + "rly_00000000-0000-4000-8000-000000000052" + ), + payload + }, {} as never) as Effect.Effect + ).pipe( + (effect) => provide(pushHandler, effect), + Effect.flip + ) + assert.instanceOf(denied, PeerRpcError.AccessDenied) + assert.strictEqual(yield* Ref.get(admitCalls), 0) + yield* Fiber.interrupt(openFiber) + }))) + + it.effect("rejects Open when authentication expires during relay authorization", () => + Effect.scoped(Effect.gen(function*() { + const authorizationStarted = yield* Deferred.make() + const authorizationRelease = yield* Deferred.make() + let authorizationCalls = 0 + const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ + authorize: (request) => + Effect.gen(function*() { + authorizationCalls++ + if (authorizationCalls === 1) { + yield* Deferred.succeed(authorizationStarted, undefined) + yield* Deferred.await(authorizationRelease) + } + return yield* authorizeRelay(request) + }), + authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode + }) + const testClock = yield* TestClock.testClockWith(Effect.succeed) + const context = yield* makeRelayHandlerContext( + authorization, + makeRelayStore() + ).pipe(Effect.provideService(Clock.Clock, testClock)) + const handler = context.mapUnsafe.get( + PeerRelayRpc.OpenRelayRpc.key + ) as Rpc.Handler<"OpenRelay"> + const now = yield* Clock.currentTimeMillis + const fiber = yield* Stream.runHead( + handler.handler( + relayOpenRequest, + {} as never + ) as Stream.Stream + ).pipe( + Effect.provideContext( + Context.add( + Context.add( + handler.context, + PeerAuthentication.AuthenticatedPeer, + { + principal: relayPrincipal, + validUntil: now + 1_000, + invalidated: Effect.never + } + ), + Clock.Clock, + testClock + ) + ), + Effect.exit, + Effect.forkScoped + ) + yield* Deferred.await(authorizationStarted) + yield* TestClock.adjust(1_001) + yield* Deferred.succeed(authorizationRelease, undefined) + const exit = yield* Fiber.join(fiber) + assert.isTrue(Exit.isFailure(exit)) + if (Exit.isFailure(exit)) { + assert.instanceOf(Cause.squash(exit.cause), PeerRpcError.AuthenticationFailure) + } + }))) + + it.effect("requires a live unsafe decode grant before relay Push admission", () => + Effect.scoped(Effect.gen(function*() { + let sendMode: "Deny" | "Stale" = "Deny" + const unsafeRequests = yield* Ref.make< + Array + >([]) + const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ + authorize: authorizeRelay, + authorizeUnsafeUnboundedAutomerge3Decode: (request) => + Ref.update(unsafeRequests, (requests) => [...requests, request]).pipe( + Effect.andThen( + request.direction === "Receive" + ? authorizeUnsafeRelayDecode(request) + : sendMode === "Deny" + ? Effect.fail(new PeerRpcError.AccessDenied()) + : authorizeUnsafeRelayDecode(request).pipe( + Effect.map((grant) => ({ + ...grant, + invalidated: Effect.void + })) + ) + ) + ) + }) + const admitCalls = yield* Ref.make(0) + const store = makeRelayStore({ + admit: (input) => + Ref.update(admitCalls, (count) => count + 1).pipe( + Effect.as({ + status: "Accepted" as const, + channel: input.channel, + ready: false, + nextEligibleAt: 0, + lane: "New" as const + }) + ) + }) + const context = yield* makeRelayHandlerContext(authorization, store) + const openHandler = context.mapUnsafe.get( + PeerRelayRpc.OpenRelayRpc.key + ) as Rpc.Handler<"OpenRelay"> + const pushHandler = context.mapUnsafe.get( + PeerRelayRpc.PushRelayRpc.key + ) as Rpc.Handler<"PushRelay"> + const events = yield* Queue.unbounded() + const openFiber = yield* Stream.runForEach( + openHandler.handler( + relayOpenRequest, + {} as never + ) as Stream.Stream, + (event) => Queue.offer(events, event) + ).pipe( + (effect) => relayHandlerEffect(openHandler, effect), + Effect.forkScoped + ) + const opened = yield* Queue.take(events) + assert.strictEqual(opened._tag, "RelayOpened") + const sessionId = (opened as PeerRelayRpc.RelayOpened).sessionId + const { payload } = yield* makeRelayPayload + const push = (relayMessageId: Identity.RelayMessageId) => + pushHandler.handler({ + sessionId, + relayMessageId, + payload + }, {} as never) as Effect.Effect + + const denied = yield* push( + Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000041") + ).pipe( + (effect) => relayHandlerEffect(pushHandler, effect), + Effect.flip + ) + assert.instanceOf(denied, PeerRpcError.AccessDenied) + assert.strictEqual(yield* Ref.get(admitCalls), 0) + + sendMode = "Stale" + const stale = yield* push( + Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000042") + ).pipe( + (effect) => relayHandlerEffect(pushHandler, effect), + Effect.flip + ) + assert.instanceOf(stale, PeerRpcError.AccessDenied) + assert.strictEqual(yield* Ref.get(admitCalls), 0) + + const sendRequests = (yield* Ref.get(unsafeRequests)).filter( + (request) => request.direction === "Send" + ) + assert.deepStrictEqual(sendRequests, [ + { + risk: PeerRelayAuthorization.unsafeUnboundedAutomerge3DecodeRisk, + direction: "Send", + principal: relayPrincipal, + remote: relayOpenRequest.remote, + documents: [{ documentType: Task.name, documentId: taskId }] + }, + { + risk: PeerRelayAuthorization.unsafeUnboundedAutomerge3DecodeRisk, + direction: "Send", + principal: relayPrincipal, + remote: relayOpenRequest.remote, + documents: [{ documentType: Task.name, documentId: taskId }] + } + ]) + yield* Fiber.interrupt(openFiber) + }))) + + it.effect("requires unsafe Receive trust before relay claim or payload read", () => + Effect.scoped(Effect.gen(function*() { + const receiveDenied = yield* Deferred.make() + const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ + authorize: authorizeRelay, + authorizeUnsafeUnboundedAutomerge3Decode: (request) => + request.direction === "Send" + ? authorizeUnsafeRelayDecode(request) + : Deferred.succeed(receiveDenied, undefined).pipe( + Effect.andThen(Effect.fail(new PeerRpcError.AccessDenied())) + ) + }) + const claimCalls = yield* Ref.make(0) + const payloadReads = yield* Ref.make(0) + const store = makeRelayStore({ + claim: () => + Ref.update(claimCalls, (count) => count + 1).pipe( + Effect.as({ + message: Option.none(), + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" as const + }) + ), + loadClaimedPayload: () => + Ref.update(payloadReads, (count) => count + 1).pipe( + Effect.as(new Uint8Array()) + ) + }) + const context = yield* makeRelayHandlerContext(authorization, store) + const openHandler = context.mapUnsafe.get( + PeerRelayRpc.OpenRelayRpc.key + ) as Rpc.Handler<"OpenRelay"> + const events = yield* Queue.unbounded() + const openFiber = yield* Stream.runForEach( + openHandler.handler( + relayOpenRequest, + {} as never + ) as Stream.Stream, + (event) => Queue.offer(events, event) + ).pipe( + (effect) => relayHandlerEffect(openHandler, effect), + Effect.forkScoped + ) + assert.strictEqual((yield* Queue.take(events))._tag, "RelayOpened") + yield* Deferred.await(receiveDenied) + yield* Effect.yieldNow + assert.strictEqual(yield* Ref.get(claimCalls), 0) + assert.strictEqual(yield* Ref.get(payloadReads), 0) + assert.isTrue(Option.isNone(yield* Queue.poll(events))) + yield* Fiber.interrupt(openFiber) + }))) + + it.effect("drains admitted Push while revocation denies later operations", () => + Effect.scoped(Effect.gen(function*() { + const sendInvalidated = yield* Deferred.make() + let pushing = false + const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ + authorize: (request) => + authorizeRelay(request).pipe( + Effect.map((result) => ({ + ...result, + invalidated: pushing && request.direction === "Send" + ? Deferred.await(sendInvalidated) + : Effect.never + })) + ), + authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode + }) + const admitStarted = yield* Deferred.make() + const allowCommit = yield* Deferred.make() + const commits = yield* Ref.make(0) + const store = makeRelayStore({ + admit: (input) => + Effect.uninterruptible( + Deferred.succeed(admitStarted, undefined).pipe( + Effect.andThen(Deferred.await(allowCommit)), + Effect.andThen(Ref.update(commits, (count) => count + 1)), + Effect.as({ + status: "Accepted", + channel: input.channel, + ready: false, + nextEligibleAt: 0, + lane: "New" + }) + ) + ) + }) + const context = yield* makeRelayHandlerContext(authorization, store) + const openHandler = context.mapUnsafe.get( + PeerRelayRpc.OpenRelayRpc.key + ) as Rpc.Handler<"OpenRelay"> + const pushHandler = context.mapUnsafe.get( + PeerRelayRpc.PushRelayRpc.key + ) as Rpc.Handler<"PushRelay"> + const events = yield* Queue.unbounded() + const openFiber = yield* Stream.runForEach( + openHandler.handler( + relayOpenRequest, + {} as never + ) as Stream.Stream, + (event) => Queue.offer(events, event) + ).pipe( + (effect) => relayHandlerEffect(openHandler, effect), + Effect.forkScoped + ) + const opened = yield* Queue.take(events) + assert.strictEqual(opened._tag, "RelayOpened") + pushing = true + const { payload } = yield* makeRelayPayload + const pushFiber = yield* ( + pushHandler.handler({ + sessionId: (opened as PeerRelayRpc.RelayOpened).sessionId, + relayMessageId: Identity.RelayMessageId.make( + "rly_00000000-0000-4000-8000-000000000043" + ), + payload + }, {} as never) as Effect.Effect + ).pipe( + (effect) => relayHandlerEffect(pushHandler, effect), + Effect.exit, + Effect.forkScoped + ) + yield* Deferred.await(admitStarted) + yield* Deferred.succeed(sendInvalidated, undefined) + yield* Effect.yieldNow + yield* Deferred.succeed(allowCommit, undefined) + const pushExit = yield* Fiber.join(pushFiber) + assert.isTrue(Exit.isSuccess(pushExit)) + assert.strictEqual(yield* Ref.get(commits), 1) + const later = yield* ( + pushHandler.handler({ + sessionId: (opened as PeerRelayRpc.RelayOpened).sessionId, + relayMessageId: Identity.RelayMessageId.make( + "rly_00000000-0000-4000-8000-000000000046" + ), + payload + }, {} as never) as Effect.Effect + ).pipe( + (effect) => relayHandlerEffect(pushHandler, effect), + Effect.flip + ) + assert.instanceOf(later, PeerRpcError.AccessDenied) + assert.strictEqual(yield* Ref.get(commits), 1) + yield* Fiber.interrupt(openFiber) + }))) + + it.effect("denies terminal mutation when fresh Receive revocation wins admission", () => + Effect.scoped(Effect.gen(function*() { + let receiveAuthorizations = 0 + const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ + authorize: (request) => + authorizeRelay(request).pipe( + Effect.map((result) => { + if (request.direction === "Receive") receiveAuthorizations += 1 + return { + ...result, + invalidated: request.direction === "Receive" && receiveAuthorizations === 3 + ? Effect.void + : Effect.never + } + }) + ), + authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode + }) + const acknowledgeCalls = yield* Ref.make(0) + const context = yield* makeTerminalRelayContext( + Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000050"), + PeerRelayRpc.ClaimToken.make("clm_00000000-0000-4000-8000-000000000050"), + authorization, + () => + Ref.update(acknowledgeCalls, (count) => count + 1).pipe( + Effect.as(relayTransition) + ) + ) + const openHandler = context.mapUnsafe.get( + PeerRelayRpc.OpenRelayRpc.key + ) as Rpc.Handler<"OpenRelay"> + const acknowledgeHandler = context.mapUnsafe.get( + PeerRelayRpc.AcknowledgeRelayRpc.key + ) as Rpc.Handler<"AcknowledgeRelay"> + const events = yield* Queue.unbounded() + const openFiber = yield* Stream.runForEach( + openHandler.handler( + relayOpenRequest, + {} as never + ) as Stream.Stream, + (event) => Queue.offer(events, event) + ).pipe( + (effect) => relayHandlerEffect(openHandler, effect), + Effect.forkScoped + ) + const opened = yield* Queue.take(events) + const stored = yield* Queue.take(events) + if (opened._tag !== "RelayOpened" || stored._tag !== "StoredMessage") { + return assert.fail("Expected relay open and stored message events") + } + const denied = yield* ( + acknowledgeHandler.handler({ + sessionId: opened.sessionId, + relayMessageId: stored.relayMessageId, + claimToken: stored.claimToken, + messageHash: stored.messageHash + }, {} as never) as Effect.Effect + ).pipe( + (effect) => relayHandlerEffect(acknowledgeHandler, effect), + Effect.flip + ) + assert.instanceOf(denied, PeerRpcError.AccessDenied) + assert.strictEqual(yield* Ref.get(acknowledgeCalls), 0) + yield* Fiber.interrupt(openFiber) + }))) + + it.effect("drains an admitted terminal mutation through revocation", () => + Effect.scoped(Effect.gen(function*() { + const terminalInvalidated = yield* Deferred.make() + let receiveAuthorizations = 0 + const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ + authorize: (request) => + authorizeRelay(request).pipe( + Effect.map((result) => { + if (request.direction === "Receive") receiveAuthorizations += 1 + return { + ...result, + invalidated: request.direction === "Receive" && receiveAuthorizations === 3 + ? Deferred.await(terminalInvalidated) + : Effect.never + } + }) + ), + authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode + }) + const acknowledgeStarted = yield* Deferred.make() + const allowAcknowledge = yield* Deferred.make() + const acknowledgeCalls = yield* Ref.make(0) + const context = yield* makeTerminalRelayContext( + Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000051"), + PeerRelayRpc.ClaimToken.make("clm_00000000-0000-4000-8000-000000000051"), + authorization, + () => + Effect.uninterruptible( + Deferred.succeed(acknowledgeStarted, undefined).pipe( + Effect.andThen(Deferred.await(allowAcknowledge)), + Effect.andThen(Ref.update(acknowledgeCalls, (count) => count + 1)), + Effect.as(relayTransition) + ) + ) + ) + const openHandler = context.mapUnsafe.get( + PeerRelayRpc.OpenRelayRpc.key + ) as Rpc.Handler<"OpenRelay"> + const acknowledgeHandler = context.mapUnsafe.get( + PeerRelayRpc.AcknowledgeRelayRpc.key + ) as Rpc.Handler<"AcknowledgeRelay"> + const runtime = Context.get(context, PeerRpcServer.PeerRelayServerRuntime) + const events = yield* Queue.unbounded() + const openFiber = yield* Stream.runForEach( + openHandler.handler( + relayOpenRequest, + {} as never + ) as Stream.Stream, + (event) => Queue.offer(events, event) + ).pipe( + (effect) => relayHandlerEffect(openHandler, effect), + Effect.forkScoped + ) + const opened = yield* Queue.take(events) + const stored = yield* Queue.take(events) + if (opened._tag !== "RelayOpened" || stored._tag !== "StoredMessage") { + return assert.fail("Expected relay open and stored message events") + } + const terminalFiber = yield* ( + acknowledgeHandler.handler({ + sessionId: opened.sessionId, + relayMessageId: stored.relayMessageId, + claimToken: stored.claimToken, + messageHash: stored.messageHash + }, {} as never) as Effect.Effect + ).pipe( + (effect) => relayHandlerEffect(acknowledgeHandler, effect), + Effect.exit, + Effect.forkScoped + ) + yield* Deferred.await(acknowledgeStarted) + yield* Deferred.succeed(terminalInvalidated, undefined) + yield* Effect.yieldNow + yield* Deferred.succeed(allowAcknowledge, undefined) + assert.isTrue(Exit.isSuccess(yield* Fiber.join(terminalFiber))) + assert.strictEqual(yield* Ref.get(acknowledgeCalls), 1) + assert.strictEqual((yield* runtime.usage).activeClaims, 0) + yield* Fiber.interrupt(openFiber) + }))) + + it.effect("drains admitted delivery through outbound offer during revocation", () => + Effect.scoped(Effect.gen(function*() { + const relayMessageId = Identity.RelayMessageId.make( + "rly_00000000-0000-4000-8000-000000000047" + ) + const claimToken = PeerRelayRpc.ClaimToken.make( + "clm_00000000-0000-4000-8000-000000000047" + ) + const { messageHash, payload } = yield* makeRelayPayload + const outerEnvelopeDigest = yield* makeRelayOuterEnvelopeDigest( + relayMessageId, + messageHash, + payload + ) + const receiveInvalidated = yield* Deferred.make() + let receiveAuthorizations = 0 + const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ + authorize: (request) => + authorizeRelay(request).pipe( + Effect.map((result) => { + if (request.direction === "Receive") receiveAuthorizations += 1 + return { + ...result, + invalidated: request.direction === "Receive" && receiveAuthorizations > 1 + ? Deferred.await(receiveInvalidated) + : Effect.never + } + }) + ), + authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode + }) + const claimStarted = yield* Deferred.make() + const allowClaimCommit = yield* Deferred.make() + const payloadReads = yield* Ref.make(0) + let claimed = false + const store = makeRelayStore({ + claim: (request) => + Effect.uninterruptible( + Deferred.succeed(claimStarted, undefined).pipe( + Effect.andThen(Deferred.await(allowClaimCommit)), + Effect.map(() => { + if (claimed) { + return { + message: Option.none(), + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" as const + } + } + claimed = true + return { + message: Option.some(makeRelayClaim( + relayMessageId, + claimToken, + messageHash, + outerEnvelopeDigest, + payload.byteLength, + request.sessionGeneration + )), + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" as const + } + }) + ) + ), + loadClaimedPayload: () => + Ref.update(payloadReads, (count) => count + 1).pipe( + Effect.as(payload) + ) + }) + const ingress = PeerRelayIngress.PeerRelayIngress.of({ + address: { _tag: "UnixAddress", path: "relay-test" }, + reserveOutbound: (bytes) => + Effect.succeed({ + bytes, + release: Effect.void, + transferToCurrentRequest: Effect.void + }), + usage: Effect.succeed({ + connections: 0, + reservedBytes: 0, + byteReservationWaiters: 0 + }), + await: Effect.never + }) + const context = yield* makeRelayHandlerContext( + authorization, + store, + PeerRelayLimits.defaults, + ingress + ) + const openHandler = context.mapUnsafe.get( + PeerRelayRpc.OpenRelayRpc.key + ) as Rpc.Handler<"OpenRelay"> + const runtime = Context.get(context, PeerRpcServer.PeerRelayServerRuntime) + const events = yield* Queue.unbounded() + const openFiber = yield* Stream.runForEach( + openHandler.handler( + relayOpenRequest, + {} as never + ) as Stream.Stream, + (event) => Queue.offer(events, event) + ).pipe( + (effect) => relayHandlerEffect(openHandler, effect), + Effect.forkScoped + ) + assert.strictEqual((yield* Queue.take(events))._tag, "RelayOpened") + yield* Deferred.await(claimStarted) + yield* Deferred.succeed(receiveInvalidated, undefined) + yield* Effect.yieldNow + yield* Deferred.succeed(allowClaimCommit, undefined) + assert.strictEqual((yield* Queue.take(events))._tag, "StoredMessage") + assert.strictEqual(yield* Ref.get(payloadReads), 1) + assert.strictEqual((yield* runtime.usage).activeClaims, 1) + yield* Fiber.interrupt(openFiber) + }))) + + it.effect("does not emit a dequeued delivery after session revocation", () => + Effect.scoped(Effect.gen(function*() { + const relayMessageId = Identity.RelayMessageId.make( + "rly_00000000-0000-4000-8000-000000000048" + ) + const claimToken = PeerRelayRpc.ClaimToken.make( + "clm_00000000-0000-4000-8000-000000000048" + ) + const { messageHash, payload } = yield* makeRelayPayload + const outerEnvelopeDigest = yield* makeRelayOuterEnvelopeDigest( + relayMessageId, + messageHash, + payload + ) + const sessionInvalidated = yield* Deferred.make() + let receiveAuthorizations = 0 + const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ + authorize: (request) => + authorizeRelay(request).pipe( + Effect.map((result) => { + if (request.direction === "Receive") receiveAuthorizations += 1 + return { + ...result, + invalidated: request.direction === "Receive" && receiveAuthorizations === 1 + ? Deferred.await(sessionInvalidated) + : Effect.never + } + }) + ), + authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode + }) + let claimed = false + const store = makeRelayStore({ + claim: (request) => + Effect.sync(() => { + if (claimed) { + return { + message: Option.none(), + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" as const + } + } + claimed = true + return { + message: Option.some(makeRelayClaim( + relayMessageId, + claimToken, + messageHash, + outerEnvelopeDigest, + payload.byteLength, + request.sessionGeneration + )), + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" as const + } + }), + loadClaimedPayload: () => Effect.succeed(payload) + }) + const transferStarted = yield* Deferred.make() + const transferGate = yield* Deferred.make() + const reservationReleases = yield* Ref.make(0) + const ingress = PeerRelayIngress.PeerRelayIngress.of({ + address: { _tag: "UnixAddress", path: "relay-test" }, + reserveOutbound: (bytes) => + Effect.succeed({ + bytes, + release: Ref.update(reservationReleases, (count) => count + 1), + transferToCurrentRequest: Deferred.succeed(transferStarted, undefined).pipe( + Effect.andThen(Deferred.await(transferGate)) + ) + }), + usage: Effect.succeed({ + connections: 0, + reservedBytes: 0, + byteReservationWaiters: 0 + }), + await: Effect.never + }) + const context = yield* makeRelayHandlerContext( + authorization, + store, + PeerRelayLimits.defaults, + ingress + ) + const handler = context.mapUnsafe.get( + PeerRelayRpc.OpenRelayRpc.key + ) as Rpc.Handler<"OpenRelay"> + const runtime = Context.get(context, PeerRpcServer.PeerRelayServerRuntime) + const events = yield* Queue.unbounded() + const openFiber = yield* Stream.runForEach( + handler.handler( + relayOpenRequest, + {} as never + ) as Stream.Stream, + (event) => Queue.offer(events, event) + ).pipe( + (effect) => relayHandlerEffect(handler, effect), + Effect.exit, + Effect.forkScoped + ) + assert.strictEqual((yield* Queue.take(events))._tag, "RelayOpened") + yield* Deferred.await(transferStarted) + yield* Deferred.succeed(sessionInvalidated, undefined) + for (let index = 0; index < 20; index++) yield* Effect.yieldNow + assert.strictEqual((yield* runtime.usage).sessions, 0) + yield* Deferred.succeed(transferGate, undefined) + yield* Fiber.join(openFiber) + assert.isTrue(Option.isNone(yield* Queue.poll(events))) + assert.strictEqual(yield* Ref.get(reservationReleases), 1) + }))) + + it.effect("treats a stale claimed payload as a local delivery race", () => + Effect.scoped(Effect.gen(function*() { + const relayMessageId = Identity.RelayMessageId.make( + "rly_00000000-0000-4000-8000-000000000044" + ) + const claimToken = PeerRelayRpc.ClaimToken.make( + "clm_00000000-0000-4000-8000-000000000044" + ) + const { messageHash, payload } = yield* makeRelayPayload + const outerEnvelopeDigest = yield* makeRelayOuterEnvelopeDigest( + relayMessageId, + messageHash, + payload + ) + let claimed = false + const released = yield* Deferred.make() + const store = makeRelayStore({ + claim: (request) => + Effect.sync(() => { + if (claimed) { + return { + message: Option.none(), + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" as const + } + } + claimed = true + return { + message: Option.some(makeRelayClaim( + relayMessageId, + claimToken, + messageHash, + outerEnvelopeDigest, + payload.byteLength, + request.sessionGeneration + )), + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" as const + } + }), + loadClaimedPayload: () => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.ProtocolMismatch({ + expected: "active relay claim", + observed: "stale relay claim" + }) + }) + ), + release: () => + Deferred.succeed(released, undefined).pipe( + Effect.as(relayTransition) + ) + }) + const ingress = PeerRelayIngress.PeerRelayIngress.of({ + address: { _tag: "UnixAddress", path: "relay-test" }, + reserveOutbound: (bytes) => + Effect.succeed({ + bytes, + release: Effect.void, + transferToCurrentRequest: Effect.void + }), + usage: Effect.succeed({ + connections: 0, + reservedBytes: 0, + byteReservationWaiters: 0 + }), + await: Effect.never + }) + const context = yield* makeRelayHandlerContext( + PeerRelayAuthorization.PeerRelayAuthorization.of({ + authorize: authorizeRelay, + authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode + }), + store, + PeerRelayLimits.defaults, + ingress + ) + const openHandler = context.mapUnsafe.get( + PeerRelayRpc.OpenRelayRpc.key + ) as Rpc.Handler<"OpenRelay"> + const runtime = Context.get(context, PeerRpcServer.PeerRelayServerRuntime) + const events = yield* Queue.unbounded() + const openFiber = yield* Stream.runForEach( + openHandler.handler( + relayOpenRequest, + {} as never + ) as Stream.Stream, + (event) => Queue.offer(events, event) + ).pipe( + (effect) => relayHandlerEffect(openHandler, effect), + Effect.forkScoped + ) + assert.strictEqual((yield* Queue.take(events))._tag, "RelayOpened") + yield* Deferred.await(released) + yield* Effect.yieldNow + yield* Effect.yieldNow + yield* runtime.health + assert.strictEqual((yield* runtime.usage).activeClaims, 0) + assert.isTrue(Option.isNone(yield* Queue.poll(events))) + yield* Fiber.interrupt(openFiber) + }))) + + it.effect("clears local claim ownership after a stale terminal transition", () => + Effect.scoped(Effect.gen(function*() { + const relayMessageId = Identity.RelayMessageId.make( + "rly_00000000-0000-4000-8000-000000000045" + ) + const claimToken = PeerRelayRpc.ClaimToken.make( + "clm_00000000-0000-4000-8000-000000000045" + ) + const { messageHash, payload } = yield* makeRelayPayload + const outerEnvelopeDigest = yield* makeRelayOuterEnvelopeDigest( + relayMessageId, + messageHash, + payload + ) + let claimed = false + const store = makeRelayStore({ + claim: (request) => + Effect.sync(() => { + if (claimed) { + return { + message: Option.none(), + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" as const + } + } + claimed = true + return { + message: Option.some(makeRelayClaim( + relayMessageId, + claimToken, + messageHash, + outerEnvelopeDigest, + payload.byteLength, + request.sessionGeneration + )), + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" as const + } + }), + loadClaimedPayload: () => Effect.succeed(payload), + acknowledge: () => + Effect.succeed({ + status: "Stale", + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" + }) + }) + const ingress = PeerRelayIngress.PeerRelayIngress.of({ + address: { _tag: "UnixAddress", path: "relay-test" }, + reserveOutbound: (bytes) => + Effect.succeed({ + bytes, + release: Effect.void, + transferToCurrentRequest: Effect.void + }), + usage: Effect.succeed({ + connections: 0, + reservedBytes: 0, + byteReservationWaiters: 0 + }), + await: Effect.never + }) + const context = yield* makeRelayHandlerContext( + PeerRelayAuthorization.PeerRelayAuthorization.of({ + authorize: authorizeRelay, + authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode + }), + store, + PeerRelayLimits.defaults, + ingress + ) + const openHandler = context.mapUnsafe.get( + PeerRelayRpc.OpenRelayRpc.key + ) as Rpc.Handler<"OpenRelay"> + const acknowledgeHandler = context.mapUnsafe.get( + PeerRelayRpc.AcknowledgeRelayRpc.key + ) as Rpc.Handler<"AcknowledgeRelay"> + const runtime = Context.get(context, PeerRpcServer.PeerRelayServerRuntime) + const events = yield* Queue.unbounded() + const openFiber = yield* Stream.runForEach( + openHandler.handler( + relayOpenRequest, + {} as never + ) as Stream.Stream, + (event) => Queue.offer(events, event) + ).pipe( + (effect) => relayHandlerEffect(openHandler, effect), + Effect.forkScoped + ) + const opened = yield* Queue.take(events) + assert.strictEqual(opened._tag, "RelayOpened") + const stored = yield* Queue.take(events) + assert.strictEqual(stored._tag, "StoredMessage") + if (opened._tag !== "RelayOpened" || stored._tag !== "StoredMessage") { + return assert.fail("Expected relay open and stored message events") + } + const error = yield* ( + acknowledgeHandler.handler({ + sessionId: opened.sessionId, + relayMessageId: stored.relayMessageId, + claimToken: stored.claimToken, + messageHash: stored.messageHash + }, {} as never) as Effect.Effect + ).pipe( + (effect) => relayHandlerEffect(acknowledgeHandler, effect), + Effect.flip + ) + assert.instanceOf(error, PeerRpcError.SessionUnavailable) + yield* runtime.health + assert.deepStrictEqual(yield* runtime.usage, { + accepting: true, + sessions: 1, + subjects: 1, + activeClaims: 0, + queuedChannels: 0 + }) + yield* Fiber.interrupt(openFiber) + }))) + + it.effect("drains an admitted empty claim after session revocation", () => + Effect.scoped(Effect.gen(function*() { + const sessionInvalidated = yield* Deferred.make() + let receiveAuthorizations = 0 + const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ + authorize: (request) => + authorizeRelay(request).pipe( + Effect.map((result) => { + if (request.direction === "Receive") receiveAuthorizations += 1 + return { + ...result, + invalidated: request.direction === "Receive" && receiveAuthorizations === 1 + ? Deferred.await(sessionInvalidated) + : Effect.never + } + }) + ), + authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode + }) + const claimStarted = yield* Deferred.make() + const allowClaim = yield* Deferred.make() + const claimCompleted = yield* Deferred.make() + const store = makeRelayStore({ + claim: () => + Deferred.succeed(claimStarted, undefined).pipe( + Effect.andThen(Deferred.await(allowClaim)), + Effect.andThen(Deferred.succeed(claimCompleted, undefined)), + Effect.as({ + message: Option.none(), + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" + }) + ) + }) + const context = yield* makeRelayHandlerContext(authorization, store) + const openHandler = context.mapUnsafe.get( + PeerRelayRpc.OpenRelayRpc.key + ) as Rpc.Handler<"OpenRelay"> + const runtime = Context.get(context, PeerRpcServer.PeerRelayServerRuntime) + const events = yield* Queue.unbounded() + const openFiber = yield* Stream.runForEach( + openHandler.handler( + relayOpenRequest, + {} as never + ) as Stream.Stream, + (event) => Queue.offer(events, event) + ).pipe( + (effect) => relayHandlerEffect(openHandler, effect), + Effect.forkScoped + ) + assert.strictEqual((yield* Queue.take(events))._tag, "RelayOpened") + yield* Deferred.await(claimStarted) + yield* Deferred.succeed(sessionInvalidated, undefined) + yield* Effect.yieldNow + assert.strictEqual((yield* runtime.usage).sessions, 0) + yield* Deferred.succeed(allowClaim, undefined) + yield* Deferred.await(claimCompleted) + yield* Effect.yieldNow + yield* runtime.health + assert.deepStrictEqual(yield* runtime.usage, { + accepting: true, + sessions: 0, + subjects: 0, + activeClaims: 0, + queuedChannels: 0 + }) + yield* Fiber.interrupt(openFiber) + }))) + it.effect("reports the logical remote recipient separately from the relay endpoint", () => Effect.scoped(Effect.gen(function*() { const localPrincipal: PeerAuthentication.PeerPrincipal = { @@ -2787,7 +4195,8 @@ describe("PeerRpcServer relay", () => { validUntil: Number.MAX_SAFE_INTEGER, invalidated: Effect.never }) - : Effect.die(authorizationDefect) + : Effect.die(authorizationDefect), + authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode }) const transition: PeerRelayStore.TransitionResult = { status: "Changed", @@ -2929,144 +4338,856 @@ describe("PeerRpcServer relay", () => { (opened as PeerRelayRpc.RelayOpened).remotePeerId, remotePeerB ) - const relayMessageId = Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000001") - const message = Uint8Array.of(1, 2, 3) - const messageHash = yield* Canonical.digest(message).pipe( - Effect.provideService(Crypto.Crypto, crypto) + const relayMessageId = Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000001") + const message = Uint8Array.of(1, 2, 3) + const messageHash = yield* Canonical.digest(message).pipe( + Effect.provideService(Crypto.Crypto, crypto) + ) + const payload = new TextEncoder().encode( + yield* Schema.encodeEffect(RelaySyncEnvelopeJson)({ + connectionEpoch: "epoch", + sequence: 0, + documentId: taskId, + documentType: Task.name, + messageHash, + message, + lineage: Identity.genesisLineage, + writerProvenance: [] + }) + ) + const expectedDigest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope({ + domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, + version: PeerSyncEnvelope.relayOuterEnvelopeVersion, + expectedLocal: localPrincipal, + remote: { + tenantId: "tenant", + subjectId: "recipient", + peerId: remotePeerB + }, + relayPeerId: serverPeerId, + relayMessageId, + protocolVersion: PeerRelayRpc.protocolVersion, + payloadVersion: 1, + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + senderConnectionEpoch: "epoch", + senderSequence: 0, + document: { + documentId: taskId, + documentType: Task.name + }, + lineage: Identity.genesisLineage, + writerProvenance: [], + messageHash, + payload + }).pipe(Effect.provideService(Crypto.Crypto, crypto)) + yield* (pushHandler.handler({ + sessionId: (opened as PeerRelayRpc.RelayOpened).sessionId, + relayMessageId, + payload + }, {} as never) as Effect.Effect).pipe( + Effect.provideContext(Context.add( + pushHandler.context, + PeerAuthentication.AuthenticatedPeer, + authenticated + )) + ) + assert.strictEqual((yield* Deferred.await(admitted)).outerEnvelopeDigest, expectedDigest) + const defect = new Error("relay authorization defect") + authorizationDefect = defect + const defectExit = yield* Stream.runHead( + (openHandler.handler( + openRequest, + {} as never + ) as Stream.Stream).pipe( + Stream.provideContext(Context.add( + openHandler.context, + PeerAuthentication.AuthenticatedPeer, + authenticated + )) + ) + ).pipe(Effect.exit) + assert.isTrue(Exit.isFailure(defectExit)) + if (Exit.isFailure(defectExit)) { + assert.strictEqual(Cause.squash(defectExit.cause), defect) + } + authorizationDefect = undefined + const ownerContext = yield* Layer.build(Layer.fresh(handlers)) + const ownerRuntime = Context.get(ownerContext, PeerRpcServer.PeerRelayServerRuntime) + const serverContext = Context.add( + Context.add( + Context.add( + ownerContext, + RpcServer.Protocol, + protocol + ), + PeerRelayIngress.PeerRelayIngress, + { + ...relayIngress, + await: Effect.never.pipe( + Effect.ensuring(Deferred.succeed(ingressStopped, undefined)) + ) + } + ), + PeerAuthentication.PeerAuthentication, + PeerAuthentication.PeerAuthentication.of((effect) => + Effect.provideService(effect, PeerAuthentication.AuthenticatedPeer, { + principal: localPrincipal, + validUntil: Number.MAX_SAFE_INTEGER, + invalidated: Effect.never + }) + ) + ) + yield* Layer.build( + PeerRpcServer.layerRelayServer.pipe( + Layer.provide(Layer.succeedContext(serverContext)) + ) + ) + const ownerDefect = new Error("relay maintenance defect") + maintenanceDefect = ownerDefect + yield* TestClock.adjust(PeerRelayLimits.defaults.maintenanceIntervalMillis) + const ownerExit = yield* ownerRuntime.owner.pipe(Effect.exit) + assert.isTrue(Exit.isFailure(ownerExit)) + if (Exit.isFailure(ownerExit)) { + assert.strictEqual(Cause.squash(ownerExit.cause), ownerDefect) + } + yield* Effect.yieldNow + yield* Effect.yieldNow + assert.isTrue(Option.isSome(yield* Deferred.poll(protocolStopped))) + assert.isTrue(Option.isSome(yield* Deferred.poll(ingressStopped))) + yield* runtime.shutdown + yield* runtime.shutdown + const stale = yield* (acknowledgeHandler.handler({ + sessionId: (opened as PeerRelayRpc.RelayOpened).sessionId, + relayMessageId, + claimToken: PeerRelayRpc.ClaimToken.make("clm_00000000-0000-4000-8000-000000000001"), + messageHash + }, {} as never) as Effect.Effect).pipe( + Effect.provideContext(Context.add( + acknowledgeHandler.context, + PeerAuthentication.AuthenticatedPeer, + authenticated + )), + Effect.flip + ) + assert.strictEqual(stale._tag, "SessionUnavailable") + assert.strictEqual(yield* Ref.get(acknowledgeCalls), 0) + assert.deepStrictEqual(yield* runtime.usage, { + accepting: false, + sessions: 0, + subjects: 0, + activeClaims: 0, + queuedChannels: 0 + }) + yield* Fiber.interrupt(openFiber) + }))) + + it.effect("does not deliver a document type excluded by fresh Receive authorization", () => + Effect.scoped(Effect.gen(function*() { + const relayMessageId = Identity.RelayMessageId.make( + "rly_00000000-0000-4000-8000-000000000021" + ) + const claimToken = PeerRelayRpc.ClaimToken.make( + "clm_00000000-0000-4000-8000-000000000021" + ) + const { messageHash, payload } = yield* makeRelayPayload + const outerEnvelopeDigest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope({ + domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, + version: PeerSyncEnvelope.relayOuterEnvelopeVersion, + expectedLocal: relayPrincipal, + remote: { + tenantId: "tenant", + subjectId: relayOpenRequest.remote.subjectId, + peerId: relayOpenRequest.remote.peerId + }, + relayPeerId: serverPeerId, + relayMessageId, + protocolVersion: PeerRelayRpc.protocolVersion, + payloadVersion: 1, + senderReplicaIncarnation: relayOpenRequest.senderReplicaIncarnation, + senderConnectionEpoch: "epoch", + senderSequence: 0, + document: { + documentId: taskId, + documentType: Task.name + }, + lineage: Identity.genesisLineage, + writerProvenance: [], + messageHash, + payload + }).pipe(Effect.provideService(Crypto.Crypto, crypto)) + let receiveAuthorizations = 0 + const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ + authorize: (request) => + Effect.sync(() => { + if (request.direction === "Receive") receiveAuthorizations++ + return { + remote: { + tenantId: request.principal.tenantId, + subjectId: request.remote.subjectId, + peerId: request.remote.peerId + }, + documents: request.documents.map((document) => ({ + document: request.direction === "Receive" && receiveAuthorizations > 1 + ? Note + : Task, + documentId: document.documentId + })), + validUntil: Number.MAX_SAFE_INTEGER, + invalidated: Effect.never + } + }), + authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode + }) + let claimed = false + const storeReleaseCalls = yield* Ref.make(0) + const reservationReleaseCalls = yield* Ref.make(0) + const reservationTransferCalls = yield* Ref.make(0) + const deliverySettled = yield* Deferred.make<"Released" | "Transferred">() + const store = makeRelayStore({ + claim: (request) => + Effect.sync(() => { + if (claimed) { + return { + message: Option.none(), + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" as const + } + } + claimed = true + return { + message: Option.some(PeerRelayStore.ClaimedMessage.make({ + rowId: 1, + channel: { + tenantId: "tenant", + senderSubjectId: relayPrincipal.subjectId, + senderPeerId: relayPrincipal.peerId, + senderReplicaIncarnation: relayOpenRequest.senderReplicaIncarnation, + recipientSubjectId: relayOpenRequest.remote.subjectId, + recipientPeerId: relayOpenRequest.remote.peerId + }, + relayMessageId, + relayPeerId: serverPeerId, + senderConnectionEpoch: "epoch", + senderSequence: 0, + documentIds: [taskId], + payloadVersion: 1, + messageHash, + outerEnvelopeDigest, + payloadBytes: payload.byteLength, + claimToken, + claimDeadline: 1, + sessionGeneration: request.sessionGeneration + })), + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" as const + } + }), + loadClaimedPayload: () => Effect.succeed(payload), + release: () => + Ref.update(storeReleaseCalls, (count) => count + 1).pipe( + Effect.andThen(Deferred.succeed(deliverySettled, "Released")), + Effect.as(relayTransition) + ) + }) + const ingress = PeerRelayIngress.PeerRelayIngress.of({ + address: { _tag: "UnixAddress", path: "relay-test" }, + reserveOutbound: (bytes) => + Effect.succeed({ + bytes, + release: Ref.update(reservationReleaseCalls, (count) => count + 1), + transferToCurrentRequest: Ref.update( + reservationTransferCalls, + (count) => count + 1 + ).pipe( + Effect.andThen(Deferred.succeed(deliverySettled, "Transferred")), + Effect.asVoid + ) + }), + usage: Effect.succeed({ + connections: 0, + reservedBytes: 0, + byteReservationWaiters: 0 + }), + await: Effect.never + }) + const context = yield* makeRelayHandlerContext( + authorization, + store, + PeerRelayLimits.defaults, + ingress + ) + const handler = context.mapUnsafe.get( + PeerRelayRpc.OpenRelayRpc.key + ) as Rpc.Handler<"OpenRelay"> + const events = yield* Queue.unbounded() + const openFiber = yield* Stream.runForEach( + handler.handler( + relayOpenRequest, + {} as never + ) as Stream.Stream, + (event) => Queue.offer(events, event) + ).pipe( + (effect) => relayHandlerEffect(handler, effect), + Effect.forkScoped + ) + const opened = yield* Queue.take(events) + assert.strictEqual(opened._tag, "RelayOpened") + assert.strictEqual(yield* Deferred.await(deliverySettled), "Released") + assert.strictEqual(yield* Ref.get(storeReleaseCalls), 1) + assert.strictEqual(yield* Ref.get(reservationReleaseCalls), 1) + assert.strictEqual(yield* Ref.get(reservationTransferCalls), 0) + assert.isTrue(Option.isNone(yield* Queue.poll(events))) + yield* Fiber.interrupt(openFiber) + }))) + + it.effect("keeps the sole subject Open slot owned after another Open is rejected", () => + Effect.scoped(Effect.gen(function*() { + const authorizationStarted = yield* Deferred.make() + const releaseAuthorization = yield* Deferred.make() + const authorizationCalls = yield* Ref.make(0) + const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ + authorize: (request) => + Ref.updateAndGet(authorizationCalls, (count) => count + 1).pipe( + Effect.tap((count) => + count === 1 + ? Deferred.succeed(authorizationStarted, undefined) + : Effect.void + ), + Effect.andThen(Deferred.await(releaseAuthorization)), + Effect.as({ + remote: { + tenantId: request.principal.tenantId, + subjectId: request.remote.subjectId, + peerId: request.remote.peerId + }, + documents: request.documents.map((document) => ({ + document: Task, + documentId: document.documentId + })), + validUntil: Number.MAX_SAFE_INTEGER, + invalidated: Effect.never + }) + ), + authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode + }) + const context = yield* makeRelayHandlerContext( + authorization, + makeRelayStore(), + PeerRelayLimits.Values.make({ + ...PeerRelayLimits.defaults, + maxInFlightOpen: 3, + maxInFlightOpenPerSubject: 1, + openRatePerSecond: 1_000, + openBurst: 100 + }) + ) + const handler = context.mapUnsafe.get( + PeerRelayRpc.OpenRelayRpc.key + ) as Rpc.Handler<"OpenRelay"> + const invoke = Stream.runHead( + handler.handler( + relayOpenRequest, + {} as never + ) as Stream.Stream + ).pipe( + (effect) => relayHandlerEffect(handler, effect), + Effect.asVoid ) - const payload = new TextEncoder().encode( - yield* Schema.encodeEffect(SyncEnvelopeJson)({ - connectionEpoch: "epoch", - sequence: 0, - documentId: taskId, - documentType: Task.name, - messageHash, - message, - writerProvenance: [] - }) + const first = yield* invoke.pipe(Effect.forkScoped) + yield* Deferred.await(authorizationStarted) + const second = yield* invoke.pipe(Effect.flip) + assert.instanceOf(second, PeerRpcError.RequestCapacityExceeded) + const third = yield* invoke.pipe(Effect.flip) + assert.instanceOf(third, PeerRpcError.RequestCapacityExceeded) + assert.strictEqual(yield* Ref.get(authorizationCalls), 1) + yield* Deferred.succeed(releaseAuthorization, undefined) + yield* Fiber.join(first) + }))) + + it.effect("removes a newly registered Open when replacement cleanup is interrupted", () => + Effect.scoped(Effect.gen(function*() { + const lateMonitors = yield* Ref.make(0) + const lateMonitorSignal = yield* Deferred.make() + let trackLateMonitors = false + let trackedAuthorizations = 0 + const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ + authorize: (request) => { + const tracked = trackLateMonitors + if (tracked) trackedAuthorizations += 1 + return Effect.succeed({ + remote: { + tenantId: request.principal.tenantId, + subjectId: request.remote.subjectId, + peerId: request.remote.peerId + }, + documents: request.documents.map((document) => ({ + document: Task, + documentId: document.documentId + })), + validUntil: Number.MAX_SAFE_INTEGER, + invalidated: tracked + ? Deferred.await(lateMonitorSignal).pipe( + Effect.tap(() => Ref.update(lateMonitors, (count) => count + 1)) + ) + : Effect.never + }) + }, + authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode + }) + const relayMessageId = Identity.RelayMessageId.make( + "rly_00000000-0000-4000-8000-000000000031" ) - const expectedDigest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope({ + const claimToken = PeerRelayRpc.ClaimToken.make( + "clm_00000000-0000-4000-8000-000000000031" + ) + const { messageHash, payload } = yield* makeRelayPayload + const outerEnvelopeDigest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope({ domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, version: PeerSyncEnvelope.relayOuterEnvelopeVersion, - expectedLocal: localPrincipal, + expectedLocal: relayPrincipal, remote: { tenantId: "tenant", - subjectId: "recipient", - peerId: remotePeerB + subjectId: relayOpenRequest.remote.subjectId, + peerId: relayOpenRequest.remote.peerId }, relayPeerId: serverPeerId, relayMessageId, protocolVersion: PeerRelayRpc.protocolVersion, payloadVersion: 1, - senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + senderReplicaIncarnation: relayOpenRequest.senderReplicaIncarnation, senderConnectionEpoch: "epoch", senderSequence: 0, document: { documentId: taskId, documentType: Task.name }, + lineage: Identity.genesisLineage, writerProvenance: [], messageHash, payload }).pipe(Effect.provideService(Crypto.Crypto, crypto)) - yield* (pushHandler.handler({ - sessionId: (opened as PeerRelayRpc.RelayOpened).sessionId, - relayMessageId, - payload - }, {} as never) as Effect.Effect).pipe( - Effect.provideContext(Context.add( - pushHandler.context, - PeerAuthentication.AuthenticatedPeer, - authenticated - )) + let claimed = false + let releaseStarted = yield* Deferred.make() + let releaseGate = yield* Deferred.make() + const store = makeRelayStore({ + claim: (request) => + Effect.sync(() => { + if (claimed) { + return { + message: Option.none(), + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" as const + } + } + claimed = true + return { + message: Option.some(PeerRelayStore.ClaimedMessage.make({ + rowId: 1, + channel: { + tenantId: "tenant", + senderSubjectId: relayPrincipal.subjectId, + senderPeerId: relayPrincipal.peerId, + senderReplicaIncarnation: relayOpenRequest.senderReplicaIncarnation, + recipientSubjectId: relayOpenRequest.remote.subjectId, + recipientPeerId: relayOpenRequest.remote.peerId + }, + relayMessageId, + relayPeerId: serverPeerId, + senderConnectionEpoch: "epoch", + senderSequence: 0, + documentIds: [taskId], + payloadVersion: 1, + messageHash, + outerEnvelopeDigest, + payloadBytes: payload.byteLength, + claimToken, + claimDeadline: 1, + sessionGeneration: request.sessionGeneration + })), + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" as const + } + }), + loadClaimedPayload: () => Effect.succeed(payload), + release: () => + Deferred.succeed(releaseStarted, undefined).pipe( + Effect.andThen(Deferred.await(releaseGate)), + Effect.as(relayTransition) + ) + }) + const ingress = PeerRelayIngress.PeerRelayIngress.of({ + address: { _tag: "UnixAddress", path: "relay-test" }, + reserveOutbound: (bytes) => + Effect.succeed({ + bytes, + release: Effect.void, + transferToCurrentRequest: Effect.void + }), + usage: Effect.succeed({ + connections: 0, + reservedBytes: 0, + byteReservationWaiters: 0 + }), + await: Effect.never + }) + const context = yield* makeRelayHandlerContext( + authorization, + store, + PeerRelayLimits.defaults, + ingress ) - assert.strictEqual((yield* Deferred.await(admitted)).outerEnvelopeDigest, expectedDigest) - const defect = new Error("relay authorization defect") - authorizationDefect = defect - const defectExit = yield* Stream.runHead( - (openHandler.handler( - openRequest, + const handler = context.mapUnsafe.get( + PeerRelayRpc.OpenRelayRpc.key + ) as Rpc.Handler<"OpenRelay"> + const runtime = Context.get(context, PeerRpcServer.PeerRelayServerRuntime) + const events = yield* Queue.unbounded() + const first = yield* Stream.runForEach( + handler.handler( + relayOpenRequest, {} as never - ) as Stream.Stream).pipe( - Stream.provideContext(Context.add( - openHandler.context, - PeerAuthentication.AuthenticatedPeer, - authenticated - )) - ) - ).pipe(Effect.exit) - assert.isTrue(Exit.isFailure(defectExit)) - if (Exit.isFailure(defectExit)) { - assert.strictEqual(Cause.squash(defectExit.cause), defect) - } - authorizationDefect = undefined - const ownerContext = yield* Layer.build(Layer.fresh(handlers)) - const ownerRuntime = Context.get(ownerContext, PeerRpcServer.PeerRelayServerRuntime) - const serverContext = Context.add( - Context.add( - Context.add( - ownerContext, - RpcServer.Protocol, - protocol - ), - PeerRelayIngress.PeerRelayIngress, - { - ...relayIngress, - await: Effect.never.pipe( - Effect.ensuring(Deferred.succeed(ingressStopped, undefined)) - ) - } - ), - PeerAuthentication.PeerAuthentication, - PeerAuthentication.PeerAuthentication.of((effect) => - Effect.provideService(effect, PeerAuthentication.AuthenticatedPeer, { - principal: localPrincipal, - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.never - }) - ) + ) as Stream.Stream, + (event) => Queue.offer(events, event) + ).pipe( + (effect) => relayHandlerEffect(handler, effect), + Effect.forkScoped ) - yield* Layer.build( - PeerRpcServer.layerRelayServer.pipe( - Layer.provide(Layer.succeedContext(serverContext)) - ) + assert.strictEqual((yield* Queue.take(events))._tag, "RelayOpened") + assert.strictEqual((yield* Queue.take(events))._tag, "StoredMessage") + assert.strictEqual((yield* runtime.usage).activeClaims, 1) + assert.strictEqual((yield* Metric.value(PeerRpcObservability.relayActiveClaims())).value, 1) + assert.strictEqual( + (yield* Metric.value(PeerRpcObservability.relayWorkers())).value, + PeerRelayLimits.defaults.relayWorkerConcurrency ) - const ownerDefect = new Error("relay maintenance defect") - maintenanceDefect = ownerDefect - yield* TestClock.adjust(PeerRelayLimits.defaults.maintenanceIntervalMillis) - const ownerExit = yield* ownerRuntime.owner.pipe(Effect.exit) - assert.isTrue(Exit.isFailure(ownerExit)) - if (Exit.isFailure(ownerExit)) { - assert.strictEqual(Cause.squash(ownerExit.cause), ownerDefect) - } - yield* Effect.yieldNow + assert.strictEqual( + (yield* Metric.value( + PeerRpcObservability.relayOutcomes("RelayClaim", "Receive", "Delivered") + )).count, + 1 + ) + assert.strictEqual( + (yield* Metric.value( + PeerRpcObservability.relayBytes("RelayClaim", "Receive", "Delivered", 1) + )).count, + 1 + ) + assert.strictEqual((yield* Metric.value(PeerRpcObservability.relayPendingItems())).value, 0) + assert.strictEqual((yield* Metric.value(PeerRpcObservability.relayPendingBytes())).value, 0) + const second = yield* Stream.runHead( + handler.handler( + relayOpenRequest, + {} as never + ) as Stream.Stream + ).pipe( + (effect) => relayHandlerEffect(handler, effect), + Effect.forkScoped + ) + yield* Deferred.await(releaseStarted) + const interrupted = yield* Fiber.interrupt(second).pipe(Effect.forkScoped) yield* Effect.yieldNow - assert.isTrue(Option.isSome(yield* Deferred.poll(protocolStopped))) - assert.isTrue(Option.isSome(yield* Deferred.poll(ingressStopped))) - yield* runtime.shutdown + yield* Deferred.succeed(releaseGate, undefined) + yield* Fiber.join(interrupted) + assert.deepStrictEqual(yield* runtime.usage, { + accepting: true, + sessions: 0, + subjects: 0, + activeClaims: 0, + queuedChannels: 0 + }) + assert.strictEqual((yield* Metric.value(PeerRpcObservability.relayActiveClaims())).value, 0) + yield* Fiber.interrupt(first) + + claimed = false + releaseStarted = yield* Deferred.make() + releaseGate = yield* Deferred.make() + const nextEvents = yield* Queue.unbounded() + const nextIncumbent = yield* Stream.runForEach( + handler.handler( + relayOpenRequest, + {} as never + ) as Stream.Stream, + (event) => Queue.offer(nextEvents, event) + ).pipe( + (effect) => relayHandlerEffect(handler, effect), + Effect.forkScoped + ) + assert.strictEqual((yield* Queue.take(nextEvents))._tag, "RelayOpened") + assert.strictEqual((yield* Queue.take(nextEvents))._tag, "StoredMessage") + trackLateMonitors = true + const staleOpen = yield* Stream.runHead( + handler.handler( + relayOpenRequest, + {} as never + ) as Stream.Stream + ).pipe( + (effect) => relayHandlerEffect(handler, effect), + Effect.forkScoped + ) + yield* Deferred.await(releaseStarted) + assert.strictEqual(trackedAuthorizations, 2) + trackLateMonitors = false + yield* Stream.runHead( + handler.handler( + relayOpenRequest, + {} as never + ) as Stream.Stream + ).pipe((effect) => relayHandlerEffect(handler, effect)) + yield* Deferred.succeed(releaseGate, undefined) + yield* Fiber.join(staleOpen) + yield* Effect.forEach( + Array.from({ length: 100 }), + () => Effect.yieldNow, + { discard: true } + ) + yield* Deferred.succeed(lateMonitorSignal, undefined) + yield* Effect.forEach( + Array.from({ length: 100 }), + () => Effect.yieldNow, + { discard: true } + ) + assert.strictEqual(yield* Ref.get(lateMonitors), 0) + yield* Fiber.interrupt(nextIncumbent) + yield* runtime.shutdown - const stale = yield* (acknowledgeHandler.handler({ - sessionId: (opened as PeerRelayRpc.RelayOpened).sessionId, - relayMessageId, - claimToken: PeerRelayRpc.ClaimToken.make("clm_00000000-0000-4000-8000-000000000001"), - messageHash - }, {} as never) as Effect.Effect).pipe( - Effect.provideContext(Context.add( - acknowledgeHandler.context, - PeerAuthentication.AuthenticatedPeer, - authenticated - )), + assert.strictEqual((yield* Metric.value(PeerRpcObservability.relayWorkers())).value, 0) + assert.strictEqual((yield* Metric.value(PeerRpcObservability.relayReadyQueueItems("New"))).value, 0) + assert.strictEqual((yield* Metric.value(PeerRpcObservability.relayReadyQueueItems("Retry"))).value, 0) + })).pipe( + Effect.provideService(Metric.MetricRegistry, new Map()) + )) + + it.effect("drains admitted Push after a session authorization monitor defects", () => + Effect.scoped(Effect.gen(function*() { + const authorizationDefect = yield* Deferred.make() + const blockingFinalizer = yield* Deferred.make() + let authorizationCalls = 0 + const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ + authorize: (request) => + Effect.sync(() => { + authorizationCalls++ + return { + remote: { + tenantId: request.principal.tenantId, + subjectId: request.remote.subjectId, + peerId: request.remote.peerId + }, + documents: request.documents.map((document) => ({ + document: Task, + documentId: document.documentId + })), + validUntil: Number.MAX_SAFE_INTEGER, + invalidated: authorizationCalls === 1 + ? Deferred.await(authorizationDefect) + : authorizationCalls === 2 + ? Effect.never.pipe( + Effect.ensuring(Deferred.await(blockingFinalizer)) + ) + : Effect.never + } + }), + authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode + }) + const admitStarted = yield* Deferred.make() + const allowAdmit = yield* Deferred.make() + const admitCalls = yield* Ref.make(0) + const store = makeRelayStore({ + admit: (input) => + Deferred.succeed(admitStarted, undefined).pipe( + Effect.andThen(Deferred.await(allowAdmit)), + Effect.andThen(Ref.update(admitCalls, (count) => count + 1)), + Effect.as({ + status: "Accepted", + channel: input.channel, + ready: false, + nextEligibleAt: 0, + lane: "New" + }) + ) + }) + const context = yield* makeRelayHandlerContext(authorization, store) + const openHandler = context.mapUnsafe.get( + PeerRelayRpc.OpenRelayRpc.key + ) as Rpc.Handler<"OpenRelay"> + const pushHandler = context.mapUnsafe.get( + PeerRelayRpc.PushRelayRpc.key + ) as Rpc.Handler<"PushRelay"> + const runtime = Context.get(context, PeerRpcServer.PeerRelayServerRuntime) + const events = yield* Queue.unbounded() + const openFiber = yield* Stream.runForEach( + openHandler.handler( + relayOpenRequest, + {} as never + ) as Stream.Stream, + (event) => Queue.offer(events, event) + ).pipe( + (effect) => relayHandlerEffect(openHandler, effect), + Effect.forkScoped + ) + const opened = yield* Queue.take(events) + assert.strictEqual(opened._tag, "RelayOpened") + const { payload } = yield* makeRelayPayload + const pushFiber = yield* ( + pushHandler.handler({ + sessionId: (opened as PeerRelayRpc.RelayOpened).sessionId, + relayMessageId: Identity.RelayMessageId.make( + "rly_00000000-0000-4000-8000-000000000041" + ), + payload + }, {} as never) as Effect.Effect + ).pipe( + (effect) => relayHandlerEffect(pushHandler, effect), + Effect.exit, + Effect.forkScoped + ) + yield* Deferred.await(admitStarted) + const defect = new Error("authorization invalidation defect") + yield* Deferred.die(authorizationDefect, defect) + yield* Effect.yieldNow + assert.strictEqual((yield* runtime.usage).sessions, 0) + yield* Deferred.succeed(allowAdmit, undefined) + const pushExit = yield* Fiber.join(pushFiber) + assert.isTrue(Exit.isSuccess(pushExit)) + assert.strictEqual(yield* Ref.get(admitCalls), 1) + const later = yield* ( + pushHandler.handler({ + sessionId: (opened as PeerRelayRpc.RelayOpened).sessionId, + relayMessageId: Identity.RelayMessageId.make( + "rly_00000000-0000-4000-8000-000000000049" + ), + payload + }, {} as never) as Effect.Effect + ).pipe( + (effect) => relayHandlerEffect(pushHandler, effect), Effect.flip ) - assert.strictEqual(stale._tag, "SessionUnavailable") - assert.strictEqual(yield* Ref.get(acknowledgeCalls), 0) + assert.instanceOf(later, PeerRpcError.SessionUnavailable) + assert.strictEqual(yield* Ref.get(admitCalls), 1) + yield* Effect.yieldNow assert.deepStrictEqual(yield* runtime.usage, { - accepting: false, + accepting: true, sessions: 0, subjects: 0, activeClaims: 0, queuedChannels: 0 }) + yield* Deferred.succeed(blockingFinalizer, undefined) + yield* Fiber.interrupt(openFiber) + }))) + + it.effect("completes queued and permit-waiting SQL submissions during shutdown", () => + Effect.scoped(Effect.gen(function*() { + const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ + authorize: (request) => + Effect.succeed({ + remote: { + tenantId: request.principal.tenantId, + subjectId: request.remote.subjectId, + peerId: request.remote.peerId + }, + documents: request.documents.map((document) => ({ + document: Task, + documentId: document.documentId + })), + validUntil: Number.MAX_SAFE_INTEGER, + invalidated: Effect.never + }), + authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode + }) + const admitStarted = yield* Deferred.make() + const admitCalls = yield* Ref.make(0) + const store = makeRelayStore({ + admit: (input) => + Ref.updateAndGet(admitCalls, (count) => count + 1).pipe( + Effect.tap(() => Deferred.succeed(admitStarted, undefined)), + Effect.andThen(Effect.never), + Effect.as({ + status: "Accepted", + channel: input.channel, + ready: false, + nextEligibleAt: 0, + lane: "New" + }) + ) + }) + const context = yield* makeRelayHandlerContext( + authorization, + store, + PeerRelayLimits.Values.make({ + ...PeerRelayLimits.defaults, + maxInFlightPush: 4, + maxInFlightPushPerSubject: 4, + admissionRatePerSecond: 1_000, + admissionBurst: 100, + maxInFlightSqlTransactions: 2, + maxInFlightSqlAdmission: 1, + sqlAdmissionQueueCapacity: 4 + }) + ) + const openHandler = context.mapUnsafe.get( + PeerRelayRpc.OpenRelayRpc.key + ) as Rpc.Handler<"OpenRelay"> + const pushHandler = context.mapUnsafe.get( + PeerRelayRpc.PushRelayRpc.key + ) as Rpc.Handler<"PushRelay"> + const runtime = Context.get(context, PeerRpcServer.PeerRelayServerRuntime) + const events = yield* Queue.unbounded() + const openFiber = yield* Stream.runForEach( + openHandler.handler( + relayOpenRequest, + {} as never + ) as Stream.Stream, + (event) => Queue.offer(events, event) + ).pipe( + (effect) => relayHandlerEffect(openHandler, effect), + Effect.forkScoped + ) + const opened = yield* Queue.take(events) + assert.strictEqual(opened._tag, "RelayOpened") + const sessionId = (opened as PeerRelayRpc.RelayOpened).sessionId + const { payload } = yield* makeRelayPayload + const invoke = (relayMessageId: Identity.RelayMessageId) => + pushHandler.handler({ + sessionId, + relayMessageId, + payload + }, {} as never) as Effect.Effect + const calls = yield* Effect.forEach( + [ + "rly_00000000-0000-4000-8000-000000000011", + "rly_00000000-0000-4000-8000-000000000012", + "rly_00000000-0000-4000-8000-000000000013" + ], + (id) => + Effect.gen(function*() { + const completed = yield* Deferred.make>() + const fiber = yield* relayHandlerEffect( + pushHandler, + invoke(Identity.RelayMessageId.make(id)) + ).pipe( + Effect.exit, + Effect.flatMap((exit) => Deferred.succeed(completed, exit)), + Effect.forkScoped + ) + return { completed, fiber } + }) + ) + yield* Effect.yieldNow + yield* Deferred.await(admitStarted) + yield* Effect.yieldNow + yield* Effect.yieldNow + assert.strictEqual(yield* Ref.get(admitCalls), 1) + yield* runtime.shutdown + yield* Effect.yieldNow + yield* Effect.yieldNow + const completed = yield* Effect.forEach(calls, (call) => Deferred.poll(call.completed)) + yield* Effect.forEach(calls, (call) => Fiber.interrupt(call.fiber), { + discard: true + }) + assert.isTrue(completed.every(Option.isSome)) yield* Fiber.interrupt(openFiber) }))) }) diff --git a/packages/local-rpc/test/PublicApi.types.test.ts b/packages/local-rpc/test/PublicApi.types.test.ts index 471202f..64206f9 100644 --- a/packages/local-rpc/test/PublicApi.types.test.ts +++ b/packages/local-rpc/test/PublicApi.types.test.ts @@ -4,8 +4,6 @@ import * as PublicApi from "../src/index.js" describe("public API", () => { it("exports the version 3 relay protocol without widening version 2", () => { - const publicApi: Record = PublicApi - for ( const name of [ "PeerRelayAuthorization", @@ -15,12 +13,10 @@ describe("public API", () => { "PeerRelayStore" ] ) { - assert.property(publicApi, name) + assert.property(PublicApi, name) } - const peerRelayRpc = publicApi.PeerRelayRpc as Record - const relayOpened = peerRelayRpc.RelayOpened as Schema.Schema - const decoded = Schema.decodeUnknownSync(relayOpened)({ + const decoded = Schema.decodeUnknownSync(PublicApi.PeerRelayRpc.RelayOpened)({ _tag: "RelayOpened", version: 3, sessionId: "ses_00000000-0000-4000-8000-000000000001", diff --git a/packages/local-rpc/test/RpcPeerTransport.test.ts b/packages/local-rpc/test/RpcPeerTransport.test.ts index aeef705..e80bb3f 100644 --- a/packages/local-rpc/test/RpcPeerTransport.test.ts +++ b/packages/local-rpc/test/RpcPeerTransport.test.ts @@ -1,14 +1,18 @@ +import * as Automerge from "@automerge/automerge" import { NodeCrypto } from "@effect/platform-node" import { assert, describe, it } from "@effect/vitest" import * as PeerRelayClientRuntime from "@lucas-barake/effect-local-sql/PeerRelayClientRuntime" import type * as PeerRelayOutbox from "@lucas-barake/effect-local-sql/PeerRelayOutbox" import * as PeerSyncEnvelope from "@lucas-barake/effect-local-sql/PeerSyncEnvelope" +import * as TestReplica from "@lucas-barake/effect-local-test/TestReplica" +import * as Canonical from "@lucas-barake/effect-local/Canonical" import * as Document from "@lucas-barake/effect-local/Document" import * as DocumentSet from "@lucas-barake/effect-local/DocumentSet" import * as Identity from "@lucas-barake/effect-local/Identity" import * as PeerTransport from "@lucas-barake/effect-local/PeerTransport" import * as ReplicaDefinition from "@lucas-barake/effect-local/ReplicaDefinition" import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" +import * as ReplicaLimits from "@lucas-barake/effect-local/ReplicaLimits" import * as Cause from "effect/Cause" import * as Context from "effect/Context" import * as Deferred from "effect/Deferred" @@ -46,6 +50,7 @@ const relayMessageId = Identity.RelayMessageId.make("rly_00000000-0000-4000-8000 const senderReplicaIncarnation = Identity.ReplicaIncarnation.make(1) const claimToken = PeerRelayRpc.ClaimToken.make("clm_00000000-0000-4000-8000-000000000001") const relayMessageHash = "1".repeat(64) +const rewrittenLineage = Identity.DocumentLineage.make("lin_00000000-0000-4000-8000-000000000001") const documents = [{ document: Task, documentId }] const definition = ReplicaDefinition.make({ name: "rpc-peer-transport-test", @@ -162,8 +167,48 @@ const relayEntry = (payload: Uint8Array): PeerRelayOutbox.Entry => ({ nextAttemptAt: "2026-07-25T00:00:00.000Z" }) -const makeStoredMessage = (payload: Uint8Array) => +const makeStoredMessage = ( + payloadSeed: Uint8Array, + lineage: Identity.DocumentLineage = Identity.genesisLineage +) => Effect.gen(function*() { + let source = Automerge.from( + { value: [...payloadSeed] }, + { actor: "a".repeat(32) } + ) + const remote = Automerge.init() + const handshake = Automerge.generateSyncMessage( + remote, + Automerge.initSyncState() + )[1]! + const received = Automerge.receiveSyncMessage( + source, + Automerge.initSyncState(), + handshake + ) + source = received[0] + const message = Automerge.generateSyncMessage(source, received[1])[1]! + const writerProvenance = Automerge.getAllChanges(source).map((bytes) => { + const change = Automerge.decodeChange(bytes) + return { + changeHash: change.hash, + writerSchemaVersion: 1, + writerDefinitionHash: definition.hash + } + }) + Automerge.free(source) + Automerge.free(remote) + const messageHash = yield* Canonical.digest(message) + const payload = yield* PeerSyncEnvelope.encodeSyncEnvelope({ + connectionEpoch: "remote-epoch", + sequence: 3, + documentId, + documentType: Task.name, + messageHash, + message, + lineage, + writerProvenance + }) const envelope: PeerSyncEnvelope.RelayOuterEnvelope = { domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, version: PeerSyncEnvelope.relayOuterEnvelopeVersion, @@ -181,8 +226,9 @@ const makeStoredMessage = (payload: Uint8Array) => senderConnectionEpoch: "remote-epoch", senderSequence: 3, document: { documentType: Task.name, documentId }, - writerProvenance: [], - messageHash: relayMessageHash, + lineage, + writerProvenance, + messageHash, payload } const outerEnvelopeDigest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope(envelope) @@ -257,7 +303,10 @@ const connectRelay = ( RpcPeerTransport.layerStoreAndForward(client, relayOptions).pipe( Layer.provide(Layer.merge( Layer.succeed(PeerRelayClientRuntime.PeerRelayClientRuntime, runtime), - NodeCrypto.layer + Layer.merge( + NodeCrypto.layer, + ReplicaLimits.layer(TestReplica.defaultLimits) + ) )) ) ) @@ -1277,7 +1326,10 @@ describe("RpcPeerTransport store and forward", () => { maximumPendingHorizon: () => Effect.succeed(relayOptions.senderRetryHorizonMillis + 1) }) ).pipe(Effect.flip) - assert.strictEqual(error.reason._tag, "ProtocolMismatch") + assert.strictEqual(error._tag, "ReplicaError") + if (error._tag === "ReplicaError") { + assert.strictEqual(error.reason._tag, "ProtocolMismatch") + } assert.strictEqual(yield* Ref.get(opens), 0) }))) @@ -1298,7 +1350,10 @@ describe("RpcPeerTransport store and forward", () => { ) ) const error = yield* connectRelay(client, makeRuntime()).pipe(Effect.flip) - assert.strictEqual(error.reason._tag, "ProtocolMismatch") + assert.strictEqual(error._tag, "ReplicaError") + if (error._tag === "ReplicaError") { + assert.strictEqual(error.reason._tag, "ProtocolMismatch") + } assert.strictEqual(yield* Ref.get(finalized), 1) }))) @@ -1326,6 +1381,25 @@ describe("RpcPeerTransport store and forward", () => { }).pipe(Effect.provide(NodeCrypto.layer)) )) + it.effect("reconstructs a non-genesis lineage when verifying the outer digest", () => + Effect.scoped( + Effect.gen(function*() { + const stored = yield* makeStoredMessage( + Uint8Array.of(2, 3, 4), + rewrittenLineage + ) + const client = makeRelayClient( + () => Stream.fromIterable([relayOpened, stored]).pipe(Stream.rechunk(1)) + ) + const connection = yield* connectRelay(client, makeRuntime()) + const delivery = yield* Stream.runHead( + connection.receiveWithAcknowledgement! + ).pipe(Effect.map(Option.getOrThrow)) + assert.deepStrictEqual(delivery.message, stored.payload) + yield* connection.close + }).pipe(Effect.provide(NodeCrypto.layer)) + )) + it.effect("defers Ack until the delivery consumer commits and closes on Ack failure", () => Effect.scoped( Effect.gen(function*() { @@ -1344,7 +1418,7 @@ describe("RpcPeerTransport store and forward", () => { assert.strictEqual(request.sessionId, sessionId) assert.strictEqual(request.relayMessageId, relayMessageId) assert.strictEqual(request.claimToken, claimToken) - assert.strictEqual(request.messageHash, relayMessageHash) + assert.strictEqual(request.messageHash, stored.messageHash) return Ref.update(acknowledgements, (count) => count + 1).pipe( Effect.andThen(Effect.fail(new PeerRpcError.ServerUnavailable())) ) @@ -1420,7 +1494,7 @@ describe("RpcPeerTransport store and forward", () => { sessionId, relayMessageId, claimToken, - messageHash: relayMessageHash, + messageHash: stored.messageHash, reason: "ApplicationRejected" }) yield* connection.close @@ -1488,6 +1562,107 @@ describe("RpcPeerTransport store and forward", () => { } }))) + it.effect("records fixed adapter acknowledgement outcomes and bounded facts", () => { + const spans: Array = [] + const tracer = Tracer.make({ + span: (options) => { + const span = new Tracer.NativeSpan(options) + spans.push(span) + return span + } + }) + return Effect.scoped(Effect.gen(function*() { + const stored = yield* makeStoredMessage(Uint8Array.of(7, 8, 9)) + const ackClient = makeRelayClient( + () => Stream.fromIterable([relayOpened, stored]).pipe(Stream.rechunk(1)) + ) + const ackConnection = yield* connectRelay(ackClient, makeRuntime()) + const acknowledged = yield* Stream.runHead( + ackConnection.receiveWithAcknowledgement! + ).pipe(Effect.map(Option.getOrThrow)) + yield* acknowledged.acknowledge + yield* ackConnection.close + + const rejectClient = makeRelayClient( + () => Stream.fromIterable([relayOpened, stored]).pipe(Stream.rechunk(1)) + ) + const rejectConnection = yield* connectRelay(rejectClient, makeRuntime()) + const rejected = yield* Stream.runHead( + rejectConnection.receiveWithAcknowledgement! + ).pipe(Effect.map(Option.getOrThrow)) + yield* rejected.reject("ApplicationRejected") + yield* rejectConnection.close + + const unavailableClient = makeRelayClient( + () => Stream.fromIterable([relayOpened, stored]).pipe(Stream.rechunk(1)), + undefined, + () => Effect.fail(new PeerRpcError.ServerUnavailable()) + ) + const unavailableConnection = yield* connectRelay( + unavailableClient, + makeRuntime() + ) + const unavailable = yield* Stream.runHead( + unavailableConnection.receiveWithAcknowledgement! + ).pipe(Effect.map(Option.getOrThrow)) + assert.isTrue(Exit.isFailure(yield* unavailable.acknowledge.pipe(Effect.exit))) + + const terminalSpans = spans.filter((span) => span.name === "effect_local_rpc.adapter.relay_acknowledge") + assert.strictEqual(terminalSpans.length, 3) + assert.deepStrictEqual( + terminalSpans.map((span) => { + const attributes = Object.fromEntries(span.attributes) + return { + name: span.name, + operation: attributes["rpc.operation"], + direction: attributes["rpc.direction"], + result: attributes["rpc.result"], + bytes: attributes["rpc.bytes"], + items: attributes["rpc.items"], + version: attributes["rpc.version"], + latencyMillis: attributes["rpc.latency_millis"] + } + }), + [ + { + name: "effect_local_rpc.adapter.relay_acknowledge", + operation: "AdapterAcknowledge", + direction: "Receive", + result: "Acknowledged", + bytes: stored.payload.byteLength, + items: 1, + version: String(stored.payloadVersion), + latencyMillis: undefined + }, + { + name: "effect_local_rpc.adapter.relay_acknowledge", + operation: "AdapterAcknowledge", + direction: "Receive", + result: "DeadLettered", + bytes: stored.payload.byteLength, + items: 1, + version: String(stored.payloadVersion), + latencyMillis: undefined + }, + { + name: "effect_local_rpc.adapter.relay_acknowledge", + operation: "AdapterAcknowledge", + direction: "Receive", + result: "Unavailable", + bytes: stored.payload.byteLength, + items: 1, + version: String(stored.payloadVersion), + latencyMillis: undefined + } + ] + ) + })).pipe( + Effect.provideService(Metric.MetricRegistry, new Map()), + Effect.provideService(Tracer.Tracer, tracer), + Effect.provide(NodeCrypto.layer) + ) + }) + it.effect("records relay boundaries without endpoint payload or claim values", () => { const spans: Array = [] const tracer = Tracer.make({ diff --git a/packages/local-sql/README.md b/packages/local-sql/README.md index a691858..8c6d113 100644 --- a/packages/local-sql/README.md +++ b/packages/local-sql/README.md @@ -2,4 +2,13 @@ SQLite persistence, Automerge history, Effect Cluster command execution, and Effect Workflow operations for Effect Local. -See the [Effect Local documentation](https://github.com/lucas-barake/effect-local#readme) for durable composition, projections, recovery, compaction, sync, and API reference. +Optional relay support adds the stable sender outbox, sender scoped recipient receipts, quota usage, maintenance, and +restore fencing used by store and forward. Direct SQL composition remains unchanged. +Relay infrastructure keeps Automerge bytes opaque. Relay enabled `PeerSync` owns the one semantic decode and its +existing change, dependency, and operation limits. +Relay SQLite commits are not atomic with an external policy authority. The RPC server local gate decides whether +revocation prevents admission or drains an already admitted bounded operation. + +See the [Effect Local documentation](https://github.com/lucas-barake/effect-local#readme) for durable composition, +projections, recovery, compaction, sync, and API reference. The relay durability boundaries are documented in +[Store and forward](https://github.com/lucas-barake/effect-local/blob/main/docs/store-and-forward.md). diff --git a/packages/local-sql/src/BackupStore.ts b/packages/local-sql/src/BackupStore.ts index ae8a96f..c79bcd0 100644 --- a/packages/local-sql/src/BackupStore.ts +++ b/packages/local-sql/src/BackupStore.ts @@ -784,6 +784,10 @@ export const layer = (definition: ReplicaDefinition.Any): Layer.Layer< yield* sql`DELETE FROM effect_local_peer_relay_outbox` yield* sql`DELETE FROM effect_local_peer_relay_outbox_remote_usage` yield* sql`DELETE FROM effect_local_peer_relay_outbox_replica_usage` + yield* sql`INSERT INTO effect_local_peer_relay_receipt_delete_tokens (receipt_row_id) + SELECT row_id + FROM effect_local_peer_receipts + WHERE relay_message_id IS NOT NULL` yield* sql`DELETE FROM effect_local_peer_receipts WHERE relay_message_id IS NOT NULL` yield* sql`DELETE FROM effect_local_peer_relay_receipt_usage` diff --git a/packages/local-sql/src/Compaction.ts b/packages/local-sql/src/Compaction.ts index 459c7e6..4da6734 100644 --- a/packages/local-sql/src/Compaction.ts +++ b/packages/local-sql/src/Compaction.ts @@ -91,6 +91,18 @@ const InstalledDocumentRow = Schema.Struct({ }) const CheckpointHashRow = Schema.Struct({ checkpoint_hash: Schema.String }) const CountRow = Schema.Struct({ count: Schema.Int }) +const RelayReceiptRewriteRow = Schema.Struct({ + encoded_size: Schema.Int.check(Schema.isGreaterThan(0)), + replica_incarnation: Identity.ReplicaIncarnation, + row_id: Schema.Int.check(Schema.isGreaterThan(0)), + sender_peer_id: Identity.PeerId, + sender_subject_id: Schema.String, + sender_tenant_id: Schema.String +}) +const RelayReceiptUsageRow = Schema.Struct({ + encoded_bytes: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + receipt_count: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)) +}) /** Only the columns `rewriteHistory`'s in-transaction guard and compare-and-swap read. */ const RewriteGuardRow = Schema.Struct({ accepted_heads: Schema.String, @@ -153,9 +165,11 @@ export class Compaction extends Context.Service + sql`SELECT COUNT(*) AS count FROM effect_local_peer_relay_outbox + WHERE document_id = ${documentId}` + }) + const unexpiredRelayReceiptCount = SqlSchema.findOneOption({ + Request: Schema.Struct({ + documentId: Identity.DocumentId, + expiresAt: Schema.String + }), + Result: CountRow, + execute: (request) => + sql`SELECT COUNT(*) AS count FROM effect_local_peer_receipts + WHERE document_id = ${request.documentId} + AND relay_message_id IS NOT NULL + AND relay_receipt_expires_at > ${request.expiresAt}` + }) + const expiredRelayReceipts = SqlSchema.findAll({ + Request: Schema.Struct({ + documentId: Identity.DocumentId, + expiresAt: Schema.String + }), + Result: RelayReceiptRewriteRow, + execute: (request) => + sql`SELECT relay_encoded_size AS encoded_size, replica_incarnation, row_id, + relay_sender_peer_id AS sender_peer_id, + relay_sender_subject_id AS sender_subject_id, + relay_sender_tenant_id AS sender_tenant_id + FROM effect_local_peer_receipts + WHERE document_id = ${request.documentId} + AND relay_message_id IS NOT NULL + AND pending_message IS NULL + AND relay_receipt_expires_at <= ${request.expiresAt} + ORDER BY replica_incarnation, relay_sender_tenant_id, relay_sender_subject_id, + relay_sender_peer_id, relay_message_id, row_id` + }) + const decrementRelayReceiptUsage = SqlSchema.findAll({ + Request: Schema.Struct({ + encodedBytes: Schema.Int.check(Schema.isGreaterThan(0)), + receiptCount: Schema.Int.check(Schema.isGreaterThan(0)), + replicaIncarnation: Identity.ReplicaIncarnation, + senderPeerId: Identity.PeerId, + senderSubjectId: Schema.String, + senderTenantId: Schema.String + }), + Result: RelayReceiptUsageRow, + execute: (request) => + sql`UPDATE effect_local_peer_relay_receipt_usage + SET receipt_count = receipt_count - ${request.receiptCount}, + encoded_bytes = encoded_bytes - ${request.encodedBytes} + WHERE replica_incarnation = ${request.replicaIncarnation} + AND sender_tenant_id = ${request.senderTenantId} + AND sender_subject_id = ${request.senderSubjectId} + AND sender_peer_id = ${request.senderPeerId} + AND receipt_count >= ${request.receiptCount} + AND encoded_bytes >= ${request.encodedBytes} + RETURNING receipt_count, encoded_bytes` + }) + const deleteZeroRelayReceiptUsage = SqlSchema.findAll({ + Request: Schema.Struct({ + replicaIncarnation: Identity.ReplicaIncarnation, + senderPeerId: Identity.PeerId, + senderSubjectId: Schema.String, + senderTenantId: Schema.String + }), + Result: Schema.Struct({ sender_peer_id: Identity.PeerId }), + execute: (request) => + sql`DELETE FROM effect_local_peer_relay_receipt_usage + WHERE replica_incarnation = ${request.replicaIncarnation} + AND sender_tenant_id = ${request.senderTenantId} + AND sender_subject_id = ${request.senderSubjectId} + AND sender_peer_id = ${request.senderPeerId} + AND receipt_count = 0 + AND encoded_bytes = 0 + RETURNING sender_peer_id` + }) + const authorizeRelayReceiptDelete = SqlSchema.findAll({ + Request: Schema.Int.check(Schema.isGreaterThan(0)), + Result: Schema.Struct({ receipt_row_id: Schema.Int.check(Schema.isGreaterThan(0)) }), + execute: (rowId) => + sql`INSERT INTO effect_local_peer_relay_receipt_delete_tokens (receipt_row_id) + VALUES (${rowId}) + RETURNING receipt_row_id` + }) + const deleteExpiredRelayReceipt = SqlSchema.findAll({ + Request: Schema.Struct({ + documentId: Identity.DocumentId, + expiresAt: Schema.String, + rowId: Schema.Int.check(Schema.isGreaterThan(0)) + }), + Result: Schema.Struct({ row_id: Schema.Int.check(Schema.isGreaterThan(0)) }), + execute: (request) => + sql`DELETE FROM effect_local_peer_receipts + WHERE row_id = ${request.rowId} + AND document_id = ${request.documentId} + AND relay_message_id IS NOT NULL + AND pending_message IS NULL + AND relay_receipt_expires_at <= ${request.expiresAt} + RETURNING row_id` + }) /** * The fail closed fence for a history rewrite. * @@ -1112,16 +1230,20 @@ export const layer: Layer.Layer< const row = guard.value // Re-read from SQL rather than reused from the `recover` above: that read committed its // own transaction and released the connection, so anything it observed is stale here. - const [unapplied, receipts, outbox, priorChanges] = yield* Effect.all([ + const receiptExpiryCutoff = DateTime.formatIso(yield* DateTime.now) + const [unapplied, receipts, outbox, relayOutbox, unexpiredRelayReceipts, priorChanges] = yield* Effect.all([ pendingCount(documentId), pendingReceiptCount(documentId), pendingOutboxCount(documentId), + relayOutboxCount(documentId), + unexpiredRelayReceiptCount({ documentId, expiresAt: receiptExpiryCutoff }), changeCount(documentId) ]) const nonZero = (count: Option.Option) => Option.exists(count, (row) => row.count !== 0) if ( row.document_type !== document.name || row.materialized_heads !== row.accepted_heads || - nonZero(unapplied) || nonZero(receipts) || nonZero(outbox) + nonZero(unapplied) || nonZero(receipts) || nonZero(outbox) || nonZero(relayOutbox) || + nonZero(unexpiredRelayReceipts) ) { return yield* new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageCorrupt({ @@ -1168,10 +1290,81 @@ export const layer: Layer.Layer< Effect.map((row) => row.commit_sequence), Effect.catchTag("NoSuchElementError", () => Effect.die(new Error("Replica metadata was not initialized"))) ) + const settledRelayReceipts = yield* expiredRelayReceipts({ + documentId, + expiresAt: receiptExpiryCutoff + }) yield* sql`DELETE FROM effect_local_changes WHERE document_id = ${documentId}` yield* sql`DELETE FROM effect_local_checkpoints WHERE document_id = ${documentId}` yield* sql`DELETE FROM effect_local_peer_outbox WHERE document_id = ${documentId}` - yield* sql`DELETE FROM effect_local_peer_receipts WHERE document_id = ${documentId}` + // Relay receipts are quota reservations as well as duplicate evidence. Decrement each + // exact sender reservation before deleting its expired receipts. Any mismatch fails the + // transaction, which restores the document root, receipts, and usage together. + const usage = new Map() + for (const receipt of settledRelayReceipts) { + const key = JSON.stringify([ + receipt.replica_incarnation, + receipt.sender_tenant_id, + receipt.sender_subject_id, + receipt.sender_peer_id + ]) + const current = usage.get(key) + usage.set(key, { + encodedBytes: (current?.encodedBytes ?? 0) + receipt.encoded_size, + receiptCount: (current?.receiptCount ?? 0) + 1, + replicaIncarnation: receipt.replica_incarnation, + senderPeerId: receipt.sender_peer_id, + senderSubjectId: receipt.sender_subject_id, + senderTenantId: receipt.sender_tenant_id + }) + } + const zeroUsage: Array<{ + readonly replicaIncarnation: Identity.ReplicaIncarnation + readonly senderPeerId: Identity.PeerId + readonly senderSubjectId: string + readonly senderTenantId: string + }> = [] + for (const entry of usage.values()) { + const updated = yield* decrementRelayReceiptUsage(entry) + if (updated.length !== 1) { + return yield* failStorageCorrupt(new Error("Relay receipt usage is inconsistent")) + } + if (updated[0]!.receipt_count === 0) { + zeroUsage.push(entry) + } + } + // Remove zero reservations before the receipts they accounted for. Both statements are + // still invisible outside this transaction, and a later delete failure rolls them back. + for (const entry of zeroUsage) { + const deleted = yield* deleteZeroRelayReceiptUsage(entry) + if (deleted.length !== 1) { + return yield* failStorageCorrupt(new Error("Zero relay receipt usage disappeared")) + } + } + for (const receipt of settledRelayReceipts) { + const authorized = yield* authorizeRelayReceiptDelete(receipt.row_id) + if (authorized.length !== 1) { + return yield* failStorageCorrupt(new Error("Relay receipt deletion was not authorized")) + } + const deleted = yield* deleteExpiredRelayReceipt({ + documentId, + expiresAt: receiptExpiryCutoff, + rowId: receipt.row_id + }) + if (deleted.length !== 1) { + return yield* failStorageCorrupt(new Error("Relay receipt disappeared during history rewrite")) + } + } + yield* sql`DELETE FROM effect_local_peer_receipts + WHERE document_id = ${documentId} + AND relay_message_id IS NULL` // `effect_local_command_receipts` is deliberately retained. Its `heads` column goes stale, // but nothing reads it: `CommandExecutor` selects only `request_hash`, `mutation_name` and // `result`. Deleting the rows would break command idempotency, so a retried command that diff --git a/packages/local-sql/src/Migrations.ts b/packages/local-sql/src/Migrations.ts index b674ec0..ad0143c 100644 --- a/packages/local-sql/src/Migrations.ts +++ b/packages/local-sql/src/Migrations.ts @@ -18,7 +18,7 @@ export const peerWriterProvenanceChecksum = "sha256:effect-local-peer-writer-pro export const replicaHealthIndexesChecksum = "sha256:effect-local-replica-health-indexes-v1" export const documentLineageChecksum = "sha256:effect-local-document-lineage-v1" export const historyRewriteMarkersChecksum = "sha256:effect-local-history-rewrite-markers-v1" -export const peerRelayStateChecksum = "sha256:effect-local-peer-relay-state-v2" +export const peerRelayStateChecksum = "sha256:effect-local-peer-relay-state-v3" const migration = Effect.gen(function*() { const sql = yield* SqlClient.SqlClient @@ -526,6 +526,27 @@ const peerRelayStateMigration = Effect.gen(function*() { row_id ) WHERE relay_message_id IS NOT NULL` + yield* sql`CREATE TABLE effect_local_peer_relay_receipt_delete_tokens ( + receipt_row_id INTEGER PRIMARY KEY + )` + yield* sql`CREATE TRIGGER effect_local_peer_relay_receipt_delete_guard + BEFORE DELETE ON effect_local_peer_receipts + WHEN OLD.relay_message_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 + FROM effect_local_peer_relay_receipt_delete_tokens + WHERE receipt_row_id = OLD.row_id + ) + BEGIN + SELECT RAISE(ABORT, 'Relay receipt deletion requires an exact token'); + END` + yield* sql`CREATE TRIGGER effect_local_peer_relay_receipt_delete_consume + AFTER DELETE ON effect_local_peer_receipts + WHEN OLD.relay_message_id IS NOT NULL + BEGIN + DELETE FROM effect_local_peer_relay_receipt_delete_tokens + WHERE receipt_row_id = OLD.row_id; + END` yield* sql`CREATE TABLE effect_local_peer_relay_receipt_usage ( replica_incarnation INTEGER NOT NULL CHECK (replica_incarnation >= 0), sender_tenant_id TEXT NOT NULL CHECK (length(sender_tenant_id) > 0), diff --git a/packages/local-sql/src/PeerRelayOutbox.ts b/packages/local-sql/src/PeerRelayOutbox.ts index bbfdf45..0296e99 100644 --- a/packages/local-sql/src/PeerRelayOutbox.ts +++ b/packages/local-sql/src/PeerRelayOutbox.ts @@ -494,6 +494,7 @@ const make = Effect.gen(function*() { documentId: row.document_id, documentType: row.document_type }, + lineage: syncEnvelope.lineage, writerProvenance: row.writer_provenance, messageHash: row.message_hash, payload: row.payload @@ -588,6 +589,7 @@ const make = Effect.gen(function*() { senderConnectionEpoch: syncEnvelope.connectionEpoch, senderSequence: syncEnvelope.sequence, document: PeerSyncEnvelope.syncEnvelopeDocument(syncEnvelope), + lineage: syncEnvelope.lineage, writerProvenance: syncEnvelope.writerProvenance, messageHash: syncEnvelope.messageHash, payload: input.payload diff --git a/packages/local-sql/src/PeerSession.ts b/packages/local-sql/src/PeerSession.ts index a8adabd..0f1200e 100644 --- a/packages/local-sql/src/PeerSession.ts +++ b/packages/local-sql/src/PeerSession.ts @@ -42,6 +42,10 @@ export interface SupervisedPeerSession extends PeerSession { readonly awaitDisconnect: Effect.Effect } +class RelayProtocolInvalid extends Schema.TaggedErrorClass( + "@lucas-barake/effect-local-sql/PeerSession/RelayProtocolInvalid" +)("RelayProtocolInvalid", {}) {} + export const SyncEnvelope = Schema.Struct({ connectionEpoch: Schema.NonEmptyString, sequence: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), @@ -152,7 +156,12 @@ const makeWithTerminal = ( } const decodeRelay = (bytes: Uint8Array) => PeerSyncEnvelope.decodeSyncEnvelope(bytes, limits).pipe( - Effect.provideService(Crypto.Crypto, crypto) + Effect.provideService(Crypto.Crypto, crypto), + Effect.catchReason( + "ReplicaError", + "ProtocolMismatch", + () => Effect.fail(new RelayProtocolInvalid()) + ) ) const selected = new Set(options.documents.map((entry) => key(entry.document.name, entry.documentId))) const selectedDocumentIds = new Set(options.documents.map((entry) => entry.documentId)) @@ -457,6 +466,14 @@ const makeWithTerminal = ( delivery?: PeerTransport.AcknowledgedDelivery ) => Effect.gen(function*() { + const protocolInvalid = (expected: string, observed: string) => + Effect.fail( + delivery === undefined + ? new ReplicaError.ReplicaError({ + reason: new ReplicaError.ProtocolMismatch({ expected, observed }) + }) + : new RelayProtocolInvalid() + ) const envelope = yield* ( delivery === undefined ? decodeDirect(bytes).pipe( @@ -465,37 +482,27 @@ const makeWithTerminal = ( lineage: envelope.lineage ?? Identity.genesisLineage })) ) - : decodeRelay(bytes).pipe( - Effect.map((envelope) => ({ - ...envelope, - lineage: Identity.genesisLineage - })) - ) + : decodeRelay(bytes) ) const boundEpoch = yield* bindRemoteEpoch(envelope.connectionEpoch, delivery !== undefined) const selectedDocument = selected.has(key(envelope.documentType, envelope.documentId)) // Dropped before the digest and entity dispatch, not after. Nothing can restore the refused // lineage inside this session, so hashing and dispatching every further message the peer // sends for it would be a peer driven retry loop over CPU, allocation, and storage. - if (selectedDocument && (yield* Ref.get(refused)).has(envelope.documentId)) return + if (selectedDocument && (yield* Ref.get(refused)).has(envelope.documentId)) { + return "ApplicationRejected" as const + } const messageHash = yield* Canonical.digest(envelope.message).pipe( Effect.provideService(Crypto.Crypto, crypto) ) if (messageHash !== envelope.messageHash) { - return yield* new ReplicaError.ReplicaError({ - reason: new ReplicaError.ProtocolMismatch({ - expected: messageHash, - observed: envelope.messageHash - }) - }) + return yield* protocolInvalid(messageHash, envelope.messageHash) } if (!selectedDocument) { - return yield* new ReplicaError.ReplicaError({ - reason: new ReplicaError.ProtocolMismatch({ - expected: "selected whole document", - observed: `${envelope.documentType}:${envelope.documentId}` - }) - }) + return yield* protocolInvalid( + "selected whole document", + `${envelope.documentType}:${envelope.documentId}` + ) } const result = yield* withSyncLock( envelope.documentId, @@ -504,9 +511,10 @@ const makeWithTerminal = ( const permit = yield* gate.shared if (permit.incarnation !== session.replicaIncarnation) { return yield* new ReplicaError.ReplicaError({ - reason: new ReplicaError.ProtocolMismatch({ - expected: String(session.replicaIncarnation), - observed: String(permit.incarnation) + reason: new ReplicaError.StorageUnavailable({ + cause: new Error( + `Replica incarnation changed from ${session.replicaIncarnation} to ${permit.incarnation}` + ) }) }) } @@ -519,9 +527,8 @@ const makeWithTerminal = ( : yield* Effect.gen(function*() { if (relayReceiptLimits._tag === "None") { return yield* new ReplicaError.ReplicaError({ - reason: new ReplicaError.ProtocolMismatch({ - expected: "relay-enabled peer session", - observed: "direct peer session" + reason: new ReplicaError.StorageUnavailable({ + cause: new Error("Relay receipt limits are unavailable") }) }) } @@ -530,39 +537,19 @@ const makeWithTerminal = ( delivery.receiptRetentionMillis <= 0 || delivery.receiptRetentionMillis > relayReceiptLimits.value.receiptRetentionMillis ) { - return yield* new ReplicaError.ReplicaError({ - reason: new ReplicaError.ProtocolMismatch({ - expected: "bounded relay receipt retention", - observed: String(delivery.receiptRetentionMillis) - }) - }) + return yield* new RelayProtocolInvalid() } if ( connection.relayPeerId === undefined || delivery.identity.relayPeerId !== connection.relayPeerId ) { - return yield* new ReplicaError.ReplicaError({ - reason: new ReplicaError.ProtocolMismatch({ - expected: connection.relayPeerId ?? "relay peer identity", - observed: delivery.identity.relayPeerId - }) - }) + return yield* new RelayProtocolInvalid() } if (delivery.identity.senderPeerId !== connection.peerId) { - return yield* new ReplicaError.ReplicaError({ - reason: new ReplicaError.ProtocolMismatch({ - expected: connection.peerId, - observed: delivery.identity.senderPeerId - }) - }) + return yield* new RelayProtocolInvalid() } if (delivery.identity.messageHash !== envelope.messageHash) { - return yield* new ReplicaError.ReplicaError({ - reason: new ReplicaError.ProtocolMismatch({ - expected: envelope.messageHash, - observed: delivery.identity.messageHash - }) - }) + return yield* new RelayProtocolInvalid() } const nowMillis = yield* Clock.currentTimeMillis return { @@ -571,7 +558,7 @@ const makeWithTerminal = ( encodedSize: bytes.byteLength } satisfies PeerSync.RelayReceipt }) - const result = yield* client.ApplySync({ + const applySync = client.ApplySync({ replicaIncarnation: incarnation, peerId: connection.peerId, connectionEpoch: boundEpoch, @@ -598,6 +585,30 @@ const makeWithTerminal = ( ) ) ) + const result = yield* ( + delivery === undefined + ? applySync + : applySync.pipe( + Effect.catchReason( + "ReplicaError", + "ProtocolMismatch", + () => + Effect.gen(function*() { + const current = yield* gate.current + if (current.incarnation === session.replicaIncarnation) { + return yield* new RelayProtocolInvalid() + } + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ + cause: new Error( + `Replica incarnation changed from ${session.replicaIncarnation} to ${current.incarnation}` + ) + }) + }) + }) + ) + ) + ) yield* Ref.update(observed, (values) => { const current = values.get(envelope.documentId) if ((current?.revision ?? 0) !== observationRevision) return values @@ -618,12 +629,13 @@ const makeWithTerminal = ( (reason) => refuse(envelope.documentId, reason).pipe(Effect.as(null)) ) ) - if (result === null) return + if (result === null) return "ApplicationRejected" as const yield* publisher.publishPending if (result.reply !== null) { yield* sync.enqueue(session, result.reply).pipe(Effect.flatMap(schedule)) yield* Queue.offer(flushRequests, undefined) } + return "Applied" as const }) const relayCall = ( @@ -645,12 +657,30 @@ const makeWithTerminal = ( const receiveAcknowledged = (delivery: PeerTransport.AcknowledgedDelivery) => processReceive(delivery.message, delivery).pipe( - Effect.andThen(relayCall("relay acknowledge", delivery.acknowledge)), - Effect.catchIf( - (error) => error.reason._tag === "ProtocolMismatch", + Effect.flatMap((outcome) => + outcome === "ApplicationRejected" + ? relayCall("relay reject", delivery.reject("ApplicationRejected")) + : relayCall("relay acknowledge", delivery.acknowledge) + ), + Effect.catchTag( + "RelayProtocolInvalid", () => relayCall("relay reject", delivery.reject("ProtocolInvalid")) ) ) + const receiveDirect = (bytes: Uint8Array) => + processReceive(bytes).pipe( + Effect.catchTag( + "RelayProtocolInvalid", + () => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ + cause: new Error("Direct receive entered relay protocol classification") + }) + }) + ) + ) + ) yield* Effect.addFinalizer(() => Effect.gen(function*() { @@ -675,7 +705,7 @@ const makeWithTerminal = ( terminalFailure, ( connection.receiveWithAcknowledgement === undefined - ? Stream.runForEach(connection.receive, processReceive) + ? Stream.runForEach(connection.receive, receiveDirect) : Stream.runForEach(connection.receiveWithAcknowledgement, receiveAcknowledged) ).pipe( Effect.andThen( diff --git a/packages/local-sql/src/PeerSync.ts b/packages/local-sql/src/PeerSync.ts index a5104a6..6bde1a4 100644 --- a/packages/local-sql/src/PeerSync.ts +++ b/packages/local-sql/src/PeerSync.ts @@ -843,6 +843,8 @@ const make = ( } } for (const row of rows) { + yield* sql`INSERT INTO effect_local_peer_relay_receipt_delete_tokens (receipt_row_id) + VALUES (${row.row_id})` const deleted = yield* deleteRelayReceipt({ rowId: row.row_id }) if (deleted.length !== 1) { return yield* failStorageCorrupt(new Error("Relay receipt disappeared during pruning")) @@ -1496,8 +1498,7 @@ const make = ( relay !== undefined && ( !("relay_outer_envelope_digest" in receipt) || - receipt.relay_outer_envelope_digest !== relay.outerEnvelopeDigest || - receipt.relay_encoded_size !== relay.encodedSize + receipt.relay_outer_envelope_digest !== relay.outerEnvelopeDigest ) ) { return yield* new ReplicaError.ReplicaError({ @@ -2251,6 +2252,16 @@ const make = ( messageHash: yield* digest(generated[1]), heads: materializedHeads } + const pendingMessage = unresolvedBytes === 0 ? null : message + const encodedWriterProvenance = Schema.encodeSync( + WriterProvenance.StoredChangeProvenances + )(writerProvenance) + const relayRetainedSize = relay === undefined + ? null + : relay.encodedSize + + (reply?.message.byteLength ?? 0) + + (pendingMessage?.byteLength ?? 0) + + new TextEncoder().encode(encodedWriterProvenance).byteLength yield* sql`INSERT INTO effect_local_peer_receipts ( replica_incarnation, peer_id, connection_epoch, receive_sequence, document_id, message_hash, reply, reply_hash, pending_message, @@ -2262,13 +2273,13 @@ const make = ( ${receiptSession.replicaIncarnation}, ${receiptSession.peerId}, ${receiptSession.connectionEpoch}, ${receiveSequence}, ${documentId}, ${messageHash}, ${reply?.message ?? null}, ${reply?.messageHash ?? null}, - ${unresolvedBytes === 0 ? null : message}, ${Schema.encodeSync(Heads)(materializedHeads)}, + ${pendingMessage}, ${Schema.encodeSync(Heads)(materializedHeads)}, ${Schema.encodeSync(Heads)(acceptedHeads)}, ${commitSequence}, ${acceptedAt}, - ${Schema.encodeSync(WriterProvenance.StoredChangeProvenances)(writerProvenance)}, + ${encodedWriterProvenance}, ${relay?.senderTenantId ?? null}, ${relay?.senderSubjectId ?? null}, ${relay?.senderPeerId ?? null}, ${relay?.relayMessageId ?? null}, ${relay?.outerEnvelopeDigest ?? null}, ${relay?.receiptExpiresAt ?? null}, - ${relay?.encodedSize ?? null} + ${relayRetainedSize} )` if (relay !== undefined) { yield* sql`INSERT INTO effect_local_peer_relay_receipt_usage ( @@ -2276,7 +2287,7 @@ const make = ( receipt_count, encoded_bytes ) VALUES ( ${receiptSession.replicaIncarnation}, ${relay.senderTenantId}, ${relay.senderSubjectId}, - ${relay.senderPeerId}, 1, ${relay.encodedSize} + ${relay.senderPeerId}, 1, ${relayRetainedSize!} ) ON CONFLICT(replica_incarnation, sender_tenant_id, sender_subject_id, sender_peer_id) DO UPDATE SET receipt_count = effect_local_peer_relay_receipt_usage.receipt_count + 1, diff --git a/packages/local-sql/src/PeerSyncEnvelope.ts b/packages/local-sql/src/PeerSyncEnvelope.ts index 6d28735..18938fd 100644 --- a/packages/local-sql/src/PeerSyncEnvelope.ts +++ b/packages/local-sql/src/PeerSyncEnvelope.ts @@ -1,9 +1,8 @@ -import * as Automerge from "@automerge/automerge" import * as Canonical from "@lucas-barake/effect-local/Canonical" import * as Identity from "@lucas-barake/effect-local/Identity" import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" import type * as ReplicaLimits from "@lucas-barake/effect-local/ReplicaLimits" -import * as Crypto from "effect/Crypto" +import type * as Crypto from "effect/Crypto" import * as Effect from "effect/Effect" import * as Schema from "effect/Schema" import * as WriterProvenance from "./internal/writerProvenance.js" @@ -26,12 +25,13 @@ export const SyncEnvelope = Schema.Struct({ documentType: BoundedName, messageHash: MessageHash, message: Schema.Uint8ArrayFromBase64, + lineage: Identity.DocumentLineage, writerProvenance: WriterProvenance.ChangeProvenances }) export type SyncEnvelope = typeof SyncEnvelope.Type -export interface SyncEnvelopeLimits - extends Pick< +export interface SyncEnvelopeLimits extends + Pick< ReplicaLimits.Values, | "maxSyncMessageBytes" | "maxSyncChangesPerMessage" @@ -63,27 +63,6 @@ export const encodeSyncEnvelope = ( Effect.mapError(() => protocolMismatch("encodable sync envelope", "invalid sync envelope")) ) -const decodeSyncChanges = (message: Uint8Array): ReadonlyArray => { - const changes = new Map() - for (const chunk of Automerge.decodeSyncMessage(message).changes) { - try { - const change = Automerge.decodeChange(chunk) - changes.set(change.hash, change) - } catch { - const document = Automerge.load(chunk) - try { - for (const bytes of Automerge.getAllChanges(document)) { - const change = Automerge.decodeChange(bytes) - changes.set(change.hash, change) - } - } finally { - Automerge.free(document) - } - } - } - return [...changes.values()] -} - export const validateSyncEnvelope = ( envelope: SyncEnvelope, limits: SyncEnvelopeLimits @@ -101,40 +80,6 @@ export const validateSyncEnvelope = ( "excess writer provenance" ) } - const changes = yield* Effect.try({ - try: () => decodeSyncChanges(envelope.message), - catch: () => protocolMismatch("valid Automerge sync message", "invalid Automerge sync message") - }) - if (changes.length > limits.maxSyncChangesPerMessage) { - return yield* protocolMismatch( - `at most ${limits.maxSyncChangesPerMessage} sync changes`, - "excess sync changes" - ) - } - const dependencyEdges = changes.reduce((total, change) => total + change.deps.length, 0) - if (dependencyEdges > limits.maxSyncDependencyEdgesPerMessage) { - return yield* protocolMismatch( - `at most ${limits.maxSyncDependencyEdgesPerMessage} sync dependency edges`, - "excess sync dependency edges" - ) - } - const operations = changes.reduce((total, change) => total + change.ops.length, 0) - if (operations > limits.maxSyncOperationsPerMessage) { - return yield* protocolMismatch( - `at most ${limits.maxSyncOperationsPerMessage} sync operations`, - "excess sync operations" - ) - } - yield* Effect.try({ - try: () => WriterProvenance.validateExact( - changes.map((change) => change.hash), - envelope.writerProvenance - ), - catch: () => protocolMismatch( - "one canonical writer provenance entry per sync change", - "invalid writer provenance" - ) - }) const messageHash = yield* Canonical.digest(envelope.message) if (messageHash !== envelope.messageHash) { return yield* protocolMismatch("matching sync message hash", "conflicting sync message hash") @@ -194,6 +139,7 @@ export const RelayOuterEnvelope = Schema.Struct({ documentId: Identity.DocumentId, documentType: BoundedName }), + lineage: Identity.DocumentLineage, writerProvenance: WriterProvenance.ChangeProvenances.check( Schema.isMaxLength(maximumWriterProvenanceEntries) ), diff --git a/packages/local-sql/test/BackupStore.test.ts b/packages/local-sql/test/BackupStore.test.ts index 4fe5393..c5bacd2 100644 --- a/packages/local-sql/test/BackupStore.test.ts +++ b/packages/local-sql/test/BackupStore.test.ts @@ -658,6 +658,7 @@ describe("BackupStore", () => { }) const rows = yield* sql<{ + readonly deleteTokens: number readonly messages: number readonly outbox: number readonly outboxRemoteUsage: number @@ -672,8 +673,10 @@ describe("BackupStore", () => { (SELECT COUNT(*) FROM effect_local_peer_relay_outbox_remote_usage) AS outboxRemoteUsage, (SELECT COUNT(*) FROM effect_local_peer_relay_outbox_replica_usage) AS outboxReplicaUsage, (SELECT COUNT(*) FROM effect_local_peer_receipts WHERE relay_message_id IS NOT NULL) AS receipts, - (SELECT COUNT(*) FROM effect_local_peer_relay_receipt_usage) AS receiptUsage` + (SELECT COUNT(*) FROM effect_local_peer_relay_receipt_usage) AS receiptUsage, + (SELECT COUNT(*) FROM effect_local_peer_relay_receipt_delete_tokens) AS deleteTokens` assert.deepStrictEqual(rows[0], { + deleteTokens: 0, messages: 0, outbox: 0, outboxRemoteUsage: 0, @@ -740,6 +743,7 @@ describe("BackupStore", () => { ` assert.strictEqual(installations.length, 0) const relayRows = yield* sql<{ + readonly deleteTokens: number readonly outbox: number readonly outboxRemoteUsage: number readonly outboxReplicaUsage: number @@ -750,8 +754,10 @@ describe("BackupStore", () => { (SELECT COUNT(*) FROM effect_local_peer_relay_outbox_remote_usage) AS outboxRemoteUsage, (SELECT COUNT(*) FROM effect_local_peer_relay_outbox_replica_usage) AS outboxReplicaUsage, (SELECT COUNT(*) FROM effect_local_peer_receipts WHERE relay_message_id IS NOT NULL) AS receipts, - (SELECT COUNT(*) FROM effect_local_peer_relay_receipt_usage) AS receiptUsage` + (SELECT COUNT(*) FROM effect_local_peer_relay_receipt_usage) AS receiptUsage, + (SELECT COUNT(*) FROM effect_local_peer_relay_receipt_delete_tokens) AS deleteTokens` assert.deepStrictEqual(relayRows, [{ + deleteTokens: 0, outbox: 1, outboxRemoteUsage: 1, outboxReplicaUsage: 1, diff --git a/packages/local-sql/test/Compaction.test.ts b/packages/local-sql/test/Compaction.test.ts index 4f55b51..710dff6 100644 --- a/packages/local-sql/test/Compaction.test.ts +++ b/packages/local-sql/test/Compaction.test.ts @@ -745,6 +745,198 @@ describe("Compaction", () => { assert.strictEqual((yield* documentRowOf(documentId)).lineage, "") }).pipe(Effect.provide(Services))) + it.effect("refuses to rewrite while a relay outbox row still carries the document lineage", () => + Effect.gen(function*() { + const compaction = yield* Compaction.Compaction + const gate = yield* ReplicaGate.ReplicaGate + const sql = yield* SqlClient.SqlClient + const store = yield* DocumentStore.DocumentStore + const documentId = yield* Identity.makeDocumentId + const created = yield* store.create(Task, documentId, { title: "one", labels: [] }) + InternalAutomerge.free(created.automerge) + yield* compaction.compact(Task, documentId) + const permit = yield* gate.current + const relayMessageId = yield* Identity.makeRelayMessageId + const payload = new Uint8Array([1, 2, 3]) + yield* sql`INSERT INTO effect_local_peer_relay_outbox ( + replica_id, replica_incarnation, writer_generation, + expected_local_tenant_id, expected_local_subject_id, expected_local_peer_id, + remote_tenant_id, remote_subject_id, remote_peer_id, relay_peer_id, + relay_message_id, outer_envelope_digest, protocol_version, payload_version, + sender_connection_epoch, sender_sequence, document_id, document_type, + writer_provenance, message_hash, payload, encoded_size, created_at, + retry_deadline, next_attempt_at, custody_state + ) VALUES ( + ${permit.replicaId}, ${permit.incarnation}, ${permit.writerGeneration}, + ${"tenant-a"}, ${"subject-local"}, ${"peer-local"}, + ${"tenant-a"}, ${"subject-remote"}, ${"peer-remote"}, ${"peer-relay"}, + ${relayMessageId}, ${"a".repeat(64)}, ${3}, ${1}, + ${"epoch-1"}, ${0}, ${documentId}, ${Task.name}, + ${"[]"}, ${"b".repeat(64)}, ${payload}, ${payload.byteLength}, + ${"2020-01-01T00:00:00.000Z"}, ${"2999-01-01T00:00:00.000Z"}, + ${"2020-01-01T00:00:00.000Z"}, ${"Pending"} + )` + + const error = yield* Effect.flip(compaction.rewriteHistory(Task, documentId, operationId)) + assert.strictEqual(error.reason._tag, "StorageCorrupt") + const retained = yield* sql<{ readonly relay_message_id: string }>` + SELECT relay_message_id FROM effect_local_peer_relay_outbox + WHERE document_id = ${documentId} + ` + assert.deepStrictEqual(retained, [{ relay_message_id: relayMessageId as string }]) + assert.strictEqual((yield* documentRowOf(documentId)).lineage, "") + }).pipe(Effect.provide(Services))) + + it.effect("refuses to rewrite while an unexpired relay receipt preserves duplicate evidence", () => + Effect.gen(function*() { + const compaction = yield* Compaction.Compaction + const gate = yield* ReplicaGate.ReplicaGate + const sql = yield* SqlClient.SqlClient + const store = yield* DocumentStore.DocumentStore + const documentId = yield* Identity.makeDocumentId + const created = yield* store.create(Task, documentId, { title: "one", labels: [] }) + InternalAutomerge.free(created.automerge) + yield* compaction.compact(Task, documentId) + const permit = yield* gate.current + const peerId = yield* Identity.makePeerId + const senderPeerId = yield* Identity.makePeerId + const relayMessageId = yield* Identity.makeRelayMessageId + yield* sql`INSERT INTO effect_local_peer_receipts ( + replica_incarnation, peer_id, connection_epoch, receive_sequence, document_id, + message_hash, reply, reply_hash, pending_message, heads, accepted_heads, + commit_sequence, accepted_at, writer_provenance, + relay_sender_tenant_id, relay_sender_subject_id, relay_sender_peer_id, + relay_message_id, relay_outer_envelope_digest, relay_receipt_expires_at, + relay_encoded_size + ) VALUES ( + ${permit.incarnation}, ${peerId}, ${"epoch-1"}, ${1}, ${documentId}, + ${"c".repeat(64)}, NULL, NULL, NULL, ${"[]"}, ${"[]"}, + ${1}, ${"2020-01-01T00:00:00.000Z"}, ${"[]"}, + ${"tenant-a"}, ${"subject-a"}, ${senderPeerId}, + ${relayMessageId}, ${"d".repeat(64)}, ${"2999-01-01T00:00:00.000Z"}, ${128} + )` + yield* sql`INSERT INTO effect_local_peer_relay_receipt_usage ( + replica_incarnation, sender_tenant_id, sender_subject_id, sender_peer_id, + receipt_count, encoded_bytes + ) VALUES ( + ${permit.incarnation}, ${"tenant-a"}, ${"subject-a"}, ${senderPeerId}, ${1}, ${128} + )` + + const error = yield* Effect.flip(compaction.rewriteHistory(Task, documentId, operationId)) + assert.strictEqual(error.reason._tag, "StorageCorrupt") + const retained = yield* sql<{ + readonly encoded_bytes: number + readonly receipt_count: number + readonly relay_message_id: string + }>`SELECT relay_message_id, + (SELECT receipt_count FROM effect_local_peer_relay_receipt_usage + WHERE sender_peer_id = ${senderPeerId}) AS receipt_count, + (SELECT encoded_bytes FROM effect_local_peer_relay_receipt_usage + WHERE sender_peer_id = ${senderPeerId}) AS encoded_bytes + FROM effect_local_peer_receipts + WHERE document_id = ${documentId}` + assert.deepStrictEqual(retained, [{ + encoded_bytes: 128, + receipt_count: 1, + relay_message_id: relayMessageId as string + }]) + assert.strictEqual((yield* documentRowOf(documentId)).lineage, "") + }).pipe(Effect.provide(Services))) + + it.effect("conserves relay receipts and sender usage when removing expired document receipts", () => + Effect.gen(function*() { + const compaction = yield* Compaction.Compaction + const gate = yield* ReplicaGate.ReplicaGate + const sql = yield* SqlClient.SqlClient + const store = yield* DocumentStore.DocumentStore + const documentId = yield* Identity.makeDocumentId + const survivorDocumentId = yield* Identity.makeDocumentId + const created = yield* store.create(Task, documentId, { title: "one", labels: [] }) + const survivor = yield* store.create(Task, survivorDocumentId, { title: "two", labels: [] }) + InternalAutomerge.free(survivor.automerge) + InternalAutomerge.free(created.automerge) + yield* compaction.compact(Task, documentId) + const permit = yield* gate.current + const peerId = yield* Identity.makePeerId + const senderAPeerId = yield* Identity.makePeerId + const senderBPeerId = yield* Identity.makePeerId + const targetA = yield* Identity.makeRelayMessageId + const targetB = yield* Identity.makeRelayMessageId + const survivorA = yield* Identity.makeRelayMessageId + for ( + const [receiveSequence, receiptDocumentId, senderPeerId, senderSubjectId, relayMessageId, encodedSize] of [ + [1, documentId, senderAPeerId, "subject-a", targetA, 120], + [2, documentId, senderBPeerId, "subject-b", targetB, 40], + [3, survivorDocumentId, senderAPeerId, "subject-a", survivorA, 80] + ] as const + ) { + yield* sql`INSERT INTO effect_local_peer_receipts ( + replica_incarnation, peer_id, connection_epoch, receive_sequence, document_id, + message_hash, reply, reply_hash, pending_message, heads, accepted_heads, + commit_sequence, accepted_at, writer_provenance, + relay_sender_tenant_id, relay_sender_subject_id, relay_sender_peer_id, + relay_message_id, relay_outer_envelope_digest, relay_receipt_expires_at, + relay_encoded_size + ) VALUES ( + ${permit.incarnation}, ${peerId}, ${"epoch-1"}, ${receiveSequence}, ${receiptDocumentId}, + ${relayMessageId.slice(4).replaceAll("-", "")}, NULL, NULL, NULL, ${"[]"}, ${"[]"}, + ${1}, ${"2020-01-01T00:00:00.000Z"}, ${"[]"}, + ${"tenant-a"}, ${senderSubjectId}, ${senderPeerId}, + ${relayMessageId}, ${"e".repeat(64)}, ${"1970-01-01T00:00:00.000Z"}, ${encodedSize} + )` + } + yield* sql`INSERT INTO effect_local_peer_relay_receipt_usage ( + replica_incarnation, sender_tenant_id, sender_subject_id, sender_peer_id, + receipt_count, encoded_bytes + ) VALUES + (${permit.incarnation}, ${"tenant-a"}, ${"subject-a"}, ${senderAPeerId}, ${2}, ${200}), + (${permit.incarnation}, ${"tenant-a"}, ${"subject-b"}, ${senderBPeerId}, ${1}, ${40})` + yield* sql.unsafe(`CREATE TRIGGER fail_target_relay_receipt_delete + BEFORE DELETE ON effect_local_peer_receipts + WHEN OLD.relay_message_id = '${targetB}' + BEGIN + SELECT RAISE(ABORT, 'delete failed'); + END`) + + const error = yield* Effect.flip(compaction.rewriteHistory(Task, documentId, operationId)) + assert.strictEqual(error.reason._tag, "StorageUnavailable") + assert.deepStrictEqual( + yield* sql`SELECT receipt_count, encoded_bytes + FROM effect_local_peer_relay_receipt_usage + ORDER BY sender_subject_id`, + [{ receipt_count: 2, encoded_bytes: 200 }, { receipt_count: 1, encoded_bytes: 40 }] + ) + assert.strictEqual( + (yield* sql<{ readonly count: number }>`SELECT COUNT(*) AS count FROM effect_local_peer_receipts`)[0]!.count, + 3 + ) + assert.strictEqual((yield* documentRowOf(documentId)).lineage, "") + + yield* sql`DROP TRIGGER fail_target_relay_receipt_delete` + const lineage = yield* compaction.rewriteHistory(Task, documentId, operationId) + assert.notStrictEqual(lineage, "") + assert.deepStrictEqual( + yield* sql`SELECT relay_message_id, document_id, relay_encoded_size + FROM effect_local_peer_receipts ORDER BY relay_message_id`, + [{ + relay_message_id: survivorA, + document_id: survivorDocumentId, + relay_encoded_size: 80 + }] + ) + assert.deepStrictEqual( + yield* sql`SELECT sender_subject_id, sender_peer_id, receipt_count, encoded_bytes + FROM effect_local_peer_relay_receipt_usage ORDER BY sender_subject_id`, + [{ + sender_subject_id: "subject-a", + sender_peer_id: senderAPeerId, + receipt_count: 1, + encoded_bytes: 80 + }] + ) + assert.strictEqual((yield* documentRowOf(documentId)).lineage, lineage) + }).pipe(Effect.provide(Services))) + it.effect("refuses to rewrite when the document advances between recovery and the transaction", () => Effect.gen(function*() { const compaction = yield* Compaction.Compaction diff --git a/packages/local-sql/test/DocumentEntity.test.ts b/packages/local-sql/test/DocumentEntity.test.ts index 0a875e7..e7f0df7 100644 --- a/packages/local-sql/test/DocumentEntity.test.ts +++ b/packages/local-sql/test/DocumentEntity.test.ts @@ -404,7 +404,10 @@ describe("DocumentEntity", () => { const sync = peerSync((_document, _documentId, _session, input) => Effect.all([ Ref.set(receivedProvenance, input.writerProvenance), - Ref.update(receivedLineages, (lineages) => [...lineages, input.lineage]), + Ref.update(receivedLineages, (lineages) => [ + ...lineages, + input.lineage ?? Identity.genesisLineage + ]), Ref.set(receivedRelay, input.relay) ]).pipe(Effect.as(syncResult)) ) diff --git a/packages/local-sql/test/Migrations.test.ts b/packages/local-sql/test/Migrations.test.ts index c116bc6..3796450 100644 --- a/packages/local-sql/test/Migrations.test.ts +++ b/packages/local-sql/test/Migrations.test.ts @@ -560,6 +560,77 @@ describe("Migrations", () => { assert.deepStrictEqual(replicaUsage, { message_count: 1, encoded_bytes: 3 }) }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:", disableWAL: true })))) + it.effect("prevents legacy receipt cleanup from deleting relay receipts", () => + Effect.gen(function*() { + yield* Migrations.run + const sql = yield* SqlClient.SqlClient + yield* sql`INSERT INTO effect_local_documents ( + document_id, document_type, schema_version, observed_versions, materialized_heads, + accepted_heads, tombstone, projection_status + ) VALUES ( + 'doc_00000000-0000-4000-8000-000000000001', + 'Task', + 1, + '[]', + '[]', + '[]', + 0, + 'Ready' + )` + yield* sql`INSERT INTO effect_local_peer_receipts ( + replica_incarnation, peer_id, connection_epoch, receive_sequence, document_id, + message_hash, heads, accepted_heads, commit_sequence, accepted_at, writer_provenance, + relay_sender_tenant_id, relay_sender_subject_id, relay_sender_peer_id, + relay_message_id, relay_outer_envelope_digest, relay_receipt_expires_at, relay_encoded_size + ) VALUES ( + 0, 'peer-relay', 'epoch-relay', 0, + 'doc_00000000-0000-4000-8000-000000000001', + ${"a".repeat(64)}, '[]', '[]', 0, '2026-01-01T00:00:00.000Z', '[]', + 'tenant-1', 'subject-1', 'peer-sender', 'relay-id-1', + ${"b".repeat(64)}, '2026-01-09T00:00:00.000Z', 32 + )` + yield* sql`INSERT INTO effect_local_peer_relay_receipt_usage ( + replica_incarnation, sender_tenant_id, sender_subject_id, sender_peer_id, + receipt_count, encoded_bytes + ) VALUES (0, 'tenant-1', 'subject-1', 'peer-sender', 1, 32)` + yield* sql`INSERT INTO effect_local_peer_receipts ( + replica_incarnation, peer_id, connection_epoch, receive_sequence, document_id, + message_hash, heads, accepted_heads, commit_sequence, accepted_at, writer_provenance + ) VALUES ( + 0, 'peer-direct', 'epoch-direct', 0, + 'doc_00000000-0000-4000-8000-000000000001', + ${"c".repeat(64)}, '[]', '[]', 0, '2026-01-01T00:00:00.000Z', '[]' + )` + + assert.strictEqual( + (yield* Effect.exit(sql`DELETE FROM effect_local_peer_receipts + WHERE pending_message IS NULL`))._tag, + "Failure" + ) + assert.strictEqual( + (yield* Effect.exit(sql`DELETE FROM effect_local_documents + WHERE document_id = 'doc_00000000-0000-4000-8000-000000000001'`))._tag, + "Failure" + ) + yield* sql`DELETE FROM effect_local_peer_receipts + WHERE relay_message_id IS NULL` + + assert.deepStrictEqual( + yield* sql`SELECT relay_message_id FROM effect_local_peer_receipts`, + [{ relay_message_id: "relay-id-1" }] + ) + assert.deepStrictEqual( + yield* sql`SELECT receipt_count, encoded_bytes + FROM effect_local_peer_relay_receipt_usage`, + [{ receipt_count: 1, encoded_bytes: 32 }] + ) + assert.deepStrictEqual( + yield* sql`SELECT receipt_row_id + FROM effect_local_peer_relay_receipt_delete_tokens`, + [] + ) + }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:", disableWAL: true })))) + it.effect("rejects a changed relay migration checksum", () => Effect.gen(function*() { yield* Migrations.run diff --git a/packages/local-sql/test/PeerRelayClientRuntime.test.ts b/packages/local-sql/test/PeerRelayClientRuntime.test.ts index 9fd3e0d..9b3d8d8 100644 --- a/packages/local-sql/test/PeerRelayClientRuntime.test.ts +++ b/packages/local-sql/test/PeerRelayClientRuntime.test.ts @@ -86,6 +86,7 @@ describe("PeerRelayClientRuntime", () => { maintenanceIntervalMillis: 1_000 } const documentId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000011") + const lineage = Identity.DocumentLineage.make("lin_00000000-0000-4000-8000-000000000015") const endpoint: PeerRelayOutbox.Endpoint = { expectedLocal: { tenantId: "tenant-a", @@ -103,6 +104,8 @@ describe("PeerRelayClientRuntime", () => { const fakePeerSync = ( pruneRelayReceipts: PeerSync.PeerSync["Service"]["pruneRelayReceipts"] ): PeerSync.PeerSync["Service"] => ({ + withDocumentInvalidation: (_documentId, effect) => effect, + invalidateDocument: () => Effect.void, open: () => Effect.die(new Error("unused")), reset: () => Effect.die(new Error("unused")), generate: () => Effect.die(new Error("unused")), @@ -170,6 +173,7 @@ describe("PeerRelayClientRuntime", () => { documentType: "Task", messageHash: yield* Canonical.digest(message), message, + lineage, writerProvenance: changes.map((change) => ({ changeHash: change.hash, writerSchemaVersion: 1, diff --git a/packages/local-sql/test/PeerRelayOutbox.test.ts b/packages/local-sql/test/PeerRelayOutbox.test.ts index a3c4422..02c7970 100644 --- a/packages/local-sql/test/PeerRelayOutbox.test.ts +++ b/packages/local-sql/test/PeerRelayOutbox.test.ts @@ -76,6 +76,7 @@ describe("PeerRelayOutbox", () => { pruneBatchSize: 10 } const documentId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000001") + const lineage = Identity.DocumentLineage.make("lin_00000000-0000-4000-8000-000000000005") const endpoint: PeerRelayOutbox.Endpoint = { expectedLocal: { tenantId: "tenant-a", @@ -142,6 +143,7 @@ describe("PeerRelayOutbox", () => { documentType: "Task", messageHash: yield* Canonical.digest(message), message, + lineage, writerProvenance: changes.map((change) => ({ changeHash: change.hash, writerSchemaVersion: 1, @@ -159,6 +161,29 @@ describe("PeerRelayOutbox", () => { const outbox = yield* PeerRelayOutbox.PeerRelayOutbox const payload = yield* makePayload(1) const first = yield* outbox.admit({ ...endpoint, payload, retryHorizonMillis: 30_000 }) + const decoded = yield* PeerSyncEnvelope.decodeSyncEnvelope(payload, replicaLimits) + assert.strictEqual(decoded.lineage, lineage) + assert.strictEqual( + first.outerEnvelopeDigest, + yield* PeerSyncEnvelope.digestRelayOuterEnvelope({ + domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, + version: PeerSyncEnvelope.relayOuterEnvelopeVersion, + expectedLocal: first.expectedLocal, + remote: first.remote, + relayPeerId: first.relayPeerId, + relayMessageId: first.relayMessageId, + protocolVersion: first.protocolVersion, + payloadVersion: first.payloadVersion, + senderReplicaIncarnation: first.replicaIncarnation, + senderConnectionEpoch: first.senderConnectionEpoch, + senderSequence: first.senderSequence, + document: first.document, + lineage: decoded.lineage, + writerProvenance: first.writerProvenance, + messageHash: first.messageHash, + payload: first.payload + }) + ) yield* TestClock.adjust("10 seconds") const duplicate = yield* outbox.admit({ ...endpoint, diff --git a/packages/local-sql/test/PeerSession.test.ts b/packages/local-sql/test/PeerSession.test.ts index 148b683..3a44554 100644 --- a/packages/local-sql/test/PeerSession.test.ts +++ b/packages/local-sql/test/PeerSession.test.ts @@ -18,6 +18,7 @@ import * as Exit from "effect/Exit" import * as Fiber from "effect/Fiber" import * as Layer from "effect/Layer" import * as Option from "effect/Option" +import * as PlatformError from "effect/PlatformError" import * as Queue from "effect/Queue" import * as Ref from "effect/Ref" import * as Schema from "effect/Schema" @@ -33,6 +34,7 @@ import * as DocumentEntity from "../src/DocumentEntity.js" import * as PeerRelayReceiptLimits from "../src/PeerRelayReceiptLimits.js" import * as PeerSession from "../src/PeerSession.js" import * as PeerSync from "../src/PeerSync.js" +import * as PeerSyncEnvelope from "../src/PeerSyncEnvelope.js" import * as ReplicaBootstrap from "../src/ReplicaBootstrap.js" import * as ReplicaGate from "../src/ReplicaGate.js" @@ -2439,11 +2441,41 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { Effect.gen(function*() { const deliveries = yield* Queue.unbounded() const events = yield* Queue.unbounded() + const closed = yield* Deferred.make() const resets = yield* Ref.make>([]) + const rejections = yield* Ref.make>([]) const received = yield* Ref.make>([]) + const receivedLineages = yield* Ref.make>([]) const documentId = yield* Identity.makeDocumentId + const rotatedDocumentId = yield* Identity.makeDocumentId const senderPeerId = yield* Identity.makePeerId const relayPeerId = yield* Identity.makePeerId + const lineage = Identity.DocumentLineage.make("lin_00000000-0000-4000-8000-000000000020") + const currentPermit = yield* Ref.make(permit) + const rotateAfterSharedRelease = yield* Ref.make(false) + const relayGate = ReplicaGate.ReplicaGate.of({ + current: Ref.get(currentPermit), + claiming: Effect.succeed(false), + shared: Effect.acquireRelease( + Ref.get(currentPermit), + () => + Ref.getAndSet(rotateAfterSharedRelease, false).pipe( + Effect.flatMap((rotate) => + rotate + ? Ref.set(currentPermit, { + ...permit, + incarnation: Identity.ReplicaIncarnation.make(2), + writerGeneration: Identity.WriterGeneration.make(3) + }) + : Effect.void + ) + ) + ), + admit: Effect.acquireRelease(Ref.get(currentPermit), () => Effect.void), + claim: (use) => Ref.get(currentPermit).pipe(Effect.flatMap(use)), + refresh: Ref.get(currentPermit), + validate: () => Effect.void + }) const message = yield* Effect.acquireUseRelease( Effect.sync(() => Automerge.init()), (document) => @@ -2457,6 +2489,8 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { const messageHash = yield* Canonical.digest(message) const reply = { documentId, message, messageHash, heads: [] } const sync = PeerSync.PeerSync.of({ + withDocumentInvalidation: (_documentId, effect) => effect, + invalidateDocument: () => Effect.void, open: (peerId) => Effect.succeed({ peerId, @@ -2472,7 +2506,7 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { receive: () => Effect.succeed(result), enqueue: (_session, value) => Queue.offer(events, "enqueue").pipe( - Effect.as({ ...value, sendSequence: 0, writerProvenance: [] }) + Effect.as({ ...value, sendSequence: 0, lineage, writerProvenance: [] }) ), pending: () => Effect.succeed([]), markSent: () => Effect.succeed(true), @@ -2488,7 +2522,7 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { receive: Stream.never, receiveWithAcknowledgement: Stream.fromQueue(deliveries), send: () => Effect.void, - close: Effect.void + close: Deferred.succeed(closed, undefined).pipe(Effect.asVoid) }) }) const publisher = CommitPublisher.CommitPublisher.of({ @@ -2500,22 +2534,28 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { events: Stream.never }) }) - const envelope = (connectionEpoch: string, sequence: number) => - encode({ + const envelope = ( + connectionEpoch: string, + sequence: number, + selectedDocumentId = documentId + ) => + PeerSyncEnvelope.encodeSyncEnvelope({ connectionEpoch, sequence, - documentId, + documentId: selectedDocumentId, documentType: Task.name, messageHash, message, + lineage, writerProvenance: [] }) const delivery = ( connectionEpoch: string, sequence: number, - relayMessageId: Identity.RelayMessageId - ): Effect.Effect => - envelope(connectionEpoch, sequence).pipe( + relayMessageId: Identity.RelayMessageId, + selectedDocumentId = documentId + ): Effect.Effect => + envelope(connectionEpoch, sequence, selectedDocumentId).pipe( Effect.map((bytes) => ({ message: bytes, identity: { @@ -2530,23 +2570,59 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { }, receiptRetentionMillis: PeerRelayReceiptLimits.defaults.receiptRetentionMillis, acknowledge: Queue.offer(events, "ack").pipe(Effect.asVoid), - reject: () => Queue.offer(events, "reject").pipe(Effect.asVoid) + reject: (reason) => + Ref.update(rejections, (current) => [...current, reason]).pipe( + Effect.andThen(Queue.offer(events, `reject:${reason}`)), + Effect.asVoid + ) })) ) yield* Effect.scoped( Effect.gen(function*() { yield* PeerSession.makeTestClient( - { peerId: senderPeerId, documents: [{ document: Task, documentId }] }, + { + peerId: senderPeerId, + documents: [ + { document: Task, documentId }, + { document: Task, documentId: rotatedDocumentId } + ] + }, () => Effect.succeed({ ApplySync: (request: typeof DocumentEntity.ApplySync.payloadSchema.Type) => Queue.offer(events, "apply").pipe( + Effect.andThen(Ref.get(currentPermit)), + Effect.flatMap((current) => + request.replicaIncarnation === current.incarnation + ? Effect.void + : Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.ProtocolMismatch({ + expected: String(current.incarnation), + observed: String(request.replicaIncarnation) + }) + }) + ) + ), Effect.andThen( request.relay === undefined ? Effect.die("Expected relay receipt") : Ref.update(received, (current) => [...current, request.relay!]) ), - Effect.as({ ...result, reply }) + Effect.andThen(Ref.update(receivedLineages, (current) => [...current, request.lineage!])), + Effect.andThen( + request.receiveSequence === 1 + ? Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.DocumentLineageChanged({ + documentId, + localLineage: Identity.genesisLineage, + remoteLineage: request.lineage! + }) + }) + ) + : Effect.succeed({ ...result, reply }) + ) ) } as never) ) @@ -2562,6 +2638,7 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { yield* Effect.forEach([0, 1, 2, 3], () => Queue.take(events)), ["apply", "publish", "enqueue", "ack"] ) + assert.isTrue(Option.isNone(yield* Deferred.poll(closed))) yield* Queue.offer( deliveries, yield* delivery( @@ -2574,14 +2651,62 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { yield* Effect.forEach([0, 1, 2, 3, 4], () => Queue.take(events)), ["reset:sender-epoch-a", "apply", "publish", "enqueue", "ack"] ) + assert.isTrue(Option.isNone(yield* Deferred.poll(closed))) assert.deepStrictEqual( (yield* Ref.get(received)).map((entry) => entry.senderPeerId), [senderPeerId, senderPeerId] ) + assert.deepStrictEqual(yield* Ref.get(receivedLineages), [lineage, lineage]) + + yield* Queue.offer( + deliveries, + yield* delivery( + "sender-epoch-b", + 1, + Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000003") + ) + ) + assert.deepStrictEqual( + yield* Effect.forEach([0, 1], () => Queue.take(events)), + ["apply", "reject:ApplicationRejected"] + ) + yield* Queue.offer( + deliveries, + yield* delivery( + "sender-epoch-b", + 2, + Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000004") + ) + ) + assert.strictEqual(yield* Queue.take(events), "reject:ApplicationRejected") + assert.deepStrictEqual(yield* Ref.get(receivedLineages), [lineage, lineage, lineage]) + + yield* Ref.set(rotateAfterSharedRelease, true) + yield* Queue.offer( + deliveries, + yield* delivery( + "sender-epoch-b", + 0, + Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000005"), + rotatedDocumentId + ) + ) + assert.strictEqual(yield* Queue.take(events), "apply") + assert.strictEqual( + yield* Effect.raceFirst( + Deferred.await(closed).pipe(Effect.as("close")), + Queue.take(events) + ), + "close" + ) + assert.deepStrictEqual(yield* Ref.get(rejections), [ + "ApplicationRejected", + "ApplicationRejected" + ]) }).pipe( Effect.provideService(PeerTransport.PeerTransport, transport), Effect.provideService(PeerSync.PeerSync, sync), - Effect.provideService(ReplicaGate.ReplicaGate, gate), + Effect.provideService(ReplicaGate.ReplicaGate, relayGate), Effect.provideService(CommitPublisher.CommitPublisher, publisher), Effect.provideService(ReplicaLimits.ReplicaLimits, limits), Effect.provideService( @@ -2596,6 +2721,132 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { ) }).pipe(Effect.provide(NodeCrypto.layer))) + it.effect("does not permanently reject relay delivery when local digest infrastructure fails", () => + Effect.gen(function*() { + const deliveries = yield* Queue.unbounded() + const outcomes = yield* Queue.unbounded() + const closed = yield* Deferred.make() + const documentId = yield* Identity.makeDocumentId + const senderPeerId = yield* Identity.makePeerId + const relayPeerId = yield* Identity.makePeerId + const message = yield* Effect.acquireUseRelease( + Effect.sync(() => Automerge.init()), + (document) => + Effect.sync(() => { + const encoded = Automerge.generateSyncMessage(document, Automerge.initSyncState())[1] + if (encoded === null) throw new TypeError("Expected an initial sync message") + return encoded + }), + (document) => Effect.sync(() => Automerge.free(document)) + ) + const messageHash = yield* Canonical.digest(message) + const encoded = yield* PeerSyncEnvelope.encodeSyncEnvelope({ + connectionEpoch: "sender-epoch", + sequence: 0, + documentId, + documentType: Task.name, + messageHash, + message, + lineage: Identity.genesisLineage, + writerProvenance: [] + }) + const failingCrypto = Crypto.make({ + randomBytes: (size) => new Uint8Array(size), + digest: () => + Effect.fail( + PlatformError.systemError({ + module: "Crypto", + method: "digest", + _tag: "Unknown", + description: "Injected digest failure" + }) + ) + }) + const sync = PeerSync.PeerSync.of({ + withDocumentInvalidation: (_documentId, effect) => effect, + invalidateDocument: () => Effect.void, + open: (peerId) => + Effect.succeed({ + peerId, + connectionEpoch: "local-epoch", + replicaIncarnation: permit.incarnation + }), + reset: () => Effect.void, + generate: () => Effect.succeed({ outbound: null, observedByPeer: false, dirty: false }), + receive: () => Effect.die("Receive must not run after digest failure"), + enqueue: () => Effect.die("Enqueue must not run after digest failure"), + pending: () => Effect.succeed([]), + markSent: () => Effect.succeed(true), + pruneRelayReceipts: Effect.succeed(0) + }) + const transport = PeerTransport.PeerTransport.of({ + capabilities: { storeAndForward: true }, + connect: () => + Effect.succeed({ + peerId: senderPeerId, + relayPeerId, + capabilities: { storeAndForward: true }, + receive: Stream.never, + receiveWithAcknowledgement: Stream.fromQueue(deliveries), + send: () => Effect.void, + close: Deferred.succeed(closed, undefined).pipe(Effect.asVoid) + }) + }) + yield* Effect.scoped( + Effect.gen(function*() { + yield* PeerSession.makeTestClient( + { peerId: senderPeerId, documents: [{ document: Task, documentId }] }, + () => Effect.die("ApplySync must not run after digest failure") + ) + yield* Queue.offer(deliveries, { + message: encoded, + identity: { + relayMessageId: yield* Identity.makeRelayMessageId, + relayPeerId, + senderTenantId: "tenant", + senderSubjectId: "sender", + senderPeerId, + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + messageHash, + outerEnvelopeDigest: "a".repeat(64) + }, + receiptRetentionMillis: PeerRelayReceiptLimits.defaults.receiptRetentionMillis, + acknowledge: Queue.offer(outcomes, "ack").pipe(Effect.asVoid), + reject: (reason) => Queue.offer(outcomes, `reject:${reason}`).pipe(Effect.asVoid) + }) + assert.strictEqual( + yield* Effect.raceFirst( + Deferred.await(closed).pipe(Effect.as("close")), + Queue.take(outcomes) + ), + "close" + ) + }).pipe( + Effect.provideService(Crypto.Crypto, failingCrypto), + Effect.provideService(PeerTransport.PeerTransport, transport), + Effect.provideService(PeerSync.PeerSync, sync), + Effect.provideService(ReplicaGate.ReplicaGate, gate), + Effect.provideService( + CommitPublisher.CommitPublisher, + CommitPublisher.CommitPublisher.of({ + publishPending: Effect.succeed(0), + invalidate: () => Effect.void, + subscribe: Effect.succeed({ + watermark: Identity.CommitSequence.make(0), + refreshGeneration: 0, + events: Stream.never + }) + }) + ), + Effect.provideService(ReplicaLimits.ReplicaLimits, limits), + Effect.provideService( + PeerRelayReceiptLimits.PeerRelayReceiptLimits, + PeerRelayReceiptLimits.defaults + ) + ) + ) + }).pipe(Effect.provide(NodeCrypto.layer))) + it.effect("retries every unsent dirty output after a send fails", () => Effect.scoped(Effect.gen(function*() { const firstDocumentId = yield* Identity.makeDocumentId diff --git a/packages/local-sql/test/PeerSync.test.ts b/packages/local-sql/test/PeerSync.test.ts index b027fb7..1cee774 100644 --- a/packages/local-sql/test/PeerSync.test.ts +++ b/packages/local-sql/test/PeerSync.test.ts @@ -99,6 +99,16 @@ describe("PeerSync", () => { ) const RelaySyncService = PeerSync.layerRelay.pipe(Layer.provide(RelayServices)) const RelayTestLayer = Layer.merge(RelayServices, RelaySyncService) + const TightRelayServices = Layer.merge( + Services, + PeerRelayReceiptLimits.layer({ + ...PeerRelayReceiptLimits.defaults, + maxEncodedBytesPerRemote: 256, + maxEncodedBytesPerReplica: 256 + }) + ) + const TightRelaySyncService = PeerSync.layerRelay.pipe(Layer.provide(TightRelayServices)) + const TightRelayTestLayer = Layer.merge(TightRelayServices, TightRelaySyncService) const RecoveryService = Recovery.layer.pipe(Layer.provide(Services)) const CompactionService = Compaction.layer.pipe( Layer.provide(Layer.merge(Services, RecoveryService)) @@ -291,14 +301,40 @@ describe("PeerSync", () => { const duplicate = yield* sync.receive(Task, documentId, sessionOne, { ...input, - relay: relay(senderOne, "sender-one", "a".repeat(64)) + relay: { + ...relay(senderOne, "sender-one", "a".repeat(64)), + encodedSize: message.byteLength + 10_000 + } }) assert.isTrue(duplicate.duplicate) + const originalCharge = yield* sql<{ + readonly pendingMessage: Uint8Array | null + readonly reply: Uint8Array | null + readonly retainedBytes: number + readonly writerProvenance: string + }>`SELECT + pending_message AS pendingMessage, + reply, + relay_encoded_size AS retainedBytes, + writer_provenance AS writerProvenance + FROM effect_local_peer_receipts + WHERE relay_sender_peer_id = ${senderOne} + AND relay_message_id = ${relayMessageId}` + assert.strictEqual(originalCharge.length, 1) + assert.strictEqual( + originalCharge[0]!.retainedBytes, + message.byteLength + + (originalCharge[0]!.reply?.byteLength ?? 0) + + (originalCharge[0]!.pendingMessage?.byteLength ?? 0) + + new TextEncoder().encode(originalCharge[0]!.writerProvenance).byteLength + ) const beforePrune = yield* sql<{ + readonly deleteTokens: number readonly directReceipts: number readonly relayBytes: number readonly relayReceipts: number + readonly retainedBytes: number readonly usageRows: number }>`SELECT (SELECT COUNT(*) FROM effect_local_peer_receipts @@ -307,20 +343,49 @@ describe("PeerSync", () => { WHERE relay_message_id IS NOT NULL) AS relayReceipts, (SELECT COUNT(*) FROM effect_local_peer_relay_receipt_usage) AS usageRows, (SELECT COALESCE(SUM(encoded_bytes), 0) - FROM effect_local_peer_relay_receipt_usage) AS relayBytes` - assert.deepStrictEqual(beforePrune, [{ + FROM effect_local_peer_relay_receipt_usage) AS relayBytes, + (SELECT COALESCE(SUM(relay_encoded_size), 0) + FROM effect_local_peer_receipts WHERE relay_message_id IS NOT NULL) AS retainedBytes, + (SELECT COUNT(*) FROM effect_local_peer_relay_receipt_delete_tokens) AS deleteTokens` + assert.deepStrictEqual(beforePrune.map(({ relayBytes: _, retainedBytes: __, ...row }) => row), [{ + deleteTokens: 0, directReceipts: 1, - relayBytes: message.byteLength * 2, relayReceipts: 2, usageRows: 2 }]) + assert.strictEqual(beforePrune[0]!.relayBytes, beforePrune[0]!.retainedBytes) + assert.isAbove(beforePrune[0]!.relayBytes, message.byteLength * 2) yield* sql`UPDATE effect_local_peer_receipts SET relay_receipt_expires_at = '1970-01-01T00:00:00.000Z' WHERE relay_message_id IS NOT NULL` + yield* sql`CREATE TRIGGER fail_relay_receipt_prune + BEFORE DELETE ON effect_local_peer_receipts + WHEN OLD.relay_message_id IS NOT NULL + BEGIN + SELECT RAISE(ABORT, 'Injected relay receipt prune failure'); + END` + assert.strictEqual((yield* Effect.exit(sync.pruneRelayReceipts!))._tag, "Failure") + yield* sql`DROP TRIGGER fail_relay_receipt_prune` + assert.deepStrictEqual( + yield* sql`SELECT + (SELECT COUNT(*) FROM effect_local_peer_receipts + WHERE relay_message_id IS NOT NULL) AS relayReceipts, + (SELECT COUNT(*) FROM effect_local_peer_relay_receipt_usage) AS usageRows, + (SELECT COALESCE(SUM(encoded_bytes), 0) + FROM effect_local_peer_relay_receipt_usage) AS relayBytes, + (SELECT COUNT(*) FROM effect_local_peer_relay_receipt_delete_tokens) AS deleteTokens`, + [{ + deleteTokens: 0, + relayBytes: beforePrune[0]!.relayBytes, + relayReceipts: 2, + usageRows: 2 + }] + ) assert.strictEqual(yield* sync.pruneRelayReceipts!, 1) const afterFirstBatch = yield* sql<{ + readonly deleteTokens: number readonly directReceipts: number readonly relayReceipts: number readonly usageRows: number @@ -329,8 +394,10 @@ describe("PeerSync", () => { WHERE relay_message_id IS NULL) AS directReceipts, (SELECT COUNT(*) FROM effect_local_peer_receipts WHERE relay_message_id IS NOT NULL) AS relayReceipts, - (SELECT COUNT(*) FROM effect_local_peer_relay_receipt_usage) AS usageRows` + (SELECT COUNT(*) FROM effect_local_peer_relay_receipt_usage) AS usageRows, + (SELECT COUNT(*) FROM effect_local_peer_relay_receipt_delete_tokens) AS deleteTokens` assert.deepStrictEqual(afterFirstBatch, [{ + deleteTokens: 0, directReceipts: 1, relayReceipts: 1, usageRows: 1 @@ -338,6 +405,7 @@ describe("PeerSync", () => { assert.strictEqual(yield* sync.pruneRelayReceipts!, 1) const afterPrune = yield* sql<{ + readonly deleteTokens: number readonly directReceipts: number readonly relayReceipts: number readonly usageRows: number @@ -346,8 +414,10 @@ describe("PeerSync", () => { WHERE relay_message_id IS NULL) AS directReceipts, (SELECT COUNT(*) FROM effect_local_peer_receipts WHERE relay_message_id IS NOT NULL) AS relayReceipts, - (SELECT COUNT(*) FROM effect_local_peer_relay_receipt_usage) AS usageRows` + (SELECT COUNT(*) FROM effect_local_peer_relay_receipt_usage) AS usageRows, + (SELECT COUNT(*) FROM effect_local_peer_relay_receipt_delete_tokens) AS deleteTokens` assert.deepStrictEqual(afterPrune, [{ + deleteTokens: 0, directReceipts: 1, relayReceipts: 0, usageRows: 0 @@ -356,6 +426,81 @@ describe("PeerSync", () => { InternalAutomerge.free(created.automerge) }).pipe(Effect.provide(RelayTestLayer))) + it.effect("charges retained relay replies before committing receipt quota usage", () => + Effect.gen(function*() { + const store = yield* DocumentStore.DocumentStore + const sync = yield* PeerSync.PeerSync + const sql = yield* SqlClient.SqlClient + const documentId = yield* Identity.makeDocumentId + const senderPeerId = yield* Identity.makePeerId + const relayPeerId = yield* Identity.makePeerId + const created = yield* store.create(Task, documentId, { + title: "x".repeat(8_000), + labels: [] + }) + const remote = Automerge.init }>>() + const generated = Automerge.generateSyncMessage(remote, Automerge.initSyncState()) + assert.isNotNull(generated[1]) + const message = generated[1]! + assert.isBelow(message.byteLength, 256) + const before = yield* sql<{ + readonly changes: number + readonly commitOutbox: number + readonly receipts: number + readonly usageRows: number + }>`SELECT + (SELECT COUNT(*) FROM effect_local_changes) AS changes, + (SELECT COUNT(*) FROM effect_local_commit_outbox) AS commitOutbox, + (SELECT COUNT(*) FROM effect_local_peer_receipts) AS receipts, + (SELECT COUNT(*) FROM effect_local_peer_relay_receipt_usage) AS usageRows` + const session = yield* sync.open(senderPeerId) + const messageHash = yield* Canonical.digest(message) + const error = yield* Effect.flip(sync.receive(Task, documentId, session, { + remoteConnectionEpoch: "sender-epoch", + receiveSequence: 0, + message, + lineage: Identity.genesisLineage, + writerProvenance: [], + relay: { + relayMessageId: yield* Identity.makeRelayMessageId, + relayPeerId, + senderTenantId: "tenant", + senderSubjectId: "sender", + senderPeerId, + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + messageHash, + outerEnvelopeDigest: "a".repeat(64), + receiptExpiresAt: new Date( + (yield* Clock.currentTimeMillis) + PeerRelayReceiptLimits.defaults.receiptRetentionMillis + ).toISOString(), + encodedSize: message.byteLength + } + })) + assert.strictEqual(error.reason._tag, "QuotaExceeded") + if (error.reason._tag === "QuotaExceeded") { + assert.strictEqual(error.reason.resource, "relay receipt bytes per remote") + } + const after = yield* sql<{ + readonly changes: number + readonly commitOutbox: number + readonly receipts: number + readonly usageRows: number + }>`SELECT + (SELECT COUNT(*) FROM effect_local_changes) AS changes, + (SELECT COUNT(*) FROM effect_local_commit_outbox) AS commitOutbox, + (SELECT COUNT(*) FROM effect_local_peer_receipts) AS receipts, + (SELECT COUNT(*) FROM effect_local_peer_relay_receipt_usage) AS usageRows` + assert.deepStrictEqual(after, before) + const reloaded = yield* store.load(Task, documentId) + assert.deepStrictEqual(reloaded.snapshot.value, { + title: "x".repeat(8_000), + labels: [] + }) + InternalAutomerge.free(reloaded.automerge) + InternalAutomerge.free(remote) + InternalAutomerge.free(created.automerge) + }).pipe(Effect.provide(TightRelayTestLayer))) + it.effect("persists inbound application and exact retransmission replies", () => Effect.gen(function*() { const store = yield* DocumentStore.DocumentStore diff --git a/packages/local-sql/test/PeerSyncEnvelope.test.ts b/packages/local-sql/test/PeerSyncEnvelope.test.ts index 36f1643..b685f48 100644 --- a/packages/local-sql/test/PeerSyncEnvelope.test.ts +++ b/packages/local-sql/test/PeerSyncEnvelope.test.ts @@ -1,8 +1,8 @@ import * as Automerge from "@automerge/automerge" -import * as Canonical from "@lucas-barake/effect-local/Canonical" -import * as Identity from "@lucas-barake/effect-local/Identity" import * as NodeCrypto from "@effect/platform-node/NodeCrypto" import { assert, describe, it } from "@effect/vitest" +import * as Canonical from "@lucas-barake/effect-local/Canonical" +import * as Identity from "@lucas-barake/effect-local/Identity" import * as Effect from "effect/Effect" import * as Schema from "effect/Schema" import * as PeerSyncEnvelope from "../src/PeerSyncEnvelope.js" @@ -14,6 +14,8 @@ const limits: PeerSyncEnvelope.SyncEnvelopeLimits = { maxSyncOperationsPerMessage: 10_000 } +const rewrittenLineage = Identity.DocumentLineage.make("lin_00000000-0000-4000-8000-000000000011") + const makeSyncEnvelope = Effect.gen(function*() { let source = Automerge.from( { value: { title: "one" }, tombstone: false }, @@ -37,6 +39,7 @@ const makeSyncEnvelope = Effect.gen(function*() { documentType: "Task", messageHash: yield* Canonical.digest(message), message, + lineage: rewrittenLineage, writerProvenance }) Automerge.free(source) @@ -51,13 +54,44 @@ describe("PeerSyncEnvelope", () => { const bytes = yield* PeerSyncEnvelope.encodeSyncEnvelope(envelope) const decoded = yield* PeerSyncEnvelope.decodeSyncEnvelope(bytes, limits) assert.deepStrictEqual(decoded, envelope) + assert.strictEqual(decoded.lineage, rewrittenLineage) assert.deepStrictEqual(PeerSyncEnvelope.syncEnvelopeDocument(decoded), { documentId: envelope.documentId, documentType: "Task" }) }).pipe(Effect.provide(NodeCrypto.layer))) - it.effect("rejects oversized, malformed, conflicting, and unprovenanced payloads", () => + it.effect("accepts an opaque standard V2 Document sync envelope without semantic expansion", () => + Effect.gen(function*() { + const envelope = yield* makeSyncEnvelope + const chunks = Automerge.decodeSyncMessage(envelope.message).changes + assert.strictEqual(chunks.length, 1) + assert.throws(() => Automerge.decodeChange(chunks[0]!)) + const document = Automerge.load(chunks[0]!) + try { + assert.strictEqual(Automerge.getAllChanges(document).length, 1) + } finally { + Automerge.free(document) + } + const bytes = yield* PeerSyncEnvelope.encodeSyncEnvelope(envelope) + const decoded = yield* PeerSyncEnvelope.decodeSyncEnvelope(bytes, { + ...limits, + maxSyncOperationsPerMessage: 0 + }) + assert.deepStrictEqual(decoded, envelope) + assert.deepStrictEqual( + yield* PeerSyncEnvelope.validateSyncEnvelope({ + ...envelope, + writerProvenance: [] + }, limits), + { + ...envelope, + writerProvenance: [] + } + ) + }).pipe(Effect.provide(NodeCrypto.layer))) + + it.effect("rejects oversized, malformed, conflicting, and excess-provenance payloads", () => Effect.gen(function*() { const envelope = yield* makeSyncEnvelope const bytes = yield* PeerSyncEnvelope.encodeSyncEnvelope(envelope) @@ -82,7 +116,10 @@ describe("PeerSyncEnvelope", () => { assert.strictEqual( (yield* Effect.exit(PeerSyncEnvelope.validateSyncEnvelope({ ...envelope, - writerProvenance: [] + writerProvenance: Array.from( + { length: limits.maxSyncChangesPerMessage + 1 }, + () => envelope.writerProvenance[0]! + ) }, limits)))._tag, "Failure" ) @@ -114,6 +151,7 @@ describe("RelayOuterEnvelope", () => { documentId: Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000005"), documentType: "Task" }, + lineage: rewrittenLineage, writerProvenance: [ { changeHash: "b".repeat(64), @@ -145,7 +183,7 @@ describe("RelayOuterEnvelope", () => { expectedLocal: { ...base.expectedLocal }, remote: { ...base.remote }, document: { ...base.document }, - writerProvenance: [...base.writerProvenance].reverse() + writerProvenance: base.writerProvenance.toReversed() }) assert.deepStrictEqual( yield* PeerSyncEnvelope.encodeRelayOuterEnvelope(base), @@ -156,7 +194,7 @@ describe("RelayOuterEnvelope", () => { yield* PeerSyncEnvelope.encodeRelayOuterEnvelope(recipientReplay) ) const digest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope(base) - assert.strictEqual(digest, "8e52edf067d2fdbc9be0c5f266f53cb9f12c4557c976e47bdc567c4d4f4efa1b") + assert.strictEqual(digest, "bf6947413c2a682c4a99718e168954df367d445fdbb5524050df97f39cfe1db0") assert.strictEqual( yield* PeerSyncEnvelope.digestRelayOuterEnvelope(relayAdmission), digest @@ -167,31 +205,39 @@ describe("RelayOuterEnvelope", () => { ) assert.deepStrictEqual(PeerSyncEnvelope.relayOuterEnvelopeDocument(base), base.document) assert.strictEqual( - (yield* Effect.exit(Schema.decodeUnknownEffect(PeerSyncEnvelope.RelayOuterEnvelope)({ - ...base, - domain: "unrelated-domain" - })))._tag, + (yield* Effect.exit( + Schema.decodeUnknownEffect(PeerSyncEnvelope.RelayOuterEnvelope)({ + ...base, + domain: "unrelated-domain" + }) + ))._tag, "Failure" ) assert.strictEqual( - (yield* Effect.exit(Schema.decodeUnknownEffect(PeerSyncEnvelope.RelayOuterEnvelope)({ - ...base, - version: 2 - })))._tag, + (yield* Effect.exit( + Schema.decodeUnknownEffect(PeerSyncEnvelope.RelayOuterEnvelope)({ + ...base, + version: 2 + }) + ))._tag, "Failure" ) assert.strictEqual( - (yield* Effect.exit(Schema.decodeUnknownEffect(PeerSyncEnvelope.RelayOuterEnvelope)({ - ...base, - protocolVersion: 4 - })))._tag, + (yield* Effect.exit( + Schema.decodeUnknownEffect(PeerSyncEnvelope.RelayOuterEnvelope)({ + ...base, + protocolVersion: 4 + }) + ))._tag, "Failure" ) assert.strictEqual( - (yield* Effect.exit(Schema.decodeUnknownEffect(PeerSyncEnvelope.RelayOuterEnvelope)({ - ...base, - payloadVersion: 2 - })))._tag, + (yield* Effect.exit( + Schema.decodeUnknownEffect(PeerSyncEnvelope.RelayOuterEnvelope)({ + ...base, + payloadVersion: 2 + }) + ))._tag, "Failure" ) }).pipe(Effect.provide(NodeCrypto.layer))) @@ -237,6 +283,10 @@ describe("RelayOuterEnvelope", () => { documentId: Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000010") } }, + { + ...base, + lineage: Identity.DocumentLineage.make("lin_00000000-0000-4000-8000-000000000012") + }, { ...base, writerProvenance: [{ From 6aefe955182f96c45cf796e4ecdbecd78198ecaa Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Sun, 26 Jul 2026 12:02:35 -0500 Subject: [PATCH 03/29] Refactor durable relay around generic SQL storage --- README.md | 573 +-- docs/architecture.md | 20 +- docs/durability.md | 2 +- docs/store-and-forward.md | 84 +- docs/sync.md | 18 +- knip.json | 3 - packages/local-rpc/README.md | 12 +- .../local-rpc/bench/PeerRelayStore.bench.ts | 7 +- packages/local-rpc/bench/PeerRpc.bench.ts | 56 +- .../bench/PeerRpcServerPerformance.bench.ts | 157 - .../local-rpc/bench/RpcPeerTransport.bench.ts | 133 - .../base-admission-instrumentation.patch | 51 - .../candidate-admission-instrumentation.patch | 75 - .../bench/vitest.admission.config.ts | 8 - packages/local-rpc/package.json | 12 +- packages/local-rpc/src/PeerAuthentication.ts | 8 +- packages/local-rpc/src/PeerAuthorization.ts | 42 - packages/local-rpc/src/PeerRelayLimits.ts | 15 +- packages/local-rpc/src/PeerRelayRpc.ts | 149 - packages/local-rpc/src/PeerRelayStore.ts | 1722 +-------- packages/local-rpc/src/PeerRpc.ts | 143 +- packages/local-rpc/src/PeerRpcLimits.ts | 108 - packages/local-rpc/src/PeerRpcServer.ts | 1310 +------ packages/local-rpc/src/RpcPeerTransport.ts | 227 +- packages/local-rpc/src/SqlPeerRelayStore.ts | 1891 ++++++++++ packages/local-rpc/src/index.ts | 4 +- .../src/internal/peerRelayMigrations.ts | 1094 ++++-- ...nsaction.ts => peerRelaySqlTransaction.ts} | 36 +- .../src/internal/peerRelayStoreErrors.ts | 3 +- .../local-rpc/src/internal/peerRpcProtocol.ts | 4 + .../local-rpc/test/PeerAuthentication.test.ts | 85 +- .../local-rpc/test/PeerAuthorization.test.ts | 99 - .../local-rpc/test/PeerRelayLimits.test.ts | 12 +- packages/local-rpc/test/PeerRelayRpc.test.ts | 191 - .../test/PeerRelaySqliteTransaction.test.ts | 2 +- .../local-rpc/test/PeerRelayStore.test.ts | 111 +- .../local-rpc/test/PeerRelayStoreContract.ts | 239 ++ packages/local-rpc/test/PeerRpc.test.ts | 406 +-- .../local-rpc/test/PeerRpcIntegration.test.ts | 296 ++ packages/local-rpc/test/PeerRpcLimits.test.ts | 169 - packages/local-rpc/test/PeerRpcServer.test.ts | 3133 ++--------------- .../PeerRpcServerSessionTransport.test.ts | 862 ----- .../local-rpc/test/PeerRpcWebSocket.test.ts | 745 ---- .../local-rpc/test/PublicApi.types.test.ts | 28 +- .../local-rpc/test/RpcPeerTransport.test.ts | 990 +----- .../local-rpc/test/SqlPeerRelayStore.test.ts | 255 ++ packages/local-rpc/vitest.config.ts | 12 +- packages/local-sql/README.md | 5 +- packages/local-sql/src/Migrations.ts | 2 +- packages/local-sql/src/PeerRelayOutbox.ts | 17 +- packages/local-sql/src/PeerSession.ts | 234 +- packages/local-sql/src/PeerSyncEnvelope.ts | 2 +- packages/local-sql/test/BackupStore.test.ts | 2 +- packages/local-sql/test/Compaction.test.ts | 2 +- packages/local-sql/test/Migrations.test.ts | 2 +- .../test/PeerRelayOutboxLimits.test.ts | 34 +- .../test/PeerRelayReceiptLimits.test.ts | 34 +- packages/local-sql/test/PeerSession.test.ts | 344 +- .../test/PeerSessionCoverage.test.ts | 6 +- .../local-sql/test/PeerSyncEnvelope.test.ts | 4 +- packages/local-test/src/TestPeer.ts | 52 +- .../local-test/test/TestPeerCoverage.test.ts | 4 +- packages/local/src/PeerTransport.ts | 6 +- packages/local/test/PublicApi.types.test.ts | 52 +- pnpm-lock.yaml | 1316 +++++++ 65 files changed, 6186 insertions(+), 11534 deletions(-) delete mode 100644 packages/local-rpc/bench/PeerRpcServerPerformance.bench.ts delete mode 100644 packages/local-rpc/bench/RpcPeerTransport.bench.ts delete mode 100644 packages/local-rpc/bench/base-admission-instrumentation.patch delete mode 100644 packages/local-rpc/bench/candidate-admission-instrumentation.patch delete mode 100644 packages/local-rpc/bench/vitest.admission.config.ts delete mode 100644 packages/local-rpc/src/PeerAuthorization.ts delete mode 100644 packages/local-rpc/src/PeerRelayRpc.ts delete mode 100644 packages/local-rpc/src/PeerRpcLimits.ts create mode 100644 packages/local-rpc/src/SqlPeerRelayStore.ts rename packages/local-rpc/src/internal/{peerRelaySqliteTransaction.ts => peerRelaySqlTransaction.ts} (89%) create mode 100644 packages/local-rpc/src/internal/peerRpcProtocol.ts delete mode 100644 packages/local-rpc/test/PeerAuthorization.test.ts delete mode 100644 packages/local-rpc/test/PeerRelayRpc.test.ts create mode 100644 packages/local-rpc/test/PeerRelayStoreContract.ts create mode 100644 packages/local-rpc/test/PeerRpcIntegration.test.ts delete mode 100644 packages/local-rpc/test/PeerRpcLimits.test.ts delete mode 100644 packages/local-rpc/test/PeerRpcServerSessionTransport.test.ts delete mode 100644 packages/local-rpc/test/PeerRpcWebSocket.test.ts create mode 100644 packages/local-rpc/test/SqlPeerRelayStore.test.ts diff --git a/README.md b/README.md index 37ae692..90afa19 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,8 @@ WebSocket, serialization, routing, and server ownership to Effect and the applic > **Beta:** The library targets Effect `4.0.0-beta.99` and Automerge `3.3.2`. Durable formats, > worker protocols, and public APIs can still change. Read [Limits and security](#limits-and-security) before adopting -> it for user data. Direct external RPC remains live only. Optional store and forward provides single authority -> SQLite custody with an at least once delivery contract. It does not claim a globally replicated relay service. +> it for user data. External RPC uses one durable version `1` relay topology with an at least once delivery contract. +> The built in SQL custody store supports SQLite, PostgreSQL, and MySQL. It is not a managed global relay service. ## Retrieval contract @@ -55,14 +55,14 @@ sessions, peer sessions, and presence connect processes without becoming another This separates concerns that are often collapsed into one client state library: -| Concern | Responsibility | What it does not own | -| ------------------------ | -------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------- | -| Canonical recovery state | Automerge changes, heads, checkpoints, tombstones, and command receipts | Physical storage bytes or UI cache state | -| Physical persistence | SQLite transactions and database bytes, stored in OPFS in the browser | Merge semantics or permanent browser retention | -| Local execution | Serialized commands, durable replies, maintenance workflows, and recovery | Replicated convergence | -| Derived durable views | SQL projection tables optimized for application queries | Canonical history | -| Ephemeral views | Atom caches, RPC leases, and presence | Durable facts | -| Connectivity | Worker RPC, peer sessions, application supplied transports, and optional relay custody | Identity issuance, application routing policy, platform socket or server ownership | +| Concern | Responsibility | What it does not own | +| ------------------------ | ------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------- | +| Canonical recovery state | Automerge changes, heads, checkpoints, tombstones, and command receipts | Physical storage bytes or UI cache state | +| Physical persistence | SQLite transactions and database bytes, stored in OPFS in the browser | Merge semantics or permanent browser retention | +| Local execution | Serialized commands, durable replies, maintenance workflows, and recovery | Replicated convergence | +| Derived durable views | SQL projection tables optimized for application queries | Canonical history | +| Ephemeral views | Atom caches, RPC leases, and presence | Durable facts | +| Connectivity | Worker RPC, durable peer sessions, application supplied transports, and relay custody | Identity issuance, application routing policy, platform socket or server ownership | The distinction follows Automerge's separation between a [CRDT document and its storage adapter](https://automerge.org/docs/reference/repositories/storage/) and SQLite's @@ -94,8 +94,7 @@ SQLite does not decide how concurrent document changes merge. Automerge `save` e | **Invalidation key** | Notification metadata for refreshing a document or projection read. It is not canonical data or a durability acknowledgement. | | **Presence** | Expiring best effort metadata about connected peers or tabs. Presence is never durable state and must never authorize an operation. | | **Hosted canonical replica** | One ordinary `SqlReplica` running in a server process. It participates as a CRDT peer. It is not a transaction authority for other replicas. | -| **Direct RPC** | The version `2` `Open` stream and `Push` path owned by `PeerRpcServer.layerHandlers`. It has no durable custody or replay log. | -| **Store and forward relay** | The optional version `3` protocol on a separate bounded listener. A stable sender outbox transfers custody to one SQLite relay authority for at least once reconnect delivery. | +| **Store and forward relay** | The version `1` RPC topology. A stable sender outbox transfers custody to an injected durable backend store for at least once reconnect delivery. | | **Relay acknowledgement** | A fenced recipient response sent only after the production SQL sync workflow and sender scoped replay receipt commit. It can retire only the exact live claim. | | **Principal** | A request scoped `tenantId`, `subjectId`, and stable `peerId` produced by `PeerAuthenticator`. It is derived from a credential on every RPC operation. | | **Authorization lease** | A bounded grant for exactly the requested whole document set. Expiry or invalidation terminates the session. It is not a distributed lock. | @@ -168,15 +167,14 @@ and [Atom](https://github.com/Effect-TS/effect/blob/eb9b10256c8558881b441c2fef83 Transport neutral synchronization has three layers. The optional RPC package adds a contract and adapter: -| Layer | Responsibility | -| ------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `PeerTransport` | Application supplied connection, peer identity, routing, security, receive stream, send operation, and close operation. | -| `PeerSync` | Persistent receive identity, epoch scoped outbox, sequencing, replay, and received change application. Automerge `SyncState` remains in memory for one connection. | -| `PeerSession` | Scoped orchestration that binds one transport connection to one replica incarnation and a selected set of whole document instances. | -| `PeerRpc` | Versioned `Open` and `Push` contract plus the generated Effect RPC client. It does not own an Effect `RpcClient.Protocol`, `RpcServer.Protocol`, serializer, platform socket, or server. | -| `RpcPeerTransport` | Adapter from the generated RPC client to `PeerTransport`, then to the existing SQL `PeerSession`. | -| `PeerRelayRpc` | Optional version `3` `OpenRelay`, `PushRelay`, `AcknowledgeRelay`, and `RejectRelay` contract. It runs on a separate bounded protocol and listener. | -| Relay SQL state | Stable sender outbox, single authority relay custody, and sender scoped recipient receipts for at least once reconnect delivery. | +| Layer | Responsibility | +| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `PeerTransport` | Application supplied connection, peer identity, routing, security, receive stream, send operation, and close operation. | +| `PeerSync` | Persistent receive identity, epoch scoped outbox, sequencing, replay, and received change application. Automerge `SyncState` remains in memory for one connection. | +| `PeerSession` | Scoped orchestration that binds one transport connection to one replica incarnation and a selected set of whole document instances. | +| `PeerRpc` | Durable version `1` `Open`, `Push`, `Acknowledge`, and `Reject` contract plus the generated Effect RPC client. | +| `RpcPeerTransport` | Adapter from the generated RPC client to `PeerTransport`, then to the existing SQL `PeerSession`. | +| Relay SQL state | Frontend SQLite outbox and receipts plus injected backend custody for at least once reconnect delivery. | `observedByPeer` means Automerge's per peer sync state indicates that the peer has all local changes. It does not prove remote storage durability. Effect Local therefore keeps durable confirmation separate and currently reports it as @@ -310,7 +308,7 @@ Package roles: | `@lucas-barake/effect-local-sql` | SQLite persistence, durable execution, recovery, compaction, peer sync, sender relay outbox, and relay receipts | | `@lucas-barake/effect-local-browser` | Effect Worker and RPC composition, OPFS ports, sessions, presence, and Atom builders | | `@lucas-barake/effect-local-test` | In memory production shaped replicas and deterministic bounded peer faults | -| `@lucas-barake/effect-local-rpc` | Direct RPC plus optional bounded store and forward protocol, custody, policies, server, and client adapter | +| `@lucas-barake/effect-local-rpc` | Durable relay protocol, injectable custody, policies, bounded server, and client adapter | Package dependency direction is: @@ -342,15 +340,14 @@ The browser suites currently exercise Chromium. Other engines are not claimed as worker, locking, reload, and browser test suite passes there. Browser storage starts as best effort. Request `navigator.storage.persist()`, report the result, and provide backup and restore controls. -The RPC package itself is platform neutral. A direct network deployment requires an Effect `RpcServer.Protocol` and +The RPC package itself is platform neutral. A network deployment requires an Effect `RpcServer.Protocol` and `RpcClient.Protocol`, the same `RpcSerialization` on both ends, a platform `Socket`, an HTTP server, TLS, WebSocket upgrade routing, Origin policy, ingress byte and connection limits, credential issuance, secret rotation, tenant routing, process supervision, and graceful shutdown. None of those responsibilities are inferred from `PeerRpcServer.layerHandlers`. -The optional relay uses its own `PeerRelayIngress` protocol and distinct socket listener. It must not reuse the direct -WebSocket MessagePack protocol or listener. The application still owns TLS, routing, identity, policy, process -supervision, and the single SQLite process and durable volume that hold one relay shard. +The durable relay uses the bounded `PeerRelayIngress` protocol. The application owns TLS, routing, identity, policy, +process supervision, and the SQLite, PostgreSQL, or MySQL deployment that holds one relay shard. ## Composition recipes @@ -899,8 +896,8 @@ backup limits. For a custom transport, the application provides `PeerTransport` and owns its identity, authentication, authorization, encryption, discovery, and routing. For Effect RPC, do not implement another `PeerTransport`. `RpcPeerTransport` -supplies it from `PeerRpc.RpcClient`. `PeerAuthentication`, `PeerAuthorization`, and the application owned Effect RPC -protocol retain those responsibilities. +supplies it from `PeerRpc.RpcClient`. `PeerAuthentication`, `PeerRelayAuthorization`, and the application owned socket +composition retain those responsibilities. ```ts import * as Identity from "@lucas-barake/effect-local/Identity" @@ -915,7 +912,7 @@ declare const openAuthenticatedConnection: ( ) => Effect.Effect export const TransportLive = Layer.succeed(PeerTransport.PeerTransport, { - capabilities: { storeAndForward: false }, + capabilities: { lineageAware: true }, connect: ({ peerId }) => openAuthenticatedConnection(peerId) }) ``` @@ -992,234 +989,34 @@ responsibility. Expiry is enforced lazily when `values` reads. Entries stay resident until then, so a peer that only writes never reclaims expired entries. -### 12. Host a canonical replica over Effect RPC +### 12. Host a durable relay over Effect RPC -The hosted process runs the same `SqlReplica` composition as any Node replica. The RPC package adds a live transport -surface. It does not introduce a server database model, second Automerge repository, or durable relay. +The RPC package exposes one protocol, `PeerRpc` version `1`. Its procedures are `Open`, `Push`, +`Acknowledge`, and `Reject`. Every connection uses durable custody. There is no direct in memory RPC topology. -The application provides two Effect capabilities: +Applications provide `PeerAuthenticator`, `PeerRelayAuthorization`, `PeerRelayLimits`, +`PeerRelayIngress`, and `PeerRelayStore`. The semantic store is injectable. The built in +`SqlPeerRelayStore.layer` requires a generic `SqlClient` and supports SQLite, PostgreSQL, and MySQL. The frontend +replica, sender outbox, and recipient receipts remain SQLite. -- `PeerAuthenticator` verifies one redacted credential on every `Open` and `Push`. It returns a request scoped - `PeerPrincipal`, a finite validity deadline, and an invalidation Effect. -- `PeerAuthorization` maps the principal and requested document identities to the server's actual document - definitions. Authorization is exact and all or nothing. It returns its own finite validity deadline and invalidation - Effect. - -`PeerCredentials`, `PeerAuthenticator`, and `PeerAuthorization` are Context services because credential retrieval and -policy evaluation are application capabilities. They can be asynchronous, fail in the typed channel, depend on other -services, and be replaced by Layers without changing middleware constructors. - -`invalidated` is a nonfailing revocation signal. It must remain pending while the authentication or authorization -decision remains valid and complete once that decision is revoked. Use `Effect.never` when no early invalidation signal -exists. It must not fail or defect. The `validUntil` deadline and `maximumReauthorizationInterval` remain independent -upper bounds. - -`PeerPrincipal.peerId` identifies the authenticated calling replica. `PeerRpcServer.layerHandlers({ peerId })` -identifies the hosted server replica. On the client, `RpcPeerTransport.makeSession({ peerId })` must receive that hosted -server replica ID. Name these values `authenticatedClientPeerId` and `hostedServerPeerId`. Do not derive one from the -other. - -The following excerpt is the public server composition. - -```ts -import * as PeerAuthentication from "@lucas-barake/effect-local-rpc/PeerAuthentication" -import * as PeerAuthenticator from "@lucas-barake/effect-local-rpc/PeerAuthenticator" -import * as PeerAuthorization from "@lucas-barake/effect-local-rpc/PeerAuthorization" -import * as PeerRpc from "@lucas-barake/effect-local-rpc/PeerRpc" -import * as PeerRpcError from "@lucas-barake/effect-local-rpc/PeerRpcError" -import * as PeerRpcLimits from "@lucas-barake/effect-local-rpc/PeerRpcLimits" -import * as PeerRpcServer from "@lucas-barake/effect-local-rpc/PeerRpcServer" -import * as Identity from "@lucas-barake/effect-local/Identity" -import * as Effect from "effect/Effect" -import * as Layer from "effect/Layer" -import * as Redacted from "effect/Redacted" -import * as HttpRouter from "effect/unstable/http/HttpRouter" -import * as RpcSerialization from "effect/unstable/rpc/RpcSerialization" -import * as RpcServer from "effect/unstable/rpc/RpcServer" -import { definition, Task } from "./domain.js" - -declare const authenticatedClientPeerId: Identity.PeerId -declare const hostedServerPeerId: Identity.PeerId -declare const documentId: Identity.DocumentId -declare const expectedCredential: string - -const PoliciesLive = Layer.mergeAll( - PeerRpcLimits.layerDefaults, - Layer.succeed(PeerAuthenticator.PeerAuthenticator)({ - authenticate: (credential) => - Redacted.value(credential) === expectedCredential - ? Effect.succeed({ - principal: PeerAuthentication.PeerPrincipal.make({ - tenantId: "acme", - subjectId: "user-42", - peerId: authenticatedClientPeerId - }), - validUntil: Date.now() + 60_000, - invalidated: Effect.never - }) - : Effect.fail(new PeerRpcError.AuthenticationFailure()) - }), - PeerAuthorization.layer((request) => { - const requested = request.documents[0] - return request.principal.tenantId === "acme" && - request.documents.length === 1 && - requested?.documentType === Task.name && - requested.documentId === documentId - ? Effect.succeed({ - documents: [{ document: Task, documentId }], - validUntil: Date.now() + 60_000, - invalidated: Effect.never - }) - : Effect.fail(new PeerRpcError.AccessDenied()) - }) -) - -const AuthenticationLive = PeerAuthentication.layerServer.pipe( - Layer.provide(PoliciesLive) -) - -const HandlersLive = PeerRpcServer.layerHandlers({ - tenantId: "acme", - peerId: hostedServerPeerId, - definition -}).pipe(Layer.provide(PoliciesLive)) - -const WsProtocol = RpcServer.layerProtocolWebsocket({ path: "/rpc" }).pipe( - Layer.provide(HttpRouter.layer) -) - -const PeerRpcLive = RpcServer.layer(PeerRpc.Rpcs, { - disableFatalDefects: true -}).pipe( - Layer.provide([HandlersLive, AuthenticationLive]), - Layer.provideMerge(WsProtocol), - Layer.provide(HttpRouter.serve(WsProtocol)), - Layer.provide(RpcSerialization.layerMsgPack) -) -``` - -The application must still provide `HttpServer` and the platform WebSocket implementation required by the Effect -server protocol. `PeerRpcServer.layerHandlers` specifically requires `CommitPublisher`, `PeerRpcLimits`, core -`ReplicaLimits`, `PeerAuthorization`, `Crypto`, `PeerSync`, `ReplicaGate`, and `Sharding`. `disableFatalDefects: true` -is required so one unexpected request defect is encoded as the fixed `InternalError` sentinel instead of terminating -unrelated requests multiplexed over the connection. Typed policy failures remain `PeerRpcError` values. - -One `PeerRpcServer` Layer instance serves exactly one configured tenant and one canonical SQL replica. The canonical -schema has no tenant column. Multi tenant deployment therefore requires isolated replicas, databases, schemas, or -processes. Do not route principals from different tenants into one handler Layer. - -`layerHandlers` owns one server scoped `CommitPublisher` subscription, not one subscription per peer. It indexes -document interest, coalesces dirty notifications, and flushes interested sessions with bounded worker concurrency. Its -registry enforces one active incarnation per tenant and peer, per subject session and in flight quotas, subject token -buckets, item capacity, per session bytes, total buffered bytes, lease invalidation, and bounded shutdown cleanup -concurrency. `PeerAuthentication.layerServer` separately owns connection authentication buckets keyed by the Effect RPC -client ID. Reopening the same authenticated peer replaces and closes the prior incarnation. Cleanup has bounded -parallelism but no time bound because each session scope is closed uninterruptibly. +`PeerRpcServer.layerHandlers({ tenantId, peerId })` builds the handlers. `PeerRpcServer.layerServer` supervises the +RPC server and bounded ingress. A successful `Push` means the backend store committed custody before replying. ### 13. Connect an RPC peer -The client composes the generated client with an application owned Effect RPC protocol. `layerClient` retrieves a -fresh redacted credential from `PeerCredentials` and overwrites any credential present in the request payload. The -preferred `RpcPeerTransport.makeSession` path adapts that generated client to the transport neutral SQL -`PeerSession.makeLive` implementation. +Create `PeerRpc.RpcClient` through `PeerRpc.makeRpcClient`, then adapt it with +`RpcPeerTransport.layer` or `RpcPeerTransport.makeSession`. Options bind the expected local principal, relay peer, +remote peer, selected documents, replica incarnation, receipt retention, retry horizon, and replay batch size. -```ts -import { NodeSocket } from "@effect/platform-node" -import * as PeerAuthentication from "@lucas-barake/effect-local-rpc/PeerAuthentication" -import * as PeerCredentials from "@lucas-barake/effect-local-rpc/PeerCredentials" -import * as PeerRpc from "@lucas-barake/effect-local-rpc/PeerRpc" -import * as RpcPeerTransport from "@lucas-barake/effect-local-rpc/RpcPeerTransport" -import * as Identity from "@lucas-barake/effect-local/Identity" -import * as Effect from "effect/Effect" -import * as Layer from "effect/Layer" -import * as Redacted from "effect/Redacted" -import * as RpcClient from "effect/unstable/rpc/RpcClient" -import * as RpcSerialization from "effect/unstable/rpc/RpcSerialization" -import { definition, Task } from "./domain.js" -import { EngineLive } from "./replica.js" +The connection always exposes the relay peer and an acknowledged delivery stream. `PeerSession` acknowledges only +after the production SQL sync workflow and sender scoped receipt commit. Lost responses or expired claims can +redeliver a message, so delivery is at least once and receipt suppression is durable. -declare const documentId: Identity.DocumentId -declare const hostedServerPeerId: Identity.PeerId - -const CredentialsLive = Layer.succeed(PeerCredentials.PeerCredentials)({ - get: Effect.succeed(Redacted.make("rotating-credential")) -}) - -const AuthenticationLive = PeerAuthentication.layerClient.pipe( - Layer.provide(CredentialsLive) -) - -const ProtocolLive = RpcClient.layerProtocolSocket().pipe( - Layer.provide([ - NodeSocket.layerWebSocket("wss://sync.example.com/rpc"), - RpcSerialization.layerMsgPack - ]) -) - -const synchronize = Effect.scoped(Effect.gen(function*() { - const client = yield* PeerRpc.makeRpcClient - const session = yield* RpcPeerTransport.makeSession(client, { - peerId: hostedServerPeerId, - documents: [{ document: Task, documentId }], - definition - }) - yield* session.flush - return yield* session.observedByPeer(documentId) -})).pipe(Effect.provide(Layer.mergeAll(EngineLive, ProtocolLive, AuthenticationLive))) -``` +Automerge `3.3.2` does not expose an allocation bounded semantic decode API. Relay admission therefore requires the +separate exact `UnsafeUnboundedAutomerge3DecodeGrant`. The recommended default remains +`PeerRelayAuthorization.denyUnsafeUnboundedAutomerge3Decode`. -`EngineLive` is the local `SqlReplica` Layer from recipe 6. It supplies the `CommitPublisher`, `PeerSync`, `ReplicaGate`, -`ReplicaLimits`, `Crypto`, and `Sharding` requirements listed in -[RPC environment requirements](#rpc-environment-requirements). - -The complete scoped synchronization Effect is the reconnect unit. Real applications keep all session use inside that -scope. Effect's socket protocol may reconnect the socket, but it cannot replay an already failed `Open` response -stream. On retry, recreate the client scope and peer session. Reuse the same stable remote `PeerId` and requested -document identities. A fresh Automerge sync state generates new messages from persisted canonical causal history. -The prior epoch's SQL outbox is not resumed. Do not cache `SessionId`, connection epoch, stream, or middleware lease -across scopes. During a graceful server restart, close client connection scopes or otherwise close upgraded WebSocket -connections before awaiting the HTTP server shutdown. - -Closing an `RpcPeerTransport` connection interrupts any fiber parked consuming its `receive` stream, so that consumer -observes an interrupt only `Exit` rather than a failure, and does not wait for the enclosing scope to close. A consumer -inside its element handler is not interrupted mid handler. It finishes that handler, may still receive a payload the -transport already pulled from the peer, and then observes the interrupt on its next pull. Payloads that arrive after -close are not delivered. - -`RpcPeerTransport.isRetryable` classifies only `ReplicaError(StorageUnavailable)` as retryable. Authentication, -authorization, version, peer identity, request shape, declared limit, and document lineage failures map to -`ReplicaError(ProtocolMismatch)` and require configuration or policy correction. A remote -`PeerRpcError(DocumentLineageChanged)` arrives as `ProtocolMismatch` with the tag in `observed`, because the wire -error carries no document identity and the adapter will not invent one. Retrying it cannot succeed. `Push` success -means only that the current server session accepted the bytes into bounded memory. `storeAndForward` and -`durableConfirmation` are both false. Inbound item or byte overflow is terminal for that incarnation. An outbound capacity timeout during initial -synchronization, a Push reply, or commit driven flush is also terminal. The triggering operation and active `Open` -stream fail with `SessionOverloaded`. Later pushes for that session receive `SessionUnavailable`, so recovery creates a -fresh connection scope and session. - -### Optional store and forward - -Direct protocol version `2` stays source compatible and live only. Store and forward is a separate opt in composition -using relay protocol version `3`, a distinct bounded socket listener, stable sender outbox rows, one SQLite custody -authority per shard, and sender scoped recipient receipts. - -`PeerRpcServer.layerStoreAndForwardDeployment` merges an existing direct deployment with the separate relay listener. -The client uses `SqlReplica.layerRelay`, `PeerRelayOutbox.layerSql`, `PeerRelayClientRuntime.layer`, and -`RpcPeerTransport.makeStoreAndForwardSession`. Relay acceptance proves durable custody. Recipient acknowledgement is -attempted only after the production SQL sync workflow and replay receipt commit. Delivery remains at least once. - -Automerge `3.3.2` does not expose an allocation bounded semantic decode API. Relay use therefore requires ordinary -endpoint and document authorization plus a separate explicit unsafe resource trust grant for the exact principal, -remote, direction, and documents. The recommended default is -`PeerRelayAuthorization.denyUnsafeUnboundedAutomerge3Decode`. Authentication and ordinary document access do not -imply that grant. - -Recipient attempts are durable and bounded by `PeerRelayLimits.maximumDeliveryAttempts`, which defaults to `16`. -Reaching the cap dead letters the message and erases its payload. - -See [Store and forward](docs/store-and-forward.md) for the exact Layer composition, protocol, acknowledgement -boundary, ordering scope, expiry, retention, quotas, security, single authority deployment boundary, failure limits, -and exposed usage values. +See [Store and forward](docs/store-and-forward.md) for complete composition, custody, security, and failure semantics. ### 14. Run compaction and recovery workflows @@ -1403,7 +1200,7 @@ it.layer(NodeCrypto.layer)("replica", (it) => { Every `TestReplica` constructor installs the SQL binding Layers passed in `projections`. Consumer Layers provide only mutation and query handlers. `layer` and `layerWithLimits` use the full durable runtime. `layerWithSync` and -`layerWithSyncAndLimits` use the direct protocol graph for deterministic peer tests. +`layerWithSyncAndLimits` use the lower level sync graph for deterministic peer tests. For sync tests use `TestReplica.layerWithSync`, `TestPeer.layer`, and `FaultInjection.layerSequence`. Fault decisions can deterministically drop, duplicate, delay, reorder, partition, heal, and flush bounded peer traffic. Effect's @@ -1427,36 +1224,34 @@ validate their bounds in the Effect error channel with the tagged `InvalidOption | Presence and tab sessions | Browser process | No | Best effort transport only | Not applicable | | RPC principals, leases, sessions, queues | RPC process scope | No | No | Recreated by authentication and synchronization | | Sender relay outbox | Sender SQLite replica | Yes | No | Removed after relay custody or retry horizon | -| Relay custody and terminal evidence | Relay SQLite authority | Yes | No | Active payload is not reconstructable after expiry | +| Relay custody and terminal evidence | Backend relay SQL store | Yes | No | Active payload is not reconstructable after expiry | | Recipient relay receipts | Recipient SQLite replica | Yes | No | Bounded replay evidence, not canonical CRDT history | ### Boundary guarantees Consistency guarantees: -| Boundary | Guarantee | -| ------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | -| One document command | Serialized through its Cluster entity and committed with canonical state, projections, receipt, sequence, and stored reply | -| Command retry | Within one replica incarnation, the same command ID and canonical request returns the durable result | -| Different command input | Within one replica incarnation, reusing a command ID for different input fails | -| Query | Reads local projection state under the replica operation gate | -| Multi document invariant | Not transactional. Model one aggregate document or an explicit Workflow | -| Peer convergence | Replicas converge after receiving the same valid Automerge change set | -| Cross lineage sync | Refused, never merged. A rewritten document stops synchronizing with every peer that holds the superseded lineage | -| Restore | Exclusive, fenced, staged, schema checked, and projection rebuilding | -| Atom invalidation | Reactive cache refresh, not a durability acknowledgement | -| Presence | Expiring best effort state with no durability guarantee | -| Direct peer send | Local SQL outbox exists before send. A successful send permits local `markSent`. It does not prove remote application | -| Automerge observation | The peer's sync state reports all current local changes observed. It does not prove remote storage durability | -| RPC `Open` handshake | One authenticated, authorized, bounded live session exists for exactly the selected whole documents | -| RPC `Push` success | Bytes were accepted into the current bounded in memory session. No custody, replay, or remote durability is implied | -| Relay `PushRelay` success | The single SQLite relay authority committed the complete envelope and quota reservation before replying | -| Relay recipient ack | The recipient SQL sync workflow and sender scoped receipt committed before the fenced acknowledgement was attempted | -| Relay delivery | At least once within configured retry, expiry, retention, authorization, and capacity boundaries | -| Relay poison bound | Delivery attempts are durable. Reaching `maximumDeliveryAttempts`, default `16`, dead letters the row and erases its payload | -| Effect RPC stream ack | The response chunk was acknowledged by the RPC protocol. It is neither a byte credit nor an application receipt | -| WebSocket frame order | Frames are ordered within one live RFC 6455 connection. Reconnect, replay, authorization, and persistence are separate | -| `durableConfirmation` | Always `false` in the shipped peer transports | +| Boundary | Guarantee | +| ------------------------ | ---------------------------------------------------------------------------------------------------------------------------- | +| One document command | Serialized through its Cluster entity and committed with canonical state, projections, receipt, sequence, and stored reply | +| Command retry | Within one replica incarnation, the same command ID and canonical request returns the durable result | +| Different command input | Within one replica incarnation, reusing a command ID for different input fails | +| Query | Reads local projection state under the replica operation gate | +| Multi document invariant | Not transactional. Model one aggregate document or an explicit Workflow | +| Peer convergence | Replicas converge after receiving the same valid Automerge change set | +| Cross lineage sync | Refused, never merged. A rewritten document stops synchronizing with every peer that holds the superseded lineage | +| Restore | Exclusive, fenced, staged, schema checked, and projection rebuilding | +| Atom invalidation | Reactive cache refresh, not a durability acknowledgement | +| Presence | Expiring best effort state with no durability guarantee | +| Automerge observation | The peer's sync state reports all current local changes observed. It does not prove remote storage durability | +| RPC `Open` handshake | One authenticated, authorized, bounded durable session exists for exactly the selected whole documents | +| RPC `Push` success | The backend relay store committed the complete envelope and quota reservation before replying | +| Relay recipient ack | The recipient SQL sync workflow and sender scoped receipt committed before the fenced acknowledgement was attempted | +| Relay delivery | At least once within configured retry, expiry, retention, authorization, and capacity boundaries | +| Relay poison bound | Delivery attempts are durable. Reaching `maximumDeliveryAttempts`, default `16`, dead letters the row and erases its payload | +| Effect RPC stream ack | The response chunk was acknowledged by the RPC protocol. It is neither a byte credit nor an application receipt | +| WebSocket frame order | Frames are ordered within one live RFC 6455 connection. Reconnect, replay, authorization, and persistence are separate | +| `durableConfirmation` | Always `false` in the shipped peer transports | ### Retry and ambiguity @@ -1549,9 +1344,8 @@ Consistency guarantees: This beta deliberately provides building blocks rather than a complete collaboration product. -- The RPC package provides a live direct protocol and an optional single authority SQLite store and forward relay. It - does not provide a managed backend, peer discovery, replicated relay, account system, credential issuer, or tenant - registry. +- The RPC package provides one durable store and forward protocol. It does not provide a managed backend, peer + discovery, account system, credential issuer, or tenant registry. - The application owns credential issuance and rotation, authenticator and authorization implementations, stable peer identity assignment, tenant routing, TLS, Origin validation, ingress controls, logging policy, and end to end encryption when required. @@ -1566,9 +1360,8 @@ This beta deliberately provides building blocks rather than a complete collabora - Automerge `3.3.2` does not expose allocation bounded semantic decode. Relay policy must deny `authorizeUnsafeUnboundedAutomerge3Decode` unless the application controls and resource trusts the producer bytes. Ordinary authentication and document authorization are insufficient. The exact grant binds principal, remote, - direction, documents, finite expiry, and revocation. Direct protocol version `2` retains the preexisting authorized - peer allocation risk. A future allocation bounded Automerge API should replace this exception and remove the unsafe - grant. + direction, documents, finite expiry, and revocation. A future allocation bounded Automerge API should replace this + exception and remove the unsafe grant. - Relay revocation is not atomic with SQLite. It does not retroactively cancel an operation that already won the local gate. That bounded operation may finish its durable commit or delivery and returns its real result. Revocation drains it and prevents later operations. If revocation wins the gate first, no SQL mutation or payload emission occurs. @@ -1578,8 +1371,8 @@ This beta deliberately provides building blocks rather than a complete collabora - A missing session and a session owned by another tenant, subject, or peer both return the same fieldless `SessionUnavailable`. Applications must not wrap this result with ownership or existence detail because that creates a session enumeration oracle. -- `PeerRpcLimits.defaults` are conservative library defaults, not product capacity planning. Configure them with core - `ReplicaLimits`, process memory, expected document size, ingress limits, and subject quotas. +- `PeerRelayLimits.defaults` are conservative library defaults, not product capacity planning. Configure them with + process memory, expected document size, ingress limits, database capacity, and subject quotas. - OPFS is origin scoped. Browser storage starts as best effort, can be evicted unless persistence is granted, and is deleted when the user clears the site's storage. - Existing secondary tabs do not yet promote themselves when the provisioning tab disappears. A new attachment can @@ -1594,9 +1387,8 @@ This beta deliberately provides building blocks rather than a complete collabora but stale peer from silently resurrecting discarded history. It is not evidence about the peer and must not be used as one. A refusal is not proof that the local document is stale. Confirm locally that a rewrite ran first. A forged lineage costs the forger that one document in that one session and nothing else. -- Direct `RpcPeerTransport` reports store and forward as false. The separate - `RpcPeerTransport.layerStoreAndForward` and `makeStoreAndForwardSession` constructors report it as true only after - the version `3` relay handshake. +- `RpcPeerTransport.layer` and `makeSession` expose only the durable acknowledged topology after the version `1` + handshake. - Conflict inspection, history browsing, sharing policy, and resolution UI belong to the application. - Presence is not durable awareness and must not carry authorization decisions. - Limits must be selected for the product. They bound backup bytes, archive records, JSON depth, sync messages, @@ -1611,11 +1403,11 @@ This beta deliberately provides building blocks rather than a complete collabora | Owner | Required responsibility | | ---------------------- | ----------------------------------------------------------------------------------------------------------------- | -| Application | Credentials, authentication, direct and relay authorization, tenant mapping, stable peer IDs, reconnect policy | +| Application | Credentials, authentication, relay authorization, tenant mapping, stable peer IDs, reconnect policy | | Effect RPC composition | `RpcClient.Protocol`, `RpcServer.Protocol`, serialization, request and stream lifecycle | | Platform | Socket and WebSocket implementation, HTTP server, TLS, DNS, process signals | | Ingress | Origin policy, connection limits, upgrade timeout, maximum frame and request bytes, load shedding | -| Effect Local RPC | Direct version `2`, relay version `3`, required auth middleware, bounded sessions, custody adapter, and limits | +| Effect Local RPC | Durable version `1`, required auth middleware, bounded sessions, custody adapter, and limits | | SQL replica | Canonical Automerge state, durable peer outbox and receipts, fencing, recovery, projections | | Operations | Secret rotation, telemetry redaction, capacity configuration, backup policy, incident response, graceful shutdown | @@ -1681,15 +1473,15 @@ Every root package exports module namespaces. Every module is also available thr The export surface supports several assembly levels. Most applications start with everyday domain and composition modules. Feature and advanced services remain public for products that need direct control. -| Level | Modules | -| --------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Everyday domain | `Document`, `DocumentSet`, `Mutation`, `Projection`, `Query`, `ReplicaDefinition`, `Replica`, `CommandOutcome`, `Snapshot`, `Identity`, `ReplicaStatus`, `Backup` | -| Durable runtime | `SqlProjection`, `SqlReplica`, `BrowserReplica`, `BrowserSqlite` | -| Reactive and test adapters | `ReplicaAtom`, `TestReplica` | -| Optional features | `PeerTransport`, `PeerSession`, `Presence`, `ReplicaWorkflow`, `TestPeer`, `FaultInjection`, `PeerRpc`, `PeerRelayRpc`, `RpcPeerTransport` | -| RPC policy and server | `PeerCredentials`, `PeerAuthenticator`, `PeerAuthentication`, `PeerAuthorization`, `PeerRelayAuthorization`, `PeerRpcLimits`, `PeerRelayLimits`, `PeerRelayIngress`, `PeerRelayStore`, `PeerRpcServer`, `PeerRpcError` | -| Advanced assembly | `CommitPublisher`, `Compaction`, `Recovery`, `PeerSync`, `DurableRuntime`, `ReplicaClient`, `ReplicaOwner`, `SessionManager` | -| Advanced framework assembly | `CommandExecutor`, `DocumentEntity`, `DocumentStore`, `EntityReplica`, `Migrations`, `ProjectionStore`, `QueryExecutor`, `ReplicaBootstrap`, `ReplicaGate`, `ReplicaRpc` | +| Level | Modules | +| --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Everyday domain | `Document`, `DocumentSet`, `Mutation`, `Projection`, `Query`, `ReplicaDefinition`, `Replica`, `CommandOutcome`, `Snapshot`, `Identity`, `ReplicaStatus`, `Backup` | +| Durable runtime | `SqlProjection`, `SqlReplica`, `BrowserReplica`, `BrowserSqlite` | +| Reactive and test adapters | `ReplicaAtom`, `TestReplica` | +| Optional features | `PeerTransport`, `PeerSession`, `Presence`, `ReplicaWorkflow`, `TestPeer`, `FaultInjection`, `PeerRpc`, `RpcPeerTransport` | +| RPC policy and server | `PeerCredentials`, `PeerAuthenticator`, `PeerAuthentication`, `PeerRelayAuthorization`, `PeerRelayLimits`, `PeerRelayIngress`, `PeerRelayStore`, `SqlPeerRelayStore`, `PeerRpcServer`, `PeerRpcError` | +| Advanced assembly | `CommitPublisher`, `Compaction`, `Recovery`, `PeerSync`, `DurableRuntime`, `ReplicaClient`, `ReplicaOwner`, `SessionManager` | +| Advanced framework assembly | `CommandExecutor`, `DocumentEntity`, `DocumentStore`, `EntityReplica`, `Migrations`, `ProjectionStore`, `QueryExecutor`, `ReplicaBootstrap`, `ReplicaGate`, `ReplicaRpc` | These public framework assembly modules support custom runtimes and diagnostics. Paths under `internal/*` remain private and unsupported. The public assembly modules are not required for the normal application path shown in the @@ -1962,14 +1754,14 @@ Do not expose the browser owner protocol as an internet peer API. | `TestReplica` | `defaultLimits`, `layerWithLimits`, `layer`, `layerWithSyncAndLimits`, `layerWithSync` | `TestReplica.layer` and `layerWithLimits` run the full Cluster and Workflow production path against in memory Node -SQLite. `layerWithSync` and `layerWithSyncAndLimits` expose the lower level direct protocol graph. All constructors +SQLite. `layerWithSync` and `layerWithSyncAndLimits` expose the lower level sync graph. All constructors still require the domain's generated mutation and query handler services and install the supplied SQL projection bindings. `FaultInjection.Decision` controls drop, finite copy count, finite nonnegative delay, and pairwise reorder. A `layerSequence` repeats its final decision for later packets. `TestPeer` validates queue capacity, maximum copies, and maximum delay. Its methods connect peers, partition, heal, and flush traffic. `transportLayer` adapts one peer to core -`PeerTransport`; the resulting capability has no store and forward guarantee. +`PeerTransport` with the same acknowledged delivery shape used by production transports. `TestReplica.defaultLimits` is the only exported `ReplicaLimits` preset. It is intended for tests, not production capacity planning: @@ -1987,170 +1779,29 @@ planning: ### `@lucas-barake/effect-local-rpc` -| Namespace | Public API | -| ------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `PeerAuthentication` | `PeerPrincipal` schema and type, request service `AuthenticatedPeer`, required RPC middleware `PeerAuthentication`, argument free `layerServer`, argument free `layerClient` | -| `PeerAuthenticator` | `PeerAuthenticator` service. `authenticate(Redacted)` returns principal, finite `validUntil`, and `invalidated`, or `AuthenticationFailure` | -| `PeerAuthorization` | `PeerAuthorization` service and validating constructor `layer`. `authorize` returns exactly resolved `SelectedDocument` values, finite `validUntil`, and `invalidated`, or `AccessDenied` or `ServerUnavailable` | -| `PeerCredentials` | `PeerCredentials` service. `get` is an Effect that returns a rotating `Redacted` or `AuthenticationFailure` | -| `PeerRelayAuthorization` | `Direction`, `RemotePeer`, ordinary request and result contracts, `UnsafeUnboundedAutomerge3DecodeRequest`, `UnsafeUnboundedAutomerge3DecodeGrant`, `unsafeUnboundedAutomerge3DecodeRisk`, `AuthorizeUnsafeUnboundedAutomerge3Decode`, `denyUnsafeUnboundedAutomerge3Decode`, `PeerRelayAuthorization` service, validating two callback `layer` | -| `PeerRelayIngress` | `Usage`, `Reservation`, `PeerRelayIngress` service, server `layerProtocolSocketServer`, client `makeProtocolSocket` and `layerProtocolSocket` | -| `PeerRelayLimits` | `Values`, `defaults`, `InvalidPeerRelayLimits`, `PeerRelayLimits` service, `make`, `layer`, `layerDefaults` | -| `PeerRelayRpc` | Protocol version `3`, relay event schemas, four RPC definitions, `Rpcs`, generated `RpcClient`, `makeRpcClient` | -| `PeerRelayStore` | Relay channel, admission, claim, terminal, maintenance, and usage contracts, `PeerRelayStore` service, `make`, `layerSqlite` | -| `PeerRpc` | `protocolVersion`, `DefinitionHash`, `RequestedDocument`, `Opened`, `Message`, `OpenEvent`, `OpenRpc`, `PushRpc`, `Rpcs`, generated `RpcClient`, `makeRpcClient` | -| `PeerRpcError` | `AuthenticationFailure`, `AccessDenied`, `UnsupportedVersion`, `PeerMismatch`, `DefinitionMismatch`, `InvalidRequest`, `RequestLimitExceeded`, `RequestCapacityExceeded`, `SessionUnavailable`, `SessionOverloaded`, `ServerUnavailable`, `DocumentLineageChanged`, union schema and type `PeerRpcError`, fixed `Defect` schema | -| `PeerRpcLimits` | `Values` schema and type, `defaults`, `InvalidPeerRpcLimits`, `PeerRpcLimits` service, `make`, `layer`, `layerDefaults` | -| `PeerRpcServer` | Direct `layerHandlers`, relay `layerRelayHandlers` and `layerRelayServer`, `layerStoreAndForwardDeployment`, `PeerRelayServerRuntime` | -| `RpcPeerTransport` | Direct `layer` and `makeSession`, relay `StoreAndForwardOptions`, `layerStoreAndForward`, `makeStoreAndForwardSession`, `isRetryable` | - -#### RPC procedure contract - -| Procedure | Request | Success | Typed errors | Lifecycle | -| --------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------- | -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `Open` | `protocolVersion`, `expectedPeerId`, `definitionHash`, unique requested whole documents, optional client `capabilities`, middleware owned credential | Stream beginning with one `Opened`, followed only by `Message` events | Every `PeerRpcError` | Stream interruption is connection close. One active incarnation exists per authenticated peer | -| `Push` | Current `sessionId`, bounded byte payload, middleware owned credential | `void` after bounded in memory admission | Every `PeerRpcError` | Authenticated identity, current authorized session, ownership, lease, and limits are checked. Inbound overflow or any outbound capacity timeout revokes the session and fails `Open` with `SessionOverloaded` | - -Protocol version `2` requires a canonical `definitionHash` in the `def_` plus 16 lowercase hexadecimal format. Version -`1` requests remain decodable only so the server can reject them with `UnsupportedVersion`. `Opened` contains the -negotiated version, new `SessionId`, server `PeerId`, and `capabilities`. Server `capabilities` are -`{ storeAndForward: false, lineageAware: true }`. `lineageAware` is an optional key on the schema, so an `Opened` frame -from a build that predates it still decodes and reads as not lineage aware. The `Open` request carries the mirrored -optional client `capabilities: { lineageAware }`, which is how the server learns the same fact about the caller. -`Message` contains one `Uint8Array`. `OpenRpc` and `PushRpc` both carry the -required `PeerAuthentication` middleware. `makeRpcClient` requires `RpcClient.Protocol`, client authentication -middleware, and `Scope`. It deliberately does not return a package owned connection service. - -Relay procedure contract: - -| Procedure | Request | Success | Boundary | -| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `OpenRelay` | Version `3`, expected relay and local principals, sender incarnation, exact remote subject and peer, whole documents, receipt retention, sender retry horizon, middleware owned credential | Stream beginning with `RelayOpened`, followed by `StoredMessage` values | Authentication and ordinary send or receive authorization bind the exact endpoint and documents. Stream interruption releases live claims for bounded retry | -| `PushRelay` | Current `sessionId`, stable `relayMessageId`, bounded payload, middleware owned credential | `void` after SQLite custody commits | Admission also requires the exact unsafe Automerge resource trust grant. Duplicate matching admission is safe. Conflicting identity reuse and failures are typed | -| `AcknowledgeRelay` | Current `sessionId`, `relayMessageId`, `claimToken`, `messageHash`, middleware owned credential | `void` after a conditional terminal transition | Only the exact live recipient claim can acknowledge. Duplicate matching acknowledgement is safe | -| `RejectRelay` | The acknowledgement identity plus `ProtocolInvalid` or `ApplicationRejected` | `void` after a conditional dead letter transition | Permanent rejection erases the active payload and retains bounded terminal evidence | - -Every relay procedure can fail with a fieldless `PeerRpcError`. Delivery of a stored message also requires the exact -unsafe Automerge resource trust grant for `Receive`. `RelayOpened` reports the authenticated local -principal, exact remote peer, a new `SessionId`, version `3`, and -`capabilities: { storeAndForward: true }`. `StoredMessage` carries the complete delivery identity and an opaque claim -token. `PeerRelayRpc.makeRpcClient` requires the custom `PeerRelayIngress.layerProtocolSocket` client protocol, -client authentication middleware, and `Scope`. - -RPC environment requirements: - -| Constructor or Layer | Required Effect services | -| ----------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `PeerAuthentication.layerClient` | `PeerCredentials` | -| `PeerAuthentication.layerServer` | `PeerAuthenticator`, `PeerRpcLimits` | -| `PeerAuthorization.layer(authorize)` | None. The callback must return an environment free Effect. Build `PeerAuthorization` with an application `Layer.effect` instead when authorization needs other services | -| `PeerRpcLimits.make`, `layer`, `layerDefaults` | Core `ReplicaLimits` | -| `PeerRpcServer.layerHandlers` | `CommitPublisher`, `PeerRpcLimits`, core `ReplicaLimits`, `PeerAuthorization`, `Crypto`, `PeerSync`, `ReplicaGate`, `Sharding` | -| `PeerRpc.makeRpcClient` | `RpcClient.Protocol`, required client `PeerAuthentication` middleware, `Scope` | -| `RpcPeerTransport.layer` | `Scope` for each connection acquisition | -| `RpcPeerTransport.makeSession` | `Scope`, `CommitPublisher`, `Crypto`, `PeerSync`, `ReplicaGate`, core `ReplicaLimits`, `Sharding`; the adapter supplies `PeerTransport` | -| `PeerRelayAuthorization.layer(authorize, authorizeUnsafeUnboundedAutomerge3Decode)` | None. Use `denyUnsafeUnboundedAutomerge3Decode` by default. Use an application `Layer.effect` when either policy needs other services | -| `PeerRelayLimits.make`, `layer`, `layerDefaults` | None | -| `PeerRelayStore.layerSqlite` | `SqlClient`, `Crypto`, `PeerRelayLimits` | -| `PeerRelayIngress.layerProtocolSocketServer` | `PeerRelayLimits` plus the requirements of the supplied `SocketServer` Layer | -| `PeerRpcServer.layerRelayHandlers` | `Crypto`, `PeerRelayAuthorization`, `PeerRelayIngress`, `PeerRelayLimits`, `PeerRelayStore` | -| `PeerRpcServer.layerRelayServer` | Use with `layerRelayHandlers` and `PeerRelayIngress`. Together they require `PeerAuthentication` and the relay `RpcServer.Protocol` | -| `PeerRpcServer.layerStoreAndForwardDeployment` | Requirements of the supplied direct deployment and relay socket Layer, plus `PeerAuthentication`, `Crypto`, `PeerRelayAuthorization`, `PeerRelayLimits`, `PeerRelayStore` | -| `PeerRelayRpc.makeRpcClient` | Custom relay `RpcClient.Protocol`, required client `PeerAuthentication` middleware, `Scope` | -| `RpcPeerTransport.layerStoreAndForward` | `Crypto`, `PeerRelayClientRuntime`; `Scope` for each connection acquisition | -| `RpcPeerTransport.makeStoreAndForwardSession` | `Scope`, `CommitPublisher`, `Crypto`, relay enabled `PeerSync`, `ReplicaGate`, core `ReplicaLimits`, `Sharding`, `PeerRelayClientRuntime` | - -`PeerPrincipal` is `{ tenantId, subjectId, peerId }`. `RequestedDocument` is `{ documentType, documentId }`. Every -`PeerRpcError` class is a fieldless tagged error, including `DocumentLineageChanged`. A peer therefore learns that -its view of a document lineage is stale, never which document and never the lineage values, because reflecting -those back would attach document identity to the RPC boundary. `InvalidPeerRpcLimits` is separate from the wire -error union and carries `field` for relational configuration failures. Middleware supplies the redacted -`credential` field on every request. Callers must not treat a payload supplied credential as authoritative because -`layerClient` overwrites it. - -#### RPC limits contract - -`PeerRpcLimits.Values` contains these required fields: - -| Category | Fields | -| ----------------- | ----------------------------------------------------------------------------------------------- | -| Session and queue | `maxSessionsPerSubject`, `inboundItemCapacity`, `outboundItemCapacity` | -| Byte memory | `maxInboundBufferedBytesPerSession`, `maxOutboundBufferedBytesPerSession`, `maxBufferedBytes` | -| Authentication | `maxInFlightAuthentication`, `authenticationRatePerSecond`, `authenticationBurst` | -| Open admission | `maxInFlightOpen`, `maxInFlightOpenPerSubject`, `openRatePerSecond`, `openBurst` | -| Push admission | `maxInFlightPush`, `maxInFlightPushPerSubject`, `pushRatePerSecond`, `pushBurst` | -| Retained state | `maxRetainedRateLimitedConnections`, `maxRetainedRateLimitedSubjects`, `rateLimitIdleRetention` | -| Lease and workers | `maximumReauthorizationInterval`, `commitFlushConcurrency`, `shutdownCleanupConcurrency` | - -Every scalar is finite and positive. Integer fields must be positive integers. `make` also checks the relational byte -constraints against core `ReplicaLimits.maxSyncMessageBytes`. Scalar decode failures preserve Effect Schema's issue -tree. Relational failures use `InvalidPeerRpcLimits` with the failing field. The defaults are inspectable through -`PeerRpcLimits.defaults` and installable with `layerDefaults`. - -`PeerRpcLimits.defaults` has these exact values: - -| Category | Exact defaults | -| ------------------- | -------------------------------------------------------------------------------------------------------------- | -| Session and items | `maxSessionsPerSubject = 4`, `inboundItemCapacity = 1`, `outboundItemCapacity = 1` | -| Byte memory | Per session inbound `4 MiB`, per session outbound `4 MiB`, total `64 MiB` | -| Authentication | `maxInFlightAuthentication = 64`, `authenticationRatePerSecond = 16`, `authenticationBurst = 32` | -| Open | `maxInFlightOpen = 16`, per subject `2`, rate per second `2`, burst `4` | -| Push | `maxInFlightPush = 128`, per subject `8`, rate per second `64`, burst `128` | -| Retained rate state | Connections `10_000`, subjects `10_000`, idle retention `600_000 ms` | -| Policy and workers | Maximum reauthorization interval `300_000 ms`, commit flush concurrency `8`, shutdown cleanup concurrency `16` | - -#### Relay limits contract - -`PeerRelayLimits.Values` separately bounds relay storage, framing, sessions, byte reservations, rate state, terminal -responses, channel queues, workers, SQL work classes, maintenance, claim leases, retries, and shutdown. Its -`maximumDeliveryAttempts` is a positive integer and defaults to `16`. Each failed, interrupted, disconnected, or -expired claim advances the durable count. Reaching the cap moves the row to `DeadLettered`, erases its payload, and -retains only bounded terminal deduplication evidence. - -#### RPC observability contract - -RPC spans are `effect_local_rpc.authentication`, `effect_local_rpc.server.open`, -`effect_local_rpc.server.push`, `effect_local_rpc.adapter.open`, and `effect_local_rpc.adapter.push`. Every span has the -fixed `rpc.operation` and `rpc.result` attributes. Open spans may add finite `rpc.selected_documents`. Push spans may add -finite `rpc.payload_bytes`. - -The fixed operation vocabulary is `Authentication`, `Open`, `Push`, `AdapterOpen`, `AdapterPush`, `Inbound`, `Outbound`, -and `Server`. The fixed result vocabulary is `Attempt`, `Success`, `AuthenticationDenied`, `AuthorizationDenied`, -`ProtocolRejected`, `CapacityRejected`, `Overloaded`, `Failure`, `Replaced`, and `ShutdownClosed`. - -| Metric | Attributes and observation | -| ------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | -| `effect_local_rpc_boundary_total` | Counter with fixed `operation` and `result` | -| `effect_local_rpc_active_sessions` | Gauge without identity attributes | -| `effect_local_rpc_queue_items` | Gauge with fixed `operation: Inbound \| Outbound` | -| `effect_local_rpc_message_bytes` | Histogram with fixed `operation: Inbound \| Outbound` and boundaries `0, 64, 256, 1_024, 4_096, 16_384, 65_536, 262_144, 1_048_576` | -| `effect_local_rpc_selected_documents` | Histogram with fixed `operation: Open` and boundaries `0, 1, 2, 4, 8, 16, 32, 64, 128` | - -The library clears ambient metric attributes before updating these instruments. The library does not attach credential, -tenant, subject, peer, session, or document identity, document content, raw payload bytes, or unbounded failure text to -these spans or metrics. Applications and exporters must preserve the same exclusion. - -## Architecture evidence - -The repository is pinned to Effect `4.0.0-beta.99` at commit -[`eb9b10256`](https://github.com/Effect-TS/effect/tree/eb9b10256c8558881b441c2fef833b7037174400) and Automerge -`3.3.2` at commit -[`b4a1bbe9`](https://github.com/automerge/automerge/tree/b4a1bbe9fc17d26c4d3f1819f9ee3b318de3a516). -The following references are the exact upstream contracts used by this implementation. - -| Design contract | Version matched source | Consequence in Effect Local | -| ----------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | -| Services and construction | Effect [`Context.Service`](https://github.com/Effect-TS/effect/blob/eb9b10256c8558881b441c2fef833b7037174400/packages/effect/src/Context.ts#L200-L259), [`Layer`](https://github.com/Effect-TS/effect/blob/eb9b10256c8558881b441c2fef833b7037174400/packages/effect/src/Layer.ts#L974-L1064), and [`Scope`](https://github.com/Effect-TS/effect/blob/eb9b10256c8558881b441c2fef833b7037174400/packages/effect/src/Scope.ts#L366-L480) | Long lived application capabilities are services. Resource constructors are scoped Layers or Effects. Cleanup follows scope ownership | -| Durable command execution | Effect Cluster [`Persisted`, `WithTransaction`, and `Uninterruptible`](https://github.com/Effect-TS/effect/blob/eb9b10256c8558881b441c2fef833b7037174400/packages/effect/src/unstable/cluster/ClusterSchema.ts#L26-L115) and [entity transaction handling](https://github.com/Effect-TS/effect/blob/eb9b10256c8558881b441c2fef833b7037174400/packages/effect/src/unstable/cluster/internal/entityManager.ts#L205-L223) | Document commands serialize through entities and durable replies share the configured SQL transaction boundary | -| Durable maintenance | Effect [`Workflow`](https://github.com/Effect-TS/effect/blob/eb9b10256c8558881b441c2fef833b7037174400/packages/effect/src/unstable/workflow/Workflow.ts#L429-L500) and [`WorkflowEngine`](https://github.com/Effect-TS/effect/blob/eb9b10256c8558881b441c2fef833b7037174400/packages/effect/src/unstable/workflow/WorkflowEngine.ts#L37-L107) | Compaction can resume activities without changing backup or command durability semantics | -| Reactive invalidation | Effect [`Reactivity`](https://github.com/Effect-TS/effect/blob/eb9b10256c8558881b441c2fef833b7037174400/packages/effect/src/unstable/reactivity/Reactivity.ts#L41-L80) | Commit publication invalidates rebuildable views. The notification is not canonical state | -| RPC naming and ownership | Effect Cluster's generated [`Rpcs`, `RpcClient`, and `makeRpcClient`](https://github.com/Effect-TS/effect/blob/eb9b10256c8558881b441c2fef833b7037174400/packages/effect/src/unstable/cluster/Runners.ts#L472-L526) and Effect [`RpcClient.make`](https://github.com/Effect-TS/effect/blob/eb9b10256c8558881b441c2fef833b7037174400/packages/effect/src/unstable/rpc/RpcClient.ts#L620-L640) | The RPC package exports the contract and generated client directly. It does not wrap the connection in a parallel client abstraction | -| RPC middleware | Effect RPC [required client middleware fixture](https://github.com/Effect-TS/effect/blob/eb9b10256c8558881b441c2fef833b7037174400/packages/platform-node/test/fixtures/rpc-schemas.ts#L9-L31) | `PeerAuthentication` is ordinary required RPC middleware. `AuthenticatedPeer` is request scoped context | -| WebSocket ownership | Effect [`RpcServer.layerProtocolWebsocket`](https://github.com/Effect-TS/effect/blob/eb9b10256c8558881b441c2fef833b7037174400/packages/effect/src/unstable/rpc/RpcServer.ts#L927-L960) and its [Node integration test](https://github.com/Effect-TS/effect/blob/eb9b10256c8558881b441c2fef833b7037174400/packages/platform-node/test/RpcServer.test.ts#L54-L75) | Applications compose protocol, router, serialization, platform socket, and HTTP server around `PeerRpc.Rpcs` | -| Stream acknowledgements | Effect RPC [response stream acknowledgement path](https://github.com/Effect-TS/effect/blob/eb9b10256c8558881b441c2fef833b7037174400/packages/effect/src/unstable/rpc/RpcServer.ts#L394-L430) | Acknowledgements bound response chunks but do not claim byte credit, remote Automerge message application, or persistence | -| Queue overflow | Effect [`Queue.dropping`](https://github.com/Effect-TS/effect/blob/eb9b10256c8558881b441c2fef833b7037174400/packages/effect/src/Queue.ts#L538-L563) | Bounded admission reports overload instead of growing memory or silently blocking every handler. Inbound overflow terminates that session | -| Canonical document operations | Automerge JavaScript [`init` and `free`](https://github.com/automerge/automerge/blob/b4a1bbe9fc17d26c4d3f1819f9ee3b318de3a516/javascript/src/implementation.ts#L232-L353), [`change`](https://github.com/automerge/automerge/blob/b4a1bbe9fc17d26c4d3f1819f9ee3b318de3a516/javascript/src/implementation.ts#L433-L521), [`load`](https://github.com/automerge/automerge/blob/b4a1bbe9fc17d26c4d3f1819f9ee3b318de3a516/javascript/src/implementation.ts#L704-L807), [`save`](https://github.com/automerge/automerge/blob/b4a1bbe9fc17d26c4d3f1819f9ee3b318de3a516/javascript/src/implementation.ts#L809-L864), [`applyChanges`](https://github.com/automerge/automerge/blob/b4a1bbe9fc17d26c4d3f1819f9ee3b318de3a516/javascript/src/implementation.ts#L987-L1054), and [`getHeads`](https://github.com/automerge/automerge/blob/b4a1bbe9fc17d26c4d3f1819f9ee3b318de3a516/javascript/src/implementation.ts#L1279-L1291) | Create, change, save, load, replay, heads, and freeing remain Automerge operations behind the SQL adapter | -| Sync state lifetime | Automerge [`sync::State`](https://github.com/automerge/automerge/blob/b4a1bbe9fc17d26c4d3f1819f9ee3b318de3a516/rust/automerge/src/sync/state.rs#L45-L78) and JavaScript [sync API](https://github.com/automerge/automerge/blob/b4a1bbe9fc17d26c4d3f1819f9ee3b318de3a516/javascript/src/implementation.ts#L1143-L1250) | Sent hashes, in flight state, and peer knowledge are connection state. Reconnect creates a fresh sync state while persisted history drives convergence | +| Module | Public contract | +| ------------------------ | ---------------------------------------------------------------------------------------------------------- | +| `PeerAuthentication` | Required client and server middleware for every relay request | +| `PeerAuthenticator` | Application credential verification service | +| `PeerCredentials` | Client credential provider | +| `PeerRelayAuthorization` | Exact send and receive authorization plus the explicit Automerge resource trust grant | +| `PeerRelayIngress` | Bounded server and client socket protocol | +| `PeerRelayLimits` | Authentication, ingress, custody, session, worker, quota, retry, and maintenance limits | +| `PeerRpc` | Durable protocol version `1`, `Open`, `Push`, `Acknowledge`, `Reject`, events, group, and generated client | +| `PeerRelayStore` | Injectable semantic custody contract | +| `SqlPeerRelayStore` | Generic `SqlClient` implementation with `make` and `layer` for SQLite, PostgreSQL, and MySQL | +| `PeerRpcServer` | `layerHandlers`, `layerServer`, and `PeerRpcServerRuntime` | +| `RpcPeerTransport` | Durable `Options`, `layer`, `makeSession`, and `isRetryable` | + +The `Open` stream starts with `Opened` and then emits `StoredMessage` values. `Push` succeeds only after durable +custody. `Acknowledge` and `Reject` use the exact claim token and message hash. Every procedure can fail with a +fieldless `PeerRpcError`. + +`PeerAuthentication.layerClient` requires `PeerCredentials`. +`PeerAuthentication.layerServer` requires `PeerAuthenticator` and `PeerRelayLimits`. +`PeerRpcServer.layerHandlers` requires `Crypto`, `PeerRelayAuthorization`, `PeerRelayIngress`, +`PeerRelayLimits`, and `PeerRelayStore`. `SqlPeerRelayStore.layer` requires `SqlClient`, `Crypto`, and +`PeerRelayLimits`. `RpcPeerTransport.layer` additionally uses `PeerRelayClientRuntime`. ### Research basis diff --git a/docs/architecture.md b/docs/architecture.md index e751c83..e15f933 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -39,33 +39,31 @@ for reconstructing the session in place. 4. Workflow journals are durable orchestration records. 5. Atom values are reactive caches. 6. Presence and tab sessions are ephemeral. -7. Optional sender relay outbox rows are temporary durable transfer state. -8. Optional relay custody rows are temporary durable delivery state owned by one SQLite authority per shard. +7. Sender relay outbox rows are temporary durable transfer state in frontend SQLite. +8. Relay custody rows are temporary durable delivery state in the configured backend SQL database. Cluster and Workflow do not replace Automerge. They serialize local effects, resolve retry ambiguity, and resume operations after process loss. Automerge remains responsible for causal history, conflicts, and convergence. ## Relay topology -The direct RPC deployment and optional store and forward deployment have separate protocols and listeners. Direct -version `2` continues through the application owned protocol and `PeerRpcServer.layerHandlers`. Relay version `3` -uses `PeerRelayRpc.Rpcs`, the bounded `PeerRelayIngress` socket protocol, and a distinct socket Layer through -`PeerRpcServer.layerStoreAndForwardDeployment`. +Effect RPC uses one durable protocol. Version `1` uses `PeerRpc.Rpcs`, the bounded `PeerRelayIngress` socket +protocol, `PeerRpcServer.layerHandlers`, and `PeerRpcServer.layerServer`. ```mermaid flowchart LR Sender["Sender SQL replica"] --> Outbox["Stable sender relay outbox"] - Outbox --> RelayListener["Relay protocol v3 listener"] - RelayListener --> Custody["Single authority relay SQLite"] + Outbox --> RelayListener["Relay protocol v1 listener"] + RelayListener --> Custody["Injected backend SQL custody"] Custody --> RecipientListener["Recipient relay session"] RecipientListener --> Receipt["Recipient SQL sync and receipt"] Receipt --> Ack["Fenced relay acknowledgement"] Ack --> Custody ``` -One relay SQLite database is authoritative for one shard. The application must route every custody write and claim -for that shard to one process and one durable volume. This topology does not include leader election, quorum -replication, automatic failover, shard rebalance, network file system sharing, or split brain recovery. +The application must route every custody write and claim for one shard to the same logical SQL database. The supplied +`SqlClient` may target SQLite, PostgreSQL, or MySQL. This topology does not itself add leader election, quorum +replication, automatic failover, shard rebalance, or split brain recovery beyond the selected database. Relay ordering is FIFO within one exact directed peer channel. Distinct channels can progress concurrently. Claims are finite fences. A stale claim may duplicate delivery, but its old token cannot retire a newer claim. See diff --git a/docs/durability.md b/docs/durability.md index 0d10a6f..7dc68c6 100644 --- a/docs/durability.md +++ b/docs/durability.md @@ -141,7 +141,7 @@ Store and forward adds three different durable boundaries. 1. Sender admission commits a stable relay message identity and complete outer envelope to the local relay outbox before a network attempt. -2. `PushRelay` succeeds only after the relay SQLite authority commits the envelope and immutable quota reservation. +2. `Push` succeeds only after the backend relay store commits the envelope and immutable quota reservation. 3. `AcknowledgeRelay` is attempted only after the recipient commits the Automerge application result and sender scoped replay receipt through the production sync path. diff --git a/docs/store-and-forward.md b/docs/store-and-forward.md index cead894..d08b84c 100644 --- a/docs/store-and-forward.md +++ b/docs/store-and-forward.md @@ -2,12 +2,11 @@ ## Contract -Store and forward is an optional relay mode for SQL replicas. It adds durable sender admission, durable relay -custody, reconnect delivery, and durable recipient receipts. Direct RPC remains protocol version `2` with -`capabilities: { storeAndForward: false }`. Relay RPC uses protocol version `3` on a distinct listener and reports -`capabilities: { storeAndForward: true }`. +Store and forward is the RPC synchronization topology for SQL replicas. Protocol version `1` adds durable sender +admission, durable relay custody, reconnect delivery, and durable recipient receipts. There is no separate direct +protocol or topology. -The delivery guarantee is at least once. A successful relay `PushRelay` means the relay SQLite authority committed +The delivery guarantee is at least once. A successful `Push` means the relay SQL authority committed the complete envelope before replying. A recipient can therefore see a duplicate after a lost response, expired claim, interrupted acknowledgement, or reconnect. The recipient SQL path makes that duplicate safe within the negotiated receipt retention window. Effect Local does not claim exactly once delivery. @@ -15,28 +14,25 @@ negotiated receipt retention window. Effect Local does not claim exactly once de Store and forward does not make the relay authoritative for document contents. Each replica remains authoritative for its local writes. Automerge remains responsible for convergence after valid changes reach a replica. -## Opt in composition +## Composition -The application must opt in on both sides. It must also keep the direct and relay transports separate. - -On the server, `PeerRpcServer.layerHandlers` remains the direct version `2` handler Layer. The application continues -to own its direct `RpcServer.Protocol`, serializer, socket, and listener. The optional -`PeerRpcServer.layerStoreAndForwardDeployment` merges that unchanged direct deployment with a relay deployment built -from `PeerRelayRpc.Rpcs`, `PeerRelayIngress`, and a different `SocketServer` Layer. +The application supplies relay policy, custody, limits, authentication, SQL, and a socket. The server uses +`PeerRpcServer.layerHandlers` with `PeerRpc.Rpcs`, plus `PeerRpcServer.layerServer` and `PeerRelayIngress` for the +bounded listener. On the client, `SqlReplica.layerRelay` installs relay receipt support. `PeerRelayOutbox.layerSql` stores stable sender admissions. `PeerRelayClientRuntime.layer` supervises sender outbox and receipt maintenance. -`RpcPeerTransport.makeStoreAndForwardSession` binds the generated relay client to the ordinary `PeerSession`. +`RpcPeerTransport.makeSession` binds the generated relay client to the ordinary `PeerSession`. -This example shows the relay specific composition. The named application values provide the existing SQL, Crypto, -authentication, direct RPC, socket, definition, projection, and mutation or query Layers. +This example shows the server and client composition. The named application values provide SQL, Crypto, +authentication, socket, definition, projection, and mutation or query Layers. ```ts import * as PeerRelayAuthorization from "@lucas-barake/effect-local-rpc/PeerRelayAuthorization" import * as PeerRelayLimits from "@lucas-barake/effect-local-rpc/PeerRelayLimits" -import * as PeerRelayStore from "@lucas-barake/effect-local-rpc/PeerRelayStore" import * as PeerRpcServer from "@lucas-barake/effect-local-rpc/PeerRpcServer" import * as RpcPeerTransport from "@lucas-barake/effect-local-rpc/RpcPeerTransport" +import * as SqlPeerRelayStore from "@lucas-barake/effect-local-rpc/SqlPeerRelayStore" import * as PeerRelayClientRuntime from "@lucas-barake/effect-local-sql/PeerRelayClientRuntime" import * as PeerRelayOutbox from "@lucas-barake/effect-local-sql/PeerRelayOutbox" import * as PeerRelayOutboxLimits from "@lucas-barake/effect-local-sql/PeerRelayOutboxLimits" @@ -45,17 +41,16 @@ import * as SqlReplica from "@lucas-barake/effect-local-sql/SqlReplica" import { Effect, Layer } from "effect" const RelayLimitsLive = PeerRelayLimits.layer(relayLimits) -const RelayStoreLive = PeerRelayStore.layerSqlite.pipe( +const RelayStoreLive = SqlPeerRelayStore.layer.pipe( Layer.provideMerge(RelayLimitsLive) ) const RelayAuthorizationLive = PeerRelayAuthorization.layer( authorizeRelay, PeerRelayAuthorization.denyUnsafeUnboundedAutomerge3Decode ) -const RelayServerLive = PeerRpcServer.layerStoreAndForwardDeployment({ - directDeployment, - relaySocketLayer, - relay: { tenantId, peerId: relayPeerId } +const RelayHandlersLive = PeerRpcServer.layerHandlers({ + tenantId, + peerId: relayPeerId }).pipe( Layer.provideMerge(RelayLimitsLive), Layer.provideMerge(RelayStoreLive), @@ -74,7 +69,7 @@ const RelayClientLive = PeerRelayClientRuntime.layer.pipe( Layer.provideMerge(RelayOutboxLive) ) -const session = RpcPeerTransport.makeStoreAndForwardSession(relayRpcClient, { +const session = RpcPeerTransport.makeSession(relayRpcClient, { expectedLocal, senderReplicaIncarnation, expectedRelayPeerId: relayPeerId, @@ -87,8 +82,8 @@ const session = RpcPeerTransport.makeStoreAndForwardSession(relayRpcClient, { }).pipe(Effect.provide(RelayClientLive)) ``` -`relayRpcClient` must be a `PeerRelayRpc.RpcClient` created with `PeerRelayRpc.makeRpcClient` over -`PeerRelayIngress.layerProtocolSocket`. It must not use the direct WebSocket MessagePack protocol. The server +`relayRpcClient` must be a `PeerRpc.RpcClient` created with `PeerRpc.makeRpcClient` over +`PeerRelayIngress.layerProtocolSocket`. The server composition still requires the application SQL and Crypto Layers, `PeerAuthentication.layerServer`, the relay authorization Layer, and a `PeerRelayLimits` Layer. The client composition still requires the application SQL, Crypto, core replica limits, generated client authentication middleware, and a scoped relay socket. @@ -97,12 +92,12 @@ The example denies the separate unsafe Automerge decode grant. That is the requi and `authorizeRelay` document authorization do not promote a caller into resource trust. The `expectedRelayPeerId`, `expectedLocal`, exact remote subject and peer, replica incarnation, selected documents, -receipt retention, and sender retry horizon are part of the version `3` handshake. A mismatch fails the connection. +receipt retention, and sender retry horizon are part of the version `1` handshake. A mismatch fails the connection. ## Protocol and durable identity -`PeerRelayRpc.Rpcs` contains `OpenRelay`, `PushRelay`, `AcknowledgeRelay`, and `RejectRelay`. Every request uses the -same required authentication middleware as direct RPC. The first `OpenRelay` stream value is `RelayOpened`. Later +`PeerRpc.Rpcs` contains `Open`, `Push`, `Acknowledge`, and `Reject`. Every request uses the +same required authentication middleware. The first `Open` stream value is `Opened`. Later values are `StoredMessage` deliveries. Each sender admission has one stable `RelayMessageId`. The sender outbox persists that identity, the exact relay and @@ -119,7 +114,7 @@ and digest is safe. Conflicting reuse is rejected. A relay claim carries an opaque claim token and a finite deadline. A recipient acknowledgement can terminalize only the exact message, recipient principal, live session generation, token, and message hash. -`PeerSession` sends `AcknowledgeRelay` only after all of these operations succeed: +`PeerSession` sends `Acknowledge` only after all of these operations succeed: 1. The relay outer envelope and inner sync envelope are validated. 2. The Automerge message is applied through the production `DocumentEntity.ApplySync` path. @@ -131,7 +126,7 @@ This boundary proves durable recipient processing for replay suppression. It doe `PeerSession.durableConfirmation()`, which remains `false`. It also does not prove that another peer observed the change. -A stable protocol violation uses `RejectRelay` with `ProtocolInvalid`. Application code can use +A stable protocol violation uses `Reject` with `ProtocolInvalid`. Application code can use `ApplicationRejected` through the acknowledged delivery contract. A permanent rejection becomes a retained dead letter row. Infrastructure failure, interruption, timeout, disconnect, and claim expiry release or recover the message for retry. @@ -256,10 +251,9 @@ writer provenance shape. It does not interpret Automerge changes, dependency gra `PeerSync` receive path performs the one semantic Automerge decode and applies the existing change, dependency, and operation limits. -Direct protocol version `2` retains its preexisting authorized peer allocation risk because it has no separate unsafe -resource trust callback. Only connect direct peers whose produced Automerge bytes the application resource trusts. -When Automerge exposes an allocation bounded decode API, Effect Local should use it and remove this unsafe grant -instead of normalizing the exception as permanent policy. +The durable protocol always requires the separate unsafe resource trust callback. When Automerge exposes an +allocation bounded decode API, Effect Local should use it and remove this unsafe grant instead of normalizing the +exception as permanent policy. The relay outer digest binds the complete versioned envelope. It provides integrity and conflict detection. It is not encryption or a signature. The relay stores and can read the payload supplied by the client. Applications that need @@ -272,13 +266,13 @@ existence. ## Deployment boundary and limitations -One SQLite database is the custody authority for one relay shard. The application must route every write and claim -for that shard to one process and one durable volume. A second process must not open the same database through a -network file system. There is no built in leader election, quorum, replicated log, automatic failover, shard -rebalance, cross region routing, or split brain recovery. +One logical SQL database is the custody authority for one relay shard. The application must route every write and +claim for that shard to it. `SqlPeerRelayStore.layer` supports SQLite, PostgreSQL, and MySQL. Replication and failover +are properties of the selected database deployment. The relay does not add leader election, shard rebalance, cross +region routing, or split brain recovery. -When that SQLite authority is unavailable, new custody stops and delivery pauses. Durable pending rows remain -discoverable after restart on the same volume. Claims recover after their deadlines. The server also runs bounded +When that SQL authority is unavailable, new custody stops and delivery pauses. Durable pending rows remain +discoverable after recovery. Claims recover after their deadlines. The server also runs bounded compensation and maintenance so a missed process notification does not permanently strand committed work. This package supplies a bounded single authority relay building block. It does not claim WhatsApp scale, global @@ -288,12 +282,12 @@ availability, managed operations, peer discovery, end to end encryption, account The relay exposes Effect services and spans. It does not install a metrics exporter. -| Source | Exposed values | -| ------------------------------ | ----------------------------------------------------------------------------------- | -| `PeerRelayServerRuntime.usage` | `accepting`, `sessions`, `subjects`, `activeClaims`, `queuedChannels` | -| `PeerRelayIngress.usage` | `connections`, `reservedBytes`, `byteReservationWaiters` | -| `PeerRelayStore.usage` | `activeCount`, `activeBytes`, `retainedCount`, `retainedBytes` for a selected scope | -| `PeerRelayOutbox.usage` | Remote and replica `messageCount` and `encodedBytes` | +| Source | Exposed values | +| ---------------------------- | ----------------------------------------------------------------------------------- | +| `PeerRpcServerRuntime.usage` | `accepting`, `sessions`, `subjects`, `activeClaims`, `queuedChannels` | +| `PeerRelayIngress.usage` | `connections`, `reservedBytes`, `byteReservationWaiters` | +| `PeerRelayStore.usage` | `activeCount`, `activeBytes`, `retainedCount`, `retainedBytes` for a selected scope | +| `PeerRelayOutbox.usage` | Remote and replica `messageCount` and `encodedBytes` | Store operations use spans named `PeerRelayStore.admit`, `claim`, `loadClaimedPayload`, `acknowledge`, `reject`, `release`, `recover`, `expire`, `repair`, `reconcile`, `collect`, and `usage`. Client relay spans are diff --git a/docs/sync.md b/docs/sync.md index 55c4a4c..8847484 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -38,11 +38,8 @@ Effect Local does not ship a backend or a server transport. An application suppl identity, authentication, authorization, encryption, and routing. The test package supplies a bounded duplex transport with deterministic drop, delay, duplicate, reorder, partition, and reconnect behavior. -Direct Effect RPC uses protocol version `2` and remains a live exchange while both peers participate. Its application -owned listener, protocol, serializer, and `PeerRpcServer.layerHandlers` composition remain unchanged. - -Optional store and forward uses protocol version `3` on a distinct bounded listener. The sender first writes a stable -envelope to its local SQL outbox. Relay acceptance means that the single SQLite relay authority committed custody. +Effect RPC uses one durable protocol at version `1`. The sender first writes a stable envelope to its local SQLite +outbox. Relay acceptance means that the configured backend SQL store committed custody. The relay then delivers the oldest eligible message for one exact directed channel when the recipient reconnects. The recipient acknowledges only after its production sync workflow and sender scoped receipt commit. Lost acknowledgements and expired claims can duplicate delivery, so the guarantee is at least once. @@ -67,13 +64,12 @@ revocation admitted first prevents SQL mutation or payload emission. An operatio returns its real result. Revocation drains that in flight operation and prevents later work. SQLite and the external policy authority do not share an atomic transaction. -Direct version `2` retains the preexisting authorized peer allocation risk because it has no separate unsafe grant. -Applications must resource trust the Automerge bytes produced by direct peers. A future allocation bounded Automerge -decode API should remove the relay unsafe grant. +The durable protocol requires the separate unsafe grant. A future allocation bounded Automerge decode API should +remove that grant. -The relay is a single SQLite custody authority per shard. It does not provide quorum replication, automatic failover, -cross region routing, or peer discovery. See [Store and forward](store-and-forward.md) for the exact contract and -composition. +Relay custody uses the injected `PeerRelayStore`. `SqlPeerRelayStore.layer` supports SQLite, PostgreSQL, and MySQL +through a generic `SqlClient`. It does not add cross region routing or peer discovery. See +[Store and forward](store-and-forward.md) for the exact contract and composition. Automerge provides merge infrastructure, not collaboration UX. The public snapshot exposes the decoded value and head frontier. History traversal, conflict inspection, review, sharing, and conflict resolution interfaces remain diff --git a/knip.json b/knip.json index 37775de..8e3218d 100644 --- a/knip.json +++ b/knip.json @@ -6,9 +6,6 @@ "entry": ["test/crash-durability/childProcess.ts"], "ignoreDependencies": ["tsx"] }, - "packages/local-rpc": { - "ignore": ["bench/PeerRpcServerPerformance.bench.ts", "bench/vitest.admission.config.ts"] - }, "packages/local-browser": { "entry": [ "test-browser/*/src/*.{ts,tsx}", diff --git a/packages/local-rpc/README.md b/packages/local-rpc/README.md index eb095d8..ec3033d 100644 --- a/packages/local-rpc/README.md +++ b/packages/local-rpc/README.md @@ -1,8 +1,12 @@ # @lucas-barake/effect-local-rpc -Platform neutral Effect RPC building blocks for authenticated peer synchronization with an Effect Local SQL replica. -Direct protocol version `2` remains application owned and live only. Optional protocol version `3` adds a separate -bounded listener, authenticated relay authorization, and single authority SQLite store and forward custody. +Platform neutral Effect RPC building blocks for authenticated, durable peer synchronization with an Effect Local SQL +replica. Protocol version `1` uses one bounded relay topology with durable custody, reconnect delivery, and fenced +acknowledgements. + +Backend custody is injected through `PeerRelayStore`. `SqlPeerRelayStore.layer` follows Effect Workflow's SQL storage +shape and works with a supplied `SqlClient` for SQLite, PostgreSQL, or MySQL. Frontend replica state, sender outbox, +and recipient receipts remain SQLite. Automerge `3.3.2` has no allocation bounded semantic decode API, so safe relay composition explicitly passes `denyUnsafeUnboundedAutomerge3Decode`. A separate exact unsafe resource trust grant is required to proceed. Ordinary authentication and document authorization do not provide that grant. @@ -13,6 +17,6 @@ mutation or payload emission. An operation admitted first may finish. Revocation External policy and SQLite do not share an atomic transaction. See the [Effect Local documentation](https://github.com/lucas-barake/effect-local#readme) for synchronization -architecture, deployment boundaries, and API reference. The optional relay contract, composition, limits, security, +architecture, deployment boundaries, and API reference. The relay contract, composition, limits, security, and failure model are documented in [Store and forward](https://github.com/lucas-barake/effect-local/blob/main/docs/store-and-forward.md). diff --git a/packages/local-rpc/bench/PeerRelayStore.bench.ts b/packages/local-rpc/bench/PeerRelayStore.bench.ts index 5c28ff4..c7cfc60 100644 --- a/packages/local-rpc/bench/PeerRelayStore.bench.ts +++ b/packages/local-rpc/bench/PeerRelayStore.bench.ts @@ -12,8 +12,9 @@ import { tmpdir } from "node:os" import { join } from "node:path" import { afterAll, bench } from "vitest" import * as PeerRelayLimits from "../src/PeerRelayLimits.js" -import * as PeerRelayRpc from "../src/PeerRelayRpc.js" import * as PeerRelayStore from "../src/PeerRelayStore.js" +import * as PeerRpc from "../src/PeerRpc.js" +import * as SqlPeerRelayStore from "../src/SqlPeerRelayStore.js" const backlogSizes = [128, 1_024] as const const drainBatchSize = 8 @@ -66,7 +67,7 @@ const makeWorkItem = ( senderSequence: distribution === "Hot" ? index : 0, payloadVersion: 1, messageHash: `message-${messageNumber}`, - outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make(messageNumber.toString(16).padStart(64, "0")), + outerEnvelopeDigest: PeerRpc.RelayDigest.make(messageNumber.toString(16).padStart(64, "0")), payload, messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, @@ -100,7 +101,7 @@ const benchmarkContext = await Effect.runPromise( Layer.build( Layer.merge( baseLayer, - PeerRelayStore.layerSqlite.pipe(Layer.provide(baseLayer)) + SqlPeerRelayStore.layer.pipe(Layer.provide(baseLayer)) ) ).pipe(Effect.provideService(Scope.Scope, scope)) ) diff --git a/packages/local-rpc/bench/PeerRpc.bench.ts b/packages/local-rpc/bench/PeerRpc.bench.ts index 918c0be..3adcfa3 100644 --- a/packages/local-rpc/bench/PeerRpc.bench.ts +++ b/packages/local-rpc/bench/PeerRpc.bench.ts @@ -13,7 +13,17 @@ const makeBytes = (size: number) => { } const peerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000001") +const localPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000002") +const remotePeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000003") const sessionId = Identity.SessionId.make("ses_00000000-0000-4000-8000-000000000001") +const relayMessageId = Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000001") +const claimToken = PeerRpc.ClaimToken.make("clm_00000000-0000-4000-8000-000000000001") +const benchmarkDocumentId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000001") +const expectedLocal = { + tenantId: "tenant", + subjectId: "local", + peerId: localPeerId +} const credential = Redacted.make("benchmark-credential") const wireParsers = (envelope: RpcMessage.FromClientEncoded | RpcMessage.FromServerEncoded) => { @@ -37,8 +47,15 @@ const decodeOpenPayload = Schema.decodeUnknownSync(openPayloadCodec) for (const documentCount of [1, 16, 64] as const) { const decoded = PeerRpc.OpenRpc.payloadSchema.make({ protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: peerId, - definitionHash: "def_0000000000000000", + expectedRelayPeerId: peerId, + expectedLocal, + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + remote: { + subjectId: "remote", + peerId: remotePeerId + }, + receiptRetentionMillis: 1, + senderRetryHorizonMillis: 1, credential, documents: Array.from({ length: documentCount }, (_, index) => ({ documentType: "Task", @@ -114,7 +131,7 @@ for ( ] as const ) { const bytes = makeBytes(size) - const decodedPush = PeerRpc.PushRpc.payloadSchema.make({ sessionId, payload: bytes, credential }) + const decodedPush = PeerRpc.PushRpc.payloadSchema.make({ sessionId, relayMessageId, payload: bytes, credential }) const encodedPush = encodePushPayload(decodedPush) const pushEnvelope: RpcMessage.RequestEncoded = { _tag: "Request", @@ -124,7 +141,30 @@ for ( headers: [] } const pushWire = wireParsers(pushEnvelope) - const decodedChunk = [PeerRpc.Message.make({ _tag: "Message", payload: bytes })] as const + const decodedChunk = [PeerRpc.StoredMessage.make({ + _tag: "StoredMessage", + relayMessageId, + claimToken, + relayPeerId: peerId, + sender: { + tenantId: "tenant", + subjectId: "remote", + peerId: remotePeerId, + replicaIncarnation: Identity.ReplicaIncarnation.make(1), + connectionEpoch: "benchmark-epoch", + sequence: 0 + }, + recipient: expectedLocal, + payloadVersion: 1, + document: { + documentType: "Task", + documentId: benchmarkDocumentId + }, + writerProvenance: [], + messageHash: "1".repeat(64), + outerEnvelopeDigest: "2".repeat(64), + payload: bytes + })] as const const encodedChunk = nonEmptyValues(encodeOpenChunk(decodedChunk)) const chunkEnvelope: RpcMessage.ResponseChunkEncoded = { _tag: "Chunk", @@ -151,7 +191,7 @@ for ( warmupTime: 0 }) - bench(`Schema JSON codec Open Message chunk encode ${label}`, () => { + bench(`Schema JSON codec Open StoredMessage chunk encode ${label}`, () => { encodeOpenChunk(decodedChunk) }, { iterations, @@ -160,7 +200,7 @@ for ( warmupTime: 0 }) - bench(`Schema JSON codec Open Message chunk decode ${label}`, () => { + bench(`Schema JSON codec Open StoredMessage chunk decode ${label}`, () => { decodeOpenChunk(encodedChunk) }, { iterations, @@ -214,8 +254,8 @@ const openedEnvelope: RpcMessage.ResponseChunkEncoded = { _tag: "Opened", protocolVersion: PeerRpc.protocolVersion, sessionId, - peerId, - capabilities: { storeAndForward: false } + remotePeerId, + authenticatedLocal: expectedLocal }) ])) } diff --git a/packages/local-rpc/bench/PeerRpcServerPerformance.bench.ts b/packages/local-rpc/bench/PeerRpcServerPerformance.bench.ts deleted file mode 100644 index ec92ac4..0000000 --- a/packages/local-rpc/bench/PeerRpcServerPerformance.bench.ts +++ /dev/null @@ -1,157 +0,0 @@ -import * as CommitPublisher from "@lucas-barake/effect-local-sql/CommitPublisher" -import * as DocumentSet from "@lucas-barake/effect-local/DocumentSet" -import * as Identity from "@lucas-barake/effect-local/Identity" -import * as ReplicaDefinition from "@lucas-barake/effect-local/ReplicaDefinition" -import * as ReplicaLimits from "@lucas-barake/effect-local/ReplicaLimits" -import * as Effect from "effect/Effect" -import * as Exit from "effect/Exit" -import * as Layer from "effect/Layer" -import * as Scope from "effect/Scope" -import * as Stream from "effect/Stream" -import { afterAll, bench } from "vitest" -import * as PeerRpcAdmissionBenchmark from "../src/internal/peerRpcAdmissionBenchmark.js" -import * as PeerAuthentication from "../src/PeerAuthentication.js" -import * as PeerAuthenticator from "../src/PeerAuthenticator.js" -import * as PeerAuthorization from "../src/PeerAuthorization.js" -import * as PeerRpcLimits from "../src/PeerRpcLimits.js" -import * as PeerRpcServer from "../src/PeerRpcServer.js" - -const sizes = [0, 1_000, 10_000] as const -const benchDefinition = ReplicaDefinition.make({ - name: "peer-rpc-server-bench", - documents: DocumentSet.make(), - mutations: [], - projections: [], - queries: [] -}) -const now = 1 -const limits = PeerRpcLimits.Values.make({ - ...PeerRpcLimits.defaults, - authenticationRatePerSecond: 1_000_000, - authenticationBurst: 1_000_000, - openRatePerSecond: 1_000_000, - openBurst: 1_000_000, - maxRetainedRateLimitedConnections: 10_001, - maxRetainedRateLimitedSubjects: 10_001 -}) -const replicaLimits = ReplicaLimits.Values.make({ - maxBackupBytes: 1, - maxChunkBytes: 1, - maxArchiveRecords: 1, - maxJsonDepth: 1, - maxSyncMessageBytes: 1, - maxPeerSendMillis: 1, - maxSyncChangesPerMessage: 1, - maxSyncDependencyEdgesPerMessage: 1, - maxSyncOperationsPerMessage: 1, - maxPendingBytesPerDocument: 1, - maxPendingBytesPerPeer: 1, - maxPendingBytesPerReplica: 1, - maxPendingAgeMillis: 1, - maxPendingChangesPerDocument: 1, - maxPendingChangesPerPeer: 1, - maxPendingChangesPerReplica: 1, - maxPendingDependencyEdgesPerDocument: 1, - maxPendingDependencyEdgesPerPeer: 1, - maxPendingDependencyEdgesPerReplica: 1, - maxSessions: 64, - maxStreamsPerSession: 1, - maxInFlightPerSession: 1, - maxQueuedRpc: 1, - maxQueuedPermits: 1, - maxActiveRestores: 1, - maxRestoresPerSession: 1, - maxRestoreMillis: 30_000, - maxRestorePullMillis: 10_000, - maxRestoreCoalesceMillis: 25, - maxRestoreErrorBytes: 4_096 -}) - -const authenticationOperations = new Map Effect.Effect - readonly release: (clientId: number, now: number) => Effect.Effect -}>() -const subjectOperations = new Map Effect.Effect<"Unavailable" | "Capacity" | "Acquired"> - readonly release: (subjectId: string, operation: "Open" | "Push") => Effect.Effect -}>() -const serverScopes: Array = [] - -await Effect.runPromise(Effect.gen(function*() { - for (const size of sizes) { - let authenticationController: typeof authenticationOperations extends Map ? Operations - : never - PeerRpcAdmissionBenchmark.installAuthenticationCapture( - (operations) => void (authenticationController = operations) - ) - yield* Layer.build(PeerAuthentication.layerServer).pipe( - Effect.provideService(PeerAuthenticator.PeerAuthenticator, { - authenticate: () => Effect.die("unused") - }), - Effect.provideService(PeerRpcLimits.PeerRpcLimits, limits), - Effect.scoped - ) - for (let clientId = 0; clientId < size; clientId++) { - yield* authenticationController.admit(clientId, now) - yield* authenticationController.release(clientId, now) - } - yield* authenticationController.admit(-1, now) - yield* authenticationController.release(-1, now) - authenticationOperations.set(size, authenticationController) - - let subjectController: typeof subjectOperations extends Map ? Operations : never - PeerRpcAdmissionBenchmark.installSubjectCapture((operations) => void (subjectController = operations)) - const scope = yield* Scope.make() - serverScopes.push(scope) - const publisher = CommitPublisher.CommitPublisher.of({ - publishPending: Effect.succeed(0), - invalidate: () => Effect.void, - subscribe: Effect.succeed({ - watermark: Identity.CommitSequence.make(0), - refreshGeneration: 0, - events: Stream.never - }) - }) - yield* Layer.build(PeerRpcServer.layerHandlers({ - tenantId: "tenant", - peerId: Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000001"), - definition: benchDefinition - })).pipe( - Effect.provideService(Scope.Scope, scope), - Effect.provideService(CommitPublisher.CommitPublisher, publisher), - Effect.provideService(PeerRpcLimits.PeerRpcLimits, limits), - Effect.provideService(ReplicaLimits.ReplicaLimits, replicaLimits), - Effect.provideService(PeerAuthorization.PeerAuthorization, {} as PeerAuthorization.PeerAuthorization["Service"]), - Effect.provideService(Scope.Scope, scope) - ) - for (let subjectId = 0; subjectId < size; subjectId++) { - yield* subjectController.acquire(String(subjectId), "Open", now) - yield* subjectController.release(String(subjectId), "Open") - } - yield* subjectController.acquire("hot", "Open", now) - yield* subjectController.release("hot", "Open") - subjectOperations.set(size, subjectController) - } -})) - -for (const size of sizes) { - bench(`authentication hot admission with ${size} retained connections`, async () => { - const operations = authenticationOperations.get(size)! - await Effect.runPromise(operations.admit(-1, now)) - await Effect.runPromise(operations.release(-1, now)) - }, { iterations: 200, time: 0, warmupIterations: 20, warmupTime: 0 }) - - bench(`subject hot admission with ${size} retained subjects`, async () => { - const operations = subjectOperations.get(size)! - await Effect.runPromise(operations.acquire("hot", "Open", now)) - await Effect.runPromise(operations.release("hot", "Open")) - }, { iterations: 200, time: 0, warmupIterations: 20, warmupTime: 0 }) -} - -afterAll(async () => { - await Promise.all(serverScopes.map((scope) => Effect.runPromise(Scope.close(scope, Exit.void)))) -}) diff --git a/packages/local-rpc/bench/RpcPeerTransport.bench.ts b/packages/local-rpc/bench/RpcPeerTransport.bench.ts deleted file mode 100644 index 944abfd..0000000 --- a/packages/local-rpc/bench/RpcPeerTransport.bench.ts +++ /dev/null @@ -1,133 +0,0 @@ -import * as Document from "@lucas-barake/effect-local/Document" -import * as DocumentSet from "@lucas-barake/effect-local/DocumentSet" -import * as Identity from "@lucas-barake/effect-local/Identity" -import * as PeerTransport from "@lucas-barake/effect-local/PeerTransport" -import * as ReplicaDefinition from "@lucas-barake/effect-local/ReplicaDefinition" -import * as Context from "effect/Context" -import * as Effect from "effect/Effect" -import * as Exit from "effect/Exit" -import * as Layer from "effect/Layer" -import * as Redacted from "effect/Redacted" -import * as Schema from "effect/Schema" -import * as Scope from "effect/Scope" -import * as Stream from "effect/Stream" -import * as RpcTest from "effect/unstable/rpc/RpcTest" -import { afterAll, bench } from "vitest" -import * as PeerAuthentication from "../src/PeerAuthentication.js" -import * as PeerAuthenticator from "../src/PeerAuthenticator.js" -import * as PeerCredentials from "../src/PeerCredentials.js" -import * as PeerRpc from "../src/PeerRpc.js" -import * as PeerRpcLimits from "../src/PeerRpcLimits.js" -import * as RpcPeerTransport from "../src/RpcPeerTransport.js" - -const makeBytes = (size: number) => { - const bytes = new Uint8Array(size) - for (let index = 0; index < bytes.length; index++) bytes[index] = index % 251 - return bytes -} - -const Task = Document.make("BenchmarkTask", { - schema: Schema.Struct({ title: Schema.String }), - version: 1 -}) -const definition = ReplicaDefinition.make({ - name: "rpc-peer-transport-bench", - documents: DocumentSet.make(Task), - mutations: [], - projections: [], - queries: [] -}) -const documentId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000001") -const replicaId = Identity.ReplicaId.make("rep_00000000-0000-4000-8000-000000000001") -const serverPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000001") -const clientPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000002") -const sessionId = Identity.SessionId.make("ses_00000000-0000-4000-8000-000000000001") -const limits = PeerRpcLimits.Values.make({ - ...PeerRpcLimits.defaults, - authenticationRatePerSecond: 1_000_000, - authenticationBurst: 1_000_000 -}) -const scope = await Effect.runPromise(Scope.make()) -const handlers = PeerRpc.Rpcs.toLayer(PeerRpc.Rpcs.of({ - Open: () => - Stream.concat( - Stream.make(PeerRpc.Opened.make({ - _tag: "Opened", - protocolVersion: PeerRpc.protocolVersion, - sessionId, - peerId: serverPeerId, - capabilities: { storeAndForward: false } - })), - Stream.never - ), - Push: () => Effect.void -})) -const client = await Effect.runPromise( - RpcTest.makeClient(PeerRpc.Rpcs).pipe( - Effect.provide(handlers), - Effect.provide(PeerAuthentication.layerServer), - Effect.provideService(PeerAuthenticator.PeerAuthenticator, { - authenticate: () => - Effect.succeed({ - principal: PeerAuthentication.PeerPrincipal.make({ - tenantId: "tenant", - subjectId: "benchmark-client", - peerId: clientPeerId - }), - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.never - }) - }), - Effect.provide(PeerAuthentication.layerClient), - Effect.provideService(PeerCredentials.PeerCredentials, { - get: Effect.succeed(Redacted.make("benchmark-credential")) - }), - Effect.provideService(PeerRpcLimits.PeerRpcLimits, limits), - Effect.provideService(Scope.Scope, scope) - ) -) -const transportContext = await Effect.runPromise( - Layer.build(RpcPeerTransport.layer(client, { - documents: [{ document: Task, documentId }], - definition - })).pipe(Effect.provideService(Scope.Scope, scope)) -) -const connection = await Effect.runPromise( - Context.get(transportContext, PeerTransport.PeerTransport).connect({ - replicaId, - peerId: serverPeerId - }).pipe(Effect.provideService(Scope.Scope, scope)) -) - -for ( - const [label, size] of [ - ["1 KiB", 1024], - ["64 KiB", 64 * 1024], - ["1 MiB", 1024 * 1024] - ] as const -) { - const payload = makeBytes(size) - - bench(`RpcTest no-serialization generated Push ${label}`, async () => { - await Effect.runPromise(client.Push({ sessionId, payload })) - }, { - iterations: 500, - time: 0, - warmupIterations: 50, - warmupTime: 0 - }) - - bench(`RpcTest no-serialization RpcPeerTransport send ${label}`, async () => { - await Effect.runPromise(connection.send(payload)) - }, { - iterations: 500, - time: 0, - warmupIterations: 50, - warmupTime: 0 - }) -} - -afterAll(async () => { - await Effect.runPromise(connection.close) - await Effect.runPromise(Scope.close(scope, Exit.void)) -}) diff --git a/packages/local-rpc/bench/base-admission-instrumentation.patch b/packages/local-rpc/bench/base-admission-instrumentation.patch deleted file mode 100644 index b8a9b66..0000000 --- a/packages/local-rpc/bench/base-admission-instrumentation.patch +++ /dev/null @@ -1,51 +0,0 @@ -diff --git a/packages/local-rpc/src/PeerAuthentication.ts b/packages/local-rpc/src/PeerAuthentication.ts -index a0a4b7e..e28303d 100644 ---- a/packages/local-rpc/src/PeerAuthentication.ts -+++ b/packages/local-rpc/src/PeerAuthentication.ts -@@ -10,0 +11 @@ import * as RpcMiddleware from "effect/unstable/rpc/RpcMiddleware" -+import * as PeerRpcAdmissionBenchmark from "./internal/peerRpcAdmissionBenchmark.js" -@@ -107,0 +109,2 @@ export const layerServer = Layer.effect( -+ PeerRpcAdmissionBenchmark.captureAuthentication({ admit, release }) -+ -diff --git a/packages/local-rpc/src/PeerRpcServer.ts b/packages/local-rpc/src/PeerRpcServer.ts -index e20efd5..060466d 100644 ---- a/packages/local-rpc/src/PeerRpcServer.ts -+++ b/packages/local-rpc/src/PeerRpcServer.ts -@@ -19,0 +20 @@ import * as Stream from "effect/Stream" -+import * as PeerRpcAdmissionBenchmark from "./internal/peerRpcAdmissionBenchmark.js" -@@ -369,0 +371,2 @@ export const layerHandlers = (options: { readonly tenantId: string; readonly pee -+ PeerRpcAdmissionBenchmark.captureSubject({ acquire: acquireSubject, release: releaseSubject }) -+ -diff --git a/packages/local-rpc/src/internal/peerRpcAdmissionBenchmark.ts b/packages/local-rpc/src/internal/peerRpcAdmissionBenchmark.ts -new file mode 100644 -index 0000000..989e7b9 ---- /dev/null -+++ b/packages/local-rpc/src/internal/peerRpcAdmissionBenchmark.ts -@@ -0,0 +1,27 @@ -+import type * as Effect from "effect/Effect" -+ -+let authenticationCapture = (_operations: { -+ readonly admit: (clientId: number, now: number) => Effect.Effect -+ readonly release: (clientId: number, now: number) => Effect.Effect -+}) => undefined -+ -+let subjectCapture = (_operations: { -+ readonly acquire: ( -+ subjectId: string, -+ operation: "Open" | "Push", -+ now: number -+ ) => Effect.Effect<"Unavailable" | "Capacity" | "Acquired"> -+ readonly release: (subjectId: string, operation: "Open" | "Push") => Effect.Effect -+}) => undefined -+ -+export const installAuthenticationCapture = (capture: typeof authenticationCapture) => { -+ authenticationCapture = capture -+} -+ -+export const installSubjectCapture = (capture: typeof subjectCapture) => { -+ subjectCapture = capture -+} -+ -+export const captureAuthentication: typeof authenticationCapture = (operations) => authenticationCapture(operations) -+ -+export const captureSubject: typeof subjectCapture = (operations) => subjectCapture(operations) diff --git a/packages/local-rpc/bench/candidate-admission-instrumentation.patch b/packages/local-rpc/bench/candidate-admission-instrumentation.patch deleted file mode 100644 index cd36347..0000000 --- a/packages/local-rpc/bench/candidate-admission-instrumentation.patch +++ /dev/null @@ -1,75 +0,0 @@ -diff --git a/packages/local-rpc/src/PeerAuthentication.ts b/packages/local-rpc/src/PeerAuthentication.ts -index 6e359df..dc90708 100644 ---- a/packages/local-rpc/src/PeerAuthentication.ts -+++ b/packages/local-rpc/src/PeerAuthentication.ts -@@ -8,6 +8,7 @@ import * as Redacted from "effect/Redacted" - import * as Schema from "effect/Schema" - import * as Semaphore from "effect/Semaphore" - import * as RpcMiddleware from "effect/unstable/rpc/RpcMiddleware" -+import * as PeerRpcAdmissionBenchmark from "./internal/peerRpcAdmissionBenchmark.js" - import * as PeerRpcObservability from "./internal/peerRpcObservability.js" - import * as PeerAuthenticator from "./PeerAuthenticator.js" - import * as PeerCredentials from "./PeerCredentials.js" -@@ -128,6 +129,8 @@ export const layerServer = Layer.effect( - if (entry.inFlight === 0) retainInactive(clientId, entry) - })) - -+ PeerRpcAdmissionBenchmark.captureAuthentication({ admit, release }) -+ - return PeerAuthentication.of((effect, options) => - PeerRpcObservability.observe({ - effect: Effect.gen(function*() { -diff --git a/packages/local-rpc/src/PeerRpcServer.ts b/packages/local-rpc/src/PeerRpcServer.ts -index 75be7ea..269aa95 100644 ---- a/packages/local-rpc/src/PeerRpcServer.ts -+++ b/packages/local-rpc/src/PeerRpcServer.ts -@@ -16,6 +16,7 @@ import * as Queue from "effect/Queue" - import * as Scope from "effect/Scope" - import * as Semaphore from "effect/Semaphore" - import * as Stream from "effect/Stream" -+import * as PeerRpcAdmissionBenchmark from "./internal/peerRpcAdmissionBenchmark.js" - import * as PeerAuthorizationValidation from "./internal/peerAuthorization.js" - import * as PeerRpcObservability from "./internal/peerRpcObservability.js" - import * as PeerAuthentication from "./PeerAuthentication.js" -@@ -487,6 +488,8 @@ export const layerHandlers = (options: { readonly tenantId: string; readonly pee - retainInactiveSubject(subjectId, subject) - })) - -+ PeerRpcAdmissionBenchmark.captureSubject({ acquire: acquireSubject, release: releaseSubject }) -+ - const admitted = ( - subjectId: string, - operation: "Open" | "Push", -diff --git a/packages/local-rpc/src/internal/peerRpcAdmissionBenchmark.ts b/packages/local-rpc/src/internal/peerRpcAdmissionBenchmark.ts -new file mode 100644 -index 0000000..989e7b9 ---- /dev/null -+++ b/packages/local-rpc/src/internal/peerRpcAdmissionBenchmark.ts -@@ -0,0 +1,27 @@ -+import type * as Effect from "effect/Effect" -+ -+let authenticationCapture = (_operations: { -+ readonly admit: (clientId: number, now: number) => Effect.Effect -+ readonly release: (clientId: number, now: number) => Effect.Effect -+}) => undefined -+ -+let subjectCapture = (_operations: { -+ readonly acquire: ( -+ subjectId: string, -+ operation: "Open" | "Push", -+ now: number -+ ) => Effect.Effect<"Unavailable" | "Capacity" | "Acquired"> -+ readonly release: (subjectId: string, operation: "Open" | "Push") => Effect.Effect -+}) => undefined -+ -+export const installAuthenticationCapture = (capture: typeof authenticationCapture) => { -+ authenticationCapture = capture -+} -+ -+export const installSubjectCapture = (capture: typeof subjectCapture) => { -+ subjectCapture = capture -+} -+ -+export const captureAuthentication: typeof authenticationCapture = (operations) => authenticationCapture(operations) -+ -+export const captureSubject: typeof subjectCapture = (operations) => subjectCapture(operations) diff --git a/packages/local-rpc/bench/vitest.admission.config.ts b/packages/local-rpc/bench/vitest.admission.config.ts deleted file mode 100644 index 4195ca7..0000000 --- a/packages/local-rpc/bench/vitest.admission.config.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { defineConfig } from "vitest/config" - -export default defineConfig({ - test: { - name: "local-rpc-admission", - benchmark: { include: ["packages/local-rpc/bench/PeerRpcServerPerformance.bench.ts"] } - } -}) diff --git a/packages/local-rpc/package.json b/packages/local-rpc/package.json index cec63ec..53e184f 100644 --- a/packages/local-rpc/package.json +++ b/packages/local-rpc/package.json @@ -4,7 +4,13 @@ "description": "Effect RPC peer synchronization for Effect Local", "homepage": "https://github.com/lucas-barake/effect-local#readme", "bugs": "https://github.com/lucas-barake/effect-local/issues", - "keywords": ["effect", "local-first", "rpc", "websocket", "automerge"], + "keywords": [ + "effect", + "local-first", + "rpc", + "websocket", + "automerge" + ], "engines": { "node": "^22.22.2 || ^24.15.0 || >=26.0.0" }, @@ -56,8 +62,12 @@ }, "devDependencies": { "@effect/platform-node": "4.0.0-beta.99", + "@effect/sql-mysql2": "4.0.0-beta.99", + "@effect/sql-pg": "4.0.0-beta.99", "@effect/sql-sqlite-node": "4.0.0-beta.99", "@lucas-barake/effect-local-test": "workspace:^", + "@testcontainers/mysql": "11.14.0", + "@testcontainers/postgresql": "11.14.0", "effect": "4.0.0-beta.99" } } diff --git a/packages/local-rpc/src/PeerAuthentication.ts b/packages/local-rpc/src/PeerAuthentication.ts index 8952f23..fe253b9 100644 --- a/packages/local-rpc/src/PeerAuthentication.ts +++ b/packages/local-rpc/src/PeerAuthentication.ts @@ -12,8 +12,8 @@ import type { PeerPrincipal } from "./internal/peerPrincipal.js" import * as PeerRpcObservability from "./internal/peerRpcObservability.js" import * as PeerAuthenticator from "./PeerAuthenticator.js" import * as PeerCredentials from "./PeerCredentials.js" +import * as PeerRelayLimits from "./PeerRelayLimits.js" import * as PeerRpcError from "./PeerRpcError.js" -import * as PeerRpcLimits from "./PeerRpcLimits.js" export { PeerPrincipal } from "./internal/peerPrincipal.js" @@ -39,7 +39,7 @@ export const layerServer = Layer.effect( PeerAuthentication, Effect.gen(function*() { const authenticator = yield* PeerAuthenticator.PeerAuthenticator - const limits = yield* PeerRpcLimits.PeerRpcLimits + const limits = yield* PeerRelayLimits.PeerRelayLimits const verifierPermits = yield* Semaphore.make(limits.maxInFlightAuthentication) const rateStateLock = yield* Semaphore.make(1) type RateState = { @@ -54,7 +54,7 @@ export const layerServer = Layer.effect( const expireOldestInactive = (now: number) => { const oldest = inactiveRateState.entries().next().value if (oldest === undefined) return - if (now - oldest[1].lastUsedAt < limits.rateLimitIdleRetention) return + if (now - oldest[1].lastUsedAt < limits.rateLimitIdleRetentionMillis) return inactiveRateState.delete(oldest[0]) rateState.delete(oldest[0]) } @@ -70,7 +70,7 @@ export const layerServer = Layer.effect( let entry = rateState.get(clientId) if (entry?.inFlight === 0) { inactiveRateState.delete(clientId) - if (now - entry.lastUsedAt >= limits.rateLimitIdleRetention) { + if (now - entry.lastUsedAt >= limits.rateLimitIdleRetentionMillis) { rateState.delete(clientId) entry = undefined } diff --git a/packages/local-rpc/src/PeerAuthorization.ts b/packages/local-rpc/src/PeerAuthorization.ts deleted file mode 100644 index 7d7dd17..0000000 --- a/packages/local-rpc/src/PeerAuthorization.ts +++ /dev/null @@ -1,42 +0,0 @@ -import type * as PeerSession from "@lucas-barake/effect-local-sql/PeerSession" -import * as Context from "effect/Context" -import * as Effect from "effect/Effect" -import * as Layer from "effect/Layer" -import { validate, validateRequest } from "./internal/peerAuthorization.js" -import type * as PeerAuthentication from "./PeerAuthentication.js" -import type * as PeerRpcError from "./PeerRpcError.js" - -export class PeerAuthorization extends Context.Service - }) => Effect.Effect<{ - readonly documents: ReadonlyArray - readonly validUntil: number - readonly invalidated: Effect.Effect - }, PeerRpcError.AccessDenied | PeerRpcError.ServerUnavailable> -}>()("@lucas-barake/effect-local-rpc/PeerAuthorization") {} - -export const layer = ( - authorize: (request: { - readonly principal: PeerAuthentication.PeerPrincipal - readonly documents: ReadonlyArray<{ - readonly documentType: string - readonly documentId: PeerSession.SelectedDocument["documentId"] - }> - }) => Effect.Effect<{ - readonly documents: ReadonlyArray - readonly validUntil: number - readonly invalidated: Effect.Effect - }, PeerRpcError.AccessDenied | PeerRpcError.ServerUnavailable> -) => - Layer.succeed(PeerAuthorization)({ - authorize: (request) => - validateRequest(request.documents).pipe( - Effect.flatMap(() => authorize(request)), - Effect.flatMap((result) => validate(request, result)) - ) - }) diff --git a/packages/local-rpc/src/PeerRelayLimits.ts b/packages/local-rpc/src/PeerRelayLimits.ts index e8a8127..9a5ca1b 100644 --- a/packages/local-rpc/src/PeerRelayLimits.ts +++ b/packages/local-rpc/src/PeerRelayLimits.ts @@ -3,12 +3,12 @@ import * as Effect from "effect/Effect" import * as Layer from "effect/Layer" import * as Schema from "effect/Schema" import type * as SchemaIssue from "effect/SchemaIssue" -import * as PeerRelayRpc from "./PeerRelayRpc.js" +import * as PeerRpcProtocol from "./internal/peerRpcProtocol.js" const PositiveInt = Schema.Int.check(Schema.isGreaterThan(0)) const PositiveNumber = Schema.Number.check(Schema.isFinite(), Schema.isGreaterThan(0)) const NegotiatedDuration = PositiveInt.check( - Schema.isLessThanOrEqualTo(PeerRelayRpc.maximumNegotiatedDurationMillis) + Schema.isLessThanOrEqualTo(PeerRpcProtocol.maximumNegotiatedDurationMillis) ) const Percentage = PositiveNumber.check(Schema.isLessThanOrEqualTo(100)) @@ -57,6 +57,9 @@ export const Values = Schema.Struct({ incompleteFrameTimeoutMillis: PositiveInt, maximumSharedPayloadBytes: PositiveInt, maximumByteReservationWaiters: PositiveInt, + maxInFlightAuthentication: PositiveInt, + authenticationRatePerSecond: PositiveNumber, + authenticationBurst: PositiveInt, maxSessionsPerSubject: PositiveInt, maxInFlightOpen: PositiveInt, @@ -169,6 +172,9 @@ export const defaults: Values = Values.make({ incompleteFrameTimeoutMillis: 10_000, maximumSharedPayloadBytes: 64 * mebibyte, maximumByteReservationWaiters: 1_024, + maxInFlightAuthentication: 64, + authenticationRatePerSecond: 16, + authenticationBurst: 64, maxSessionsPerSubject: 4, maxInFlightOpen: 32, @@ -318,14 +324,15 @@ const validate = (values: Values) => { ], ["maximumRawChunkBytes", values.maximumRawChunkBytes <= values.maximumIncompleteFrameBytes], ["maximumDeclaredFrameBytes", values.maximumDeclaredFrameBytes + 4 <= values.maximumIncompleteFrameBytes], - ["maximumDeclaredFrameBytes", values.maximumDeclaredFrameBytes >= PeerRelayRpc.maximumRelayPayloadBytes], + ["maximumDeclaredFrameBytes", values.maximumDeclaredFrameBytes >= PeerRpcProtocol.maximumRelayPayloadBytes], ["maximumSharedPayloadBytes", values.maximumSharedPayloadBytes >= values.maximumIncompleteFrameBytes], [ "maximumSharedPayloadBytes", - values.relayWorkerConcurrency * PeerRelayRpc.maximumRelayPayloadBytes <= + values.relayWorkerConcurrency * PeerRpcProtocol.maximumRelayPayloadBytes <= values.maximumSharedPayloadBytes ], ["maximumByteReservationWaiters", values.maximumByteReservationWaiters >= values.maxRelayConnections], + ["authenticationBurst", values.authenticationBurst >= values.maxInFlightAuthentication], ["maxSessionsPerSubject", values.maxSessionsPerSubject <= values.maxRelayConnections], ["maxInFlightOpen", values.maxInFlightOpen <= values.maxRelayConnections], ["maxInFlightOpenPerSubject", values.maxInFlightOpenPerSubject <= values.maxInFlightOpen], diff --git a/packages/local-rpc/src/PeerRelayRpc.ts b/packages/local-rpc/src/PeerRelayRpc.ts deleted file mode 100644 index b429f0f..0000000 --- a/packages/local-rpc/src/PeerRelayRpc.ts +++ /dev/null @@ -1,149 +0,0 @@ -import * as Identity from "@lucas-barake/effect-local/Identity" -import * as PeerSyncEnvelope from "@lucas-barake/effect-local-sql/PeerSyncEnvelope" -import type * as Effect from "effect/Effect" -import * as Schema from "effect/Schema" -import type * as Scope from "effect/Scope" -import * as Rpc from "effect/unstable/rpc/Rpc" -import { make as makeClient } from "effect/unstable/rpc/RpcClient" -import type * as RpcClient_ from "effect/unstable/rpc/RpcClient" -import type { RpcClientError } from "effect/unstable/rpc/RpcClientError" -import * as RpcGroup from "effect/unstable/rpc/RpcGroup" -import type * as RpcMiddleware from "effect/unstable/rpc/RpcMiddleware" -import * as PeerAuthentication from "./PeerAuthentication.js" -import * as PeerRpc from "./PeerRpc.js" -import * as PeerRpcError from "./PeerRpcError.js" - -export const protocolVersion = 3 - -export const maximumNegotiatedDurationMillis = 90 * 24 * 60 * 60 * 1_000 -export const maximumRelayPayloadBytes = PeerSyncEnvelope.maximumRelayPayloadBytes -export const maximumRequestedDocuments = 1_000 - -const DurationMillis = Schema.Int.check( - Schema.isGreaterThan(0), - Schema.isLessThanOrEqualTo(maximumNegotiatedDurationMillis) -) - -const Credential = Schema.optionalKey(Schema.RedactedFromValue(Schema.String)) -const BoundedName = Schema.NonEmptyString.check(Schema.isMaxLength(256)) -const BoundedPeerPrincipal = PeerSyncEnvelope.RelayPeerPrincipal -const MessageHash = PeerSyncEnvelope.RelayOuterEnvelope.fields.messageHash -const Payload = PeerSyncEnvelope.RelayOuterEnvelope.fields.payload - -export const ClaimToken = Schema.String.check( - Schema.isPattern(/^clm_[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/) -).pipe(Schema.brand("@lucas-barake/effect-local-rpc/ClaimToken")) -export type ClaimToken = typeof ClaimToken.Type - -export const RelayDigest = Schema.String.check(Schema.isPattern(/^[0-9a-f]{64}$/)) -export type RelayDigest = typeof RelayDigest.Type - -export const RelayOpened = Schema.TaggedStruct("RelayOpened", { - version: Schema.Literal(protocolVersion), - sessionId: Identity.SessionId, - remotePeerId: Identity.PeerId, - authenticatedLocal: BoundedPeerPrincipal, - capabilities: Schema.Struct({ storeAndForward: Schema.Literal(true) }) -}) -export type RelayOpened = typeof RelayOpened.Type - -export const StoredMessage = Schema.TaggedStruct("StoredMessage", { - relayMessageId: Identity.RelayMessageId, - claimToken: ClaimToken, - relayPeerId: Identity.PeerId, - sender: Schema.Struct({ - tenantId: PeerSyncEnvelope.RelayPeerPrincipal.fields.tenantId, - subjectId: PeerSyncEnvelope.RelayPeerPrincipal.fields.subjectId, - peerId: PeerSyncEnvelope.RelayPeerPrincipal.fields.peerId, - replicaIncarnation: Identity.ReplicaIncarnation, - connectionEpoch: PeerSyncEnvelope.RelayOuterEnvelope.fields.senderConnectionEpoch, - sequence: PeerSyncEnvelope.RelayOuterEnvelope.fields.senderSequence - }), - recipient: BoundedPeerPrincipal, - payloadVersion: PeerSyncEnvelope.RelayOuterEnvelope.fields.payloadVersion, - document: PeerSyncEnvelope.RelayOuterEnvelope.fields.document, - writerProvenance: PeerSyncEnvelope.RelayOuterEnvelope.fields.writerProvenance, - messageHash: MessageHash, - outerEnvelopeDigest: RelayDigest, - payload: Payload -}) -export type StoredMessage = typeof StoredMessage.Type - -export const OpenRelayEvent = Schema.Union([RelayOpened, StoredMessage]) -export type OpenRelayEvent = typeof OpenRelayEvent.Type - -export const RejectReason = Schema.Literals(["ProtocolInvalid", "ApplicationRejected"]) -export type RejectReason = typeof RejectReason.Type - -export class OpenRelayRpc extends Rpc.make("OpenRelay", { - payload: { - version: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), - expectedRelayPeerId: Identity.PeerId, - expectedLocal: BoundedPeerPrincipal, - senderReplicaIncarnation: Identity.ReplicaIncarnation, - remote: Schema.Struct({ - subjectId: BoundedName, - peerId: Identity.PeerId - }), - documents: Schema.Array(PeerRpc.RequestedDocument).check( - Schema.isMinLength(1), - Schema.isMaxLength(maximumRequestedDocuments) - ), - receiptRetentionMillis: DurationMillis, - senderRetryHorizonMillis: DurationMillis, - credential: Credential - }, - success: OpenRelayEvent, - error: PeerRpcError.PeerRpcError, - defect: PeerRpcError.Defect, - stream: true -}) {} - -export class PushRelayRpc extends Rpc.make("PushRelay", { - payload: { - sessionId: Identity.SessionId, - relayMessageId: Identity.RelayMessageId, - payload: Payload, - credential: Credential - }, - error: PeerRpcError.PeerRpcError, - defect: PeerRpcError.Defect -}) {} - -const TerminalPayload = { - sessionId: Identity.SessionId, - relayMessageId: Identity.RelayMessageId, - claimToken: ClaimToken, - messageHash: MessageHash, - credential: Credential -} - -export class AcknowledgeRelayRpc extends Rpc.make("AcknowledgeRelay", { - payload: TerminalPayload, - error: PeerRpcError.PeerRpcError, - defect: PeerRpcError.Defect -}) {} - -export class RejectRelayRpc extends Rpc.make("RejectRelay", { - payload: { - ...TerminalPayload, - reason: RejectReason - }, - error: PeerRpcError.PeerRpcError, - defect: PeerRpcError.Defect -}) {} - -export class Rpcs extends RpcGroup.make( - OpenRelayRpc, - PushRelayRpc, - AcknowledgeRelayRpc, - RejectRelayRpc -).middleware(PeerAuthentication.PeerAuthentication) {} - -export interface RpcClient extends RpcClient_.FromGroup {} - -export const makeRpcClient: Effect.Effect< - RpcClient, - never, - RpcClient_.Protocol | RpcMiddleware.ForClient | Scope.Scope -> = makeClient(Rpcs) diff --git a/packages/local-rpc/src/PeerRelayStore.ts b/packages/local-rpc/src/PeerRelayStore.ts index 27d1c7b..dbdecae 100644 --- a/packages/local-rpc/src/PeerRelayStore.ts +++ b/packages/local-rpc/src/PeerRelayStore.ts @@ -1,27 +1,11 @@ import * as Identity from "@lucas-barake/effect-local/Identity" -import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" +import type * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" import * as Context from "effect/Context" -import * as Crypto from "effect/Crypto" -import * as Effect from "effect/Effect" -import * as Exit from "effect/Exit" -import * as Layer from "effect/Layer" -import * as Option from "effect/Option" -import * as Random from "effect/Random" +import type * as Effect from "effect/Effect" +import type * as Option from "effect/Option" import * as Schema from "effect/Schema" -import type * as Migrator from "effect/unstable/sql/Migrator" -import * as SqlClient from "effect/unstable/sql/SqlClient" -import type * as SqlError from "effect/unstable/sql/SqlError" -import * as SqlSchema from "effect/unstable/sql/SqlSchema" import { PeerPrincipal } from "./internal/peerPrincipal.js" -import * as PeerRelayMigrations from "./internal/peerRelayMigrations.js" -import { - make as makeImmediateTransaction, - type NestedPeerRelayTransactionError -} from "./internal/peerRelaySqliteTransaction.js" -import { mapStoreErrors } from "./internal/peerRelayStoreErrors.js" -import * as PeerRpcObservability from "./internal/peerRpcObservability.js" -import * as PeerRelayLimits from "./PeerRelayLimits.js" -import * as PeerRelayRpc from "./PeerRelayRpc.js" +import * as PeerRpc from "./PeerRpc.js" const PositiveInt = Schema.Int.check(Schema.isGreaterThan(0)) const NonNegativeInt = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)) @@ -46,7 +30,7 @@ export const Admission = Schema.Struct({ senderSequence: NonNegativeInt, payloadVersion: Schema.Literal(1), messageHash: Schema.NonEmptyString.check(Schema.isMaxLength(256)), - outerEnvelopeDigest: PeerRelayRpc.RelayDigest, + outerEnvelopeDigest: PeerRpc.RelayDigest, payload: Schema.Uint8Array, messageTtlMillis: PositiveInt, senderRetryHorizonMillis: PositiveInt, @@ -84,9 +68,9 @@ export const ClaimedMessage = Schema.Struct({ documentIds: DocumentIds, payloadVersion: Schema.Literal(1), messageHash: Schema.NonEmptyString, - outerEnvelopeDigest: PeerRelayRpc.RelayDigest, + outerEnvelopeDigest: PeerRpc.RelayDigest, payloadBytes: NonNegativeInt, - claimToken: PeerRelayRpc.ClaimToken, + claimToken: PeerRpc.ClaimToken, claimDeadline: NonNegativeInt, sessionGeneration: NonNegativeInt }) @@ -103,7 +87,7 @@ export const LoadClaimedPayloadRequest = Schema.Struct({ rowId: PositiveInt, channel: ChannelKey, relayMessageId: Identity.RelayMessageId, - claimToken: PeerRelayRpc.ClaimToken, + claimToken: PeerRpc.ClaimToken, sessionGeneration: NonNegativeInt, payloadBytes: NonNegativeInt }) @@ -112,7 +96,7 @@ export type LoadClaimedPayloadRequest = typeof LoadClaimedPayloadRequest.Type export const TerminalRequest = Schema.Struct({ channel: ChannelKey, relayMessageId: Identity.RelayMessageId, - claimToken: PeerRelayRpc.ClaimToken, + claimToken: PeerRpc.ClaimToken, messageHash: Schema.NonEmptyString.check(Schema.isMaxLength(256)), sessionGeneration: NonNegativeInt, recipient: PeerPrincipal @@ -121,14 +105,14 @@ export type TerminalRequest = typeof TerminalRequest.Type export const RejectRequest = Schema.Struct({ ...TerminalRequest.fields, - reason: PeerRelayRpc.RejectReason + reason: PeerRpc.RejectReason }) export type RejectRequest = typeof RejectRequest.Type export const ReleaseRequest = Schema.Struct({ channel: ChannelKey, relayMessageId: Identity.RelayMessageId, - claimToken: PeerRelayRpc.ClaimToken, + claimToken: PeerRpc.ClaimToken, sessionGeneration: NonNegativeInt }) export type ReleaseRequest = typeof ReleaseRequest.Type @@ -165,9 +149,7 @@ export interface Usage { readonly retainedBytes: number } -export type StoreError = - | ReplicaError.ReplicaError - | NestedPeerRelayTransactionError +export type StoreError = ReplicaError.ReplicaError export interface Service { readonly admit: (input: Admission) => Effect.Effect @@ -189,1683 +171,3 @@ export interface Service { export class PeerRelayStore extends Context.Service()( "@lucas-barake/effect-local-rpc/PeerRelayStore" ) {} - -type UsageKind = UsageRequest["scopeKind"] - -interface UsageScope { - readonly kind: UsageKind - readonly key: string - readonly activeCountLimit: number - readonly activeBytesLimit: number - readonly retainedCountLimit: number - readonly retainedBytesLimit: number -} - -const storageCorrupt = (cause: unknown) => - new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ cause }) - }) - -const protocolMismatch = (expected: string, observed: string) => - new ReplicaError.ReplicaError({ - reason: new ReplicaError.ProtocolMismatch({ expected, observed }) - }) - -const quotaExceeded = (resource: string, limit: number) => - new ReplicaError.ReplicaError({ - reason: new ReplicaError.QuotaExceeded({ resource, limit }) - }) - -const relayFailureResult = (exit: Exit.Exit) => { - const error = PeerRpcObservability.failure(exit) - if (error?._tag !== "ReplicaError") return "Failure" as const - switch (error.reason._tag) { - case "ProtocolMismatch": - return "ProtocolRejected" as const - case "QuotaExceeded": - return "CapacityRejected" as const - case "StorageUnavailable": - return "Unavailable" as const - default: - return "Failure" as const - } -} - -const relayQuotaDomain = ( - error: StoreError -): Option.Option => { - if (error._tag !== "ReplicaError" || error.reason._tag !== "QuotaExceeded") { - return Option.none() - } - const resource = error.reason.resource - if (resource === "relay payload bytes") return Option.some("Payload") - for ( - const domain of [ - "SenderPeer", - "RecipientPeer", - "RecipientSubject", - "Tenant", - "Shard" - ] as const - ) { - if (resource === `${domain} relay custody`) return Option.some(domain) - } - return Option.none() -} - -const recordQuotaRejection = (error: StoreError) => - Option.match(relayQuotaDomain(error), { - onNone: () => Effect.void, - onSome: (domain) => - PeerRpcObservability.recordRelayQuotaRejection(domain).pipe( - Effect.catchCause(() => Effect.void) - ) - }) - -const encodeKey = (...parts: ReadonlyArray) => JSON.stringify(parts) - -const channelScopes = ( - channel: ChannelKey, - limits: PeerRelayLimits.Values -): ReadonlyArray => [ - { - kind: "SenderPeer", - key: encodeKey(channel.tenantId, channel.senderSubjectId, channel.senderPeerId), - activeCountLimit: limits.maxActiveMessagesPerSenderPeer, - activeBytesLimit: limits.maxActiveBytesPerSenderPeer, - retainedCountLimit: limits.maxRetainedRowsPerSenderPeer, - retainedBytesLimit: limits.maxRetainedBytesPerSenderPeer - }, - { - kind: "RecipientPeer", - key: encodeKey(channel.tenantId, channel.recipientSubjectId, channel.recipientPeerId), - activeCountLimit: limits.maxActiveMessagesPerRecipientPeer, - activeBytesLimit: limits.maxActiveBytesPerRecipientPeer, - retainedCountLimit: limits.maxRetainedRowsPerRecipientPeer, - retainedBytesLimit: limits.maxRetainedBytesPerRecipientPeer - }, - { - kind: "RecipientSubject", - key: encodeKey(channel.tenantId, channel.recipientSubjectId), - activeCountLimit: limits.maxActiveMessagesPerRecipientSubject, - activeBytesLimit: limits.maxActiveBytesPerRecipientSubject, - retainedCountLimit: limits.maxRetainedRowsPerRecipientSubject, - retainedBytesLimit: limits.maxRetainedBytesPerRecipientSubject - }, - { - kind: "Tenant", - key: encodeKey(channel.tenantId), - activeCountLimit: limits.maxActiveMessagesPerTenant, - activeBytesLimit: limits.maxActiveBytesPerTenant, - retainedCountLimit: limits.maxRetainedRowsPerTenant, - retainedBytesLimit: limits.maxRetainedBytesPerTenant - }, - { - kind: "Shard", - key: encodeKey("local"), - activeCountLimit: limits.maxActiveMessagesPerShard, - activeBytesLimit: limits.maxActiveBytesPerShard, - retainedCountLimit: limits.maxRetainedRowsPerShard, - retainedBytesLimit: limits.maxRetainedBytesPerShard - } -] - -const TimeRow = Schema.Struct({ now: NonNegativeInt }) -const UnitRow = Schema.Struct({ value: Schema.Int }) - -const nowQuery = (sql: SqlClient.SqlClient) => - SqlSchema.findOne({ - Request: Schema.Void, - Result: TimeRow, - execute: () => sql`SELECT CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) AS now` - })(undefined) - -const decodeDocuments = Schema.decodeUnknownEffect( - Schema.fromJsonString(DocumentIds) -) - -const parseDocuments = (value: string) => - decodeDocuments(value).pipe( - Effect.mapError((cause) => storageCorrupt(cause)) - ) - -const validateInput = (schema: S, input: unknown) => - Schema.decodeUnknownEffect(schema)(input).pipe( - Effect.mapError((cause) => protocolMismatch("valid relay store request", String(cause))) - ) - -const checkPragmas = Effect.gen(function*() { - const sql = yield* SqlClient.SqlClient - const journal = SqlSchema.findOne({ - Request: Schema.Void, - Result: Schema.Struct({ journal_mode: Schema.String }), - execute: () => sql`PRAGMA journal_mode = WAL` - }) - const synchronous = SqlSchema.findOne({ - Request: Schema.Void, - Result: Schema.Struct({ synchronous: Schema.Int }), - execute: () => sql`PRAGMA synchronous` - }) - const mode = yield* journal(undefined) - if (mode.journal_mode.toLowerCase() !== "wal") { - return yield* storageCorrupt(new Error("Relay custody requires SQLite WAL mode")) - } - yield* sql`PRAGMA synchronous = FULL` - const setting = yield* synchronous(undefined) - if (setting.synchronous !== 2) { - return yield* storageCorrupt(new Error("Relay custody requires SQLite FULL synchronous mode")) - } -}) - -const DuplicateRow = Schema.Struct({ - channelId: PositiveInt, - tenantId: Schema.NonEmptyString, - senderSubjectId: Schema.NonEmptyString, - senderPeerId: Identity.PeerId, - senderReplicaIncarnation: Identity.ReplicaIncarnation, - recipientSubjectId: Schema.NonEmptyString, - recipientPeerId: Identity.PeerId, - outerEnvelopeDigest: PeerRelayRpc.RelayDigest, - state: Schema.Literals(["Pending", "Claimed", "Acknowledged", "DeadLettered", "Expired"]), - nextEligibleAt: NonNegativeInt -}) - -const ChannelRow = Schema.Struct({ - channelId: PositiveInt, - nextSequence: NonNegativeInt -}) - -const MessageIdRow = Schema.Struct({ messageId: PositiveInt }) - -const CandidateRow = Schema.Struct({ - messageId: PositiveInt, - channelId: PositiveInt, - tenantId: Schema.NonEmptyString, - senderSubjectId: Schema.NonEmptyString, - senderPeerId: Identity.PeerId, - senderReplicaIncarnation: Identity.ReplicaIncarnation, - recipientSubjectId: Schema.NonEmptyString, - recipientPeerId: Identity.PeerId, - relayMessageId: Identity.RelayMessageId, - relayPeerId: Identity.PeerId, - senderConnectionEpoch: Schema.NonEmptyString, - senderSequence: NonNegativeInt, - documentIds: Schema.String, - payloadVersion: Schema.Literal(1), - messageHash: Schema.NonEmptyString, - outerEnvelopeDigest: PeerRelayRpc.RelayDigest, - payloadBytes: NonNegativeInt, - createdAt: NonNegativeInt, - nextEligibleAt: NonNegativeInt, - retryCount: NonNegativeInt -}) - -const ReservationRow = Schema.Struct({ - senderPeerUsageKey: Schema.String, - recipientPeerUsageKey: Schema.String, - recipientSubjectUsageKey: Schema.String, - tenantUsageKey: Schema.String, - shardUsageKey: Schema.String, - activeCountDelta: Schema.Literal(1), - activeBytesDelta: NonNegativeInt, - retainedCountDelta: Schema.Literal(1), - retainedBytesDelta: NonNegativeInt, - activeConsumed: Schema.Literals([0, 1]), - retainedConsumed: Schema.Literals([0, 1]) -}) - -const UsageRow = Schema.Struct({ - activeCount: NonNegativeInt, - activeBytes: NonNegativeInt, - retainedCount: NonNegativeInt, - retainedBytes: NonNegativeInt -}) - -const KeyRow = Schema.Struct({ messageId: PositiveInt }) - -const makeService = Effect.gen(function*() { - const sql = yield* SqlClient.SqlClient - const crypto = yield* Crypto.Crypto - const limits = yield* PeerRelayLimits.PeerRelayLimits - yield* PeerRelayMigrations.run - yield* checkPragmas.pipe( - Effect.catchTag("NoSuchElementError", (cause) => Effect.fail(storageCorrupt(cause))) - ) - - const write = makeImmediateTransaction(sql, { - maxAcquireAttempts: limits.sqliteLockRetryMaxAttempts, - acquireRetryBaseDelayMillis: limits.sqliteLockRetryBaseDelayMillis, - acquireRetryMaximumDelayMillis: limits.sqliteLockRetryMaximumDelayMillis - }) - - const findDuplicate = SqlSchema.findOneOption({ - Request: Schema.Struct({ - tenantId: Schema.String, - senderSubjectId: Schema.String, - senderPeerId: Schema.String, - relayMessageId: Schema.String - }), - Result: DuplicateRow, - execute: (request) => - sql`SELECT - m.channel_id AS channelId, - c.tenant_id AS tenantId, - c.sender_subject_id AS senderSubjectId, - c.sender_peer_id AS senderPeerId, - c.sender_replica_incarnation AS senderReplicaIncarnation, - c.recipient_subject_id AS recipientSubjectId, - c.recipient_peer_id AS recipientPeerId, - m.outer_envelope_digest AS outerEnvelopeDigest, - m.state, - m.next_eligible_at AS nextEligibleAt - FROM effect_local_relay_messages m - JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id - JOIN effect_local_relay_reservations r ON r.message_id = m.message_id - WHERE m.tenant_id = ${request.tenantId} - AND m.sender_subject_id = ${request.senderSubjectId} - AND m.sender_peer_id = ${request.senderPeerId} - AND m.relay_message_id = ${request.relayMessageId}` - }) - - const findChannel = SqlSchema.findOne({ - Request: ChannelKey, - Result: ChannelRow, - execute: (channel) => - sql`SELECT - channel_id AS channelId, - next_sequence AS nextSequence - FROM effect_local_relay_channels - WHERE tenant_id = ${channel.tenantId} - AND sender_subject_id = ${channel.senderSubjectId} - AND sender_peer_id = ${channel.senderPeerId} - AND sender_replica_incarnation = ${channel.senderReplicaIncarnation} - AND recipient_subject_id = ${channel.recipientSubjectId} - AND recipient_peer_id = ${channel.recipientPeerId}` - }) - - const reserveUsage = ( - scope: UsageScope, - payloadBytes: number - ): Effect.Effect => - Effect.gen(function*() { - yield* sql`INSERT INTO effect_local_relay_usage ( - scope_kind, - scope_key, - active_count, - active_bytes, - retained_count, - retained_bytes - ) VALUES (${scope.kind}, ${scope.key}, 0, 0, 0, 0) - ON CONFLICT(scope_kind, scope_key) DO NOTHING` - const changed = yield* SqlSchema.findAll({ - Request: Schema.Void, - Result: UnitRow, - execute: () => - sql`UPDATE effect_local_relay_usage - SET active_count = active_count + 1, - active_bytes = active_bytes + ${payloadBytes}, - retained_count = retained_count + 1, - retained_bytes = retained_bytes + ${payloadBytes} - WHERE scope_kind = ${scope.kind} - AND scope_key = ${scope.key} - AND active_count + 1 <= ${scope.activeCountLimit} - AND active_bytes + ${payloadBytes} <= ${scope.activeBytesLimit} - AND retained_count + 1 <= ${scope.retainedCountLimit} - AND retained_bytes + ${payloadBytes} <= ${scope.retainedBytesLimit} - RETURNING 1 AS value` - })(undefined) - if (changed.length !== 1) { - return yield* quotaExceeded( - `${scope.kind} relay custody`, - Math.min(scope.activeCountLimit, scope.retainedCountLimit) - ) - } - }) - - const findReservation = SqlSchema.findOneOption({ - Request: PositiveInt, - Result: ReservationRow, - execute: (messageId) => - sql`SELECT - sender_peer_usage_key AS senderPeerUsageKey, - recipient_peer_usage_key AS recipientPeerUsageKey, - recipient_subject_usage_key AS recipientSubjectUsageKey, - tenant_usage_key AS tenantUsageKey, - shard_usage_key AS shardUsageKey, - active_count_delta AS activeCountDelta, - active_bytes_delta AS activeBytesDelta, - retained_count_delta AS retainedCountDelta, - retained_bytes_delta AS retainedBytesDelta, - active_consumed AS activeConsumed, - retained_consumed AS retainedConsumed - FROM effect_local_relay_reservations - WHERE message_id = ${messageId}` - }) - - const usageKeys = ( - reservation: typeof ReservationRow.Type - ): ReadonlyArray => [ - ["SenderPeer", reservation.senderPeerUsageKey], - ["RecipientPeer", reservation.recipientPeerUsageKey], - ["RecipientSubject", reservation.recipientSubjectUsageKey], - ["Tenant", reservation.tenantUsageKey], - ["Shard", reservation.shardUsageKey] - ] - - const releaseActiveUsage = ( - messageId: number - ): Effect.Effect => - Effect.gen(function*() { - const reservationOption = yield* findReservation(messageId) - if (Option.isNone(reservationOption)) { - return yield* storageCorrupt(new Error("Missing relay quota reservation")) - } - const reservation = reservationOption.value - if (reservation.activeConsumed === 1) { - return - } - for (const [kind, key] of usageKeys(reservation)) { - const changed = yield* SqlSchema.findAll({ - Request: Schema.Void, - Result: UnitRow, - execute: () => - sql`UPDATE effect_local_relay_usage - SET active_count = active_count - ${reservation.activeCountDelta}, - active_bytes = active_bytes - ${reservation.activeBytesDelta} - WHERE scope_kind = ${kind} - AND scope_key = ${key} - AND active_count >= ${reservation.activeCountDelta} - AND active_bytes >= ${reservation.activeBytesDelta} - RETURNING 1 AS value` - })(undefined) - if (changed.length !== 1) { - return yield* storageCorrupt(new Error("Invalid active relay quota reservation")) - } - yield* sql`DELETE FROM effect_local_relay_usage - WHERE scope_kind = ${kind} - AND scope_key = ${key} - AND active_count = 0 - AND active_bytes = 0 - AND retained_count = 0 - AND retained_bytes = 0` - } - yield* sql`UPDATE effect_local_relay_reservations - SET active_consumed = 1 - WHERE message_id = ${messageId} AND active_consumed = 0` - }) - - const releaseRetainedUsage = ( - messageId: number - ): Effect.Effect => - Effect.gen(function*() { - const reservationOption = yield* findReservation(messageId) - if (Option.isNone(reservationOption)) { - return yield* storageCorrupt(new Error("Missing relay quota reservation")) - } - const reservation = reservationOption.value - if (reservation.retainedConsumed === 1) { - return - } - for (const [kind, key] of usageKeys(reservation)) { - const changed = yield* SqlSchema.findAll({ - Request: Schema.Void, - Result: UnitRow, - execute: () => - sql`UPDATE effect_local_relay_usage - SET retained_count = retained_count - ${reservation.retainedCountDelta}, - retained_bytes = retained_bytes - ${reservation.retainedBytesDelta} - WHERE scope_kind = ${kind} - AND scope_key = ${key} - AND retained_count >= ${reservation.retainedCountDelta} - AND retained_bytes >= ${reservation.retainedBytesDelta} - RETURNING 1 AS value` - })(undefined) - if (changed.length !== 1) { - return yield* storageCorrupt(new Error("Invalid retained relay quota reservation")) - } - yield* sql`DELETE FROM effect_local_relay_usage - WHERE scope_kind = ${kind} - AND scope_key = ${key} - AND active_count = 0 - AND active_bytes = 0 - AND retained_count = 0 - AND retained_bytes = 0` - } - yield* sql`UPDATE effect_local_relay_reservations - SET retained_consumed = 1 - WHERE message_id = ${messageId} AND retained_consumed = 0` - }) - - const terminalize = ( - messageId: number, - state: "Acknowledged" | "DeadLettered" | "Expired", - now: number, - terminal: { - readonly token?: PeerRelayRpc.ClaimToken - readonly sessionGeneration?: number - readonly reason: string - } - ) => - Effect.gen(function*() { - yield* releaseActiveUsage(messageId) - yield* sql`UPDATE effect_local_relay_messages - SET state = ${state}, - payload = NULL, - payload_length = 0, - claim_token = NULL, - claim_session_generation = NULL, - claim_deadline = NULL, - terminal_at = ${now}, - terminal_claim_token = ${terminal.token ?? null}, - terminal_session_generation = ${terminal.sessionGeneration ?? null}, - terminal_reason = ${terminal.reason}, - deduplicate_until = MAX( - deduplicate_until, - ${now + limits.minimumTerminalRetentionMillis} - ) - WHERE message_id = ${messageId} - AND state IN ('Pending', 'Claimed')` - yield* sql`UPDATE effect_local_relay_channels - SET claimed_message_id = NULL, - claim_session_generation = NULL, - claim_token = NULL, - claim_deadline = NULL - WHERE claimed_message_id = ${messageId}` - }) - - const admit: Service["admit"] = (unsafeInput) => { - let observedBytes: number | undefined - let observedVersion: number | undefined - const effect = mapStoreErrors(Effect.gen(function*() { - const input = yield* validateInput(Admission, unsafeInput) - const payloadBytes = input.payload.byteLength - observedBytes = payloadBytes - observedVersion = input.payloadVersion - if ( - input.messageTtlMillis > limits.messageTtlMillis || - input.senderRetryHorizonMillis > limits.maximumSenderRetryHorizonMillis || - input.minimumTerminalRetentionMillis < limits.minimumTerminalRetentionMillis - ) { - return yield* protocolMismatch("configured relay retention horizon", "unsupported horizon") - } - if ( - payloadBytes > limits.maxActiveBytesPerSenderPeer || - payloadBytes > limits.maxActiveBytesPerRecipientPeer || - payloadBytes > limits.maxActiveBytesPerRecipientSubject || - payloadBytes > limits.maxActiveBytesPerTenant || - payloadBytes > limits.maxActiveBytesPerShard - ) { - return yield* quotaExceeded("relay payload bytes", limits.maxActiveBytesPerShard) - } - return yield* write(Effect.gen(function*() { - const now = (yield* nowQuery(sql)).now - const existing = yield* findDuplicate({ - tenantId: input.channel.tenantId, - senderSubjectId: input.channel.senderSubjectId, - senderPeerId: input.channel.senderPeerId, - relayMessageId: input.relayMessageId - }) - if (Option.isSome(existing)) { - if (existing.value.outerEnvelopeDigest !== input.outerEnvelopeDigest) { - return yield* protocolMismatch( - existing.value.outerEnvelopeDigest, - input.outerEnvelopeDigest - ) - } - return AdmissionResult.make({ - status: "Duplicate", - channel: ChannelKey.make({ - tenantId: existing.value.tenantId, - senderSubjectId: existing.value.senderSubjectId, - senderPeerId: existing.value.senderPeerId, - senderReplicaIncarnation: existing.value.senderReplicaIncarnation, - recipientSubjectId: existing.value.recipientSubjectId, - recipientPeerId: existing.value.recipientPeerId - }), - ready: existing.value.state === "Pending" && existing.value.nextEligibleAt <= now, - nextEligibleAt: existing.value.nextEligibleAt, - lane: "New" - }) - } - yield* sql`INSERT INTO effect_local_relay_channels ( - tenant_id, - sender_subject_id, - sender_peer_id, - sender_replica_incarnation, - recipient_subject_id, - recipient_peer_id - ) VALUES ( - ${input.channel.tenantId}, - ${input.channel.senderSubjectId}, - ${input.channel.senderPeerId}, - ${input.channel.senderReplicaIncarnation}, - ${input.channel.recipientSubjectId}, - ${input.channel.recipientPeerId} - ) - ON CONFLICT ( - tenant_id, - sender_subject_id, - sender_peer_id, - sender_replica_incarnation, - recipient_subject_id, - recipient_peer_id - ) DO NOTHING` - const channel = yield* findChannel(input.channel) - const scopes = channelScopes(input.channel, limits) - for (const scope of scopes) { - yield* reserveUsage(scope, payloadBytes) - } - const duplicateHorizon = Math.max( - input.messageTtlMillis, - input.senderRetryHorizonMillis - ) - const inserted = yield* SqlSchema.findOne({ - Request: Schema.Void, - Result: MessageIdRow, - execute: () => - sql`INSERT INTO effect_local_relay_messages ( - channel_id, - channel_sequence, - tenant_id, - sender_subject_id, - sender_peer_id, - recipient_subject_id, - recipient_peer_id, - relay_message_id, - relay_peer_id, - sender_connection_epoch, - sender_sequence, - document_ids, - payload_version, - message_hash, - outer_envelope_digest, - payload, - payload_length, - state, - created_at, - expires_at, - deduplicate_until, - next_eligible_at - ) VALUES ( - ${channel.channelId}, - ${channel.nextSequence}, - ${input.channel.tenantId}, - ${input.channel.senderSubjectId}, - ${input.channel.senderPeerId}, - ${input.channel.recipientSubjectId}, - ${input.channel.recipientPeerId}, - ${input.relayMessageId}, - ${input.relayPeerId}, - ${input.senderConnectionEpoch}, - ${input.senderSequence}, - ${JSON.stringify(input.documentIds)}, - ${input.payloadVersion}, - ${input.messageHash}, - ${input.outerEnvelopeDigest}, - ${input.payload}, - ${payloadBytes}, - 'Pending', - ${now}, - ${now + input.messageTtlMillis}, - ${now + duplicateHorizon}, - ${now} - ) - RETURNING message_id AS messageId` - })(undefined) - yield* sql`INSERT INTO effect_local_relay_reservations ( - message_id, - sender_peer_usage_key, - recipient_peer_usage_key, - recipient_subject_usage_key, - tenant_usage_key, - shard_usage_key, - active_count_delta, - active_bytes_delta, - retained_count_delta, - retained_bytes_delta - ) VALUES ( - ${inserted.messageId}, - ${scopes[0]!.key}, - ${scopes[1]!.key}, - ${scopes[2]!.key}, - ${scopes[3]!.key}, - ${scopes[4]!.key}, - 1, - ${payloadBytes}, - 1, - ${payloadBytes} - )` - yield* sql`UPDATE effect_local_relay_channels - SET next_sequence = next_sequence + 1 - WHERE channel_id = ${channel.channelId} - AND next_sequence = ${channel.nextSequence}` - return AdmissionResult.make({ - status: "Accepted", - channel: input.channel, - ready: true, - nextEligibleAt: now, - lane: "New" - }) - })) - })).pipe(Effect.tapError(recordQuotaRejection)) - return PeerRpcObservability.observeRelay({ - effect, - operation: "RelayAdmit", - direction: "Send", - facts: () => ({ - ...(observedBytes === undefined ? {} : { bytes: observedBytes }), - ...(observedVersion === undefined ? {} : { version: observedVersion }), - items: observedBytes === undefined ? 0 : 1 - }), - result: (exit) => - Exit.isSuccess(exit) - ? exit.value.status - : relayFailureResult(exit) - }) - } - - const findCandidate = SqlSchema.findOneOption({ - Request: Schema.Struct({ - tenantId: Schema.String, - recipientSubjectId: Schema.String, - recipientPeerId: Schema.String, - senderSubjectId: Schema.String, - senderPeerId: Schema.String, - authorizedDocumentIds: Schema.String, - now: NonNegativeInt - }), - Result: CandidateRow, - execute: (request) => - sql`SELECT - m.message_id AS messageId, - m.channel_id AS channelId, - c.tenant_id AS tenantId, - c.sender_subject_id AS senderSubjectId, - c.sender_peer_id AS senderPeerId, - c.sender_replica_incarnation AS senderReplicaIncarnation, - c.recipient_subject_id AS recipientSubjectId, - c.recipient_peer_id AS recipientPeerId, - m.relay_message_id AS relayMessageId, - m.relay_peer_id AS relayPeerId, - m.sender_connection_epoch AS senderConnectionEpoch, - m.sender_sequence AS senderSequence, - m.document_ids AS documentIds, - m.payload_version AS payloadVersion, - m.message_hash AS messageHash, - m.outer_envelope_digest AS outerEnvelopeDigest, - m.payload_length AS payloadBytes, - m.created_at AS createdAt, - m.next_eligible_at AS nextEligibleAt, - m.retry_count AS retryCount - FROM effect_local_relay_messages m - JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id - JOIN effect_local_relay_reservations r ON r.message_id = m.message_id - WHERE m.tenant_id = ${request.tenantId} - AND m.sender_subject_id = ${request.senderSubjectId} - AND m.sender_peer_id = ${request.senderPeerId} - AND m.recipient_subject_id = ${request.recipientSubjectId} - AND m.recipient_peer_id = ${request.recipientPeerId} - AND c.tenant_id = ${request.tenantId} - AND c.recipient_subject_id = ${request.recipientSubjectId} - AND c.recipient_peer_id = ${request.recipientPeerId} - AND c.sender_subject_id = ${request.senderSubjectId} - AND c.sender_peer_id = ${request.senderPeerId} - AND m.tenant_id = c.tenant_id - AND m.sender_subject_id = c.sender_subject_id - AND m.sender_peer_id = c.sender_peer_id - AND m.recipient_subject_id = c.recipient_subject_id - AND m.recipient_peer_id = c.recipient_peer_id - AND c.claimed_message_id IS NULL - AND m.state = 'Pending' - AND m.next_eligible_at <= ${request.now} - AND m.expires_at > ${request.now} - AND m.payload IS NOT NULL - AND m.payload_length = length(m.payload) - AND r.active_consumed = 0 - AND NOT EXISTS ( - SELECT 1 - FROM effect_local_relay_messages earlier - WHERE earlier.channel_id = m.channel_id - AND earlier.channel_sequence < m.channel_sequence - AND earlier.state IN ('Pending', 'Claimed') - ) - AND NOT EXISTS ( - SELECT 1 - FROM json_each(m.document_ids) document - WHERE document.value NOT IN ( - SELECT value FROM json_each(${request.authorizedDocumentIds}) - ) - ) - ORDER BY m.created_at, m.message_id - LIMIT 1` - }) - - const claim: Service["claim"] = (unsafeInput) => { - let observedAttempt: number | undefined - const effect = mapStoreErrors(Effect.gen(function*() { - const input = yield* validateInput(ClaimRequest, unsafeInput) - return yield* write(Effect.gen(function*() { - const now = (yield* nowQuery(sql)).now - const candidateOption = yield* findCandidate({ - tenantId: input.recipient.tenantId, - recipientSubjectId: input.recipient.subjectId, - recipientPeerId: input.recipient.peerId, - senderSubjectId: input.sender.subjectId, - senderPeerId: input.sender.peerId, - authorizedDocumentIds: JSON.stringify(input.authorizedDocumentIds), - now - }) - if (Option.isNone(candidateOption)) { - return { - message: Option.none(), - ready: false, - nextEligibleAt: Option.none(), - lane: "New" as const - } - } - const candidate = candidateOption.value - observedAttempt = candidate.retryCount + 1 - const documentIds = yield* parseDocuments(candidate.documentIds) - const uuid = yield* crypto.randomUUIDv4 - const token = PeerRelayRpc.ClaimToken.make(`clm_${uuid}`) - const deadline = now + limits.claimLeaseMillis - const claimed = yield* SqlSchema.findAll({ - Request: Schema.Void, - Result: UnitRow, - execute: () => - sql`UPDATE effect_local_relay_messages - SET state = 'Claimed', - claim_token = ${token}, - claim_session_generation = ${input.sessionGeneration}, - claim_deadline = ${deadline} - WHERE message_id = ${candidate.messageId} - AND state = 'Pending' - RETURNING 1 AS value` - })(undefined) - if (claimed.length !== 1) { - return yield* storageCorrupt(new Error("Relay claim compare and swap failed")) - } - const channelClaimed = yield* SqlSchema.findAll({ - Request: Schema.Void, - Result: UnitRow, - execute: () => - sql`UPDATE effect_local_relay_channels - SET claimed_message_id = ${candidate.messageId}, - claim_session_generation = ${input.sessionGeneration}, - claim_token = ${token}, - claim_deadline = ${deadline} - WHERE channel_id = ${candidate.channelId} - AND claimed_message_id IS NULL - RETURNING 1 AS value` - })(undefined) - if (channelClaimed.length !== 1) { - return yield* storageCorrupt(new Error("Relay channel claim compare and swap failed")) - } - return { - message: Option.some(ClaimedMessage.make({ - rowId: candidate.messageId, - channel: ChannelKey.make({ - tenantId: candidate.tenantId, - senderSubjectId: candidate.senderSubjectId, - senderPeerId: candidate.senderPeerId, - senderReplicaIncarnation: candidate.senderReplicaIncarnation, - recipientSubjectId: candidate.recipientSubjectId, - recipientPeerId: candidate.recipientPeerId - }), - relayMessageId: candidate.relayMessageId, - relayPeerId: candidate.relayPeerId, - senderConnectionEpoch: candidate.senderConnectionEpoch, - senderSequence: candidate.senderSequence, - documentIds, - payloadVersion: candidate.payloadVersion, - messageHash: candidate.messageHash, - outerEnvelopeDigest: candidate.outerEnvelopeDigest, - payloadBytes: candidate.payloadBytes, - claimToken: token, - claimDeadline: deadline, - sessionGeneration: input.sessionGeneration - })), - ready: false, - nextEligibleAt: Option.none(), - lane: candidate.retryCount === 0 ? "New" as const : "Retry" as const - } - })) - })) - return PeerRpcObservability.observeRelay({ - effect, - operation: "RelayClaim", - direction: "Receive", - facts: (exit) => { - if (Exit.isFailure(exit) || Option.isNone(exit.value.message)) { - return { items: 0 } - } - const message = exit.value.message.value - return { - bytes: message.payloadBytes, - items: 1, - ...(observedAttempt === undefined ? {} : { attempt: observedAttempt }), - version: message.payloadVersion - } - }, - result: (exit) => - Exit.isSuccess(exit) - ? Option.isSome(exit.value.message) ? "Claimed" : "Empty" - : relayFailureResult(exit) - }) - } - - const loadClaimedPayload: Service["loadClaimedPayload"] = (unsafeInput) => - mapStoreErrors(Effect.gen(function*() { - const input = yield* validateInput(LoadClaimedPayloadRequest, unsafeInput) - const rows = yield* SqlSchema.findAll({ - Request: Schema.Void, - Result: Schema.Struct({ payload: Schema.Uint8Array }), - execute: () => - sql`SELECT m.payload - FROM effect_local_relay_messages m - JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id - WHERE m.message_id = ${input.rowId} - AND m.channel_id = c.channel_id - AND c.tenant_id = ${input.channel.tenantId} - AND c.sender_subject_id = ${input.channel.senderSubjectId} - AND c.sender_peer_id = ${input.channel.senderPeerId} - AND c.sender_replica_incarnation = ${input.channel.senderReplicaIncarnation} - AND c.recipient_subject_id = ${input.channel.recipientSubjectId} - AND c.recipient_peer_id = ${input.channel.recipientPeerId} - AND m.tenant_id = c.tenant_id - AND m.sender_subject_id = c.sender_subject_id - AND m.sender_peer_id = c.sender_peer_id - AND m.relay_message_id = ${input.relayMessageId} - AND m.state = 'Claimed' - AND m.claim_token = ${input.claimToken} - AND m.claim_session_generation = ${input.sessionGeneration} - AND c.claimed_message_id = m.message_id - AND c.claim_token = m.claim_token - AND c.claim_session_generation = m.claim_session_generation - AND c.claim_deadline = m.claim_deadline - AND m.claim_deadline > CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) - AND m.expires_at > CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) - AND m.payload_length = ${input.payloadBytes} - AND length(m.payload) = ${input.payloadBytes} - AND m.payload_length <= ${limits.maxActiveBytesPerShard}` - })(undefined) - if (rows.length !== 1) { - return yield* protocolMismatch("active relay claim", "stale relay claim") - } - return rows[0]!.payload - })) - - const TerminalRow = Schema.Struct({ - messageId: PositiveInt, - channelId: PositiveInt, - state: Schema.Literals(["Pending", "Claimed", "Acknowledged", "DeadLettered", "Expired"]), - messageHash: Schema.NonEmptyString, - claimToken: Schema.NullOr(PeerRelayRpc.ClaimToken), - claimSessionGeneration: Schema.NullOr(NonNegativeInt), - claimDeadline: Schema.NullOr(NonNegativeInt), - createdAt: NonNegativeInt, - expiresAt: NonNegativeInt, - channelClaimedMessageId: Schema.NullOr(PositiveInt), - channelClaimToken: Schema.NullOr(PeerRelayRpc.ClaimToken), - channelClaimSessionGeneration: Schema.NullOr(NonNegativeInt), - channelClaimDeadline: Schema.NullOr(NonNegativeInt), - terminalClaimToken: Schema.NullOr(PeerRelayRpc.ClaimToken), - terminalSessionGeneration: Schema.NullOr(NonNegativeInt), - terminalReason: Schema.NullOr(Schema.String) - }) - - const findTerminal = SqlSchema.findOneOption({ - Request: Schema.Struct({ - channel: ChannelKey, - relayMessageId: Identity.RelayMessageId - }), - Result: TerminalRow, - execute: ({ channel, relayMessageId }) => - sql`SELECT - m.message_id AS messageId, - m.channel_id AS channelId, - m.state, - m.message_hash AS messageHash, - m.claim_token AS claimToken, - m.claim_session_generation AS claimSessionGeneration, - m.claim_deadline AS claimDeadline, - m.created_at AS createdAt, - m.expires_at AS expiresAt, - c.claimed_message_id AS channelClaimedMessageId, - c.claim_token AS channelClaimToken, - c.claim_session_generation AS channelClaimSessionGeneration, - c.claim_deadline AS channelClaimDeadline, - m.terminal_claim_token AS terminalClaimToken, - m.terminal_session_generation AS terminalSessionGeneration, - m.terminal_reason AS terminalReason - FROM effect_local_relay_messages m - JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id - WHERE c.tenant_id = ${channel.tenantId} - AND c.sender_subject_id = ${channel.senderSubjectId} - AND c.sender_peer_id = ${channel.senderPeerId} - AND c.sender_replica_incarnation = ${channel.senderReplicaIncarnation} - AND c.recipient_subject_id = ${channel.recipientSubjectId} - AND c.recipient_peer_id = ${channel.recipientPeerId} - AND m.relay_message_id = ${relayMessageId}` - }) - - const ReadyRow = Schema.Struct({ - nextEligibleAt: NonNegativeInt, - retryCount: NonNegativeInt - }) - - const nextReady = ( - channel: ChannelKey, - now: number - ): Effect.Effect< - { readonly ready: boolean; readonly nextEligibleAt: Option.Option; readonly lane: "New" | "Retry" }, - SqlError.SqlError | Schema.SchemaError - > => - SqlSchema.findOneOption({ - Request: Schema.Void, - Result: ReadyRow, - execute: () => - sql`SELECT - m.next_eligible_at AS nextEligibleAt, - m.retry_count AS retryCount - FROM effect_local_relay_messages m - JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id - WHERE c.tenant_id = ${channel.tenantId} - AND c.sender_subject_id = ${channel.senderSubjectId} - AND c.sender_peer_id = ${channel.senderPeerId} - AND c.recipient_subject_id = ${channel.recipientSubjectId} - AND c.recipient_peer_id = ${channel.recipientPeerId} - AND c.claimed_message_id IS NULL - AND m.state = 'Pending' - AND m.expires_at > ${now} - AND NOT EXISTS ( - SELECT 1 - FROM effect_local_relay_messages earlier - WHERE earlier.channel_id = m.channel_id - AND earlier.channel_sequence < m.channel_sequence - AND earlier.state IN ('Pending', 'Claimed') - ) - ORDER BY m.next_eligible_at, m.created_at, m.message_id - LIMIT 1` - })(undefined).pipe( - Effect.map((row) => - Option.isNone(row) - ? { - ready: false, - nextEligibleAt: Option.none(), - lane: "New" as const - } - : { - ready: row.value.nextEligibleAt <= now, - nextEligibleAt: Option.some(row.value.nextEligibleAt), - lane: row.value.retryCount === 0 ? "New" as const : "Retry" as const - } - ) - ) - - const terminalTransition = ( - unsafeInput: TerminalRequest, - state: "Acknowledged" | "DeadLettered", - reason: string, - onChanged?: (latencyMillis: number) => void - ): Effect.Effect => - mapStoreErrors(Effect.gen(function*() { - const input = yield* validateInput(TerminalRequest, unsafeInput) - if ( - input.recipient.tenantId !== input.channel.tenantId || - input.recipient.subjectId !== input.channel.recipientSubjectId || - input.recipient.peerId !== input.channel.recipientPeerId - ) { - return { - status: "Stale", - ready: false, - nextEligibleAt: Option.none(), - lane: "New" - } as const - } - return yield* write(Effect.gen(function*() { - const now = (yield* nowQuery(sql)).now - const rowOption = yield* findTerminal({ - channel: input.channel, - relayMessageId: input.relayMessageId - }) - if (Option.isNone(rowOption)) { - return { - status: "Stale", - ready: false, - nextEligibleAt: Option.none(), - lane: "New" - } as const - } - const row = rowOption.value - const active = row.state === "Claimed" && - row.messageHash === input.messageHash && - row.claimToken === input.claimToken && - row.claimSessionGeneration === input.sessionGeneration && - row.claimDeadline !== null && - row.claimDeadline > now && - row.expiresAt > now && - row.channelClaimedMessageId === row.messageId && - row.channelClaimToken === row.claimToken && - row.channelClaimSessionGeneration === row.claimSessionGeneration && - row.channelClaimDeadline === row.claimDeadline - if (active) { - yield* terminalize(row.messageId, state, now, { - token: input.claimToken, - sessionGeneration: input.sessionGeneration, - reason - }) - if (onChanged !== undefined) { - yield* Effect.sync(() => onChanged(now - row.createdAt)) - } - const hint = yield* nextReady(input.channel, now) - return { status: "Changed", ...hint } as const - } - const duplicate = row.state === state && - row.messageHash === input.messageHash && - row.terminalClaimToken === input.claimToken && - row.terminalSessionGeneration === input.sessionGeneration && - row.terminalReason === reason - if (duplicate) { - const hint = yield* nextReady(input.channel, now) - return { status: "Duplicate", ...hint } as const - } - return { - status: "Stale", - ready: false, - nextEligibleAt: Option.none(), - lane: "New" - } as const - })) - })) - - const acknowledge: Service["acknowledge"] = (input) => { - let latencyMillis: number | undefined - const effect = terminalTransition( - input, - "Acknowledged", - "Acknowledged", - (latency) => { - latencyMillis = latency - } - ) - return PeerRpcObservability.observeRelay({ - effect, - operation: "RelayAcknowledge", - direction: "Receive", - facts: (exit) => ({ - items: Exit.isSuccess(exit) && exit.value.status !== "Stale" ? 1 : 0, - ...(Exit.isSuccess(exit) && - exit.value.status === "Changed" && - latencyMillis !== undefined - ? { latencyMillis } - : {}) - }), - result: (exit) => - Exit.isSuccess(exit) - ? exit.value.status === "Changed" - ? "Acknowledged" - : exit.value.status - : relayFailureResult(exit) - }) - } - - const reject: Service["reject"] = (unsafeInput) => { - const effect = mapStoreErrors(Effect.gen(function*() { - const input = yield* validateInput(RejectRequest, unsafeInput) - return yield* terminalTransition( - TerminalRequest.make({ - channel: input.channel, - relayMessageId: input.relayMessageId, - claimToken: input.claimToken, - messageHash: input.messageHash, - sessionGeneration: input.sessionGeneration, - recipient: input.recipient - }), - "DeadLettered", - input.reason - ) - })) - return PeerRpcObservability.observeRelay({ - effect, - operation: "RelayAcknowledge", - direction: "Receive", - facts: (exit) => ({ - items: Exit.isSuccess(exit) && exit.value.status !== "Stale" ? 1 : 0 - }), - result: (exit) => - Exit.isSuccess(exit) - ? exit.value.status === "Changed" - ? "DeadLettered" - : exit.value.status - : relayFailureResult(exit) - }) - } - - const release: Service["release"] = (unsafeInput) => { - const effect = mapStoreErrors(Effect.gen(function*() { - const input = yield* validateInput(ReleaseRequest, unsafeInput) - return yield* write(Effect.gen(function*() { - const now = (yield* nowQuery(sql)).now - const rowOption = yield* findTerminal({ - channel: input.channel, - relayMessageId: input.relayMessageId - }) - if (Option.isNone(rowOption)) { - return { - status: "Stale", - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" - } as const - } - const row = rowOption.value - if ( - row.state !== "Claimed" || - row.claimToken !== input.claimToken || - row.claimSessionGeneration !== input.sessionGeneration - ) { - return { - status: "Stale", - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" - } as const - } - const retry = yield* SqlSchema.findOne({ - Request: Schema.Void, - Result: Schema.Struct({ retryCount: PositiveInt }), - execute: () => - sql`SELECT retry_count + 1 AS retryCount - FROM effect_local_relay_messages - WHERE message_id = ${row.messageId}` - })(undefined) - if (retry.retryCount >= limits.maximumDeliveryAttempts) { - yield* sql`UPDATE effect_local_relay_messages - SET retry_count = ${retry.retryCount} - WHERE message_id = ${row.messageId} - AND state = 'Claimed' - AND claim_token = ${input.claimToken} - AND claim_session_generation = ${input.sessionGeneration}` - yield* terminalize(row.messageId, "DeadLettered", now, { - reason: "MaximumDeliveryAttempts" - }) - return { - status: "Changed", - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" - } as const - } - const random = yield* Random.next - const maximum = Math.min( - limits.retryMaximumDelayMillis, - limits.retryBaseDelayMillis * 2 ** Math.min(retry.retryCount - 1, 30) - ) - const delay = Math.max(1, Math.floor(maximum / 2 + random * maximum / 2)) - const nextEligibleAt = now + delay - yield* sql`UPDATE effect_local_relay_messages - SET state = 'Pending', - retry_count = ${retry.retryCount}, - next_eligible_at = ${nextEligibleAt}, - claim_token = NULL, - claim_session_generation = NULL, - claim_deadline = NULL - WHERE message_id = ${row.messageId} - AND state = 'Claimed' - AND claim_token = ${input.claimToken} - AND claim_session_generation = ${input.sessionGeneration}` - yield* sql`UPDATE effect_local_relay_channels - SET claimed_message_id = NULL, - claim_session_generation = NULL, - claim_token = NULL, - claim_deadline = NULL - WHERE channel_id = ${row.channelId} - AND claimed_message_id = ${row.messageId} - AND claim_token = ${input.claimToken} - AND claim_session_generation = ${input.sessionGeneration}` - return { - status: "Changed", - ready: false, - nextEligibleAt: Option.some(nextEligibleAt), - lane: "Retry" - } as const - })) - })) - return PeerRpcObservability.observeRelay({ - effect, - operation: "RelayRelease", - direction: "Receive", - facts: (exit) => ({ - items: Exit.isSuccess(exit) && exit.value.status === "Changed" ? 1 : 0 - }), - result: (exit) => - Exit.isSuccess(exit) - ? exit.value.status === "Changed" ? "Released" : exit.value.status - : relayFailureResult(exit) - }) - } - - const maintenanceResult = ( - ids: ReadonlyArray, - requestedBatchSize: number - ): MaintenanceResult => { - const processedIds = ids.slice(0, requestedBatchSize) - const cursor = processedIds.at(-1) - return cursor === undefined - ? { processed: 0, hasMore: false } - : { - cursor, - processed: processedIds.length, - hasMore: ids.length > requestedBatchSize - } - } - - const recover: Service["recover"] = (unsafeInput) => - Effect.suspend(() => { - let deadLettered = 0 - const effect = mapStoreErrors(Effect.gen(function*() { - const input = yield* validateInput(MaintenanceRequest, unsafeInput) - const effectiveBatch = Math.min(input.batchSize, limits.claimRecoveryBatchSize) - return yield* write(Effect.gen(function*() { - const now = (yield* nowQuery(sql)).now - const rows = yield* SqlSchema.findAll({ - Request: Schema.Void, - Result: Schema.Struct({ - messageId: PositiveInt, - retryCount: NonNegativeInt - }), - execute: () => - sql`SELECT - message_id AS messageId, - retry_count AS retryCount - FROM effect_local_relay_messages - WHERE state = 'Claimed' - AND claim_deadline <= ${now} - ORDER BY claim_deadline, message_id - LIMIT ${effectiveBatch + 1}` - })(undefined) - for (const row of rows.slice(0, effectiveBatch)) { - const retryCount = row.retryCount + 1 - if (retryCount >= limits.maximumDeliveryAttempts) { - deadLettered += 1 - yield* sql`UPDATE effect_local_relay_messages - SET retry_count = ${retryCount} - WHERE message_id = ${row.messageId} - AND state = 'Claimed' - AND claim_deadline <= ${now}` - yield* terminalize(row.messageId, "DeadLettered", now, { - reason: "MaximumDeliveryAttempts" - }) - continue - } - const random = yield* Random.next - const maximum = Math.min( - limits.retryMaximumDelayMillis, - limits.retryBaseDelayMillis * 2 ** Math.min(retryCount - 1, 30) - ) - const delay = Math.max(1, Math.floor(maximum / 2 + random * maximum / 2)) - yield* sql`UPDATE effect_local_relay_messages - SET state = 'Pending', - retry_count = ${retryCount}, - next_eligible_at = ${now + delay}, - claim_token = NULL, - claim_session_generation = NULL, - claim_deadline = NULL - WHERE message_id = ${row.messageId} - AND state = 'Claimed' - AND claim_deadline <= ${now}` - yield* sql`UPDATE effect_local_relay_channels - SET claimed_message_id = NULL, - claim_session_generation = NULL, - claim_token = NULL, - claim_deadline = NULL - WHERE claimed_message_id = ${row.messageId} - AND claim_deadline <= ${now}` - } - return maintenanceResult(rows.map((row) => row.messageId), effectiveBatch) - })) - })) - return PeerRpcObservability.observeRelay({ - effect, - operation: "RelayMaintenance", - direction: "Receive", - stage: "Recover", - facts: (exit) => ({ - items: Exit.isSuccess(exit) ? exit.value.processed : 0 - }), - result: (exit) => - Exit.isSuccess(exit) - ? deadLettered > 0 ? "DeadLettered" : "Released" - : relayFailureResult(exit) - }) - }) - - const expire: Service["expire"] = (unsafeInput) => { - const effect = mapStoreErrors(Effect.gen(function*() { - const input = yield* validateInput(MaintenanceRequest, unsafeInput) - const effectiveBatch = Math.min(input.batchSize, limits.expiryBatchSize) - return yield* write(Effect.gen(function*() { - const now = (yield* nowQuery(sql)).now - const rows = yield* SqlSchema.findAll({ - Request: Schema.Void, - Result: KeyRow, - execute: () => - sql`SELECT message_id AS messageId - FROM effect_local_relay_messages - WHERE state IN ('Pending', 'Claimed') - AND expires_at <= ${now} - ORDER BY expires_at, message_id - LIMIT ${effectiveBatch + 1}` - })(undefined) - for (const row of rows.slice(0, effectiveBatch)) { - yield* terminalize(row.messageId, "Expired", now, { reason: "Expired" }) - } - return maintenanceResult(rows.map((row) => row.messageId), effectiveBatch) - })) - })) - return PeerRpcObservability.observeRelay({ - effect, - operation: "RelayMaintenance", - direction: "Receive", - stage: "Expire", - facts: (exit) => ({ - items: Exit.isSuccess(exit) ? exit.value.processed : 0 - }), - result: (exit) => Exit.isSuccess(exit) ? "Expired" : relayFailureResult(exit) - }) - } - - const IntegrityRow = Schema.Struct({ - messageId: PositiveInt, - state: Schema.String, - messageTenantId: Schema.NonEmptyString, - channelTenantId: Schema.NullOr(Schema.NonEmptyString), - messageSenderSubjectId: Schema.NonEmptyString, - channelSenderSubjectId: Schema.NullOr(Schema.NonEmptyString), - messageSenderPeerId: Schema.String, - channelSenderPeerId: Schema.NullOr(Schema.String), - messageRecipientSubjectId: Schema.NullOr(Schema.String), - channelRecipientSubjectId: Schema.NullOr(Schema.String), - messageRecipientPeerId: Schema.NullOr(Schema.String), - channelRecipientPeerId: Schema.NullOr(Schema.String), - payloadLength: NonNegativeInt, - actualLength: Schema.NullOr(NonNegativeInt), - reservationPresent: Schema.Literals([0, 1]), - activeConsumed: Schema.NullOr(Schema.Literals([0, 1])), - retainedConsumed: Schema.NullOr(Schema.Literals([0, 1])) - }) - - const repair: Service["repair"] = (unsafeInput) => { - const effect = mapStoreErrors(Effect.gen(function*() { - const input = yield* validateInput(MaintenanceRequest, unsafeInput) - const effectiveBatch = Math.min(input.batchSize, limits.integrityBatchSize) - return yield* write(Effect.gen(function*() { - const now = (yield* nowQuery(sql)).now - const rows = yield* SqlSchema.findAll({ - Request: Schema.Void, - Result: IntegrityRow, - execute: () => - sql`SELECT - m.message_id AS messageId, - m.state, - m.tenant_id AS messageTenantId, - c.tenant_id AS channelTenantId, - m.sender_subject_id AS messageSenderSubjectId, - c.sender_subject_id AS channelSenderSubjectId, - m.sender_peer_id AS messageSenderPeerId, - c.sender_peer_id AS channelSenderPeerId, - m.recipient_subject_id AS messageRecipientSubjectId, - c.recipient_subject_id AS channelRecipientSubjectId, - m.recipient_peer_id AS messageRecipientPeerId, - c.recipient_peer_id AS channelRecipientPeerId, - m.payload_length AS payloadLength, - length(m.payload) AS actualLength, - CASE WHEN r.message_id IS NULL THEN 0 ELSE 1 END AS reservationPresent, - r.active_consumed AS activeConsumed, - r.retained_consumed AS retainedConsumed - FROM effect_local_relay_messages m - LEFT JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id - LEFT JOIN effect_local_relay_reservations r ON r.message_id = m.message_id - WHERE m.message_id > ${input.cursor ?? 0} - ORDER BY m.message_id - LIMIT ${effectiveBatch + 1}` - })(undefined) - for (const row of rows.slice(0, effectiveBatch)) { - if ( - row.reservationPresent === 0 || - row.activeConsumed === null || - row.retainedConsumed === null - ) { - return yield* storageCorrupt(new Error("Relay reservation reconciliation required")) - } - const active = row.state === "Pending" || row.state === "Claimed" - const terminal = row.state === "Acknowledged" || - row.state === "DeadLettered" || - row.state === "Expired" - if ( - (active && row.activeConsumed !== 0) || - (terminal && row.activeConsumed !== 1) - ) { - return yield* storageCorrupt(new Error("Corrupt relay reservation entitlement")) - } - const corrupt = (!active && !terminal) || - row.messageTenantId !== row.channelTenantId || - row.messageSenderSubjectId !== row.channelSenderSubjectId || - row.messageSenderPeerId !== row.channelSenderPeerId || - row.messageRecipientSubjectId !== row.channelRecipientSubjectId || - row.messageRecipientPeerId !== row.channelRecipientPeerId || - (active && ( - row.actualLength === null || - row.actualLength !== row.payloadLength - )) || - (terminal && ( - row.actualLength !== null || - row.payloadLength !== 0 - )) - if (corrupt && active) { - yield* terminalize(row.messageId, "DeadLettered", now, { reason: "Corrupt" }) - } else if (corrupt) { - return yield* storageCorrupt(new Error("Corrupt terminal relay row")) - } - } - return maintenanceResult(rows.map((row) => row.messageId), effectiveBatch) - })) - })) - return PeerRpcObservability.observeRelay({ - effect, - operation: "RelayMaintenance", - direction: "Receive", - stage: "Repair", - facts: (exit) => ({ - items: Exit.isSuccess(exit) ? exit.value.processed : 0 - }), - result: (exit) => Exit.isSuccess(exit) ? "DeadLettered" : relayFailureResult(exit) - }) - } - - const reconcile: Service["reconcile"] = (unsafeInput) => { - const effect = mapStoreErrors(Effect.gen(function*() { - const input = yield* validateInput(MaintenanceRequest, unsafeInput) - const effectiveBatch = Math.min(input.batchSize, limits.reconciliationBatchSize) - return yield* write(Effect.gen(function*() { - const messageRows = yield* SqlSchema.findAll({ - Request: Schema.Void, - Result: KeyRow, - execute: () => - sql`SELECT message_id AS messageId - FROM effect_local_relay_messages - WHERE message_id > ${input.cursor ?? 0} - ORDER BY message_id - LIMIT ${effectiveBatch + 1}` - })(undefined) - const reservationRows = yield* SqlSchema.findAll({ - Request: Schema.Void, - Result: KeyRow, - execute: () => - sql`SELECT message_id AS messageId - FROM effect_local_relay_reservations - WHERE message_id > ${input.cursor ?? 0} - ORDER BY message_id - LIMIT ${effectiveBatch + 1}` - })(undefined) - if ( - messageRows.length !== reservationRows.length || - messageRows.some((row, index) => row.messageId !== reservationRows[index]?.messageId) - ) { - return yield* storageCorrupt(new Error("Relay reservation bijection is corrupt")) - } - return maintenanceResult(messageRows.map((row) => row.messageId), effectiveBatch) - })) - })) - return PeerRpcObservability.observeRelay({ - effect, - operation: "RelayMaintenance", - direction: "Receive", - stage: "Reconcile", - facts: (exit) => ({ - items: Exit.isSuccess(exit) ? exit.value.processed : 0 - }), - result: (exit) => Exit.isSuccess(exit) ? "Success" : relayFailureResult(exit) - }) - } - - const collect: Service["collect"] = (unsafeInput) => { - const effect = mapStoreErrors(Effect.gen(function*() { - const input = yield* validateInput(MaintenanceRequest, unsafeInput) - const effectiveBatch = Math.min(input.batchSize, limits.terminalCollectionBatchSize) - return yield* write(Effect.gen(function*() { - const now = (yield* nowQuery(sql)).now - const rows = yield* SqlSchema.findAll({ - Request: Schema.Void, - Result: Schema.Struct({ - messageId: PositiveInt, - channelId: PositiveInt - }), - execute: () => - sql`SELECT - message_id AS messageId, - channel_id AS channelId - FROM effect_local_relay_messages - WHERE state IN ('Acknowledged', 'DeadLettered', 'Expired') - AND deduplicate_until <= ${now} - ORDER BY deduplicate_until, message_id - LIMIT ${effectiveBatch + 1}` - })(undefined) - for (const row of rows.slice(0, effectiveBatch)) { - yield* releaseRetainedUsage(row.messageId) - const reservationDeleted = yield* SqlSchema.findAll({ - Request: Schema.Void, - Result: UnitRow, - execute: () => - sql`DELETE FROM effect_local_relay_reservations - WHERE message_id = ${row.messageId} - AND active_consumed = 1 - AND retained_consumed = 1 - RETURNING 1 AS value` - })(undefined) - if (reservationDeleted.length !== 1) { - return yield* storageCorrupt(new Error("Invalid collected relay quota reservation")) - } - yield* sql`DELETE FROM effect_local_relay_messages - WHERE message_id = ${row.messageId} - AND state IN ('Acknowledged', 'DeadLettered', 'Expired') - AND deduplicate_until <= ${now}` - yield* sql`DELETE FROM effect_local_relay_channels - WHERE channel_id = ${row.channelId} - AND claimed_message_id IS NULL - AND NOT EXISTS ( - SELECT 1 - FROM effect_local_relay_messages m - WHERE m.channel_id = ${row.channelId} - )` - } - return maintenanceResult(rows.map((row) => row.messageId), effectiveBatch) - })) - })) - return PeerRpcObservability.observeRelay({ - effect, - operation: "RelayMaintenance", - direction: "Receive", - stage: "Collect", - facts: (exit) => ({ - items: Exit.isSuccess(exit) ? exit.value.processed : 0 - }), - result: (exit) => Exit.isSuccess(exit) ? "Success" : relayFailureResult(exit) - }) - } - - const usage: Service["usage"] = (unsafeInput) => { - const exactShard = unsafeInput === undefined - const effect = mapStoreErrors(Effect.gen(function*() { - const input = unsafeInput === undefined - ? UsageRequest.make({ scopeKind: "Shard", scopeKey: encodeKey("local") }) - : yield* validateInput(UsageRequest, unsafeInput) - const row = yield* SqlSchema.findOneOption({ - Request: Schema.Void, - Result: UsageRow, - execute: () => - sql`SELECT - active_count AS activeCount, - active_bytes AS activeBytes, - retained_count AS retainedCount, - retained_bytes AS retainedBytes - FROM effect_local_relay_usage - WHERE scope_kind = ${input.scopeKind} - AND scope_key = ${input.scopeKey}` - })(undefined) - return Option.getOrElse(row, (): Usage => ({ - activeCount: 0, - activeBytes: 0, - retainedCount: 0, - retainedBytes: 0 - })) - })) - return PeerRpcObservability.observeRelay({ - effect, - operation: "RelayMaintenance", - direction: "Receive", - stage: "Usage", - facts: (exit) => ({ - items: Exit.isSuccess(exit) ? exit.value.activeCount : 0, - ...(Exit.isSuccess(exit) ? { bytes: exit.value.activeBytes, version: 1 } : {}) - }), - result: (exit) => Exit.isSuccess(exit) ? "Success" : relayFailureResult(exit) - }).pipe( - Effect.tap((value) => - exactShard - ? PeerRpcObservability.setRelayPending( - value.activeCount, - value.activeBytes - ).pipe(Effect.catchCause(() => Effect.void)) - : Effect.void - ) - ) - } - - return PeerRelayStore.of({ - admit, - claim, - loadClaimedPayload, - acknowledge, - reject, - release, - recover, - expire, - repair, - reconcile, - collect, - usage - }) -}) - -export const make = makeService - -export const layerSqlite: Layer.Layer< - PeerRelayStore, - | Migrator.MigrationError - | SqlError.SqlError - | Schema.SchemaError - | ReplicaError.ReplicaError, - SqlClient.SqlClient | Crypto.Crypto | PeerRelayLimits.PeerRelayLimits -> = Layer.effect(PeerRelayStore, makeService) diff --git a/packages/local-rpc/src/PeerRpc.ts b/packages/local-rpc/src/PeerRpc.ts index 106b8c9..1cefad5 100644 --- a/packages/local-rpc/src/PeerRpc.ts +++ b/packages/local-rpc/src/PeerRpc.ts @@ -1,3 +1,4 @@ +import * as PeerSyncEnvelope from "@lucas-barake/effect-local-sql/PeerSyncEnvelope" import * as Identity from "@lucas-barake/effect-local/Identity" import type * as Effect from "effect/Effect" import * as Schema from "effect/Schema" @@ -8,10 +9,26 @@ import type * as RpcClient_ from "effect/unstable/rpc/RpcClient" import type { RpcClientError } from "effect/unstable/rpc/RpcClientError" import * as RpcGroup from "effect/unstable/rpc/RpcGroup" import type * as RpcMiddleware from "effect/unstable/rpc/RpcMiddleware" +import * as PeerRpcProtocol from "./internal/peerRpcProtocol.js" import * as PeerAuthentication from "./PeerAuthentication.js" import * as PeerRpcError from "./PeerRpcError.js" -export const protocolVersion = 2 +export const protocolVersion = PeerSyncEnvelope.relayProtocolVersion + +export const maximumNegotiatedDurationMillis = PeerRpcProtocol.maximumNegotiatedDurationMillis +export const maximumRelayPayloadBytes = PeerRpcProtocol.maximumRelayPayloadBytes +export const maximumRequestedDocuments = 1_000 + +const DurationMillis = Schema.Int.check( + Schema.isGreaterThan(0), + Schema.isLessThanOrEqualTo(maximumNegotiatedDurationMillis) +) + +const Credential = Schema.optionalKey(Schema.RedactedFromValue(Schema.String)) +const BoundedName = Schema.NonEmptyString.check(Schema.isMaxLength(256)) +const BoundedPeerPrincipal = PeerSyncEnvelope.RelayPeerPrincipal +const MessageHash = PeerSyncEnvelope.RelayOuterEnvelope.fields.messageHash +const Payload = PeerSyncEnvelope.RelayOuterEnvelope.fields.payload export const RequestedDocument = Schema.Struct({ documentType: Schema.NonEmptyString, @@ -19,59 +36,68 @@ export const RequestedDocument = Schema.Struct({ }) export type RequestedDocument = typeof RequestedDocument.Type +export const ClaimToken = Schema.String.check( + Schema.isPattern(/^clm_[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/) +).pipe(Schema.brand("@lucas-barake/effect-local-rpc/ClaimToken")) +export type ClaimToken = typeof ClaimToken.Type + +export const RelayDigest = Schema.String.check(Schema.isPattern(/^[0-9a-f]{64}$/)) +export type RelayDigest = typeof RelayDigest.Type + export const Opened = Schema.TaggedStruct("Opened", { protocolVersion: Schema.Literal(protocolVersion), sessionId: Identity.SessionId, - peerId: Identity.PeerId, - // `lineageAware` is `optionalKey` and never `Schema.Boolean` alone: this struct decodes every - // `Opened` frame, including one from a peer built before lineage existed, and a required key - // would make that frame fail to decode instead of reading as "not lineage aware". - capabilities: Schema.Struct({ - storeAndForward: Schema.Literal(false), - lineageAware: Schema.optionalKey(Schema.Boolean) - }) + remotePeerId: Identity.PeerId, + authenticatedLocal: BoundedPeerPrincipal }) export type Opened = typeof Opened.Type -export const Message = Schema.TaggedStruct("Message", { - payload: Schema.Uint8Array +export const StoredMessage = Schema.TaggedStruct("StoredMessage", { + relayMessageId: Identity.RelayMessageId, + claimToken: ClaimToken, + relayPeerId: Identity.PeerId, + sender: Schema.Struct({ + tenantId: PeerSyncEnvelope.RelayPeerPrincipal.fields.tenantId, + subjectId: PeerSyncEnvelope.RelayPeerPrincipal.fields.subjectId, + peerId: PeerSyncEnvelope.RelayPeerPrincipal.fields.peerId, + replicaIncarnation: Identity.ReplicaIncarnation, + connectionEpoch: PeerSyncEnvelope.RelayOuterEnvelope.fields.senderConnectionEpoch, + sequence: PeerSyncEnvelope.RelayOuterEnvelope.fields.senderSequence + }), + recipient: BoundedPeerPrincipal, + payloadVersion: PeerSyncEnvelope.RelayOuterEnvelope.fields.payloadVersion, + document: PeerSyncEnvelope.RelayOuterEnvelope.fields.document, + writerProvenance: PeerSyncEnvelope.RelayOuterEnvelope.fields.writerProvenance, + messageHash: MessageHash, + outerEnvelopeDigest: RelayDigest, + payload: Payload }) -export type Message = typeof Message.Type +export type StoredMessage = typeof StoredMessage.Type -export const OpenEvent = Schema.Union([Opened, Message]) +export const OpenEvent = Schema.Union([Opened, StoredMessage]) export type OpenEvent = typeof OpenEvent.Type -const Credential = Schema.optionalKey(Schema.RedactedFromValue(Schema.String)) - -export const DefinitionHash = Schema.String.check(Schema.isPattern(/^def_[0-9a-f]{16}$/)) - -const OpenPayload = Schema.Struct({ - protocolVersion: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), - expectedPeerId: Identity.PeerId, - definitionHash: Schema.optionalKey(DefinitionHash), - documents: Schema.Array(RequestedDocument), - // What the opening client claims about itself, and the only thing that tells the server whether - // this peer compares document lineage before it merges. `optionalKey` at both levels and never a - // required key: a client built before lineage sends neither, and a required key would make its - // `Open` fail to decode instead of reading as "not lineage aware". - // - // `protocolVersion` was deliberately not bumped for lineage, so an older build passes the version - // check and this advertisement is the only thing that distinguishes it. Absent is the fail closed - // answer: such a client unions whatever it is given, so a rewritten document sent to it comes - // straight back carrying the history the rewrite discarded. - capabilities: Schema.optionalKey(Schema.Struct({ - lineageAware: Schema.optionalKey(Schema.Boolean) - })), - credential: Credential -}).check( - Schema.makeFilter( - (request) => request.protocolVersion !== protocolVersion || request.definitionHash !== undefined, - { expected: `definitionHash for protocol version ${protocolVersion}` } - ) -) +export const RejectReason = Schema.Literals(["ProtocolInvalid", "ApplicationRejected"]) +export type RejectReason = typeof RejectReason.Type export class OpenRpc extends Rpc.make("Open", { - payload: OpenPayload, + payload: { + protocolVersion: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), + expectedRelayPeerId: Identity.PeerId, + expectedLocal: BoundedPeerPrincipal, + senderReplicaIncarnation: Identity.ReplicaIncarnation, + remote: Schema.Struct({ + subjectId: BoundedName, + peerId: Identity.PeerId + }), + documents: Schema.Array(RequestedDocument).check( + Schema.isMinLength(1), + Schema.isMaxLength(maximumRequestedDocuments) + ), + receiptRetentionMillis: DurationMillis, + senderRetryHorizonMillis: DurationMillis, + credential: Credential + }, success: OpenEvent, error: PeerRpcError.PeerRpcError, defect: PeerRpcError.Defect, @@ -81,14 +107,43 @@ export class OpenRpc extends Rpc.make("Open", { export class PushRpc extends Rpc.make("Push", { payload: { sessionId: Identity.SessionId, - payload: Schema.Uint8Array, + relayMessageId: Identity.RelayMessageId, + payload: Payload, credential: Credential }, error: PeerRpcError.PeerRpcError, defect: PeerRpcError.Defect }) {} -export class Rpcs extends RpcGroup.make(OpenRpc, PushRpc).middleware(PeerAuthentication.PeerAuthentication) {} +const TerminalPayload = { + sessionId: Identity.SessionId, + relayMessageId: Identity.RelayMessageId, + claimToken: ClaimToken, + messageHash: MessageHash, + credential: Credential +} + +export class AcknowledgeRpc extends Rpc.make("Acknowledge", { + payload: TerminalPayload, + error: PeerRpcError.PeerRpcError, + defect: PeerRpcError.Defect +}) {} + +export class RejectRpc extends Rpc.make("Reject", { + payload: { + ...TerminalPayload, + reason: RejectReason + }, + error: PeerRpcError.PeerRpcError, + defect: PeerRpcError.Defect +}) {} + +export class Rpcs extends RpcGroup.make( + OpenRpc, + PushRpc, + AcknowledgeRpc, + RejectRpc +).middleware(PeerAuthentication.PeerAuthentication) {} export interface RpcClient extends RpcClient_.FromGroup {} diff --git a/packages/local-rpc/src/PeerRpcLimits.ts b/packages/local-rpc/src/PeerRpcLimits.ts deleted file mode 100644 index 0ff9cef..0000000 --- a/packages/local-rpc/src/PeerRpcLimits.ts +++ /dev/null @@ -1,108 +0,0 @@ -import * as PeerSession from "@lucas-barake/effect-local-sql/PeerSession" -import * as ReplicaLimits from "@lucas-barake/effect-local/ReplicaLimits" -import * as Context from "effect/Context" -import * as Effect from "effect/Effect" -import * as Layer from "effect/Layer" -import * as Schema from "effect/Schema" - -const PositiveInt = Schema.Int.check(Schema.isGreaterThan(0)) -const PositiveNumber = Schema.Number.check(Schema.isFinite(), Schema.isGreaterThan(0)) - -export const Values = Schema.Struct({ - maxSessionsPerSubject: PositiveInt, - inboundItemCapacity: PositiveInt, - outboundItemCapacity: PositiveInt, - maxInboundBufferedBytesPerSession: PositiveInt, - maxOutboundBufferedBytesPerSession: PositiveInt, - maxBufferedBytes: PositiveInt, - maxInFlightAuthentication: PositiveInt, - authenticationRatePerSecond: PositiveNumber, - authenticationBurst: PositiveInt, - maxInFlightOpen: PositiveInt, - maxInFlightOpenPerSubject: PositiveInt, - maxInFlightPush: PositiveInt, - maxInFlightPushPerSubject: PositiveInt, - openRatePerSecond: PositiveNumber, - openBurst: PositiveInt, - pushRatePerSecond: PositiveNumber, - pushBurst: PositiveInt, - maxRetainedRateLimitedConnections: PositiveInt, - maxRetainedRateLimitedSubjects: PositiveInt, - rateLimitIdleRetention: PositiveInt, - maximumReauthorizationInterval: PositiveInt, - commitFlushConcurrency: PositiveInt, - shutdownCleanupConcurrency: PositiveInt -}) -export type Values = typeof Values.Type - -export const defaults: Values = Values.make({ - maxSessionsPerSubject: 4, - inboundItemCapacity: 1, - outboundItemCapacity: 1, - maxInboundBufferedBytesPerSession: 4 * 1_024 * 1_024, - maxOutboundBufferedBytesPerSession: 4 * 1_024 * 1_024, - maxBufferedBytes: 64 * 1_024 * 1_024, - maxInFlightAuthentication: 64, - authenticationRatePerSecond: 16, - authenticationBurst: 32, - maxInFlightOpen: 16, - maxInFlightOpenPerSubject: 2, - maxInFlightPush: 128, - maxInFlightPushPerSubject: 8, - openRatePerSecond: 2, - openBurst: 4, - pushRatePerSecond: 64, - pushBurst: 128, - maxRetainedRateLimitedConnections: 10_000, - maxRetainedRateLimitedSubjects: 10_000, - rateLimitIdleRetention: 10 * 60_000, - maximumReauthorizationInterval: 5 * 60_000, - commitFlushConcurrency: 8, - shutdownCleanupConcurrency: 16 -}) - -export class InvalidPeerRpcLimits extends Schema.TaggedErrorClass( - "@lucas-barake/effect-local-rpc/PeerRpcLimits/InvalidPeerRpcLimits" -)("InvalidPeerRpcLimits", { field: Schema.String }) {} - -export class PeerRpcLimits extends Context.Service()( - "@lucas-barake/effect-local-rpc/PeerRpcLimits" -) {} - -const validate = (values: Values, replicaLimits: ReplicaLimits.Values) => { - const envelope = PeerSession.maximumSyncEnvelopeBytes( - replicaLimits.maxSyncMessageBytes, - replicaLimits.maxSyncChangesPerMessage - ) - const checks: ReadonlyArray = [ - ["maxInboundBufferedBytesPerSession", values.maxInboundBufferedBytesPerSession >= envelope], - ["maxOutboundBufferedBytesPerSession", values.maxOutboundBufferedBytesPerSession >= envelope], - [ - "inboundItemCapacity", - values.inboundItemCapacity * envelope <= values.maxInboundBufferedBytesPerSession - ], - [ - "outboundItemCapacity", - values.outboundItemCapacity * envelope <= values.maxOutboundBufferedBytesPerSession - ], - [ - "maxBufferedBytes", - values.maxBufferedBytes >= envelope - ] - ] - const invalid = checks.find(([, valid]) => !valid) - return invalid === undefined - ? Effect.succeed(values) - : Effect.fail(new InvalidPeerRpcLimits({ field: invalid[0] })) -} - -export const make = (values: Values) => - Values.makeEffect(values).pipe( - Effect.flatMap((validated) => - ReplicaLimits.ReplicaLimits.use((replicaLimits) => validate(validated, replicaLimits)) - ) - ) - -export const layer = (values: Values) => Layer.effect(PeerRpcLimits, make(values)) - -export const layerDefaults = layer(defaults) diff --git a/packages/local-rpc/src/PeerRpcServer.ts b/packages/local-rpc/src/PeerRpcServer.ts index cb9c8c7..cf4b426 100644 --- a/packages/local-rpc/src/PeerRpcServer.ts +++ b/packages/local-rpc/src/PeerRpcServer.ts @@ -1,19 +1,11 @@ -import * as CommitPublisher from "@lucas-barake/effect-local-sql/CommitPublisher" -import * as PeerSession from "@lucas-barake/effect-local-sql/PeerSession" import * as PeerSyncEnvelope from "@lucas-barake/effect-local-sql/PeerSyncEnvelope" import * as Canonical from "@lucas-barake/effect-local/Canonical" import * as Identity from "@lucas-barake/effect-local/Identity" -import * as PeerTransport from "@lucas-barake/effect-local/PeerTransport" -import type * as ReplicaDefinition from "@lucas-barake/effect-local/ReplicaDefinition" -import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" -import * as ReplicaLimits from "@lucas-barake/effect-local/ReplicaLimits" -import * as Arr from "effect/Array" -import * as Cause from "effect/Cause" +import type * as Cause from "effect/Cause" import * as Clock from "effect/Clock" import * as Context from "effect/Context" import * as Crypto from "effect/Crypto" import * as Deferred from "effect/Deferred" -import * as Duration from "effect/Duration" import * as Effect from "effect/Effect" import * as Exit from "effect/Exit" import * as Fiber from "effect/Fiber" @@ -25,1210 +17,14 @@ import * as Scope from "effect/Scope" import * as Semaphore from "effect/Semaphore" import * as Stream from "effect/Stream" import * as RpcServer from "effect/unstable/rpc/RpcServer" -import type * as SocketServer from "effect/unstable/socket/SocketServer" -import * as PeerAuthorizationValidation from "./internal/peerAuthorization.js" import * as PeerRpcObservability from "./internal/peerRpcObservability.js" import * as PeerAuthentication from "./PeerAuthentication.js" -import * as PeerAuthorization from "./PeerAuthorization.js" import * as PeerRelayAuthorization from "./PeerRelayAuthorization.js" import * as PeerRelayIngress from "./PeerRelayIngress.js" import * as PeerRelayLimits from "./PeerRelayLimits.js" -import * as PeerRelayRpc from "./PeerRelayRpc.js" import * as PeerRelayStore from "./PeerRelayStore.js" import * as PeerRpc from "./PeerRpc.js" import * as PeerRpcError from "./PeerRpcError.js" -import * as PeerRpcLimits from "./PeerRpcLimits.js" - -interface InboundItem { - readonly id: number - readonly payload: Uint8Array -} - -interface OutboundItem { - readonly id: number - readonly payload: Uint8Array -} - -interface Entry { - readonly tenantId: string - readonly subjectId: string - readonly peerId: Identity.PeerId - readonly sessionId: Identity.SessionId - /** - * What the client advertised on its own `Open`, never anything inferred from the protocol version. - * - * `PeerRpc.protocolVersion` was not bumped when lineage was added, so a client running an older - * build passes the version check and would be inferred lineage aware. It would then be sent a - * rewritten document, union it, and reply with the history the rewrite discarded. Absent means - * not lineage aware. - */ - readonly lineageAware: boolean - readonly validUntil: number - readonly scope: Scope.Closeable - readonly inbound: Queue.Queue - readonly outbound: Queue.Queue - readonly outboundPermits: Semaphore.Semaphore - readonly inboundConsumerStarted: Deferred.Deferred - readonly closed: Deferred.Deferred - readonly terminal: Deferred.Deferred - readonly documents: ReadonlyArray - readonly selectedIds: ReadonlySet - readonly dirty: Set - readonly inboundReservations: Map - readonly outboundReservations: Map - outboundWaiter: ByteCapacityWaiter | undefined - active: boolean - cleanupStarted: boolean - queued: boolean - inboundBytes: number - session: PeerSession.PeerSession | undefined - watcher: Fiber.Fiber | undefined - requestFiber: Fiber.Fiber | undefined -} - -interface SubjectState { - openTokens: number - pushTokens: number - openUpdatedAt: number - pushUpdatedAt: number - lastUsedAt: number - openInFlight: number - pushInFlight: number - activeSessions: number -} - -interface Registry { - accepting: boolean - bufferedBytes: number - readonly sessions: Map - readonly cleanups: Map - readonly peers: Map - readonly documents: Map> - readonly subjects: Map - readonly outboundWaiters: Map -} - -interface ByteCapacityWaiter { - readonly id: number - readonly bytes: number - readonly ready: Deferred.Deferred - readonly active: () => boolean - readonly entry: Entry - state: "Waiting" | "Reserved" | "Registered" | "Cancelled" -} - -interface Cleanup { - readonly entry: Entry - readonly activeSession: boolean - readonly inboundItems: number - readonly outboundItems: number - readonly outboundBytes: number - readonly watcher: Fiber.Fiber | undefined - readonly requestFiber: Fiber.Fiber | undefined - readonly completed: Deferred.Deferred - readonly error: PeerRpcError.PeerRpcError | undefined - readonly fromWatcher: boolean - readonly interruptRequest: boolean - started: boolean -} - -const replicaFailure = () => - new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageUnavailable({ cause: new Error("RPC peer session unavailable") }) - }) - -const sessionFailure = (cause: Cause.Cause) => - Cause.findErrorOption(cause).pipe( - Option.match({ - onNone: () => new PeerRpcError.ServerUnavailable(), - onSome: (error) => { - // A rewritten document lineage is a peer visible protocol outcome, not a server fault, and - // the peer must be able to stop retrying against a history it can no longer reach. - if (error.reason._tag === "DocumentLineageChanged") { - return new PeerRpcError.DocumentLineageChanged() - } - return error.reason._tag === "StorageUnavailable" && Cause.isTimeoutError(error.reason.cause) - ? new PeerRpcError.SessionOverloaded() - : new PeerRpcError.ServerUnavailable() - } - }) - ) - -const authorizationFailure = ( - cause: Cause.Cause -) => { - if (Cause.hasInterruptsOnly(cause)) return Effect.failCause(cause) - if (cause.reasons.length === 1 && Cause.isFailReason(cause.reasons[0])) { - const error = cause.reasons[0].error - if (error._tag === "AccessDenied" || error._tag === "ServerUnavailable") { - return Effect.fail(error) - } - } - return Effect.fail(new PeerRpcError.ServerUnavailable()) -} - -export const layerHandlers = ( - options: { - readonly tenantId: string - readonly peerId: Identity.PeerId - readonly definition: ReplicaDefinition.Any - } -) => - PeerRpc.Rpcs.toLayer(Effect.gen(function*() { - const serverScope = yield* Scope.Scope - const runtimeScope = yield* Scope.fork(serverScope, "parallel") - const publisher = yield* CommitPublisher.CommitPublisher - const limits = yield* PeerRpcLimits.PeerRpcLimits - const replicaLimits = yield* ReplicaLimits.ReplicaLimits - const authorization = yield* PeerAuthorization.PeerAuthorization - const subscription = yield* publisher.subscribe - const lock = yield* Semaphore.make(1) - const openPermits = yield* Semaphore.make(limits.maxInFlightOpen) - const pushPermits = yield* Semaphore.make(limits.maxInFlightPush) - const bufferedBytes = yield* Semaphore.make(limits.maxBufferedBytes) - const readySessions = yield* Queue.bounded(replicaLimits.maxSessions) - const registry: Registry = { - accepting: true, - bufferedBytes: 0, - sessions: new Map(), - cleanups: new Map(), - peers: new Map(), - documents: new Map(), - subjects: new Map(), - outboundWaiters: new Map() - } - const inactiveSubjects = new Map() - let reservationId = 0 - let outboundWaiterId = 0 - - const peerKey = (tenantId: string, peerId: Identity.PeerId) => `${tenantId}:${peerId}` - - const retainInactiveSubject = (subjectId: string, subject: SubjectState) => { - if (subject.activeSessions !== 0 || subject.openInFlight !== 0 || subject.pushInFlight !== 0) return - inactiveSubjects.delete(subjectId) - inactiveSubjects.set(subjectId, subject) - } - - const drainByteCapacity = () => { - while (registry.outboundWaiters.size > 0) { - const waiter = registry.outboundWaiters.values().next().value! - if (!registry.accepting || !waiter.active()) { - registry.outboundWaiters.delete(waiter.id) - waiter.state = "Cancelled" - if (waiter.entry.outboundWaiter === waiter) waiter.entry.outboundWaiter = undefined - Deferred.doneUnsafe(waiter.ready, Effect.succeed(false)) - continue - } - if (registry.bufferedBytes + waiter.bytes > limits.maxBufferedBytes) return - registry.outboundWaiters.delete(waiter.id) - registry.bufferedBytes += waiter.bytes - waiter.state = "Reserved" - Deferred.doneUnsafe(waiter.ready, Effect.succeed(true)) - } - } - - const cancelByteCapacityWaiter = (waiter: ByteCapacityWaiter): Effect.Effect => - lock.withPermit(Effect.sync(() => { - if (waiter.state === "Cancelled" || waiter.state === "Registered") return - if (waiter.state === "Waiting") registry.outboundWaiters.delete(waiter.id) - else registry.bufferedBytes -= waiter.bytes - waiter.state = "Cancelled" - if (waiter.entry.outboundWaiter === waiter) waiter.entry.outboundWaiter = undefined - Deferred.doneUnsafe(waiter.ready, Effect.succeed(false)) - drainByteCapacity() - })) - - const reserveByteCapacity = ( - active: () => boolean, - entry: Entry, - bytes: number, - interruptible: (effect: Effect.Effect) => Effect.Effect - ) => - lock.withPermit(Effect.sync(() => { - if (!active()) return false as const - if (registry.outboundWaiters.size === 0 && registry.bufferedBytes + bytes <= limits.maxBufferedBytes) { - registry.bufferedBytes += bytes - return true as const - } - if (registry.outboundWaiters.size >= replicaLimits.maxSessions) { - return false as const - } - const waiter: ByteCapacityWaiter = { - id: outboundWaiterId++, - bytes, - ready: Deferred.makeUnsafe(), - active, - entry, - state: "Waiting" - } - registry.outboundWaiters.set(waiter.id, waiter) - entry.outboundWaiter = waiter - return waiter - })).pipe( - Effect.flatMap((result): Effect.Effect => { - if (typeof result === "boolean") return Effect.succeed(result) - return interruptible(Deferred.await(result.ready)).pipe( - Effect.onInterrupt(() => cancelByteCapacityWaiter(result)), - Effect.map((granted) => granted ? result : false) - ) - }) - ) - - const releaseByteCapacity = (bytes: number) => - lock.withPermit(Effect.sync(() => { - registry.bufferedBytes -= bytes - drainByteCapacity() - })) - - const releaseReservations = (cleanup: Cleanup) => - cleanup.outboundBytes === 0 - ? Effect.void - : Effect.all([ - bufferedBytes.release(cleanup.outboundBytes), - cleanup.entry.outboundPermits.release(cleanup.outboundBytes) - ], { discard: true }) - - const detach = ( - entry: Entry, - error: PeerRpcError.PeerRpcError | undefined, - fromWatcher: boolean, - interruptRequest: boolean - ): Cleanup | undefined => { - if (entry.cleanupStarted) return undefined - const activeSession = entry.active - if (error !== undefined) Deferred.doneUnsafe(entry.terminal, Effect.fail(error)) - Deferred.doneUnsafe(entry.closed, Effect.void) - entry.cleanupStarted = true - entry.active = false - if (registry.sessions.get(entry.sessionId) === entry) { - registry.sessions.delete(entry.sessionId) - if (registry.peers.get(peerKey(entry.tenantId, entry.peerId)) === entry.sessionId) { - registry.peers.delete(peerKey(entry.tenantId, entry.peerId)) - } - for (const documentId of entry.selectedIds) { - const sessions = registry.documents.get(documentId) - sessions?.delete(entry.sessionId) - if (sessions?.size === 0) registry.documents.delete(documentId) - } - const subject = registry.subjects.get(entry.subjectId) - if (subject !== undefined) { - subject.activeSessions -= 1 - retainInactiveSubject(entry.subjectId, subject) - } - } - const inboundItems = entry.inboundReservations.size - const inboundBytes = [...entry.inboundReservations.values()].reduce((total, bytes) => total + bytes, 0) - const outboundItems = entry.outboundReservations.size - const outboundBytes = [...entry.outboundReservations.values()].reduce((total, bytes) => total + bytes, 0) - entry.inboundReservations.clear() - entry.outboundReservations.clear() - entry.inboundBytes = 0 - registry.bufferedBytes -= inboundBytes + outboundBytes - const outboundWaiter = entry.outboundWaiter - if (outboundWaiter !== undefined && outboundWaiter.state !== "Registered") { - if (outboundWaiter.state === "Waiting") registry.outboundWaiters.delete(outboundWaiter.id) - else if (outboundWaiter.state === "Reserved") registry.bufferedBytes -= outboundWaiter.bytes - outboundWaiter.state = "Cancelled" - entry.outboundWaiter = undefined - Deferred.doneUnsafe(outboundWaiter.ready, Effect.succeed(false)) - } - drainByteCapacity() - const cleanup = { - entry, - activeSession, - inboundItems, - outboundItems, - outboundBytes, - watcher: entry.watcher, - requestFiber: entry.requestFiber, - completed: Deferred.makeUnsafe(), - error, - fromWatcher, - interruptRequest, - started: false - } - registry.cleanups.set(entry.sessionId, cleanup) - return cleanup - } - - const finishCleanup = (cleanup: Cleanup) => - Effect.uninterruptible( - lock.withPermit(Effect.sync(() => { - if (cleanup.started) return false - cleanup.started = true - return true - })).pipe( - Effect.flatMap((owner) => - owner - ? Effect.gen(function*() { - if (cleanup.activeSession) yield* PeerRpcObservability.modifyActiveSessions(-1) - yield* PeerRpcObservability.modifyQueueItems("Inbound", -cleanup.inboundItems) - yield* PeerRpcObservability.modifyQueueItems("Outbound", -cleanup.outboundItems) - if (cleanup.interruptRequest && cleanup.requestFiber !== undefined) { - yield* Effect.sync(() => cleanup.requestFiber?.interruptUnsafe(cleanup.requestFiber.id)) - } - if (cleanup.error !== undefined) { - yield* Queue.fail(cleanup.entry.inbound, replicaFailure()) - yield* Queue.fail(cleanup.entry.outbound, cleanup.error) - } - yield* Queue.shutdown(cleanup.entry.inbound) - yield* Queue.shutdown(cleanup.entry.outbound) - yield* releaseReservations(cleanup) - if (!cleanup.fromWatcher && cleanup.watcher !== undefined) yield* Fiber.interrupt(cleanup.watcher) - yield* Scope.close( - cleanup.entry.scope, - cleanup.error === undefined ? Exit.void : Exit.fail(cleanup.error) - ) - }).pipe( - Effect.ensuring(lock.withPermit(Effect.sync(() => { - if (registry.cleanups.get(cleanup.entry.sessionId) === cleanup) { - registry.cleanups.delete(cleanup.entry.sessionId) - } - Deferred.doneUnsafe(cleanup.completed, Effect.void) - }))) - ) - : Deferred.await(cleanup.completed) - ) - ) - ) - - const finishCleanups = (cleanups: ReadonlyArray) => - Effect.forEach(cleanups, finishCleanup, { - concurrency: limits.shutdownCleanupConcurrency, - discard: true - }) - - const revoke = ( - entry: Entry, - error: PeerRpcError.PeerRpcError | undefined, - fromWatcher = false, - interruptRequest = error !== undefined - ) => - lock.withPermit(Effect.sync(() => detach(entry, error, fromWatcher, interruptRequest))).pipe( - Effect.flatMap((cleanup) => cleanup === undefined ? Effect.void : finishCleanup(cleanup)) - ) - - const stopAll = (error: PeerRpcError.PeerRpcError, shutdown: boolean) => - lock.withPermit(Effect.sync(() => { - registry.accepting = false - const entries = Array.from(registry.sessions.values()) - let shutdownCloses = 0 - for (const entry of entries) { - const cleanup = detach(entry, error, false, true) - if (cleanup?.activeSession === true) shutdownCloses += 1 - } - const cleanups = [...registry.cleanups.values()] - return { cleanups, shutdownCloses } - })).pipe( - Effect.flatMap(({ cleanups, shutdownCloses }) => - (shutdown && shutdownCloses > 0 - ? PeerRpcObservability.record("Server", "ShutdownClosed", shutdownCloses) - : Effect.void).pipe( - Effect.andThen(Effect.forEach( - cleanups, - (cleanup) => - cleanup.interruptRequest && cleanup.requestFiber !== undefined - ? Fiber.interrupt(cleanup.requestFiber) - : Effect.void, - { concurrency: limits.shutdownCleanupConcurrency, discard: true } - )), - Effect.andThen(finishCleanups(cleanups)) - ) - ) - ) - - const subjectState = (subjectId: string, now: number) => { - const oldest = inactiveSubjects.entries().next().value - if (oldest !== undefined) { - if (now - oldest[1].lastUsedAt >= limits.rateLimitIdleRetention) { - inactiveSubjects.delete(oldest[0]) - registry.subjects.delete(oldest[0]) - } - } - let subject = registry.subjects.get(subjectId) - if ( - subject !== undefined && subject.activeSessions === 0 && subject.openInFlight === 0 && - subject.pushInFlight === 0 - ) { - inactiveSubjects.delete(subjectId) - if (now - subject.lastUsedAt >= limits.rateLimitIdleRetention) { - registry.subjects.delete(subjectId) - subject = undefined - } - } - if (subject !== undefined) return subject - if (registry.subjects.size >= limits.maxRetainedRateLimitedSubjects) { - const evictable = inactiveSubjects.keys().next().value - if (evictable === undefined) return undefined - inactiveSubjects.delete(evictable) - registry.subjects.delete(evictable) - } - subject = { - openTokens: limits.openBurst, - pushTokens: limits.pushBurst, - openUpdatedAt: now, - pushUpdatedAt: now, - lastUsedAt: now, - openInFlight: 0, - pushInFlight: 0, - activeSessions: 0 - } - registry.subjects.set(subjectId, subject) - return subject - } - - const acquireSubject = (subjectId: string, operation: "Open" | "Push", now: number) => - lock.withPermit(Effect.sync(() => { - if (!registry.accepting) return "Unavailable" as const - const subject = subjectState(subjectId, now) - if (subject === undefined) return "Capacity" as const - const storedUpdatedAt = operation === "Open" ? subject.openUpdatedAt : subject.pushUpdatedAt - const effectiveNow = Math.max(now, subject.openUpdatedAt, subject.pushUpdatedAt) - const elapsed = effectiveNow - storedUpdatedAt - if (operation === "Open") { - subject.openTokens = Math.min( - limits.openBurst, - subject.openTokens + elapsed / 1_000 * limits.openRatePerSecond - ) - subject.openUpdatedAt = effectiveNow - if (subject.openTokens < 1 || subject.openInFlight >= limits.maxInFlightOpenPerSubject) { - retainInactiveSubject(subjectId, subject) - return "Capacity" as const - } - subject.openTokens -= 1 - subject.openInFlight += 1 - } else { - subject.pushTokens = Math.min( - limits.pushBurst, - subject.pushTokens + elapsed / 1_000 * limits.pushRatePerSecond - ) - subject.pushUpdatedAt = effectiveNow - if (subject.pushTokens < 1 || subject.pushInFlight >= limits.maxInFlightPushPerSubject) { - retainInactiveSubject(subjectId, subject) - return "Capacity" as const - } - subject.pushTokens -= 1 - subject.pushInFlight += 1 - } - subject.lastUsedAt = effectiveNow - return "Acquired" as const - })) - - const releaseSubject = (subjectId: string, operation: "Open" | "Push") => - lock.withPermit(Effect.sync(() => { - const subject = registry.subjects.get(subjectId) - if (subject === undefined) return - if (operation === "Open") subject.openInFlight -= 1 - else subject.pushInFlight -= 1 - retainInactiveSubject(subjectId, subject) - })) - - const admitted = ( - subjectId: string, - operation: "Open" | "Push", - effect: Effect.Effect - ): Effect.Effect< - A, - E | PeerRpcError.RequestCapacityExceeded | PeerRpcError.ServerUnavailable, - R - > => - Clock.currentTimeMillis.pipe( - Effect.flatMap((now) => acquireSubject(subjectId, operation, now)), - Effect.flatMap((admission) => - Effect.gen(function*() { - if (admission === "Unavailable") return yield* new PeerRpcError.ServerUnavailable() - if (admission === "Capacity") return yield* new PeerRpcError.RequestCapacityExceeded() - return yield* (operation === "Open" ? openPermits : pushPermits).withPermitsIfAvailable(1)(effect).pipe( - Effect.flatMap( - Option.match({ - onNone: () => Effect.fail(new PeerRpcError.RequestCapacityExceeded()), - onSome: Effect.succeed - }) - ), - Effect.ensuring(releaseSubject(subjectId, operation)) - ) - }) - ) - ) - - const releaseInbound = (entry: Entry, id: number) => - lock.withPermit(Effect.gen(function*() { - const bytes = yield* Effect.sync(() => { - const bytes = entry.inboundReservations.get(id) - if (bytes === undefined) return undefined - entry.inboundReservations.delete(id) - entry.inboundBytes -= bytes - registry.bufferedBytes -= bytes - drainByteCapacity() - return bytes - }) - if (bytes !== undefined) yield* PeerRpcObservability.modifyQueueItems("Inbound", -1) - })) - - const releaseOutbound = (entry: Entry, id: number) => - lock.withPermit(Effect.gen(function*() { - const bytes = yield* Effect.sync(() => { - const bytes = entry.outboundReservations.get(id) - if (bytes === undefined) return undefined - entry.outboundReservations.delete(id) - registry.bufferedBytes -= bytes - drainByteCapacity() - return bytes - }) - if (bytes !== undefined) yield* PeerRpcObservability.modifyQueueItems("Outbound", -1) - return bytes - })).pipe( - Effect.flatMap((bytes) => - bytes === undefined - ? Effect.void - : Effect.all([ - bufferedBytes.release(bytes), - entry.outboundPermits.release(bytes) - ], { discard: true }) - ) - ) - - const sessionTransport = (entry: Entry) => { - let currentInbound: number | undefined - let started = false - const closeRequested = Deferred.makeUnsafe() - const receive = Stream.fromPull(Effect.succeed( - Effect.gen(function*() { - if (currentInbound !== undefined) { - yield* releaseInbound(entry, currentInbound) - currentInbound = undefined - } - if (!started) { - started = true - yield* Deferred.succeed(entry.inboundConsumerStarted, undefined) - } - const item = yield* Effect.raceFirst(Deferred.await(closeRequested), Queue.take(entry.inbound)) - currentInbound = item.id - return Arr.of(item.payload) - }) - )).pipe( - Stream.ensuring( - Effect.suspend(() => currentInbound === undefined ? Effect.void : releaseInbound(entry, currentInbound)) - ) - ) - const send = (payload: Uint8Array) => { - const bytes = payload.byteLength - if ( - bytes > PeerSession.maximumSyncEnvelopeBytes( - replicaLimits.maxSyncMessageBytes, - replicaLimits.maxSyncChangesPerMessage - ) - ) { - return PeerRpcObservability.record("Outbound", "Overloaded", 1).pipe( - Effect.andThen(Effect.fail(replicaFailure())) - ) - } - const releaseUntracked = releaseByteCapacity(bytes) - const reserve = Effect.uninterruptibleMask((restore) => { - return Effect.gen(function*() { - yield* restore(entry.outboundPermits.take(bytes)) - const capacityReserved = yield* reserveByteCapacity( - () => entry.active, - entry, - bytes, - (effect) => restore(effect) - ).pipe( - Effect.onInterrupt(() => entry.outboundPermits.release(bytes).pipe(Effect.asVoid)) - ) - if (!capacityReserved) { - yield* entry.outboundPermits.release(bytes) - yield* PeerRpcObservability.record("Outbound", "Overloaded", 1) - return yield* replicaFailure() - } - yield* bufferedBytes.take(bytes) - const id = reservationId++ - const registration = yield* lock.withPermit(Effect.gen(function*() { - if (!entry.active || registry.sessions.get(entry.sessionId) !== entry) { - return { registered: false, cleanup: undefined } - } - if ((yield* Clock.currentTimeMillis) >= entry.validUntil) { - return { - registered: false, - cleanup: detach(entry, new PeerRpcError.SessionUnavailable(), false, true) - } - } - const registered = yield* Effect.sync(() => { - if (typeof capacityReserved === "object") { - if (capacityReserved.state !== "Reserved") return false - capacityReserved.state = "Registered" - if (entry.outboundWaiter === capacityReserved) entry.outboundWaiter = undefined - } - entry.outboundReservations.set(id, bytes) - return true - }) - if (registered) yield* PeerRpcObservability.modifyQueueItems("Outbound", 1) - return { registered, cleanup: undefined } - })) - if (!registration.registered) { - yield* bufferedBytes.release(bytes) - yield* entry.outboundPermits.release(bytes) - if (typeof capacityReserved !== "object" || capacityReserved.state === "Reserved") { - yield* releaseUntracked - } - if (registration.cleanup !== undefined) { - yield* finishCleanup(registration.cleanup).pipe(Effect.forkIn(runtimeScope)) - } - return yield* replicaFailure() - } - const offered = yield* restore(Queue.offer(entry.outbound, { id, payload })).pipe( - Effect.onInterrupt(() => releaseOutbound(entry, id)) - ) - if (!offered) { - yield* releaseOutbound(entry, id) - yield* PeerRpcObservability.record("Outbound", "Overloaded", 1) - return yield* replicaFailure() - } - yield* PeerRpcObservability.recordBytes("Outbound", bytes) - }) - }) - return reserve - } - return PeerTransport.PeerTransport.of({ - // Both `capabilities` here describe the REMOTE peer, which is what the server side - // `PeerSession` reads before it lets `PeerSync.generate` emit a rewritten document. The - // value is read off the client's own `Open` advertisement and is never inferred from the - // shared protocol version, which lineage did not bump and which therefore cannot tell this - // build apart from one that would union a rewritten document. - capabilities: { storeAndForward: false, lineageAware: entry.lineageAware }, - connect: (connectOptions) => - connectOptions.peerId !== entry.peerId - ? Effect.fail( - new ReplicaError.ReplicaError({ - reason: new ReplicaError.ProtocolMismatch({ - expected: entry.peerId, - observed: connectOptions.peerId - }) - }) - ) - : Effect.succeed({ - peerId: entry.peerId, - capabilities: { storeAndForward: false, lineageAware: entry.lineageAware }, - receive, - send, - close: Deferred.interrupt(closeRequested).pipe(Effect.asVoid) - }) - }) - } - - const register = (entry: Entry) => - lock.withPermit(Effect.sync(() => { - if (!registry.accepting) return { _tag: "Unavailable" as const } - if (entry.cleanupStarted) return { _tag: "Revoked" as const } - const previousId = registry.peers.get(peerKey(entry.tenantId, entry.peerId)) - const previous = previousId === undefined ? undefined : registry.sessions.get(previousId) - const subject = registry.subjects.get(entry.subjectId) - if (subject === undefined) return { _tag: "Capacity" as const } - const effectiveSessions = registry.sessions.size - (previous === undefined ? 0 : 1) - const effectiveSubjectSessions = subject.activeSessions - - (previous?.subjectId === entry.subjectId ? 1 : 0) - if ( - effectiveSessions >= replicaLimits.maxSessions || - effectiveSubjectSessions >= limits.maxSessionsPerSubject - ) { - return { _tag: "Overloaded" as const } - } - const previousCleanup = previous === undefined - ? undefined - : detach(previous, new PeerRpcError.SessionUnavailable(), false, true) - entry.active = true - registry.sessions.set(entry.sessionId, entry) - registry.peers.set(peerKey(entry.tenantId, entry.peerId), entry.sessionId) - subject.activeSessions += 1 - inactiveSubjects.delete(entry.subjectId) - for (const documentId of entry.selectedIds) { - const sessions = registry.documents.get(documentId) ?? new Set() - sessions.add(entry.sessionId) - registry.documents.set(documentId, sessions) - } - return { _tag: "Registered" as const, previousCleanup } - })).pipe( - Effect.tap((result) => - result._tag !== "Registered" - ? Effect.void - : PeerRpcObservability.modifyActiveSessions(1).pipe( - Effect.andThen( - result.previousCleanup === undefined - ? Effect.void - : PeerRpcObservability.record("Open", "Replaced", 1) - ) - ) - ) - ) - - const markReady = (entry: Entry, session: PeerSession.PeerSession) => - lock.withPermit(Effect.sync(() => { - if (!entry.active || registry.sessions.get(entry.sessionId) !== entry) return false - entry.session = session - if (entry.dirty.size === 0 || entry.queued) return false - entry.queued = true - return true - })).pipe( - Effect.flatMap((enqueue) => - enqueue ? Queue.offer(readySessions, entry.sessionId).pipe(Effect.asVoid) : Effect.void - ) - ) - - const revokeForSessionFailure = (entry: Entry, cause: Cause.Cause) => { - const error = sessionFailure(cause) - return (error._tag === "SessionOverloaded" - ? PeerRpcObservability.record("Outbound", "Overloaded", 1) - : Effect.void).pipe( - Effect.andThen(revoke(entry, error, false, true)) - ) - } - - const responseStream = (entry: Entry) => { - let opened = false - let currentOutbound: number | undefined - const checkDelivery = lock.withPermit(Effect.gen(function*() { - if (!entry.active || registry.sessions.get(entry.sessionId) !== entry) { - return { _tag: "Inactive" as const } - } - if ((yield* Clock.currentTimeMillis) >= entry.validUntil) { - return { - _tag: "Expired" as const, - cleanup: detach(entry, new PeerRpcError.SessionUnavailable(), false, false) - } - } - return { _tag: "Active" as const } - })) - const requireDelivery = Effect.gen(function*() { - const delivery = yield* checkDelivery - if (delivery._tag === "Active") return - if (delivery._tag === "Expired" && delivery.cleanup !== undefined) { - yield* finishCleanup(delivery.cleanup) - } - return yield* Deferred.await(entry.terminal) - }) - const pull = Effect.withFiber((fiber) => - lock.withPermit(Effect.sync(() => { - if (!entry.cleanupStarted && entry.requestFiber === undefined) entry.requestFiber = fiber - })).pipe( - Effect.as( - Effect.gen(function*() { - if (currentOutbound !== undefined) { - yield* releaseOutbound(entry, currentOutbound) - currentOutbound = undefined - } - if (!opened) { - yield* Effect.raceFirst( - Deferred.await(entry.terminal), - Deferred.await(entry.inboundConsumerStarted) - ) - return yield* Effect.uninterruptible(Effect.gen(function*() { - yield* requireDelivery - opened = true - return Arr.of(PeerRpc.Opened.make({ - _tag: "Opened", - protocolVersion: PeerRpc.protocolVersion, - sessionId: entry.sessionId, - peerId: options.peerId, - // The only place the flag reaches a remote peer. Without it the client reads an - // absent capability, and its own `generate` refuses to emit any rewritten - // document toward this server. - capabilities: { storeAndForward: false, lineageAware: true } - })) - })) - } - const item = yield* Effect.raceFirst( - Deferred.await(entry.terminal), - Queue.take(entry.outbound) - ) - return yield* Effect.uninterruptible(Effect.gen(function*() { - const delivery = yield* checkDelivery - if (delivery._tag === "Active") { - currentOutbound = item.id - return Arr.of(PeerRpc.Message.make({ _tag: "Message", payload: item.payload })) - } - yield* releaseOutbound(entry, item.id) - if (delivery._tag === "Expired" && delivery.cleanup !== undefined) { - yield* finishCleanup(delivery.cleanup) - } - return yield* Deferred.await(entry.terminal) - })) - }).pipe( - Effect.catchCauseIf( - (cause) => cause.reasons.every(Cause.isInterruptReason), - (cause) => - Deferred.poll(entry.terminal).pipe( - Effect.flatMap(Option.match({ - onNone: () => Effect.failCause(cause), - onSome: (terminal) => terminal - })) - ) - ) - ) - ) - ) - ) - return Stream.fromPull(pull).pipe( - Stream.ensuring( - Effect.suspend(() => - (currentOutbound === undefined ? Effect.void : releaseOutbound(entry, currentOutbound)).pipe( - Effect.andThen(revoke(entry, undefined, false, false)) - ) - ) - ) - ) - } - - const openUnobserved = (request: typeof PeerRpc.OpenRpc.payloadSchema.Type) => - Effect.gen(function*() { - const authenticated = yield* PeerAuthentication.AuthenticatedPeer - const now = yield* Clock.currentTimeMillis - if (!Number.isFinite(authenticated.validUntil) || authenticated.validUntil <= now) { - return yield* new PeerRpcError.AuthenticationFailure() - } - if (authenticated.principal.tenantId !== options.tenantId) return yield* new PeerRpcError.AccessDenied() - if (request.protocolVersion !== PeerRpc.protocolVersion) return yield* new PeerRpcError.UnsupportedVersion() - if (request.expectedPeerId !== options.peerId) return yield* new PeerRpcError.PeerMismatch() - if (request.definitionHash === undefined) return yield* new PeerRpcError.InvalidRequest() - if (request.definitionHash !== options.definition.hash) return yield* new PeerRpcError.DefinitionMismatch() - if (request.documents.length === 0) return yield* new PeerRpcError.InvalidRequest() - const requested = new Set(request.documents.map((entry) => `${entry.documentType}:${entry.documentId}`)) - if (requested.size !== request.documents.length) return yield* new PeerRpcError.InvalidRequest() - const requestedDocumentIds = new Set(request.documents.map((entry) => entry.documentId)) - if (requestedDocumentIds.size !== request.documents.length) return yield* new PeerRpcError.InvalidRequest() - if (request.documents.length > replicaLimits.maxStreamsPerSession) { - return yield* new PeerRpcError.RequestLimitExceeded() - } - - return yield* admitted( - authenticated.principal.subjectId, - "Open", - Effect.gen(function*() { - const authorizationRequest = { - principal: authenticated.principal, - documents: request.documents - } - const authorized = yield* authorization.authorize(authorizationRequest).pipe( - Effect.flatMap((result) => PeerAuthorizationValidation.validate(authorizationRequest, result)), - Effect.catchCause(authorizationFailure) - ) - if ( - authorized.documents.some((entry) => - options.definition.documents.byName.get(entry.document.name) !== entry.document - ) - ) { - return yield* new PeerRpcError.AccessDenied() - } - const sessionId = yield* Identity.makeSessionId.pipe( - Effect.catchTag("PlatformError", () => Effect.fail(new PeerRpcError.ServerUnavailable())) - ) - const now = yield* Clock.currentTimeMillis - const entry: Entry = yield* Effect.uninterruptible(Effect.gen(function*() { - const scope = yield* Scope.fork(runtimeScope, "parallel") - return { - tenantId: options.tenantId, - subjectId: authenticated.principal.subjectId, - peerId: authenticated.principal.peerId, - sessionId, - // Read, not inferred. An omitted advertisement is exactly the older client this - // must stay false for. - lineageAware: request.capabilities?.lineageAware === true, - validUntil: Math.min( - authenticated.validUntil, - authorized.validUntil, - now + limits.maximumReauthorizationInterval - ), - scope, - inbound: yield* Queue.dropping( - limits.inboundItemCapacity - ), - outbound: yield* Queue.bounded( - limits.outboundItemCapacity - ), - outboundPermits: yield* Semaphore.make(limits.maxOutboundBufferedBytesPerSession), - inboundConsumerStarted: yield* Deferred.make(), - closed: yield* Deferred.make(), - terminal: yield* Deferred.make(), - documents: authorized.documents, - selectedIds: new Set(authorized.documents.map((document) => document.documentId)), - dirty: new Set(), - inboundReservations: new Map(), - outboundReservations: new Map(), - outboundWaiter: undefined, - active: false, - cleanupStarted: false, - queued: false, - inboundBytes: 0, - session: undefined, - watcher: undefined, - requestFiber: undefined - } - })) - return yield* Effect.gen(function*() { - // Per-branch so revocation is not gated on the race awaiting the losing monitors' finalizers. - const revokeOnInvalidation = revoke(entry, new PeerRpcError.SessionUnavailable(), true, true) - const leaseWatcher = Effect.raceAllFirst([ - authenticated.invalidated, - authorized.invalidated, - Effect.sleep(Duration.millis(Math.max(0, authenticated.validUntil - now))), - Effect.sleep(Duration.millis(Math.max(0, authorized.validUntil - now))), - Effect.sleep(Duration.millis(limits.maximumReauthorizationInterval)) - ].map((trigger) => trigger.pipe(Effect.ensuring(revokeOnInvalidation)))) - entry.watcher = yield* Effect.forkIn(leaseWatcher, runtimeScope, { startImmediately: true }) - const registered = yield* register(entry) - if (registered._tag !== "Registered") { - const error = registered._tag === "Overloaded" - ? new PeerRpcError.SessionOverloaded() - : registered._tag === "Revoked" - ? new PeerRpcError.SessionUnavailable() - : registered._tag === "Unavailable" - ? new PeerRpcError.ServerUnavailable() - : new PeerRpcError.RequestCapacityExceeded() - yield* revoke(entry, error, false, false) - return yield* error - } - if (registered.previousCleanup !== undefined) { - yield* finishCleanup(registered.previousCleanup) - } - yield* PeerSession.makeSupervised({ peerId: entry.peerId, documents: entry.documents }).pipe( - Effect.provideService(PeerTransport.PeerTransport, sessionTransport(entry)), - Effect.provideService(Scope.Scope, entry.scope), - Effect.tap((session) => - markReady(entry, session).pipe( - Effect.andThen( - Effect.raceFirst( - session.awaitDisconnect.pipe( - Effect.matchEffect({ - onFailure: (error) => revokeForSessionFailure(entry, Cause.fail(error)), - onSuccess: () => Effect.void - }) - ), - Deferred.await(entry.closed) - ).pipe( - Effect.forkIn(runtimeScope, { startImmediately: true }), - Effect.asVoid - ) - ) - ) - ), - Effect.onError((cause) => - entry.cleanupStarted - ? Effect.void - : revokeForSessionFailure(entry, cause).pipe( - Effect.forkIn(runtimeScope), - Effect.asVoid - ) - ), - Effect.forkIn(entry.scope, { startImmediately: true }) - ) - return responseStream(entry) - }).pipe( - Effect.onExitIf(Exit.isFailure, () => revoke(entry, new PeerRpcError.ServerUnavailable(), false, false)) - ) - }) - ) - }) - - const pushUnobserved = (request: typeof PeerRpc.PushRpc.payloadSchema.Type) => - Effect.gen(function*() { - const authenticated = yield* PeerAuthentication.AuthenticatedPeer - if ( - request.payload.byteLength > PeerSession.maximumSyncEnvelopeBytes( - replicaLimits.maxSyncMessageBytes, - replicaLimits.maxSyncChangesPerMessage - ) - ) { - return yield* new PeerRpcError.RequestLimitExceeded() - } - return yield* admitted( - authenticated.principal.subjectId, - "Push", - Effect.gen(function*() { - const error = new PeerRpcError.SessionOverloaded() - const expired = new PeerRpcError.SessionUnavailable() - const outcome = yield* lock.withPermit(Effect.gen(function*() { - const now = yield* Clock.currentTimeMillis - if (!Number.isFinite(authenticated.validUntil) || now >= authenticated.validUntil) { - return { _tag: "AuthenticationExpired" as const } - } - if (!registry.accepting) return { _tag: "Unavailable" as const } - const entry = registry.sessions.get(request.sessionId) - if ( - entry === undefined || !entry.active || entry.tenantId !== authenticated.principal.tenantId || - entry.subjectId !== authenticated.principal.subjectId || entry.peerId !== authenticated.principal.peerId - ) { - return { _tag: "Unavailable" as const } - } - if (now >= entry.validUntil) { - return { _tag: "Expired" as const, cleanup: detach(entry, expired, false, true) } - } - const bytes = request.payload.byteLength - if ( - entry.inboundReservations.size >= limits.inboundItemCapacity || - entry.inboundBytes + bytes > limits.maxInboundBufferedBytesPerSession || - registry.bufferedBytes + bytes > limits.maxBufferedBytes - ) { - return { _tag: "Overloaded" as const, cleanup: detach(entry, error, false, true) } - } - const id = reservationId++ - entry.inboundReservations.set(id, bytes) - entry.inboundBytes += bytes - registry.bufferedBytes += bytes - yield* PeerRpcObservability.modifyQueueItems("Inbound", 1) - const offered = yield* Queue.offer(entry.inbound, { id, payload: request.payload }) - if (!offered) { - return { _tag: "Overloaded" as const, cleanup: detach(entry, error, false, true) } - } - yield* PeerRpcObservability.recordBytes("Inbound", bytes) - return { _tag: "Accepted" as const } - })) - if (outcome._tag === "Accepted") return - if (outcome._tag === "AuthenticationExpired") { - return yield* new PeerRpcError.AuthenticationFailure() - } - if (outcome._tag === "Unavailable") return yield* new PeerRpcError.SessionUnavailable() - if (outcome._tag === "Expired") { - if (outcome.cleanup !== undefined) yield* finishCleanup(outcome.cleanup) - return yield* expired - } - if (outcome.cleanup !== undefined) { - yield* finishCleanup(outcome.cleanup) - } - return yield* error - }) - ) - }) - - const handlerResult = (exit: Exit.Exit) => { - const error = PeerRpcObservability.failure(exit) - switch (error?._tag) { - case "AuthenticationFailure": - return "AuthenticationDenied" as const - case "AccessDenied": - return "AuthorizationDenied" as const - case "UnsupportedVersion": - case "PeerMismatch": - case "DefinitionMismatch": - case "InvalidRequest": - case "RequestLimitExceeded": - // Peer caused, so it must not read as a server fault in effect_local_rpc_boundary_total. - case "DocumentLineageChanged": - return "ProtocolRejected" as const - case "RequestCapacityExceeded": - return "CapacityRejected" as const - case "SessionOverloaded": - return "Overloaded" as const - case "SessionUnavailable": - case "ServerUnavailable": - return "Failure" as const - case undefined: - return Exit.isSuccess(exit) ? "Success" as const : "Failure" as const - } - } - - const open = (request: typeof PeerRpc.OpenRpc.payloadSchema.Type) => - PeerRpcObservability.observe({ - effect: PeerRpcObservability.recordSelectedDocuments(request.documents.length).pipe( - Effect.andThen(openUnobserved(request)) - ), - operation: "Open", - spanName: "effect_local_rpc.server.open", - attributes: { "rpc.selected_documents": request.documents.length }, - result: handlerResult - }) - - const push = (request: typeof PeerRpc.PushRpc.payloadSchema.Type) => - PeerRpcObservability.observe({ - effect: pushUnobserved(request), - operation: "Push", - spanName: "effect_local_rpc.server.push", - attributes: { "rpc.payload_bytes": request.payload.byteLength }, - result: handlerResult - }) - - yield* Stream.runForEach(subscription.events, (event) => - lock.withPermit(Effect.sync(() => { - const entries = event._tag === "FullRefreshRequired" - ? [...registry.sessions.values()] - : [...(registry.documents.get(event.documentId) ?? [])].flatMap((sessionId) => { - const entry = registry.sessions.get(sessionId) - return entry === undefined ? [] : [entry] - }) - const enqueue: Array = [] - for (const entry of entries) { - if (!entry.active) continue - if (event._tag === "FullRefreshRequired") { - for (const documentId of entry.selectedIds) entry.dirty.add(documentId) - } else { - entry.dirty.add(event.documentId) - } - if (entry.session !== undefined && !entry.queued) { - entry.queued = true - enqueue.push(entry.sessionId) - } - } - return enqueue - })).pipe( - Effect.flatMap((entries) => Queue.offerAll(readySessions, entries)), - Effect.asVoid - )).pipe( - Effect.andThen(stopAll(new PeerRpcError.ServerUnavailable(), false)), - Effect.catchCause(() => stopAll(new PeerRpcError.ServerUnavailable(), false)), - Effect.forkIn(runtimeScope, { startImmediately: true }) - ) - - for (let index = 0; index < limits.commitFlushConcurrency; index++) { - yield* Effect.gen(function*() { - while (true) { - const sessionId = yield* Queue.take(readySessions) - const work = yield* lock.withPermit(Effect.gen(function*() { - const entry = registry.sessions.get(sessionId) - if (entry === undefined || !entry.active || entry.session === undefined) return undefined - if ((yield* Clock.currentTimeMillis) >= entry.validUntil) { - return { - _tag: "Expired" as const, - cleanup: detach(entry, new PeerRpcError.SessionUnavailable(), false, true) - } - } - const dirty = [...entry.dirty] - entry.dirty.clear() - return { _tag: "Ready" as const, entry, session: entry.session, dirty } - })) - if (work === undefined) continue - if (work._tag === "Expired") { - if (work.cleanup !== undefined) yield* finishCleanup(work.cleanup) - continue - } - const exit = yield* Effect.forEach(work.dirty, work.session.markDirty, { discard: true }).pipe( - Effect.andThen(work.session.flush), - Effect.exit - ) - if (Exit.isFailure(exit)) { - yield* revokeForSessionFailure(work.entry, exit.cause) - continue - } - const requeue = yield* lock.withPermit(Effect.sync(() => { - if (!work.entry.active) return false - work.entry.queued = false - if (work.entry.dirty.size === 0) return false - work.entry.queued = true - return true - })) - if (requeue) yield* Queue.offer(readySessions, work.entry.sessionId) - } - }).pipe(Effect.forkIn(runtimeScope, { startImmediately: true })) - } - - yield* Effect.addFinalizer(() => - stopAll(new PeerRpcError.ServerUnavailable(), true).pipe( - Effect.andThen(Queue.shutdown(readySessions)), - Effect.asVoid - ) - ) - - return PeerRpc.Rpcs.of({ - Open: (request) => Stream.unwrap(open(request)), - Push: push - }) - })) type RelaySqlLane = "Admission" | "Terminal" | "Delivery" | "Maintenance" @@ -1508,7 +304,7 @@ const makeRelayAdmissionLane = (options: { } }) -export interface PeerRelayServerUsage { +export interface PeerRpcServerUsage { readonly accepting: boolean readonly sessions: number readonly subjects: number @@ -1516,16 +312,16 @@ export interface PeerRelayServerUsage { readonly queuedChannels: number } -export class PeerRelayServerRuntime extends Context.Service readonly owner: Effect.Effect readonly shutdown: Effect.Effect - readonly usage: Effect.Effect -}>()("@lucas-barake/effect-local-rpc/PeerRelayServerRuntime") {} + readonly usage: Effect.Effect +}>()("@lucas-barake/effect-local-rpc/PeerRpcServerRuntime") {} -class PeerRelayFatalSignal extends Context.Service) => Effect.Effect -}>()("@lucas-barake/effect-local-rpc/PeerRelayFatalSignal") {} +}>()("@lucas-barake/effect-local-rpc/PeerRpcFatalSignal") {} interface RelayWorkSelector { readonly recipient: { @@ -1540,7 +336,7 @@ interface RelayWorkSelector { } interface RelayOutboundItem { - readonly event: PeerRelayRpc.StoredMessage + readonly event: PeerRpc.StoredMessage readonly reservation: PeerRelayIngress.Reservation transferred: boolean } @@ -1648,7 +444,7 @@ const decodeRelayEnvelope = (payload: Uint8Array) => Effect.mapError(() => new PeerRpcError.InvalidRequest()) ) -export const layerRelayHandlers = ( +export const layerHandlers = ( options: { readonly tenantId: string readonly peerId: Identity.PeerId @@ -2037,7 +833,7 @@ export const layerRelayHandlers = ( }, relayPeerId: claim.relayPeerId, relayMessageId: claim.relayMessageId, - protocolVersion: PeerRelayRpc.protocolVersion, + protocolVersion: PeerRpc.protocolVersion, payloadVersion: claim.payloadVersion, senderReplicaIncarnation: claim.channel.senderReplicaIncarnation, senderConnectionEpoch: claim.senderConnectionEpoch, @@ -2201,7 +997,7 @@ export const layerRelayHandlers = ( ) return } - const event = PeerRelayRpc.StoredMessage.make({ + const event = PeerRpc.StoredMessage.make({ _tag: "StoredMessage", relayMessageId: claim.relayMessageId, claimToken: claim.claimToken, @@ -2447,7 +1243,7 @@ export const layerRelayHandlers = ( ) const open = ( - request: typeof PeerRelayRpc.OpenRelayRpc.payloadSchema.Type + request: typeof PeerRpc.OpenRpc.payloadSchema.Type ) => Effect.gen(function*() { const authenticated = yield* PeerAuthentication.AuthenticatedPeer @@ -2459,7 +1255,7 @@ export const layerRelayHandlers = ( if (authenticated.validUntil <= now) { return yield* new PeerRpcError.AuthenticationFailure() } - if (request.version !== PeerRelayRpc.protocolVersion) { + if (request.protocolVersion !== PeerRpc.protocolVersion) { return yield* new PeerRpcError.UnsupportedVersion() } if ( @@ -2555,8 +1351,7 @@ export const layerRelayHandlers = ( ) return { entry, replaced } })) - return yield* restore(Effect.gen(function*() { - if (entry.replaced !== undefined) yield* detachEntry(entry.replaced, false) + const start = restore(Effect.gen(function*() { const monitorStartedAt = yield* Clock.currentTimeMillis if (authenticated.validUntil <= monitorStartedAt) { return yield* new PeerRpcError.AuthenticationFailure() @@ -2618,13 +1413,12 @@ export const layerRelayHandlers = ( Effect.uninterruptible ) yield* notify(relaySelectorForEntry(entry.entry), "Retry") - const opened = PeerRelayRpc.RelayOpened.make({ - _tag: "RelayOpened", - version: PeerRelayRpc.protocolVersion, + const opened = PeerRpc.Opened.make({ + _tag: "Opened", + protocolVersion: PeerRpc.protocolVersion, sessionId, remotePeerId: entry.entry.remote.peerId, - authenticatedLocal: principal, - capabilities: { storeAndForward: true } + authenticatedLocal: principal }) const deliveries = Stream.fromQueue(entry.entry.outbound).pipe( Stream.mapEffect((item) => @@ -2646,7 +1440,12 @@ export const layerRelayHandlers = ( return Stream.concat(Stream.make(opened), deliveries).pipe( Stream.ensuring(detachEntry(entry.entry, false)) ) - })).pipe( + })) + return yield* ( + entry.replaced === undefined + ? start + : detachEntry(entry.replaced, false).pipe(Effect.andThen(start)) + ).pipe( Effect.onExitIf( Exit.isFailure, () => detachEntry(entry.entry, false) @@ -2658,7 +1457,7 @@ export const layerRelayHandlers = ( ) }) - const push = (request: typeof PeerRelayRpc.PushRelayRpc.payloadSchema.Type) => + const push = (request: typeof PeerRpc.PushRpc.payloadSchema.Type) => Effect.gen(function*() { const authenticated = yield* PeerAuthentication.AuthenticatedPeer return yield* pushLane.run( @@ -2699,7 +1498,7 @@ export const layerRelayHandlers = ( }, relayPeerId: options.peerId, relayMessageId: request.relayMessageId, - protocolVersion: PeerRelayRpc.protocolVersion, + protocolVersion: PeerRpc.protocolVersion, payloadVersion: 1, senderReplicaIncarnation: entry.senderReplicaIncarnation, senderConnectionEpoch: envelope.connectionEpoch, @@ -2767,9 +1566,9 @@ export const layerRelayHandlers = ( const terminal = ( request: - | typeof PeerRelayRpc.AcknowledgeRelayRpc.payloadSchema.Type - | typeof PeerRelayRpc.RejectRelayRpc.payloadSchema.Type, - reason: PeerRelayRpc.RejectReason | undefined + | typeof PeerRpc.AcknowledgeRpc.payloadSchema.Type + | typeof PeerRpc.RejectRpc.payloadSchema.Type, + reason: PeerRpc.RejectReason | undefined ) => Effect.gen(function*() { const authenticated = yield* PeerAuthentication.AuthenticatedPeer @@ -2888,7 +1687,7 @@ export const layerRelayHandlers = ( serverScope ) - const runtime = PeerRelayServerRuntime.of({ + const runtime = PeerRpcServerRuntime.of({ health: Deferred.poll(fatal).pipe( Effect.flatMap((exit) => Option.isNone(exit) @@ -2906,25 +1705,25 @@ export const layerRelayHandlers = ( queuedChannels: workOwners.size }))) }) - const handlerContext = yield* PeerRelayRpc.Rpcs.toHandlers(PeerRelayRpc.Rpcs.of({ - OpenRelay: (request) => Stream.unwrap(open(request)), - PushRelay: push, - AcknowledgeRelay: (request) => terminal(request, undefined), - RejectRelay: (request) => terminal(request, request.reason) + const handlerContext = yield* PeerRpc.Rpcs.toHandlers(PeerRpc.Rpcs.of({ + Open: (request) => Stream.unwrap(open(request)), + Push: push, + Acknowledge: (request) => terminal(request, undefined), + Reject: (request) => terminal(request, request.reason) })) - return Context.add(handlerContext, PeerRelayServerRuntime, runtime).pipe( - Context.add(PeerRelayFatalSignal, PeerRelayFatalSignal.of({ signal: signalFatal })) + return Context.add(handlerContext, PeerRpcServerRuntime, runtime).pipe( + Context.add(PeerRpcFatalSignal, PeerRpcFatalSignal.of({ signal: signalFatal })) ) })) -export const layerRelayServer = Layer.effectDiscard(Effect.gen(function*() { +export const layerServer = Layer.effectDiscard(Effect.gen(function*() { const scope = yield* Scope.Scope - const runtime = yield* PeerRelayServerRuntime - const fatalSignal = yield* PeerRelayFatalSignal + const runtime = yield* PeerRpcServerRuntime + const fatalSignal = yield* PeerRpcFatalSignal const ingress = yield* PeerRelayIngress.PeerRelayIngress let stopping = false const serverFiber = yield* Effect.forkIn( - RpcServer.make(PeerRelayRpc.Rpcs, { disableFatalDefects: true }), + RpcServer.make(PeerRpc.Rpcs, { disableFatalDefects: true }), scope ) const ingressFiber = yield* Effect.forkIn(ingress.await, scope) @@ -2955,30 +1754,3 @@ export const layerRelayServer = Layer.effectDiscard(Effect.gen(function*() { ) yield* Effect.addFinalizer(() => stop) })) - -export const layerStoreAndForwardDeployment = < - DirectOut, - DirectError, - DirectRequirements, - RelaySocketError, - RelaySocketRequirements, ->(options: { - readonly directDeployment: Layer.Layer - readonly relaySocketLayer: Layer.Layer< - SocketServer.SocketServer, - RelaySocketError, - RelaySocketRequirements - > - readonly relay: { - readonly tenantId: string - readonly peerId: Identity.PeerId - } -}) => { - const ingress = PeerRelayIngress.layerProtocolSocketServer(options.relaySocketLayer) - const handlers = layerRelayHandlers(options.relay) - const relay = layerRelayServer.pipe( - Layer.provideMerge(handlers), - Layer.provideMerge(ingress) - ) - return Layer.merge(options.directDeployment, relay) -} diff --git a/packages/local-rpc/src/RpcPeerTransport.ts b/packages/local-rpc/src/RpcPeerTransport.ts index 2eba82d..9310a39 100644 --- a/packages/local-rpc/src/RpcPeerTransport.ts +++ b/packages/local-rpc/src/RpcPeerTransport.ts @@ -19,7 +19,6 @@ import * as Sink from "effect/Sink" import * as Stream from "effect/Stream" import type { RpcClientError } from "effect/unstable/rpc/RpcClientError" import * as PeerRpcObservability from "./internal/peerRpcObservability.js" -import * as PeerRelayRpc from "./PeerRelayRpc.js" import * as PeerRpc from "./PeerRpc.js" import type * as PeerRpcError from "./PeerRpcError.js" @@ -108,177 +107,7 @@ const adapterAcknowledgeResult = ( } } -export const layer = ( - client: PeerRpc.RpcClient, - options: { - readonly documents: ReadonlyArray - readonly definition: ReplicaDefinition.Any - } -) => - Layer.succeed(PeerTransport.PeerTransport, { - // The connection level value below is whatever the server advertised in its `Opened` frame. - // This one describes the local adapter, which compares lineage on every inbound message it - // hands to `PeerSession`, so it is true for the same reason the server side is. - capabilities: { storeAndForward: false, lineageAware: true }, - connect: (connectOptions) => - PeerRpcObservability.observe({ - effect: Effect.gen(function*() { - yield* validateDocuments(options.documents, options.definition) - const parentScope = yield* Scope.Scope - return yield* Effect.uninterruptibleMask((restore) => - Effect.gen(function*() { - const lifetimeScope = yield* Scope.fork(parentScope, "sequential") - const connectionScope = yield* Scope.make("parallel") - const stateLock = yield* Semaphore.make(1) - const closeCompleted = yield* Deferred.make() - // Effect.interrupt is required. A trigger that succeeds ends the stream normally, - // and PeerSession reports a normally ended receive stream as a retryable StorageUnavailable. - const interruptOnClose = Deferred.await(closeCompleted).pipe(Effect.andThen(Effect.interrupt)) - let closing = false - const closeConnection = (exit: Exit.Exit) => - Effect.sync(() => { - if (closing) return false - closing = true - return true - }).pipe( - stateLock.withPermit, - Effect.flatMap((owner) => - owner - ? Scope.close(connectionScope, exit).pipe( - Effect.ensuring(Deferred.succeed(closeCompleted, undefined)) - ) - : Deferred.await(closeCompleted) - ), - Effect.uninterruptible - ) - const closeWithExit = (exit: Exit.Exit) => - closeConnection(exit).pipe( - Effect.ensuring(Scope.close(lifetimeScope, exit)) - ) - yield* Scope.addFinalizerExit(lifetimeScope, closeConnection) - return yield* restore(Effect.gen(function*() { - const openCompleted = yield* Deferred.make< - Exit.Exit< - readonly [ - ReadonlyArray, - Stream.Stream - ], - ReplicaError.ReplicaError - > - >() - const openRequest = client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: connectOptions.peerId, - definitionHash: options.definition.hash, - // Truthful for the same reason the adapter level `capabilities` above is: this - // build compares lineage on every inbound message before it hands one to - // `PeerSession`. The server has no other way to tell this build from an older one - // on the same protocol version, and without the claim it refuses to emit any - // rewritten document toward this replica. - capabilities: { lineageAware: true }, - documents: options.documents.map((entry) => ({ - documentType: entry.document.name, - documentId: entry.documentId - })) - }, { streamBufferSize: 1 }).pipe( - Stream.mapError(mapError), - Stream.peel(Sink.take(1)), - Effect.provideService(Scope.Scope, connectionScope), - Effect.onExit((exit) => Deferred.succeed(openCompleted, exit).pipe(Effect.asVoid)) - ) - const openFiber = yield* stateLock.withPermit( - Effect.suspend(() => - closing - ? Effect.fail(unavailable()) - : Effect.forkIn(openRequest, connectionScope) - ) - ) - const [first, remainder] = yield* Deferred.await(openCompleted).pipe( - Effect.onInterrupt(() => Fiber.interrupt(openFiber)), - Effect.flatten - ) - const handshake = first[0] - if (handshake === undefined || handshake._tag !== "Opened") { - return yield* protocolFailure(handshake?._tag ?? "Open stream ended before handshake") - } - if (handshake.peerId !== connectOptions.peerId) { - return yield* new ReplicaError.ReplicaError({ - reason: new ReplicaError.ProtocolMismatch({ - expected: connectOptions.peerId, - observed: handshake.peerId - }) - }) - } - yield* stateLock.withPermit( - Effect.suspend(() => closing ? Effect.fail(unavailable()) : Effect.void) - ) - const sendLock = yield* Semaphore.make(1) - const send = (message: Uint8Array) => - PeerRpcObservability.observe({ - effect: Effect.uninterruptibleMask((restoreSend) => - stateLock.withPermit(Effect.gen(function*() { - if (closing) return yield* unavailable() - const completed = yield* Deferred.make>() - const fiber = yield* client.Push({ sessionId: handshake.sessionId, payload: message }).pipe( - Effect.mapError(mapError), - sendLock.withPermit, - Effect.onExit((exit) => Deferred.succeed(completed, exit).pipe(Effect.asVoid)), - Effect.forkIn(connectionScope, { startImmediately: true }) - ) - return [fiber, completed] as const - })).pipe( - Effect.flatMap(([fiber, completed]) => - Deferred.await(completed).pipe( - restoreSend, - Effect.flatten, - Effect.onInterrupt(() => Fiber.interrupt(fiber)) - ) - ) - ) - ), - operation: "AdapterPush", - spanName: "effect_local_rpc.adapter.push", - attributes: { "rpc.payload_bytes": message.byteLength }, - result: adapterResult - }) - return { - peerId: handshake.peerId, - capabilities: handshake.capabilities, - receive: remainder.pipe( - Stream.mapEffect((event) => - event._tag === "Message" - ? Effect.succeed(event.payload) - : Effect.fail(protocolFailure(event._tag)) - ), - Stream.interruptWhen(interruptOnClose) - ), - send, - close: closeWithExit(Exit.void) - } - })).pipe(Effect.onExitIf(Exit.isFailure, closeWithExit)) - }) - ) - }), - operation: "AdapterOpen", - spanName: "effect_local_rpc.adapter.open", - attributes: { "rpc.selected_documents": options.documents.length }, - result: adapterResult - }) - }) - -export const makeSession = ( - client: PeerRpc.RpcClient, - options: { - readonly peerId: Identity.PeerId - readonly documents: ReadonlyArray - readonly definition: ReplicaDefinition.Any - } -) => - PeerSession.makeLive(options).pipe( - Effect.provide(layer(client, { documents: options.documents, definition: options.definition })) - ) - -export interface StoreAndForwardOptions { +export interface Options { readonly expectedLocal: PeerSyncEnvelope.RelayPeerPrincipal readonly senderReplicaIncarnation: Identity.ReplicaIncarnation readonly expectedRelayPeerId: Identity.PeerId @@ -301,7 +130,7 @@ const samePrincipal = ( left.subjectId === right.subjectId && left.peerId === right.peerId -const validateRelayOptions = (options: StoreAndForwardOptions) => +const validateRelayOptions = (options: Options) => Effect.suspend(() => { for ( const [name, value] of [ @@ -312,7 +141,7 @@ const validateRelayOptions = (options: StoreAndForwardOptions) => if ( !Number.isSafeInteger(value) || value <= 0 || - value > PeerRelayRpc.maximumNegotiatedDurationMillis + value > PeerRpc.maximumNegotiatedDurationMillis ) { return Effect.fail(protocolFailure(`valid ${name}`)) } @@ -324,8 +153,8 @@ const validateRelayOptions = (options: StoreAndForwardOptions) => }) const validateStoredMessage = ( - event: PeerRelayRpc.StoredMessage, - options: StoreAndForwardOptions, + event: PeerRpc.StoredMessage, + options: Options, crypto: Crypto.Crypto, limits: ReplicaLimits.Values ) => @@ -361,7 +190,7 @@ const validateStoredMessage = ( remote: expectedRecipient, relayPeerId: event.relayPeerId, relayMessageId: event.relayMessageId, - protocolVersion: PeerRelayRpc.protocolVersion, + protocolVersion: PeerRpc.protocolVersion, payloadVersion: event.payloadVersion, senderReplicaIncarnation: event.sender.replicaIncarnation, senderConnectionEpoch: event.sender.connectionEpoch, @@ -377,9 +206,9 @@ const validateStoredMessage = ( } }) -export const layerStoreAndForward = ( - client: PeerRelayRpc.RpcClient, - options: StoreAndForwardOptions +export const layer = ( + client: PeerRpc.RpcClient, + options: Options ) => Layer.effect( PeerTransport.PeerTransport, @@ -398,7 +227,7 @@ export const layerStoreAndForward = ( } as const return { - capabilities: { storeAndForward: true }, + capabilities: { lineageAware: true }, connect: (connectOptions) => PeerRpcObservability.observe({ effect: Effect.gen(function*() { @@ -500,17 +329,17 @@ export const layerStoreAndForward = ( const openCompleted = yield* Deferred.make< Exit.Exit< readonly [ - ReadonlyArray, + ReadonlyArray, Stream.Stream< - PeerRelayRpc.OpenRelayEvent, + PeerRpc.OpenEvent, ReplicaError.ReplicaError > ], ReplicaError.ReplicaError > >() - const openRequest = client.OpenRelay({ - version: PeerRelayRpc.protocolVersion, + const openRequest = client.Open({ + protocolVersion: PeerRpc.protocolVersion, expectedRelayPeerId: options.expectedRelayPeerId, expectedLocal: options.expectedLocal, senderReplicaIncarnation: options.senderReplicaIncarnation, @@ -523,7 +352,7 @@ export const layerStoreAndForward = ( senderRetryHorizonMillis: advertisedRetryHorizon }, { streamBufferSize: 1 }).pipe( Stream.mapError(mapError), - Stream.peel(Sink.take(1)), + Stream.peel(Sink.take(1)), Effect.provideService(Scope.Scope, connectionScope), Effect.onExit((exit) => Deferred.succeed(openCompleted, exit).pipe(Effect.asVoid)) ) @@ -544,10 +373,9 @@ export const layerStoreAndForward = ( const handshake = first[0] if ( handshake === undefined || - handshake._tag !== "RelayOpened" || - handshake.version !== PeerRelayRpc.protocolVersion || + handshake._tag !== "Opened" || + handshake.protocolVersion !== PeerRpc.protocolVersion || handshake.remotePeerId !== options.remote.peerId || - handshake.capabilities.storeAndForward !== true || !samePrincipal(handshake.authenticatedLocal, options.expectedLocal) ) { return yield* protocolFailure("valid relay handshake") @@ -596,7 +424,7 @@ export const layerStoreAndForward = ( const pushEntry = ( entry: Effect.Success> ) => - client.PushRelay({ + client.Push({ sessionId: handshake.sessionId, relayMessageId: entry.relayMessageId, payload: entry.payload @@ -672,7 +500,7 @@ export const layerStoreAndForward = ( receiptRetentionMillis: options.receiptRetentionMillis, acknowledge: terminalCall( PeerRpcObservability.observeRelay({ - effect: client.AcknowledgeRelay({ + effect: client.Acknowledge({ sessionId: handshake.sessionId, relayMessageId: event.relayMessageId, claimToken: event.claimToken, @@ -694,7 +522,7 @@ export const layerStoreAndForward = ( reject: (reason: PeerTransport.PermanentRejectReason) => terminalCall( PeerRpcObservability.observeRelay({ - effect: client.RejectRelay({ + effect: client.Reject({ sessionId: handshake.sessionId, relayMessageId: event.relayMessageId, claimToken: event.claimToken, @@ -741,11 +569,8 @@ export const layerStoreAndForward = ( return { peerId: handshake.remotePeerId, relayPeerId: options.expectedRelayPeerId, - capabilities: handshake.capabilities, - receive: acknowledged.pipe( - Stream.map((delivery) => delivery.message) - ), - receiveWithAcknowledgement: acknowledged, + capabilities: { lineageAware: true }, + receive: acknowledged, send, close: closeWithExit(Exit.void) } @@ -762,13 +587,13 @@ export const layerStoreAndForward = ( }) ) -export const makeStoreAndForwardSession = ( - client: PeerRelayRpc.RpcClient, - options: StoreAndForwardOptions +export const makeSession = ( + client: PeerRpc.RpcClient, + options: Options ) => PeerSession.makeLive({ peerId: options.remote.peerId, documents: options.documents }).pipe( - Effect.provide(layerStoreAndForward(client, options)) + Effect.provide(layer(client, options)) ) diff --git a/packages/local-rpc/src/SqlPeerRelayStore.ts b/packages/local-rpc/src/SqlPeerRelayStore.ts new file mode 100644 index 0000000..6db2590 --- /dev/null +++ b/packages/local-rpc/src/SqlPeerRelayStore.ts @@ -0,0 +1,1891 @@ +import * as Identity from "@lucas-barake/effect-local/Identity" +import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" +import * as Crypto from "effect/Crypto" +import * as Effect from "effect/Effect" +import * as Exit from "effect/Exit" +import * as Layer from "effect/Layer" +import * as Option from "effect/Option" +import * as Random from "effect/Random" +import * as Schema from "effect/Schema" +import type * as Migrator from "effect/unstable/sql/Migrator" +import * as SqlClient from "effect/unstable/sql/SqlClient" +import type * as SqlError from "effect/unstable/sql/SqlError" +import * as SqlSchema from "effect/unstable/sql/SqlSchema" +import * as PeerRelayMigrations from "./internal/peerRelayMigrations.js" +import { make as makeWriteTransaction } from "./internal/peerRelaySqlTransaction.js" +import { mapStoreErrors } from "./internal/peerRelayStoreErrors.js" +import * as PeerRpcObservability from "./internal/peerRpcObservability.js" +import * as PeerRelayLimits from "./PeerRelayLimits.js" +import { + Admission, + AdmissionResult, + ChannelKey, + ClaimedMessage, + ClaimRequest, + LoadClaimedPayloadRequest, + MaintenanceRequest, + type MaintenanceResult, + PeerRelayStore, + RejectRequest, + ReleaseRequest, + type Service, + type StoreError, + TerminalRequest, + type TransitionResult, + type Usage, + UsageRequest +} from "./PeerRelayStore.js" +import * as PeerRpc from "./PeerRpc.js" + +const DatabaseInt = Schema.Union([Schema.Int, Schema.NumberFromString]).check(Schema.isInt()) +const PositiveInt = DatabaseInt.check(Schema.isGreaterThan(0)) +const NonNegativeInt = DatabaseInt.check(Schema.isGreaterThanOrEqualTo(0)) +const DatabaseReplicaIncarnation = NonNegativeInt.pipe( + Schema.brand("@lucas-barake/effect-local/ReplicaIncarnation") +) +const DocumentIds = Schema.Array(Identity.DocumentId).check(Schema.isMinLength(1)) + +type UsageKind = UsageRequest["scopeKind"] + +interface UsageScope { + readonly kind: UsageKind + readonly key: string + readonly activeCountLimit: number + readonly activeBytesLimit: number + readonly retainedCountLimit: number + readonly retainedBytesLimit: number +} + +const storageCorrupt = (cause: unknown) => + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + +const protocolMismatch = (expected: string, observed: string) => + new ReplicaError.ReplicaError({ + reason: new ReplicaError.ProtocolMismatch({ expected, observed }) + }) + +const quotaExceeded = (resource: string, limit: number) => + new ReplicaError.ReplicaError({ + reason: new ReplicaError.QuotaExceeded({ resource, limit }) + }) + +const relayFailureResult = (exit: Exit.Exit) => { + const error = PeerRpcObservability.failure(exit) + if (error?._tag !== "ReplicaError") return "Failure" as const + switch (error.reason._tag) { + case "ProtocolMismatch": + return "ProtocolRejected" as const + case "QuotaExceeded": + return "CapacityRejected" as const + case "StorageUnavailable": + return "Unavailable" as const + default: + return "Failure" as const + } +} + +const relayQuotaDomain = ( + error: StoreError +): Option.Option => { + if (error._tag !== "ReplicaError" || error.reason._tag !== "QuotaExceeded") { + return Option.none() + } + const resource = error.reason.resource + if (resource === "relay payload bytes") return Option.some("Payload") + for ( + const domain of [ + "SenderPeer", + "RecipientPeer", + "RecipientSubject", + "Tenant", + "Shard" + ] as const + ) { + if (resource === `${domain} relay custody`) return Option.some(domain) + } + return Option.none() +} + +const recordQuotaRejection = (error: StoreError) => + Option.match(relayQuotaDomain(error), { + onNone: () => Effect.void, + onSome: (domain) => + PeerRpcObservability.recordRelayQuotaRejection(domain).pipe( + Effect.catchCause(() => Effect.void) + ) + }) + +const encodeKey = (...parts: ReadonlyArray) => JSON.stringify(parts) + +const channelScopes = ( + channel: ChannelKey, + limits: PeerRelayLimits.Values +): ReadonlyArray => [ + { + kind: "SenderPeer", + key: encodeKey(channel.tenantId, channel.senderSubjectId, channel.senderPeerId), + activeCountLimit: limits.maxActiveMessagesPerSenderPeer, + activeBytesLimit: limits.maxActiveBytesPerSenderPeer, + retainedCountLimit: limits.maxRetainedRowsPerSenderPeer, + retainedBytesLimit: limits.maxRetainedBytesPerSenderPeer + }, + { + kind: "RecipientPeer", + key: encodeKey(channel.tenantId, channel.recipientSubjectId, channel.recipientPeerId), + activeCountLimit: limits.maxActiveMessagesPerRecipientPeer, + activeBytesLimit: limits.maxActiveBytesPerRecipientPeer, + retainedCountLimit: limits.maxRetainedRowsPerRecipientPeer, + retainedBytesLimit: limits.maxRetainedBytesPerRecipientPeer + }, + { + kind: "RecipientSubject", + key: encodeKey(channel.tenantId, channel.recipientSubjectId), + activeCountLimit: limits.maxActiveMessagesPerRecipientSubject, + activeBytesLimit: limits.maxActiveBytesPerRecipientSubject, + retainedCountLimit: limits.maxRetainedRowsPerRecipientSubject, + retainedBytesLimit: limits.maxRetainedBytesPerRecipientSubject + }, + { + kind: "Tenant", + key: encodeKey(channel.tenantId), + activeCountLimit: limits.maxActiveMessagesPerTenant, + activeBytesLimit: limits.maxActiveBytesPerTenant, + retainedCountLimit: limits.maxRetainedRowsPerTenant, + retainedBytesLimit: limits.maxRetainedBytesPerTenant + }, + { + kind: "Shard", + key: encodeKey("local"), + activeCountLimit: limits.maxActiveMessagesPerShard, + activeBytesLimit: limits.maxActiveBytesPerShard, + retainedCountLimit: limits.maxRetainedRowsPerShard, + retainedBytesLimit: limits.maxRetainedBytesPerShard + } +] + +const TimeRow = Schema.Struct({ now: NonNegativeInt }) +const UnitRow = Schema.Struct({ value: Schema.Int }) + +const nowQuery = (sql: SqlClient.SqlClient) => + SqlSchema.findOne({ + Request: Schema.Void, + Result: TimeRow, + execute: () => + sql.onDialectOrElse({ + pg: () => sql`SELECT CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS BIGINT) AS now`, + mysql: () => sql`SELECT CAST(UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000 AS UNSIGNED) AS now`, + orElse: () => sql`SELECT CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) AS now` + }) + })(undefined) + +const decodeDocuments = Schema.decodeUnknownEffect( + Schema.fromJsonString(DocumentIds) +) + +const parseDocuments = (value: string) => + decodeDocuments(value).pipe( + Effect.mapError((cause) => storageCorrupt(cause)) + ) + +const validateInput = (schema: S, input: unknown) => + Schema.decodeUnknownEffect(schema)(input).pipe( + Effect.mapError((cause) => protocolMismatch("valid relay store request", String(cause))) + ) + +const checkDurability = ( + sql: SqlClient.SqlClient +): Effect.Effect< + void, + SqlError.SqlError | Schema.SchemaError | ReplicaError.ReplicaError +> => + sql.onDialectOrElse({ + sqlite: () => + Effect.gen(function*() { + const journal = SqlSchema.findOne({ + Request: Schema.Void, + Result: Schema.Struct({ journal_mode: Schema.String }), + execute: () => sql`PRAGMA journal_mode = WAL` + }) + const synchronous = SqlSchema.findOne({ + Request: Schema.Void, + Result: Schema.Struct({ synchronous: Schema.Int }), + execute: () => sql`PRAGMA synchronous` + }) + const mode = yield* journal(undefined).pipe( + Effect.catchTag("NoSuchElementError", (cause) => Effect.fail(storageCorrupt(cause))) + ) + if (mode.journal_mode.toLowerCase() !== "wal") { + return yield* storageCorrupt(new Error("Relay custody requires SQLite WAL mode")) + } + yield* sql`PRAGMA synchronous = FULL` + const setting = yield* synchronous(undefined).pipe( + Effect.catchTag("NoSuchElementError", (cause) => Effect.fail(storageCorrupt(cause))) + ) + if (setting.synchronous !== 2) { + return yield* storageCorrupt(new Error("Relay custody requires SQLite FULL synchronous mode")) + } + }), + orElse: () => Effect.void + }) + +const DuplicateRow = Schema.Struct({ + channelId: PositiveInt, + tenantId: Schema.NonEmptyString, + senderSubjectId: Schema.NonEmptyString, + senderPeerId: Identity.PeerId, + senderReplicaIncarnation: DatabaseReplicaIncarnation, + recipientSubjectId: Schema.NonEmptyString, + recipientPeerId: Identity.PeerId, + outerEnvelopeDigest: PeerRpc.RelayDigest, + state: Schema.Literals(["Pending", "Claimed", "Acknowledged", "DeadLettered", "Expired"]), + nextEligibleAt: NonNegativeInt +}) + +const ChannelRow = Schema.Struct({ + channelId: PositiveInt, + nextSequence: NonNegativeInt +}) + +const MessageIdRow = Schema.Struct({ messageId: PositiveInt }) + +const CandidateRow = Schema.Struct({ + messageId: PositiveInt, + channelId: PositiveInt, + tenantId: Schema.NonEmptyString, + senderSubjectId: Schema.NonEmptyString, + senderPeerId: Identity.PeerId, + senderReplicaIncarnation: DatabaseReplicaIncarnation, + recipientSubjectId: Schema.NonEmptyString, + recipientPeerId: Identity.PeerId, + relayMessageId: Identity.RelayMessageId, + relayPeerId: Identity.PeerId, + senderConnectionEpoch: Schema.NonEmptyString, + senderSequence: NonNegativeInt, + documentIds: Schema.String, + payloadVersion: Schema.Literal(1), + messageHash: Schema.NonEmptyString, + outerEnvelopeDigest: PeerRpc.RelayDigest, + payloadBytes: NonNegativeInt, + createdAt: NonNegativeInt, + nextEligibleAt: NonNegativeInt, + retryCount: NonNegativeInt +}) + +const ReservationRow = Schema.Struct({ + senderPeerUsageKey: Schema.String, + recipientPeerUsageKey: Schema.String, + recipientSubjectUsageKey: Schema.String, + tenantUsageKey: Schema.String, + shardUsageKey: Schema.String, + activeCountDelta: Schema.Literal(1), + activeBytesDelta: NonNegativeInt, + retainedCountDelta: Schema.Literal(1), + retainedBytesDelta: NonNegativeInt, + activeConsumed: Schema.Literals([0, 1]), + retainedConsumed: Schema.Literals([0, 1]) +}) + +const UsageRow = Schema.Struct({ + activeCount: NonNegativeInt, + activeBytes: NonNegativeInt, + retainedCount: NonNegativeInt, + retainedBytes: NonNegativeInt +}) + +const KeyRow = Schema.Struct({ messageId: PositiveInt }) + +const makeService = Effect.gen(function*() { + const sql = (yield* SqlClient.SqlClient).withoutTransforms() + const crypto = yield* Crypto.Crypto + const limits = yield* PeerRelayLimits.PeerRelayLimits + yield* PeerRelayMigrations.run + yield* checkDurability(sql) + + const write = makeWriteTransaction(sql, { + maxAcquireAttempts: limits.sqliteLockRetryMaxAttempts, + acquireRetryBaseDelayMillis: limits.sqliteLockRetryBaseDelayMillis, + acquireRetryMaximumDelayMillis: limits.sqliteLockRetryMaximumDelayMillis + }) + + const findDuplicate = SqlSchema.findOneOption({ + Request: Schema.Struct({ + tenantId: Schema.String, + senderSubjectId: Schema.String, + senderPeerId: Schema.String, + relayMessageId: Schema.String + }), + Result: DuplicateRow, + execute: (request) => + sql`SELECT + m.channel_id AS "channelId", + c.tenant_id AS "tenantId", + c.sender_subject_id AS "senderSubjectId", + c.sender_peer_id AS "senderPeerId", + c.sender_replica_incarnation AS "senderReplicaIncarnation", + c.recipient_subject_id AS "recipientSubjectId", + c.recipient_peer_id AS "recipientPeerId", + m.outer_envelope_digest AS "outerEnvelopeDigest", + m.state, + m.next_eligible_at AS "nextEligibleAt" + FROM effect_local_relay_messages m + JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id + JOIN effect_local_relay_reservations r ON r.message_id = m.message_id + WHERE m.tenant_id = ${request.tenantId} + AND m.sender_subject_id = ${request.senderSubjectId} + AND m.sender_peer_id = ${request.senderPeerId} + AND m.relay_message_id = ${request.relayMessageId}` + }) + + const findChannel = SqlSchema.findOne({ + Request: ChannelKey, + Result: ChannelRow, + execute: (channel) => + sql`SELECT + channel_id AS "channelId", + next_sequence AS "nextSequence" + FROM effect_local_relay_channels + WHERE tenant_id = ${channel.tenantId} + AND sender_subject_id = ${channel.senderSubjectId} + AND sender_peer_id = ${channel.senderPeerId} + AND sender_replica_incarnation = ${channel.senderReplicaIncarnation} + AND recipient_subject_id = ${channel.recipientSubjectId} + AND recipient_peer_id = ${channel.recipientPeerId}` + }) + + const insertUsage = sql.onDialectOrElse({ + mysql: () => (scope: UsageScope) => + sql`INSERT IGNORE INTO effect_local_relay_usage ( + scope_kind, + scope_key, + active_count, + active_bytes, + retained_count, + retained_bytes + ) VALUES (${scope.kind}, ${scope.key}, 0, 0, 0, 0)`, + orElse: () => (scope: UsageScope) => + sql`INSERT INTO effect_local_relay_usage ( + scope_kind, + scope_key, + active_count, + active_bytes, + retained_count, + retained_bytes + ) VALUES (${scope.kind}, ${scope.key}, 0, 0, 0, 0) + ON CONFLICT(scope_kind, scope_key) DO NOTHING` + }) + + const mutationCount = ( + mysql: Effect.Effect, + returning: Effect.Effect, SqlError.SqlError | Schema.SchemaError> + ) => + sql.onDialectOrElse({ + mysql: () => + mysql.pipe( + Effect.flatMap((result) => + Schema.decodeUnknownEffect( + Schema.Struct({ affectedRows: NonNegativeInt }) + )(result) + ), + Effect.map((result) => result.affectedRows) + ), + orElse: () => returning.pipe(Effect.map((rows) => rows.length)) + }) + + const reserveUsage = ( + scope: UsageScope, + payloadBytes: number + ): Effect.Effect => + Effect.gen(function*() { + yield* insertUsage(scope) + const changed = yield* mutationCount( + sql`UPDATE effect_local_relay_usage + SET active_count = active_count + 1, + active_bytes = active_bytes + ${payloadBytes}, + retained_count = retained_count + 1, + retained_bytes = retained_bytes + ${payloadBytes} + WHERE scope_kind = ${scope.kind} + AND scope_key = ${scope.key} + AND active_count + 1 <= ${scope.activeCountLimit} + AND active_bytes + ${payloadBytes} <= ${scope.activeBytesLimit} + AND retained_count + 1 <= ${scope.retainedCountLimit} + AND retained_bytes + ${payloadBytes} <= ${scope.retainedBytesLimit}`.raw, + SqlSchema.findAll({ + Request: Schema.Void, + Result: UnitRow, + execute: () => + sql`UPDATE effect_local_relay_usage + SET active_count = active_count + 1, + active_bytes = active_bytes + ${payloadBytes}, + retained_count = retained_count + 1, + retained_bytes = retained_bytes + ${payloadBytes} + WHERE scope_kind = ${scope.kind} + AND scope_key = ${scope.key} + AND active_count + 1 <= ${scope.activeCountLimit} + AND active_bytes + ${payloadBytes} <= ${scope.activeBytesLimit} + AND retained_count + 1 <= ${scope.retainedCountLimit} + AND retained_bytes + ${payloadBytes} <= ${scope.retainedBytesLimit} + RETURNING 1 AS value` + })(undefined) + ) + if (changed !== 1) { + return yield* quotaExceeded( + `${scope.kind} relay custody`, + Math.min(scope.activeCountLimit, scope.retainedCountLimit) + ) + } + }) + + const findReservation = SqlSchema.findOneOption({ + Request: PositiveInt, + Result: ReservationRow, + execute: (messageId) => + sql`SELECT + sender_peer_usage_key AS "senderPeerUsageKey", + recipient_peer_usage_key AS "recipientPeerUsageKey", + recipient_subject_usage_key AS "recipientSubjectUsageKey", + tenant_usage_key AS "tenantUsageKey", + shard_usage_key AS "shardUsageKey", + active_count_delta AS "activeCountDelta", + active_bytes_delta AS "activeBytesDelta", + retained_count_delta AS "retainedCountDelta", + retained_bytes_delta AS "retainedBytesDelta", + active_consumed AS "activeConsumed", + retained_consumed AS "retainedConsumed" + FROM effect_local_relay_reservations + WHERE message_id = ${messageId}` + }) + + const usageKeys = ( + reservation: typeof ReservationRow.Type + ): ReadonlyArray => [ + ["SenderPeer", reservation.senderPeerUsageKey], + ["RecipientPeer", reservation.recipientPeerUsageKey], + ["RecipientSubject", reservation.recipientSubjectUsageKey], + ["Tenant", reservation.tenantUsageKey], + ["Shard", reservation.shardUsageKey] + ] + + const releaseActiveUsage = ( + messageId: number + ): Effect.Effect => + Effect.gen(function*() { + const reservationOption = yield* findReservation(messageId) + if (Option.isNone(reservationOption)) { + return yield* storageCorrupt(new Error("Missing relay quota reservation")) + } + const reservation = reservationOption.value + if (reservation.activeConsumed === 1) { + return + } + for (const [kind, key] of usageKeys(reservation)) { + const changed = yield* mutationCount( + sql`UPDATE effect_local_relay_usage + SET active_count = active_count - ${reservation.activeCountDelta}, + active_bytes = active_bytes - ${reservation.activeBytesDelta} + WHERE scope_kind = ${kind} + AND scope_key = ${key} + AND active_count >= ${reservation.activeCountDelta} + AND active_bytes >= ${reservation.activeBytesDelta}`.raw, + SqlSchema.findAll({ + Request: Schema.Void, + Result: UnitRow, + execute: () => + sql`UPDATE effect_local_relay_usage + SET active_count = active_count - ${reservation.activeCountDelta}, + active_bytes = active_bytes - ${reservation.activeBytesDelta} + WHERE scope_kind = ${kind} + AND scope_key = ${key} + AND active_count >= ${reservation.activeCountDelta} + AND active_bytes >= ${reservation.activeBytesDelta} + RETURNING 1 AS value` + })(undefined) + ) + if (changed !== 1) { + return yield* storageCorrupt(new Error("Invalid active relay quota reservation")) + } + yield* sql`DELETE FROM effect_local_relay_usage + WHERE scope_kind = ${kind} + AND scope_key = ${key} + AND active_count = 0 + AND active_bytes = 0 + AND retained_count = 0 + AND retained_bytes = 0` + } + yield* sql`UPDATE effect_local_relay_reservations + SET active_consumed = 1 + WHERE message_id = ${messageId} AND active_consumed = 0` + }) + + const releaseRetainedUsage = ( + messageId: number + ): Effect.Effect => + Effect.gen(function*() { + const reservationOption = yield* findReservation(messageId) + if (Option.isNone(reservationOption)) { + return yield* storageCorrupt(new Error("Missing relay quota reservation")) + } + const reservation = reservationOption.value + if (reservation.retainedConsumed === 1) { + return + } + for (const [kind, key] of usageKeys(reservation)) { + const changed = yield* mutationCount( + sql`UPDATE effect_local_relay_usage + SET retained_count = retained_count - ${reservation.retainedCountDelta}, + retained_bytes = retained_bytes - ${reservation.retainedBytesDelta} + WHERE scope_kind = ${kind} + AND scope_key = ${key} + AND retained_count >= ${reservation.retainedCountDelta} + AND retained_bytes >= ${reservation.retainedBytesDelta}`.raw, + SqlSchema.findAll({ + Request: Schema.Void, + Result: UnitRow, + execute: () => + sql`UPDATE effect_local_relay_usage + SET retained_count = retained_count - ${reservation.retainedCountDelta}, + retained_bytes = retained_bytes - ${reservation.retainedBytesDelta} + WHERE scope_kind = ${kind} + AND scope_key = ${key} + AND retained_count >= ${reservation.retainedCountDelta} + AND retained_bytes >= ${reservation.retainedBytesDelta} + RETURNING 1 AS value` + })(undefined) + ) + if (changed !== 1) { + return yield* storageCorrupt(new Error("Invalid retained relay quota reservation")) + } + yield* sql`DELETE FROM effect_local_relay_usage + WHERE scope_kind = ${kind} + AND scope_key = ${key} + AND active_count = 0 + AND active_bytes = 0 + AND retained_count = 0 + AND retained_bytes = 0` + } + yield* sql`UPDATE effect_local_relay_reservations + SET retained_consumed = 1 + WHERE message_id = ${messageId} AND retained_consumed = 0` + }) + + const greatest = sql.onDialectOrElse({ + sqlite: () => "MAX", + orElse: () => "GREATEST" + }) + + const terminalize = ( + messageId: number, + state: "Acknowledged" | "DeadLettered" | "Expired", + now: number, + terminal: { + readonly token?: PeerRpc.ClaimToken + readonly sessionGeneration?: number + readonly reason: string + } + ) => + Effect.gen(function*() { + yield* releaseActiveUsage(messageId) + yield* sql`UPDATE effect_local_relay_messages + SET state = ${state}, + payload = NULL, + payload_length = 0, + claim_token = NULL, + claim_session_generation = NULL, + claim_deadline = NULL, + terminal_at = ${now}, + terminal_claim_token = ${terminal.token ?? null}, + terminal_session_generation = ${terminal.sessionGeneration ?? null}, + terminal_reason = ${terminal.reason}, + deduplicate_until = ${sql.literal(greatest)}( + deduplicate_until, + ${now + limits.minimumTerminalRetentionMillis} + ) + WHERE message_id = ${messageId} + AND state IN ('Pending', 'Claimed')` + yield* sql`UPDATE effect_local_relay_channels + SET claimed_message_id = NULL, + claim_session_generation = NULL, + claim_token = NULL, + claim_deadline = NULL + WHERE claimed_message_id = ${messageId}` + }) + + const insertChannel = sql.onDialectOrElse({ + mysql: () => (channel: ChannelKey) => + sql`INSERT IGNORE INTO effect_local_relay_channels ( + tenant_id, + sender_subject_id, + sender_peer_id, + sender_replica_incarnation, + recipient_subject_id, + recipient_peer_id + ) VALUES ( + ${channel.tenantId}, + ${channel.senderSubjectId}, + ${channel.senderPeerId}, + ${channel.senderReplicaIncarnation}, + ${channel.recipientSubjectId}, + ${channel.recipientPeerId} + )`, + orElse: () => (channel: ChannelKey) => + sql`INSERT INTO effect_local_relay_channels ( + tenant_id, + sender_subject_id, + sender_peer_id, + sender_replica_incarnation, + recipient_subject_id, + recipient_peer_id + ) VALUES ( + ${channel.tenantId}, + ${channel.senderSubjectId}, + ${channel.senderPeerId}, + ${channel.senderReplicaIncarnation}, + ${channel.recipientSubjectId}, + ${channel.recipientPeerId} + ) + ON CONFLICT ( + tenant_id, + sender_subject_id, + sender_peer_id, + sender_replica_incarnation, + recipient_subject_id, + recipient_peer_id + ) DO NOTHING` + }) + + const admit: Service["admit"] = (unsafeInput) => { + let observedBytes: number | undefined + let observedVersion: number | undefined + const effect = mapStoreErrors(Effect.gen(function*() { + const input = yield* validateInput(Admission, unsafeInput) + const payloadBytes = input.payload.byteLength + observedBytes = payloadBytes + observedVersion = input.payloadVersion + if ( + input.messageTtlMillis > limits.messageTtlMillis || + input.senderRetryHorizonMillis > limits.maximumSenderRetryHorizonMillis || + input.minimumTerminalRetentionMillis < limits.minimumTerminalRetentionMillis + ) { + return yield* protocolMismatch("configured relay retention horizon", "unsupported horizon") + } + if ( + payloadBytes > limits.maxActiveBytesPerSenderPeer || + payloadBytes > limits.maxActiveBytesPerRecipientPeer || + payloadBytes > limits.maxActiveBytesPerRecipientSubject || + payloadBytes > limits.maxActiveBytesPerTenant || + payloadBytes > limits.maxActiveBytesPerShard + ) { + return yield* quotaExceeded("relay payload bytes", limits.maxActiveBytesPerShard) + } + return yield* write(Effect.gen(function*() { + const now = (yield* nowQuery(sql)).now + const existing = yield* findDuplicate({ + tenantId: input.channel.tenantId, + senderSubjectId: input.channel.senderSubjectId, + senderPeerId: input.channel.senderPeerId, + relayMessageId: input.relayMessageId + }) + if (Option.isSome(existing)) { + if (existing.value.outerEnvelopeDigest !== input.outerEnvelopeDigest) { + return yield* protocolMismatch( + existing.value.outerEnvelopeDigest, + input.outerEnvelopeDigest + ) + } + return AdmissionResult.make({ + status: "Duplicate", + channel: ChannelKey.make({ + tenantId: existing.value.tenantId, + senderSubjectId: existing.value.senderSubjectId, + senderPeerId: existing.value.senderPeerId, + senderReplicaIncarnation: existing.value.senderReplicaIncarnation, + recipientSubjectId: existing.value.recipientSubjectId, + recipientPeerId: existing.value.recipientPeerId + }), + ready: existing.value.state === "Pending" && existing.value.nextEligibleAt <= now, + nextEligibleAt: existing.value.nextEligibleAt, + lane: "New" + }) + } + yield* insertChannel(input.channel) + const channel = yield* findChannel(input.channel) + const scopes = channelScopes(input.channel, limits) + for (const scope of scopes) { + yield* reserveUsage(scope, payloadBytes) + } + const duplicateHorizon = Math.max( + input.messageTtlMillis, + input.senderRetryHorizonMillis + ) + const insertMessage = sql`INSERT INTO effect_local_relay_messages ( + channel_id, + channel_sequence, + tenant_id, + sender_subject_id, + sender_peer_id, + recipient_subject_id, + recipient_peer_id, + relay_message_id, + relay_peer_id, + sender_connection_epoch, + sender_sequence, + document_ids, + payload_version, + message_hash, + outer_envelope_digest, + payload, + payload_length, + state, + created_at, + expires_at, + deduplicate_until, + next_eligible_at + ) VALUES ( + ${channel.channelId}, + ${channel.nextSequence}, + ${input.channel.tenantId}, + ${input.channel.senderSubjectId}, + ${input.channel.senderPeerId}, + ${input.channel.recipientSubjectId}, + ${input.channel.recipientPeerId}, + ${input.relayMessageId}, + ${input.relayPeerId}, + ${input.senderConnectionEpoch}, + ${input.senderSequence}, + ${JSON.stringify(input.documentIds)}, + ${input.payloadVersion}, + ${input.messageHash}, + ${input.outerEnvelopeDigest}, + ${input.payload}, + ${payloadBytes}, + 'Pending', + ${now}, + ${now + input.messageTtlMillis}, + ${now + duplicateHorizon}, + ${now} + )` + const inserted = yield* sql.onDialectOrElse({ + mysql: () => + insertMessage.raw.pipe( + Effect.flatMap((result) => + Schema.decodeUnknownEffect( + Schema.Struct({ insertId: PositiveInt }) + )(result) + ), + Effect.map((result) => MessageIdRow.make({ messageId: result.insertId })) + ), + orElse: () => + SqlSchema.findOne({ + Request: Schema.Void, + Result: MessageIdRow, + execute: () => sql`${insertMessage} RETURNING message_id AS "messageId"` + })(undefined) + }) + yield* sql`INSERT INTO effect_local_relay_reservations ( + message_id, + sender_peer_usage_key, + recipient_peer_usage_key, + recipient_subject_usage_key, + tenant_usage_key, + shard_usage_key, + active_count_delta, + active_bytes_delta, + retained_count_delta, + retained_bytes_delta + ) VALUES ( + ${inserted.messageId}, + ${scopes[0]!.key}, + ${scopes[1]!.key}, + ${scopes[2]!.key}, + ${scopes[3]!.key}, + ${scopes[4]!.key}, + 1, + ${payloadBytes}, + 1, + ${payloadBytes} + )` + yield* sql`UPDATE effect_local_relay_channels + SET next_sequence = next_sequence + 1 + WHERE channel_id = ${channel.channelId} + AND next_sequence = ${channel.nextSequence}` + return AdmissionResult.make({ + status: "Accepted", + channel: input.channel, + ready: true, + nextEligibleAt: now, + lane: "New" + }) + })) + })).pipe(Effect.tapError(recordQuotaRejection)) + return PeerRpcObservability.observeRelay({ + effect, + operation: "RelayAdmit", + direction: "Send", + facts: () => ({ + ...(observedBytes === undefined ? {} : { bytes: observedBytes }), + ...(observedVersion === undefined ? {} : { version: observedVersion }), + items: observedBytes === undefined ? 0 : 1 + }), + result: (exit) => + Exit.isSuccess(exit) + ? exit.value.status + : relayFailureResult(exit) + }) + } + + const findCandidate = SqlSchema.findOneOption({ + Request: Schema.Struct({ + tenantId: Schema.String, + recipientSubjectId: Schema.String, + recipientPeerId: Schema.String, + senderSubjectId: Schema.String, + senderPeerId: Schema.String, + authorizedDocumentIds: Schema.String, + now: NonNegativeInt + }), + Result: CandidateRow, + execute: (request) => { + const unauthorizedDocument = sql.onDialectOrElse({ + pg: () => + sql`SELECT 1 + FROM jsonb_array_elements_text(CAST(m.document_ids AS JSONB)) document(value) + WHERE document.value NOT IN ( + SELECT value + FROM jsonb_array_elements_text(CAST(${request.authorizedDocumentIds} AS JSONB)) authorized(value) + )`, + mysql: () => + sql`SELECT 1 + FROM JSON_TABLE( + m.document_ids, + '$[*]' COLUMNS(value VARCHAR(256) PATH '$') + ) document + WHERE document.value NOT IN ( + SELECT authorized.value + FROM JSON_TABLE( + CAST(${request.authorizedDocumentIds} AS JSON), + '$[*]' COLUMNS(value VARCHAR(256) PATH '$') + ) authorized + )`, + orElse: () => + sql`SELECT 1 + FROM json_each(m.document_ids) document + WHERE document.value NOT IN ( + SELECT value FROM json_each(${request.authorizedDocumentIds}) + )` + }) + return sql`SELECT + m.message_id AS "messageId", + m.channel_id AS "channelId", + c.tenant_id AS "tenantId", + c.sender_subject_id AS "senderSubjectId", + c.sender_peer_id AS "senderPeerId", + c.sender_replica_incarnation AS "senderReplicaIncarnation", + c.recipient_subject_id AS "recipientSubjectId", + c.recipient_peer_id AS "recipientPeerId", + m.relay_message_id AS "relayMessageId", + m.relay_peer_id AS "relayPeerId", + m.sender_connection_epoch AS "senderConnectionEpoch", + m.sender_sequence AS "senderSequence", + m.document_ids AS "documentIds", + m.payload_version AS "payloadVersion", + m.message_hash AS "messageHash", + m.outer_envelope_digest AS "outerEnvelopeDigest", + m.payload_length AS "payloadBytes", + m.created_at AS "createdAt", + m.next_eligible_at AS "nextEligibleAt", + m.retry_count AS "retryCount" + FROM effect_local_relay_messages m + JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id + JOIN effect_local_relay_reservations r ON r.message_id = m.message_id + WHERE m.tenant_id = ${request.tenantId} + AND m.sender_subject_id = ${request.senderSubjectId} + AND m.sender_peer_id = ${request.senderPeerId} + AND m.recipient_subject_id = ${request.recipientSubjectId} + AND m.recipient_peer_id = ${request.recipientPeerId} + AND c.tenant_id = ${request.tenantId} + AND c.recipient_subject_id = ${request.recipientSubjectId} + AND c.recipient_peer_id = ${request.recipientPeerId} + AND c.sender_subject_id = ${request.senderSubjectId} + AND c.sender_peer_id = ${request.senderPeerId} + AND m.tenant_id = c.tenant_id + AND m.sender_subject_id = c.sender_subject_id + AND m.sender_peer_id = c.sender_peer_id + AND m.recipient_subject_id = c.recipient_subject_id + AND m.recipient_peer_id = c.recipient_peer_id + AND c.claimed_message_id IS NULL + AND m.state = 'Pending' + AND m.next_eligible_at <= ${request.now} + AND m.expires_at > ${request.now} + AND m.payload IS NOT NULL + AND m.payload_length = length(m.payload) + AND r.active_consumed = 0 + AND NOT EXISTS ( + SELECT 1 + FROM effect_local_relay_messages earlier + WHERE earlier.channel_id = m.channel_id + AND earlier.channel_sequence < m.channel_sequence + AND earlier.state IN ('Pending', 'Claimed') + ) + AND NOT EXISTS ( + ${unauthorizedDocument} + ) + ORDER BY m.created_at, m.message_id + LIMIT 1` + } + }) + + const claim: Service["claim"] = (unsafeInput) => { + let observedAttempt: number | undefined + const effect = mapStoreErrors(Effect.gen(function*() { + const input = yield* validateInput(ClaimRequest, unsafeInput) + return yield* write(Effect.gen(function*() { + const now = (yield* nowQuery(sql)).now + const candidateOption = yield* findCandidate({ + tenantId: input.recipient.tenantId, + recipientSubjectId: input.recipient.subjectId, + recipientPeerId: input.recipient.peerId, + senderSubjectId: input.sender.subjectId, + senderPeerId: input.sender.peerId, + authorizedDocumentIds: JSON.stringify(input.authorizedDocumentIds), + now + }) + if (Option.isNone(candidateOption)) { + return { + message: Option.none(), + ready: false, + nextEligibleAt: Option.none(), + lane: "New" as const + } + } + const candidate = candidateOption.value + observedAttempt = candidate.retryCount + 1 + const documentIds = yield* parseDocuments(candidate.documentIds) + const uuid = yield* crypto.randomUUIDv4 + const token = PeerRpc.ClaimToken.make(`clm_${uuid}`) + const deadline = now + limits.claimLeaseMillis + const claimed = yield* mutationCount( + sql`UPDATE effect_local_relay_messages + SET state = 'Claimed', + claim_token = ${token}, + claim_session_generation = ${input.sessionGeneration}, + claim_deadline = ${deadline} + WHERE message_id = ${candidate.messageId} + AND state = 'Pending'`.raw, + SqlSchema.findAll({ + Request: Schema.Void, + Result: UnitRow, + execute: () => + sql`UPDATE effect_local_relay_messages + SET state = 'Claimed', + claim_token = ${token}, + claim_session_generation = ${input.sessionGeneration}, + claim_deadline = ${deadline} + WHERE message_id = ${candidate.messageId} + AND state = 'Pending' + RETURNING 1 AS value` + })(undefined) + ) + if (claimed !== 1) { + return yield* storageCorrupt(new Error("Relay claim compare and swap failed")) + } + const channelClaimed = yield* mutationCount( + sql`UPDATE effect_local_relay_channels + SET claimed_message_id = ${candidate.messageId}, + claim_session_generation = ${input.sessionGeneration}, + claim_token = ${token}, + claim_deadline = ${deadline} + WHERE channel_id = ${candidate.channelId} + AND claimed_message_id IS NULL`.raw, + SqlSchema.findAll({ + Request: Schema.Void, + Result: UnitRow, + execute: () => + sql`UPDATE effect_local_relay_channels + SET claimed_message_id = ${candidate.messageId}, + claim_session_generation = ${input.sessionGeneration}, + claim_token = ${token}, + claim_deadline = ${deadline} + WHERE channel_id = ${candidate.channelId} + AND claimed_message_id IS NULL + RETURNING 1 AS value` + })(undefined) + ) + if (channelClaimed !== 1) { + return yield* storageCorrupt(new Error("Relay channel claim compare and swap failed")) + } + return { + message: Option.some(ClaimedMessage.make({ + rowId: candidate.messageId, + channel: ChannelKey.make({ + tenantId: candidate.tenantId, + senderSubjectId: candidate.senderSubjectId, + senderPeerId: candidate.senderPeerId, + senderReplicaIncarnation: candidate.senderReplicaIncarnation, + recipientSubjectId: candidate.recipientSubjectId, + recipientPeerId: candidate.recipientPeerId + }), + relayMessageId: candidate.relayMessageId, + relayPeerId: candidate.relayPeerId, + senderConnectionEpoch: candidate.senderConnectionEpoch, + senderSequence: candidate.senderSequence, + documentIds, + payloadVersion: candidate.payloadVersion, + messageHash: candidate.messageHash, + outerEnvelopeDigest: candidate.outerEnvelopeDigest, + payloadBytes: candidate.payloadBytes, + claimToken: token, + claimDeadline: deadline, + sessionGeneration: input.sessionGeneration + })), + ready: false, + nextEligibleAt: Option.none(), + lane: candidate.retryCount === 0 ? "New" as const : "Retry" as const + } + })) + })) + return PeerRpcObservability.observeRelay({ + effect, + operation: "RelayClaim", + direction: "Receive", + facts: (exit) => { + if (Exit.isFailure(exit) || Option.isNone(exit.value.message)) { + return { items: 0 } + } + const message = exit.value.message.value + return { + bytes: message.payloadBytes, + items: 1, + ...(observedAttempt === undefined ? {} : { attempt: observedAttempt }), + version: message.payloadVersion + } + }, + result: (exit) => + Exit.isSuccess(exit) + ? Option.isSome(exit.value.message) ? "Claimed" : "Empty" + : relayFailureResult(exit) + }) + } + + const loadClaimedPayload: Service["loadClaimedPayload"] = (unsafeInput) => + mapStoreErrors(Effect.gen(function*() { + const input = yield* validateInput(LoadClaimedPayloadRequest, unsafeInput) + const now = (yield* nowQuery(sql)).now + const rows = yield* SqlSchema.findAll({ + Request: Schema.Void, + Result: Schema.Struct({ payload: Schema.Uint8Array }), + execute: () => + sql`SELECT m.payload + FROM effect_local_relay_messages m + JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id + WHERE m.message_id = ${input.rowId} + AND m.channel_id = c.channel_id + AND c.tenant_id = ${input.channel.tenantId} + AND c.sender_subject_id = ${input.channel.senderSubjectId} + AND c.sender_peer_id = ${input.channel.senderPeerId} + AND c.sender_replica_incarnation = ${input.channel.senderReplicaIncarnation} + AND c.recipient_subject_id = ${input.channel.recipientSubjectId} + AND c.recipient_peer_id = ${input.channel.recipientPeerId} + AND m.tenant_id = c.tenant_id + AND m.sender_subject_id = c.sender_subject_id + AND m.sender_peer_id = c.sender_peer_id + AND m.relay_message_id = ${input.relayMessageId} + AND m.state = 'Claimed' + AND m.claim_token = ${input.claimToken} + AND m.claim_session_generation = ${input.sessionGeneration} + AND c.claimed_message_id = m.message_id + AND c.claim_token = m.claim_token + AND c.claim_session_generation = m.claim_session_generation + AND c.claim_deadline = m.claim_deadline + AND m.claim_deadline > ${now} + AND m.expires_at > ${now} + AND m.payload_length = ${input.payloadBytes} + AND length(m.payload) = ${input.payloadBytes} + AND m.payload_length <= ${limits.maxActiveBytesPerShard}` + })(undefined) + if (rows.length !== 1) { + return yield* protocolMismatch("active relay claim", "stale relay claim") + } + return rows[0]!.payload + })) + + const TerminalRow = Schema.Struct({ + messageId: PositiveInt, + channelId: PositiveInt, + state: Schema.Literals(["Pending", "Claimed", "Acknowledged", "DeadLettered", "Expired"]), + messageHash: Schema.NonEmptyString, + claimToken: Schema.NullOr(PeerRpc.ClaimToken), + claimSessionGeneration: Schema.NullOr(NonNegativeInt), + claimDeadline: Schema.NullOr(NonNegativeInt), + createdAt: NonNegativeInt, + expiresAt: NonNegativeInt, + channelClaimedMessageId: Schema.NullOr(PositiveInt), + channelClaimToken: Schema.NullOr(PeerRpc.ClaimToken), + channelClaimSessionGeneration: Schema.NullOr(NonNegativeInt), + channelClaimDeadline: Schema.NullOr(NonNegativeInt), + terminalClaimToken: Schema.NullOr(PeerRpc.ClaimToken), + terminalSessionGeneration: Schema.NullOr(NonNegativeInt), + terminalReason: Schema.NullOr(Schema.String) + }) + + const findTerminal = SqlSchema.findOneOption({ + Request: Schema.Struct({ + channel: ChannelKey, + relayMessageId: Identity.RelayMessageId + }), + Result: TerminalRow, + execute: ({ channel, relayMessageId }) => + sql`SELECT + m.message_id AS "messageId", + m.channel_id AS "channelId", + m.state, + m.message_hash AS "messageHash", + m.claim_token AS "claimToken", + m.claim_session_generation AS "claimSessionGeneration", + m.claim_deadline AS "claimDeadline", + m.created_at AS "createdAt", + m.expires_at AS "expiresAt", + c.claimed_message_id AS "channelClaimedMessageId", + c.claim_token AS "channelClaimToken", + c.claim_session_generation AS "channelClaimSessionGeneration", + c.claim_deadline AS "channelClaimDeadline", + m.terminal_claim_token AS "terminalClaimToken", + m.terminal_session_generation AS "terminalSessionGeneration", + m.terminal_reason AS "terminalReason" + FROM effect_local_relay_messages m + JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id + WHERE c.tenant_id = ${channel.tenantId} + AND c.sender_subject_id = ${channel.senderSubjectId} + AND c.sender_peer_id = ${channel.senderPeerId} + AND c.sender_replica_incarnation = ${channel.senderReplicaIncarnation} + AND c.recipient_subject_id = ${channel.recipientSubjectId} + AND c.recipient_peer_id = ${channel.recipientPeerId} + AND m.relay_message_id = ${relayMessageId}` + }) + + const ReadyRow = Schema.Struct({ + nextEligibleAt: NonNegativeInt, + retryCount: NonNegativeInt + }) + + const nextReady = ( + channel: ChannelKey, + now: number + ): Effect.Effect< + { readonly ready: boolean; readonly nextEligibleAt: Option.Option; readonly lane: "New" | "Retry" }, + SqlError.SqlError | Schema.SchemaError + > => + SqlSchema.findOneOption({ + Request: Schema.Void, + Result: ReadyRow, + execute: () => + sql`SELECT + m.next_eligible_at AS "nextEligibleAt", + m.retry_count AS "retryCount" + FROM effect_local_relay_messages m + JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id + WHERE c.tenant_id = ${channel.tenantId} + AND c.sender_subject_id = ${channel.senderSubjectId} + AND c.sender_peer_id = ${channel.senderPeerId} + AND c.recipient_subject_id = ${channel.recipientSubjectId} + AND c.recipient_peer_id = ${channel.recipientPeerId} + AND c.claimed_message_id IS NULL + AND m.state = 'Pending' + AND m.expires_at > ${now} + AND NOT EXISTS ( + SELECT 1 + FROM effect_local_relay_messages earlier + WHERE earlier.channel_id = m.channel_id + AND earlier.channel_sequence < m.channel_sequence + AND earlier.state IN ('Pending', 'Claimed') + ) + ORDER BY m.next_eligible_at, m.created_at, m.message_id + LIMIT 1` + })(undefined).pipe( + Effect.map((row) => + Option.isNone(row) + ? { + ready: false, + nextEligibleAt: Option.none(), + lane: "New" as const + } + : { + ready: row.value.nextEligibleAt <= now, + nextEligibleAt: Option.some(row.value.nextEligibleAt), + lane: row.value.retryCount === 0 ? "New" as const : "Retry" as const + } + ) + ) + + const terminalTransition = ( + unsafeInput: TerminalRequest, + state: "Acknowledged" | "DeadLettered", + reason: string, + onChanged?: (latencyMillis: number) => void + ): Effect.Effect => + mapStoreErrors(Effect.gen(function*() { + const input = yield* validateInput(TerminalRequest, unsafeInput) + if ( + input.recipient.tenantId !== input.channel.tenantId || + input.recipient.subjectId !== input.channel.recipientSubjectId || + input.recipient.peerId !== input.channel.recipientPeerId + ) { + return { + status: "Stale", + ready: false, + nextEligibleAt: Option.none(), + lane: "New" + } as const + } + return yield* write(Effect.gen(function*() { + const now = (yield* nowQuery(sql)).now + const rowOption = yield* findTerminal({ + channel: input.channel, + relayMessageId: input.relayMessageId + }) + if (Option.isNone(rowOption)) { + return { + status: "Stale", + ready: false, + nextEligibleAt: Option.none(), + lane: "New" + } as const + } + const row = rowOption.value + const active = row.state === "Claimed" && + row.messageHash === input.messageHash && + row.claimToken === input.claimToken && + row.claimSessionGeneration === input.sessionGeneration && + row.claimDeadline !== null && + row.claimDeadline > now && + row.expiresAt > now && + row.channelClaimedMessageId === row.messageId && + row.channelClaimToken === row.claimToken && + row.channelClaimSessionGeneration === row.claimSessionGeneration && + row.channelClaimDeadline === row.claimDeadline + if (active) { + yield* terminalize(row.messageId, state, now, { + token: input.claimToken, + sessionGeneration: input.sessionGeneration, + reason + }) + if (onChanged !== undefined) { + yield* Effect.sync(() => onChanged(now - row.createdAt)) + } + const hint = yield* nextReady(input.channel, now) + return { status: "Changed", ...hint } as const + } + const duplicate = row.state === state && + row.messageHash === input.messageHash && + row.terminalClaimToken === input.claimToken && + row.terminalSessionGeneration === input.sessionGeneration && + row.terminalReason === reason + if (duplicate) { + const hint = yield* nextReady(input.channel, now) + return { status: "Duplicate", ...hint } as const + } + return { + status: "Stale", + ready: false, + nextEligibleAt: Option.none(), + lane: "New" + } as const + })) + })) + + const acknowledge: Service["acknowledge"] = (input) => { + let latencyMillis: number | undefined + const effect = terminalTransition( + input, + "Acknowledged", + "Acknowledged", + (latency) => { + latencyMillis = latency + } + ) + return PeerRpcObservability.observeRelay({ + effect, + operation: "RelayAcknowledge", + direction: "Receive", + facts: (exit) => ({ + items: Exit.isSuccess(exit) && exit.value.status !== "Stale" ? 1 : 0, + ...(Exit.isSuccess(exit) && + exit.value.status === "Changed" && + latencyMillis !== undefined + ? { latencyMillis } + : {}) + }), + result: (exit) => + Exit.isSuccess(exit) + ? exit.value.status === "Changed" + ? "Acknowledged" + : exit.value.status + : relayFailureResult(exit) + }) + } + + const reject: Service["reject"] = (unsafeInput) => { + const effect = mapStoreErrors(Effect.gen(function*() { + const input = yield* validateInput(RejectRequest, unsafeInput) + return yield* terminalTransition( + TerminalRequest.make({ + channel: input.channel, + relayMessageId: input.relayMessageId, + claimToken: input.claimToken, + messageHash: input.messageHash, + sessionGeneration: input.sessionGeneration, + recipient: input.recipient + }), + "DeadLettered", + input.reason + ) + })) + return PeerRpcObservability.observeRelay({ + effect, + operation: "RelayAcknowledge", + direction: "Receive", + facts: (exit) => ({ + items: Exit.isSuccess(exit) && exit.value.status !== "Stale" ? 1 : 0 + }), + result: (exit) => + Exit.isSuccess(exit) + ? exit.value.status === "Changed" + ? "DeadLettered" + : exit.value.status + : relayFailureResult(exit) + }) + } + + const release: Service["release"] = (unsafeInput) => { + const effect = mapStoreErrors(Effect.gen(function*() { + const input = yield* validateInput(ReleaseRequest, unsafeInput) + return yield* write(Effect.gen(function*() { + const now = (yield* nowQuery(sql)).now + const rowOption = yield* findTerminal({ + channel: input.channel, + relayMessageId: input.relayMessageId + }) + if (Option.isNone(rowOption)) { + return { + status: "Stale", + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" + } as const + } + const row = rowOption.value + if ( + row.state !== "Claimed" || + row.claimToken !== input.claimToken || + row.claimSessionGeneration !== input.sessionGeneration + ) { + return { + status: "Stale", + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" + } as const + } + const retry = yield* SqlSchema.findOne({ + Request: Schema.Void, + Result: Schema.Struct({ retryCount: PositiveInt }), + execute: () => + sql`SELECT retry_count + 1 AS "retryCount" + FROM effect_local_relay_messages + WHERE message_id = ${row.messageId}` + })(undefined) + if (retry.retryCount >= limits.maximumDeliveryAttempts) { + yield* sql`UPDATE effect_local_relay_messages + SET retry_count = ${retry.retryCount} + WHERE message_id = ${row.messageId} + AND state = 'Claimed' + AND claim_token = ${input.claimToken} + AND claim_session_generation = ${input.sessionGeneration}` + yield* terminalize(row.messageId, "DeadLettered", now, { + reason: "MaximumDeliveryAttempts" + }) + return { + status: "Changed", + ready: false, + nextEligibleAt: Option.none(), + lane: "Retry" + } as const + } + const random = yield* Random.next + const maximum = Math.min( + limits.retryMaximumDelayMillis, + limits.retryBaseDelayMillis * 2 ** Math.min(retry.retryCount - 1, 30) + ) + const delay = Math.max(1, Math.floor(maximum / 2 + random * maximum / 2)) + const nextEligibleAt = now + delay + yield* sql`UPDATE effect_local_relay_messages + SET state = 'Pending', + retry_count = ${retry.retryCount}, + next_eligible_at = ${nextEligibleAt}, + claim_token = NULL, + claim_session_generation = NULL, + claim_deadline = NULL + WHERE message_id = ${row.messageId} + AND state = 'Claimed' + AND claim_token = ${input.claimToken} + AND claim_session_generation = ${input.sessionGeneration}` + yield* sql`UPDATE effect_local_relay_channels + SET claimed_message_id = NULL, + claim_session_generation = NULL, + claim_token = NULL, + claim_deadline = NULL + WHERE channel_id = ${row.channelId} + AND claimed_message_id = ${row.messageId} + AND claim_token = ${input.claimToken} + AND claim_session_generation = ${input.sessionGeneration}` + return { + status: "Changed", + ready: false, + nextEligibleAt: Option.some(nextEligibleAt), + lane: "Retry" + } as const + })) + })) + return PeerRpcObservability.observeRelay({ + effect, + operation: "RelayRelease", + direction: "Receive", + facts: (exit) => ({ + items: Exit.isSuccess(exit) && exit.value.status === "Changed" ? 1 : 0 + }), + result: (exit) => + Exit.isSuccess(exit) + ? exit.value.status === "Changed" ? "Released" : exit.value.status + : relayFailureResult(exit) + }) + } + + const maintenanceResult = ( + ids: ReadonlyArray, + requestedBatchSize: number + ): MaintenanceResult => { + const processedIds = ids.slice(0, requestedBatchSize) + const cursor = processedIds.at(-1) + return cursor === undefined + ? { processed: 0, hasMore: false } + : { + cursor, + processed: processedIds.length, + hasMore: ids.length > requestedBatchSize + } + } + + const recover: Service["recover"] = (unsafeInput) => + Effect.suspend(() => { + let deadLettered = 0 + const effect = mapStoreErrors(Effect.gen(function*() { + const input = yield* validateInput(MaintenanceRequest, unsafeInput) + const effectiveBatch = Math.min(input.batchSize, limits.claimRecoveryBatchSize) + return yield* write(Effect.gen(function*() { + const now = (yield* nowQuery(sql)).now + const rows = yield* SqlSchema.findAll({ + Request: Schema.Void, + Result: Schema.Struct({ + messageId: PositiveInt, + retryCount: NonNegativeInt + }), + execute: () => + sql`SELECT + message_id AS "messageId", + retry_count AS "retryCount" + FROM effect_local_relay_messages + WHERE state = 'Claimed' + AND claim_deadline <= ${now} + ORDER BY claim_deadline, message_id + LIMIT ${sql.literal(String(effectiveBatch + 1))}` + })(undefined) + for (const row of rows.slice(0, effectiveBatch)) { + const retryCount = row.retryCount + 1 + if (retryCount >= limits.maximumDeliveryAttempts) { + deadLettered += 1 + yield* sql`UPDATE effect_local_relay_messages + SET retry_count = ${retryCount} + WHERE message_id = ${row.messageId} + AND state = 'Claimed' + AND claim_deadline <= ${now}` + yield* terminalize(row.messageId, "DeadLettered", now, { + reason: "MaximumDeliveryAttempts" + }) + continue + } + const random = yield* Random.next + const maximum = Math.min( + limits.retryMaximumDelayMillis, + limits.retryBaseDelayMillis * 2 ** Math.min(retryCount - 1, 30) + ) + const delay = Math.max(1, Math.floor(maximum / 2 + random * maximum / 2)) + yield* sql`UPDATE effect_local_relay_messages + SET state = 'Pending', + retry_count = ${retryCount}, + next_eligible_at = ${now + delay}, + claim_token = NULL, + claim_session_generation = NULL, + claim_deadline = NULL + WHERE message_id = ${row.messageId} + AND state = 'Claimed' + AND claim_deadline <= ${now}` + yield* sql`UPDATE effect_local_relay_channels + SET claimed_message_id = NULL, + claim_session_generation = NULL, + claim_token = NULL, + claim_deadline = NULL + WHERE claimed_message_id = ${row.messageId} + AND claim_deadline <= ${now}` + } + return maintenanceResult(rows.map((row) => row.messageId), effectiveBatch) + })) + })) + return PeerRpcObservability.observeRelay({ + effect, + operation: "RelayMaintenance", + direction: "Receive", + stage: "Recover", + facts: (exit) => ({ + items: Exit.isSuccess(exit) ? exit.value.processed : 0 + }), + result: (exit) => + Exit.isSuccess(exit) + ? deadLettered > 0 ? "DeadLettered" : "Released" + : relayFailureResult(exit) + }) + }) + + const expire: Service["expire"] = (unsafeInput) => { + const effect = mapStoreErrors(Effect.gen(function*() { + const input = yield* validateInput(MaintenanceRequest, unsafeInput) + const effectiveBatch = Math.min(input.batchSize, limits.expiryBatchSize) + return yield* write(Effect.gen(function*() { + const now = (yield* nowQuery(sql)).now + const rows = yield* SqlSchema.findAll({ + Request: Schema.Void, + Result: KeyRow, + execute: () => + sql`SELECT message_id AS "messageId" + FROM effect_local_relay_messages + WHERE state IN ('Pending', 'Claimed') + AND expires_at <= ${now} + ORDER BY expires_at, message_id + LIMIT ${sql.literal(String(effectiveBatch + 1))}` + })(undefined) + for (const row of rows.slice(0, effectiveBatch)) { + yield* terminalize(row.messageId, "Expired", now, { reason: "Expired" }) + } + return maintenanceResult(rows.map((row) => row.messageId), effectiveBatch) + })) + })) + return PeerRpcObservability.observeRelay({ + effect, + operation: "RelayMaintenance", + direction: "Receive", + stage: "Expire", + facts: (exit) => ({ + items: Exit.isSuccess(exit) ? exit.value.processed : 0 + }), + result: (exit) => Exit.isSuccess(exit) ? "Expired" : relayFailureResult(exit) + }) + } + + const IntegrityRow = Schema.Struct({ + messageId: PositiveInt, + state: Schema.String, + messageTenantId: Schema.NonEmptyString, + channelTenantId: Schema.NullOr(Schema.NonEmptyString), + messageSenderSubjectId: Schema.NonEmptyString, + channelSenderSubjectId: Schema.NullOr(Schema.NonEmptyString), + messageSenderPeerId: Schema.String, + channelSenderPeerId: Schema.NullOr(Schema.String), + messageRecipientSubjectId: Schema.NullOr(Schema.String), + channelRecipientSubjectId: Schema.NullOr(Schema.String), + messageRecipientPeerId: Schema.NullOr(Schema.String), + channelRecipientPeerId: Schema.NullOr(Schema.String), + payloadLength: NonNegativeInt, + actualLength: Schema.NullOr(NonNegativeInt), + reservationPresent: Schema.Literals([0, 1]), + activeConsumed: Schema.NullOr(Schema.Literals([0, 1])), + retainedConsumed: Schema.NullOr(Schema.Literals([0, 1])) + }) + + const repair: Service["repair"] = (unsafeInput) => { + const effect = mapStoreErrors(Effect.gen(function*() { + const input = yield* validateInput(MaintenanceRequest, unsafeInput) + const effectiveBatch = Math.min(input.batchSize, limits.integrityBatchSize) + return yield* write(Effect.gen(function*() { + const now = (yield* nowQuery(sql)).now + const rows = yield* SqlSchema.findAll({ + Request: Schema.Void, + Result: IntegrityRow, + execute: () => + sql`SELECT + m.message_id AS "messageId", + m.state, + m.tenant_id AS "messageTenantId", + c.tenant_id AS "channelTenantId", + m.sender_subject_id AS "messageSenderSubjectId", + c.sender_subject_id AS "channelSenderSubjectId", + m.sender_peer_id AS "messageSenderPeerId", + c.sender_peer_id AS "channelSenderPeerId", + m.recipient_subject_id AS "messageRecipientSubjectId", + c.recipient_subject_id AS "channelRecipientSubjectId", + m.recipient_peer_id AS "messageRecipientPeerId", + c.recipient_peer_id AS "channelRecipientPeerId", + m.payload_length AS "payloadLength", + length(m.payload) AS "actualLength", + CASE WHEN r.message_id IS NULL THEN 0 ELSE 1 END AS "reservationPresent", + r.active_consumed AS "activeConsumed", + r.retained_consumed AS "retainedConsumed" + FROM effect_local_relay_messages m + LEFT JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id + LEFT JOIN effect_local_relay_reservations r ON r.message_id = m.message_id + WHERE m.message_id > ${input.cursor ?? 0} + ORDER BY m.message_id + LIMIT ${sql.literal(String(effectiveBatch + 1))}` + })(undefined) + for (const row of rows.slice(0, effectiveBatch)) { + if ( + row.reservationPresent === 0 || + row.activeConsumed === null || + row.retainedConsumed === null + ) { + return yield* storageCorrupt(new Error("Relay reservation reconciliation required")) + } + const active = row.state === "Pending" || row.state === "Claimed" + const terminal = row.state === "Acknowledged" || + row.state === "DeadLettered" || + row.state === "Expired" + if ( + (active && row.activeConsumed !== 0) || + (terminal && row.activeConsumed !== 1) + ) { + return yield* storageCorrupt(new Error("Corrupt relay reservation entitlement")) + } + const corrupt = (!active && !terminal) || + row.messageTenantId !== row.channelTenantId || + row.messageSenderSubjectId !== row.channelSenderSubjectId || + row.messageSenderPeerId !== row.channelSenderPeerId || + row.messageRecipientSubjectId !== row.channelRecipientSubjectId || + row.messageRecipientPeerId !== row.channelRecipientPeerId || + (active && ( + row.actualLength === null || + row.actualLength !== row.payloadLength + )) || + (terminal && ( + row.actualLength !== null || + row.payloadLength !== 0 + )) + if (corrupt && active) { + yield* terminalize(row.messageId, "DeadLettered", now, { reason: "Corrupt" }) + } else if (corrupt) { + return yield* storageCorrupt(new Error("Corrupt terminal relay row")) + } + } + return maintenanceResult(rows.map((row) => row.messageId), effectiveBatch) + })) + })) + return PeerRpcObservability.observeRelay({ + effect, + operation: "RelayMaintenance", + direction: "Receive", + stage: "Repair", + facts: (exit) => ({ + items: Exit.isSuccess(exit) ? exit.value.processed : 0 + }), + result: (exit) => Exit.isSuccess(exit) ? "DeadLettered" : relayFailureResult(exit) + }) + } + + const reconcile: Service["reconcile"] = (unsafeInput) => { + const effect = mapStoreErrors(Effect.gen(function*() { + const input = yield* validateInput(MaintenanceRequest, unsafeInput) + const effectiveBatch = Math.min(input.batchSize, limits.reconciliationBatchSize) + return yield* write(Effect.gen(function*() { + const messageRows = yield* SqlSchema.findAll({ + Request: Schema.Void, + Result: KeyRow, + execute: () => + sql`SELECT message_id AS "messageId" + FROM effect_local_relay_messages + WHERE message_id > ${input.cursor ?? 0} + ORDER BY message_id + LIMIT ${sql.literal(String(effectiveBatch + 1))}` + })(undefined) + const reservationRows = yield* SqlSchema.findAll({ + Request: Schema.Void, + Result: KeyRow, + execute: () => + sql`SELECT message_id AS "messageId" + FROM effect_local_relay_reservations + WHERE message_id > ${input.cursor ?? 0} + ORDER BY message_id + LIMIT ${sql.literal(String(effectiveBatch + 1))}` + })(undefined) + if ( + messageRows.length !== reservationRows.length || + messageRows.some((row, index) => row.messageId !== reservationRows[index]?.messageId) + ) { + return yield* storageCorrupt(new Error("Relay reservation bijection is corrupt")) + } + return maintenanceResult(messageRows.map((row) => row.messageId), effectiveBatch) + })) + })) + return PeerRpcObservability.observeRelay({ + effect, + operation: "RelayMaintenance", + direction: "Receive", + stage: "Reconcile", + facts: (exit) => ({ + items: Exit.isSuccess(exit) ? exit.value.processed : 0 + }), + result: (exit) => Exit.isSuccess(exit) ? "Success" : relayFailureResult(exit) + }) + } + + const collect: Service["collect"] = (unsafeInput) => { + const effect = mapStoreErrors(Effect.gen(function*() { + const input = yield* validateInput(MaintenanceRequest, unsafeInput) + const effectiveBatch = Math.min(input.batchSize, limits.terminalCollectionBatchSize) + return yield* write(Effect.gen(function*() { + const now = (yield* nowQuery(sql)).now + const rows = yield* SqlSchema.findAll({ + Request: Schema.Void, + Result: Schema.Struct({ + messageId: PositiveInt, + channelId: PositiveInt + }), + execute: () => + sql`SELECT + message_id AS "messageId", + channel_id AS "channelId" + FROM effect_local_relay_messages + WHERE state IN ('Acknowledged', 'DeadLettered', 'Expired') + AND deduplicate_until <= ${now} + ORDER BY deduplicate_until, message_id + LIMIT ${sql.literal(String(effectiveBatch + 1))}` + })(undefined) + for (const row of rows.slice(0, effectiveBatch)) { + yield* releaseRetainedUsage(row.messageId) + const reservationDeleted = yield* mutationCount( + sql`DELETE FROM effect_local_relay_reservations + WHERE message_id = ${row.messageId} + AND active_consumed = 1 + AND retained_consumed = 1`.raw, + SqlSchema.findAll({ + Request: Schema.Void, + Result: UnitRow, + execute: () => + sql`DELETE FROM effect_local_relay_reservations + WHERE message_id = ${row.messageId} + AND active_consumed = 1 + AND retained_consumed = 1 + RETURNING 1 AS value` + })(undefined) + ) + if (reservationDeleted !== 1) { + return yield* storageCorrupt(new Error("Invalid collected relay quota reservation")) + } + yield* sql`DELETE FROM effect_local_relay_messages + WHERE message_id = ${row.messageId} + AND state IN ('Acknowledged', 'DeadLettered', 'Expired') + AND deduplicate_until <= ${now}` + yield* sql`DELETE FROM effect_local_relay_channels + WHERE channel_id = ${row.channelId} + AND claimed_message_id IS NULL + AND NOT EXISTS ( + SELECT 1 + FROM effect_local_relay_messages m + WHERE m.channel_id = ${row.channelId} + )` + } + return maintenanceResult(rows.map((row) => row.messageId), effectiveBatch) + })) + })) + return PeerRpcObservability.observeRelay({ + effect, + operation: "RelayMaintenance", + direction: "Receive", + stage: "Collect", + facts: (exit) => ({ + items: Exit.isSuccess(exit) ? exit.value.processed : 0 + }), + result: (exit) => Exit.isSuccess(exit) ? "Success" : relayFailureResult(exit) + }) + } + + const usage: Service["usage"] = (unsafeInput) => { + const exactShard = unsafeInput === undefined + const effect = mapStoreErrors(Effect.gen(function*() { + const input = unsafeInput === undefined + ? UsageRequest.make({ scopeKind: "Shard", scopeKey: encodeKey("local") }) + : yield* validateInput(UsageRequest, unsafeInput) + const row = yield* SqlSchema.findOneOption({ + Request: Schema.Void, + Result: UsageRow, + execute: () => + sql`SELECT + active_count AS "activeCount", + active_bytes AS "activeBytes", + retained_count AS "retainedCount", + retained_bytes AS "retainedBytes" + FROM effect_local_relay_usage + WHERE scope_kind = ${input.scopeKind} + AND scope_key = ${input.scopeKey}` + })(undefined) + return Option.getOrElse(row, (): Usage => ({ + activeCount: 0, + activeBytes: 0, + retainedCount: 0, + retainedBytes: 0 + })) + })) + return PeerRpcObservability.observeRelay({ + effect, + operation: "RelayMaintenance", + direction: "Receive", + stage: "Usage", + facts: (exit) => ({ + items: Exit.isSuccess(exit) ? exit.value.activeCount : 0, + ...(Exit.isSuccess(exit) ? { bytes: exit.value.activeBytes, version: 1 } : {}) + }), + result: (exit) => Exit.isSuccess(exit) ? "Success" : relayFailureResult(exit) + }).pipe( + Effect.tap((value) => + exactShard + ? PeerRpcObservability.setRelayPending( + value.activeCount, + value.activeBytes + ).pipe(Effect.catchCause(() => Effect.void)) + : Effect.void + ) + ) + } + + return PeerRelayStore.of({ + admit, + claim, + loadClaimedPayload, + acknowledge, + reject, + release, + recover, + expire, + repair, + reconcile, + collect, + usage + }) +}) + +export const make = makeService + +export const layer: Layer.Layer< + PeerRelayStore, + | Migrator.MigrationError + | SqlError.SqlError + | Schema.SchemaError + | ReplicaError.ReplicaError, + SqlClient.SqlClient | Crypto.Crypto | PeerRelayLimits.PeerRelayLimits +> = Layer.effect(PeerRelayStore, makeService) diff --git a/packages/local-rpc/src/index.ts b/packages/local-rpc/src/index.ts index 2515a66..92f3337 100644 --- a/packages/local-rpc/src/index.ts +++ b/packages/local-rpc/src/index.ts @@ -1,14 +1,12 @@ export * as PeerAuthentication from "./PeerAuthentication.js" export * as PeerAuthenticator from "./PeerAuthenticator.js" -export * as PeerAuthorization from "./PeerAuthorization.js" export * as PeerCredentials from "./PeerCredentials.js" export * as PeerRelayAuthorization from "./PeerRelayAuthorization.js" export * as PeerRelayIngress from "./PeerRelayIngress.js" export * as PeerRelayLimits from "./PeerRelayLimits.js" -export * as PeerRelayRpc from "./PeerRelayRpc.js" export * as PeerRelayStore from "./PeerRelayStore.js" export * as PeerRpc from "./PeerRpc.js" export * as PeerRpcError from "./PeerRpcError.js" -export * as PeerRpcLimits from "./PeerRpcLimits.js" export * as PeerRpcServer from "./PeerRpcServer.js" export * as RpcPeerTransport from "./RpcPeerTransport.js" +export * as SqlPeerRelayStore from "./SqlPeerRelayStore.js" diff --git a/packages/local-rpc/src/internal/peerRelayMigrations.ts b/packages/local-rpc/src/internal/peerRelayMigrations.ts index 5a68521..6aba9f0 100644 --- a/packages/local-rpc/src/internal/peerRelayMigrations.ts +++ b/packages/local-rpc/src/internal/peerRelayMigrations.ts @@ -4,278 +4,874 @@ import * as Schema from "effect/Schema" import * as Migrator from "effect/unstable/sql/Migrator" import * as SqlClient from "effect/unstable/sql/SqlClient" import type * as SqlError from "effect/unstable/sql/SqlError" -import * as SqlSchema from "effect/unstable/sql/SqlSchema" - -export const relayCustodyChecksum = "sha256:effect-local-relay-custody-v1" -export const relayMaintenanceIndexesChecksum = "sha256:effect-local-relay-maintenance-indexes-v1" -export const relayClaimAdmissionIndexChecksum = "sha256:effect-local-relay-claim-admission-index-v1" -export const relayClaimRecipientRouteChecksum = "sha256:effect-local-relay-claim-recipient-route-v1" - -const NonNegativeInt = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)) - -const custody = Effect.gen(function*() { - const sql = yield* SqlClient.SqlClient - yield* sql`CREATE TABLE effect_local_relay_migration_catalog ( - migration_id INTEGER PRIMARY KEY, - name TEXT NOT NULL, - checksum TEXT NOT NULL - )` - yield* sql`CREATE TABLE effect_local_relay_channels ( - channel_id INTEGER PRIMARY KEY AUTOINCREMENT, - tenant_id TEXT NOT NULL, - sender_subject_id TEXT NOT NULL, - sender_peer_id TEXT NOT NULL, - sender_replica_incarnation INTEGER NOT NULL, - recipient_subject_id TEXT NOT NULL, - recipient_peer_id TEXT NOT NULL, - next_sequence INTEGER NOT NULL DEFAULT 0 CHECK (next_sequence >= 0), - claimed_message_id INTEGER, - claim_session_generation INTEGER, - claim_token TEXT, - claim_deadline INTEGER, - UNIQUE ( - tenant_id, - sender_subject_id, - sender_peer_id, - sender_replica_incarnation, - recipient_subject_id, - recipient_peer_id - ) - )` - yield* sql`CREATE TABLE effect_local_relay_messages ( - message_id INTEGER PRIMARY KEY AUTOINCREMENT, - channel_id INTEGER NOT NULL REFERENCES effect_local_relay_channels(channel_id) ON DELETE CASCADE, - channel_sequence INTEGER NOT NULL CHECK (channel_sequence >= 0), - tenant_id TEXT NOT NULL, - sender_subject_id TEXT NOT NULL, - sender_peer_id TEXT NOT NULL, - relay_message_id TEXT NOT NULL, - relay_peer_id TEXT NOT NULL, - sender_connection_epoch TEXT NOT NULL, - sender_sequence INTEGER NOT NULL CHECK (sender_sequence >= 0), - document_ids TEXT NOT NULL, - payload_version INTEGER NOT NULL, - message_hash TEXT NOT NULL, - outer_envelope_digest TEXT NOT NULL, - payload BLOB, - payload_length INTEGER NOT NULL CHECK (payload_length >= 0), - state TEXT NOT NULL CHECK ( - state IN ('Pending', 'Claimed', 'Acknowledged', 'DeadLettered', 'Expired') - ), - created_at INTEGER NOT NULL, - expires_at INTEGER NOT NULL, - deduplicate_until INTEGER NOT NULL, - next_eligible_at INTEGER NOT NULL, - retry_count INTEGER NOT NULL DEFAULT 0 CHECK (retry_count >= 0), - claim_token TEXT, - claim_session_generation INTEGER, - claim_deadline INTEGER, - terminal_at INTEGER, - terminal_claim_token TEXT, - terminal_session_generation INTEGER, - terminal_reason TEXT, - UNIQUE(channel_id, channel_sequence), - UNIQUE(tenant_id, sender_subject_id, sender_peer_id, relay_message_id) - )` - yield* sql`CREATE TABLE effect_local_relay_usage ( - scope_kind TEXT NOT NULL CHECK ( - scope_kind IN ('SenderPeer', 'RecipientPeer', 'RecipientSubject', 'Tenant', 'Shard') - ), - scope_key TEXT NOT NULL, - active_count INTEGER NOT NULL CHECK (active_count >= 0), - active_bytes INTEGER NOT NULL CHECK (active_bytes >= 0), - retained_count INTEGER NOT NULL CHECK (retained_count >= 0), - retained_bytes INTEGER NOT NULL CHECK (retained_bytes >= 0), - PRIMARY KEY(scope_kind, scope_key) - )` - yield* sql`CREATE TABLE effect_local_relay_reservations ( - message_id INTEGER PRIMARY KEY REFERENCES effect_local_relay_messages(message_id) ON DELETE CASCADE, - sender_peer_usage_key TEXT NOT NULL, - recipient_peer_usage_key TEXT NOT NULL, - recipient_subject_usage_key TEXT NOT NULL, - tenant_usage_key TEXT NOT NULL, - shard_usage_key TEXT NOT NULL, - active_count_delta INTEGER NOT NULL CHECK (active_count_delta = 1), - active_bytes_delta INTEGER NOT NULL CHECK (active_bytes_delta >= 0), - retained_count_delta INTEGER NOT NULL CHECK (retained_count_delta = 1), - retained_bytes_delta INTEGER NOT NULL CHECK (retained_bytes_delta >= 0), - active_consumed INTEGER NOT NULL DEFAULT 0 CHECK (active_consumed IN (0, 1)), - retained_consumed INTEGER NOT NULL DEFAULT 0 CHECK (retained_consumed IN (0, 1)) - )` - yield* sql`INSERT INTO effect_local_relay_migration_catalog (migration_id, name, checksum) - VALUES (1, 'relay_custody', ${relayCustodyChecksum})` -}) - -const maintenanceIndexes = Effect.gen(function*() { - const sql = yield* SqlClient.SqlClient - yield* sql`CREATE INDEX effect_local_relay_messages_channel_head - ON effect_local_relay_messages(channel_id, channel_sequence, state, next_eligible_at)` - yield* sql`CREATE INDEX effect_local_relay_channels_discovery - ON effect_local_relay_channels( - tenant_id, - recipient_subject_id, - recipient_peer_id, - sender_subject_id, - sender_peer_id, - channel_id - )` - yield* sql`CREATE INDEX effect_local_relay_messages_admission_order - ON effect_local_relay_messages(created_at, message_id, channel_id, channel_sequence)` - yield* sql`CREATE INDEX effect_local_relay_messages_recovery - ON effect_local_relay_messages(state, claim_deadline, message_id)` - yield* sql`CREATE INDEX effect_local_relay_messages_expiry - ON effect_local_relay_messages(expires_at, message_id) - WHERE state IN ('Pending', 'Claimed')` - yield* sql`CREATE INDEX effect_local_relay_messages_collection - ON effect_local_relay_messages(deduplicate_until, message_id) - WHERE state IN ('Acknowledged', 'DeadLettered', 'Expired')` - yield* sql`CREATE INDEX effect_local_relay_channels_claim - ON effect_local_relay_channels(claimed_message_id, channel_id)` - yield* sql`INSERT INTO effect_local_relay_migration_catalog (migration_id, name, checksum) - VALUES (2, 'relay_maintenance_indexes', ${relayMaintenanceIndexesChecksum})` -}) -const claimAdmissionIndex = Effect.gen(function*() { - const sql = yield* SqlClient.SqlClient - yield* sql`CREATE INDEX effect_local_relay_messages_claim_admission - ON effect_local_relay_messages( - tenant_id, - sender_subject_id, - sender_peer_id, - created_at, - message_id - ) - WHERE state = 'Pending' AND payload IS NOT NULL` - yield* sql`INSERT INTO effect_local_relay_migration_catalog (migration_id, name, checksum) - VALUES (3, 'relay_claim_admission_index', ${relayClaimAdmissionIndexChecksum})` -}) +const executeStatements = ( + sql: SqlClient.SqlClient, + source: string +) => + Effect.forEach( + source.split(";").map((statement) => statement.trim()).filter((statement) => statement.length > 0), + (statement) => sql.unsafe(statement), + { discard: true } + ) -const claimRecipientRoute = Effect.gen(function*() { - const sql = yield* SqlClient.SqlClient - yield* sql`DROP INDEX effect_local_relay_messages_claim_admission` - yield* sql`ALTER TABLE effect_local_relay_messages - ADD COLUMN recipient_subject_id TEXT` - yield* sql`ALTER TABLE effect_local_relay_messages - ADD COLUMN recipient_peer_id TEXT` - yield* sql`UPDATE effect_local_relay_messages - SET recipient_subject_id = ( - SELECT c.recipient_subject_id - FROM effect_local_relay_channels c - WHERE c.channel_id = effect_local_relay_messages.channel_id - ), - recipient_peer_id = ( - SELECT c.recipient_peer_id - FROM effect_local_relay_channels c - WHERE c.channel_id = effect_local_relay_messages.channel_id - )` - const invalid = yield* SqlSchema.findOne({ - Request: Schema.Void, - Result: Schema.Struct({ count: NonNegativeInt }), - execute: () => - sql`SELECT COUNT(*) AS count - FROM effect_local_relay_messages - WHERE recipient_subject_id IS NULL OR recipient_peer_id IS NULL` - })(undefined) - if (invalid.count !== 0) { - return yield* new Migrator.MigrationError({ - kind: "BadState", - message: "Relay recipient route backfill is incomplete" - }) +const mysqlIndexes = [ + { + table: "effect_local_relay_messages", + name: "effect_local_relay_messages_channel_head", + columns: ["channel_id", "channel_sequence", "state", "next_eligible_at"], + statement: `CREATE INDEX effect_local_relay_messages_channel_head + ON effect_local_relay_messages(channel_id, channel_sequence, state, next_eligible_at)` + }, + { + table: "effect_local_relay_channels", + name: "effect_local_relay_channels_discovery", + columns: ["recipient_peer_id", "sender_peer_id", "channel_id"], + statement: `CREATE INDEX effect_local_relay_channels_discovery + ON effect_local_relay_channels(recipient_peer_id, sender_peer_id, channel_id)` + }, + { + table: "effect_local_relay_messages", + name: "effect_local_relay_messages_admission_order", + columns: ["created_at", "message_id", "channel_id", "channel_sequence"], + statement: `CREATE INDEX effect_local_relay_messages_admission_order + ON effect_local_relay_messages(created_at, message_id, channel_id, channel_sequence)` + }, + { + table: "effect_local_relay_messages", + name: "effect_local_relay_messages_recovery", + columns: ["state", "claim_deadline", "message_id"], + statement: `CREATE INDEX effect_local_relay_messages_recovery + ON effect_local_relay_messages(state, claim_deadline, message_id)` + }, + { + table: "effect_local_relay_messages", + name: "effect_local_relay_messages_expiry", + columns: ["state", "expires_at", "message_id"], + statement: `CREATE INDEX effect_local_relay_messages_expiry + ON effect_local_relay_messages(state, expires_at, message_id)` + }, + { + table: "effect_local_relay_messages", + name: "effect_local_relay_messages_collection", + columns: ["state", "deduplicate_until", "message_id"], + statement: `CREATE INDEX effect_local_relay_messages_collection + ON effect_local_relay_messages(state, deduplicate_until, message_id)` + }, + { + table: "effect_local_relay_channels", + name: "effect_local_relay_channels_claim", + columns: ["claimed_message_id", "channel_id"], + statement: `CREATE INDEX effect_local_relay_channels_claim + ON effect_local_relay_channels(claimed_message_id, channel_id)` + }, + { + table: "effect_local_relay_messages", + name: "effect_local_relay_messages_claim_admission", + columns: ["state", "tenant_id", "sender_peer_id", "recipient_peer_id", "created_at", "message_id"], + statement: `CREATE INDEX effect_local_relay_messages_claim_admission + ON effect_local_relay_messages( + state, + tenant_id, + sender_peer_id, + recipient_peer_id, + created_at, + message_id + )` } - yield* sql`CREATE INDEX effect_local_relay_messages_claim_admission - ON effect_local_relay_messages( - tenant_id, - sender_subject_id, - sender_peer_id, - recipient_subject_id, - recipient_peer_id, - created_at, - message_id +] as const + +const migrationFailure = (message: string, cause?: unknown) => + new Migrator.MigrationError({ + kind: "Failed", + message, + ...(cause === undefined ? {} : { cause }) + }) + +const DatabaseInt = Schema.Union([Schema.Int, Schema.NumberFromString]).check(Schema.isInt()) + +const decodeMysqlRows = ( + schema: S, + rows: unknown, + message: string +) => + Schema.decodeUnknownEffect(schema)(rows).pipe( + Effect.mapError((cause) => migrationFailure(message, cause)) + ) + +const createMysqlIndexes = (sql: SqlClient.SqlClient) => + Effect.gen(function*() { + const rows = yield* sql.unsafe(` + SELECT table_name AS tableName, index_name AS indexName + FROM information_schema.statistics + WHERE table_schema = DATABASE() + `) + const existing = yield* decodeMysqlRows( + Schema.Array(Schema.Struct({ + tableName: Schema.String, + indexName: Schema.String + })), + rows, + "Could not inspect MySQL relay indexes" ) - WHERE state = 'Pending' AND payload IS NOT NULL` - yield* sql`INSERT INTO effect_local_relay_migration_catalog (migration_id, name, checksum) - VALUES (4, 'relay_claim_recipient_route', ${relayClaimRecipientRouteChecksum})` -}) + const names = new Set(existing.map((row) => `${row.tableName}.${row.indexName}`)) + for (const index of mysqlIndexes) { + if (!names.has(`${index.table}.${index.name}`)) { + yield* sql.unsafe(index.statement) + } + } + }) + +const createRelayTables = Effect.gen(function*() { + const sql = (yield* SqlClient.SqlClient).withoutTransforms() + + yield* sql.onDialectOrElse({ + pg: () => + executeStatements( + sql, + ` + CREATE TABLE effect_local_relay_write_lock ( + lock_id INTEGER PRIMARY KEY CHECK (lock_id = 1) + ); + INSERT INTO effect_local_relay_write_lock (lock_id) VALUES (1); + + CREATE TABLE effect_local_relay_channels ( + channel_id BIGSERIAL PRIMARY KEY, + tenant_id VARCHAR(256) NOT NULL, + sender_subject_id VARCHAR(256) NOT NULL, + sender_peer_id VARCHAR(64) NOT NULL, + sender_replica_incarnation BIGINT NOT NULL, + recipient_subject_id VARCHAR(256) NOT NULL, + recipient_peer_id VARCHAR(64) NOT NULL, + next_sequence BIGINT NOT NULL DEFAULT 0 CHECK (next_sequence >= 0), + claimed_message_id BIGINT, + claim_session_generation BIGINT, + claim_token VARCHAR(256), + claim_deadline BIGINT, + UNIQUE ( + tenant_id, + sender_subject_id, + sender_peer_id, + sender_replica_incarnation, + recipient_subject_id, + recipient_peer_id + ) + ); + + CREATE TABLE effect_local_relay_messages ( + message_id BIGSERIAL PRIMARY KEY, + channel_id BIGINT NOT NULL REFERENCES effect_local_relay_channels(channel_id) ON DELETE CASCADE, + channel_sequence BIGINT NOT NULL CHECK (channel_sequence >= 0), + tenant_id VARCHAR(256) NOT NULL, + sender_subject_id VARCHAR(256) NOT NULL, + sender_peer_id VARCHAR(64) NOT NULL, + recipient_subject_id VARCHAR(256) NOT NULL, + recipient_peer_id VARCHAR(64) NOT NULL, + relay_message_id VARCHAR(64) NOT NULL, + relay_peer_id VARCHAR(64) NOT NULL, + sender_connection_epoch VARCHAR(256) NOT NULL, + sender_sequence BIGINT NOT NULL CHECK (sender_sequence >= 0), + document_ids TEXT NOT NULL, + payload_version INTEGER NOT NULL, + message_hash VARCHAR(256) NOT NULL, + outer_envelope_digest VARCHAR(256) NOT NULL, + payload BYTEA, + payload_length BIGINT NOT NULL CHECK (payload_length >= 0), + state VARCHAR(32) NOT NULL CHECK ( + state IN ('Pending', 'Claimed', 'Acknowledged', 'DeadLettered', 'Expired') + ), + created_at BIGINT NOT NULL, + expires_at BIGINT NOT NULL, + deduplicate_until BIGINT NOT NULL, + next_eligible_at BIGINT NOT NULL, + retry_count INTEGER NOT NULL DEFAULT 0 CHECK (retry_count >= 0), + claim_token VARCHAR(256), + claim_session_generation BIGINT, + claim_deadline BIGINT, + terminal_at BIGINT, + terminal_claim_token VARCHAR(256), + terminal_session_generation BIGINT, + terminal_reason VARCHAR(256), + UNIQUE(channel_id, channel_sequence), + UNIQUE(tenant_id, sender_subject_id, sender_peer_id, relay_message_id) + ); + + CREATE TABLE effect_local_relay_usage ( + scope_kind VARCHAR(32) NOT NULL, + scope_key VARCHAR(1024) NOT NULL, + active_count BIGINT NOT NULL CHECK (active_count >= 0), + active_bytes BIGINT NOT NULL CHECK (active_bytes >= 0), + retained_count BIGINT NOT NULL CHECK (retained_count >= 0), + retained_bytes BIGINT NOT NULL CHECK (retained_bytes >= 0), + PRIMARY KEY(scope_kind, scope_key) + ); + + CREATE TABLE effect_local_relay_reservations ( + message_id BIGINT PRIMARY KEY REFERENCES effect_local_relay_messages(message_id) ON DELETE CASCADE, + sender_peer_usage_key VARCHAR(1024) NOT NULL, + recipient_peer_usage_key VARCHAR(1024) NOT NULL, + recipient_subject_usage_key VARCHAR(1024) NOT NULL, + tenant_usage_key VARCHAR(1024) NOT NULL, + shard_usage_key VARCHAR(1024) NOT NULL, + active_count_delta INTEGER NOT NULL CHECK (active_count_delta = 1), + active_bytes_delta BIGINT NOT NULL CHECK (active_bytes_delta >= 0), + retained_count_delta INTEGER NOT NULL CHECK (retained_count_delta = 1), + retained_bytes_delta BIGINT NOT NULL CHECK (retained_bytes_delta >= 0), + active_consumed INTEGER NOT NULL DEFAULT 0 CHECK (active_consumed IN (0, 1)), + retained_consumed INTEGER NOT NULL DEFAULT 0 CHECK (retained_consumed IN (0, 1)) + ) + ` + ), + mysql: () => + executeStatements( + sql, + ` + CREATE TABLE IF NOT EXISTS effect_local_relay_write_lock ( + lock_id INT PRIMARY KEY, + CHECK (lock_id = 1) + ); + INSERT IGNORE INTO effect_local_relay_write_lock (lock_id) VALUES (1); + + CREATE TABLE IF NOT EXISTS effect_local_relay_channels ( + channel_id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, + tenant_id VARCHAR(256) COLLATE utf8mb4_bin NOT NULL, + sender_subject_id VARCHAR(256) COLLATE utf8mb4_bin NOT NULL, + sender_peer_id VARCHAR(64) COLLATE utf8mb4_bin NOT NULL, + sender_replica_incarnation BIGINT NOT NULL, + recipient_subject_id VARCHAR(256) COLLATE utf8mb4_bin NOT NULL, + recipient_peer_id VARCHAR(64) COLLATE utf8mb4_bin NOT NULL, + channel_identity CHAR(64) COLLATE utf8mb4_bin GENERATED ALWAYS AS ( + SHA2(JSON_ARRAY( + tenant_id, + sender_subject_id, + sender_peer_id, + sender_replica_incarnation, + recipient_subject_id, + recipient_peer_id + ), 256) + ) STORED, + next_sequence BIGINT NOT NULL DEFAULT 0, + claimed_message_id BIGINT, + claim_session_generation BIGINT, + claim_token VARCHAR(256) COLLATE utf8mb4_bin, + claim_deadline BIGINT, + UNIQUE KEY effect_local_relay_channels_identity (channel_identity), + CHECK (next_sequence >= 0) + ); + + CREATE TABLE IF NOT EXISTS effect_local_relay_messages ( + message_id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, + channel_id BIGINT NOT NULL, + channel_sequence BIGINT NOT NULL, + tenant_id VARCHAR(256) COLLATE utf8mb4_bin NOT NULL, + sender_subject_id VARCHAR(256) COLLATE utf8mb4_bin NOT NULL, + sender_peer_id VARCHAR(64) COLLATE utf8mb4_bin NOT NULL, + recipient_subject_id VARCHAR(256) COLLATE utf8mb4_bin NOT NULL, + recipient_peer_id VARCHAR(64) COLLATE utf8mb4_bin NOT NULL, + relay_message_id VARCHAR(64) COLLATE utf8mb4_bin NOT NULL, + relay_peer_id VARCHAR(64) COLLATE utf8mb4_bin NOT NULL, + sender_connection_epoch VARCHAR(256) COLLATE utf8mb4_bin NOT NULL, + sender_sequence BIGINT NOT NULL, + sender_message_identity CHAR(64) COLLATE utf8mb4_bin GENERATED ALWAYS AS ( + SHA2(JSON_ARRAY( + tenant_id, + sender_subject_id, + sender_peer_id, + relay_message_id + ), 256) + ) STORED, + document_ids LONGTEXT NOT NULL, + payload_version INT NOT NULL, + message_hash VARCHAR(256) COLLATE utf8mb4_bin NOT NULL, + outer_envelope_digest VARCHAR(256) COLLATE utf8mb4_bin NOT NULL, + payload LONGBLOB, + payload_length BIGINT NOT NULL, + state VARCHAR(32) COLLATE utf8mb4_bin NOT NULL, + created_at BIGINT NOT NULL, + expires_at BIGINT NOT NULL, + deduplicate_until BIGINT NOT NULL, + next_eligible_at BIGINT NOT NULL, + retry_count INT NOT NULL DEFAULT 0, + claim_token VARCHAR(256) COLLATE utf8mb4_bin, + claim_session_generation BIGINT, + claim_deadline BIGINT, + terminal_at BIGINT, + terminal_claim_token VARCHAR(256) COLLATE utf8mb4_bin, + terminal_session_generation BIGINT, + terminal_reason VARCHAR(256) COLLATE utf8mb4_bin, + UNIQUE KEY effect_local_relay_messages_channel_sequence(channel_id, channel_sequence), + UNIQUE KEY effect_local_relay_messages_sender_identity(sender_message_identity), + CONSTRAINT effect_local_relay_messages_channel_fk + FOREIGN KEY(channel_id) REFERENCES effect_local_relay_channels(channel_id) ON DELETE CASCADE, + CHECK (channel_sequence >= 0), + CHECK (sender_sequence >= 0), + CHECK (payload_length >= 0), + CHECK (retry_count >= 0), + CHECK (state IN ('Pending', 'Claimed', 'Acknowledged', 'DeadLettered', 'Expired')) + ); + + CREATE TABLE IF NOT EXISTS effect_local_relay_usage ( + scope_kind VARCHAR(32) COLLATE utf8mb4_bin NOT NULL, + scope_key VARCHAR(700) COLLATE utf8mb4_bin NOT NULL, + active_count BIGINT NOT NULL, + active_bytes BIGINT NOT NULL, + retained_count BIGINT NOT NULL, + retained_bytes BIGINT NOT NULL, + PRIMARY KEY(scope_kind, scope_key), + CHECK (active_count >= 0), + CHECK (active_bytes >= 0), + CHECK (retained_count >= 0), + CHECK (retained_bytes >= 0) + ); + + CREATE TABLE IF NOT EXISTS effect_local_relay_reservations ( + message_id BIGINT PRIMARY KEY, + sender_peer_usage_key TEXT NOT NULL, + recipient_peer_usage_key TEXT NOT NULL, + recipient_subject_usage_key TEXT NOT NULL, + tenant_usage_key TEXT NOT NULL, + shard_usage_key TEXT NOT NULL, + active_count_delta INT NOT NULL, + active_bytes_delta BIGINT NOT NULL, + retained_count_delta INT NOT NULL, + retained_bytes_delta BIGINT NOT NULL, + active_consumed INT NOT NULL DEFAULT 0, + retained_consumed INT NOT NULL DEFAULT 0, + CONSTRAINT effect_local_relay_reservations_message_fk + FOREIGN KEY(message_id) REFERENCES effect_local_relay_messages(message_id) ON DELETE CASCADE, + CHECK (active_count_delta = 1), + CHECK (active_bytes_delta >= 0), + CHECK (retained_count_delta = 1), + CHECK (retained_bytes_delta >= 0), + CHECK (active_consumed IN (0, 1)), + CHECK (retained_consumed IN (0, 1)) + ) + ` + ), + orElse: () => + executeStatements( + sql, + ` + CREATE TABLE effect_local_relay_write_lock ( + lock_id INTEGER PRIMARY KEY CHECK (lock_id = 1) + ); + INSERT INTO effect_local_relay_write_lock (lock_id) VALUES (1); + + CREATE TABLE effect_local_relay_channels ( + channel_id INTEGER PRIMARY KEY AUTOINCREMENT, + tenant_id TEXT NOT NULL, + sender_subject_id TEXT NOT NULL, + sender_peer_id TEXT NOT NULL, + sender_replica_incarnation INTEGER NOT NULL, + recipient_subject_id TEXT NOT NULL, + recipient_peer_id TEXT NOT NULL, + next_sequence INTEGER NOT NULL DEFAULT 0 CHECK (next_sequence >= 0), + claimed_message_id INTEGER, + claim_session_generation INTEGER, + claim_token TEXT, + claim_deadline INTEGER, + UNIQUE ( + tenant_id, + sender_subject_id, + sender_peer_id, + sender_replica_incarnation, + recipient_subject_id, + recipient_peer_id + ) + ); + + CREATE TABLE effect_local_relay_messages ( + message_id INTEGER PRIMARY KEY AUTOINCREMENT, + channel_id INTEGER NOT NULL REFERENCES effect_local_relay_channels(channel_id) ON DELETE CASCADE, + channel_sequence INTEGER NOT NULL CHECK (channel_sequence >= 0), + tenant_id TEXT NOT NULL, + sender_subject_id TEXT NOT NULL, + sender_peer_id TEXT NOT NULL, + recipient_subject_id TEXT NOT NULL, + recipient_peer_id TEXT NOT NULL, + relay_message_id TEXT NOT NULL, + relay_peer_id TEXT NOT NULL, + sender_connection_epoch TEXT NOT NULL, + sender_sequence INTEGER NOT NULL CHECK (sender_sequence >= 0), + document_ids TEXT NOT NULL, + payload_version INTEGER NOT NULL, + message_hash TEXT NOT NULL, + outer_envelope_digest TEXT NOT NULL, + payload BLOB, + payload_length INTEGER NOT NULL CHECK (payload_length >= 0), + state TEXT NOT NULL CHECK ( + state IN ('Pending', 'Claimed', 'Acknowledged', 'DeadLettered', 'Expired') + ), + created_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + deduplicate_until INTEGER NOT NULL, + next_eligible_at INTEGER NOT NULL, + retry_count INTEGER NOT NULL DEFAULT 0 CHECK (retry_count >= 0), + claim_token TEXT, + claim_session_generation INTEGER, + claim_deadline INTEGER, + terminal_at INTEGER, + terminal_claim_token TEXT, + terminal_session_generation INTEGER, + terminal_reason TEXT, + UNIQUE(channel_id, channel_sequence), + UNIQUE(tenant_id, sender_subject_id, sender_peer_id, relay_message_id) + ); -export const loader = Migrator.fromRecord({ - "1_relay_custody": custody, - "2_relay_maintenance_indexes": maintenanceIndexes, - "3_relay_claim_admission_index": claimAdmissionIndex, - "4_relay_claim_recipient_route": claimRecipientRoute + CREATE TABLE effect_local_relay_usage ( + scope_kind TEXT NOT NULL, + scope_key TEXT NOT NULL, + active_count INTEGER NOT NULL CHECK (active_count >= 0), + active_bytes INTEGER NOT NULL CHECK (active_bytes >= 0), + retained_count INTEGER NOT NULL CHECK (retained_count >= 0), + retained_bytes INTEGER NOT NULL CHECK (retained_bytes >= 0), + PRIMARY KEY(scope_kind, scope_key) + ); + + CREATE TABLE effect_local_relay_reservations ( + message_id INTEGER PRIMARY KEY REFERENCES effect_local_relay_messages(message_id) ON DELETE CASCADE, + sender_peer_usage_key TEXT NOT NULL, + recipient_peer_usage_key TEXT NOT NULL, + recipient_subject_usage_key TEXT NOT NULL, + tenant_usage_key TEXT NOT NULL, + shard_usage_key TEXT NOT NULL, + active_count_delta INTEGER NOT NULL CHECK (active_count_delta = 1), + active_bytes_delta INTEGER NOT NULL CHECK (active_bytes_delta >= 0), + retained_count_delta INTEGER NOT NULL CHECK (retained_count_delta = 1), + retained_bytes_delta INTEGER NOT NULL CHECK (retained_bytes_delta >= 0), + active_consumed INTEGER NOT NULL DEFAULT 0 CHECK (active_consumed IN (0, 1)), + retained_consumed INTEGER NOT NULL DEFAULT 0 CHECK (retained_consumed IN (0, 1)) + ) + ` + ) + }) + + yield* sql.onDialectOrElse({ + mysql: () => createMysqlIndexes(sql), + orElse: () => + executeStatements( + sql, + ` + CREATE INDEX effect_local_relay_messages_channel_head + ON effect_local_relay_messages(channel_id, channel_sequence, state, next_eligible_at); + CREATE INDEX effect_local_relay_channels_discovery + ON effect_local_relay_channels( + tenant_id, + recipient_subject_id, + recipient_peer_id, + sender_subject_id, + sender_peer_id, + channel_id + ); + CREATE INDEX effect_local_relay_messages_admission_order + ON effect_local_relay_messages(created_at, message_id, channel_id, channel_sequence); + CREATE INDEX effect_local_relay_messages_recovery + ON effect_local_relay_messages(state, claim_deadline, message_id); + CREATE INDEX effect_local_relay_messages_expiry + ON effect_local_relay_messages(expires_at, message_id); + CREATE INDEX effect_local_relay_messages_collection + ON effect_local_relay_messages(deduplicate_until, message_id); + CREATE INDEX effect_local_relay_channels_claim + ON effect_local_relay_channels(claimed_message_id, channel_id); + CREATE INDEX effect_local_relay_messages_claim_admission + ON effect_local_relay_messages( + tenant_id, + sender_subject_id, + sender_peer_id, + recipient_subject_id, + recipient_peer_id, + created_at, + message_id + ) + ` + ) + }) }) -const migrate = Migrator.make({})({ - loader, - table: "effect_local_relay_migrations" +const migrations = Migrator.fromRecord({ + "0001_create_relay_tables": createRelayTables }) -const expectedCatalog = [ +const mysqlExpectedColumns = { + effect_local_relay_write_lock: [ + "lock_id:int:NO" + ], + effect_local_relay_channels: [ + "channel_id:bigint:NO", + "tenant_id:varchar:NO", + "sender_subject_id:varchar:NO", + "sender_peer_id:varchar:NO", + "sender_replica_incarnation:bigint:NO", + "recipient_subject_id:varchar:NO", + "recipient_peer_id:varchar:NO", + "channel_identity:char:YES", + "next_sequence:bigint:NO", + "claimed_message_id:bigint:YES", + "claim_session_generation:bigint:YES", + "claim_token:varchar:YES", + "claim_deadline:bigint:YES" + ], + effect_local_relay_messages: [ + "message_id:bigint:NO", + "channel_id:bigint:NO", + "channel_sequence:bigint:NO", + "tenant_id:varchar:NO", + "sender_subject_id:varchar:NO", + "sender_peer_id:varchar:NO", + "recipient_subject_id:varchar:NO", + "recipient_peer_id:varchar:NO", + "relay_message_id:varchar:NO", + "relay_peer_id:varchar:NO", + "sender_connection_epoch:varchar:NO", + "sender_sequence:bigint:NO", + "sender_message_identity:char:YES", + "document_ids:longtext:NO", + "payload_version:int:NO", + "message_hash:varchar:NO", + "outer_envelope_digest:varchar:NO", + "payload:longblob:YES", + "payload_length:bigint:NO", + "state:varchar:NO", + "created_at:bigint:NO", + "expires_at:bigint:NO", + "deduplicate_until:bigint:NO", + "next_eligible_at:bigint:NO", + "retry_count:int:NO", + "claim_token:varchar:YES", + "claim_session_generation:bigint:YES", + "claim_deadline:bigint:YES", + "terminal_at:bigint:YES", + "terminal_claim_token:varchar:YES", + "terminal_session_generation:bigint:YES", + "terminal_reason:varchar:YES" + ], + effect_local_relay_usage: [ + "scope_kind:varchar:NO", + "scope_key:varchar:NO", + "active_count:bigint:NO", + "active_bytes:bigint:NO", + "retained_count:bigint:NO", + "retained_bytes:bigint:NO" + ], + effect_local_relay_reservations: [ + "message_id:bigint:NO", + "sender_peer_usage_key:text:NO", + "recipient_peer_usage_key:text:NO", + "recipient_subject_usage_key:text:NO", + "tenant_usage_key:text:NO", + "shard_usage_key:text:NO", + "active_count_delta:int:NO", + "active_bytes_delta:bigint:NO", + "retained_count_delta:int:NO", + "retained_bytes_delta:bigint:NO", + "active_consumed:int:NO", + "retained_consumed:int:NO" + ] +} as const + +const mysqlBinaryColumns = new Set([ + "effect_local_relay_channels.tenant_id", + "effect_local_relay_channels.sender_subject_id", + "effect_local_relay_channels.sender_peer_id", + "effect_local_relay_channels.recipient_subject_id", + "effect_local_relay_channels.recipient_peer_id", + "effect_local_relay_channels.channel_identity", + "effect_local_relay_channels.claim_token", + "effect_local_relay_messages.tenant_id", + "effect_local_relay_messages.sender_subject_id", + "effect_local_relay_messages.sender_peer_id", + "effect_local_relay_messages.recipient_subject_id", + "effect_local_relay_messages.recipient_peer_id", + "effect_local_relay_messages.relay_message_id", + "effect_local_relay_messages.relay_peer_id", + "effect_local_relay_messages.sender_connection_epoch", + "effect_local_relay_messages.sender_message_identity", + "effect_local_relay_messages.message_hash", + "effect_local_relay_messages.outer_envelope_digest", + "effect_local_relay_messages.state", + "effect_local_relay_messages.claim_token", + "effect_local_relay_messages.terminal_claim_token", + "effect_local_relay_messages.terminal_reason", + "effect_local_relay_usage.scope_kind", + "effect_local_relay_usage.scope_key" +]) + +const mysqlRequiredIndexes = [ + ...mysqlIndexes, + { + table: "effect_local_relay_write_lock", + name: "PRIMARY", + columns: ["lock_id"] + }, + { + table: "effect_local_relay_channels", + name: "PRIMARY", + columns: ["channel_id"] + }, { - id: 1, - name: "relay_custody", - checksum: relayCustodyChecksum, - label: "Relay custody" + table: "effect_local_relay_channels", + name: "effect_local_relay_channels_identity", + columns: ["channel_identity"] }, { - id: 2, - name: "relay_maintenance_indexes", - checksum: relayMaintenanceIndexesChecksum, - label: "Relay maintenance indexes" + table: "effect_local_relay_messages", + name: "PRIMARY", + columns: ["message_id"] }, { - id: 3, - name: "relay_claim_admission_index", - checksum: relayClaimAdmissionIndexChecksum, - label: "Relay claim admission index" + table: "effect_local_relay_messages", + name: "effect_local_relay_messages_channel_sequence", + columns: ["channel_id", "channel_sequence"] }, { - id: 4, - name: "relay_claim_recipient_route", - checksum: relayClaimRecipientRouteChecksum, - label: "Relay claim recipient route" + table: "effect_local_relay_messages", + name: "effect_local_relay_messages_sender_identity", + columns: ["sender_message_identity"] + }, + { + table: "effect_local_relay_usage", + name: "PRIMARY", + columns: ["scope_kind", "scope_key"] + }, + { + table: "effect_local_relay_reservations", + name: "PRIMARY", + columns: ["message_id"] } ] as const -export const run = Effect.gen(function*() { - const sql = yield* SqlClient.SqlClient - const findCatalog = SqlSchema.findAll({ - Request: Schema.Int, - Result: Schema.Struct({ - name: Schema.String, - checksum: Schema.String - }), - execute: (migrationId) => - sql`SELECT name, checksum - FROM effect_local_relay_migration_catalog - WHERE migration_id = ${migrationId}` - }) - return yield* sql.withTransaction(Effect.gen(function*() { - const applied = yield* migrate - for (const expected of expectedCatalog) { - const rows = yield* findCatalog(expected.id) - const row = rows[0] +const verifyMysqlSchema = (sql: SqlClient.SqlClient) => + Effect.gen(function*() { + const columnRows = yield* sql.unsafe(` + SELECT + table_name AS tableName, + column_name AS columnName, + data_type AS dataType, + is_nullable AS isNullable, + collation_name AS collationName + FROM information_schema.columns + WHERE table_schema = DATABASE() + AND table_name IN ( + 'effect_local_relay_write_lock', + 'effect_local_relay_channels', + 'effect_local_relay_messages', + 'effect_local_relay_usage', + 'effect_local_relay_reservations' + ) + ORDER BY table_name, ordinal_position + `) + const columns = yield* decodeMysqlRows( + Schema.Array(Schema.Struct({ + tableName: Schema.String, + columnName: Schema.String, + dataType: Schema.String, + isNullable: Schema.String, + collationName: Schema.NullOr(Schema.String) + })), + columnRows, + "Could not inspect MySQL relay columns" + ) + for (const [table, expected] of Object.entries(mysqlExpectedColumns)) { + const actual = columns + .filter((column) => column.tableName === table) + .map((column) => `${column.columnName}:${column.dataType}:${column.isNullable}`) + if (actual.length !== expected.length || actual.some((value, index) => value !== expected[index])) { + return yield* migrationFailure(`MySQL relay table "${table}" does not match migration 1`) + } + } + for (const column of columns) { if ( - rows.length !== 1 || - row?.name !== expected.name || - row.checksum !== expected.checksum + mysqlBinaryColumns.has(`${column.tableName}.${column.columnName}`) && + column.collationName !== "utf8mb4_bin" ) { - return yield* new Migrator.MigrationError({ - kind: "BadState", - message: `${expected.label} migration checksum mismatch` - }) + return yield* migrationFailure( + `MySQL relay column "${column.tableName}.${column.columnName}" must use utf8mb4_bin` + ) } } - return applied - })) -}).pipe( - Effect.catchTag("SchemaError", (cause) => - Effect.fail( - new Migrator.MigrationError({ - kind: "BadState", - message: `Invalid relay migration catalog: ${cause}` - }) - )) -) + + const indexRows = yield* sql.unsafe(` + SELECT + table_name AS tableName, + index_name AS indexName, + column_name AS columnName, + seq_in_index AS sequence + FROM information_schema.statistics + WHERE table_schema = DATABASE() + ORDER BY table_name, index_name, seq_in_index + `) + const indexes = yield* decodeMysqlRows( + Schema.Array(Schema.Struct({ + tableName: Schema.String, + indexName: Schema.String, + columnName: Schema.String, + sequence: DatabaseInt + })), + indexRows, + "Could not inspect MySQL relay indexes" + ) + for (const expected of mysqlRequiredIndexes) { + const actual = indexes + .filter((index) => index.tableName === expected.table && index.indexName === expected.name) + .toSorted((left, right) => left.sequence - right.sequence) + .map((index) => index.columnName) + if ( + actual.length !== expected.columns.length || + actual.some((column, index) => column !== expected.columns[index]) + ) { + return yield* migrationFailure(`MySQL relay index "${expected.name}" is missing or invalid`) + } + } + + const foreignKeyRows = yield* sql.unsafe(` + SELECT + usage_table.table_name AS tableName, + usage_table.column_name AS columnName, + usage_table.referenced_table_name AS referencedTableName, + usage_table.referenced_column_name AS referencedColumnName, + rules.delete_rule AS deleteRule + FROM information_schema.key_column_usage usage_table + JOIN information_schema.referential_constraints rules + ON rules.constraint_schema = usage_table.constraint_schema + AND rules.constraint_name = usage_table.constraint_name + WHERE usage_table.constraint_schema = DATABASE() + AND usage_table.referenced_table_name IS NOT NULL + AND usage_table.table_name IN ( + 'effect_local_relay_messages', + 'effect_local_relay_reservations' + ) + ORDER BY usage_table.table_name + `) + const foreignKeys = yield* decodeMysqlRows( + Schema.Array(Schema.Struct({ + tableName: Schema.String, + columnName: Schema.String, + referencedTableName: Schema.String, + referencedColumnName: Schema.String, + deleteRule: Schema.String + })), + foreignKeyRows, + "Could not inspect MySQL relay foreign keys" + ) + const actualForeignKeys = foreignKeys.map((foreignKey) => + [ + foreignKey.tableName, + foreignKey.columnName, + foreignKey.referencedTableName, + foreignKey.referencedColumnName, + foreignKey.deleteRule + ].join(":") + ) + const expectedForeignKeys = [ + "effect_local_relay_messages:channel_id:effect_local_relay_channels:channel_id:CASCADE", + "effect_local_relay_reservations:message_id:effect_local_relay_messages:message_id:CASCADE" + ] + if ( + actualForeignKeys.length !== expectedForeignKeys.length || + actualForeignKeys.some((foreignKey, index) => foreignKey !== expectedForeignKeys[index]) + ) { + return yield* migrationFailure("MySQL relay foreign keys do not match migration 1") + } + + const checkRows = yield* sql.unsafe(` + SELECT constraint_name AS constraintName + FROM information_schema.table_constraints + WHERE constraint_schema = DATABASE() + AND constraint_type = 'CHECK' + AND table_name IN ( + 'effect_local_relay_write_lock', + 'effect_local_relay_channels', + 'effect_local_relay_messages', + 'effect_local_relay_usage', + 'effect_local_relay_reservations' + ) + `) + const checks = yield* decodeMysqlRows( + Schema.Array(Schema.Struct({ constraintName: Schema.String })), + checkRows, + "Could not inspect MySQL relay check constraints" + ) + if (checks.length !== 17) { + return yield* migrationFailure("MySQL relay check constraints do not match migration 1") + } + }) + +const genericRun = Migrator.make({})({ + loader: migrations, + table: "effect_local_relay_migrations" +}) + +const runMysql = (sql: SqlClient.SqlClient) => + sql.withTransaction( + Effect.acquireUseRelease( + Effect.gen(function*() { + const lockRows = yield* sql`SELECT GET_LOCK( + 'effect_local_relay_migrations', + 30 + ) AS acquired` + const lock = yield* decodeMysqlRows( + Schema.Array(Schema.Struct({ acquired: Schema.NullOr(DatabaseInt) })), + lockRows, + "Could not decode the MySQL relay migration lock result" + ) + if (lock.length !== 1 || lock[0]!.acquired !== 1) { + return yield* new Migrator.MigrationError({ + kind: "Locked", + message: "Could not acquire the MySQL relay migration lock" + }) + } + }), + () => + Effect.gen(function*() { + yield* sql` + CREATE TABLE IF NOT EXISTS effect_local_relay_migrations ( + migration_id INTEGER UNSIGNED NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + name VARCHAR(255) NOT NULL, + PRIMARY KEY (migration_id) + ) + ` + const markerRows = yield* sql` + SELECT migration_id AS migrationId + FROM effect_local_relay_migrations + WHERE migration_id = 1 + ` + const markers = yield* decodeMysqlRows( + Schema.Array(Schema.Struct({ migrationId: DatabaseInt })), + markerRows, + "Could not decode MySQL relay migration state" + ) + if (markers.length > 1) { + return yield* migrationFailure("MySQL relay migration state contains duplicate markers") + } + if (markers.length === 0) { + yield* createRelayTables + } + yield* verifyMysqlSchema(sql) + if (markers.length === 0) { + yield* sql` + INSERT INTO effect_local_relay_migrations (migration_id, name) + VALUES (1, 'create_relay_tables') + ` + } + return markers.length === 0 + ? [[1, "create_relay_tables"] as const] + : [] + }), + () => + sql`SELECT RELEASE_LOCK('effect_local_relay_migrations')`.pipe( + Effect.asVoid + ) + ) + ) + +export const run = Effect.gen(function*() { + const sql = (yield* SqlClient.SqlClient).withoutTransforms() + return yield* sql.onDialectOrElse({ + mysql: () => runMysql(sql), + orElse: () => genericRun + }) +}) export const layer: Layer.Layer< never, diff --git a/packages/local-rpc/src/internal/peerRelaySqliteTransaction.ts b/packages/local-rpc/src/internal/peerRelaySqlTransaction.ts similarity index 89% rename from packages/local-rpc/src/internal/peerRelaySqliteTransaction.ts rename to packages/local-rpc/src/internal/peerRelaySqlTransaction.ts index 019d9c4..a55ab55 100644 --- a/packages/local-rpc/src/internal/peerRelaySqliteTransaction.ts +++ b/packages/local-rpc/src/internal/peerRelaySqlTransaction.ts @@ -9,7 +9,7 @@ import type * as SqlConnection from "effect/unstable/sql/SqlConnection" import type * as SqlError from "effect/unstable/sql/SqlError" export class NestedPeerRelayTransactionError extends Schema.TaggedErrorClass( - "@lucas-barake/effect-local-rpc/internal/peerRelaySqliteTransaction/NestedPeerRelayTransactionError" + "@lucas-barake/effect-local-rpc/internal/peerRelaySqlTransaction/NestedPeerRelayTransactionError" )("NestedPeerRelayTransactionError", {}) {} export interface Options { @@ -106,7 +106,7 @@ const acquireImmediate = ( return attempt(options.maxAcquireAttempts, 0) }) -export const make = ( +export const makeSqlite = ( sql: SqlClient.SqlClient, options: Options ) => { @@ -204,3 +204,35 @@ export const make = ( }) }) } + +const makeServer = (sql: SqlClient.SqlClient) => +( + effect: Effect.Effect +): Effect.Effect< + A, + E | SqlError.SqlError | NestedPeerRelayTransactionError, + R +> => + Effect.gen(function*() { + const active = yield* Effect.serviceOption(sql.transactionService) + if (Option.isSome(active)) { + return yield* new NestedPeerRelayTransactionError() + } + return yield* sql.withTransaction( + sql`SELECT lock_id + FROM effect_local_relay_write_lock + WHERE lock_id = 1 + FOR UPDATE`.pipe(Effect.andThen(effect)) + ) + }) + +export const make = ( + sql: SqlClient.SqlClient, + options: Options +) => + sql.onDialectOrElse({ + sqlite: () => makeSqlite(sql, options), + pg: () => makeServer(sql), + mysql: () => makeServer(sql), + orElse: () => makeServer(sql) + }) diff --git a/packages/local-rpc/src/internal/peerRelayStoreErrors.ts b/packages/local-rpc/src/internal/peerRelayStoreErrors.ts index db8f8c4..3907906 100644 --- a/packages/local-rpc/src/internal/peerRelayStoreErrors.ts +++ b/packages/local-rpc/src/internal/peerRelayStoreErrors.ts @@ -4,7 +4,7 @@ import * as Effect from "effect/Effect" import type * as PlatformError from "effect/PlatformError" import type * as Schema from "effect/Schema" import type * as SqlError from "effect/unstable/sql/SqlError" -import type { NestedPeerRelayTransactionError } from "./peerRelaySqliteTransaction.js" +import type { NestedPeerRelayTransactionError } from "./peerRelaySqlTransaction.js" export type StoreBoundaryError = | ReplicaError.ReplicaError @@ -34,6 +34,7 @@ export const mapStoreErrors = ( switch (error._tag) { case "SqlError": case "PlatformError": + case "NestedPeerRelayTransactionError": return storageUnavailable(error) case "SchemaError": case "NoSuchElementError": diff --git a/packages/local-rpc/src/internal/peerRpcProtocol.ts b/packages/local-rpc/src/internal/peerRpcProtocol.ts new file mode 100644 index 0000000..48c8424 --- /dev/null +++ b/packages/local-rpc/src/internal/peerRpcProtocol.ts @@ -0,0 +1,4 @@ +import * as PeerSyncEnvelope from "@lucas-barake/effect-local-sql/PeerSyncEnvelope" + +export const maximumNegotiatedDurationMillis = 90 * 24 * 60 * 60 * 1_000 +export const maximumRelayPayloadBytes = PeerSyncEnvelope.maximumRelayPayloadBytes diff --git a/packages/local-rpc/test/PeerAuthentication.test.ts b/packages/local-rpc/test/PeerAuthentication.test.ts index 46b5bb9..e584ad7 100644 --- a/packages/local-rpc/test/PeerAuthentication.test.ts +++ b/packages/local-rpc/test/PeerAuthentication.test.ts @@ -24,13 +24,20 @@ import * as PeerRpcObservability from "../src/internal/peerRpcObservability.js" import * as PeerAuthentication from "../src/PeerAuthentication.js" import * as PeerAuthenticator from "../src/PeerAuthenticator.js" import * as PeerCredentials from "../src/PeerCredentials.js" +import * as PeerRelayLimits from "../src/PeerRelayLimits.js" import * as PeerRpc from "../src/PeerRpc.js" import * as PeerRpcError from "../src/PeerRpcError.js" -import * as PeerRpcLimits from "../src/PeerRpcLimits.js" const peerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000001") +const remotePeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000002") const principal = PeerAuthentication.PeerPrincipal.make({ tenantId: "tenant", subjectId: "subject", peerId }) const sessionId = Identity.SessionId.make("ses_00000000-0000-4000-8000-000000000001") +const relayMessageId = Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000001") +const documentId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000001") +const terminalHandlers = { + Acknowledge: () => Effect.void, + Reject: () => Effect.void +} const invoke = ( middleware: PeerAuthentication.PeerAuthentication["Service"], @@ -63,7 +70,8 @@ describe("PeerAuthentication", () => { const authenticatedWith = yield* Deferred.make>() const handlers = PeerRpc.Rpcs.toLayer(PeerRpc.Rpcs.of({ Open: () => Stream.empty, - Push: () => Effect.void + Push: () => Effect.void, + ...terminalHandlers })) const client = yield* RpcTest.makeClient(PeerRpc.Rpcs).pipe( Effect.provide(handlers), @@ -75,10 +83,10 @@ describe("PeerAuthentication", () => { Effect.provideService(PeerCredentials.PeerCredentials, { get: Effect.succeed(Redacted.make("constant")) }), - Effect.provideService(PeerRpcLimits.PeerRpcLimits, PeerRpcLimits.defaults) + Effect.provideService(PeerRelayLimits.PeerRelayLimits, PeerRelayLimits.defaults) ) - yield* client.Push({ sessionId, payload: Uint8Array.of(1) }) + yield* client.Push({ sessionId, relayMessageId, payload: Uint8Array.of(1) }) assert.strictEqual(Redacted.value(yield* Deferred.await(authenticatedWith)), "constant") })) @@ -110,12 +118,13 @@ describe("PeerAuthentication", () => { _tag: "Opened", protocolVersion: PeerRpc.protocolVersion, sessionId, - peerId, - capabilities: { storeAndForward: false } + remotePeerId, + authenticatedLocal: principal }) }) ), - Push: () => PeerAuthentication.AuthenticatedPeer.pipe(Effect.asVoid) + Push: () => PeerAuthentication.AuthenticatedPeer.pipe(Effect.asVoid), + ...terminalHandlers })) const client = yield* RpcTest.makeClient(PeerRpc.Rpcs).pipe( Effect.provide(handlers), @@ -134,18 +143,25 @@ describe("PeerAuthentication", () => { get: Effect.sync(() => Redacted.make(credential)) }) ), - Effect.provideService(PeerRpcLimits.PeerRpcLimits, PeerRpcLimits.defaults) + Effect.provideService(PeerRelayLimits.PeerRelayLimits, PeerRelayLimits.defaults) ) yield* client.Open({ protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: peerId, - definitionHash: "def_0000000000000000", - documents: [], + expectedRelayPeerId: peerId, + expectedLocal: principal, + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + remote: { + subjectId: "remote", + peerId: remotePeerId + }, + documents: [{ documentType: "Task", documentId }], + receiptRetentionMillis: 1, + senderRetryHorizonMillis: 1, credential: Redacted.make("caller") }).pipe(Stream.runDrain) credential = "second" - yield* client.Push({ sessionId, payload: Uint8Array.of(1) }) + yield* client.Push({ sessionId, relayMessageId, payload: Uint8Array.of(1) }) assert.deepStrictEqual(credentials, ["first", "second"]) })) @@ -156,7 +172,8 @@ describe("PeerAuthentication", () => { const authenticatedWith = yield* Deferred.make>() const handlers = PeerRpc.Rpcs.toLayer(PeerRpc.Rpcs.of({ Open: () => Stream.empty, - Push: () => Effect.void + Push: () => Effect.void, + ...terminalHandlers })) const client = yield* RpcTest.makeClient(PeerRpc.Rpcs).pipe( Effect.provide(handlers), @@ -168,10 +185,12 @@ describe("PeerAuthentication", () => { Effect.provideService(PeerCredentials.PeerCredentials, { get: Deferred.succeed(requested, undefined).pipe(Effect.andThen(Deferred.await(credential))) }), - Effect.provideService(PeerRpcLimits.PeerRpcLimits, PeerRpcLimits.defaults) + Effect.provideService(PeerRelayLimits.PeerRelayLimits, PeerRelayLimits.defaults) ) - const request = yield* client.Push({ sessionId, payload: Uint8Array.of(1) }).pipe(Effect.forkChild) + const request = yield* client.Push({ sessionId, relayMessageId, payload: Uint8Array.of(1) }).pipe( + Effect.forkChild + ) yield* Deferred.await(requested) assert.isTrue(Option.isNone(yield* Deferred.poll(authenticatedWith))) yield* Deferred.succeed(credential, Redacted.make("asynchronous")) @@ -194,7 +213,7 @@ describe("PeerAuthentication", () => { Effect.provideService(PeerAuthenticator.PeerAuthenticator, { authenticate: () => Effect.succeed(authenticated) }), - Effect.provideService(PeerRpcLimits.PeerRpcLimits, PeerRpcLimits.defaults) + Effect.provideService(PeerRelayLimits.PeerRelayLimits, PeerRelayLimits.defaults) )) it.effect("rejects verifier concurrency immediately", () => @@ -211,8 +230,8 @@ describe("PeerAuthentication", () => { return authenticated }) }), - Effect.provideService(PeerRpcLimits.PeerRpcLimits, { - ...PeerRpcLimits.defaults, + Effect.provideService(PeerRelayLimits.PeerRelayLimits, { + ...PeerRelayLimits.defaults, maxInFlightAuthentication: 1 }) ) @@ -251,8 +270,8 @@ describe("PeerAuthentication", () => { Effect.provideService(PeerAuthenticator.PeerAuthenticator, { authenticate: () => Deferred.succeed(authenticatedOnce, undefined).pipe(Effect.as(authenticated)) }), - Effect.provideService(PeerRpcLimits.PeerRpcLimits, { - ...PeerRpcLimits.defaults, + Effect.provideService(PeerRelayLimits.PeerRelayLimits, { + ...PeerRelayLimits.defaults, authenticationRatePerSecond: 1, authenticationBurst: 1 }) @@ -286,7 +305,7 @@ describe("PeerAuthentication", () => { Effect.provideService(PeerAuthenticator.PeerAuthenticator, { authenticate: () => Deferred.succeed(started, undefined).pipe(Effect.andThen(Effect.never)) }), - Effect.provideService(PeerRpcLimits.PeerRpcLimits, PeerRpcLimits.defaults) + Effect.provideService(PeerRelayLimits.PeerRelayLimits, PeerRelayLimits.defaults) ) const fiber = yield* invoke(middleware, { credential: Redacted.make("credential") }, 1).pipe(Effect.forkChild) @@ -322,8 +341,8 @@ describe("PeerAuthentication", () => { }) : Effect.succeed(authenticated) }), - Effect.provideService(PeerRpcLimits.PeerRpcLimits, { - ...PeerRpcLimits.defaults, + Effect.provideService(PeerRelayLimits.PeerRelayLimits, { + ...PeerRelayLimits.defaults, authenticationBurst: 1, maxRetainedRateLimitedConnections: 1 }) @@ -344,8 +363,8 @@ describe("PeerAuthentication", () => { Effect.provideService(PeerAuthenticator.PeerAuthenticator, { authenticate: () => Effect.succeed(authenticated) }), - Effect.provideService(PeerRpcLimits.PeerRpcLimits, { - ...PeerRpcLimits.defaults, + Effect.provideService(PeerRelayLimits.PeerRelayLimits, { + ...PeerRelayLimits.defaults, authenticationRatePerSecond: 1, authenticationBurst: 1, maxRetainedRateLimitedConnections: 3 @@ -380,11 +399,11 @@ describe("PeerAuthentication", () => { Effect.provideService(PeerAuthenticator.PeerAuthenticator, { authenticate: () => Effect.succeed(authenticated) }), - Effect.provideService(PeerRpcLimits.PeerRpcLimits, { - ...PeerRpcLimits.defaults, + Effect.provideService(PeerRelayLimits.PeerRelayLimits, { + ...PeerRelayLimits.defaults, authenticationRatePerSecond: Number.MIN_VALUE, authenticationBurst: 1, - rateLimitIdleRetention: 1_000, + rateLimitIdleRetentionMillis: 1_000, maxRetainedRateLimitedConnections: 8 }) ) @@ -414,8 +433,8 @@ describe("PeerAuthentication", () => { Effect.provideService(PeerAuthenticator.PeerAuthenticator, { authenticate: () => Effect.succeed(authenticated) }), - Effect.provideService(PeerRpcLimits.PeerRpcLimits, { - ...PeerRpcLimits.defaults, + Effect.provideService(PeerRelayLimits.PeerRelayLimits, { + ...PeerRelayLimits.defaults, authenticationBurst: 1 }) )) @@ -430,7 +449,7 @@ describe("PeerAuthentication", () => { Effect.provideService(PeerAuthenticator.PeerAuthenticator, { authenticate: () => Effect.succeed({ ...authenticated, validUntil: 0 }) }), - Effect.provideService(PeerRpcLimits.PeerRpcLimits, PeerRpcLimits.defaults) + Effect.provideService(PeerRelayLimits.PeerRelayLimits, PeerRelayLimits.defaults) )) it.effect("records authentication denial without credential identity or verifier defects", () => { @@ -492,7 +511,7 @@ describe("PeerAuthentication", () => { Effect.andThen(Effect.die(new Error("verifier-forbidden-value"))) ) }), - Effect.provideService(PeerRpcLimits.PeerRpcLimits, PeerRpcLimits.defaults), + Effect.provideService(PeerRelayLimits.PeerRelayLimits, PeerRelayLimits.defaults), Effect.provideService(Metric.MetricRegistry, new Map()), Effect.provideService(Tracer.Tracer, tracer) ) @@ -509,7 +528,7 @@ describe("PeerAuthentication", () => { Effect.provideService(PeerAuthenticator.PeerAuthenticator, { authenticate: () => Effect.succeed({ ...authenticated, validUntil }) }), - Effect.provideService(PeerRpcLimits.PeerRpcLimits, PeerRpcLimits.defaults) + Effect.provideService(PeerRelayLimits.PeerRelayLimits, PeerRelayLimits.defaults) )) } }) diff --git a/packages/local-rpc/test/PeerAuthorization.test.ts b/packages/local-rpc/test/PeerAuthorization.test.ts deleted file mode 100644 index 7b46042..0000000 --- a/packages/local-rpc/test/PeerAuthorization.test.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { assert, describe, it } from "@effect/vitest" -import * as Document from "@lucas-barake/effect-local/Document" -import * as Identity from "@lucas-barake/effect-local/Identity" -import * as Effect from "effect/Effect" -import * as Schema from "effect/Schema" -import * as PeerAuthentication from "../src/PeerAuthentication.js" -import * as PeerAuthorization from "../src/PeerAuthorization.js" -import * as PeerRpcError from "../src/PeerRpcError.js" - -const document = Document.make("Task", { schema: Schema.Struct({ title: Schema.String }), version: 1 }) -const documentId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000001") -const otherDocument = Document.make("Note", { schema: Schema.Struct({ body: Schema.String }), version: 1 }) -const otherDocumentId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000002") -const principal = PeerAuthentication.PeerPrincipal.make({ - tenantId: "tenant", - subjectId: "subject", - peerId: Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000001") -}) - -describe("PeerAuthorization", () => { - it.effect("resolves an exactly matching requested document set", () => - Effect.gen(function*() { - const authorization = yield* PeerAuthorization.PeerAuthorization - const result = yield* authorization.authorize({ - principal, - documents: [{ documentType: document.name, documentId }] - }) - assert.deepStrictEqual(result.documents, [{ document, documentId }]) - }).pipe( - Effect.provide(PeerAuthorization.layer(() => - Effect.succeed({ - documents: [{ document, documentId }], - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.void - }) - )) - )) - - it.effect("rejects duplicate requests before policy evaluation", () => { - let calls = 0 - return Effect.gen(function*() { - const authorization = yield* PeerAuthorization.PeerAuthorization - const error = yield* authorization.authorize({ - principal, - documents: [ - { documentType: document.name, documentId }, - { documentType: document.name, documentId } - ] - }).pipe(Effect.flip) - assert.instanceOf(error, PeerRpcError.AccessDenied) - assert.strictEqual(calls, 0) - }).pipe( - Effect.provide(PeerAuthorization.layer(() => { - calls++ - return Effect.never - })) - ) - }) - - it.effect("rejects missing extra duplicate and substituted authorization results", () => - Effect.gen(function*() { - const requested = [{ documentType: document.name, documentId }] - const cases = [ - [], - [{ document, documentId }, { document: otherDocument, documentId: otherDocumentId }], - [{ document, documentId }, { document, documentId }], - [{ document: otherDocument, documentId }], - [{ document, documentId: otherDocumentId }] - ] - - for (const documents of cases) { - const authorization = yield* PeerAuthorization.PeerAuthorization.pipe( - Effect.provide( - PeerAuthorization.layer(() => - Effect.succeed({ documents, validUntil: Number.MAX_SAFE_INTEGER, invalidated: Effect.void }) - ) - ) - ) - const error = yield* authorization.authorize({ principal, documents: requested }).pipe(Effect.flip) - assert.instanceOf(error, PeerRpcError.AccessDenied) - } - })) - - it.effect("rejects an expired authorization lease", () => - Effect.gen(function*() { - const authorization = yield* PeerAuthorization.PeerAuthorization - const error = yield* authorization.authorize({ - principal, - documents: [{ documentType: document.name, documentId }] - }).pipe(Effect.flip) - assert.instanceOf(error, PeerRpcError.AccessDenied) - }).pipe( - Effect.provide( - PeerAuthorization.layer(() => - Effect.succeed({ documents: [{ document, documentId }], validUntil: 0, invalidated: Effect.void }) - ) - ) - )) -}) diff --git a/packages/local-rpc/test/PeerRelayLimits.test.ts b/packages/local-rpc/test/PeerRelayLimits.test.ts index 9551019..ce23610 100644 --- a/packages/local-rpc/test/PeerRelayLimits.test.ts +++ b/packages/local-rpc/test/PeerRelayLimits.test.ts @@ -2,14 +2,14 @@ import { assert, describe, it } from "@effect/vitest" import * as Effect from "effect/Effect" import * as Layer from "effect/Layer" import * as PeerRelayLimits from "../src/PeerRelayLimits.js" -import * as PeerRelayRpc from "../src/PeerRelayRpc.js" +import * as PeerRpc from "../src/PeerRpc.js" describe("PeerRelayLimits", () => { it.effect("publishes the complete validated production defaults through its Layer", () => Effect.gen(function*() { const limits = yield* PeerRelayLimits.PeerRelayLimits assert.deepStrictEqual(limits, PeerRelayLimits.defaults) - assert.strictEqual(Object.keys(limits).length, 94) + assert.strictEqual(Object.keys(limits).length, 97) assert.strictEqual(limits.maximumDeliveryAttempts, 16) }).pipe(Effect.provide(PeerRelayLimits.layerDefaults))) @@ -27,9 +27,9 @@ describe("PeerRelayLimits", () => { ], [ "maximumReceiptRetentionMillis", - PeerRelayRpc.maximumNegotiatedDurationMillis + 1, - `Expected a value less than or equal to ${PeerRelayRpc.maximumNegotiatedDurationMillis}, got ${ - PeerRelayRpc.maximumNegotiatedDurationMillis + 1 + PeerRpc.maximumNegotiatedDurationMillis + 1, + `Expected a value less than or equal to ${PeerRpc.maximumNegotiatedDurationMillis}, got ${ + PeerRpc.maximumNegotiatedDurationMillis + 1 }` ], ["sqliteCapacityHeadroomPercent", 101, "Expected a value less than or equal to 100, got 101"], @@ -81,7 +81,7 @@ describe("PeerRelayLimits", () => { ], [ "maximumDeclaredFrameBytes", - { maximumDeclaredFrameBytes: PeerRelayRpc.maximumRelayPayloadBytes - 1 } + { maximumDeclaredFrameBytes: PeerRpc.maximumRelayPayloadBytes - 1 } ], [ "maximumSharedPayloadBytes", diff --git a/packages/local-rpc/test/PeerRelayRpc.test.ts b/packages/local-rpc/test/PeerRelayRpc.test.ts deleted file mode 100644 index dc41bd7..0000000 --- a/packages/local-rpc/test/PeerRelayRpc.test.ts +++ /dev/null @@ -1,191 +0,0 @@ -import { assert, describe, it } from "@effect/vitest" -import * as Identity from "@lucas-barake/effect-local/Identity" -import * as Schema from "effect/Schema" -import * as PeerRelayRpc from "../src/PeerRelayRpc.js" -import * as PeerRpc from "../src/PeerRpc.js" - -const localPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000001") -const remotePeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000002") -const relayPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000003") -const sessionId = Identity.SessionId.make("ses_00000000-0000-4000-8000-000000000001") -const relayMessageId = Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000001") -const documentId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000001") -const claimToken = PeerRelayRpc.ClaimToken.make("clm_00000000-0000-4000-8000-000000000001") -const hash = "0".repeat(64) - -describe("PeerRelayRpc", () => { - it("roundtrips the strict version 3 handshake", () => { - const opened = PeerRelayRpc.RelayOpened.make({ - _tag: "RelayOpened", - version: PeerRelayRpc.protocolVersion, - sessionId, - remotePeerId, - authenticatedLocal: { - tenantId: "tenant", - subjectId: "subject", - peerId: localPeerId - }, - capabilities: { storeAndForward: true } - }) - - assert.deepStrictEqual(Schema.decodeUnknownSync(PeerRelayRpc.OpenRelayEvent)(opened), opened) - assert.throws(() => Schema.decodeUnknownSync(PeerRpc.OpenEvent)(opened)) - }) - - it("roundtrips every relay request and stored message field", () => { - const open = PeerRelayRpc.OpenRelayRpc.payloadSchema.make({ - version: PeerRelayRpc.protocolVersion, - expectedRelayPeerId: relayPeerId, - expectedLocal: { - tenantId: "tenant", - subjectId: "subject", - peerId: localPeerId - }, - senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), - remote: { - subjectId: "remote", - peerId: remotePeerId - }, - documents: [{ documentType: "Task", documentId }], - receiptRetentionMillis: 8 * 24 * 60 * 60 * 1_000, - senderRetryHorizonMillis: 7 * 24 * 60 * 60 * 1_000 - }) - const stored = PeerRelayRpc.StoredMessage.make({ - _tag: "StoredMessage", - relayMessageId, - claimToken, - relayPeerId, - sender: { - tenantId: "tenant", - subjectId: "subject", - peerId: localPeerId, - replicaIncarnation: Identity.ReplicaIncarnation.make(1), - connectionEpoch: "epoch", - sequence: 0 - }, - recipient: { - tenantId: "tenant", - subjectId: "remote", - peerId: remotePeerId - }, - payloadVersion: 1, - document: { documentType: "Task", documentId }, - writerProvenance: [], - messageHash: hash, - outerEnvelopeDigest: hash, - payload: Uint8Array.of(1, 2, 3) - }) - const push = PeerRelayRpc.PushRelayRpc.payloadSchema.make({ - sessionId, - relayMessageId, - payload: Uint8Array.of(1, 2, 3) - }) - const acknowledge = PeerRelayRpc.AcknowledgeRelayRpc.payloadSchema.make({ - sessionId, - relayMessageId, - claimToken, - messageHash: hash - }) - const reject = PeerRelayRpc.RejectRelayRpc.payloadSchema.make({ - ...acknowledge, - reason: "ProtocolInvalid" - }) - - assert.deepStrictEqual(Schema.decodeUnknownSync(PeerRelayRpc.OpenRelayRpc.payloadSchema)(open), open) - assert.deepStrictEqual(Schema.decodeUnknownSync(PeerRelayRpc.OpenRelayEvent)(stored), stored) - assert.deepStrictEqual(Schema.decodeUnknownSync(PeerRelayRpc.PushRelayRpc.payloadSchema)(push), push) - assert.deepStrictEqual( - Schema.decodeUnknownSync(PeerRelayRpc.AcknowledgeRelayRpc.payloadSchema)(acknowledge), - acknowledge - ) - assert.deepStrictEqual(Schema.decodeUnknownSync(PeerRelayRpc.RejectRelayRpc.payloadSchema)(reject), reject) - }) - - it("rejects invalid durations, empty document sets, identifiers, hashes, and oversized payloads", () => { - const baseOpen = { - version: PeerRelayRpc.protocolVersion, - expectedRelayPeerId: relayPeerId, - expectedLocal: { - tenantId: "tenant", - subjectId: "subject", - peerId: localPeerId - }, - senderReplicaIncarnation: 1, - remote: { - subjectId: "remote", - peerId: remotePeerId - }, - documents: [{ documentType: "Task", documentId }], - receiptRetentionMillis: 1, - senderRetryHorizonMillis: 1 - } - - for (const duration of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) { - assert.throws(() => - Schema.decodeUnknownSync(PeerRelayRpc.OpenRelayRpc.payloadSchema)({ - ...baseOpen, - receiptRetentionMillis: duration - }) - ) - } - assert.throws(() => - Schema.decodeUnknownSync(PeerRelayRpc.OpenRelayRpc.payloadSchema)({ ...baseOpen, documents: [] }) - ) - assert.throws(() => - Schema.decodeUnknownSync(PeerRelayRpc.AcknowledgeRelayRpc.payloadSchema)({ - sessionId, - relayMessageId: "rly_invalid", - claimToken, - messageHash: hash - }) - ) - assert.throws(() => - Schema.decodeUnknownSync(PeerRelayRpc.AcknowledgeRelayRpc.payloadSchema)({ - sessionId, - relayMessageId, - claimToken, - messageHash: "not-a-hash" - }) - ) - assert.throws(() => - Schema.decodeUnknownSync(PeerRelayRpc.PushRelayRpc.payloadSchema)({ - sessionId, - relayMessageId, - payload: new Uint8Array(PeerRelayRpc.maximumRelayPayloadBytes + 1) - }) - ) - assert.throws(() => - Schema.decodeUnknownSync(PeerRelayRpc.OpenRelayEvent)( - PeerRelayRpc.StoredMessage.make({ - _tag: "StoredMessage", - relayMessageId, - claimToken, - relayPeerId, - sender: { - tenantId: "tenant", - subjectId: "subject", - peerId: localPeerId, - replicaIncarnation: Identity.ReplicaIncarnation.make(1), - connectionEpoch: "epoch", - sequence: 0 - }, - recipient: { - tenantId: "tenant", - subjectId: "remote", - peerId: remotePeerId - }, - payloadVersion: 1, - document: { documentType: "Task", documentId }, - writerProvenance: [{ - changeHash: hash, - writerSchemaVersion: 1, - writerDefinitionHash: "invalid\"definition" - }], - messageHash: hash, - outerEnvelopeDigest: hash, - payload: Uint8Array.of(1) - }) - ) - ) - }) -}) diff --git a/packages/local-rpc/test/PeerRelaySqliteTransaction.test.ts b/packages/local-rpc/test/PeerRelaySqliteTransaction.test.ts index 340e2c2..8de51b4 100644 --- a/packages/local-rpc/test/PeerRelaySqliteTransaction.test.ts +++ b/packages/local-rpc/test/PeerRelaySqliteTransaction.test.ts @@ -8,7 +8,7 @@ import * as Scope from "effect/Scope" import * as SqlClient from "effect/unstable/sql/SqlClient" import type * as SqlConnection from "effect/unstable/sql/SqlConnection" import * as SqlError from "effect/unstable/sql/SqlError" -import { cleanupAcquisition, make } from "../src/internal/peerRelaySqliteTransaction.js" +import { cleanupAcquisition, makeSqlite as make } from "../src/internal/peerRelaySqlTransaction.js" describe("peerRelaySqliteTransaction", () => { it.effect("preserves interruption together with rollback failure during acquisition", () => diff --git a/packages/local-rpc/test/PeerRelayStore.test.ts b/packages/local-rpc/test/PeerRelayStore.test.ts index 85a75b1..ad0ca84 100644 --- a/packages/local-rpc/test/PeerRelayStore.test.ts +++ b/packages/local-rpc/test/PeerRelayStore.test.ts @@ -13,8 +13,9 @@ import { tmpdir } from "node:os" import { join } from "node:path" import * as PeerRpcObservability from "../src/internal/peerRpcObservability.js" import * as PeerRelayLimits from "../src/PeerRelayLimits.js" -import * as PeerRelayRpc from "../src/PeerRelayRpc.js" import * as PeerRelayStore from "../src/PeerRelayStore.js" +import * as PeerRpc from "../src/PeerRpc.js" +import * as SqlPeerRelayStore from "../src/SqlPeerRelayStore.js" const peer = (value: string) => Identity.PeerId.make(`peer_00000000-0000-4000-8000-${value}`) const relayId = (value: string) => Identity.RelayMessageId.make(`rly_00000000-0000-4000-8000-${value}`) @@ -29,7 +30,7 @@ const makeLayer = ( NodeCrypto.layer, PeerRelayLimits.layer(limits) ) - const store = PeerRelayStore.layerSqlite.pipe(Layer.provide(base)) + const store = SqlPeerRelayStore.layer.pipe(Layer.provide(base)) return Layer.merge(base, store) } @@ -55,7 +56,7 @@ const withStore = ( describe("PeerRelayStore", () => { it.effect("records fixed relay outcomes, acknowledgement latency, and exact pending gauges", () => { - const registry: Metric.MetricRegistry = new Map() + const registry = new Map() const limits = PeerRelayLimits.Values.make({ ...PeerRelayLimits.defaults, maxActiveMessagesPerSenderPeer: 1, @@ -86,7 +87,7 @@ describe("PeerRelayStore", () => { senderSequence: 0, payloadVersion: 1, messageHash: "message-hash-observe", - outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make("5".repeat(64)), + outerEnvelopeDigest: PeerRpc.RelayDigest.make("5".repeat(64)), payload: new Uint8Array([5]), messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, @@ -128,7 +129,7 @@ describe("PeerRelayStore", () => { const rejected = yield* store.admit(PeerRelayStore.Admission.make({ ...admission, relayMessageId: relayId("000000000054"), - outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make("4".repeat(64)) + outerEnvelopeDigest: PeerRpc.RelayDigest.make("4".repeat(64)) })).pipe(Effect.exit) assert.strictEqual(Exit.isFailure(rejected), true) assert.strictEqual( @@ -182,84 +183,6 @@ describe("PeerRelayStore", () => { ).pipe(Effect.provideService(Metric.MetricRegistry, registry)) }) - it.effect("upgrades the exact version 3 claim index and backfills recipient routes", () => - Effect.gen(function*() { - const filename = join(tmpdir(), `effect-local-relay-${globalThis.crypto.randomUUID()}.sqlite`) - yield* Effect.addFinalizer(() => - Effect.sync(() => { - rmSync(filename, { force: true }) - rmSync(`${filename}-shm`, { force: true }) - rmSync(`${filename}-wal`, { force: true }) - }) - ) - const channel = PeerRelayStore.ChannelKey.make({ - tenantId: "tenant-upgrade", - senderSubjectId: "sender-upgrade", - senderPeerId: peer("000000000091"), - senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), - recipientSubjectId: "recipient-upgrade", - recipientPeerId: peer("000000000092") - }) - yield* Effect.scoped( - Effect.gen(function*() { - const store = yield* PeerRelayStore.PeerRelayStore - const sql = yield* SqlClient.SqlClient - yield* store.admit(PeerRelayStore.Admission.make({ - channel, - relayMessageId: relayId("000000000091"), - relayPeerId: peer("000000000093"), - documentIds: [documentId("000000000091")], - senderConnectionEpoch: "epoch-upgrade", - senderSequence: 0, - payloadVersion: 1, - messageHash: "message-hash-upgrade", - outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make("9".repeat(64)), - payload: new Uint8Array([9]), - messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, - senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, - minimumTerminalRetentionMillis: PeerRelayLimits.defaults.minimumTerminalRetentionMillis - })) - yield* sql`DROP INDEX effect_local_relay_messages_claim_admission` - yield* sql`ALTER TABLE effect_local_relay_messages DROP COLUMN recipient_peer_id` - yield* sql`ALTER TABLE effect_local_relay_messages DROP COLUMN recipient_subject_id` - yield* sql`CREATE INDEX effect_local_relay_messages_claim_admission - ON effect_local_relay_messages( - tenant_id, - sender_subject_id, - sender_peer_id, - created_at, - message_id - ) - WHERE state = 'Pending' AND payload IS NOT NULL` - yield* sql`DELETE FROM effect_local_relay_migration_catalog WHERE migration_id = 4` - yield* sql`DELETE FROM effect_local_relay_migrations WHERE migration_id = 4` - }).pipe(Effect.provide(makeLayer(filename))) - ) - yield* Effect.scoped( - Effect.gen(function*() { - const sql = yield* SqlClient.SqlClient - const routes = yield* sql<{ - readonly recipientSubjectId: string - readonly recipientPeerId: string - }>`SELECT - recipient_subject_id AS recipientSubjectId, - recipient_peer_id AS recipientPeerId - FROM effect_local_relay_messages` - assert.deepStrictEqual(routes, [{ - recipientSubjectId: channel.recipientSubjectId, - recipientPeerId: channel.recipientPeerId - }]) - const catalog = yield* sql<{ readonly count: number }>` - SELECT COUNT(*) AS count - FROM effect_local_relay_migration_catalog - WHERE migration_id = 4 - AND name = 'relay_claim_recipient_route' - ` - assert.strictEqual(catalog[0]?.count, 1) - }).pipe(Effect.provide(makeLayer(filename))) - ) - })) - it.effect("migrates a real WAL database and fences claim payload loading and terminal duplicates", () => Effect.gen(function*() { const filename = join(tmpdir(), `effect-local-relay-${globalThis.crypto.randomUUID()}.sqlite`) @@ -292,7 +215,7 @@ describe("PeerRelayStore", () => { senderSequence: 0, payloadVersion: 1, messageHash: "message-hash", - outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make("a".repeat(64)), + outerEnvelopeDigest: PeerRpc.RelayDigest.make("a".repeat(64)), payload, messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, @@ -398,7 +321,7 @@ describe("PeerRelayStore", () => { const expiringAdmission = PeerRelayStore.Admission.make({ ...admission, relayMessageId: relayId("000000000002"), - outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make("b".repeat(64)) + outerEnvelopeDigest: PeerRpc.RelayDigest.make("b".repeat(64)) }) yield* store.admit(expiringAdmission) const expiringClaim = yield* store.claim({ @@ -463,7 +386,7 @@ describe("PeerRelayStore", () => { const corruptAdmission = PeerRelayStore.Admission.make({ ...admission, relayMessageId: relayId("000000000003"), - outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make("c".repeat(64)) + outerEnvelopeDigest: PeerRpc.RelayDigest.make("c".repeat(64)) }) yield* store.admit(corruptAdmission) yield* sql`UPDATE effect_local_relay_messages @@ -494,7 +417,7 @@ describe("PeerRelayStore", () => { ...admission, channel: restartedChannel, relayMessageId: relayId("000000000004"), - outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make("d".repeat(64)) + outerEnvelopeDigest: PeerRpc.RelayDigest.make("d".repeat(64)) }) yield* store.admit(restartedAdmission) const discovered = yield* store.claim({ @@ -548,7 +471,7 @@ describe("PeerRelayStore", () => { senderSequence: 0, payloadVersion: 1, messageHash: "message-hash", - outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make("e".repeat(64)), + outerEnvelopeDigest: PeerRpc.RelayDigest.make("e".repeat(64)), payload: new Uint8Array([1]), messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, @@ -633,7 +556,7 @@ describe("PeerRelayStore", () => { senderSequence: 0, payloadVersion: 1, messageHash: "message-hash-orphan", - outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make("8".repeat(64)), + outerEnvelopeDigest: PeerRpc.RelayDigest.make("8".repeat(64)), payload: new Uint8Array([8]), messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, @@ -689,7 +612,7 @@ describe("PeerRelayStore", () => { senderSequence: 0, payloadVersion: 1, messageHash: "message-hash-release-cap", - outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make("7".repeat(64)), + outerEnvelopeDigest: PeerRpc.RelayDigest.make("7".repeat(64)), payload: new Uint8Array([7]), messageTtlMillis: limits.messageTtlMillis, senderRetryHorizonMillis: limits.maximumSenderRetryHorizonMillis, @@ -814,7 +737,7 @@ describe("PeerRelayStore", () => { })) it.effect("dead letters an abandoned claim at the delivery attempt cap exactly once", () => { - const registry: Metric.MetricRegistry = new Map() + const registry = new Map() const metricValue = (metric: Metric.Metric) => Metric.value(metric).pipe( Effect.provideService(Metric.CurrentMetricAttributes, {}) @@ -840,7 +763,7 @@ describe("PeerRelayStore", () => { senderSequence: 0, payloadVersion: 1, messageHash: "message-hash-recover-cap", - outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make("6".repeat(64)), + outerEnvelopeDigest: PeerRpc.RelayDigest.make("6".repeat(64)), payload: new Uint8Array([6]), messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, @@ -947,7 +870,7 @@ describe("PeerRelayStore", () => { senderSequence: 0, payloadVersion: 1, messageHash: "message-hash", - outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make("f".repeat(64)), + outerEnvelopeDigest: PeerRpc.RelayDigest.make("f".repeat(64)), payload: new Uint8Array([1]), messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, @@ -1029,7 +952,7 @@ describe("PeerRelayStore", () => { senderSequence: index, payloadVersion: 1, messageHash: `message-hash-${index}`, - outerEnvelopeDigest: PeerRelayRpc.RelayDigest.make(String(index).repeat(64)), + outerEnvelopeDigest: PeerRpc.RelayDigest.make(String(index).repeat(64)), payload: new Uint8Array([index]), messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, diff --git a/packages/local-rpc/test/PeerRelayStoreContract.ts b/packages/local-rpc/test/PeerRelayStoreContract.ts new file mode 100644 index 0000000..61a46e5 --- /dev/null +++ b/packages/local-rpc/test/PeerRelayStoreContract.ts @@ -0,0 +1,239 @@ +import { assert } from "@effect/vitest" +import * as Identity from "@lucas-barake/effect-local/Identity" +import * as Effect from "effect/Effect" +import * as Exit from "effect/Exit" +import * as Option from "effect/Option" +import * as SqlClient from "effect/unstable/sql/SqlClient" +import * as PeerRelayLimits from "../src/PeerRelayLimits.js" +import * as PeerRelayStore from "../src/PeerRelayStore.js" +import * as PeerRpc from "../src/PeerRpc.js" +import * as SqlPeerRelayStore from "../src/SqlPeerRelayStore.js" + +const peerId = (suffix: string) => Identity.PeerId.make(`peer_00000000-0000-4000-8000-${suffix}`) + +const relayMessageId = (suffix: string) => Identity.RelayMessageId.make(`rly_00000000-0000-4000-8000-${suffix}`) + +const documentId = (suffix: string) => Identity.DocumentId.make(`doc_00000000-0000-4000-8000-${suffix}`) + +const makeFixture = (index: number, overrides: Partial = {}) => { + const suffix = String(index).padStart(12, "0") + const channel = PeerRelayStore.ChannelKey.make({ + tenantId: `tenant-contract-${index}`, + senderSubjectId: `sender-contract-${index}`, + senderPeerId: peerId(suffix), + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + recipientSubjectId: `recipient-contract-${index}`, + recipientPeerId: peerId(String(index + 100).padStart(12, "0")) + }) + const admission = PeerRelayStore.Admission.make({ + channel, + relayMessageId: relayMessageId(suffix), + relayPeerId: peerId(String(index + 200).padStart(12, "0")), + documentIds: [documentId(suffix)], + senderConnectionEpoch: `epoch-contract-${index}`, + senderSequence: 0, + payloadVersion: 1, + messageHash: `message-hash-contract-${index}`, + outerEnvelopeDigest: PeerRpc.RelayDigest.make( + (index % 16).toString(16).repeat(64) + ), + payload: new Uint8Array([index]), + messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, + senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, + minimumTerminalRetentionMillis: PeerRelayLimits.defaults.minimumTerminalRetentionMillis, + ...overrides + }) + const claim = (sessionGeneration: number): PeerRelayStore.ClaimRequest => + PeerRelayStore.ClaimRequest.make({ + recipient: { + tenantId: admission.channel.tenantId, + subjectId: admission.channel.recipientSubjectId, + peerId: admission.channel.recipientPeerId + }, + sender: { + subjectId: admission.channel.senderSubjectId, + peerId: admission.channel.senderPeerId + }, + sessionGeneration, + authorizedDocumentIds: admission.documentIds + }) + return { admission, channel: admission.channel, claim } +} + +const terminalRequest = ( + message: PeerRelayStore.ClaimedMessage +): PeerRelayStore.TerminalRequest => + PeerRelayStore.TerminalRequest.make({ + channel: message.channel, + relayMessageId: message.relayMessageId, + claimToken: message.claimToken, + messageHash: message.messageHash, + sessionGeneration: message.sessionGeneration, + recipient: { + tenantId: message.channel.tenantId, + subjectId: message.channel.recipientSubjectId, + peerId: message.channel.recipientPeerId + } + }) + +const claimedMessage = ( + result: PeerRelayStore.ClaimResult +): PeerRelayStore.ClaimedMessage => { + assert.strictEqual(Option.isSome(result.message), true) + if (Option.isNone(result.message)) { + throw new Error("Expected the relay message to be claimed") + } + return result.message.value +} + +export const runPeerRelayStoreContract = Effect.gen(function*() { + const store = yield* PeerRelayStore.PeerRelayStore + const sql = yield* SqlClient.SqlClient + + assert.deepStrictEqual(yield* store.usage(), { + activeCount: 0, + activeBytes: 0, + retainedCount: 0, + retainedBytes: 0 + }) + + const lifecycle = makeFixture(1) + const accepted = yield* store.admit(lifecycle.admission) + assert.strictEqual(accepted.status, "Accepted") + assert.strictEqual((yield* store.admit(lifecycle.admission)).status, "Duplicate") + + const overQuota = PeerRelayStore.Admission.make({ + ...lifecycle.admission, + relayMessageId: relayMessageId("000000000002"), + senderSequence: 1, + messageHash: "message-hash-contract-quota", + outerEnvelopeDigest: PeerRpc.RelayDigest.make("f".repeat(64)) + }) + const quotaFailure = yield* Effect.flip(store.admit(overQuota)) + assert.strictEqual(quotaFailure.reason._tag, "QuotaExceeded") + assert.deepStrictEqual(yield* store.usage(), { + activeCount: 1, + activeBytes: 1, + retainedCount: 1, + retainedBytes: 1 + }) + + const firstClaim = claimedMessage(yield* store.claim(lifecycle.claim(1))) + assert.deepStrictEqual( + yield* store.loadClaimedPayload(PeerRelayStore.LoadClaimedPayloadRequest.make({ + rowId: firstClaim.rowId, + channel: firstClaim.channel, + relayMessageId: firstClaim.relayMessageId, + claimToken: firstClaim.claimToken, + sessionGeneration: firstClaim.sessionGeneration, + payloadBytes: firstClaim.payloadBytes + })), + lifecycle.admission.payload + ) + const firstTerminal = terminalRequest(firstClaim) + const released = yield* store.release(PeerRelayStore.ReleaseRequest.make({ + channel: firstClaim.channel, + relayMessageId: firstClaim.relayMessageId, + claimToken: firstClaim.claimToken, + sessionGeneration: firstClaim.sessionGeneration + })) + assert.strictEqual(released.status, "Changed") + assert.strictEqual(released.lane, "Retry") + yield* sql`UPDATE effect_local_relay_messages + SET next_eligible_at = 0 + WHERE message_id = ${firstClaim.rowId}` + + const retryResult = yield* store.claim(lifecycle.claim(2)) + assert.strictEqual(retryResult.lane, "Retry") + const retryClaim = claimedMessage(retryResult) + assert.strictEqual((yield* store.acknowledge(firstTerminal)).status, "Stale") + const retryTerminal = terminalRequest(retryClaim) + assert.strictEqual((yield* store.acknowledge(retryTerminal)).status, "Changed") + assert.strictEqual((yield* store.acknowledge(retryTerminal)).status, "Duplicate") + + const reopened = yield* SqlPeerRelayStore.make + assert.deepStrictEqual(yield* reopened.usage(), { + activeCount: 0, + activeBytes: 0, + retainedCount: 1, + retainedBytes: 1 + }) + + const rejectedFixture = makeFixture(3) + yield* store.admit(rejectedFixture.admission) + const rejectedClaim = claimedMessage(yield* store.claim(rejectedFixture.claim(1))) + const rejection = PeerRelayStore.RejectRequest.make({ + ...terminalRequest(rejectedClaim), + reason: "ApplicationRejected" + }) + assert.strictEqual((yield* store.reject(rejection)).status, "Changed") + assert.strictEqual((yield* store.reject(rejection)).status, "Duplicate") + const reopenedAfterReject = yield* SqlPeerRelayStore.make + assert.deepStrictEqual(yield* reopenedAfterReject.usage(), { + activeCount: 0, + activeBytes: 0, + retainedCount: 2, + retainedBytes: 2 + }) + + const recoveredFixture = makeFixture(4) + yield* store.admit(recoveredFixture.admission) + const abandoned = claimedMessage(yield* store.claim(recoveredFixture.claim(1))) + yield* sql`UPDATE effect_local_relay_messages + SET claim_deadline = 0 + WHERE message_id = ${abandoned.rowId}` + yield* sql`UPDATE effect_local_relay_channels + SET claim_deadline = 0 + WHERE claimed_message_id = ${abandoned.rowId}` + assert.strictEqual((yield* store.recover({ batchSize: 1 })).processed, 1) + yield* sql`UPDATE effect_local_relay_messages + SET next_eligible_at = 0 + WHERE message_id = ${abandoned.rowId}` + const recoveredResult = yield* store.claim(recoveredFixture.claim(2)) + assert.strictEqual(recoveredResult.lane, "Retry") + const recoveredClaim = claimedMessage(recoveredResult) + assert.strictEqual((yield* store.acknowledge(terminalRequest(recoveredClaim))).status, "Changed") + + const expiringFixture = makeFixture(5) + yield* store.admit(expiringFixture.admission) + const expiringClaim = claimedMessage(yield* store.claim(expiringFixture.claim(1))) + yield* sql`UPDATE effect_local_relay_messages + SET expires_at = 0 + WHERE message_id = ${expiringClaim.rowId}` + assert.strictEqual((yield* store.expire({ batchSize: 1 })).processed, 1) + assert.strictEqual((yield* store.acknowledge(terminalRequest(expiringClaim))).status, "Stale") + + yield* sql`UPDATE effect_local_relay_messages + SET deduplicate_until = 0 + WHERE message_id = ${rejectedClaim.rowId}` + assert.strictEqual((yield* store.collect({ batchSize: 1 })).processed, 1) + assert.strictEqual( + Exit.isFailure( + yield* Effect.exit(store.loadClaimedPayload({ + rowId: rejectedClaim.rowId, + channel: rejectedClaim.channel, + relayMessageId: rejectedClaim.relayMessageId, + claimToken: rejectedClaim.claimToken, + sessionGeneration: rejectedClaim.sessionGeneration, + payloadBytes: rejectedClaim.payloadBytes + })) + ), + true + ) + + const corruptFixture = makeFixture(6) + yield* store.admit(corruptFixture.admission) + yield* sql`UPDATE effect_local_relay_messages + SET tenant_id = ${"corrupt-contract-tenant"} + WHERE relay_message_id = ${corruptFixture.admission.relayMessageId}` + const repaired = yield* store.repair({ batchSize: 100 }) + assert.strictEqual(repaired.processed > 0, true) + assert.strictEqual( + Option.isNone((yield* store.claim(corruptFixture.claim(1))).message), + true + ) + + const reconciled = yield* store.reconcile({ batchSize: 1 }) + assert.strictEqual(reconciled.processed, 1) + assert.strictEqual(reconciled.hasMore, true) +}) diff --git a/packages/local-rpc/test/PeerRpc.test.ts b/packages/local-rpc/test/PeerRpc.test.ts index b42d07c..9e87984 100644 --- a/packages/local-rpc/test/PeerRpc.test.ts +++ b/packages/local-rpc/test/PeerRpc.test.ts @@ -1,284 +1,186 @@ import { assert, describe, it } from "@effect/vitest" import * as Identity from "@lucas-barake/effect-local/Identity" -import * as Deferred from "effect/Deferred" -import * as Effect from "effect/Effect" -import * as Option from "effect/Option" -import * as Queue from "effect/Queue" -import * as Redacted from "effect/Redacted" import * as Schema from "effect/Schema" -import * as Stream from "effect/Stream" -import * as RpcClient from "effect/unstable/rpc/RpcClient" -import type * as RpcMessage from "effect/unstable/rpc/RpcMessage" -import * as RpcServer from "effect/unstable/rpc/RpcServer" -import * as PeerAuthentication from "../src/PeerAuthentication.js" -import * as PeerAuthenticator from "../src/PeerAuthenticator.js" -import * as PeerCredentials from "../src/PeerCredentials.js" import * as PeerRpc from "../src/PeerRpc.js" -import * as PeerRpcError from "../src/PeerRpcError.js" -import * as PeerRpcLimits from "../src/PeerRpcLimits.js" -const documentId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000001") -const peerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000001") +const localPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000001") +const remotePeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000002") +const relayPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000003") const sessionId = Identity.SessionId.make("ses_00000000-0000-4000-8000-000000000001") +const relayMessageId = Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000001") +const documentId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000001") +const claimToken = PeerRpc.ClaimToken.make("clm_00000000-0000-4000-8000-000000000001") +const hash = "0".repeat(64) describe("PeerRpc", () => { - it.effect("executes the generated client through handlers and client authentication middleware", () => - Effect.scoped(Effect.gen(function*() { - const openRequest = yield* Deferred.make() - const pushRequest = yield* Deferred.make() - const disconnects = yield* Queue.unbounded() - const clients = new Set() - let sendToServer: (clientId: number, request: RpcMessage.FromClientEncoded) => Effect.Effect = () => - Effect.void - let sendToClient: (clientId: number, response: RpcMessage.FromServerEncoded) => Effect.Effect = () => - Effect.void - const serverProtocol = yield* RpcServer.Protocol.make((writeRequest) => - Effect.sync(() => { - sendToServer = writeRequest - return { - disconnects, - send: (clientId, response) => sendToClient(clientId, response), - end: () => Effect.void, - clientIds: Effect.succeed(clients), - initialMessage: Effect.succeed(Option.none()), - supportsAck: true, - supportsTransferables: false, - supportsSpanPropagation: true - } - }) - ) - const clientProtocol = yield* RpcClient.Protocol.make((writeResponse, clientIds) => - Effect.sync(() => { - clients.clear() - for (const clientId of clientIds) clients.add(clientId) - sendToClient = writeResponse - return { - send: (clientId, request) => sendToServer(clientId, request), - supportsAck: true, - supportsTransferables: false - } - }) - ) - const authenticated = { - principal: PeerAuthentication.PeerPrincipal.make({ - tenantId: "tenant", - subjectId: "subject", - peerId - }), - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.void + it("roundtrips the strict version 1 handshake", () => { + const opened = PeerRpc.Opened.make({ + _tag: "Opened", + protocolVersion: PeerRpc.protocolVersion, + sessionId, + remotePeerId, + authenticatedLocal: { + tenantId: "tenant", + subjectId: "subject", + peerId: localPeerId } - const handlers = PeerRpc.Rpcs.toLayer(PeerRpc.Rpcs.of({ - Open: (request) => - Stream.fromEffect( - Deferred.succeed(openRequest, request).pipe( - Effect.as(PeerRpc.Opened.make({ - _tag: "Opened", - protocolVersion: PeerRpc.protocolVersion, - sessionId, - peerId, - capabilities: { storeAndForward: false } - })) - ) - ), - Push: (request) => Deferred.succeed(pushRequest, request).pipe(Effect.asVoid) - })) - yield* RpcServer.make(PeerRpc.Rpcs).pipe( - Effect.provideService(RpcServer.Protocol, serverProtocol), - Effect.provide(handlers), - Effect.provide(PeerAuthentication.layerServer), - Effect.provideService(PeerAuthenticator.PeerAuthenticator, { - authenticate: (credential) => { - assert.strictEqual(Redacted.value(credential), "secret") - return Effect.succeed(authenticated) - } - }), - Effect.provideService(PeerRpcLimits.PeerRpcLimits, PeerRpcLimits.defaults), - Effect.forkScoped - ) - const client = yield* PeerRpc.makeRpcClient.pipe( - Effect.provideService(RpcClient.Protocol, clientProtocol), - Effect.provide(PeerAuthentication.layerClient), - Effect.provideService(PeerCredentials.PeerCredentials, { - get: Effect.succeed(Redacted.make("secret")) - }) - ) - const events = yield* client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: peerId, - definitionHash: "def_0000000000000000", - documents: [{ documentType: "Task", documentId }] - }).pipe(Stream.runCollect) - yield* client.Push({ sessionId, payload: Uint8Array.of(4, 5, 6) }) - assert.deepStrictEqual(events, [PeerRpc.Opened.make({ - _tag: "Opened", - protocolVersion: PeerRpc.protocolVersion, - sessionId, - peerId, - capabilities: { storeAndForward: false } - })]) - assert.strictEqual(Redacted.value((yield* Deferred.await(openRequest)).credential!), "secret") - assert.strictEqual(Redacted.value((yield* Deferred.await(pushRequest)).credential!), "secret") - }))) + }) - it("roundtrips every current-version request and response event", () => { + assert.deepStrictEqual(Schema.decodeUnknownSync(PeerRpc.OpenEvent)(opened), opened) + }) + + it("roundtrips every relay request and stored message field", () => { const open = PeerRpc.OpenRpc.payloadSchema.make({ protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: peerId, - definitionHash: "def_0000000000000000", - capabilities: { lineageAware: true }, - documents: [{ documentType: "Task", documentId }] + expectedRelayPeerId: relayPeerId, + expectedLocal: { + tenantId: "tenant", + subjectId: "subject", + peerId: localPeerId + }, + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + remote: { + subjectId: "remote", + peerId: remotePeerId + }, + documents: [{ documentType: "Task", documentId }], + receiptRetentionMillis: 8 * 24 * 60 * 60 * 1_000, + senderRetryHorizonMillis: 7 * 24 * 60 * 60 * 1_000 }) - const opened = PeerRpc.Opened.make({ - _tag: "Opened", - protocolVersion: PeerRpc.protocolVersion, + const stored = PeerRpc.StoredMessage.make({ + _tag: "StoredMessage", + relayMessageId, + claimToken, + relayPeerId, + sender: { + tenantId: "tenant", + subjectId: "subject", + peerId: localPeerId, + replicaIncarnation: Identity.ReplicaIncarnation.make(1), + connectionEpoch: "epoch", + sequence: 0 + }, + recipient: { + tenantId: "tenant", + subjectId: "remote", + peerId: remotePeerId + }, + payloadVersion: 1, + document: { documentType: "Task", documentId }, + writerProvenance: [], + messageHash: hash, + outerEnvelopeDigest: hash, + payload: Uint8Array.of(1, 2, 3) + }) + const push = PeerRpc.PushRpc.payloadSchema.make({ + sessionId, + relayMessageId, + payload: Uint8Array.of(1, 2, 3) + }) + const acknowledge = PeerRpc.AcknowledgeRpc.payloadSchema.make({ sessionId, - peerId, - capabilities: { storeAndForward: false } + relayMessageId, + claimToken, + messageHash: hash + }) + const reject = PeerRpc.RejectRpc.payloadSchema.make({ + ...acknowledge, + reason: "ProtocolInvalid" }) - const message = PeerRpc.Message.make({ _tag: "Message", payload: Uint8Array.of(1, 2, 3) }) - const push = PeerRpc.PushRpc.payloadSchema.make({ sessionId, payload: Uint8Array.of(4, 5, 6) }) assert.deepStrictEqual(Schema.decodeUnknownSync(PeerRpc.OpenRpc.payloadSchema)(open), open) - assert.deepStrictEqual(Schema.decodeUnknownSync(PeerRpc.OpenEvent)(opened), opened) - assert.deepStrictEqual(Schema.decodeUnknownSync(PeerRpc.OpenEvent)(message), message) + assert.deepStrictEqual(Schema.decodeUnknownSync(PeerRpc.OpenEvent)(stored), stored) assert.deepStrictEqual(Schema.decodeUnknownSync(PeerRpc.PushRpc.payloadSchema)(push), push) - assert.strictEqual(PeerRpc.OpenRpc._tag, "Open") - assert.strictEqual(PeerRpc.PushRpc._tag, "Push") + assert.deepStrictEqual( + Schema.decodeUnknownSync(PeerRpc.AcknowledgeRpc.payloadSchema)(acknowledge), + acknowledge + ) + assert.deepStrictEqual(Schema.decodeUnknownSync(PeerRpc.RejectRpc.payloadSchema)(reject), reject) }) - it("roundtrips every tagged wire error with its exact tag", () => { - const errors = [ - new PeerRpcError.AuthenticationFailure(), - new PeerRpcError.AccessDenied(), - new PeerRpcError.UnsupportedVersion(), - new PeerRpcError.PeerMismatch(), - new PeerRpcError.DefinitionMismatch(), - new PeerRpcError.InvalidRequest(), - new PeerRpcError.RequestLimitExceeded(), - new PeerRpcError.RequestCapacityExceeded(), - new PeerRpcError.SessionUnavailable(), - new PeerRpcError.SessionOverloaded(), - new PeerRpcError.ServerUnavailable(), - new PeerRpcError.DocumentLineageChanged() - ] - - for (const error of errors) { - const encoded = Schema.encodeSync(PeerRpcError.PeerRpcError)(error) - assert.deepStrictEqual(encoded, { _tag: error._tag }) - assert.strictEqual(Schema.decodeUnknownSync(PeerRpcError.PeerRpcError)(encoded)._tag, error._tag) + it("rejects invalid durations, empty document sets, identifiers, hashes, and oversized payloads", () => { + const baseOpen = { + protocolVersion: PeerRpc.protocolVersion, + expectedRelayPeerId: relayPeerId, + expectedLocal: { + tenantId: "tenant", + subjectId: "subject", + peerId: localPeerId + }, + senderReplicaIncarnation: 1, + remote: { + subjectId: "remote", + peerId: remotePeerId + }, + documents: [{ documentType: "Task", documentId }], + receiptRetentionMillis: 1, + senderRetryHorizonMillis: 1 } - }) - - it("keeps the lineage rewrite wire error free of document identity", () => { - const encoded = Schema.encodeSync(PeerRpcError.PeerRpcError)(new PeerRpcError.DocumentLineageChanged()) - assert.deepStrictEqual(encoded, { _tag: "DocumentLineageChanged" }) - }) - it("redacts defects to one fixed sentinel", () => { - assert.deepStrictEqual(Schema.encodeSync(PeerRpcError.Defect)(new Error("secret")), { _tag: "InternalError" }) - assert.isUndefined(Schema.decodeUnknownSync(PeerRpcError.Defect)({ _tag: "InternalError", secret: "ignored" })) - }) - - it("rejects malformed identities and empty document types", () => { - assert.throws(() => - Schema.decodeUnknownSync(PeerRpc.OpenRpc.payloadSchema)({ - protocolVersion: 1, - expectedPeerId: "peer_invalid", - definitionHash: "def_0000000000000000", - documents: [{ documentType: "", documentId }] - }) - ) - }) - - it("rejects malformed definition hashes at the wire boundary", () => { - for ( - const definitionHash of [ - "", - "def_short", - "bad_0000000000000000", - "def_000000000000000G", - "def_000000000000000A", - "def_00000000000000000", - "x".repeat(1024 * 1024) - ] - ) { + for (const duration of [0, -1, Number.NaN, Number.POSITIVE_INFINITY]) { assert.throws(() => Schema.decodeUnknownSync(PeerRpc.OpenRpc.payloadSchema)({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: peerId, - definitionHash, - documents: [{ documentType: "Task", documentId }] + ...baseOpen, + receiptRetentionMillis: duration }) ) } - }) - - it("requires a definition hash for the current protocol version", () => { + assert.throws(() => Schema.decodeUnknownSync(PeerRpc.OpenRpc.payloadSchema)({ ...baseOpen, documents: [] })) assert.throws(() => - Schema.decodeUnknownSync(PeerRpc.OpenRpc.payloadSchema)({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: peerId, - documents: [{ documentType: "Task", documentId }] + Schema.decodeUnknownSync(PeerRpc.AcknowledgeRpc.payloadSchema)({ + sessionId, + relayMessageId: "rly_invalid", + claimToken, + messageHash: hash }) ) - }) - - it("decodes an Open from a client that predates lineage without a capability advertisement", () => { - // The whole reason the field is `optionalKey`. `protocolVersion` was not bumped for lineage, so - // a client on an older build sends exactly this payload and passes the version check. A required - // key would make its `Open` fail to decode outright; absent has to read as "not lineage aware" - // instead, which is what the server then refuses to send a rewritten document to. - const legacy = Schema.decodeUnknownSync(PeerRpc.OpenRpc.payloadSchema)({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: peerId, - definitionHash: "def_0000000000000000", - documents: [{ documentType: "Task", documentId }] - }) - assert.isUndefined(legacy.capabilities) - - // An advertisement that is present but says nothing about lineage decodes the same way, so the - // server reads the flag itself rather than the presence of the object carrying it. - const silent = Schema.decodeUnknownSync(PeerRpc.OpenRpc.payloadSchema)({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: peerId, - definitionHash: "def_0000000000000000", - documents: [{ documentType: "Task", documentId }], - capabilities: {} - }) - assert.isUndefined(silent.capabilities?.lineageAware) - - const advertised = Schema.decodeUnknownSync(PeerRpc.OpenRpc.payloadSchema)({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: peerId, - definitionHash: "def_0000000000000000", - documents: [{ documentType: "Task", documentId }], - capabilities: { lineageAware: true } - }) - assert.deepStrictEqual(advertised.capabilities, { lineageAware: true }) - }) - - it("keeps version one requests decodable for a typed version rejection", () => { - const legacy = Schema.decodeUnknownSync(PeerRpc.OpenRpc.payloadSchema)({ - protocolVersion: 1, - expectedPeerId: peerId, - documents: [{ documentType: "Task", documentId }] - }) - assert.strictEqual(legacy.protocolVersion, 1) - assert.notStrictEqual(PeerRpc.protocolVersion, 1) - - const VersionOneOpen = Schema.Struct({ - protocolVersion: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), - expectedPeerId: Identity.PeerId, - documents: Schema.Array(PeerRpc.RequestedDocument) - }) - const decodedByVersionOne = Schema.decodeUnknownSync(VersionOneOpen)({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: peerId, - definitionHash: "def_0000000000000000", - documents: [{ documentType: "Task", documentId }] - }) - assert.strictEqual(decodedByVersionOne.protocolVersion, PeerRpc.protocolVersion) - assert.notStrictEqual(decodedByVersionOne.protocolVersion, 1) + assert.throws(() => + Schema.decodeUnknownSync(PeerRpc.AcknowledgeRpc.payloadSchema)({ + sessionId, + relayMessageId, + claimToken, + messageHash: "not-a-hash" + }) + ) + assert.throws(() => + Schema.decodeUnknownSync(PeerRpc.PushRpc.payloadSchema)({ + sessionId, + relayMessageId, + payload: new Uint8Array(PeerRpc.maximumRelayPayloadBytes + 1) + }) + ) + assert.throws(() => + Schema.decodeUnknownSync(PeerRpc.OpenEvent)( + PeerRpc.StoredMessage.make({ + _tag: "StoredMessage", + relayMessageId, + claimToken, + relayPeerId, + sender: { + tenantId: "tenant", + subjectId: "subject", + peerId: localPeerId, + replicaIncarnation: Identity.ReplicaIncarnation.make(1), + connectionEpoch: "epoch", + sequence: 0 + }, + recipient: { + tenantId: "tenant", + subjectId: "remote", + peerId: remotePeerId + }, + payloadVersion: 1, + document: { documentType: "Task", documentId }, + writerProvenance: [{ + changeHash: hash, + writerSchemaVersion: 1, + writerDefinitionHash: "invalid\"definition" + }], + messageHash: hash, + outerEnvelopeDigest: hash, + payload: Uint8Array.of(1) + }) + ) + ) }) }) diff --git a/packages/local-rpc/test/PeerRpcIntegration.test.ts b/packages/local-rpc/test/PeerRpcIntegration.test.ts new file mode 100644 index 0000000..3c5b0c8 --- /dev/null +++ b/packages/local-rpc/test/PeerRpcIntegration.test.ts @@ -0,0 +1,296 @@ +import { NodeCrypto, NodeFileSystem } from "@effect/platform-node" +import { SqliteClient } from "@effect/sql-sqlite-node" +import { assert, describe, it } from "@effect/vitest" +import * as PeerSyncEnvelope from "@lucas-barake/effect-local-sql/PeerSyncEnvelope" +import * as Canonical from "@lucas-barake/effect-local/Canonical" +import * as Document from "@lucas-barake/effect-local/Document" +import * as Identity from "@lucas-barake/effect-local/Identity" +import * as Context from "effect/Context" +import * as Crypto from "effect/Crypto" +import * as Effect from "effect/Effect" +import * as Fiber from "effect/Fiber" +import * as FileSystem from "effect/FileSystem" +import * as Layer from "effect/Layer" +import * as Queue from "effect/Queue" +import * as Redacted from "effect/Redacted" +import * as Ref from "effect/Ref" +import * as Schema from "effect/Schema" +import * as Stream from "effect/Stream" +import * as Reactivity from "effect/unstable/reactivity/Reactivity" +import * as RpcTest from "effect/unstable/rpc/RpcTest" +import * as SqlClient from "effect/unstable/sql/SqlClient" +import * as PeerAuthentication from "../src/PeerAuthentication.js" +import * as PeerAuthenticator from "../src/PeerAuthenticator.js" +import * as PeerCredentials from "../src/PeerCredentials.js" +import * as PeerRelayAuthorization from "../src/PeerRelayAuthorization.js" +import * as PeerRelayIngress from "../src/PeerRelayIngress.js" +import * as PeerRelayLimits from "../src/PeerRelayLimits.js" +import * as PeerRelayStore from "../src/PeerRelayStore.js" +import * as PeerRpc from "../src/PeerRpc.js" +import * as PeerRpcServer from "../src/PeerRpcServer.js" +import * as SqlPeerRelayStore from "../src/SqlPeerRelayStore.js" + +const Task = Document.make("Task", { + schema: Schema.Struct({ title: Schema.String }), + version: 1 +}) +const documentId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000001") +const relayPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000001") +const senderPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000002") +const recipientPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000003") +const relayMessageId = Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000001") + +const sender = PeerAuthentication.PeerPrincipal.make({ + tenantId: "tenant", + subjectId: "sender", + peerId: senderPeerId +}) +const recipient = PeerAuthentication.PeerPrincipal.make({ + tenantId: "tenant", + subjectId: "recipient", + peerId: recipientPeerId +}) + +const openRequest = ( + principal: PeerAuthentication.PeerPrincipal, + remote: PeerAuthentication.PeerPrincipal +) => + PeerRpc.OpenRpc.payloadSchema.make({ + protocolVersion: PeerRpc.protocolVersion, + expectedRelayPeerId: relayPeerId, + expectedLocal: principal, + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + remote: { + subjectId: remote.subjectId, + peerId: remote.peerId + }, + documents: [{ documentType: Task.name, documentId }], + receiptRetentionMillis: PeerRelayLimits.defaults.maximumReceiptRetentionMillis, + senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis + }) + +const authorize: PeerRelayAuthorization.Authorize = (request) => + Effect.succeed({ + remote: { + tenantId: request.principal.tenantId, + subjectId: request.remote.subjectId, + peerId: request.remote.peerId + }, + documents: request.documents.map((requested) => ({ + document: Task, + documentId: requested.documentId + })), + validUntil: Number.MAX_SAFE_INTEGER, + invalidated: Effect.never + }) + +const authorizeUnsafe: PeerRelayAuthorization.AuthorizeUnsafeUnboundedAutomerge3Decode = ( + request +) => + Effect.succeed({ + _tag: "UnsafeUnboundedAutomerge3DecodeGrant", + risk: PeerRelayAuthorization.unsafeUnboundedAutomerge3DecodeRisk, + principal: request.principal, + remote: { + tenantId: request.principal.tenantId, + subjectId: request.remote.subjectId, + peerId: request.remote.peerId + }, + direction: request.direction, + documents: request.documents, + validUntil: Number.MAX_SAFE_INTEGER, + invalidated: Effect.never + }) + +const ingress = PeerRelayIngress.PeerRelayIngress.of({ + address: { _tag: "UnixAddress", path: "relay-rpc-integration" }, + reserveOutbound: (bytes) => + Effect.succeed({ + bytes, + release: Effect.void, + transferToCurrentRequest: Effect.void + }), + usage: Effect.succeed({ + connections: 0, + reservedBytes: 0, + byteReservationWaiters: 0 + }), + await: Effect.never +}) + +const RelaySyncEnvelopeJson = Schema.fromJsonString( + Schema.toCodecJson(PeerSyncEnvelope.SyncEnvelope) +) + +describe("PeerRpc production composition", () => { + it.effect("round trips durable custody and owns session cleanup through RpcTest", () => + Effect.scoped(Effect.gen(function*() { + const limits = PeerRelayLimits.defaults + const crypto = yield* Crypto.Crypto.pipe(Effect.provide(NodeCrypto.layer)) + const fs = yield* FileSystem.FileSystem.pipe(Effect.provide(NodeFileSystem.layer)) + const directory = yield* fs.makeTempDirectoryScoped() + const sql = yield* SqliteClient.make({ filename: `${directory}/relay.sqlite` }).pipe( + Effect.provide(Reactivity.layer) + ) + const store = yield* SqlPeerRelayStore.make.pipe( + Effect.provideService(SqlClient.SqlClient, sql), + Effect.provideService(Crypto.Crypto, crypto), + Effect.provideService(PeerRelayLimits.PeerRelayLimits, limits) + ) + const authorization = yield* PeerRelayAuthorization.PeerRelayAuthorization.pipe( + Effect.provide(PeerRelayAuthorization.layer(authorize, authorizeUnsafe)) + ) + const handlers = yield* Layer.build( + PeerRpcServer.layerHandlers({ + tenantId: "tenant", + peerId: relayPeerId + }).pipe( + Layer.provide(Layer.mergeAll( + Layer.succeed(Crypto.Crypto, crypto), + Layer.succeed(PeerRelayLimits.PeerRelayLimits, limits), + Layer.succeed(PeerRelayAuthorization.PeerRelayAuthorization, authorization), + Layer.succeed(PeerRelayStore.PeerRelayStore, store), + Layer.succeed(PeerRelayIngress.PeerRelayIngress, ingress) + )) + ) + ) + const runtime = Context.get(handlers, PeerRpcServer.PeerRpcServerRuntime) + const credential = yield* Ref.make("sender") + const principals = new Map([ + ["sender", sender], + ["recipient", recipient] + ]) + const authentication = yield* PeerAuthentication.PeerAuthentication.pipe( + Effect.provide(PeerAuthentication.layerServer), + Effect.provideService(PeerAuthenticator.PeerAuthenticator, { + authenticate: (secret) => { + const principal = principals.get(Redacted.value(secret)) + return principal === undefined + ? Effect.die("Unknown integration credential") + : Effect.succeed({ + principal, + validUntil: Number.MAX_SAFE_INTEGER, + invalidated: Effect.never + }) + } + }), + Effect.provideService(PeerRelayLimits.PeerRelayLimits, limits) + ) + const client = yield* RpcTest.makeClient(PeerRpc.Rpcs).pipe( + Effect.provideContext( + Context.add( + handlers, + PeerAuthentication.PeerAuthentication, + authentication + ) + ), + Effect.provide(PeerAuthentication.layerClient), + Effect.provideService(PeerCredentials.PeerCredentials, { + get: Ref.get(credential).pipe(Effect.map(Redacted.make)) + }) + ) + + const invalidVersion = yield* client.Open({ + ...openRequest(sender, recipient), + protocolVersion: 2 + }).pipe(Stream.runDrain, Effect.flip) + assert.strictEqual(invalidVersion._tag, "UnsupportedVersion") + assert.strictEqual((yield* runtime.usage).sessions, 0) + + assert.throws(() => + Schema.decodeUnknownSync(PeerRpc.OpenEvent)({ + _tag: "Opened", + protocolVersion: 2, + sessionId: "ses_00000000-0000-4000-8000-000000000001", + remotePeerId: recipientPeerId, + authenticatedLocal: sender + }) + ) + assert.strictEqual((yield* runtime.usage).sessions, 0) + + const senderEvents = yield* Queue.unbounded() + yield* Ref.set(credential, "sender") + const senderOpen = yield* client.Open(openRequest(sender, recipient)).pipe( + Stream.runForEach((event) => Queue.offer(senderEvents, event)), + Effect.forkScoped + ) + const senderOpened = yield* Queue.take(senderEvents) + assert.strictEqual(senderOpened._tag, "Opened") + if (senderOpened._tag !== "Opened") return assert.fail("Expected sender Opened event") + + const recipientEvents = yield* Queue.unbounded() + yield* Ref.set(credential, "recipient") + const recipientOpen = yield* client.Open(openRequest(recipient, sender)).pipe( + Stream.runForEach((event) => Queue.offer(recipientEvents, event)), + Effect.forkScoped + ) + const recipientOpened = yield* Queue.take(recipientEvents) + assert.strictEqual(recipientOpened._tag, "Opened") + if (recipientOpened._tag !== "Opened") return assert.fail("Expected recipient Opened event") + + const message = Uint8Array.of(1, 2, 3) + const messageHash = yield* Canonical.digest(message).pipe( + Effect.provideService(Crypto.Crypto, crypto) + ) + const payload = new TextEncoder().encode( + yield* Schema.encodeEffect(RelaySyncEnvelopeJson)({ + connectionEpoch: "epoch", + sequence: 0, + documentId, + documentType: Task.name, + messageHash, + message, + lineage: Identity.genesisLineage, + writerProvenance: [] + }) + ) + + yield* Ref.set(credential, "sender") + yield* client.Push({ + sessionId: senderOpened.sessionId, + relayMessageId, + payload + }) + + const stored = yield* Queue.take(recipientEvents) + assert.strictEqual(stored._tag, "StoredMessage") + if (stored._tag !== "StoredMessage") return assert.fail("Expected durable delivery") + assert.deepStrictEqual(stored.payload, payload) + assert.strictEqual((yield* store.usage()).activeCount, 1) + + yield* Ref.set(credential, "recipient") + yield* client.Acknowledge({ + sessionId: recipientOpened.sessionId, + relayMessageId: stored.relayMessageId, + claimToken: stored.claimToken, + messageHash: stored.messageHash + }) + const afterAcknowledge = yield* store.usage() + assert.strictEqual(afterAcknowledge.activeCount, 0) + assert.strictEqual(afterAcknowledge.retainedCount, 1) + + yield* Fiber.interrupt(senderOpen) + yield* Fiber.interrupt(recipientOpen) + assert.strictEqual((yield* runtime.usage).sessions, 0) + + const incumbentEvents = yield* Queue.unbounded() + yield* Ref.set(credential, "sender") + const incumbent = yield* client.Open(openRequest(sender, recipient)).pipe( + Stream.runForEach((event) => Queue.offer(incumbentEvents, event)), + Effect.forkScoped + ) + assert.strictEqual((yield* Queue.take(incumbentEvents))._tag, "Opened") + + const replacementEvents = yield* Queue.unbounded() + const replacement = yield* client.Open(openRequest(sender, recipient)).pipe( + Stream.runForEach((event) => Queue.offer(replacementEvents, event)), + Effect.forkScoped + ) + assert.strictEqual((yield* Queue.take(replacementEvents))._tag, "Opened") + yield* Fiber.await(incumbent) + assert.strictEqual((yield* runtime.usage).sessions, 1) + + yield* Fiber.interrupt(replacement) + assert.strictEqual((yield* runtime.usage).sessions, 0) + }))) +}) diff --git a/packages/local-rpc/test/PeerRpcLimits.test.ts b/packages/local-rpc/test/PeerRpcLimits.test.ts deleted file mode 100644 index 504ee05..0000000 --- a/packages/local-rpc/test/PeerRpcLimits.test.ts +++ /dev/null @@ -1,169 +0,0 @@ -import { assert, describe, it } from "@effect/vitest" -import * as PeerSession from "@lucas-barake/effect-local-sql/PeerSession" -import * as ReplicaLimits from "@lucas-barake/effect-local/ReplicaLimits" -import * as Effect from "effect/Effect" -import * as PeerRpcLimits from "../src/PeerRpcLimits.js" - -const replicaLimits = ReplicaLimits.Values.make({ - maxBackupBytes: 1, - maxChunkBytes: 1, - maxArchiveRecords: 1, - maxJsonDepth: 1, - maxSyncMessageBytes: 1_024, - maxPeerSendMillis: 1, - maxSyncChangesPerMessage: 1, - maxSyncDependencyEdgesPerMessage: 1, - maxSyncOperationsPerMessage: 1, - maxPendingBytesPerDocument: 1, - maxPendingBytesPerPeer: 1, - maxPendingBytesPerReplica: 1, - maxPendingAgeMillis: 1, - maxPendingChangesPerDocument: 1, - maxPendingChangesPerPeer: 1, - maxPendingChangesPerReplica: 1, - maxPendingDependencyEdgesPerDocument: 1, - maxPendingDependencyEdgesPerPeer: 1, - maxPendingDependencyEdgesPerReplica: 1, - maxSessions: 1, - maxStreamsPerSession: 1, - maxInFlightPerSession: 1, - maxQueuedRpc: 1, - maxQueuedPermits: 1, - maxActiveRestores: 1, - maxRestoresPerSession: 1, - maxRestoreMillis: 30_000, - maxRestorePullMillis: 10_000, - maxRestoreCoalesceMillis: 25, - maxRestoreErrorBytes: 4_096 -}) - -describe("PeerRpcLimits", () => { - it.effect("publishes every conservative default exactly", () => - Effect.gen(function*() { - const limits = yield* PeerRpcLimits.make(PeerRpcLimits.defaults).pipe( - Effect.provideService(ReplicaLimits.ReplicaLimits, replicaLimits) - ) - assert.deepStrictEqual(limits, { - maxSessionsPerSubject: 4, - inboundItemCapacity: 1, - outboundItemCapacity: 1, - maxInboundBufferedBytesPerSession: 4 * 1_024 * 1_024, - maxOutboundBufferedBytesPerSession: 4 * 1_024 * 1_024, - maxBufferedBytes: 64 * 1_024 * 1_024, - maxInFlightAuthentication: 64, - authenticationRatePerSecond: 16, - authenticationBurst: 32, - maxInFlightOpen: 16, - maxInFlightOpenPerSubject: 2, - maxInFlightPush: 128, - maxInFlightPushPerSubject: 8, - openRatePerSecond: 2, - openBurst: 4, - pushRatePerSecond: 64, - pushBurst: 128, - maxRetainedRateLimitedConnections: 10_000, - maxRetainedRateLimitedSubjects: 10_000, - rateLimitIdleRetention: 10 * 60_000, - maximumReauthorizationInterval: 5 * 60_000, - commitFlushConcurrency: 8, - shutdownCleanupConcurrency: 16 - }) - })) - - it.effect("provides equivalent defaults through layerDefaults", () => - Effect.gen(function*() { - const limits = yield* PeerRpcLimits.PeerRpcLimits - assert.deepStrictEqual(limits, PeerRpcLimits.defaults) - }).pipe( - Effect.provide(PeerRpcLimits.layerDefaults), - Effect.provideService(ReplicaLimits.ReplicaLimits, replicaLimits) - )) - - it.effect("preserves native SchemaError trees and messages for invalid scalar limits", () => - Effect.gen(function*() { - const invalid = [ - [ - "maxSessionsPerSubject", - "Expected a value greater than 0, got 0", - { ...PeerRpcLimits.defaults, maxSessionsPerSubject: 0 } - ], - [ - "maxSessionsPerSubject", - "Expected a value greater than 0, got -1", - { ...PeerRpcLimits.defaults, maxSessionsPerSubject: -1 } - ], - [ - "authenticationRatePerSecond", - "Expected a finite number, got Infinity", - { ...PeerRpcLimits.defaults, authenticationRatePerSecond: Number.POSITIVE_INFINITY } - ], - [ - "inboundItemCapacity", - "Expected an integer, got 1.5", - { ...PeerRpcLimits.defaults, inboundItemCapacity: 1.5 } - ], - [ - "shutdownCleanupConcurrency", - "Expected a value greater than 0, got 0", - { ...PeerRpcLimits.defaults, shutdownCleanupConcurrency: 0 } - ] - ] as const - for (const [field, expected, values] of invalid) { - const error = yield* PeerRpcLimits.make(values).pipe( - Effect.provideService(ReplicaLimits.ReplicaLimits, replicaLimits), - Effect.flip - ) - assert.strictEqual(error._tag, "SchemaError") - if (error._tag !== "SchemaError") continue - assert.strictEqual(error.issue._tag, "Composite") - if (error.issue._tag !== "Composite") continue - assert.strictEqual(error.issue.issues[0]?._tag, "Pointer") - if (error.issue.issues[0]?._tag === "Pointer") { - assert.deepStrictEqual(error.issue.issues[0].path, [field]) - } - assert.strictEqual(error.message, `${expected}\n at ["${field}"]`) - } - })) - - it.effect("rejects incompatible byte budgets with the exact field", () => - Effect.gen(function*() { - const envelope = PeerSession.maximumSyncEnvelopeBytes( - replicaLimits.maxSyncMessageBytes, - replicaLimits.maxSyncChangesPerMessage - ) - assert.strictEqual(envelope, 6_656) - const cases = [ - ["maxInboundBufferedBytesPerSession", { - ...PeerRpcLimits.defaults, - maxInboundBufferedBytesPerSession: envelope - 1 - }], - ["maxOutboundBufferedBytesPerSession", { - ...PeerRpcLimits.defaults, - maxOutboundBufferedBytesPerSession: envelope - 1 - }], - ["inboundItemCapacity", { - ...PeerRpcLimits.defaults, - inboundItemCapacity: 2, - maxInboundBufferedBytesPerSession: envelope - }], - ["outboundItemCapacity", { - ...PeerRpcLimits.defaults, - outboundItemCapacity: 2, - maxOutboundBufferedBytesPerSession: envelope - }], - ["maxBufferedBytes", { - ...PeerRpcLimits.defaults, - maxBufferedBytes: envelope - 1 - }] - ] as const - - for (const [field, values] of cases) { - const error = yield* PeerRpcLimits.make(values).pipe( - Effect.provideService(ReplicaLimits.ReplicaLimits, replicaLimits), - Effect.flip - ) - assert.strictEqual(error._tag, "InvalidPeerRpcLimits") - if (error._tag === "InvalidPeerRpcLimits") assert.strictEqual(error.field, field) - } - })) -}) diff --git a/packages/local-rpc/test/PeerRpcServer.test.ts b/packages/local-rpc/test/PeerRpcServer.test.ts index b17608d..48f14ab 100644 --- a/packages/local-rpc/test/PeerRpcServer.test.ts +++ b/packages/local-rpc/test/PeerRpcServer.test.ts @@ -1,18 +1,9 @@ -import { assert, describe, it } from "@effect/vitest" -import * as CommitPublisher from "@lucas-barake/effect-local-sql/CommitPublisher" -import type * as DocumentEntity from "@lucas-barake/effect-local-sql/DocumentEntity" -import * as PeerSession from "@lucas-barake/effect-local-sql/PeerSession" -import * as PeerSync from "@lucas-barake/effect-local-sql/PeerSync" +import { assert, describe, it, vi } from "@effect/vitest" import * as PeerSyncEnvelope from "@lucas-barake/effect-local-sql/PeerSyncEnvelope" -import * as ReplicaGate from "@lucas-barake/effect-local-sql/ReplicaGate" import * as Canonical from "@lucas-barake/effect-local/Canonical" import * as Document from "@lucas-barake/effect-local/Document" -import * as DocumentSet from "@lucas-barake/effect-local/DocumentSet" import * as Identity from "@lucas-barake/effect-local/Identity" -import * as PeerTransport from "@lucas-barake/effect-local/PeerTransport" -import * as ReplicaDefinition from "@lucas-barake/effect-local/ReplicaDefinition" import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" -import * as ReplicaLimits from "@lucas-barake/effect-local/ReplicaLimits" import * as Cause from "effect/Cause" import * as Clock from "effect/Clock" import * as Context from "effect/Context" @@ -25,2741 +16,33 @@ import * as Layer from "effect/Layer" import * as Metric from "effect/Metric" import * as Option from "effect/Option" import * as Queue from "effect/Queue" -import * as Redacted from "effect/Redacted" import * as Ref from "effect/Ref" import * as Schema from "effect/Schema" -import * as Scope from "effect/Scope" import * as Stream from "effect/Stream" import * as TestClock from "effect/testing/TestClock" -import * as Tracer from "effect/Tracer" -import * as Sharding from "effect/unstable/cluster/Sharding" import type * as Rpc from "effect/unstable/rpc/Rpc" import * as RpcServer from "effect/unstable/rpc/RpcServer" -import * as RpcTest from "effect/unstable/rpc/RpcTest" import { createHash, randomBytes } from "node:crypto" import * as PeerRpcObservability from "../src/internal/peerRpcObservability.js" import * as PeerAuthentication from "../src/PeerAuthentication.js" -import * as PeerAuthenticator from "../src/PeerAuthenticator.js" -import * as PeerAuthorization from "../src/PeerAuthorization.js" -import * as PeerCredentials from "../src/PeerCredentials.js" import * as PeerRelayAuthorization from "../src/PeerRelayAuthorization.js" import * as PeerRelayIngress from "../src/PeerRelayIngress.js" import * as PeerRelayLimits from "../src/PeerRelayLimits.js" -import * as PeerRelayRpc from "../src/PeerRelayRpc.js" import * as PeerRelayStore from "../src/PeerRelayStore.js" import * as PeerRpc from "../src/PeerRpc.js" import * as PeerRpcError from "../src/PeerRpcError.js" -import * as PeerRpcLimits from "../src/PeerRpcLimits.js" import * as PeerRpcServer from "../src/PeerRpcServer.js" -import * as RpcPeerTransport from "../src/RpcPeerTransport.js" const Task = Document.make("Task", { schema: Schema.Struct({ title: Schema.String }), version: 1 }) const Note = Document.make("Note", { schema: Schema.Struct({ body: Schema.String }), version: 1 }) const taskId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000001") -const noteId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000002") const serverPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000001") -const definition = ReplicaDefinition.make({ - name: "peer-rpc-server-test", - documents: DocumentSet.make(Task, Note), - mutations: [], - projections: [], - queries: [] -}) -const definitionHash = definition.hash -const remotePeerA = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000002") -const remotePeerB = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000003") -const missingSessionId = Identity.SessionId.make("ses_00000000-0000-4000-8000-000000000001") -const permit = { - replicaId: Identity.ReplicaId.make("rep_00000000-0000-4000-8000-000000000001"), - incarnation: Identity.ReplicaIncarnation.make(1), - writerGeneration: Identity.WriterGeneration.make(1) -} -const replicaLimits = ReplicaLimits.Values.make({ - maxBackupBytes: 1_000_000, - maxChunkBytes: 64_000, - maxArchiveRecords: 1_000, - maxJsonDepth: 32, - maxSyncMessageBytes: 64_000, - maxPeerSendMillis: 1_000, - maxSyncChangesPerMessage: 100, - maxSyncDependencyEdgesPerMessage: 1_000, - maxSyncOperationsPerMessage: 1_000, - maxPendingBytesPerDocument: 1_000_000, - maxPendingBytesPerPeer: 1_000_000, - maxPendingBytesPerReplica: 2_000_000, - maxPendingAgeMillis: 60_000, - maxPendingChangesPerDocument: 1_000, - maxPendingChangesPerPeer: 1_000, - maxPendingChangesPerReplica: 2_000, - maxPendingDependencyEdgesPerDocument: 10_000, - maxPendingDependencyEdgesPerPeer: 10_000, - maxPendingDependencyEdgesPerReplica: 20_000, - maxSessions: 8, - maxStreamsPerSession: 4, - maxInFlightPerSession: 1, - maxQueuedRpc: 32, - maxQueuedPermits: 32, - maxActiveRestores: 32, - maxRestoresPerSession: 1, - maxRestoreMillis: 30_000, - maxRestorePullMillis: 10_000, - maxRestoreCoalesceMillis: 25, - maxRestoreErrorBytes: 4_096 -}) -const maximumWriterProvenance = Array.from( - { length: replicaLimits.maxSyncChangesPerMessage }, - (_, index) => ({ - changeHash: index.toString(16).padStart(64, "0"), - writerSchemaVersion: Number.MAX_SAFE_INTEGER, - writerDefinitionHash: "x".repeat(256) - }) -) -const rpcLimits = PeerRpcLimits.Values.make({ - ...PeerRpcLimits.defaults, - openRatePerSecond: 1_000, - openBurst: 1_000, - pushRatePerSecond: 1_000, - pushBurst: 1_000, - authenticationRatePerSecond: 1_000, - authenticationBurst: 1_000, - maximumReauthorizationInterval: 60_000 -}) -const crypto = Crypto.make({ - randomBytes: (size) => randomBytes(size), - digest: (algorithm, bytes) => - Effect.sync(() => new Uint8Array(createHash(algorithm.replace("-", "").toLowerCase()).update(bytes).digest())) -}) -const SyncEnvelopeJson = Schema.fromJsonString(Schema.toCodecJson(PeerSession.SyncEnvelope)) -const baseOptions = { - rpcLimits: {}, - replicaLimits: {}, - initialOutbound: null, - authorization: undefined, - authenticationValidUntil: Number.MAX_SAFE_INTEGER, - authorizationValidUntil: Number.MAX_SAFE_INTEGER, - blockInbound: false, - blockAuthorization: false, - failSessionOpen: false, - sessionOpenFailure: undefined, - manualClock: false -} - -const makeFixture = (options: { - readonly rpcLimits: Partial - readonly replicaLimits: Partial - readonly initialOutbound: PeerSync.Outbound | null - readonly authorization: - | ((request: { - readonly principal: PeerAuthentication.PeerPrincipal - readonly documents: ReadonlyArray - }) => Effect.Effect<{ - readonly documents: ReadonlyArray - readonly validUntil: number - readonly invalidated: Effect.Effect - }, PeerRpcError.AccessDenied | PeerRpcError.ServerUnavailable>) - | undefined - readonly authenticationValidUntil: number - readonly authorizationValidUntil: number - readonly blockInbound: boolean - readonly blockAuthorization: boolean - readonly failSessionOpen: boolean - readonly sessionOpenFailure: ReplicaError.Reason | undefined - readonly manualClock: boolean -}) => - Effect.gen(function*() { - const configuredReplicaLimits = ReplicaLimits.Values.make({ ...replicaLimits, ...options.replicaLimits }) - const configuredRpcLimits = PeerRpcLimits.Values.make({ ...rpcLimits, ...options.rpcLimits }) - const commits = yield* Queue.unbounded() - const generated = yield* Queue.unbounded() - const generatePeers = yield* Queue.unbounded<{ readonly lineageAware: boolean }>() - const received = yield* Queue.unbounded() - const receivedPayloads = yield* Queue.unbounded() - const sent = yield* Queue.unbounded() - const enqueued = yield* Queue.unbounded() - const pendingStarted = yield* Queue.unbounded() - const inboundRelease = yield* Deferred.make() - const inboundBlocked = yield* Deferred.make() - const authenticationInvalidated = yield* Deferred.make() - const commitProcessed = yield* Queue.unbounded() - const commitFlushStarted = yield* Deferred.make() - const commitFlushRelease = yield* Deferred.make() - const authorizationInvalidated = yield* Deferred.make() - const authorizationRelease = yield* Deferred.make() - const authorizationStarted = yield* Queue.unbounded() - const authorizationCalls = yield* Ref.make(0) - const sessionOpenCalls = yield* Ref.make(0) - const subscriptions = yield* Ref.make(0) - const publications = yield* Ref.make(0) - let activeFibers = 0 - let credential = "owner" - let initialOutbound = options.initialOutbound - let failSessionOpen = options.failSessionOpen - let reply: PeerSync.Reply | null = null - let blockCommitFlush = false - let currentTime = 0 - let currentTimeOnNextRandomBytes: number | undefined - const clock = { - currentTimeMillisUnsafe: () => currentTime, - currentTimeMillis: Effect.sync(() => currentTime), - currentTimeNanosUnsafe: () => BigInt(currentTime) * 1_000_000n, - currentTimeNanos: Effect.sync(() => BigInt(currentTime) * 1_000_000n), - sleep: () => Effect.never - } satisfies Clock.Clock - const fixtureCrypto = Crypto.make({ - randomBytes: (size) => { - if (currentTimeOnNextRandomBytes !== undefined) { - currentTime = currentTimeOnNextRandomBytes - currentTimeOnNextRandomBytes = undefined - } - return randomBytes(size) - }, - digest: crypto.digest - }) - const nextOutbound = yield* Ref.make(null) - const enqueuedOutbounds = yield* Ref.make(new Map>()) - const principals = new Map([ - [ - "owner", - PeerAuthentication.PeerPrincipal.make({ tenantId: "tenant", subjectId: "subject-a", peerId: remotePeerA }) - ], - [ - "same-subject", - PeerAuthentication.PeerPrincipal.make({ - tenantId: "tenant", - subjectId: "subject-a", - peerId: remotePeerB - }) - ], - [ - "same-peer", - PeerAuthentication.PeerPrincipal.make({ - tenantId: "tenant", - subjectId: "subject-b", - peerId: remotePeerA - }) - ], - [ - "foreign", - PeerAuthentication.PeerPrincipal.make({ - tenantId: "tenant", - subjectId: "subject-b", - peerId: remotePeerB - }) - ], - [ - "other-tenant", - PeerAuthentication.PeerPrincipal.make({ - tenantId: "other", - subjectId: "subject-c", - peerId: remotePeerB - }) - ], - [ - "third", - PeerAuthentication.PeerPrincipal.make({ - tenantId: "tenant", - subjectId: "subject-c", - peerId: Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000010") - }) - ] - ]) - const gate = ReplicaGate.ReplicaGate.of({ - current: Effect.succeed(permit), - claiming: Effect.succeed(false), - shared: Effect.acquireRelease(Effect.succeed(permit), () => Effect.void), - admit: Effect.acquireRelease(Effect.succeed(permit), () => Effect.void), - claim: (use) => use(permit), - refresh: Effect.succeed(permit), - validate: () => Effect.void - }) - const sync = PeerSync.PeerSync.of({ - withDocumentInvalidation: (_documentId, effect) => effect, - invalidateDocument: () => Effect.void, - open: (peerId) => - Ref.update(sessionOpenCalls, (count) => count + 1).pipe( - Effect.andThen( - failSessionOpen - ? Effect.fail( - new ReplicaError.ReplicaError({ - reason: options.sessionOpenFailure ?? - new ReplicaError.StorageUnavailable({ cause: new Error("session startup failed") }) - }) - ) - : Effect.succeed({ peerId, connectionEpoch: "local-epoch", replicaIncarnation: permit.incarnation }) - ) - ), - reset: () => Effect.void, - generate: (_document, documentId, _session, peer) => - Queue.offer(generated, documentId).pipe( - // What the server side session read off its own transport connection. This is the value - // that decides whether a rewritten document may be emitted toward the connected peer. - Effect.andThen(Queue.offer(generatePeers, peer)), - Effect.andThen( - blockCommitFlush - ? Deferred.succeed(commitFlushStarted, undefined).pipe( - Effect.andThen(Deferred.await(commitFlushRelease)) - ) - : Effect.void - ), - Effect.andThen(Ref.getAndSet(nextOutbound, null)), - Effect.map((outbound) => ({ outbound, observedByPeer: false, dirty: false })) - ), - receive: () => Effect.die("unexpected direct PeerSync receive"), - enqueue: (session, reply) => { - const outbound = { - ...reply, - sendSequence: 0, - lineage: Identity.genesisLineage, - writerProvenance: maximumWriterProvenance - } - return Queue.offer(enqueued, reply.documentId).pipe( - Effect.andThen(Ref.update(enqueuedOutbounds, (pendingByPeer) => { - const next = new Map(pendingByPeer) - next.set(session.peerId, [...(next.get(session.peerId) ?? []), outbound]) - return next - })), - Effect.as(outbound) - ) - }, - pending: (session) => - Queue.offer(pendingStarted, session.peerId).pipe( - Effect.andThen(Ref.modify(enqueuedOutbounds, (pendingByPeer) => { - const next = new Map(pendingByPeer) - const pending = next.get(session.peerId) ?? [] - next.delete(session.peerId) - const initial = initialOutbound === null ? [] : [initialOutbound] - initialOutbound = null - return [[...initial, ...pending], next] - })) - ), - markSent: (_session, sequence) => Queue.offer(sent, sequence).pipe(Effect.as(true)) - }) - const publisher = CommitPublisher.CommitPublisher.of({ - publishPending: Ref.updateAndGet(publications, (count) => count + 1), - invalidate: () => Effect.void, - subscribe: Ref.update(subscriptions, (count) => count + 1).pipe( - Effect.as({ - watermark: Identity.CommitSequence.make(0), - refreshGeneration: 0, - events: Stream.fromQueue(commits).pipe( - Stream.tap(() => Queue.offer(commitProcessed, undefined)) - ) - }) - ) - }) - const applyResult = { - reply: null, - heads: [], - acceptedHeads: [], - commitSequence: Identity.CommitSequence.make(1), - observedByPeer: true, - durableConfirmation: false as const, - duplicate: false - } - const sharding = Sharding.Sharding.of({ - ...({} as Sharding.Sharding["Service"]), - makeClient: () => - Effect.succeed(() => - ({ - ApplySync: (payload: typeof DocumentEntity.ApplySync.payloadSchema.Type) => - Queue.offer(receivedPayloads, payload).pipe( - Effect.andThen(Queue.offer(received, payload.receiveSequence)), - Effect.andThen( - options.blockInbound - ? Deferred.succeed(inboundBlocked, undefined).pipe(Effect.andThen(Deferred.await(inboundRelease))) - : Effect.void - ), - Effect.as({ ...applyResult, reply }) - ) - }) as never - ) - }) - const authorization = PeerAuthorization.PeerAuthorization.of({ - authorize: options.authorization ?? ((request) => - Ref.update(authorizationCalls, (count) => count + 1).pipe( - Effect.andThen(Queue.offer(authorizationStarted, request.principal.subjectId)), - Effect.andThen(options.blockAuthorization ? Deferred.await(authorizationRelease) : Effect.void), - Effect.as({ - documents: request.documents.map((requested) => ({ - document: requested.documentType === Task.name ? Task : Note, - documentId: requested.documentId - })), - validUntil: options.authorizationValidUntil, - invalidated: Deferred.await(authorizationInvalidated) - }) - )) - }) - const services = Layer.mergeAll( - Layer.succeed(Crypto.Crypto, fixtureCrypto), - Layer.succeed(CommitPublisher.CommitPublisher, publisher), - Layer.succeed(PeerSync.PeerSync, sync), - Layer.succeed(ReplicaGate.ReplicaGate, gate), - Layer.succeed(ReplicaLimits.ReplicaLimits, configuredReplicaLimits), - Layer.succeed(Sharding.Sharding, sharding), - Layer.succeed(PeerRpcLimits.PeerRpcLimits, configuredRpcLimits), - Layer.succeed(PeerAuthorization.PeerAuthorization, authorization), - options.manualClock ? Layer.succeed(Clock.Clock, clock) : Layer.empty, - Layer.succeed(Metric.FiberRuntimeMetrics, { - recordFiberStart: () => void (activeFibers += 1), - recordFiberEnd: () => void (activeFibers -= 1) - }) - ) - const handlers = PeerRpcServer.layerHandlers({ tenantId: "tenant", peerId: serverPeerId, definition }).pipe( - Layer.provide(services) - ) - const handlerScope = yield* Scope.make() - yield* Effect.addFinalizer(() => Scope.close(handlerScope, Exit.void)) - const handlerContext = yield* Layer.build(handlers).pipe(Effect.provideService(Scope.Scope, handlerScope)) - const openHandler = handlerContext.mapUnsafe.get(PeerRpc.OpenRpc.key) as Rpc.Handler<"Open"> - const pushHandler = handlerContext.mapUnsafe.get(PeerRpc.PushRpc.key) as Rpc.Handler<"Push"> - const directOpenAs = ( - principal: PeerAuthentication.PeerPrincipal, - documents: ReadonlyArray - ) => - (openHandler.handler({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents - }, {} as never) as Stream.Stream).pipe( - Stream.provideContext(Context.add( - openHandler.context, - PeerAuthentication.AuthenticatedPeer, - { - principal, - validUntil: options.authenticationValidUntil, - invalidated: Deferred.await(authenticationInvalidated) - } - )) - ) - const directOpen = (documents: ReadonlyArray) => - directOpenAs(principals.get("owner")!, documents) - const directPushAs = ( - validUntil: number, - request: typeof PeerRpc.PushRpc.payloadSchema.Type - ) => - (pushHandler.handler(request, {} as never) as Effect.Effect).pipe( - Effect.provideContext(Context.add( - pushHandler.context, - PeerAuthentication.AuthenticatedPeer, - { - principal: principals.get("owner")!, - validUntil, - invalidated: Deferred.await(authenticationInvalidated) - } - )) - ) - const directPush = (request: typeof PeerRpc.PushRpc.payloadSchema.Type) => - directPushAs(options.authenticationValidUntil, request) - const client = yield* RpcTest.makeClient(PeerRpc.Rpcs).pipe( - Effect.provide(handlerContext), - Effect.provide(PeerAuthentication.layerServer), - Effect.provideService(PeerAuthenticator.PeerAuthenticator, { - authenticate: (value) => { - const credential = Redacted.value(value) - const bulk = /^bulk-(\d+)$/.exec(credential) - const subject = /^subject-(\d+)$/.exec(credential) - const principal = principals.get(credential) ?? (bulk === null && subject === null - ? undefined - : PeerAuthentication.PeerPrincipal.make({ - tenantId: "tenant", - subjectId: subject === null ? "bulk" : `subject-${subject[1]}`, - peerId: Identity.PeerId.make( - `peer_00000000-0000-4000-8000-${Number((bulk ?? subject)![1]).toString(16).padStart(12, "0")}` - ) - })) - return principal === undefined - ? Effect.fail(new PeerRpcError.AuthenticationFailure()) - : Effect.succeed({ - principal, - validUntil: options.authenticationValidUntil, - invalidated: Deferred.await(authenticationInvalidated) - }) - } - }), - Effect.provide(PeerAuthentication.layerClient), - Effect.provideService(PeerCredentials.PeerCredentials, { - get: Effect.sync(() => Redacted.make(credential)) - }), - Effect.provideService(PeerRpcLimits.PeerRpcLimits, configuredRpcLimits) - ) - // `capabilities` is omitted unless a test supplies one, which is exactly the payload a client - // built before lineage sends. It is spread rather than passed as `undefined` because the field - // is `Schema.optionalKey`: absent and explicitly undefined are not the same wire value. - const open = ( - documents: ReadonlyArray, - capabilities?: { readonly lineageAware?: boolean } - ) => - Effect.gen(function*() { - const events = yield* Queue.unbounded() - const fiber = yield* Stream.runForEach( - client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - ...(capabilities === undefined ? {} : { capabilities }), - documents - }), - (event) => Queue.offer(events, event).pipe(Effect.asVoid) - ).pipe(Effect.forkChild) - const opened = yield* Effect.raceFirst( - Queue.take(events), - Fiber.join(fiber).pipe(Effect.andThen(Effect.die("Open stream ended before Opened"))) - ) - assert.strictEqual(opened._tag, "Opened") - return { opened: opened as PeerRpc.Opened, events, fiber } - }) - const encodeMessage = ( - sequence: number, - documentId: Identity.DocumentId, - documentType: string, - message: Uint8Array - ) => - Effect.gen(function*() { - const value = yield* Schema.encodeEffect(SyncEnvelopeJson)({ - connectionEpoch: "remote-epoch", - sequence, - documentId, - documentType, - messageHash: yield* Canonical.digest(message).pipe(Effect.provideService(Crypto.Crypto, crypto)), - message, - writerProvenance: [{ - changeHash: "a".repeat(64), - writerSchemaVersion: 1, - writerDefinitionHash: "remote-definition-hash" - }] - }) - return new TextEncoder().encode(value) - }) - const encode = (sequence: number, documentId = taskId, documentType = Task.name) => - encodeMessage(sequence, documentId, documentType, Uint8Array.of(sequence + 1)) - return { - client, - commits, - generated, - generatePeers, - received, - receivedPayloads, - sent, - enqueued, - pendingStarted, - inboundRelease, - inboundBlocked, - authenticationInvalidated, - commitProcessed, - commitFlushStarted, - commitFlushRelease, - authorizationInvalidated, - authorizationRelease, - authorizationStarted, - authorizationCalls, - sessionOpenCalls, - subscriptions, - publications, - setCredential: (value: string) => Effect.sync(() => void (credential = value)), - allowSessionOpen: Effect.sync(() => void (failSessionOpen = false)), - setPendingOutbound: (outbound: PeerSync.Outbound) => Effect.sync(() => void (initialOutbound = outbound)), - setNextOutbound: (outbound: PeerSync.Outbound) => Ref.set(nextOutbound, outbound), - setReply: (value: PeerSync.Reply) => Effect.sync(() => void (reply = value)), - setCurrentTime: (value: number) => Effect.sync(() => void (currentTime = value)), - setCurrentTimeOnNextRandomBytes: (value: number) => - Effect.sync(() => void (currentTimeOnNextRandomBytes = value)), - blockCommitGeneration: Effect.sync(() => void (blockCommitFlush = true)), - activeFiberCount: Effect.sync(() => activeFibers), - closeServer: Scope.close(handlerScope, Exit.void), - open, - directOpen, - directPush, - directPushAs, - directOpenAs: (credential: "owner" | "foreign" | "third", documents: ReadonlyArray) => - directOpenAs(principals.get(credential)!, documents), - encode, - encodeMessage - } - }) - -describe("PeerRpcServer", () => { - it.effect("emits Opened before any peer Message", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - initialOutbound: { - sendSequence: 0, - documentId: taskId, - message: Uint8Array.of(1, 2, 3), - messageHash: "hash", - heads: [], - lineage: Identity.genesisLineage, - writerProvenance: [] - } - }) - const session = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - assert.strictEqual((yield* Queue.take(session.events))._tag, "Message") - yield* Fiber.interrupt(session.fiber) - assert.strictEqual(yield* Ref.get(fixture.subscriptions), 1) - }))) - - it.effect("advertises lineage awareness through a real Open handshake and into peer session generation", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture(baseOptions) - // Both halves of the handshake through the adapter a real peer actually uses: its `Open` - // carries the client's capability advertisement, and it decodes the server's out of `Opened`. - const context = yield* Layer.build(RpcPeerTransport.layer(fixture.client, { - documents: [{ document: Task, documentId: taskId }], - definition - })) - const connection = yield* Context.get(context, PeerTransport.PeerTransport).connect({ - replicaId: permit.replicaId, - peerId: serverPeerId - }) - // The wire frame, decoded by the real client through the real `Opened` schema. Absent here - // would mean every lineage aware peer refuses to emit a rewritten document to this server. - assert.deepStrictEqual(connection.capabilities, { storeAndForward: false, lineageAware: true }) - // The server side session read the CLIENT's advertisement off its own transport connection, so - // a rewritten document may be generated toward this peer. `PeerSync.generate` is what turns a - // false here into a refusal, which its own suite covers. - assert.deepStrictEqual(yield* Queue.take(fixture.generatePeers), { lineageAware: true }) - yield* connection.close - }))) - - it.effect("treats an Open that omits the client capability as not lineage aware", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture(baseOptions) - // Exactly what a client built before lineage sends. `PeerRpc.protocolVersion` was not bumped - // for lineage, so it passes the version check; inferring awareness from that version would - // hand it a rewritten document, which it would union and reply with the discarded history. - const session = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - // The server's own claim is unconditional and independent of the client's. - assert.deepStrictEqual(session.opened.capabilities, { storeAndForward: false, lineageAware: true }) - assert.deepStrictEqual(yield* Queue.take(fixture.generatePeers), { lineageAware: false }) - yield* Fiber.interrupt(session.fiber) - }))) - - it.effect("reads the advertised flag rather than the presence of the capability object", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture(baseOptions) - const session = yield* fixture.open([{ documentType: Task.name, documentId: taskId }], {}) - assert.deepStrictEqual(yield* Queue.take(fixture.generatePeers), { lineageAware: false }) - yield* Fiber.interrupt(session.fiber) - }))) - - it.effect("hosts the existing canonical replica through the real PeerSession", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture(baseOptions) - const session = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - yield* fixture.client.Push({ sessionId: session.opened.sessionId, payload: yield* fixture.encode(0) }) - assert.strictEqual(yield* Queue.take(fixture.received), 0) - const payload = yield* Queue.take(fixture.receivedPayloads) - assert.strictEqual(payload.connectionEpoch, "remote-epoch") - assert.strictEqual(payload.localConnectionEpoch, "local-epoch") - assert.deepStrictEqual(payload.writerProvenance, [{ - changeHash: "a".repeat(64), - writerSchemaVersion: 1, - writerDefinitionHash: "remote-definition-hash" - }]) - assert.strictEqual(yield* Ref.get(fixture.publications), 1) - yield* Fiber.interrupt(session.fiber) - }))) - - it.effect("serially applies Push messages in accepted order", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture(baseOptions) - const session = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - yield* fixture.client.Push({ sessionId: session.opened.sessionId, payload: yield* fixture.encode(0) }) - assert.strictEqual(yield* Queue.take(fixture.received), 0) - yield* fixture.client.Push({ sessionId: session.opened.sessionId, payload: yield* fixture.encode(1) }) - assert.strictEqual(yield* Queue.take(fixture.received), 1) - yield* Fiber.interrupt(session.fiber) - }))) - - it.effect("replaces the prior session without letting stale cleanup remove the replacement", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture(baseOptions) - const first = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - const replacement = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - const oldExit = yield* Fiber.await(first.fiber) - assert.isTrue(Exit.isFailure(oldExit)) - if (Exit.isFailure(oldExit)) { - const streamError = Cause.findErrorOption(oldExit.cause) - assert.strictEqual(streamError._tag, "Some") - if (streamError._tag === "Some") assert.strictEqual(streamError.value._tag, "SessionUnavailable") - } - const oldError = yield* fixture.client.Push({ - sessionId: first.opened.sessionId, - payload: Uint8Array.of(1) - }).pipe(Effect.flip) - assert.instanceOf(oldError, PeerRpcError.SessionUnavailable) - yield* Fiber.interrupt(first.fiber) - yield* fixture.client.Push({ - sessionId: replacement.opened.sessionId, - payload: yield* fixture.encode(0) - }) - assert.strictEqual(yield* Queue.take(fixture.received), 0) - yield* Fiber.interrupt(replacement.fiber) - }))) - - it.effect("keeps the healthy session alive when a malformed replacement Open fails validation", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture(baseOptions) - const first = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - const authorizationCallsBeforeMalformedOpen = yield* Ref.get(fixture.authorizationCalls) - const malformedError = yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [ - { documentType: Task.name, documentId: taskId }, - { documentType: Note.name, documentId: taskId } - ] - }).pipe(Stream.runDrain, Effect.flip) - yield* fixture.client.Push({ - sessionId: first.opened.sessionId, - payload: yield* fixture.encode(0) - }) - assert.strictEqual(yield* Queue.take(fixture.received), 0) - assert.instanceOf(malformedError, PeerRpcError.InvalidRequest) - assert.strictEqual(yield* Ref.get(fixture.authorizationCalls), authorizationCallsBeforeMalformedOpen) - yield* Fiber.interrupt(first.fiber) - }))) - - it.effect("does not consume Open admission for a cross type duplicate document id", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - manualClock: true, - rpcLimits: { openRatePerSecond: Number.MIN_VALUE, openBurst: 1 } - }) - const malformedError = yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [ - { documentType: Task.name, documentId: taskId }, - { documentType: Note.name, documentId: taskId } - ] - }).pipe(Stream.runDrain, Effect.flip) - assert.instanceOf(malformedError, PeerRpcError.InvalidRequest) - - const valid = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - assert.strictEqual(yield* Ref.get(fixture.authorizationCalls), 1) - yield* Fiber.interrupt(valid.fiber) - }))) - - it.effect("enforces the global session limit", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ ...baseOptions, replicaLimits: { maxSessions: 1 } }) - const first = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - yield* fixture.setCredential("foreign") - const error = yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Note.name, documentId: noteId }] - }).pipe(Stream.runDrain, Effect.flip) - assert.instanceOf(error, PeerRpcError.SessionOverloaded) - yield* Fiber.interrupt(first.fiber) - }))) - - it.effect("enforces the per subject session limit", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ ...baseOptions, rpcLimits: { maxSessionsPerSubject: 1 } }) - const first = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - yield* fixture.setCredential("same-subject") - const error = yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Note.name, documentId: noteId }] - }).pipe(Stream.runDrain, Effect.flip) - assert.instanceOf(error, PeerRpcError.SessionOverloaded) - yield* Fiber.interrupt(first.fiber) - }))) - - it.effect("retains touched subjects while evicting the oldest inactive subject", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - rpcLimits: { - openRatePerSecond: 1, - openBurst: 1, - maxRetainedRateLimitedSubjects: 3 - }, - authorization: () => Effect.fail(new PeerRpcError.AccessDenied()) - }) - const attempt = (subject: number) => - fixture.setCredential(`subject-${subject}`).pipe( - Effect.andThen( - fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Task.name, documentId: taskId }] - }).pipe(Stream.runDrain, Effect.flip) - ) - ) - for (const subject of [1, 2, 3]) { - assert.instanceOf(yield* attempt(subject), PeerRpcError.AccessDenied) - } - assert.instanceOf(yield* attempt(2), PeerRpcError.RequestCapacityExceeded) - assert.instanceOf(yield* attempt(4), PeerRpcError.AccessDenied) - assert.instanceOf(yield* attempt(1), PeerRpcError.AccessDenied) - assert.instanceOf(yield* attempt(2), PeerRpcError.RequestCapacityExceeded) - assert.instanceOf(yield* attempt(3), PeerRpcError.AccessDenied) - }))) - - it.effect("retains an active subject when the subject state bound is full", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - rpcLimits: { maxRetainedRateLimitedSubjects: 1 } - }) - const active = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - yield* fixture.setCredential("foreign") - const full = yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Note.name, documentId: noteId }] - }).pipe(Stream.runDrain, Effect.flip) - assert.instanceOf(full, PeerRpcError.RequestCapacityExceeded) - yield* Fiber.interrupt(active.fiber) - const replacement = yield* fixture.open([{ documentType: Note.name, documentId: noteId }]) - yield* Fiber.interrupt(replacement.fiber) - }))) - - for ( - const [description, elapsed, expected] of [ - ["retains an exhausted inactive subject just before idle expiry", 999, "RequestCapacityExceeded"], - ["refreshes an exhausted inactive subject exactly at idle expiry", 1_000, "AccessDenied"] - ] as const - ) { - it.effect(description, () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - rpcLimits: { - openRatePerSecond: Number.MIN_VALUE, - openBurst: 1, - rateLimitIdleRetention: 1_000, - maxRetainedRateLimitedSubjects: 8 - }, - authorization: () => Effect.fail(new PeerRpcError.AccessDenied()) - }) - const attempt = () => - fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Task.name, documentId: taskId }] - }).pipe(Stream.runDrain, Effect.flip) - assert.instanceOf(yield* attempt(), PeerRpcError.AccessDenied) - yield* TestClock.adjust(elapsed) - assert.strictEqual((yield* attempt())._tag, expected) - }))) - } - - it.effect("serves outbound byte waiters in FIFO order without bypassing a large head", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - rpcLimits: { - maxBufferedBytes: PeerSession.maximumSyncEnvelopeBytes( - replicaLimits.maxSyncMessageBytes, - replicaLimits.maxSyncChangesPerMessage - ) - }, - initialOutbound: { - sendSequence: 0, - documentId: taskId, - message: new Uint8Array(48_000), - messageHash: "fifo-blocker", - heads: [], - lineage: Identity.genesisLineage, - writerProvenance: maximumWriterProvenance - } - }) - const opened = yield* Queue.unbounded() - const blockerReady = yield* Deferred.make() - const blocker = yield* Effect.scoped(Effect.gen(function*() { - const pull = yield* Stream.toPull(fixture.directOpen([{ documentType: Task.name, documentId: taskId }])) - assert.strictEqual((yield* pull)[0]._tag, "Opened") - assert.strictEqual((yield* pull)[0]._tag, "Message") - yield* Deferred.succeed(blockerReady, undefined) - return yield* Effect.never - })).pipe(Effect.forkChild) - yield* Deferred.await(blockerReady) - assert.strictEqual(yield* Queue.take(fixture.pendingStarted), remotePeerA) - yield* fixture.setPendingOutbound({ - sendSequence: 0, - documentId: noteId, - message: new Uint8Array(replicaLimits.maxSyncMessageBytes), - messageHash: "fifo-head", - heads: [], - lineage: Identity.genesisLineage, - writerProvenance: maximumWriterProvenance - }) - yield* fixture.setCredential("foreign") - const head = yield* Stream.runForEach( - fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Note.name, documentId: noteId }] - }), - (event) => event._tag === "Message" ? Queue.offer(opened, "head").pipe(Effect.asVoid) : Effect.void - ).pipe( - Effect.forkChild - ) - assert.strictEqual(yield* Queue.take(fixture.pendingStarted), remotePeerB) - for (let index = 0; index < 5; index++) yield* Effect.yieldNow - yield* fixture.setPendingOutbound({ - sendSequence: 0, - documentId: taskId, - message: new Uint8Array(48_000), - messageHash: "fifo-tail", - heads: [], - lineage: Identity.genesisLineage, - writerProvenance: [] - }) - yield* fixture.setCredential("subject-10") - const tail = yield* Stream.runForEach( - fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Task.name, documentId: taskId }] - }), - (event) => event._tag === "Message" ? Queue.offer(opened, "tail").pipe(Effect.asVoid) : Effect.void - ).pipe( - Effect.forkChild - ) - yield* Queue.take(fixture.pendingStarted) - for (let index = 0; index < 5; index++) yield* Effect.yieldNow - assert.strictEqual((yield* Queue.poll(opened))._tag, "None") - yield* Fiber.interrupt(blocker) - assert.strictEqual(yield* Queue.take(opened), "head") - assert.strictEqual(yield* Queue.take(opened), "tail") - yield* Fiber.interrupt(head) - yield* Fiber.interrupt(tail) - }))) - - it.effect("advances outbound byte waiters and restores capacity when the head is canceled", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - rpcLimits: { - maxBufferedBytes: PeerSession.maximumSyncEnvelopeBytes( - replicaLimits.maxSyncMessageBytes, - replicaLimits.maxSyncChangesPerMessage - ) - } - }) - yield* fixture.setPendingOutbound({ - sendSequence: 0, - documentId: taskId, - message: new Uint8Array(replicaLimits.maxSyncMessageBytes), - messageHash: "cancel-blocker", - heads: [], - lineage: Identity.genesisLineage, - writerProvenance: maximumWriterProvenance - }) - const blockerReady = yield* Deferred.make() - const blocker = yield* Effect.scoped(Effect.gen(function*() { - const pull = yield* Stream.toPull(fixture.directOpen([{ documentType: Task.name, documentId: taskId }])) - yield* pull - yield* pull - yield* Deferred.succeed(blockerReady, undefined) - return yield* Effect.never - })).pipe(Effect.forkChild) - yield* Deferred.await(blockerReady) - assert.strictEqual(yield* Queue.take(fixture.pendingStarted), remotePeerA) - yield* fixture.setPendingOutbound({ - sendSequence: 0, - documentId: noteId, - message: new Uint8Array(replicaLimits.maxSyncMessageBytes), - messageHash: "cancel-head", - heads: [], - lineage: Identity.genesisLineage, - writerProvenance: [] - }) - const head = yield* fixture.directOpenAs( - "foreign", - [{ documentType: Note.name, documentId: noteId }] - ).pipe(Stream.runDrain, Effect.forkChild) - assert.strictEqual(yield* Queue.take(fixture.pendingStarted), remotePeerB) - for (let index = 0; index < 5; index++) yield* Effect.yieldNow - yield* fixture.setPendingOutbound({ - sendSequence: 0, - documentId: taskId, - message: Uint8Array.of(1), - messageHash: "cancel-tail", - heads: [], - lineage: Identity.genesisLineage, - writerProvenance: [] - }) - const tailMessage = yield* Deferred.make() - const tail = yield* fixture.directOpenAs( - "third", - [{ documentType: Task.name, documentId: taskId }] - ).pipe( - Stream.runForEach((event) => - event._tag === "Message" ? Deferred.succeed(tailMessage, undefined).pipe(Effect.asVoid) : Effect.void - ), - Effect.forkChild - ) - yield* Queue.take(fixture.pendingStarted) - for (let index = 0; index < 5; index++) yield* Effect.yieldNow - yield* Fiber.interrupt(head) - yield* Deferred.await(tailMessage) - yield* Fiber.interrupt(tail) - yield* Fiber.interrupt(blocker) - yield* fixture.setPendingOutbound({ - sendSequence: 0, - documentId: taskId, - message: new Uint8Array(replicaLimits.maxSyncMessageBytes), - messageHash: "cancel-probe", - heads: [], - lineage: Identity.genesisLineage, - writerProvenance: [] - }) - const probeMessage = yield* Deferred.make() - const probe = yield* fixture.directOpenAs( - "third", - [{ documentType: Task.name, documentId: taskId }] - ).pipe( - Stream.runForEach((event) => - event._tag === "Message" ? Deferred.succeed(probeMessage, undefined).pipe(Effect.asVoid) : Effect.void - ), - Effect.forkChild - ) - yield* Deferred.await(probeMessage) - yield* Fiber.interrupt(probe) - }))) - - it.effect("bounds outbound byte waiters by the active session limit", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - replicaLimits: { maxSessions: 8 }, - rpcLimits: { - maxBufferedBytes: PeerSession.maximumSyncEnvelopeBytes( - replicaLimits.maxSyncMessageBytes, - replicaLimits.maxSyncChangesPerMessage - ) - } - }) - const large = new Uint8Array(replicaLimits.maxSyncMessageBytes) - yield* fixture.setPendingOutbound({ - sendSequence: 0, - documentId: taskId, - message: large, - messageHash: "bounded-blocker", - heads: [], - lineage: Identity.genesisLineage, - writerProvenance: maximumWriterProvenance - }) - const blockerReady = yield* Deferred.make() - const blocker = yield* Effect.scoped(Effect.gen(function*() { - const pull = yield* Stream.toPull(fixture.directOpen([{ documentType: Task.name, documentId: taskId }])) - yield* pull - yield* pull - yield* Deferred.succeed(blockerReady, undefined) - return yield* Effect.never - })).pipe(Effect.forkChild) - yield* Deferred.await(blockerReady) - yield* Queue.take(fixture.pendingStarted) - const waiters = [] - for (let index = 0; index < 7; index++) { - yield* fixture.setPendingOutbound({ - sendSequence: 0, - documentId: taskId, - message: large, - messageHash: `bounded-waiter-${index}`, - heads: [], - lineage: Identity.genesisLineage, - writerProvenance: [] - }) - yield* fixture.setCredential(`subject-${index + 10}`) - waiters.push( - yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Task.name, documentId: taskId }] - }).pipe(Stream.runDrain, Effect.forkChild) - ) - yield* Queue.take(fixture.pendingStarted) - for (let turn = 0; turn < 5; turn++) yield* Effect.yieldNow - } - yield* fixture.setPendingOutbound({ - sendSequence: 0, - documentId: taskId, - message: large, - messageHash: "bounded-rejected", - heads: [], - lineage: Identity.genesisLineage, - writerProvenance: [] - }) - yield* fixture.setCredential("subject-20") - const rejected = yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Task.name, documentId: taskId }] - }).pipe(Stream.runDrain, Effect.flip) - assert.instanceOf(rejected, PeerRpcError.SessionOverloaded) - for (const waiter of waiters) { - assert.isUndefined(waiter.pollUnsafe()) - yield* Fiber.interrupt(waiter) - } - yield* Fiber.interrupt(blocker) - }))) - - it.effect("bounds global Open setup while authorization is blocked", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - blockAuthorization: true, - rpcLimits: { maxInFlightOpen: 1, maxInFlightOpenPerSubject: 2 } - }) - const first = yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Task.name, documentId: taskId }] - }).pipe(Stream.runDrain, Effect.forkChild) - assert.strictEqual(yield* Queue.take(fixture.authorizationStarted), "subject-a") - yield* fixture.setCredential("foreign") - const error = yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Note.name, documentId: noteId }] - }).pipe(Stream.runDrain, Effect.flip) - assert.instanceOf(error, PeerRpcError.RequestCapacityExceeded) - yield* Deferred.succeed(fixture.authorizationRelease, undefined) - yield* Fiber.interrupt(first) - }))) - - it.effect("bounds per subject Open setup while authorization is blocked", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - blockAuthorization: true, - rpcLimits: { maxInFlightOpen: 2, maxInFlightOpenPerSubject: 1 } - }) - const first = yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Task.name, documentId: taskId }] - }).pipe(Stream.runDrain, Effect.forkChild) - assert.strictEqual(yield* Queue.take(fixture.authorizationStarted), "subject-a") - yield* fixture.setCredential("same-subject") - const error = yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Note.name, documentId: noteId }] - }).pipe(Stream.runDrain, Effect.flip) - assert.instanceOf(error, PeerRpcError.RequestCapacityExceeded) - yield* Deferred.succeed(fixture.authorizationRelease, undefined) - yield* Fiber.interrupt(first) - }))) - - it.effect("enforces the selected document limit before allocation", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ ...baseOptions, replicaLimits: { maxStreamsPerSession: 1 } }) - const error = yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [ - { documentType: Task.name, documentId: taskId }, - { documentType: Note.name, documentId: noteId } - ] - }).pipe(Stream.runDrain, Effect.flip) - assert.instanceOf(error, PeerRpcError.RequestLimitExceeded) - assert.strictEqual(yield* Ref.get(fixture.authorizationCalls), 0) - }))) - - it.effect("validates Open identity protocol and selection before authorization", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture(baseOptions) - const cases = [ - { - request: { - protocolVersion: PeerRpc.protocolVersion + 1, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Task.name, documentId: taskId }] - }, - tag: "UnsupportedVersion" - }, - { - request: { - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: remotePeerA, - definitionHash, - documents: [{ documentType: Task.name, documentId: taskId }] - }, - tag: "PeerMismatch" - }, - { - request: { - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash: "def_ffffffffffffffff", - documents: [{ documentType: Task.name, documentId: taskId }] - }, - tag: "DefinitionMismatch" - }, - { - request: { - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [] - }, - tag: "InvalidRequest" - }, - { - request: { - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [ - { documentType: Task.name, documentId: taskId }, - { documentType: Task.name, documentId: taskId } - ] - }, - tag: "InvalidRequest" - }, - { - request: { - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [ - { documentType: Task.name, documentId: taskId }, - { documentType: Note.name, documentId: taskId } - ] - }, - tag: "InvalidRequest" - }, - { - request: { - protocolVersion: PeerRpc.protocolVersion + 1, - expectedPeerId: remotePeerA, - definitionHash: "def_ffffffffffffffff", - documents: [] - }, - tag: "UnsupportedVersion" - }, - { - request: { - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: remotePeerA, - definitionHash: "def_ffffffffffffffff", - documents: [] - }, - tag: "PeerMismatch" - }, - { - request: { - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash: "def_ffffffffffffffff", - documents: [] - }, - tag: "DefinitionMismatch" - } - ] - for (const testCase of cases) { - const error = yield* fixture.client.Open(testCase.request).pipe(Stream.runDrain, Effect.flip) - assert.strictEqual(error._tag, testCase.tag) - } - const legacyError = yield* fixture.client.Open({ - protocolVersion: 1, - expectedPeerId: serverPeerId, - documents: [{ documentType: Task.name, documentId: taskId }] - }).pipe(Stream.runDrain, Effect.flip) - assert.instanceOf(legacyError, PeerRpcError.UnsupportedVersion) - yield* fixture.setCredential("other-tenant") - const tenantError = yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Task.name, documentId: taskId }] - }).pipe(Stream.runDrain, Effect.flip) - assert.instanceOf(tenantError, PeerRpcError.AccessDenied) - assert.strictEqual(yield* Ref.get(fixture.authorizationCalls), 0) - assert.strictEqual(yield* Ref.get(fixture.sessionOpenCalls), 0) - }))) - - it.effect("rejects authorized documents that do not belong to the configured definition", () => - Effect.scoped(Effect.gen(function*() { - const TaskV2 = Document.make(Task.name, { - schema: Schema.Struct({ title: Schema.String }), - version: Task.version + 1 - }) - const fixture = yield* makeFixture({ - ...baseOptions, - authorization: (request) => - Effect.succeed({ - documents: request.documents.map(({ documentId }) => ({ document: TaskV2, documentId })), - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.never - }) - }) - const exit = yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Task.name, documentId: taskId }] - }).pipe(Stream.take(1), Stream.runDrain, Effect.exit) - assert.isTrue(Exit.isFailure(exit)) - if (Exit.isFailure(exit)) { - const error = Cause.findErrorOption(exit.cause) - assert.isTrue(Option.isSome(error) && error.value._tag === "AccessDenied") - } - assert.strictEqual(yield* Ref.get(fixture.sessionOpenCalls), 0) - }))) - - it.effect("mediates every direct authorization selection before session allocation", () => - Effect.scoped(Effect.gen(function*() { - const cases = [ - [], - [{ document: Task, documentId: taskId }, { document: Note, documentId: noteId }], - [{ document: Task, documentId: taskId }, { document: Task, documentId: taskId }], - [{ document: Note, documentId: taskId }], - [{ document: Task, documentId: noteId }] - ] - for (const documents of cases) { - const fixture = yield* makeFixture({ - ...baseOptions, - authorization: () => - Effect.succeed({ - documents, - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.void - }) - }) - const error = yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Task.name, documentId: taskId }] - }).pipe(Stream.runDrain, Effect.flip) - assert.instanceOf(error, PeerRpcError.AccessDenied) - assert.strictEqual((yield* Queue.poll(fixture.pendingStarted))._tag, "None") - } - }))) - - it.effect("maps direct authorization defects and unexpected failures without exposing their contents", () => - Effect.scoped(Effect.gen(function*() { - const sentinel = "authorization-private-sentinel-836592" - for ( - const authorization of [ - () => Effect.die(new Error(sentinel)), - () => Effect.fail(new Error(sentinel) as never) - ] - ) { - const fixture = yield* makeFixture({ ...baseOptions, authorization }) - const error = yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Task.name, documentId: taskId }] - }).pipe(Stream.runDrain, Effect.flip) - assert.instanceOf(error, PeerRpcError.ServerUnavailable) - assert.isFalse(JSON.stringify(error).includes(sentinel)) - } - }))) - - it.effect("preserves declared direct authorization failures", () => - Effect.scoped(Effect.gen(function*() { - for (const expected of [new PeerRpcError.AccessDenied(), new PeerRpcError.ServerUnavailable()]) { - const fixture = yield* makeFixture({ - ...baseOptions, - authorization: () => Effect.fail(expected) - }) - const actual = yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Task.name, documentId: taskId }] - }).pipe(Stream.runDrain, Effect.flip) - assert.strictEqual(actual._tag, expected._tag) - } - }))) - - it.effect("maps mixed authorization interruption and defects without exposing their contents", () => - Effect.scoped(Effect.gen(function*() { - const sentinel = "mixed-authorization-private-sentinel-361794" - const fixture = yield* makeFixture({ - ...baseOptions, - authorization: () => - Effect.failCause(Cause.fromReasons([ - Cause.makeInterruptReason(1), - Cause.makeDieReason(new Error(sentinel)) - ])) - }) - const error = yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Task.name, documentId: taskId }] - }).pipe(Stream.runDrain, Effect.flip) - assert.instanceOf(error, PeerRpcError.ServerUnavailable) - assert.isFalse(JSON.stringify(error).includes(sentinel)) - assert.strictEqual((yield* Queue.poll(fixture.pendingStarted))._tag, "None") - }))) - - it.effect("preserves direct authorization interruption", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - authorization: () => Effect.interrupt - }) - const exit = yield* fixture.directOpen([{ documentType: Task.name, documentId: taskId }]).pipe( - Stream.runDrain, - Effect.exit - ) - assert.isTrue(Exit.isFailure(exit)) - if (Exit.isFailure(exit)) assert.isTrue(Cause.hasInterruptsOnly(exit.cause)) - }))) - - it.effect("rejects an expired direct authorization before session allocation", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - authorization: () => - Effect.succeed({ - documents: [{ document: Task, documentId: taskId }], - validUntil: 0, - invalidated: Effect.void - }) - }) - const error = yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Task.name, documentId: taskId }] - }).pipe(Stream.runDrain, Effect.flip) - assert.instanceOf(error, PeerRpcError.AccessDenied) - assert.strictEqual((yield* Queue.poll(fixture.pendingStarted))._tag, "None") - }))) - - for (const validUntil of [Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY]) { - it.effect(`rejects nonfinite direct authorization lease ${String(validUntil)} before session allocation`, () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - authorization: () => - Effect.succeed({ - documents: [{ document: Task, documentId: taskId }], - validUntil, - invalidated: Effect.void - }) - }) - const error = yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Task.name, documentId: taskId }] - }).pipe(Stream.runDrain, Effect.flip) - assert.instanceOf(error, PeerRpcError.AccessDenied) - assert.strictEqual((yield* Queue.poll(fixture.pendingStarted))._tag, "None") - }))) - } - - for (const validUntil of [Number.NaN, Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY]) { - it.effect(`rejects nonfinite direct authentication lease ${String(validUntil)} before session allocation`, () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ ...baseOptions, authenticationValidUntil: validUntil }) - const error = yield* fixture.directOpen([{ documentType: Task.name, documentId: taskId }]).pipe( - Stream.runDrain, - Effect.flip - ) - assert.instanceOf(error, PeerRpcError.AuthenticationFailure) - assert.strictEqual((yield* Queue.poll(fixture.pendingStarted))._tag, "None") - assert.strictEqual(yield* Ref.get(fixture.authorizationCalls), 0) - }))) - } - - it.effect("closes the session instead of dropping an inbound message on overflow", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ ...baseOptions, blockInbound: true }) - const session = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - yield* fixture.client.Push({ sessionId: session.opened.sessionId, payload: yield* fixture.encode(0) }) - yield* Deferred.await(fixture.inboundBlocked) - const error = yield* fixture.client.Push({ - sessionId: session.opened.sessionId, - payload: yield* fixture.encode(1) - }).pipe(Effect.flip) - assert.instanceOf(error, PeerRpcError.SessionOverloaded) - const streamExit = yield* Fiber.await(session.fiber) - assert.isTrue(Exit.isFailure(streamExit)) - if (Exit.isFailure(streamExit)) { - const streamError = Cause.findErrorOption(streamExit.cause) - assert.strictEqual(streamError._tag, "Some") - if (streamError._tag === "Some") assert.strictEqual(streamError.value._tag, "SessionOverloaded") - } - yield* Deferred.succeed(fixture.inboundRelease, undefined) - const unavailable = yield* fixture.client.Push({ - sessionId: session.opened.sessionId, - payload: yield* fixture.encode(3) - }).pipe(Effect.flip) - assert.instanceOf(unavailable, PeerRpcError.SessionUnavailable) - }))) - - it.effect("does not expose sessions across the ownership matrix", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture(baseOptions) - const missing = yield* fixture.client.Push({ sessionId: missingSessionId, payload: Uint8Array.of(1) }).pipe( - Effect.flip - ) - const session = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - assert.instanceOf(missing, PeerRpcError.SessionUnavailable) - for (const credential of ["same-peer", "same-subject", "other-tenant"] as const) { - yield* fixture.setCredential(credential) - const foreign = yield* fixture.client.Push({ - sessionId: session.opened.sessionId, - payload: Uint8Array.of(1) - }).pipe(Effect.flip) - assert.instanceOf(foreign, PeerRpcError.SessionUnavailable) - assert.strictEqual(missing._tag, foreign._tag) - } - yield* Fiber.interrupt(session.fiber) - }))) - - it.effect("rejects Push after the Open stream is interrupted", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture(baseOptions) - const session = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - yield* Fiber.interrupt(session.fiber) - const error = yield* fixture.client.Push({ - sessionId: session.opened.sessionId, - payload: Uint8Array.of(1) - }).pipe(Effect.flip) - assert.instanceOf(error, PeerRpcError.SessionUnavailable) - }))) - - it.effect("stops admission before draining sessions on Layer close", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture(baseOptions) - const session = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - yield* fixture.closeServer - const streamExit = yield* Fiber.await(session.fiber) - assert.isTrue(Exit.isFailure(streamExit)) - if (Exit.isFailure(streamExit)) { - const streamError = Cause.findErrorOption(streamExit.cause) - assert.strictEqual(streamError._tag, "Some") - if (streamError._tag === "Some") assert.strictEqual(streamError.value._tag, "ServerUnavailable") - } - const error = yield* fixture.client.Push({ - sessionId: session.opened.sessionId, - payload: Uint8Array.of(1) - }).pipe(Effect.flip) - assert.instanceOf(error, PeerRpcError.ServerUnavailable) - }))) - - it.effect("interrupts a captured Open request while the server scope is closing", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - initialOutbound: { - sendSequence: 0, - documentId: taskId, - message: Uint8Array.of(1), - messageHash: "held", - heads: [], - lineage: Identity.genesisLineage, - writerProvenance: [] - } - }) - const messageHeld = yield* Deferred.make() - const interrupted = yield* Deferred.make() - const request = yield* Effect.scoped(Effect.gen(function*() { - const pull = yield* Stream.toPull(fixture.directOpen([{ documentType: Task.name, documentId: taskId }])) - assert.strictEqual((yield* pull)[0]._tag, "Opened") - assert.strictEqual((yield* pull)[0]._tag, "Message") - yield* Deferred.succeed(messageHeld, undefined) - return yield* Effect.never - })).pipe( - Effect.ensuring(Deferred.succeed(interrupted, undefined)), - Effect.forkChild - ) - yield* Deferred.await(messageHeld) - yield* fixture.closeServer - yield* Deferred.await(interrupted) - assert.isTrue(Exit.isFailure(yield* Fiber.await(request))) - }))) - - it.effect("preserves overload when detachment follows an outbound take", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - blockInbound: true, - initialOutbound: { - sendSequence: 0, - documentId: taskId, - message: Uint8Array.of(1), - messageHash: "taken-before-overload", - heads: [], - lineage: Identity.genesisLineage, - writerProvenance: [] - } - }) - const opened = yield* Deferred.make() - const messageTaken = yield* Deferred.make() - const request = yield* Effect.scoped(Effect.gen(function*() { - const pull = yield* Stream.toPull(fixture.directOpen([{ documentType: Task.name, documentId: taskId }])) - const first = yield* pull - assert.strictEqual(first[0]._tag, "Opened") - yield* Deferred.succeed(opened, first[0] as PeerRpc.Opened) - assert.strictEqual((yield* pull)[0]._tag, "Message") - yield* Deferred.succeed(messageTaken, undefined) - yield* pull - })).pipe(Effect.forkChild) - const session = yield* Deferred.await(opened) - yield* Deferred.await(messageTaken) - yield* fixture.client.Push({ - sessionId: session.sessionId, - payload: yield* fixture.encode(0) - }) - yield* Deferred.await(fixture.inboundBlocked) - const overload = yield* fixture.client.Push({ - sessionId: session.sessionId, - payload: yield* fixture.encode(1) - }).pipe(Effect.flip) - assert.strictEqual(overload._tag, "SessionOverloaded") - yield* Deferred.succeed(fixture.inboundRelease, undefined) - const streamExit = yield* Fiber.await(request) - assert.isTrue(Exit.isFailure(streamExit)) - if (Exit.isFailure(streamExit)) { - const streamError = Cause.findErrorOption(streamExit.cause) - assert.strictEqual(streamError._tag, "Some") - if (streamError._tag === "Some") assert.instanceOf(streamError.value, PeerRpcError.SessionOverloaded) - } - }))) - - it.effect("revokes the session when an authorization lease is invalidated", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture(baseOptions) - const session = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - yield* Deferred.succeed(fixture.authorizationInvalidated, undefined) - assert.isTrue(Exit.isFailure(yield* Fiber.await(session.fiber))) - const error = yield* fixture.client.Push({ - sessionId: session.opened.sessionId, - payload: Uint8Array.of(1) - }).pipe(Effect.flip) - assert.instanceOf(error, PeerRpcError.SessionUnavailable) - }))) - - it.effect("revokes the session when the authorization invalidation monitor defects", () => - Effect.scoped(Effect.gen(function*() { - const invalidate = yield* Deferred.make() - const fixture = yield* makeFixture({ - ...baseOptions, - authorization: (request) => - Effect.succeed({ - documents: request.documents.map((document) => ({ document: Task, documentId: document.documentId })), - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Deferred.await(invalidate).pipe( - Effect.andThen(Effect.die("authorization invalidation monitor defect")) - ) - }) - }) - const session = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - yield* Deferred.succeed(invalidate, undefined) - for (let index = 0; index < 5; index++) yield* Effect.yieldNow - const exit = session.fiber.pollUnsafe() - assert.isDefined(exit) - assert.isTrue(Exit.isFailure(exit!)) - if (Exit.isFailure(exit!)) { - const error = Cause.findErrorOption(exit!.cause) - assert.strictEqual(error._tag, "Some") - if (error._tag === "Some") assert.instanceOf(error.value, PeerRpcError.SessionUnavailable) - } - }))) - - it.effect("revokes the active session when authentication is invalidated", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture(baseOptions) - const session = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - yield* Deferred.succeed(fixture.authenticationInvalidated, undefined) - assert.isTrue(Exit.isFailure(yield* Fiber.await(session.fiber))) - const error = yield* fixture.client.Push({ - sessionId: session.opened.sessionId, - payload: Uint8Array.of(1) - }).pipe(Effect.flip) - assert.instanceOf(error, PeerRpcError.SessionUnavailable) - }))) - - it.effect( - "rejects a Push admitted while the losing lease monitor is still tearing down after authentication is invalidated", - () => - Effect.scoped(Effect.gen(function*() { - const authorizationNeverInvalidates = yield* Deferred.make() - const teardownStarted = yield* Deferred.make() - const teardownRelease = yield* Deferred.make() - const fixture = yield* makeFixture({ - ...baseOptions, - authorization: (request) => - Effect.succeed({ - documents: request.documents.map((document) => ({ document: Task, documentId: document.documentId })), - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Deferred.await(authorizationNeverInvalidates).pipe( - Effect.onInterrupt(() => - Deferred.succeed(teardownStarted, undefined).pipe(Effect.andThen(Deferred.await(teardownRelease))) - ) - ) - }) - }) - const session = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - - yield* Deferred.succeed(fixture.authenticationInvalidated, undefined) - yield* Deferred.await(teardownStarted) - - yield* Effect.gen(function*() { - const rejection = yield* fixture.directPush({ - sessionId: session.opened.sessionId, - payload: Uint8Array.of(1) - }).pipe(Effect.exit) - - assert.isTrue(Exit.isFailure(rejection)) - if (Exit.isFailure(rejection)) { - const error = Cause.findErrorOption(rejection.cause) - assert.strictEqual(error._tag, "Some") - if (error._tag === "Some") assert.strictEqual(error.value._tag, "SessionUnavailable") - } - }).pipe(Effect.ensuring(Deferred.succeed(teardownRelease, undefined))) - - assert.isTrue(Exit.isFailure(yield* Fiber.await(session.fiber))) - })) - ) - - for (const lease of ["authentication", "authorization"] as const) { - it.effect(`revokes the active session when ${lease} expires`, () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - authenticationValidUntil: lease === "authentication" ? 1_000 : Number.MAX_SAFE_INTEGER, - authorizationValidUntil: lease === "authorization" ? 1_000 : Number.MAX_SAFE_INTEGER - }) - const session = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - yield* TestClock.adjust(1_000) - assert.isTrue(Exit.isFailure(yield* Fiber.await(session.fiber))) - }))) - } - - it.effect("rejects Push at its current authentication deadline", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - authenticationValidUntil: 10_000, - manualClock: true - }) - const session = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - yield* fixture.setCurrentTime(1) - - const error = yield* fixture.directPushAs(1, { - sessionId: session.opened.sessionId, - payload: yield* fixture.encode(0) - }).pipe(Effect.flip) - - assert.instanceOf(error, PeerRpcError.AuthenticationFailure) - assert.strictEqual((yield* Queue.poll(fixture.received))._tag, "None") - }))) - - it.effect("rejects every Push racing the exact earliest lease boundary", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - authenticationValidUntil: 2_000, - authorizationValidUntil: 1_000, - manualClock: true - }) - const session = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - yield* fixture.setCurrentTime(999) - yield* fixture.directPush({ - sessionId: session.opened.sessionId, - payload: yield* fixture.encode(0) - }) - assert.strictEqual(yield* Queue.take(fixture.received), 0) - yield* fixture.setCurrentTime(1_000) - const exits = yield* Effect.forEach( - Array.from({ length: 16 }, (_, sequence) => sequence + 1), - (sequence) => - Effect.gen(function*() { - const payload = yield* fixture.encode(sequence) - return yield* fixture.directPush({ - sessionId: session.opened.sessionId, - payload - }) - }).pipe(Effect.exit), - { concurrency: "unbounded" } - ) - assert.isTrue(exits.every(Exit.isFailure)) - assert.isTrue(exits.every((exit) => { - if (Exit.isSuccess(exit)) return false - const error = Cause.findErrorOption(exit.cause) - return error._tag === "Some" && error.value instanceof PeerRpcError.SessionUnavailable - })) - assert.isTrue(Exit.isFailure(yield* Fiber.await(session.fiber))) - }))) - - it.effect("rejects a queued outbound Message at the exact earliest lease boundary", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - authenticationValidUntil: 2_000, - authorizationValidUntil: 1_000, - initialOutbound: { - sendSequence: 0, - documentId: taskId, - message: Uint8Array.of(1, 2, 3), - messageHash: "lease-boundary", - heads: [], - lineage: Identity.genesisLineage, - writerProvenance: [] - }, - manualClock: true - }) - const pull = yield* Stream.toPull(fixture.directOpen([{ documentType: Task.name, documentId: taskId }])) - assert.strictEqual((yield* pull)[0]._tag, "Opened") - assert.strictEqual(yield* Queue.take(fixture.sent), 0) - yield* fixture.setCurrentTime(1_000) - const error = yield* pull.pipe(Effect.flip) - assert.instanceOf(error, PeerRpcError.SessionUnavailable) - }))) - - it.effect("rejects the first Opened event at the exact earliest lease boundary", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - authenticationValidUntil: 2_000, - authorizationValidUntil: 1_000, - manualClock: true - }) - yield* fixture.setCurrentTimeOnNextRandomBytes(1_000) - const pull = yield* Stream.toPull(fixture.directOpen([{ documentType: Task.name, documentId: taskId }])) - const error = yield* pull.pipe(Effect.flip) - assert.instanceOf(error, PeerRpcError.SessionUnavailable) - }))) - - it.effect("rejects commit flush generation at the exact earliest lease boundary", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - authenticationValidUntil: 2_000, - authorizationValidUntil: 1_000, - manualClock: true - }) - const session = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - assert.strictEqual(yield* Queue.take(fixture.generated), taskId) - yield* fixture.setNextOutbound({ - sendSequence: 0, - documentId: taskId, - message: Uint8Array.of(1, 2, 3), - messageHash: "lease-boundary", - heads: [], - lineage: Identity.genesisLineage, - writerProvenance: [] - }) - yield* fixture.setCurrentTime(1_000) - yield* Queue.offer(fixture.commits, { - _tag: "Commit", - commitSequence: Identity.CommitSequence.make(1), - documentId: taskId, - keys: [], - refreshGeneration: 0 - }) - yield* Queue.take(fixture.commitProcessed) - const error = yield* Fiber.join(session.fiber).pipe(Effect.flip) - assert.instanceOf(error, PeerRpcError.SessionUnavailable) - assert.strictEqual((yield* Queue.poll(fixture.generated))._tag, "None") - }))) - - it.effect("revokes the active session at the maximum reauthorization interval", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - rpcLimits: { maximumReauthorizationInterval: 1_000 } - }) - const session = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - yield* TestClock.adjust(1_000) - assert.isTrue(Exit.isFailure(yield* Fiber.await(session.fiber))) - }))) - - it.effect("publishes no session when a lease is invalidated during setup", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture(baseOptions) - yield* Deferred.succeed(fixture.authenticationInvalidated, undefined) - const error = yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Task.name, documentId: taskId }] - }).pipe(Stream.runDrain, Effect.flip) - assert.instanceOf(error, PeerRpcError.SessionUnavailable) - }))) - - it.effect("releases partial acquisition after session startup failure", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - failSessionOpen: true, - replicaLimits: { maxSessions: 1 } - }) - const error = yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Task.name, documentId: taskId }] - }).pipe(Stream.runDrain, Effect.flip) - assert.instanceOf(error, PeerRpcError.ServerUnavailable) - yield* fixture.allowSessionOpen - yield* fixture.setCredential("foreign") - const session = yield* fixture.open([{ documentType: Note.name, documentId: noteId }]) - yield* Fiber.interrupt(session.fiber) - }))) - - it.effect("reports a lineage rewrite session failure as its own fieldless wire error", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - failSessionOpen: true, - sessionOpenFailure: new ReplicaError.DocumentLineageChanged({ - documentId: taskId, - localLineage: Identity.DocumentLineage.make("lin_00000000-0000-4000-8000-000000000004"), - remoteLineage: Identity.genesisLineage - }) - }) - const error = yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Task.name, documentId: taskId }] - }).pipe(Stream.runDrain, Effect.flip) - assert.instanceOf(error, PeerRpcError.DocumentLineageChanged) - // The peer learns the lineage no longer matches and nothing else: no document id, no lineage. - assert.deepStrictEqual( - Schema.encodeSync(PeerRpcError.PeerRpcError)(error), - { _tag: "DocumentLineageChanged" } - ) - }))) - - it.effect("maps initialization send capacity timeout to SessionOverloaded", () => - Effect.scoped(Effect.gen(function*() { - const message = new Uint8Array(replicaLimits.maxSyncMessageBytes) - const fixture = yield* makeFixture({ - ...baseOptions, - blockInbound: true, - rpcLimits: { - maxBufferedBytes: PeerSession.maximumSyncEnvelopeBytes( - replicaLimits.maxSyncMessageBytes, - replicaLimits.maxSyncChangesPerMessage - ) - } - }) - const blocker = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - yield* Queue.take(fixture.generated) - yield* Queue.take(fixture.pendingStarted) - yield* fixture.client.Push({ - sessionId: blocker.opened.sessionId, - payload: yield* fixture.encodeMessage(0, taskId, Task.name, message) - }) - yield* Deferred.await(fixture.inboundBlocked) - yield* fixture.setPendingOutbound({ - sendSequence: 0, - documentId: noteId, - message, - messageHash: "initial-capacity", - heads: [], - lineage: Identity.genesisLineage, - writerProvenance: maximumWriterProvenance - }) - yield* fixture.setCredential("foreign") - const opening = yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Note.name, documentId: noteId }] - }).pipe(Stream.runDrain, Effect.forkChild) - assert.strictEqual(yield* Queue.take(fixture.pendingStarted), remotePeerB) - yield* TestClock.adjust(replicaLimits.maxPeerSendMillis + 1) - const error = yield* Fiber.join(opening).pipe(Effect.flip) - assert.strictEqual(error._tag, "SessionOverloaded") - yield* Deferred.succeed(fixture.inboundRelease, undefined) - yield* Fiber.interrupt(blocker.fiber) - }))) - - it.effect("maps a post ready Push reply capacity timeout to SessionOverloaded", () => - Effect.scoped(Effect.gen(function*() { - const message = new Uint8Array(replicaLimits.maxSyncMessageBytes) - const fixture = yield* makeFixture({ - ...baseOptions, - rpcLimits: { - maxBufferedBytes: PeerSession.maximumSyncEnvelopeBytes( - replicaLimits.maxSyncMessageBytes, - replicaLimits.maxSyncChangesPerMessage - ) - }, - initialOutbound: { - sendSequence: 0, - documentId: taskId, - message, - messageHash: "held-for-push-reply", - heads: [], - lineage: Identity.genesisLineage, - writerProvenance: maximumWriterProvenance - } - }) - const messageHeld = yield* Deferred.make() - const blocker = yield* Effect.scoped(Effect.gen(function*() { - const pull = yield* Stream.toPull(fixture.directOpen([{ documentType: Task.name, documentId: taskId }])) - assert.strictEqual((yield* pull)[0]._tag, "Opened") - assert.strictEqual((yield* pull)[0]._tag, "Message") - yield* Deferred.succeed(messageHeld, undefined) - return yield* Effect.never - })).pipe(Effect.forkChild) - yield* Deferred.await(messageHeld) - yield* Queue.take(fixture.generated) - assert.strictEqual(yield* Queue.take(fixture.pendingStarted), remotePeerA) - yield* fixture.setCredential("foreign") - const session = yield* fixture.open([{ documentType: Note.name, documentId: noteId }]) - yield* Queue.take(fixture.generated) - assert.strictEqual(yield* Queue.take(fixture.pendingStarted), remotePeerB) - yield* fixture.setReply({ - documentId: noteId, - message, - messageHash: "push-reply-capacity", - heads: [] - }) - yield* fixture.client.Push({ - sessionId: session.opened.sessionId, - payload: yield* fixture.encodeMessage(0, noteId, Note.name, Uint8Array.of(1)) - }) - assert.strictEqual(yield* Queue.take(fixture.enqueued), noteId) - assert.strictEqual(yield* Queue.take(fixture.pendingStarted), remotePeerB) - yield* TestClock.adjust(replicaLimits.maxPeerSendMillis + 1) - const error = yield* Fiber.join(session.fiber).pipe(Effect.flip) - assert.strictEqual(error._tag, "SessionOverloaded") - yield* Fiber.interrupt(blocker) - }))) - - it.effect("does not retain disconnected watchers after repeated startup failure", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ ...baseOptions, failSessionOpen: true }) - const baseline = yield* fixture.activeFiberCount - assert.isAbove(baseline, 0) - for (let index = 0; index < 20; index++) { - const error = yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Task.name, documentId: taskId }] - }).pipe(Stream.runDrain, Effect.flip) - assert.instanceOf(error, PeerRpcError.ServerUnavailable) - } - for (let index = 0; index < 10; index++) yield* Effect.yieldNow - assert.strictEqual(yield* fixture.activeFiberCount, baseline) - }))) - - it.effect("interrupts an overloaded Open without draining an unacknowledged message", () => - Effect.scoped(Effect.gen(function*() { - const message = new Uint8Array(replicaLimits.maxSyncMessageBytes) - const fixture = yield* makeFixture({ - ...baseOptions, - rpcLimits: { - maxOutboundBufferedBytesPerSession: PeerSession.maximumSyncEnvelopeBytes( - replicaLimits.maxSyncMessageBytes, - replicaLimits.maxSyncChangesPerMessage - ), - maxBufferedBytes: PeerSession.maximumSyncEnvelopeBytes( - replicaLimits.maxSyncMessageBytes, - replicaLimits.maxSyncChangesPerMessage - ) - }, - initialOutbound: { - sendSequence: 0, - documentId: taskId, - message, - messageHash: "first", - heads: [], - lineage: Identity.genesisLineage, - writerProvenance: maximumWriterProvenance - } - }) - const opened = yield* Deferred.make() - const messageHeld = yield* Deferred.make() - const stream = yield* Effect.scoped(Effect.gen(function*() { - const pull = yield* Stream.toPull(fixture.directOpen([{ documentType: Task.name, documentId: taskId }])) - const first = yield* pull - assert.strictEqual(first[0]._tag, "Opened") - yield* Deferred.succeed(opened, first[0] as PeerRpc.Opened) - const second = yield* pull - assert.strictEqual(second[0]._tag, "Message") - yield* Deferred.succeed(messageHeld, undefined) - return yield* Effect.never - })).pipe(Effect.forkChild) - const session = yield* Deferred.await(opened) - yield* Deferred.await(messageHeld) - yield* Queue.take(fixture.generated) - assert.strictEqual(yield* Queue.take(fixture.sent), 0) - yield* fixture.setNextOutbound({ - sendSequence: 1, - documentId: taskId, - message, - messageHash: "second", - heads: [], - lineage: Identity.genesisLineage, - writerProvenance: [] - }) - yield* Queue.offer(fixture.commits, { - _tag: "Commit", - commitSequence: Identity.CommitSequence.make(1), - documentId: taskId, - keys: [], - refreshGeneration: 0 - }) - yield* Queue.take(fixture.generated) - for (let index = 0; index < 5; index++) { - yield* Effect.yieldNow - yield* TestClock.adjust(replicaLimits.maxPeerSendMillis + 1) - } - yield* Effect.yieldNow - assert.isTrue(Exit.isFailure(yield* Fiber.await(stream))) - const error = yield* fixture.client.Push({ sessionId: session.sessionId, payload: Uint8Array.of(1) }).pipe( - Effect.flip - ) - assert.instanceOf(error, PeerRpcError.SessionUnavailable) - }))) - - it.effect("maps commit flush send capacity timeout to SessionOverloaded", () => - Effect.scoped(Effect.gen(function*() { - const message = new Uint8Array(replicaLimits.maxSyncMessageBytes) - const fixture = yield* makeFixture({ - ...baseOptions, - blockInbound: true, - rpcLimits: { - maxBufferedBytes: PeerSession.maximumSyncEnvelopeBytes( - replicaLimits.maxSyncMessageBytes, - replicaLimits.maxSyncChangesPerMessage - ) - } - }) - const blocker = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - yield* Queue.take(fixture.pendingStarted) - yield* Queue.take(fixture.generated) - yield* fixture.client.Push({ - sessionId: blocker.opened.sessionId, - payload: yield* fixture.encodeMessage(0, taskId, Task.name, message) - }) - yield* Deferred.await(fixture.inboundBlocked) - yield* fixture.setCredential("foreign") - const session = yield* fixture.open([{ documentType: Note.name, documentId: noteId }]) - yield* Queue.take(fixture.pendingStarted) - yield* Queue.take(fixture.generated) - yield* fixture.setNextOutbound({ - sendSequence: 0, - documentId: noteId, - message, - messageHash: "flush-capacity", - heads: [], - lineage: Identity.genesisLineage, - writerProvenance: maximumWriterProvenance - }) - yield* Queue.offer(fixture.commits, { - _tag: "Commit", - commitSequence: Identity.CommitSequence.make(1), - documentId: noteId, - keys: [], - refreshGeneration: 0 - }) - assert.strictEqual(yield* Queue.take(fixture.generated), noteId) - yield* TestClock.adjust(replicaLimits.maxPeerSendMillis + 1) - const error = yield* Fiber.join(session.fiber).pipe(Effect.flip) - assert.strictEqual(error._tag, "SessionOverloaded") - yield* Deferred.succeed(fixture.inboundRelease, undefined) - yield* Fiber.interrupt(blocker.fiber) - }))) - - it.effect("returns inbound byte reservations after overload cleanup", () => - Effect.scoped(Effect.gen(function*() { - const message = new Uint8Array(replicaLimits.maxSyncMessageBytes) - const fixture = yield* makeFixture({ - ...baseOptions, - blockInbound: true, - rpcLimits: { - maxBufferedBytes: PeerSession.maximumSyncEnvelopeBytes( - replicaLimits.maxSyncMessageBytes, - replicaLimits.maxSyncChangesPerMessage - ) - } - }) - const first = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - const payload = yield* fixture.encodeMessage(0, taskId, Task.name, message) - yield* fixture.client.Push({ sessionId: first.opened.sessionId, payload }) - yield* Deferred.await(fixture.inboundBlocked) - const overload = yield* fixture.client.Push({ - sessionId: first.opened.sessionId, - payload: yield* fixture.encode(1) - }).pipe(Effect.flip) - assert.strictEqual(overload._tag, "SessionOverloaded") - yield* Deferred.succeed(fixture.inboundRelease, undefined) - yield* fixture.setCredential("foreign") - const second = yield* fixture.open([{ documentType: Note.name, documentId: noteId }]) - yield* fixture.client.Push({ - sessionId: second.opened.sessionId, - payload: yield* fixture.encodeMessage(0, noteId, Note.name, message) - }) - assert.strictEqual(yield* Queue.take(fixture.received), 0) - yield* Fiber.interrupt(second.fiber) - }))) - - it.effect("survives repeated replacement followed by server shutdown", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - replicaLimits: { maxSessions: 64 }, - rpcLimits: { maxSessionsPerSubject: 64 } - }) - let current = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - for (let index = 0; index < 20; index++) { - const replacement = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - const exit = yield* Fiber.await(current.fiber) - assert.isTrue(Exit.isFailure(exit)) - if (Exit.isFailure(exit)) { - const error = Cause.findErrorOption(exit.cause) - assert.strictEqual(error._tag, "Some") - if (error._tag === "Some") assert.strictEqual(error.value._tag, "SessionUnavailable") - } - current = replacement - } - yield* fixture.closeServer - const exit = yield* Fiber.await(current.fiber) - assert.isTrue(Exit.isFailure(exit)) - if (Exit.isFailure(exit)) { - const error = Cause.findErrorOption(exit.cause) - assert.strictEqual(error._tag, "Some") - if (error._tag === "Some") assert.strictEqual(error.value._tag, "ServerUnavailable") - } - }))) - - it.effect("coalesces 256 commits while 64 sessions are active", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - replicaLimits: { maxSessions: 64 }, - rpcLimits: { maxSessionsPerSubject: 64 } - }) - const documentIds = Array.from({ length: 64 }, (_, index) => - Identity.DocumentId.make( - `doc_00000000-0000-4000-8000-${index.toString(16).padStart(12, "0")}` - )) - const sessions = yield* Effect.forEach(documentIds, (documentId, index) => - fixture.setCredential(`bulk-${index}`).pipe( - Effect.andThen(fixture.open([{ documentType: Task.name, documentId }])), - Effect.tap(() => - Queue.take(fixture.generated) - ) - )) - yield* fixture.blockCommitGeneration - const commits = Array.from({ length: 256 }, (_, index) => ({ - _tag: "Commit" as const, - commitSequence: Identity.CommitSequence.make(index + 1), - documentId: documentIds[0], - keys: [], - refreshGeneration: 0 - })) - yield* Queue.offer(fixture.commits, commits[0]) - yield* Queue.take(fixture.commitProcessed) - yield* Deferred.await(fixture.commitFlushStarted) - yield* Queue.offerAll(fixture.commits, commits.slice(1)) - for (let index = 1; index < commits.length; index++) yield* Queue.take(fixture.commitProcessed) - yield* Deferred.succeed(fixture.commitFlushRelease, undefined) - assert.strictEqual(yield* Queue.take(fixture.generated), documentIds[0]) - assert.strictEqual(yield* Queue.take(fixture.generated), documentIds[0]) - for (let index = 0; index < 10; index++) yield* Effect.yieldNow - assert.strictEqual((yield* Queue.poll(fixture.generated))._tag, "None") - yield* Effect.forEach(sessions, (session) => Fiber.interrupt(session.fiber), { - concurrency: "unbounded", - discard: true - }) - }))) - - it.effect("routes disjoint commits only to interested sessions", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture(baseOptions) - const task = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - yield* Queue.take(fixture.generated) - yield* fixture.setCredential("foreign") - const note = yield* fixture.open([{ documentType: Note.name, documentId: noteId }]) - yield* Queue.take(fixture.generated) - yield* Queue.offer(fixture.commits, { - _tag: "Commit", - commitSequence: Identity.CommitSequence.make(1), - documentId: taskId, - keys: [], - refreshGeneration: 0 - }) - assert.strictEqual(yield* Queue.take(fixture.generated), taskId) - yield* Fiber.interrupt(task.fiber) - yield* Fiber.interrupt(note.fiber) - }))) - - it.effect("records fixed cardinality metrics and safe finite spans at live boundaries", () => { - const spans: Array = [] - const tracer = Tracer.make({ - span: (options) => { - const span = new Tracer.NativeSpan(options) - spans.push(span) - return span - } - }) - return Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - initialOutbound: { - sendSequence: 0, - documentId: taskId, - message: Uint8Array.of(17, 18, 19), - messageHash: "hash", - heads: [], - lineage: Identity.genesisLineage, - writerProvenance: [] - }, - authorization: (request) => - Effect.logDebug("authorization-log-forbidden-value").pipe( - Effect.andThen( - request.documents.some((document) => document.documentType === Note.name) - ? Effect.fail(new PeerRpcError.AccessDenied()) - : Effect.succeed({ - documents: request.documents.map((document) => ({ document: Task, documentId: document.documentId })), - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.never - }) - ) - ) - }) - const session = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - assert.strictEqual((yield* Queue.take(session.events))._tag, "Message") - yield* fixture.client.Push({ sessionId: session.opened.sessionId, payload: yield* fixture.encode(0) }) - yield* Queue.take(fixture.received) - yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Note.name, documentId: noteId }] - }).pipe(Stream.runDrain, Effect.flip) - yield* fixture.client.Open({ - protocolVersion: PeerRpc.protocolVersion + 1, - expectedPeerId: serverPeerId, - definitionHash, - documents: [{ documentType: Task.name, documentId: taskId }] - }).pipe(Stream.runDrain, Effect.flip) - - assert.strictEqual((yield* Metric.value(PeerRpcObservability.boundary("Open", "Attempt"))).count, 3) - assert.strictEqual((yield* Metric.value(PeerRpcObservability.boundary("Open", "Success"))).count, 1) - assert.strictEqual( - (yield* Metric.value(PeerRpcObservability.boundary("Open", "AuthorizationDenied"))).count, - 1 - ) - assert.strictEqual( - (yield* Metric.value(PeerRpcObservability.boundary("Open", "ProtocolRejected"))).count, - 1 - ) - assert.strictEqual((yield* Metric.value(PeerRpcObservability.boundary("Push", "Attempt"))).count, 1) - assert.strictEqual((yield* Metric.value(PeerRpcObservability.boundary("Push", "Success"))).count, 1) - assert.strictEqual((yield* Metric.value(PeerRpcObservability.activeSessions())).value, 1) - assert.deepInclude(yield* Metric.value(PeerRpcObservability.selectedDocuments()), { count: 3, sum: 3 }) - assert.strictEqual((yield* Metric.value(PeerRpcObservability.bytes("Inbound"))).count, 1) - assert.strictEqual((yield* Metric.value(PeerRpcObservability.bytes("Outbound"))).count, 1) - - yield* fixture.closeServer - yield* Fiber.await(session.fiber) - assert.strictEqual((yield* Metric.value(PeerRpcObservability.activeSessions())).value, 0) - assert.strictEqual((yield* Metric.value(PeerRpcObservability.queueItems("Inbound"))).value, 0) - assert.strictEqual((yield* Metric.value(PeerRpcObservability.queueItems("Outbound"))).value, 0) - assert.strictEqual( - (yield* Metric.value(PeerRpcObservability.boundary("Server", "ShutdownClosed"))).count, - 1 - ) - - const safeSpans = spans.filter((span) => span.name.startsWith("effect_local_rpc.")) - assert.deepStrictEqual( - new Set(safeSpans.map((span) => span.name)), - new Set([ - "effect_local_rpc.authentication", - "effect_local_rpc.server.open", - "effect_local_rpc.server.push" - ]) - ) - const allowedAttributes = new Set([ - "rpc.operation", - "rpc.result", - "rpc.selected_documents", - "rpc.payload_bytes" - ]) - for (const span of safeSpans) { - for (const [key, value] of span.attributes) { - assert.isTrue(allowedAttributes.has(key)) - if (typeof value === "number") assert.isTrue(Number.isFinite(value)) - } - assert.strictEqual(span.status._tag, "Ended") - if (span.status._tag === "Ended") assert.isTrue(Exit.isSuccess(span.status.exit)) - assert.deepStrictEqual(span.events, []) - } - const telemetry = JSON.stringify(safeSpans.map((span) => ({ - name: span.name, - attributes: [...span.attributes], - status: span.status._tag === "Ended" && Exit.isFailure(span.status.exit) - ? Cause.pretty(span.status.exit.cause) - : span.status._tag - }))) + (yield* Metric.dump) - for ( - const forbidden of [ - "owner", - "authorization-log-forbidden-value", - "tenant", - "subject-a", - taskId, - noteId, - remotePeerA, - session.opened.sessionId - ] - ) assert.notInclude(telemetry, forbidden) - })).pipe( - Effect.provideService(Metric.MetricRegistry, new Map()), - Effect.provideService(Tracer.Tracer, tracer) - ) - }) - - it.effect("rejects Push beyond the per subject burst", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - rpcLimits: { pushBurst: 1, pushRatePerSecond: Number.MIN_VALUE } - }) - const session = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - yield* fixture.client.Push({ sessionId: session.opened.sessionId, payload: yield* fixture.encode(0) }) - assert.strictEqual(yield* Queue.take(fixture.received), 0) - const rejected = yield* fixture.client.Push({ - sessionId: session.opened.sessionId, - payload: yield* fixture.encode(1) - }).pipe(Effect.flip) - assert.instanceOf(rejected, PeerRpcError.RequestCapacityExceeded) - assert.strictEqual((yield* Queue.poll(fixture.received))._tag, "None") - yield* Fiber.interrupt(session.fiber) - }))) - - it.effect("keeps the subject Push bucket monotonic when a stale update lands after a fresher one", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - rpcLimits: { pushBurst: 1, pushRatePerSecond: 1 } - }) - const session = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - yield* TestClock.setTime(500) - yield* fixture.directPush({ sessionId: session.opened.sessionId, payload: yield* fixture.encode(0) }) - assert.strictEqual(yield* Queue.take(fixture.received), 0) - - yield* TestClock.setTime(0) - const stale = yield* fixture.directPush({ - sessionId: session.opened.sessionId, - payload: yield* fixture.encode(1) - }).pipe(Effect.flip) - assert.strictEqual(stale._tag, "RequestCapacityExceeded") - - yield* TestClock.setTime(1_000) - const laterExit = yield* fixture.directPush({ - sessionId: session.opened.sessionId, - payload: yield* fixture.encode(1) - }).pipe(Effect.exit) - assert.isTrue(Exit.isFailure(laterExit), "expected the third push to stay capacity rejected") - if (Exit.isFailure(laterExit)) { - const failure = Cause.findErrorOption(laterExit.cause) - assert.isTrue(Option.isSome(failure) && failure.value._tag === "RequestCapacityExceeded") - } - assert.strictEqual((yield* Queue.poll(fixture.received))._tag, "None") - - yield* Fiber.interrupt(session.fiber) - }))) - - it.effect("keeps the subject Open bucket monotonic when a stale update lands after a fresher one", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - rpcLimits: { - openBurst: 1, - openRatePerSecond: 1, - maxInFlightOpenPerSubject: 100, - maxSessionsPerSubject: 100 - } - }) - const documents = [{ documentType: Task.name, documentId: taskId }] - const attemptOpen = () => - Effect.gen(function*() { - const events = yield* Queue.unbounded() - const fiber = yield* Stream.runForEach( - fixture.directOpen(documents), - (event) => Queue.offer(events, event).pipe(Effect.asVoid) - ).pipe(Effect.forkChild) - return yield* Effect.raceFirst( - Queue.take(events).pipe(Effect.as({ _tag: "Admitted" as const, fiber })), - Fiber.await(fiber).pipe(Effect.map((exit) => ({ _tag: "Rejected" as const, exit }))) - ) - }) - - yield* TestClock.setTime(500) - const first = yield* attemptOpen() - assert.strictEqual(first._tag, "Admitted") - - yield* TestClock.setTime(0) - const stale = yield* attemptOpen() - assert.strictEqual(stale._tag, "Rejected") - if (stale._tag === "Rejected") { - assert.isTrue(Exit.isFailure(stale.exit)) - if (Exit.isFailure(stale.exit)) { - const failure = Cause.findErrorOption(stale.exit.cause) - assert.isTrue(Option.isSome(failure) && failure.value._tag === "RequestCapacityExceeded") - } - } - - yield* TestClock.setTime(1_000) - const later = yield* attemptOpen() - assert.strictEqual(later._tag, "Rejected") - if (later._tag === "Rejected") { - assert.isTrue(Exit.isFailure(later.exit)) - if (Exit.isFailure(later.exit)) { - const failure = Cause.findErrorOption(later.exit.cause) - assert.isTrue(Option.isSome(failure) && failure.value._tag === "RequestCapacityExceeded") - } - } - - if (first._tag === "Admitted") yield* Fiber.interrupt(first.fiber) - }))) - - it.effect("does not let a stale Push rewind the shared subject clock advanced by an Open", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - rpcLimits: { pushBurst: 1, pushRatePerSecond: 1 } - }) - const documents = [{ documentType: Task.name, documentId: taskId }] - const openDirect = () => - Effect.gen(function*() { - const events = yield* Queue.unbounded() - const fiber = yield* Stream.runForEach( - fixture.directOpen(documents), - (event) => Queue.offer(events, event).pipe(Effect.asVoid) - ).pipe(Effect.forkChild) - const opened = yield* Effect.raceFirst( - Queue.take(events), - Fiber.join(fiber).pipe(Effect.andThen(Effect.die("Open stream ended before Opened"))) - ) - assert.strictEqual(opened._tag, "Opened") - return { opened: opened as PeerRpc.Opened, fiber } - }) - - yield* TestClock.setTime(0) - const first = yield* openDirect() - - yield* TestClock.setTime(2_000) - yield* fixture.directPush({ sessionId: first.opened.sessionId, payload: yield* fixture.encode(0) }) - assert.strictEqual(yield* Queue.take(fixture.received), 0) - - yield* TestClock.setTime(10_000) - const second = yield* openDirect() - - yield* TestClock.setTime(3_000) - yield* fixture.directPush({ sessionId: second.opened.sessionId, payload: yield* fixture.encode(1) }) - assert.strictEqual(yield* Queue.take(fixture.received), 1) - - yield* TestClock.setTime(4_000) - const laterExit = yield* fixture.directPush({ - sessionId: second.opened.sessionId, - payload: yield* fixture.encode(2) - }).pipe(Effect.exit) - assert.isTrue(Exit.isFailure(laterExit), "expected the fourth push to stay capacity rejected") - if (Exit.isFailure(laterExit)) { - const failure = Cause.findErrorOption(laterExit.cause) - assert.isTrue(Option.isSome(failure) && failure.value._tag === "RequestCapacityExceeded") - } - assert.strictEqual((yield* Queue.poll(fixture.received))._tag, "None") - - yield* Fiber.interrupt(first.fiber) - yield* Fiber.interrupt(second.fiber) - }))) - - it.effect("does not let a stale Push rewind the shared clock advanced by a rejected Open", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - rpcLimits: { - openBurst: 1, - openRatePerSecond: Number.MIN_VALUE, - pushBurst: 1, - pushRatePerSecond: 1 - } - }) - const documents = [{ documentType: Task.name, documentId: taskId }] - const session = yield* fixture.open(documents) - - yield* TestClock.setTime(2_000) - yield* fixture.directPush({ sessionId: session.opened.sessionId, payload: yield* fixture.encode(0) }) - assert.strictEqual(yield* Queue.take(fixture.received), 0) - - yield* TestClock.setTime(10_000) - const rejectedOpen = yield* fixture.directOpen(documents).pipe(Stream.runDrain, Effect.flip) - assert.strictEqual(rejectedOpen._tag, "RequestCapacityExceeded") - - yield* TestClock.setTime(3_000) - yield* fixture.directPush({ sessionId: session.opened.sessionId, payload: yield* fixture.encode(1) }) - assert.strictEqual(yield* Queue.take(fixture.received), 1) - - yield* TestClock.setTime(4_000) - const laterExit = yield* fixture.directPush({ - sessionId: session.opened.sessionId, - payload: yield* fixture.encode(2) - }).pipe(Effect.exit) - assert.isTrue(Exit.isFailure(laterExit), "expected the later Push to remain capacity rejected") - if (Exit.isFailure(laterExit)) { - const failure = Cause.findErrorOption(laterExit.cause) - assert.isTrue(Option.isSome(failure) && failure.value._tag === "RequestCapacityExceeded") - } - assert.strictEqual((yield* Queue.poll(fixture.received))._tag, "None") - - yield* Fiber.interrupt(session.fiber) - }))) - - it.effect("does not let a stale Open ignore the shared subject clock advanced by a Push", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - rpcLimits: { - openBurst: 1, - openRatePerSecond: 1, - maxInFlightOpenPerSubject: 100, - maxSessionsPerSubject: 100 - } - }) - const documents = [{ documentType: Task.name, documentId: taskId }] - const openDirect = () => - Effect.gen(function*() { - const events = yield* Queue.unbounded() - const fiber = yield* Stream.runForEach( - fixture.directOpen(documents), - (event) => Queue.offer(events, event).pipe(Effect.asVoid) - ).pipe(Effect.forkChild) - const opened = yield* Effect.raceFirst( - Queue.take(events), - Fiber.join(fiber).pipe(Effect.andThen(Effect.die("Open stream ended before Opened"))) - ) - assert.strictEqual(opened._tag, "Opened") - return { opened: opened as PeerRpc.Opened, fiber } - }) - - yield* TestClock.setTime(0) - const first = yield* openDirect() - yield* TestClock.setTime(10_000) - yield* fixture.directPush({ sessionId: first.opened.sessionId, payload: yield* fixture.encode(0) }) - assert.strictEqual(yield* Queue.take(fixture.received), 0) - - yield* TestClock.setTime(500) - const second = yield* openDirect() - yield* fixture.directPush({ sessionId: second.opened.sessionId, payload: yield* fixture.encode(1) }) - assert.strictEqual(yield* Queue.take(fixture.received), 1) - - yield* Fiber.interrupt(first.fiber) - yield* Fiber.interrupt(second.fiber) - }))) - - it.effect("keeps inactive subject retention monotonic after a stale admitted Open", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - rpcLimits: { - openBurst: 2, - openRatePerSecond: 1, - rateLimitIdleRetention: 1_000 - }, - authorization: () => Effect.fail(new PeerRpcError.AccessDenied()) - }) - const documents = [{ documentType: Task.name, documentId: taskId }] - const attempt = () => fixture.directOpen(documents).pipe(Stream.runDrain, Effect.flip) - - assert.strictEqual((yield* attempt())._tag, "AccessDenied") - yield* TestClock.setTime(10_000) - assert.strictEqual((yield* attempt())._tag, "AccessDenied") - yield* TestClock.setTime(500) - assert.strictEqual((yield* attempt())._tag, "AccessDenied") - - yield* TestClock.setTime(1_500) - assert.strictEqual((yield* attempt())._tag, "RequestCapacityExceeded") - }))) - - it.effect("admits a Push exactly when fractional refill reaches one token", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture({ - ...baseOptions, - rpcLimits: { pushBurst: 1, pushRatePerSecond: 1 } - }) - const session = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - yield* fixture.directPush({ sessionId: session.opened.sessionId, payload: yield* fixture.encode(0) }) - assert.strictEqual(yield* Queue.take(fixture.received), 0) - - yield* TestClock.setTime(999) - const fractional = yield* fixture.directPush({ - sessionId: session.opened.sessionId, - payload: yield* fixture.encode(1) - }).pipe(Effect.flip) - assert.strictEqual(fractional._tag, "RequestCapacityExceeded") - - yield* TestClock.setTime(1_000) - yield* fixture.directPush({ sessionId: session.opened.sessionId, payload: yield* fixture.encode(1) }) - assert.strictEqual(yield* Queue.take(fixture.received), 1) - yield* Fiber.interrupt(session.fiber) - }))) - - it.effect("rejects a Push payload beyond the sync envelope limit", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture(baseOptions) - const session = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - const oversized = new Uint8Array( - PeerSession.maximumSyncEnvelopeBytes( - replicaLimits.maxSyncMessageBytes, - replicaLimits.maxSyncChangesPerMessage - ) + 1 - ) - const rejected = yield* fixture.client.Push({ - sessionId: session.opened.sessionId, - payload: oversized - }).pipe(Effect.flip) - assert.instanceOf(rejected, PeerRpcError.RequestLimitExceeded) - assert.strictEqual((yield* Queue.poll(fixture.received))._tag, "None") - yield* Fiber.interrupt(session.fiber) - }))) - - it.effect("marks every session dirty on a full refresh commit", () => - Effect.scoped(Effect.gen(function*() { - const fixture = yield* makeFixture(baseOptions) - const task = yield* fixture.open([{ documentType: Task.name, documentId: taskId }]) - assert.strictEqual(yield* Queue.take(fixture.generated), taskId) - yield* fixture.setCredential("foreign") - const note = yield* fixture.open([{ documentType: Note.name, documentId: noteId }]) - assert.strictEqual(yield* Queue.take(fixture.generated), noteId) - yield* Queue.offer(fixture.commits, { _tag: "FullRefreshRequired", refreshGeneration: 1 }) - yield* Queue.take(fixture.commitProcessed) - const refreshed = yield* Effect.all([Queue.take(fixture.generated), Queue.take(fixture.generated)]) - assert.deepStrictEqual(new Set(refreshed), new Set([taskId, noteId])) - yield* Fiber.interrupt(task.fiber) - yield* Fiber.interrupt(note.fiber) - }))) +const remotePeerA = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000002") +const remotePeerB = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000003") +const crypto = Crypto.make({ + randomBytes: (size) => randomBytes(size), + digest: (algorithm, bytes) => + Effect.sync(() => new Uint8Array(createHash(algorithm.replace("-", "").toLowerCase()).update(bytes).digest())) }) const relayTransition: PeerRelayStore.TransitionResult = { @@ -2838,7 +121,7 @@ const relayPrincipal: PeerAuthentication.PeerPrincipal = { } const relayOpenRequest = { - version: PeerRelayRpc.protocolVersion, + protocolVersion: PeerRpc.protocolVersion, expectedRelayPeerId: serverPeerId, expectedLocal: relayPrincipal, senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), @@ -2849,7 +132,7 @@ const relayOpenRequest = { documents: [{ documentType: Task.name, documentId: taskId }], receiptRetentionMillis: PeerRelayLimits.defaults.maximumReceiptRetentionMillis, senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis -} satisfies typeof PeerRelayRpc.OpenRelayRpc.payloadSchema.Type +} satisfies typeof PeerRpc.OpenRpc.payloadSchema.Type const relayAuthenticated = { principal: relayPrincipal, @@ -2916,9 +199,9 @@ const makeRelayPayload = Effect.gen(function*() { const makeRelayClaim = ( relayMessageId: Identity.RelayMessageId, - claimToken: PeerRelayRpc.ClaimToken, + claimToken: PeerRpc.ClaimToken, messageHash: string, - outerEnvelopeDigest: PeerRelayRpc.RelayDigest, + outerEnvelopeDigest: PeerRpc.RelayDigest, payloadBytes: number, sessionGeneration: number ) => @@ -2962,7 +245,7 @@ const makeRelayOuterEnvelopeDigest = ( }, relayPeerId: serverPeerId, relayMessageId, - protocolVersion: PeerRelayRpc.protocolVersion, + protocolVersion: PeerRpc.protocolVersion, payloadVersion: 1, senderReplicaIncarnation: relayOpenRequest.senderReplicaIncarnation, senderConnectionEpoch: "epoch", @@ -2993,7 +276,7 @@ const makeRelayHandlerContext = ( } ) => Layer.build( - PeerRpcServer.layerRelayHandlers({ + PeerRpcServer.layerHandlers({ tenantId: "tenant", peerId: serverPeerId }).pipe( @@ -3021,7 +304,7 @@ const relayHandlerEffect = ( const makeTerminalRelayContext = ( relayMessageId: Identity.RelayMessageId, - claimToken: PeerRelayRpc.ClaimToken, + claimToken: PeerRpc.ClaimToken, authorization: Context.Service.Shape, acknowledge: PeerRelayStore.Service["acknowledge"] ) => @@ -3085,7 +368,7 @@ const makeTerminalRelayContext = ( ) }) -describe("PeerRpcServer relay", () => { +describe("PeerRpcServer", () => { it.effect("denies operation admission after an absolute grant deadline", () => Effect.scoped(Effect.gen(function*() { let now = 0 @@ -3130,11 +413,11 @@ describe("PeerRpcServer relay", () => { }) const context = yield* makeRelayHandlerContext(authorization, store) const openHandler = context.mapUnsafe.get( - PeerRelayRpc.OpenRelayRpc.key - ) as Rpc.Handler<"OpenRelay"> + PeerRpc.OpenRpc.key + ) as Rpc.Handler<"Open"> const pushHandler = context.mapUnsafe.get( - PeerRelayRpc.PushRelayRpc.key - ) as Rpc.Handler<"PushRelay"> + PeerRpc.PushRpc.key + ) as Rpc.Handler<"Push"> const provide = ( handler: Rpc.Handler, effect: Effect.Effect @@ -3152,23 +435,23 @@ describe("PeerRpcServer relay", () => { ) ) ) - const events = yield* Queue.unbounded() + const events = yield* Queue.unbounded() const openFiber = yield* Stream.runForEach( openHandler.handler( relayOpenRequest, {} as never - ) as Stream.Stream, + ) as Stream.Stream, (event) => Queue.offer(events, event) ).pipe( (effect) => provide(openHandler, effect), Effect.forkScoped ) const opened = yield* Queue.take(events) - assert.strictEqual(opened._tag, "RelayOpened") + assert.strictEqual(opened._tag, "Opened") const { payload } = yield* makeRelayPayload const denied = yield* ( pushHandler.handler({ - sessionId: (opened as PeerRelayRpc.RelayOpened).sessionId, + sessionId: (opened as PeerRpc.Opened).sessionId, relayMessageId: Identity.RelayMessageId.make( "rly_00000000-0000-4000-8000-000000000052" ), @@ -3206,14 +489,14 @@ describe("PeerRpcServer relay", () => { makeRelayStore() ).pipe(Effect.provideService(Clock.Clock, testClock)) const handler = context.mapUnsafe.get( - PeerRelayRpc.OpenRelayRpc.key - ) as Rpc.Handler<"OpenRelay"> + PeerRpc.OpenRpc.key + ) as Rpc.Handler<"Open"> const now = yield* Clock.currentTimeMillis const fiber = yield* Stream.runHead( handler.handler( relayOpenRequest, {} as never - ) as Stream.Stream + ) as Stream.Stream ).pipe( Effect.provideContext( Context.add( @@ -3282,25 +565,25 @@ describe("PeerRpcServer relay", () => { }) const context = yield* makeRelayHandlerContext(authorization, store) const openHandler = context.mapUnsafe.get( - PeerRelayRpc.OpenRelayRpc.key - ) as Rpc.Handler<"OpenRelay"> + PeerRpc.OpenRpc.key + ) as Rpc.Handler<"Open"> const pushHandler = context.mapUnsafe.get( - PeerRelayRpc.PushRelayRpc.key - ) as Rpc.Handler<"PushRelay"> - const events = yield* Queue.unbounded() + PeerRpc.PushRpc.key + ) as Rpc.Handler<"Push"> + const events = yield* Queue.unbounded() const openFiber = yield* Stream.runForEach( openHandler.handler( relayOpenRequest, {} as never - ) as Stream.Stream, + ) as Stream.Stream, (event) => Queue.offer(events, event) ).pipe( (effect) => relayHandlerEffect(openHandler, effect), Effect.forkScoped ) const opened = yield* Queue.take(events) - assert.strictEqual(opened._tag, "RelayOpened") - const sessionId = (opened as PeerRelayRpc.RelayOpened).sessionId + assert.strictEqual(opened._tag, "Opened") + const sessionId = (opened as PeerRpc.Opened).sessionId const { payload } = yield* makeRelayPayload const push = (relayMessageId: Identity.RelayMessageId) => pushHandler.handler({ @@ -3381,20 +664,20 @@ describe("PeerRpcServer relay", () => { }) const context = yield* makeRelayHandlerContext(authorization, store) const openHandler = context.mapUnsafe.get( - PeerRelayRpc.OpenRelayRpc.key - ) as Rpc.Handler<"OpenRelay"> - const events = yield* Queue.unbounded() + PeerRpc.OpenRpc.key + ) as Rpc.Handler<"Open"> + const events = yield* Queue.unbounded() const openFiber = yield* Stream.runForEach( openHandler.handler( relayOpenRequest, {} as never - ) as Stream.Stream, + ) as Stream.Stream, (event) => Queue.offer(events, event) ).pipe( (effect) => relayHandlerEffect(openHandler, effect), Effect.forkScoped ) - assert.strictEqual((yield* Queue.take(events))._tag, "RelayOpened") + assert.strictEqual((yield* Queue.take(events))._tag, "Opened") yield* Deferred.await(receiveDenied) yield* Effect.yieldNow assert.strictEqual(yield* Ref.get(claimCalls), 0) @@ -3440,29 +723,29 @@ describe("PeerRpcServer relay", () => { }) const context = yield* makeRelayHandlerContext(authorization, store) const openHandler = context.mapUnsafe.get( - PeerRelayRpc.OpenRelayRpc.key - ) as Rpc.Handler<"OpenRelay"> + PeerRpc.OpenRpc.key + ) as Rpc.Handler<"Open"> const pushHandler = context.mapUnsafe.get( - PeerRelayRpc.PushRelayRpc.key - ) as Rpc.Handler<"PushRelay"> - const events = yield* Queue.unbounded() + PeerRpc.PushRpc.key + ) as Rpc.Handler<"Push"> + const events = yield* Queue.unbounded() const openFiber = yield* Stream.runForEach( openHandler.handler( relayOpenRequest, {} as never - ) as Stream.Stream, + ) as Stream.Stream, (event) => Queue.offer(events, event) ).pipe( (effect) => relayHandlerEffect(openHandler, effect), Effect.forkScoped ) const opened = yield* Queue.take(events) - assert.strictEqual(opened._tag, "RelayOpened") + assert.strictEqual(opened._tag, "Opened") pushing = true const { payload } = yield* makeRelayPayload const pushFiber = yield* ( pushHandler.handler({ - sessionId: (opened as PeerRelayRpc.RelayOpened).sessionId, + sessionId: (opened as PeerRpc.Opened).sessionId, relayMessageId: Identity.RelayMessageId.make( "rly_00000000-0000-4000-8000-000000000043" ), @@ -3482,7 +765,7 @@ describe("PeerRpcServer relay", () => { assert.strictEqual(yield* Ref.get(commits), 1) const later = yield* ( pushHandler.handler({ - sessionId: (opened as PeerRelayRpc.RelayOpened).sessionId, + sessionId: (opened as PeerRpc.Opened).sessionId, relayMessageId: Identity.RelayMessageId.make( "rly_00000000-0000-4000-8000-000000000046" ), @@ -3518,7 +801,7 @@ describe("PeerRpcServer relay", () => { const acknowledgeCalls = yield* Ref.make(0) const context = yield* makeTerminalRelayContext( Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000050"), - PeerRelayRpc.ClaimToken.make("clm_00000000-0000-4000-8000-000000000050"), + PeerRpc.ClaimToken.make("clm_00000000-0000-4000-8000-000000000050"), authorization, () => Ref.update(acknowledgeCalls, (count) => count + 1).pipe( @@ -3526,17 +809,17 @@ describe("PeerRpcServer relay", () => { ) ) const openHandler = context.mapUnsafe.get( - PeerRelayRpc.OpenRelayRpc.key - ) as Rpc.Handler<"OpenRelay"> + PeerRpc.OpenRpc.key + ) as Rpc.Handler<"Open"> const acknowledgeHandler = context.mapUnsafe.get( - PeerRelayRpc.AcknowledgeRelayRpc.key - ) as Rpc.Handler<"AcknowledgeRelay"> - const events = yield* Queue.unbounded() + PeerRpc.AcknowledgeRpc.key + ) as Rpc.Handler<"Acknowledge"> + const events = yield* Queue.unbounded() const openFiber = yield* Stream.runForEach( openHandler.handler( relayOpenRequest, {} as never - ) as Stream.Stream, + ) as Stream.Stream, (event) => Queue.offer(events, event) ).pipe( (effect) => relayHandlerEffect(openHandler, effect), @@ -3544,7 +827,7 @@ describe("PeerRpcServer relay", () => { ) const opened = yield* Queue.take(events) const stored = yield* Queue.take(events) - if (opened._tag !== "RelayOpened" || stored._tag !== "StoredMessage") { + if (opened._tag !== "Opened" || stored._tag !== "StoredMessage") { return assert.fail("Expected relay open and stored message events") } const denied = yield* ( @@ -3587,7 +870,7 @@ describe("PeerRpcServer relay", () => { const acknowledgeCalls = yield* Ref.make(0) const context = yield* makeTerminalRelayContext( Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000051"), - PeerRelayRpc.ClaimToken.make("clm_00000000-0000-4000-8000-000000000051"), + PeerRpc.ClaimToken.make("clm_00000000-0000-4000-8000-000000000051"), authorization, () => Effect.uninterruptible( @@ -3599,18 +882,18 @@ describe("PeerRpcServer relay", () => { ) ) const openHandler = context.mapUnsafe.get( - PeerRelayRpc.OpenRelayRpc.key - ) as Rpc.Handler<"OpenRelay"> + PeerRpc.OpenRpc.key + ) as Rpc.Handler<"Open"> const acknowledgeHandler = context.mapUnsafe.get( - PeerRelayRpc.AcknowledgeRelayRpc.key - ) as Rpc.Handler<"AcknowledgeRelay"> - const runtime = Context.get(context, PeerRpcServer.PeerRelayServerRuntime) - const events = yield* Queue.unbounded() + PeerRpc.AcknowledgeRpc.key + ) as Rpc.Handler<"Acknowledge"> + const runtime = Context.get(context, PeerRpcServer.PeerRpcServerRuntime) + const events = yield* Queue.unbounded() const openFiber = yield* Stream.runForEach( openHandler.handler( relayOpenRequest, {} as never - ) as Stream.Stream, + ) as Stream.Stream, (event) => Queue.offer(events, event) ).pipe( (effect) => relayHandlerEffect(openHandler, effect), @@ -3618,7 +901,7 @@ describe("PeerRpcServer relay", () => { ) const opened = yield* Queue.take(events) const stored = yield* Queue.take(events) - if (opened._tag !== "RelayOpened" || stored._tag !== "StoredMessage") { + if (opened._tag !== "Opened" || stored._tag !== "StoredMessage") { return assert.fail("Expected relay open and stored message events") } const terminalFiber = yield* ( @@ -3648,7 +931,7 @@ describe("PeerRpcServer relay", () => { const relayMessageId = Identity.RelayMessageId.make( "rly_00000000-0000-4000-8000-000000000047" ) - const claimToken = PeerRelayRpc.ClaimToken.make( + const claimToken = PeerRpc.ClaimToken.make( "clm_00000000-0000-4000-8000-000000000047" ) const { messageHash, payload } = yield* makeRelayPayload @@ -3736,21 +1019,21 @@ describe("PeerRpcServer relay", () => { ingress ) const openHandler = context.mapUnsafe.get( - PeerRelayRpc.OpenRelayRpc.key - ) as Rpc.Handler<"OpenRelay"> - const runtime = Context.get(context, PeerRpcServer.PeerRelayServerRuntime) - const events = yield* Queue.unbounded() + PeerRpc.OpenRpc.key + ) as Rpc.Handler<"Open"> + const runtime = Context.get(context, PeerRpcServer.PeerRpcServerRuntime) + const events = yield* Queue.unbounded() const openFiber = yield* Stream.runForEach( openHandler.handler( relayOpenRequest, {} as never - ) as Stream.Stream, + ) as Stream.Stream, (event) => Queue.offer(events, event) ).pipe( (effect) => relayHandlerEffect(openHandler, effect), Effect.forkScoped ) - assert.strictEqual((yield* Queue.take(events))._tag, "RelayOpened") + assert.strictEqual((yield* Queue.take(events))._tag, "Opened") yield* Deferred.await(claimStarted) yield* Deferred.succeed(receiveInvalidated, undefined) yield* Effect.yieldNow @@ -3766,7 +1049,7 @@ describe("PeerRpcServer relay", () => { const relayMessageId = Identity.RelayMessageId.make( "rly_00000000-0000-4000-8000-000000000048" ) - const claimToken = PeerRelayRpc.ClaimToken.make( + const claimToken = PeerRpc.ClaimToken.make( "clm_00000000-0000-4000-8000-000000000048" ) const { messageHash, payload } = yield* makeRelayPayload @@ -3848,22 +1131,22 @@ describe("PeerRpcServer relay", () => { ingress ) const handler = context.mapUnsafe.get( - PeerRelayRpc.OpenRelayRpc.key - ) as Rpc.Handler<"OpenRelay"> - const runtime = Context.get(context, PeerRpcServer.PeerRelayServerRuntime) - const events = yield* Queue.unbounded() + PeerRpc.OpenRpc.key + ) as Rpc.Handler<"Open"> + const runtime = Context.get(context, PeerRpcServer.PeerRpcServerRuntime) + const events = yield* Queue.unbounded() const openFiber = yield* Stream.runForEach( handler.handler( relayOpenRequest, {} as never - ) as Stream.Stream, + ) as Stream.Stream, (event) => Queue.offer(events, event) ).pipe( (effect) => relayHandlerEffect(handler, effect), Effect.exit, Effect.forkScoped ) - assert.strictEqual((yield* Queue.take(events))._tag, "RelayOpened") + assert.strictEqual((yield* Queue.take(events))._tag, "Opened") yield* Deferred.await(transferStarted) yield* Deferred.succeed(sessionInvalidated, undefined) for (let index = 0; index < 20; index++) yield* Effect.yieldNow @@ -3879,7 +1162,7 @@ describe("PeerRpcServer relay", () => { const relayMessageId = Identity.RelayMessageId.make( "rly_00000000-0000-4000-8000-000000000044" ) - const claimToken = PeerRelayRpc.ClaimToken.make( + const claimToken = PeerRpc.ClaimToken.make( "clm_00000000-0000-4000-8000-000000000044" ) const { messageHash, payload } = yield* makeRelayPayload @@ -3955,21 +1238,21 @@ describe("PeerRpcServer relay", () => { ingress ) const openHandler = context.mapUnsafe.get( - PeerRelayRpc.OpenRelayRpc.key - ) as Rpc.Handler<"OpenRelay"> - const runtime = Context.get(context, PeerRpcServer.PeerRelayServerRuntime) - const events = yield* Queue.unbounded() + PeerRpc.OpenRpc.key + ) as Rpc.Handler<"Open"> + const runtime = Context.get(context, PeerRpcServer.PeerRpcServerRuntime) + const events = yield* Queue.unbounded() const openFiber = yield* Stream.runForEach( openHandler.handler( relayOpenRequest, {} as never - ) as Stream.Stream, + ) as Stream.Stream, (event) => Queue.offer(events, event) ).pipe( (effect) => relayHandlerEffect(openHandler, effect), Effect.forkScoped ) - assert.strictEqual((yield* Queue.take(events))._tag, "RelayOpened") + assert.strictEqual((yield* Queue.take(events))._tag, "Opened") yield* Deferred.await(released) yield* Effect.yieldNow yield* Effect.yieldNow @@ -3984,7 +1267,7 @@ describe("PeerRpcServer relay", () => { const relayMessageId = Identity.RelayMessageId.make( "rly_00000000-0000-4000-8000-000000000045" ) - const claimToken = PeerRelayRpc.ClaimToken.make( + const claimToken = PeerRpc.ClaimToken.make( "clm_00000000-0000-4000-8000-000000000045" ) const { messageHash, payload } = yield* makeRelayPayload @@ -4054,28 +1337,28 @@ describe("PeerRpcServer relay", () => { ingress ) const openHandler = context.mapUnsafe.get( - PeerRelayRpc.OpenRelayRpc.key - ) as Rpc.Handler<"OpenRelay"> + PeerRpc.OpenRpc.key + ) as Rpc.Handler<"Open"> const acknowledgeHandler = context.mapUnsafe.get( - PeerRelayRpc.AcknowledgeRelayRpc.key - ) as Rpc.Handler<"AcknowledgeRelay"> - const runtime = Context.get(context, PeerRpcServer.PeerRelayServerRuntime) - const events = yield* Queue.unbounded() + PeerRpc.AcknowledgeRpc.key + ) as Rpc.Handler<"Acknowledge"> + const runtime = Context.get(context, PeerRpcServer.PeerRpcServerRuntime) + const events = yield* Queue.unbounded() const openFiber = yield* Stream.runForEach( openHandler.handler( relayOpenRequest, {} as never - ) as Stream.Stream, + ) as Stream.Stream, (event) => Queue.offer(events, event) ).pipe( (effect) => relayHandlerEffect(openHandler, effect), Effect.forkScoped ) const opened = yield* Queue.take(events) - assert.strictEqual(opened._tag, "RelayOpened") + assert.strictEqual(opened._tag, "Opened") const stored = yield* Queue.take(events) assert.strictEqual(stored._tag, "StoredMessage") - if (opened._tag !== "RelayOpened" || stored._tag !== "StoredMessage") { + if (opened._tag !== "Opened" || stored._tag !== "StoredMessage") { return assert.fail("Expected relay open and stored message events") } const error = yield* ( @@ -4138,21 +1421,21 @@ describe("PeerRpcServer relay", () => { }) const context = yield* makeRelayHandlerContext(authorization, store) const openHandler = context.mapUnsafe.get( - PeerRelayRpc.OpenRelayRpc.key - ) as Rpc.Handler<"OpenRelay"> - const runtime = Context.get(context, PeerRpcServer.PeerRelayServerRuntime) - const events = yield* Queue.unbounded() + PeerRpc.OpenRpc.key + ) as Rpc.Handler<"Open"> + const runtime = Context.get(context, PeerRpcServer.PeerRpcServerRuntime) + const events = yield* Queue.unbounded() const openFiber = yield* Stream.runForEach( openHandler.handler( relayOpenRequest, {} as never - ) as Stream.Stream, + ) as Stream.Stream, (event) => Queue.offer(events, event) ).pipe( (effect) => relayHandlerEffect(openHandler, effect), Effect.forkScoped ) - assert.strictEqual((yield* Queue.take(events))._tag, "RelayOpened") + assert.strictEqual((yield* Queue.take(events))._tag, "Opened") yield* Deferred.await(claimStarted) yield* Deferred.succeed(sessionInvalidated, undefined) yield* Effect.yieldNow @@ -4259,7 +1542,7 @@ describe("PeerRpcServer relay", () => { usage: Effect.succeed({ connections: 0, reservedBytes: 0, byteReservationWaiters: 0 }), await: Effect.never }) - const handlers = PeerRpcServer.layerRelayHandlers({ + const handlers = PeerRpcServer.layerHandlers({ tenantId: "tenant", peerId: serverPeerId }).pipe( @@ -4272,12 +1555,12 @@ describe("PeerRpcServer relay", () => { )) ) const context = yield* Layer.build(handlers) - const openHandler = context.mapUnsafe.get(PeerRelayRpc.OpenRelayRpc.key) as Rpc.Handler<"OpenRelay"> - const pushHandler = context.mapUnsafe.get(PeerRelayRpc.PushRelayRpc.key) as Rpc.Handler<"PushRelay"> + const openHandler = context.mapUnsafe.get(PeerRpc.OpenRpc.key) as Rpc.Handler<"Open"> + const pushHandler = context.mapUnsafe.get(PeerRpc.PushRpc.key) as Rpc.Handler<"Push"> const acknowledgeHandler = context.mapUnsafe.get( - PeerRelayRpc.AcknowledgeRelayRpc.key - ) as Rpc.Handler<"AcknowledgeRelay"> - const runtime = Context.get(context, PeerRpcServer.PeerRelayServerRuntime) + PeerRpc.AcknowledgeRpc.key + ) as Rpc.Handler<"Acknowledge"> + const runtime = Context.get(context, PeerRpcServer.PeerRpcServerRuntime) const protocolStopped = yield* Deferred.make() const ingressStopped = yield* Deferred.make() const baseProtocol = yield* RpcServer.Protocol.make(() => @@ -4300,7 +1583,7 @@ describe("PeerRpcServer relay", () => { ) } const openRequest = { - version: PeerRelayRpc.protocolVersion, + protocolVersion: PeerRpc.protocolVersion, expectedRelayPeerId: serverPeerId, expectedLocal: localPrincipal, senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), @@ -4311,7 +1594,7 @@ describe("PeerRpcServer relay", () => { documents: [{ documentType: Task.name, documentId: taskId }], receiptRetentionMillis: PeerRelayLimits.defaults.maximumReceiptRetentionMillis, senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis - } satisfies typeof PeerRelayRpc.OpenRelayRpc.payloadSchema.Type + } satisfies typeof PeerRpc.OpenRpc.payloadSchema.Type const authenticated = { principal: localPrincipal, validUntil: Number.MAX_SAFE_INTEGER, @@ -4320,8 +1603,8 @@ describe("PeerRpcServer relay", () => { const stream = openHandler.handler( openRequest, {} as never - ) as Stream.Stream - const events = yield* Queue.unbounded() + ) as Stream.Stream + const events = yield* Queue.unbounded() const openFiber = yield* Stream.runForEach( stream.pipe( Stream.provideContext(Context.add( @@ -4333,9 +1616,9 @@ describe("PeerRpcServer relay", () => { (event) => Queue.offer(events, event) ).pipe(Effect.forkScoped) const opened = yield* Queue.take(events) - assert.strictEqual(opened._tag, "RelayOpened") + assert.strictEqual(opened._tag, "Opened") assert.strictEqual( - (opened as PeerRelayRpc.RelayOpened).remotePeerId, + (opened as PeerRpc.Opened).remotePeerId, remotePeerB ) const relayMessageId = Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000001") @@ -4366,7 +1649,7 @@ describe("PeerRpcServer relay", () => { }, relayPeerId: serverPeerId, relayMessageId, - protocolVersion: PeerRelayRpc.protocolVersion, + protocolVersion: PeerRpc.protocolVersion, payloadVersion: 1, senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), senderConnectionEpoch: "epoch", @@ -4381,7 +1664,7 @@ describe("PeerRpcServer relay", () => { payload }).pipe(Effect.provideService(Crypto.Crypto, crypto)) yield* (pushHandler.handler({ - sessionId: (opened as PeerRelayRpc.RelayOpened).sessionId, + sessionId: (opened as PeerRpc.Opened).sessionId, relayMessageId, payload }, {} as never) as Effect.Effect).pipe( @@ -4398,7 +1681,7 @@ describe("PeerRpcServer relay", () => { (openHandler.handler( openRequest, {} as never - ) as Stream.Stream).pipe( + ) as Stream.Stream).pipe( Stream.provideContext(Context.add( openHandler.context, PeerAuthentication.AuthenticatedPeer, @@ -4412,7 +1695,7 @@ describe("PeerRpcServer relay", () => { } authorizationDefect = undefined const ownerContext = yield* Layer.build(Layer.fresh(handlers)) - const ownerRuntime = Context.get(ownerContext, PeerRpcServer.PeerRelayServerRuntime) + const ownerRuntime = Context.get(ownerContext, PeerRpcServer.PeerRpcServerRuntime) const serverContext = Context.add( Context.add( Context.add( @@ -4438,7 +1721,7 @@ describe("PeerRpcServer relay", () => { ) ) yield* Layer.build( - PeerRpcServer.layerRelayServer.pipe( + PeerRpcServer.layerServer.pipe( Layer.provide(Layer.succeedContext(serverContext)) ) ) @@ -4457,9 +1740,9 @@ describe("PeerRpcServer relay", () => { yield* runtime.shutdown yield* runtime.shutdown const stale = yield* (acknowledgeHandler.handler({ - sessionId: (opened as PeerRelayRpc.RelayOpened).sessionId, + sessionId: (opened as PeerRpc.Opened).sessionId, relayMessageId, - claimToken: PeerRelayRpc.ClaimToken.make("clm_00000000-0000-4000-8000-000000000001"), + claimToken: PeerRpc.ClaimToken.make("clm_00000000-0000-4000-8000-000000000001"), messageHash }, {} as never) as Effect.Effect).pipe( Effect.provideContext(Context.add( @@ -4486,7 +1769,7 @@ describe("PeerRpcServer relay", () => { const relayMessageId = Identity.RelayMessageId.make( "rly_00000000-0000-4000-8000-000000000021" ) - const claimToken = PeerRelayRpc.ClaimToken.make( + const claimToken = PeerRpc.ClaimToken.make( "clm_00000000-0000-4000-8000-000000000021" ) const { messageHash, payload } = yield* makeRelayPayload @@ -4501,7 +1784,7 @@ describe("PeerRpcServer relay", () => { }, relayPeerId: serverPeerId, relayMessageId, - protocolVersion: PeerRelayRpc.protocolVersion, + protocolVersion: PeerRpc.protocolVersion, payloadVersion: 1, senderReplicaIncarnation: relayOpenRequest.senderReplicaIncarnation, senderConnectionEpoch: "epoch", @@ -4619,21 +1902,21 @@ describe("PeerRpcServer relay", () => { ingress ) const handler = context.mapUnsafe.get( - PeerRelayRpc.OpenRelayRpc.key - ) as Rpc.Handler<"OpenRelay"> - const events = yield* Queue.unbounded() + PeerRpc.OpenRpc.key + ) as Rpc.Handler<"Open"> + const events = yield* Queue.unbounded() const openFiber = yield* Stream.runForEach( handler.handler( relayOpenRequest, {} as never - ) as Stream.Stream, + ) as Stream.Stream, (event) => Queue.offer(events, event) ).pipe( (effect) => relayHandlerEffect(handler, effect), Effect.forkScoped ) const opened = yield* Queue.take(events) - assert.strictEqual(opened._tag, "RelayOpened") + assert.strictEqual(opened._tag, "Opened") assert.strictEqual(yield* Deferred.await(deliverySettled), "Released") assert.strictEqual(yield* Ref.get(storeReleaseCalls), 1) assert.strictEqual(yield* Ref.get(reservationReleaseCalls), 1) @@ -4684,13 +1967,13 @@ describe("PeerRpcServer relay", () => { }) ) const handler = context.mapUnsafe.get( - PeerRelayRpc.OpenRelayRpc.key - ) as Rpc.Handler<"OpenRelay"> + PeerRpc.OpenRpc.key + ) as Rpc.Handler<"Open"> const invoke = Stream.runHead( handler.handler( relayOpenRequest, {} as never - ) as Stream.Stream + ) as Stream.Stream ).pipe( (effect) => relayHandlerEffect(handler, effect), Effect.asVoid @@ -4706,6 +1989,94 @@ describe("PeerRpcServer relay", () => { yield* Fiber.join(first) }))) + it.effect("detaches the incumbent when an interrupted replacement acquires the registry lock", () => + Effect.scoped(Effect.gen(function*() { + const replacementAuthorized = yield* Deferred.make() + let trackReplacement = false + let replacementAuthorizationCalls = 0 + const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ + authorize: (request) => { + if (trackReplacement) { + replacementAuthorizationCalls += 1 + if (replacementAuthorizationCalls === 2) { + Deferred.doneUnsafe(replacementAuthorized, Effect.void) + } + } + return authorizeRelay(request) + }, + authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode + }) + const context = yield* makeRelayHandlerContext(authorization, makeRelayStore()) + const handler = context.mapUnsafe.get( + PeerRpc.OpenRpc.key + ) as Rpc.Handler<"Open"> + const runtime = Context.get(context, PeerRpcServer.PeerRpcServerRuntime) + const invoke = (request: typeof PeerRpc.OpenRpc.payloadSchema.Type) => + Stream.runForEach( + handler.handler( + request, + {} as never + ) as Stream.Stream, + () => Effect.void + ).pipe((effect) => relayHandlerEffect(handler, effect)) + const incumbent = yield* invoke(relayOpenRequest).pipe(Effect.forkScoped) + const blockerRequest = { + ...relayOpenRequest, + remote: { + subjectId: "registry-blocker", + peerId: remotePeerB + } + } + const blocker = yield* invoke(blockerRequest).pipe(Effect.forkScoped) + while ((yield* runtime.usage).sessions !== 2) { + yield* Effect.yieldNow + } + + const registryLocked = yield* Deferred.make() + const releaseRegistry = yield* Deferred.make() + const setRelayActiveClaims = PeerRpcObservability.setRelayActiveClaims + const metricSpy = vi.spyOn(PeerRpcObservability, "setRelayActiveClaims") + .mockImplementation((amount) => + Deferred.succeed(registryLocked, undefined).pipe( + Effect.andThen(Deferred.await(releaseRegistry)), + Effect.andThen(setRelayActiveClaims(amount)) + ) + ) + + yield* Effect.gen(function*() { + const interruptBlocker = yield* Fiber.interrupt(blocker).pipe(Effect.forkScoped) + yield* Deferred.await(registryLocked) + + trackReplacement = true + const replacement = yield* invoke(relayOpenRequest).pipe(Effect.forkScoped) + yield* Deferred.await(replacementAuthorized) + yield* Effect.forEach( + Array.from({ length: 10 }), + () => Effect.yieldNow, + { discard: true } + ) + const interruptReplacement = yield* Fiber.interrupt(replacement).pipe(Effect.forkScoped) + yield* Effect.forEach( + Array.from({ length: 10 }), + () => Effect.yieldNow, + { discard: true } + ) + assert.isUndefined(interruptReplacement.pollUnsafe()) + + yield* Deferred.succeed(releaseRegistry, undefined) + yield* Fiber.join(interruptBlocker) + yield* Fiber.join(interruptReplacement) + assert.strictEqual((yield* runtime.usage).sessions, 0) + yield* Fiber.interrupt(incumbent) + }).pipe( + Effect.ensuring( + Deferred.succeed(releaseRegistry, undefined).pipe( + Effect.andThen(Effect.sync(() => metricSpy.mockRestore())) + ) + ) + ) + }))) + it.effect("removes a newly registered Open when replacement cleanup is interrupted", () => Effect.scoped(Effect.gen(function*() { const lateMonitors = yield* Ref.make(0) @@ -4739,7 +2110,7 @@ describe("PeerRpcServer relay", () => { const relayMessageId = Identity.RelayMessageId.make( "rly_00000000-0000-4000-8000-000000000031" ) - const claimToken = PeerRelayRpc.ClaimToken.make( + const claimToken = PeerRpc.ClaimToken.make( "clm_00000000-0000-4000-8000-000000000031" ) const { messageHash, payload } = yield* makeRelayPayload @@ -4754,7 +2125,7 @@ describe("PeerRpcServer relay", () => { }, relayPeerId: serverPeerId, relayMessageId, - protocolVersion: PeerRelayRpc.protocolVersion, + protocolVersion: PeerRpc.protocolVersion, payloadVersion: 1, senderReplicaIncarnation: relayOpenRequest.senderReplicaIncarnation, senderConnectionEpoch: "epoch", @@ -4841,21 +2212,21 @@ describe("PeerRpcServer relay", () => { ingress ) const handler = context.mapUnsafe.get( - PeerRelayRpc.OpenRelayRpc.key - ) as Rpc.Handler<"OpenRelay"> - const runtime = Context.get(context, PeerRpcServer.PeerRelayServerRuntime) - const events = yield* Queue.unbounded() + PeerRpc.OpenRpc.key + ) as Rpc.Handler<"Open"> + const runtime = Context.get(context, PeerRpcServer.PeerRpcServerRuntime) + const events = yield* Queue.unbounded() const first = yield* Stream.runForEach( handler.handler( relayOpenRequest, {} as never - ) as Stream.Stream, + ) as Stream.Stream, (event) => Queue.offer(events, event) ).pipe( (effect) => relayHandlerEffect(handler, effect), Effect.forkScoped ) - assert.strictEqual((yield* Queue.take(events))._tag, "RelayOpened") + assert.strictEqual((yield* Queue.take(events))._tag, "Opened") assert.strictEqual((yield* Queue.take(events))._tag, "StoredMessage") assert.strictEqual((yield* runtime.usage).activeClaims, 1) assert.strictEqual((yield* Metric.value(PeerRpcObservability.relayActiveClaims())).value, 1) @@ -4881,7 +2252,7 @@ describe("PeerRpcServer relay", () => { handler.handler( relayOpenRequest, {} as never - ) as Stream.Stream + ) as Stream.Stream ).pipe( (effect) => relayHandlerEffect(handler, effect), Effect.forkScoped @@ -4904,25 +2275,25 @@ describe("PeerRpcServer relay", () => { claimed = false releaseStarted = yield* Deferred.make() releaseGate = yield* Deferred.make() - const nextEvents = yield* Queue.unbounded() + const nextEvents = yield* Queue.unbounded() const nextIncumbent = yield* Stream.runForEach( handler.handler( relayOpenRequest, {} as never - ) as Stream.Stream, + ) as Stream.Stream, (event) => Queue.offer(nextEvents, event) ).pipe( (effect) => relayHandlerEffect(handler, effect), Effect.forkScoped ) - assert.strictEqual((yield* Queue.take(nextEvents))._tag, "RelayOpened") + assert.strictEqual((yield* Queue.take(nextEvents))._tag, "Opened") assert.strictEqual((yield* Queue.take(nextEvents))._tag, "StoredMessage") trackLateMonitors = true const staleOpen = yield* Stream.runHead( handler.handler( relayOpenRequest, {} as never - ) as Stream.Stream + ) as Stream.Stream ).pipe( (effect) => relayHandlerEffect(handler, effect), Effect.forkScoped @@ -4934,7 +2305,7 @@ describe("PeerRpcServer relay", () => { handler.handler( relayOpenRequest, {} as never - ) as Stream.Stream + ) as Stream.Stream ).pipe((effect) => relayHandlerEffect(handler, effect)) yield* Deferred.succeed(releaseGate, undefined) yield* Fiber.join(staleOpen) @@ -5010,29 +2381,29 @@ describe("PeerRpcServer relay", () => { }) const context = yield* makeRelayHandlerContext(authorization, store) const openHandler = context.mapUnsafe.get( - PeerRelayRpc.OpenRelayRpc.key - ) as Rpc.Handler<"OpenRelay"> + PeerRpc.OpenRpc.key + ) as Rpc.Handler<"Open"> const pushHandler = context.mapUnsafe.get( - PeerRelayRpc.PushRelayRpc.key - ) as Rpc.Handler<"PushRelay"> - const runtime = Context.get(context, PeerRpcServer.PeerRelayServerRuntime) - const events = yield* Queue.unbounded() + PeerRpc.PushRpc.key + ) as Rpc.Handler<"Push"> + const runtime = Context.get(context, PeerRpcServer.PeerRpcServerRuntime) + const events = yield* Queue.unbounded() const openFiber = yield* Stream.runForEach( openHandler.handler( relayOpenRequest, {} as never - ) as Stream.Stream, + ) as Stream.Stream, (event) => Queue.offer(events, event) ).pipe( (effect) => relayHandlerEffect(openHandler, effect), Effect.forkScoped ) const opened = yield* Queue.take(events) - assert.strictEqual(opened._tag, "RelayOpened") + assert.strictEqual(opened._tag, "Opened") const { payload } = yield* makeRelayPayload const pushFiber = yield* ( pushHandler.handler({ - sessionId: (opened as PeerRelayRpc.RelayOpened).sessionId, + sessionId: (opened as PeerRpc.Opened).sessionId, relayMessageId: Identity.RelayMessageId.make( "rly_00000000-0000-4000-8000-000000000041" ), @@ -5054,7 +2425,7 @@ describe("PeerRpcServer relay", () => { assert.strictEqual(yield* Ref.get(admitCalls), 1) const later = yield* ( pushHandler.handler({ - sessionId: (opened as PeerRelayRpc.RelayOpened).sessionId, + sessionId: (opened as PeerRpc.Opened).sessionId, relayMessageId: Identity.RelayMessageId.make( "rly_00000000-0000-4000-8000-000000000049" ), @@ -5128,26 +2499,26 @@ describe("PeerRpcServer relay", () => { }) ) const openHandler = context.mapUnsafe.get( - PeerRelayRpc.OpenRelayRpc.key - ) as Rpc.Handler<"OpenRelay"> + PeerRpc.OpenRpc.key + ) as Rpc.Handler<"Open"> const pushHandler = context.mapUnsafe.get( - PeerRelayRpc.PushRelayRpc.key - ) as Rpc.Handler<"PushRelay"> - const runtime = Context.get(context, PeerRpcServer.PeerRelayServerRuntime) - const events = yield* Queue.unbounded() + PeerRpc.PushRpc.key + ) as Rpc.Handler<"Push"> + const runtime = Context.get(context, PeerRpcServer.PeerRpcServerRuntime) + const events = yield* Queue.unbounded() const openFiber = yield* Stream.runForEach( openHandler.handler( relayOpenRequest, {} as never - ) as Stream.Stream, + ) as Stream.Stream, (event) => Queue.offer(events, event) ).pipe( (effect) => relayHandlerEffect(openHandler, effect), Effect.forkScoped ) const opened = yield* Queue.take(events) - assert.strictEqual(opened._tag, "RelayOpened") - const sessionId = (opened as PeerRelayRpc.RelayOpened).sessionId + assert.strictEqual(opened._tag, "Opened") + const sessionId = (opened as PeerRpc.Opened).sessionId const { payload } = yield* makeRelayPayload const invoke = (relayMessageId: Identity.RelayMessageId) => pushHandler.handler({ diff --git a/packages/local-rpc/test/PeerRpcServerSessionTransport.test.ts b/packages/local-rpc/test/PeerRpcServerSessionTransport.test.ts deleted file mode 100644 index 24249c6..0000000 --- a/packages/local-rpc/test/PeerRpcServerSessionTransport.test.ts +++ /dev/null @@ -1,862 +0,0 @@ -import { assert, describe, it } from "@effect/vitest" -import * as CommitPublisher from "@lucas-barake/effect-local-sql/CommitPublisher" -import type * as DocumentEntity from "@lucas-barake/effect-local-sql/DocumentEntity" -import * as PeerSession from "@lucas-barake/effect-local-sql/PeerSession" -import * as PeerSync from "@lucas-barake/effect-local-sql/PeerSync" -import * as ReplicaGate from "@lucas-barake/effect-local-sql/ReplicaGate" -import * as Document from "@lucas-barake/effect-local/Document" -import * as DocumentSet from "@lucas-barake/effect-local/DocumentSet" -import * as Identity from "@lucas-barake/effect-local/Identity" -import * as PeerTransport from "@lucas-barake/effect-local/PeerTransport" -import * as ReplicaDefinition from "@lucas-barake/effect-local/ReplicaDefinition" -import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" -import * as ReplicaLimits from "@lucas-barake/effect-local/ReplicaLimits" -import * as Cause from "effect/Cause" -import * as Context from "effect/Context" -import * as Crypto from "effect/Crypto" -import * as Deferred from "effect/Deferred" -import * as Effect from "effect/Effect" -import * as Exit from "effect/Exit" -import * as Fiber from "effect/Fiber" -import * as Layer from "effect/Layer" -import * as Metric from "effect/Metric" -import * as Option from "effect/Option" -import * as Queue from "effect/Queue" -import * as Redacted from "effect/Redacted" -import * as Ref from "effect/Ref" -import * as Schema from "effect/Schema" -import * as Scope from "effect/Scope" -import * as Stream from "effect/Stream" -import * as Sharding from "effect/unstable/cluster/Sharding" -import type * as Rpc from "effect/unstable/rpc/Rpc" -import * as RpcTest from "effect/unstable/rpc/RpcTest" -import { createHash, randomBytes } from "node:crypto" -import { vi } from "vitest" -import * as PeerRpcObservability from "../src/internal/peerRpcObservability.js" -import * as PeerAuthentication from "../src/PeerAuthentication.js" -import * as PeerAuthenticator from "../src/PeerAuthenticator.js" -import * as PeerAuthorization from "../src/PeerAuthorization.js" -import * as PeerCredentials from "../src/PeerCredentials.js" -import * as PeerRpc from "../src/PeerRpc.js" -import * as PeerRpcError from "../src/PeerRpcError.js" -import * as PeerRpcLimits from "../src/PeerRpcLimits.js" -import * as PeerRpcServer from "../src/PeerRpcServer.js" - -/** - * The stand-in for `PeerSession.makeSupervised`, which is the only export this file replaces. - * - * It runs in the very context production hands the real one at `PeerRpcServer.ts:978`: the ambient - * `Scope` is `entry.scope` and the ambient `PeerTransport` is `sessionTransport(entry)`. So the - * `Connection` it publishes to the test is the production connection built at - * `PeerRpcServer.ts:695-701`, including the `close` under test at `:700`. - */ -type SessionStandIn = ( - options: { - readonly peerId: Identity.PeerId - readonly documents: ReadonlyArray - } -) => Effect.Effect< - PeerSession.SupervisedPeerSession, - ReplicaError.ReplicaError, - Scope.Scope | PeerTransport.PeerTransport | ReplicaGate.ReplicaGate -> - -const sessionMock = vi.hoisted(() => ({ - /** - * Incremented inside the replacement `makeSupervised`, never in the factory. Only - * `PeerRpcServer.ts:978` calls `makeSupervised` in this suite, so a non-zero count is evidence - * that the specifier imported at `PeerRpcServer.ts:2` was the one `vi.mock` intercepted. A - * factory that merely runs would only prove this test file's own import was rewritten. - */ - intercepted: 0, - standIn: undefined as SessionStandIn | undefined -})) - -vi.mock("@lucas-barake/effect-local-sql/PeerSession", async (importActual) => { - const actual = await importActual() - const makeSupervised: typeof actual.makeSupervised = (options) => { - sessionMock.intercepted += 1 - const standIn = sessionMock.standIn - return standIn === undefined - ? Effect.die(new Error("PeerSession.makeSupervised stand-in was not installed")) - : standIn(options) - } - // Every other export, notably `maximumSyncEnvelopeBytes`, stays the real one: production reads it - // at `PeerRpcServer.ts:604` and `:1021`, and `PeerRpcLimits.ts:73` validates against it. - return { ...actual, makeSupervised } -}) - -const Task = Document.make("Task", { schema: Schema.Struct({ title: Schema.String }), version: 1 }) -const taskId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000001") -const serverPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000001") -const remotePeerA = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000002") -const definition = ReplicaDefinition.make({ - name: "peer-rpc-server-session-transport-test", - documents: DocumentSet.make(Task), - mutations: [], - projections: [], - queries: [] -}) -const definitionHash = definition.hash -const selected = [{ documentType: Task.name, documentId: taskId }] -const permit = { - replicaId: Identity.ReplicaId.make("rep_00000000-0000-4000-8000-000000000001"), - incarnation: Identity.ReplicaIncarnation.make(1), - writerGeneration: Identity.WriterGeneration.make(1) -} -// Production values, built through the real constructors, so the bounds `PeerRpcLimits.ts:80-83` -// validates are the ones the inbound quota accounting depends on. -const replicaLimits = ReplicaLimits.Values.make({ - maxBackupBytes: 1_000_000, - maxChunkBytes: 64_000, - maxArchiveRecords: 1_000, - maxJsonDepth: 32, - maxSyncMessageBytes: 64_000, - maxPeerSendMillis: 1_000, - maxSyncChangesPerMessage: 100, - maxSyncDependencyEdgesPerMessage: 1_000, - maxSyncOperationsPerMessage: 1_000, - maxPendingBytesPerDocument: 1_000_000, - maxPendingBytesPerPeer: 1_000_000, - maxPendingBytesPerReplica: 2_000_000, - maxPendingAgeMillis: 60_000, - maxPendingChangesPerDocument: 1_000, - maxPendingChangesPerPeer: 1_000, - maxPendingChangesPerReplica: 2_000, - maxPendingDependencyEdgesPerDocument: 10_000, - maxPendingDependencyEdgesPerPeer: 10_000, - maxPendingDependencyEdgesPerReplica: 20_000, - maxSessions: 8, - maxStreamsPerSession: 4, - maxInFlightPerSession: 1, - maxQueuedRpc: 32, - maxQueuedPermits: 32, - maxActiveRestores: 32, - maxRestoresPerSession: 1, - maxRestoreMillis: 30_000, - maxRestorePullMillis: 10_000, - maxRestoreCoalesceMillis: 25, - maxRestoreErrorBytes: 4_096 -}) -const rpcLimits = PeerRpcLimits.Values.make({ - ...PeerRpcLimits.defaults, - openRatePerSecond: 1_000, - openBurst: 1_000, - pushRatePerSecond: 1_000, - pushBurst: 1_000, - authenticationRatePerSecond: 1_000, - authenticationBurst: 1_000, - maximumReauthorizationInterval: 60_000 -}) -const crypto = Crypto.make({ - randomBytes: (size) => randomBytes(size), - digest: (algorithm, bytes) => - Effect.sync(() => new Uint8Array(createHash(algorithm.replace("-", "").toLowerCase()).update(bytes).digest())) -}) -const principal = PeerAuthentication.PeerPrincipal.make({ - tenantId: "tenant", - subjectId: "subject-a", - peerId: remotePeerA -}) - -const makeHarness = (options?: { - readonly rpcLimits?: Partial - /** - * Runs inside the stand-in's element handler, after the payload has been recorded and `received` - * signalled, so a test can hold the handler open exactly the way - * `RpcPeerTransport.test.ts:707-711` does. - */ - readonly onPayload?: (payload: Uint8Array) => Effect.Effect - /** Drives `close` before the consumer's very first pull. */ - readonly closeBeforeConsumer?: boolean -}) => - Effect.gen(function*() { - const configuredRpcLimits = PeerRpcLimits.Values.make({ ...rpcLimits, ...options?.rpcLimits }) - const connectionReady = yield* Deferred.make() - const consumerReady = yield* Deferred.make>() - const delivered = yield* Queue.unbounded() - const received = yield* Deferred.make() - // Test owned, so a test can fail it on demand. Criterion 11 requires `close` never to be the - // sole terminator: the real monitor at `PeerRpcServer.ts:984-995` only revokes once this - // resolves. - const disconnect = yield* Deferred.make() - - const standIn: SessionStandIn = (standInOptions) => - Effect.gen(function*() { - const gate = yield* ReplicaGate.ReplicaGate - const claimed = yield* gate.shared - const transport = yield* PeerTransport.PeerTransport - const connection = yield* transport.connect({ - replicaId: claimed.replicaId, - peerId: standInOptions.peerId - }) - yield* Deferred.succeed(connectionReady, connection) - if (options?.closeBeforeConsumer === true) yield* connection.close - // Exactly one consumer of `connection.receive`, for the whole suite. It closes over the - // transport's mutable `currentInbound` and `started` (`PeerRpcServer.ts:579-580`), so a - // second consumer would clobber the inbound reservation accounting. - const consumer = yield* Stream.runForEach(connection.receive, (payload) => - Queue.offer(delivered, payload).pipe( - Effect.andThen(Deferred.succeed(received, undefined)), - Effect.andThen(options?.onPayload === undefined ? Effect.void : options.onPayload(payload)) - )).pipe(Effect.forkScoped) - yield* Deferred.succeed(consumerReady, consumer) - return { - peerId: connection.peerId, - connectionEpoch: "stand-in-epoch", - markDirty: () => Effect.void, - flush: Effect.void, - observedByPeer: () => Effect.succeed(false), - durableConfirmation: () => Effect.succeed(false as const), - awaitDisconnect: Deferred.await(disconnect) - } - }) - yield* Effect.sync(() => void (sessionMock.standIn = standIn)) - yield* Effect.addFinalizer(() => Effect.sync(() => void (sessionMock.standIn = undefined))) - - const gate = ReplicaGate.ReplicaGate.of({ - current: Effect.succeed(permit), - claiming: Effect.succeed(false), - shared: Effect.acquireRelease(Effect.succeed(permit), () => Effect.void), - admit: Effect.acquireRelease(Effect.succeed(permit), () => Effect.void), - claim: (use) => use(permit), - refresh: Effect.succeed(permit), - validate: () => Effect.void - }) - const sync = PeerSync.PeerSync.of({ - withDocumentInvalidation: (_documentId, effect) => effect, - invalidateDocument: () => Effect.void, - open: (peerId) => - Effect.succeed({ peerId, connectionEpoch: "local-epoch", replicaIncarnation: permit.incarnation }), - reset: () => Effect.void, - generate: () => Effect.succeed({ outbound: null, observedByPeer: false, dirty: false }), - receive: () => Effect.die("unexpected direct PeerSync receive"), - enqueue: (_session, reply) => - Effect.succeed({ - ...reply, - sendSequence: 0, - lineage: Identity.genesisLineage, - writerProvenance: [] - }), - pending: () => Effect.succeed([]), - markSent: () => Effect.succeed(true) - }) - const publisher = CommitPublisher.CommitPublisher.of({ - publishPending: Effect.succeed(0), - invalidate: () => Effect.void, - subscribe: Effect.succeed({ - watermark: Identity.CommitSequence.make(0), - refreshGeneration: 0, - events: Stream.never - }) - }) - const sharding = Sharding.Sharding.of({ - ...({} as Sharding.Sharding["Service"]), - makeClient: () => - Effect.succeed(() => - ({ - ApplySync: (_payload: typeof DocumentEntity.ApplySync.payloadSchema.Type) => - Effect.die("unexpected DocumentEntity ApplySync") - }) as never - ) - }) - const authorization = PeerAuthorization.PeerAuthorization.of({ - authorize: (request) => - Effect.succeed({ - documents: request.documents.map((requested) => ({ - document: Task, - documentId: requested.documentId - })), - validUntil: Number.MAX_SAFE_INTEGER, - // Nothing in this suite invalidates a lease, and the lease watcher - // (`PeerRpcServer.ts:955-961`) only ever awaits this arm. `Effect.never` states that - // directly instead of parking on a `Deferred` no test ever completes. - invalidated: Effect.never - }) - }) - const services = Layer.mergeAll( - Layer.succeed(Crypto.Crypto, crypto), - Layer.succeed(CommitPublisher.CommitPublisher, publisher), - Layer.succeed(PeerSync.PeerSync, sync), - Layer.succeed(ReplicaGate.ReplicaGate, gate), - Layer.succeed(ReplicaLimits.ReplicaLimits, replicaLimits), - Layer.succeed(Sharding.Sharding, sharding), - Layer.succeed(PeerRpcLimits.PeerRpcLimits, configuredRpcLimits), - Layer.succeed(PeerAuthorization.PeerAuthorization, authorization), - Layer.succeed(Metric.FiberRuntimeMetrics, { - recordFiberStart: () => {}, - recordFiberEnd: () => {} - }) - ) - // The subject under test is built through the real Layer, at the real call site, never - // hand-assembled: `sessionTransport(entry)` is only reachable from `layerHandlers`. - const handlers = PeerRpcServer.layerHandlers({ tenantId: "tenant", peerId: serverPeerId, definition }).pipe( - Layer.provide(services) - ) - const handlerScope = yield* Scope.make() - yield* Effect.addFinalizer(() => Scope.close(handlerScope, Exit.void)) - const handlerContext = yield* Layer.build(handlers).pipe(Effect.provideService(Scope.Scope, handlerScope)) - const pushHandler = handlerContext.mapUnsafe.get(PeerRpc.PushRpc.key) as Rpc.Handler<"Push"> - // The direct handler shape, for `Push` only: it is the observed production handler - // (`PeerRpcServer.ts:1216` -> `:1125`), and it surfaces the typed `PeerRpcError` these tests - // discriminate on rather than a serialized client failure. - const directPush = (request: typeof PeerRpc.PushRpc.payloadSchema.Type) => - (pushHandler.handler(request, {} as never) as Effect.Effect).pipe( - Effect.provideContext(Context.add( - pushHandler.context, - PeerAuthentication.AuthenticatedPeer, - { - principal, - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.never - } - )) - ) - const client = yield* RpcTest.makeClient(PeerRpc.Rpcs).pipe( - Effect.provide(handlerContext), - Effect.provide(PeerAuthentication.layerServer), - Effect.provideService(PeerAuthenticator.PeerAuthenticator, { - authenticate: (value) => - Redacted.value(value) === "owner" - ? Effect.succeed({ - principal, - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.never - }) - : Effect.fail(new PeerRpcError.AuthenticationFailure()) - }), - Effect.provide(PeerAuthentication.layerClient), - Effect.provideService(PeerCredentials.PeerCredentials, { - get: Effect.sync(() => Redacted.make("owner")) - }), - Effect.provideService(PeerRpcLimits.PeerRpcLimits, configuredRpcLimits) - ) - const open = (documents: ReadonlyArray) => - Effect.gen(function*() { - const events = yield* Queue.unbounded() - const fiber = yield* Stream.runForEach( - client.Open({ - protocolVersion: PeerRpc.protocolVersion, - expectedPeerId: serverPeerId, - definitionHash, - documents - }), - (event) => Queue.offer(events, event).pipe(Effect.asVoid) - ).pipe(Effect.forkChild) - const first = yield* Effect.raceFirst( - Queue.take(events), - Fiber.join(fiber).pipe(Effect.andThen(Effect.die("Open stream ended before Opened"))) - ) - if (first._tag !== "Opened") return yield* Effect.die("the first Open event was not Opened") - return { opened: first, events, fiber } - }) - return { - connectionReady, - consumerReady, - delivered, - received, - disconnect, - open, - directPush, - closeServer: Scope.close(handlerScope, Exit.void) - } - }) - -/** - * Every await in this suite is guarded, so a regression fails with a named message well inside the - * `it.live` budget instead of hanging until vitest kills the file. `assert.isTrue` does not narrow, - * so the `Option` narrowing lives here once rather than at every call site. - */ -const within = (effect: Effect.Effect, message: string) => - effect.pipe( - Effect.timeoutOption("4 seconds"), - Effect.map((result) => { - assert.isTrue(Option.isSome(result), message) - return Option.getOrThrow(result) - }) - ) - -/** The captured production `Connection`, built at `PeerRpcServer.ts:695-701`. */ -const attach = (harness: { readonly connectionReady: Deferred.Deferred }) => - within(Deferred.await(harness.connectionReady), "the stand-in never published the production connection") - -const consumerOf = (harness: { - readonly consumerReady: Deferred.Deferred> -}) => within(Deferred.await(harness.consumerReady), "the stand-in never published its receive consumer") - -/** - * Criterion 2: an interrupt-only `Exit`, not a normal end (which `PeerSession.ts:544-553` converts - * into a retryable `StorageUnavailable`) and not a manufactured failure. The expanded form of this - * check stays inline in the first test, which is where the distinction is established. - */ -const awaitInterrupted = (fiber: Fiber.Fiber, message: string) => - within(Fiber.await(fiber), message).pipe( - Effect.map((exit) => - assert.isTrue( - Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause), - "the receive consumer's Exit was not interrupt-only" - ) - ) - ) - -/** - * Suspended, not a hoisted constant: a `Metric` instance caches its metadata against the first - * registry it resolves against (`node_modules/effect/src/Metric.ts:1762-1771`), so a shared - * instance would keep reading the first test's registry once each test provides a fresh one. - */ -const inboundGauge = Effect.suspend(() => Metric.value(PeerRpcObservability.queueItems("Inbound"))).pipe( - Effect.map((gauge) => gauge.value) -) - -describe("PeerRpcServer sessionTransport", () => { - it.live( - "intercepts PeerSession.makeSupervised while every other export stays real", - () => - Effect.scoped(Effect.gen(function*() { - const harness = yield* makeHarness() - const session = yield* harness.open(selected) - - assert.isTrue( - sessionMock.intercepted > 0, - "vi.mock did not intercept @lucas-barake/effect-local-sql/PeerSession as imported by PeerRpcServer.ts:2" - ) - - assert.strictEqual( - typeof PeerSession.maximumSyncEnvelopeBytes, - "function", - "importActual did not survive the partial mock" - ) - const oversized = new Uint8Array( - PeerSession.maximumSyncEnvelopeBytes( - replicaLimits.maxSyncMessageBytes, - replicaLimits.maxSyncChangesPerMessage - ) + 1 - ) - const rejected = yield* harness.directPush({ - sessionId: session.opened.sessionId, - payload: oversized - }).pipe(Effect.exit) - assert.isTrue(Exit.isFailure(rejected), "an oversized Push was not rejected") - assert.strictEqual( - PeerRpcObservability.failure(rejected)?._tag, - "RequestLimitExceeded", - "production did not read the real maximumSyncEnvelopeBytes" - ) - - assert.strictEqual(session.opened._tag, "Opened", "the client never observed Opened") - })).pipe(Effect.provideService(Metric.MetricRegistry, new Map())), - 30000 - ) - - it.live( - "terminates a parked receive consumer when the server-hosted connection closes", - () => - Effect.scoped(Effect.gen(function*() { - const harness = yield* makeHarness() - const session = yield* harness.open(selected) - const connection = yield* attach(harness) - const receiving = yield* consumerOf(harness) - - yield* harness.directPush({ sessionId: session.opened.sessionId, payload: Uint8Array.of(1, 2, 3) }) - yield* within(Deferred.await(harness.received), "the receive consumer never observed a delivered payload") - - yield* Effect.yieldNow - assert.isUndefined(receiving.pollUnsafe(), "the receive consumer terminated before close ran") - - const closing = yield* connection.close.pipe(Effect.forkChild) - yield* within(Fiber.await(closing), "connection.close did not complete") - - // Deliberately expanded rather than folded into `awaitInterrupted`: this is the criterion-2 - // detector, and the two halves it separates - "did not end normally" and "was not a - // manufactured failure" - are the whole reason `close` interrupts the race arm instead of - // completing it. - const exit = yield* within(Fiber.await(receiving), "close did not terminate the receive consumer") - assert.isTrue(Exit.isFailure(exit), "the receive consumer ended normally instead of being interrupted") - if (Exit.isFailure(exit)) { - assert.isTrue(Cause.hasInterruptsOnly(exit.cause), "the receive consumer's Exit was not interrupt-only") - } - - yield* within(harness.closeServer, "the server scope did not close") - assert.strictEqual( - yield* inboundGauge, - 0, - "the delivered payload's inbound reservation was not reclaimed" - ) - })).pipe(Effect.provideService(Metric.MetricRegistry, new Map())), - 30000 - ) - - it.live( - "completes every close call without suspending", - () => - Effect.scoped(Effect.gen(function*() { - const harness = yield* makeHarness() - const session = yield* harness.open(selected) - const connection = yield* attach(harness) - const receiving = yield* consumerOf(harness) - - yield* harness.directPush({ sessionId: session.opened.sessionId, payload: Uint8Array.of(1) }) - yield* within(Deferred.await(harness.received), "the receive consumer never observed a delivered payload") - - // The one assertion that fails if `close` is ever made to await its own cleanup the way - // `RpcPeerTransport.ts:124` does. Criterion 3: `close` runs on the receive fiber itself - // (`PeerSession.ts:557`), so a suspending `close` would deadlock that fiber against itself. - const closing = yield* connection.close.pipe(Effect.forkChild) - yield* Effect.yieldNow - const first = closing.pollUnsafe() - assert.isDefined(first, "close suspended instead of completing on the calling fiber") - if (first === undefined) return yield* Effect.die("unreachable") - assert.isTrue(Exit.isSuccess(first), "the first close call did not succeed") - - const second = yield* connection.close.pipe(Effect.exit) - const third = yield* connection.close.pipe(Effect.exit) - assert.isTrue(Exit.isSuccess(second), "the second sequential close call did not succeed") - assert.isTrue(Exit.isSuccess(third), "the third sequential close call did not succeed") - - const concurrent = yield* Effect.all( - [connection.close.pipe(Effect.exit), connection.close.pipe(Effect.exit)], - { concurrency: "unbounded" } - ) - assert.isTrue(concurrent.every(Exit.isSuccess), "a concurrent close call did not succeed") - - yield* awaitInterrupted(receiving, "close did not terminate the receive consumer") - })).pipe(Effect.provideService(Metric.MetricRegistry, new Map())), - 30000 - ) - - it.live( - "does not interrupt an in-flight element handler, and terminates at the next pull", - () => - Effect.scoped(Effect.gen(function*() { - const inHandler = yield* Deferred.make() - const releaseHandler = yield* Deferred.make() - const handlerInterrupted = yield* Ref.make(false) - const harness = yield* makeHarness({ - onPayload: () => - Deferred.succeed(inHandler, undefined).pipe( - Effect.andThen(Deferred.await(releaseHandler)), - Effect.onInterrupt(() => Ref.set(handlerInterrupted, true)) - ) - }) - const session = yield* harness.open(selected) - const connection = yield* attach(harness) - const receiving = yield* consumerOf(harness) - - yield* harness.directPush({ sessionId: session.opened.sessionId, payload: Uint8Array.of(1, 2, 3) }) - yield* within(Deferred.await(inHandler), "the consumer never entered its element handler") - - // Criterion 6: the reservation for item N is held for the whole of item N's handler. Under - // the rejected `Stream.interruptWhen` design the producer would already have re-entered the - // pull and run `releaseInbound` (`PeerRpcServer.ts:585`), so this would read 0. It is also - // the gate the `:1051` item-capacity check reads, which is why no separate test is needed - // to prove a concurrent Push is rejected while this reservation is held. - assert.strictEqual( - yield* inboundGauge, - 1, - "the inbound reservation was released while the element handler was still in flight" - ) - - yield* connection.close - yield* Effect.yieldNow - assert.isUndefined(receiving.pollUnsafe(), "close terminated the consumer inside its element handler") - assert.isFalse(yield* Ref.get(handlerInterrupted), "close interrupted the in-flight element handler") - - yield* Deferred.succeed(releaseHandler, undefined) - yield* awaitInterrupted(receiving, "the consumer did not terminate at its next pull") - assert.isFalse(yield* Ref.get(handlerInterrupted), "the released element handler was interrupted after all") - assert.strictEqual( - yield* inboundGauge, - 0, - "the next pull did not release the handled item's reservation before teardown" - ) - })).pipe(Effect.provideService(Metric.MetricRegistry, new Map())), - 30000 - ) - - it.live( - "does not revoke, detach, or terminate the outbound stream when close runs", - () => - Effect.scoped(Effect.gen(function*() { - const harness = yield* makeHarness() - const session = yield* harness.open(selected) - const connection = yield* attach(harness) - - yield* harness.directPush({ sessionId: session.opened.sessionId, payload: Uint8Array.of(1) }) - yield* within(Deferred.await(harness.received), "the receive consumer never observed a delivered payload") - - yield* connection.close - - // Offered at `PeerRpcServer.ts:665` and drained by `responseStream` at `:827`. A `detach` - // would have failed `entry.terminal` at `:286`, which surfaces at `:825-827`. - const outbound = Uint8Array.of(9, 8, 7) - yield* connection.send(outbound) - const message = yield* within(Queue.take(session.events), "the Open stream stopped delivering after close") - assert.strictEqual(message._tag, "Message") - if (message._tag === "Message") { - assert.deepStrictEqual(message.payload, outbound) - } - - yield* Effect.yieldNow - assert.isUndefined(session.fiber.pollUnsafe(), "close terminated the client's Open stream") - assert.strictEqual( - (yield* Metric.value(PeerRpcObservability.activeSessions())).value, - 1, - "close revoked the session" - ) - })).pipe(Effect.provideService(Metric.MetricRegistry, new Map())), - 30000 - ) - - it.live( - "accepts the first post-close Push, then overloads on the next", - () => - Effect.scoped(Effect.gen(function*() { - // The default `inboundItemCapacity` of 1 (`PeerRpcLimits.ts:40`), so the second post-close - // Push is the overflowing one. It is rejected by the item-capacity gate at - // `PeerRpcServer.ts:1051-1052`, not by the `!offered` branch at `:1064`: raising only - // `entry.inbound`'s own capacity (`:927-929`) leaves this test green, while raising only - // the gate hands the same rejection to `:1064`. Both bounds are `inboundItemCapacity` and - // every queued item holds a reservation, so no capacity separates them - the gate always - // short-circuits first. - const harness = yield* makeHarness() - const session = yield* harness.open(selected) - const connection = yield* attach(harness) - const receiving = yield* consumerOf(harness) - - yield* harness.directPush({ sessionId: session.opened.sessionId, payload: Uint8Array.of(1) }) - yield* within(Deferred.await(harness.received), "the receive consumer never observed a delivered payload") - - yield* connection.close - // The consumer's exit is the barrier that its reservation is gone: released either by the - // next pull (`PeerRpcServer.ts:585`) or by `Stream.ensuring` (`:597-599`). - yield* within(Fiber.await(receiving), "close did not terminate the receive consumer") - - // The rejected `Queue.shutdown` design made `Queue.offer` return false here - // (`Queue.ts:632-639`), which fabricates `SessionOverloaded` at `PeerRpcServer.ts:1065`. - const accepted = yield* harness.directPush({ - sessionId: session.opened.sessionId, - payload: Uint8Array.of(2) - }).pipe(Effect.exit) - assert.isTrue(Exit.isSuccess(accepted), "the first post-close Push was rejected") - assert.strictEqual( - (yield* Metric.value(PeerRpcObservability.boundary("Push", "Success"))).count, - 2, - "the post-close Push was not recorded as a successful Push" - ) - assert.strictEqual( - (yield* Metric.value(PeerRpcObservability.boundary("Push", "Overloaded"))).count, - 0, - "close reclassified a Push as Overloaded" - ) - - // Nothing consumes the accepted item now, so this one finds the only slot reserved and hits - // the pre-existing gate at `PeerRpcServer.ts:1051`, detaching at `:1056`. Criterion 7 - // accepts this; the assertion pins it so a future change cannot silently widen it. - const overflow = yield* harness.directPush({ - sessionId: session.opened.sessionId, - payload: Uint8Array.of(3) - }).pipe(Effect.exit) - assert.strictEqual( - PeerRpcObservability.failure(overflow)?._tag, - "SessionOverloaded", - "the overflowing post-close Push was not classified as SessionOverloaded" - ) - assert.strictEqual( - (yield* Metric.value(PeerRpcObservability.boundary("Push", "Overloaded"))).count, - 1, - "the overflowing Push was not recorded as Overloaded" - ) - - yield* within(harness.closeServer, "the server scope did not close") - assert.strictEqual(yield* inboundGauge, 0, "the post-close Push's reservation was not reclaimed") - })).pipe(Effect.provideService(Metric.MetricRegistry, new Map())), - 30000 - ) - - it.live( - "emits Opened even when close precedes the first pull, and still terminates", - () => - Effect.scoped(Effect.gen(function*() { - const harness = yield* makeHarness({ closeBeforeConsumer: true }) - // Criterion 8: `inboundConsumerStarted` (`PeerRpcServer.ts:590`) is completed before the - // race, so the `Opened` gate at `:808` still resolves when close won the very first pull. - const session = yield* harness.open(selected) - assert.strictEqual(session.opened._tag, "Opened", "the client never observed Opened") - - const receiving = yield* consumerOf(harness) - yield* awaitInterrupted(receiving, "close before the first pull did not terminate the consumer") - - // Criterion 11: close is never the sole terminator. Nothing fails `entry.terminal` on its - // own, so without this the Open stream would park forever at `PeerRpcServer.ts:825-828`. - yield* Deferred.fail( - harness.disconnect, - new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageUnavailable({ cause: new Error("peer session lost") }) - }) - ) - const ended = yield* within(Fiber.await(session.fiber), "the monitor never revoked the session") - assert.strictEqual( - PeerRpcObservability.failure(ended)?._tag, - "ServerUnavailable", - "the Open stream did not terminate with the classified PeerRpcError" - ) - })).pipe(Effect.provideService(Metric.MetricRegistry, new Map())), - 30000 - ) - - it.live( - "does not take a queued item once close has been requested", - () => - Effect.scoped(Effect.gen(function*() { - const inHandler = yield* Deferred.make() - const releaseHandler = yield* Deferred.make() - const harness = yield* makeHarness({ - rpcLimits: { inboundItemCapacity: 2 }, - onPayload: (payload) => - payload[0] === 1 - ? Deferred.succeed(inHandler, undefined).pipe(Effect.andThen(Deferred.await(releaseHandler))) - : Effect.void - }) - const session = yield* harness.open(selected) - const connection = yield* attach(harness) - const receiving = yield* consumerOf(harness) - - yield* harness.directPush({ sessionId: session.opened.sessionId, payload: Uint8Array.of(1) }) - yield* within(Deferred.await(inHandler), "the consumer never entered its element handler") - // Lands in `entry.inbound` while the consumer is inside the handler for the first item, so - // the next pull starts with an item already buffered and close already requested. - yield* harness.directPush({ sessionId: session.opened.sessionId, payload: Uint8Array.of(2) }) - - yield* connection.close - yield* Deferred.succeed(releaseHandler, undefined) - - yield* awaitInterrupted(receiving, "close did not terminate the receive consumer") - // `Effect.raceFirst` is `raceAllFirst([self, that])` (`internal/effect.ts:1611`), which forks - // the arms in order and breaks at `internal/effect.ts:1515` as soon as one settles, and both - // arms settle synchronously here. Signal first means the buffered item is never taken; - // reversing the arms would drain the queue before observing close. - assert.deepStrictEqual( - yield* Queue.takeAll(harness.delivered), - [Uint8Array.of(1)], - "the pull took a buffered item after close was requested" - ) - })).pipe(Effect.provideService(Metric.MetricRegistry, new Map())), - 30000 - ) - - it.live( - "terminates the consumer after many delivered messages", - () => - Effect.scoped(Effect.gen(function*() { - const messageCount = 50 - const acknowledged: Array> = [] - for (let index = 0; index < messageCount; index++) { - acknowledged.push(yield* Deferred.make()) - } - // Two slots, so message N+1 can be admitted while the handler for message N still holds its - // reservation. That is the only outstanding-reservation state this loop ever reaches, so - // the sequence stays deterministic without a single sleep. - const harness = yield* makeHarness({ - rpcLimits: { inboundItemCapacity: 2 }, - onPayload: (payload) => Deferred.succeed(acknowledged[payload[0]!]!, undefined).pipe(Effect.asVoid) - }) - const session = yield* harness.open(selected) - const connection = yield* attach(harness) - const receiving = yield* consumerOf(harness) - - // Each pull installs a fresh `Deferred.await(closeRequested)` arm in the race at - // `PeerRpcServer.ts:592`. If a losing arm were left resumable across pulls, `close` would - // become a no-op once the stale resume had been consumed, which is the original bug - // reappearing only under load. - for (let index = 0; index < messageCount; index++) { - yield* harness.directPush({ sessionId: session.opened.sessionId, payload: Uint8Array.of(index) }) - yield* within(Deferred.await(acknowledged[index]!), `message ${index} was never delivered`) - } - - yield* connection.close - yield* awaitInterrupted(receiving, `close did not terminate the consumer after ${messageCount} messages`) - - yield* within(harness.closeServer, "the server scope did not close") - assert.strictEqual( - yield* inboundGauge, - 0, - "a delivered message's inbound reservation survived teardown" - ) - })).pipe(Effect.provideService(Metric.MetricRegistry, new Map())), - 30000 - ) - - it.live( - "leaves the entry active and buffering when close runs with no other terminator", - () => - Effect.scoped(Effect.gen(function*() { - const harness = yield* makeHarness() - const session = yield* harness.open(selected) - const connection = yield* attach(harness) - const receiving = yield* consumerOf(harness) - - yield* harness.directPush({ sessionId: session.opened.sessionId, payload: Uint8Array.of(1) }) - yield* within(Deferred.await(harness.received), "the receive consumer never observed a delivered payload") - - // `harness.disconnect` is left unresolved, so the monitor at `PeerRpcServer.ts:984-995` - // never fires and `close` is the only thing that ran. Criterion 11 accepts exactly this - // hazard, and this test is what pins it: the entry stays registered, the client keeps its - // Open stream, and inbound work keeps being admitted with nothing left to consume it. - yield* connection.close - yield* within(Fiber.await(receiving), "close did not terminate the receive consumer") - - assert.strictEqual( - (yield* Metric.value(PeerRpcObservability.activeSessions())).value, - 1, - "close revoked the session even though nothing else terminated it" - ) - yield* Effect.yieldNow - assert.isUndefined(session.fiber.pollUnsafe(), "close failed the client's Open stream") - - const accepted = yield* harness.directPush({ - sessionId: session.opened.sessionId, - payload: Uint8Array.of(2) - }).pipe(Effect.exit) - assert.isTrue(Exit.isSuccess(accepted), "the entry stopped admitting Pushes after close") - assert.strictEqual( - yield* inboundGauge, - 1, - "the admitted Push was not buffered against the still-active entry" - ) - })).pipe(Effect.provideService(Metric.MetricRegistry, new Map())), - 30000 - ) - - it.live( - "leaves a classified receive failure intact when close never ran", - () => - Effect.scoped(Effect.gen(function*() { - const harness = yield* makeHarness() - const session = yield* harness.open(selected) - const receiving = yield* consumerOf(harness) - - yield* harness.directPush({ sessionId: session.opened.sessionId, payload: Uint8Array.of(1) }) - yield* within(Deferred.await(harness.received), "the receive consumer never observed a delivered payload") - - // No `close` anywhere on this path. Real teardown runs `finishCleanup`, whose - // `Queue.fail(entry.inbound, replicaFailure())` at `PeerRpcServer.ts:358` is what the - // consumer must observe. D3 only holds because `close` cannot preempt it here. - yield* within(harness.closeServer, "the server scope did not close") - - const exit = yield* within(Fiber.await(receiving), "teardown did not terminate the receive consumer") - assert.isTrue(Exit.isFailure(exit), "teardown ended the receive consumer normally") - if (Exit.isFailure(exit)) { - assert.isFalse( - Cause.hasInterruptsOnly(exit.cause), - "the classified receive failure was replaced by an interrupt" - ) - const error = Cause.findErrorOption(exit.cause) - assert.isTrue(Option.isSome(error), "the consumer did not observe a ReplicaError") - if (Option.isSome(error)) { - assert.strictEqual(error.value.reason._tag, "StorageUnavailable") - } - } - })).pipe(Effect.provideService(Metric.MetricRegistry, new Map())), - 30000 - ) -}) diff --git a/packages/local-rpc/test/PeerRpcWebSocket.test.ts b/packages/local-rpc/test/PeerRpcWebSocket.test.ts deleted file mode 100644 index 4160880..0000000 --- a/packages/local-rpc/test/PeerRpcWebSocket.test.ts +++ /dev/null @@ -1,745 +0,0 @@ -import { NodeCrypto, NodeHttpServer, NodeSocket } from "@effect/platform-node" -import { assert, describe, it } from "@effect/vitest" -import * as DocumentEntity from "@lucas-barake/effect-local-sql/DocumentEntity" -import * as TestReplica from "@lucas-barake/effect-local-test/TestReplica" -import * as CommandOutcome from "@lucas-barake/effect-local/CommandOutcome" -import * as Document from "@lucas-barake/effect-local/Document" -import * as DocumentSet from "@lucas-barake/effect-local/DocumentSet" -import * as Identity from "@lucas-barake/effect-local/Identity" -import * as Mutation from "@lucas-barake/effect-local/Mutation" -import * as Replica from "@lucas-barake/effect-local/Replica" -import * as ReplicaDefinition from "@lucas-barake/effect-local/ReplicaDefinition" -import * as ReplicaLimits from "@lucas-barake/effect-local/ReplicaLimits" -import * as Cause from "effect/Cause" -import * as Context from "effect/Context" -import * as Deferred from "effect/Deferred" -import * as Effect from "effect/Effect" -import * as Exit from "effect/Exit" -import * as Fiber from "effect/Fiber" -import * as Layer from "effect/Layer" -import * as Option from "effect/Option" -import * as Redacted from "effect/Redacted" -import * as Ref from "effect/Ref" -import * as Schema from "effect/Schema" -import * as Scope from "effect/Scope" -import * as Stream from "effect/Stream" -import * as Sharding from "effect/unstable/cluster/Sharding" -import * as HttpRouter from "effect/unstable/http/HttpRouter" -import * as HttpServer from "effect/unstable/http/HttpServer" -import * as RpcClient from "effect/unstable/rpc/RpcClient" -import * as RpcSerialization from "effect/unstable/rpc/RpcSerialization" -import * as RpcServer from "effect/unstable/rpc/RpcServer" -import * as Http from "node:http" -import * as PeerAuthentication from "../src/PeerAuthentication.js" -import * as PeerAuthenticator from "../src/PeerAuthenticator.js" -import * as PeerAuthorization from "../src/PeerAuthorization.js" -import * as PeerCredentials from "../src/PeerCredentials.js" -import * as PeerRpc from "../src/PeerRpc.js" -import * as PeerRpcError from "../src/PeerRpcError.js" -import * as PeerRpcLimits from "../src/PeerRpcLimits.js" -import * as PeerRpcServer from "../src/PeerRpcServer.js" -import * as RpcPeerTransport from "../src/RpcPeerTransport.js" - -const Task = Document.make("WebSocketTask", { - schema: Schema.Struct({ title: Schema.String, labels: Schema.Array(Schema.String) }), - version: 1 -}) - -const RenameTask = Mutation.make("WebSocketTask.Rename", { - document: Task, - payload: Schema.String -}) - -const AddLabel = Mutation.make("WebSocketTask.AddLabel", { - document: Task, - payload: Schema.String -}) - -const definition = ReplicaDefinition.make({ - name: "rpc-websocket-test", - documents: DocumentSet.make(Task), - mutations: [RenameTask, AddLabel], - projections: [], - queries: [] -}) - -const HandlersLive = Layer.mergeAll( - RenameTask.toLayer(({ draft, payload }) => { - draft.title = payload - return undefined - }), - AddLabel.toLayer(({ draft, payload }) => { - draft.labels.push(payload) - return undefined - }) -) - -const ReplicaLive = TestReplica.layer(definition, { projections: [] }).pipe( - Layer.provide(HandlersLive) -) - -const serverPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000001") -const clientPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000002") -const missingSessionId = Identity.SessionId.make("ses_00000000-0000-4000-8000-000000000001") - -const repeatUntil = (step: Effect.Effect) => { - const failure = new Error("Observable WebSocket condition did not complete") - return Effect.gen(function*() { - for (let attempt = 0; attempt < 500; attempt++) { - if (yield* step) return - } - return yield* Effect.die(failure) - }) -} - -const makeWebSocketServer = ( - documentId: Identity.DocumentId, - document: Document.Any = Task, - replicaDefinition: ReplicaDefinition.Any = definition -) => { - const ServerDependencies = Layer.mergeAll( - PeerRpcLimits.layerDefaults, - Layer.succeed(PeerAuthenticator.PeerAuthenticator)({ - authenticate: () => - Effect.succeed({ - principal: PeerAuthentication.PeerPrincipal.make({ - tenantId: "tenant", - subjectId: "websocket-client", - peerId: clientPeerId - }), - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.never - }) - }), - PeerAuthorization.layer(() => - Effect.succeed({ - documents: [{ document, documentId }], - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.never - }) - ) - ) - const AuthenticationLive = PeerAuthentication.layerServer.pipe(Layer.provide(ServerDependencies)) - const PeerHandlersLive = PeerRpcServer.layerHandlers({ - tenantId: "tenant", - peerId: serverPeerId, - definition: replicaDefinition - }).pipe( - Layer.provide(ServerDependencies) - ) - const WsProtocol = RpcServer.layerProtocolWebsocket({ path: "/rpc" }).pipe(Layer.provide(HttpRouter.layer)) - const RpcLive = RpcServer.layer(PeerRpc.Rpcs, { disableFatalDefects: true }).pipe( - Layer.provide([PeerHandlersLive, AuthenticationLive]) - ) - return RpcLive.pipe( - Layer.provideMerge(WsProtocol), - Layer.provide(HttpRouter.serve(WsProtocol, { disableListenLog: true, disableLogger: true })) - ) -} - -describe("PeerRpc WebSocket", () => { - it.live( - "synchronizes two replicas through the application owned WebSocket server", - () => - Effect.scoped(Effect.gen(function*() { - const serverEngineContext = yield* Layer.build(ReplicaLive) - const clientEngineContext = yield* Layer.build(ReplicaLive) - const serverReplica = Context.get(serverEngineContext, Replica.Replica) - const clientReplica = Context.get(clientEngineContext, Replica.Replica) - const serverSharding = Context.get(serverEngineContext, Sharding.Sharding) - const clientSharding = Context.get(clientEngineContext, Sharding.Sharding) - - const created = yield* serverReplica.create(Task, { - commandId: yield* Identity.makeCommandId, - value: { title: "before WebSocket sync", labels: [] } - }) - const documentId = yield* CommandOutcome.committedOrFail(created) - const archive = yield* serverReplica.exportBackup({ - maxBytes: TestReplica.defaultLimits.maxBackupBytes - }).pipe(Stream.runCollect) - yield* clientReplica.restoreBackup({ - source: Stream.fromIterable(archive), - mode: "clone", - maxBytes: TestReplica.defaultLimits.maxBackupBytes, - expectedDefinitionHash: definition.hash, - installationId: yield* Identity.makeBackupInstallationId - }) - - const credentials: Array = [] - let credential = "client-payload-credential" - let authorizationCalls = 0 - const ServerDependencies = Layer.mergeAll( - PeerRpcLimits.layerDefaults, - Layer.succeed(PeerAuthenticator.PeerAuthenticator)({ - authenticate: (value) => { - const received = Redacted.value(value) - credentials.push(received) - return received === "defect-credential" - ? Effect.die(new Error("raw-authentication-secret")) - : Effect.succeed({ - principal: PeerAuthentication.PeerPrincipal.make({ - tenantId: "tenant", - subjectId: "websocket-client", - peerId: clientPeerId - }), - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.never - }) - } - }), - PeerAuthorization.layer((request) => { - authorizationCalls++ - assert.strictEqual(request.principal.tenantId, "tenant") - assert.strictEqual(request.principal.subjectId, "websocket-client") - assert.strictEqual(request.principal.peerId, clientPeerId) - assert.deepStrictEqual(request.documents, [{ documentType: Task.name, documentId }]) - return Effect.succeed({ - documents: [{ document: Task, documentId }], - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.never - }) - }) - ) - const AuthenticationLive = PeerAuthentication.layerServer.pipe( - Layer.provide(ServerDependencies) - ) - const PeerHandlersLive = PeerRpcServer.layerHandlers({ - tenantId: "tenant", - peerId: serverPeerId, - definition - }).pipe( - Layer.provide(ServerDependencies) - ) - const WsProtocol = RpcServer.layerProtocolWebsocket({ path: "/rpc" }).pipe( - Layer.provide(HttpRouter.layer) - ) - const RpcLive = RpcServer.layer(PeerRpc.Rpcs, { disableFatalDefects: true }).pipe( - Layer.provide([PeerHandlersLive, AuthenticationLive]) - ) - const WebSocketServerLive = RpcLive.pipe( - Layer.provideMerge(WsProtocol), - Layer.provide(HttpRouter.serve(WsProtocol, { disableListenLog: true, disableLogger: true })) - ) - const serverContext = yield* Layer.build(WebSocketServerLive.pipe( - Layer.provideMerge(NodeHttpServer.layerTest), - Layer.provide(RpcSerialization.layerMsgPack) - )).pipe( - Effect.provide(serverEngineContext) - ) - const address = Context.get(serverContext, HttpServer.HttpServer).address - if (address._tag !== "TcpAddress") return yield* Effect.die("Expected a TCP test server address") - - let connections = 0 - let disconnections = 0 - yield* Effect.scoped( - Effect.gen(function*() { - const CredentialsLive = Layer.succeed(PeerCredentials.PeerCredentials)({ - get: Effect.sync(() => Redacted.make(credential)) - }) - const ClientAuthenticationLive = PeerAuthentication.layerClient.pipe( - Layer.provide(CredentialsLive) - ) - const ConnectionHooksLive = Layer.succeed(RpcClient.ConnectionHooks)({ - onConnect: Effect.sync(() => connections++).pipe(Effect.asVoid), - onDisconnect: Effect.sync(() => disconnections++).pipe(Effect.asVoid) - }) - const ClientProtocolLive = RpcClient.layerProtocolSocket().pipe( - Layer.provide([ - NodeSocket.layerWebSocket(`ws://127.0.0.1:${address.port}/rpc`), - RpcSerialization.layerMsgPack, - ConnectionHooksLive - ]) - ) - const clientContext = yield* Layer.build(Layer.merge(ClientProtocolLive, ClientAuthenticationLive)) - const client = yield* PeerRpc.makeRpcClient.pipe(Effect.provide(clientContext)) - const session = yield* RpcPeerTransport.makeSession(client, { - peerId: serverPeerId, - documents: [{ document: Task, documentId }], - definition - }).pipe(Effect.provide(clientEngineContext)) - - assert.strictEqual(session.peerId, serverPeerId) - assert.strictEqual(authorizationCalls, 1) - - yield* clientReplica.mutate(RenameTask, { - commandId: yield* Identity.makeCommandId, - documentId, - payload: "renamed by the WebSocket client" - }) - - yield* repeatUntil(Effect.gen(function*() { - yield* session.flush - yield* Effect.all([serverSharding.pollStorage, clientSharding.pollStorage], { discard: true }) - return (yield* serverReplica.get(Task, documentId)).value.title === "renamed by the WebSocket client" - })) - - yield* serverReplica.mutate(AddLabel, { - commandId: yield* Identity.makeCommandId, - documentId, - payload: "server" - }) - - yield* repeatUntil(Effect.gen(function*() { - yield* session.flush - yield* Effect.all([serverSharding.pollStorage, clientSharding.pollStorage], { discard: true }) - const serverSnapshot = yield* serverReplica.get(Task, documentId) - const clientSnapshot = yield* clientReplica.get(Task, documentId) - return JSON.stringify(serverSnapshot.value) === JSON.stringify(clientSnapshot.value) && - JSON.stringify(serverSnapshot.heads) === JSON.stringify(clientSnapshot.heads) - })) - - yield* session.markDirty(documentId) - yield* repeatUntil(Effect.gen(function*() { - yield* session.flush - yield* Effect.all([serverSharding.pollStorage, clientSharding.pollStorage], { discard: true }) - return yield* session.observedByPeer(documentId) - })) - - const serverSnapshot = yield* serverReplica.get(Task, documentId) - const clientSnapshot = yield* clientReplica.get(Task, documentId) - assert.deepStrictEqual(clientSnapshot.value, { - title: "renamed by the WebSocket client", - labels: ["server"] - }) - assert.deepStrictEqual(clientSnapshot.heads, serverSnapshot.heads) - assert.isFalse(yield* session.durableConfirmation(documentId)) - assert.isAtLeast(credentials.length, 2) - assert.isTrue(credentials.every((value) => value === "client-payload-credential")) - - credential = "defect-credential" - const authenticationError = yield* client.Push({ - sessionId: missingSessionId, - payload: Uint8Array.of(1) - }).pipe(Effect.flip) - assert.instanceOf(authenticationError, PeerRpcError.AuthenticationFailure) - assert.notInclude(String(authenticationError), "raw-authentication-secret") - }).pipe(Effect.provide(clientEngineContext)) - ) - - assert.strictEqual(credentials.at(-1), "defect-credential") - assert.strictEqual(connections, 1) - assert.strictEqual(disconnections, 1) - })).pipe(Effect.provide([ - NodeCrypto.layer, - ReplicaLimits.layer(TestReplica.defaultLimits) - ])), - 15_000 - ) - - it.live( - "replays acknowledged, unacknowledged, and offline changes through a fresh session after coordinated restart", - () => - Effect.scoped(Effect.gen(function*() { - const serverEngineContext = yield* Layer.build(ReplicaLive) - const clientEngineContext = yield* Layer.build(ReplicaLive) - const serverReplica = Context.get(serverEngineContext, Replica.Replica) - const clientReplica = Context.get(clientEngineContext, Replica.Replica) - const serverSharding = Context.get(serverEngineContext, Sharding.Sharding) - const clientSharding = Context.get(clientEngineContext, Sharding.Sharding) - - const created = yield* serverReplica.create(Task, { - commandId: yield* Identity.makeCommandId, - value: { title: "before restart", labels: [] } - }) - const documentId = yield* CommandOutcome.committedOrFail(created) - const archive = yield* serverReplica.exportBackup({ - maxBytes: TestReplica.defaultLimits.maxBackupBytes - }).pipe(Stream.runCollect) - yield* clientReplica.restoreBackup({ - source: Stream.fromIterable(archive), - mode: "clone", - maxBytes: TestReplica.defaultLimits.maxBackupBytes, - expectedDefinitionHash: definition.hash, - installationId: yield* Identity.makeBackupInstallationId - }) - - const WebSocketServerLive = makeWebSocketServer(documentId) - - const buildServer = ( - port: number, - engineContext = serverEngineContext - ) => - Effect.gen(function*() { - const scope = yield* Scope.make() - yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)) - const context = yield* Layer.build( - WebSocketServerLive.pipe( - Layer.provideMerge(NodeHttpServer.layer(() => Http.createServer(), { port })), - Layer.provide(RpcSerialization.layerMsgPack) - ) - ).pipe( - Effect.provide(engineContext), - Effect.provideService(Scope.Scope, scope) - ) - const address = Context.get(context, HttpServer.HttpServer).address - if (address._tag !== "TcpAddress") return yield* Effect.die("Expected a TCP test server address") - return { scope, port: address.port } - }) - - const connectSession = (port: number) => - Effect.gen(function*() { - const CredentialsLive = Layer.succeed(PeerCredentials.PeerCredentials)({ - get: Effect.sync(() => Redacted.make("client-payload-credential")) - }) - const ClientAuthenticationLive = PeerAuthentication.layerClient.pipe(Layer.provide(CredentialsLive)) - const ClientProtocolLive = RpcClient.layerProtocolSocket().pipe( - Layer.provide([ - NodeSocket.layerWebSocket(`ws://127.0.0.1:${port}/rpc`), - RpcSerialization.layerMsgPack - ]) - ) - const clientContext = yield* Layer.build(Layer.merge(ClientProtocolLive, ClientAuthenticationLive)) - const client = yield* PeerRpc.makeRpcClient.pipe(Effect.provide(clientContext)) - return yield* RpcPeerTransport.makeSession(client, { - peerId: serverPeerId, - documents: [{ document: Task, documentId }], - definition - }).pipe(Effect.provide(clientEngineContext)) - }) - - const addLabel = (payload: string) => - Effect.gen(function*() { - yield* clientReplica.mutate(AddLabel, { - commandId: yield* Identity.makeCommandId, - documentId, - payload - }) - }) - - const pendingApplyStarted = yield* Deferred.make() - const blockNextApply = yield* Ref.make(false) - const blockingSharding = Sharding.Sharding.of( - { - ...serverSharding, - makeClient: (entity) => - serverSharding.makeClient(entity).pipe( - Effect.map((makeClient) => (entityId) => { - const client = makeClient(entityId) - if (entity.type !== DocumentEntity.DocumentEntity.type) return client - const documentClient = client as ReturnType< - Effect.Success - > - return { - ...documentClient, - ApplySync: (request: typeof DocumentEntity.ApplySync.payloadSchema.Type) => - Ref.getAndSet(blockNextApply, false).pipe( - Effect.flatMap((block) => - block - ? Deferred.succeed(pendingApplyStarted, undefined).pipe(Effect.andThen(Effect.never)) - : documentClient.ApplySync(request) - ) - ) - } as typeof client - }) - ) - } as Sharding.Sharding["Service"] - ) - const server1 = yield* buildServer( - 0, - Context.add(serverEngineContext, Sharding.Sharding, blockingSharding) - ) - const assignedPort = server1.port - - const session1Scope = yield* Scope.make() - yield* Effect.addFinalizer(() => Scope.close(session1Scope, Exit.void)) - const session1 = yield* connectSession(assignedPort).pipe( - Effect.provideService(Scope.Scope, session1Scope) - ) - - yield* addLabel("committed-before-restart") - yield* repeatUntil(Effect.gen(function*() { - yield* session1.flush - yield* Effect.all([serverSharding.pollStorage, clientSharding.pollStorage], { discard: true }) - return (yield* serverReplica.get(Task, documentId)).value.labels.includes("committed-before-restart") - })) - yield* Ref.set(blockNextApply, true) - yield* addLabel("pending-at-restart") - yield* session1.markDirty(documentId) - yield* session1.flush - yield* Deferred.await(pendingApplyStarted) - assert.notInclude((yield* serverReplica.get(Task, documentId)).value.labels, "pending-at-restart") - - const closingServer = yield* Scope.close(server1.scope, Exit.void).pipe(Effect.forkChild) - // The client scope owns the upgraded socket. Close it while the server - // drains so both sides finish before the same port is rebound. - yield* Scope.close(session1Scope, Exit.void) - yield* Fiber.join(closingServer).pipe(Effect.timeout("5 seconds")) - - yield* addLabel("committed-while-offline") - - const server2 = yield* buildServer(assignedPort) - assert.strictEqual(server2.port, assignedPort) - - const session2Scope = yield* Scope.make() - yield* Effect.addFinalizer(() => Scope.close(session2Scope, Exit.void)) - const session2 = yield* connectSession(assignedPort).pipe( - Effect.provideService(Scope.Scope, session2Scope) - ) - - yield* repeatUntil(Effect.gen(function*() { - yield* Effect.all([serverSharding.pollStorage, clientSharding.pollStorage], { discard: true }) - const server = yield* serverReplica.get(Task, documentId) - const client = yield* clientReplica.get(Task, documentId) - const converged = server.value.labels.includes("committed-while-offline") && - JSON.stringify(server.value) === JSON.stringify(client.value) && - JSON.stringify(server.heads) === JSON.stringify(client.heads) - if (!converged) yield* session2.flush - return converged - })) - const serverSnapshot = yield* serverReplica.get(Task, documentId) - const clientSnapshot = yield* clientReplica.get(Task, documentId) - assert.deepStrictEqual(serverSnapshot.value, clientSnapshot.value) - assert.deepStrictEqual(serverSnapshot.heads, clientSnapshot.heads) - // Every mutation is applied exactly once: no work lost across the restart, and the - // change the server already held before the restart is not applied twice on redelivery. - assert.deepStrictEqual([...serverSnapshot.value.labels].toSorted(), [ - "committed-before-restart", - "committed-while-offline", - "pending-at-restart" - ]) - - yield* Scope.close(session2Scope, Exit.void) - yield* Scope.close(server2.scope, Exit.void) - })).pipe(Effect.provide([ - NodeCrypto.layer, - ReplicaLimits.layer(TestReplica.defaultLimits) - ])), - 30_000 - ) - - it.live( - "rejects a version-skewed reconnect with a typed error and applies no data", - () => - Effect.scoped(Effect.gen(function*() { - const serverEngineContext = yield* Layer.build(ReplicaLive) - const clientEngineContext = yield* Layer.build(ReplicaLive) - const serverReplica = Context.get(serverEngineContext, Replica.Replica) - const clientReplica = Context.get(clientEngineContext, Replica.Replica) - const serverSharding = Context.get(serverEngineContext, Sharding.Sharding) - const clientSharding = Context.get(clientEngineContext, Sharding.Sharding) - - const created = yield* serverReplica.create(Task, { - commandId: yield* Identity.makeCommandId, - value: { title: "matched build", labels: [] } - }) - const documentId = yield* CommandOutcome.committedOrFail(created) - const archive = yield* serverReplica.exportBackup({ - maxBytes: TestReplica.defaultLimits.maxBackupBytes - }).pipe(Stream.runCollect) - yield* clientReplica.restoreBackup({ - source: Stream.fromIterable(archive), - mode: "clone", - maxBytes: TestReplica.defaultLimits.maxBackupBytes, - expectedDefinitionHash: definition.hash, - installationId: yield* Identity.makeBackupInstallationId - }) - - const serverContext = yield* Layer.build( - makeWebSocketServer(documentId).pipe( - Layer.provideMerge(NodeHttpServer.layerTest), - Layer.provide(RpcSerialization.layerMsgPack) - ) - ).pipe(Effect.provide(serverEngineContext)) - const address = Context.get(serverContext, HttpServer.HttpServer).address - if (address._tag !== "TcpAddress") return yield* Effect.die("Expected a TCP test server address") - - const CredentialsLive = Layer.succeed(PeerCredentials.PeerCredentials)({ - get: Effect.sync(() => Redacted.make("client-payload-credential")) - }) - const ClientAuthenticationLive = PeerAuthentication.layerClient.pipe(Layer.provide(CredentialsLive)) - const buildClient = Effect.gen(function*() { - const ClientProtocolLive = RpcClient.layerProtocolSocket().pipe( - Layer.provide([ - NodeSocket.layerWebSocket(`ws://127.0.0.1:${address.port}/rpc`), - RpcSerialization.layerMsgPack - ]) - ) - const clientContext = yield* Layer.build(Layer.merge(ClientProtocolLive, ClientAuthenticationLive)) - return yield* PeerRpc.makeRpcClient.pipe(Effect.provide(clientContext)) - }) - - yield* Effect.scoped(Effect.gen(function*() { - const client = yield* buildClient - const session = yield* RpcPeerTransport.makeSession(client, { - peerId: serverPeerId, - documents: [{ document: Task, documentId }], - definition - }).pipe(Effect.provide(clientEngineContext)) - yield* clientReplica.mutate(AddLabel, { - commandId: yield* Identity.makeCommandId, - documentId, - payload: "matched" - }) - yield* repeatUntil(Effect.gen(function*() { - yield* session.flush - yield* Effect.all([serverSharding.pollStorage, clientSharding.pollStorage], { discard: true }) - return (yield* serverReplica.get(Task, documentId)).value.labels.includes("matched") - })) - })) - - const baseline = yield* serverReplica.get(Task, documentId) - - // A peer built against a newer wire protocol reconnects. The transport pins the - // current protocol version, so drive Open directly to model the skewed build. - yield* Effect.scoped(Effect.gen(function*() { - const client = yield* buildClient - const error = yield* client.Open({ - protocolVersion: PeerRpc.protocolVersion + 1, - expectedPeerId: serverPeerId, - definitionHash: definition.hash, - documents: [{ documentType: Task.name, documentId }] - }).pipe(Stream.runDrain, Effect.flip) - assert.strictEqual(error._tag, "UnsupportedVersion") - })) - - yield* Effect.all([serverSharding.pollStorage, clientSharding.pollStorage], { discard: true }) - const afterSkew = yield* serverReplica.get(Task, documentId) - assert.deepStrictEqual(afterSkew.value, baseline.value) - assert.deepStrictEqual(afterSkew.heads, baseline.heads) - - // A matched-build client still converges afterwards: the skew was rejected, not corrupting. - yield* Effect.scoped(Effect.gen(function*() { - const client = yield* buildClient - const session = yield* RpcPeerTransport.makeSession(client, { - peerId: serverPeerId, - documents: [{ document: Task, documentId }], - definition - }).pipe(Effect.provide(clientEngineContext)) - yield* clientReplica.mutate(AddLabel, { - commandId: yield* Identity.makeCommandId, - documentId, - payload: "after-skew" - }) - yield* repeatUntil(Effect.gen(function*() { - yield* session.flush - yield* Effect.all([serverSharding.pollStorage, clientSharding.pollStorage], { discard: true }) - const serverSnapshot = yield* serverReplica.get(Task, documentId) - const clientSnapshot = yield* clientReplica.get(Task, documentId) - return serverSnapshot.value.labels.includes("after-skew") && - JSON.stringify(serverSnapshot.value) === JSON.stringify(clientSnapshot.value) - })) - })) - })).pipe(Effect.provide([ - NodeCrypto.layer, - ReplicaLimits.layer(TestReplica.defaultLimits) - ])), - 20_000 - ) - - it.live( - "rejects a definition-hash-skewed peer at session open and syncs no data", - () => - Effect.scoped(Effect.gen(function*() { - const SkewTask = Document.make("WebSocketSkewTask", { - schema: Schema.Struct({ title: Schema.String, labels: Schema.Array(Schema.String) }), - version: 1 - }) - const SkewAddLabel = Mutation.make("WebSocketSkewTask.AddLabel", { document: SkewTask, payload: Schema.String }) - const serverDefinition = ReplicaDefinition.make({ - name: "rpc-skew-test", - documents: DocumentSet.make(SkewTask), - mutations: [SkewAddLabel], - projections: [], - queries: [] - }) - const SkewTaskV2 = Document.make("WebSocketSkewTask", { - schema: Schema.Struct({ title: Schema.String, labels: Schema.Array(Schema.String) }), - version: 2 - }) - const SkewAddLabelV2 = Mutation.make("WebSocketSkewTask.AddLabel", { - document: SkewTaskV2, - payload: Schema.String - }) - const clientDefinition = ReplicaDefinition.make({ - name: "rpc-skew-test", - documents: DocumentSet.make(SkewTaskV2), - mutations: [SkewAddLabelV2], - projections: [], - queries: [] - }) - assert.notStrictEqual(serverDefinition.hash, clientDefinition.hash) - - const ServerReplicaLive = TestReplica.layer(serverDefinition, { projections: [] }).pipe( - Layer.provide(SkewAddLabel.toLayer(({ draft, payload }) => { - draft.labels.push(payload) - return undefined - })) - ) - const ClientReplicaLive = TestReplica.layer(clientDefinition, { projections: [] }).pipe( - Layer.provide(SkewAddLabelV2.toLayer(({ draft, payload }) => { - draft.labels.push(payload) - return undefined - })) - ) - const serverEngineContext = yield* Layer.build(ServerReplicaLive) - const clientEngineContext = yield* Layer.build(ClientReplicaLive) - const serverReplica = Context.get(serverEngineContext, Replica.Replica) - const clientReplica = Context.get(clientEngineContext, Replica.Replica) - const serverSharding = Context.get(serverEngineContext, Sharding.Sharding) - const clientSharding = Context.get(clientEngineContext, Sharding.Sharding) - - // Two skewed builds own the same logical document: seed the same deterministic id on both. - const sharedCommandId = yield* Identity.makeCommandId - const createdServer = yield* serverReplica.create(SkewTask, { - commandId: sharedCommandId, - value: { title: "seed", labels: [] } - }) - const documentId = yield* CommandOutcome.committedOrFail(createdServer) - const createdClient = yield* clientReplica.create(SkewTaskV2, { - commandId: sharedCommandId, - value: { title: "seed", labels: [] } - }) - assert.strictEqual(documentId, yield* CommandOutcome.committedOrFail(createdClient)) - - const serverContext = yield* Layer.build( - makeWebSocketServer(documentId, SkewTask, serverDefinition).pipe( - Layer.provideMerge(NodeHttpServer.layerTest), - Layer.provide(RpcSerialization.layerMsgPack) - ) - ).pipe(Effect.provide(serverEngineContext)) - const address = Context.get(serverContext, HttpServer.HttpServer).address - if (address._tag !== "TcpAddress") return yield* Effect.die("Expected a TCP test server address") - - const CredentialsLive = Layer.succeed(PeerCredentials.PeerCredentials)({ - get: Effect.sync(() => Redacted.make("client-payload-credential")) - }) - const ClientAuthenticationLive = PeerAuthentication.layerClient.pipe(Layer.provide(CredentialsLive)) - - const openExit = yield* Effect.scoped(Effect.gen(function*() { - const ClientProtocolLive = RpcClient.layerProtocolSocket().pipe( - Layer.provide([ - NodeSocket.layerWebSocket(`ws://127.0.0.1:${address.port}/rpc`), - RpcSerialization.layerMsgPack - ]) - ) - const clientContext = yield* Layer.build(Layer.merge(ClientProtocolLive, ClientAuthenticationLive)) - const client = yield* PeerRpc.makeRpcClient.pipe(Effect.provide(clientContext)) - return yield* RpcPeerTransport.makeSession(client, { - peerId: serverPeerId, - documents: [{ document: SkewTaskV2, documentId }], - definition: clientDefinition - }).pipe(Effect.provide(clientEngineContext)) - })).pipe(Effect.exit) - - assert.isTrue(Exit.isFailure(openExit), "skewed session must be rejected at open") - if (Exit.isFailure(openExit)) { - const error = Cause.findErrorOption(openExit.cause) - assert.isTrue(Option.isSome(error) && error.value.reason._tag === "ProtocolMismatch") - } - - // No data crossed the rejected boundary: a client mutation never reaches the server. - yield* clientReplica.mutate(SkewAddLabelV2, { - commandId: yield* Identity.makeCommandId, - documentId, - payload: "from-skewed-client" - }) - yield* Effect.all([serverSharding.pollStorage, clientSharding.pollStorage], { discard: true }) - const serverSnapshot = yield* serverReplica.get(SkewTask, documentId) - assert.deepStrictEqual(serverSnapshot.value.labels, []) - })).pipe(Effect.provide([ - NodeCrypto.layer, - ReplicaLimits.layer(TestReplica.defaultLimits) - ])), - 30_000 - ) -}) diff --git a/packages/local-rpc/test/PublicApi.types.test.ts b/packages/local-rpc/test/PublicApi.types.test.ts index 64206f9..3cc8a2a 100644 --- a/packages/local-rpc/test/PublicApi.types.test.ts +++ b/packages/local-rpc/test/PublicApi.types.test.ts @@ -3,46 +3,46 @@ import * as Schema from "effect/Schema" import * as PublicApi from "../src/index.js" describe("public API", () => { - it("exports the version 3 relay protocol without widening version 2", () => { + it("exports the durable protocol as version 1", () => { for ( const name of [ "PeerRelayAuthorization", "PeerRelayIngress", "PeerRelayLimits", - "PeerRelayRpc", - "PeerRelayStore" + "PeerRpc", + "PeerRelayStore", + "PeerRpcServer", + "SqlPeerRelayStore" ] ) { assert.property(PublicApi, name) } - const decoded = Schema.decodeUnknownSync(PublicApi.PeerRelayRpc.RelayOpened)({ - _tag: "RelayOpened", - version: 3, + for (const name of ["PeerRelayRpc", "PeerAuthorization", "PeerRpcLimits"]) { + assert.notProperty(PublicApi, name) + } + + const decoded = Schema.decodeUnknownSync(PublicApi.PeerRpc.Opened)({ + _tag: "Opened", + protocolVersion: 1, sessionId: "ses_00000000-0000-4000-8000-000000000001", remotePeerId: "peer_00000000-0000-4000-8000-000000000002", authenticatedLocal: { tenantId: "tenant", subjectId: "subject", peerId: "peer_00000000-0000-4000-8000-000000000001" - }, - capabilities: { - storeAndForward: true } }) assert.deepStrictEqual(decoded, { - _tag: "RelayOpened", - version: 3, + _tag: "Opened", + protocolVersion: 1, sessionId: "ses_00000000-0000-4000-8000-000000000001", remotePeerId: "peer_00000000-0000-4000-8000-000000000002", authenticatedLocal: { tenantId: "tenant", subjectId: "subject", peerId: "peer_00000000-0000-4000-8000-000000000001" - }, - capabilities: { - storeAndForward: true } }) }) diff --git a/packages/local-rpc/test/RpcPeerTransport.test.ts b/packages/local-rpc/test/RpcPeerTransport.test.ts index e80bb3f..3b3e135 100644 --- a/packages/local-rpc/test/RpcPeerTransport.test.ts +++ b/packages/local-rpc/test/RpcPeerTransport.test.ts @@ -25,30 +25,22 @@ import * as Option from "effect/Option" import * as Queue from "effect/Queue" import * as Ref from "effect/Ref" import * as Schema from "effect/Schema" -import * as Scope from "effect/Scope" import * as Stream from "effect/Stream" import * as Tracer from "effect/Tracer" -import { RpcClientDefect, RpcClientError } from "effect/unstable/rpc/RpcClientError" -import * as PeerRpcObservability from "../src/internal/peerRpcObservability.js" -import * as PeerRelayRpc from "../src/PeerRelayRpc.js" import * as PeerRpc from "../src/PeerRpc.js" import * as PeerRpcError from "../src/PeerRpcError.js" import * as RpcPeerTransport from "../src/RpcPeerTransport.js" const Task = Document.make("Task", { schema: Schema.Struct({ title: Schema.String }), version: 1 }) -const TaskV2 = Document.make("Task", { schema: Schema.Struct({ title: Schema.String }), version: 2 }) const documentId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000001") const replicaId = Identity.ReplicaId.make("rep_00000000-0000-4000-8000-000000000001") -const serverPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000001") -const otherPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000002") const sessionId = Identity.SessionId.make("ses_00000000-0000-4000-8000-000000000001") -const otherSessionId = Identity.SessionId.make("ses_00000000-0000-4000-8000-000000000002") const relayPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000003") const localPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000004") const remotePeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000005") const relayMessageId = Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000001") const senderReplicaIncarnation = Identity.ReplicaIncarnation.make(1) -const claimToken = PeerRelayRpc.ClaimToken.make("clm_00000000-0000-4000-8000-000000000001") +const claimToken = PeerRpc.ClaimToken.make("clm_00000000-0000-4000-8000-000000000001") const relayMessageHash = "1".repeat(64) const rewrittenLineage = Identity.DocumentLineage.make("lin_00000000-0000-4000-8000-000000000001") const documents = [{ document: Task, documentId }] @@ -60,36 +52,7 @@ const definition = ReplicaDefinition.make({ queries: [] }) -const opened = (peerId: Identity.PeerId, openSessionId: Identity.SessionId) => - PeerRpc.Opened.make({ - _tag: "Opened", - protocolVersion: PeerRpc.protocolVersion, - sessionId: openSessionId, - peerId, - capabilities: { storeAndForward: false } - }) -const serverOpened = opened(serverPeerId, sessionId) - -const makeClient = ( - open: ( - request: typeof PeerRpc.OpenRpc.payloadSchema.Type, - options: { readonly streamBufferSize?: number | undefined } - ) => Stream.Stream, - push: (request: typeof PeerRpc.PushRpc.payloadSchema.Type) => Effect.Effect -): PeerRpc.RpcClient => ({ Open: open, Push: push }) as never - -const connect = (client: PeerRpc.RpcClient, peerId: Identity.PeerId) => - Effect.gen(function*() { - const context = yield* Layer.build(RpcPeerTransport.layer(client, { documents, definition })) - return yield* Context.get(context, PeerTransport.PeerTransport).connect({ replicaId, peerId }) - }) - -const liveOpen = (event: PeerRpc.OpenEvent) => Stream.concat(Stream.make(event), Stream.never) - -const openWithMessage = (payload: Uint8Array) => - Stream.fromIterable([serverOpened, PeerRpc.Message.make({ _tag: "Message", payload })]).pipe(Stream.rechunk(1)) - -const relayOptions: RpcPeerTransport.StoreAndForwardOptions = { +const relayOptions: RpcPeerTransport.Options = { expectedLocal: { tenantId: "tenant", subjectId: "local-subject", @@ -108,35 +71,34 @@ const relayOptions: RpcPeerTransport.StoreAndForwardOptions = { replayBatchSize: 16 } -const relayOpened = PeerRelayRpc.RelayOpened.make({ - _tag: "RelayOpened", - version: PeerRelayRpc.protocolVersion, +const relayOpened = PeerRpc.Opened.make({ + _tag: "Opened", + protocolVersion: PeerRpc.protocolVersion, sessionId, remotePeerId, - authenticatedLocal: relayOptions.expectedLocal, - capabilities: { storeAndForward: true } + authenticatedLocal: relayOptions.expectedLocal }) const makeRelayClient = ( open: ( - request: typeof PeerRelayRpc.OpenRelayRpc.payloadSchema.Type, + request: typeof PeerRpc.OpenRpc.payloadSchema.Type, options: { readonly streamBufferSize?: number | undefined } - ) => Stream.Stream, + ) => Stream.Stream, push: ( - request: typeof PeerRelayRpc.PushRelayRpc.payloadSchema.Type + request: typeof PeerRpc.PushRpc.payloadSchema.Type ) => Effect.Effect = () => Effect.void, acknowledge: ( - request: typeof PeerRelayRpc.AcknowledgeRelayRpc.payloadSchema.Type + request: typeof PeerRpc.AcknowledgeRpc.payloadSchema.Type ) => Effect.Effect = () => Effect.void, reject: ( - request: typeof PeerRelayRpc.RejectRelayRpc.payloadSchema.Type + request: typeof PeerRpc.RejectRpc.payloadSchema.Type ) => Effect.Effect = () => Effect.void -): PeerRelayRpc.RpcClient => +): PeerRpc.RpcClient => ({ - OpenRelay: open, - PushRelay: push, - AcknowledgeRelay: acknowledge, - RejectRelay: reject + Open: open, + Push: push, + Acknowledge: acknowledge, + Reject: reject }) as never const relayEntry = (payload: Uint8Array): PeerRelayOutbox.Entry => ({ @@ -153,7 +115,7 @@ const relayEntry = (payload: Uint8Array): PeerRelayOutbox.Entry => ({ relayPeerId, relayMessageId, outerEnvelopeDigest: "2".repeat(64), - protocolVersion: 3, + protocolVersion: PeerRpc.protocolVersion, payloadVersion: 1, senderConnectionEpoch: "sender-epoch", senderSequence: 0, @@ -220,7 +182,7 @@ const makeStoredMessage = ( remote: relayOptions.expectedLocal, relayPeerId, relayMessageId, - protocolVersion: PeerRelayRpc.protocolVersion, + protocolVersion: PeerRpc.protocolVersion, payloadVersion: PeerSyncEnvelope.syncEnvelopeVersion, senderReplicaIncarnation, senderConnectionEpoch: "remote-epoch", @@ -232,7 +194,7 @@ const makeStoredMessage = ( payload } const outerEnvelopeDigest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope(envelope) - return PeerRelayRpc.StoredMessage.make({ + return PeerRpc.StoredMessage.make({ _tag: "StoredMessage", relayMessageId, claimToken, @@ -294,13 +256,13 @@ const makeRuntime = (overrides: RuntimeOverrides = {}) => }) const connectRelay = ( - client: PeerRelayRpc.RpcClient, + client: PeerRpc.RpcClient, runtime: ReturnType, peerId: Identity.PeerId = remotePeerId ) => Effect.gen(function*() { const context = yield* Layer.build( - RpcPeerTransport.layerStoreAndForward(client, relayOptions).pipe( + RpcPeerTransport.layer(client, relayOptions).pipe( Layer.provide(Layer.merge( Layer.succeed(PeerRelayClientRuntime.PeerRelayClientRuntime, runtime), Layer.merge( @@ -317,891 +279,12 @@ const connectRelay = ( }) describe("RpcPeerTransport", () => { - it.effect("validates the first streamed event as the handshake and uses response capacity one", () => - Effect.scoped(Effect.gen(function*() { - const payload = Uint8Array.of(1, 2, 3) - const client = makeClient( - (request, options) => { - assert.strictEqual(options.streamBufferSize, 1) - assert.strictEqual(request.expectedPeerId, serverPeerId) - assert.deepStrictEqual(request.documents, [{ documentType: Task.name, documentId }]) - return openWithMessage(payload) - }, - () => Effect.void - ) - const connection = yield* connect(client, serverPeerId) - assert.strictEqual(connection.peerId, serverPeerId) - assert.deepStrictEqual(yield* Stream.runCollect(connection.receive), [payload]) - yield* connection.close - }))) - - it.effect("rejects selected documents that do not belong to the supplied definition", () => - Effect.scoped(Effect.gen(function*() { - const opens = yield* Ref.make(0) - const client = makeClient( - () => Ref.update(opens, (count) => count + 1).pipe(Effect.as(liveOpen(serverOpened)), Stream.unwrap), - () => Effect.void - ) - const context = yield* Layer.build(RpcPeerTransport.layer(client, { - documents: [{ document: TaskV2, documentId }], - definition - })) - const exit = yield* Context.get(context, PeerTransport.PeerTransport).connect({ - replicaId, - peerId: serverPeerId - }).pipe(Effect.exit) - assert.isTrue(Exit.isFailure(exit)) - if (Exit.isFailure(exit)) { - const error = Cause.findErrorOption(exit.cause) - assert.isTrue(Option.isSome(error) && error.value.reason._tag === "ProtocolMismatch") - } - assert.strictEqual(yield* Ref.get(opens), 0) - }))) - - it.effect("records adapter boundaries without peer session document or payload values", () => { - const spans: Array = [] - const tracer = Tracer.make({ - span: (options) => { - const span = new Tracer.NativeSpan(options) - spans.push(span) - return span - } - }) - return Effect.scoped(Effect.gen(function*() { - const client = makeClient( - () => liveOpen(serverOpened), - () => Effect.logDebug("adapter-log-forbidden-value") - ) - const connection = yield* connect(client, serverPeerId) - yield* connection.send(Uint8Array.of(201, 202, 203)) - yield* connection.close - assert.strictEqual( - (yield* Metric.value(PeerRpcObservability.boundary("AdapterOpen", "Success"))).count, - 1 - ) - assert.strictEqual( - (yield* Metric.value(PeerRpcObservability.boundary("AdapterPush", "Success"))).count, - 1 - ) - const safeSpans = spans.filter((span) => span.name.startsWith("effect_local_rpc.adapter.")) - assert.deepStrictEqual( - new Set(safeSpans.map((span) => span.name)), - new Set(["effect_local_rpc.adapter.open", "effect_local_rpc.adapter.push"]) - ) - for (const span of safeSpans) { - assert.strictEqual(span.status._tag, "Ended") - if (span.status._tag === "Ended") assert.isTrue(Exit.isSuccess(span.status.exit)) - assert.deepStrictEqual(span.events, []) - } - const telemetry = JSON.stringify(safeSpans.map((span) => ({ - name: span.name, - attributes: [...span.attributes] - }))) + (yield* Metric.dump) - for (const forbidden of [serverPeerId, sessionId, documentId, "201,202,203", "adapter-log-forbidden-value"]) { - assert.notInclude(telemetry, forbidden) - } - })).pipe( - Effect.provideService(Metric.MetricRegistry, new Map()), - Effect.provideService(Tracer.Tracer, tracer) - ) - }) - - it.effect("rejects Message before Opened and closes the child scope", () => - Effect.scoped(Effect.gen(function*() { - const finalized = yield* Ref.make(0) - const client = makeClient( - () => - Stream.make(PeerRpc.Message.make({ _tag: "Message", payload: Uint8Array.of(1) })).pipe( - Stream.ensuring(Ref.update(finalized, (count) => count + 1)) - ), - () => Effect.void - ) - const error = yield* connect(client, serverPeerId).pipe(Effect.flip) - assert.strictEqual(error.reason._tag, "ProtocolMismatch") - assert.strictEqual(yield* Ref.get(finalized), 1) - }))) - - it.effect("rejects an Open stream that ends before the handshake", () => - Effect.scoped(Effect.gen(function*() { - const client = makeClient(() => Stream.empty, () => Effect.void) - const error = yield* connect(client, serverPeerId).pipe(Effect.flip) - assert.strictEqual(error.reason._tag, "ProtocolMismatch") - }))) - - it.effect("rejects a mismatched server peer identity", () => - Effect.scoped(Effect.gen(function*() { - const finalized = yield* Ref.make(0) - const client = makeClient( - () => - liveOpen(opened(otherPeerId, sessionId)).pipe( - Stream.ensuring(Ref.update(finalized, (count) => count + 1)) - ), - () => Effect.void - ) - const error = yield* connect(client, serverPeerId).pipe(Effect.flip) - assert.strictEqual(error.reason._tag, "ProtocolMismatch") - if (error.reason._tag === "ProtocolMismatch") { - assert.strictEqual(error.reason.expected, serverPeerId) - assert.strictEqual(error.reason.observed, otherPeerId) - } - assert.strictEqual(yield* Ref.get(finalized), 1) - }))) - - it.effect("rejects another Opened event after the handshake", () => - Effect.scoped(Effect.gen(function*() { - const client = makeClient( - () => Stream.fromIterable([serverOpened, opened(serverPeerId, otherSessionId)]).pipe(Stream.rechunk(1)), - () => Effect.void - ) - const connection = yield* connect(client, serverPeerId) - const error = yield* Stream.runDrain(connection.receive).pipe(Effect.flip) - assert.strictEqual(error.reason._tag, "ProtocolMismatch") - yield* connection.close - }))) - - it.effect("serializes concurrent send calls", () => - Effect.scoped(Effect.gen(function*() { - const starts = yield* Queue.unbounded() - const release = yield* Deferred.make() - const client = makeClient( - () => liveOpen(serverOpened), - (request) => - Queue.offer(starts, request.payload[0]).pipe( - Effect.andThen(request.payload[0] === 1 ? Deferred.await(release) : Effect.void) - ) - ) - const connection = yield* connect(client, serverPeerId) - const first = yield* connection.send(Uint8Array.of(1)).pipe(Effect.forkChild) - assert.strictEqual(yield* Queue.take(starts), 1) - const second = yield* connection.send(Uint8Array.of(2)).pipe(Effect.forkChild) - yield* Effect.yieldNow - assert.isTrue(Option.isNone(yield* Queue.poll(starts))) - yield* Deferred.succeed(release, undefined) - yield* Fiber.join(first) - assert.strictEqual(yield* Queue.take(starts), 2) - yield* Fiber.join(second) - yield* connection.close - }))) - - it.effect("maps typed RPC failures to stable public ReplicaError values", () => - Effect.scoped(Effect.gen(function*() { - const permanent = [ - new PeerRpcError.AuthenticationFailure(), - new PeerRpcError.AccessDenied(), - new PeerRpcError.UnsupportedVersion(), - new PeerRpcError.PeerMismatch(), - new PeerRpcError.InvalidRequest(), - new PeerRpcError.DocumentLineageChanged() - ] - for (const rpcError of permanent) { - const client = makeClient(() => Stream.fail(rpcError), () => Effect.void) - const error = yield* connect(client, serverPeerId).pipe(Effect.flip) - assert.strictEqual(error.reason._tag, "ProtocolMismatch") - // The wire tag stays diagnosable without inventing the document identity it withholds. - if (error.reason._tag === "ProtocolMismatch") { - assert.strictEqual(error.reason.observed, rpcError._tag) - } - assert.isFalse(RpcPeerTransport.isRetryable(error)) - } - const limited = makeClient(() => Stream.fail(new PeerRpcError.RequestLimitExceeded()), () => Effect.void) - const limitError = yield* connect(limited, serverPeerId).pipe(Effect.flip) - assert.strictEqual(limitError.reason._tag, "ProtocolMismatch") - assert.isFalse(RpcPeerTransport.isRetryable(limitError)) - }))) - - it.effect("ends receive with a retryable public error when the Open stream fails", () => - Effect.scoped(Effect.gen(function*() { - const events = yield* Queue.unbounded() - yield* Queue.offer(events, serverOpened) - const client = makeClient( - () => Stream.fromQueue(events), - () => Effect.void - ) - const connection = yield* connect(client, serverPeerId) - yield* Queue.fail(events, new PeerRpcError.ServerUnavailable()) - const receiveExit = yield* Effect.exit(Stream.runDrain(connection.receive)) - assert.isTrue(Exit.isFailure(receiveExit)) - if (Exit.isFailure(receiveExit)) { - const error = Cause.findErrorOption(receiveExit.cause) - assert.isTrue(Option.isSome(error)) - if (Option.isSome(error)) { - assert.strictEqual(error.value.reason._tag, "StorageUnavailable") - assert.isTrue(RpcPeerTransport.isRetryable(error.value)) - } - } - }))) - - it.effect("interrupts the Open stream exactly once when closed", () => - Effect.scoped(Effect.gen(function*() { - const finalized = yield* Ref.make(0) - const client = makeClient( - () => liveOpen(serverOpened).pipe(Stream.ensuring(Ref.update(finalized, (count) => count + 1))), - () => Effect.void - ) - const connection = yield* connect(client, serverPeerId) - yield* connection.close - yield* connection.close - assert.strictEqual(yield* Ref.get(finalized), 1) - }))) - - it.effect("does not send after close", () => - Effect.scoped(Effect.gen(function*() { - const pushes = yield* Ref.make(0) - const client = makeClient( - () => liveOpen(serverOpened), - () => Ref.update(pushes, (count) => count + 1) - ) - const connection = yield* connect(client, serverPeerId) - yield* connection.close - const error = yield* connection.send(Uint8Array.of(1)).pipe(Effect.flip) - assert.strictEqual(error.reason._tag, "StorageUnavailable") - assert.strictEqual(yield* Ref.get(pushes), 0) - }))) - - it.effect("does not send after the supplied ambient scope closes", () => - Effect.gen(function*() { - const ambient = yield* Scope.make() - const pushes = yield* Ref.make(0) - const client = makeClient( - () => liveOpen(serverOpened), - () => Ref.update(pushes, (count) => count + 1) - ) - const connection = yield* connect(client, serverPeerId).pipe(Effect.provideService(Scope.Scope, ambient)) - yield* Scope.close(ambient, Exit.void) - const error = yield* connection.send(Uint8Array.of(1)).pipe(Effect.flip) - assert.strictEqual(error.reason._tag, "StorageUnavailable") - assert.strictEqual(yield* Ref.get(pushes), 0) - })) - - it.effect("interrupts a pending Open when the supplied ambient scope closes", () => - Effect.gen(function*() { - const ambient = yield* Scope.make() - const openStarted = yield* Deferred.make() - const openFinalized = yield* Ref.make(0) - const client = makeClient( - () => - Stream.fromEffect( - Deferred.succeed(openStarted, undefined).pipe( - Effect.andThen(Effect.never) - ) - ).pipe( - Stream.ensuring(Ref.update(openFinalized, (count) => count + 1)) - ), - () => Effect.void - ) - const connecting = yield* connect(client, serverPeerId).pipe( - Effect.provideService(Scope.Scope, ambient), - Effect.forkChild - ) - yield* Deferred.await(openStarted) - yield* Scope.close(ambient, Exit.void) - assert.isTrue(Exit.isFailure(yield* Fiber.await(connecting))) - assert.strictEqual(yield* Ref.get(openFinalized), 1) - })) - - it.effect("does not complete Open after the supplied ambient scope closes during request construction", () => - Effect.gen(function*() { - const ambient = yield* Scope.make() - const pulled = yield* Ref.make(0) - const client = makeClient( - () => { - Effect.runSync(Scope.close(ambient, Exit.void)) - return Stream.fromEffect( - Ref.updateAndGet(pulled, (count) => count + 1).pipe( - Effect.as(serverOpened), - Effect.uninterruptible - ) - ) - }, - () => Effect.void - ) - const error = yield* connect(client, serverPeerId).pipe( - Effect.provideService(Scope.Scope, ambient), - Effect.flip - ) - assert.strictEqual(error.reason._tag, "StorageUnavailable") - assert.strictEqual(yield* Ref.get(pulled), 0) - })) - - it.effect("rejects a send racing supplied ambient scope cleanup", () => - Effect.gen(function*() { - const ambient = yield* Scope.make() - const pushStarted = yield* Deferred.make() - const cleanupStarted = yield* Deferred.make() - const cleanupRelease = yield* Deferred.make() - const pushes = yield* Ref.make(0) - const client = makeClient( - () => liveOpen(serverOpened), - () => - Ref.update(pushes, (count) => count + 1).pipe( - Effect.andThen(Deferred.succeed(pushStarted, undefined)), - Effect.andThen(Effect.never), - Effect.ensuring( - Deferred.succeed(cleanupStarted, undefined).pipe( - Effect.andThen(Deferred.await(cleanupRelease)) - ) - ) - ) - ) - const connection = yield* connect(client, serverPeerId).pipe(Effect.provideService(Scope.Scope, ambient)) - const sending = yield* connection.send(Uint8Array.of(1)).pipe(Effect.forkChild) - yield* Deferred.await(pushStarted) - const closing = yield* Scope.close(ambient, Exit.void).pipe(Effect.forkChild) - yield* Deferred.await(cleanupStarted) - const error = yield* connection.send(Uint8Array.of(2)).pipe(Effect.flip) - assert.strictEqual(error.reason._tag, "StorageUnavailable") - assert.strictEqual(yield* Ref.get(pushes), 1) - yield* Deferred.succeed(cleanupRelease, undefined) - yield* Fiber.join(closing) - assert.isTrue(Exit.isFailure(yield* Fiber.await(sending))) - })) - - it.effect("joins explicit and ambient close races through one child cleanup", () => - Effect.gen(function*() { - const ambient = yield* Scope.make() - const pushStarted = yield* Deferred.make() - const pushCleanupStarted = yield* Deferred.make() - const pushCleanupRelease = yield* Deferred.make() - const childFinalized = yield* Ref.make(0) - const openFinalized = yield* Ref.make(0) - const pushFinalized = yield* Ref.make(0) - const pushes = yield* Ref.make(0) - const client = makeClient( - () => - Stream.unwrap(Effect.gen(function*() { - const scope = yield* Scope.Scope - yield* Scope.addFinalizer(scope, Ref.update(childFinalized, (count) => count + 1)) - return liveOpen(serverOpened).pipe( - Stream.ensuring(Ref.update(openFinalized, (count) => count + 1)) - ) - })), - () => - Ref.update(pushes, (count) => count + 1).pipe( - Effect.andThen(Deferred.succeed(pushStarted, undefined)), - Effect.andThen(Effect.never), - Effect.ensuring( - Deferred.succeed(pushCleanupStarted, undefined).pipe( - Effect.andThen(Deferred.await(pushCleanupRelease)), - Effect.andThen(Ref.update(pushFinalized, (count) => count + 1)) - ) - ) - ) - ) - const connection = yield* connect(client, serverPeerId).pipe(Effect.provideService(Scope.Scope, ambient)) - const sending = yield* connection.send(Uint8Array.of(1)).pipe(Effect.forkChild) - yield* Deferred.await(pushStarted) - const explicitClose = yield* connection.close.pipe(Effect.forkChild) - yield* Deferred.await(pushCleanupStarted) - const ambientClose = yield* Scope.close(ambient, Exit.void).pipe(Effect.forkChild) - yield* Effect.yieldNow - assert.isUndefined(explicitClose.pollUnsafe()) - assert.isUndefined(ambientClose.pollUnsafe()) - yield* Deferred.succeed(pushCleanupRelease, undefined) - yield* Fiber.join(explicitClose) - yield* Fiber.join(ambientClose) - assert.isTrue(Exit.isFailure(yield* Fiber.await(sending))) - assert.strictEqual(yield* Ref.get(childFinalized), 1) - assert.strictEqual(yield* Ref.get(openFinalized), 1) - assert.strictEqual(yield* Ref.get(pushFinalized), 1) - const error = yield* connection.send(Uint8Array.of(2)).pipe(Effect.flip) - assert.strictEqual(error.reason._tag, "StorageUnavailable") - assert.strictEqual(yield* Ref.get(pushes), 1) - })) - - it.effect("does not let a queued Push begin after close", () => - Effect.scoped(Effect.gen(function*() { - const starts = yield* Queue.unbounded() - const client = makeClient( - () => liveOpen(serverOpened), - (request) => - Queue.offer(starts, request.payload[0]).pipe( - Effect.andThen(Effect.never) - ) - ) - const connection = yield* connect(client, serverPeerId) - const first = yield* connection.send(Uint8Array.of(1)).pipe(Effect.forkChild) - assert.strictEqual(yield* Queue.take(starts), 1) - const second = yield* connection.send(Uint8Array.of(2)).pipe(Effect.forkChild) - yield* Effect.yieldNow - assert.isTrue(Option.isNone(yield* Queue.poll(starts))) - yield* connection.close - assert.isTrue(Exit.isFailure(yield* Fiber.await(first))) - assert.isTrue(Exit.isFailure(yield* Fiber.await(second))) - assert.isTrue(Option.isNone(yield* Queue.poll(starts))) - }))) - - it.effect("maps Push failures through the same stable public categories", () => - Effect.scoped(Effect.gen(function*() { - const client = makeClient( - () => liveOpen(serverOpened), - () => Effect.fail(new PeerRpcError.SessionOverloaded()) - ) - const connection = yield* connect(client, serverPeerId) - const error = yield* connection.send(Uint8Array.of(1)).pipe(Effect.flip) - assert.strictEqual(error.reason._tag, "StorageUnavailable") - assert.isTrue(RpcPeerTransport.isRetryable(error)) - yield* connection.close - }))) - - it.effect("reports storeAndForward false and its own lineage awareness", () => - Effect.scoped(Effect.gen(function*() { - const client = makeClient(() => liveOpen(serverOpened), () => Effect.void) - const context = yield* Layer.build(RpcPeerTransport.layer(client, { documents, definition })) - const transport = Context.get(context, PeerTransport.PeerTransport) - assert.isFalse(transport.capabilities.storeAndForward) - assert.isTrue(transport.capabilities.lineageAware) - const connection = yield* transport.connect({ replicaId, peerId: serverPeerId }) - assert.isFalse(connection.capabilities.storeAndForward) - // The connection level value is the server's, never the adapter's. This stub `Opened` omits - // the key the way a peer built before lineage would, and it must stay absent rather than - // inherit the adapter's own true. - assert.isUndefined(connection.capabilities.lineageAware) - yield* connection.close - }))) - - it.effect("advertises its own lineage awareness on the Open it sends", () => - Effect.scoped(Effect.gen(function*() { - const requests = yield* Queue.unbounded() - const client = makeClient( - (request) => Queue.offer(requests, request).pipe(Effect.as(liveOpen(serverOpened)), Stream.unwrap), - () => Effect.void - ) - const connection = yield* connect(client, serverPeerId) - // The server infers nothing from the protocol version, which lineage deliberately did not - // bump. This claim is the only thing that separates this build from an older one that would - // union a rewritten document and push the discarded history back. - assert.deepStrictEqual((yield* Queue.take(requests)).capabilities, { lineageAware: true }) - yield* connection.close - }))) - - it.effect("passes a lineage aware server handshake through to the connection", () => - Effect.scoped(Effect.gen(function*() { - const client = makeClient( - () => - liveOpen(PeerRpc.Opened.make({ - _tag: "Opened", - protocolVersion: PeerRpc.protocolVersion, - sessionId, - peerId: serverPeerId, - capabilities: { storeAndForward: false, lineageAware: true } - })), - () => Effect.void - ) - const connection = yield* connect(client, serverPeerId) - assert.isTrue(connection.capabilities.lineageAware) - yield* connection.close - }))) - - it.effect("opens a fresh RPC session after reconnect", () => - Effect.scoped(Effect.gen(function*() { - const opens = yield* Ref.make(0) - const pushedSessions = yield* Queue.unbounded() - const client = makeClient( - () => - Ref.updateAndGet(opens, (count) => count + 1).pipe( - Effect.map((count) => liveOpen(opened(serverPeerId, count === 1 ? sessionId : otherSessionId))), - Stream.unwrap - ), - (request) => Queue.offer(pushedSessions, request.sessionId).pipe(Effect.asVoid) - ) - const first = yield* connect(client, serverPeerId) - yield* first.send(Uint8Array.of(1)) - yield* first.close - const second = yield* connect(client, serverPeerId) - yield* second.send(Uint8Array.of(2)) - yield* second.close - assert.strictEqual(yield* Ref.get(opens), 2) - assert.deepStrictEqual(yield* Queue.takeAll(pushedSessions), [sessionId, otherSessionId]) - }))) - - it.effect("closes only the Open child scope and preserves ambient resources", () => - Effect.gen(function*() { - const ambient = yield* Scope.make() - const ambientClosed = yield* Ref.make(false) - yield* Scope.addFinalizer(ambient, Ref.set(ambientClosed, true)) - const client = makeClient(() => liveOpen(serverOpened), () => Effect.void) - const connection = yield* connect(client, serverPeerId).pipe(Effect.provideService(Scope.Scope, ambient)) - yield* connection.close - assert.isFalse(yield* Ref.get(ambientClosed)) - yield* Scope.close(ambient, Exit.void) - assert.isTrue(yield* Ref.get(ambientClosed)) - })) - - it.effect("interrupts and joins a Deferred blocked Push when close runs", () => - Effect.scoped(Effect.gen(function*() { - const started = yield* Deferred.make() - const stopped = yield* Deferred.make() - const client = makeClient( - () => liveOpen(serverOpened), - () => - Deferred.succeed(started, undefined).pipe( - Effect.andThen(Effect.never), - Effect.ensuring(Deferred.succeed(stopped, undefined)) - ) - ) - const connection = yield* connect(client, serverPeerId) - const sending = yield* connection.send(Uint8Array.of(1)).pipe(Effect.forkChild) - yield* Deferred.await(started) - yield* connection.close - yield* Deferred.await(stopped) - assert.isTrue(Exit.isFailure(yield* Fiber.await(sending))) - }))) - - it.effect("makes concurrent close callers await the same Push cleanup", () => - Effect.scoped(Effect.gen(function*() { - const pushStarted = yield* Deferred.make() - const cleanupStarted = yield* Deferred.make() - const cleanupRelease = yield* Deferred.make() - const client = makeClient( - () => liveOpen(serverOpened), - () => - Deferred.succeed(pushStarted, undefined).pipe( - Effect.andThen(Effect.never), - Effect.ensuring( - Deferred.succeed(cleanupStarted, undefined).pipe( - Effect.andThen(Deferred.await(cleanupRelease)) - ) - ) - ) - ) - const connection = yield* connect(client, serverPeerId) - const sending = yield* connection.send(Uint8Array.of(1)).pipe(Effect.forkChild) - yield* Deferred.await(pushStarted) - const firstClose = yield* connection.close.pipe(Effect.forkChild) - yield* Deferred.await(cleanupStarted) - const secondClose = yield* connection.close.pipe(Effect.forkChild) - yield* Effect.yieldNow - assert.isUndefined(secondClose.pollUnsafe()) - yield* Deferred.succeed(cleanupRelease, undefined) - yield* Fiber.join(firstClose) - yield* Fiber.join(secondClose) - assert.isTrue(Exit.isFailure(yield* Fiber.await(sending))) - }))) - - it.live("interrupts a parked receive consumer when close runs", () => - Effect.scoped(Effect.gen(function*() { - const payload = Uint8Array.of(1, 2, 3) - const received = yield* Deferred.make() - const client = makeClient(() => openWithMessage(payload).pipe(Stream.concat(Stream.never)), () => Effect.void) - const connection = yield* connect(client, serverPeerId) - const receiving = yield* Stream.runForEach( - connection.receive, - () => Deferred.succeed(received, undefined) - ).pipe(Effect.forkChild) - - const arrived = yield* Deferred.await(received).pipe(Effect.timeoutOption("4 seconds")) - assert.isTrue(Option.isSome(arrived), "consumer never received a Message") - - const closing = yield* connection.close.pipe(Effect.forkChild) - const closed = yield* Fiber.await(closing).pipe(Effect.timeoutOption("4 seconds")) - assert.isTrue(Option.isSome(closed), "connection.close did not complete") - - const exit = yield* Fiber.await(receiving).pipe(Effect.timeoutOption("4 seconds")) - assert.isTrue(Option.isSome(exit), "close did not terminate the receive consumer") - if (Option.isSome(exit)) { - assert.isTrue(Exit.isFailure(exit.value)) - if (Exit.isFailure(exit.value)) { - assert.isTrue(Cause.hasInterruptsOnly(exit.value.cause)) - } - } - - const late = yield* Stream.runDrain(connection.receive).pipe(Effect.forkChild) - const lateExit = yield* Fiber.await(late).pipe(Effect.timeoutOption("4 seconds")) - assert.isTrue(Option.isSome(lateExit), "receive did not terminate for a consumer started after close") - if (Option.isSome(lateExit)) { - assert.isTrue(Exit.isFailure(lateExit.value), "receive ended normally for a consumer started after close") - if (Exit.isFailure(lateExit.value)) { - assert.isTrue(Cause.hasInterruptsOnly(lateExit.value.cause)) - } - } - })), 30000) - - it.live( - "finishes an in-flight receive handler and terminates at the next pull when close runs", - () => - Effect.scoped(Effect.gen(function*() { - const first = Uint8Array.of(1) - const second = Uint8Array.of(2) - const delivered = yield* Queue.unbounded() - const inHandler = yield* Deferred.make() - const releaseHandler = yield* Deferred.make() - const accepted = yield* Deferred.make() - const handlerInterrupted = yield* Ref.make(false) - const client = makeClient( - () => - Stream.fromIterable([ - serverOpened, - PeerRpc.Message.make({ _tag: "Message", payload: first }), - PeerRpc.Message.make({ _tag: "Message", payload: second }) - ]).pipe( - Stream.rechunk(1), - Stream.tap((event) => - event._tag === "Message" && event.payload === second - ? Deferred.succeed(accepted, undefined) - : Effect.void - ), - Stream.concat(Stream.never) - ), - () => Effect.void - ) - const connection = yield* connect(client, serverPeerId) - const receiving = yield* Stream.runForEach( - connection.receive, - (payload) => - Queue.offer(delivered, payload[0]!).pipe( - Effect.andThen( - payload[0] === 1 - ? Deferred.succeed(inHandler, undefined).pipe( - Effect.andThen(Deferred.await(releaseHandler)), - Effect.onInterrupt(() => Ref.set(handlerInterrupted, true)) - ) - : Effect.void - ) - ) - ).pipe(Effect.forkChild) - - const entered = yield* Deferred.await(inHandler).pipe(Effect.timeoutOption("4 seconds")) - assert.isTrue(Option.isSome(entered), "consumer never entered its element handler") - const pulled = yield* Deferred.await(accepted).pipe(Effect.timeoutOption("4 seconds")) - assert.isTrue(Option.isSome(pulled), "the transport never accepted the second payload") - yield* Effect.yieldNow - - yield* connection.close - assert.isUndefined(receiving.pollUnsafe(), "close terminated the consumer inside its element handler") - assert.isFalse(yield* Ref.get(handlerInterrupted), "close interrupted the in-flight element handler") - - yield* Deferred.succeed(releaseHandler, undefined) - const exit = yield* Fiber.await(receiving).pipe(Effect.timeoutOption("4 seconds")) - assert.isTrue(Option.isSome(exit), "consumer did not terminate at its next pull") - if (Option.isSome(exit)) { - assert.isTrue(Exit.isFailure(exit.value)) - if (Exit.isFailure(exit.value)) { - assert.isTrue(Cause.hasInterruptsOnly(exit.value.cause)) - } - } - // Channel.merge hands the consumer a rendezvous queue. Whether the payload the transport - // already pulled is still handed off depends on whether its Queue.offer registered before - // the close trigger failed that queue, so both [1] and [1, 2] are legal deliveries. - const payloads = yield* Queue.takeAll(delivered) - assert.deepStrictEqual(payloads, [1, 2].slice(0, payloads.length)) - assert.strictEqual(payloads[0], 1) - })), - 30000 - ) - - it.live( - "makes concurrent close callers interrupt one parked receive consumer", - () => - Effect.scoped(Effect.gen(function*() { - const received = yield* Deferred.make() - const pushStarted = yield* Deferred.make() - const cleanupStarted = yield* Deferred.make() - const cleanupRelease = yield* Deferred.make() - const finalized = yield* Ref.make(0) - const client = makeClient( - () => - openWithMessage(Uint8Array.of(1, 2, 3)).pipe( - Stream.concat(Stream.never), - Stream.ensuring(Ref.update(finalized, (count) => count + 1)) - ), - () => - Deferred.succeed(pushStarted, undefined).pipe( - Effect.andThen(Effect.never), - Effect.ensuring( - Deferred.succeed(cleanupStarted, undefined).pipe( - Effect.andThen(Deferred.await(cleanupRelease)) - ) - ) - ) - ) - const connection = yield* connect(client, serverPeerId) - const receiving = yield* Stream.runForEach( - connection.receive, - () => Deferred.succeed(received, undefined) - ).pipe(Effect.forkChild) - const arrived = yield* Deferred.await(received).pipe(Effect.timeoutOption("4 seconds")) - assert.isTrue(Option.isSome(arrived), "consumer never received a Message") - - const sending = yield* connection.send(Uint8Array.of(1)).pipe(Effect.forkChild) - yield* Deferred.await(pushStarted) - const firstClose = yield* connection.close.pipe(Effect.forkChild) - yield* Deferred.await(cleanupStarted) - const secondClose = yield* connection.close.pipe(Effect.forkChild) - yield* Effect.yieldNow - assert.isUndefined(secondClose.pollUnsafe()) - - yield* Deferred.succeed(cleanupRelease, undefined) - yield* Fiber.join(firstClose) - yield* Fiber.join(secondClose) - - const exit = yield* Fiber.await(receiving).pipe(Effect.timeoutOption("4 seconds")) - assert.isTrue(Option.isSome(exit), "concurrent close did not terminate the receive consumer") - if (Option.isSome(exit)) { - assert.isTrue(Exit.isFailure(exit.value)) - if (Exit.isFailure(exit.value)) { - assert.isTrue(Cause.hasInterruptsOnly(exit.value.cause)) - } - } - assert.isTrue(Exit.isFailure(yield* Fiber.await(sending))) - assert.strictEqual(yield* Ref.get(finalized), 1) - })), - 30000 - ) - - it.live("interrupts a parked receive consumer when the supplied ambient scope closes", () => - Effect.gen(function*() { - const ambient = yield* Scope.make() - const received = yield* Deferred.make() - const client = makeClient( - () => openWithMessage(Uint8Array.of(1, 2, 3)).pipe(Stream.concat(Stream.never)), - () => Effect.void - ) - const connection = yield* connect(client, serverPeerId).pipe(Effect.provideService(Scope.Scope, ambient)) - const receiving = yield* Stream.runForEach( - connection.receive, - () => Deferred.succeed(received, undefined) - ).pipe(Effect.forkChild) - const arrived = yield* Deferred.await(received).pipe(Effect.timeoutOption("4 seconds")) - assert.isTrue(Option.isSome(arrived), "consumer never received a Message") - - yield* Scope.close(ambient, Exit.void) - - const exit = yield* Fiber.await(receiving).pipe(Effect.timeoutOption("4 seconds")) - assert.isTrue(Option.isSome(exit), "ambient scope close did not terminate the receive consumer") - if (Option.isSome(exit)) { - assert.isTrue(Exit.isFailure(exit.value)) - if (Exit.isFailure(exit.value)) { - assert.isTrue(Cause.hasInterruptsOnly(exit.value.cause)) - } - } - }), 30000) - - it.live("drops payloads the peer emits after close", () => - Effect.scoped(Effect.gen(function*() { - const before = Uint8Array.of(1) - const after = Uint8Array.of(2) - const events = yield* Queue.unbounded() - yield* Queue.offer(events, serverOpened) - const delivered = yield* Queue.unbounded() - const received = yield* Deferred.make() - const client = makeClient(() => Stream.fromQueue(events), () => Effect.void) - const connection = yield* connect(client, serverPeerId) - const receiving = yield* Stream.runForEach( - connection.receive, - (payload) => Queue.offer(delivered, payload).pipe(Effect.andThen(Deferred.succeed(received, undefined))) - ).pipe(Effect.forkChild) - - yield* Queue.offer(events, PeerRpc.Message.make({ _tag: "Message", payload: before })) - const arrived = yield* Deferred.await(received).pipe(Effect.timeoutOption("4 seconds")) - assert.isTrue(Option.isSome(arrived), "consumer never received the pre-close Message") - - yield* connection.close - yield* Queue.offer(events, PeerRpc.Message.make({ _tag: "Message", payload: after })) - - const exit = yield* Fiber.await(receiving).pipe(Effect.timeoutOption("4 seconds")) - assert.isTrue(Option.isSome(exit), "close did not terminate the receive consumer") - if (Option.isSome(exit)) { - assert.isTrue(Exit.isFailure(exit.value)) - if (Exit.isFailure(exit.value)) { - assert.isTrue(Cause.hasInterruptsOnly(exit.value.cause)) - } - } - assert.deepStrictEqual(yield* Queue.takeAll(delivered), [before]) - })), 30000) - - it.effect("interrupts and joins a canceled send before a later send begins", () => - Effect.scoped(Effect.gen(function*() { - const starts = yield* Queue.unbounded() - const firstStopped = yield* Deferred.make() - const client = makeClient( - () => liveOpen(serverOpened), - (request) => - Queue.offer(starts, request.payload[0]).pipe( - Effect.andThen(request.payload[0] === 1 ? Effect.never : Effect.void), - Effect.ensuring( - request.payload[0] === 1 ? Deferred.succeed(firstStopped, undefined) : Effect.void - ) - ) - ) - const connection = yield* connect(client, serverPeerId) - const first = yield* connection.send(Uint8Array.of(1)).pipe(Effect.forkChild) - assert.strictEqual(yield* Queue.take(starts), 1) - yield* Fiber.interrupt(first) - assert.isTrue(yield* Deferred.isDone(firstStopped)) - yield* connection.send(Uint8Array.of(2)) - assert.strictEqual(yield* Queue.take(starts), 2) - yield* connection.close - }))) - - it.effect("classifies transient capacity and session failures as retryable", () => - Effect.scoped(Effect.gen(function*() { - const transient = [ - new PeerRpcError.RequestCapacityExceeded(), - new PeerRpcError.SessionUnavailable(), - new PeerRpcError.SessionOverloaded(), - new PeerRpcError.ServerUnavailable() - ] - for (const rpcError of transient) { - const client = makeClient(() => Stream.fail(rpcError), () => Effect.void) - const error = yield* connect(client, serverPeerId).pipe(Effect.flip) - assert.strictEqual(error.reason._tag, "StorageUnavailable") - assert.isTrue(RpcPeerTransport.isRetryable(error)) - } - const disconnected = makeClient( - () => - Stream.fail( - new RpcClientError({ - reason: new RpcClientDefect({ message: "disconnected", cause: new Error("private transport detail") }) - }) - ), - () => Effect.void - ) - const disconnectError = yield* connect(disconnected, serverPeerId).pipe(Effect.flip) - assert.strictEqual(disconnectError.reason._tag, "StorageUnavailable") - assert.isTrue(RpcPeerTransport.isRetryable(disconnectError)) - }))) - - it.effect("builds the transport composition used by makeSession", () => - Effect.scoped(Effect.gen(function*() { - const pushed = yield* Deferred.make() - const client = makeClient( - () => liveOpen(serverOpened), - (request) => Deferred.succeed(pushed, request).pipe(Effect.asVoid) - ) - const context = yield* Layer.build(RpcPeerTransport.layer(client, { documents, definition })) - const connection = yield* Context.get(context, PeerTransport.PeerTransport).connect({ - replicaId, - peerId: serverPeerId - }) - yield* connection.send(Uint8Array.of(9)) - assert.deepStrictEqual(yield* Deferred.await(pushed), { - sessionId, - payload: Uint8Array.of(9) - }) - yield* connection.close - }))) - - it("keeps nonretryable ReplicaError reasons stable", () => { - assert.isFalse(RpcPeerTransport.isRetryable( - new ReplicaError.ReplicaError({ - reason: new ReplicaError.ProtocolMismatch({ expected: "expected", observed: "observed" }) - }) - )) - assert.isFalse(RpcPeerTransport.isRetryable( - new ReplicaError.ReplicaError({ - reason: new ReplicaError.QuotaExceeded({ resource: "request", limit: 1 }) - }) - )) - }) -}) - -describe("RpcPeerTransport store and forward", () => { - it.effect("opens strict version 3 relay mode with distinct logical and relay identities", () => + it.effect("opens strict version 1 durable mode with distinct logical and relay identities", () => Effect.scoped(Effect.gen(function*() { const configurations = yield* Ref.make(0) const client = makeRelayClient((request, options) => { assert.strictEqual(options.streamBufferSize, 1) - assert.strictEqual(request.version, PeerRelayRpc.protocolVersion) + assert.strictEqual(request.protocolVersion, PeerRpc.protocolVersion) assert.strictEqual(request.expectedRelayPeerId, relayPeerId) assert.deepStrictEqual(request.expectedLocal, relayOptions.expectedLocal) assert.strictEqual(request.senderReplicaIncarnation, senderReplicaIncarnation) @@ -1225,7 +308,6 @@ describe("RpcPeerTransport store and forward", () => { const connection = yield* connectRelay(client, runtime) assert.strictEqual(connection.peerId, remotePeerId) assert.strictEqual(connection.relayPeerId, relayPeerId) - assert.isTrue(connection.capabilities.storeAndForward) assert.strictEqual(yield* Ref.get(configurations), 1) yield* connection.close }))) @@ -1333,7 +415,7 @@ describe("RpcPeerTransport store and forward", () => { assert.strictEqual(yield* Ref.get(opens), 0) }))) - it.effect("rejects a mismatched RelayOpened before exposing a connection", () => + it.effect("rejects a mismatched Opened before exposing a connection", () => Effect.scoped(Effect.gen(function*() { const finalized = yield* Ref.make(0) const mismatched = { @@ -1342,7 +424,7 @@ describe("RpcPeerTransport store and forward", () => { ...relayOpened.authenticatedLocal, subjectId: "other-subject" } - } as PeerRelayRpc.RelayOpened + } as PeerRpc.Opened const client = makeRelayClient( () => Stream.concat(Stream.make(mismatched), Stream.never).pipe( @@ -1361,7 +443,7 @@ describe("RpcPeerTransport store and forward", () => { Effect.scoped( Effect.gen(function*() { const valid = yield* makeStoredMessage(Uint8Array.of(1, 2, 3)) - const invalid = PeerRelayRpc.StoredMessage.make({ + const invalid = PeerRpc.StoredMessage.make({ ...valid, outerEnvelopeDigest: "0".repeat(64) }) @@ -1373,7 +455,7 @@ describe("RpcPeerTransport store and forward", () => { ) const connection = yield* connectRelay(client, makeRuntime()) const error = yield* Stream.runHead( - connection.receiveWithAcknowledgement! + connection.receive ).pipe(Effect.flip) assert.strictEqual(error.reason._tag, "ProtocolMismatch") assert.strictEqual(yield* Ref.get(acknowledgements), 0) @@ -1393,7 +475,7 @@ describe("RpcPeerTransport store and forward", () => { ) const connection = yield* connectRelay(client, makeRuntime()) const delivery = yield* Stream.runHead( - connection.receiveWithAcknowledgement! + connection.receive ).pipe(Effect.map(Option.getOrThrow)) assert.deepStrictEqual(delivery.message, stored.payload) yield* connection.close @@ -1425,7 +507,7 @@ describe("RpcPeerTransport store and forward", () => { } ) const connection = yield* connectRelay(client, makeRuntime()) - const deliveryOption = yield* Stream.runHead(connection.receiveWithAcknowledgement!) + const deliveryOption = yield* Stream.runHead(connection.receive) assert.isTrue(Option.isSome(deliveryOption)) if (Option.isNone(deliveryOption)) return const delivery = deliveryOption.value @@ -1461,7 +543,7 @@ describe("RpcPeerTransport store and forward", () => { }) ) const delivery = yield* Stream.runHead( - connection.receiveWithAcknowledgement! + connection.receive ).pipe(Effect.map(Option.getOrThrow)) assert.strictEqual(yield* Ref.get(acknowledgements), 0) assert.strictEqual(yield* Ref.get(pruneSignals), 0) @@ -1477,7 +559,7 @@ describe("RpcPeerTransport store and forward", () => { Effect.gen(function*() { const stored = yield* makeStoredMessage(Uint8Array.of(6, 7, 8)) const rejected = yield* Deferred.make< - typeof PeerRelayRpc.RejectRelayRpc.payloadSchema.Type + typeof PeerRpc.RejectRpc.payloadSchema.Type >() const client = makeRelayClient( () => Stream.fromIterable([relayOpened, stored]).pipe(Stream.rechunk(1)), @@ -1487,7 +569,7 @@ describe("RpcPeerTransport store and forward", () => { ) const connection = yield* connectRelay(client, makeRuntime()) const delivery = yield* Stream.runHead( - connection.receiveWithAcknowledgement! + connection.receive ).pipe(Effect.map(Option.getOrThrow)) yield* delivery.reject("ApplicationRejected") assert.deepStrictEqual(yield* Deferred.await(rejected), { @@ -1523,7 +605,7 @@ describe("RpcPeerTransport store and forward", () => { }) ) const receiving = yield* Stream.runDrain( - connection.receiveWithAcknowledgement! + connection.receive ).pipe(Effect.forkChild) const sending = yield* connection.send(Uint8Array.of(9)).pipe( Effect.forkChild @@ -1578,7 +660,7 @@ describe("RpcPeerTransport store and forward", () => { ) const ackConnection = yield* connectRelay(ackClient, makeRuntime()) const acknowledged = yield* Stream.runHead( - ackConnection.receiveWithAcknowledgement! + ackConnection.receive ).pipe(Effect.map(Option.getOrThrow)) yield* acknowledged.acknowledge yield* ackConnection.close @@ -1588,7 +670,7 @@ describe("RpcPeerTransport store and forward", () => { ) const rejectConnection = yield* connectRelay(rejectClient, makeRuntime()) const rejected = yield* Stream.runHead( - rejectConnection.receiveWithAcknowledgement! + rejectConnection.receive ).pipe(Effect.map(Option.getOrThrow)) yield* rejected.reject("ApplicationRejected") yield* rejectConnection.close @@ -1603,7 +685,7 @@ describe("RpcPeerTransport store and forward", () => { makeRuntime() ) const unavailable = yield* Stream.runHead( - unavailableConnection.receiveWithAcknowledgement! + unavailableConnection.receive ).pipe(Effect.map(Option.getOrThrow)) assert.isTrue(Exit.isFailure(yield* unavailable.acknowledge.pipe(Effect.exit))) diff --git a/packages/local-rpc/test/SqlPeerRelayStore.test.ts b/packages/local-rpc/test/SqlPeerRelayStore.test.ts new file mode 100644 index 0000000..d6f6c14 --- /dev/null +++ b/packages/local-rpc/test/SqlPeerRelayStore.test.ts @@ -0,0 +1,255 @@ +import { NodeCrypto, NodeFileSystem } from "@effect/platform-node" +import { MysqlClient } from "@effect/sql-mysql2" +import { PgClient } from "@effect/sql-pg" +import { SqliteClient } from "@effect/sql-sqlite-node" +import { assert, describe, it } from "@effect/vitest" +import * as Identity from "@lucas-barake/effect-local/Identity" +import { MySqlContainer, type StartedMySqlContainer } from "@testcontainers/mysql" +import { PostgreSqlContainer } from "@testcontainers/postgresql" +import { Context, Data, Effect, Exit, FileSystem, Layer, Option, Redacted } from "effect" +import * as SqlClient from "effect/unstable/sql/SqlClient" +import * as PeerRelayLimits from "../src/PeerRelayLimits.js" +import * as PeerRelayStore from "../src/PeerRelayStore.js" +import * as PeerRpc from "../src/PeerRpc.js" +import * as SqlPeerRelayStore from "../src/SqlPeerRelayStore.js" +import { runPeerRelayStoreContract } from "./PeerRelayStoreContract.js" + +class ContainerError extends Data.TaggedError("ContainerError")<{ + readonly cause: unknown +}> {} + +class PgContainer extends Context.Service()( + "@lucas-barake/effect-local-rpc/test/PgContainer", + { + make: Effect.acquireRelease( + Effect.tryPromise({ + try: () => new PostgreSqlContainer("postgres:alpine").start(), + catch: (cause) => new ContainerError({ cause }) + }), + (container) => Effect.promise(() => container.stop()) + ) + } +) { + static readonly layer = Layer.effect(this)(this.make) + + static readonly layerClient = Layer.unwrap( + Effect.gen(function*() { + const container = yield* PgContainer + return PgClient.layer({ + url: Redacted.make(container.getConnectionUri()) + }) + }) + ).pipe(Layer.provide(this.layer)) +} + +class MysqlContainer extends Context.Service< + MysqlContainer, + StartedMySqlContainer +>()("@lucas-barake/effect-local-rpc/test/MysqlContainer") { + static readonly layer = Layer.effect(this)( + Effect.acquireRelease( + Effect.tryPromise({ + try: () => new MySqlContainer("mysql:lts").start(), + catch: (cause) => new ContainerError({ cause }) + }), + (container) => Effect.promise(() => container.stop()) + ) + ) + + static readonly layerClient = Layer.unwrap( + Effect.gen(function*() { + const container = yield* MysqlContainer + return MysqlClient.layer({ + url: Redacted.make(container.getConnectionUri()) + }) + }) + ).pipe(Layer.provide(this.layer)) +} + +const SqliteLayer = Effect.gen(function*() { + const fs = yield* FileSystem.FileSystem + const directory = yield* fs.makeTempDirectoryScoped() + return SqliteClient.layer({ + filename: `${directory}/relay.sqlite` + }) +}).pipe(Layer.unwrap, Layer.provide(NodeFileSystem.layer)) + +const StoreLive = SqlPeerRelayStore.layer.pipe( + Layer.provideMerge(NodeCrypto.layer), + Layer.provideMerge(PeerRelayLimits.layer(PeerRelayLimits.Values.make({ + ...PeerRelayLimits.defaults, + maxActiveMessagesPerSenderPeer: 1 + }))) +) + +const MysqlMigrationTestLive = Layer.mergeAll( + MysqlContainer.layerClient, + NodeCrypto.layer, + PeerRelayLimits.layerDefaults +) + +const peerId = (suffix: string) => Identity.PeerId.make(`peer_00000000-0000-4000-8000-${suffix}`) + +const relayMessageId = (suffix: string) => Identity.RelayMessageId.make(`rly_00000000-0000-4000-8000-${suffix}`) + +const documentId = (suffix: string) => Identity.DocumentId.make(`doc_00000000-0000-4000-8000-${suffix}`) + +describe("SqlPeerRelayStore", () => { + it.layer(MysqlMigrationTestLive, { + timeout: 120_000 + })("mysql migrations", (it) => { + it.effect("recovers after DDL commits a partial first migration", () => + Effect.gen(function*() { + const sql = (yield* SqlClient.SqlClient).withoutTransforms() + + yield* sql.unsafe(` + CREATE TABLE effect_local_relay_channels ( + malformed INT NOT NULL + ) + `) + + const firstAttempt = yield* Effect.exit(SqlPeerRelayStore.make) + assert.strictEqual(Exit.isFailure(firstAttempt), true) + + const recordedAfterFailure = yield* sql.unsafe( + "SELECT migration_id FROM effect_local_relay_migrations WHERE migration_id = 1" + ) + assert.strictEqual(recordedAfterFailure.length, 0) + + yield* sql.unsafe("DROP TABLE effect_local_relay_channels") + + yield* SqlPeerRelayStore.make + + const recordedAfterRecovery = yield* sql.unsafe( + "SELECT migration_id FROM effect_local_relay_migrations WHERE migration_id = 1" + ) + assert.strictEqual(recordedAfterRecovery.length, 1) + + const relayTables = yield* sql.unsafe(` + SELECT table_name + FROM information_schema.tables + WHERE table_schema = DATABASE() + AND table_name IN ( + 'effect_local_relay_write_lock', + 'effect_local_relay_channels', + 'effect_local_relay_messages', + 'effect_local_relay_usage', + 'effect_local_relay_reservations' + ) + `) + assert.strictEqual(relayTables.length, 5) + })) + }) + ;([ + ["pg", Layer.orDie(PgContainer.layerClient)], + ["mysql", Layer.orDie(MysqlContainer.layerClient)], + ["sqlite", Layer.orDie(SqliteLayer)] + ] as const).forEach(([label, layer]) => { + it.layer(StoreLive.pipe(Layer.provideMerge(layer)), { + timeout: 120_000 + })(label, (it) => { + it.effect("conforms to the relay custody contract", () => runPeerRelayStoreContract) + + it.effect("keeps case-distinct routing custody and usage isolated", () => + Effect.gen(function*() { + const store = yield* PeerRelayStore.PeerRelayStore + const senderPeerId = peerId("000000000011") + const recipientPeerId = peerId("000000000012") + const upperChannel = PeerRelayStore.ChannelKey.make({ + tenantId: "Tenant-Case-Distinct", + senderSubjectId: "Sender-Case-Distinct", + senderPeerId, + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + recipientSubjectId: "Recipient-Case-Distinct", + recipientPeerId + }) + const lowerChannel = PeerRelayStore.ChannelKey.make({ + tenantId: upperChannel.tenantId.toLowerCase(), + senderSubjectId: upperChannel.senderSubjectId.toLowerCase(), + senderPeerId, + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + recipientSubjectId: upperChannel.recipientSubjectId.toLowerCase(), + recipientPeerId + }) + const makeAdmission = ( + channel: PeerRelayStore.ChannelKey, + relayId: Identity.RelayMessageId, + digest: string + ) => + PeerRelayStore.Admission.make({ + channel, + relayMessageId: relayId, + relayPeerId: peerId("000000000013"), + documentIds: [documentId("000000000011")], + senderConnectionEpoch: "epoch-case-distinct", + senderSequence: 0, + payloadVersion: 1, + messageHash: `message-hash-${digest}`, + outerEnvelopeDigest: PeerRpc.RelayDigest.make(digest.repeat(64)), + payload: new Uint8Array([1]), + messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, + senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, + minimumTerminalRetentionMillis: PeerRelayLimits.defaults.minimumTerminalRetentionMillis + }) + + assert.strictEqual( + (yield* store.admit( + makeAdmission(upperChannel, relayMessageId("000000000011"), "b") + )).status, + "Accepted" + ) + assert.strictEqual( + (yield* store.admit( + makeAdmission(lowerChannel, relayMessageId("000000000012"), "c") + )).status, + "Accepted" + ) + + for (const channel of [upperChannel, lowerChannel]) { + assert.deepStrictEqual( + yield* store.usage(PeerRelayStore.UsageRequest.make({ + scopeKind: "Tenant", + scopeKey: JSON.stringify([channel.tenantId]) + })), + { + activeCount: 1, + activeBytes: 1, + retainedCount: 1, + retainedBytes: 1 + } + ) + assert.deepStrictEqual( + yield* store.usage(PeerRelayStore.UsageRequest.make({ + scopeKind: "RecipientSubject", + scopeKey: JSON.stringify([channel.tenantId, channel.recipientSubjectId]) + })), + { + activeCount: 1, + activeBytes: 1, + retainedCount: 1, + retainedBytes: 1 + } + ) + } + + const lowerClaim = yield* store.claim(PeerRelayStore.ClaimRequest.make({ + recipient: { + tenantId: lowerChannel.tenantId, + subjectId: lowerChannel.recipientSubjectId, + peerId: lowerChannel.recipientPeerId + }, + sender: { + subjectId: lowerChannel.senderSubjectId, + peerId: lowerChannel.senderPeerId + }, + sessionGeneration: 1, + authorizedDocumentIds: [documentId("000000000011")] + })) + assert.strictEqual(Option.isSome(lowerClaim.message), true) + if (Option.isSome(lowerClaim.message)) { + assert.deepStrictEqual(lowerClaim.message.value.channel, lowerChannel) + } + })) + }) + }) +}) diff --git a/packages/local-rpc/vitest.config.ts b/packages/local-rpc/vitest.config.ts index 219d245..2076ce1 100644 --- a/packages/local-rpc/vitest.config.ts +++ b/packages/local-rpc/vitest.config.ts @@ -3,16 +3,6 @@ import { defineProject } from "vitest/config" export default defineProject({ test: { name: "local-rpc", - include: ["test/**/*.test.ts"], - benchmark: { - exclude: [ - "**/node_modules/**", - "**/dist/**", - "**/cypress/**", - "**/.{idea,git,cache,output,temp}/**", - "**/{karma,rollup,webpack,vite,vitest,jest,ava,babel,nyc,cypress,tsup,build,eslint,prettier}.config.*", - "bench/PeerRpcServerPerformance.bench.ts" - ] - } + include: ["test/**/*.test.ts"] } }) diff --git a/packages/local-sql/README.md b/packages/local-sql/README.md index 8c6d113..15b200e 100644 --- a/packages/local-sql/README.md +++ b/packages/local-sql/README.md @@ -2,8 +2,9 @@ SQLite persistence, Automerge history, Effect Cluster command execution, and Effect Workflow operations for Effect Local. -Optional relay support adds the stable sender outbox, sender scoped recipient receipts, quota usage, maintenance, and -restore fencing used by store and forward. Direct SQL composition remains unchanged. +Durable relay support adds the stable sender outbox, sender scoped recipient receipts, quota usage, maintenance, and +restore fencing used by store and forward. These frontend and replica stores remain SQLite. The backend relay custody +store is supplied separately by `@lucas-barake/effect-local-rpc`. Relay infrastructure keeps Automerge bytes opaque. Relay enabled `PeerSync` owns the one semantic decode and its existing change, dependency, and operation limits. Relay SQLite commits are not atomic with an external policy authority. The RPC server local gate decides whether diff --git a/packages/local-sql/src/Migrations.ts b/packages/local-sql/src/Migrations.ts index ad0143c..3e2787c 100644 --- a/packages/local-sql/src/Migrations.ts +++ b/packages/local-sql/src/Migrations.ts @@ -574,7 +574,7 @@ const peerRelayStateMigration = Effect.gen(function*() { relay_peer_id TEXT NOT NULL CHECK (length(relay_peer_id) > 0), relay_message_id TEXT NOT NULL CHECK (length(relay_message_id) > 0), outer_envelope_digest TEXT NOT NULL CHECK (length(outer_envelope_digest) = 64), - protocol_version INTEGER NOT NULL CHECK (protocol_version = 3), + protocol_version INTEGER NOT NULL CHECK (protocol_version = 1), payload_version INTEGER NOT NULL CHECK (payload_version = 1), sender_connection_epoch TEXT NOT NULL CHECK (length(sender_connection_epoch) > 0), sender_sequence INTEGER NOT NULL CHECK (sender_sequence >= 0), diff --git a/packages/local-sql/src/PeerRelayOutbox.ts b/packages/local-sql/src/PeerRelayOutbox.ts index 0296e99..bfe95f7 100644 --- a/packages/local-sql/src/PeerRelayOutbox.ts +++ b/packages/local-sql/src/PeerRelayOutbox.ts @@ -43,8 +43,8 @@ export interface Entry extends Endpoint { readonly writerGeneration: Identity.WriterGeneration readonly relayMessageId: Identity.RelayMessageId readonly outerEnvelopeDigest: string - readonly protocolVersion: 3 - readonly payloadVersion: 1 + readonly protocolVersion: typeof PeerSyncEnvelope.relayProtocolVersion + readonly payloadVersion: typeof PeerSyncEnvelope.syncEnvelopeVersion readonly senderConnectionEpoch: string readonly senderSequence: number readonly document: { @@ -96,8 +96,8 @@ const Row = Schema.Struct({ relay_peer_id: Identity.PeerId, relay_message_id: Identity.RelayMessageId, outer_envelope_digest: RelayDigest, - protocol_version: Schema.Literal(3), - payload_version: Schema.Literal(1), + protocol_version: Schema.Literal(PeerSyncEnvelope.relayProtocolVersion), + payload_version: Schema.Literal(PeerSyncEnvelope.syncEnvelopeVersion), sender_connection_epoch: Schema.String, sender_sequence: NonNegativeInt, document_id: Identity.DocumentId, @@ -308,7 +308,8 @@ const make = Effect.gen(function*() { ${request.replicaId}, ${request.replicaIncarnation}, ${request.writerGeneration}, ${request.expectedLocalTenantId}, ${request.expectedLocalSubjectId}, ${request.expectedLocalPeerId}, ${request.remoteTenantId}, ${request.remoteSubjectId}, ${request.remotePeerId}, ${request.relayPeerId}, - ${request.relayMessageId}, ${request.outerEnvelopeDigest}, 3, 1, + ${request.relayMessageId}, ${request.outerEnvelopeDigest}, + ${PeerSyncEnvelope.relayProtocolVersion}, ${PeerSyncEnvelope.syncEnvelopeVersion}, ${request.senderConnectionEpoch}, ${request.senderSequence}, ${request.documentId}, ${request.documentType}, ${request.writerProvenance}, ${request.messageHash}, ${request.payload}, ${request.encodedSize}, ${request.createdAt}, ${request.retryDeadline}, @@ -583,7 +584,7 @@ const make = Effect.gen(function*() { remote: endpoint.remote, relayPeerId: endpoint.relayPeerId, relayMessageId, - protocolVersion: 3, + protocolVersion: PeerSyncEnvelope.relayProtocolVersion, payloadVersion: PeerSyncEnvelope.syncEnvelopeVersion, senderReplicaIncarnation: permit.incarnation, senderConnectionEpoch: syncEnvelope.connectionEpoch, @@ -703,8 +704,8 @@ const make = Effect.gen(function*() { relayPeerId: endpoint.relayPeerId, relayMessageId, outerEnvelopeDigest, - protocolVersion: 3 as const, - payloadVersion: 1 as const, + protocolVersion: PeerSyncEnvelope.relayProtocolVersion as typeof PeerSyncEnvelope.relayProtocolVersion, + payloadVersion: PeerSyncEnvelope.syncEnvelopeVersion as typeof PeerSyncEnvelope.syncEnvelopeVersion, senderConnectionEpoch: syncEnvelope.connectionEpoch, senderSequence: syncEnvelope.sequence, document: PeerSyncEnvelope.syncEnvelopeDocument(syncEnvelope), diff --git a/packages/local-sql/src/PeerSession.ts b/packages/local-sql/src/PeerSession.ts index 0f1200e..e6ede98 100644 --- a/packages/local-sql/src/PeerSession.ts +++ b/packages/local-sql/src/PeerSession.ts @@ -1,6 +1,5 @@ -import * as Canonical from "@lucas-barake/effect-local/Canonical" import type * as Document from "@lucas-barake/effect-local/Document" -import * as Identity from "@lucas-barake/effect-local/Identity" +import type * as Identity from "@lucas-barake/effect-local/Identity" import * as PeerTransport from "@lucas-barake/effect-local/PeerTransport" import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" import * as ReplicaLimits from "@lucas-barake/effect-local/ReplicaLimits" @@ -18,7 +17,6 @@ import * as Stream from "effect/Stream" import type * as Sharding from "effect/unstable/cluster/Sharding" import * as CommitPublisher from "./CommitPublisher.js" import * as DocumentEntity from "./DocumentEntity.js" -import * as WriterProvenance from "./internal/writerProvenance.js" import * as PeerRelayReceiptLimits from "./PeerRelayReceiptLimits.js" import * as PeerSync from "./PeerSync.js" import * as PeerSyncEnvelope from "./PeerSyncEnvelope.js" @@ -46,42 +44,8 @@ class RelayProtocolInvalid extends Schema.TaggedErrorClass "@lucas-barake/effect-local-sql/PeerSession/RelayProtocolInvalid" )("RelayProtocolInvalid", {}) {} -export const SyncEnvelope = Schema.Struct({ - connectionEpoch: Schema.NonEmptyString, - sequence: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)), - documentId: Identity.DocumentId, - documentType: Schema.String, - messageHash: Schema.String, - message: Schema.Uint8ArrayFromBase64, - // `optionalKey`, so an envelope from a peer built before lineage still decodes. An absent key - // decodes as absent and never as the genesis lineage, so every reader normalizes it with - // `?? Identity.genesisLineage` at the point the envelope enters the system. - lineage: Schema.optionalKey(Identity.DocumentLineage), - writerProvenance: WriterProvenance.ChangeProvenances -}) -// The lineage field is bounded by its own schema at 40 characters, so it costs at most ~60 bytes of -// JSON on top of an envelope that already reserves 4 KiB of fixed headroom. -export const maximumSyncEnvelopeBytes = ( - maxSyncMessageBytes: number, - maxSyncChangesPerMessage: number -) => maxSyncMessageBytes * 2 + maxSyncChangesPerMessage * 512 + 4_096 -const SyncEnvelopeJson = Schema.fromJsonString(Schema.toCodecJson(SyncEnvelope)) - const key = (documentType: string, documentId: Identity.DocumentId) => `${documentType}:${documentId}` -const encodeDirect = (envelope: typeof SyncEnvelope.Type) => - Schema.encodeEffect(SyncEnvelopeJson)(envelope).pipe( - Effect.map((value) => new TextEncoder().encode(value)), - Effect.mapError((cause) => - new ReplicaError.ReplicaError({ - reason: new ReplicaError.ProtocolMismatch({ - expected: "encodable sync envelope", - observed: String(cause) - }) - }) - ) - ) - const supervise = ( terminalFailure: Deferred.Deferred, effect: Effect.Effect @@ -128,32 +92,6 @@ const makeWithTerminal = ( const sync = yield* PeerSync.PeerSync const crypto = yield* Crypto.Crypto const relayReceiptLimits = yield* Effect.serviceOption(PeerRelayReceiptLimits.PeerRelayReceiptLimits) - const decodeDirect = (bytes: Uint8Array) => { - const maximumBytes = maximumSyncEnvelopeBytes( - limits.maxSyncMessageBytes, - limits.maxSyncChangesPerMessage - ) - if (bytes.byteLength > maximumBytes) { - return Effect.fail( - new ReplicaError.ReplicaError({ - reason: new ReplicaError.ProtocolMismatch({ - expected: `sync envelope at most ${maximumBytes} bytes`, - observed: String(bytes.byteLength) - }) - }) - ) - } - return Schema.decodeUnknownEffect(SyncEnvelopeJson)(new TextDecoder().decode(bytes)).pipe( - Effect.mapError(() => - new ReplicaError.ReplicaError({ - reason: new ReplicaError.ProtocolMismatch({ - expected: "sync envelope", - observed: "invalid sync envelope" - }) - }) - ) - ) - } const decodeRelay = (bytes: Uint8Array) => PeerSyncEnvelope.decodeSyncEnvelope(bytes, limits).pipe( Effect.provideService(Crypto.Crypto, crypto), @@ -267,11 +205,7 @@ const makeWithTerminal = ( Effect.gen(function*() { if (!(yield* Ref.get(active))) return const entry = yield* selectedById(outbound.documentId) - const bytes = yield* ( - connection.receiveWithAcknowledgement === undefined - ? encodeDirect - : PeerSyncEnvelope.encodeSyncEnvelope - )({ + const bytes = yield* PeerSyncEnvelope.encodeSyncEnvelope({ connectionEpoch: session.connectionEpoch, sequence: outbound.sendSequence, documentId: outbound.documentId, @@ -463,41 +397,19 @@ const makeWithTerminal = ( const processReceive = ( bytes: Uint8Array, - delivery?: PeerTransport.AcknowledgedDelivery + delivery: PeerTransport.AcknowledgedDelivery ) => Effect.gen(function*() { - const protocolInvalid = (expected: string, observed: string) => - Effect.fail( - delivery === undefined - ? new ReplicaError.ReplicaError({ - reason: new ReplicaError.ProtocolMismatch({ expected, observed }) - }) - : new RelayProtocolInvalid() - ) - const envelope = yield* ( - delivery === undefined - ? decodeDirect(bytes).pipe( - Effect.map((envelope) => ({ - ...envelope, - lineage: envelope.lineage ?? Identity.genesisLineage - })) - ) - : decodeRelay(bytes) - ) - const boundEpoch = yield* bindRemoteEpoch(envelope.connectionEpoch, delivery !== undefined) + const protocolInvalid = (expected: string, observed: string) => Effect.fail(new RelayProtocolInvalid()) + const envelope = yield* decodeRelay(bytes) + const boundEpoch = yield* bindRemoteEpoch(envelope.connectionEpoch, true) const selectedDocument = selected.has(key(envelope.documentType, envelope.documentId)) - // Dropped before the digest and entity dispatch, not after. Nothing can restore the refused - // lineage inside this session, so hashing and dispatching every further message the peer - // sends for it would be a peer driven retry loop over CPU, allocation, and storage. + // Dropped after envelope validation but before entity dispatch. Nothing can restore the + // refused lineage inside this session, so dispatching every further message the peer sends + // for it would be a peer driven retry loop over allocation and storage. if (selectedDocument && (yield* Ref.get(refused)).has(envelope.documentId)) { return "ApplicationRejected" as const } - const messageHash = yield* Canonical.digest(envelope.message).pipe( - Effect.provideService(Crypto.Crypto, crypto) - ) - if (messageHash !== envelope.messageHash) { - return yield* protocolInvalid(messageHash, envelope.messageHash) - } if (!selectedDocument) { return yield* protocolInvalid( "selected whole document", @@ -522,42 +434,37 @@ const makeWithTerminal = ( })) const observationRevision = (yield* Ref.get(observed)).get(envelope.documentId)?.revision ?? 0 const client = yield* entity(envelope.documentId) - const relay = delivery === undefined - ? undefined - : yield* Effect.gen(function*() { - if (relayReceiptLimits._tag === "None") { - return yield* new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageUnavailable({ - cause: new Error("Relay receipt limits are unavailable") - }) + const relay = yield* Effect.gen(function*() { + if (relayReceiptLimits._tag === "None") { + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ + cause: new Error("Relay receipt limits are unavailable") }) - } - if ( - !Number.isSafeInteger(delivery.receiptRetentionMillis) || - delivery.receiptRetentionMillis <= 0 || - delivery.receiptRetentionMillis > relayReceiptLimits.value.receiptRetentionMillis - ) { - return yield* new RelayProtocolInvalid() - } - if ( - connection.relayPeerId === undefined || - delivery.identity.relayPeerId !== connection.relayPeerId - ) { - return yield* new RelayProtocolInvalid() - } - if (delivery.identity.senderPeerId !== connection.peerId) { - return yield* new RelayProtocolInvalid() - } - if (delivery.identity.messageHash !== envelope.messageHash) { - return yield* new RelayProtocolInvalid() - } - const nowMillis = yield* Clock.currentTimeMillis - return { - ...delivery.identity, - receiptExpiresAt: new Date(nowMillis + delivery.receiptRetentionMillis).toISOString(), - encodedSize: bytes.byteLength - } satisfies PeerSync.RelayReceipt - }) + }) + } + if ( + !Number.isSafeInteger(delivery.receiptRetentionMillis) || + delivery.receiptRetentionMillis <= 0 || + delivery.receiptRetentionMillis > relayReceiptLimits.value.receiptRetentionMillis + ) { + return yield* new RelayProtocolInvalid() + } + if (delivery.identity.relayPeerId !== connection.relayPeerId) { + return yield* new RelayProtocolInvalid() + } + if (delivery.identity.senderPeerId !== connection.peerId) { + return yield* new RelayProtocolInvalid() + } + if (delivery.identity.messageHash !== envelope.messageHash) { + return yield* new RelayProtocolInvalid() + } + const nowMillis = yield* Clock.currentTimeMillis + return { + ...delivery.identity, + receiptExpiresAt: new Date(nowMillis + delivery.receiptRetentionMillis).toISOString(), + encodedSize: bytes.byteLength + } satisfies PeerSync.RelayReceipt + }) const applySync = client.ApplySync({ replicaIncarnation: incarnation, peerId: connection.peerId, @@ -571,7 +478,7 @@ const makeWithTerminal = ( // absent key of a pre lineage peer becomes the genesis lineage. lineage: envelope.lineage, writerProvenance: envelope.writerProvenance, - ...(relay === undefined ? {} : { relay }) + relay }).pipe( Effect.catchTag( ["MailboxFull", "AlreadyProcessingMessage", "PersistenceError"], @@ -585,29 +492,25 @@ const makeWithTerminal = ( ) ) ) - const result = yield* ( - delivery === undefined - ? applySync - : applySync.pipe( - Effect.catchReason( - "ReplicaError", - "ProtocolMismatch", - () => - Effect.gen(function*() { - const current = yield* gate.current - if (current.incarnation === session.replicaIncarnation) { - return yield* new RelayProtocolInvalid() - } - return yield* new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageUnavailable({ - cause: new Error( - `Replica incarnation changed from ${session.replicaIncarnation} to ${current.incarnation}` - ) - }) - }) + const result = yield* applySync.pipe( + Effect.catchReason( + "ReplicaError", + "ProtocolMismatch", + () => + Effect.gen(function*() { + const current = yield* gate.current + if (current.incarnation === session.replicaIncarnation) { + return yield* new RelayProtocolInvalid() + } + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ + cause: new Error( + `Replica incarnation changed from ${session.replicaIncarnation} to ${current.incarnation}` + ) }) - ) - ) + }) + }) + ) ) yield* Ref.update(observed, (values) => { const current = values.get(envelope.documentId) @@ -667,21 +570,6 @@ const makeWithTerminal = ( () => relayCall("relay reject", delivery.reject("ProtocolInvalid")) ) ) - const receiveDirect = (bytes: Uint8Array) => - processReceive(bytes).pipe( - Effect.catchTag( - "RelayProtocolInvalid", - () => - Effect.fail( - new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageUnavailable({ - cause: new Error("Direct receive entered relay protocol classification") - }) - }) - ) - ) - ) - yield* Effect.addFinalizer(() => Effect.gen(function*() { const boundEpoch = yield* Ref.get(remoteEpoch) @@ -703,11 +591,7 @@ const makeWithTerminal = ( ) yield* supervise( terminalFailure, - ( - connection.receiveWithAcknowledgement === undefined - ? Stream.runForEach(connection.receive, receiveDirect) - : Stream.runForEach(connection.receiveWithAcknowledgement, receiveAcknowledged) - ).pipe( + Stream.runForEach(connection.receive, receiveAcknowledged).pipe( Effect.andThen( Effect.fail( new ReplicaError.ReplicaError({ diff --git a/packages/local-sql/src/PeerSyncEnvelope.ts b/packages/local-sql/src/PeerSyncEnvelope.ts index 18938fd..80eb9ff 100644 --- a/packages/local-sql/src/PeerSyncEnvelope.ts +++ b/packages/local-sql/src/PeerSyncEnvelope.ts @@ -10,7 +10,7 @@ import * as WriterProvenance from "./internal/writerProvenance.js" export const syncEnvelopeVersion = 1 export const relayOuterEnvelopeVersion = 1 export const relayOuterEnvelopeDomain = "effect-local/relay-outer-envelope" -export const relayProtocolVersion = 3 +export const relayProtocolVersion = 1 export const maximumWriterProvenanceEntries = 1_000 export const maximumRelayPayloadBytes = 4 * 1_024 * 1_024 diff --git a/packages/local-sql/test/BackupStore.test.ts b/packages/local-sql/test/BackupStore.test.ts index c5bacd2..4f0de71 100644 --- a/packages/local-sql/test/BackupStore.test.ts +++ b/packages/local-sql/test/BackupStore.test.ts @@ -186,7 +186,7 @@ describe("BackupStore", () => { ${current.replicaId}, ${current.incarnation}, ${current.writerGeneration}, 'tenant', 'local', ${senderPeerId}, 'tenant', 'remote', ${remotePeerId}, ${senderPeerId}, ${relayMessageId}, - ${"b".repeat(64)}, 3, 1, 'sender-epoch', 0, ${documentId}, ${Task.name}, '[]', + ${"b".repeat(64)}, 1, 1, 'sender-epoch', 0, ${documentId}, ${Task.name}, '[]', ${"a".repeat(64)}, ${Uint8Array.of(1)}, 1, '2026-07-25T00:00:00.000Z', '2026-08-01T00:00:00.000Z', '2026-07-25T00:00:00.000Z', 'Pending' )` diff --git a/packages/local-sql/test/Compaction.test.ts b/packages/local-sql/test/Compaction.test.ts index 710dff6..6f2c9d1 100644 --- a/packages/local-sql/test/Compaction.test.ts +++ b/packages/local-sql/test/Compaction.test.ts @@ -770,7 +770,7 @@ describe("Compaction", () => { ${permit.replicaId}, ${permit.incarnation}, ${permit.writerGeneration}, ${"tenant-a"}, ${"subject-local"}, ${"peer-local"}, ${"tenant-a"}, ${"subject-remote"}, ${"peer-remote"}, ${"peer-relay"}, - ${relayMessageId}, ${"a".repeat(64)}, ${3}, ${1}, + ${relayMessageId}, ${"a".repeat(64)}, ${1}, ${1}, ${"epoch-1"}, ${0}, ${documentId}, ${Task.name}, ${"[]"}, ${"b".repeat(64)}, ${payload}, ${payload.byteLength}, ${"2020-01-01T00:00:00.000Z"}, ${"2999-01-01T00:00:00.000Z"}, diff --git a/packages/local-sql/test/Migrations.test.ts b/packages/local-sql/test/Migrations.test.ts index 3796450..ad23d68 100644 --- a/packages/local-sql/test/Migrations.test.ts +++ b/packages/local-sql/test/Migrations.test.ts @@ -493,7 +493,7 @@ describe("Migrations", () => { 'rep_00000000-0000-4000-8000-000000000011', 0, ${writerGeneration}, 'tenant-1', 'subject-local', 'peer-local', 'tenant-1', 'subject-remote', 'peer-remote', 'peer-relay', - ${relayMessageId}, ${"d".repeat(64)}, 3, 1, + ${relayMessageId}, ${"d".repeat(64)}, 1, 1, 'epoch-outbound', ${senderSequence}, 'doc_00000000-0000-4000-8000-000000000001', 'Task', '[]', ${"e".repeat(64)}, ${payload}, ${payload.byteLength}, diff --git a/packages/local-sql/test/PeerRelayOutboxLimits.test.ts b/packages/local-sql/test/PeerRelayOutboxLimits.test.ts index 4dbf870..e18da02 100644 --- a/packages/local-sql/test/PeerRelayOutboxLimits.test.ts +++ b/packages/local-sql/test/PeerRelayOutboxLimits.test.ts @@ -13,22 +13,24 @@ describe("PeerRelayOutboxLimits", () => { it.effect("rejects invalid scalar and aggregate relationships", () => Effect.gen(function*() { - for (const values of [ - { ...PeerRelayOutboxLimits.defaults, maxMessagesPerRemote: 0 }, - { ...PeerRelayOutboxLimits.defaults, pruneRowsPerSecond: Number.NaN }, - { - ...PeerRelayOutboxLimits.defaults, - maxRetryHorizonMillis: PeerRelayOutboxLimits.maximumRetryHorizonMillis + 1 - }, - { - ...PeerRelayOutboxLimits.defaults, - maxMessagesPerRemote: PeerRelayOutboxLimits.defaults.maxMessagesPerReplica + 1 - }, - { - ...PeerRelayOutboxLimits.defaults, - maxEncodedBytesPerRemote: PeerRelayOutboxLimits.defaults.maxEncodedBytesPerReplica + 1 - } - ]) { + for ( + const values of [ + { ...PeerRelayOutboxLimits.defaults, maxMessagesPerRemote: 0 }, + { ...PeerRelayOutboxLimits.defaults, pruneRowsPerSecond: Number.NaN }, + { + ...PeerRelayOutboxLimits.defaults, + maxRetryHorizonMillis: PeerRelayOutboxLimits.maximumRetryHorizonMillis + 1 + }, + { + ...PeerRelayOutboxLimits.defaults, + maxMessagesPerRemote: PeerRelayOutboxLimits.defaults.maxMessagesPerReplica + 1 + }, + { + ...PeerRelayOutboxLimits.defaults, + maxEncodedBytesPerRemote: PeerRelayOutboxLimits.defaults.maxEncodedBytesPerReplica + 1 + } + ] + ) { assert.strictEqual( (yield* Effect.exit(PeerRelayOutboxLimits.make( values as PeerRelayOutboxLimits.Values diff --git a/packages/local-sql/test/PeerRelayReceiptLimits.test.ts b/packages/local-sql/test/PeerRelayReceiptLimits.test.ts index 2353c11..e695f78 100644 --- a/packages/local-sql/test/PeerRelayReceiptLimits.test.ts +++ b/packages/local-sql/test/PeerRelayReceiptLimits.test.ts @@ -13,22 +13,24 @@ describe("PeerRelayReceiptLimits", () => { it.effect("rejects invalid scalar and aggregate relationships", () => Effect.gen(function*() { - for (const values of [ - { ...PeerRelayReceiptLimits.defaults, maxReceiptsPerRemote: 0 }, - { ...PeerRelayReceiptLimits.defaults, maxEncodedBytesPerRemote: Number.POSITIVE_INFINITY }, - { - ...PeerRelayReceiptLimits.defaults, - receiptRetentionMillis: PeerRelayReceiptLimits.maximumReceiptRetentionMillis + 1 - }, - { - ...PeerRelayReceiptLimits.defaults, - maxReceiptsPerRemote: PeerRelayReceiptLimits.defaults.maxReceiptsPerReplica + 1 - }, - { - ...PeerRelayReceiptLimits.defaults, - maxEncodedBytesPerRemote: PeerRelayReceiptLimits.defaults.maxEncodedBytesPerReplica + 1 - } - ]) { + for ( + const values of [ + { ...PeerRelayReceiptLimits.defaults, maxReceiptsPerRemote: 0 }, + { ...PeerRelayReceiptLimits.defaults, maxEncodedBytesPerRemote: Number.POSITIVE_INFINITY }, + { + ...PeerRelayReceiptLimits.defaults, + receiptRetentionMillis: PeerRelayReceiptLimits.maximumReceiptRetentionMillis + 1 + }, + { + ...PeerRelayReceiptLimits.defaults, + maxReceiptsPerRemote: PeerRelayReceiptLimits.defaults.maxReceiptsPerReplica + 1 + }, + { + ...PeerRelayReceiptLimits.defaults, + maxEncodedBytesPerRemote: PeerRelayReceiptLimits.defaults.maxEncodedBytesPerReplica + 1 + } + ] + ) { assert.strictEqual( (yield* Effect.exit(PeerRelayReceiptLimits.make( values as PeerRelayReceiptLimits.Values diff --git a/packages/local-sql/test/PeerSession.test.ts b/packages/local-sql/test/PeerSession.test.ts index 3a44554..61cbaaf 100644 --- a/packages/local-sql/test/PeerSession.test.ts +++ b/packages/local-sql/test/PeerSession.test.ts @@ -38,7 +38,7 @@ import * as PeerSyncEnvelope from "../src/PeerSyncEnvelope.js" import * as ReplicaBootstrap from "../src/ReplicaBootstrap.js" import * as ReplicaGate from "../src/ReplicaGate.js" -it.layer(NodeCrypto.layer)("PeerSession", (it) => { +it.layer(Layer.merge(NodeCrypto.layer, PeerRelayReceiptLimits.layerDefaults))("PeerSession", (it) => { const Task = Document.make("Task", { schema: Schema.Struct({ title: Schema.String }), version: 1 }) const definition = ReplicaDefinition.make({ name: "tasks", @@ -122,6 +122,49 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { refresh: Effect.succeed(permit), validate: () => Effect.void }) + const testRelayPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000099") + let relayMessageSequence = 0 + const acknowledged = ( + peerId: Identity.PeerId, + stream: Stream.Stream + ): Stream.Stream => + Stream.map(stream, (message) => { + let messageHash = "0".repeat(64) + try { + const decoded = JSON.parse(new TextDecoder().decode(message)) as { readonly messageHash?: unknown } + if (typeof decoded.messageHash === "string") messageHash = decoded.messageHash + } catch { + // The production decoder must classify malformed bytes, not this test adapter. + } + relayMessageSequence += 1 + const suffix = relayMessageSequence.toString(16).padStart(12, "0").slice(-12) + return { + message, + identity: { + relayMessageId: Identity.RelayMessageId.make(`rly_00000000-0000-4000-8000-${suffix}`), + relayPeerId: testRelayPeerId, + senderTenantId: "tenant", + senderSubjectId: "subject", + senderPeerId: peerId, + senderReplicaIncarnation: permit.incarnation, + messageHash, + outerEnvelopeDigest: "0".repeat(64) + }, + receiptRetentionMillis: PeerRelayReceiptLimits.defaults.receiptRetentionMillis, + acknowledge: Effect.void, + reject: (reason) => + reason === "ApplicationRejected" + ? Effect.void + : Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.ProtocolMismatch({ + expected: "valid relay delivery", + observed: "invalid relay delivery" + }) + }) + ) + } + }) const makeLiveFixture = (documents: ReadonlyArray) => Effect.gen(function*() { @@ -173,11 +216,12 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { markSent: () => Effect.succeed(true) }) const transport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: false }, + capabilities: {}, connect: () => Effect.succeed({ peerId, - capabilities: { storeAndForward: false }, + relayPeerId: testRelayPeerId, + capabilities: {}, receive: Stream.fromEffect(Deferred.await(receiveFailure)), send: () => Effect.void, close: Ref.update(closed, (count) => count + 1) @@ -226,14 +270,19 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { } }) - const SyncEnvelopeJson = Schema.fromJsonString(Schema.toCodecJson(PeerSession.SyncEnvelope)) + const SyncEnvelopeJson = Schema.fromJsonString(Schema.toCodecJson(PeerSyncEnvelope.SyncEnvelope)) // The longest value the lineage schema admits, so the envelope size bound is checked against the // worst case rather than against an absent or genesis lineage. const maximalLineage = Identity.DocumentLineage.make("lin_ffffffff-ffff-4fff-bfff-ffffffffffff") - const encode = (envelope: typeof PeerSession.SyncEnvelope.Type) => - Schema.encodeEffect(SyncEnvelopeJson)(envelope).pipe( - Effect.map((value) => new TextEncoder().encode(value)) - ) + const encode = ( + envelope: Omit & { + readonly lineage?: Identity.DocumentLineage + } + ) => + PeerSyncEnvelope.encodeSyncEnvelope({ + ...envelope, + lineage: envelope.lineage ?? Identity.genesisLineage + }) it.effect("bounds a maximum size sync envelope including writer provenance", () => Effect.gen(function*() { @@ -256,14 +305,14 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { }) assert.isAtMost( encoded.byteLength, - PeerSession.maximumSyncEnvelopeBytes( + PeerSyncEnvelope.maximumSyncEnvelopeBytes( limits.maxSyncMessageBytes, limits.maxSyncChangesPerMessage ) ) })) - it.effect("decodes a sync envelope with no lineage field as the genesis lineage", () => + it.effect("requires a lineage in the durable sync envelope", () => Effect.gen(function*() { const documentId = yield* Identity.makeDocumentId const message = Uint8Array.of(1, 2, 3) @@ -278,13 +327,11 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { writerProvenance: [] }) const wire = JSON.parse(new TextDecoder().decode(encoded)) - // The compatibility claim itself: an envelope from a peer built before lineage carries no - // such key at all, rather than carrying an empty one. - assert.isFalse("lineage" in wire) - - const withoutLineage = yield* Schema.decodeUnknownEffect(SyncEnvelopeJson)(JSON.stringify(wire)) - assert.isUndefined(withoutLineage.lineage) - assert.strictEqual(withoutLineage.lineage ?? Identity.genesisLineage, Identity.genesisLineage) + delete wire.lineage + assert.strictEqual( + (yield* Effect.exit(Schema.decodeUnknownEffect(SyncEnvelopeJson)(JSON.stringify(wire))))._tag, + "Failure" + ) const withLineage = yield* Schema.decodeUnknownEffect(SyncEnvelopeJson)( JSON.stringify({ ...wire, lineage: maximalLineage }) @@ -327,7 +374,7 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { const json = JSON.stringify(tampered) assert.isBelow( new TextEncoder().encode(json).byteLength, - PeerSession.maximumSyncEnvelopeBytes(limits.maxSyncMessageBytes, limits.maxSyncChangesPerMessage) + PeerSyncEnvelope.maximumSyncEnvelopeBytes(limits.maxSyncMessageBytes, limits.maxSyncChangesPerMessage) ) const exit = yield* Effect.exit(Schema.decodeUnknownEffect(SyncEnvelopeJson)(json)) assert.strictEqual(exit._tag, "Failure") @@ -357,12 +404,13 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { markSent: () => Effect.succeed(true) }) const transport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: false }, + capabilities: {}, connect: () => Ref.updateAndGet(connectCalls, (count) => count + 1).pipe( Effect.as({ peerId, - capabilities: { storeAndForward: false }, + relayPeerId: testRelayPeerId, + capabilities: {}, receive: Stream.never, send: () => Effect.void, close: Effect.void @@ -448,12 +496,13 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { markSent: () => Effect.succeed(true) }) const transport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: false }, + capabilities: {}, connect: () => Effect.succeed({ peerId, - capabilities: { storeAndForward: false }, - receive: Stream.fromQueue(inbound), + relayPeerId: testRelayPeerId, + capabilities: {}, + receive: acknowledged(peerId, Stream.fromQueue(inbound)), send: () => Effect.void, close: Effect.void }) @@ -577,12 +626,13 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { markSent: () => Effect.succeed(true) }) const transport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: false }, + capabilities: {}, connect: () => Effect.succeed({ peerId, - capabilities: { storeAndForward: false }, - receive: Stream.fromQueue(inbound), + relayPeerId: testRelayPeerId, + capabilities: {}, + receive: acknowledged(peerId, Stream.fromQueue(inbound)), send: () => Effect.void, close: Effect.void }) @@ -741,12 +791,13 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { markSent: () => Ref.set(pending, []).pipe(Effect.as(true)) }) const transport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: false }, + capabilities: {}, connect: () => Effect.succeed({ peerId, - capabilities: { storeAndForward: false }, - receive: Stream.fromQueue(inbound), + relayPeerId: testRelayPeerId, + capabilities: {}, + receive: acknowledged(peerId, Stream.fromQueue(inbound)), send: (bytes) => Effect.gen(function*() { assert.isFalse(yield* Ref.get(gateReleased)) @@ -922,12 +973,13 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { ) }) const transport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: false }, + capabilities: {}, connect: () => Effect.succeed({ peerId, - capabilities: { storeAndForward: false }, - receive: Stream.fromQueue(inbound), + relayPeerId: testRelayPeerId, + capabilities: {}, + receive: acknowledged(peerId, Stream.fromQueue(inbound)), send: (bytes) => Schema.decodeUnknownEffect(SyncEnvelopeJson)(new TextDecoder().decode(bytes)).pipe( Effect.tap((envelope) => Queue.offer(sent, envelope.sequence)), @@ -1057,11 +1109,12 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { markSent: () => Ref.set(pendingCount, 0).pipe(Effect.as(true)) }) const transport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: false }, + capabilities: {}, connect: () => Effect.succeed({ peerId, - capabilities: { storeAndForward: false }, + relayPeerId: testRelayPeerId, + capabilities: {}, receive: Stream.never, send: (bytes) => Schema.decodeUnknownEffect(SyncEnvelopeJson)(new TextDecoder().decode(bytes)).pipe( @@ -1146,11 +1199,12 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { markSent: () => Effect.succeed(true) }) const transport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: false }, + capabilities: {}, connect: () => Effect.succeed({ peerId, - capabilities: { storeAndForward: false }, + relayPeerId: testRelayPeerId, + capabilities: {}, receive: Stream.never, send: () => Ref.updateAndGet(sendCalls, (count) => count + 1).pipe( @@ -1227,12 +1281,13 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { markSent: () => Effect.succeed(true) }) const transport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: false }, + capabilities: {}, connect: () => Effect.succeed({ peerId, - capabilities: { storeAndForward: false }, - receive: Stream.fromQueue(inbound), + relayPeerId: testRelayPeerId, + capabilities: {}, + receive: acknowledged(peerId, Stream.fromQueue(inbound)), send: () => Effect.die(new Error("automatic send defect")), close: Deferred.succeed(closed, undefined).pipe(Effect.asVoid) }) @@ -1425,12 +1480,13 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { markSent: () => Effect.succeed(true) }) const transport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: false }, + capabilities: {}, connect: () => Effect.succeed({ peerId, - capabilities: { storeAndForward: false }, - receive: Stream.fromQueue(inbound), + relayPeerId: testRelayPeerId, + capabilities: {}, + receive: acknowledged(peerId, Stream.fromQueue(inbound)), send: () => Effect.die(`unexpected send for ${testCase.name}`), close: Deferred.succeed(closed, undefined).pipe(Effect.asVoid) }) @@ -1498,12 +1554,13 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { markSent: () => Effect.succeed(true) }) const transport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: false }, + capabilities: {}, connect: () => Effect.acquireRelease( Effect.succeed({ peerId, - capabilities: { storeAndForward: false }, + relayPeerId: testRelayPeerId, + capabilities: {}, receive: Stream.never, send: () => Effect.void, close: Ref.set(closed, true) @@ -1591,14 +1648,15 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { markSent: () => Effect.succeed(true) }) const transport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: false }, + capabilities: {}, connect: ({ replicaId }) => Ref.get(sharedHeld).pipe( Effect.flatMap((held) => held ? Effect.void : Ref.set(current, nextPermit)), Effect.andThen(Ref.set(connectedReplica, replicaId)), Effect.as({ peerId, - capabilities: { storeAndForward: false }, + relayPeerId: testRelayPeerId, + capabilities: {}, receive: Stream.never, send: () => Effect.void, close: Effect.void @@ -1652,11 +1710,12 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { markSent: () => Effect.succeed(true) }) const transport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: false }, + capabilities: {}, connect: () => Effect.succeed({ peerId, - capabilities: { storeAndForward: false }, + relayPeerId: testRelayPeerId, + capabilities: {}, receive: Stream.empty, send: () => Effect.void, close: Ref.set(closed, true) @@ -1722,11 +1781,12 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { markSent: () => Effect.succeed(true) }) const transport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: false }, + capabilities: {}, connect: () => Effect.succeed({ peerId, - capabilities: { storeAndForward: false }, + relayPeerId: testRelayPeerId, + capabilities: {}, receive: Stream.fromEffect(Deferred.await(endReceive)).pipe(Stream.drain), send: () => Effect.void, close: Deferred.succeed(closeStarted, undefined).pipe( @@ -1795,12 +1855,13 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { markSent: () => Effect.succeed(true) }) const transport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: false }, + capabilities: {}, connect: () => Effect.succeed({ peerId, - capabilities: { storeAndForward: false }, - receive: Stream.fromQueue(inbound), + relayPeerId: testRelayPeerId, + capabilities: {}, + receive: acknowledged(peerId, Stream.fromQueue(inbound)), send: () => Ref.update(sends, (count) => count + 1), close: Effect.void }) @@ -1927,11 +1988,12 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { markSent: () => Effect.succeed(true) }) const transport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: false }, + capabilities: {}, connect: () => Effect.succeed({ peerId, - capabilities: { storeAndForward: false }, + relayPeerId: testRelayPeerId, + capabilities: {}, receive: Stream.never, send: () => Deferred.succeed(sendStarted, undefined).pipe(Effect.andThen(Effect.never)), close: Effect.void @@ -2015,11 +2077,12 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { markSent: () => Ref.update(marked, (count) => count + 1).pipe(Effect.as(true)) }) const transport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: false }, + capabilities: {}, connect: () => Effect.succeed({ peerId, - capabilities: { storeAndForward: false }, + relayPeerId: testRelayPeerId, + capabilities: {}, receive: Stream.never, send: () => Effect.gen(function*() { @@ -2154,11 +2217,12 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { markSent: () => Effect.succeed(true) }) const transport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: false }, + capabilities: {}, connect: () => Effect.succeed({ peerId, - capabilities: { storeAndForward: false }, + relayPeerId: testRelayPeerId, + capabilities: {}, receive: Stream.never, send: () => Ref.update(sends, (count) => count + 1), close: Ref.set(closed, true) @@ -2262,11 +2326,12 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { markSent: () => Effect.succeed(true) }) const transport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: false }, + capabilities: {}, connect: () => Effect.succeed({ peerId, - capabilities: { storeAndForward: false }, + relayPeerId: testRelayPeerId, + capabilities: {}, receive: Stream.never, send: () => Effect.void, close: Ref.set(closed, true) @@ -2319,124 +2384,6 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { } }).pipe(Effect.provide(NodeCrypto.layer))) - it.effect("binds one remote epoch and resets both session directions", () => - Effect.gen(function*() { - const inbound = yield* Queue.unbounded() - const firstReceived = yield* Deferred.make() - const receiveEnded = yield* Deferred.make() - const receives = yield* Ref.make(0) - const resets = yield* Ref.make>([]) - const documentId = yield* Identity.makeDocumentId - const peerId = yield* Identity.makePeerId - const message = new Uint8Array([1]) - const messageHash = yield* Canonical.digest(message) - const sync = PeerSync.PeerSync.of({ - withDocumentInvalidation: (_documentId, effect) => effect, - invalidateDocument: () => Effect.void, - open: (id) => - Effect.succeed({ - peerId: id, - connectionEpoch: "local-epoch", - replicaIncarnation: permit.incarnation - }), - reset: (session) => Ref.update(resets, (current) => [...current, session]), - generate: () => Effect.succeed({ outbound: null, observedByPeer: false, dirty: false }), - receive: () => - Ref.updateAndGet(receives, (count) => count + 1).pipe( - Effect.tap(() => Deferred.succeed(firstReceived, undefined)), - Effect.as(result) - ), - enqueue: (_session, reply) => - Effect.succeed({ ...reply, sendSequence: 0, lineage: Identity.genesisLineage, writerProvenance: [] }), - pending: () => Effect.succeed([]), - markSent: () => Effect.succeed(true) - }) - const transport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: false }, - connect: () => - Effect.succeed({ - peerId, - capabilities: { storeAndForward: false }, - receive: Stream.fromQueue(inbound).pipe( - Stream.ensuring(Deferred.succeed(receiveEnded, undefined)) - ), - send: () => Effect.void, - close: Effect.void - }) - }) - yield* Effect.scoped( - Effect.gen(function*() { - yield* PeerSession.makeTestClient( - { peerId, documents: [{ document: Task, documentId }] }, - () => - Effect.succeed({ - ApplySync: () => - sync.receive( - Task, - documentId, - { - peerId, - connectionEpoch: "local-epoch", - replicaIncarnation: permit.incarnation - }, - { - remoteConnectionEpoch: "remote-epoch", - receiveSequence: 0, - lineage: Identity.genesisLineage, - message, - writerProvenance: [{ - changeHash: "a".repeat(64), - writerSchemaVersion: Task.version, - writerDefinitionHash: definition.hash - }] - } - ) - } as never) - ) - const envelope = (connectionEpoch: string) => - encode({ - connectionEpoch, - sequence: 0, - documentId, - documentType: Task.name, - messageHash, - message, - writerProvenance: [{ - changeHash: "a".repeat(64), - writerSchemaVersion: Task.version, - writerDefinitionHash: definition.hash - }] - }) - yield* Queue.offer(inbound, yield* envelope("remote-epoch")) - yield* Deferred.await(firstReceived) - yield* Queue.offer(inbound, yield* envelope("changed-epoch")) - yield* Deferred.await(receiveEnded) - assert.strictEqual(yield* Ref.get(receives), 1) - }).pipe( - Effect.provideService(PeerTransport.PeerTransport, transport), - Effect.provideService(PeerSync.PeerSync, sync), - Effect.provideService(ReplicaGate.ReplicaGate, gate), - Effect.provideService( - CommitPublisher.CommitPublisher, - CommitPublisher.CommitPublisher.of({ - publishPending: Effect.succeed(0), - invalidate: () => Effect.void, - subscribe: Effect.succeed({ - watermark: Identity.CommitSequence.make(0), - refreshGeneration: 0, - events: Stream.never - }) - }) - ), - Effect.provideService(ReplicaLimits.ReplicaLimits, limits) - ) - ) - assert.deepStrictEqual( - (yield* Ref.get(resets)).map((session) => session.connectionEpoch).toSorted(), - ["local-epoch", "remote-epoch"] - ) - }).pipe(Effect.provide(NodeCrypto.layer))) - it.effect("rotates relay sender epochs and acknowledges only after the durable receive workflow", () => Effect.gen(function*() { const deliveries = yield* Queue.unbounded() @@ -2513,14 +2460,13 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { pruneRelayReceipts: Effect.succeed(0) }) const transport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: true }, + capabilities: {}, connect: () => Effect.succeed({ peerId: senderPeerId, relayPeerId, - capabilities: { storeAndForward: true }, - receive: Stream.never, - receiveWithAcknowledgement: Stream.fromQueue(deliveries), + capabilities: {}, + receive: Stream.fromQueue(deliveries), send: () => Effect.void, close: Deferred.succeed(closed, undefined).pipe(Effect.asVoid) }) @@ -2780,14 +2726,13 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { pruneRelayReceipts: Effect.succeed(0) }) const transport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: true }, + capabilities: {}, connect: () => Effect.succeed({ peerId: senderPeerId, relayPeerId, - capabilities: { storeAndForward: true }, - receive: Stream.never, - receiveWithAcknowledgement: Stream.fromQueue(deliveries), + capabilities: {}, + receive: Stream.fromQueue(deliveries), send: () => Effect.void, close: Deferred.succeed(closed, undefined).pipe(Effect.asVoid) }) @@ -2892,11 +2837,12 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { markSent: () => Effect.succeed(true) }) const transport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: false }, + capabilities: {}, connect: () => Effect.succeed({ peerId, - capabilities: { storeAndForward: false }, + relayPeerId: testRelayPeerId, + capabilities: {}, receive: Stream.never, send: (bytes) => Schema.decodeUnknownEffect(SyncEnvelopeJson)(new TextDecoder().decode(bytes)).pipe( @@ -3313,12 +3259,13 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { markSent: () => Effect.succeed(true) }) const transport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: false, lineageAware: true }, + capabilities: { lineageAware: true }, connect: () => Effect.succeed({ peerId, - capabilities: { storeAndForward: false, lineageAware: true }, - receive: Stream.fromQueue(inbound), + relayPeerId: testRelayPeerId, + capabilities: { lineageAware: true }, + receive: acknowledged(peerId, Stream.fromQueue(inbound)), send: () => Effect.void, close: Ref.update(closed, (count) => count + 1).pipe( Effect.andThen(Deferred.succeed(closeStarted, undefined)), @@ -3372,14 +3319,14 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { yield* Queue.offer(inbound, yield* envelope(0, refusedId)) assert.strictEqual(yield* whileOpen(Queue.take(dispatched)), refusedId) yield* Ref.set(digestCalls, 0) - // The peer keeps pushing the refused document. The second message must be dropped ahead of - // the digest and entity, so a hostile peer cannot make the session repeat either cost at will. + // The peer keeps pushing the refused document. Its envelope is still validated, but it must + // be dropped before entity dispatch. yield* Queue.offer(inbound, yield* envelope(1, refusedId)) yield* Queue.offer(inbound, yield* envelope(2, survivingId)) // Taken after two further messages, so it is only reachable if the refusal neither ended the // session nor stopped the receive loop. assert.strictEqual(yield* whileOpen(Queue.take(dispatched)), survivingId) - assert.strictEqual(yield* Ref.get(digestCalls), 1) + assert.strictEqual(yield* Ref.get(digestCalls), 2) assert.strictEqual(yield* Queue.size(dispatched), 0) // Only the applied document published, and the connection is still open. assert.strictEqual(yield* Ref.get(publications), 1) @@ -3443,12 +3390,13 @@ it.layer(NodeCrypto.layer)("PeerSession", (it) => { markSent: () => Effect.succeed(true) }) const transport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: false }, + capabilities: {}, connect: () => Effect.succeed({ peerId, - capabilities: { storeAndForward: false }, - receive: Stream.fromQueue(inbound), + relayPeerId: testRelayPeerId, + capabilities: {}, + receive: acknowledged(peerId, Stream.fromQueue(inbound)), send: () => Effect.void, close: Effect.void }) diff --git a/packages/local-sql/test/PeerSessionCoverage.test.ts b/packages/local-sql/test/PeerSessionCoverage.test.ts index 7592b64..0bc18a7 100644 --- a/packages/local-sql/test/PeerSessionCoverage.test.ts +++ b/packages/local-sql/test/PeerSessionCoverage.test.ts @@ -55,6 +55,7 @@ it.layer(NodeCrypto.layer)("PeerSession coverage", (it) => { writerGeneration: Identity.WriterGeneration.make(2), definitionHash: "hash" } + const testRelayPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000099") const gate = ReplicaGate.ReplicaGate.of({ current: Effect.succeed(permit), claiming: Effect.succeed(false), @@ -88,12 +89,13 @@ it.layer(NodeCrypto.layer)("PeerSession coverage", (it) => { }) const makeScopedTransport = (peerId: Identity.PeerId, closed: Ref.Ref) => PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: false }, + capabilities: {}, connect: () => Effect.acquireRelease( Effect.succeed({ peerId, - capabilities: { storeAndForward: false } as const, + relayPeerId: testRelayPeerId, + capabilities: {} as const, receive: Stream.never, send: () => Effect.void, close: Ref.update(closed, (count) => count + 1) diff --git a/packages/local-sql/test/PeerSyncEnvelope.test.ts b/packages/local-sql/test/PeerSyncEnvelope.test.ts index b685f48..b6a24d3 100644 --- a/packages/local-sql/test/PeerSyncEnvelope.test.ts +++ b/packages/local-sql/test/PeerSyncEnvelope.test.ts @@ -142,7 +142,7 @@ describe("RelayOuterEnvelope", () => { }, relayPeerId: Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000003"), relayMessageId: Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000004"), - protocolVersion: 3, + protocolVersion: PeerSyncEnvelope.relayProtocolVersion, payloadVersion: 1, senderReplicaIncarnation: Identity.ReplicaIncarnation.make(2), senderConnectionEpoch: "epoch-7", @@ -194,7 +194,7 @@ describe("RelayOuterEnvelope", () => { yield* PeerSyncEnvelope.encodeRelayOuterEnvelope(recipientReplay) ) const digest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope(base) - assert.strictEqual(digest, "bf6947413c2a682c4a99718e168954df367d445fdbb5524050df97f39cfe1db0") + assert.strictEqual(digest, "126090efed19a37f5b4844c5964fcaa35cd0dea5a352c73f0477a6b843caee1e") assert.strictEqual( yield* PeerSyncEnvelope.digestRelayOuterEnvelope(relayAdmission), digest diff --git a/packages/local-test/src/TestPeer.ts b/packages/local-test/src/TestPeer.ts index 343b1dc..46d6e81 100644 --- a/packages/local-test/src/TestPeer.ts +++ b/packages/local-test/src/TestPeer.ts @@ -78,6 +78,43 @@ interface Scheduled { const route = (from: Identity.PeerId, to: Identity.PeerId) => `${from}\u0000${to}` +const fallbackMessageHash = (message: Uint8Array): string => + Array.from(message, (byte) => byte.toString(16).padStart(2, "0")) + .join("") + .padEnd(64, "0") + .slice(0, 64) + +const deliveryIdentity = ( + message: Uint8Array, + relayPeerId: Identity.PeerId, + senderPeerId: Identity.PeerId +): PeerTransport.RelayDeliveryIdentity => { + let messageHash = fallbackMessageHash(message) + try { + const decoded = JSON.parse(new TextDecoder().decode(message)) as { readonly messageHash?: unknown } + if (typeof decoded.messageHash === "string" && /^[0-9a-f]{64}$/.test(decoded.messageHash)) { + messageHash = decoded.messageHash + } + } catch { + // Raw packets are supported by the low level test transport. + } + const relayMessageId = Identity.RelayMessageId.make( + `rly_${messageHash.slice(0, 8)}-${messageHash.slice(8, 12)}-4${messageHash.slice(13, 16)}-8${ + messageHash.slice(17, 20) + }-${messageHash.slice(20, 32)}` + ) + return { + relayMessageId, + relayPeerId, + senderTenantId: "effect-local-test", + senderSubjectId: senderPeerId, + senderPeerId, + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(0), + messageHash, + outerEnvelopeDigest: messageHash + } +} + const toValidatedMillis = (input: Duration.Input) => typeof input === "number" && Number.isNaN(input) ? Number.NaN @@ -297,7 +334,7 @@ export const make = ( transport: (peerId) => ({ // Both ends of this transport are this build, which compares lineage before it merges, so // the advertisement is a statement of fact rather than a test convenience. - capabilities: { storeAndForward: false, lineageAware: true }, + capabilities: { lineageAware: true }, connect: ({ peerId: remotePeerId }) => connect(peerId, remotePeerId).pipe( Effect.tap(() => @@ -308,8 +345,17 @@ export const make = ( ), Effect.map((connection) => ({ peerId: remotePeerId, - capabilities: { storeAndForward: false, lineageAware: true }, - receive: connection.receive, + relayPeerId: peerId, + capabilities: { lineageAware: true }, + receive: connection.receive.pipe( + Stream.map((message): PeerTransport.AcknowledgedDelivery => ({ + message, + identity: deliveryIdentity(message, peerId, remotePeerId), + receiptRetentionMillis: 24 * 60 * 60 * 1_000, + acknowledge: Effect.void, + reject: () => Effect.void + })) + ), send: (message) => connection.send(message).pipe( Effect.mapError((error) => diff --git a/packages/local-test/test/TestPeerCoverage.test.ts b/packages/local-test/test/TestPeerCoverage.test.ts index 9c05ca4..b9c31eb 100644 --- a/packages/local-test/test/TestPeerCoverage.test.ts +++ b/packages/local-test/test/TestPeerCoverage.test.ts @@ -32,9 +32,9 @@ describe("TestPeer coverage", () => { assert.strictEqual(a.peerId, rightId) assert.strictEqual(b.peerId, leftId) yield* a.send(bytes(1)) - assert.deepStrictEqual(Option.getOrThrow(yield* Stream.runHead(b.receive)), bytes(1)) + assert.deepStrictEqual(Option.getOrThrow(yield* Stream.runHead(b.receive)).message, bytes(1)) yield* b.send(bytes(2)) - assert.deepStrictEqual(Option.getOrThrow(yield* Stream.runHead(a.receive)), bytes(2)) + assert.deepStrictEqual(Option.getOrThrow(yield* Stream.runHead(a.receive)).message, bytes(2)) yield* a.close const error = yield* Effect.flip(a.send(bytes(3))) assert.strictEqual(error._tag, "ReplicaError") diff --git a/packages/local/src/PeerTransport.ts b/packages/local/src/PeerTransport.ts index 54ddafe..abd5a67 100644 --- a/packages/local/src/PeerTransport.ts +++ b/packages/local/src/PeerTransport.ts @@ -6,7 +6,6 @@ import type * as Identity from "./Identity.js" import type * as ReplicaError from "./ReplicaError.js" export interface Capabilities { - readonly storeAndForward: boolean /** * Whether the peer compares document lineage before it merges a sync message. * @@ -43,10 +42,9 @@ export interface AcknowledgedDelivery { export interface Connection { readonly peerId: Identity.PeerId - readonly relayPeerId?: Identity.PeerId + readonly relayPeerId: Identity.PeerId readonly capabilities: Capabilities - readonly receive: Stream.Stream - readonly receiveWithAcknowledgement?: Stream.Stream + readonly receive: Stream.Stream readonly send: (message: Uint8Array) => Effect.Effect /** * Terminates `receive`. A fiber parked consuming it observes an interrupt only `Exit`, never a normal end and diff --git a/packages/local/test/PublicApi.types.test.ts b/packages/local/test/PublicApi.types.test.ts index d09c794..c95055a 100644 --- a/packages/local/test/PublicApi.types.test.ts +++ b/packages/local/test/PublicApi.types.test.ts @@ -91,22 +91,10 @@ describe("public API types", () => { assert.isDefined(descriptor) }) - it("adds acknowledged relay delivery without changing the direct transport shape", () => { + it("requires acknowledged durable relay delivery", () => { const peerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000002") const relayMessageId = Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000003") const relayPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000004") - const directConnection = { - peerId, - capabilities: { storeAndForward: false }, - receive: Stream.empty, - send: () => Effect.void, - close: Effect.void - } satisfies PeerTransport.Connection - const directConnectionApi: PeerTransport.Connection = directConnection - const directTransport = PeerTransport.PeerTransport.of({ - capabilities: { storeAndForward: false }, - connect: () => Effect.succeed(directConnection) - }) const identity: PeerTransport.RelayDeliveryIdentity = { relayMessageId, relayPeerId: peerId, @@ -124,31 +112,33 @@ describe("public API types", () => { acknowledge: Effect.void, reject: (_reason) => Effect.void } - const relayConnection = { - ...directConnection, + const connection = { + peerId, relayPeerId, - capabilities: { storeAndForward: true }, - receiveWithAcknowledgement: Stream.make(delivery) + capabilities: {}, + receive: Stream.make(delivery), + send: () => Effect.void, + close: Effect.void } satisfies PeerTransport.Connection - const optionalAcknowledgedReceive: Equal< - PeerTransport.Connection["receiveWithAcknowledgement"], - Stream.Stream | undefined + const acknowledgedReceive: Equal< + PeerTransport.Connection["receive"], + Stream.Stream > = true - const optionalRelayPeerId: Equal< + const requiredRelayPeerId: Equal< PeerTransport.Connection["relayPeerId"], - Identity.PeerId | undefined + Identity.PeerId > = true + const transport = PeerTransport.PeerTransport.of({ + capabilities: {}, + connect: () => Effect.succeed(connection) + }) const protocolRejection = delivery.reject("ProtocolInvalid") const applicationRejection = delivery.reject("ApplicationRejected") - assert.isDefined(directTransport) - assert.isUndefined(directConnectionApi.relayPeerId) - assert.isUndefined(directConnectionApi.receiveWithAcknowledgement) - assert.strictEqual(relayConnection.relayPeerId, relayPeerId) - assert.isDefined(relayConnection.receiveWithAcknowledgement) - assert.isTrue(optionalAcknowledgedReceive) - assert.isTrue(optionalRelayPeerId) - assert.isFalse(directConnection.capabilities.storeAndForward) - assert.isTrue(relayConnection.capabilities.storeAndForward) + assert.isDefined(transport) + assert.strictEqual(connection.relayPeerId, relayPeerId) + assert.isDefined(connection.receive) + assert.isTrue(acknowledgedReceive) + assert.isTrue(requiredRelayPeerId) assert.isDefined(protocolRejection) assert.isDefined(applicationRejection) }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6fb44be..469427c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -139,12 +139,24 @@ importers: '@effect/platform-node': specifier: 4.0.0-beta.99 version: 4.0.0-beta.99(effect@4.0.0-beta.99)(ioredis@5.11.1) + '@effect/sql-mysql2': + specifier: 4.0.0-beta.99 + version: 4.0.0-beta.99(@types/node@25.9.5)(effect@4.0.0-beta.99) + '@effect/sql-pg': + specifier: 4.0.0-beta.99 + version: 4.0.0-beta.99(effect@4.0.0-beta.99) '@effect/sql-sqlite-node': specifier: 4.0.0-beta.99 version: 4.0.0-beta.99(effect@4.0.0-beta.99) '@lucas-barake/effect-local-test': specifier: workspace:^ version: link:../local-test + '@testcontainers/mysql': + specifier: 11.14.0 + version: 11.14.0 + '@testcontainers/postgresql': + specifier: 11.14.0 + version: 11.14.0 effect: specifier: 4.0.0-beta.99 version: 4.0.0-beta.99 @@ -219,6 +231,9 @@ packages: resolution: {integrity: sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==} engines: {node: '>=6.9.0'} + '@balena/dockerignore@1.0.2': + resolution: {integrity: sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==} + '@bcoe/v8-coverage@1.0.2': resolution: {integrity: sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==} engines: {node: '>=18'} @@ -396,6 +411,16 @@ packages: effect: ^4.0.0-beta.99 ioredis: ^5.7.0 + '@effect/sql-mysql2@4.0.0-beta.99': + resolution: {integrity: sha512-XY7nI1YbiufzTIDjMolGJLtegF56yK/0G0+4jBznq0PmltwTZ6rCbNLJnDAtfqTt/z4nggemaNCFINVGc1oKyw==} + peerDependencies: + effect: ^4.0.0-beta.99 + + '@effect/sql-pg@4.0.0-beta.99': + resolution: {integrity: sha512-ZZCKdArioVX29YcH+0HjiYeZiTAEnngU8ao3cBGZXrUOZwcvzknGWNOS7aGzES7hSK/HDBKPu2t14LGgtSgsCA==} + peerDependencies: + effect: ^4.0.0-beta.99 + '@effect/sql-sqlite-node@4.0.0-beta.99': resolution: {integrity: sha512-43fKlLxWADCDdRYU+ZTig/xeYfyXQOR+j+A0Tp1dZRpztfH8xIkz7o8XxAgcAZMlW3a0EmaHdlgFSU7itziRog==} peerDependencies: @@ -743,6 +768,20 @@ packages: cpu: [x64] os: [win32] + '@grpc/grpc-js@1.14.4': + resolution: {integrity: sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ==} + engines: {node: '>=12.10.0'} + + '@grpc/proto-loader@0.7.15': + resolution: {integrity: sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==} + engines: {node: '>=6'} + hasBin: true + + '@grpc/proto-loader@0.8.1': + resolution: {integrity: sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==} + engines: {node: '>=6'} + hasBin: true + '@inquirer/external-editor@1.0.3': resolution: {integrity: sha512-RWbSrDiYmO4LbejWY7ttpxczuwQyZLBUyygsA9Nsv95hpzUWwnNTVQmAq3xuh7vNwCp07UTmE5i11XAEExx4RA==} engines: {node: '>=18'} @@ -755,6 +794,10 @@ packages: '@ioredis/commands@1.10.0': resolution: {integrity: sha512-UmeW7z4LfctwoQ5wkhVzgq8tXkreED2xZGpX+Bg+zA+WJFZCT6c062AfCK/Dfk81xZnnwdhJCUMkitihRaoC2Q==} + '@isaacs/cliui@8.0.2': + resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} + engines: {node: '>=12'} + '@jridgewell/resolve-uri@3.1.2': resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} engines: {node: '>=6.0.0'} @@ -765,6 +808,12 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@js-sdsl/ordered-map@4.4.2': + resolution: {integrity: sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==} + + '@kwsites/file-exists@1.1.1': + resolution: {integrity: sha512-m9/5YGR18lIwxSFDwfE3oA7bWuq9kdau6ugN4H2rJeyhFQZcG9AgSHkQtSD15a8WvTgfz9aikZMrKPHvbpqFiw==} + '@manypkg/find-root@1.1.0': resolution: {integrity: sha512-mki5uBvhHzO8kYYix/WRy2WX8S3B5wdVSc9D6KcU5lQNglP2yt58/VfLuAK49glRXChosY8ap2oJ1qgma3GUVA==} @@ -1076,11 +1125,42 @@ packages: cpu: [x64] os: [win32] + '@pkgjs/parseargs@0.11.0': + resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} + engines: {node: '>=14'} + '@playwright/test@1.61.1': resolution: {integrity: sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig==} engines: {node: '>=18'} hasBin: true + '@protobufjs/aspromise@1.1.2': + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + + '@protobufjs/base64@1.1.2': + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + + '@protobufjs/codegen@2.0.5': + resolution: {integrity: sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==} + + '@protobufjs/eventemitter@1.1.1': + resolution: {integrity: sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==} + + '@protobufjs/fetch@1.1.1': + resolution: {integrity: sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==} + + '@protobufjs/float@1.0.2': + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + + '@protobufjs/path@1.1.2': + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + + '@protobufjs/pool@1.1.0': + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + + '@protobufjs/utf8@1.1.2': + resolution: {integrity: sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==} + '@rollup/rollup-android-arm-eabi@4.62.2': resolution: {integrity: sha512-6o7ZLZK+BeenkZCFNDXqpbjw9bD6nuWonvS/lwQJp7NoVVxm6p3qE7qQ5jGuBjiFsgvqjD8mZAU5oWxTmbOeOg==} cpu: [arm] @@ -1209,6 +1289,12 @@ packages: '@standard-schema/spec@1.1.0': resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} + '@testcontainers/mysql@11.14.0': + resolution: {integrity: sha512-0/OKd1gOvnl0qS+RIpmT6J0XKWpjkVyIbOtS3cFTVR8t6Ps7W9TV0U+zp7FPxCmitWuWYPXwH/+RQXF+1jZuZQ==} + + '@testcontainers/postgresql@11.14.0': + resolution: {integrity: sha512-wYbJn8GRTj8qfqzfVubxioYWlHJU/ImIjuzPwyy9C5Qfo6g3GLduPZAj+BifvqTZjgT3gd4gFVLCPhBji7dc1w==} + '@ts-graphviz/adapter@2.0.6': resolution: {integrity: sha512-kJ10lIMSWMJkLkkCG5gt927SnGZcBuG0s0HHswGzcHTgvtUe7yk5/3zTEr0bafzsodsOq5Gi6FhQeV775nC35Q==} engines: {node: '>=18'} @@ -1234,12 +1320,21 @@ packages: '@types/deep-eql@4.0.2': resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} + '@types/docker-modem@3.0.6': + resolution: {integrity: sha512-yKpAGEuKRSS8wwx0joknWxsmLha78wNMe9R2S3UNsVOkZded8UqOrV8KoeDXoXsjndxwyF3eIhyClGbO1SEhEg==} + + '@types/dockerode@4.0.1': + resolution: {integrity: sha512-cmUpB+dPN955PxBEuXE3f6lKO1hHiIGYJA46IVF3BJpNsZGvtBDcRnlrHYHtOH/B6vtDOyl2kZ2ShAu3mgc27Q==} + '@types/estree@1.0.9': resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} '@types/node@12.20.55': resolution: {integrity: sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==} + '@types/node@18.19.130': + resolution: {integrity: sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==} + '@types/node@25.9.5': resolution: {integrity: sha512-OScDchr2fwuUmWdf4kZ9h7PcJiYDVInhJizG/biAq3cAvqwYktuy/TYGGdZNMtNTFUP7rnb0NU4TUdm82kt4Rg==} @@ -1251,6 +1346,15 @@ packages: '@types/react@19.2.17': resolution: {integrity: sha512-MXfmqaVPEVgkBT/aY0aGCkRWWtByiYQXo3xdQ8r5RzuFrPiRn8Gar2tQdXSUQ2GKV3bkXckek89V8wQBY2Q/Aw==} + '@types/ssh2-streams@0.1.13': + resolution: {integrity: sha512-faHyY3brO9oLEA0QlcO8N2wT7R0+1sHWZvQ+y3rMLwdY1ZyS1z0W3t65j9PqT4HmQ6ALzNe7RZlNuCNE0wBSWA==} + + '@types/ssh2@0.5.52': + resolution: {integrity: sha512-lbLLlXxdCZOSJMCInKH2+9V/77ET2J6NPQHpFI0kda61Dd1KglJs+fPQBchizmzYSOJBgdTajhPqBO1xxLywvg==} + + '@types/ssh2@1.15.5': + resolution: {integrity: sha512-N1ASjp/nXH3ovBHddRJpli4ozpk6UdDYIX4RJWFa9L1YKnzdhTlVmiGHm4DZnj/jLbqZpes4aeR30EFGQtvhQQ==} + '@types/ws@8.18.1': resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==} @@ -1372,6 +1476,10 @@ packages: '@vue/shared@3.5.40': resolution: {integrity: sha512-WxnBtruIqOoV3rA4jeKDWzrYI5h7Cp4+pjwDi8kWGHz+IslhiN+wguLVVhtv2l8VoU02rzDCVfDjgCl1lNpZVg==} + abort-controller@3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} + engines: {node: '>=6.5'} + ansi-colors@4.1.3: resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==} engines: {node: '>=6'} @@ -1380,16 +1488,32 @@ packages: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} + ansi-regex@6.2.2: + resolution: {integrity: sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==} + engines: {node: '>=12'} + ansi-styles@4.3.0: resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} engines: {node: '>=8'} + ansi-styles@6.2.3: + resolution: {integrity: sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==} + engines: {node: '>=12'} + any-promise@1.3.0: resolution: {integrity: sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==} app-module-path@2.2.0: resolution: {integrity: sha512-gkco+qxENJV+8vFcDiiFhuoSvRXb2a/QPqpSoWhVz829VNJfOTnELbBmPmNKFxf3xdNnw4DWCkzkDaavcX/1YQ==} + archiver-utils@5.0.2: + resolution: {integrity: sha512-wuLJMmIBQYCsGZgYLTy5FIB2pF6Lfb6cXMSF8Qywwk3t20zWnAi7zLcQFdKQmIB8wyZpY5ER38x08GbwtR2cLA==} + engines: {node: '>= 14'} + + archiver@7.0.1: + resolution: {integrity: sha512-ZcbTaIqJOfCc03QwD468Unz/5Ir8ATtvAHsK+FdXbDIbGfihqh9mrvdcYunQzqn4HrvWWaFyaxJhGZagaJJpPQ==} + engines: {node: '>= 14'} + argparse@1.0.10: resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==} @@ -1400,6 +1524,9 @@ packages: resolution: {integrity: sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==} engines: {node: '>=8'} + asn1@0.2.6: + resolution: {integrity: sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==} + assertion-error@2.0.1: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} @@ -1411,13 +1538,74 @@ packages: ast-v8-to-istanbul@1.0.5: resolution: {integrity: sha512-UPAgKJFSEGMWSDr3LX4tqnAb4f7KGT8O40Tyx8wbYmmZ/yn58lNCm8h3svs3eXgiGd5AXxz8NDOvXWvicq+rJA==} + async-lock@1.4.1: + resolution: {integrity: sha512-Az2ZTpuytrtqENulXwO3GGv1Bztugx6TT37NIo7imr/Qo0gsYiGtSdBa2B6fsXhTpVZDNfu1Qn3pk531e3q+nQ==} + + async@3.2.6: + resolution: {integrity: sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==} + + aws-ssl-profiles@1.1.2: + resolution: {integrity: sha512-NZKeq9AfyQvEeNlN0zSYAaWrmBffJh3IELMZfRpJVWgrpEbtEpnjvzqBPf+mxoI287JohRDoa+/nsfqqiZmF6g==} + engines: {node: '>= 6.0.0'} + + b4a@1.8.1: + resolution: {integrity: sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw==} + peerDependencies: + react-native-b4a: '*' + peerDependenciesMeta: + react-native-b4a: + optional: true + + balanced-match@1.0.2: + resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + balanced-match@4.0.4: resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==} engines: {node: 18 || 20 || >=22} + bare-events@2.9.1: + resolution: {integrity: sha512-Z0oHEHAFDZkffN8Qc39zNZjQlMDkPJRyyyZieU1VH7u8c5S+qHZ2S8ixdKIAxEjfHO7FJxXmJWgteOghVanIsg==} + peerDependencies: + bare-abort-controller: '*' + peerDependenciesMeta: + bare-abort-controller: + optional: true + + bare-fs@4.7.4: + resolution: {integrity: sha512-y1kC+ffIx/tPLdTE693uNjHfzTfr+ravR5tvWlMXe25nELbkqV400S71qHDwbkAQ1FVEZobB1NFRzFbCCcyBCQ==} + engines: {bare: '>=1.16.0'} + peerDependencies: + bare-buffer: '*' + peerDependenciesMeta: + bare-buffer: + optional: true + + bare-path@3.1.1: + resolution: {integrity: sha512-JprUlveX3QjApC1cTpsUOiscADftCGVWkzitbHsRqv84hzYwYHw2mbluddsq5TvI8mH/8Ov1f4BiMAdcB0oYnQ==} + + bare-stream@2.13.3: + resolution: {integrity: sha512-Kc+brLqvEqGkjyfiwJmImAOqLZL7OsoLKuavx+hJjgVV3nLTOjloJyPMFxjUPerGGHrNH0fLU06jjykMLWrERQ==} + peerDependencies: + bare-abort-controller: '*' + bare-buffer: '*' + bare-events: '*' + peerDependenciesMeta: + bare-abort-controller: + optional: true + bare-buffer: + optional: true + bare-events: + optional: true + + bare-url@2.4.6: + resolution: {integrity: sha512-iQxPClE07hETVpbRoX7JXX3v/ZQViCxe/SYCxylRLzdEx1xJAufPptfiOqR8tqiCtmbtMDANKWszzjLu1PMAZQ==} + base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + bcrypt-pbkdf@1.0.2: + resolution: {integrity: sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==} + better-path-resolve@1.0.0: resolution: {integrity: sha512-pbnl5XzGBdrFU/wT4jqmJVPn2B6UHPBOhzMQkY/SPUPB6QtUXtmBHBIwCbXJol93mOpGMnQyP/+BB19q04xj7g==} engines: {node: '>=4'} @@ -1425,6 +1613,9 @@ packages: bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} + brace-expansion@2.1.2: + resolution: {integrity: sha512-w5JZcKgdhDOgOwm8H+KgbosopHMuGcl6qbulwjtz3SM7I7P3yW1eAjzMPLrIE+NQ9vjgANKHWeMHnrT0OXW1oA==} + brace-expansion@5.0.7: resolution: {integrity: sha512-7oFy703dxfY3/NLxC1fh2SUCQ0H9rmAY+5EpDVfXjUTTs+HEwR2nYaqLv+GWcTsumwxPfiz6CzCNkwXwBUwqCA==} engines: {node: 18 || 20 || >=22} @@ -1433,9 +1624,24 @@ packages: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} engines: {node: '>=8'} + buffer-crc32@1.0.0: + resolution: {integrity: sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==} + engines: {node: '>=8.0.0'} + buffer@5.7.1: resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} + buffer@6.0.3: + resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} + + buildcheck@0.0.7: + resolution: {integrity: sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA==} + engines: {node: '>=10.0.0'} + + byline@5.0.0: + resolution: {integrity: sha512-s6webAy+R4SR8XVuJWt2V2rGvhnrhxN+9S15GNuTK3wKPOXFF6RNc+8ug2XhH+2s4f+uudG4kUVYmYOQWL2g0Q==} + engines: {node: '>=0.10.0'} + chai@6.2.2: resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} engines: {node: '>=18'} @@ -1447,6 +1653,9 @@ packages: chardet@2.2.0: resolution: {integrity: sha512-rddelWYNPRrXq6PtNEN2S3f6t9ILzvqaN5pVgi4kqt9jHQaXIial9PznB5iSPVlQSLNaaH22ItWz3EJtQ10+OA==} + chownr@1.1.4: + resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} + cli-cursor@3.1.0: resolution: {integrity: sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==} engines: {node: '>=8'} @@ -1455,6 +1664,10 @@ packages: resolution: {integrity: sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==} engines: {node: '>=6'} + cliui@8.0.1: + resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} + engines: {node: '>=12'} + clone@1.0.4: resolution: {integrity: sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==} engines: {node: '>=0.8'} @@ -1481,9 +1694,29 @@ packages: commondir@1.0.1: resolution: {integrity: sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==} + compress-commons@6.0.2: + resolution: {integrity: sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==} + engines: {node: '>= 14'} + convert-source-map@2.0.0: resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} + core-util-is@1.0.3: + resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} + + cpu-features@0.0.10: + resolution: {integrity: sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==} + engines: {node: '>=10.0.0'} + + crc-32@1.2.2: + resolution: {integrity: sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==} + engines: {node: '>=0.8'} + hasBin: true + + crc32-stream@6.0.0: + resolution: {integrity: sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==} + engines: {node: '>= 14'} + cross-spawn@7.0.6: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} @@ -1574,6 +1807,18 @@ packages: resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==} engines: {node: '>=8'} + docker-compose@1.4.2: + resolution: {integrity: sha512-rPHigTKGaEHpkUmfd69QgaOp+Os5vGJwG/Ry8lcr8W/382AmI+z/D7qoa9BybKIkqNppaIbs8RYeHSevdQjWww==} + engines: {node: '>= 6.0.0'} + + docker-modem@5.0.7: + resolution: {integrity: sha512-XJgGhoR/CLpqshm4d3L7rzH6t8NgDFUIIpztYlLHIApeJjMZKYJMz2zxPsYxnejq5h3ELYSw/RBsi3t5h7gNTA==} + engines: {node: '>= 8.0'} + + dockerode@4.0.12: + resolution: {integrity: sha512-/bCZd6KlGcjZO8Buqmi/vXuqEGVEZ0PNjx/biBNqJD3MhK9DmdiAuKxqfNhflgDESDIiBz3qF+0e55+CpnrUcw==} + engines: {node: '>= 8.0'} + dotenv@8.6.0: resolution: {integrity: sha512-IrPdXQsk2BbzvCBGBOTmmSH5SodmqZNt4ERAZDmW4CT+tL8VtvinqywuANaFu4bOMWki16nqf0e4oC0QIaDr/g==} engines: {node: '>=10'} @@ -1582,9 +1827,21 @@ packages: resolution: {integrity: sha512-1d4D4SB9KiD2qFnBWbl3aoYjLvPVpZoth28yxTT00xJv2yXugdz0Xoyv7sGG0w0hzE6hQZMgd6B333GnySDpGQ==} hasBin: true + eastasianwidth@0.2.0: + resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + effect@4.0.0-beta.99: resolution: {integrity: sha512-hP1C61uzINfLl/4kKMwcqksxd34s4sQ3VSjsWjhGrkx9CRlXaqnfOK9dpTEKynQ6rA7wU9rb3c+48eDYw7uzxA==} + emoji-regex@8.0.0: + resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + + emoji-regex@9.2.2: + resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==} + + end-of-stream@1.4.5: + resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} + enhanced-resolve@5.24.3: resolution: {integrity: sha512-PwKooW9JUzh5chmYfHM3IQl5OkK2u2Nm011MgeZrss3JmFraUx/fqrf78kk8GUMYoibx/14MdwTl/1WKkG7TpQ==} engines: {node: '>=10.13.0'} @@ -1614,6 +1871,10 @@ packages: engines: {node: '>=18'} hasBin: true + escalade@3.2.0: + resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} + engines: {node: '>=6'} + escodegen@2.1.0: resolution: {integrity: sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==} engines: {node: '>=6.0'} @@ -1642,6 +1903,17 @@ packages: resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} engines: {node: '>=0.10.0'} + event-target-shim@5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} + engines: {node: '>=6'} + + events-universal@1.0.1: + resolution: {integrity: sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==} + + events@3.3.0: + resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} + engines: {node: '>=0.8.x'} + expect-type@1.4.0: resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} engines: {node: '>=12.0.0'} @@ -1653,6 +1925,9 @@ packages: resolution: {integrity: sha512-7ms6T7SybUev/PQITciI0yLM2pOSFy5zpG8Ty7tQofcVaQUvrMXp6CBwqF6fThLCLOrfBtuHAtwq6Yu4XPCllg==} engines: {node: '>=12.17.0'} + fast-fifo@1.3.2: + resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} + fast-glob@3.3.3: resolution: {integrity: sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==} engines: {node: '>=8.6.0'} @@ -1688,11 +1963,18 @@ packages: resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==} engines: {node: '>=8'} + foreground-child@3.3.1: + resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} + engines: {node: '>=14'} + formatly@0.3.0: resolution: {integrity: sha512-9XNj/o4wrRFyhSMJOvsuyMwy8aUfBaZ1VrqHVfohyXf0Sw0e+yfKG+xZaY3arGCOMdwFsqObtzVOc1gU9KiT9w==} engines: {node: '>=18.3.0'} hasBin: true + fs-constants@1.0.0: + resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} + fs-extra@7.0.1: resolution: {integrity: sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==} engines: {node: '>=6 <7 || >=8'} @@ -1714,13 +1996,24 @@ packages: function-bind@1.1.2: resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + generate-function@2.3.1: + resolution: {integrity: sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==} + get-amd-module-type@6.0.2: resolution: {integrity: sha512-7zShVYAYtMnj9S65CfN+hvpBCByfuB1OY8xID01nZEzXTZbx4YyysAfi+nMl95JSR6odt4q8TCj2W63KAoyVLQ==} engines: {node: '>=18'} + get-caller-file@2.0.5: + resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} + engines: {node: 6.* || 8.* || >= 10.*} + get-own-enumerable-property-symbols@3.0.2: resolution: {integrity: sha512-I0UBV/XOz1XkIJHEUDMZAbzCThU/H8DxmSfmdGcKPnVhu2VfFqr34jr9777IyaTYvxjedWhqVIilEDsCdP5G6g==} + get-port@7.2.0: + resolution: {integrity: sha512-afP4W205ONCuMoPBqcR6PSXnzX35KTcJygfJfcp+QY+uwm3p20p1YczWXhlICIzGMCxYBQcySEcOgsJcrkyobg==} + engines: {node: '>=16'} + get-tsconfig@4.14.0: resolution: {integrity: sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==} @@ -1728,6 +2021,11 @@ packages: resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} engines: {node: '>= 6'} + glob@10.5.0: + resolution: {integrity: sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==} + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me + hasBin: true + glob@13.0.6: resolution: {integrity: sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==} engines: {node: 18 || 20 || >=22} @@ -1792,6 +2090,10 @@ packages: resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} engines: {node: '>=0.10.0'} + is-fullwidth-code-point@3.0.0: + resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} + engines: {node: '>=8'} + is-glob@4.0.3: resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} engines: {node: '>=0.10.0'} @@ -1808,10 +2110,17 @@ packages: resolution: {integrity: sha512-l4RyHgRqGN4Y3+9JHVrNqO+tN0rV5My76uW5/nuO4K1b6vw5G8d/cmFjP9tRfEsdhZNt0IFdZuK/c2Vr4Nb+Qg==} engines: {node: '>=0.10.0'} + is-property@1.0.2: + resolution: {integrity: sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==} + is-regexp@1.0.0: resolution: {integrity: sha512-7zjFAPO4/gwyQAAgRRmqeEeyIICSdmCqa3tsVHMdBzaXXRiqopZL4Cyghg/XulGWrtABTpbnYYzzIRffLkP4oA==} engines: {node: '>=0.10.0'} + is-stream@2.0.1: + resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} + engines: {node: '>=8'} + is-subdir@1.2.0: resolution: {integrity: sha512-2AT6j+gXe/1ueqbW6fLZJiIw3F8iXGJtt0yDrZaBhAZEG1raiTxKWU+IPqMCzQAXOUCKdA4UDMgacKH25XG2Cw==} engines: {node: '>=4'} @@ -1828,6 +2137,9 @@ packages: resolution: {integrity: sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==} engines: {node: '>=0.10.0'} + isarray@1.0.0: + resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==} + isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} @@ -1843,6 +2155,9 @@ packages: resolution: {integrity: sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==} engines: {node: '>=8'} + jackspeak@3.4.3: + resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} + jiti@2.7.0: resolution: {integrity: sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==} hasBin: true @@ -1874,21 +2189,41 @@ packages: kubernetes-types@1.30.0: resolution: {integrity: sha512-Dew1okvhM/SQcIa2rcgujNndZwU8VnSapDgdxlYoB84ZlpAD43U6KLAFqYo17ykSFGHNPrg0qry0bP+GJd9v7Q==} + lazystream@1.0.1: + resolution: {integrity: sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==} + engines: {node: '>= 0.6.3'} + locate-path@5.0.0: resolution: {integrity: sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==} engines: {node: '>=8'} + lodash.camelcase@4.3.0: + resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==} + lodash.startcase@4.4.0: resolution: {integrity: sha512-+WKqsK294HMSc2jEbNgpHpd0JfIBhp7rEV4aqXWqFr6AlXov+SlcgB1Fv01y2kGe3Gc8nMW7VA0SrGuSkRfIEg==} + lodash@4.18.1: + resolution: {integrity: sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==} + log-symbols@4.1.0: resolution: {integrity: sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==} engines: {node: '>=10'} + long@5.3.2: + resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} + + lru-cache@10.4.3: + resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} + lru-cache@11.5.2: resolution: {integrity: sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g==} engines: {node: 20 || >=22} + lru.min@1.1.4: + resolution: {integrity: sha512-DqC6n3QQ77zdFpCMASA1a3Jlb64Hv2N2DciFGkO/4L9+q/IpIAuRlKOvCXabtRW6cQf8usbmM6BE/TOPysCdIA==} + engines: {bun: '>=1.0.0', deno: '>=1.30.0', node: '>=8.0.0'} + lucide-react@0.468.0: resolution: {integrity: sha512-6koYRhnM2N0GGZIdXzSeiNwguv1gt/FAjZOiPl76roBi3xKEXa4WmfpxgQwTTL4KipXjefrnf3oV4IsYhi4JFA==} peerDependencies: @@ -1935,6 +2270,14 @@ packages: resolution: {integrity: sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==} engines: {node: 18 || 20 || >=22} + minimatch@5.1.9: + resolution: {integrity: sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==} + engines: {node: '>=10'} + + minimatch@9.0.9: + resolution: {integrity: sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==} + engines: {node: '>=16 || 14 >=14.17'} + minimist@1.2.8: resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} @@ -1942,6 +2285,14 @@ packages: resolution: {integrity: sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==} engines: {node: '>=16 || 14 >=14.17'} + mkdirp-classic@0.5.3: + resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} + + mkdirp@3.0.1: + resolution: {integrity: sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==} + engines: {node: '>=10'} + hasBin: true + module-definition@6.0.2: resolution: {integrity: sha512-SvAU3lB0+Yjbq55yHY3wkRZBOh+fhU1SnIF3IFbTewv6mtAh7yUT8ACHAJ2mGIJ7tCes2QuCL/cl6m0JSZ/ArA==} engines: {node: '>=18'} @@ -1969,6 +2320,19 @@ packages: multipasta@0.2.8: resolution: {integrity: sha512-ZPWuMKyv0cSO29f7hozp+k6+crZbQijV8ipMvxNxRf2SwtYGTX1ZX89Kd20VV4H9Znonx+EQn+iy1wGQsJ+b+Q==} + mysql2@3.23.1: + resolution: {integrity: sha512-tTuRnC7qCet2IOfSNMYZ5SwXuBnfvBPAcIA28P0gtruXyZlU1LMxA6uha32kYypoFgyYklMqhLWwt4laYwXR/Q==} + engines: {node: '>= 8.0'} + peerDependencies: + '@types/node': '>= 8' + + named-placeholders@1.1.6: + resolution: {integrity: sha512-Tz09sEL2EEuv5fFowm419c1+a/jSMiBjI9gHxVLrVdbUkkNUUfjsVYs9pVZu5oCon/kmRh9TfLEObFtkVxmY0w==} + engines: {node: '>=8.0.0'} + + nan@2.28.0: + resolution: {integrity: sha512-fTsDz99OTq2sVePhGdp4qQhggZFtKr64ZNVyVajRKtMOkJxYekplBh577PiJB12v/D3s2E5cGtOI45LWp6rnLQ==} + nanoid@3.3.16: resolution: {integrity: sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} @@ -1991,10 +2355,20 @@ packages: resolution: {integrity: sha512-71kFFjYaSshDTA8/a2HiTYPLdASWjLJxUyJxGE+ffxU+KhxSBtM9kiLUX+R2yooFdSFKMFpi4n3PFtDy6qXv8A==} engines: {node: '>=18'} + normalize-path@3.0.0: + resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} + engines: {node: '>=0.10.0'} + + obuf@1.1.2: + resolution: {integrity: sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==} + obug@2.1.4: resolution: {integrity: sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA==} engines: {node: '>=12.20.0'} + once@1.4.0: + resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + onetime@5.1.2: resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==} engines: {node: '>=6'} @@ -2043,6 +2417,9 @@ packages: resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} engines: {node: '>=6'} + package-json-from-dist@1.0.1: + resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} + package-manager-detector@0.2.11: resolution: {integrity: sha512-BEnLolu+yuz22S56CU1SUKq3XC3PkwD5wv4ikR4MfGvnRVcmzXR9DwSlW2fEamyTPyXHomBJRzgapeuBvRNzJQ==} @@ -2061,6 +2438,10 @@ packages: path-parse@1.0.7: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} + path-scurry@1.11.1: + resolution: {integrity: sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==} + engines: {node: '>=16 || 14 >=14.18'} + path-scurry@2.0.2: resolution: {integrity: sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==} engines: {node: 18 || 20 || >=22} @@ -2072,6 +2453,53 @@ packages: pathe@2.0.3: resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + pg-cloudflare@1.4.0: + resolution: {integrity: sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==} + + pg-connection-string@2.14.0: + resolution: {integrity: sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==} + + pg-cursor@2.21.0: + resolution: {integrity: sha512-IYvk/j+Suhtbo/C3uOf4JLsLK/gWxOTUOmYbDsbKnLaVJDq+KwhwK6ngpRfiCk8eDMS3AmGQABZCv0cREEzHQw==} + peerDependencies: + pg: ^8 + + pg-int8@1.0.1: + resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} + engines: {node: '>=4.0.0'} + + pg-numeric@1.0.2: + resolution: {integrity: sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==} + engines: {node: '>=4'} + + pg-pool@3.14.0: + resolution: {integrity: sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==} + peerDependencies: + pg: '>=8.0' + + pg-protocol@1.15.0: + resolution: {integrity: sha512-cq9sECI5s0+uPUXjbz8ioyPJni6RzsRib0US67i5IoTZKw8fNeYlVE7u8F4dG7vEJJtc5wdD1K189lCCUwqWTQ==} + + pg-types@2.2.0: + resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} + engines: {node: '>=4'} + + pg-types@4.1.0: + resolution: {integrity: sha512-o2XFanIMy/3+mThw69O8d4n1E5zsLhdO+OPqswezu7Z5ekP4hYDqlDjlmOpYMbzY2Br0ufCwJLdDIXeNVwcWFg==} + engines: {node: '>=10'} + + pg@8.22.0: + resolution: {integrity: sha512-8wih1vVIBMxoUM2oB4soJsD9tDnDpLv4OXBJ+EJzFsvycD+lfyIreC2gGHq78f8jbLLt+bvlPTFdFZfJkOuzAA==} + engines: {node: '>= 16.0.0'} + peerDependencies: + pg-native: '>=3.0.1' + peerDependenciesMeta: + pg-native: + optional: true + + pgpass@1.0.5: + resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} + picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -2111,6 +2539,41 @@ packages: resolution: {integrity: sha512-v4sDNP3fdNiWMfabO7OwOQdOX8TiQSztKyT1Wj0w+j7LDallJThJRBBBmzVGyYj0crMh7jlV4zepPkiNu9UwDQ==} engines: {node: ^10 || ^12 || >=14} + postgres-array@2.0.0: + resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} + engines: {node: '>=4'} + + postgres-array@3.0.4: + resolution: {integrity: sha512-nAUSGfSDGOaOAEGwqsRY27GPOea7CNipJPOA7lPbdEpx5Kg3qzdP0AaWC5MlhTWV9s4hFX39nomVZ+C4tnGOJQ==} + engines: {node: '>=12'} + + postgres-bytea@1.0.1: + resolution: {integrity: sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==} + engines: {node: '>=0.10.0'} + + postgres-bytea@3.0.0: + resolution: {integrity: sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw==} + engines: {node: '>= 6'} + + postgres-date@1.0.7: + resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} + engines: {node: '>=0.10.0'} + + postgres-date@2.1.0: + resolution: {integrity: sha512-K7Juri8gtgXVcDfZttFKVmhglp7epKb1K4pgrkLxehjqkrgPhfG6OO8LHLkfaqkbpjNRnra018XwAr1yQFWGcA==} + engines: {node: '>=12'} + + postgres-interval@1.2.0: + resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} + engines: {node: '>=0.10.0'} + + postgres-interval@3.0.0: + resolution: {integrity: sha512-BSNDnbyZCXSxgA+1f5UU2GmwhoI0aU5yMxRGO8CdFEcY2BQF9xm/7MqKnYoM1nJDk8nONNWDk9WeSmePFhQdlw==} + engines: {node: '>=12'} + + postgres-range@1.1.4: + resolution: {integrity: sha512-i/hbxIE9803Alj/6ytL7UHQxRvZkI9O4Sy+J3HGc4F4oo/2eQAjTSNJ0bfxyse3bH0nuVesCk+3IRLaMtG3H6w==} + precinct@12.3.2: resolution: {integrity: sha512-JbJevI1K80z8e/WIyDt/4vUN/4qcfBSKKqOjJA4mosPPPb7zODKRJQV7YN7apVWN3k58nZYm/vEsLgEGYmnxwg==} engines: {node: '>=18'} @@ -2125,6 +2588,27 @@ packages: resolution: {integrity: sha512-973driJZvxiGOQ5ONsFhOF/DtzPMOMtgC11kCpUrPGMTgqp2q/1gwzCquocrN33is0VZ5GFHXZYMM9l6h67v2Q==} engines: {node: '>=10'} + process-nextick-args@2.0.1: + resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} + + process@0.11.10: + resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} + engines: {node: '>= 0.6.0'} + + proper-lockfile@4.1.2: + resolution: {integrity: sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==} + + properties-reader@3.0.1: + resolution: {integrity: sha512-WPn+h9RGEExOKdu4bsF4HksG/uzd3cFq3MFtq8PsFeExPse5Ha/VOjQNyHhjboBFwGXGev6muJYTSPAOkROq2g==} + engines: {node: '>=18'} + + protobufjs@7.6.5: + resolution: {integrity: sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==} + engines: {node: '>=12.0.0'} + + pump@3.0.4: + resolution: {integrity: sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==} + pure-rand@8.4.2: resolution: {integrity: sha512-vvuOGgcuPJAirlHvuQw1TrOiw7ptaIXXmIbNuiNOY6lNGJJH49PQ1Kj4nd783nPdQhQdicgOjVI2yI/9BD6/Ng==} @@ -2154,10 +2638,20 @@ packages: resolution: {integrity: sha512-VIMnQi/Z4HT2Fxuwg5KrY174U1VdUIASQVWXXyqtNRtxSr9IYkn1rsI6Tb6HsrHCmB7gVpNwX6JxPTHcH6IoTA==} engines: {node: '>=6'} + readable-stream@2.3.8: + resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==} + readable-stream@3.6.2: resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} engines: {node: '>= 6'} + readable-stream@4.7.0: + resolution: {integrity: sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + + readdir-glob@1.1.3: + resolution: {integrity: sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==} + redis-errors@1.2.0: resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} engines: {node: '>=4'} @@ -2166,6 +2660,10 @@ packages: resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} engines: {node: '>=4'} + require-directory@2.1.1: + resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} + engines: {node: '>=0.10.0'} + requirejs-config-file@4.0.0: resolution: {integrity: sha512-jnIre8cbWOyvr8a5F2KuqBnY+SDA4NXr/hzEZJG79Mxm2WiFQz2dzhC8ibtPJS7zkmBEl1mxSwp5HhC1W4qpxw==} engines: {node: '>=10.13.0'} @@ -2195,6 +2693,10 @@ packages: resolution: {integrity: sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==} engines: {node: '>=8'} + retry@0.12.0: + resolution: {integrity: sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==} + engines: {node: '>= 4'} + reusify@1.1.0: resolution: {integrity: sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} @@ -2207,6 +2709,9 @@ packages: run-parallel@1.2.0: resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} + safe-buffer@5.1.2: + resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} + safe-buffer@5.2.1: resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} @@ -2263,9 +2768,27 @@ packages: spawndamnit@3.0.1: resolution: {integrity: sha512-MmnduQUuHCoFckZoWnXsTg7JaiLBJrKFj9UI2MbRPGaJeVpsLcVBu6P/IGZovziM/YBsellCmsprgNA+w0CzVg==} + split-ca@1.0.1: + resolution: {integrity: sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==} + + split2@4.2.0: + resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} + engines: {node: '>= 10.x'} + sprintf-js@1.0.3: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} + sql-escaper@1.5.1: + resolution: {integrity: sha512-4toX5E1fQbBrpfXidaHnF0669nkAdETeIPTs2SUjxxD7RRIs9ICG4gtpmfc68JCEKehsdwLFqBu9VlQqZ1P1gg==} + engines: {bun: '>=1.0.0', deno: '>=2.0.0', node: '>=12.0.0'} + + ssh-remote-port-forward@1.0.4: + resolution: {integrity: sha512-x0LV1eVDwjf1gmG7TTnfqIzf+3VPRz7vrNIjX6oYLbeCrf/PeVY6hkT68Mg+q02qXxQhrLjB0jfgvhevoCRmLQ==} + + ssh2@1.17.0: + resolution: {integrity: sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==} + engines: {node: '>=10.16.0'} + stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} @@ -2278,6 +2801,20 @@ packages: stream-to-array@2.3.0: resolution: {integrity: sha512-UsZtOYEn4tWU2RGLOXr/o/xjRBftZRlG3dEWoaHr8j4GuypJ3isitGbVyjQKAuMu+xbiop8q224TjiZWc4XTZA==} + streamx@2.28.0: + resolution: {integrity: sha512-1Yowhzjf0ivGMrTIkY9hav5TxobO9qIVqUE41fiCGMGgc3CLlf4MY+9AHmZqBWgDTue0fY9zWjYFVyf6Diuobw==} + + string-width@4.2.3: + resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} + engines: {node: '>=8'} + + string-width@5.1.2: + resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==} + engines: {node: '>=12'} + + string_decoder@1.1.1: + resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==} + string_decoder@1.3.0: resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} @@ -2289,6 +2826,10 @@ packages: resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} engines: {node: '>=8'} + strip-ansi@7.2.0: + resolution: {integrity: sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==} + engines: {node: '>=12'} + strip-bom@3.0.0: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} engines: {node: '>=4'} @@ -2318,10 +2859,32 @@ packages: resolution: {integrity: sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==} engines: {node: '>=6'} + tar-fs@2.1.5: + resolution: {integrity: sha512-OboTd8mmMhZDNPV+UjQcK9yKAatXu2aJ+r1w4im1Otd4M4fl2hwvdoXUxIYHFTHWK/3y3FarBP70v3vwmGlOxw==} + + tar-fs@3.1.3: + resolution: {integrity: sha512-/hU4AXnIdZu+Gvl1pk0oI5f5HxWsCJRtY2aFaJdk9VvyL48DWU6iU5WAIPG+wIi1YvWA6eTJvIviP/tMAZZNwQ==} + + tar-stream@2.2.0: + resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} + engines: {node: '>=6'} + + tar-stream@3.2.0: + resolution: {integrity: sha512-ojzvCvVaNp6aOTFmG7jaRD0meowIAuPc3cMMhSgKiVWws1GyHbGd/xvnyuRKcKlMpt3qvxx6r0hreCNITP9hIg==} + + teex@1.0.1: + resolution: {integrity: sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==} + term-size@2.2.1: resolution: {integrity: sha512-wK0Ri4fOGjv/XPy8SBHZChl8CM7uMc5VML7SqiQ0zG7+J5Vr+RMQDoHa2CNT6KHUnTGIXH34UDMkPzAUyapBZg==} engines: {node: '>=8'} + testcontainers@11.14.0: + resolution: {integrity: sha512-r9pniwv/iwzyHaI7gwAvAm4Y+IvjJg3vBWdjrUCaDMc2AXIr4jKbq7jJO18Mw2ybs73pZy1Aj7p/4RVBGMRWjg==} + + text-decoder@1.2.7: + resolution: {integrity: sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==} + tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} @@ -2337,6 +2900,10 @@ packages: resolution: {integrity: sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==} engines: {node: '>=14.0.0'} + tmp@0.2.7: + resolution: {integrity: sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw==} + engines: {node: '>=14.14'} + to-regex-range@5.0.1: resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} engines: {node: '>=8.0'} @@ -2370,6 +2937,9 @@ packages: engines: {node: '>=18.0.0'} hasBin: true + tweetnacl@0.14.5: + resolution: {integrity: sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==} + typescript@5.9.3: resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} engines: {node: '>=14.17'} @@ -2384,9 +2954,16 @@ packages: resolution: {integrity: sha512-3cudTErfToSc4Ggv8XGXVNVli/xHKUtUZvaY5UVwhOcUPbQGz7PeaEnT/SAVgNziZtX67KEN9swMUYkLghxA1w==} engines: {node: '>=14'} + undici-types@5.26.5: + resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + undici-types@7.24.6: resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} + undici@7.29.0: + resolution: {integrity: sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==} + engines: {node: '>=20.18.1'} + undici@8.8.0: resolution: {integrity: sha512-ubshXMXwF3MQIMF1y/WxZdNBnjEKeSg2wF5mcGUtU55YTw34tnVVpKRlLf7ruDXZ5344KokPVX4RBx1wJm64Bw==} engines: {node: '>=22.19.0'} @@ -2398,6 +2975,11 @@ packages: util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + uuid@10.0.0: + resolution: {integrity: sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==} + deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028). + hasBin: true + uuid@14.0.1: resolution: {integrity: sha512-6ZxzVpzDXDa3bJWaHilVayA+BH/1zmxCJoVgvmqJnid/gPoKHxUrS/aC/T6LGQtNHT+XHG9fXPJB4d+IrU30Ew==} hasBin: true @@ -2515,6 +3097,17 @@ packages: engines: {node: '>=8'} hasBin: true + wrap-ansi@7.0.0: + resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} + engines: {node: '>=10'} + + wrap-ansi@8.1.0: + resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==} + engines: {node: '>=12'} + + wrappy@1.0.2: + resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + ws@8.21.1: resolution: {integrity: sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw==} engines: {node: '>=10.0.0'} @@ -2527,11 +3120,31 @@ packages: utf-8-validate: optional: true + xtend@4.0.2: + resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} + engines: {node: '>=0.4'} + + y18n@5.0.8: + resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} + engines: {node: '>=10'} + yaml@2.9.0: resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==} engines: {node: '>= 14.6'} hasBin: true + yargs-parser@21.1.1: + resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} + engines: {node: '>=12'} + + yargs@17.7.3: + resolution: {integrity: sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==} + engines: {node: '>=12'} + + zip-stream@6.0.1: + resolution: {integrity: sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==} + engines: {node: '>= 14'} + zod@4.4.3: resolution: {integrity: sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==} @@ -2554,6 +3167,8 @@ snapshots: '@babel/helper-string-parser': 7.29.7 '@babel/helper-validator-identifier': 7.29.7 + '@balena/dockerignore@1.0.2': {} + '@bcoe/v8-coverage@1.0.2': {} '@changesets/apply-release-plan@7.1.1': @@ -2799,6 +3414,24 @@ snapshots: - bufferutil - utf-8-validate + '@effect/sql-mysql2@4.0.0-beta.99(@types/node@25.9.5)(effect@4.0.0-beta.99)': + dependencies: + effect: 4.0.0-beta.99 + mysql2: 3.23.1(@types/node@25.9.5) + transitivePeerDependencies: + - '@types/node' + + '@effect/sql-pg@4.0.0-beta.99(effect@4.0.0-beta.99)': + dependencies: + effect: 4.0.0-beta.99 + pg: 8.22.0 + pg-connection-string: 2.14.0 + pg-cursor: 2.21.0(pg@8.22.0) + pg-pool: 3.14.0(pg@8.22.0) + pg-types: 4.1.0 + transitivePeerDependencies: + - pg-native + '@effect/sql-sqlite-node@4.0.0-beta.99(effect@4.0.0-beta.99)': dependencies: effect: 4.0.0-beta.99 @@ -2998,6 +3631,25 @@ snapshots: '@esbuild/win32-x64@0.28.1': optional: true + '@grpc/grpc-js@1.14.4': + dependencies: + '@grpc/proto-loader': 0.8.1 + '@js-sdsl/ordered-map': 4.4.2 + + '@grpc/proto-loader@0.7.15': + dependencies: + lodash.camelcase: 4.3.0 + long: 5.3.2 + protobufjs: 7.6.5 + yargs: 17.7.3 + + '@grpc/proto-loader@0.8.1': + dependencies: + lodash.camelcase: 4.3.0 + long: 5.3.2 + protobufjs: 7.6.5 + yargs: 17.7.3 + '@inquirer/external-editor@1.0.3(@types/node@25.9.5)': dependencies: chardet: 2.2.0 @@ -3007,6 +3659,15 @@ snapshots: '@ioredis/commands@1.10.0': {} + '@isaacs/cliui@8.0.2': + dependencies: + string-width: 5.1.2 + string-width-cjs: string-width@4.2.3 + strip-ansi: 7.2.0 + strip-ansi-cjs: strip-ansi@6.0.1 + wrap-ansi: 8.1.0 + wrap-ansi-cjs: wrap-ansi@7.0.0 + '@jridgewell/resolve-uri@3.1.2': {} '@jridgewell/sourcemap-codec@1.5.5': {} @@ -3016,6 +3677,14 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 + '@js-sdsl/ordered-map@4.4.2': {} + + '@kwsites/file-exists@1.1.1': + dependencies: + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + '@manypkg/find-root@1.1.0': dependencies: '@babel/runtime': 7.29.7 @@ -3227,10 +3896,33 @@ snapshots: '@oxlint/win32-x64@1.42.0': optional: true + '@pkgjs/parseargs@0.11.0': + optional: true + '@playwright/test@1.61.1': dependencies: playwright: 1.61.1 + '@protobufjs/aspromise@1.1.2': {} + + '@protobufjs/base64@1.1.2': {} + + '@protobufjs/codegen@2.0.5': {} + + '@protobufjs/eventemitter@1.1.1': {} + + '@protobufjs/fetch@1.1.1': + dependencies: + '@protobufjs/aspromise': 1.1.2 + + '@protobufjs/float@1.0.2': {} + + '@protobufjs/path@1.1.2': {} + + '@protobufjs/pool@1.1.0': {} + + '@protobufjs/utf8@1.1.2': {} + '@rollup/rollup-android-arm-eabi@4.62.2': optional: true @@ -3308,6 +4000,24 @@ snapshots: '@standard-schema/spec@1.1.0': {} + '@testcontainers/mysql@11.14.0': + dependencies: + testcontainers: 11.14.0 + transitivePeerDependencies: + - bare-abort-controller + - bare-buffer + - react-native-b4a + - supports-color + + '@testcontainers/postgresql@11.14.0': + dependencies: + testcontainers: 11.14.0 + transitivePeerDependencies: + - bare-abort-controller + - bare-buffer + - react-native-b4a + - supports-color + '@ts-graphviz/adapter@2.0.6': dependencies: '@ts-graphviz/common': 2.1.5 @@ -3335,10 +4045,25 @@ snapshots: '@types/deep-eql@4.0.2': {} + '@types/docker-modem@3.0.6': + dependencies: + '@types/node': 25.9.5 + '@types/ssh2': 1.15.5 + + '@types/dockerode@4.0.1': + dependencies: + '@types/docker-modem': 3.0.6 + '@types/node': 25.9.5 + '@types/ssh2': 1.15.5 + '@types/estree@1.0.9': {} '@types/node@12.20.55': {} + '@types/node@18.19.130': + dependencies: + undici-types: 5.26.5 + '@types/node@25.9.5': dependencies: undici-types: 7.24.6 @@ -3351,6 +4076,19 @@ snapshots: dependencies: csstype: 3.2.3 + '@types/ssh2-streams@0.1.13': + dependencies: + '@types/node': 25.9.5 + + '@types/ssh2@0.5.52': + dependencies: + '@types/node': 25.9.5 + '@types/ssh2-streams': 0.1.13 + + '@types/ssh2@1.15.5': + dependencies: + '@types/node': 18.19.130 + '@types/ws@8.18.1': dependencies: '@types/node': 25.9.5 @@ -3508,18 +4246,50 @@ snapshots: '@vue/shared@3.5.40': {} + abort-controller@3.0.0: + dependencies: + event-target-shim: 5.0.1 + ansi-colors@4.1.3: {} ansi-regex@5.0.1: {} + ansi-regex@6.2.2: {} + ansi-styles@4.3.0: dependencies: color-convert: 2.0.1 + ansi-styles@6.2.3: {} + any-promise@1.3.0: {} app-module-path@2.2.0: {} + archiver-utils@5.0.2: + dependencies: + glob: 10.5.0 + graceful-fs: 4.2.11 + is-stream: 2.0.1 + lazystream: 1.0.1 + lodash: 4.18.1 + normalize-path: 3.0.0 + readable-stream: 4.7.0 + + archiver@7.0.1: + dependencies: + archiver-utils: 5.0.2 + async: 3.2.6 + buffer-crc32: 1.0.0 + readable-stream: 4.7.0 + readdir-glob: 1.1.3 + tar-stream: 3.2.0 + zip-stream: 6.0.1 + transitivePeerDependencies: + - bare-abort-controller + - bare-buffer + - react-native-b4a + argparse@1.0.10: dependencies: sprintf-js: 1.0.3 @@ -3528,6 +4298,10 @@ snapshots: array-union@2.1.0: {} + asn1@0.2.6: + dependencies: + safer-buffer: 2.1.2 + assertion-error@2.0.1: {} ast-module-types@6.0.2: {} @@ -3538,10 +4312,53 @@ snapshots: estree-walker: 3.0.3 js-tokens: 10.0.0 + async-lock@1.4.1: {} + + async@3.2.6: {} + + aws-ssl-profiles@1.1.2: {} + + b4a@1.8.1: {} + + balanced-match@1.0.2: {} + balanced-match@4.0.4: {} + bare-events@2.9.1: {} + + bare-fs@4.7.4: + dependencies: + bare-events: 2.9.1 + bare-path: 3.1.1 + bare-stream: 2.13.3(bare-events@2.9.1) + bare-url: 2.4.6 + fast-fifo: 1.3.2 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + + bare-path@3.1.1: {} + + bare-stream@2.13.3(bare-events@2.9.1): + dependencies: + b4a: 1.8.1 + streamx: 2.28.0 + teex: 1.0.1 + optionalDependencies: + bare-events: 2.9.1 + transitivePeerDependencies: + - react-native-b4a + + bare-url@2.4.6: + dependencies: + bare-path: 3.1.1 + base64-js@1.5.1: {} + bcrypt-pbkdf@1.0.2: + dependencies: + tweetnacl: 0.14.5 + better-path-resolve@1.0.0: dependencies: is-windows: 1.0.2 @@ -3552,6 +4369,10 @@ snapshots: inherits: 2.0.4 readable-stream: 3.6.2 + brace-expansion@2.1.2: + dependencies: + balanced-match: 1.0.2 + brace-expansion@5.0.7: dependencies: balanced-match: 4.0.4 @@ -3560,11 +4381,23 @@ snapshots: dependencies: fill-range: 7.1.1 + buffer-crc32@1.0.0: {} + buffer@5.7.1: dependencies: base64-js: 1.5.1 ieee754: 1.2.1 + buffer@6.0.3: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + + buildcheck@0.0.7: + optional: true + + byline@5.0.0: {} + chai@6.2.2: {} chalk@4.1.2: @@ -3574,12 +4407,20 @@ snapshots: chardet@2.2.0: {} + chownr@1.1.4: {} + cli-cursor@3.1.0: dependencies: restore-cursor: 3.1.0 cli-spinners@2.9.2: {} + cliui@8.0.1: + dependencies: + string-width: 4.2.3 + strip-ansi: 6.0.1 + wrap-ansi: 7.0.0 + clone@1.0.4: {} cluster-key-slot@1.1.1: {} @@ -3596,8 +4437,31 @@ snapshots: commondir@1.0.1: {} + compress-commons@6.0.2: + dependencies: + crc-32: 1.2.2 + crc32-stream: 6.0.0 + is-stream: 2.0.1 + normalize-path: 3.0.0 + readable-stream: 4.7.0 + convert-source-map@2.0.0: {} + core-util-is@1.0.3: {} + + cpu-features@0.0.10: + dependencies: + buildcheck: 0.0.7 + nan: 2.28.0 + optional: true + + crc-32@1.2.2: {} + + crc32-stream@6.0.0: + dependencies: + crc-32: 1.2.2 + readable-stream: 4.7.0 + cross-spawn@7.0.6: dependencies: path-key: 3.1.1 @@ -3695,6 +4559,31 @@ snapshots: dependencies: path-type: 4.0.0 + docker-compose@1.4.2: + dependencies: + yaml: 2.9.0 + + docker-modem@5.0.7: + dependencies: + debug: 4.4.3 + readable-stream: 3.6.2 + split-ca: 1.0.1 + ssh2: 1.17.0 + transitivePeerDependencies: + - supports-color + + dockerode@4.0.12: + dependencies: + '@balena/dockerignore': 1.0.2 + '@grpc/grpc-js': 1.14.4 + '@grpc/proto-loader': 0.7.15 + docker-modem: 5.0.7 + protobufjs: 7.6.5 + tar-fs: 2.1.5 + uuid: 10.0.0 + transitivePeerDependencies: + - supports-color + dotenv@8.6.0: {} dprint@0.55.2: @@ -3715,6 +4604,8 @@ snapshots: '@dprint/win32-arm64': 0.55.2 '@dprint/win32-x64': 0.55.2 + eastasianwidth@0.2.0: {} + effect@4.0.0-beta.99: dependencies: '@standard-schema/spec': 1.1.0 @@ -3728,6 +4619,14 @@ snapshots: uuid: 14.0.1 yaml: 2.9.0 + emoji-regex@8.0.0: {} + + emoji-regex@9.2.2: {} + + end-of-stream@1.4.5: + dependencies: + once: 1.4.0 + enhanced-resolve@5.24.3: dependencies: graceful-fs: 4.2.11 @@ -3802,6 +4701,8 @@ snapshots: '@esbuild/win32-ia32': 0.28.1 '@esbuild/win32-x64': 0.28.1 + escalade@3.2.0: {} + escodegen@2.1.0: dependencies: esprima: 4.0.1 @@ -3824,6 +4725,16 @@ snapshots: esutils@2.0.3: {} + event-target-shim@5.0.1: {} + + events-universal@1.0.1: + dependencies: + bare-events: 2.9.1 + transitivePeerDependencies: + - bare-abort-controller + + events@3.3.0: {} + expect-type@1.4.0: {} extendable-error@0.1.7: {} @@ -3832,6 +4743,8 @@ snapshots: dependencies: pure-rand: 8.4.2 + fast-fifo@1.3.2: {} + fast-glob@3.3.3: dependencies: '@nodelib/fs.stat': 2.0.5 @@ -3877,10 +4790,17 @@ snapshots: locate-path: 5.0.0 path-exists: 4.0.0 + foreground-child@3.3.1: + dependencies: + cross-spawn: 7.0.6 + signal-exit: 4.1.0 + formatly@0.3.0: dependencies: fd-package-json: 2.0.0 + fs-constants@1.0.0: {} + fs-extra@7.0.1: dependencies: graceful-fs: 4.2.11 @@ -3901,13 +4821,21 @@ snapshots: function-bind@1.1.2: {} + generate-function@2.3.1: + dependencies: + is-property: 1.0.2 + get-amd-module-type@6.0.2: dependencies: ast-module-types: 6.0.2 node-source-walk: 7.0.2 + get-caller-file@2.0.5: {} + get-own-enumerable-property-symbols@3.0.2: {} + get-port@7.2.0: {} + get-tsconfig@4.14.0: dependencies: resolve-pkg-maps: 1.0.0 @@ -3916,6 +4844,15 @@ snapshots: dependencies: is-glob: 4.0.3 + glob@10.5.0: + dependencies: + foreground-child: 3.3.1 + jackspeak: 3.4.3 + minimatch: 9.0.9 + minipass: 7.1.3 + package-json-from-dist: 1.0.1 + path-scurry: 1.11.1 + glob@13.0.6: dependencies: minimatch: 10.2.5 @@ -3979,6 +4916,8 @@ snapshots: is-extglob@2.1.1: {} + is-fullwidth-code-point@3.0.0: {} + is-glob@4.0.3: dependencies: is-extglob: 2.1.1 @@ -3989,8 +4928,12 @@ snapshots: is-obj@1.0.1: {} + is-property@1.0.2: {} + is-regexp@1.0.0: {} + is-stream@2.0.1: {} + is-subdir@1.2.0: dependencies: better-path-resolve: 1.0.0 @@ -4001,6 +4944,8 @@ snapshots: is-windows@1.0.2: {} + isarray@1.0.0: {} + isexe@2.0.0: {} istanbul-lib-coverage@3.2.2: {} @@ -4016,6 +4961,12 @@ snapshots: html-escaper: 2.0.2 istanbul-lib-report: 3.0.1 + jackspeak@3.4.3: + dependencies: + '@isaacs/cliui': 8.0.2 + optionalDependencies: + '@pkgjs/parseargs': 0.11.0 + jiti@2.7.0: {} js-tokens@10.0.0: {} @@ -4053,19 +5004,33 @@ snapshots: kubernetes-types@1.30.0: {} + lazystream@1.0.1: + dependencies: + readable-stream: 2.3.8 + locate-path@5.0.0: dependencies: p-locate: 4.1.0 + lodash.camelcase@4.3.0: {} + lodash.startcase@4.4.0: {} + lodash@4.18.1: {} + log-symbols@4.1.0: dependencies: chalk: 4.1.2 is-unicode-supported: 0.1.0 + long@5.3.2: {} + + lru-cache@10.4.3: {} + lru-cache@11.5.2: {} + lru.min@1.1.4: {} + lucide-react@0.468.0(react@19.2.7): dependencies: react: 19.2.7 @@ -4118,10 +5083,22 @@ snapshots: dependencies: brace-expansion: 5.0.7 + minimatch@5.1.9: + dependencies: + brace-expansion: 2.1.2 + + minimatch@9.0.9: + dependencies: + brace-expansion: 2.1.2 + minimist@1.2.8: {} minipass@7.1.3: {} + mkdirp-classic@0.5.3: {} + + mkdirp@3.0.1: {} + module-definition@6.0.2: dependencies: ast-module-types: 6.0.2 @@ -4155,6 +5132,25 @@ snapshots: multipasta@0.2.8: {} + mysql2@3.23.1(@types/node@25.9.5): + dependencies: + '@types/node': 25.9.5 + aws-ssl-profiles: 1.1.2 + denque: 2.1.0 + generate-function: 2.3.1 + iconv-lite: 0.7.3 + long: 5.3.2 + lru.min: 1.1.4 + named-placeholders: 1.1.6 + sql-escaper: 1.5.1 + + named-placeholders@1.1.6: + dependencies: + lru.min: 1.1.4 + + nan@2.28.0: + optional: true + nanoid@3.3.16: {} node-fetch@2.7.0: @@ -4170,8 +5166,16 @@ snapshots: dependencies: '@babel/parser': 7.29.7 + normalize-path@3.0.0: {} + + obuf@1.1.2: {} + obug@2.1.4: {} + once@1.4.0: + dependencies: + wrappy: 1.0.2 + onetime@5.1.2: dependencies: mimic-fn: 2.1.0 @@ -4264,6 +5268,8 @@ snapshots: p-try@2.2.0: {} + package-json-from-dist@1.0.1: {} + package-manager-detector@0.2.11: dependencies: quansync: 0.2.11 @@ -4276,6 +5282,11 @@ snapshots: path-parse@1.0.7: {} + path-scurry@1.11.1: + dependencies: + lru-cache: 10.4.3 + minipass: 7.1.3 + path-scurry@2.0.2: dependencies: lru-cache: 11.5.2 @@ -4285,6 +5296,57 @@ snapshots: pathe@2.0.3: {} + pg-cloudflare@1.4.0: + optional: true + + pg-connection-string@2.14.0: {} + + pg-cursor@2.21.0(pg@8.22.0): + dependencies: + pg: 8.22.0 + + pg-int8@1.0.1: {} + + pg-numeric@1.0.2: {} + + pg-pool@3.14.0(pg@8.22.0): + dependencies: + pg: 8.22.0 + + pg-protocol@1.15.0: {} + + pg-types@2.2.0: + dependencies: + pg-int8: 1.0.1 + postgres-array: 2.0.0 + postgres-bytea: 1.0.1 + postgres-date: 1.0.7 + postgres-interval: 1.2.0 + + pg-types@4.1.0: + dependencies: + pg-int8: 1.0.1 + pg-numeric: 1.0.2 + postgres-array: 3.0.4 + postgres-bytea: 3.0.0 + postgres-date: 2.1.0 + postgres-interval: 3.0.0 + postgres-range: 1.1.4 + + pg@8.22.0: + dependencies: + pg-connection-string: 2.14.0 + pg-pool: 3.14.0(pg@8.22.0) + pg-protocol: 1.15.0 + pg-types: 2.2.0 + pgpass: 1.0.5 + optionalDependencies: + pg-cloudflare: 1.4.0 + + pgpass@1.0.5: + dependencies: + split2: 4.2.0 + picocolors@1.1.1: {} picomatch@2.3.2: {} @@ -4316,6 +5378,28 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 + postgres-array@2.0.0: {} + + postgres-array@3.0.4: {} + + postgres-bytea@1.0.1: {} + + postgres-bytea@3.0.0: + dependencies: + obuf: 1.1.2 + + postgres-date@1.0.7: {} + + postgres-date@2.1.0: {} + + postgres-interval@1.2.0: + dependencies: + xtend: 4.0.2 + + postgres-interval@3.0.0: {} + + postgres-range@1.1.4: {} + precinct@12.3.2: dependencies: '@dependents/detective-less': 5.0.3 @@ -4342,6 +5426,42 @@ snapshots: dependencies: parse-ms: 2.1.0 + process-nextick-args@2.0.1: {} + + process@0.11.10: {} + + proper-lockfile@4.1.2: + dependencies: + graceful-fs: 4.2.11 + retry: 0.12.0 + signal-exit: 3.0.7 + + properties-reader@3.0.1: + dependencies: + '@kwsites/file-exists': 1.1.1 + mkdirp: 3.0.1 + transitivePeerDependencies: + - supports-color + + protobufjs@7.6.5: + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.5 + '@protobufjs/eventemitter': 1.1.1 + '@protobufjs/fetch': 1.1.1 + '@protobufjs/float': 1.0.2 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.2 + '@types/node': 25.9.5 + long: 5.3.2 + + pump@3.0.4: + dependencies: + end-of-stream: 1.4.5 + once: 1.4.0 + pure-rand@8.4.2: {} quansync@0.2.11: {} @@ -4371,18 +5491,42 @@ snapshots: pify: 4.0.1 strip-bom: 3.0.0 + readable-stream@2.3.8: + dependencies: + core-util-is: 1.0.3 + inherits: 2.0.4 + isarray: 1.0.0 + process-nextick-args: 2.0.1 + safe-buffer: 5.1.2 + string_decoder: 1.1.1 + util-deprecate: 1.0.2 + readable-stream@3.6.2: dependencies: inherits: 2.0.4 string_decoder: 1.3.0 util-deprecate: 1.0.2 + readable-stream@4.7.0: + dependencies: + abort-controller: 3.0.0 + buffer: 6.0.3 + events: 3.3.0 + process: 0.11.10 + string_decoder: 1.3.0 + + readdir-glob@1.1.3: + dependencies: + minimatch: 5.1.9 + redis-errors@1.2.0: {} redis-parser@3.0.0: dependencies: redis-errors: 1.2.0 + require-directory@2.1.1: {} + requirejs-config-file@4.0.0: dependencies: esprima: 4.0.1 @@ -4408,6 +5552,8 @@ snapshots: onetime: 5.1.2 signal-exit: 3.0.7 + retry@0.12.0: {} + reusify@1.1.0: {} rollup@4.62.2: @@ -4445,6 +5591,8 @@ snapshots: dependencies: queue-microtask: 1.2.3 + safe-buffer@5.1.2: {} + safe-buffer@5.2.1: {} safer-buffer@2.1.2: {} @@ -4484,8 +5632,27 @@ snapshots: cross-spawn: 7.0.6 signal-exit: 4.1.0 + split-ca@1.0.1: {} + + split2@4.2.0: {} + sprintf-js@1.0.3: {} + sql-escaper@1.5.1: {} + + ssh-remote-port-forward@1.0.4: + dependencies: + '@types/ssh2': 0.5.52 + ssh2: 1.17.0 + + ssh2@1.17.0: + dependencies: + asn1: 0.2.6 + bcrypt-pbkdf: 1.0.2 + optionalDependencies: + cpu-features: 0.0.10 + nan: 2.28.0 + stackback@0.0.2: {} standard-as-callback@2.1.0: {} @@ -4496,6 +5663,31 @@ snapshots: dependencies: any-promise: 1.3.0 + streamx@2.28.0: + dependencies: + events-universal: 1.0.1 + fast-fifo: 1.3.2 + text-decoder: 1.2.7 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + + string-width@4.2.3: + dependencies: + emoji-regex: 8.0.0 + is-fullwidth-code-point: 3.0.0 + strip-ansi: 6.0.1 + + string-width@5.1.2: + dependencies: + eastasianwidth: 0.2.0 + emoji-regex: 9.2.2 + strip-ansi: 7.2.0 + + string_decoder@1.1.1: + dependencies: + safe-buffer: 5.1.2 + string_decoder@1.3.0: dependencies: safe-buffer: 5.2.1 @@ -4510,6 +5702,10 @@ snapshots: dependencies: ansi-regex: 5.0.1 + strip-ansi@7.2.0: + dependencies: + ansi-regex: 6.2.2 + strip-bom@3.0.0: {} strip-json-comments@2.0.1: {} @@ -4528,8 +5724,82 @@ snapshots: tapable@2.3.3: {} + tar-fs@2.1.5: + dependencies: + chownr: 1.1.4 + mkdirp-classic: 0.5.3 + pump: 3.0.4 + tar-stream: 2.2.0 + + tar-fs@3.1.3: + dependencies: + pump: 3.0.4 + tar-stream: 3.2.0 + optionalDependencies: + bare-fs: 4.7.4 + bare-path: 3.1.1 + transitivePeerDependencies: + - bare-abort-controller + - bare-buffer + - react-native-b4a + + tar-stream@2.2.0: + dependencies: + bl: 4.1.0 + end-of-stream: 1.4.5 + fs-constants: 1.0.0 + inherits: 2.0.4 + readable-stream: 3.6.2 + + tar-stream@3.2.0: + dependencies: + b4a: 1.8.1 + bare-fs: 4.7.4 + fast-fifo: 1.3.2 + streamx: 2.28.0 + transitivePeerDependencies: + - bare-abort-controller + - bare-buffer + - react-native-b4a + + teex@1.0.1: + dependencies: + streamx: 2.28.0 + transitivePeerDependencies: + - bare-abort-controller + - react-native-b4a + term-size@2.2.1: {} + testcontainers@11.14.0: + dependencies: + '@balena/dockerignore': 1.0.2 + '@types/dockerode': 4.0.1 + archiver: 7.0.1 + async-lock: 1.4.1 + byline: 5.0.0 + debug: 4.4.3 + docker-compose: 1.4.2 + dockerode: 4.0.12 + get-port: 7.2.0 + proper-lockfile: 4.1.2 + properties-reader: 3.0.1 + ssh-remote-port-forward: 1.0.4 + tar-fs: 3.1.3 + tmp: 0.2.7 + undici: 7.29.0 + transitivePeerDependencies: + - bare-abort-controller + - bare-buffer + - react-native-b4a + - supports-color + + text-decoder@1.2.7: + dependencies: + b4a: 1.8.1 + transitivePeerDependencies: + - react-native-b4a + tinybench@2.9.0: {} tinyexec@1.2.4: {} @@ -4541,6 +5811,8 @@ snapshots: tinyrainbow@3.1.0: {} + tmp@0.2.7: {} + to-regex-range@5.0.1: dependencies: is-number: 7.0.0 @@ -4576,20 +5848,28 @@ snapshots: optionalDependencies: fsevents: 2.3.3 + tweetnacl@0.14.5: {} + typescript@5.9.3: {} typescript@6.0.3: {} unbash@4.0.3: {} + undici-types@5.26.5: {} + undici-types@7.24.6: {} + undici@7.29.0: {} + undici@8.8.0: {} universalify@0.1.2: {} util-deprecate@1.0.2: {} + uuid@10.0.0: {} + uuid@14.0.1: {} vite-plugin-wasm@3.6.0(vite@7.3.6(@types/node@25.9.5)(jiti@2.7.0)(tsx@4.20.6)(yaml@2.9.0)): @@ -4663,8 +5943,44 @@ snapshots: siginfo: 2.0.0 stackback: 0.0.2 + wrap-ansi@7.0.0: + dependencies: + ansi-styles: 4.3.0 + string-width: 4.2.3 + strip-ansi: 6.0.1 + + wrap-ansi@8.1.0: + dependencies: + ansi-styles: 6.2.3 + string-width: 5.1.2 + strip-ansi: 7.2.0 + + wrappy@1.0.2: {} + ws@8.21.1: {} + xtend@4.0.2: {} + + y18n@5.0.8: {} + yaml@2.9.0: {} + yargs-parser@21.1.1: {} + + yargs@17.7.3: + dependencies: + cliui: 8.0.1 + escalade: 3.2.0 + get-caller-file: 2.0.5 + require-directory: 2.1.1 + string-width: 4.2.3 + y18n: 5.0.8 + yargs-parser: 21.1.1 + + zip-stream@6.0.1: + dependencies: + archiver-utils: 5.0.2 + compress-commons: 6.0.2 + readable-stream: 4.7.0 + zod@4.4.3: {} From 203881d82922901d563641f3821c67639b6d3193 Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Sun, 26 Jul 2026 12:10:23 -0500 Subject: [PATCH 04/29] Stabilize SQLite relay claim planning --- packages/local-rpc/src/SqlPeerRelayStore.ts | 9 ++++++++- packages/local-rpc/test/PeerRelayStore.test.ts | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/local-rpc/src/SqlPeerRelayStore.ts b/packages/local-rpc/src/SqlPeerRelayStore.ts index 6db2590..8c3568b 100644 --- a/packages/local-rpc/src/SqlPeerRelayStore.ts +++ b/packages/local-rpc/src/SqlPeerRelayStore.ts @@ -834,6 +834,13 @@ const makeService = Effect.gen(function*() { }) } + const claimMessages = sql.onDialectOrElse({ + sqlite: () => + sql`effect_local_relay_messages m + INDEXED BY effect_local_relay_messages_claim_admission`, + orElse: () => sql`effect_local_relay_messages m` + }) + const findCandidate = SqlSchema.findOneOption({ Request: Schema.Struct({ tenantId: Schema.String, @@ -895,7 +902,7 @@ const makeService = Effect.gen(function*() { m.created_at AS "createdAt", m.next_eligible_at AS "nextEligibleAt", m.retry_count AS "retryCount" - FROM effect_local_relay_messages m + FROM ${claimMessages} JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id JOIN effect_local_relay_reservations r ON r.message_id = m.message_id WHERE m.tenant_id = ${request.tenantId} diff --git a/packages/local-rpc/test/PeerRelayStore.test.ts b/packages/local-rpc/test/PeerRelayStore.test.ts index ad0ca84..993004d 100644 --- a/packages/local-rpc/test/PeerRelayStore.test.ts +++ b/packages/local-rpc/test/PeerRelayStore.test.ts @@ -1097,6 +1097,7 @@ describe("PeerRelayStore", () => { EXPLAIN QUERY PLAN SELECT m.message_id FROM effect_local_relay_messages m + INDEXED BY effect_local_relay_messages_claim_admission JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id JOIN effect_local_relay_reservations r ON r.message_id = m.message_id WHERE m.tenant_id = 'tenant-plan' From a842bb6f6942d27d58892215dcb6ac56c2fe3ab8 Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Sun, 26 Jul 2026 12:50:47 -0500 Subject: [PATCH 05/29] Address durable relay review findings --- packages/local-rpc/src/PeerRelayIngress.ts | 4 +-- packages/local-rpc/src/PeerRelayLimits.ts | 35 +++++++++++++++++++ packages/local-rpc/src/SqlPeerRelayStore.ts | 13 +++---- .../src/internal/peerRelayMigrations.ts | 8 ----- .../src/internal/peerRpcObservability.ts | 2 +- .../local-rpc/test/PeerRelayLimits.test.ts | 32 +++++++++++++++++ packages/local-sql/src/PeerRelayOutbox.ts | 29 +++++++++------ .../local-sql/test/PeerRelayOutbox.test.ts | 32 +++++++++++++++++ 8 files changed, 125 insertions(+), 30 deletions(-) diff --git a/packages/local-rpc/src/PeerRelayIngress.ts b/packages/local-rpc/src/PeerRelayIngress.ts index d82daf3..51e1bb6 100644 --- a/packages/local-rpc/src/PeerRelayIngress.ts +++ b/packages/local-rpc/src/PeerRelayIngress.ts @@ -611,7 +611,7 @@ const makeServerProtocol = ( yield* socket.runRaw( () => Effect.void, { onOpen: write(new Socket.CloseEvent(1013)).pipe(Effect.ignore) } - ).pipe(Effect.catch(() => Effect.void)) + ).pipe(Effect.ignore) }) } connections++ @@ -676,7 +676,7 @@ const makeServerProtocol = ( }) ) } - ).pipe(Effect.catch(() => Effect.void)) + ).pipe(Effect.ignore) }).pipe( Effect.ensuring( Effect.gen(function*() { diff --git a/packages/local-rpc/src/PeerRelayLimits.ts b/packages/local-rpc/src/PeerRelayLimits.ts index 9a5ca1b..0cd8a53 100644 --- a/packages/local-rpc/src/PeerRelayLimits.ts +++ b/packages/local-rpc/src/PeerRelayLimits.ts @@ -365,11 +365,46 @@ const validate = (values: Values) => { values.maxInFlightSqlMaintenance <= values.maxInFlightSqlTransactions ], + [ + "maintenanceIntervalMillis", + [ + values.claimRecoveryBatchSize, + values.expiryBatchSize, + values.integrityBatchSize, + values.reconciliationBatchSize, + values.terminalCollectionBatchSize + ].every((batchSize) => batchSize * 1_000 / values.maintenanceIntervalMillis >= values.admissionRatePerSecond) + ], ["claimRecoveryRowsPerSecond", values.claimRecoveryRowsPerSecond >= values.admissionRatePerSecond], + [ + "claimRecoveryRowsPerSecond", + values.claimRecoveryRowsPerSecond >= + values.claimRecoveryBatchSize * 1_000 / values.maintenanceIntervalMillis + ], ["expiryRowsPerSecond", values.expiryRowsPerSecond >= values.admissionRatePerSecond], + [ + "expiryRowsPerSecond", + values.expiryRowsPerSecond >= + values.expiryBatchSize * 1_000 / values.maintenanceIntervalMillis + ], ["integrityRowsPerSecond", values.integrityRowsPerSecond >= values.admissionRatePerSecond], + [ + "integrityRowsPerSecond", + values.integrityRowsPerSecond >= + values.integrityBatchSize * 1_000 / values.maintenanceIntervalMillis + ], ["reconciliationRowsPerSecond", values.reconciliationRowsPerSecond >= values.admissionRatePerSecond], + [ + "reconciliationRowsPerSecond", + values.reconciliationRowsPerSecond >= + values.reconciliationBatchSize * 1_000 / values.maintenanceIntervalMillis + ], ["terminalCollectionRowsPerSecond", values.terminalCollectionRowsPerSecond >= values.admissionRatePerSecond], + [ + "terminalCollectionRowsPerSecond", + values.terminalCollectionRowsPerSecond >= + values.terminalCollectionBatchSize * 1_000 / values.maintenanceIntervalMillis + ], ["orphanChannelCleanupRowsPerSecond", values.orphanChannelCleanupRowsPerSecond >= values.admissionRatePerSecond], ["shutdownReleaseConcurrency", values.shutdownReleaseConcurrency <= values.maxInFlightSqlTransactions], ["shutdownReleaseTimeoutMillis", values.shutdownReleaseTimeoutMillis < values.claimLeaseMillis] diff --git a/packages/local-rpc/src/SqlPeerRelayStore.ts b/packages/local-rpc/src/SqlPeerRelayStore.ts index 8c3568b..10c3f40 100644 --- a/packages/local-rpc/src/SqlPeerRelayStore.ts +++ b/packages/local-rpc/src/SqlPeerRelayStore.ts @@ -111,10 +111,7 @@ const relayQuotaDomain = ( const recordQuotaRejection = (error: StoreError) => Option.match(relayQuotaDomain(error), { onNone: () => Effect.void, - onSome: (domain) => - PeerRpcObservability.recordRelayQuotaRejection(domain).pipe( - Effect.catchCause(() => Effect.void) - ) + onSome: (domain) => PeerRpcObservability.recordRelayQuotaRejection(domain).pipe(Effect.ignoreCause) }) const encodeKey = (...parts: ReadonlyArray) => JSON.stringify(parts) @@ -296,7 +293,7 @@ const UsageRow = Schema.Struct({ const KeyRow = Schema.Struct({ messageId: PositiveInt }) -const makeService = Effect.gen(function*() { +export const make = Effect.gen(function*() { const sql = (yield* SqlClient.SqlClient).withoutTransforms() const crypto = yield* Crypto.Crypto const limits = yield* PeerRelayLimits.PeerRelayLimits @@ -1864,7 +1861,7 @@ const makeService = Effect.gen(function*() { ? PeerRpcObservability.setRelayPending( value.activeCount, value.activeBytes - ).pipe(Effect.catchCause(() => Effect.void)) + ).pipe(Effect.ignoreCause) : Effect.void ) ) @@ -1886,8 +1883,6 @@ const makeService = Effect.gen(function*() { }) }) -export const make = makeService - export const layer: Layer.Layer< PeerRelayStore, | Migrator.MigrationError @@ -1895,4 +1890,4 @@ export const layer: Layer.Layer< | Schema.SchemaError | ReplicaError.ReplicaError, SqlClient.SqlClient | Crypto.Crypto | PeerRelayLimits.PeerRelayLimits -> = Layer.effect(PeerRelayStore, makeService) +> = Layer.effect(PeerRelayStore, make) diff --git a/packages/local-rpc/src/internal/peerRelayMigrations.ts b/packages/local-rpc/src/internal/peerRelayMigrations.ts index 6aba9f0..828a813 100644 --- a/packages/local-rpc/src/internal/peerRelayMigrations.ts +++ b/packages/local-rpc/src/internal/peerRelayMigrations.ts @@ -1,9 +1,7 @@ import * as Effect from "effect/Effect" -import * as Layer from "effect/Layer" import * as Schema from "effect/Schema" import * as Migrator from "effect/unstable/sql/Migrator" import * as SqlClient from "effect/unstable/sql/SqlClient" -import type * as SqlError from "effect/unstable/sql/SqlError" const executeStatements = ( sql: SqlClient.SqlClient, @@ -872,9 +870,3 @@ export const run = Effect.gen(function*() { orElse: () => genericRun }) }) - -export const layer: Layer.Layer< - never, - Migrator.MigrationError | SqlError.SqlError, - SqlClient.SqlClient -> = Layer.effectDiscard(run) diff --git a/packages/local-rpc/src/internal/peerRpcObservability.ts b/packages/local-rpc/src/internal/peerRpcObservability.ts index b729633..2660f38 100644 --- a/packages/local-rpc/src/internal/peerRpcObservability.ts +++ b/packages/local-rpc/src/internal/peerRpcObservability.ts @@ -274,7 +274,7 @@ export const recordSelectedDocuments = (amount: number) => const withoutMetricAttributes = (effect: Effect.Effect) => effect.pipe(Effect.provideService(Metric.CurrentMetricAttributes, {})) -const bestEffort = (effect: Effect.Effect) => effect.pipe(Effect.catchCause(() => Effect.void)) +const bestEffort = (effect: Effect.Effect) => effect.pipe(Effect.ignoreCause) const safeSpan = (span: Tracer.Span): Tracer.Span => { let status = span.status diff --git a/packages/local-rpc/test/PeerRelayLimits.test.ts b/packages/local-rpc/test/PeerRelayLimits.test.ts index ce23610..5832a59 100644 --- a/packages/local-rpc/test/PeerRelayLimits.test.ts +++ b/packages/local-rpc/test/PeerRelayLimits.test.ts @@ -167,4 +167,36 @@ describe("PeerRelayLimits", () => { }) ) )) + + it.effect("rejects a maintenance interval that exceeds the declared row rate", () => + Effect.gen(function*() { + const exit = yield* Effect.exit(PeerRelayLimits.make({ + ...PeerRelayLimits.defaults, + maintenanceIntervalMillis: 1 + })) + assert.strictEqual(exit._tag, "Failure") + if (exit._tag === "Failure") { + const error = yield* Effect.failCause(exit.cause).pipe(Effect.flip) + assert.strictEqual(error._tag, "InvalidPeerRelayLimits") + if (error._tag === "InvalidPeerRelayLimits") { + assert.strictEqual(error.field, "claimRecoveryRowsPerSecond") + } + } + })) + + it.effect("rejects a maintenance interval that cannot keep up with admission", () => + Effect.gen(function*() { + const exit = yield* Effect.exit(PeerRelayLimits.make({ + ...PeerRelayLimits.defaults, + maintenanceIntervalMillis: 2_000 + })) + assert.strictEqual(exit._tag, "Failure") + if (exit._tag === "Failure") { + const error = yield* Effect.failCause(exit.cause).pipe(Effect.flip) + assert.strictEqual(error._tag, "InvalidPeerRelayLimits") + if (error._tag === "InvalidPeerRelayLimits") { + assert.strictEqual(error.field, "maintenanceIntervalMillis") + } + } + })) }) diff --git a/packages/local-sql/src/PeerRelayOutbox.ts b/packages/local-sql/src/PeerRelayOutbox.ts index bfe95f7..14c33d9 100644 --- a/packages/local-sql/src/PeerRelayOutbox.ts +++ b/packages/local-sql/src/PeerRelayOutbox.ts @@ -124,6 +124,7 @@ const UsageRow = Schema.Struct({ encoded_bytes: NonNegativeInt }) const HorizonRow = Schema.Struct({ horizon_millis: Schema.NullOr(Schema.Number) }) +const replayRowBatchSize = 500 const replicaFailure = (reason: ReplicaError.Reason): ReplicaError.ReplicaError => new ReplicaError.ReplicaError({ reason }) @@ -253,11 +254,11 @@ const make = Effect.gen(function*() { LIMIT ${request.maximum}` }) - const findByRowId = SqlSchema.findAll({ + const findByRowIds = SqlSchema.findAll({ Request: Schema.Struct({ replicaId: Identity.ReplicaId, replicaIncarnation: Identity.ReplicaIncarnation, - rowId: PositiveInt + rowIds: Schema.Array(PositiveInt).check(Schema.isMinLength(1)) }), Result: Row, execute: (request) => @@ -265,7 +266,7 @@ const make = Effect.gen(function*() { FROM effect_local_peer_relay_outbox WHERE replica_id = ${request.replicaId} AND replica_incarnation = ${request.replicaIncarnation} - AND row_id = ${request.rowId}` + AND ${sql.in("row_id", request.rowIds)}` }) const insertRow = SqlSchema.findAll({ @@ -751,6 +752,18 @@ const make = Effect.gen(function*() { now, maximum: input.maximum }) + if (metadata.length === 0) return [] + const rows: Array = [] + for (let offset = 0; offset < metadata.length; offset += replayRowBatchSize) { + rows.push( + ...yield* findByRowIds({ + replicaId: permit.replicaId, + replicaIncarnation: permit.incarnation, + rowIds: metadata.slice(offset, offset + replayRowBatchSize).map((item) => item.row_id) + }) + ) + } + const rowsById = new Map(rows.map((row) => [row.row_id, row])) const entries: Array = [] for (const item of metadata) { if ( @@ -762,15 +775,11 @@ const make = Effect.gen(function*() { ) { return yield* storageCorrupt(new Error("Relay outbox payload length mismatch")) } - const rows = yield* findByRowId({ - replicaId: permit.replicaId, - replicaIncarnation: permit.incarnation, - rowId: item.row_id - }) - if (rows.length !== 1) { + const row = rowsById.get(item.row_id) + if (row === undefined) { return yield* storageCorrupt(new Error("Relay outbox row disappeared during replay")) } - const entry = yield* validateRow(rows[0]!, permit) + const entry = yield* validateRow(row, permit) if ( entry.expectedLocal.tenantId !== endpoint.expectedLocal.tenantId || entry.expectedLocal.subjectId !== endpoint.expectedLocal.subjectId || diff --git a/packages/local-sql/test/PeerRelayOutbox.test.ts b/packages/local-sql/test/PeerRelayOutbox.test.ts index 02c7970..c345ce7 100644 --- a/packages/local-sql/test/PeerRelayOutbox.test.ts +++ b/packages/local-sql/test/PeerRelayOutbox.test.ts @@ -12,6 +12,7 @@ import * as Effect from "effect/Effect" import * as Layer from "effect/Layer" import * as Schema from "effect/Schema" import { TestClock } from "effect/testing" +import * as Tracer from "effect/Tracer" import * as SqlClient from "effect/unstable/sql/SqlClient" import { mkdtempSync, rmSync } from "node:fs" import { tmpdir } from "node:os" @@ -286,6 +287,37 @@ describe("PeerRelayOutbox", () => { }) }).pipe(Effect.provide(layer(":memory:")))) + it.effect("loads replay rows in bounded batches instead of one query per row", () => + Effect.gen(function*() { + yield* insertDocument + const outbox = yield* PeerRelayOutbox.PeerRelayOutbox + for (let sequence = 1; sequence <= 8; sequence++) { + yield* outbox.admit({ + ...endpoint, + payload: yield* makePayload(sequence), + retryHorizonMillis: 30_000 + }) + } + + let queryCount = 0 + const tracer = Tracer.make({ + span: (options) => { + if (options.name === "sql.execute") queryCount++ + return new Tracer.NativeSpan(options) + } + }) + const entries = yield* outbox.dueForEndpoint({ ...endpoint, maximum: 8 }).pipe( + Effect.provideService(Tracer.Tracer, tracer) + ) + + assert.strictEqual(entries.length, 8) + assert.isAtMost(queryCount, 2) + }).pipe(Effect.provide(layer(":memory:", { + ...outboxLimits, + maxMessagesPerRemote: 8, + maxMessagesPerReplica: 8 + })))) + it.effect("fences every replay operation after the replica incarnation changes", () => Effect.gen(function*() { yield* insertDocument From 1170faa56297a1d22dfb6544e82bd6d007575ef6 Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Sun, 26 Jul 2026 13:14:28 -0500 Subject: [PATCH 06/29] Handle Effect errors at call sites --- RULES.md | 2 + .../local-rpc/src/PeerRelayAuthorization.ts | 33 +- packages/local-rpc/src/PeerRelayIngress.ts | 47 +- packages/local-rpc/src/SqlPeerRelayStore.ts | 338 ++++++++++++-- .../src/internal/peerRelayStoreErrors.ts | 46 -- .../test/PeerRelayStoreErrors.test.ts | 41 -- packages/local-sql/src/Compaction.ts | 83 +++- packages/local-sql/src/PeerRelayOutbox.ts | 213 ++++++--- packages/local-sql/src/PeerSync.ts | 416 ++++++++++++++---- packages/local-sql/src/ReplicaBootstrap.ts | 21 +- 10 files changed, 919 insertions(+), 321 deletions(-) delete mode 100644 packages/local-rpc/src/internal/peerRelayStoreErrors.ts delete mode 100644 packages/local-rpc/test/PeerRelayStoreErrors.test.ts diff --git a/RULES.md b/RULES.md index cf2e34c..1a78ea6 100644 --- a/RULES.md +++ b/RULES.md @@ -25,6 +25,8 @@ Read this file before any work. Treat these rules as required for every package. - Define every library owned typed error with `Schema.TaggedErrorClass(identifier)("Tag", fields)`. - Give every typed failure a stable `_tag`. Do not add untagged or `Data.TaggedError` library error types. - Discriminate tagged errors with `_tag`. Do not use `instanceof` for error discrimination. +- Do not define shared error transformers such as `toSomeError`, `mapErrors`, or helpers that accept `unknown` and return another error. Handle each inferred failure directly at the call site with explicit `_tag` based `Effect.catchTag`, `catchTags`, `catchReason`, or `catchReasons` branches. Use `catchCause` at the call site only when the full Cause must be preserved. Accept small duplication so error channels stay narrow and every handled failure remains explicit. +- Do not use generic `E` intersections, structural `_tag` probes, `typeof error === "object"`, or property existence checks to claim that an Effect failure is a specific tagged error. Let the Effect error channel prove the tag and use the matching typed catch combinator. If the combinator does not typecheck, the call site has not proved that it can fail with that error. - When a consumer error can share an infrastructure `_tag`, use the infrastructure package's precise guard instead of `catchTag`. - Use `Effect.catchTag` or `catchTags` for `_tag` failures. Use `catchReason` or `catchReasons` for nested tagged reasons. Use `catchIf`, `catchFilter`, or a specialized combinator for other selected typed failures. - Use `Effect.catch` only when one handler intentionally covers the complete typed failure channel. It does not catch defects or interruption. diff --git a/packages/local-rpc/src/PeerRelayAuthorization.ts b/packages/local-rpc/src/PeerRelayAuthorization.ts index 3bb42e9..f613cd8 100644 --- a/packages/local-rpc/src/PeerRelayAuthorization.ts +++ b/packages/local-rpc/src/PeerRelayAuthorization.ts @@ -87,23 +87,13 @@ export class PeerRelayAuthorization extends Context.Service()("@lucas-barake/effect-local-rpc/PeerRelayAuthorization") {} -const accessDeniedOnSchemaError = (effect: Effect.Effect) => - effect.pipe( - Effect.catchIf( - (error): error is E & { readonly _tag: "SchemaError" } => - typeof error === "object" && error !== null && "_tag" in error && error._tag === "SchemaError", - () => new PeerRpcError.AccessDenied() - ) - ) - const validateRequestShape = (request: Request) => - accessDeniedOnSchemaError( - Effect.all([ - Direction.makeEffect(request.direction), - PeerAuthentication.PeerPrincipal.makeEffect(request.principal), - RemotePeer.makeEffect(request.remote) - ]) - ).pipe( + Effect.all([ + Direction.makeEffect(request.direction), + PeerAuthentication.PeerPrincipal.makeEffect(request.principal), + RemotePeer.makeEffect(request.remote) + ]).pipe( + Effect.catchTag("SchemaError", () => new PeerRpcError.AccessDenied()), Effect.flatMap(() => validateRequest(request.documents)), Effect.flatMap((requested) => new Set(request.documents.map((document) => document.documentId)).size === @@ -114,7 +104,8 @@ const validateRequestShape = (request: Request) => ) const validateResult = (request: Request, result: Result) => - accessDeniedOnSchemaError(PeerAuthentication.PeerPrincipal.makeEffect(result.remote)).pipe( + PeerAuthentication.PeerPrincipal.makeEffect(result.remote).pipe( + Effect.catchTag("SchemaError", () => new PeerRpcError.AccessDenied()), Effect.flatMap((remote) => remote.tenantId === request.principal.tenantId && remote.subjectId === request.remote.subjectId && @@ -149,7 +140,8 @@ const canonicalDocuments = ( }) const validateUnsafeRequest = (request: UnsafeUnboundedAutomerge3DecodeRequest) => - accessDeniedOnSchemaError(UnsafeUnboundedAutomerge3DecodeRequest.makeEffect(request)).pipe( + UnsafeUnboundedAutomerge3DecodeRequest.makeEffect(request).pipe( + Effect.catchTag("SchemaError", () => new PeerRpcError.AccessDenied()), Effect.flatMap((validated) => validateUnsafeDocuments(validated.documents).pipe(Effect.as(validated))) ) @@ -157,9 +149,8 @@ const validateUnsafeGrant = ( request: UnsafeUnboundedAutomerge3DecodeRequest, result: UnsafeUnboundedAutomerge3DecodeGrant ) => - accessDeniedOnSchemaError( - Schema.decodeUnknownEffect(UnsafeUnboundedAutomerge3DecodeGrant)(result) - ).pipe( + Schema.decodeUnknownEffect(UnsafeUnboundedAutomerge3DecodeGrant)(result).pipe( + Effect.catchTag("SchemaError", () => new PeerRpcError.AccessDenied()), Effect.flatMap((grant) => Effect.gen(function*() { const requested = yield* validateUnsafeDocuments(request.documents) diff --git a/packages/local-rpc/src/PeerRelayIngress.ts b/packages/local-rpc/src/PeerRelayIngress.ts index 51e1bb6..5f1b978 100644 --- a/packages/local-rpc/src/PeerRelayIngress.ts +++ b/packages/local-rpc/src/PeerRelayIngress.ts @@ -724,11 +724,6 @@ export const layerProtocolSocketServer = ( }) ) as Layer.Layer -const toClientError = (message: string, cause: unknown) => - new RpcClientError.RpcClientError({ - reason: new RpcClientError.RpcClientDefect({ message, cause }) - }) - export const makeProtocolSocket = Effect.gen(function*() { const socket = yield* Socket.Socket const limits = yield* PeerRelayLimits @@ -784,7 +779,12 @@ export const makeProtocolSocket = Effect.gen(function*() { ), Effect.tapCause((cause) => { if (Cause.hasInterruptsOnly(cause)) return Effect.void - currentError = toClientError("Relay socket protocol failed", Cause.squash(cause)) + currentError = new RpcClientError.RpcClientError({ + reason: new RpcClientError.RpcClientDefect({ + message: "Relay socket protocol failed", + cause: Cause.squash(cause) + }) + }) return Effect.forEach( clientIds, (clientId) => @@ -812,20 +812,47 @@ export const makeProtocolSocket = Effect.gen(function*() { try { encoded = encodeFrame(request, limits.maximumDeclaredFrameBytes) } catch (cause) { - return yield* Effect.fail(toClientError("Relay request frame is invalid", cause)) + return yield* new RpcClientError.RpcClientError({ + reason: new RpcClientError.RpcClientDefect({ + message: "Relay request frame is invalid", + cause + }) + }) } yield* reservation.shrinkTo(encoded.bodyBytes) if (request._tag === "Request") requestClients.set(request.id, clientId) yield* write(encoded.frame).pipe( - Effect.mapError((cause) => toClientError("Relay socket write failed", cause)) + Effect.catchTag("SocketError", (cause) => + Effect.fail( + new RpcClientError.RpcClientError({ + reason: new RpcClientError.RpcClientDefect({ + message: "Relay socket write failed", + cause + }) + }) + )) ) }) ).pipe( Effect.catchTags({ RequestLimitExceeded: (cause) => - Effect.fail(toClientError("Relay request capacity is exhausted", cause)), + Effect.fail( + new RpcClientError.RpcClientError({ + reason: new RpcClientError.RpcClientDefect({ + message: "Relay request capacity is exhausted", + cause + }) + }) + ), RequestCapacityExceeded: (cause) => - Effect.fail(toClientError("Relay request capacity is exhausted", cause)) + Effect.fail( + new RpcClientError.RpcClientError({ + reason: new RpcClientError.RpcClientDefect({ + message: "Relay request capacity is exhausted", + cause + }) + }) + ) }) ) }), diff --git a/packages/local-rpc/src/SqlPeerRelayStore.ts b/packages/local-rpc/src/SqlPeerRelayStore.ts index 10c3f40..eeac994 100644 --- a/packages/local-rpc/src/SqlPeerRelayStore.ts +++ b/packages/local-rpc/src/SqlPeerRelayStore.ts @@ -1,5 +1,6 @@ import * as Identity from "@lucas-barake/effect-local/Identity" import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" +import * as Cause from "effect/Cause" import * as Crypto from "effect/Crypto" import * as Effect from "effect/Effect" import * as Exit from "effect/Exit" @@ -13,7 +14,6 @@ import type * as SqlError from "effect/unstable/sql/SqlError" import * as SqlSchema from "effect/unstable/sql/SqlSchema" import * as PeerRelayMigrations from "./internal/peerRelayMigrations.js" import { make as makeWriteTransaction } from "./internal/peerRelaySqlTransaction.js" -import { mapStoreErrors } from "./internal/peerRelayStoreErrors.js" import * as PeerRpcObservability from "./internal/peerRpcObservability.js" import * as PeerRelayLimits from "./PeerRelayLimits.js" import { @@ -56,11 +56,6 @@ interface UsageScope { readonly retainedBytesLimit: number } -const storageCorrupt = (cause: unknown) => - new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ cause }) - }) - const protocolMismatch = (expected: string, observed: string) => new ReplicaError.ReplicaError({ reason: new ReplicaError.ProtocolMismatch({ expected, observed }) @@ -183,7 +178,12 @@ const decodeDocuments = Schema.decodeUnknownEffect( const parseDocuments = (value: string) => decodeDocuments(value).pipe( - Effect.mapError((cause) => storageCorrupt(cause)) + Effect.catchTag("SchemaError", (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + )) ) const validateInput = (schema: S, input: unknown) => @@ -211,17 +211,35 @@ const checkDurability = ( execute: () => sql`PRAGMA synchronous` }) const mode = yield* journal(undefined).pipe( - Effect.catchTag("NoSuchElementError", (cause) => Effect.fail(storageCorrupt(cause))) + Effect.catchTag("NoSuchElementError", (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + )) ) if (mode.journal_mode.toLowerCase() !== "wal") { - return yield* storageCorrupt(new Error("Relay custody requires SQLite WAL mode")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Relay custody requires SQLite WAL mode") + }) + }) } yield* sql`PRAGMA synchronous = FULL` const setting = yield* synchronous(undefined).pipe( - Effect.catchTag("NoSuchElementError", (cause) => Effect.fail(storageCorrupt(cause))) + Effect.catchTag("NoSuchElementError", (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + )) ) if (setting.synchronous !== 2) { - return yield* storageCorrupt(new Error("Relay custody requires SQLite FULL synchronous mode")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Relay custody requires SQLite FULL synchronous mode") + }) + }) } }), orElse: () => Effect.void @@ -470,7 +488,11 @@ export const make = Effect.gen(function*() { Effect.gen(function*() { const reservationOption = yield* findReservation(messageId) if (Option.isNone(reservationOption)) { - return yield* storageCorrupt(new Error("Missing relay quota reservation")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Missing relay quota reservation") + }) + }) } const reservation = reservationOption.value if (reservation.activeConsumed === 1) { @@ -500,7 +522,11 @@ export const make = Effect.gen(function*() { })(undefined) ) if (changed !== 1) { - return yield* storageCorrupt(new Error("Invalid active relay quota reservation")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Invalid active relay quota reservation") + }) + }) } yield* sql`DELETE FROM effect_local_relay_usage WHERE scope_kind = ${kind} @@ -521,7 +547,11 @@ export const make = Effect.gen(function*() { Effect.gen(function*() { const reservationOption = yield* findReservation(messageId) if (Option.isNone(reservationOption)) { - return yield* storageCorrupt(new Error("Missing relay quota reservation")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Missing relay quota reservation") + }) + }) } const reservation = reservationOption.value if (reservation.retainedConsumed === 1) { @@ -551,7 +581,11 @@ export const make = Effect.gen(function*() { })(undefined) ) if (changed !== 1) { - return yield* storageCorrupt(new Error("Invalid retained relay quota reservation")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Invalid retained relay quota reservation") + }) + }) } yield* sql`DELETE FROM effect_local_relay_usage WHERE scope_kind = ${kind} @@ -654,7 +688,7 @@ export const make = Effect.gen(function*() { const admit: Service["admit"] = (unsafeInput) => { let observedBytes: number | undefined let observedVersion: number | undefined - const effect = mapStoreErrors(Effect.gen(function*() { + const effect = Effect.gen(function*() { const input = yield* validateInput(Admission, unsafeInput) const payloadBytes = input.payload.byteLength observedBytes = payloadBytes @@ -814,7 +848,27 @@ export const make = Effect.gen(function*() { lane: "New" }) })) - })).pipe(Effect.tapError(recordQuotaRejection)) + }).pipe( + Effect.catchCause((cause) => + Effect.failCause(Cause.map(cause, (error) => { + switch (error._tag) { + case "SqlError": + case "NestedPeerRelayTransactionError": + return new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause: error }) + }) + case "SchemaError": + case "NoSuchElementError": + return new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause: error }) + }) + default: + return error + } + })) + ), + Effect.tapError(recordQuotaRejection) + ) return PeerRpcObservability.observeRelay({ effect, operation: "RelayAdmit", @@ -941,7 +995,7 @@ export const make = Effect.gen(function*() { const claim: Service["claim"] = (unsafeInput) => { let observedAttempt: number | undefined - const effect = mapStoreErrors(Effect.gen(function*() { + const effect = Effect.gen(function*() { const input = yield* validateInput(ClaimRequest, unsafeInput) return yield* write(Effect.gen(function*() { const now = (yield* nowQuery(sql)).now @@ -991,7 +1045,11 @@ export const make = Effect.gen(function*() { })(undefined) ) if (claimed !== 1) { - return yield* storageCorrupt(new Error("Relay claim compare and swap failed")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Relay claim compare and swap failed") + }) + }) } const channelClaimed = yield* mutationCount( sql`UPDATE effect_local_relay_channels @@ -1016,7 +1074,11 @@ export const make = Effect.gen(function*() { })(undefined) ) if (channelClaimed !== 1) { - return yield* storageCorrupt(new Error("Relay channel claim compare and swap failed")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Relay channel claim compare and swap failed") + }) + }) } return { message: Option.some(ClaimedMessage.make({ @@ -1047,7 +1109,25 @@ export const make = Effect.gen(function*() { lane: candidate.retryCount === 0 ? "New" as const : "Retry" as const } })) - })) + }).pipe(Effect.catchCause((cause) => + Effect.failCause(Cause.map(cause, (error) => { + switch (error._tag) { + case "SqlError": + case "PlatformError": + case "NestedPeerRelayTransactionError": + return new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause: error }) + }) + case "SchemaError": + case "NoSuchElementError": + return new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause: error }) + }) + default: + return error + } + })) + )) return PeerRpcObservability.observeRelay({ effect, operation: "RelayClaim", @@ -1072,7 +1152,7 @@ export const make = Effect.gen(function*() { } const loadClaimedPayload: Service["loadClaimedPayload"] = (unsafeInput) => - mapStoreErrors(Effect.gen(function*() { + Effect.gen(function*() { const input = yield* validateInput(LoadClaimedPayloadRequest, unsafeInput) const now = (yield* nowQuery(sql)).now const rows = yield* SqlSchema.findAll({ @@ -1111,6 +1191,25 @@ export const make = Effect.gen(function*() { return yield* protocolMismatch("active relay claim", "stale relay claim") } return rows[0]!.payload + }).pipe(Effect.catchTags({ + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ), + NoSuchElementError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) })) const TerminalRow = Schema.Struct({ @@ -1227,7 +1326,7 @@ export const make = Effect.gen(function*() { reason: string, onChanged?: (latencyMillis: number) => void ): Effect.Effect => - mapStoreErrors(Effect.gen(function*() { + Effect.gen(function*() { const input = yield* validateInput(TerminalRequest, unsafeInput) if ( input.recipient.tenantId !== input.channel.tenantId || @@ -1295,7 +1394,24 @@ export const make = Effect.gen(function*() { lane: "New" } as const })) - })) + }).pipe(Effect.catchCause((cause) => + Effect.failCause(Cause.map(cause, (error) => { + switch (error._tag) { + case "SqlError": + case "NestedPeerRelayTransactionError": + return new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause: error }) + }) + case "SchemaError": + case "NoSuchElementError": + return new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause: error }) + }) + default: + return error + } + })) + )) const acknowledge: Service["acknowledge"] = (input) => { let latencyMillis: number | undefined @@ -1329,7 +1445,7 @@ export const make = Effect.gen(function*() { } const reject: Service["reject"] = (unsafeInput) => { - const effect = mapStoreErrors(Effect.gen(function*() { + const effect = Effect.gen(function*() { const input = yield* validateInput(RejectRequest, unsafeInput) return yield* terminalTransition( TerminalRequest.make({ @@ -1343,7 +1459,7 @@ export const make = Effect.gen(function*() { "DeadLettered", input.reason ) - })) + }) return PeerRpcObservability.observeRelay({ effect, operation: "RelayAcknowledge", @@ -1361,7 +1477,7 @@ export const make = Effect.gen(function*() { } const release: Service["release"] = (unsafeInput) => { - const effect = mapStoreErrors(Effect.gen(function*() { + const effect = Effect.gen(function*() { const input = yield* validateInput(ReleaseRequest, unsafeInput) return yield* write(Effect.gen(function*() { const now = (yield* nowQuery(sql)).now @@ -1449,7 +1565,24 @@ export const make = Effect.gen(function*() { lane: "Retry" } as const })) - })) + }).pipe(Effect.catchCause((cause) => + Effect.failCause(Cause.map(cause, (error) => { + switch (error._tag) { + case "SqlError": + case "NestedPeerRelayTransactionError": + return new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause: error }) + }) + case "SchemaError": + case "NoSuchElementError": + return new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause: error }) + }) + default: + return error + } + })) + )) return PeerRpcObservability.observeRelay({ effect, operation: "RelayRelease", @@ -1482,7 +1615,7 @@ export const make = Effect.gen(function*() { const recover: Service["recover"] = (unsafeInput) => Effect.suspend(() => { let deadLettered = 0 - const effect = mapStoreErrors(Effect.gen(function*() { + const effect = Effect.gen(function*() { const input = yield* validateInput(MaintenanceRequest, unsafeInput) const effectiveBatch = Math.min(input.batchSize, limits.claimRecoveryBatchSize) return yield* write(Effect.gen(function*() { @@ -1543,7 +1676,24 @@ export const make = Effect.gen(function*() { } return maintenanceResult(rows.map((row) => row.messageId), effectiveBatch) })) - })) + }).pipe(Effect.catchCause((cause) => + Effect.failCause(Cause.map(cause, (error) => { + switch (error._tag) { + case "SqlError": + case "NestedPeerRelayTransactionError": + return new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause: error }) + }) + case "SchemaError": + case "NoSuchElementError": + return new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause: error }) + }) + default: + return error + } + })) + )) return PeerRpcObservability.observeRelay({ effect, operation: "RelayMaintenance", @@ -1560,7 +1710,7 @@ export const make = Effect.gen(function*() { }) const expire: Service["expire"] = (unsafeInput) => { - const effect = mapStoreErrors(Effect.gen(function*() { + const effect = Effect.gen(function*() { const input = yield* validateInput(MaintenanceRequest, unsafeInput) const effectiveBatch = Math.min(input.batchSize, limits.expiryBatchSize) return yield* write(Effect.gen(function*() { @@ -1581,7 +1731,24 @@ export const make = Effect.gen(function*() { } return maintenanceResult(rows.map((row) => row.messageId), effectiveBatch) })) - })) + }).pipe(Effect.catchCause((cause) => + Effect.failCause(Cause.map(cause, (error) => { + switch (error._tag) { + case "SqlError": + case "NestedPeerRelayTransactionError": + return new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause: error }) + }) + case "SchemaError": + case "NoSuchElementError": + return new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause: error }) + }) + default: + return error + } + })) + )) return PeerRpcObservability.observeRelay({ effect, operation: "RelayMaintenance", @@ -1615,7 +1782,7 @@ export const make = Effect.gen(function*() { }) const repair: Service["repair"] = (unsafeInput) => { - const effect = mapStoreErrors(Effect.gen(function*() { + const effect = Effect.gen(function*() { const input = yield* validateInput(MaintenanceRequest, unsafeInput) const effectiveBatch = Math.min(input.batchSize, limits.integrityBatchSize) return yield* write(Effect.gen(function*() { @@ -1655,7 +1822,11 @@ export const make = Effect.gen(function*() { row.activeConsumed === null || row.retainedConsumed === null ) { - return yield* storageCorrupt(new Error("Relay reservation reconciliation required")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Relay reservation reconciliation required") + }) + }) } const active = row.state === "Pending" || row.state === "Claimed" const terminal = row.state === "Acknowledged" || @@ -1665,7 +1836,11 @@ export const make = Effect.gen(function*() { (active && row.activeConsumed !== 0) || (terminal && row.activeConsumed !== 1) ) { - return yield* storageCorrupt(new Error("Corrupt relay reservation entitlement")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Corrupt relay reservation entitlement") + }) + }) } const corrupt = (!active && !terminal) || row.messageTenantId !== row.channelTenantId || @@ -1684,12 +1859,33 @@ export const make = Effect.gen(function*() { if (corrupt && active) { yield* terminalize(row.messageId, "DeadLettered", now, { reason: "Corrupt" }) } else if (corrupt) { - return yield* storageCorrupt(new Error("Corrupt terminal relay row")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Corrupt terminal relay row") + }) + }) } } return maintenanceResult(rows.map((row) => row.messageId), effectiveBatch) })) - })) + }).pipe(Effect.catchCause((cause) => + Effect.failCause(Cause.map(cause, (error) => { + switch (error._tag) { + case "SqlError": + case "NestedPeerRelayTransactionError": + return new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause: error }) + }) + case "SchemaError": + case "NoSuchElementError": + return new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause: error }) + }) + default: + return error + } + })) + )) return PeerRpcObservability.observeRelay({ effect, operation: "RelayMaintenance", @@ -1703,7 +1899,7 @@ export const make = Effect.gen(function*() { } const reconcile: Service["reconcile"] = (unsafeInput) => { - const effect = mapStoreErrors(Effect.gen(function*() { + const effect = Effect.gen(function*() { const input = yield* validateInput(MaintenanceRequest, unsafeInput) const effectiveBatch = Math.min(input.batchSize, limits.reconciliationBatchSize) return yield* write(Effect.gen(function*() { @@ -1731,11 +1927,31 @@ export const make = Effect.gen(function*() { messageRows.length !== reservationRows.length || messageRows.some((row, index) => row.messageId !== reservationRows[index]?.messageId) ) { - return yield* storageCorrupt(new Error("Relay reservation bijection is corrupt")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Relay reservation bijection is corrupt") + }) + }) } return maintenanceResult(messageRows.map((row) => row.messageId), effectiveBatch) })) - })) + }).pipe(Effect.catchCause((cause) => + Effect.failCause(Cause.map(cause, (error) => { + switch (error._tag) { + case "SqlError": + case "NestedPeerRelayTransactionError": + return new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause: error }) + }) + case "SchemaError": + return new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause: error }) + }) + default: + return error + } + })) + )) return PeerRpcObservability.observeRelay({ effect, operation: "RelayMaintenance", @@ -1749,7 +1965,7 @@ export const make = Effect.gen(function*() { } const collect: Service["collect"] = (unsafeInput) => { - const effect = mapStoreErrors(Effect.gen(function*() { + const effect = Effect.gen(function*() { const input = yield* validateInput(MaintenanceRequest, unsafeInput) const effectiveBatch = Math.min(input.batchSize, limits.terminalCollectionBatchSize) return yield* write(Effect.gen(function*() { @@ -1789,7 +2005,11 @@ export const make = Effect.gen(function*() { })(undefined) ) if (reservationDeleted !== 1) { - return yield* storageCorrupt(new Error("Invalid collected relay quota reservation")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Invalid collected relay quota reservation") + }) + }) } yield* sql`DELETE FROM effect_local_relay_messages WHERE message_id = ${row.messageId} @@ -1806,7 +2026,24 @@ export const make = Effect.gen(function*() { } return maintenanceResult(rows.map((row) => row.messageId), effectiveBatch) })) - })) + }).pipe(Effect.catchCause((cause) => + Effect.failCause(Cause.map(cause, (error) => { + switch (error._tag) { + case "SqlError": + case "NestedPeerRelayTransactionError": + return new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause: error }) + }) + case "SchemaError": + case "NoSuchElementError": + return new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause: error }) + }) + default: + return error + } + })) + )) return PeerRpcObservability.observeRelay({ effect, operation: "RelayMaintenance", @@ -1821,7 +2058,7 @@ export const make = Effect.gen(function*() { const usage: Service["usage"] = (unsafeInput) => { const exactShard = unsafeInput === undefined - const effect = mapStoreErrors(Effect.gen(function*() { + const effect = Effect.gen(function*() { const input = unsafeInput === undefined ? UsageRequest.make({ scopeKind: "Shard", scopeKey: encodeKey("local") }) : yield* validateInput(UsageRequest, unsafeInput) @@ -1844,6 +2081,19 @@ export const make = Effect.gen(function*() { retainedCount: 0, retainedBytes: 0 })) + }).pipe(Effect.catchTags({ + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) })) return PeerRpcObservability.observeRelay({ effect, diff --git a/packages/local-rpc/src/internal/peerRelayStoreErrors.ts b/packages/local-rpc/src/internal/peerRelayStoreErrors.ts deleted file mode 100644 index 3907906..0000000 --- a/packages/local-rpc/src/internal/peerRelayStoreErrors.ts +++ /dev/null @@ -1,46 +0,0 @@ -import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" -import * as Cause from "effect/Cause" -import * as Effect from "effect/Effect" -import type * as PlatformError from "effect/PlatformError" -import type * as Schema from "effect/Schema" -import type * as SqlError from "effect/unstable/sql/SqlError" -import type { NestedPeerRelayTransactionError } from "./peerRelaySqlTransaction.js" - -export type StoreBoundaryError = - | ReplicaError.ReplicaError - | NestedPeerRelayTransactionError - | SqlError.SqlError - | Schema.SchemaError - | PlatformError.PlatformError - | Cause.NoSuchElementError - -const storageUnavailable = (cause: unknown) => - new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageUnavailable({ cause }) - }) - -const storageCorrupt = (cause: unknown) => - new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ cause }) - }) - -export const mapStoreErrors = ( - effect: Effect.Effect -) => - Effect.catchCause( - effect, - (cause) => - Effect.failCause(Cause.map(cause, (error) => { - switch (error._tag) { - case "SqlError": - case "PlatformError": - case "NestedPeerRelayTransactionError": - return storageUnavailable(error) - case "SchemaError": - case "NoSuchElementError": - return storageCorrupt(error) - default: - return error - } - })) - ) diff --git a/packages/local-rpc/test/PeerRelayStoreErrors.test.ts b/packages/local-rpc/test/PeerRelayStoreErrors.test.ts deleted file mode 100644 index 6e1dcd7..0000000 --- a/packages/local-rpc/test/PeerRelayStoreErrors.test.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { assert, describe, it } from "@effect/vitest" -import * as Cause from "effect/Cause" -import * as Effect from "effect/Effect" -import * as Exit from "effect/Exit" -import * as SqlError from "effect/unstable/sql/SqlError" -import { mapStoreErrors } from "../src/internal/peerRelayStoreErrors.js" - -describe("PeerRelayStore errors", () => { - it.effect("maps typed SQL failures while preserving rollback defects", () => - Effect.gen(function*() { - const sqlFailure = new SqlError.SqlError({ - reason: new SqlError.UnknownError({ cause: new Error("commit failed") }) - }) - const rollbackDefect = new Error("rollback failed") - const exit = yield* mapStoreErrors( - Effect.failCause( - Cause.combine( - Cause.fail(sqlFailure), - Cause.die(rollbackDefect) - ) - ) - ).pipe(Effect.exit) - assert.strictEqual(Exit.isFailure(exit), true) - if (Exit.isFailure(exit)) { - const failures = exit.cause.reasons.filter(Cause.isFailReason) - assert.strictEqual(failures.length, 1) - const mapped = failures[0]!.error - assert.strictEqual(mapped._tag, "ReplicaError") - if (mapped._tag === "ReplicaError") { - assert.strictEqual(mapped.reason._tag, "StorageUnavailable") - if (mapped.reason._tag === "StorageUnavailable") { - assert.strictEqual(mapped.reason.cause, sqlFailure) - } - } - assert.strictEqual( - exit.cause.reasons.some((reason) => Cause.isDieReason(reason) && reason.defect === rollbackDefect), - true - ) - } - })) -}) diff --git a/packages/local-sql/src/Compaction.ts b/packages/local-sql/src/Compaction.ts index 4da6734..3c19fb9 100644 --- a/packages/local-sql/src/Compaction.ts +++ b/packages/local-sql/src/Compaction.ts @@ -204,20 +204,6 @@ export class Compaction extends Context.Service - Effect.fail( - new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageUnavailable({ cause }) - }) - ) - -const failStorageCorrupt = (cause: unknown) => - Effect.fail( - new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ cause }) - }) - ) - export const layer: Layer.Layer< Compaction, never, @@ -356,7 +342,12 @@ export const layer: Layer.Layer< // durable is wrong, the operation simply could not start. const makeLineage = Identity.makeDocumentLineage.pipe( Effect.provideService(Crypto.Crypto, crypto), - Effect.catchTag("PlatformError", failStorageUnavailable) + Effect.catchTag("PlatformError", (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + )) ) const findDefinitionHash = SqlSchema.findOne({ Request: Schema.Void, @@ -885,7 +876,20 @@ export const layer: Layer.Layer< if (removed < receiptPruneBatchSize) return deleted } }).pipe( - Effect.catchTags({ SqlError: failStorageUnavailable, SchemaError: failStorageCorrupt }), + Effect.catchTags({ + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) + }), Effect.tap((deleted) => Effect.annotateCurrentSpan({ "receipts.pruned": deleted })), Effect.withSpan("Compaction.pruneCommandReceipts", { attributes: { "receipts.batch_size": receiptPruneBatchSize } @@ -1170,9 +1174,11 @@ export const layer: Layer.Layer< const rootChanges = InternalAutomerge.changesSince(rebuilt, []) const rootChange = rootChanges[0] if (rootChanges.length !== 1 || rootChange === undefined) { - return yield* failStorageCorrupt( - new Error(`Re-rooted document produced ${rootChanges.length} changes instead of one`) - ) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error(`Re-rooted document produced ${rootChanges.length} changes instead of one`) + }) + }) } const definitionHash = yield* findDefinitionHash(undefined).pipe( Effect.map((row) => row.definition_hash), @@ -1334,7 +1340,11 @@ export const layer: Layer.Layer< for (const entry of usage.values()) { const updated = yield* decrementRelayReceiptUsage(entry) if (updated.length !== 1) { - return yield* failStorageCorrupt(new Error("Relay receipt usage is inconsistent")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Relay receipt usage is inconsistent") + }) + }) } if (updated[0]!.receipt_count === 0) { zeroUsage.push(entry) @@ -1345,13 +1355,21 @@ export const layer: Layer.Layer< for (const entry of zeroUsage) { const deleted = yield* deleteZeroRelayReceiptUsage(entry) if (deleted.length !== 1) { - return yield* failStorageCorrupt(new Error("Zero relay receipt usage disappeared")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Zero relay receipt usage disappeared") + }) + }) } } for (const receipt of settledRelayReceipts) { const authorized = yield* authorizeRelayReceiptDelete(receipt.row_id) if (authorized.length !== 1) { - return yield* failStorageCorrupt(new Error("Relay receipt deletion was not authorized")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Relay receipt deletion was not authorized") + }) + }) } const deleted = yield* deleteExpiredRelayReceipt({ documentId, @@ -1359,7 +1377,11 @@ export const layer: Layer.Layer< rowId: receipt.row_id }) if (deleted.length !== 1) { - return yield* failStorageCorrupt(new Error("Relay receipt disappeared during history rewrite")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Relay receipt disappeared during history rewrite") + }) + }) } } yield* sql`DELETE FROM effect_local_peer_receipts @@ -1480,7 +1502,20 @@ export const layer: Layer.Layer< return lineage })) })).pipe( - Effect.catchTags({ SqlError: failStorageUnavailable, SchemaError: failStorageCorrupt }), + Effect.catchTags({ + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) + }), // Byte and row counts only, annotated from inside. The document's value, its heads and the // lineage itself never reach the span. Effect.withSpan("Compaction.rewriteHistory") diff --git a/packages/local-sql/src/PeerRelayOutbox.ts b/packages/local-sql/src/PeerRelayOutbox.ts index 14c33d9..03204ae 100644 --- a/packages/local-sql/src/PeerRelayOutbox.ts +++ b/packages/local-sql/src/PeerRelayOutbox.ts @@ -7,9 +7,7 @@ import * as Crypto from "effect/Crypto" import * as Effect from "effect/Effect" import * as Layer from "effect/Layer" import * as Schema from "effect/Schema" -import type * as SchemaError from "effect/SchemaError" import * as SqlClient from "effect/unstable/sql/SqlClient" -import type * as SqlError from "effect/unstable/sql/SqlError" import * as SqlSchema from "effect/unstable/sql/SqlSchema" import * as WriterProvenance from "./internal/writerProvenance.js" import * as PeerRelayOutboxLimits from "./PeerRelayOutboxLimits.js" @@ -126,19 +124,15 @@ const UsageRow = Schema.Struct({ const HorizonRow = Schema.Struct({ horizon_millis: Schema.NullOr(Schema.Number) }) const replayRowBatchSize = 500 -const replicaFailure = (reason: ReplicaError.Reason): ReplicaError.ReplicaError => - new ReplicaError.ReplicaError({ reason }) - -const storageUnavailable = (cause: unknown) => - Effect.fail(replicaFailure(new ReplicaError.StorageUnavailable({ cause }))) - -const storageCorrupt = (cause: unknown) => Effect.fail(replicaFailure(new ReplicaError.StorageCorrupt({ cause }))) - const protocolMismatch = (expected: string, observed: string) => - replicaFailure(new ReplicaError.ProtocolMismatch({ expected, observed })) + new ReplicaError.ReplicaError({ + reason: new ReplicaError.ProtocolMismatch({ expected, observed }) + }) const quotaExceeded = (resource: string, limit: number) => - replicaFailure(new ReplicaError.QuotaExceeded({ resource, limit })) + new ReplicaError.ReplicaError({ + reason: new ReplicaError.QuotaExceeded({ resource, limit }) + }) const parseIso = (value: string): number | null => { const millis = Date.parse(value) @@ -419,7 +413,11 @@ const make = Effect.gen(function*() { encodedSize: row.encoded_size }) if (remote.length !== 1 || replica.length !== 1) { - return yield* storageCorrupt(new Error("Relay outbox usage reservation mismatch")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Relay outbox usage reservation mismatch") + }) + }) } yield* sql`DELETE FROM effect_local_peer_relay_outbox_remote_usage WHERE replica_incarnation = ${row.replica_incarnation} @@ -444,7 +442,11 @@ const make = Effect.gen(function*() { row.replica_id !== permit.replicaId || row.encoded_size !== row.payload.byteLength ) { - return yield* storageCorrupt(new Error("Relay outbox metadata mismatch")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Relay outbox metadata mismatch") + }) + }) } const createdAt = parseIso(row.created_at) const retryDeadline = parseIso(row.retry_deadline) @@ -458,7 +460,11 @@ const make = Effect.gen(function*() { nextAttemptAt < createdAt || nextAttemptAt >= retryDeadline ) { - return yield* storageCorrupt(new Error("Relay outbox deadline mismatch")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Relay outbox deadline mismatch") + }) + }) } const syncEnvelope = yield* PeerSyncEnvelope.decodeSyncEnvelope(row.payload, replicaLimits).pipe( Effect.provideService(Crypto.Crypto, crypto) @@ -470,7 +476,11 @@ const make = Effect.gen(function*() { syncEnvelope.documentType !== row.document_type || syncEnvelope.messageHash !== row.message_hash ) { - return yield* storageCorrupt(new Error("Relay outbox payload metadata mismatch")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Relay outbox payload metadata mismatch") + }) + }) } const outerEnvelope: PeerSyncEnvelope.RelayOuterEnvelope = { domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, @@ -505,7 +515,11 @@ const make = Effect.gen(function*() { Effect.provideService(Crypto.Crypto, crypto) ) if (digest !== row.outer_envelope_digest) { - return yield* storageCorrupt(new Error("Relay outbox digest mismatch")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Relay outbox digest mismatch") + }) + }) } return { rowId: row.row_id, @@ -532,20 +546,6 @@ const make = Effect.gen(function*() { } }) - const mapStorageFailures = ( - effect: Effect.Effect< - A, - ReplicaError.ReplicaError | SchemaError.SchemaError | SqlError.SqlError, - R - > - ): Effect.Effect => - effect.pipe( - Effect.catchTags({ - SqlError: storageUnavailable, - SchemaError: storageCorrupt - }) - ) - const validateReplicaIncarnation = (expected: Identity.ReplicaIncarnation) => gate.current.pipe( Effect.flatMap((permit) => @@ -576,7 +576,12 @@ const make = Effect.gen(function*() { ) const relayMessageId = yield* Identity.makeRelayMessageId.pipe( Effect.provideService(Crypto.Crypto, crypto), - Effect.mapError((cause) => replicaFailure(new ReplicaError.StorageUnavailable({ cause }))) + Effect.catchTag("PlatformError", (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + )) ) const outerEnvelope: PeerSyncEnvelope.RelayOuterEnvelope = { domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, @@ -612,10 +617,14 @@ const make = Effect.gen(function*() { senderConnectionEpoch: syncEnvelope.connectionEpoch, senderSequence: syncEnvelope.sequence } - return yield* mapStorageFailures(sql.withTransaction(Effect.gen(function*() { + return yield* sql.withTransaction(Effect.gen(function*() { const existing = yield* findSource(request) if (existing.length > 1) { - return yield* storageCorrupt(new Error("Duplicate relay source operation")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Duplicate relay source operation") + }) + }) } if (existing.length === 1) { const entry = yield* validateRow(existing[0]!, permit) @@ -692,7 +701,11 @@ const make = Effect.gen(function*() { nextAttemptAt: createdAt }) if (inserted.length !== 1) { - return yield* storageCorrupt(new Error("Relay outbox insert did not return one row")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Relay outbox insert did not return one row") + }) + }) } yield* gate.validate(permit) return { @@ -718,7 +731,20 @@ const make = Effect.gen(function*() { retryDeadline, nextAttemptAt: createdAt } - }))) + })).pipe(Effect.catchTags({ + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) + })) })) const dueForEndpoint = (input: ReplayInput) => @@ -738,7 +764,7 @@ const make = Effect.gen(function*() { } const permit = yield* gate.shared const now = new Date(yield* Clock.currentTimeMillis).toISOString() - return yield* mapStorageFailures(sql.withTransaction(Effect.gen(function*() { + return yield* sql.withTransaction(Effect.gen(function*() { const metadata = yield* findDueMetadata({ replicaId: permit.replicaId, replicaIncarnation: permit.incarnation, @@ -773,11 +799,19 @@ const make = Effect.gen(function*() { replicaLimits.maxSyncChangesPerMessage ) ) { - return yield* storageCorrupt(new Error("Relay outbox payload length mismatch")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Relay outbox payload length mismatch") + }) + }) } const row = rowsById.get(item.row_id) if (row === undefined) { - return yield* storageCorrupt(new Error("Relay outbox row disappeared during replay")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Relay outbox row disappeared during replay") + }) + }) } const entry = yield* validateRow(row, permit) if ( @@ -785,12 +819,29 @@ const make = Effect.gen(function*() { entry.expectedLocal.subjectId !== endpoint.expectedLocal.subjectId || entry.expectedLocal.peerId !== endpoint.expectedLocal.peerId ) { - return yield* storageCorrupt(new Error("Relay outbox local endpoint mismatch")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Relay outbox local endpoint mismatch") + }) + }) } entries.push(entry) } return entries - }))) + })).pipe(Effect.catchTags({ + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) + })) })) const maximumPendingHorizon = (input: Endpoint) => @@ -815,12 +866,29 @@ const make = Effect.gen(function*() { AND remote_subject_id = ${endpoint.remote.subjectId} AND remote_peer_id = ${endpoint.remote.peerId}` }) - const rows = yield* mapStorageFailures(query(undefined)) + const rows = yield* query(undefined).pipe(Effect.catchTags({ + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) + })) const horizon = rows[0]?.horizon_millis ?? null if (horizon === null) return null const rounded = Math.round(horizon) if (rounded <= 0 || rounded > limits.maxRetryHorizonMillis) { - return yield* storageCorrupt(new Error("Relay outbox horizon mismatch")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Relay outbox horizon mismatch") + }) + }) } return rounded })) @@ -828,7 +896,7 @@ const make = Effect.gen(function*() { const markCustody = (input: CustodyInput) => Effect.scoped(Effect.gen(function*() { const permit = yield* gate.shared - yield* mapStorageFailures(sql.withTransaction(Effect.gen(function*() { + yield* sql.withTransaction(Effect.gen(function*() { const rows = yield* findRow({ replicaId: permit.replicaId, replicaIncarnation: permit.incarnation, @@ -839,7 +907,11 @@ const make = Effect.gen(function*() { return } if (rows.length !== 1) { - return yield* storageCorrupt(new Error("Duplicate relay message identity")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Duplicate relay message identity") + }) + }) } const row = rows[0]! yield* validateRow(row, permit) @@ -856,13 +928,26 @@ const make = Effect.gen(function*() { AND relay_message_id = ${input.relayMessageId} AND outer_envelope_digest = ${input.outerEnvelopeDigest}` yield* gate.validate(permit) - }))) + })).pipe(Effect.catchTags({ + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) + })) })) const pruneExpired = Effect.scoped(Effect.gen(function*() { const permit = yield* gate.shared const now = new Date(yield* Clock.currentTimeMillis).toISOString() - return yield* mapStorageFailures(sql.withTransaction(Effect.gen(function*() { + return yield* sql.withTransaction(Effect.gen(function*() { const query = SqlSchema.findAll({ Request: Schema.Void, Result: Row, @@ -886,7 +971,20 @@ const make = Effect.gen(function*() { } yield* gate.validate(permit) return rows.length - }))) + })).pipe(Effect.catchTags({ + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) + })) })) const usage = (input: Endpoint) => @@ -912,9 +1010,22 @@ const make = Effect.gen(function*() { FROM effect_local_peer_relay_outbox_replica_usage WHERE replica_incarnation = ${permit.incarnation}` }) - const [remoteRows, replicaRows] = yield* mapStorageFailures( - sql.withTransaction(Effect.all([remoteQuery(undefined), replicaQuery(undefined)])) - ) + const [remoteRows, replicaRows] = yield* sql.withTransaction( + Effect.all([remoteQuery(undefined), replicaQuery(undefined)]) + ).pipe(Effect.catchTags({ + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) + })) const remote = remoteRows[0] ?? { message_count: 0, encoded_bytes: 0 } const replica = replicaRows[0] ?? { message_count: 0, encoded_bytes: 0 } return { diff --git a/packages/local-sql/src/PeerSync.ts b/packages/local-sql/src/PeerSync.ts index 6bde1a4..fc62960 100644 --- a/packages/local-sql/src/PeerSync.ts +++ b/packages/local-sql/src/PeerSync.ts @@ -245,20 +245,6 @@ const decodeSyncChanges = ( return [...changes.values()] } -const failStorageUnavailable = (cause: unknown) => - Effect.fail( - new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageUnavailable({ cause }) - }) - ) - -const failStorageCorrupt = (cause: unknown) => - Effect.fail( - new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ cause }) - }) - ) - export class PeerSync extends Context.Service Effect.Effect readonly reset: (session: Session) => Effect.Effect @@ -530,8 +516,18 @@ const make = ( // A document this replica does not hold has no history for a rewrite to have discarded, // so it is on the genesis lineage exactly as a never rewritten document is. NoSuchElementError: () => Effect.succeed(Identity.genesisLineage), - SqlError: failStorageUnavailable, - SchemaError: failStorageCorrupt + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) }) ) const findPendingOutbox = SqlSchema.findAll({ @@ -839,7 +835,11 @@ const make = ( replicaIncarnation }) if (updated.length !== 1) { - return yield* failStorageCorrupt(new Error("Relay receipt usage is inconsistent")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Relay receipt usage is inconsistent") + }) + }) } } for (const row of rows) { @@ -847,7 +847,11 @@ const make = ( VALUES (${row.row_id})` const deleted = yield* deleteRelayReceipt({ rowId: row.row_id }) if (deleted.length !== 1) { - return yield* failStorageCorrupt(new Error("Relay receipt disappeared during pruning")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Relay receipt disappeared during pruning") + }) + }) } } yield* sql`DELETE FROM effect_local_peer_relay_receipt_usage @@ -886,8 +890,18 @@ const make = ( } })).pipe( Effect.catchTags({ - SchemaError: failStorageCorrupt, - SqlError: failStorageUnavailable + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ), + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ) }) ) @@ -1040,8 +1054,18 @@ const make = ( findCheckpointProvenance(documentId) ]).pipe( Effect.catchTags({ - SqlError: failStorageUnavailable, - SchemaError: failStorageCorrupt + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) }) ) return yield* Effect.try({ @@ -1139,8 +1163,18 @@ const make = ( messageHash: reply.messageHash }).pipe( Effect.catchTags({ - SqlError: failStorageUnavailable, - SchemaError: failStorageCorrupt + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) }) ) const existing = rows[0] @@ -1159,8 +1193,18 @@ const make = ( persistOutbound(session, reply.documentId, reply.message, reply.heads) ).pipe( Effect.catchTags({ - SqlError: failStorageUnavailable, - SchemaError: failStorageCorrupt + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) }) ) })) @@ -1204,8 +1248,18 @@ const make = ( documentId }).pipe( Effect.catchTags({ - SqlError: failStorageUnavailable, - SchemaError: failStorageCorrupt + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) }) ) if ((existing[0]?.count ?? 0) > 0) { @@ -1259,8 +1313,18 @@ const make = ( return outbound })).pipe( Effect.catchTags({ - SqlError: failStorageUnavailable, - SchemaError: failStorageCorrupt + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) }) ) return outbound === null @@ -1425,15 +1489,30 @@ const make = ( documentId, acceptedAt, new Date(nowMillis - limits.maxPendingAgeMillis).toISOString() - ).pipe(Effect.catchTag("SqlError", failStorageUnavailable)) + ).pipe(Effect.catchTag("SqlError", (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ))) if (relayReceiptLimits !== null) { yield* sql.withTransaction(Effect.gen(function*() { yield* pruneRelayReceiptsInTransaction(permit.incarnation, acceptedAt) yield* gate.validate(permit) })).pipe( Effect.catchTags({ - SqlError: failStorageUnavailable, - SchemaError: failStorageCorrupt + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) }) ) } @@ -1511,8 +1590,18 @@ const make = ( }) const receiptRows = yield* loadReceipt().pipe( Effect.catchTags({ - SqlError: failStorageUnavailable, - SchemaError: failStorageCorrupt + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) }) ) const receipt = receiptRows[0] @@ -1528,8 +1617,18 @@ const make = ( documentId }).pipe( Effect.catchTags({ - SqlError: failStorageUnavailable, - SchemaError: failStorageCorrupt + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) }) ) const receiptTotal = receiptTotals[0] @@ -1771,16 +1870,36 @@ const make = ( })) }).pipe( Effect.catchTags({ - SqlError: failStorageUnavailable, - SchemaError: failStorageCorrupt + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) }) ) yield* validateExistingChanges(existingChanges) const validatePendingQuota = Effect.gen(function*() { const pendingTotals = yield* findDocumentPendingChangeTotals(documentId).pipe( Effect.catchTags({ - SqlError: failStorageUnavailable, - SchemaError: failStorageCorrupt + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) }) ) const receiptPending = yield* findDocumentPendingReceiptTotals({ @@ -1788,8 +1907,18 @@ const make = ( documentId }).pipe( Effect.catchTags({ - SqlError: failStorageUnavailable, - SchemaError: failStorageCorrupt + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) }) ) if ( @@ -1826,8 +1955,18 @@ const make = ( } const peerTotals = yield* findPeerPendingChangeTotals(receiptSession.peerId).pipe( Effect.catchTags({ - SqlError: failStorageUnavailable, - SchemaError: failStorageCorrupt + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) }) ) const peerReceiptPending = yield* findPeerPendingReceiptTotals({ @@ -1835,8 +1974,18 @@ const make = ( peerId: receiptSession.peerId }).pipe( Effect.catchTags({ - SqlError: failStorageUnavailable, - SchemaError: failStorageCorrupt + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) }) ) if ( @@ -1873,16 +2022,36 @@ const make = ( } const replicaTotals = yield* findReplicaPendingChangeTotals(undefined).pipe( Effect.catchTags({ - SqlError: failStorageUnavailable, - SchemaError: failStorageCorrupt + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) }) ) const replicaReceiptPending = yield* findReplicaPendingReceiptTotals( receiptSession.replicaIncarnation ).pipe( Effect.catchTags({ - SqlError: failStorageUnavailable, - SchemaError: failStorageCorrupt + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) }) ) if ( @@ -1931,8 +2100,18 @@ const make = ( }) const pendingRows = yield* findPendingChanges(documentId).pipe( Effect.catchTags({ - SqlError: failStorageUnavailable, - SchemaError: failStorageCorrupt + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) }) ) const pendingReceiptRows = yield* findPendingReceipts({ @@ -1940,14 +2119,34 @@ const make = ( documentId }).pipe( Effect.catchTags({ - SqlError: failStorageUnavailable, - SchemaError: failStorageCorrupt + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) }) ) const checkpointProvenanceRows = yield* findCheckpointProvenance(documentId).pipe( Effect.catchTags({ - SqlError: failStorageUnavailable, - SchemaError: failStorageCorrupt + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) }) ) const checkpointProvenanceByHash = new Map< @@ -1963,9 +2162,13 @@ const make = ( existing.writerDefinitionHash !== entry.writerDefinitionHash ) ) { - return yield* failStorageCorrupt( - new Error(`Conflicting checkpoint writer provenance for change ${entry.changeHash}`) - ) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error( + `Conflicting checkpoint writer provenance for change ${entry.changeHash}` + ) + }) + }) } checkpointProvenanceByHash.set(entry.changeHash, entry) } @@ -2005,9 +2208,13 @@ const make = ( existing.writerDefinitionHash !== entry.writerDefinitionHash ) ) { - return yield* failStorageCorrupt( - new Error(`Conflicting pending writer provenance for change ${entry.changeHash}`) - ) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error( + `Conflicting pending writer provenance for change ${entry.changeHash}` + ) + }) + }) } pendingProvenanceByHash.set(entry.changeHash, entry) } @@ -2086,8 +2293,18 @@ const make = ( const bytes = InternalAutomerge.save(staged) const durableRows = yield* findDocumentChangeProvenance(documentId).pipe( Effect.catchTags({ - SqlError: failStorageUnavailable, - SchemaError: failStorageCorrupt + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) }) ) const checkpointWriterProvenance = yield* Effect.try({ @@ -2216,7 +2433,11 @@ const make = ( writerProvenance: checkpoint.writerProvenance }) if (installed.length !== 1) { - return yield* failStorageCorrupt(new Error("Checkpoint identity collision")) + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: new Error("Checkpoint identity collision") + }) + }) } yield* sql`DELETE FROM effect_local_checkpoints WHERE document_id = ${documentId} @@ -2301,8 +2522,18 @@ const make = ( return { _tag: "Committed" as const, commitSequence, reply } })).pipe( Effect.catchTags({ - SqlError: failStorageUnavailable, - SchemaError: failStorageCorrupt + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) }) ) if (result._tag === "Committed") { @@ -2354,8 +2585,18 @@ const make = ( return pruned })).pipe( Effect.catchTags({ - SqlError: failStorageUnavailable, - SchemaError: failStorageCorrupt + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) }) ) ) @@ -2391,7 +2632,12 @@ const make = ( AND peer_id = ${session.peerId} AND connection_epoch = ${session.connectionEpoch} AND relay_message_id IS NULL` - })).pipe(Effect.catchTag("SqlError", failStorageUnavailable)) + })).pipe(Effect.catchTag("SqlError", (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ))) yield* Ref.update(generation, (current) => current + 1) yield* removeState(session) })) @@ -2432,8 +2678,18 @@ const make = ( })) ), Effect.catchTags({ - SqlError: failStorageUnavailable, - SchemaError: failStorageCorrupt + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) }) ) })), @@ -2461,8 +2717,18 @@ const make = ( return true })).pipe( Effect.catchTags({ - SqlError: failStorageUnavailable, - SchemaError: failStorageCorrupt + SqlError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause }) + }) + ), + SchemaError: (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ) }) ) ) diff --git a/packages/local-sql/src/ReplicaBootstrap.ts b/packages/local-sql/src/ReplicaBootstrap.ts index 350b101..ba853a6 100644 --- a/packages/local-sql/src/ReplicaBootstrap.ts +++ b/packages/local-sql/src/ReplicaBootstrap.ts @@ -35,13 +35,6 @@ const metadataMissing = () => }) }) -const failStorageCorrupt = (cause: unknown) => - Effect.fail( - new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ cause }) - }) - ) - const unsupportedStorageFormat = (observedVersion: number) => new ReplicaError.ReplicaError({ reason: new ReplicaError.UnsupportedStorageFormatVersion({ @@ -102,7 +95,12 @@ export const make = (definition: ReplicaDefinition.Any) => } if (stored.storage_format_version === storageFormatVersion) return return yield* unsupportedStorageFormat(stored.storage_format_version) - }).pipe(Effect.catchTag("SchemaError", failStorageCorrupt)) + }).pipe(Effect.catchTag("SchemaError", (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ))) yield* Migrations.run const findMetadata = SqlSchema.findAll({ Request: Schema.Void, @@ -217,7 +215,12 @@ export const make = (definition: ReplicaDefinition.Any) => writerGeneration: row.writer_generation, definitionHash: definition.hash } - })).pipe(Effect.catchTag("SchemaError", failStorageCorrupt)) + })).pipe(Effect.catchTag("SchemaError", (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause }) + }) + ))) }) export const layer = ( From 1be91ee809be1523519577cb59cd2b9efb434f04 Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Sun, 26 Jul 2026 23:42:46 -0500 Subject: [PATCH 07/29] Move task tracking to LEDGER.md and tighten service rules The repository rules duplicated the global agent rules by naming AGENTS.local.md as a second task ledger, so the two could drift and an agent reading one would miss the other. Task tracking now lives only in the worktree LEDGER.md the global rules already mandate. Also close a loophole in the service rules. "Capture every service used later by its methods or expose that service as a Layer requirement" let an implementation push its dependencies into every method's requirement channel, where they spread to every transitive caller and let a method run under a context the Layer never validated. Dependencies are now resolved once at construction, and leaking a requirement needs a stated per-call reason. --- AGENTS.md | 2 -- CLAUDE.md | 2 -- RULES.md | 12 ++++++------ 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3527dc9..03a81ff 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,3 @@ # Agent Instructions Read `RULES.md` before any work. - -Read the untracked task ledger in `AGENTS.local.md`. Only the main task owner edits or replaces it. Review agents report their findings directly and do not modify the ledger. diff --git a/CLAUDE.md b/CLAUDE.md index 3527dc9..03a81ff 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,5 +1,3 @@ # Agent Instructions Read `RULES.md` before any work. - -Read the untracked task ledger in `AGENTS.local.md`. Only the main task owner edits or replaces it. Review agents report their findings directly and do not modify the ledger. diff --git a/RULES.md b/RULES.md index 1a78ea6..3f88e78 100644 --- a/RULES.md +++ b/RULES.md @@ -4,13 +4,10 @@ Read this file before any work. Treat these rules as required for every package. ## Coordination -- Read `AGENTS.local.md` before editing. -- Only the main task owner edits or replaces `AGENTS.local.md`, and only when the repository level task changes to unrelated work. -- The main task owner records the task, branch, PR, ownership, checkpoints, commands, results, decisions, and next actions. -- The main task owner checks `git status` and the remote branch before mutation, rebases before every push, and pushes each independently verified fix. +- Check `git status` and the remote branch before mutation, rebase before every push, and push each independently verified fix. - Never overwrite, revert, or reformat another contributor's work. - Keep commits narrow. -- Use one production owner for overlapping files. Review agents must not edit production code, tests, shared ledgers, commits, branches, or PR state. +- Use one production owner for overlapping files. Review agents must not edit production code, tests, commits, branches, or PR state. ## Effect Source Of Truth @@ -41,7 +38,10 @@ Read this file before any work. Treat these rules as required for every package. - For a service that requires cleanup, acquire it with `Effect.acquireRelease` or another scoped `Effect`, and provide it with `Layer.effect`. - Keep Layer construction configurable. Do not hide lifecycle, persistence, concurrency, or security choices behind defaults. - Do not capture consumer specific runtime values in module global state. Pass them through service implementations, Layer constructors, or operation arguments. -- A Layer must capture every service used later by its methods or expose that service as a Layer requirement. +- Resolve dependencies once while the Layer is being built and close over them. A service method must not leave a service in its own requirement channel. +- A service's methods should be `Effect`. Leak a requirement only when the dependency is genuinely per call, such as a request scoped context or an ambient `Scope` the caller owns, and say why at the definition. +- A leaked requirement is a real cost: it spreads to every transitive caller, it lets a method be invoked under a context the Layer never validated, and it hides which services an implementation actually needs behind the call sites instead of stating it once at construction. +- A Layer must capture every service used later by its methods. - Export a constructor or use `Layer.fresh` when a context sensitive Layer must build independently more than once under one memo map. ## Composition And Effects From ecf8f271e825b4aec5854bae200ac5d79a9446cf Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Sun, 26 Jul 2026 23:43:34 -0500 Subject: [PATCH 08/29] Add the cluster relay inbox entity and its store contract The existing relay keeps endpoint ownership, the claim ledger, delivery wakeups and admission counters in process local state, so only one server process can own a custody shard. Modelling each recipient device as a cluster entity moves all of that to sharding, which elects exactly one owner per key across every runner. Being the sole writer for its device is what makes the store contract small. Claim tokens, lease deadlines, session generations and work ownership exist only to arbitrate between competing processes, and there are none. There is also no in-flight state: a message stays Pending until terminal and what is being delivered lives only in memory, so losing a runner loses that memory and nothing else and the next owner redelivers. At-least-once falls out of the schema instead of a recovery protocol. Delivery is stop-and-wait per sender channel and concurrent across channels, so a stalled sender cannot hold up other senders to the same device, and the connection epoch is part of the channel identity because the sender's sequence restarts at zero on every reconnect. Two details are load bearing and easy to lose. Subscribe is forked so it does not hold the entity's single handler permit for the life of a session, which would leave the settlement that ends each delivery unable to run and stall the inbox with no error anywhere. And a session's liveness is bounded by a server side deadline rather than by a disconnect notification, because the cross node interrupt is best effort and is never sent at all when the front door's own node fails. --- packages/local-rpc/src/RelayInbox.ts | 544 ++++++++++++++++++ packages/local-rpc/src/RelayInboxStore.ts | 270 +++++++++ .../local-rpc/src/internal/relayInboxKey.ts | 48 ++ 3 files changed, 862 insertions(+) create mode 100644 packages/local-rpc/src/RelayInbox.ts create mode 100644 packages/local-rpc/src/RelayInboxStore.ts create mode 100644 packages/local-rpc/src/internal/relayInboxKey.ts diff --git a/packages/local-rpc/src/RelayInbox.ts b/packages/local-rpc/src/RelayInbox.ts new file mode 100644 index 0000000..22c686c --- /dev/null +++ b/packages/local-rpc/src/RelayInbox.ts @@ -0,0 +1,544 @@ +import * as Identity from "@lucas-barake/effect-local/Identity" +import type * as Cause from "effect/Cause" +import * as Clock from "effect/Clock" +import * as Deferred from "effect/Deferred" +import * as Effect from "effect/Effect" +import * as Exit from "effect/Exit" +import * as Latch from "effect/Latch" +import * as Option from "effect/Option" +import * as Queue from "effect/Queue" +import * as Ref from "effect/Ref" +import * as Schedule from "effect/Schedule" +import * as Schema from "effect/Schema" +import * as Scope from "effect/Scope" +import * as Semaphore from "effect/Semaphore" +import * as Entity from "effect/unstable/cluster/Entity" +import * as Rpc from "effect/unstable/rpc/Rpc" +import * as PeerRpc from "./PeerRpc.js" +import * as PeerRpcError from "./PeerRpcError.js" +import * as RelayInboxStore from "./RelayInboxStore.js" + +/** + * One cluster entity per recipient device. The cluster guarantees a single live owner per entity + * id across every runner, which is what makes the relay horizontally scalable: endpoint ownership, + * routing, and cross node wakeups come from sharding rather than from process local registries. + * + * Because the entity is the sole writer for its device, custody needs no distributed protocol. A + * message is `Pending` in the store until it reaches a terminal state, and what is currently being + * delivered is tracked only in memory. Losing the owning runner loses that memory and nothing + * else, so the next owner finds the same `Pending` rows and delivers them again. + */ + +const InboxError = Schema.Union([ + PeerRpcError.SessionUnavailable, + PeerRpcError.InvalidRequest, + PeerRpcError.RequestCapacityExceeded, + PeerRpcError.ServerUnavailable +]) + +/** + * Accepts a message into the recipient's durable inbox. + * + * Deliberately volatile rather than `ClusterSchema.Persisted`: durability is the store write this + * handler performs, so persisting the cluster message as well would hold the payload twice under + * two independent retry ledgers. The at-least-once guarantee therefore rests on this rpc failing + * whenever the durable write did not land, which leaves the sender's outbox entry intact to be + * retried under the same `relayMessageId`. + */ +export class DeliverRpc extends Rpc.make("Deliver", { + payload: { + channel: RelayInboxStore.ChannelKey, + envelope: RelayInboxStore.InboxEnvelope, + /** + * The horizon the sending session negotiated at `Open`. + * + * Carried per message because the deduplication horizon is derived from it: a message's + * identity must outlive the window in which its sender may still replay it, and that window is + * a property of the sender's session rather than of this server's configuration. + */ + senderRetryHorizonMillis: Schema.Int.pipe(Schema.check(Schema.isGreaterThan(0))) + }, + error: InboxError +}) {} + +/** + * Opens the recipient's live session and streams its durable inbox. + * + * Replaces any previous session for this device, which is how the cluster enforces the "one live + * session per endpoint" rule that the previous implementation kept in a process local map. + */ +export class SubscribeRpc extends Rpc.make("Subscribe", { + payload: { sessionId: Identity.SessionId }, + success: PeerRpc.StoredMessage, + error: InboxError, + stream: true +}) {} + +/** + * Terminally settles one delivered message. Both acknowledgement and rejection settle: the + * recipient has made a durable decision either way, so neither should be redelivered. + */ +export class SettleRpc extends Rpc.make("Settle", { + payload: { + sessionId: Identity.SessionId, + relayMessageId: Identity.RelayMessageId, + claimToken: PeerRpc.ClaimToken, + messageHash: PeerRpc.StoredMessage.fields.messageHash, + outcome: RelayInboxStore.TerminalOutcome + }, + error: InboxError +}) {} + +/** + * Extends the session's liveness deadline. + * + * Driven by the front door while it holds a live socket, never by the client, so it stays off the + * public wire contract. A cluster cannot rely on disconnects announcing themselves: the cross node + * interrupt is best effort and is never sent at all when the front door's own node fails. The + * deadline bounds how long a failed front door can hold a device's channels open. + */ +export class HeartbeatRpc extends Rpc.make("Heartbeat", { + payload: { sessionId: Identity.SessionId }, + error: InboxError +}) {} + +export class EndSessionRpc extends Rpc.make("EndSession", { + payload: { sessionId: Identity.SessionId }, + error: InboxError +}) {} + +export const RelayInbox = Entity.make("EffectLocalRelayInbox", [ + DeliverRpc, + SubscribeRpc, + SettleRpc, + HeartbeatRpc, + EndSessionRpc +]) + +export interface Options { + /** + * How many times a message may be handed to a session before it is dead lettered. + * + * Counts deliveries rather than failures. A recipient that reads a message and then disconnects + * without settling it is not a failure at any layer, yet left uncounted it would redeliver + * forever and block its channel, so the budget has to be spent by the attempt itself. + */ + readonly maxDeliveries: number + /** How long an undelivered message survives before it is expired. */ + readonly messageTtlMillis: number + /** + * How long a settled message's identity is retained. Must cover the sender's retry horizon, or a + * replay could outlive the deduplication record and be applied twice. + */ + readonly terminalRetentionMillis: number + /** How long a session survives without a heartbeat. */ + readonly sessionDeadlineMillis: number + /** How often the entity checks for a lapsed session deadline. */ + readonly sessionSweepMillis: number + /** Maximum channels delivered concurrently to one session. */ + readonly maxConcurrentChannels: number + /** Backoff after a store failure, so a failing database cannot spin the dispatcher. */ + readonly storeRetryMillis: number + /** Per-inbox admission caps, enforced inside the admission transaction. */ + readonly maxPendingMessages: number + readonly maxPendingBytes: number + /** + * Entity mailbox depth. Surfaces to callers as `MailboxFull`. + * + * Note the forked `Subscribe` handler occupies one slot for the whole life of a session, so the + * usable depth for `Deliver` and `Settle` is one less than this. + */ + readonly mailboxCapacity: number + /** How long an idle entity survives before the cluster passivates it. */ + readonly maxIdleTimeMillis: number +} + +interface Settlement { + readonly claimToken: PeerRpc.ClaimToken + readonly messageHash: string + /** Completed by the recipient's `Settle` call. */ + readonly requested: Deferred.Deferred + /** + * Completed by the delivering fiber once the terminal transition is durable. + * + * `Settle` waits on this before replying. Replying earlier would tell the recipient its + * acknowledgement is safe while the row is still `Pending`, and the recipient prunes its own + * receipt on that reply. + */ + readonly durable: Deferred.Deferred +} + +interface Session { + readonly sessionId: Identity.SessionId + readonly outbound: Queue.Queue + readonly settlements: Map + /** Channels with a delivery in flight. One message per channel at a time preserves order. */ + readonly busyChannels: Set + /** Opened whenever there may be new work: a fresh admission, or a channel falling idle. */ + readonly wake: Latch.Latch + readonly scope: Scope.Closeable + readonly deadlineAt: Ref.Ref +} + +const makeClaimToken = Effect.sync(() => `clm_${crypto.randomUUID()}` as PeerRpc.ClaimToken) + +const channelId = (channel: RelayInboxStore.ChannelKey): string => + JSON.stringify([ + channel.tenantId, + channel.senderSubjectId, + channel.senderPeerId, + channel.senderReplicaIncarnation + ]) + +export const layer = (options: Options) => + RelayInbox.toLayer( + Effect.gen(function*() { + const address = yield* Entity.CurrentAddress + const inboxKey = address.entityId + const store = yield* RelayInboxStore.RelayInboxStore + + const sessionRef = yield* Ref.make(Option.none()) + // Serializes session installation so two concurrent reconnects cannot both install + // themselves, which would leave two dispatchers draining one inbox. + const sessionLock = yield* Semaphore.make(1) + + /** + * Closes a session and waits for it to finish closing. + * + * Awaiting matters: closing the scope interrupts the dispatcher and every in flight channel + * delivery. Returning before that completes would let a replacement session start delivering + * while the outgoing one is still mid delivery, and the same message could then be offered + * to two sessions at once. + */ + const closeSession = (session: Session) => + Effect.gen(function*() { + yield* Ref.update( + sessionRef, + Option.filter((current) => current.sessionId !== session.sessionId) + ) + for (const settlement of session.settlements.values()) { + yield* Deferred.interrupt(settlement.requested) + yield* Deferred.interrupt(settlement.durable) + } + session.settlements.clear() + yield* Scope.close(session.scope, Exit.void) + yield* Queue.end(session.outbound) + }) + + const closeCurrentSession = Ref.get(sessionRef).pipe( + Effect.flatMap(Option.match({ onNone: () => Effect.void, onSome: closeSession })) + ) + + /** + * Delivers one channel's head message and waits for the recipient to settle it. + * + * Interruption — a disconnect, a replaced session, a lapsed deadline — simply abandons the + * attempt. The row is still `Pending`, so the next session picks it up. Nothing has to be + * released and no lease has to expire first. + */ + const deliverHead = (session: Session, head: RelayInboxStore.PendingMessage) => + Effect.gen(function*() { + const claimToken = yield* makeClaimToken + const requested = yield* Deferred.make() + const durable = yield* Deferred.make() + // Registered before the message is offered. The recipient can settle as soon as it reads + // the message, and it does so on a different fiber, so the settlement must already be + // addressable by the time the message becomes visible. + session.settlements.set(head.relayMessageId, { + claimToken, + messageHash: head.envelope.messageHash, + requested, + durable + }) + + // The outbound queue is a rendezvous, so this suspends until the recipient actually + // takes the message. Everything before it is bookkeeping that costs the message nothing. + yield* Queue.offer(session.outbound, { + _tag: "StoredMessage" as const, + ...head.envelope, + claimToken + }) + + // Charged only now, because the message has provably reached the transport. Counting at + // the start would spend the budget on messages that were prepared for a channel and then + // abandoned when the recipient disconnected, which is ordinary behaviour on a flaky + // connection and would dead letter messages that were never transmitted once. + const deliveredAt = yield* Clock.currentTimeMillis + const record = yield* store.recordDelivery(inboxKey, head.relayMessageId, { + maxDeliveries: options.maxDeliveries, + now: deliveredAt + }) + if (record._tag === "DeadLettered") { + yield* Effect.logWarning( + "Relay inbox message exhausted its delivery budget and was dead lettered" + ).pipe( + Effect.annotateLogs({ + inboxKey, + relayMessageId: head.relayMessageId, + deliveries: record.deliveries + }) + ) + return + } + + const outcome = yield* Deferred.await(requested) + const settledAt = yield* Clock.currentTimeMillis + const settled = yield* store.settle(inboxKey, head.relayMessageId, { + outcome, + messageHash: head.envelope.messageHash, + now: settledAt, + terminalRetentionMillis: options.terminalRetentionMillis + }) + if (settled !== "Settled") { + yield* Effect.logWarning("Relay inbox settlement did not apply").pipe( + Effect.annotateLogs({ inboxKey, relayMessageId: head.relayMessageId, settled }) + ) + yield* Deferred.fail(durable, new PeerRpcError.ServerUnavailable()) + return + } + yield* Deferred.succeed(durable, void 0) + }).pipe( + // Only the store's typed failure is recovered here. Defects and interruption stay + // untouched: a defect is a bug that must surface, and an interruption simply abandons the + // attempt, leaving the row `Pending` for the next session to pick up. + Effect.catchTag("ReplicaError", (error) => + Effect.logWarning("Relay inbox delivery failed; message stays pending").pipe( + Effect.annotateLogs({ + inboxKey, + relayMessageId: head.relayMessageId, + reason: error.reason._tag + }), + Effect.andThen(Effect.sleep(options.storeRetryMillis)) + )), + // Fails any settle still waiting on this delivery so the recipient retries rather than + // believing an acknowledgement landed. + Effect.ensuring( + Deferred.fail( + session.settlements.get(head.relayMessageId)?.durable ?? + Deferred.makeUnsafe(), + new PeerRpcError.ServerUnavailable() + ).pipe(Effect.ignore) + ), + Effect.ensuring(Effect.sync(() => { + session.settlements.delete(head.relayMessageId) + session.busyChannels.delete(channelId(head.channel)) + })), + // Frees this channel and re-polls, which is also how the next message in the channel and + // any newly admitted message get picked up. + Effect.ensuring(session.wake.open) + ) + + const dispatch = (session: Session): Effect.Effect => + Effect.gen(function*() { + while (true) { + // Closed before polling so an admission that arrives mid poll is not lost: it reopens + // the latch and the following await returns immediately. + yield* session.wake.close + const heads = yield* store.pendingHeads(inboxKey, { + limit: options.maxConcurrentChannels + }) + for (const head of heads) { + const channel = channelId(head.channel) + if (session.busyChannels.has(channel)) continue + if (session.busyChannels.size >= options.maxConcurrentChannels) break + session.busyChannels.add(channel) + yield* Effect.forkIn(deliverHead(session, head), session.scope) + } + yield* session.wake.await + } + }).pipe( + // Recovers only the store's typed failure. A defect here is a bug in the dispatcher and + // must propagate rather than restart invisibly; interruption ends the session normally. + Effect.catchTag("ReplicaError", (error) => + Effect.logWarning("Relay inbox dispatch failed; retrying").pipe( + Effect.annotateLogs({ inboxKey, reason: error.reason._tag }), + Effect.andThen(Effect.sleep(options.storeRetryMillis)), + Effect.andThen(dispatch(session)) + )) + ) as Effect.Effect + + // A single long lived sweeper rather than a fiber inside each session scope, so that closing + // a session is never performed by a fiber the close itself must interrupt. + yield* Effect.gen(function*() { + const current = yield* Ref.get(sessionRef) + if (Option.isNone(current)) return + const now = yield* Clock.currentTimeMillis + const deadlineAt = yield* Ref.get(current.value.deadlineAt) + if (now < deadlineAt) return + yield* sessionLock.withPermit( + Effect.gen(function*() { + const latest = yield* Ref.get(sessionRef) + if (Option.isNone(latest) || latest.value.sessionId !== current.value.sessionId) return + yield* Effect.logInfo("Relay inbox session deadline lapsed; releasing session").pipe( + Effect.annotateLogs({ inboxKey, sessionId: current.value.sessionId }) + ) + yield* closeSession(latest.value) + }) + ) + }).pipe( + Effect.repeat(Schedule.spaced(options.sessionSweepMillis)), + Effect.forkScoped + ) + + // Runs before the framework's own termination handshake, which otherwise waits the full + // `entityTerminationTimeout` for a forked streaming handler that never returns on its own, + // rejecting deliveries and settlements for the whole of that window on every rebalance. + yield* Effect.addFinalizer(() => Effect.orDie(closeCurrentSession)) + + const currentSession = (sessionId: Identity.SessionId) => + Ref.get(sessionRef).pipe( + Effect.flatMap(Option.match({ + onNone: () => Effect.fail(new PeerRpcError.SessionUnavailable()), + onSome: (session) => + session.sessionId === sessionId + ? Effect.succeed(session) + : Effect.fail(new PeerRpcError.SessionUnavailable()) + })) + ) + + const extendDeadline = (session: Session) => + Clock.currentTimeMillis.pipe( + Effect.flatMap((now) => Ref.set(session.deadlineAt, now + options.sessionDeadlineMillis)) + ) + + return RelayInbox.of({ + Deliver: ({ payload }) => + Effect.gen(function*() { + const now = yield* Clock.currentTimeMillis + const result = yield* store.admit({ + inboxKey, + channel: payload.channel, + envelope: payload.envelope, + now, + messageTtlMillis: options.messageTtlMillis, + senderRetryHorizonMillis: payload.senderRetryHorizonMillis, + quota: { + maxPendingMessages: options.maxPendingMessages, + maxPendingBytes: options.maxPendingBytes + } + }) + switch (result._tag) { + case "Conflict": + return yield* new PeerRpcError.InvalidRequest() + case "QuotaExceeded": + return yield* new PeerRpcError.RequestCapacityExceeded() + case "Duplicate": + // A duplicate of a message this inbox already gave up on must not be reported as + // success: the sender deletes its outbox entry on success, and that entry is the + // only remaining copy of a message the relay can no longer deliver. + if (result.state === "DeadLettered" || result.state === "Expired") { + return yield* new PeerRpcError.ServerUnavailable() + } + break + case "Admitted": + break + } + // Duplicates still wake the dispatcher: the original may be sitting undelivered. + const session = yield* Ref.get(sessionRef) + if (Option.isSome(session)) { + yield* session.value.wake.open + } + }).pipe( + // Discriminated per reason rather than collapsed. Reporting a permanent fault as a + // transient one makes the sender's outbox retry it forever. + Effect.catchReason("ReplicaError", "QuotaExceeded", () => new PeerRpcError.RequestCapacityExceeded()), + Effect.catchReason("ReplicaError", "StorageUnavailable", () => new PeerRpcError.ServerUnavailable()), + // Corruption is permanent. Reporting it as transient would have the sender's outbox + // retry it until the message expires. + Effect.catchReason("ReplicaError", "StorageCorrupt", (reason) => + Effect.logError("Relay inbox storage is corrupt").pipe( + Effect.annotateLogs({ inboxKey, reason: reason._tag }), + Effect.andThen(new PeerRpcError.InvalidRequest()) + )), + Effect.catchTag("ReplicaError", (error) => + Effect.logWarning("Relay inbox admission failed").pipe( + Effect.annotateLogs({ inboxKey, reason: error.reason._tag }), + Effect.andThen(new PeerRpcError.ServerUnavailable()) + )) + ), + + // Forked so it does not hold the entity's sequential handler permit. Without this the + // session would occupy the only permit for its whole lifetime and the settlement that ends + // each delivery could never be handled, stalling the inbox with no error anywhere. + Subscribe: ({ payload }) => + Rpc.fork( + sessionLock.withPermit( + // Installation is uninterruptible up to the point the session becomes reachable + // through `sessionRef`. An interrupt in between would strand the scope — nothing + // else holds a reference to close it — and could leave a dispatcher polling for a + // session no one owns, delivering alongside its own replacement. + Effect.uninterruptibleMask((restore) => + Effect.gen(function*() { + yield* restore(closeCurrentSession) + + const scope = yield* Scope.make() + const install = Effect.gen(function*() { + const outbound = yield* Queue.bounded(0) + const now = yield* Clock.currentTimeMillis + const deadlineAt = yield* Ref.make(now + options.sessionDeadlineMillis) + const session: Session = { + sessionId: payload.sessionId, + outbound, + settlements: new Map(), + busyChannels: new Set(), + wake: yield* Latch.make(true), + scope, + deadlineAt + } + yield* Effect.forkIn(dispatch(session), scope) + yield* Ref.set(sessionRef, Option.some(session)) + return outbound + }) + return yield* Effect.onError( + install, + () => Effect.orDie(Scope.close(scope, Exit.void)) + ) + }) + ) + ) + ), + + Settle: ({ payload }) => + Effect.gen(function*() { + const session = yield* currentSession(payload.sessionId) + const settlement = session.settlements.get(payload.relayMessageId) + if (settlement === undefined) { + return yield* new PeerRpcError.SessionUnavailable() + } + // The claim token is minted per delivery attempt, so this also rejects a token + // replayed from an earlier attempt of the same message. + if ( + settlement.claimToken !== payload.claimToken || + settlement.messageHash !== payload.messageHash + ) { + return yield* new PeerRpcError.SessionUnavailable() + } + yield* extendDeadline(session) + yield* Deferred.succeed(settlement.requested, payload.outcome) + // Replies only once the terminal transition is durable. The recipient prunes its own + // relay receipt on this reply, so a premature success would leave neither side holding + // the message. + yield* Deferred.await(settlement.durable) + }), + + Heartbeat: ({ payload }) => currentSession(payload.sessionId).pipe(Effect.flatMap(extendDeadline)), + + EndSession: ({ payload }) => + sessionLock.withPermit( + currentSession(payload.sessionId).pipe( + Effect.flatMap(closeSession), + Effect.catchTag("SessionUnavailable", () => Effect.void) + ) + ) + }) + }), + { + // Stated rather than inherited. Serialising the entity's handlers is what lets `Deliver` and + // `Settle` reason about session state without their own locking, and it is why `Subscribe` + // has to be forked — so the value is load bearing and belongs in the source, not in a + // framework default that could change underneath these invariants. + concurrency: 1, + mailboxCapacity: options.mailboxCapacity, + maxIdleTime: options.maxIdleTimeMillis + } + ) diff --git a/packages/local-rpc/src/RelayInboxStore.ts b/packages/local-rpc/src/RelayInboxStore.ts new file mode 100644 index 0000000..5fab7f1 --- /dev/null +++ b/packages/local-rpc/src/RelayInboxStore.ts @@ -0,0 +1,270 @@ +import * as Identity from "@lucas-barake/effect-local/Identity" +import type * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" +import * as Context from "effect/Context" +import type * as Effect from "effect/Effect" +import * as Schema from "effect/Schema" +import * as PeerRpc from "./PeerRpc.js" + +/** + * Durable storage for one peer device's relay inbox. + * + * The owning cluster entity is the sole writer for a given `inboxKey`, and that single fact is + * what keeps this contract small. Distributed custody transfer — claim tokens, lease deadlines, + * session generations, work ownership, a global write lock — exists only to arbitrate between + * competing processes, and there are none here. Sharding elects one owner per device and the store + * only has to remember what that owner has not yet finished. + * + * There is deliberately no in-flight state. A message stays `Pending` until it reaches a terminal + * state, and the owner tracks what it is currently delivering in memory. If the owning runner + * dies, that memory is lost and the rows are still `Pending`, so the next owner simply delivers + * them again. At-least-once delivery therefore falls out of the schema rather than out of a lease + * recovery protocol. + */ + +/** + * A message at rest in a recipient's durable inbox. + * + * This is the wire `StoredMessage` without its claim token. The claim token is minted per delivery + * attempt rather than stored, so a token from an earlier attempt can never settle a later one. + */ +export const InboxEnvelope = Schema.Struct({ + relayMessageId: Identity.RelayMessageId, + relayPeerId: Identity.PeerId, + sender: PeerRpc.StoredMessage.fields.sender, + recipient: PeerRpc.StoredMessage.fields.recipient, + payloadVersion: PeerRpc.StoredMessage.fields.payloadVersion, + document: PeerRpc.StoredMessage.fields.document, + writerProvenance: PeerRpc.StoredMessage.fields.writerProvenance, + messageHash: PeerRpc.StoredMessage.fields.messageHash, + outerEnvelopeDigest: PeerRpc.RelayDigest, + payload: PeerRpc.StoredMessage.fields.payload +}) +export type InboxEnvelope = typeof InboxEnvelope.Type + +export const InboxState = Schema.Literals([ + "Pending", + "Acknowledged", + "Rejected", + "DeadLettered", + "Expired" +]) +export type InboxState = typeof InboxState.Type + +export const TerminalOutcome = Schema.Literals(["Acknowledged", "Rejected"]) +export type TerminalOutcome = typeof TerminalOutcome.Type + +/** + * Identifies the ordered stream a message belongs to. + * + * Messages are ordered within a channel and independent across channels, so one stalled sender + * cannot hold up another sender's messages to the same device. A single queue per device would + * serialize every sender behind whichever message happens to be at the head. + */ +export const ChannelKey = Schema.Struct({ + tenantId: Schema.NonEmptyString, + senderSubjectId: Schema.NonEmptyString, + senderPeerId: Identity.PeerId, + senderReplicaIncarnation: Identity.ReplicaIncarnation, + /** + * Part of the channel identity, not merely payload. + * + * The sender allocates `senderSequence` per connection epoch, so it restarts at zero every time + * the sender reconnects. Without the epoch in the key, one channel would hold two runs of + * sequences and head selection would be arbitrary — and interleaving two epochs makes the + * recipient repeatedly reset the other epoch's sync state, discarding its pending replies. + */ + senderConnectionEpoch: Schema.NonEmptyString +}) +export type ChannelKey = typeof ChannelKey.Type + +export interface AdmissionRequest { + readonly inboxKey: string + readonly channel: ChannelKey + readonly envelope: InboxEnvelope + /** Wall clock at admission. Passed in so the caller owns the clock. */ + readonly now: number + /** How long an undelivered message survives before it is expired. */ + readonly messageTtlMillis: number + /** + * The retry horizon this sender negotiated for its session. + * + * Threaded per message rather than read from static configuration because the sender advertises + * it at `Open` and it can exceed any fixed server value. The deduplication horizon is derived + * from it, so a message's identity always outlives the window in which its sender may replay it. + * Without this, a replay can arrive after the identity was collected and be applied twice. + */ + readonly senderRetryHorizonMillis: number + /** Evaluated in the same transaction as the insert. */ + readonly quota: AdmissionQuota +} + +/** + * `Admitted` covers both a first admission and the revival of a message whose previous row had + * already been given up on. A sender that replays an identity still holds custody of it, and that + * is better evidence than the relay's own earlier decision to expire or dead-letter it. + * + * `Duplicate` reports the state of the row that is already present. It is only safe to treat as + * success while that row can still be delivered or has already been settled — on `Push` success + * the sender deletes its outbox entry, so reporting success for a `DeadLettered` or `Expired` row + * would make the sender's own recovery replay destroy the last durable copy of the message. + * + * `Conflict` means the same identity arrived carrying different content, which is never a legal + * retry and must not silently overwrite the stored message. + */ +export const AdmissionResult = Schema.Union([ + Schema.TaggedStruct("Admitted", {}), + Schema.TaggedStruct("Duplicate", { state: InboxState }), + Schema.TaggedStruct("Conflict", {}), + Schema.TaggedStruct("QuotaExceeded", {}) +]) +export type AdmissionResult = typeof AdmissionResult.Type + +export interface PendingMessage { + readonly relayMessageId: Identity.RelayMessageId + readonly channel: ChannelKey + readonly senderSequence: number + readonly deliveries: number + readonly envelope: InboxEnvelope +} + +/** + * Reports whether recording this delivery attempt exhausted the message's budget. + * + * A message that is repeatedly delivered but never settled — a recipient that reads it and then + * disconnects every time — would otherwise block its channel forever, because a disconnect is an + * interruption and interruptions are deliberately not counted as failures. Counting deliveries + * rather than failures is what bounds that, and `DeadLettered` keeps the outcome observable + * instead of silently dropping the message. + */ +export const DeliveryRecord = Schema.Union([ + Schema.TaggedStruct("Recorded", { deliveries: Schema.Int }), + Schema.TaggedStruct("DeadLettered", { deliveries: Schema.Int }) +]) +export type DeliveryRecord = typeof DeliveryRecord.Type + +/** + * Whether a settlement actually transitioned a row. + * + * `NotPending` means the message had already left `Pending` — settled by an earlier attempt, or + * abandoned by dead-lettering or expiry. `HashMismatch` means the identity matched but the content + * did not, which is never a legal settlement. + */ +export const SettleResult = Schema.Literals(["Settled", "NotPending", "HashMismatch"]) +export type SettleResult = typeof SettleResult.Type + +export interface AbandonedMessage { + readonly relayMessageId: Identity.RelayMessageId + readonly state: InboxState + readonly deliveries: number + readonly terminalAt: number +} + +export interface Usage { + readonly pendingCount: number + readonly pendingBytes: number + readonly retainedCount: number +} + +/** + * Caps evaluated inside the admission transaction. + * + * Passed with the request rather than read beforehand: the entity's rpc lane and its forked + * dispatcher fibers write to the same inbox concurrently, so a quota read followed by a separate + * insert is a race. The predicate has to live in the write itself. + */ +export interface AdmissionQuota { + readonly maxPendingMessages: number + readonly maxPendingBytes: number +} + +export type StoreError = ReplicaError.ReplicaError + +export class RelayInboxStore extends Context.Service Effect.Effect + + /** + * The oldest undelivered message of every channel in this inbox. + * + * One head per channel: the owner delivers heads concurrently across channels and strictly in + * order within a channel. + */ + readonly pendingHeads: ( + inboxKey: string, + options: { readonly limit: number } + ) => Effect.Effect, StoreError> + + readonly recordDelivery: ( + inboxKey: string, + relayMessageId: Identity.RelayMessageId, + options: { readonly maxDeliveries: number; readonly now: number } + ) => Effect.Effect + + /** + * Terminally settles a message. + * + * Takes the `messageHash` as part of the write predicate rather than trusting an in-memory + * check: the delivering entity's memory is lost on runner death and on defect restart, and the + * durable transition must not depend on state that can vanish. + * + * Reports whether it applied. A settle that silently matched no row — because the message was + * already dead-lettered, expired, or settled — must be distinguishable from success, otherwise + * the acknowledgement the recipient believes is durable never happened. + * + * `deduplicate_until` only ever grows. Shrinking it would reopen a deduplication window that a + * sender may still be replaying into. + */ + readonly settle: ( + inboxKey: string, + relayMessageId: Identity.RelayMessageId, + options: { + readonly outcome: TerminalOutcome + readonly messageHash: string + readonly now: number + readonly terminalRetentionMillis: number + } + ) => Effect.Effect + + readonly usage: (inboxKey: string) => Effect.Effect + + /** + * Messages this inbox has given up on. + * + * Dead-lettering and expiry destroy a message the sender already released custody of, so both + * have to be answerable after the fact rather than only appearing once in a log line. + */ + readonly abandoned: ( + inboxKey: string, + options: { readonly limit: number } + ) => Effect.Effect, StoreError> + + /** + * Moves undelivered messages past their TTL to `Expired`. + * + * Takes the retention window because expiry is a terminal transition like any other: the + * identity has to stay deduplicated for as long as its sender may still replay it. + */ + readonly expire: ( + options: { + readonly now: number + readonly limit: number + readonly terminalRetentionMillis: number + } + ) => Effect.Effect + + /** Removes terminal rows whose deduplication horizon has lapsed. */ + readonly collect: ( + options: { readonly now: number; readonly limit: number } + ) => Effect.Effect +}>()("@lucas-barake/effect-local-rpc/RelayInboxStore") {} diff --git a/packages/local-rpc/src/internal/relayInboxKey.ts b/packages/local-rpc/src/internal/relayInboxKey.ts new file mode 100644 index 0000000..bb3d556 --- /dev/null +++ b/packages/local-rpc/src/internal/relayInboxKey.ts @@ -0,0 +1,48 @@ +import * as Canonical from "@lucas-barake/effect-local/Canonical" +import type * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" +import type * as Crypto from "effect/Crypto" +import type * as Effect from "effect/Effect" +import type { PeerPrincipal } from "./peerPrincipal.js" + +/** + * The durable inbox of a peer device is addressed by a cluster `EntityId` derived from its full + * principal. Two properties are load bearing and neither is optional. + * + * Injectivity. `subjectId` is caller influenced text of up to 256 characters, so any delimiter + * joined encoding is forgeable: a subject containing the delimiter can be crafted to collide with + * a different tenant and subject pair. A collision routes one device's messages into another + * device's inbox and merges their deduplication namespaces, so it is a confidentiality boundary + * rather than a routing convenience. + * + * Bounded length. The backing `PersistedQueue` stores its queue name in a `VARCHAR(100)` column, + * and a principal alone can exceed that, so the encoding must be fixed width. + * + * Canonical digesting satisfies both. The principal is serialized as a structured value, which + * removes delimiter ambiguity entirely, and the digest is a fixed 64 character hex string. + */ +export const encodeInboxKey = ( + principal: PeerPrincipal +): Effect.Effect => + Canonical.digest({ + domain: inboxKeyDomain, + tenantId: principal.tenantId, + subjectId: principal.subjectId, + peerId: principal.peerId + }) + +/** + * Domain separator. Keeps inbox keys from ever coinciding with another digest in the system that + * happens to cover the same fields. + */ +export const inboxKeyDomain = "effect-local/relay-inbox-key" + +/** + * Deduplication identity for one message inside one inbox. + * + * `PersistedQueue` enforces uniqueness on the element id alone, not on `(queue_name, id)`, so ids + * from different inboxes share one namespace. Without the inbox key prefix, the same + * `relayMessageId` addressed to two devices would be treated as a duplicate and the second device + * would silently never receive its copy. + */ +export const encodeInboxElementId = (inboxKey: string, relayMessageId: string): string => + `${inboxKey}:${relayMessageId}` From 55d9041551002aaa3b8c51f4fd04a91921ecc072 Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Sun, 26 Jul 2026 23:43:44 -0500 Subject: [PATCH 09/29] Implement the relay inbox store over Effect SQL Backs the inbox contract with a single table over a generic SqlClient, so the operator chooses the database. Supports SQLite, PostgreSQL and MySQL, matching the coverage of the store this replaces. Every guard is a predicate inside the write it protects. The owning entity is the sole writer for an inbox, but its rpc lane and its forked dispatcher fibers issue statements concurrently, so a read followed by a separate write is still a race within one process. Ordering is indexed on a channel digest rather than the six raw channel columns, which would exceed MySQL's key length limit once the subject is sized for real input. The index is unique because head selection picks the lowest sequence in a channel, and two rows sharing one would make that choice arbitrary and could starve either indefinitely. Replaying an identity the inbox already abandoned revives it rather than reporting a duplicate. A duplicate is a success, and the sender deletes its outbox entry on success, so reporting one for a dead lettered or expired row would make the sender's own recovery destroy the last durable copy of the message. Deduplication horizons only ever grow, since shrinking one reopens a window a sender may still be replaying into. Integers are decoded through a schema that accepts strings because PostgreSQL returns BIGINT, COUNT and SUM that way and installs no default parser. The MySQL index path checks information_schema instead of ignoring errors: that dialect commits DDL implicitly and the migrator writes its completion marker before running the body, so a swallowed failure would leave a half built schema that later runs treat as done. --- packages/local-rpc/src/SqlRelayInboxStore.ts | 588 ++++++++++++++++++ .../src/internal/relayInboxMigrations.ts | 196 ++++++ 2 files changed, 784 insertions(+) create mode 100644 packages/local-rpc/src/SqlRelayInboxStore.ts create mode 100644 packages/local-rpc/src/internal/relayInboxMigrations.ts diff --git a/packages/local-rpc/src/SqlRelayInboxStore.ts b/packages/local-rpc/src/SqlRelayInboxStore.ts new file mode 100644 index 0000000..5c765dd --- /dev/null +++ b/packages/local-rpc/src/SqlRelayInboxStore.ts @@ -0,0 +1,588 @@ +import * as Canonical from "@lucas-barake/effect-local/Canonical" +import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" +import * as Crypto from "effect/Crypto" +import * as Effect from "effect/Effect" +import * as Layer from "effect/Layer" +import * as Option from "effect/Option" +import * as Schema from "effect/Schema" +import * as SqlClient from "effect/unstable/sql/SqlClient" +import * as SqlSchema from "effect/unstable/sql/SqlSchema" +import * as Migrations from "./internal/relayInboxMigrations.js" +import * as RelayInboxStore from "./RelayInboxStore.js" + +/** + * `RelayInboxStore` over a generic `SqlClient`, so the operator chooses the database. + * + * Every guard is expressed as a predicate inside the write that depends on it. The owning entity is + * the sole writer for an inbox, but its rpc lane and its forked dispatcher fibers issue statements + * concurrently, so a read followed by a separate write is still a race within one process. + */ + +const EnvelopeJson = Schema.fromJsonString( + Schema.toCodecJson(RelayInboxStore.InboxEnvelope) +) + +/** + * PostgreSQL returns `BIGINT`, `COUNT(*)` and `SUM(...)` as strings, and no default type parser is + * installed, so a bare `Schema.Number` would fail to decode every row on that dialect. + */ +const DatabaseInt = Schema.Union([Schema.Int, Schema.NumberFromString]).check(Schema.isInt()) + +/** Rows are decoded, never asserted: the database is an external boundary like any other. */ +const PendingRow = Schema.Struct({ + relay_message_id: Schema.String, + tenant_id: Schema.String, + sender_subject_id: Schema.String, + sender_peer_id: Schema.String, + sender_replica_incarnation: DatabaseInt, + sender_connection_epoch: Schema.String, + sender_sequence: DatabaseInt, + deliveries: DatabaseInt, + envelope: Schema.String, + message_hash: Schema.String, + outer_envelope_digest: Schema.String +}) + +const ExistingRow = Schema.Struct({ + state: Schema.String, + terminal_at: Schema.NullOr(DatabaseInt), + outer_envelope_digest: Schema.String, + message_hash: Schema.String +}) + +const CountRow = Schema.Struct({ + pending_count: DatabaseInt, + pending_bytes: DatabaseInt +}) + +const RetainedRow = Schema.Struct({ retained_count: DatabaseInt }) + +const AbandonedRow = Schema.Struct({ + relay_message_id: Schema.String, + state: Schema.String, + deliveries: DatabaseInt, + terminal_at: Schema.NullOr(DatabaseInt) +}) + +const StateRow = Schema.Struct({ state: Schema.String, deliveries: DatabaseInt }) + +const IdRow = Schema.Struct({ inbox_key: Schema.String, relay_message_id: Schema.String }) + +/** + * A channel's identity as a fixed width digest. + * + * Ordering is indexed on this rather than on the raw channel columns because a composite index over + * them would exceed MySQL's key length limit once the subject is sized for real input. + */ +const channelDigest = (channel: RelayInboxStore.ChannelKey) => + Canonical.digest({ + domain: "effect-local/relay-inbox-channel", + tenantId: channel.tenantId, + senderSubjectId: channel.senderSubjectId, + senderPeerId: channel.senderPeerId, + senderReplicaIncarnation: channel.senderReplicaIncarnation, + senderConnectionEpoch: channel.senderConnectionEpoch + }) + +const table = Migrations.tableName + +export const make = Effect.gen(function*() { + const sql = yield* SqlClient.SqlClient + // Captured once so the service's own methods carry no requirements of their own. + const crypto = yield* Crypto.Crypto + + yield* Migrations.run.pipe( + Effect.mapError((cause) => + new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) }) + ) + ) + + const decodeEnvelope = Schema.decodeUnknownEffect(EnvelopeJson) + const encodeEnvelope = Schema.encodeEffect(EnvelopeJson) + + const findExisting = SqlSchema.findOneOption({ + Request: Schema.Struct({ inboxKey: Schema.String, relayMessageId: Schema.String }), + Result: ExistingRow, + execute: (request) => + sql` + SELECT state, terminal_at, outer_envelope_digest, message_hash FROM ${sql(table)} + WHERE inbox_key = ${request.inboxKey} AND relay_message_id = ${request.relayMessageId} + ` + }) + + const findUsage = SqlSchema.findOne({ + Request: Schema.String, + Result: CountRow, + execute: (inboxKey) => + sql` + SELECT COUNT(*) AS pending_count, COALESCE(SUM(byte_size), 0) AS pending_bytes + FROM ${sql(table)} WHERE inbox_key = ${inboxKey} AND state = 'Pending' + ` + }) + + // SQLite spells the two argument maximum `MAX`; PostgreSQL and MySQL spell it `GREATEST`. + const greatestFn = sql.onDialectOrElse({ + orElse: () => "MAX", + pg: () => "GREATEST", + mysql: () => "GREATEST" + }) + + const admit = (request: RelayInboxStore.AdmissionRequest) => + sql.withTransaction(Effect.gen(function*() { + // The channel is redundant with the envelope's own sender fields, so a disagreement means + // the message would be filed under a fabricated ordering stream — the unique index would + // guard nothing and per channel ordering would silently break. + const sender = request.envelope.sender + if ( + request.channel.tenantId !== sender.tenantId || + request.channel.senderSubjectId !== sender.subjectId || + request.channel.senderPeerId !== sender.peerId || + request.channel.senderReplicaIncarnation !== sender.replicaIncarnation || + request.channel.senderConnectionEpoch !== sender.connectionEpoch + ) { + return { _tag: "Conflict" } as const + } + + const existing = yield* findExisting({ + inboxKey: request.inboxKey, + relayMessageId: request.envelope.relayMessageId + }) + + if (Option.isSome(existing)) { + const row = existing.value + // Conflicts are detected on the outer envelope digest, which binds sender, recipient, + // relay, sequence, epoch, document, lineage and provenance. The message hash covers only + // the inner payload, and both peers key their own deduplication on the digest. + if (row.outer_envelope_digest !== request.envelope.outerEnvelopeDigest) { + return { _tag: "Conflict" } as const + } + const state = yield* Schema.decodeUnknownEffect(RelayInboxStore.InboxState)(row.state).pipe( + Effect.mapError(() => + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause: `unknown inbox state ${row.state}` }) + }) + ) + ) + if (state !== "DeadLettered" && state !== "Expired") { + return { _tag: "Duplicate", state } as const + } + // The sender still holds custody and is replaying, which is better evidence than this + // inbox's earlier decision to give up. Revived as a fresh attempt so the message is not + // lost the moment the sender drops it from its outbox. + // + // A revive creates a `Pending` row, so it is charged against the same caps as a first + // admission; otherwise replaying abandoned identities would restore unbounded pending work. + const revivedUsage = yield* findUsage(request.inboxKey) + if (revivedUsage.pending_count + 1 > request.quota.maxPendingMessages) { + return { _tag: "QuotaExceeded" } as const + } + yield* sql` + UPDATE ${sql(table)} + SET state = 'Pending', deliveries = 0, terminal_at = NULL, + expires_at = ${request.now + request.messageTtlMillis}, + deduplicate_until = ${sql.literal(greatestFn)}( + deduplicate_until, + ${request.now + Math.max(request.messageTtlMillis, request.senderRetryHorizonMillis)} + ) + WHERE inbox_key = ${request.inboxKey} + AND relay_message_id = ${request.envelope.relayMessageId} + ` + return { _tag: "Admitted" } as const + } + + const usage = yield* findUsage(request.inboxKey) + const encoded = yield* encodeEnvelope(request.envelope).pipe( + Effect.mapError(() => + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause: "relay inbox envelope failed to encode" }) + }) + ) + ) + const byteSize = encoded.length + if ( + usage.pending_count + 1 > request.quota.maxPendingMessages || + usage.pending_bytes + byteSize > request.quota.maxPendingBytes + ) { + return { _tag: "QuotaExceeded" } as const + } + + const channelKey = yield* channelDigest(request.channel).pipe( + Effect.provideService(Crypto.Crypto, crypto) + ) + // The identity outlives the window in which the sender may still replay it. Derived from the + // horizon the sender negotiated rather than a fixed server value, because a shorter one would + // let a replay arrive after the record was collected and be applied a second time. + const deduplicateUntil = request.now + + Math.max(request.messageTtlMillis, request.senderRetryHorizonMillis) + + yield* sql` + INSERT INTO ${sql(table)} ( + inbox_key, relay_message_id, channel_key, tenant_id, sender_subject_id, sender_peer_id, + sender_replica_incarnation, sender_connection_epoch, sender_sequence, state, deliveries, + envelope, message_hash, outer_envelope_digest, byte_size, created_at, expires_at, + deduplicate_until, terminal_at + ) VALUES ( + ${request.inboxKey}, ${request.envelope.relayMessageId}, ${channelKey}, + ${request.channel.tenantId}, ${request.channel.senderSubjectId}, + ${request.channel.senderPeerId}, ${request.channel.senderReplicaIncarnation}, + ${request.channel.senderConnectionEpoch}, ${request.envelope.sender.sequence}, + 'Pending', 0, ${encoded}, ${request.envelope.messageHash}, + ${request.envelope.outerEnvelopeDigest}, ${byteSize}, ${request.now}, + ${request.now + request.messageTtlMillis}, ${deduplicateUntil}, NULL + ) + ` + return { _tag: "Admitted" } as const + })).pipe( + Effect.catchTag("SqlError", (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) }) + )), + Effect.catchTag("SchemaError", (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageCorrupt({ cause }) }) + )), + Effect.catchTag("NoSuchElementError", (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageCorrupt({ cause }) }) + )) + ) + + const findHeads = SqlSchema.findAll({ + Request: Schema.Struct({ inboxKey: Schema.String, limit: Schema.Int }), + Result: PendingRow, + execute: (request) => + sql` + SELECT relay_message_id, tenant_id, sender_subject_id, sender_peer_id, + sender_replica_incarnation, sender_connection_epoch, sender_sequence, + deliveries, envelope, message_hash, outer_envelope_digest + FROM ( + SELECT *, ROW_NUMBER() OVER ( + PARTITION BY channel_key ORDER BY sender_sequence ASC + ) AS rn + FROM ${sql(table)} + WHERE inbox_key = ${request.inboxKey} AND state = 'Pending' + ) heads + WHERE rn = 1 + ORDER BY created_at ASC, channel_key ASC + LIMIT ${sql.literal(String(request.limit))} + ` + }) + + const pendingHeads = (inboxKey: string, options: { readonly limit: number }) => + findHeads({ inboxKey, limit: options.limit }).pipe( + Effect.mapError((cause) => + new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) }) + ), + Effect.flatMap(Effect.forEach((row) => + Effect.gen(function*() { + const envelope = yield* decodeEnvelope(row.envelope).pipe( + Effect.mapError(() => + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause: `undecodable envelope for ${row.relay_message_id}` }) + }) + ) + ) + // The redundant columns exist to be checked. A row whose envelope disagrees with the + // identity and ordering it was filed under cannot be replayed safely. + if (envelope.messageHash !== row.message_hash) { + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause: `message hash mismatch for ${row.relay_message_id}` }) + }) + } + if (envelope.relayMessageId !== row.relay_message_id) { + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: `envelope identity disagrees with key column for ${row.relay_message_id}` + }) + }) + } + if (envelope.outerEnvelopeDigest !== row.outer_envelope_digest) { + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: `outer envelope digest mismatch for ${row.relay_message_id}` + }) + }) + } + if (envelope.sender.sequence !== row.sender_sequence) { + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause: `sequence mismatch for ${row.relay_message_id}` }) + }) + } + const channel = yield* Schema.decodeUnknownEffect(RelayInboxStore.ChannelKey)({ + tenantId: row.tenant_id, + senderSubjectId: row.sender_subject_id, + senderPeerId: row.sender_peer_id, + senderReplicaIncarnation: row.sender_replica_incarnation, + senderConnectionEpoch: row.sender_connection_epoch + }).pipe( + Effect.mapError(() => + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause: `invalid channel for ${row.relay_message_id}` }) + }) + ) + ) + return { + relayMessageId: envelope.relayMessageId, + channel, + senderSequence: row.sender_sequence, + deliveries: row.deliveries, + envelope + } satisfies RelayInboxStore.PendingMessage + }) + )) + ) + + const findState = SqlSchema.findOneOption({ + Request: Schema.Struct({ inboxKey: Schema.String, relayMessageId: Schema.String }), + Result: StateRow, + execute: (request) => + sql` + SELECT state, deliveries FROM ${sql(table)} + WHERE inbox_key = ${request.inboxKey} AND relay_message_id = ${request.relayMessageId} + ` + }) + + const recordDelivery = ( + inboxKey: string, + relayMessageId: string, + options: { readonly maxDeliveries: number; readonly now: number } + ) => + sql.withTransaction(Effect.gen(function*() { + // Incremented only for rows that are still `Pending`, so a message settled concurrently by + // its recipient cannot be charged for a delivery it no longer needs. + yield* sql` + UPDATE ${sql(table)} SET deliveries = deliveries + 1 + WHERE inbox_key = ${inboxKey} AND relay_message_id = ${relayMessageId} + AND state = 'Pending' + ` + const current = yield* findState({ inboxKey, relayMessageId }) + if (Option.isNone(current)) { + return { _tag: "Recorded", deliveries: 0 } as const + } + const deliveries = current.value.deliveries + if (current.value.state === "Pending" && deliveries >= options.maxDeliveries) { + yield* sql` + UPDATE ${sql(table)} + SET state = 'DeadLettered', terminal_at = ${options.now} + WHERE inbox_key = ${inboxKey} AND relay_message_id = ${relayMessageId} + AND state = 'Pending' + ` + return { _tag: "DeadLettered", deliveries } as const + } + return { _tag: "Recorded", deliveries } as const + })).pipe( + Effect.catchTag("SqlError", (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) }) + )), + Effect.catchTag("SchemaError", (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageCorrupt({ cause }) }) + )) + ) + + const settle = ( + inboxKey: string, + relayMessageId: string, + options: { + readonly outcome: RelayInboxStore.TerminalOutcome + readonly messageHash: string + readonly now: number + readonly terminalRetentionMillis: number + } + ) => + sql.withTransaction(Effect.gen(function*() { + const current = yield* findExisting({ inboxKey, relayMessageId }) + if (Option.isNone(current)) return "NotPending" as const + if (current.value.state !== "Pending") return "NotPending" as const + // Checked against the stored row rather than the delivering fiber's memory, which is lost on + // runner death and on defect restart. + if (current.value.message_hash !== options.messageHash) return "HashMismatch" as const + + const horizon = options.now + options.terminalRetentionMillis + // The horizon only ever grows. Shrinking it would reopen a deduplication window a sender may + // still be replaying into. + const greatest = greatestFn + yield* sql` + UPDATE ${sql(table)} + SET state = ${options.outcome}, terminal_at = ${options.now}, + deduplicate_until = ${sql.literal(greatest)}(deduplicate_until, ${horizon}) + WHERE inbox_key = ${inboxKey} AND relay_message_id = ${relayMessageId} + AND state = 'Pending' AND message_hash = ${options.messageHash} + ` + // Re-read rather than trusting the earlier one. The read above takes no lock at the default + // isolation of PostgreSQL and MySQL, so a sweeper could have moved the row out of `Pending` + // in between; reporting success then would tell the recipient an acknowledgement is durable + // when it never applied. + const after = yield* findExisting({ inboxKey, relayMessageId }) + if (Option.isNone(after)) return "NotPending" as const + return after.value.state === options.outcome && after.value.terminal_at === options.now + ? "Settled" as const + : "NotPending" as const + })).pipe( + Effect.catchTag("SqlError", (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) }) + )), + Effect.catchTag("SchemaError", (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageCorrupt({ cause }) }) + )) + ) + + const findRetained = SqlSchema.findOne({ + Request: Schema.String, + Result: RetainedRow, + execute: (inboxKey) => + sql` + SELECT COUNT(*) AS retained_count FROM ${sql(table)} + WHERE inbox_key = ${inboxKey} AND state <> 'Pending' + ` + }) + + const usage = (inboxKey: string) => + Effect.all([findUsage(inboxKey), findRetained(inboxKey)]).pipe( + Effect.mapError((cause) => + new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) }) + ), + Effect.map(([pending, retained]) => ({ + pendingCount: pending.pending_count, + pendingBytes: pending.pending_bytes, + retainedCount: retained.retained_count + })) + ) + + const findAbandoned = SqlSchema.findAll({ + Request: Schema.Struct({ inboxKey: Schema.String, limit: Schema.Int }), + Result: AbandonedRow, + execute: (request) => + sql` + SELECT relay_message_id, state, deliveries, terminal_at FROM ${sql(table)} + WHERE inbox_key = ${request.inboxKey} AND state IN ('DeadLettered', 'Expired') + ORDER BY terminal_at DESC + LIMIT ${sql.literal(String(request.limit))} + ` + }) + + const abandoned = (inboxKey: string, options: { readonly limit: number }) => + findAbandoned({ inboxKey, limit: options.limit }).pipe( + Effect.mapError((cause) => + new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) }) + ), + Effect.flatMap(Effect.forEach((row) => + Schema.decodeUnknownEffect(RelayInboxStore.InboxState)(row.state).pipe( + Effect.mapError(() => + new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ cause: `unknown inbox state ${row.state}` }) + }) + ), + Effect.map((state) => ({ + relayMessageId: row.relay_message_id as RelayInboxStore.AbandonedMessage["relayMessageId"], + state, + deliveries: row.deliveries, + terminalAt: row.terminal_at ?? 0 + })) + ) + )) + ) + + const selectExpired = SqlSchema.findAll({ + Request: Schema.Struct({ now: Schema.Number, limit: Schema.Int }), + Result: IdRow, + execute: (request) => + sql` + SELECT inbox_key, relay_message_id FROM ${sql(table)} + WHERE state = 'Pending' AND expires_at <= ${request.now} + LIMIT ${sql.literal(String(request.limit))} + ` + }) + + // `LIMIT` is not portable inside `UPDATE`/`DELETE`, so the batch is chosen by key first. + const expire = ( + options: { + readonly now: number + readonly limit: number + readonly terminalRetentionMillis: number + } + ) => + Effect.gen(function*() { + const rows = yield* selectExpired({ now: options.now, limit: options.limit }).pipe( + Effect.mapError((cause) => + new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) }) + ) + ) + if (rows.length === 0) return 0 + // Re-applies every predicate from the selection. Filtering on `relay_message_id` alone would + // reach rows in other inboxes, because the key is `(inbox_key, relay_message_id)` and the + // same identity can legitimately be addressed to more than one device. + yield* sql` + UPDATE ${sql(table)} + SET state = 'Expired', terminal_at = ${options.now}, + deduplicate_until = ${sql.literal(greatestFn)}( + deduplicate_until, ${options.now + options.terminalRetentionMillis} + ) + WHERE state = 'Pending' AND expires_at <= ${options.now} + AND ${ + sql.or(rows.map((row) => sql`(inbox_key = ${row.inbox_key} AND relay_message_id = ${row.relay_message_id})`)) + } + `.pipe(Effect.mapError((cause) => + new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) }) + )) + yield* Effect.logWarning("Relay inbox expired undelivered messages").pipe( + Effect.annotateLogs({ expired: rows.length }) + ) + return rows.length + }) + + const selectCollectable = SqlSchema.findAll({ + Request: Schema.Struct({ now: Schema.Number, limit: Schema.Int }), + Result: IdRow, + execute: (request) => + sql` + SELECT inbox_key, relay_message_id FROM ${sql(table)} + WHERE state <> 'Pending' AND deduplicate_until <= ${request.now} + LIMIT ${sql.literal(String(request.limit))} + ` + }) + + const collect = (options: { readonly now: number; readonly limit: number }) => + Effect.gen(function*() { + const rows = yield* selectCollectable({ now: options.now, limit: options.limit }).pipe( + Effect.mapError((cause) => + new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) }) + ) + ) + if (rows.length === 0) return 0 + yield* sql` + DELETE FROM ${sql(table)} + WHERE state <> 'Pending' AND deduplicate_until <= ${options.now} + AND ${ + sql.or(rows.map((row) => sql`(inbox_key = ${row.inbox_key} AND relay_message_id = ${row.relay_message_id})`)) + } + `.pipe(Effect.mapError((cause) => + new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) }) + )) + return rows.length + }) + + return RelayInboxStore.RelayInboxStore.of({ + admit, + pendingHeads, + recordDelivery, + settle, + usage, + abandoned, + expire, + collect + }) +}) satisfies Effect.Effect< + RelayInboxStore.RelayInboxStore["Service"], + ReplicaError.ReplicaError, + SqlClient.SqlClient | Crypto.Crypto +> + +export const layer: Layer.Layer< + RelayInboxStore.RelayInboxStore, + ReplicaError.ReplicaError, + SqlClient.SqlClient | Crypto.Crypto +> = Layer.effect(RelayInboxStore.RelayInboxStore)(make) diff --git a/packages/local-rpc/src/internal/relayInboxMigrations.ts b/packages/local-rpc/src/internal/relayInboxMigrations.ts new file mode 100644 index 0000000..136a8bb --- /dev/null +++ b/packages/local-rpc/src/internal/relayInboxMigrations.ts @@ -0,0 +1,196 @@ +import * as Effect from "effect/Effect" +import * as Schema from "effect/Schema" +import * as Migrator from "effect/unstable/sql/Migrator" +import * as SqlClient from "effect/unstable/sql/SqlClient" +import * as SqlSchema from "effect/unstable/sql/SqlSchema" + +/** + * Schema for the relay inbox. + * + * The lineage is pinned to its own table. The client replica lineage owns `effect_local_migrations` + * and the cluster's own storage owns `effect_sql_migrations`, and a deployment can put all three in + * one database, so sharing a table would collide migration ids across unrelated lineages. + */ +export const migratorTable = "effect_local_relay_inbox_migrations" + +export const tableName = "effect_local_relay_inbox" + +/** + * Identity columns are compared byte for byte. + * + * MySQL's default collation is case insensitive, which would let two principals that differ only by + * case address the same inbox — a route crossover between tenants rather than a cosmetic issue. + */ +const mysqlIdentity = (length: number) => `VARCHAR(${length}) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin` + +const createTable = Effect.gen(function*() { + const sql = yield* SqlClient.SqlClient + + /** + * MySQL has no `CREATE INDEX IF NOT EXISTS`, and swallowing the error instead would also hide a + * lock wait timeout, a dropped connection, or a key limit breach. Because MySQL commits DDL + * implicitly and the migrator writes its completion marker before running the migration, a + * silently skipped index would never be retried: the lineage would look applied while the unique + * constraint that makes head selection deterministic was simply missing. + */ + const countIndex = SqlSchema.findOne({ + Request: Schema.String, + Result: Schema.Struct({ + count: Schema.Union([Schema.Int, Schema.NumberFromString]).check(Schema.isInt()) + }), + execute: (name) => + sql` + SELECT COUNT(*) AS count FROM information_schema.statistics + WHERE table_schema = DATABASE() AND table_name = ${tableName} AND index_name = ${name} + ` + }) + + const createIndexMysql = (name: string, columns: string, unique: boolean) => + Effect.gen(function*() { + const existing = yield* countIndex(name) + if (existing.count > 0) return + yield* sql.unsafe( + `CREATE ${unique ? "UNIQUE " : ""}INDEX ${name} ON ${tableName} (${columns})` + ) + }) + + yield* sql.onDialectOrElse({ + orElse: () => + sql.unsafe(` + CREATE TABLE IF NOT EXISTS ${tableName} ( + inbox_key TEXT NOT NULL, + relay_message_id TEXT NOT NULL, + channel_key TEXT NOT NULL, + tenant_id TEXT NOT NULL, + sender_subject_id TEXT NOT NULL, + sender_peer_id TEXT NOT NULL, + sender_replica_incarnation INTEGER NOT NULL, + sender_connection_epoch TEXT NOT NULL, + sender_sequence INTEGER NOT NULL, + state TEXT NOT NULL, + deliveries INTEGER NOT NULL DEFAULT 0, + envelope TEXT NOT NULL, + message_hash TEXT NOT NULL, + outer_envelope_digest TEXT NOT NULL, + byte_size INTEGER NOT NULL, + created_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + deduplicate_until INTEGER NOT NULL, + terminal_at INTEGER, + PRIMARY KEY (inbox_key, relay_message_id) + ) + `), + pg: () => + sql.unsafe(` + CREATE TABLE IF NOT EXISTS ${tableName} ( + inbox_key TEXT NOT NULL, + relay_message_id TEXT NOT NULL, + channel_key TEXT NOT NULL, + tenant_id TEXT NOT NULL, + sender_subject_id TEXT NOT NULL, + sender_peer_id TEXT NOT NULL, + sender_replica_incarnation BIGINT NOT NULL, + sender_connection_epoch TEXT NOT NULL, + sender_sequence BIGINT NOT NULL, + state TEXT NOT NULL, + deliveries INTEGER NOT NULL DEFAULT 0, + envelope TEXT NOT NULL, + message_hash TEXT NOT NULL, + outer_envelope_digest TEXT NOT NULL, + byte_size BIGINT NOT NULL, + created_at BIGINT NOT NULL, + expires_at BIGINT NOT NULL, + deduplicate_until BIGINT NOT NULL, + terminal_at BIGINT, + PRIMARY KEY (inbox_key, relay_message_id) + ) + `), + mysql: () => + sql.unsafe(` + CREATE TABLE IF NOT EXISTS ${tableName} ( + inbox_key ${mysqlIdentity(64)} NOT NULL, + relay_message_id ${mysqlIdentity(64)} NOT NULL, + channel_key ${mysqlIdentity(64)} NOT NULL, + tenant_id ${mysqlIdentity(128)} NOT NULL, + sender_subject_id ${mysqlIdentity(256)} NOT NULL, + sender_peer_id ${mysqlIdentity(64)} NOT NULL, + sender_replica_incarnation BIGINT NOT NULL, + sender_connection_epoch ${mysqlIdentity(128)} NOT NULL, + sender_sequence BIGINT NOT NULL, + state ${mysqlIdentity(16)} NOT NULL, + deliveries INT NOT NULL DEFAULT 0, + envelope LONGTEXT NOT NULL, + message_hash ${mysqlIdentity(64)} NOT NULL, + outer_envelope_digest ${mysqlIdentity(64)} NOT NULL, + byte_size BIGINT NOT NULL, + created_at BIGINT NOT NULL, + expires_at BIGINT NOT NULL, + deduplicate_until BIGINT NOT NULL, + terminal_at BIGINT, + PRIMARY KEY (inbox_key, relay_message_id) + ) + `) + }) + + // Channel ordering. Indexed on the channel digest rather than the six raw channel columns: a + // composite index over them would exceed MySQL's key length limit once `sender_subject_id` is + // sized for real input, and the digest is fixed width. + // + // Unique because head selection picks the lowest `sender_sequence` in a channel. Two rows sharing + // a sequence would make that choice arbitrary and could starve one of them indefinitely. + yield* sql.onDialectOrElse({ + orElse: () => + sql.unsafe( + `CREATE UNIQUE INDEX IF NOT EXISTS ${tableName}_channel_sequence + ON ${tableName} (inbox_key, channel_key, sender_sequence)` + ), + mysql: () => + createIndexMysql( + `${tableName}_channel_sequence`, + "inbox_key, channel_key, sender_sequence", + true + ) + }) + + yield* sql.onDialectOrElse({ + orElse: () => + sql.unsafe( + `CREATE INDEX IF NOT EXISTS ${tableName}_pending_head + ON ${tableName} (inbox_key, state, channel_key, sender_sequence)` + ), + mysql: () => + createIndexMysql( + `${tableName}_pending_head`, + "inbox_key, state, channel_key, sender_sequence", + false + ) + }) + + yield* sql.onDialectOrElse({ + orElse: () => + sql.unsafe( + `CREATE INDEX IF NOT EXISTS ${tableName}_expiry ON ${tableName} (state, expires_at)` + ), + mysql: () => createIndexMysql(`${tableName}_expiry`, "state, expires_at", false) + }) + + yield* sql.onDialectOrElse({ + orElse: () => + sql.unsafe( + `CREATE INDEX IF NOT EXISTS ${tableName}_collect ON ${tableName} (state, deduplicate_until)` + ), + mysql: () => createIndexMysql(`${tableName}_collect`, "state, deduplicate_until", false) + }) +}) + +export const loader = Migrator.fromRecord({ + "1_create_relay_inbox": createTable +}) + +/** + * MySQL commits DDL implicitly, and the migrator inserts its completion marker *before* running the + * migration body, so an interrupted run leaves a partially built schema that later runs treat as + * complete. Every statement above is therefore individually idempotent and individually verified + * rather than relying on transactional rollback, which this dialect does not provide for DDL. + */ +export const run = Migrator.make({})({ loader, table: migratorTable }) From e9faf30c6781ed8e47eb95e9e70a03116386a345 Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Sun, 26 Jul 2026 23:54:49 -0500 Subject: [PATCH 10/29] Add the relay front door over the existing wire contract Terminates the client socket, authenticates and authorizes each request, and forwards it to the cluster entity that owns the addressed device. The wire contract is unchanged, so this is a swap of the server's internals rather than a new protocol. Which inbox a request reaches is derived per operation and never taken from the request body. Open, Acknowledge and Reject address the caller's own inbox, computed from the authenticated principal, so settling another device's message is not expressible. Push addresses the counterparty the handshake authorized, so a session can only ever send where it was allowed to. Handshake state is per connection. A socket lives on one node and dies with it, so a client that loses this node reconnects and opens again. What could not be process local is the recipient's delivery session, because a message accepted on any node has to reach whichever node holds that recipient's socket, and that is exactly what the entity provides. The sender's message hash is recomputed rather than believed, since it is the identity the recipient deduplicates on. Every entity call is bounded by a timeout: sharding retries an unassigned entity or an unavailable runner indefinitely, so an unbounded call would hang through a rebalance and leak a fiber per in-flight request. A live session is kept alive by a heartbeat driven from here, because what must be proven alive is this node and its socket rather than the peer at the other end. --- packages/local-rpc/src/RelayServer.ts | 396 ++++++++++++++++++++++++++ 1 file changed, 396 insertions(+) create mode 100644 packages/local-rpc/src/RelayServer.ts diff --git a/packages/local-rpc/src/RelayServer.ts b/packages/local-rpc/src/RelayServer.ts new file mode 100644 index 0000000..38330d1 --- /dev/null +++ b/packages/local-rpc/src/RelayServer.ts @@ -0,0 +1,396 @@ +import * as PeerSyncEnvelope from "@lucas-barake/effect-local-sql/PeerSyncEnvelope" +import * as Canonical from "@lucas-barake/effect-local/Canonical" +import * as Identity from "@lucas-barake/effect-local/Identity" +import * as Clock from "effect/Clock" +import * as Crypto from "effect/Crypto" +import * as Effect from "effect/Effect" +import * as Layer from "effect/Layer" +import * as Schedule from "effect/Schedule" +import * as Schema from "effect/Schema" +import * as Stream from "effect/Stream" +import { encodeInboxKey } from "./internal/relayInboxKey.js" +import * as PeerAuthentication from "./PeerAuthentication.js" +import * as PeerRelayAuthorization from "./PeerRelayAuthorization.js" +import * as PeerRelayLimits from "./PeerRelayLimits.js" +import * as PeerRpc from "./PeerRpc.js" +import * as PeerRpcError from "./PeerRpcError.js" +import * as RelayInbox from "./RelayInbox.js" + +/** + * The relay's public front door. + * + * Terminates the client socket, authenticates and authorizes each request, and forwards it to the + * cluster entity that owns the addressed device. It holds no custody of its own: every durable + * decision belongs to an entity, so any node can serve any client and a node failure costs only the + * connections it was terminating. + * + * Handshake state is deliberately per connection rather than in the cluster. A socket lives on + * exactly one node and dies with it, so a client that loses this node simply reconnects and opens + * again. What could not be process local is the *recipient's* delivery session, because a message + * accepted anywhere has to reach whichever node holds that recipient's socket — and that is what + * the entity provides. + */ + +export interface Options { + /** The tenant this relay serves. Requests authenticated into any other tenant are refused. */ + readonly tenantId: string + /** The relay's own peer identity, which clients pin at handshake. */ + readonly peerId: Identity.PeerId + /** + * How often a live session is kept alive with the entity that owns it. + * + * Driven from here rather than by the client, because what has to be proven alive is this node + * and its socket, not the peer at the other end. + */ + readonly heartbeatIntervalMillis: number + /** + * How long to wait on an entity call before giving up. + * + * Sharding retries `EntityNotAssignedToRunner` and `RunnerUnavailable` indefinitely, so without a + * bound a request during a rebalance would hang forever and leak a fiber per in-flight call. + */ + readonly entityCallTimeoutMillis: number +} + +/** Everything the handshake established, held for the life of one client connection. */ +interface Session { + readonly sessionId: Identity.SessionId + readonly principal: PeerAuthentication.PeerPrincipal + readonly remote: PeerAuthentication.PeerPrincipal + readonly documents: ReadonlyArray + readonly senderReplicaIncarnation: Identity.ReplicaIncarnation + readonly senderRetryHorizonMillis: number + readonly inboxKeySelf: string + readonly inboxKeyRemote: string +} + +const RelayEnvelopeJson = Schema.fromJsonString( + Schema.toCodecJson(PeerSyncEnvelope.SyncEnvelope) +) + +export const layerHandlers = (options: Options) => + PeerRpc.Rpcs.toLayer(Effect.gen(function*() { + const authorization = yield* PeerRelayAuthorization.PeerRelayAuthorization + const limits = yield* PeerRelayLimits.PeerRelayLimits + const inboxClient = yield* RelayInbox.RelayInbox.client + // Resolved here so no handler carries a service in its own requirement channel. + const crypto = yield* Crypto.Crypto + + const sessions = new Map() + const decodeEnvelope = Schema.decodeUnknownEffect(RelayEnvelopeJson) + + /** + * Bounds an entity call and reports exhaustion as unavailability. + * + * Applied at each call site rather than wrapped around the client, so the timeout is visible + * where the request is made. + */ + const bounded = (effect: Effect.Effect) => + effect.pipe( + Effect.timeoutOrElse({ + duration: options.entityCallTimeoutMillis, + orElse: () => Effect.fail(new PeerRpcError.ServerUnavailable()) + }) + ) + + /** + * Resolves the caller's session. + * + * A session the front door does not know is reported as unavailable rather than as an + * authentication problem: the credential may be perfectly valid and simply belong to a session + * this node never opened or has already replaced. + */ + const sessionFor = (sessionId: Identity.SessionId, principal: PeerAuthentication.PeerPrincipal) => + Effect.suspend(() => { + const session = sessions.get(sessionId) + if (session === undefined) return Effect.fail(new PeerRpcError.SessionUnavailable()) + // A session id is unguessable, but it must still belong to the caller presenting it. + if ( + session.principal.tenantId !== principal.tenantId || + session.principal.subjectId !== principal.subjectId || + session.principal.peerId !== principal.peerId + ) { + return Effect.fail(new PeerRpcError.SessionUnavailable()) + } + return Effect.succeed(session) + }) + + const settle = ( + payload: { + readonly sessionId: Identity.SessionId + readonly relayMessageId: Identity.RelayMessageId + readonly claimToken: PeerRpc.ClaimToken + readonly messageHash: string + }, + outcome: "Acknowledged" | "Rejected" + ) => + Effect.gen(function*() { + const authenticated = yield* PeerAuthentication.AuthenticatedPeer + const session = yield* sessionFor(payload.sessionId, authenticated.principal) + // Keyed by the caller's own inbox, so settling another device's message is not expressible. + yield* bounded( + inboxClient(session.inboxKeySelf).Settle({ + sessionId: payload.sessionId, + relayMessageId: payload.relayMessageId, + claimToken: payload.claimToken, + messageHash: payload.messageHash, + outcome + }) + ).pipe( + Effect.catchTag("MailboxFull", () => new PeerRpcError.SessionOverloaded()), + Effect.catchTag("AlreadyProcessingMessage", () => new PeerRpcError.SessionOverloaded()), + Effect.catchTag("PersistenceError", () => new PeerRpcError.ServerUnavailable()) + ) + }) + + return PeerRpc.Rpcs.of({ + Open: (payload) => + Stream.unwrap(Effect.gen(function*() { + const authenticated = yield* PeerAuthentication.AuthenticatedPeer + const principal = authenticated.principal + const now = yield* Clock.currentTimeMillis + + if (authenticated.validUntil <= now) { + return yield* new PeerRpcError.AuthenticationFailure() + } + if (payload.protocolVersion !== PeerRpc.protocolVersion) { + return yield* new PeerRpcError.UnsupportedVersion() + } + // The client states who it believes it is and which relay it believes it reached. Both + // are checked against the authenticated identity rather than trusted, so a credential can + // never be used to open a session for another device. + if ( + payload.expectedRelayPeerId !== options.peerId || + payload.expectedLocal.tenantId !== principal.tenantId || + payload.expectedLocal.subjectId !== principal.subjectId || + payload.expectedLocal.peerId !== principal.peerId + ) { + return yield* new PeerRpcError.PeerMismatch() + } + if (principal.tenantId !== options.tenantId) { + return yield* new PeerRpcError.AccessDenied() + } + // The negotiated windows have to nest: a receipt must outlive both the message and the + // window in which its sender may replay it, or a redelivery could be applied twice. + if ( + payload.senderRetryHorizonMillis > limits.maximumSenderRetryHorizonMillis || + payload.receiptRetentionMillis > limits.maximumReceiptRetentionMillis || + payload.receiptRetentionMillis < + Math.max(limits.messageTtlMillis, payload.senderRetryHorizonMillis) + + limits.minimumTerminalRetentionMillis + ) { + return yield* new PeerRpcError.InvalidRequest() + } + + const send = yield* authorization.authorize({ + direction: "Send", + principal, + remote: payload.remote, + documents: payload.documents + }) + const receive = yield* authorization.authorize({ + direction: "Receive", + principal, + remote: payload.remote, + documents: payload.documents + }) + // Authorization is not instantaneous, so both grants and the credential are rechecked + // against the clock afterwards rather than against the time the request arrived. + const authorizedAt = yield* Clock.currentTimeMillis + if (authenticated.validUntil <= authorizedAt) { + return yield* new PeerRpcError.AuthenticationFailure() + } + if (send.validUntil <= authorizedAt || receive.validUntil <= authorizedAt) { + return yield* new PeerRpcError.AccessDenied() + } + + const sessionId = yield* Identity.makeSessionId.pipe( + Effect.provideService(Crypto.Crypto, crypto), + // The platform's randomness failing is an environment fault, not something the client + // can act on, so it is reported as the relay being unavailable. + Effect.catchTag("PlatformError", () => new PeerRpcError.ServerUnavailable()) + ) + const inboxKeySelf = yield* encodeInboxKey(principal).pipe( + Effect.provideService(Crypto.Crypto, crypto), + Effect.catchTag("ReplicaError", () => new PeerRpcError.ServerUnavailable()) + ) + const inboxKeyRemote = yield* encodeInboxKey(send.remote).pipe( + Effect.provideService(Crypto.Crypto, crypto), + Effect.catchTag("ReplicaError", () => new PeerRpcError.ServerUnavailable()) + ) + + const session: Session = { + sessionId, + principal, + remote: send.remote, + documents: payload.documents, + senderReplicaIncarnation: payload.senderReplicaIncarnation, + senderRetryHorizonMillis: payload.senderRetryHorizonMillis, + inboxKeySelf, + inboxKeyRemote + } + + sessions.set(sessionId, session) + yield* Effect.addFinalizer(() => + Effect.sync(() => { + sessions.delete(sessionId) + }).pipe( + // Best effort: the entity's own liveness deadline is what actually bounds an + // abandoned session, because this never runs when the node itself fails. + Effect.andThen(Effect.ignore(inboxClient(inboxKeySelf).EndSession({ sessionId }))) + ) + ) + + // Proves this node and its socket are still alive. Stopping is the signal. + yield* Effect.ignore(inboxClient(inboxKeySelf).Heartbeat({ sessionId })).pipe( + Effect.repeat(Schedule.spaced(options.heartbeatIntervalMillis)), + Effect.forkScoped + ) + + const opened: PeerRpc.OpenEvent = { + _tag: "Opened", + protocolVersion: PeerRpc.protocolVersion, + sessionId, + remotePeerId: send.remote.peerId, + authenticatedLocal: principal + } + + // Emitted only once the subscription exists, so a client that sees `Opened` has a session + // that is genuinely receiving rather than one that failed to attach. + return Stream.concat( + Stream.succeed(opened), + inboxClient(inboxKeySelf).Subscribe({ sessionId }).pipe( + // Cluster level failures are not part of the public contract, so each is reported as + // the wire error that tells the client what to do about it. + Stream.catchTag("MailboxFull", () => Stream.fail(new PeerRpcError.SessionOverloaded())), + Stream.catchTag( + "AlreadyProcessingMessage", + () => Stream.fail(new PeerRpcError.SessionOverloaded()) + ), + Stream.catchTag( + "PersistenceError", + () => Stream.fail(new PeerRpcError.ServerUnavailable()) + ) + ) + ) + })), + + Push: (payload) => + Effect.gen(function*() { + const authenticated = yield* PeerAuthentication.AuthenticatedPeer + const session = yield* sessionFor(payload.sessionId, authenticated.principal) + + // The recipient is fixed by the handshake, never taken from the request, so a session can + // only ever send to the counterparty it was authorized for. + const envelope = yield* decodeEnvelope(new TextDecoder().decode(payload.payload)).pipe( + Effect.catchTag("SchemaError", () => new PeerRpcError.InvalidRequest()) + ) + const document = session.documents.find((candidate) => + candidate.documentId === envelope.documentId && + candidate.documentType === envelope.documentType + ) + if (document === undefined) { + return yield* new PeerRpcError.InvalidRequest() + } + // The sender's own hash is recomputed rather than believed: it is the identity the + // recipient will deduplicate on, so an unchecked one lets a sender mislabel its content. + const messageHash = yield* Canonical.digest(envelope.message).pipe( + Effect.provideService(Crypto.Crypto, crypto), + Effect.catchTag("ReplicaError", () => new PeerRpcError.ServerUnavailable()) + ) + if (messageHash !== envelope.messageHash) { + return yield* new PeerRpcError.InvalidRequest() + } + + const send = yield* authorization.authorize({ + direction: "Send", + principal: session.principal, + remote: session.remote, + documents: [document] + }) + if (send.documents.length === 0) { + return yield* new PeerRpcError.AccessDenied() + } + + const outerEnvelopeDigest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope({ + domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, + version: PeerSyncEnvelope.relayOuterEnvelopeVersion, + expectedLocal: session.principal, + remote: session.remote, + relayPeerId: options.peerId, + relayMessageId: payload.relayMessageId, + protocolVersion: PeerRpc.protocolVersion, + payloadVersion: 1, + senderReplicaIncarnation: session.senderReplicaIncarnation, + senderConnectionEpoch: envelope.connectionEpoch, + senderSequence: envelope.sequence, + document: { + documentId: envelope.documentId, + documentType: envelope.documentType + }, + lineage: envelope.lineage, + writerProvenance: envelope.writerProvenance, + messageHash: envelope.messageHash, + payload: payload.payload + }).pipe( + Effect.provideService(Crypto.Crypto, crypto), + Effect.catchTag("ReplicaError", () => new PeerRpcError.ServerUnavailable()) + ) + + const sender = { + tenantId: session.principal.tenantId, + subjectId: session.principal.subjectId, + peerId: session.principal.peerId, + replicaIncarnation: session.senderReplicaIncarnation, + connectionEpoch: envelope.connectionEpoch, + sequence: envelope.sequence + } + + yield* bounded( + inboxClient(session.inboxKeyRemote).Deliver({ + channel: { + tenantId: session.principal.tenantId, + senderSubjectId: session.principal.subjectId, + senderPeerId: session.principal.peerId, + senderReplicaIncarnation: session.senderReplicaIncarnation, + senderConnectionEpoch: envelope.connectionEpoch + }, + envelope: { + relayMessageId: payload.relayMessageId, + relayPeerId: options.peerId, + sender, + recipient: session.remote, + payloadVersion: 1, + document: { + documentId: envelope.documentId, + documentType: envelope.documentType + }, + writerProvenance: envelope.writerProvenance, + messageHash: envelope.messageHash, + outerEnvelopeDigest, + payload: payload.payload + }, + senderRetryHorizonMillis: session.senderRetryHorizonMillis + }) + ).pipe( + Effect.catchTag("MailboxFull", () => new PeerRpcError.SessionOverloaded()), + Effect.catchTag("AlreadyProcessingMessage", () => new PeerRpcError.SessionOverloaded()), + Effect.catchTag("PersistenceError", () => new PeerRpcError.ServerUnavailable()) + ) + }), + + Acknowledge: (payload) => settle(payload, "Acknowledged"), + + Reject: (payload) => settle(payload, "Rejected") + }) + })) + +/** + * The relay server: the front door plus the entity behaviour it forwards to. + * + * Requires a `Sharding` but never builds one, so the deployment shape stays the consumer's choice — + * an in-memory single process for tests, one runner over SQL, or many runners sharded across + * machines — without the relay changing. + */ +export const layer = (options: Options & { readonly inbox: RelayInbox.Options }) => + Layer.merge(layerHandlers(options), RelayInbox.layer(options.inbox)) From 6bdab08c6e48725bb057be8686483557888bedc0 Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Mon, 27 Jul 2026 00:23:06 -0500 Subject: [PATCH 11/29] Enforce authorization for the life of a relay session Two guards from the previous server were missing, and both let an unauthorized peer keep receiving. Receive authorization only ran at the handshake and its result was then discarded, so the delivery stream was piped unfiltered. Anything holding a Send grant to a device can write into that device's inbox, and the entity has no concept of grants, so a recipient received documents it never requested and was never authorized to receive. Every delivery is now re-authorized, and a message that fails is withheld rather than emitted, staying durable for a later session instead of erroring the stream. A session also outlived the credential and grants that created it. All three expose an invalidated signal and a deadline, none of which the client observes, so the relay is the only place that can act on them. Nothing did, and the relay's own heartbeat kept the session alive for as long as the socket stayed open. A session now ends as soon as any of its authority is withdrawn or lapses. Also: an Open beyond the per subject session cap is refused, since sessions were the front door's only unbounded resource and an authenticated client can open them in a loop; an envelope that decodes but cannot be canonicalized is reported as permanently invalid rather than as unavailability, which had the sender's outbox retrying it forever at the cost of a canonical encode each time; and a heartbeat interval that cannot outrun the entity's session deadline is refused at construction, because that combination silently reaps every session. --- packages/local-rpc/src/RelayServer.ts | 94 ++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 3 deletions(-) diff --git a/packages/local-rpc/src/RelayServer.ts b/packages/local-rpc/src/RelayServer.ts index 38330d1..1fff951 100644 --- a/packages/local-rpc/src/RelayServer.ts +++ b/packages/local-rpc/src/RelayServer.ts @@ -204,6 +204,21 @@ export const layerHandlers = (options: Options) => return yield* new PeerRpcError.AccessDenied() } + // Sessions are the front door's only unbounded resource, and an authenticated client can + // open them in a loop. Checked before minting an id so a refusal costs nothing. + let heldBySubject = 0 + for (const held of sessions.values()) { + if ( + held.principal.tenantId === principal.tenantId && + held.principal.subjectId === principal.subjectId + ) { + heldBySubject += 1 + } + } + if (heldBySubject >= limits.maxSessionsPerSubject) { + return yield* new PeerRpcError.RequestCapacityExceeded() + } + const sessionId = yield* Identity.makeSessionId.pipe( Effect.provideService(Crypto.Crypto, crypto), // The platform's randomness failing is an environment fault, not something the client @@ -241,6 +256,31 @@ export const layerHandlers = (options: Options) => ) ) + // A session must not outlive what authorized it. The credential and both grants each + // expose an `invalidated` signal and a deadline, and none of them is observed by the + // client, so the relay is the only place that can end the session when authority is + // withdrawn. Without this the heartbeat below would keep an unauthorized session alive + // for as long as the socket stays open. + const authorityLapsesAt = Math.min( + authenticated.validUntil, + send.validUntil, + receive.validUntil + ) + yield* Effect.raceAll([ + authenticated.invalidated, + send.invalidated, + receive.invalidated, + Effect.sleep(Math.max(0, authorityLapsesAt - authorizedAt)) + ]).pipe( + Effect.andThen( + Effect.logInfo("Relay session authority withdrawn; ending session").pipe( + Effect.annotateLogs({ sessionId }) + ) + ), + Effect.andThen(Effect.ignore(inboxClient(inboxKeySelf).EndSession({ sessionId }))), + Effect.forkScoped + ) + // Proves this node and its socket are still alive. Stopping is the signal. yield* Effect.ignore(inboxClient(inboxKeySelf).Heartbeat({ sessionId })).pipe( Effect.repeat(Schedule.spaced(options.heartbeatIntervalMillis)), @@ -255,11 +295,39 @@ export const layerHandlers = (options: Options) => authenticatedLocal: principal } - // Emitted only once the subscription exists, so a client that sees `Opened` has a session - // that is genuinely receiving rather than one that failed to attach. + // `Stream.concat` builds the second stream only after the first completes, so the + // subscription attaches after `Opened` reaches the wire. A client that sees `Opened` has + // a session the relay accepted, not yet one the owning entity has attached. return Stream.concat( Stream.succeed(opened), inboxClient(inboxKeySelf).Subscribe({ sessionId }).pipe( + // Re-authorized per message rather than once at handshake. A grant can be narrowed + // or revoked mid session, and anything with a Send grant to this device can write + // into its inbox, so the handshake's Receive decision cannot stand in for the + // recipient's right to see a particular document. A message that fails here is left + // unsettled and simply not emitted, so it stays durable for a later session. + Stream.filterEffect((message) => + authorization.authorize({ + direction: "Receive", + principal: session.principal, + remote: session.remote, + documents: [{ + documentType: message.document.documentType, + documentId: message.document.documentId + }] + }).pipe( + Effect.as(true), + Effect.catchTag("AccessDenied", () => + Effect.logInfo("Withheld a delivery the recipient may no longer receive").pipe( + Effect.annotateLogs({ + sessionId, + documentId: message.document.documentId + }), + Effect.as(false) + )), + Effect.catchTag("ServerUnavailable", () => Effect.succeed(false)) + ) + ), // Cluster level failures are not part of the public contract, so each is reported as // the wire error that tells the client what to do about it. Stream.catchTag("MailboxFull", () => Stream.fail(new PeerRpcError.SessionOverloaded())), @@ -334,6 +402,10 @@ export const layerHandlers = (options: Options) => payload: payload.payload }).pipe( Effect.provideService(Crypto.Crypto, crypto), + // A client can produce an envelope that decodes yet fails to canonicalize, so this is + // a permanent rejection. Reporting it as unavailability would have the sender's outbox + // retry it forever, paying a full canonical encode on the relay each time. + Effect.catchReason("ReplicaError", "ProtocolMismatch", () => new PeerRpcError.InvalidRequest()), Effect.catchTag("ReplicaError", () => new PeerRpcError.ServerUnavailable()) ) @@ -393,4 +465,20 @@ export const layerHandlers = (options: Options) => * machines — without the relay changing. */ export const layer = (options: Options & { readonly inbox: RelayInbox.Options }) => - Layer.merge(layerHandlers(options), RelayInbox.layer(options.inbox)) + Layer.merge(layerHandlers(options), RelayInbox.layer(options.inbox)).pipe( + // Two independently configured values have to agree or every session is reaped on a fixed + // cycle and no client can hold a delivery stream — a total outage that only appears under a + // particular configuration, so it is refused at construction rather than discovered in + // production. The factor of two leaves room for one lost heartbeat. + Layer.provide(Layer.effectDiscard( + options.heartbeatIntervalMillis * 2 < options.inbox.sessionDeadlineMillis && + options.entityCallTimeoutMillis > 0 + ? Effect.void + : Effect.die( + new Error( + "RelayServer: heartbeatIntervalMillis * 2 must be less than " + + "inbox.sessionDeadlineMillis, and entityCallTimeoutMillis must be positive" + ) + ) + )) + ) From cdf6e3c4500492f72566856c7488cdc4e7356a9c Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Mon, 27 Jul 2026 00:33:35 -0500 Subject: [PATCH 12/29] Cover the relay inbox store against real SQLite Nineteen cases over the production store and its real migrations, so the properties the design depends on are asserted rather than argued. Most of these exist because a review found the corresponding defect by running the store rather than reading it. The cross inbox sweep is the clearest: the key is (inbox, id), the same relay message id can legally address two devices, and a sweep filtered on the id alone destroyed a message in an inbox that had most of its lifetime left. Writing them found one more. The delivery budget off by one had been reported and I believed I had fixed it, but the edit never applied and nothing failed; a budget of one dead lettered the message on its first delivery, wasting the only attempt the recipient could have settled. The test is what noticed, which is the argument for having it. Also covered: reviving a message the inbox gave up on rather than reporting a duplicate the sender would treat as permission to drop its last copy; charging that revive against the quota; keeping its identity deduplicated across the sender's replay window; refusing a message whose channel disagrees with its envelope; and reaching every channel regardless of its own sequence numbering. --- packages/local-rpc/src/SqlRelayInboxStore.ts | 4 +- .../local-rpc/test/RelayInboxStore.test.ts | 395 ++++++++++++++++++ 2 files changed, 398 insertions(+), 1 deletion(-) create mode 100644 packages/local-rpc/test/RelayInboxStore.test.ts diff --git a/packages/local-rpc/src/SqlRelayInboxStore.ts b/packages/local-rpc/src/SqlRelayInboxStore.ts index 5c765dd..14692a8 100644 --- a/packages/local-rpc/src/SqlRelayInboxStore.ts +++ b/packages/local-rpc/src/SqlRelayInboxStore.ts @@ -360,7 +360,9 @@ export const make = Effect.gen(function*() { return { _tag: "Recorded", deliveries: 0 } as const } const deliveries = current.value.deliveries - if (current.value.state === "Pending" && deliveries >= options.maxDeliveries) { + // Strictly greater: the attempt that spends the last of the budget is still a real delivery + // the recipient can settle, so dead lettering at equality would waste it. + if (current.value.state === "Pending" && deliveries > options.maxDeliveries) { yield* sql` UPDATE ${sql(table)} SET state = 'DeadLettered', terminal_at = ${options.now} diff --git a/packages/local-rpc/test/RelayInboxStore.test.ts b/packages/local-rpc/test/RelayInboxStore.test.ts new file mode 100644 index 0000000..9b55c38 --- /dev/null +++ b/packages/local-rpc/test/RelayInboxStore.test.ts @@ -0,0 +1,395 @@ +import { NodeCrypto } from "@effect/platform-node" +import { SqliteClient } from "@effect/sql-sqlite-node" +import { assert, describe, it } from "@effect/vitest" +import * as Identity from "@lucas-barake/effect-local/Identity" +import * as Effect from "effect/Effect" +import * as Layer from "effect/Layer" +import * as RelayInboxStore from "../src/RelayInboxStore.js" +import * as SqlRelayInboxStore from "../src/SqlRelayInboxStore.js" + +const peer = (value: string) => Identity.PeerId.make(`peer_00000000-0000-4000-8000-${value}`) +const relayId = (value: string) => Identity.RelayMessageId.make(`rly_00000000-0000-4000-8000-${value}`) +const documentId = (value: string) => Identity.DocumentId.make(`doc_00000000-0000-4000-8000-${value}`) + +const layer = SqlRelayInboxStore.layer.pipe( + Layer.provide(SqliteClient.layer({ filename: ":memory:", disableWAL: true })), + Layer.provide(NodeCrypto.layer), + Layer.orDie +) + +const quota = { maxPendingMessages: 100, maxPendingBytes: 10_000_000 } + +const channel = (options?: { readonly epoch?: string; readonly subject?: string }) => ({ + tenantId: "tenant-a", + senderSubjectId: options?.subject ?? "sender-a", + senderPeerId: peer("00000000aaa1"), + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + senderConnectionEpoch: options?.epoch ?? "epoch-1" +}) + +const envelope = (options: { + readonly id: string + readonly sequence: number + readonly digest?: string + readonly epoch?: string + readonly subject?: string +}) => { + const source = channel({ epoch: options.epoch, subject: options.subject }) + return { + relayMessageId: relayId(options.id), + relayPeerId: peer("00000000ffff"), + sender: { + tenantId: source.tenantId, + subjectId: source.senderSubjectId, + peerId: source.senderPeerId, + replicaIncarnation: source.senderReplicaIncarnation, + connectionEpoch: source.senderConnectionEpoch, + sequence: options.sequence + }, + recipient: { + tenantId: "tenant-a", + subjectId: "recipient-a", + peerId: peer("00000000bbb1") + }, + payloadVersion: 1 as const, + document: { documentId: documentId("00000000dddd"), documentType: "note" }, + writerProvenance: [], + messageHash: "a".repeat(64), + outerEnvelopeDigest: (options.digest ?? "b").repeat(64).slice(0, 64), + payload: new Uint8Array([1, 2, 3]) + } +} + +const admission = (options: { + readonly inboxKey: string + readonly id: string + readonly sequence: number + readonly now: number + readonly ttl?: number + readonly horizon?: number + readonly digest?: string + readonly epoch?: string + readonly subject?: string + readonly quota?: RelayInboxStore.AdmissionQuota +}) => ({ + inboxKey: options.inboxKey, + channel: channel({ epoch: options.epoch, subject: options.subject }), + envelope: envelope(options), + now: options.now, + messageTtlMillis: options.ttl ?? 1_000, + senderRetryHorizonMillis: options.horizon ?? 1_000, + quota: options.quota ?? quota +}) + +describe("RelayInboxStore", () => { + it.effect("admits a message and reports it as the pending head", () => + Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + const result = yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0 })) + assert.strictEqual(result._tag, "Admitted") + + const heads = yield* store.pendingHeads("a", { limit: 10 }) + assert.strictEqual(heads.length, 1) + assert.strictEqual(heads[0]!.relayMessageId, relayId("000000000001")) + }).pipe(Effect.provide(layer))) + + it.effect("reports a replay of the same identity and digest as a duplicate", () => + Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0 })) + const replay = yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 1 })) + + assert.strictEqual(replay._tag, "Duplicate") + assert.strictEqual(replay._tag === "Duplicate" ? replay.state : "", "Pending") + const heads = yield* store.pendingHeads("a", { limit: 10 }) + assert.strictEqual(heads.length, 1, "a replay must not create a second row") + }).pipe(Effect.provide(layer))) + + it.effect("rejects the same identity carrying a different envelope digest", () => + Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0 })) + const conflict = yield* store.admit( + admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 1, digest: "c" }) + ) + assert.strictEqual(conflict._tag, "Conflict") + }).pipe(Effect.provide(layer))) + + it.effect("refuses a message whose channel disagrees with its envelope", () => + Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + const request = admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0 }) + const forged = { + ...request, + channel: { ...request.channel, senderConnectionEpoch: "another-epoch" } + } + const result = yield* store.admit(forged) + assert.strictEqual( + result._tag, + "Conflict", + "a fabricated channel would file the message under an ordering stream it does not belong to" + ) + }).pipe(Effect.provide(layer))) + + it.effect("expiring one inbox leaves another inbox's copy of the same identity untouched", () => + Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + // The same relay message id addressed to two devices. The key is (inbox, id), so a sweep + // that filters on the id alone would reach across inboxes. + yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0, ttl: 1_000 })) + yield* store.admit( + admission({ inboxKey: "b", id: "000000000001", sequence: 0, now: 100_000, ttl: 1_000_000 }) + ) + + const expired = yield* store.expire({ now: 5_000, limit: 10, terminalRetentionMillis: 1_000 }) + assert.strictEqual(expired, 1) + + const stale = yield* store.pendingHeads("a", { limit: 10 }) + assert.strictEqual(stale.length, 0, "the overdue message is expired") + const live = yield* store.pendingHeads("b", { limit: 10 }) + assert.strictEqual(live.length, 1, "the other inbox still holds a message with time left") + }).pipe(Effect.provide(layer))) + + it.effect("delivers one head per channel and does not starve a channel with high sequences", () => + Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + // `senderSequence` restarts per connection epoch, so it cannot order heads across channels. + yield* store.admit( + admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0, subject: "sender-a" }) + ) + yield* store.admit( + admission({ inboxKey: "a", id: "000000000002", sequence: 1, now: 1, subject: "sender-b" }) + ) + yield* store.admit( + admission({ inboxKey: "a", id: "000000000003", sequence: 500, now: 2, subject: "sender-c" }) + ) + + const heads = yield* store.pendingHeads("a", { limit: 2 }) + assert.strictEqual(heads.length, 2) + + // Oldest waiting head first, so every channel is reachable regardless of its own numbering. + const all = yield* store.pendingHeads("a", { limit: 10 }) + assert.deepStrictEqual( + all.map((head) => head.channel.senderSubjectId).toSorted(), + ["sender-a", "sender-b", "sender-c"] + ) + }).pipe(Effect.provide(layer))) + + it.effect("orders within a channel by sender sequence", () => + Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit(admission({ inboxKey: "a", id: "000000000002", sequence: 5, now: 0 })) + yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 1, now: 1 })) + + const heads = yield* store.pendingHeads("a", { limit: 10 }) + assert.strictEqual(heads.length, 1, "one head per channel") + assert.strictEqual(heads[0]!.relayMessageId, relayId("000000000001")) + }).pipe(Effect.provide(layer))) + + it.effect("spends the full delivery budget before dead lettering", () => + Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0 })) + + const first = yield* store.recordDelivery("a", relayId("000000000001"), { + maxDeliveries: 1, + now: 1 + }) + assert.strictEqual( + first._tag, + "Recorded", + "the first delivery of a budget of one must be allowed to be settled" + ) + + const second = yield* store.recordDelivery("a", relayId("000000000001"), { + maxDeliveries: 1, + now: 2 + }) + assert.strictEqual(second._tag, "DeadLettered") + + const heads = yield* store.pendingHeads("a", { limit: 10 }) + assert.strictEqual(heads.length, 0, "a dead lettered message stops blocking its channel") + }).pipe(Effect.provide(layer))) + + it.effect("makes an abandoned message answerable after the fact", () => + Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0 })) + yield* store.recordDelivery("a", relayId("000000000001"), { maxDeliveries: 1, now: 1 }) + yield* store.recordDelivery("a", relayId("000000000001"), { maxDeliveries: 1, now: 2 }) + + const abandoned = yield* store.abandoned("a", { limit: 10 }) + assert.strictEqual(abandoned.length, 1) + assert.strictEqual(abandoned[0]!.state, "DeadLettered") + }).pipe(Effect.provide(layer))) + + it.effect("settles on acknowledgement and does not redeliver", () => + Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0 })) + + const settled = yield* store.settle("a", relayId("000000000001"), { + outcome: "Acknowledged", + messageHash: "a".repeat(64), + now: 10, + terminalRetentionMillis: 1_000 + }) + assert.strictEqual(settled, "Settled") + + const heads = yield* store.pendingHeads("a", { limit: 10 }) + assert.strictEqual(heads.length, 0) + }).pipe(Effect.provide(layer))) + + it.effect("refuses a settlement whose content does not match", () => + Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0 })) + + const settled = yield* store.settle("a", relayId("000000000001"), { + outcome: "Acknowledged", + messageHash: "f".repeat(64), + now: 10, + terminalRetentionMillis: 1_000 + }) + assert.strictEqual(settled, "HashMismatch") + + const heads = yield* store.pendingHeads("a", { limit: 10 }) + assert.strictEqual(heads.length, 1, "the message stays deliverable") + }).pipe(Effect.provide(layer))) + + it.effect("reports a settlement that no longer applies", () => + Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0 })) + const options = { + outcome: "Acknowledged" as const, + messageHash: "a".repeat(64), + now: 10, + terminalRetentionMillis: 1_000 + } + yield* store.settle("a", relayId("000000000001"), options) + const again = yield* store.settle("a", relayId("000000000001"), options) + assert.strictEqual(again, "NotPending") + }).pipe(Effect.provide(layer))) + + it.effect("still deduplicates a replay that arrives after settlement", () => + Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit( + admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0, horizon: 100_000 }) + ) + yield* store.settle("a", relayId("000000000001"), { + outcome: "Acknowledged", + messageHash: "a".repeat(64), + now: 10, + terminalRetentionMillis: 1_000 + }) + + const replay = yield* store.admit( + admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 20, horizon: 100_000 }) + ) + assert.strictEqual(replay._tag, "Duplicate") + const heads = yield* store.pendingHeads("a", { limit: 10 }) + assert.strictEqual(heads.length, 0, "an acknowledged message must not become deliverable again") + }).pipe(Effect.provide(layer))) + + it.effect("revives a message the inbox had given up on rather than telling the sender it landed", () => + Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0, ttl: 1_000 })) + yield* store.expire({ now: 5_000, limit: 10, terminalRetentionMillis: 1_000 }) + + // The sender still holds custody and is replaying. Reporting a duplicate would make it drop + // the last copy of a message this inbox can no longer deliver. + const replay = yield* store.admit( + admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 6_000, ttl: 1_000 }) + ) + assert.strictEqual(replay._tag, "Admitted") + + const heads = yield* store.pendingHeads("a", { limit: 10 }) + assert.strictEqual(heads.length, 1, "the revived message is deliverable again") + }).pipe(Effect.provide(layer))) + + it.effect("charges a revived message against the inbox quota", () => + Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0, ttl: 1_000 })) + yield* store.expire({ now: 5_000, limit: 10, terminalRetentionMillis: 1_000 }) + + const revived = yield* store.admit(admission({ + inboxKey: "a", + id: "000000000001", + sequence: 0, + now: 6_000, + ttl: 1_000, + quota: { maxPendingMessages: 0, maxPendingBytes: 0 } + })) + assert.strictEqual( + revived._tag, + "QuotaExceeded", + "reviving creates pending work, so it cannot bypass the cap a first admission obeys" + ) + }).pipe(Effect.provide(layer))) + + it.effect("keeps a revived message deduplicated for the sender's replay window", () => + Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit( + admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0, ttl: 1_000, horizon: 1_000 }) + ) + yield* store.expire({ now: 2_000, limit: 10, terminalRetentionMillis: 1_000 }) + yield* store.admit( + admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 2_000, ttl: 1_000, horizon: 10_000 }) + ) + yield* store.expire({ now: 3_100, limit: 10, terminalRetentionMillis: 1_000 }) + + // The horizon was extended by the revive, so collection at this point must not remove it. + const collected = yield* store.collect({ now: 3_200, limit: 10 }) + assert.strictEqual( + collected, + 0, + "collecting inside the sender's retry window would let a replay be applied twice" + ) + }).pipe(Effect.provide(layer))) + + it.effect("removes terminal rows only once their deduplication horizon lapses", () => + Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit( + admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0, horizon: 1_000 }) + ) + yield* store.settle("a", relayId("000000000001"), { + outcome: "Acknowledged", + messageHash: "a".repeat(64), + now: 10, + terminalRetentionMillis: 100 + }) + + assert.strictEqual(yield* store.collect({ now: 500, limit: 10 }), 0) + assert.strictEqual(yield* store.collect({ now: 5_000, limit: 10 }), 1) + }).pipe(Effect.provide(layer))) + + it.effect("refuses admission beyond the inbox quota", () => + Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0 })) + const refused = yield* store.admit(admission({ + inboxKey: "a", + id: "000000000002", + sequence: 1, + now: 1, + quota: { maxPendingMessages: 1, maxPendingBytes: 10_000_000 } + })) + assert.strictEqual(refused._tag, "QuotaExceeded") + }).pipe(Effect.provide(layer))) + + it.effect("counts a replay once against usage", () => + Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0 })) + yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 1 })) + + const usage = yield* store.usage("a") + assert.strictEqual(usage.pendingCount, 1, "quota is derived from rows, so replays cannot inflate it") + }).pipe(Effect.provide(layer))) +}) From eba9c80bbe41407b02535388551cd2a25abb08ae Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Mon, 27 Jul 2026 00:54:43 -0500 Subject: [PATCH 13/29] Unbreak the type check on the relay inbox store suite `pnpm check` has been failing on the branch since the suite landed, so the `check` CI job was red while the test job passed. `channel` declared its options as `readonly epoch?: string`, but every call site forwards `options.epoch`, which is `string | undefined`. Under `exactOptionalPropertyTypes` an optional property and an explicitly-undefined one are different types, so passing the latter is an error. Widen the parameter to accept `undefined` rather than making the call sites strip it, because forwarding an absent field is exactly what these helpers are for. --- packages/local-rpc/test/RelayInboxStore.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/local-rpc/test/RelayInboxStore.test.ts b/packages/local-rpc/test/RelayInboxStore.test.ts index 9b55c38..fbcab1e 100644 --- a/packages/local-rpc/test/RelayInboxStore.test.ts +++ b/packages/local-rpc/test/RelayInboxStore.test.ts @@ -19,7 +19,9 @@ const layer = SqlRelayInboxStore.layer.pipe( const quota = { maxPendingMessages: 100, maxPendingBytes: 10_000_000 } -const channel = (options?: { readonly epoch?: string; readonly subject?: string }) => ({ +const channel = ( + options?: { readonly epoch?: string | undefined; readonly subject?: string | undefined } +) => ({ tenantId: "tenant-a", senderSubjectId: options?.subject ?? "sender-a", senderPeerId: peer("00000000aaa1"), From 6e96ef0a949ed863295271517d45353cfcd57b77 Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Mon, 27 Jul 2026 01:18:38 -0500 Subject: [PATCH 14/29] Cover the relay inbox entity and fix what that exposed Adds test/RelayInbox.test.ts against the explicit in-memory cluster and the real SQL store over SQLite. Two harness notes worth keeping: TestRunner.layer bakes ShardingConfig defaults that stall every test under virtual time, and shard acquisition runs on scheduled fibers, so the clock has to be advanced once before any entity is reachable. The suite exposed three defects, each with a regression test that fails for its own reason before the corresponding change. deliverHead built the finalizer that fails a pending settlement as an eagerly evaluated argument to Effect.ensuring. It ran when deliverHead was called, before the body registered the settlement, so the lookup always missed and the failure went to a throwaway Deferred. A recipient's Settle then waited forever whenever the durable write did not land, and because Settle runs on the entity's serialized lane that hung request blocked every later Deliver and Settle for the device. Suspended so the map is read at finalization time. channelId omitted senderConnectionEpoch while ChannelKey treats it as identity. The store partitions heads by the full key, so two epochs of one sender are two heads at once, but the coarser in-memory key marked both busy from a single delivery. One unsettled head in a stale epoch held the live epoch's entire stream for the life of the session, which is the reconnect path. closeSession interrupted the Deferred that Settle awaits. Settle declares a closed error union, and an interrupt is not in it: the front door cannot catch it, so releasing a session took down the peer's acknowledgement fiber instead of returning a retryable failure. It now fails with ServerUnavailable, while the deferred only the delivering fiber awaits is still interrupted. --- packages/local-rpc/src/RelayInbox.ts | 38 +- packages/local-rpc/test/RelayInbox.test.ts | 651 +++++++++++++++++++++ 2 files changed, 680 insertions(+), 9 deletions(-) create mode 100644 packages/local-rpc/test/RelayInbox.test.ts diff --git a/packages/local-rpc/src/RelayInbox.ts b/packages/local-rpc/src/RelayInbox.ts index 22c686c..23bc557 100644 --- a/packages/local-rpc/src/RelayInbox.ts +++ b/packages/local-rpc/src/RelayInbox.ts @@ -182,12 +182,21 @@ interface Session { const makeClaimToken = Effect.sync(() => `clm_${crypto.randomUUID()}` as PeerRpc.ClaimToken) +/** + * The in-flight key for a channel. + * + * Must name every component of `ChannelKey`. The store partitions heads by the full key including + * the connection epoch, so two epochs of one sender are two heads at once; a coarser key here marks + * both busy from one delivery and lets a stale epoch's unsettled head hold the live epoch's stream + * for the life of the session. + */ const channelId = (channel: RelayInboxStore.ChannelKey): string => JSON.stringify([ channel.tenantId, channel.senderSubjectId, channel.senderPeerId, - channel.senderReplicaIncarnation + channel.senderReplicaIncarnation, + channel.senderConnectionEpoch ]) export const layer = (options: Options) => @@ -217,8 +226,15 @@ export const layer = (options: Options) => Option.filter((current) => current.sessionId !== session.sessionId) ) for (const settlement of session.settlements.values()) { + // Only the delivering fiber awaits this, and closing the scope interrupts it anyway. yield* Deferred.interrupt(settlement.requested) - yield* Deferred.interrupt(settlement.durable) + // The recipient awaits this one through the `Settle` rpc, which declares a closed error + // union. Interrupting it answers the caller outside that union: an interrupt cannot be + // caught by the front door's typed handlers, so instead of a retryable failure it takes + // down the peer's acknowledgement fiber. + yield* Effect.ignore( + Deferred.fail(settlement.durable, new PeerRpcError.ServerUnavailable()) + ) } session.settlements.clear() yield* Scope.close(session.scope, Exit.void) @@ -312,13 +328,17 @@ export const layer = (options: Options) => )), // Fails any settle still waiting on this delivery so the recipient retries rather than // believing an acknowledgement landed. - Effect.ensuring( - Deferred.fail( - session.settlements.get(head.relayMessageId)?.durable ?? - Deferred.makeUnsafe(), - new PeerRpcError.ServerUnavailable() - ).pipe(Effect.ignore) - ), + // + // Suspended because the settlement is registered by the body above, which has not run + // when this pipeline is built. Reading the map eagerly here always found nothing, and the + // failure went to a throwaway deferred while the recipient's `Settle` waited forever — + // holding the entity's single handler permit against every other request for the device. + Effect.ensuring(Effect.suspend(() => { + const settlement = session.settlements.get(head.relayMessageId) + return settlement === undefined + ? Effect.void + : Effect.ignore(Deferred.fail(settlement.durable, new PeerRpcError.ServerUnavailable())) + })), Effect.ensuring(Effect.sync(() => { session.settlements.delete(head.relayMessageId) session.busyChannels.delete(channelId(head.channel)) diff --git a/packages/local-rpc/test/RelayInbox.test.ts b/packages/local-rpc/test/RelayInbox.test.ts new file mode 100644 index 0000000..e718b4a --- /dev/null +++ b/packages/local-rpc/test/RelayInbox.test.ts @@ -0,0 +1,651 @@ +import { NodeCrypto } from "@effect/platform-node" +import { SqliteClient } from "@effect/sql-sqlite-node" +import { assert, describe, it } from "@effect/vitest" +import * as Identity from "@lucas-barake/effect-local/Identity" +import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" +import * as Effect from "effect/Effect" +import * as Fiber from "effect/Fiber" +import * as Layer from "effect/Layer" +import * as Stream from "effect/Stream" +import { TestClock } from "effect/testing" +import * as MessageStorage from "effect/unstable/cluster/MessageStorage" +import * as RunnerHealth from "effect/unstable/cluster/RunnerHealth" +import * as Runners from "effect/unstable/cluster/Runners" +import * as RunnerStorage from "effect/unstable/cluster/RunnerStorage" +import * as Sharding from "effect/unstable/cluster/Sharding" +import * as ShardingConfig from "effect/unstable/cluster/ShardingConfig" +import * as PeerRpc from "../src/PeerRpc.js" +import * as RelayInbox from "../src/RelayInbox.js" +import * as RelayInboxStore from "../src/RelayInboxStore.js" +import * as SqlRelayInboxStore from "../src/SqlRelayInboxStore.js" + +const peer = (value: string) => Identity.PeerId.make(`peer_00000000-0000-4000-8000-${value}`) +const relayId = (value: string) => Identity.RelayMessageId.make(`rly_00000000-0000-4000-8000-${value}`) +const documentId = (value: string) => Identity.DocumentId.make(`doc_00000000-0000-4000-8000-${value}`) +const sessionId = (value: string) => Identity.SessionId.make(`ses_00000000-0000-4000-8000-${value}`) + +const inboxKey = "inbox-a" + +const baseOptions: RelayInbox.Options = { + maxDeliveries: 10, + messageTtlMillis: 600_000, + terminalRetentionMillis: 600_000, + sessionDeadlineMillis: 90_000, + sessionSweepMillis: 1_000, + maxConcurrentChannels: 4, + storeRetryMillis: 0, + maxPendingMessages: 100, + maxPendingBytes: 10_000_000, + mailboxCapacity: 16, + maxIdleTimeMillis: 3_600_000 +} + +/** + * The explicit cluster composition rather than `TestRunner.layer`. + * + * `TestRunner` bakes `ShardingConfig` defaults that cannot be overridden, and two of them — a 15s + * entity termination timeout and a 10s message poll interval — stall every test under the virtual + * clock `it.effect` installs. + */ +const TestShardingConfig = ShardingConfig.layer({ + entityTerminationTimeout: 0, + entityMessagePollInterval: 5000, + sendRetryInterval: 100 +}) + +/** + * The real SQL store over in-memory SQLite, so the entity is exercised against the durability it + * actually ships with. A hand written in-memory stand-in would let the entity's custody claims pass + * against semantics no production deployment has. + */ +const sqliteStore = SqlRelayInboxStore.layer.pipe( + Layer.provide(SqliteClient.layer({ filename: ":memory:", disableWAL: true })), + Layer.provide(NodeCrypto.layer), + Layer.orDie +) + +const relay = ( + options?: { + readonly entity?: Partial + readonly store?: Layer.Layer + } +) => + RelayInbox.layer({ ...baseOptions, ...options?.entity }).pipe( + Layer.provideMerge(Sharding.layer), + Layer.provide(Runners.layerNoop), + Layer.provideMerge(MessageStorage.layerMemory), + Layer.provide(RunnerStorage.layerMemory), + Layer.provide(RunnerHealth.layerNoop), + Layer.provide(TestShardingConfig), + // Merged rather than provided so a test can assert against the same durable rows the entity + // wrote. Delivery budgets and terminal states are durable guarantees, not internals. + Layer.provideMerge(options?.store ?? sqliteStore) + ) + +const layer = relay() + +/** + * Replaces one store operation with a failure, leaving the rest of the real store intact. + * + * Lives here rather than in `src` on purpose: the durable store is the relay's only custody, and a + * fault injecting one must never be reachable from a published entry point. + */ +const storeFailing = ( + override: ( + real: RelayInboxStore.RelayInboxStore["Service"] + ) => Partial +) => + Layer.effect(RelayInboxStore.RelayInboxStore)( + Effect.gen(function*() { + const real = yield* RelayInboxStore.RelayInboxStore + return RelayInboxStore.RelayInboxStore.of({ ...real, ...override(real) }) + }) + ).pipe(Layer.provide(sqliteStore)) + +const unavailable = new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageUnavailable({ cause: new Error("injected") }) +}) + +const channel = (options?: { readonly epoch?: string; readonly subject?: string }) => ({ + tenantId: "tenant-a", + senderSubjectId: options?.subject ?? "sender-a", + senderPeerId: peer("00000000aaa1"), + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + senderConnectionEpoch: options?.epoch ?? "epoch-1" +}) + +interface Message { + readonly id: string + readonly sequence: number + readonly epoch?: string + readonly subject?: string +} + +const deliver = (message: Message) => { + const source = channel(message) + return { + channel: source, + envelope: { + relayMessageId: relayId(message.id), + relayPeerId: peer("00000000ffff"), + sender: { + tenantId: source.tenantId, + subjectId: source.senderSubjectId, + peerId: source.senderPeerId, + replicaIncarnation: source.senderReplicaIncarnation, + connectionEpoch: source.senderConnectionEpoch, + sequence: message.sequence + }, + recipient: { + tenantId: "tenant-a", + subjectId: "recipient-a", + peerId: peer("00000000bbb1") + }, + payloadVersion: 1 as const, + document: { documentId: documentId("00000000dddd"), documentType: "note" }, + writerProvenance: [], + messageHash: message.id.padStart(64, "a"), + outerEnvelopeDigest: message.id.padStart(64, "b"), + payload: new Uint8Array([1, 2, 3]) + }, + senderRetryHorizonMillis: 60_000 + } +} + +const inboxFor = Effect.map(RelayInbox.RelayInbox.client, (make) => make(inboxKey)) +type Inbox = Effect.Success + +/** + * Brings the runner up: shard assignment and acquisition are driven by scheduled fibers, so under + * the virtual clock no entity is reachable until time is advanced. + */ +const inbox = Effect.gen(function*() { + yield* TestClock.adjust(5000) + return yield* inboxFor +}) + +/** Receives `count` messages and settles each, keeping the session alive across the settlements. */ +const receiveAndSettle = ( + client: Inbox, + session: Identity.SessionId, + count: number, + outcome: RelayInboxStore.TerminalOutcome = "Acknowledged" +) => + client.Subscribe({ sessionId: session }).pipe( + Stream.take(count), + Stream.mapEffect((message) => + client.Settle({ + sessionId: session, + relayMessageId: message.relayMessageId, + claimToken: message.claimToken, + messageHash: message.messageHash, + outcome + }).pipe(Effect.as(message)) + ), + Stream.runCollect + ) + +/** Receives `count` messages and drops the session without settling any of them. */ +const receiveOnly = (client: Inbox, session: Identity.SessionId, count: number) => + client.Subscribe({ sessionId: session }).pipe(Stream.take(count), Stream.runCollect) + +const ids = (messages: ReadonlyArray) => messages.map((message) => message.relayMessageId) + +describe("RelayInbox", () => { + it.effect("delivers a message admitted while the recipient had no session", () => + Effect.gen(function*() { + const client = yield* inbox + // Admitted with nobody listening. This is the store-and-forward promise: the sender is never + // required to overlap with the recipient. + yield* client.Deliver(deliver({ id: "000000000001", sequence: 0 })) + + const received = yield* receiveOnly(client, sessionId("000000000001"), 1) + + assert.deepStrictEqual(ids(received), [relayId("000000000001")]) + }).pipe(Effect.provide(layer))) + + it.effect("redelivers a message whose session dropped without settling it", () => + Effect.gen(function*() { + const client = yield* inbox + yield* client.Deliver(deliver({ id: "000000000001", sequence: 0 })) + + const first = yield* receiveOnly(client, sessionId("000000000001"), 1) + const second = yield* receiveOnly(client, sessionId("000000000002"), 1) + + assert.deepStrictEqual(ids(first), [relayId("000000000001")]) + assert.deepStrictEqual( + ids(second), + [relayId("000000000001")], + "an unsettled message is still this inbox's responsibility" + ) + }).pipe(Effect.provide(layer))) + + it.effect("treats acknowledgement and rejection as equally terminal", () => + Effect.gen(function*() { + const client = yield* inbox + yield* client.Deliver(deliver({ id: "000000000001", sequence: 0, subject: "sender-a" })) + yield* client.Deliver(deliver({ id: "000000000002", sequence: 0, subject: "sender-b" })) + + const settled = yield* receiveAndSettle(client, sessionId("000000000001"), 1, "Acknowledged").pipe( + Effect.zipWith( + receiveAndSettle(client, sessionId("000000000002"), 1, "Rejected"), + (first, second) => [...first, ...second] + ) + ) + assert.strictEqual(settled.length, 2) + + const store = yield* RelayInboxStore.RelayInboxStore + const pending = yield* store.pendingHeads(inboxKey, { limit: 10 }) + assert.strictEqual( + pending.length, + 0, + "the recipient made a durable decision either way, so neither is redelivered" + ) + }).pipe(Effect.provide(layer))) + + it.effect("admits nothing when the durable write fails", () => + Effect.gen(function*() { + const client = yield* inbox + // The at-least-once guarantee rests entirely on this seam: `Deliver` is deliberately not a + // persisted cluster message, so a write that did not land must surface as a failed rpc and + // leave the sender holding the only copy. + const exit = yield* client.Deliver(deliver({ id: "000000000001", sequence: 0 })).pipe(Effect.exit) + assert.isTrue(exit._tag === "Failure") + + const store = yield* RelayInboxStore.RelayInboxStore + const pending = yield* store.pendingHeads(inboxKey, { limit: 10 }) + assert.strictEqual(pending.length, 0, "a failed admission must leave no durable trace") + }).pipe(Effect.provide(relay({ + store: storeFailing(() => ({ admit: () => Effect.fail(unavailable) })) + })))) + + it.effect("does not let one unsettled channel block another", () => + Effect.gen(function*() { + const client = yield* inbox + yield* client.Deliver(deliver({ id: "000000000001", sequence: 0, subject: "sender-a" })) + yield* client.Deliver(deliver({ id: "000000000002", sequence: 0, subject: "sender-b" })) + + // Neither is settled. A single ordered queue per device would stall on the first. + const received = yield* receiveOnly(client, sessionId("000000000001"), 2) + + assert.deepStrictEqual( + ids(received).toSorted(), + [relayId("000000000001"), relayId("000000000002")].toSorted() + ) + }).pipe(Effect.provide(layer))) + + it.effect("does not let one connection epoch hold another epoch's stream", () => + Effect.gen(function*() { + const client = yield* inbox + // Two epochs of the SAME sender, so they differ only by the field the in-memory busy-channel + // key is most likely to drop. The store partitions heads by epoch, so both are heads at once; + // if the entity's own key is coarser than the store's, the stale epoch's unsettled head holds + // the live epoch's entire stream for the life of the session. + yield* client.Deliver(deliver({ id: "000000000001", sequence: 0, epoch: "epoch-1" })) + yield* client.Deliver(deliver({ id: "000000000002", sequence: 0, epoch: "epoch-2" })) + + const received = yield* receiveOnly(client, sessionId("000000000001"), 2) + assert.deepStrictEqual( + ids(received).toSorted(), + [relayId("000000000001"), relayId("000000000002")].toSorted() + ) + }).pipe(Effect.provide(layer))) + + it.effect("answers a settle inside its declared failure channel when the session is released", () => + Effect.gen(function*() { + const client = yield* inbox + yield* client.Deliver(deliver({ id: "000000000001", sequence: 0 })) + + const session = sessionId("000000000001") + const delivered = yield* receiveOnly(client, session, 1) + + // Parked on the durable write when its session is released. `Settle` declares a closed error + // union, so the recipient must be able to discriminate the outcome and retry. Answering with + // an interrupt escapes that union entirely: it is uncatchable at the front door and takes the + // peer's acknowledgement fiber down with it instead of producing a retryable failure. + const settling = yield* Effect.forkChild( + client.Settle({ + sessionId: session, + relayMessageId: delivered[0]!.relayMessageId, + claimToken: delivered[0]!.claimToken, + messageHash: delivered[0]!.messageHash, + outcome: "Acknowledged" + // Covers the complete declared failure channel, so only an out-of-channel outcome fails. + }).pipe(Effect.catch(() => Effect.void), Effect.exit) + ) + yield* TestClock.adjust(10) + yield* TestClock.adjust(baseOptions.sessionDeadlineMillis + baseOptions.sessionSweepMillis * 3) + + const outcome = yield* Fiber.join(settling) + assert.strictEqual( + outcome._tag, + "Success", + "releasing a session must fail a waiting settle, not interrupt it" + ) + }).pipe(Effect.provide(relay({ + // Never completes, so the settlement is still in flight when the deadline releases it. + store: storeFailing(() => ({ settle: () => Effect.never })) + })))) + + it.effect("keeps a channel in sender sequence order across a redelivery", () => + Effect.gen(function*() { + const client = yield* inbox + yield* client.Deliver(deliver({ id: "000000000001", sequence: 0 })) + yield* client.Deliver(deliver({ id: "000000000002", sequence: 1 })) + + const dropped = yield* receiveOnly(client, sessionId("000000000001"), 1) + assert.deepStrictEqual(ids(dropped), [relayId("000000000001")], "the head goes first") + + const resumed = yield* receiveAndSettle(client, sessionId("000000000002"), 2) + assert.deepStrictEqual( + ids(resumed), + [relayId("000000000001"), relayId("000000000002")], + "a redelivery resumes at the head rather than skipping past it" + ) + }).pipe(Effect.provide(layer))) + + it.effect("orders each connection epoch of one sender by its own sequence", () => + Effect.gen(function*() { + const client = yield* inbox + // `senderSequence` restarts at zero on every reconnect, so both epochs carry {0, 1}. Treated + // as one channel, one epoch's run would be ordered behind the other's and a message could be + // starved. This pins per-epoch order only; that the two epochs are independently deliverable + // at all is owned by "does not let one connection epoch hold another epoch's stream", because + // relative order within an epoch survives any interleaving and cannot detect a merged key. + yield* client.Deliver(deliver({ id: "000000000001", sequence: 0, epoch: "epoch-1" })) + yield* client.Deliver(deliver({ id: "000000000002", sequence: 1, epoch: "epoch-1" })) + yield* client.Deliver(deliver({ id: "000000000003", sequence: 0, epoch: "epoch-2" })) + yield* client.Deliver(deliver({ id: "000000000004", sequence: 1, epoch: "epoch-2" })) + + const received = yield* receiveAndSettle(client, sessionId("000000000001"), 4) + + const perEpoch = (epoch: string) => + received + .filter((message) => message.sender.connectionEpoch === epoch) + .map((message) => message.sender.sequence) + assert.deepStrictEqual(perEpoch("epoch-1"), [0, 1]) + assert.deepStrictEqual(perEpoch("epoch-2"), [0, 1]) + }).pipe(Effect.provide(layer))) + + it.effect("delivers each message exactly once across a session replacement", () => + Effect.gen(function*() { + const client = yield* inbox + yield* client.Deliver(deliver({ id: "000000000001", sequence: 0 })) + yield* client.Deliver(deliver({ id: "000000000002", sequence: 1 })) + yield* client.Deliver(deliver({ id: "000000000003", sequence: 2 })) + + const first = yield* receiveAndSettle(client, sessionId("000000000001"), 1) + const second = yield* receiveAndSettle(client, sessionId("000000000002"), 2) + + const seen = [...ids(first), ...ids(second)] + assert.deepStrictEqual( + seen.toSorted(), + [relayId("000000000001"), relayId("000000000002"), relayId("000000000003")], + "nothing is stranded by the replacement and nothing is delivered twice" + ) + }).pipe(Effect.provide(layer))) + + it.effect("leaves no dispatcher behind when a subscribe is interrupted", () => + Effect.gen(function*() { + const client = yield* inbox + yield* client.Deliver(deliver({ id: "000000000001", sequence: 0 })) + yield* client.Deliver(deliver({ id: "000000000002", sequence: 1 })) + + // An interrupt between the session scope being created and the session becoming reachable + // would strand a dispatcher that no one can close, and it would keep delivering alongside + // whichever session replaces it. + for (const attempt of ["000000000001", "000000000002", "000000000003"]) { + const fiber = yield* Effect.forkChild( + Stream.runDrain(client.Subscribe({ sessionId: sessionId(attempt) })) + ) + yield* Fiber.interrupt(fiber) + } + + const received = yield* receiveAndSettle(client, sessionId("000000000009"), 2) + assert.deepStrictEqual( + ids(received), + [relayId("000000000001"), relayId("000000000002")], + "an orphaned dispatcher would have consumed or duplicated these" + ) + }).pipe(Effect.provide(layer))) + + it.effect("releases a session whose liveness deadline lapses", () => + Effect.gen(function*() { + const client = yield* inbox + const session = sessionId("000000000001") + + const subscriber = yield* Effect.forkChild( + Stream.runCollect(client.Subscribe({ sessionId: session })) + ) + yield* TestClock.adjust(10) + + // Nothing heartbeats it, so the entity's own deadline is what bounds it. A cluster cannot + // rely on disconnects announcing themselves. + yield* TestClock.adjust(baseOptions.sessionDeadlineMillis + baseOptions.sessionSweepMillis * 2) + + const collected = yield* Fiber.join(subscriber) + assert.strictEqual(collected.length, 0, "the released session's stream ends") + + const exit = yield* client.Heartbeat({ sessionId: session }).pipe(Effect.exit) + assert.isTrue(exit._tag === "Failure") + }).pipe(Effect.provide(layer))) + + it.effect("refuses a settlement carrying a claim token from another attempt", () => + Effect.gen(function*() { + const client = yield* inbox + yield* client.Deliver(deliver({ id: "000000000001", sequence: 0 })) + + const session = sessionId("000000000001") + const outcome = yield* client.Subscribe({ sessionId: session }).pipe( + Stream.take(1), + Stream.mapEffect((message) => + client.Settle({ + sessionId: session, + relayMessageId: message.relayMessageId, + // Minted per delivery attempt, so a stale token is also a replay of an earlier attempt. + claimToken: PeerRpc.ClaimToken.make("clm_00000000-0000-4000-8000-000000000000"), + messageHash: message.messageHash, + outcome: "Acknowledged" + }).pipe(Effect.exit) + ), + Stream.runCollect + ) + assert.isTrue(outcome[0]!._tag === "Failure") + + const redelivered = yield* receiveOnly(client, sessionId("000000000002"), 1) + assert.deepStrictEqual( + ids(redelivered), + [relayId("000000000001")], + "a refused settlement leaves the message this inbox's responsibility" + ) + }).pipe(Effect.provide(layer))) + + it.effect("refuses a settlement whose message hash does not match", () => + Effect.gen(function*() { + const client = yield* inbox + yield* client.Deliver(deliver({ id: "000000000001", sequence: 0 })) + + const session = sessionId("000000000001") + const outcome = yield* client.Subscribe({ sessionId: session }).pipe( + Stream.take(1), + Stream.mapEffect((message) => + client.Settle({ + sessionId: session, + relayMessageId: message.relayMessageId, + claimToken: message.claimToken, + messageHash: "f".repeat(64), + outcome: "Acknowledged" + }).pipe(Effect.exit) + ), + Stream.runCollect + ) + assert.isTrue(outcome[0]!._tag === "Failure") + + const store = yield* RelayInboxStore.RelayInboxStore + const pending = yield* store.pendingHeads(inboxKey, { limit: 10 }) + assert.strictEqual(pending.length, 1, "the message stays deliverable") + }).pipe(Effect.provide(layer))) + + it.effect("refuses a settlement presented under a session that is not the live one", () => + Effect.gen(function*() { + const client = yield* inbox + yield* client.Deliver(deliver({ id: "000000000001", sequence: 0 })) + + // Presented under a session id that was never opened, while the real session is still live + // and the delivery in flight. The claim token and hash both match, so the session identity + // check is the only thing that can refuse it — settling against a replaced session instead + // would be refused by the stale token and prove nothing about this guard. + const session = sessionId("000000000001") + const outcome = yield* client.Subscribe({ sessionId: session }).pipe( + Stream.take(1), + Stream.mapEffect((message) => + client.Settle({ + sessionId: sessionId("000000000099"), + relayMessageId: message.relayMessageId, + claimToken: message.claimToken, + messageHash: message.messageHash, + outcome: "Acknowledged" + }).pipe(Effect.exit) + ), + Stream.runCollect + ) + assert.isTrue(outcome[0]!._tag === "Failure") + + const store = yield* RelayInboxStore.RelayInboxStore + const pending = yield* store.pendingHeads(inboxKey, { limit: 10 }) + assert.strictEqual(pending.length, 1, "the message stays this inbox's responsibility") + }).pipe(Effect.provide(layer))) + + it.effect("does not report an acknowledgement the durable write never made", () => + Effect.gen(function*() { + const client = yield* inbox + yield* client.Deliver(deliver({ id: "000000000001", sequence: 0 })) + + const session = sessionId("000000000001") + // The recipient prunes its own relay receipt on a successful acknowledgement, so replying + // before the row is terminal would leave neither side holding the message. + const outcome = yield* client.Subscribe({ sessionId: session }).pipe( + Stream.take(1), + Stream.mapEffect((message) => + client.Settle({ + sessionId: session, + relayMessageId: message.relayMessageId, + claimToken: message.claimToken, + messageHash: message.messageHash, + outcome: "Acknowledged" + }).pipe(Effect.exit) + ), + Stream.runCollect + ) + assert.isTrue(outcome[0]!._tag === "Failure") + }).pipe(Effect.provide(relay({ + store: storeFailing(() => ({ settle: () => Effect.fail(unavailable) })) + })))) + + it.effect("dead letters a message that exhausts its delivery budget and unblocks its channel", () => + Effect.gen(function*() { + const client = yield* inbox + yield* client.Deliver(deliver({ id: "000000000001", sequence: 0 })) + yield* client.Deliver(deliver({ id: "000000000002", sequence: 1 })) + + // A recipient that reads and disconnects without settling is not a failure at any layer, so + // without a budget the head would be redelivered forever and its channel would never advance. + const first = yield* receiveOnly(client, sessionId("000000000001"), 1) + assert.deepStrictEqual(ids(first), [relayId("000000000001")]) + + const second = yield* receiveOnly(client, sessionId("000000000002"), 2) + assert.deepStrictEqual( + ids(second), + [relayId("000000000001"), relayId("000000000002")], + "the exhausted head is dead lettered and the channel moves on" + ) + + const store = yield* RelayInboxStore.RelayInboxStore + const abandoned = yield* store.abandoned(inboxKey, { limit: 10 }) + assert.deepStrictEqual( + abandoned.map((message) => [message.relayMessageId, message.state]), + [[relayId("000000000001"), "DeadLettered"]], + "an abandoned message stays answerable, because its sender already released custody" + ) + }).pipe(Effect.provide(relay({ entity: { maxDeliveries: 1 } })))) + + it.effect("does not charge a delivery the recipient never took", () => + Effect.gen(function*() { + const client = yield* inbox + yield* client.Deliver(deliver({ id: "000000000001", sequence: 0, subject: "sender-a" })) + + const session = sessionId("000000000001") + const received = yield* receiveOnly(client, session, 1) + assert.deepStrictEqual(ids(received), [relayId("000000000001")]) + + // Admitted after the recipient stopped reading. It is prepared for its channel and offered to + // a rendezvous nobody is taking from, so it never reaches the transport. Charging it here is + // what would dead letter messages on an ordinary flaky connection. + yield* client.Deliver(deliver({ id: "000000000002", sequence: 0, subject: "sender-b" })) + // Hands the prior session's dispatcher a scheduler pass so it really does prepare the message + // and park on the offer. Nothing is due at this point, so the amount is irrelevant. + yield* TestClock.adjust(10) + + const store = yield* RelayInboxStore.RelayInboxStore + const untaken = (yield* store.pendingHeads(inboxKey, { limit: 10 })) + .find((message) => message.relayMessageId === relayId("000000000002")) + assert.strictEqual(untaken?.deliveries, 0, "an untransmitted attempt costs the message nothing") + + // The one transmission it ever gets. Asserting the positive as well keeps this test honest if + // the nudge above ever stops landing: a charge taken while it sat unread shows up as a second. + const later = yield* receiveOnly(client, sessionId("000000000002"), 2) + assert.isTrue(ids(later).includes(relayId("000000000002"))) + + const transmitted = (yield* store.pendingHeads(inboxKey, { limit: 10 })) + .find((message) => message.relayMessageId === relayId("000000000002")) + assert.strictEqual( + transmitted?.deliveries, + 1, + "only the transmission that actually reached the recipient is charged" + ) + }).pipe(Effect.provide(layer))) + + it.effect("charges one delivery per transmission and none for merely reaching the head", () => + Effect.gen(function*() { + const client = yield* inbox + yield* client.Deliver(deliver({ id: "000000000001", sequence: 0, subject: "sender-a" })) + yield* client.Deliver(deliver({ id: "000000000002", sequence: 1, subject: "sender-a" })) + yield* client.Deliver(deliver({ id: "000000000003", sequence: 0, subject: "sender-b" })) + + // One session, two channels. `sender-a` is stop-and-wait, so its second message never + // becomes a head while the first is unsettled and must therefore cost nothing. + const received = yield* receiveOnly(client, sessionId("000000000001"), 2) + assert.strictEqual(received.length, 2) + + const store = yield* RelayInboxStore.RelayInboxStore + const pending = yield* store.pendingHeads(inboxKey, { limit: 10 }) + assert.deepStrictEqual( + pending.map((message) => [message.relayMessageId, message.deliveries]).toSorted(), + [ + [relayId("000000000001"), 1], + [relayId("000000000003"), 1] + ].toSorted(), + "each transmitted head is charged exactly once" + ) + + const queued = yield* store.usage(inboxKey) + assert.strictEqual(queued.pendingCount, 3, "the queued message is still waiting behind its head") + + // `pendingHeads` returns one row per channel, so the queued message is structurally absent + // from the assertion above and its count went unobserved. Settling the head promotes it, and + // only then can "none for merely reaching the head" actually be checked. + yield* store.settle(inboxKey, relayId("000000000001"), { + outcome: "Acknowledged", + messageHash: "000000000001".padStart(64, "a"), + now: 1_000, + terminalRetentionMillis: baseOptions.terminalRetentionMillis + }) + const promoted = (yield* store.pendingHeads(inboxKey, { limit: 10 })) + .find((message) => message.relayMessageId === relayId("000000000002")) + assert.strictEqual( + promoted?.deliveries, + 0, + "a message queued behind its channel head is never charged while it waits" + ) + }).pipe(Effect.provide(layer))) +}) From a3a0ec74f7cfaa12d07a3c607200494b0a096dbb Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Mon, 27 Jul 2026 01:41:34 -0500 Subject: [PATCH 15/29] Prove the relay wire format over a real socket Both RpcTest.makeClient and Entity.makeTestClient hand the handler an already decoded value, so every other relay suite exercises the schemas without ever serializing them. That is the one thing this PR changed most: the bespoke length-prefixed framing is gone and the relay now speaks standard Effect RPC. This drives a full Open/Push/Acknowledge round trip between two clients and a real RpcServer over a TCP socket with RpcSerialization.layerJson, and asserts the payload survives byte identically in both directions. The payload carries high bytes on purpose, since a Uint8Array is the part of the contract a JSON codec is most likely to mangle. The no-redelivery check pushes a second message and requires it to arrive first, rather than sleeping and asserting nothing showed up: a redelivery of the acknowledged message would be the older head and would necessarily arrive ahead of it. --- .../local-rpc/test/RelayWireFormat.test.ts | 322 ++++++++++++++++++ 1 file changed, 322 insertions(+) create mode 100644 packages/local-rpc/test/RelayWireFormat.test.ts diff --git a/packages/local-rpc/test/RelayWireFormat.test.ts b/packages/local-rpc/test/RelayWireFormat.test.ts new file mode 100644 index 0000000..136d9b4 --- /dev/null +++ b/packages/local-rpc/test/RelayWireFormat.test.ts @@ -0,0 +1,322 @@ +import { NodeCrypto, NodeSocket, NodeSocketServer } from "@effect/platform-node" +import { SqliteClient } from "@effect/sql-sqlite-node" +import { assert, describe, it } from "@effect/vitest" +import * as PeerSyncEnvelope from "@lucas-barake/effect-local-sql/PeerSyncEnvelope" +import * as Canonical from "@lucas-barake/effect-local/Canonical" +import * as Document from "@lucas-barake/effect-local/Document" +import * as Identity from "@lucas-barake/effect-local/Identity" +import * as Context from "effect/Context" +import * as Crypto from "effect/Crypto" +import * as Effect from "effect/Effect" +import * as Layer from "effect/Layer" +import * as Queue from "effect/Queue" +import * as Redacted from "effect/Redacted" +import * as Ref from "effect/Ref" +import * as Schema from "effect/Schema" +import * as Stream from "effect/Stream" +import * as MessageStorage from "effect/unstable/cluster/MessageStorage" +import * as RunnerHealth from "effect/unstable/cluster/RunnerHealth" +import * as Runners from "effect/unstable/cluster/Runners" +import * as RunnerStorage from "effect/unstable/cluster/RunnerStorage" +import * as Sharding from "effect/unstable/cluster/Sharding" +import * as ShardingConfig from "effect/unstable/cluster/ShardingConfig" +import * as RpcClient from "effect/unstable/rpc/RpcClient" +import * as RpcSerialization from "effect/unstable/rpc/RpcSerialization" +import * as RpcServer from "effect/unstable/rpc/RpcServer" +import * as SocketServer from "effect/unstable/socket/SocketServer" +import * as PeerAuthentication from "../src/PeerAuthentication.js" +import * as PeerAuthenticator from "../src/PeerAuthenticator.js" +import * as PeerCredentials from "../src/PeerCredentials.js" +import * as PeerRelayAuthorization from "../src/PeerRelayAuthorization.js" +import * as PeerRelayLimits from "../src/PeerRelayLimits.js" +import * as PeerRpc from "../src/PeerRpc.js" +import * as PeerRpcError from "../src/PeerRpcError.js" +import * as RelayInbox from "../src/RelayInbox.js" +import * as RelayServer from "../src/RelayServer.js" +import * as SqlRelayInboxStore from "../src/SqlRelayInboxStore.js" + +/** + * The wire format, end to end. + * + * Every other relay suite drives the front door through `RpcTest.makeClient` or the entity through + * `Entity.makeTestClient`, and both of those bypass serialization entirely — they hand the handler + * the decoded value. So nothing else in this package proves that the relay's schemas survive a real + * encode and decode, which is exactly what changed when the bespoke length-prefixed framing was + * replaced by standard Effect RPC. This is that proof, over a real socket with real JSON codecs. + */ + +const Task = Document.make("Task", { + schema: Schema.Struct({ title: Schema.String }), + version: 1 +}) + +const documentId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000001") +const relayPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000001") +const senderPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000002") +const recipientPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000003") +const relayMessageId = Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000001") +const secondRelayMessageId = Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000002") + +const sender = PeerAuthentication.PeerPrincipal.make({ + tenantId: "tenant", + subjectId: "sender", + peerId: senderPeerId +}) +const recipient = PeerAuthentication.PeerPrincipal.make({ + tenantId: "tenant", + subjectId: "recipient", + peerId: recipientPeerId +}) + +const inboxOptions: RelayInbox.Options = { + maxDeliveries: 10, + messageTtlMillis: 600_000, + terminalRetentionMillis: 600_000, + sessionDeadlineMillis: 90_000, + sessionSweepMillis: 1_000, + maxConcurrentChannels: 4, + storeRetryMillis: 0, + maxPendingMessages: 100, + maxPendingBytes: 10_000_000, + mailboxCapacity: 16, + maxIdleTimeMillis: 3_600_000 +} + +const openRequest = ( + principal: PeerAuthentication.PeerPrincipal, + remote: PeerAuthentication.PeerPrincipal +) => + PeerRpc.OpenRpc.payloadSchema.make({ + protocolVersion: PeerRpc.protocolVersion, + expectedRelayPeerId: relayPeerId, + expectedLocal: principal, + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + remote: { subjectId: remote.subjectId, peerId: remote.peerId }, + documents: [{ documentType: Task.name, documentId }], + receiptRetentionMillis: PeerRelayLimits.defaults.maximumReceiptRetentionMillis, + senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis + }) + +const RelaySyncEnvelopeJson = Schema.fromJsonString( + Schema.toCodecJson(PeerSyncEnvelope.SyncEnvelope) +) + +const tcpPort = (address: SocketServer.Address) => { + assert.strictEqual(address._tag, "TcpAddress") + return (address as SocketServer.TcpAddress).port +} + +describe("relay wire format", () => { + it.live( + "round trips Open, Push and Acknowledge over a socket with real serialization", + () => + Effect.scoped(Effect.gen(function*() { + const scope = yield* Effect.scope + const crypto = yield* Crypto.Crypto.pipe(Effect.provide(NodeCrypto.layer)) + const limits = PeerRelayLimits.defaults + + const authorization = yield* PeerRelayAuthorization.PeerRelayAuthorization.pipe( + Effect.provide(PeerRelayAuthorization.layer( + (request) => + Effect.succeed({ + remote: { + tenantId: request.principal.tenantId, + subjectId: request.remote.subjectId, + peerId: request.remote.peerId + }, + documents: request.documents.map((requested) => ({ + document: Task, + documentId: requested.documentId + })), + validUntil: Number.MAX_SAFE_INTEGER, + invalidated: Effect.never + }), + PeerRelayAuthorization.denyUnsafeUnboundedAutomerge3Decode + )) + ) + + const principals = new Map([["sender", sender], ["recipient", recipient]]) + const authenticator = PeerAuthenticator.PeerAuthenticator.of({ + authenticate: (secret) => { + const principal = principals.get(Redacted.value(secret)) + return principal === undefined + ? Effect.fail(new PeerRpcError.AuthenticationFailure()) + : Effect.succeed({ + principal, + validUntil: Number.MAX_SAFE_INTEGER, + invalidated: Effect.never + }) + } + }) + + const cluster = RelayInbox.layer(inboxOptions).pipe( + Layer.provideMerge(Sharding.layer), + Layer.provide(Runners.layerNoop), + Layer.provideMerge(MessageStorage.layerMemory), + Layer.provide(RunnerStorage.layerMemory), + Layer.provide(RunnerHealth.layerNoop), + Layer.provide(ShardingConfig.layer({ entityTerminationTimeout: 0 })), + Layer.provide( + SqlRelayInboxStore.layer.pipe( + Layer.provide(SqliteClient.layer({ filename: ":memory:", disableWAL: true })), + Layer.provide(Layer.succeed(Crypto.Crypto)(crypto)), + Layer.orDie + ) + ) + ) + + // The socket server is built first so its assigned port can be handed to the client. + const listener = yield* Layer.buildWithScope(NodeSocketServer.layer({ port: 0 }), scope) + const socketServer = Context.get(listener, SocketServer.SocketServer) + const port = tcpPort(socketServer.address) + + yield* Layer.buildWithScope( + RpcServer.layer(PeerRpc.Rpcs).pipe( + Layer.provide(RelayServer.layerHandlers({ + tenantId: "tenant", + peerId: relayPeerId, + heartbeatIntervalMillis: 30_000, + entityCallTimeoutMillis: 30_000 + })), + Layer.provide(PeerAuthentication.layerServer), + Layer.provide(RpcServer.layerProtocolSocketServer), + Layer.provide(Layer.succeed(SocketServer.SocketServer)(socketServer)), + // The real codec. This is the whole point of the test. + Layer.provide(RpcSerialization.layerJson), + Layer.provide(cluster), + Layer.provide(Layer.mergeAll( + Layer.succeed(Crypto.Crypto)(crypto), + Layer.succeed(PeerRelayLimits.PeerRelayLimits)(limits), + Layer.succeed(PeerRelayAuthorization.PeerRelayAuthorization)(authorization), + Layer.succeed(PeerAuthenticator.PeerAuthenticator)(authenticator) + )) + ), + scope + ) + + const clientFor = (subject: string) => + Effect.gen(function*() { + const credential = yield* Ref.make(subject) + const context = yield* Layer.buildWithScope( + RpcClient.layerProtocolSocket().pipe( + Layer.provide(NodeSocket.layerNet({ port })), + Layer.provide(RpcSerialization.layerJson) + ), + scope + ) + return yield* PeerRpc.makeRpcClient.pipe( + Effect.provideService(RpcClient.Protocol, Context.get(context, RpcClient.Protocol)), + Effect.provide(PeerAuthentication.layerClient), + Effect.provideService(PeerCredentials.PeerCredentials, { + get: Ref.get(credential).pipe(Effect.map(Redacted.make)) + }) + ) + }) + + const senderClient = yield* clientFor("sender") + const recipientClient = yield* clientFor("recipient") + + const senderEvents = yield* Queue.unbounded() + yield* senderClient.Open(openRequest(sender, recipient)).pipe( + Stream.runForEach((event) => Queue.offer(senderEvents, event)), + Effect.forkScoped + ) + const senderOpened = yield* Queue.take(senderEvents) + assert.strictEqual(senderOpened._tag, "Opened") + if (senderOpened._tag !== "Opened") return + + const recipientEvents = yield* Queue.unbounded() + yield* recipientClient.Open(openRequest(recipient, sender)).pipe( + Stream.runForEach((event) => Queue.offer(recipientEvents, event)), + Effect.forkScoped + ) + const recipientOpened = yield* Queue.take(recipientEvents) + assert.strictEqual(recipientOpened._tag, "Opened") + + const message = Uint8Array.of(1, 2, 3, 250, 251, 252) + const messageHash = yield* Canonical.digest(message).pipe( + Effect.provideService(Crypto.Crypto, crypto) + ) + const payload = new TextEncoder().encode( + yield* Schema.encodeEffect(RelaySyncEnvelopeJson)({ + connectionEpoch: "epoch", + sequence: 0, + documentId, + documentType: Task.name, + messageHash, + message, + lineage: Identity.genesisLineage, + writerProvenance: [] + }) + ) + + yield* senderClient.Push({ + sessionId: senderOpened.sessionId, + relayMessageId, + payload + }) + + const stored = yield* Queue.take(recipientEvents) + assert.strictEqual(stored._tag, "StoredMessage") + if (stored._tag !== "StoredMessage") return + + // Bytes, not a structural near-miss. A `Uint8Array` has to survive JSON in both directions, + // and the high bytes above are there to catch an encoding that mangles them. + assert.deepStrictEqual(stored.payload, payload) + assert.strictEqual(stored.relayMessageId, relayMessageId) + assert.strictEqual(stored.relayPeerId, relayPeerId) + assert.strictEqual(stored.messageHash, messageHash) + assert.deepStrictEqual(stored.recipient, recipient) + assert.strictEqual(stored.sender.peerId, senderPeerId) + + yield* recipientClient.Acknowledge({ + sessionId: (recipientOpened as PeerRpc.Opened).sessionId, + relayMessageId: stored.relayMessageId, + claimToken: stored.claimToken, + messageHash: stored.messageHash + }) + + // Reconnecting proves the acknowledgement crossed the wire and was applied durably. Rather + // than waiting a while and asserting nothing arrived, push a second message and require it + // to be the first thing the new session sees: a redelivery of the acknowledged message + // would necessarily arrive ahead of it, being the older head of the same channel. + const afterEvents = yield* Queue.unbounded() + yield* recipientClient.Open(openRequest(recipient, sender)).pipe( + Stream.runForEach((event) => Queue.offer(afterEvents, event)), + Effect.forkScoped + ) + assert.strictEqual((yield* Queue.take(afterEvents))._tag, "Opened") + + const second = Uint8Array.of(9, 8, 7) + const secondHash = yield* Canonical.digest(second).pipe( + Effect.provideService(Crypto.Crypto, crypto) + ) + yield* senderClient.Push({ + sessionId: senderOpened.sessionId, + relayMessageId: secondRelayMessageId, + payload: new TextEncoder().encode( + yield* Schema.encodeEffect(RelaySyncEnvelopeJson)({ + connectionEpoch: "epoch", + sequence: 1, + documentId, + documentType: Task.name, + messageHash: secondHash, + message: second, + lineage: Identity.genesisLineage, + writerProvenance: [] + }) + ) + }) + + const next = yield* Queue.take(afterEvents) + assert.strictEqual(next._tag, "StoredMessage") + if (next._tag !== "StoredMessage") return + assert.strictEqual( + next.relayMessageId, + secondRelayMessageId, + "an acknowledged message must not be redelivered" + ) + })), + { timeout: 30_000 } + ) +}) From 4ca281dd2d458560ebd4e218f230c4556d76d87d Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Mon, 27 Jul 2026 01:45:18 -0500 Subject: [PATCH 16/29] Run relay inbox retention as a cluster singleton expire and collect are global sweeps that take no inbox key, and the entity owning an inbox is passivated as soon as it goes idle, so no entity can host them: the inboxes that most need expiring are exactly the ones with nobody connected. Nothing called either operation, which made messageTtlMillis and terminalRetentionMillis inert - undelivered messages were never expired, terminal rows were never collected, and the retained row count would climb until admission was refused for good. A cluster singleton gives it one owner at a time that moves with the shard rather than being pinned to a process. Interval, batch limit, retention window and enabled are all required: enabling retention silently would have every deployment delete durable rows on a schedule it never chose, and defaulting it off would leave TTL quietly dead. A sweep failure is caught and logged rather than allowed to escape. Sharding converts a failed singleton into a defect, which would stop retention for the whole deployment until a rebalance happened to move ownership. index.ts is regenerated by pnpm codegen; it had not been run since the relay modules landed, so five of them were missing from the package entry point. --- .../local-rpc/src/RelayInboxMaintenance.ts | 98 +++++++++++++ packages/local-rpc/src/index.ts | 5 + .../test/RelayInboxMaintenance.test.ts | 136 ++++++++++++++++++ 3 files changed, 239 insertions(+) create mode 100644 packages/local-rpc/src/RelayInboxMaintenance.ts create mode 100644 packages/local-rpc/test/RelayInboxMaintenance.test.ts diff --git a/packages/local-rpc/src/RelayInboxMaintenance.ts b/packages/local-rpc/src/RelayInboxMaintenance.ts new file mode 100644 index 0000000..88c7c1d --- /dev/null +++ b/packages/local-rpc/src/RelayInboxMaintenance.ts @@ -0,0 +1,98 @@ +import * as Clock from "effect/Clock" +import * as Effect from "effect/Effect" +import * as Layer from "effect/Layer" +import * as Schedule from "effect/Schedule" +import type * as Sharding from "effect/unstable/cluster/Sharding" +import * as Singleton from "effect/unstable/cluster/Singleton" +import * as RelayInboxStore from "./RelayInboxStore.js" + +/** + * Retention for every relay inbox in the deployment. + * + * `expire` and `collect` are global sweeps rather than per-inbox operations, and the entity that + * owns an inbox is passivated as soon as it goes idle, so no entity can host them: the inboxes that + * most need expiring are exactly the ones with nobody connected to run the sweep. This is therefore + * a cluster singleton — one runner owns it at a time, and ownership moves with the shard rather than + * being pinned to a particular process. + * + * Without it `messageTtlMillis` and `terminalRetentionMillis` are inert: undelivered messages are + * never expired, terminal rows are never collected, the table grows without bound, and the retained + * row count climbs until admission is permanently refused. + */ + +export interface Options { + /** How often the sweep runs. */ + readonly intervalMillis: number + /** + * Rows touched per sweep, per operation. + * + * Bounded so one sweep cannot hold long write transactions over a large backlog; the loop simply + * runs again on the next interval. + */ + readonly batchLimit: number + /** + * How long an expired message's identity is retained. + * + * Expiry is a terminal transition like settlement, so the identity has to outlive the window in + * which its sender may still replay it. The store only ever grows this horizon, never shrinks it. + */ + readonly terminalRetentionMillis: number + /** + * Whether this deployment runs retention at all. + * + * Required rather than defaulted. Enabling it silently would have every deployment delete durable + * rows on a schedule it never chose, and defaulting it off would make TTL quietly inert. + */ + readonly enabled: boolean +} + +const sweep = ( + store: RelayInboxStore.RelayInboxStore["Service"], + options: Options +) => + Effect.gen(function*() { + const now = yield* Clock.currentTimeMillis + const expired = yield* store.expire({ + now, + limit: options.batchLimit, + terminalRetentionMillis: options.terminalRetentionMillis + }) + const collected = yield* store.collect({ now, limit: options.batchLimit }) + if (expired > 0 || collected > 0) { + // Expiry destroys a message whose sender was told the relay had taken custody, so it is + // reported rather than performed silently. + yield* Effect.logInfo("Relay inbox retention swept").pipe( + Effect.annotateLogs({ expired, collected }) + ) + } + }).pipe( + // A sweep failure must not end the singleton. `Sharding` turns a failed singleton into a + // defect, and retention would then stop for the whole deployment until a rebalance happened to + // move ownership — a silent, unbounded outage of the only thing bounding table growth. + Effect.catchTag("ReplicaError", (error) => + Effect.logWarning("Relay inbox retention sweep failed; retrying on the next interval").pipe( + Effect.annotateLogs({ reason: error.reason._tag }) + )), + Effect.withSpan("RelayInboxMaintenance.sweep") + ) + +/** + * Registers the retention sweep as a cluster singleton. + * + * Requires `Sharding` but never builds one, so the deployment shape stays the consumer's choice. + */ +export const layer = ( + options: Options +): Layer.Layer => + options.enabled + ? Singleton.make( + "EffectLocalRelayInboxMaintenance", + Effect.gen(function*() { + // Resolved once here rather than per sweep, so the loop carries no requirement of its own. + const store = yield* RelayInboxStore.RelayInboxStore + yield* sweep(store, options).pipe( + Effect.repeat(Schedule.spaced(options.intervalMillis)) + ) + }) + ) + : Layer.empty diff --git a/packages/local-rpc/src/index.ts b/packages/local-rpc/src/index.ts index 92f3337..8c5c4a9 100644 --- a/packages/local-rpc/src/index.ts +++ b/packages/local-rpc/src/index.ts @@ -8,5 +8,10 @@ export * as PeerRelayStore from "./PeerRelayStore.js" export * as PeerRpc from "./PeerRpc.js" export * as PeerRpcError from "./PeerRpcError.js" export * as PeerRpcServer from "./PeerRpcServer.js" +export * as RelayInbox from "./RelayInbox.js" +export * as RelayInboxMaintenance from "./RelayInboxMaintenance.js" +export * as RelayInboxStore from "./RelayInboxStore.js" +export * as RelayServer from "./RelayServer.js" export * as RpcPeerTransport from "./RpcPeerTransport.js" export * as SqlPeerRelayStore from "./SqlPeerRelayStore.js" +export * as SqlRelayInboxStore from "./SqlRelayInboxStore.js" diff --git a/packages/local-rpc/test/RelayInboxMaintenance.test.ts b/packages/local-rpc/test/RelayInboxMaintenance.test.ts new file mode 100644 index 0000000..a754588 --- /dev/null +++ b/packages/local-rpc/test/RelayInboxMaintenance.test.ts @@ -0,0 +1,136 @@ +import { NodeCrypto } from "@effect/platform-node" +import { SqliteClient } from "@effect/sql-sqlite-node" +import { assert, describe, it } from "@effect/vitest" +import * as Identity from "@lucas-barake/effect-local/Identity" +import * as Clock from "effect/Clock" +import * as Effect from "effect/Effect" +import * as Layer from "effect/Layer" +import { TestClock } from "effect/testing" +import * as MessageStorage from "effect/unstable/cluster/MessageStorage" +import * as RunnerHealth from "effect/unstable/cluster/RunnerHealth" +import * as Runners from "effect/unstable/cluster/Runners" +import * as RunnerStorage from "effect/unstable/cluster/RunnerStorage" +import * as Sharding from "effect/unstable/cluster/Sharding" +import * as ShardingConfig from "effect/unstable/cluster/ShardingConfig" +import * as RelayInboxMaintenance from "../src/RelayInboxMaintenance.js" +import * as RelayInboxStore from "../src/RelayInboxStore.js" +import * as SqlRelayInboxStore from "../src/SqlRelayInboxStore.js" + +const peer = (value: string) => Identity.PeerId.make(`peer_00000000-0000-4000-8000-${value}`) +const relayId = (value: string) => Identity.RelayMessageId.make(`rly_00000000-0000-4000-8000-${value}`) +const documentId = (value: string) => Identity.DocumentId.make(`doc_00000000-0000-4000-8000-${value}`) + +const inboxKey = "inbox-a" + +const options: RelayInboxMaintenance.Options = { + intervalMillis: 1_000, + batchLimit: 100, + terminalRetentionMillis: 10_000, + enabled: true +} + +const TestShardingConfig = ShardingConfig.layer({ + entityTerminationTimeout: 0, + entityMessagePollInterval: 5000, + sendRetryInterval: 100 +}) + +const store = SqlRelayInboxStore.layer.pipe( + Layer.provide(SqliteClient.layer({ filename: ":memory:", disableWAL: true })), + Layer.provide(NodeCrypto.layer), + Layer.orDie +) + +const relay = (maintenance: RelayInboxMaintenance.Options) => + RelayInboxMaintenance.layer(maintenance).pipe( + Layer.provideMerge(Sharding.layer), + Layer.provide(Runners.layerNoop), + Layer.provideMerge(MessageStorage.layerMemory), + Layer.provide(RunnerStorage.layerMemory), + Layer.provide(RunnerHealth.layerNoop), + Layer.provide(TestShardingConfig), + Layer.provideMerge(store) + ) + +const channel = { + tenantId: "tenant-a", + senderSubjectId: "sender-a", + senderPeerId: peer("00000000aaa1"), + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + senderConnectionEpoch: "epoch-1" +} + +const admission = (now: number, ttl: number) => ({ + inboxKey, + channel, + envelope: { + relayMessageId: relayId("000000000001"), + relayPeerId: peer("00000000ffff"), + sender: { + tenantId: channel.tenantId, + subjectId: channel.senderSubjectId, + peerId: channel.senderPeerId, + replicaIncarnation: channel.senderReplicaIncarnation, + connectionEpoch: channel.senderConnectionEpoch, + sequence: 0 + }, + recipient: { tenantId: "tenant-a", subjectId: "recipient-a", peerId: peer("00000000bbb1") }, + payloadVersion: 1 as const, + document: { documentId: documentId("00000000dddd"), documentType: "note" }, + writerProvenance: [], + messageHash: "a".repeat(64), + outerEnvelopeDigest: "b".repeat(64), + payload: new Uint8Array([1, 2, 3]) + }, + now, + messageTtlMillis: ttl, + senderRetryHorizonMillis: 1_000, + quota: { maxPendingMessages: 100, maxPendingBytes: 10_000_000 } +}) + +describe("RelayInboxMaintenance", () => { + it.effect("expires overdue messages and later collects them, with no inbox connected", () => + Effect.gen(function*() { + // Shard acquisition has to complete before the singleton is started by its owner. + yield* TestClock.adjust(5_000) + const inbox = yield* RelayInboxStore.RelayInboxStore + + const now = yield* Clock.currentTimeMillis + yield* inbox.admit(admission(now, 2_000)) + assert.strictEqual((yield* inbox.pendingHeads(inboxKey, { limit: 10 })).length, 1) + + // Nothing is subscribed to this inbox and its entity is passivated, which is precisely the + // state in which TTL matters: only a cluster-wide owner can sweep it. + yield* TestClock.adjust(4_000) + assert.strictEqual( + (yield* inbox.pendingHeads(inboxKey, { limit: 10 })).length, + 0, + "an overdue message is expired by the singleton" + ) + const expired = yield* inbox.usage(inboxKey) + assert.strictEqual(expired.retainedCount, 1, "its identity is retained for the replay window") + + // Past the retention horizon the identity itself is collected, which is what bounds the table. + yield* TestClock.adjust(20_000) + const collected = yield* inbox.usage(inboxKey) + assert.strictEqual(collected.retainedCount, 0) + }).pipe(Effect.provide(relay(options)))) + + it.effect("sweeps nothing when retention is disabled", () => + Effect.gen(function*() { + yield* TestClock.adjust(5_000) + const inbox = yield* RelayInboxStore.RelayInboxStore + + const now = yield* Clock.currentTimeMillis + yield* inbox.admit(admission(now, 2_000)) + yield* TestClock.adjust(30_000) + + // The flag is load bearing in both directions: a deployment that did not ask for retention + // must not have its durable rows deleted on a schedule it never chose. + assert.strictEqual( + (yield* inbox.pendingHeads(inboxKey, { limit: 10 })).length, + 1, + "no sweep runs when retention is disabled" + ) + }).pipe(Effect.provide(relay({ ...options, enabled: false })))) +}) From d8ba935f4fd2366a5f08f5975ab87f80dfb2ea98 Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Mon, 27 Jul 2026 01:47:55 -0500 Subject: [PATCH 17/29] Span the relay inbox I/O and entity boundaries RULES.md requires spans at meaningful workflow, I/O and remote boundaries, and the new relay modules had none, so a stalled inbox or a slow sweep was invisible. Every store operation and every entity handler now carries a stable span name. The store is spanned where the service is constructed rather than inside each body, so the names sit together and the database round trips are all visible at one boundary. Attributes are limited to identifiers: inbox_key is a digest, and relay message ids and outcomes are opaque. The envelope and payload are never recorded. Subscribe spans the installation of the session rather than its lifetime, since the handler returns as soon as the session is reachable and the stream long outlives it. --- packages/local-rpc/src/RelayInbox.ts | 98 +++++++++++++------- packages/local-rpc/src/SqlRelayInboxStore.ts | 53 +++++++++-- 2 files changed, 108 insertions(+), 43 deletions(-) diff --git a/packages/local-rpc/src/RelayInbox.ts b/packages/local-rpc/src/RelayInbox.ts index 23bc557..19e3804 100644 --- a/packages/local-rpc/src/RelayInbox.ts +++ b/packages/local-rpc/src/RelayInbox.ts @@ -474,7 +474,15 @@ export const layer = (options: Options) => Effect.logWarning("Relay inbox admission failed").pipe( Effect.annotateLogs({ inboxKey, reason: error.reason._tag }), Effect.andThen(new PeerRpcError.ServerUnavailable()) - )) + )), + // `inboxKey` is a digest and the ids are opaque, so both are safe to record. The + // envelope and its payload never become attributes. + Effect.withSpan("RelayInbox.Deliver", { + attributes: { + inbox_key: inboxKey, + relay_message_id: payload.envelope.relayMessageId + } + }) ), // Forked so it does not hold the entity's sequential handler permit. Without this the @@ -482,39 +490,45 @@ export const layer = (options: Options) => // each delivery could never be handled, stalling the inbox with no error anywhere. Subscribe: ({ payload }) => Rpc.fork( - sessionLock.withPermit( - // Installation is uninterruptible up to the point the session becomes reachable - // through `sessionRef`. An interrupt in between would strand the scope — nothing - // else holds a reference to close it — and could leave a dispatcher polling for a - // session no one owns, delivering alongside its own replacement. - Effect.uninterruptibleMask((restore) => - Effect.gen(function*() { - yield* restore(closeCurrentSession) - - const scope = yield* Scope.make() - const install = Effect.gen(function*() { - const outbound = yield* Queue.bounded(0) - const now = yield* Clock.currentTimeMillis - const deadlineAt = yield* Ref.make(now + options.sessionDeadlineMillis) - const session: Session = { - sessionId: payload.sessionId, - outbound, - settlements: new Map(), - busyChannels: new Set(), - wake: yield* Latch.make(true), - scope, - deadlineAt - } - yield* Effect.forkIn(dispatch(session), scope) - yield* Ref.set(sessionRef, Option.some(session)) - return outbound + // Spans the installation of the session, not its lifetime: the handler returns the + // queue as soon as the session is reachable, and the stream outlives it. + Effect.withSpan( + sessionLock.withPermit( + // Installation is uninterruptible up to the point the session becomes reachable + // through `sessionRef`. An interrupt in between would strand the scope — nothing + // else holds a reference to close it — and could leave a dispatcher polling for a + // session no one owns, delivering alongside its own replacement. + Effect.uninterruptibleMask((restore) => + Effect.gen(function*() { + yield* restore(closeCurrentSession) + + const scope = yield* Scope.make() + const install = Effect.gen(function*() { + const outbound = yield* Queue.bounded(0) + const now = yield* Clock.currentTimeMillis + const deadlineAt = yield* Ref.make(now + options.sessionDeadlineMillis) + const session: Session = { + sessionId: payload.sessionId, + outbound, + settlements: new Map(), + busyChannels: new Set(), + wake: yield* Latch.make(true), + scope, + deadlineAt + } + yield* Effect.forkIn(dispatch(session), scope) + yield* Ref.set(sessionRef, Option.some(session)) + return outbound + }) + return yield* Effect.onError( + install, + () => Effect.orDie(Scope.close(scope, Exit.void)) + ) }) - return yield* Effect.onError( - install, - () => Effect.orDie(Scope.close(scope, Exit.void)) - ) - }) - ) + ) + ), + "RelayInbox.Subscribe", + { attributes: { inbox_key: inboxKey } } ) ), @@ -539,9 +553,21 @@ export const layer = (options: Options) => // relay receipt on this reply, so a premature success would leave neither side holding // the message. yield* Deferred.await(settlement.durable) - }), + }).pipe( + Effect.withSpan("RelayInbox.Settle", { + attributes: { + inbox_key: inboxKey, + relay_message_id: payload.relayMessageId, + outcome: payload.outcome + } + }) + ), - Heartbeat: ({ payload }) => currentSession(payload.sessionId).pipe(Effect.flatMap(extendDeadline)), + Heartbeat: ({ payload }) => + currentSession(payload.sessionId).pipe( + Effect.flatMap(extendDeadline), + Effect.withSpan("RelayInbox.Heartbeat", { attributes: { inbox_key: inboxKey } }) + ), EndSession: ({ payload }) => sessionLock.withPermit( @@ -549,6 +575,8 @@ export const layer = (options: Options) => Effect.flatMap(closeSession), Effect.catchTag("SessionUnavailable", () => Effect.void) ) + ).pipe( + Effect.withSpan("RelayInbox.EndSession", { attributes: { inbox_key: inboxKey } }) ) }) }), diff --git a/packages/local-rpc/src/SqlRelayInboxStore.ts b/packages/local-rpc/src/SqlRelayInboxStore.ts index 14692a8..a655f52 100644 --- a/packages/local-rpc/src/SqlRelayInboxStore.ts +++ b/packages/local-rpc/src/SqlRelayInboxStore.ts @@ -567,15 +567,52 @@ export const make = Effect.gen(function*() { return rows.length }) + // Spanned at the boundary rather than inside each body, so every database round trip is visible + // with a stable name. `inbox_key` is a digest and safe to record; the envelope and payload are + // never attributes. return RelayInboxStore.RelayInboxStore.of({ - admit, - pendingHeads, - recordDelivery, - settle, - usage, - abandoned, - expire, - collect + admit: (request) => + admit(request).pipe( + Effect.withSpan("SqlRelayInboxStore.admit", { + attributes: { inbox_key: request.inboxKey, relay_message_id: request.envelope.relayMessageId } + }) + ), + pendingHeads: (inboxKey, options) => + pendingHeads(inboxKey, options).pipe( + Effect.withSpan("SqlRelayInboxStore.pendingHeads", { + attributes: { inbox_key: inboxKey, limit: options.limit } + }) + ), + recordDelivery: (inboxKey, relayMessageId, options) => + recordDelivery(inboxKey, relayMessageId, options).pipe( + Effect.withSpan("SqlRelayInboxStore.recordDelivery", { + attributes: { inbox_key: inboxKey, relay_message_id: relayMessageId, max_deliveries: options.maxDeliveries } + }) + ), + settle: (inboxKey, relayMessageId, options) => + settle(inboxKey, relayMessageId, options).pipe( + Effect.withSpan("SqlRelayInboxStore.settle", { + attributes: { inbox_key: inboxKey, relay_message_id: relayMessageId, outcome: options.outcome } + }) + ), + usage: (inboxKey) => + usage(inboxKey).pipe( + Effect.withSpan("SqlRelayInboxStore.usage", { attributes: { inbox_key: inboxKey } }) + ), + abandoned: (inboxKey, options) => + abandoned(inboxKey, options).pipe( + Effect.withSpan("SqlRelayInboxStore.abandoned", { + attributes: { inbox_key: inboxKey, limit: options.limit } + }) + ), + expire: (options) => + expire(options).pipe( + Effect.withSpan("SqlRelayInboxStore.expire", { attributes: { limit: options.limit } }) + ), + collect: (options) => + collect(options).pipe( + Effect.withSpan("SqlRelayInboxStore.collect", { attributes: { limit: options.limit } }) + ) }) }) satisfies Effect.Effect< RelayInboxStore.RelayInboxStore["Service"], From 87c1260bc39a78c99adb67c71f977ed8f60f898d Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Mon, 27 Jul 2026 01:52:28 -0500 Subject: [PATCH 18/29] Restore the authorization the front door had dropped Two guarantees the old server made were missing from the rebuild. Neither was untested by accident; both were simply gone. Settling was not authorized at all. The handler resolved the session and forwarded to the entity, so a peer whose Receive authority had been withdrawn could still perform a durable terminal mutation. The session monitor does end a session whose grants lapse, but asynchronously, which leaves a window and reports SessionUnavailable where the contract calls for AccessDenied. Receive is now re-checked on every settle. The unbounded-decode risk acknowledgement was never requested. The old server required it before admitting a Push and before a delivery, on its own port that denies by default, because relaying a document commits its recipient to a decode that is not allocation bounded on the Automerge version this protocol targets. The new front door inherited that risk from the ordinary Send grant instead, which is not the same decision. It is now required on Push, on each delivery, and on settle. Also spans the four front-door RPCs, and makes retention part of RelayServer.layer rather than a separate layer a deployment can forget: TTL and collection are inert without it, and the failure mode is a table that grows until admission stops succeeding. The digest assertion now recomputes the outer envelope digest independently and compares it, rather than checking it looks like a hex string - a digest over the wrong inputs matches that pattern just as well as a correct one. --- packages/local-rpc/src/RelayServer.ts | 448 ++++++----- packages/local-rpc/test/RelayServer.test.ts | 710 ++++++++++++++++++ .../local-rpc/test/RelayWireFormat.test.ts | 16 +- 3 files changed, 989 insertions(+), 185 deletions(-) create mode 100644 packages/local-rpc/test/RelayServer.test.ts diff --git a/packages/local-rpc/src/RelayServer.ts b/packages/local-rpc/src/RelayServer.ts index 1fff951..b9be35e 100644 --- a/packages/local-rpc/src/RelayServer.ts +++ b/packages/local-rpc/src/RelayServer.ts @@ -15,6 +15,7 @@ import * as PeerRelayLimits from "./PeerRelayLimits.js" import * as PeerRpc from "./PeerRpc.js" import * as PeerRpcError from "./PeerRpcError.js" import * as RelayInbox from "./RelayInbox.js" +import * as RelayInboxMaintenance from "./RelayInboxMaintenance.js" /** * The relay's public front door. @@ -127,6 +128,29 @@ export const layerHandlers = (options: Options) => Effect.gen(function*() { const authenticated = yield* PeerAuthentication.AuthenticatedPeer const session = yield* sessionFor(payload.sessionId, authenticated.principal) + + // Re-authorized here, not merely at the handshake. Settling is a durable mutation of the + // relay's state, and a principal whose Receive authority has been withdrawn must not be + // able to perform one. The session monitor also ends a session whose grants lapse, but that + // is asynchronous, so on its own it leaves a window in which a revoked peer can still + // settle — and it reports the wrong thing, `SessionUnavailable` rather than `AccessDenied`. + const receive = yield* authorization.authorize({ + direction: "Receive", + principal: session.principal, + remote: session.remote, + documents: session.documents + }) + if (receive.documents.length === 0) { + return yield* new PeerRpcError.AccessDenied() + } + yield* authorization.authorizeUnsafeUnboundedAutomerge3Decode({ + risk: PeerRelayAuthorization.unsafeUnboundedAutomerge3DecodeRisk, + direction: "Receive", + principal: session.principal, + remote: session.remote, + documents: session.documents + }) + // Keyed by the caller's own inbox, so settling another device's message is not expressible. yield* bounded( inboxClient(session.inboxKeySelf).Settle({ @@ -141,207 +165,231 @@ export const layerHandlers = (options: Options) => Effect.catchTag("AlreadyProcessingMessage", () => new PeerRpcError.SessionOverloaded()), Effect.catchTag("PersistenceError", () => new PeerRpcError.ServerUnavailable()) ) - }) + }).pipe( + // Identifiers only. The session id is unguessable but not secret, and the message hash is + // already a digest; the payload is never an attribute. + Effect.withSpan("RelayServer.Settle", { + attributes: { relay_message_id: payload.relayMessageId, outcome } + }) + ) return PeerRpc.Rpcs.of({ Open: (payload) => - Stream.unwrap(Effect.gen(function*() { - const authenticated = yield* PeerAuthentication.AuthenticatedPeer - const principal = authenticated.principal - const now = yield* Clock.currentTimeMillis - - if (authenticated.validUntil <= now) { - return yield* new PeerRpcError.AuthenticationFailure() - } - if (payload.protocolVersion !== PeerRpc.protocolVersion) { - return yield* new PeerRpcError.UnsupportedVersion() - } - // The client states who it believes it is and which relay it believes it reached. Both - // are checked against the authenticated identity rather than trusted, so a credential can - // never be used to open a session for another device. - if ( - payload.expectedRelayPeerId !== options.peerId || - payload.expectedLocal.tenantId !== principal.tenantId || - payload.expectedLocal.subjectId !== principal.subjectId || - payload.expectedLocal.peerId !== principal.peerId - ) { - return yield* new PeerRpcError.PeerMismatch() - } - if (principal.tenantId !== options.tenantId) { - return yield* new PeerRpcError.AccessDenied() - } - // The negotiated windows have to nest: a receipt must outlive both the message and the - // window in which its sender may replay it, or a redelivery could be applied twice. - if ( - payload.senderRetryHorizonMillis > limits.maximumSenderRetryHorizonMillis || - payload.receiptRetentionMillis > limits.maximumReceiptRetentionMillis || - payload.receiptRetentionMillis < - Math.max(limits.messageTtlMillis, payload.senderRetryHorizonMillis) + - limits.minimumTerminalRetentionMillis - ) { - return yield* new PeerRpcError.InvalidRequest() - } + // Spans the handshake, not the session: the stream it returns outlives this effect. + Stream.unwrap( + Effect.withSpan("RelayServer.Open", { + attributes: { remote_peer_id: payload.remote.peerId } + })(Effect.gen(function*() { + const authenticated = yield* PeerAuthentication.AuthenticatedPeer + const principal = authenticated.principal + const now = yield* Clock.currentTimeMillis - const send = yield* authorization.authorize({ - direction: "Send", - principal, - remote: payload.remote, - documents: payload.documents - }) - const receive = yield* authorization.authorize({ - direction: "Receive", - principal, - remote: payload.remote, - documents: payload.documents - }) - // Authorization is not instantaneous, so both grants and the credential are rechecked - // against the clock afterwards rather than against the time the request arrived. - const authorizedAt = yield* Clock.currentTimeMillis - if (authenticated.validUntil <= authorizedAt) { - return yield* new PeerRpcError.AuthenticationFailure() - } - if (send.validUntil <= authorizedAt || receive.validUntil <= authorizedAt) { - return yield* new PeerRpcError.AccessDenied() - } - - // Sessions are the front door's only unbounded resource, and an authenticated client can - // open them in a loop. Checked before minting an id so a refusal costs nothing. - let heldBySubject = 0 - for (const held of sessions.values()) { + if (authenticated.validUntil <= now) { + return yield* new PeerRpcError.AuthenticationFailure() + } + if (payload.protocolVersion !== PeerRpc.protocolVersion) { + return yield* new PeerRpcError.UnsupportedVersion() + } + // The client states who it believes it is and which relay it believes it reached. Both + // are checked against the authenticated identity rather than trusted, so a credential can + // never be used to open a session for another device. if ( - held.principal.tenantId === principal.tenantId && - held.principal.subjectId === principal.subjectId + payload.expectedRelayPeerId !== options.peerId || + payload.expectedLocal.tenantId !== principal.tenantId || + payload.expectedLocal.subjectId !== principal.subjectId || + payload.expectedLocal.peerId !== principal.peerId ) { - heldBySubject += 1 + return yield* new PeerRpcError.PeerMismatch() + } + if (principal.tenantId !== options.tenantId) { + return yield* new PeerRpcError.AccessDenied() + } + // The negotiated windows have to nest: a receipt must outlive both the message and the + // window in which its sender may replay it, or a redelivery could be applied twice. + if ( + payload.senderRetryHorizonMillis > limits.maximumSenderRetryHorizonMillis || + payload.receiptRetentionMillis > limits.maximumReceiptRetentionMillis || + payload.receiptRetentionMillis < + Math.max(limits.messageTtlMillis, payload.senderRetryHorizonMillis) + + limits.minimumTerminalRetentionMillis + ) { + return yield* new PeerRpcError.InvalidRequest() } - } - if (heldBySubject >= limits.maxSessionsPerSubject) { - return yield* new PeerRpcError.RequestCapacityExceeded() - } - const sessionId = yield* Identity.makeSessionId.pipe( - Effect.provideService(Crypto.Crypto, crypto), - // The platform's randomness failing is an environment fault, not something the client - // can act on, so it is reported as the relay being unavailable. - Effect.catchTag("PlatformError", () => new PeerRpcError.ServerUnavailable()) - ) - const inboxKeySelf = yield* encodeInboxKey(principal).pipe( - Effect.provideService(Crypto.Crypto, crypto), - Effect.catchTag("ReplicaError", () => new PeerRpcError.ServerUnavailable()) - ) - const inboxKeyRemote = yield* encodeInboxKey(send.remote).pipe( - Effect.provideService(Crypto.Crypto, crypto), - Effect.catchTag("ReplicaError", () => new PeerRpcError.ServerUnavailable()) - ) + const send = yield* authorization.authorize({ + direction: "Send", + principal, + remote: payload.remote, + documents: payload.documents + }) + const receive = yield* authorization.authorize({ + direction: "Receive", + principal, + remote: payload.remote, + documents: payload.documents + }) + // Authorization is not instantaneous, so both grants and the credential are rechecked + // against the clock afterwards rather than against the time the request arrived. + const authorizedAt = yield* Clock.currentTimeMillis + if (authenticated.validUntil <= authorizedAt) { + return yield* new PeerRpcError.AuthenticationFailure() + } + if (send.validUntil <= authorizedAt || receive.validUntil <= authorizedAt) { + return yield* new PeerRpcError.AccessDenied() + } - const session: Session = { - sessionId, - principal, - remote: send.remote, - documents: payload.documents, - senderReplicaIncarnation: payload.senderReplicaIncarnation, - senderRetryHorizonMillis: payload.senderRetryHorizonMillis, - inboxKeySelf, - inboxKeyRemote - } + // Sessions are the front door's only unbounded resource, and an authenticated client can + // open them in a loop. Checked before minting an id so a refusal costs nothing. + let heldBySubject = 0 + for (const held of sessions.values()) { + if ( + held.principal.tenantId === principal.tenantId && + held.principal.subjectId === principal.subjectId + ) { + heldBySubject += 1 + } + } + if (heldBySubject >= limits.maxSessionsPerSubject) { + return yield* new PeerRpcError.RequestCapacityExceeded() + } - sessions.set(sessionId, session) - yield* Effect.addFinalizer(() => - Effect.sync(() => { - sessions.delete(sessionId) - }).pipe( - // Best effort: the entity's own liveness deadline is what actually bounds an - // abandoned session, because this never runs when the node itself fails. - Effect.andThen(Effect.ignore(inboxClient(inboxKeySelf).EndSession({ sessionId }))) + const sessionId = yield* Identity.makeSessionId.pipe( + Effect.provideService(Crypto.Crypto, crypto), + // The platform's randomness failing is an environment fault, not something the client + // can act on, so it is reported as the relay being unavailable. + Effect.catchTag("PlatformError", () => new PeerRpcError.ServerUnavailable()) + ) + const inboxKeySelf = yield* encodeInboxKey(principal).pipe( + Effect.provideService(Crypto.Crypto, crypto), + Effect.catchTag("ReplicaError", () => new PeerRpcError.ServerUnavailable()) + ) + const inboxKeyRemote = yield* encodeInboxKey(send.remote).pipe( + Effect.provideService(Crypto.Crypto, crypto), + Effect.catchTag("ReplicaError", () => new PeerRpcError.ServerUnavailable()) ) - ) - // A session must not outlive what authorized it. The credential and both grants each - // expose an `invalidated` signal and a deadline, and none of them is observed by the - // client, so the relay is the only place that can end the session when authority is - // withdrawn. Without this the heartbeat below would keep an unauthorized session alive - // for as long as the socket stays open. - const authorityLapsesAt = Math.min( - authenticated.validUntil, - send.validUntil, - receive.validUntil - ) - yield* Effect.raceAll([ - authenticated.invalidated, - send.invalidated, - receive.invalidated, - Effect.sleep(Math.max(0, authorityLapsesAt - authorizedAt)) - ]).pipe( - Effect.andThen( - Effect.logInfo("Relay session authority withdrawn; ending session").pipe( - Effect.annotateLogs({ sessionId }) + const session: Session = { + sessionId, + principal, + remote: send.remote, + documents: payload.documents, + senderReplicaIncarnation: payload.senderReplicaIncarnation, + senderRetryHorizonMillis: payload.senderRetryHorizonMillis, + inboxKeySelf, + inboxKeyRemote + } + + sessions.set(sessionId, session) + yield* Effect.addFinalizer(() => + Effect.sync(() => { + sessions.delete(sessionId) + }).pipe( + // Best effort: the entity's own liveness deadline is what actually bounds an + // abandoned session, because this never runs when the node itself fails. + Effect.andThen(Effect.ignore(inboxClient(inboxKeySelf).EndSession({ sessionId }))) ) - ), - Effect.andThen(Effect.ignore(inboxClient(inboxKeySelf).EndSession({ sessionId }))), - Effect.forkScoped - ) + ) - // Proves this node and its socket are still alive. Stopping is the signal. - yield* Effect.ignore(inboxClient(inboxKeySelf).Heartbeat({ sessionId })).pipe( - Effect.repeat(Schedule.spaced(options.heartbeatIntervalMillis)), - Effect.forkScoped - ) + // A session must not outlive what authorized it. The credential and both grants each + // expose an `invalidated` signal and a deadline, and none of them is observed by the + // client, so the relay is the only place that can end the session when authority is + // withdrawn. Without this the heartbeat below would keep an unauthorized session alive + // for as long as the socket stays open. + const authorityLapsesAt = Math.min( + authenticated.validUntil, + send.validUntil, + receive.validUntil + ) + yield* Effect.raceAll([ + authenticated.invalidated, + send.invalidated, + receive.invalidated, + Effect.sleep(Math.max(0, authorityLapsesAt - authorizedAt)) + ]).pipe( + Effect.andThen( + Effect.logInfo("Relay session authority withdrawn; ending session").pipe( + Effect.annotateLogs({ sessionId }) + ) + ), + Effect.andThen(Effect.ignore(inboxClient(inboxKeySelf).EndSession({ sessionId }))), + Effect.forkScoped + ) - const opened: PeerRpc.OpenEvent = { - _tag: "Opened", - protocolVersion: PeerRpc.protocolVersion, - sessionId, - remotePeerId: send.remote.peerId, - authenticatedLocal: principal - } + // Proves this node and its socket are still alive. Stopping is the signal. + yield* Effect.ignore(inboxClient(inboxKeySelf).Heartbeat({ sessionId })).pipe( + Effect.repeat(Schedule.spaced(options.heartbeatIntervalMillis)), + Effect.forkScoped + ) - // `Stream.concat` builds the second stream only after the first completes, so the - // subscription attaches after `Opened` reaches the wire. A client that sees `Opened` has - // a session the relay accepted, not yet one the owning entity has attached. - return Stream.concat( - Stream.succeed(opened), - inboxClient(inboxKeySelf).Subscribe({ sessionId }).pipe( - // Re-authorized per message rather than once at handshake. A grant can be narrowed - // or revoked mid session, and anything with a Send grant to this device can write - // into its inbox, so the handshake's Receive decision cannot stand in for the - // recipient's right to see a particular document. A message that fails here is left - // unsettled and simply not emitted, so it stays durable for a later session. - Stream.filterEffect((message) => - authorization.authorize({ - direction: "Receive", - principal: session.principal, - remote: session.remote, - documents: [{ - documentType: message.document.documentType, - documentId: message.document.documentId - }] - }).pipe( - Effect.as(true), - Effect.catchTag("AccessDenied", () => - Effect.logInfo("Withheld a delivery the recipient may no longer receive").pipe( - Effect.annotateLogs({ - sessionId, + const opened: PeerRpc.OpenEvent = { + _tag: "Opened", + protocolVersion: PeerRpc.protocolVersion, + sessionId, + remotePeerId: send.remote.peerId, + authenticatedLocal: principal + } + + // `Stream.concat` builds the second stream only after the first completes, so the + // subscription attaches after `Opened` reaches the wire. A client that sees `Opened` has + // a session the relay accepted, not yet one the owning entity has attached. + return Stream.concat( + Stream.succeed(opened), + inboxClient(inboxKeySelf).Subscribe({ sessionId }).pipe( + // Re-authorized per message rather than once at handshake. A grant can be narrowed + // or revoked mid session, and anything with a Send grant to this device can write + // into its inbox, so the handshake's Receive decision cannot stand in for the + // recipient's right to see a particular document. A message that fails here is left + // unsettled and simply not emitted, so it stays durable for a later session. + Stream.filterEffect((message) => + authorization.authorize({ + direction: "Receive", + principal: session.principal, + remote: session.remote, + documents: [{ + documentType: message.document.documentType, + documentId: message.document.documentId + }] + }).pipe( + // Delivering commits this peer to decoding the document, so the same risk + // acknowledgement the sender needed is required again on the receiving side, per + // message rather than once at the handshake. + Effect.andThen(authorization.authorizeUnsafeUnboundedAutomerge3Decode({ + risk: PeerRelayAuthorization.unsafeUnboundedAutomerge3DecodeRisk, + direction: "Receive", + principal: session.principal, + remote: session.remote, + documents: [{ + documentType: message.document.documentType, documentId: message.document.documentId - }), - Effect.as(false) - )), - Effect.catchTag("ServerUnavailable", () => Effect.succeed(false)) + }] + })), + Effect.as(true), + Effect.catchTag("AccessDenied", () => + Effect.logInfo("Withheld a delivery the recipient may no longer receive").pipe( + Effect.annotateLogs({ + sessionId, + documentId: message.document.documentId + }), + Effect.as(false) + )), + Effect.catchTag("ServerUnavailable", () => Effect.succeed(false)) + ) + ), + // Cluster level failures are not part of the public contract, so each is reported as + // the wire error that tells the client what to do about it. + Stream.catchTag("MailboxFull", () => Stream.fail(new PeerRpcError.SessionOverloaded())), + Stream.catchTag( + "AlreadyProcessingMessage", + () => Stream.fail(new PeerRpcError.SessionOverloaded()) + ), + Stream.catchTag( + "PersistenceError", + () => Stream.fail(new PeerRpcError.ServerUnavailable()) ) - ), - // Cluster level failures are not part of the public contract, so each is reported as - // the wire error that tells the client what to do about it. - Stream.catchTag("MailboxFull", () => Stream.fail(new PeerRpcError.SessionOverloaded())), - Stream.catchTag( - "AlreadyProcessingMessage", - () => Stream.fail(new PeerRpcError.SessionOverloaded()) - ), - Stream.catchTag( - "PersistenceError", - () => Stream.fail(new PeerRpcError.ServerUnavailable()) ) ) - ) - })), + })) + ), Push: (payload) => Effect.gen(function*() { @@ -379,6 +427,17 @@ export const layerHandlers = (options: Options) => if (send.documents.length === 0) { return yield* new PeerRpcError.AccessDenied() } + // Relaying a document commits its recipient to decoding it, and that decode is not + // allocation bounded on the Automerge version this protocol targets. The deployment has + // to acknowledge that risk explicitly for this document rather than inherit it from the + // ordinary Send grant, which is why it is a separate port that denies by default. + yield* authorization.authorizeUnsafeUnboundedAutomerge3Decode({ + risk: PeerRelayAuthorization.unsafeUnboundedAutomerge3DecodeRisk, + direction: "Send", + principal: session.principal, + remote: session.remote, + documents: [document] + }) const outerEnvelopeDigest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope({ domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, @@ -449,7 +508,11 @@ export const layerHandlers = (options: Options) => Effect.catchTag("AlreadyProcessingMessage", () => new PeerRpcError.SessionOverloaded()), Effect.catchTag("PersistenceError", () => new PeerRpcError.ServerUnavailable()) ) - }), + }).pipe( + Effect.withSpan("RelayServer.Push", { + attributes: { relay_message_id: payload.relayMessageId } + }) + ), Acknowledge: (payload) => settle(payload, "Acknowledged"), @@ -464,8 +527,25 @@ export const layerHandlers = (options: Options) => * an in-memory single process for tests, one runner over SQL, or many runners sharded across * machines — without the relay changing. */ -export const layer = (options: Options & { readonly inbox: RelayInbox.Options }) => - Layer.merge(layerHandlers(options), RelayInbox.layer(options.inbox)).pipe( +export const layer = ( + options: Options & { + readonly inbox: RelayInbox.Options + /** + * Retention for every inbox in the deployment. + * + * Required rather than optional, and composed here rather than left to the consumer, because + * `messageTtlMillis` and `terminalRetentionMillis` are inert without it: a deployment that + * forgot to add the singleton separately would expire nothing and collect nothing, and would + * only find out when the table had grown past the point where admission still succeeded. + */ + readonly maintenance: RelayInboxMaintenance.Options + } +) => + Layer.mergeAll( + layerHandlers(options), + RelayInbox.layer(options.inbox), + RelayInboxMaintenance.layer(options.maintenance) + ).pipe( // Two independently configured values have to agree or every session is reaped on a fixed // cycle and no client can hold a delivery stream — a total outage that only appears under a // particular configuration, so it is refused at construction rather than discovered in diff --git a/packages/local-rpc/test/RelayServer.test.ts b/packages/local-rpc/test/RelayServer.test.ts new file mode 100644 index 0000000..7d6b6e2 --- /dev/null +++ b/packages/local-rpc/test/RelayServer.test.ts @@ -0,0 +1,710 @@ +import { NodeCrypto } from "@effect/platform-node" +import { SqliteClient } from "@effect/sql-sqlite-node" +import { assert, describe, it } from "@effect/vitest" +import * as PeerSyncEnvelope from "@lucas-barake/effect-local-sql/PeerSyncEnvelope" +import * as Canonical from "@lucas-barake/effect-local/Canonical" +import * as Document from "@lucas-barake/effect-local/Document" +import * as Identity from "@lucas-barake/effect-local/Identity" +import * as Context from "effect/Context" +import * as Crypto from "effect/Crypto" +import * as Effect from "effect/Effect" +import * as Fiber from "effect/Fiber" +import * as Layer from "effect/Layer" +import * as Queue from "effect/Queue" +import * as Redacted from "effect/Redacted" +import * as Ref from "effect/Ref" +import * as Schema from "effect/Schema" +import * as Stream from "effect/Stream" +import { TestClock } from "effect/testing" +import * as MessageStorage from "effect/unstable/cluster/MessageStorage" +import * as RunnerHealth from "effect/unstable/cluster/RunnerHealth" +import * as Runners from "effect/unstable/cluster/Runners" +import * as RunnerStorage from "effect/unstable/cluster/RunnerStorage" +import * as Sharding from "effect/unstable/cluster/Sharding" +import * as ShardingConfig from "effect/unstable/cluster/ShardingConfig" +import * as RpcTest from "effect/unstable/rpc/RpcTest" +import { encodeInboxKey } from "../src/internal/relayInboxKey.js" +import * as PeerAuthentication from "../src/PeerAuthentication.js" +import * as PeerAuthenticator from "../src/PeerAuthenticator.js" +import * as PeerCredentials from "../src/PeerCredentials.js" +import * as PeerRelayAuthorization from "../src/PeerRelayAuthorization.js" +import * as PeerRelayLimits from "../src/PeerRelayLimits.js" +import * as PeerRpc from "../src/PeerRpc.js" +import * as PeerRpcError from "../src/PeerRpcError.js" +import * as RelayInbox from "../src/RelayInbox.js" +import * as RelayInboxStore from "../src/RelayInboxStore.js" +import * as RelayServer from "../src/RelayServer.js" +import * as SqlRelayInboxStore from "../src/SqlRelayInboxStore.js" + +const Task = Document.make("Task", { + schema: Schema.Struct({ title: Schema.String }), + version: 1 +}) + +const documentId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000001") +const otherDocumentId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000002") +const relayPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000001") +const senderPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000002") +const recipientPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000003") +const relayId = (value: string) => Identity.RelayMessageId.make(`rly_00000000-0000-4000-8000-${value}`) + +const sender = PeerAuthentication.PeerPrincipal.make({ + tenantId: "tenant", + subjectId: "sender", + peerId: senderPeerId +}) +const recipient = PeerAuthentication.PeerPrincipal.make({ + tenantId: "tenant", + subjectId: "recipient", + peerId: recipientPeerId +}) + +const serverOptions: RelayServer.Options = { + tenantId: "tenant", + peerId: relayPeerId, + heartbeatIntervalMillis: 30_000, + entityCallTimeoutMillis: 30_000 +} + +const inboxOptions: RelayInbox.Options = { + maxDeliveries: 10, + messageTtlMillis: 600_000, + terminalRetentionMillis: 600_000, + sessionDeadlineMillis: 90_000, + sessionSweepMillis: 1_000, + maxConcurrentChannels: 4, + storeRetryMillis: 0, + maxPendingMessages: 100, + maxPendingBytes: 10_000_000, + mailboxCapacity: 16, + maxIdleTimeMillis: 3_600_000 +} + +const TestShardingConfig = ShardingConfig.layer({ + entityTerminationTimeout: 0, + entityMessagePollInterval: 5000, + sendRetryInterval: 100 +}) + +const RelaySyncEnvelopeJson = Schema.fromJsonString( + Schema.toCodecJson(PeerSyncEnvelope.SyncEnvelope) +) + +const openRequest = ( + principal: PeerAuthentication.PeerPrincipal, + remote: PeerAuthentication.PeerPrincipal, + documents: ReadonlyArray = [{ documentType: Task.name, documentId }] +) => + PeerRpc.OpenRpc.payloadSchema.make({ + protocolVersion: PeerRpc.protocolVersion, + expectedRelayPeerId: relayPeerId, + expectedLocal: principal, + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + remote: { subjectId: remote.subjectId, peerId: remote.peerId }, + documents, + receiptRetentionMillis: PeerRelayLimits.defaults.maximumReceiptRetentionMillis, + senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis + }) + +/** + * Mutable knobs the tests turn. + * + * Authorization and credential validity are the two things the relay is supposed to keep watching + * for the life of a session, so they have to be changeable while a session is open. + */ +interface Knobs { + allow: (request: PeerRelayAuthorization.Request) => boolean + /** + * The separate acknowledgement that relaying a document commits its recipient to an + * allocation-unbounded decode. It denies by default in production, so it is its own knob. + */ + allowRisk: boolean + authority: { validUntil: number; invalidated: Effect.Effect } +} + +const harness = (options?: { + readonly limits?: PeerRelayLimits.Values + readonly knobs?: Partial +}) => + Effect.gen(function*() { + const knobs: Knobs = { + allow: options?.knobs?.allow ?? (() => true), + allowRisk: options?.knobs?.allowRisk ?? true, + authority: options?.knobs?.authority ?? { + validUntil: Number.MAX_SAFE_INTEGER, + invalidated: Effect.never + } + } + const limits = options?.limits ?? PeerRelayLimits.defaults + const crypto = yield* Crypto.Crypto.pipe(Effect.provide(NodeCrypto.layer)) + + const authorization = yield* PeerRelayAuthorization.PeerRelayAuthorization.pipe( + Effect.provide(PeerRelayAuthorization.layer( + (request) => + knobs.allow(request) + ? Effect.succeed({ + remote: { + tenantId: request.principal.tenantId, + subjectId: request.remote.subjectId, + peerId: request.remote.peerId + }, + documents: request.documents.map((requested) => ({ + document: Task, + documentId: requested.documentId + })), + validUntil: Number.MAX_SAFE_INTEGER, + invalidated: Effect.never + }) + : Effect.fail(new PeerRpcError.AccessDenied()), + (request) => + knobs.allowRisk && knobs.allow({ + direction: request.direction, + principal: request.principal, + remote: request.remote, + documents: request.documents + }) + ? Effect.succeed({ + _tag: "UnsafeUnboundedAutomerge3DecodeGrant" as const, + risk: PeerRelayAuthorization.unsafeUnboundedAutomerge3DecodeRisk, + principal: request.principal, + remote: { + tenantId: request.principal.tenantId, + subjectId: request.remote.subjectId, + peerId: request.remote.peerId + }, + direction: request.direction, + documents: request.documents, + validUntil: Number.MAX_SAFE_INTEGER, + invalidated: Effect.never + }) + : Effect.fail(new PeerRpcError.AccessDenied()) + )) + ) + + // The real durable store behind the real entity, so the front door is exercised against the + // custody it actually forwards to rather than a stand-in that always says yes. + const cluster = RelayInbox.layer(inboxOptions).pipe( + Layer.provideMerge(Sharding.layer), + Layer.provide(Runners.layerNoop), + Layer.provideMerge(MessageStorage.layerMemory), + Layer.provide(RunnerStorage.layerMemory), + Layer.provide(RunnerHealth.layerNoop), + Layer.provide(TestShardingConfig), + Layer.provideMerge( + SqlRelayInboxStore.layer.pipe( + Layer.provide(SqliteClient.layer({ filename: ":memory:", disableWAL: true })), + Layer.provide(Layer.succeed(Crypto.Crypto)(crypto)), + Layer.orDie + ) + ) + ) + + const context = yield* Layer.build( + RelayServer.layerHandlers(serverOptions).pipe( + Layer.provideMerge(cluster), + Layer.provide(Layer.mergeAll( + Layer.succeed(Crypto.Crypto)(crypto), + Layer.succeed(PeerRelayLimits.PeerRelayLimits)(limits), + Layer.succeed(PeerRelayAuthorization.PeerRelayAuthorization)(authorization) + )) + ) + ) + + const credential = yield* Ref.make("sender") + const principals = new Map([["sender", sender], ["recipient", recipient]]) + const authentication = yield* PeerAuthentication.PeerAuthentication.pipe( + Effect.provide(PeerAuthentication.layerServer), + Effect.provideService(PeerAuthenticator.PeerAuthenticator, { + authenticate: (secret) => { + const principal = principals.get(Redacted.value(secret)) + return principal === undefined + ? Effect.fail(new PeerRpcError.AuthenticationFailure()) + : Effect.succeed({ + principal, + validUntil: knobs.authority.validUntil, + invalidated: knobs.authority.invalidated + }) + } + }), + Effect.provideService(PeerRelayLimits.PeerRelayLimits, limits) + ) + + const client = yield* RpcTest.makeClient(PeerRpc.Rpcs).pipe( + Effect.provideContext( + Context.add(context, PeerAuthentication.PeerAuthentication, authentication) + ), + Effect.provide(PeerAuthentication.layerClient), + Effect.provideService(PeerCredentials.PeerCredentials, { + get: Ref.get(credential).pipe(Effect.map(Redacted.make)) + }) + ) + + // Shard assignment and acquisition run on scheduled fibers, so no entity is reachable under + // virtual time until the clock is advanced past them. + yield* TestClock.adjust(5000) + + const store = Context.get(context, RelayInboxStore.RelayInboxStore) + return { client, credential, crypto, knobs, store } + }) + +type Harness = Effect.Success> + +/** Opens a session and drains its events into a queue, leaving the stream live. */ +const open = ( + peer: Harness, + principal: PeerAuthentication.PeerPrincipal, + remote: PeerAuthentication.PeerPrincipal, + documents?: ReadonlyArray +) => + Effect.gen(function*() { + yield* Ref.set(peer.credential, principal.subjectId) + const events = yield* Queue.unbounded() + const fiber = yield* peer.client.Open(openRequest(principal, remote, documents)).pipe( + Stream.runForEach((event) => Queue.offer(events, event)), + Effect.forkScoped + ) + const opened = yield* Queue.take(events) + assert.strictEqual(opened._tag, "Opened") + if (opened._tag !== "Opened") return yield* Effect.die("expected Opened") + return { opened, events, fiber } + }) + +const encodePayload = (peer: Harness, options?: { + readonly message?: Uint8Array + readonly hash?: string + readonly documentId?: Identity.DocumentId + readonly sequence?: number +}) => + Effect.gen(function*() { + const message = options?.message ?? Uint8Array.of(1, 2, 3) + const messageHash = options?.hash ?? (yield* Canonical.digest(message).pipe( + Effect.provideService(Crypto.Crypto, peer.crypto) + )) + return new TextEncoder().encode( + yield* Schema.encodeEffect(RelaySyncEnvelopeJson)({ + connectionEpoch: "epoch", + sequence: options?.sequence ?? 0, + documentId: options?.documentId ?? documentId, + documentType: Task.name, + messageHash, + message, + lineage: Identity.genesisLineage, + writerProvenance: [] + }) + ) + }) + +/** The inbox a device's messages land in, derived exactly as the front door derives it. */ +const inboxKeyOf = (peer: Harness, principal: PeerAuthentication.PeerPrincipal) => + encodeInboxKey(principal).pipe( + Effect.provideService(Crypto.Crypto, peer.crypto), + Effect.orDie + ) + +const push = (peer: Harness, sessionId: Identity.SessionId, payload: Uint8Array, id = "000000000001") => + Ref.set(peer.credential, "sender").pipe( + Effect.andThen(peer.client.Push({ sessionId, relayMessageId: relayId(id), payload })) + ) + +describe("RelayServer", () => { + it.effect("refuses an unsupported protocol version and opens no session", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + const failure = yield* peer.client.Open({ + ...openRequest(sender, recipient), + protocolVersion: 2 + }).pipe(Stream.runDrain, Effect.flip) + assert.strictEqual(failure._tag, "UnsupportedVersion") + }))) + + it.effect("answers Open with a distinct session and the logical counterparty", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + const first = yield* open(peer, sender, recipient) + + assert.strictEqual(first.opened.protocolVersion, PeerRpc.protocolVersion) + assert.deepStrictEqual(first.opened.authenticatedLocal, sender) + // The counterparty the client asked for, not the relay it is talking to. Reporting the relay + // here would make the client address its own peer as the relay. + assert.strictEqual(first.opened.remotePeerId, recipientPeerId) + assert.notStrictEqual(first.opened.remotePeerId, relayPeerId) + + const second = yield* open(peer, recipient, sender) + assert.notStrictEqual(first.opened.sessionId, second.opened.sessionId) + }))) + + it.effect("refuses a handshake that names another relay or another local peer", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + + const wrongRelay = yield* peer.client.Open({ + ...openRequest(sender, recipient), + expectedRelayPeerId: recipientPeerId + }).pipe(Stream.runDrain, Effect.flip) + assert.strictEqual(wrongRelay._tag, "PeerMismatch") + + // The client states who it believes it is; the credential decides. A mismatch must not be + // resolved in favour of the payload, or a credential could open a session for another device. + const wrongLocal = yield* peer.client.Open({ + ...openRequest(sender, recipient), + expectedLocal: recipient + }).pipe(Stream.runDrain, Effect.flip) + assert.strictEqual(wrongLocal._tag, "PeerMismatch") + }))) + + it.effect("delivers a pushed message byte identically to the addressed recipient", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + const senderSession = yield* open(peer, sender, recipient) + const recipientSession = yield* open(peer, recipient, sender) + + const payload = yield* encodePayload(peer) + yield* push(peer, senderSession.opened.sessionId, payload) + + const stored = yield* Queue.take(recipientSession.events) + assert.strictEqual(stored._tag, "StoredMessage") + if (stored._tag !== "StoredMessage") return + assert.deepStrictEqual(stored.payload, payload) + assert.strictEqual(stored.relayPeerId, relayPeerId) + assert.strictEqual(stored.sender.peerId, senderPeerId) + assert.deepStrictEqual(stored.recipient, recipient) + + // Computed here independently and compared, rather than merely checked for shape. The digest + // binds sender, recipient, relay, sequence, epoch, document, lineage and provenance, and it + // is what the recipient verifies provenance against, so a digest over the wrong inputs is + // exactly as wrong as a malformed one while still matching a hex pattern. + const expected = yield* PeerSyncEnvelope.digestRelayOuterEnvelope({ + domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, + version: PeerSyncEnvelope.relayOuterEnvelopeVersion, + expectedLocal: sender, + remote: recipient, + relayPeerId, + relayMessageId: relayId("000000000001"), + protocolVersion: PeerRpc.protocolVersion, + payloadVersion: 1, + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + senderConnectionEpoch: "epoch", + senderSequence: 0, + document: { documentId, documentType: Task.name }, + lineage: Identity.genesisLineage, + writerProvenance: [], + messageHash: stored.messageHash, + payload + }).pipe(Effect.provideService(Crypto.Crypto, peer.crypto)) + assert.strictEqual(stored.outerEnvelopeDigest, expected) + }))) + + it.effect("retires an acknowledged message so a fresh session does not see it again", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + const senderSession = yield* open(peer, sender, recipient) + const recipientSession = yield* open(peer, recipient, sender) + + yield* push(peer, senderSession.opened.sessionId, yield* encodePayload(peer)) + const stored = yield* Queue.take(recipientSession.events) + if (stored._tag !== "StoredMessage") return assert.fail("expected a delivery") + + yield* Ref.set(peer.credential, "recipient") + yield* peer.client.Acknowledge({ + sessionId: recipientSession.opened.sessionId, + relayMessageId: stored.relayMessageId, + claimToken: stored.claimToken, + messageHash: stored.messageHash + }) + + const pending = yield* peer.store.pendingHeads( + yield* inboxKeyOf(peer, recipient), + { limit: 10 } + ) + assert.strictEqual(pending.length, 0, "an acknowledged message is terminal") + }))) + + it.effect("honours a rejection end to end", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + const senderSession = yield* open(peer, sender, recipient) + const recipientSession = yield* open(peer, recipient, sender) + + yield* push(peer, senderSession.opened.sessionId, yield* encodePayload(peer)) + const stored = yield* Queue.take(recipientSession.events) + if (stored._tag !== "StoredMessage") return assert.fail("expected a delivery") + + // A rejection is a durable decision too: the recipient looked at the message and refused it, + // so redelivering it would loop forever. + yield* Ref.set(peer.credential, "recipient") + yield* peer.client.Reject({ + sessionId: recipientSession.opened.sessionId, + relayMessageId: stored.relayMessageId, + claimToken: stored.claimToken, + messageHash: stored.messageHash, + reason: "ApplicationRejected" + }) + + const pending = yield* peer.store.pendingHeads( + yield* inboxKeyOf(peer, recipient), + { limit: 10 } + ) + assert.strictEqual(pending.length, 0, "a rejected message is terminal") + }))) + + it.effect("replaces the incumbent session and refuses its session id afterwards", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + const incumbent = yield* open(peer, sender, recipient) + const replacement = yield* open(peer, sender, recipient) + assert.notStrictEqual(incumbent.opened.sessionId, replacement.opened.sessionId) + + // Server initiated: the client never asked for the incumbent to end. + yield* Fiber.await(incumbent.fiber) + + const payload = yield* encodePayload(peer) + const stale = yield* push(peer, incumbent.opened.sessionId, payload).pipe(Effect.flip) + assert.strictEqual(stale._tag, "SessionUnavailable") + + const staleAck = yield* peer.client.Acknowledge({ + sessionId: incumbent.opened.sessionId, + relayMessageId: relayId("000000000001"), + claimToken: PeerRpc.ClaimToken.make("clm_00000000-0000-4000-8000-000000000000"), + messageHash: "a".repeat(64) + }).pipe(Effect.flip) + assert.strictEqual(staleAck._tag, "SessionUnavailable") + }))) + + it.effect("fails Open with an authentication failure when the credential has expired", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness({ knobs: { authority: { validUntil: 0, invalidated: Effect.never } } }) + const failure = yield* peer.client.Open(openRequest(sender, recipient)).pipe( + Stream.runDrain, + Effect.flip + ) + // Expiry at the handshake is an authentication problem, not an authorization one. + assert.strictEqual(failure._tag, "AuthenticationFailure") + }))) + + it.effect("refuses a push whose send grant has been withdrawn and admits nothing", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + const senderSession = yield* open(peer, sender, recipient) + + peer.knobs.allow = (request) => request.direction !== "Send" + const payload = yield* encodePayload(peer) + const denied = yield* push(peer, senderSession.opened.sessionId, payload).pipe(Effect.flip) + assert.strictEqual(denied._tag, "AccessDenied") + + const pending = yield* peer.store.pendingHeads( + yield* inboxKeyOf(peer, recipient), + { limit: 10 } + ) + assert.strictEqual(pending.length, 0, "a denied push must not reach durable custody") + }))) + + it.effect("commits an admitted push exactly once even if the grant is withdrawn afterwards", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + const senderSession = yield* open(peer, sender, recipient) + + const payload = yield* encodePayload(peer) + yield* push(peer, senderSession.opened.sessionId, payload) + // Revoked after admission. The message is already the relay's responsibility; withdrawing the + // grant must not retroactively destroy custody the sender was told the relay had taken. + peer.knobs.allow = () => false + + const inbox = yield* inboxKeyOf(peer, recipient) + const usage = yield* peer.store.usage(inbox) + assert.strictEqual(usage.pendingCount, 1, "the admitted message survives the revocation") + }))) + + it.effect("withholds a delivery the recipient may no longer receive without erroring the stream", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + const senderSession = yield* open(peer, sender, recipient) + const recipientSession = yield* open(peer, recipient, sender) + + // Anything holding a Send grant to this device can write into its inbox, and the entity knows + // nothing of grants, so the handshake's Receive decision cannot stand in for the recipient's + // right to see this particular document. + peer.knobs.allow = (request) => request.direction !== "Receive" + yield* push(peer, senderSession.opened.sessionId, yield* encodePayload(peer)) + yield* TestClock.adjust(100) + + assert.strictEqual(yield* Queue.size(recipientSession.events), 0, "withheld, not delivered") + assert.strictEqual( + recipientSession.fiber.pollUnsafe(), + undefined, + "and the stream neither ends nor errors" + ) + + const pending = yield* peer.store.pendingHeads( + yield* inboxKeyOf(peer, recipient), + { limit: 10 } + ) + assert.strictEqual(pending.length, 1, "the message stays durable for a later session") + + // Withheld, not lost. Restoring the grant and reconnecting must produce the message, which is + // what distinguishes withholding from silently dropping it. + peer.knobs.allow = () => true + const restored = yield* open(peer, recipient, sender) + const delivered = yield* Queue.take(restored.events) + assert.strictEqual(delivered._tag, "StoredMessage") + }))) + + it.effect("refuses a push whose payload does not decode or does not match its own hash", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + const senderSession = yield* open(peer, sender, recipient) + const session = senderSession.opened.sessionId + + const undecodable = yield* push(peer, session, new TextEncoder().encode("{")).pipe(Effect.flip) + assert.strictEqual(undecodable._tag, "InvalidRequest") + + // The hash is the identity the recipient deduplicates on, so the relay recomputes it rather + // than believing the sender's label. + const mislabelled = yield* encodePayload(peer, { hash: "a".repeat(64) }) + const wrongHash = yield* push(peer, session, mislabelled, "000000000002").pipe(Effect.flip) + assert.strictEqual(wrongHash._tag, "InvalidRequest") + + const undeclared = yield* encodePayload(peer, { documentId: otherDocumentId }) + const notInSession = yield* push(peer, session, undeclared, "000000000003").pipe(Effect.flip) + assert.strictEqual(notInSession._tag, "InvalidRequest") + + const pending = yield* peer.store.pendingHeads( + yield* inboxKeyOf(peer, recipient), + { limit: 10 } + ) + assert.strictEqual(pending.length, 0, "no malformed push reaches durable custody") + }))) + + it.effect("keys the delivery stream by the authenticated caller", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + const senderSession = yield* open(peer, sender, recipient) + const recipientSession = yield* open(peer, recipient, sender) + + yield* push(peer, senderSession.opened.sessionId, yield* encodePayload(peer)) + + // Waits for the delivery rather than nudging the clock, so the sender's stream is checked at + // a point where the message has provably already been routed somewhere. + const stored = yield* Queue.take(recipientSession.events) + assert.strictEqual(stored._tag, "StoredMessage") + + // The message is addressed to the recipient, so the sender's own stream must stay empty no + // matter what it put in any payload field: its inbox key comes from its credential. + assert.strictEqual(yield* Queue.size(senderSession.events), 0) + }))) + + it.effect("refuses an unauthenticated push and admits nothing", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + const senderSession = yield* open(peer, sender, recipient) + + const payload = yield* encodePayload(peer) + yield* Ref.set(peer.credential, "nobody") + const failure = yield* peer.client.Push({ + sessionId: senderSession.opened.sessionId, + relayMessageId: relayId("000000000001"), + payload + }).pipe(Effect.flip) + assert.strictEqual(failure._tag, "AuthenticationFailure") + + const pending = yield* peer.store.pendingHeads( + yield* inboxKeyOf(peer, recipient), + { limit: 10 } + ) + assert.strictEqual(pending.length, 0, "an unauthenticated push creates no inbox state") + }))) + + it.effect("refuses to open more sessions than a subject is allowed", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness({ + limits: PeerRelayLimits.Values.make({ + ...PeerRelayLimits.defaults, + maxSessionsPerSubject: 1 + }) + }) + yield* open(peer, sender, recipient) + + // Sessions are the front door's only unbounded resource and an authenticated client can open + // them in a loop, so the cap has to be refused rather than absorbed. + const failure = yield* peer.client.Open(openRequest(sender, recipient)).pipe( + Stream.runDrain, + Effect.flip + ) + assert.strictEqual(failure._tag, "RequestCapacityExceeded") + }))) + + it.effect("refuses a push when the unbounded decode risk is not acknowledged", () => + Effect.scoped(Effect.gen(function*() { + // Ordinary Send authorization is granted; only the separate risk acknowledgement is withheld. + // Relaying commits the recipient to an allocation-unbounded decode, so it is gated on its own + // port that denies by default, and an ordinary grant must not be able to stand in for it. + const peer = yield* harness({ knobs: { allowRisk: false } }) + const senderSession = yield* open(peer, sender, recipient) + + const payload = yield* encodePayload(peer) + const denied = yield* push(peer, senderSession.opened.sessionId, payload).pipe(Effect.flip) + assert.strictEqual(denied._tag, "AccessDenied") + + const pending = yield* peer.store.pendingHeads( + yield* inboxKeyOf(peer, recipient), + { limit: 10 } + ) + assert.strictEqual(pending.length, 0, "an unacknowledged risk admits nothing") + }))) + + it.effect("refuses a settlement once the recipient's receive authority is withdrawn", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + const senderSession = yield* open(peer, sender, recipient) + const recipientSession = yield* open(peer, recipient, sender) + + yield* push(peer, senderSession.opened.sessionId, yield* encodePayload(peer)) + const stored = yield* Queue.take(recipientSession.events) + if (stored._tag !== "StoredMessage") return assert.fail("expected a delivery") + + // Settling is a durable mutation. A peer whose Receive authority has been withdrawn must not + // be able to perform one, and the refusal has to say so rather than blaming the session: the + // session is perfectly live, it is the authority behind it that is gone. + peer.knobs.allow = (request) => request.direction !== "Receive" + yield* Ref.set(peer.credential, "recipient") + const denied = yield* peer.client.Acknowledge({ + sessionId: recipientSession.opened.sessionId, + relayMessageId: stored.relayMessageId, + claimToken: stored.claimToken, + messageHash: stored.messageHash + }).pipe(Effect.flip) + assert.strictEqual(denied._tag, "AccessDenied") + + const pending = yield* peer.store.pendingHeads( + yield* inboxKeyOf(peer, recipient), + { limit: 10 } + ) + assert.strictEqual(pending.length, 1, "the terminal transition never ran") + }))) + + it.effect("pins the handshake protocol version so a foreign version fails to decode", () => + Effect.gen(function*() { + // `Opened` carries a literal, not a number, so a client cannot be talked into continuing + // against a relay speaking a version it does not implement. + assert.throws(() => + Schema.decodeUnknownSync(PeerRpc.OpenEvent)({ + _tag: "Opened", + protocolVersion: PeerRpc.protocolVersion + 1, + sessionId: "ses_00000000-0000-4000-8000-000000000001", + remotePeerId: recipientPeerId, + authenticatedLocal: sender + }) + ) + })) + + it.effect("refuses a handshake whose retention window cannot cover its retry horizon", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + // A receipt has to outlive both the message and the window in which its sender may replay it, + // or a redelivery lands after the deduplication record is gone and is applied twice. + const failure = yield* peer.client.Open({ + ...openRequest(sender, recipient), + receiptRetentionMillis: 1_000 + }).pipe(Stream.runDrain, Effect.flip) + assert.strictEqual(failure._tag, "InvalidRequest") + }))) +}) diff --git a/packages/local-rpc/test/RelayWireFormat.test.ts b/packages/local-rpc/test/RelayWireFormat.test.ts index 136d9b4..c4ecd5a 100644 --- a/packages/local-rpc/test/RelayWireFormat.test.ts +++ b/packages/local-rpc/test/RelayWireFormat.test.ts @@ -131,7 +131,21 @@ describe("relay wire format", () => { validUntil: Number.MAX_SAFE_INTEGER, invalidated: Effect.never }), - PeerRelayAuthorization.denyUnsafeUnboundedAutomerge3Decode + (request) => + Effect.succeed({ + _tag: "UnsafeUnboundedAutomerge3DecodeGrant" as const, + risk: PeerRelayAuthorization.unsafeUnboundedAutomerge3DecodeRisk, + principal: request.principal, + remote: { + tenantId: request.principal.tenantId, + subjectId: request.remote.subjectId, + peerId: request.remote.peerId + }, + direction: request.direction, + documents: request.documents, + validUntil: Number.MAX_SAFE_INTEGER, + invalidated: Effect.never + }) )) ) From 1750464595125069707eb87d7921b0b7a8966314 Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Mon, 27 Jul 2026 01:54:45 -0500 Subject: [PATCH 19/29] Cover the remaining front-door contract guarantees A broken authorization backend must not be reported as AccessDenied: that tells a legitimate client it is unauthorized and makes it stop retrying, when the truth is the relay cannot answer. The wire contract encodes a defect as InternalError to keep the two distinct, so the test asserts the cause carries a defect and no typed failure at all. Credential invalidation ends the session by closing the stream rather than erroring it. Nothing but the relay observes the credential, so it is the only thing that can end a session when authority is withdrawn, and an orderly close lets the client reconnect on its own terms. The delivered-but-unsettled message stays durable across the revocation. --- packages/local-rpc/test/RelayServer.test.ts | 57 +++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/packages/local-rpc/test/RelayServer.test.ts b/packages/local-rpc/test/RelayServer.test.ts index 7d6b6e2..b8427f0 100644 --- a/packages/local-rpc/test/RelayServer.test.ts +++ b/packages/local-rpc/test/RelayServer.test.ts @@ -5,10 +5,12 @@ import * as PeerSyncEnvelope from "@lucas-barake/effect-local-sql/PeerSyncEnvelo import * as Canonical from "@lucas-barake/effect-local/Canonical" import * as Document from "@lucas-barake/effect-local/Document" import * as Identity from "@lucas-barake/effect-local/Identity" +import * as Cause from "effect/Cause" import * as Context from "effect/Context" import * as Crypto from "effect/Crypto" import * as Effect from "effect/Effect" import * as Fiber from "effect/Fiber" +import * as Latch from "effect/Latch" import * as Layer from "effect/Layer" import * as Queue from "effect/Queue" import * as Redacted from "effect/Redacted" @@ -681,6 +683,61 @@ describe("RelayServer", () => { assert.strictEqual(pending.length, 1, "the terminal transition never ran") }))) + it.effect("surfaces an internal authorization fault as a defect rather than a typed error", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness({ + knobs: { + allow: () => { + throw new Error("authorization backend is broken") + } + } + }) + + // A broken authorization backend is not a decision about this caller. Reporting it as + // AccessDenied would tell a legitimate client it is unauthorized and make it stop retrying, + // and the wire contract encodes a defect as InternalError precisely to keep them distinct. + const exit = yield* peer.client.Open(openRequest(sender, recipient)).pipe( + Stream.runDrain, + Effect.exit + ) + assert.isTrue(exit._tag === "Failure") + if (exit._tag !== "Failure") return + assert.isTrue(Cause.hasDies(exit.cause), "the fault stays a defect") + assert.isFalse( + Cause.hasFails(exit.cause), + "and is never laundered into a typed PeerRpcError" + ) + }))) + + it.effect("ends a session cleanly when its credential is invalidated", () => + Effect.scoped(Effect.gen(function*() { + const revoked = yield* Latch.make(false) + const peer = yield* harness({ + knobs: { + authority: { validUntil: Number.MAX_SAFE_INTEGER, invalidated: revoked.await } + } + }) + const senderSession = yield* open(peer, sender, recipient) + const recipientSession = yield* open(peer, recipient, sender) + + yield* push(peer, senderSession.opened.sessionId, yield* encodePayload(peer)) + const stored = yield* Queue.take(recipientSession.events) + assert.strictEqual(stored._tag, "StoredMessage") + + // Nothing observes the credential except the relay, so it is the only thing that can end the + // session when authority is withdrawn. The stream must end, not error: revocation is an + // orderly close, and the client reconnects on its own terms. + yield* revoked.open + yield* Fiber.join(recipientSession.fiber) + + // The message was delivered but never settled, so it is still the relay's responsibility. + const pending = yield* peer.store.pendingHeads( + yield* inboxKeyOf(peer, recipient), + { limit: 10 } + ) + assert.strictEqual(pending.length, 1, "an unsettled message survives the revocation") + }))) + it.effect("pins the handshake protocol version so a foreign version fails to decode", () => Effect.gen(function*() { // `Opened` carries a literal, not a number, so a client cannot be talked into continuing From 6cd97d529b93004c0b2d5d63379c365a8801e875 Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Mon, 27 Jul 2026 02:03:36 -0500 Subject: [PATCH 20/29] Delete the single-process relay and cover the store on every dialect Removes PeerRpcServer, PeerRelayIngress, PeerRelayStore, SqlPeerRelayStore and their migration and transaction helpers, along with the tests and bench that exercised them. The packages are unpublished, so the cluster relay replaces them outright rather than living alongside. All 25 client-visible contract guarantees the deleted tests encoded were audited first. Twenty are re-expressed against the new stack; five are waived because their subject no longer exists - process-local runtime shutdown is now cluster rebalance and passivation, there is no claim or payload-load split to race, and the front door has no admission queue to drain across a revocation, so exactly-once there is structural rather than a drain guarantee. The store contract moves into its own module and now runs against SQLite, PostgreSQL and MySQL. Multi-dialect parity was a decision, and a single dialect cannot establish this contract: PostgreSQL returns BIGINT, COUNT and SUM as strings, which made the store non-functional there while SQLite stayed green, and MySQL compares identity columns case-insensitively without a binary collation. Both were real defects in this store. Each check owns a distinct inbox key and none asserts a sweep's returned count, because expire and collect are deployment-wide and the dialect containers are shared across the block. Asserting the effect on the check's own inbox is the real guarantee anyway. --- .../local-rpc/bench/PeerRelayStore.bench.ts | 194 -- packages/local-rpc/src/PeerRelayIngress.ts | 870 ------ packages/local-rpc/src/PeerRelayStore.ts | 173 -- packages/local-rpc/src/PeerRpcServer.ts | 1756 ----------- packages/local-rpc/src/SqlPeerRelayStore.ts | 2143 -------------- packages/local-rpc/src/index.ts | 4 - .../src/internal/peerRelayMigrations.ts | 872 ------ .../src/internal/peerRelaySqlTransaction.ts | 238 -- .../local-rpc/test/PeerRelayIngress.test.ts | 737 ----- .../test/PeerRelaySqliteTransaction.test.ts | 278 -- .../local-rpc/test/PeerRelayStore.test.ts | 1189 -------- .../local-rpc/test/PeerRelayStoreContract.ts | 239 -- .../local-rpc/test/PeerRpcIntegration.test.ts | 296 -- packages/local-rpc/test/PeerRpcServer.test.ts | 2564 ----------------- .../local-rpc/test/PublicApi.types.test.ts | 23 +- .../local-rpc/test/RelayInboxStore.test.ts | 479 +-- .../local-rpc/test/RelayInboxStoreContract.ts | 462 +++ .../local-rpc/test/SqlPeerRelayStore.test.ts | 255 -- 18 files changed, 573 insertions(+), 12199 deletions(-) delete mode 100644 packages/local-rpc/bench/PeerRelayStore.bench.ts delete mode 100644 packages/local-rpc/src/PeerRelayIngress.ts delete mode 100644 packages/local-rpc/src/PeerRelayStore.ts delete mode 100644 packages/local-rpc/src/PeerRpcServer.ts delete mode 100644 packages/local-rpc/src/SqlPeerRelayStore.ts delete mode 100644 packages/local-rpc/src/internal/peerRelayMigrations.ts delete mode 100644 packages/local-rpc/src/internal/peerRelaySqlTransaction.ts delete mode 100644 packages/local-rpc/test/PeerRelayIngress.test.ts delete mode 100644 packages/local-rpc/test/PeerRelaySqliteTransaction.test.ts delete mode 100644 packages/local-rpc/test/PeerRelayStore.test.ts delete mode 100644 packages/local-rpc/test/PeerRelayStoreContract.ts delete mode 100644 packages/local-rpc/test/PeerRpcIntegration.test.ts delete mode 100644 packages/local-rpc/test/PeerRpcServer.test.ts create mode 100644 packages/local-rpc/test/RelayInboxStoreContract.ts delete mode 100644 packages/local-rpc/test/SqlPeerRelayStore.test.ts diff --git a/packages/local-rpc/bench/PeerRelayStore.bench.ts b/packages/local-rpc/bench/PeerRelayStore.bench.ts deleted file mode 100644 index c7cfc60..0000000 --- a/packages/local-rpc/bench/PeerRelayStore.bench.ts +++ /dev/null @@ -1,194 +0,0 @@ -import { NodeCrypto } from "@effect/platform-node" -import { SqliteClient } from "@effect/sql-sqlite-node" -import * as Identity from "@lucas-barake/effect-local/Identity" -import * as Context from "effect/Context" -import * as Effect from "effect/Effect" -import * as Exit from "effect/Exit" -import * as Layer from "effect/Layer" -import * as Option from "effect/Option" -import * as Scope from "effect/Scope" -import { mkdtempSync, rmSync } from "node:fs" -import { tmpdir } from "node:os" -import { join } from "node:path" -import { afterAll, bench } from "vitest" -import * as PeerRelayLimits from "../src/PeerRelayLimits.js" -import * as PeerRelayStore from "../src/PeerRelayStore.js" -import * as PeerRpc from "../src/PeerRpc.js" -import * as SqlPeerRelayStore from "../src/SqlPeerRelayStore.js" - -const backlogSizes = [128, 1_024] as const -const drainBatchSize = 8 -const payloadBytes = 1_024 -const benchmarkOptions = { - iterations: 8, - time: 0, - warmupIterations: 2, - warmupTime: 0 -} as const - -const suffix = (value: number) => String(value).padStart(12, "0") -const peerId = (value: number) => Identity.PeerId.make(`peer_00000000-0000-4000-8000-${suffix(value)}`) -const relayMessageId = (value: number) => Identity.RelayMessageId.make(`rly_00000000-0000-4000-8000-${suffix(value)}`) -const documentId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000001") -const relayPeerId = peerId(1) -const recipientPeerId = peerId(2) -const payload = Uint8Array.from({ length: payloadBytes }, (_, index) => index % 251) - -let nextMessage = 1 -let nextSenderPeer = 10 - -interface WorkItem { - readonly admission: PeerRelayStore.Admission - readonly claim: PeerRelayStore.ClaimRequest -} - -const makeWorkItem = ( - workload: string, - index: number, - distribution: "Hot" | "Distributed" -): WorkItem => { - const messageNumber = nextMessage++ - const senderPeerId = distribution === "Hot" ? peerId(3) : peerId(nextSenderPeer++) - const channel = PeerRelayStore.ChannelKey.make({ - tenantId: `benchmark-${workload}`, - senderSubjectId: "sender", - senderPeerId, - senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), - recipientSubjectId: "recipient", - recipientPeerId - }) - return { - admission: PeerRelayStore.Admission.make({ - channel, - relayMessageId: relayMessageId(messageNumber), - relayPeerId, - documentIds: [documentId], - senderConnectionEpoch: `epoch-${workload}`, - senderSequence: distribution === "Hot" ? index : 0, - payloadVersion: 1, - messageHash: `message-${messageNumber}`, - outerEnvelopeDigest: PeerRpc.RelayDigest.make(messageNumber.toString(16).padStart(64, "0")), - payload, - messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, - senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, - minimumTerminalRetentionMillis: PeerRelayLimits.defaults.minimumTerminalRetentionMillis - }), - claim: PeerRelayStore.ClaimRequest.make({ - recipient: { - tenantId: channel.tenantId, - subjectId: channel.recipientSubjectId, - peerId: channel.recipientPeerId - }, - sender: { - subjectId: channel.senderSubjectId, - peerId: channel.senderPeerId - }, - sessionGeneration: 1, - authorizedDocumentIds: [documentId] - }) - } -} - -const temporaryDirectory = mkdtempSync(join(tmpdir(), "effect-local-peer-relay-bench-")) -const filename = join(temporaryDirectory, "relay.sqlite") -const scope = await Effect.runPromise(Scope.make()) -const baseLayer = Layer.mergeAll( - SqliteClient.layer({ filename }), - NodeCrypto.layer, - PeerRelayLimits.layerDefaults -) -const benchmarkContext = await Effect.runPromise( - Layer.build( - Layer.merge( - baseLayer, - SqlPeerRelayStore.layer.pipe(Layer.provide(baseLayer)) - ) - ).pipe(Effect.provideService(Scope.Scope, scope)) -) -const store = Context.get(benchmarkContext, PeerRelayStore.PeerRelayStore) - -const workloads = new Map>() - -for (const backlogSize of backlogSizes) { - for (const distribution of ["Hot", "Distributed"] as const) { - const workload = `${distribution.toLowerCase()}-${backlogSize}` - const items = Array.from( - { length: backlogSize }, - (_, index) => makeWorkItem(workload, index, distribution) - ) - workloads.set(workload, items) - } -} - -await Effect.runPromise( - Effect.forEach( - Array.from(workloads.values()).flat(), - ({ admission }) => store.admit(admission), - { concurrency: 1, discard: true } - ) -) - -const drain = ( - items: ReadonlyArray, - cursor: { value: number } -) => - Effect.gen(function*() { - for (let offset = 0; offset < drainBatchSize; offset++) { - const item = items[cursor.value++] - if (item === undefined) { - return yield* Effect.die(new Error("PeerRelayStore benchmark exhausted its prepared backlog")) - } - const claimed = yield* store.claim(item.claim) - if (Option.isNone(claimed.message)) { - return yield* Effect.die(new Error("PeerRelayStore benchmark could not claim prepared work")) - } - const message = claimed.message.value - const acknowledged = yield* store.acknowledge({ - channel: message.channel, - relayMessageId: message.relayMessageId, - claimToken: message.claimToken, - messageHash: message.messageHash, - sessionGeneration: message.sessionGeneration, - recipient: item.claim.recipient - }) - if (acknowledged.status !== "Changed") { - return yield* Effect.die(new Error("PeerRelayStore benchmark did not acknowledge its active claim")) - } - } - }) - -for (const backlogSize of backlogSizes) { - for (const distribution of ["Hot", "Distributed"] as const) { - const workload = `${distribution.toLowerCase()}-${backlogSize}` - const items = workloads.get(workload)! - const cursor = { value: 0 } - - bench( - `SQLite drain batch=${drainBatchSize}, backlog=${backlogSize}, channels=${distribution.toLowerCase()}, payload=${payloadBytes} bytes`, - async () => { - await Effect.runPromise(drain(items, cursor)) - }, - benchmarkOptions - ) - } -} - -for (const distribution of ["Hot", "Distributed"] as const) { - let sequence = 0 - const workload = `admission-${distribution.toLowerCase()}` - - bench( - `SQLite admission, channels=${distribution.toLowerCase()}, payload=${payloadBytes} bytes`, - async () => { - await Effect.runPromise( - store.admit(makeWorkItem(workload, sequence++, distribution).admission) - ) - }, - benchmarkOptions - ) -} - -afterAll(async () => { - await Effect.runPromise(Scope.close(scope, Exit.void)) - rmSync(temporaryDirectory, { recursive: true, force: true }) -}) diff --git a/packages/local-rpc/src/PeerRelayIngress.ts b/packages/local-rpc/src/PeerRelayIngress.ts deleted file mode 100644 index 5f1b978..0000000 --- a/packages/local-rpc/src/PeerRelayIngress.ts +++ /dev/null @@ -1,870 +0,0 @@ -import { Cause, Context, Deferred, Effect, Fiber, Layer, Queue, Scope } from "effect" -import * as RpcClient from "effect/unstable/rpc/RpcClient" -import * as RpcClientError from "effect/unstable/rpc/RpcClientError" -import type * as RpcMessage from "effect/unstable/rpc/RpcMessage" -import * as RpcServer from "effect/unstable/rpc/RpcServer" -import { Socket, SocketServer } from "effect/unstable/socket" -import { PeerRelayLimits } from "./PeerRelayLimits.ts" -import * as PeerRpcError from "./PeerRpcError.ts" - -export interface Usage { - readonly connections: number - readonly reservedBytes: number - readonly byteReservationWaiters: number -} - -export interface Reservation { - readonly bytes: number - readonly release: Effect.Effect - readonly transferToCurrentRequest: Effect.Effect -} - -export class PeerRelayIngress extends Context.Service< - PeerRelayIngress, - { - readonly address: SocketServer.Address - readonly reserveOutbound: ( - bytes: number - ) => Effect.Effect< - Reservation, - PeerRpcError.RequestLimitExceeded | PeerRpcError.RequestCapacityExceeded - > - readonly usage: Effect.Effect - readonly await: Effect.Effect - } ->()("@lucas-barake/effect-local-rpc/PeerRelayIngress") {} - -interface RequestKey { - readonly clientId: number - readonly requestId: string | number -} - -const CurrentRequestKey = Context.Reference( - "@lucas-barake/effect-local-rpc/PeerRelayIngress/CurrentRequestKey", - { defaultValue: () => undefined } -) - -interface InternalReservation extends Reservation { - readonly releaseUnsafe: () => void - readonly shrinkTo: (bytes: number) => Effect.Effect - readonly transfer: (key: RequestKey) => Effect.Effect -} - -interface ByteWaiter { - readonly bytes: number - readonly deferred: Deferred.Deferred - state: "Waiting" | "Reserved" | "Delivered" | "Cancelled" -} - -const makeByteBudget = ( - capacity: number, - maximumWaiters: number, - outbound: Map>, - isClientActive: (clientId: number) => boolean = () => true -) => { - let reservedBytes = 0 - let waiterId = 0 - const waiters = new Map() - - const makeReservation = (bytes: number): InternalReservation => { - let reserved = bytes - let released = false - let transferred = false - const releaseUnsafe = () => { - if (released) return - released = true - reservedBytes -= reserved - drain() - } - const release = Effect.sync(releaseUnsafe) - const shrinkTo = (target: number) => - Effect.sync(() => { - if ( - released || - transferred || - !Number.isSafeInteger(target) || - target <= 0 || - target > reserved - ) { - throw new Error("Invalid relay byte reservation shrink") - } - if (target === reserved) return - const releasedBytes = reserved - target - reserved = target - reservedBytes -= releasedBytes - drain() - }) - const transfer = (key: RequestKey) => - Effect.suspend(() => { - if (released || transferred || !isClientActive(key.clientId)) { - return Effect.fail(new PeerRpcError.SessionUnavailable()) - } - let client = outbound.get(key.clientId) - if (!client) { - client = new Map() - outbound.set(key.clientId, client) - } - if (client.has(key.requestId)) { - return Effect.fail(new PeerRpcError.SessionUnavailable()) - } - client.set(key.requestId, reservation) - transferred = true - return Effect.void - }) - const reservation: InternalReservation = { - get bytes() { - return reserved - }, - release, - releaseUnsafe, - shrinkTo, - transfer, - transferToCurrentRequest: Effect.flatMap(CurrentRequestKey, (key) => - key === undefined - ? Effect.fail(new PeerRpcError.SessionUnavailable()) - : transfer(key)) - } - return reservation - } - - const drain = () => { - for (const [id, waiter] of waiters) { - if (waiter.state !== "Waiting") { - waiters.delete(id) - continue - } - if (reservedBytes + waiter.bytes > capacity) return - waiters.delete(id) - waiter.state = "Reserved" - reservedBytes += waiter.bytes - Deferred.doneUnsafe(waiter.deferred, Effect.succeed(makeReservation(waiter.bytes))) - } - } - - const cancelWaiter = (id: number, waiter: ByteWaiter) => - Effect.sync(() => { - if (waiter.state === "Waiting") { - waiter.state = "Cancelled" - waiters.delete(id) - drain() - } else if (waiter.state === "Reserved") { - waiter.state = "Cancelled" - reservedBytes -= waiter.bytes - drain() - } - }) - - const acquire = ( - bytes: number - ): Effect.Effect< - InternalReservation, - PeerRpcError.RequestLimitExceeded | PeerRpcError.RequestCapacityExceeded - > => - Effect.suspend< - InternalReservation, - PeerRpcError.RequestLimitExceeded | PeerRpcError.RequestCapacityExceeded, - never - >(() => { - if (!Number.isSafeInteger(bytes) || bytes <= 0 || bytes > capacity) { - return Effect.fail(new PeerRpcError.RequestLimitExceeded()) - } - if (waiters.size === 0 && reservedBytes + bytes <= capacity) { - reservedBytes += bytes - return Effect.succeed(makeReservation(bytes)) - } - if (waiters.size >= maximumWaiters) { - return Effect.fail(new PeerRpcError.RequestCapacityExceeded()) - } - const id = waiterId++ - const waiter: ByteWaiter = { - bytes, - deferred: Deferred.makeUnsafe(), - state: "Waiting" - } - waiters.set(id, waiter) - return Effect.interruptible(Deferred.await(waiter.deferred)).pipe( - Effect.onInterrupt(() => cancelWaiter(id, waiter)), - Effect.tap(() => - Effect.sync(() => { - waiter.state = "Delivered" - }) - ) - ) - }) - - const reserve = ( - bytes: number - ): Effect.Effect< - InternalReservation, - PeerRpcError.RequestLimitExceeded | PeerRpcError.RequestCapacityExceeded - > => - Effect.suspend(() => { - let acquired: InternalReservation | undefined - return Effect.uninterruptibleMask(() => - acquire(bytes).pipe( - Effect.tap((reservation) => - Effect.sync(() => { - acquired = reservation - }) - ) - ) - ).pipe( - Effect.onInterrupt(() => acquired?.release ?? Effect.void) - ) - }) - - const use = ( - bytes: number, - f: (reservation: InternalReservation) => Effect.Effect - ): Effect.Effect< - A, - E | PeerRpcError.RequestLimitExceeded | PeerRpcError.RequestCapacityExceeded, - R - > => - Effect.uninterruptibleMask((restore) => - acquire(bytes).pipe( - Effect.flatMap((reservation) => - restore(f(reservation)).pipe( - Effect.ensuring(reservation.release) - ) - ) - ) - ) - - return { - reserve, - use, - usage: () => ({ - reservedBytes, - byteReservationWaiters: waiters.size - }) - } -} - -const invalidFrame = () => - new Socket.SocketError({ - reason: new Socket.SocketCloseError({ code: 1009 }) - }) - -const utf8Length = (value: string): number => { - let bytes = 0 - for (let index = 0; index < value.length; index++) { - const code = value.charCodeAt(index) - if (code < 0x80) { - bytes++ - } else if (code < 0x800) { - bytes += 2 - } else if (code >= 0xd800 && code <= 0xdbff && index + 1 < value.length) { - const next = value.charCodeAt(index + 1) - if (next >= 0xdc00 && next <= 0xdfff) { - bytes += 4 - index++ - } else { - bytes += 3 - } - } else { - bytes += 3 - } - } - return bytes -} - -const encodeFrame = (value: unknown, maximumFrameBytes: number) => { - const body = new TextEncoder().encode(JSON.stringify(value)) - if (body.byteLength === 0 || body.byteLength > maximumFrameBytes) { - throw invalidFrame() - } - const frame = new Uint8Array(4 + body.byteLength) - new DataView(frame.buffer).setUint32(0, body.byteLength, false) - frame.set(body, 4) - return { bodyBytes: body.byteLength, frame } -} - -const makeFrameReader = ( - socket: Socket.Socket, - limits: { - readonly maximumRawChunkBytes: number - readonly maximumDeclaredFrameBytes: number - readonly maximumIncompleteFrameBytes: number - readonly incompleteFrameTimeoutMillis: number - }, - reserve: ( - bytes: number - ) => Effect.Effect< - InternalReservation, - PeerRpcError.RequestLimitExceeded | PeerRpcError.RequestCapacityExceeded - >, - onFrame: ( - value: unknown, - reservation: InternalReservation - ) => Effect.Effect -) => - Effect.gen(function*() { - const scope = yield* Effect.scope - const timeout = Deferred.makeUnsafe() - const header = new Uint8Array(4) - const decoder = new TextDecoder("utf-8", { fatal: true }) - let headerBytes = 0 - let body: Uint8Array | undefined - let bodyBytes = 0 - let reservation: InternalReservation | undefined - let timer: Fiber.Fiber | undefined - let chunkActive = false - let chunkRejected = false - - const stopTimer = Effect.suspend(() => { - if (timer === undefined) return Effect.void - const current = timer - timer = undefined - return Fiber.interrupt(current) - }) - - const startTimer = Effect.suspend(() => { - if (timer !== undefined) return Effect.void - return Effect.forkIn( - Effect.sleep(limits.incompleteFrameTimeoutMillis).pipe( - Effect.andThen(Effect.sync(() => { - Deferred.doneUnsafe(timeout, Effect.fail(invalidFrame())) - })) - ), - scope - ).pipe( - Effect.tap((fiber) => - Effect.sync(() => { - timer = fiber - }) - ), - Effect.asVoid - ) - }) - - const resetFrame = () => { - headerBytes = 0 - body = undefined - bodyBytes = 0 - reservation = undefined - } - - const consume = (raw: string | Uint8Array) => - Effect.gen(function*() { - const rawBytes = typeof raw === "string" ? utf8Length(raw) : raw.byteLength - if (rawBytes === 0) return - if (rawBytes > limits.maximumRawChunkBytes) { - return yield* Effect.fail(invalidFrame()) - } - const chunk = typeof raw === "string" ? new TextEncoder().encode(raw) : raw - if (chunk.byteLength !== rawBytes) { - return yield* Effect.fail(invalidFrame()) - } - let offset = 0 - while (offset < chunk.byteLength) { - if (headerBytes === 0 && body === undefined) { - yield* startTimer - } - if (headerBytes < 4) { - const count = Math.min(4 - headerBytes, chunk.byteLength - offset) - header.set(chunk.subarray(offset, offset + count), headerBytes) - headerBytes += count - offset += count - if (headerBytes < 4) continue - - const declared = new DataView(header.buffer).getUint32(0, false) - if ( - declared === 0 || - declared > limits.maximumDeclaredFrameBytes || - declared + 4 > limits.maximumIncompleteFrameBytes - ) { - return yield* Effect.fail(invalidFrame()) - } - reservation = yield* reserve(declared).pipe( - Effect.mapError(() => invalidFrame()) - ) - body = new Uint8Array(declared) - } - - const count = Math.min(body!.byteLength - bodyBytes, chunk.byteLength - offset) - body!.set(chunk.subarray(offset, offset + count), bodyBytes) - bodyBytes += count - offset += count - if (bodyBytes < body!.byteLength) continue - - yield* stopTimer - const retained = reservation! - let value: unknown - try { - value = JSON.parse(decoder.decode(body!)) - } catch { - yield* retained.release - return yield* Effect.fail(invalidFrame()) - } - resetFrame() - yield* onFrame(value, retained).pipe( - Effect.onExit((exit) => exit._tag === "Failure" ? retained.release : Effect.void) - ) - } - }) - - return yield* socket.runRaw((chunk) => { - if (chunkRejected) return - if (chunkActive) { - chunkRejected = true - return Effect.fail(invalidFrame()) - } - chunkActive = true - return consume(chunk).pipe( - Effect.ensuring( - Effect.sync(() => { - chunkActive = false - }) - ) - ) - }).pipe( - Effect.raceFirst(Deferred.await(timeout)), - Effect.ensuring( - Effect.andThen(stopTimer, Effect.suspend(() => reservation === undefined ? Effect.void : reservation.release)) - ) - ) - }) - -const removeReservation = ( - reservations: Map>, - key: RequestKey -) => { - const client = reservations.get(key.clientId) - const reservation = client?.get(key.requestId) - if (reservation === undefined) return undefined - client!.delete(key.requestId) - if (client!.size === 0) reservations.delete(key.clientId) - return reservation -} - -const removeAndReleaseReservation = ( - reservations: Map>, - key: RequestKey -) => - Effect.sync(() => { - removeReservation(reservations, key)?.releaseUnsafe() - }) - -const releaseClientReservations = ( - reservations: Map>, - clientId: number -) => - Effect.sync(() => { - const client = reservations.get(clientId) - if (client === undefined) return - reservations.delete(clientId) - for (const reservation of client.values()) { - reservation.releaseUnsafe() - } - }) - -const makeServerProtocol = ( - server: SocketServer.SocketServer["Service"], - limits: Context.Service.Shape, - childScope: Scope.Closeable -) => - Effect.gen(function*() { - const disconnects = yield* Queue.bounded(limits.maxRelayConnections) - const clients = new Map Effect.Effect - }>() - const clientIds = new Set() - const inbound = new Map>() - const outbound = new Map>() - const budget = makeByteBudget( - limits.maximumSharedPayloadBytes, - limits.maximumByteReservationWaiters, - outbound, - (clientId) => clientIds.has(clientId) - ) - let connections = 0 - let nextClientId = 0 - let protocolRunners = 0 - let writeRequest!: ( - clientId: number, - message: RpcMessage.FromClientEncoded - ) => Effect.Effect - - const cleanupClient = (clientId: number) => - Effect.gen(function*() { - clients.delete(clientId) - const wasRegistered = clientIds.delete(clientId) - yield* releaseClientReservations(inbound, clientId) - yield* releaseClientReservations(outbound, clientId) - if (wasRegistered) { - yield* Queue.offer(disconnects, clientId) - } - }) - - const baseProtocol = yield* RpcServer.Protocol.make((writeRequest_) => { - writeRequest = writeRequest_ - return Effect.succeed({ - disconnects, - send: (clientId, response) => - Effect.uninterruptibleMask((restore) => - Effect.flatMap(CurrentRequestKey, (currentRequest) => - Effect.sync(() => { - const key = "requestId" in response - ? { clientId, requestId: response.requestId } - : response._tag === "Defect" - ? currentRequest - : undefined - const transferred = key === undefined - ? undefined - : removeReservation(outbound, key) - return { key, transferred } - })).pipe( - Effect.flatMap(({ key, transferred }) => - restore(Effect.gen(function*() { - const client = clients.get(clientId) - if (client === undefined) { - return - } - - const maximumAdditional = transferred === undefined - ? limits.maximumDeclaredFrameBytes - : Math.max(0, limits.maximumDeclaredFrameBytes - transferred.bytes) - const sendWithExtra = (extra: InternalReservation | undefined) => - Effect.gen(function*() { - let encoded: ReturnType - try { - encoded = encodeFrame(response, limits.maximumDeclaredFrameBytes) - } catch { - yield* client.write(new Socket.CloseEvent(1009)).pipe(Effect.ignore) - return - } - - const additional = transferred === undefined - ? encoded.bodyBytes - : Math.max(0, encoded.bodyBytes - transferred.bytes) - if (extra !== undefined && additional > 0) { - yield* extra.shrinkTo(additional) - } else if (extra !== undefined) { - yield* extra.release - } - - yield* client.write(encoded.frame).pipe(Effect.ignore) - - if ( - (response._tag === "Exit" || response._tag === "Defect") && - key !== undefined - ) { - yield* removeAndReleaseReservation(inbound, key) - } - }) - if (maximumAdditional === 0) { - yield* sendWithExtra(undefined) - } else { - yield* budget.use( - maximumAdditional, - sendWithExtra - ).pipe( - Effect.catch(() => client.write(new Socket.CloseEvent(1013)).pipe(Effect.ignore)) - ) - } - })).pipe(Effect.ensuring(transferred?.release ?? Effect.void)) - ) - ) - ), - end: (clientId) => - Effect.gen(function*() { - const client = clients.get(clientId) - if (client !== undefined) { - yield* client.write(new Socket.CloseEvent()).pipe(Effect.ignore) - } - yield* cleanupClient(clientId) - }), - clientIds: Effect.sync(() => clientIds), - initialMessage: Effect.succeedNone, - supportsAck: true, - supportsTransferables: false, - supportsSpanPropagation: true - }) - }) - const protocol: RpcServer.Protocol["Service"] = { - ...baseProtocol, - run: (handler) => - Effect.acquireUseRelease( - Effect.sync(() => { - protocolRunners++ - }), - () => baseProtocol.run(handler), - () => - Effect.sync(() => { - protocolRunners-- - }) - ) - } - - const onSocket = (socket: Socket.Socket) => - Effect.scoped( - Effect.suspend(() => { - if ( - protocolRunners === 0 || - connections + Queue.sizeUnsafe(disconnects) >= limits.maxRelayConnections - ) { - return Effect.gen(function*() { - const write = yield* socket.writer - yield* socket.runRaw( - () => Effect.void, - { onOpen: write(new Socket.CloseEvent(1013)).pipe(Effect.ignore) } - ).pipe(Effect.ignore) - }) - } - connections++ - const clientId = nextClientId++ - return Effect.gen(function*() { - const write = yield* socket.writer - clients.set(clientId, { write }) - clientIds.add(clientId) - - yield* makeFrameReader( - socket, - limits, - budget.reserve, - (value, reservation): Effect.Effect => { - if ( - typeof value !== "object" || - value === null || - Array.isArray(value) || - !("_tag" in value) - ) { - return Effect.andThen(reservation.release, Effect.fail(invalidFrame())) - } - const tag = value._tag - if ( - tag !== "Request" && - tag !== "Ack" && - tag !== "Interrupt" && - tag !== "Ping" && - tag !== "Eof" - ) { - return Effect.andThen(reservation.release, Effect.fail(invalidFrame())) - } - const message = value as RpcMessage.FromClientEncoded - if (message._tag !== "Request") { - return Effect.provideService( - writeRequest(clientId, message), - CurrentRequestKey, - undefined - ).pipe(Effect.ensuring(reservation.release)) - } - if (typeof message.id !== "string" && typeof message.id !== "number") { - return Effect.andThen(reservation.release, Effect.fail(invalidFrame())) - } - let client = inbound.get(clientId) - if (client === undefined) { - client = new Map() - inbound.set(clientId, client) - } - if (client.has(message.id)) { - return Effect.andThen(reservation.release, Effect.fail(invalidFrame())) - } - client.set(message.id, reservation) - const key = { clientId, requestId: message.id } - return Effect.provideService( - writeRequest(clientId, message), - CurrentRequestKey, - key - ).pipe( - Effect.onExit((exit) => { - if (exit._tag === "Success") return Effect.void - return removeAndReleaseReservation(inbound, key) - }) - ) - } - ).pipe(Effect.ignore) - }).pipe( - Effect.ensuring( - Effect.gen(function*() { - yield* cleanupClient(clientId) - connections-- - }) - ) - ) - }) - ) - - const acceptFiber = yield* Effect.forkIn(server.run(onSocket), childScope) - const service = PeerRelayIngress.of({ - address: server.address, - reserveOutbound: budget.reserve, - usage: Effect.sync(() => ({ - connections, - ...budget.usage() - })), - await: Fiber.join(acceptFiber) - }) - return { protocol, service } - }) - -export const layerProtocolSocketServer = ( - socketLayer: Layer.Layer -): Layer.Layer< - PeerRelayIngress | RpcServer.Protocol, - E, - R | PeerRelayLimits -> => - Layer.effectContext( - Effect.gen(function*() { - const limits = yield* PeerRelayLimits - const childScope = yield* Effect.acquireRelease( - Scope.make("sequential"), - (scope, exit) => Scope.close(scope, exit) - ) - const socketContext = yield* Layer.buildWithScope(socketLayer, childScope) - const server = Context.get(socketContext, SocketServer.SocketServer) - const { protocol, service } = yield* makeServerProtocol(server, limits, childScope) - return Context.make(PeerRelayIngress, service).pipe( - Context.add(RpcServer.Protocol, protocol) - ) - }) - ) as Layer.Layer - -export const makeProtocolSocket = Effect.gen(function*() { - const socket = yield* Socket.Socket - const limits = yield* PeerRelayLimits - const outbound = new Map>() - const budget = makeByteBudget( - limits.maximumSharedPayloadBytes, - limits.maximumByteReservationWaiters, - outbound - ) - - return yield* RpcClient.Protocol.make((writeResponse, clientIds) => - Effect.gen(function*() { - const write = yield* socket.writer - const requestClients = new Map() - let currentError: RpcClientError.RpcClientError | undefined - - const read = makeFrameReader( - socket, - limits, - budget.reserve, - (value, reservation) => { - if ( - typeof value !== "object" || - value === null || - Array.isArray(value) || - !("_tag" in value) - ) { - return Effect.andThen(reservation.release, Effect.fail(invalidFrame())) - } - const response = value as RpcMessage.FromServerEncoded - const requestClient = "requestId" in response - ? requestClients.get(response.requestId) - : undefined - const effect = requestClient === undefined - ? Effect.forEach(clientIds, (clientId) => writeResponse(clientId, response), { - discard: true - }) - : Effect.suspend(() => { - if (response._tag === "Exit" && "requestId" in response) { - requestClients.delete(response.requestId) - } - return writeResponse(requestClient, response) - }) - return Effect.ensuring(effect, reservation.release) - } - ).pipe( - Effect.flatMap(() => - Effect.fail( - new Socket.SocketError({ - reason: new Socket.SocketCloseError({ code: 1000 }) - }) - ) - ), - Effect.tapCause((cause) => { - if (Cause.hasInterruptsOnly(cause)) return Effect.void - currentError = new RpcClientError.RpcClientError({ - reason: new RpcClientError.RpcClientDefect({ - message: "Relay socket protocol failed", - cause: Cause.squash(cause) - }) - }) - return Effect.forEach( - clientIds, - (clientId) => - writeResponse(clientId, { - _tag: "ClientProtocolError", - error: currentError! - }), - { discard: true } - ) - }), - Effect.catchCause((cause) => Cause.hasInterruptsOnly(cause) ? Effect.failCause(cause) : Effect.void), - Effect.forkScoped - ) - yield* read - - return { - send: (clientId: number, request: RpcMessage.FromClientEncoded) => - Effect.gen(function*() { - if (currentError !== undefined) return yield* Effect.fail(currentError) - yield* budget.use( - limits.maximumDeclaredFrameBytes, - (reservation) => - Effect.gen(function*() { - let encoded: ReturnType - try { - encoded = encodeFrame(request, limits.maximumDeclaredFrameBytes) - } catch (cause) { - return yield* new RpcClientError.RpcClientError({ - reason: new RpcClientError.RpcClientDefect({ - message: "Relay request frame is invalid", - cause - }) - }) - } - yield* reservation.shrinkTo(encoded.bodyBytes) - if (request._tag === "Request") requestClients.set(request.id, clientId) - yield* write(encoded.frame).pipe( - Effect.catchTag("SocketError", (cause) => - Effect.fail( - new RpcClientError.RpcClientError({ - reason: new RpcClientError.RpcClientDefect({ - message: "Relay socket write failed", - cause - }) - }) - )) - ) - }) - ).pipe( - Effect.catchTags({ - RequestLimitExceeded: (cause) => - Effect.fail( - new RpcClientError.RpcClientError({ - reason: new RpcClientError.RpcClientDefect({ - message: "Relay request capacity is exhausted", - cause - }) - }) - ), - RequestCapacityExceeded: (cause) => - Effect.fail( - new RpcClientError.RpcClientError({ - reason: new RpcClientError.RpcClientDefect({ - message: "Relay request capacity is exhausted", - cause - }) - }) - ) - }) - ) - }), - supportsAck: true, - supportsTransferables: false - } - }) - ) -}) - -export const layerProtocolSocket: Layer.Layer< - RpcClient.Protocol, - never, - Socket.Socket | PeerRelayLimits -> = Layer.effect(RpcClient.Protocol, makeProtocolSocket) diff --git a/packages/local-rpc/src/PeerRelayStore.ts b/packages/local-rpc/src/PeerRelayStore.ts deleted file mode 100644 index dbdecae..0000000 --- a/packages/local-rpc/src/PeerRelayStore.ts +++ /dev/null @@ -1,173 +0,0 @@ -import * as Identity from "@lucas-barake/effect-local/Identity" -import type * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" -import * as Context from "effect/Context" -import type * as Effect from "effect/Effect" -import type * as Option from "effect/Option" -import * as Schema from "effect/Schema" -import { PeerPrincipal } from "./internal/peerPrincipal.js" -import * as PeerRpc from "./PeerRpc.js" - -const PositiveInt = Schema.Int.check(Schema.isGreaterThan(0)) -const NonNegativeInt = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)) -const DocumentIds = Schema.Array(Identity.DocumentId).check(Schema.isMinLength(1)) - -export const ChannelKey = Schema.Struct({ - tenantId: Schema.NonEmptyString, - senderSubjectId: Schema.NonEmptyString, - senderPeerId: Identity.PeerId, - senderReplicaIncarnation: Identity.ReplicaIncarnation, - recipientSubjectId: Schema.NonEmptyString, - recipientPeerId: Identity.PeerId -}) -export type ChannelKey = typeof ChannelKey.Type - -export const Admission = Schema.Struct({ - channel: ChannelKey, - relayMessageId: Identity.RelayMessageId, - relayPeerId: Identity.PeerId, - documentIds: DocumentIds, - senderConnectionEpoch: Schema.NonEmptyString.check(Schema.isMaxLength(256)), - senderSequence: NonNegativeInt, - payloadVersion: Schema.Literal(1), - messageHash: Schema.NonEmptyString.check(Schema.isMaxLength(256)), - outerEnvelopeDigest: PeerRpc.RelayDigest, - payload: Schema.Uint8Array, - messageTtlMillis: PositiveInt, - senderRetryHorizonMillis: PositiveInt, - minimumTerminalRetentionMillis: PositiveInt -}) -export type Admission = typeof Admission.Type - -export const AdmissionResult = Schema.Struct({ - status: Schema.Literals(["Accepted", "Duplicate"]), - channel: ChannelKey, - ready: Schema.Boolean, - nextEligibleAt: NonNegativeInt, - lane: Schema.Literal("New") -}) -export type AdmissionResult = typeof AdmissionResult.Type - -export const ClaimRequest = Schema.Struct({ - recipient: PeerPrincipal, - sender: Schema.Struct({ - subjectId: Schema.NonEmptyString, - peerId: Identity.PeerId - }), - sessionGeneration: NonNegativeInt, - authorizedDocumentIds: DocumentIds -}) -export type ClaimRequest = typeof ClaimRequest.Type - -export const ClaimedMessage = Schema.Struct({ - rowId: PositiveInt, - channel: ChannelKey, - relayMessageId: Identity.RelayMessageId, - relayPeerId: Identity.PeerId, - senderConnectionEpoch: Schema.NonEmptyString, - senderSequence: NonNegativeInt, - documentIds: DocumentIds, - payloadVersion: Schema.Literal(1), - messageHash: Schema.NonEmptyString, - outerEnvelopeDigest: PeerRpc.RelayDigest, - payloadBytes: NonNegativeInt, - claimToken: PeerRpc.ClaimToken, - claimDeadline: NonNegativeInt, - sessionGeneration: NonNegativeInt -}) -export type ClaimedMessage = typeof ClaimedMessage.Type - -export interface ClaimResult { - readonly message: Option.Option - readonly ready: boolean - readonly nextEligibleAt: Option.Option - readonly lane: "New" | "Retry" -} - -export const LoadClaimedPayloadRequest = Schema.Struct({ - rowId: PositiveInt, - channel: ChannelKey, - relayMessageId: Identity.RelayMessageId, - claimToken: PeerRpc.ClaimToken, - sessionGeneration: NonNegativeInt, - payloadBytes: NonNegativeInt -}) -export type LoadClaimedPayloadRequest = typeof LoadClaimedPayloadRequest.Type - -export const TerminalRequest = Schema.Struct({ - channel: ChannelKey, - relayMessageId: Identity.RelayMessageId, - claimToken: PeerRpc.ClaimToken, - messageHash: Schema.NonEmptyString.check(Schema.isMaxLength(256)), - sessionGeneration: NonNegativeInt, - recipient: PeerPrincipal -}) -export type TerminalRequest = typeof TerminalRequest.Type - -export const RejectRequest = Schema.Struct({ - ...TerminalRequest.fields, - reason: PeerRpc.RejectReason -}) -export type RejectRequest = typeof RejectRequest.Type - -export const ReleaseRequest = Schema.Struct({ - channel: ChannelKey, - relayMessageId: Identity.RelayMessageId, - claimToken: PeerRpc.ClaimToken, - sessionGeneration: NonNegativeInt -}) -export type ReleaseRequest = typeof ReleaseRequest.Type - -export interface TransitionResult { - readonly status: "Changed" | "Duplicate" | "Stale" - readonly ready: boolean - readonly nextEligibleAt: Option.Option - readonly lane: "New" | "Retry" -} - -export const MaintenanceRequest = Schema.Struct({ - cursor: Schema.optionalKey(NonNegativeInt), - batchSize: PositiveInt -}) -export type MaintenanceRequest = typeof MaintenanceRequest.Type - -export interface MaintenanceResult { - readonly cursor?: number - readonly processed: number - readonly hasMore: boolean -} - -export const UsageRequest = Schema.Struct({ - scopeKind: Schema.Literals(["SenderPeer", "RecipientPeer", "RecipientSubject", "Tenant", "Shard"]), - scopeKey: Schema.String -}) -export type UsageRequest = typeof UsageRequest.Type - -export interface Usage { - readonly activeCount: number - readonly activeBytes: number - readonly retainedCount: number - readonly retainedBytes: number -} - -export type StoreError = ReplicaError.ReplicaError - -export interface Service { - readonly admit: (input: Admission) => Effect.Effect - readonly claim: (input: ClaimRequest) => Effect.Effect - readonly loadClaimedPayload: ( - input: LoadClaimedPayloadRequest - ) => Effect.Effect - readonly acknowledge: (input: TerminalRequest) => Effect.Effect - readonly reject: (input: RejectRequest) => Effect.Effect - readonly release: (input: ReleaseRequest) => Effect.Effect - readonly recover: (input: MaintenanceRequest) => Effect.Effect - readonly expire: (input: MaintenanceRequest) => Effect.Effect - readonly repair: (input: MaintenanceRequest) => Effect.Effect - readonly reconcile: (input: MaintenanceRequest) => Effect.Effect - readonly collect: (input: MaintenanceRequest) => Effect.Effect - readonly usage: (input?: UsageRequest) => Effect.Effect -} - -export class PeerRelayStore extends Context.Service()( - "@lucas-barake/effect-local-rpc/PeerRelayStore" -) {} diff --git a/packages/local-rpc/src/PeerRpcServer.ts b/packages/local-rpc/src/PeerRpcServer.ts deleted file mode 100644 index cf4b426..0000000 --- a/packages/local-rpc/src/PeerRpcServer.ts +++ /dev/null @@ -1,1756 +0,0 @@ -import * as PeerSyncEnvelope from "@lucas-barake/effect-local-sql/PeerSyncEnvelope" -import * as Canonical from "@lucas-barake/effect-local/Canonical" -import * as Identity from "@lucas-barake/effect-local/Identity" -import type * as Cause from "effect/Cause" -import * as Clock from "effect/Clock" -import * as Context from "effect/Context" -import * as Crypto from "effect/Crypto" -import * as Deferred from "effect/Deferred" -import * as Effect from "effect/Effect" -import * as Exit from "effect/Exit" -import * as Fiber from "effect/Fiber" -import * as Layer from "effect/Layer" -import * as Option from "effect/Option" -import * as Queue from "effect/Queue" -import * as Schema from "effect/Schema" -import * as Scope from "effect/Scope" -import * as Semaphore from "effect/Semaphore" -import * as Stream from "effect/Stream" -import * as RpcServer from "effect/unstable/rpc/RpcServer" -import * as PeerRpcObservability from "./internal/peerRpcObservability.js" -import * as PeerAuthentication from "./PeerAuthentication.js" -import * as PeerRelayAuthorization from "./PeerRelayAuthorization.js" -import * as PeerRelayIngress from "./PeerRelayIngress.js" -import * as PeerRelayLimits from "./PeerRelayLimits.js" -import * as PeerRelayStore from "./PeerRelayStore.js" -import * as PeerRpc from "./PeerRpc.js" -import * as PeerRpcError from "./PeerRpcError.js" - -type RelaySqlLane = "Admission" | "Terminal" | "Delivery" | "Maintenance" - -interface RelaySqlTask { - state: "Queued" | "Acquired" | "Cancelled" | "Completed" - readonly cancel: Deferred.Deferred - readonly cancelResult: () => void - readonly execute: Effect.Effect -} - -interface RelaySqlScheduler { - readonly submit: ( - lane: RelaySqlLane, - effect: Effect.Effect - ) => Effect.Effect - readonly shutdown: Effect.Effect -} - -const drainRelayQueue = ( - queue: Queue.Dequeue -): Effect.Effect> => - Effect.gen(function*() { - const items: Array = [] - while (true) { - const item = yield* Queue.poll(queue) - if (Option.isNone(item)) return items - items.push(item.value) - } - }) - -const makeRelaySqlScheduler = ( - limits: PeerRelayLimits.Values, - runtimeScope: Scope.Scope, - onFatal: (cause: Cause.Cause) => Effect.Effect -): Effect.Effect => - Effect.gen(function*() { - const capacities: Record = { - Admission: limits.sqlAdmissionQueueCapacity, - Terminal: limits.sqlTerminalQueueCapacity, - Delivery: limits.sqlDeliveryQueueCapacity, - Maintenance: limits.sqlMaintenanceQueueCapacity - } - const queues = { - Admission: yield* Queue.dropping(capacities.Admission), - Terminal: yield* Queue.dropping(capacities.Terminal), - Delivery: yield* Queue.dropping(capacities.Delivery), - Maintenance: yield* Queue.dropping(capacities.Maintenance) - } satisfies Record> - const lock = yield* Semaphore.make(1) - const globalPermits = yield* Semaphore.make(limits.maxInFlightSqlTransactions) - const laneConcurrency = { - Admission: limits.maxInFlightSqlAdmission, - Terminal: limits.maxInFlightSqlTerminal, - Delivery: limits.maxInFlightSqlDelivery, - Maintenance: limits.maxInFlightSqlMaintenance - } satisfies Record - const activeTasks = new Set() - let accepting = true - - const cancelTask = (task: RelaySqlTask) => - lock.withPermit(Effect.sync(() => { - if (task.state === "Completed" || task.state === "Cancelled") return - task.state = "Cancelled" - Deferred.doneUnsafe(task.cancel, Effect.void) - task.cancelResult() - activeTasks.delete(task) - })) - - const submit: RelaySqlScheduler["submit"] = (lane, effect) => - Effect.uninterruptibleMask((restore) => - Effect.gen(function*() { - const result = yield* Deferred.make, Effect.Error>() - const cancel = yield* Deferred.make() - const task: RelaySqlTask = { - state: "Queued", - cancel, - cancelResult: () => { - Deferred.doneUnsafe(result, Effect.interrupt) - }, - execute: Effect.uninterruptibleMask((restoreTask) => - lock.withPermit(Effect.sync(() => { - if (task.state !== "Queued") return false - task.state = "Acquired" - return true - })).pipe( - Effect.flatMap((acquired) => { - if (!acquired) return Effect.void - const cancelled = Deferred.await(cancel).pipe(Effect.flatMap(() => Effect.interrupt)) - return Effect.raceFirst(restoreTask(effect), cancelled).pipe( - Effect.exit, - Effect.flatMap((exit) => Deferred.done(result, exit)), - Effect.ensuring(lock.withPermit(Effect.sync(() => { - task.state = "Completed" - activeTasks.delete(task) - }))) - ) - }) - ) - ) - } - const offered = yield* lock.withPermit( - Effect.suspend(() => { - if (!accepting) return Effect.succeed(false) - return Queue.offer(queues[lane], task).pipe( - Effect.tap((offered) => - Effect.sync(() => { - if (offered) activeTasks.add(task) - }) - ) - ) - }) - ) - if (!offered) return yield* new PeerRpcError.RequestCapacityExceeded() - return yield* restore(Deferred.await(result)).pipe( - Effect.onInterrupt(() => cancelTask(task)) - ) - }) - ) - - const workerFibers: Array> = [] - for (const lane of Object.keys(queues) as Array) { - const worker = Effect.forever( - Queue.take(queues[lane]).pipe( - Effect.flatMap((task) => task.execute.pipe(globalPermits.withPermits(1))) - ) - ) - for (let index = 0; index < laneConcurrency[lane]; index++) { - const fiber = yield* Effect.forkIn(worker, runtimeScope) - workerFibers.push(fiber) - yield* Effect.forkIn( - Fiber.await(fiber).pipe( - Effect.flatMap((exit) => - lock.withPermit(Effect.sync(() => accepting)).pipe( - Effect.flatMap((wasAccepting) => - wasAccepting && Exit.isFailure(exit) - ? onFatal(exit.cause) - : Effect.void - ) - ) - ) - ), - runtimeScope - ) - } - } - - const shutdown = Effect.uninterruptible( - lock.withPermit(Effect.sync(() => { - accepting = false - return [...activeTasks] - })).pipe( - Effect.flatMap((tasks) => Effect.forEach(tasks, cancelTask, { discard: true })), - Effect.andThen(Effect.forEach( - Object.values(queues), - (queue) => Queue.shutdown(queue), - { discard: true } - )), - Effect.andThen(Effect.forEach(workerFibers, Fiber.interrupt, { discard: true })) - ) - ).pipe(Effect.ignore) - - return { submit, shutdown } - }) - -interface RelaySubjectState { - tokens: number - updatedAt: number - lastUsedAt: number - inFlight: number -} - -interface RelayAdmissionLane { - readonly run: ( - subjectId: string, - effect: Effect.Effect - ) => Effect.Effect - readonly clear: Effect.Effect -} - -const makeRelayAdmissionLane = (options: { - readonly maximumInFlight: number - readonly maximumInFlightPerSubject: number - readonly ratePerSecond: number - readonly burst: number - readonly maximumSubjects: number - readonly idleRetentionMillis: number -}): Effect.Effect => - Effect.gen(function*() { - const lock = yield* Semaphore.make(1) - const permits = yield* Semaphore.make(options.maximumInFlight) - const subjects = new Map() - const inactive = new Map() - - const removeExpired = (now: number) => { - while (inactive.size > 0) { - const oldest = inactive.entries().next().value! - if (now - oldest[1].lastUsedAt < options.idleRetentionMillis) return - inactive.delete(oldest[0]) - subjects.delete(oldest[0]) - } - } - - const admit = (subjectId: string, now: number) => - lock.withPermit(Effect.sync(() => { - removeExpired(now) - let state = subjects.get(subjectId) - if (state?.inFlight === 0) inactive.delete(subjectId) - if (state === undefined) { - while (subjects.size >= options.maximumSubjects) { - const evictable = inactive.entries().next().value - if (evictable === undefined) return false - inactive.delete(evictable[0]) - subjects.delete(evictable[0]) - } - state = { - tokens: options.burst, - updatedAt: now, - lastUsedAt: now, - inFlight: 0 - } - subjects.set(subjectId, state) - } - const effectiveNow = Math.max(now, state.updatedAt, state.lastUsedAt) - state.tokens = Math.min( - options.burst, - state.tokens + ((effectiveNow - state.updatedAt) / 1_000) * options.ratePerSecond - ) - state.updatedAt = effectiveNow - state.lastUsedAt = effectiveNow - if (state.inFlight >= options.maximumInFlightPerSubject || state.tokens < 1) { - if (state.inFlight === 0) inactive.set(subjectId, state) - return false - } - state.tokens -= 1 - state.inFlight += 1 - return true - })) - - const release = (subjectId: string) => - Clock.currentTimeMillis.pipe( - Effect.flatMap((now) => - lock.withPermit(Effect.sync(() => { - const state = subjects.get(subjectId) - if (state === undefined) return - state.inFlight -= 1 - state.lastUsedAt = Math.max(now, state.lastUsedAt) - if (state.inFlight === 0) { - inactive.delete(subjectId) - inactive.set(subjectId, state) - } - })) - ) - ) - - const run: RelayAdmissionLane["run"] = (subjectId, effect) => - Effect.uninterruptibleMask((restore) => - Effect.gen(function*() { - const admittedAt = yield* Clock.currentTimeMillis - if (!(yield* admit(subjectId, admittedAt))) { - return yield* new PeerRpcError.RequestCapacityExceeded() - } - return yield* restore( - effect.pipe( - permits.withPermitsIfAvailable(1), - Effect.flatMap(Effect.fromOption(() => new PeerRpcError.RequestCapacityExceeded())) - ) - ).pipe(Effect.ensuring(release(subjectId))) - }) - ) - - return { - run, - clear: lock.withPermit(Effect.sync(() => { - subjects.clear() - inactive.clear() - })) - } - }) - -export interface PeerRpcServerUsage { - readonly accepting: boolean - readonly sessions: number - readonly subjects: number - readonly activeClaims: number - readonly queuedChannels: number -} - -export class PeerRpcServerRuntime extends Context.Service - readonly owner: Effect.Effect - readonly shutdown: Effect.Effect - readonly usage: Effect.Effect -}>()("@lucas-barake/effect-local-rpc/PeerRpcServerRuntime") {} - -class PeerRpcFatalSignal extends Context.Service) => Effect.Effect -}>()("@lucas-barake/effect-local-rpc/PeerRpcFatalSignal") {} - -interface RelayWorkSelector { - readonly recipient: { - readonly tenantId: string - readonly subjectId: string - readonly peerId: Identity.PeerId - } - readonly sender: { - readonly subjectId: string - readonly peerId: Identity.PeerId - } -} - -interface RelayOutboundItem { - readonly event: PeerRpc.StoredMessage - readonly reservation: PeerRelayIngress.Reservation - transferred: boolean -} - -interface RelayEntry { - readonly sessionId: Identity.SessionId - readonly generation: number - readonly principal: PeerAuthentication.PeerPrincipal - readonly senderReplicaIncarnation: Identity.ReplicaIncarnation - readonly remote: PeerRelayAuthorization.RemotePeer - readonly documents: ReadonlyArray - readonly receiptRetentionMillis: number - readonly senderRetryHorizonMillis: number - readonly outbound: Queue.Queue - readonly claims: Map - readonly revoked: Deferred.Deferred - readonly monitorFibers: Array> - watcher: Fiber.Fiber | undefined - active: boolean -} - -type RelayWorkOwner = - | { - readonly _tag: "Worker" - readonly lane: "New" | "Retry" - pending: boolean - } - | { - readonly _tag: "Entry" - readonly generation: number - pending: boolean - } - | { - readonly _tag: "Terminal" - } - -const relaySelectorKey = (selector: RelayWorkSelector) => - JSON.stringify([ - selector.recipient.tenantId, - selector.recipient.subjectId, - selector.recipient.peerId, - selector.sender.subjectId, - selector.sender.peerId - ]) - -const relayEntryKey = ( - principal: PeerAuthentication.PeerPrincipal, - remote: PeerRelayAuthorization.RemotePeer -) => - JSON.stringify([ - principal.tenantId, - principal.subjectId, - principal.peerId, - remote.subjectId, - remote.peerId - ]) - -const relayIncarnationKey = ( - principal: PeerAuthentication.PeerPrincipal, - incarnation: Identity.ReplicaIncarnation, - remote: PeerRelayAuthorization.RemotePeer -) => - JSON.stringify([ - principal.tenantId, - principal.subjectId, - principal.peerId, - incarnation, - remote.subjectId, - remote.peerId - ]) - -const relaySelectorForEntry = (entry: RelayEntry): RelayWorkSelector => ({ - recipient: entry.principal, - sender: entry.remote -}) - -const sameRelayPrincipal = ( - left: PeerAuthentication.PeerPrincipal, - right: PeerAuthentication.PeerPrincipal -) => - left.tenantId === right.tenantId && - left.subjectId === right.subjectId && - left.peerId === right.peerId - -const relayStoreFailure = (error: PeerRelayStore.StoreError): PeerRpcError.PeerRpcError => { - if (error._tag !== "ReplicaError") { - return new PeerRpcError.ServerUnavailable() - } - switch (error.reason._tag) { - case "QuotaExceeded": - return new PeerRpcError.RequestCapacityExceeded() - case "ProtocolMismatch": - return new PeerRpcError.InvalidRequest() - default: - return new PeerRpcError.ServerUnavailable() - } -} - -const RelaySyncEnvelopeJson = Schema.fromJsonString( - Schema.toCodecJson(PeerSyncEnvelope.SyncEnvelope) -) - -const decodeRelayEnvelope = (payload: Uint8Array) => - Schema.decodeUnknownEffect(RelaySyncEnvelopeJson)(new TextDecoder().decode(payload)).pipe( - Effect.mapError(() => new PeerRpcError.InvalidRequest()) - ) - -export const layerHandlers = ( - options: { - readonly tenantId: string - readonly peerId: Identity.PeerId - } -) => - Layer.effectContext(Effect.gen(function*() { - const serverScope = yield* Scope.Scope - const runtimeScope = yield* Scope.fork(serverScope, "sequential") - const authorization = yield* PeerRelayAuthorization.PeerRelayAuthorization - const store = yield* PeerRelayStore.PeerRelayStore - const limits = yield* PeerRelayLimits.PeerRelayLimits - const ingress = yield* PeerRelayIngress.PeerRelayIngress - yield* Crypto.Crypto - const lock = yield* Semaphore.make(1) - const fatal = yield* Deferred.make() - const shutdownStarted = yield* Deferred.make() - const shutdownFinished = yield* Deferred.make() - const sessions = new Map() - const endpoints = new Map() - const incarnations = new Map() - const subjectSessions = new Map() - const workOwners = new Map() - const selectors = new Map() - const newWork = yield* Queue.dropping(limits.newWorkQueueCapacity) - const retryWork = yield* Queue.dropping(limits.retryQueueCapacity) - const newWorkLock = yield* Semaphore.make(1) - const retryWorkLock = yield* Semaphore.make(1) - const workWake = yield* Queue.dropping( - limits.newWorkQueueCapacity + limits.retryQueueCapacity - ) - let accepting = true - let generation = 0 - let compensationCursor = 0 - - const signalFatal = (cause: Cause.Cause) => Deferred.done(fatal, Exit.failCause(cause)).pipe(Effect.asVoid) - - const sql = yield* makeRelaySqlScheduler(limits, runtimeScope, signalFatal) - const openLane = yield* makeRelayAdmissionLane({ - maximumInFlight: limits.maxInFlightOpen, - maximumInFlightPerSubject: limits.maxInFlightOpenPerSubject, - ratePerSecond: limits.openRatePerSecond, - burst: limits.openBurst, - maximumSubjects: limits.maxRetainedRateLimitedSubjects, - idleRetentionMillis: limits.rateLimitIdleRetentionMillis - }) - const pushLane = yield* makeRelayAdmissionLane({ - maximumInFlight: limits.maxInFlightPush, - maximumInFlightPerSubject: limits.maxInFlightPushPerSubject, - ratePerSecond: limits.admissionRatePerSecond, - burst: limits.admissionBurst, - maximumSubjects: limits.maxRetainedRateLimitedSubjects, - idleRetentionMillis: limits.rateLimitIdleRetentionMillis - }) - const terminalLane = yield* makeRelayAdmissionLane({ - maximumInFlight: limits.maxInFlightTerminalResponses, - maximumInFlightPerSubject: limits.maxInFlightTerminalResponsesPerSubject, - ratePerSecond: limits.terminalResponseRatePerSecond, - burst: limits.terminalResponseBurst, - maximumSubjects: limits.maxRetainedTerminalResponseSubjects, - idleRetentionMillis: limits.terminalResponseSubjectIdleRetentionMillis - }) - - const storeEffect = (effect: Effect.Effect) => - effect.pipe(Effect.mapError(relayStoreFailure)) - - const activeClaimCount = () => [...sessions.values()].reduce((sum, entry) => sum + entry.claims.size, 0) - - const refreshRelayPending = sql.submit( - "Maintenance", - storeEffect(store.usage()) - ).pipe( - Effect.flatMap((usage) => PeerRpcObservability.setRelayPending(usage.activeCount, usage.activeBytes)) - ) - - yield* refreshRelayPending - yield* PeerRpcObservability.setRelayActiveClaims(0) - yield* PeerRpcObservability.setRelayWorkers(0) - yield* PeerRpcObservability.setRelayReadyQueueItems("New", 0) - yield* PeerRpcObservability.setRelayReadyQueueItems("Retry", 0) - - const releaseClaim = ( - entry: RelayEntry, - claim: PeerRelayStore.ClaimedMessage - ) => - sql.submit( - "Terminal", - storeEffect(store.release({ - channel: claim.channel, - relayMessageId: claim.relayMessageId, - claimToken: claim.claimToken, - sessionGeneration: entry.generation - })) - ).pipe( - Effect.timeoutOrElse({ - duration: limits.shutdownReleaseTimeoutMillis, - orElse: () => Effect.succeed(undefined) - }), - Effect.ignore - ) - - const detachEntry = ( - entry: RelayEntry, - fromWatcher: boolean - ): Effect.Effect => - Effect.uninterruptible(Effect.gen(function*() { - yield* Deferred.succeed(entry.revoked, undefined) - const cleanup = yield* lock.withPermit(Effect.gen(function*() { - if (!entry.active) return undefined - entry.active = false - sessions.delete(entry.sessionId) - const endpoint = relayEntryKey(entry.principal, entry.remote) - if (endpoints.get(endpoint) === entry.sessionId) endpoints.delete(endpoint) - const incarnation = relayIncarnationKey( - entry.principal, - entry.senderReplicaIncarnation, - entry.remote - ) - if (incarnations.get(incarnation) === entry.sessionId) incarnations.delete(incarnation) - const current = subjectSessions.get(entry.principal.subjectId) ?? 0 - if (current <= 1) subjectSessions.delete(entry.principal.subjectId) - else subjectSessions.set(entry.principal.subjectId, current - 1) - const selector = relaySelectorForEntry(entry) - const key = relaySelectorKey(selector) - const owner = workOwners.get(key) - if (owner?._tag === "Entry" && owner.generation === entry.generation) { - workOwners.delete(key) - } - if (endpoints.get(endpoint) === undefined) { - selectors.delete(key) - } - const claims = [...entry.claims.values()] - entry.claims.clear() - yield* PeerRpcObservability.setRelayActiveClaims(activeClaimCount()) - return { claims } - })) - if (cleanup === undefined) return - const buffered = yield* drainRelayQueue(entry.outbound) - yield* Effect.forEach(buffered, (item) => item.reservation.release, { - concurrency: 1, - discard: true - }) - yield* Queue.fail(entry.outbound, new PeerRpcError.SessionUnavailable()) - yield* Queue.shutdown(entry.outbound) - yield* Effect.forEach( - cleanup.claims, - (claim) => releaseClaim(entry, claim), - { concurrency: limits.shutdownReleaseConcurrency, discard: true } - ) - const watcherFibers = fromWatcher || entry.watcher === undefined - ? entry.monitorFibers - : [...entry.monitorFibers, entry.watcher] - yield* Effect.forEach( - watcherFibers, - (fiber) => Effect.forkIn(Fiber.interrupt(fiber), runtimeScope), - { discard: true } - ) - })) - - const notify = ( - selector: RelayWorkSelector, - lane: "New" | "Retry" - ): Effect.Effect => - Effect.gen(function*() { - const key = relaySelectorKey(selector) - const shouldOffer = yield* lock.withPermit(Effect.sync(() => { - if (!accepting) return false - if ( - endpoints.get(relayEntryKey(selector.recipient, selector.sender)) === undefined - ) { - return false - } - selectors.set(key, selector) - const current = workOwners.get(key) - if (current === undefined || current._tag === "Terminal") { - workOwners.set(key, { _tag: "Worker", lane, pending: false }) - return true - } - current.pending = true - return false - })) - if (!shouldOffer) return - const queue = lane === "New" ? newWork : retryWork - const queueLock = lane === "New" ? newWorkLock : retryWorkLock - const offered = yield* queueLock.withPermit(Effect.gen(function*() { - const offered = yield* Queue.offer(queue, selector) - if (offered) { - const size = yield* Queue.size(queue) - yield* PeerRpcObservability.setRelayReadyQueueItems(lane, size) - } - return offered - })) - if (!offered) { - yield* lock.withPermit(Effect.sync(() => { - const current = workOwners.get(key) - if (current?._tag === "Worker" && current.lane === lane) { - workOwners.delete(key) - } - })) - return - } - yield* Queue.offer(workWake, undefined) - }) - - const ensureCurrentEntry = (entry: RelayEntry) => - lock.withPermit(Effect.gen(function*() { - const revoked = yield* Deferred.poll(entry.revoked) - if ( - Option.isSome(revoked) || - !entry.active || - sessions.get(entry.sessionId) !== entry || - endpoints.get(relayEntryKey(entry.principal, entry.remote)) !== entry.sessionId || - incarnations.get( - relayIncarnationKey( - entry.principal, - entry.senderReplicaIncarnation, - entry.remote - ) - ) !== entry.sessionId - ) { - return yield* new PeerRpcError.SessionUnavailable() - } - return entry - })) - - const freshEntry = ( - sessionId: Identity.SessionId, - authenticated: Context.Service.Shape - ) => - Effect.gen(function*() { - const now = yield* Clock.currentTimeMillis - if (authenticated.validUntil <= now) { - return yield* new PeerRpcError.SessionUnavailable() - } - const entry = yield* lock.withPermit(Effect.sync(() => sessions.get(sessionId))) - if ( - entry === undefined || - !sameRelayPrincipal(entry.principal, authenticated.principal) - ) { - return yield* new PeerRpcError.SessionUnavailable() - } - return yield* ensureCurrentEntry(entry) - }) - - const raceRevocation = ( - entry: RelayEntry, - effect: Effect.Effect - ) => - Effect.raceFirst( - effect, - Deferred.await(entry.revoked).pipe( - Effect.andThen(Effect.fail(new PeerRpcError.SessionUnavailable())) - ) - ) - - const authorizeEntry = ( - entry: RelayEntry, - direction: PeerRelayAuthorization.Direction, - documents: ReadonlyArray - ) => - authorization.authorize({ - direction, - principal: entry.principal, - remote: entry.remote, - documents - }) - - const withRelayGrantAdmission = ( - entry: RelayEntry, - grants: ReadonlyArray<{ - readonly validUntil: number - readonly invalidated: Effect.Effect - }>, - effect: Effect.Effect - ) => - Effect.gen(function*() { - const now = yield* Clock.currentTimeMillis - const validUntil = Math.min(...grants.map((grant) => grant.validUntil)) - if (validUntil <= now) { - return yield* new PeerRpcError.AccessDenied() - } - const admissionGate = yield* Semaphore.make(1) - let admitted = false - let invalidated = false - const invalidate = admissionGate.withPermit( - Effect.sync(() => { - if (!admitted) invalidated = true - }) - ) - const enter = admissionGate.withPermit( - Effect.gen(function*() { - const currentTime = yield* Clock.currentTimeMillis - if (invalidated || validUntil <= currentTime) return false - admitted = true - return true - }) - ).pipe( - Effect.flatMap((entered) => - entered - ? Effect.void - : Effect.fail(new PeerRpcError.AccessDenied()) - ) - ) - return yield* Effect.acquireUseRelease( - Effect.forEach( - [ - ...grants.map((grant) => grant.invalidated), - Deferred.await(entry.revoked), - Effect.sleep(Math.max(0, validUntil - now)) - ], - (monitor) => - monitor.pipe( - Effect.exit, - Effect.andThen(invalidate), - Effect.forkIn(runtimeScope) - ) - ), - () => - Effect.gen(function*() { - yield* Effect.yieldNow - yield* ensureCurrentEntry(entry) - yield* enter - return yield* effect - }), - (fibers) => - Effect.forEach( - fibers, - (fiber) => Effect.forkIn(Fiber.interrupt(fiber), runtimeScope), - { discard: true } - ) - ) - }) - - const withRelayAuthorizationGrants = ( - entry: RelayEntry, - direction: PeerRelayAuthorization.Direction, - normalGrant: PeerRelayAuthorization.Result, - documents: PeerRelayAuthorization.UnsafeUnboundedAutomerge3DecodeRequest["documents"], - effect: Effect.Effect - ) => - Effect.gen(function*() { - const grant = yield* authorization.authorizeUnsafeUnboundedAutomerge3Decode({ - risk: PeerRelayAuthorization.unsafeUnboundedAutomerge3DecodeRisk, - direction, - principal: entry.principal, - remote: entry.remote, - documents - }) - return yield* withRelayGrantAdmission( - entry, - [normalGrant, grant], - effect - ) - }) - - const validateClaimPayload = ( - claim: PeerRelayStore.ClaimedMessage, - payload: Uint8Array - ) => - Effect.gen(function*() { - if (payload.byteLength !== claim.payloadBytes) { - return yield* new PeerRpcError.ServerUnavailable() - } - const envelope = yield* decodeRelayEnvelope(payload) - if ( - claim.relayPeerId !== options.peerId || - envelope.connectionEpoch !== claim.senderConnectionEpoch || - envelope.sequence !== claim.senderSequence || - envelope.messageHash !== claim.messageHash || - claim.documentIds.length !== 1 || - envelope.documentId !== claim.documentIds[0] - ) { - return yield* new PeerRpcError.ServerUnavailable() - } - const digest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope({ - domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, - version: PeerSyncEnvelope.relayOuterEnvelopeVersion, - expectedLocal: { - tenantId: claim.channel.tenantId, - subjectId: claim.channel.senderSubjectId, - peerId: claim.channel.senderPeerId - }, - remote: { - tenantId: claim.channel.tenantId, - subjectId: claim.channel.recipientSubjectId, - peerId: claim.channel.recipientPeerId - }, - relayPeerId: claim.relayPeerId, - relayMessageId: claim.relayMessageId, - protocolVersion: PeerRpc.protocolVersion, - payloadVersion: claim.payloadVersion, - senderReplicaIncarnation: claim.channel.senderReplicaIncarnation, - senderConnectionEpoch: claim.senderConnectionEpoch, - senderSequence: claim.senderSequence, - document: { - documentId: envelope.documentId, - documentType: envelope.documentType - }, - lineage: envelope.lineage, - writerProvenance: envelope.writerProvenance, - messageHash: envelope.messageHash, - payload - }).pipe(Effect.mapError(relayStoreFailure)) - if (digest !== claim.outerEnvelopeDigest) { - return yield* new PeerRpcError.ServerUnavailable() - } - return envelope - }) - - const abandonClaim = ( - entry: RelayEntry, - claim: PeerRelayStore.ClaimedMessage - ) => - lock.withPermit(Effect.gen(function*() { - if (entry.claims.get(claim.relayMessageId) === claim) { - entry.claims.delete(claim.relayMessageId) - const key = relaySelectorKey(relaySelectorForEntry(entry)) - const owner = workOwners.get(key) - if ( - owner?._tag === "Worker" || - owner?._tag === "Entry" && owner.generation === entry.generation - ) { - workOwners.delete(key) - } - yield* PeerRpcObservability.setRelayActiveClaims(activeClaimCount()) - return true - } - return false - })).pipe( - Effect.flatMap((removed) => removed ? releaseClaim(entry, claim) : Effect.void) - ) - - const deliver = (selector: RelayWorkSelector) => - Effect.gen(function*() { - const key = relaySelectorKey(selector) - const entry = yield* lock.withPermit(Effect.sync(() => { - const sessionId = endpoints.get(relayEntryKey(selector.recipient, selector.sender)) - return sessionId === undefined ? undefined : sessions.get(sessionId) - })) - if (entry === undefined || !entry.active) { - yield* lock.withPermit(Effect.sync(() => { - workOwners.delete(key) - })) - return - } - const allowed = yield* authorizeEntry(entry, "Receive", entry.documents) - const authorizedDocumentIds = allowed.documents.map((document) => document.documentId) - const unsafeDocuments = allowed.documents.map((document) => ({ - documentType: document.document.name, - documentId: document.documentId - })) - let ownedClaim: PeerRelayStore.ClaimedMessage | undefined - yield* withRelayAuthorizationGrants( - entry, - "Receive", - allowed, - unsafeDocuments, - Effect.gen(function*() { - type ClaimAcquisition = { - readonly claimed: PeerRelayStore.ClaimResult - readonly claim: PeerRelayStore.ClaimedMessage | undefined - readonly recorded: boolean - } - const acquisition = yield* Effect.uninterruptibleMask((restore) => - restore(sql.submit( - "Delivery", - storeEffect(store.claim({ - recipient: selector.recipient, - sender: selector.sender, - sessionGeneration: entry.generation, - authorizedDocumentIds - })) - )).pipe( - Effect.flatMap((claimed) => { - if (Option.isNone(claimed.message)) { - return Effect.succeed({ - claimed, - claim: undefined, - recorded: true - } as ClaimAcquisition) - } - const claim = claimed.message.value - return lock.withPermit(Effect.gen(function*() { - if ( - sessions.get(entry.sessionId) !== entry || - !entry.active || - endpoints.get(relayEntryKey(entry.principal, entry.remote)) !== entry.sessionId - ) { - return { claimed, claim, recorded: false } as ClaimAcquisition - } - entry.claims.set(claim.relayMessageId, claim) - yield* PeerRpcObservability.setRelayActiveClaims(activeClaimCount()) - return { claimed, claim, recorded: true } as ClaimAcquisition - })) - }) - ) - ) - if (acquisition.claim === undefined) { - const pending = yield* lock.withPermit(Effect.sync(() => { - const current = workOwners.get(key) - workOwners.delete(key) - return current?._tag === "Worker" && current.pending - })) - if (pending) yield* notify(selector, acquisition.claimed.lane) - return - } - const claim = acquisition.claim - if (!acquisition.recorded) { - yield* releaseClaim(entry, claim) - return - } - ownedClaim = claim - let pendingReservation: PeerRelayIngress.Reservation | undefined - yield* Effect.gen(function*() { - const reservation = yield* ingress.reserveOutbound(claim.payloadBytes).pipe( - Effect.onError(() => abandonClaim(entry, claim)) - ) - pendingReservation = reservation - const payload = yield* sql.submit( - "Delivery", - storeEffect(store.loadClaimedPayload({ - channel: claim.channel, - rowId: claim.rowId, - relayMessageId: claim.relayMessageId, - claimToken: claim.claimToken, - sessionGeneration: entry.generation, - payloadBytes: claim.payloadBytes - })) - ).pipe( - Effect.catchTag( - "InvalidRequest", - () => Effect.fail(new PeerRpcError.SessionUnavailable()) - ), - Effect.onError(() => Effect.andThen(reservation.release, abandonClaim(entry, claim))) - ) - const envelope = yield* validateClaimPayload(claim, payload).pipe( - Effect.onError(() => Effect.andThen(reservation.release, abandonClaim(entry, claim))) - ) - const authorized = allowed.documents.some((document) => - document.documentId === envelope.documentId && - document.document.name === envelope.documentType - ) - if (!authorized) { - yield* Effect.uninterruptible( - Effect.sync(() => { - pendingReservation = undefined - }).pipe( - Effect.andThen(reservation.release), - Effect.ensuring(abandonClaim(entry, claim)) - ) - ) - return - } - const event = PeerRpc.StoredMessage.make({ - _tag: "StoredMessage", - relayMessageId: claim.relayMessageId, - claimToken: claim.claimToken, - relayPeerId: claim.relayPeerId, - sender: { - tenantId: claim.channel.tenantId, - subjectId: claim.channel.senderSubjectId, - peerId: claim.channel.senderPeerId, - replicaIncarnation: claim.channel.senderReplicaIncarnation, - connectionEpoch: claim.senderConnectionEpoch, - sequence: claim.senderSequence - }, - recipient: selector.recipient, - payloadVersion: 1, - document: { - documentType: envelope.documentType, - documentId: envelope.documentId - }, - writerProvenance: envelope.writerProvenance, - messageHash: envelope.messageHash, - outerEnvelopeDigest: claim.outerEnvelopeDigest, - payload - }) - const transferred = yield* lock.withPermit(Effect.sync(() => { - const currentSession = sessions.get(entry.sessionId) - const owner = workOwners.get(key) - if ( - currentSession !== entry || - !entry.active || - owner?._tag !== "Worker" - ) { - return false - } - workOwners.set(key, { - _tag: "Entry", - generation: entry.generation, - pending: owner.pending - }) - return true - })) - if (!transferred) { - yield* reservation.release - yield* abandonClaim(entry, claim) - return - } - const offered = yield* Queue.offer(entry.outbound, { - event, - reservation, - transferred: false - }) - if (!offered) { - yield* lock.withPermit(Effect.sync(() => { - const owner = workOwners.get(key) - if (owner?._tag === "Entry" && owner.generation === entry.generation) { - workOwners.delete(key) - } - })) - yield* reservation.release - yield* abandonClaim(entry, claim) - return - } - pendingReservation = undefined - ownedClaim = undefined - yield* PeerRpcObservability.recordRelayOutcome({ - operation: "RelayClaim", - direction: "Receive", - result: "Delivered", - facts: { - bytes: payload.byteLength, - items: 1, - version: claim.payloadVersion - } - }) - }).pipe( - Effect.onInterrupt(() => - pendingReservation === undefined - ? Effect.void - : Effect.andThen( - pendingReservation.release, - abandonClaim(entry, claim) - ) - ) - ) - ownedClaim = undefined - }).pipe( - Effect.onExitIf( - Exit.isFailure, - () => ownedClaim === undefined ? Effect.void : abandonClaim(entry, ownedClaim) - ) - ) - ) - }).pipe( - Effect.catchTags({ - AccessDenied: () => - lock.withPermit(Effect.sync(() => { - workOwners.delete(relaySelectorKey(selector)) - })), - RequestCapacityExceeded: () => - lock.withPermit(Effect.sync(() => { - workOwners.delete(relaySelectorKey(selector)) - })), - SessionUnavailable: () => - lock.withPermit(Effect.sync(() => { - workOwners.delete(relaySelectorKey(selector)) - })) - }), - Effect.catchCause((cause) => signalFatal(cause)) - ) - - const workOrder = [ - ...Array.from({ length: limits.newWorkWeight }, () => "New" as const), - ...Array.from({ length: limits.retryWorkWeight }, () => "Retry" as const) - ] - const makeWorker = Effect.suspend(() => { - let cursor = 0 - const take = (): Effect.Effect => - Effect.gen(function*() { - for (let offset = 0; offset < workOrder.length; offset++) { - const index = (cursor + offset) % workOrder.length - const lane = workOrder[index]! - const queue = lane === "New" ? newWork : retryWork - const queueLock = lane === "New" ? newWorkLock : retryWorkLock - const item = yield* queueLock.withPermit(Effect.gen(function*() { - const item = yield* Queue.poll(queue) - if (Option.isSome(item)) { - const size = yield* Queue.size(queue) - yield* PeerRpcObservability.setRelayReadyQueueItems(lane, size) - } - return item - })) - if (Option.isSome(item)) { - cursor = (index + 1) % workOrder.length - return item.value - } - } - yield* Queue.take(workWake) - return yield* take() - }) - return Effect.forever(take().pipe(Effect.flatMap(deliver))) - }) - - const workerFibers = new Set>() - let workerCount = 0 - for (let index = 0; index < limits.relayWorkerConcurrency; index++) { - const countedWorker = Effect.acquireUseRelease( - lock.withPermit(Effect.gen(function*() { - workerCount += 1 - yield* PeerRpcObservability.setRelayWorkers(workerCount) - })), - () => makeWorker, - () => - lock.withPermit(Effect.gen(function*() { - workerCount -= 1 - yield* PeerRpcObservability.setRelayWorkers(workerCount) - })) - ) - const fiber = yield* Effect.forkIn(countedWorker, runtimeScope).pipe( - Effect.tap((fiber) => - lock.withPermit(Effect.sync(() => { - workerFibers.add(fiber) - })) - ), - Effect.uninterruptible - ) - yield* Effect.forkIn( - Fiber.await(fiber).pipe( - Effect.flatMap((exit) => - lock.withPermit(Effect.sync(() => accepting)).pipe( - Effect.flatMap((wasAccepting) => - wasAccepting && Exit.isFailure(exit) - ? signalFatal(exit.cause) - : Effect.void - ) - ) - ) - ), - runtimeScope - ) - } - - const compensationFiber = yield* Effect.forkIn( - Effect.forever( - Effect.sleep(limits.compensationIntervalMillis).pipe( - Effect.andThen(lock.withPermit(Effect.sync(() => { - const active = [...selectors.values()] - if (active.length === 0) return [] - const batch: Array = [] - for ( - let index = 0; - index < Math.min(limits.compensationBatchSize, active.length); - index++ - ) { - batch.push(active[(compensationCursor + index) % active.length]!) - } - compensationCursor = (compensationCursor + batch.length) % active.length - return batch - }))), - Effect.flatMap((batch) => - Effect.forEach(batch, (selector) => notify(selector, "Retry"), { - discard: true - }) - ) - ) - ), - runtimeScope - ) - - type MaintenanceStage = { - cursor: number | undefined - readonly batchSize: number - readonly run: ( - request: PeerRelayStore.MaintenanceRequest - ) => Effect.Effect - } - const maintenanceStages: Array = [ - { cursor: undefined, batchSize: limits.claimRecoveryBatchSize, run: store.recover }, - { cursor: undefined, batchSize: limits.expiryBatchSize, run: store.expire }, - { cursor: undefined, batchSize: limits.integrityBatchSize, run: store.repair }, - { cursor: undefined, batchSize: limits.reconciliationBatchSize, run: store.reconcile }, - { cursor: undefined, batchSize: limits.terminalCollectionBatchSize, run: store.collect } - ] - const maintenanceFiber = yield* Effect.forkIn( - Effect.forever( - Effect.sleep(limits.maintenanceIntervalMillis).pipe( - Effect.andThen(Effect.forEach(maintenanceStages, (stage) => - sql.submit( - "Maintenance", - storeEffect(stage.run({ - ...(stage.cursor === undefined ? {} : { cursor: stage.cursor }), - batchSize: stage.batchSize - })) - ).pipe( - Effect.tap((result) => - Effect.sync(() => { - stage.cursor = result.hasMore ? result.cursor : undefined - }) - ) - ), { discard: true })), - Effect.andThen(refreshRelayPending) - ) - ).pipe(Effect.catchCause(signalFatal)), - runtimeScope - ) - - const open = ( - request: typeof PeerRpc.OpenRpc.payloadSchema.Type - ) => - Effect.gen(function*() { - const authenticated = yield* PeerAuthentication.AuthenticatedPeer - return yield* openLane.run( - authenticated.principal.subjectId, - Effect.gen(function*() { - const principal = authenticated.principal - const now = yield* Clock.currentTimeMillis - if (authenticated.validUntil <= now) { - return yield* new PeerRpcError.AuthenticationFailure() - } - if (request.protocolVersion !== PeerRpc.protocolVersion) { - return yield* new PeerRpcError.UnsupportedVersion() - } - if ( - request.expectedRelayPeerId !== options.peerId || - request.expectedLocal.tenantId !== principal.tenantId || - request.expectedLocal.subjectId !== principal.subjectId || - request.expectedLocal.peerId !== principal.peerId - ) { - return yield* new PeerRpcError.PeerMismatch() - } - if (principal.tenantId !== options.tenantId) { - return yield* new PeerRpcError.AccessDenied() - } - if ( - request.senderRetryHorizonMillis > limits.maximumSenderRetryHorizonMillis || - request.receiptRetentionMillis > limits.maximumReceiptRetentionMillis || - request.receiptRetentionMillis < - Math.max(limits.messageTtlMillis, request.senderRetryHorizonMillis) + - limits.minimumTerminalRetentionMillis - ) { - return yield* new PeerRpcError.InvalidRequest() - } - const send = yield* authorization.authorize({ - direction: "Send", - principal, - remote: request.remote, - documents: request.documents - }) - const receive = yield* authorization.authorize({ - direction: "Receive", - principal, - remote: request.remote, - documents: request.documents - }) - const authorizationCompletedAt = yield* Clock.currentTimeMillis - if (authenticated.validUntil <= authorizationCompletedAt) { - return yield* new PeerRpcError.AuthenticationFailure() - } - if ( - send.validUntil <= authorizationCompletedAt || - receive.validUntil <= authorizationCompletedAt - ) { - return yield* new PeerRpcError.AccessDenied() - } - return yield* Effect.uninterruptibleMask((restore) => - Effect.gen(function*() { - const outbound = yield* Queue.dropping< - RelayOutboundItem, - PeerRpcError.PeerRpcError | Cause.Done - >(1) - const sessionId = yield* Identity.makeSessionId.pipe( - Effect.mapError(() => new PeerRpcError.ServerUnavailable()) - ) - const revoked = yield* Deferred.make() - const entry = yield* lock.withPermit(Effect.gen(function*() { - if (!accepting) return yield* new PeerRpcError.ServerUnavailable() - const current = subjectSessions.get(principal.subjectId) ?? 0 - const endpoint = relayEntryKey(principal, request.remote) - const replacedId = endpoints.get(endpoint) - const replaced = replacedId === undefined ? undefined : sessions.get(replacedId) - if (replaced === undefined && current >= limits.maxSessionsPerSubject) { - return yield* new PeerRpcError.RequestCapacityExceeded() - } - if (replaced === undefined && endpoints.size >= limits.maxActiveChannels) { - return yield* new PeerRpcError.RequestCapacityExceeded() - } - const entry: RelayEntry = { - sessionId, - generation: generation++, - principal, - senderReplicaIncarnation: request.senderReplicaIncarnation, - remote: request.remote, - documents: request.documents, - receiptRetentionMillis: request.receiptRetentionMillis, - senderRetryHorizonMillis: request.senderRetryHorizonMillis, - outbound, - claims: new Map(), - revoked, - monitorFibers: [], - watcher: undefined, - active: true - } - sessions.set(sessionId, entry) - endpoints.set(endpoint, sessionId) - incarnations.set( - relayIncarnationKey(principal, request.senderReplicaIncarnation, request.remote), - sessionId - ) - subjectSessions.set(principal.subjectId, current + 1) - selectors.set( - relaySelectorKey(relaySelectorForEntry(entry)), - relaySelectorForEntry(entry) - ) - return { entry, replaced } - })) - const start = restore(Effect.gen(function*() { - const monitorStartedAt = yield* Clock.currentTimeMillis - if (authenticated.validUntil <= monitorStartedAt) { - return yield* new PeerRpcError.AuthenticationFailure() - } - if ( - send.validUntil <= monitorStartedAt || - receive.validUntil <= monitorStartedAt - ) { - return yield* new PeerRpcError.AccessDenied() - } - const validUntil = Math.min( - authenticated.validUntil, - send.validUntil, - receive.validUntil - ) - const monitors = [ - authenticated.invalidated, - send.invalidated, - receive.invalidated, - Effect.sleep(Math.max(0, validUntil - monitorStartedAt)) - ] - for (const monitor of monitors) { - yield* monitor.pipe( - Effect.exit, - Effect.andThen(Deferred.succeed(entry.entry.revoked, undefined)), - Effect.asVoid, - Effect.forkIn(runtimeScope), - Effect.tap((fiber) => - lock.withPermit(Effect.sync(() => { - if ( - entry.entry.active && - sessions.get(entry.entry.sessionId) === entry.entry - ) { - entry.entry.monitorFibers.push(fiber) - return true - } - return false - })).pipe( - Effect.flatMap((owned) => - owned - ? Effect.void - : Effect.forkIn(Fiber.interrupt(fiber), runtimeScope).pipe( - Effect.asVoid - ) - ) - ) - ), - Effect.uninterruptible - ) - } - yield* Deferred.await(entry.entry.revoked).pipe( - Effect.andThen(detachEntry(entry.entry, true)), - Effect.forkIn(runtimeScope), - Effect.tap((fiber) => - Effect.sync(() => { - entry.entry.watcher = fiber - }) - ), - Effect.uninterruptible - ) - yield* notify(relaySelectorForEntry(entry.entry), "Retry") - const opened = PeerRpc.Opened.make({ - _tag: "Opened", - protocolVersion: PeerRpc.protocolVersion, - sessionId, - remotePeerId: entry.entry.remote.peerId, - authenticatedLocal: principal - }) - const deliveries = Stream.fromQueue(entry.entry.outbound).pipe( - Stream.mapEffect((item) => - raceRevocation( - entry.entry, - item.reservation.transferToCurrentRequest - ).pipe( - Effect.tap(() => - Effect.sync(() => { - item.transferred = true - }) - ), - Effect.andThen(ensureCurrentEntry(entry.entry)), - Effect.as(item.event), - Effect.onError(() => item.reservation.release) - ) - ) - ) - return Stream.concat(Stream.make(opened), deliveries).pipe( - Stream.ensuring(detachEntry(entry.entry, false)) - ) - })) - return yield* ( - entry.replaced === undefined - ? start - : detachEntry(entry.replaced, false).pipe(Effect.andThen(start)) - ).pipe( - Effect.onExitIf( - Exit.isFailure, - () => detachEntry(entry.entry, false) - ) - ) - }) - ) - }) - ) - }) - - const push = (request: typeof PeerRpc.PushRpc.payloadSchema.Type) => - Effect.gen(function*() { - const authenticated = yield* PeerAuthentication.AuthenticatedPeer - return yield* pushLane.run( - authenticated.principal.subjectId, - Effect.gen(function*() { - const entry = yield* freshEntry(request.sessionId, authenticated) - const envelope = yield* decodeRelayEnvelope(request.payload) - const document = entry.documents.find((candidate) => - candidate.documentId === envelope.documentId && - candidate.documentType === envelope.documentType - ) - if (document === undefined || !/^[0-9a-f]{64}$/.test(envelope.messageHash)) { - return yield* new PeerRpcError.InvalidRequest() - } - const messageHash = yield* Canonical.digest(envelope.message).pipe( - Effect.mapError(relayStoreFailure) - ) - if (messageHash !== envelope.messageHash) { - return yield* new PeerRpcError.InvalidRequest() - } - const send = yield* authorizeEntry(entry, "Send", [document]) - const sendDocument = send.documents.find((candidate) => - candidate.documentId === document.documentId && - candidate.document.name === document.documentType - ) - if (sendDocument === undefined) { - return yield* new PeerRpcError.AccessDenied() - } - yield* ensureCurrentEntry(entry) - const outerEnvelopeDigest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope({ - domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, - version: PeerSyncEnvelope.relayOuterEnvelopeVersion, - expectedLocal: entry.principal, - remote: { - tenantId: entry.principal.tenantId, - subjectId: entry.remote.subjectId, - peerId: entry.remote.peerId - }, - relayPeerId: options.peerId, - relayMessageId: request.relayMessageId, - protocolVersion: PeerRpc.protocolVersion, - payloadVersion: 1, - senderReplicaIncarnation: entry.senderReplicaIncarnation, - senderConnectionEpoch: envelope.connectionEpoch, - senderSequence: envelope.sequence, - document: { - documentId: envelope.documentId, - documentType: envelope.documentType - }, - lineage: envelope.lineage, - writerProvenance: envelope.writerProvenance, - messageHash: envelope.messageHash, - payload: request.payload - }).pipe(Effect.mapError(relayStoreFailure)) - const channel: PeerRelayStore.ChannelKey = { - tenantId: entry.principal.tenantId, - senderSubjectId: entry.principal.subjectId, - senderPeerId: entry.principal.peerId, - senderReplicaIncarnation: entry.senderReplicaIncarnation, - recipientSubjectId: entry.remote.subjectId, - recipientPeerId: entry.remote.peerId - } - const result = yield* withRelayAuthorizationGrants( - entry, - "Send", - send, - [{ - documentType: sendDocument.document.name, - documentId: sendDocument.documentId - }], - sql.submit( - "Admission", - storeEffect(store.admit({ - channel, - relayMessageId: request.relayMessageId, - relayPeerId: options.peerId, - documentIds: [envelope.documentId], - senderConnectionEpoch: envelope.connectionEpoch, - senderSequence: envelope.sequence, - payloadVersion: 1, - messageHash: envelope.messageHash, - outerEnvelopeDigest, - payload: request.payload, - messageTtlMillis: limits.messageTtlMillis, - senderRetryHorizonMillis: entry.senderRetryHorizonMillis, - minimumTerminalRetentionMillis: limits.minimumTerminalRetentionMillis - })) - ) - ) - if (result.ready) { - yield* notify({ - recipient: { - tenantId: channel.tenantId, - subjectId: channel.recipientSubjectId, - peerId: channel.recipientPeerId - }, - sender: { - subjectId: channel.senderSubjectId, - peerId: channel.senderPeerId - } - }, "New") - } - }) - ) - }) - - const terminal = ( - request: - | typeof PeerRpc.AcknowledgeRpc.payloadSchema.Type - | typeof PeerRpc.RejectRpc.payloadSchema.Type, - reason: PeerRpc.RejectReason | undefined - ) => - Effect.gen(function*() { - const authenticated = yield* PeerAuthentication.AuthenticatedPeer - return yield* terminalLane.run( - authenticated.principal.subjectId, - Effect.gen(function*() { - const entry = yield* freshEntry(request.sessionId, authenticated) - const claim = yield* lock.withPermit(Effect.sync(() => entry.claims.get(request.relayMessageId))) - if ( - claim === undefined || - claim.claimToken !== request.claimToken || - claim.messageHash !== request.messageHash - ) { - return yield* new PeerRpcError.SessionUnavailable() - } - const document = entry.documents.filter((candidate) => claim.documentIds.includes(candidate.documentId)) - const receive = yield* authorizeEntry(entry, "Receive", document) - const transition = yield* withRelayGrantAdmission( - entry, - [receive], - sql.submit( - "Terminal", - storeEffect( - reason === undefined - ? store.acknowledge({ - channel: claim.channel, - relayMessageId: claim.relayMessageId, - claimToken: claim.claimToken, - messageHash: claim.messageHash, - sessionGeneration: entry.generation, - recipient: entry.principal - }) - : store.reject({ - channel: claim.channel, - relayMessageId: claim.relayMessageId, - claimToken: claim.claimToken, - messageHash: claim.messageHash, - sessionGeneration: entry.generation, - recipient: entry.principal, - reason - }) - ) - ) - ) - const selector = relaySelectorForEntry(entry) - const shouldNotify = yield* lock.withPermit(Effect.gen(function*() { - entry.claims.delete(claim.relayMessageId) - const key = relaySelectorKey(selector) - const owner = workOwners.get(key) - if (owner?._tag === "Entry" && owner.generation === entry.generation) { - workOwners.delete(key) - } - yield* PeerRpcObservability.setRelayActiveClaims(activeClaimCount()) - return transition.ready || owner?._tag === "Entry" && owner.pending - })) - if (transition.status === "Stale") { - return yield* new PeerRpcError.SessionUnavailable() - } - if (shouldNotify) yield* notify(selector, transition.lane) - }) - ) - }) - - const shutdown = Effect.uninterruptibleMask(() => - Effect.gen(function*() { - const first = yield* Deferred.succeed(shutdownStarted, undefined) - if (!first) return yield* Deferred.await(shutdownFinished) - yield* lock.withPermit(Effect.sync(() => { - accepting = false - })) - yield* Fiber.interrupt(compensationFiber) - yield* Fiber.interrupt(maintenanceFiber) - const activeWorkers = yield* lock.withPermit(Effect.sync(() => [...workerFibers])) - yield* Effect.forEach(activeWorkers, Fiber.interrupt, { discard: true }) - yield* lock.withPermit(Effect.gen(function*() { - workerFibers.clear() - yield* PeerRpcObservability.setRelayWorkers(workerCount) - })) - yield* newWorkLock.withPermit( - Queue.shutdown(newWork).pipe( - Effect.andThen(Queue.size(newWork)), - Effect.flatMap((size) => PeerRpcObservability.setRelayReadyQueueItems("New", size)) - ) - ) - yield* retryWorkLock.withPermit( - Queue.shutdown(retryWork).pipe( - Effect.andThen(Queue.size(retryWork)), - Effect.flatMap((size) => PeerRpcObservability.setRelayReadyQueueItems("Retry", size)) - ) - ) - yield* Queue.shutdown(workWake) - const active = yield* lock.withPermit(Effect.sync(() => [...sessions.values()])) - yield* Effect.forEach(active, (entry) => detachEntry(entry, false), { - concurrency: limits.shutdownReleaseConcurrency, - discard: true - }) - yield* lock.withPermit(Effect.sync(() => { - workOwners.clear() - selectors.clear() - })) - yield* PeerRpcObservability.setRelayActiveClaims(0) - yield* sql.shutdown - yield* openLane.clear - yield* pushLane.clear - yield* terminalLane.clear - yield* Scope.close(runtimeScope, Exit.void) - yield* Deferred.succeed(shutdownFinished, undefined) - }) - ) - - yield* Effect.addFinalizer(() => shutdown) - yield* Effect.forkIn( - Deferred.await(fatal).pipe( - Effect.ensuring(shutdown) - ), - serverScope - ) - - const runtime = PeerRpcServerRuntime.of({ - health: Deferred.poll(fatal).pipe( - Effect.flatMap((exit) => - Option.isNone(exit) - ? Effect.void - : Effect.fail(new PeerRpcError.ServerUnavailable()) - ) - ), - owner: Deferred.await(fatal), - shutdown, - usage: lock.withPermit(Effect.sync(() => ({ - accepting, - sessions: sessions.size, - subjects: subjectSessions.size, - activeClaims: [...sessions.values()].reduce((sum, entry) => sum + entry.claims.size, 0), - queuedChannels: workOwners.size - }))) - }) - const handlerContext = yield* PeerRpc.Rpcs.toHandlers(PeerRpc.Rpcs.of({ - Open: (request) => Stream.unwrap(open(request)), - Push: push, - Acknowledge: (request) => terminal(request, undefined), - Reject: (request) => terminal(request, request.reason) - })) - return Context.add(handlerContext, PeerRpcServerRuntime, runtime).pipe( - Context.add(PeerRpcFatalSignal, PeerRpcFatalSignal.of({ signal: signalFatal })) - ) - })) - -export const layerServer = Layer.effectDiscard(Effect.gen(function*() { - const scope = yield* Scope.Scope - const runtime = yield* PeerRpcServerRuntime - const fatalSignal = yield* PeerRpcFatalSignal - const ingress = yield* PeerRelayIngress.PeerRelayIngress - let stopping = false - const serverFiber = yield* Effect.forkIn( - RpcServer.make(PeerRpc.Rpcs, { disableFatalDefects: true }), - scope - ) - const ingressFiber = yield* Effect.forkIn(ingress.await, scope) - const stop = Effect.uninterruptible( - Effect.sync(() => { - stopping = true - }).pipe( - Effect.andThen(runtime.shutdown), - Effect.andThen(Fiber.interrupt(serverFiber)), - Effect.andThen(Fiber.interrupt(ingressFiber)) - ) - ) - const observe = (fiber: Fiber.Fiber) => - Fiber.await(fiber).pipe( - Effect.flatMap((exit) => - stopping || Exit.isSuccess(exit) - ? Effect.void - : fatalSignal.signal(exit.cause) - ) - ) - yield* Effect.forkIn(observe(serverFiber), scope) - yield* Effect.forkIn(observe(ingressFiber), scope) - yield* Effect.forkIn( - runtime.owner.pipe( - Effect.catchCause(() => stop) - ), - scope - ) - yield* Effect.addFinalizer(() => stop) -})) diff --git a/packages/local-rpc/src/SqlPeerRelayStore.ts b/packages/local-rpc/src/SqlPeerRelayStore.ts deleted file mode 100644 index eeac994..0000000 --- a/packages/local-rpc/src/SqlPeerRelayStore.ts +++ /dev/null @@ -1,2143 +0,0 @@ -import * as Identity from "@lucas-barake/effect-local/Identity" -import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" -import * as Cause from "effect/Cause" -import * as Crypto from "effect/Crypto" -import * as Effect from "effect/Effect" -import * as Exit from "effect/Exit" -import * as Layer from "effect/Layer" -import * as Option from "effect/Option" -import * as Random from "effect/Random" -import * as Schema from "effect/Schema" -import type * as Migrator from "effect/unstable/sql/Migrator" -import * as SqlClient from "effect/unstable/sql/SqlClient" -import type * as SqlError from "effect/unstable/sql/SqlError" -import * as SqlSchema from "effect/unstable/sql/SqlSchema" -import * as PeerRelayMigrations from "./internal/peerRelayMigrations.js" -import { make as makeWriteTransaction } from "./internal/peerRelaySqlTransaction.js" -import * as PeerRpcObservability from "./internal/peerRpcObservability.js" -import * as PeerRelayLimits from "./PeerRelayLimits.js" -import { - Admission, - AdmissionResult, - ChannelKey, - ClaimedMessage, - ClaimRequest, - LoadClaimedPayloadRequest, - MaintenanceRequest, - type MaintenanceResult, - PeerRelayStore, - RejectRequest, - ReleaseRequest, - type Service, - type StoreError, - TerminalRequest, - type TransitionResult, - type Usage, - UsageRequest -} from "./PeerRelayStore.js" -import * as PeerRpc from "./PeerRpc.js" - -const DatabaseInt = Schema.Union([Schema.Int, Schema.NumberFromString]).check(Schema.isInt()) -const PositiveInt = DatabaseInt.check(Schema.isGreaterThan(0)) -const NonNegativeInt = DatabaseInt.check(Schema.isGreaterThanOrEqualTo(0)) -const DatabaseReplicaIncarnation = NonNegativeInt.pipe( - Schema.brand("@lucas-barake/effect-local/ReplicaIncarnation") -) -const DocumentIds = Schema.Array(Identity.DocumentId).check(Schema.isMinLength(1)) - -type UsageKind = UsageRequest["scopeKind"] - -interface UsageScope { - readonly kind: UsageKind - readonly key: string - readonly activeCountLimit: number - readonly activeBytesLimit: number - readonly retainedCountLimit: number - readonly retainedBytesLimit: number -} - -const protocolMismatch = (expected: string, observed: string) => - new ReplicaError.ReplicaError({ - reason: new ReplicaError.ProtocolMismatch({ expected, observed }) - }) - -const quotaExceeded = (resource: string, limit: number) => - new ReplicaError.ReplicaError({ - reason: new ReplicaError.QuotaExceeded({ resource, limit }) - }) - -const relayFailureResult = (exit: Exit.Exit) => { - const error = PeerRpcObservability.failure(exit) - if (error?._tag !== "ReplicaError") return "Failure" as const - switch (error.reason._tag) { - case "ProtocolMismatch": - return "ProtocolRejected" as const - case "QuotaExceeded": - return "CapacityRejected" as const - case "StorageUnavailable": - return "Unavailable" as const - default: - return "Failure" as const - } -} - -const relayQuotaDomain = ( - error: StoreError -): Option.Option => { - if (error._tag !== "ReplicaError" || error.reason._tag !== "QuotaExceeded") { - return Option.none() - } - const resource = error.reason.resource - if (resource === "relay payload bytes") return Option.some("Payload") - for ( - const domain of [ - "SenderPeer", - "RecipientPeer", - "RecipientSubject", - "Tenant", - "Shard" - ] as const - ) { - if (resource === `${domain} relay custody`) return Option.some(domain) - } - return Option.none() -} - -const recordQuotaRejection = (error: StoreError) => - Option.match(relayQuotaDomain(error), { - onNone: () => Effect.void, - onSome: (domain) => PeerRpcObservability.recordRelayQuotaRejection(domain).pipe(Effect.ignoreCause) - }) - -const encodeKey = (...parts: ReadonlyArray) => JSON.stringify(parts) - -const channelScopes = ( - channel: ChannelKey, - limits: PeerRelayLimits.Values -): ReadonlyArray => [ - { - kind: "SenderPeer", - key: encodeKey(channel.tenantId, channel.senderSubjectId, channel.senderPeerId), - activeCountLimit: limits.maxActiveMessagesPerSenderPeer, - activeBytesLimit: limits.maxActiveBytesPerSenderPeer, - retainedCountLimit: limits.maxRetainedRowsPerSenderPeer, - retainedBytesLimit: limits.maxRetainedBytesPerSenderPeer - }, - { - kind: "RecipientPeer", - key: encodeKey(channel.tenantId, channel.recipientSubjectId, channel.recipientPeerId), - activeCountLimit: limits.maxActiveMessagesPerRecipientPeer, - activeBytesLimit: limits.maxActiveBytesPerRecipientPeer, - retainedCountLimit: limits.maxRetainedRowsPerRecipientPeer, - retainedBytesLimit: limits.maxRetainedBytesPerRecipientPeer - }, - { - kind: "RecipientSubject", - key: encodeKey(channel.tenantId, channel.recipientSubjectId), - activeCountLimit: limits.maxActiveMessagesPerRecipientSubject, - activeBytesLimit: limits.maxActiveBytesPerRecipientSubject, - retainedCountLimit: limits.maxRetainedRowsPerRecipientSubject, - retainedBytesLimit: limits.maxRetainedBytesPerRecipientSubject - }, - { - kind: "Tenant", - key: encodeKey(channel.tenantId), - activeCountLimit: limits.maxActiveMessagesPerTenant, - activeBytesLimit: limits.maxActiveBytesPerTenant, - retainedCountLimit: limits.maxRetainedRowsPerTenant, - retainedBytesLimit: limits.maxRetainedBytesPerTenant - }, - { - kind: "Shard", - key: encodeKey("local"), - activeCountLimit: limits.maxActiveMessagesPerShard, - activeBytesLimit: limits.maxActiveBytesPerShard, - retainedCountLimit: limits.maxRetainedRowsPerShard, - retainedBytesLimit: limits.maxRetainedBytesPerShard - } -] - -const TimeRow = Schema.Struct({ now: NonNegativeInt }) -const UnitRow = Schema.Struct({ value: Schema.Int }) - -const nowQuery = (sql: SqlClient.SqlClient) => - SqlSchema.findOne({ - Request: Schema.Void, - Result: TimeRow, - execute: () => - sql.onDialectOrElse({ - pg: () => sql`SELECT CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS BIGINT) AS now`, - mysql: () => sql`SELECT CAST(UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000 AS UNSIGNED) AS now`, - orElse: () => sql`SELECT CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) AS now` - }) - })(undefined) - -const decodeDocuments = Schema.decodeUnknownEffect( - Schema.fromJsonString(DocumentIds) -) - -const parseDocuments = (value: string) => - decodeDocuments(value).pipe( - Effect.catchTag("SchemaError", (cause) => - Effect.fail( - new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ cause }) - }) - )) - ) - -const validateInput = (schema: S, input: unknown) => - Schema.decodeUnknownEffect(schema)(input).pipe( - Effect.mapError((cause) => protocolMismatch("valid relay store request", String(cause))) - ) - -const checkDurability = ( - sql: SqlClient.SqlClient -): Effect.Effect< - void, - SqlError.SqlError | Schema.SchemaError | ReplicaError.ReplicaError -> => - sql.onDialectOrElse({ - sqlite: () => - Effect.gen(function*() { - const journal = SqlSchema.findOne({ - Request: Schema.Void, - Result: Schema.Struct({ journal_mode: Schema.String }), - execute: () => sql`PRAGMA journal_mode = WAL` - }) - const synchronous = SqlSchema.findOne({ - Request: Schema.Void, - Result: Schema.Struct({ synchronous: Schema.Int }), - execute: () => sql`PRAGMA synchronous` - }) - const mode = yield* journal(undefined).pipe( - Effect.catchTag("NoSuchElementError", (cause) => - Effect.fail( - new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ cause }) - }) - )) - ) - if (mode.journal_mode.toLowerCase() !== "wal") { - return yield* new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ - cause: new Error("Relay custody requires SQLite WAL mode") - }) - }) - } - yield* sql`PRAGMA synchronous = FULL` - const setting = yield* synchronous(undefined).pipe( - Effect.catchTag("NoSuchElementError", (cause) => - Effect.fail( - new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ cause }) - }) - )) - ) - if (setting.synchronous !== 2) { - return yield* new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ - cause: new Error("Relay custody requires SQLite FULL synchronous mode") - }) - }) - } - }), - orElse: () => Effect.void - }) - -const DuplicateRow = Schema.Struct({ - channelId: PositiveInt, - tenantId: Schema.NonEmptyString, - senderSubjectId: Schema.NonEmptyString, - senderPeerId: Identity.PeerId, - senderReplicaIncarnation: DatabaseReplicaIncarnation, - recipientSubjectId: Schema.NonEmptyString, - recipientPeerId: Identity.PeerId, - outerEnvelopeDigest: PeerRpc.RelayDigest, - state: Schema.Literals(["Pending", "Claimed", "Acknowledged", "DeadLettered", "Expired"]), - nextEligibleAt: NonNegativeInt -}) - -const ChannelRow = Schema.Struct({ - channelId: PositiveInt, - nextSequence: NonNegativeInt -}) - -const MessageIdRow = Schema.Struct({ messageId: PositiveInt }) - -const CandidateRow = Schema.Struct({ - messageId: PositiveInt, - channelId: PositiveInt, - tenantId: Schema.NonEmptyString, - senderSubjectId: Schema.NonEmptyString, - senderPeerId: Identity.PeerId, - senderReplicaIncarnation: DatabaseReplicaIncarnation, - recipientSubjectId: Schema.NonEmptyString, - recipientPeerId: Identity.PeerId, - relayMessageId: Identity.RelayMessageId, - relayPeerId: Identity.PeerId, - senderConnectionEpoch: Schema.NonEmptyString, - senderSequence: NonNegativeInt, - documentIds: Schema.String, - payloadVersion: Schema.Literal(1), - messageHash: Schema.NonEmptyString, - outerEnvelopeDigest: PeerRpc.RelayDigest, - payloadBytes: NonNegativeInt, - createdAt: NonNegativeInt, - nextEligibleAt: NonNegativeInt, - retryCount: NonNegativeInt -}) - -const ReservationRow = Schema.Struct({ - senderPeerUsageKey: Schema.String, - recipientPeerUsageKey: Schema.String, - recipientSubjectUsageKey: Schema.String, - tenantUsageKey: Schema.String, - shardUsageKey: Schema.String, - activeCountDelta: Schema.Literal(1), - activeBytesDelta: NonNegativeInt, - retainedCountDelta: Schema.Literal(1), - retainedBytesDelta: NonNegativeInt, - activeConsumed: Schema.Literals([0, 1]), - retainedConsumed: Schema.Literals([0, 1]) -}) - -const UsageRow = Schema.Struct({ - activeCount: NonNegativeInt, - activeBytes: NonNegativeInt, - retainedCount: NonNegativeInt, - retainedBytes: NonNegativeInt -}) - -const KeyRow = Schema.Struct({ messageId: PositiveInt }) - -export const make = Effect.gen(function*() { - const sql = (yield* SqlClient.SqlClient).withoutTransforms() - const crypto = yield* Crypto.Crypto - const limits = yield* PeerRelayLimits.PeerRelayLimits - yield* PeerRelayMigrations.run - yield* checkDurability(sql) - - const write = makeWriteTransaction(sql, { - maxAcquireAttempts: limits.sqliteLockRetryMaxAttempts, - acquireRetryBaseDelayMillis: limits.sqliteLockRetryBaseDelayMillis, - acquireRetryMaximumDelayMillis: limits.sqliteLockRetryMaximumDelayMillis - }) - - const findDuplicate = SqlSchema.findOneOption({ - Request: Schema.Struct({ - tenantId: Schema.String, - senderSubjectId: Schema.String, - senderPeerId: Schema.String, - relayMessageId: Schema.String - }), - Result: DuplicateRow, - execute: (request) => - sql`SELECT - m.channel_id AS "channelId", - c.tenant_id AS "tenantId", - c.sender_subject_id AS "senderSubjectId", - c.sender_peer_id AS "senderPeerId", - c.sender_replica_incarnation AS "senderReplicaIncarnation", - c.recipient_subject_id AS "recipientSubjectId", - c.recipient_peer_id AS "recipientPeerId", - m.outer_envelope_digest AS "outerEnvelopeDigest", - m.state, - m.next_eligible_at AS "nextEligibleAt" - FROM effect_local_relay_messages m - JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id - JOIN effect_local_relay_reservations r ON r.message_id = m.message_id - WHERE m.tenant_id = ${request.tenantId} - AND m.sender_subject_id = ${request.senderSubjectId} - AND m.sender_peer_id = ${request.senderPeerId} - AND m.relay_message_id = ${request.relayMessageId}` - }) - - const findChannel = SqlSchema.findOne({ - Request: ChannelKey, - Result: ChannelRow, - execute: (channel) => - sql`SELECT - channel_id AS "channelId", - next_sequence AS "nextSequence" - FROM effect_local_relay_channels - WHERE tenant_id = ${channel.tenantId} - AND sender_subject_id = ${channel.senderSubjectId} - AND sender_peer_id = ${channel.senderPeerId} - AND sender_replica_incarnation = ${channel.senderReplicaIncarnation} - AND recipient_subject_id = ${channel.recipientSubjectId} - AND recipient_peer_id = ${channel.recipientPeerId}` - }) - - const insertUsage = sql.onDialectOrElse({ - mysql: () => (scope: UsageScope) => - sql`INSERT IGNORE INTO effect_local_relay_usage ( - scope_kind, - scope_key, - active_count, - active_bytes, - retained_count, - retained_bytes - ) VALUES (${scope.kind}, ${scope.key}, 0, 0, 0, 0)`, - orElse: () => (scope: UsageScope) => - sql`INSERT INTO effect_local_relay_usage ( - scope_kind, - scope_key, - active_count, - active_bytes, - retained_count, - retained_bytes - ) VALUES (${scope.kind}, ${scope.key}, 0, 0, 0, 0) - ON CONFLICT(scope_kind, scope_key) DO NOTHING` - }) - - const mutationCount = ( - mysql: Effect.Effect, - returning: Effect.Effect, SqlError.SqlError | Schema.SchemaError> - ) => - sql.onDialectOrElse({ - mysql: () => - mysql.pipe( - Effect.flatMap((result) => - Schema.decodeUnknownEffect( - Schema.Struct({ affectedRows: NonNegativeInt }) - )(result) - ), - Effect.map((result) => result.affectedRows) - ), - orElse: () => returning.pipe(Effect.map((rows) => rows.length)) - }) - - const reserveUsage = ( - scope: UsageScope, - payloadBytes: number - ): Effect.Effect => - Effect.gen(function*() { - yield* insertUsage(scope) - const changed = yield* mutationCount( - sql`UPDATE effect_local_relay_usage - SET active_count = active_count + 1, - active_bytes = active_bytes + ${payloadBytes}, - retained_count = retained_count + 1, - retained_bytes = retained_bytes + ${payloadBytes} - WHERE scope_kind = ${scope.kind} - AND scope_key = ${scope.key} - AND active_count + 1 <= ${scope.activeCountLimit} - AND active_bytes + ${payloadBytes} <= ${scope.activeBytesLimit} - AND retained_count + 1 <= ${scope.retainedCountLimit} - AND retained_bytes + ${payloadBytes} <= ${scope.retainedBytesLimit}`.raw, - SqlSchema.findAll({ - Request: Schema.Void, - Result: UnitRow, - execute: () => - sql`UPDATE effect_local_relay_usage - SET active_count = active_count + 1, - active_bytes = active_bytes + ${payloadBytes}, - retained_count = retained_count + 1, - retained_bytes = retained_bytes + ${payloadBytes} - WHERE scope_kind = ${scope.kind} - AND scope_key = ${scope.key} - AND active_count + 1 <= ${scope.activeCountLimit} - AND active_bytes + ${payloadBytes} <= ${scope.activeBytesLimit} - AND retained_count + 1 <= ${scope.retainedCountLimit} - AND retained_bytes + ${payloadBytes} <= ${scope.retainedBytesLimit} - RETURNING 1 AS value` - })(undefined) - ) - if (changed !== 1) { - return yield* quotaExceeded( - `${scope.kind} relay custody`, - Math.min(scope.activeCountLimit, scope.retainedCountLimit) - ) - } - }) - - const findReservation = SqlSchema.findOneOption({ - Request: PositiveInt, - Result: ReservationRow, - execute: (messageId) => - sql`SELECT - sender_peer_usage_key AS "senderPeerUsageKey", - recipient_peer_usage_key AS "recipientPeerUsageKey", - recipient_subject_usage_key AS "recipientSubjectUsageKey", - tenant_usage_key AS "tenantUsageKey", - shard_usage_key AS "shardUsageKey", - active_count_delta AS "activeCountDelta", - active_bytes_delta AS "activeBytesDelta", - retained_count_delta AS "retainedCountDelta", - retained_bytes_delta AS "retainedBytesDelta", - active_consumed AS "activeConsumed", - retained_consumed AS "retainedConsumed" - FROM effect_local_relay_reservations - WHERE message_id = ${messageId}` - }) - - const usageKeys = ( - reservation: typeof ReservationRow.Type - ): ReadonlyArray => [ - ["SenderPeer", reservation.senderPeerUsageKey], - ["RecipientPeer", reservation.recipientPeerUsageKey], - ["RecipientSubject", reservation.recipientSubjectUsageKey], - ["Tenant", reservation.tenantUsageKey], - ["Shard", reservation.shardUsageKey] - ] - - const releaseActiveUsage = ( - messageId: number - ): Effect.Effect => - Effect.gen(function*() { - const reservationOption = yield* findReservation(messageId) - if (Option.isNone(reservationOption)) { - return yield* new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ - cause: new Error("Missing relay quota reservation") - }) - }) - } - const reservation = reservationOption.value - if (reservation.activeConsumed === 1) { - return - } - for (const [kind, key] of usageKeys(reservation)) { - const changed = yield* mutationCount( - sql`UPDATE effect_local_relay_usage - SET active_count = active_count - ${reservation.activeCountDelta}, - active_bytes = active_bytes - ${reservation.activeBytesDelta} - WHERE scope_kind = ${kind} - AND scope_key = ${key} - AND active_count >= ${reservation.activeCountDelta} - AND active_bytes >= ${reservation.activeBytesDelta}`.raw, - SqlSchema.findAll({ - Request: Schema.Void, - Result: UnitRow, - execute: () => - sql`UPDATE effect_local_relay_usage - SET active_count = active_count - ${reservation.activeCountDelta}, - active_bytes = active_bytes - ${reservation.activeBytesDelta} - WHERE scope_kind = ${kind} - AND scope_key = ${key} - AND active_count >= ${reservation.activeCountDelta} - AND active_bytes >= ${reservation.activeBytesDelta} - RETURNING 1 AS value` - })(undefined) - ) - if (changed !== 1) { - return yield* new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ - cause: new Error("Invalid active relay quota reservation") - }) - }) - } - yield* sql`DELETE FROM effect_local_relay_usage - WHERE scope_kind = ${kind} - AND scope_key = ${key} - AND active_count = 0 - AND active_bytes = 0 - AND retained_count = 0 - AND retained_bytes = 0` - } - yield* sql`UPDATE effect_local_relay_reservations - SET active_consumed = 1 - WHERE message_id = ${messageId} AND active_consumed = 0` - }) - - const releaseRetainedUsage = ( - messageId: number - ): Effect.Effect => - Effect.gen(function*() { - const reservationOption = yield* findReservation(messageId) - if (Option.isNone(reservationOption)) { - return yield* new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ - cause: new Error("Missing relay quota reservation") - }) - }) - } - const reservation = reservationOption.value - if (reservation.retainedConsumed === 1) { - return - } - for (const [kind, key] of usageKeys(reservation)) { - const changed = yield* mutationCount( - sql`UPDATE effect_local_relay_usage - SET retained_count = retained_count - ${reservation.retainedCountDelta}, - retained_bytes = retained_bytes - ${reservation.retainedBytesDelta} - WHERE scope_kind = ${kind} - AND scope_key = ${key} - AND retained_count >= ${reservation.retainedCountDelta} - AND retained_bytes >= ${reservation.retainedBytesDelta}`.raw, - SqlSchema.findAll({ - Request: Schema.Void, - Result: UnitRow, - execute: () => - sql`UPDATE effect_local_relay_usage - SET retained_count = retained_count - ${reservation.retainedCountDelta}, - retained_bytes = retained_bytes - ${reservation.retainedBytesDelta} - WHERE scope_kind = ${kind} - AND scope_key = ${key} - AND retained_count >= ${reservation.retainedCountDelta} - AND retained_bytes >= ${reservation.retainedBytesDelta} - RETURNING 1 AS value` - })(undefined) - ) - if (changed !== 1) { - return yield* new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ - cause: new Error("Invalid retained relay quota reservation") - }) - }) - } - yield* sql`DELETE FROM effect_local_relay_usage - WHERE scope_kind = ${kind} - AND scope_key = ${key} - AND active_count = 0 - AND active_bytes = 0 - AND retained_count = 0 - AND retained_bytes = 0` - } - yield* sql`UPDATE effect_local_relay_reservations - SET retained_consumed = 1 - WHERE message_id = ${messageId} AND retained_consumed = 0` - }) - - const greatest = sql.onDialectOrElse({ - sqlite: () => "MAX", - orElse: () => "GREATEST" - }) - - const terminalize = ( - messageId: number, - state: "Acknowledged" | "DeadLettered" | "Expired", - now: number, - terminal: { - readonly token?: PeerRpc.ClaimToken - readonly sessionGeneration?: number - readonly reason: string - } - ) => - Effect.gen(function*() { - yield* releaseActiveUsage(messageId) - yield* sql`UPDATE effect_local_relay_messages - SET state = ${state}, - payload = NULL, - payload_length = 0, - claim_token = NULL, - claim_session_generation = NULL, - claim_deadline = NULL, - terminal_at = ${now}, - terminal_claim_token = ${terminal.token ?? null}, - terminal_session_generation = ${terminal.sessionGeneration ?? null}, - terminal_reason = ${terminal.reason}, - deduplicate_until = ${sql.literal(greatest)}( - deduplicate_until, - ${now + limits.minimumTerminalRetentionMillis} - ) - WHERE message_id = ${messageId} - AND state IN ('Pending', 'Claimed')` - yield* sql`UPDATE effect_local_relay_channels - SET claimed_message_id = NULL, - claim_session_generation = NULL, - claim_token = NULL, - claim_deadline = NULL - WHERE claimed_message_id = ${messageId}` - }) - - const insertChannel = sql.onDialectOrElse({ - mysql: () => (channel: ChannelKey) => - sql`INSERT IGNORE INTO effect_local_relay_channels ( - tenant_id, - sender_subject_id, - sender_peer_id, - sender_replica_incarnation, - recipient_subject_id, - recipient_peer_id - ) VALUES ( - ${channel.tenantId}, - ${channel.senderSubjectId}, - ${channel.senderPeerId}, - ${channel.senderReplicaIncarnation}, - ${channel.recipientSubjectId}, - ${channel.recipientPeerId} - )`, - orElse: () => (channel: ChannelKey) => - sql`INSERT INTO effect_local_relay_channels ( - tenant_id, - sender_subject_id, - sender_peer_id, - sender_replica_incarnation, - recipient_subject_id, - recipient_peer_id - ) VALUES ( - ${channel.tenantId}, - ${channel.senderSubjectId}, - ${channel.senderPeerId}, - ${channel.senderReplicaIncarnation}, - ${channel.recipientSubjectId}, - ${channel.recipientPeerId} - ) - ON CONFLICT ( - tenant_id, - sender_subject_id, - sender_peer_id, - sender_replica_incarnation, - recipient_subject_id, - recipient_peer_id - ) DO NOTHING` - }) - - const admit: Service["admit"] = (unsafeInput) => { - let observedBytes: number | undefined - let observedVersion: number | undefined - const effect = Effect.gen(function*() { - const input = yield* validateInput(Admission, unsafeInput) - const payloadBytes = input.payload.byteLength - observedBytes = payloadBytes - observedVersion = input.payloadVersion - if ( - input.messageTtlMillis > limits.messageTtlMillis || - input.senderRetryHorizonMillis > limits.maximumSenderRetryHorizonMillis || - input.minimumTerminalRetentionMillis < limits.minimumTerminalRetentionMillis - ) { - return yield* protocolMismatch("configured relay retention horizon", "unsupported horizon") - } - if ( - payloadBytes > limits.maxActiveBytesPerSenderPeer || - payloadBytes > limits.maxActiveBytesPerRecipientPeer || - payloadBytes > limits.maxActiveBytesPerRecipientSubject || - payloadBytes > limits.maxActiveBytesPerTenant || - payloadBytes > limits.maxActiveBytesPerShard - ) { - return yield* quotaExceeded("relay payload bytes", limits.maxActiveBytesPerShard) - } - return yield* write(Effect.gen(function*() { - const now = (yield* nowQuery(sql)).now - const existing = yield* findDuplicate({ - tenantId: input.channel.tenantId, - senderSubjectId: input.channel.senderSubjectId, - senderPeerId: input.channel.senderPeerId, - relayMessageId: input.relayMessageId - }) - if (Option.isSome(existing)) { - if (existing.value.outerEnvelopeDigest !== input.outerEnvelopeDigest) { - return yield* protocolMismatch( - existing.value.outerEnvelopeDigest, - input.outerEnvelopeDigest - ) - } - return AdmissionResult.make({ - status: "Duplicate", - channel: ChannelKey.make({ - tenantId: existing.value.tenantId, - senderSubjectId: existing.value.senderSubjectId, - senderPeerId: existing.value.senderPeerId, - senderReplicaIncarnation: existing.value.senderReplicaIncarnation, - recipientSubjectId: existing.value.recipientSubjectId, - recipientPeerId: existing.value.recipientPeerId - }), - ready: existing.value.state === "Pending" && existing.value.nextEligibleAt <= now, - nextEligibleAt: existing.value.nextEligibleAt, - lane: "New" - }) - } - yield* insertChannel(input.channel) - const channel = yield* findChannel(input.channel) - const scopes = channelScopes(input.channel, limits) - for (const scope of scopes) { - yield* reserveUsage(scope, payloadBytes) - } - const duplicateHorizon = Math.max( - input.messageTtlMillis, - input.senderRetryHorizonMillis - ) - const insertMessage = sql`INSERT INTO effect_local_relay_messages ( - channel_id, - channel_sequence, - tenant_id, - sender_subject_id, - sender_peer_id, - recipient_subject_id, - recipient_peer_id, - relay_message_id, - relay_peer_id, - sender_connection_epoch, - sender_sequence, - document_ids, - payload_version, - message_hash, - outer_envelope_digest, - payload, - payload_length, - state, - created_at, - expires_at, - deduplicate_until, - next_eligible_at - ) VALUES ( - ${channel.channelId}, - ${channel.nextSequence}, - ${input.channel.tenantId}, - ${input.channel.senderSubjectId}, - ${input.channel.senderPeerId}, - ${input.channel.recipientSubjectId}, - ${input.channel.recipientPeerId}, - ${input.relayMessageId}, - ${input.relayPeerId}, - ${input.senderConnectionEpoch}, - ${input.senderSequence}, - ${JSON.stringify(input.documentIds)}, - ${input.payloadVersion}, - ${input.messageHash}, - ${input.outerEnvelopeDigest}, - ${input.payload}, - ${payloadBytes}, - 'Pending', - ${now}, - ${now + input.messageTtlMillis}, - ${now + duplicateHorizon}, - ${now} - )` - const inserted = yield* sql.onDialectOrElse({ - mysql: () => - insertMessage.raw.pipe( - Effect.flatMap((result) => - Schema.decodeUnknownEffect( - Schema.Struct({ insertId: PositiveInt }) - )(result) - ), - Effect.map((result) => MessageIdRow.make({ messageId: result.insertId })) - ), - orElse: () => - SqlSchema.findOne({ - Request: Schema.Void, - Result: MessageIdRow, - execute: () => sql`${insertMessage} RETURNING message_id AS "messageId"` - })(undefined) - }) - yield* sql`INSERT INTO effect_local_relay_reservations ( - message_id, - sender_peer_usage_key, - recipient_peer_usage_key, - recipient_subject_usage_key, - tenant_usage_key, - shard_usage_key, - active_count_delta, - active_bytes_delta, - retained_count_delta, - retained_bytes_delta - ) VALUES ( - ${inserted.messageId}, - ${scopes[0]!.key}, - ${scopes[1]!.key}, - ${scopes[2]!.key}, - ${scopes[3]!.key}, - ${scopes[4]!.key}, - 1, - ${payloadBytes}, - 1, - ${payloadBytes} - )` - yield* sql`UPDATE effect_local_relay_channels - SET next_sequence = next_sequence + 1 - WHERE channel_id = ${channel.channelId} - AND next_sequence = ${channel.nextSequence}` - return AdmissionResult.make({ - status: "Accepted", - channel: input.channel, - ready: true, - nextEligibleAt: now, - lane: "New" - }) - })) - }).pipe( - Effect.catchCause((cause) => - Effect.failCause(Cause.map(cause, (error) => { - switch (error._tag) { - case "SqlError": - case "NestedPeerRelayTransactionError": - return new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageUnavailable({ cause: error }) - }) - case "SchemaError": - case "NoSuchElementError": - return new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ cause: error }) - }) - default: - return error - } - })) - ), - Effect.tapError(recordQuotaRejection) - ) - return PeerRpcObservability.observeRelay({ - effect, - operation: "RelayAdmit", - direction: "Send", - facts: () => ({ - ...(observedBytes === undefined ? {} : { bytes: observedBytes }), - ...(observedVersion === undefined ? {} : { version: observedVersion }), - items: observedBytes === undefined ? 0 : 1 - }), - result: (exit) => - Exit.isSuccess(exit) - ? exit.value.status - : relayFailureResult(exit) - }) - } - - const claimMessages = sql.onDialectOrElse({ - sqlite: () => - sql`effect_local_relay_messages m - INDEXED BY effect_local_relay_messages_claim_admission`, - orElse: () => sql`effect_local_relay_messages m` - }) - - const findCandidate = SqlSchema.findOneOption({ - Request: Schema.Struct({ - tenantId: Schema.String, - recipientSubjectId: Schema.String, - recipientPeerId: Schema.String, - senderSubjectId: Schema.String, - senderPeerId: Schema.String, - authorizedDocumentIds: Schema.String, - now: NonNegativeInt - }), - Result: CandidateRow, - execute: (request) => { - const unauthorizedDocument = sql.onDialectOrElse({ - pg: () => - sql`SELECT 1 - FROM jsonb_array_elements_text(CAST(m.document_ids AS JSONB)) document(value) - WHERE document.value NOT IN ( - SELECT value - FROM jsonb_array_elements_text(CAST(${request.authorizedDocumentIds} AS JSONB)) authorized(value) - )`, - mysql: () => - sql`SELECT 1 - FROM JSON_TABLE( - m.document_ids, - '$[*]' COLUMNS(value VARCHAR(256) PATH '$') - ) document - WHERE document.value NOT IN ( - SELECT authorized.value - FROM JSON_TABLE( - CAST(${request.authorizedDocumentIds} AS JSON), - '$[*]' COLUMNS(value VARCHAR(256) PATH '$') - ) authorized - )`, - orElse: () => - sql`SELECT 1 - FROM json_each(m.document_ids) document - WHERE document.value NOT IN ( - SELECT value FROM json_each(${request.authorizedDocumentIds}) - )` - }) - return sql`SELECT - m.message_id AS "messageId", - m.channel_id AS "channelId", - c.tenant_id AS "tenantId", - c.sender_subject_id AS "senderSubjectId", - c.sender_peer_id AS "senderPeerId", - c.sender_replica_incarnation AS "senderReplicaIncarnation", - c.recipient_subject_id AS "recipientSubjectId", - c.recipient_peer_id AS "recipientPeerId", - m.relay_message_id AS "relayMessageId", - m.relay_peer_id AS "relayPeerId", - m.sender_connection_epoch AS "senderConnectionEpoch", - m.sender_sequence AS "senderSequence", - m.document_ids AS "documentIds", - m.payload_version AS "payloadVersion", - m.message_hash AS "messageHash", - m.outer_envelope_digest AS "outerEnvelopeDigest", - m.payload_length AS "payloadBytes", - m.created_at AS "createdAt", - m.next_eligible_at AS "nextEligibleAt", - m.retry_count AS "retryCount" - FROM ${claimMessages} - JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id - JOIN effect_local_relay_reservations r ON r.message_id = m.message_id - WHERE m.tenant_id = ${request.tenantId} - AND m.sender_subject_id = ${request.senderSubjectId} - AND m.sender_peer_id = ${request.senderPeerId} - AND m.recipient_subject_id = ${request.recipientSubjectId} - AND m.recipient_peer_id = ${request.recipientPeerId} - AND c.tenant_id = ${request.tenantId} - AND c.recipient_subject_id = ${request.recipientSubjectId} - AND c.recipient_peer_id = ${request.recipientPeerId} - AND c.sender_subject_id = ${request.senderSubjectId} - AND c.sender_peer_id = ${request.senderPeerId} - AND m.tenant_id = c.tenant_id - AND m.sender_subject_id = c.sender_subject_id - AND m.sender_peer_id = c.sender_peer_id - AND m.recipient_subject_id = c.recipient_subject_id - AND m.recipient_peer_id = c.recipient_peer_id - AND c.claimed_message_id IS NULL - AND m.state = 'Pending' - AND m.next_eligible_at <= ${request.now} - AND m.expires_at > ${request.now} - AND m.payload IS NOT NULL - AND m.payload_length = length(m.payload) - AND r.active_consumed = 0 - AND NOT EXISTS ( - SELECT 1 - FROM effect_local_relay_messages earlier - WHERE earlier.channel_id = m.channel_id - AND earlier.channel_sequence < m.channel_sequence - AND earlier.state IN ('Pending', 'Claimed') - ) - AND NOT EXISTS ( - ${unauthorizedDocument} - ) - ORDER BY m.created_at, m.message_id - LIMIT 1` - } - }) - - const claim: Service["claim"] = (unsafeInput) => { - let observedAttempt: number | undefined - const effect = Effect.gen(function*() { - const input = yield* validateInput(ClaimRequest, unsafeInput) - return yield* write(Effect.gen(function*() { - const now = (yield* nowQuery(sql)).now - const candidateOption = yield* findCandidate({ - tenantId: input.recipient.tenantId, - recipientSubjectId: input.recipient.subjectId, - recipientPeerId: input.recipient.peerId, - senderSubjectId: input.sender.subjectId, - senderPeerId: input.sender.peerId, - authorizedDocumentIds: JSON.stringify(input.authorizedDocumentIds), - now - }) - if (Option.isNone(candidateOption)) { - return { - message: Option.none(), - ready: false, - nextEligibleAt: Option.none(), - lane: "New" as const - } - } - const candidate = candidateOption.value - observedAttempt = candidate.retryCount + 1 - const documentIds = yield* parseDocuments(candidate.documentIds) - const uuid = yield* crypto.randomUUIDv4 - const token = PeerRpc.ClaimToken.make(`clm_${uuid}`) - const deadline = now + limits.claimLeaseMillis - const claimed = yield* mutationCount( - sql`UPDATE effect_local_relay_messages - SET state = 'Claimed', - claim_token = ${token}, - claim_session_generation = ${input.sessionGeneration}, - claim_deadline = ${deadline} - WHERE message_id = ${candidate.messageId} - AND state = 'Pending'`.raw, - SqlSchema.findAll({ - Request: Schema.Void, - Result: UnitRow, - execute: () => - sql`UPDATE effect_local_relay_messages - SET state = 'Claimed', - claim_token = ${token}, - claim_session_generation = ${input.sessionGeneration}, - claim_deadline = ${deadline} - WHERE message_id = ${candidate.messageId} - AND state = 'Pending' - RETURNING 1 AS value` - })(undefined) - ) - if (claimed !== 1) { - return yield* new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ - cause: new Error("Relay claim compare and swap failed") - }) - }) - } - const channelClaimed = yield* mutationCount( - sql`UPDATE effect_local_relay_channels - SET claimed_message_id = ${candidate.messageId}, - claim_session_generation = ${input.sessionGeneration}, - claim_token = ${token}, - claim_deadline = ${deadline} - WHERE channel_id = ${candidate.channelId} - AND claimed_message_id IS NULL`.raw, - SqlSchema.findAll({ - Request: Schema.Void, - Result: UnitRow, - execute: () => - sql`UPDATE effect_local_relay_channels - SET claimed_message_id = ${candidate.messageId}, - claim_session_generation = ${input.sessionGeneration}, - claim_token = ${token}, - claim_deadline = ${deadline} - WHERE channel_id = ${candidate.channelId} - AND claimed_message_id IS NULL - RETURNING 1 AS value` - })(undefined) - ) - if (channelClaimed !== 1) { - return yield* new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ - cause: new Error("Relay channel claim compare and swap failed") - }) - }) - } - return { - message: Option.some(ClaimedMessage.make({ - rowId: candidate.messageId, - channel: ChannelKey.make({ - tenantId: candidate.tenantId, - senderSubjectId: candidate.senderSubjectId, - senderPeerId: candidate.senderPeerId, - senderReplicaIncarnation: candidate.senderReplicaIncarnation, - recipientSubjectId: candidate.recipientSubjectId, - recipientPeerId: candidate.recipientPeerId - }), - relayMessageId: candidate.relayMessageId, - relayPeerId: candidate.relayPeerId, - senderConnectionEpoch: candidate.senderConnectionEpoch, - senderSequence: candidate.senderSequence, - documentIds, - payloadVersion: candidate.payloadVersion, - messageHash: candidate.messageHash, - outerEnvelopeDigest: candidate.outerEnvelopeDigest, - payloadBytes: candidate.payloadBytes, - claimToken: token, - claimDeadline: deadline, - sessionGeneration: input.sessionGeneration - })), - ready: false, - nextEligibleAt: Option.none(), - lane: candidate.retryCount === 0 ? "New" as const : "Retry" as const - } - })) - }).pipe(Effect.catchCause((cause) => - Effect.failCause(Cause.map(cause, (error) => { - switch (error._tag) { - case "SqlError": - case "PlatformError": - case "NestedPeerRelayTransactionError": - return new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageUnavailable({ cause: error }) - }) - case "SchemaError": - case "NoSuchElementError": - return new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ cause: error }) - }) - default: - return error - } - })) - )) - return PeerRpcObservability.observeRelay({ - effect, - operation: "RelayClaim", - direction: "Receive", - facts: (exit) => { - if (Exit.isFailure(exit) || Option.isNone(exit.value.message)) { - return { items: 0 } - } - const message = exit.value.message.value - return { - bytes: message.payloadBytes, - items: 1, - ...(observedAttempt === undefined ? {} : { attempt: observedAttempt }), - version: message.payloadVersion - } - }, - result: (exit) => - Exit.isSuccess(exit) - ? Option.isSome(exit.value.message) ? "Claimed" : "Empty" - : relayFailureResult(exit) - }) - } - - const loadClaimedPayload: Service["loadClaimedPayload"] = (unsafeInput) => - Effect.gen(function*() { - const input = yield* validateInput(LoadClaimedPayloadRequest, unsafeInput) - const now = (yield* nowQuery(sql)).now - const rows = yield* SqlSchema.findAll({ - Request: Schema.Void, - Result: Schema.Struct({ payload: Schema.Uint8Array }), - execute: () => - sql`SELECT m.payload - FROM effect_local_relay_messages m - JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id - WHERE m.message_id = ${input.rowId} - AND m.channel_id = c.channel_id - AND c.tenant_id = ${input.channel.tenantId} - AND c.sender_subject_id = ${input.channel.senderSubjectId} - AND c.sender_peer_id = ${input.channel.senderPeerId} - AND c.sender_replica_incarnation = ${input.channel.senderReplicaIncarnation} - AND c.recipient_subject_id = ${input.channel.recipientSubjectId} - AND c.recipient_peer_id = ${input.channel.recipientPeerId} - AND m.tenant_id = c.tenant_id - AND m.sender_subject_id = c.sender_subject_id - AND m.sender_peer_id = c.sender_peer_id - AND m.relay_message_id = ${input.relayMessageId} - AND m.state = 'Claimed' - AND m.claim_token = ${input.claimToken} - AND m.claim_session_generation = ${input.sessionGeneration} - AND c.claimed_message_id = m.message_id - AND c.claim_token = m.claim_token - AND c.claim_session_generation = m.claim_session_generation - AND c.claim_deadline = m.claim_deadline - AND m.claim_deadline > ${now} - AND m.expires_at > ${now} - AND m.payload_length = ${input.payloadBytes} - AND length(m.payload) = ${input.payloadBytes} - AND m.payload_length <= ${limits.maxActiveBytesPerShard}` - })(undefined) - if (rows.length !== 1) { - return yield* protocolMismatch("active relay claim", "stale relay claim") - } - return rows[0]!.payload - }).pipe(Effect.catchTags({ - SqlError: (cause) => - Effect.fail( - new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageUnavailable({ cause }) - }) - ), - SchemaError: (cause) => - Effect.fail( - new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ cause }) - }) - ), - NoSuchElementError: (cause) => - Effect.fail( - new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ cause }) - }) - ) - })) - - const TerminalRow = Schema.Struct({ - messageId: PositiveInt, - channelId: PositiveInt, - state: Schema.Literals(["Pending", "Claimed", "Acknowledged", "DeadLettered", "Expired"]), - messageHash: Schema.NonEmptyString, - claimToken: Schema.NullOr(PeerRpc.ClaimToken), - claimSessionGeneration: Schema.NullOr(NonNegativeInt), - claimDeadline: Schema.NullOr(NonNegativeInt), - createdAt: NonNegativeInt, - expiresAt: NonNegativeInt, - channelClaimedMessageId: Schema.NullOr(PositiveInt), - channelClaimToken: Schema.NullOr(PeerRpc.ClaimToken), - channelClaimSessionGeneration: Schema.NullOr(NonNegativeInt), - channelClaimDeadline: Schema.NullOr(NonNegativeInt), - terminalClaimToken: Schema.NullOr(PeerRpc.ClaimToken), - terminalSessionGeneration: Schema.NullOr(NonNegativeInt), - terminalReason: Schema.NullOr(Schema.String) - }) - - const findTerminal = SqlSchema.findOneOption({ - Request: Schema.Struct({ - channel: ChannelKey, - relayMessageId: Identity.RelayMessageId - }), - Result: TerminalRow, - execute: ({ channel, relayMessageId }) => - sql`SELECT - m.message_id AS "messageId", - m.channel_id AS "channelId", - m.state, - m.message_hash AS "messageHash", - m.claim_token AS "claimToken", - m.claim_session_generation AS "claimSessionGeneration", - m.claim_deadline AS "claimDeadline", - m.created_at AS "createdAt", - m.expires_at AS "expiresAt", - c.claimed_message_id AS "channelClaimedMessageId", - c.claim_token AS "channelClaimToken", - c.claim_session_generation AS "channelClaimSessionGeneration", - c.claim_deadline AS "channelClaimDeadline", - m.terminal_claim_token AS "terminalClaimToken", - m.terminal_session_generation AS "terminalSessionGeneration", - m.terminal_reason AS "terminalReason" - FROM effect_local_relay_messages m - JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id - WHERE c.tenant_id = ${channel.tenantId} - AND c.sender_subject_id = ${channel.senderSubjectId} - AND c.sender_peer_id = ${channel.senderPeerId} - AND c.sender_replica_incarnation = ${channel.senderReplicaIncarnation} - AND c.recipient_subject_id = ${channel.recipientSubjectId} - AND c.recipient_peer_id = ${channel.recipientPeerId} - AND m.relay_message_id = ${relayMessageId}` - }) - - const ReadyRow = Schema.Struct({ - nextEligibleAt: NonNegativeInt, - retryCount: NonNegativeInt - }) - - const nextReady = ( - channel: ChannelKey, - now: number - ): Effect.Effect< - { readonly ready: boolean; readonly nextEligibleAt: Option.Option; readonly lane: "New" | "Retry" }, - SqlError.SqlError | Schema.SchemaError - > => - SqlSchema.findOneOption({ - Request: Schema.Void, - Result: ReadyRow, - execute: () => - sql`SELECT - m.next_eligible_at AS "nextEligibleAt", - m.retry_count AS "retryCount" - FROM effect_local_relay_messages m - JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id - WHERE c.tenant_id = ${channel.tenantId} - AND c.sender_subject_id = ${channel.senderSubjectId} - AND c.sender_peer_id = ${channel.senderPeerId} - AND c.recipient_subject_id = ${channel.recipientSubjectId} - AND c.recipient_peer_id = ${channel.recipientPeerId} - AND c.claimed_message_id IS NULL - AND m.state = 'Pending' - AND m.expires_at > ${now} - AND NOT EXISTS ( - SELECT 1 - FROM effect_local_relay_messages earlier - WHERE earlier.channel_id = m.channel_id - AND earlier.channel_sequence < m.channel_sequence - AND earlier.state IN ('Pending', 'Claimed') - ) - ORDER BY m.next_eligible_at, m.created_at, m.message_id - LIMIT 1` - })(undefined).pipe( - Effect.map((row) => - Option.isNone(row) - ? { - ready: false, - nextEligibleAt: Option.none(), - lane: "New" as const - } - : { - ready: row.value.nextEligibleAt <= now, - nextEligibleAt: Option.some(row.value.nextEligibleAt), - lane: row.value.retryCount === 0 ? "New" as const : "Retry" as const - } - ) - ) - - const terminalTransition = ( - unsafeInput: TerminalRequest, - state: "Acknowledged" | "DeadLettered", - reason: string, - onChanged?: (latencyMillis: number) => void - ): Effect.Effect => - Effect.gen(function*() { - const input = yield* validateInput(TerminalRequest, unsafeInput) - if ( - input.recipient.tenantId !== input.channel.tenantId || - input.recipient.subjectId !== input.channel.recipientSubjectId || - input.recipient.peerId !== input.channel.recipientPeerId - ) { - return { - status: "Stale", - ready: false, - nextEligibleAt: Option.none(), - lane: "New" - } as const - } - return yield* write(Effect.gen(function*() { - const now = (yield* nowQuery(sql)).now - const rowOption = yield* findTerminal({ - channel: input.channel, - relayMessageId: input.relayMessageId - }) - if (Option.isNone(rowOption)) { - return { - status: "Stale", - ready: false, - nextEligibleAt: Option.none(), - lane: "New" - } as const - } - const row = rowOption.value - const active = row.state === "Claimed" && - row.messageHash === input.messageHash && - row.claimToken === input.claimToken && - row.claimSessionGeneration === input.sessionGeneration && - row.claimDeadline !== null && - row.claimDeadline > now && - row.expiresAt > now && - row.channelClaimedMessageId === row.messageId && - row.channelClaimToken === row.claimToken && - row.channelClaimSessionGeneration === row.claimSessionGeneration && - row.channelClaimDeadline === row.claimDeadline - if (active) { - yield* terminalize(row.messageId, state, now, { - token: input.claimToken, - sessionGeneration: input.sessionGeneration, - reason - }) - if (onChanged !== undefined) { - yield* Effect.sync(() => onChanged(now - row.createdAt)) - } - const hint = yield* nextReady(input.channel, now) - return { status: "Changed", ...hint } as const - } - const duplicate = row.state === state && - row.messageHash === input.messageHash && - row.terminalClaimToken === input.claimToken && - row.terminalSessionGeneration === input.sessionGeneration && - row.terminalReason === reason - if (duplicate) { - const hint = yield* nextReady(input.channel, now) - return { status: "Duplicate", ...hint } as const - } - return { - status: "Stale", - ready: false, - nextEligibleAt: Option.none(), - lane: "New" - } as const - })) - }).pipe(Effect.catchCause((cause) => - Effect.failCause(Cause.map(cause, (error) => { - switch (error._tag) { - case "SqlError": - case "NestedPeerRelayTransactionError": - return new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageUnavailable({ cause: error }) - }) - case "SchemaError": - case "NoSuchElementError": - return new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ cause: error }) - }) - default: - return error - } - })) - )) - - const acknowledge: Service["acknowledge"] = (input) => { - let latencyMillis: number | undefined - const effect = terminalTransition( - input, - "Acknowledged", - "Acknowledged", - (latency) => { - latencyMillis = latency - } - ) - return PeerRpcObservability.observeRelay({ - effect, - operation: "RelayAcknowledge", - direction: "Receive", - facts: (exit) => ({ - items: Exit.isSuccess(exit) && exit.value.status !== "Stale" ? 1 : 0, - ...(Exit.isSuccess(exit) && - exit.value.status === "Changed" && - latencyMillis !== undefined - ? { latencyMillis } - : {}) - }), - result: (exit) => - Exit.isSuccess(exit) - ? exit.value.status === "Changed" - ? "Acknowledged" - : exit.value.status - : relayFailureResult(exit) - }) - } - - const reject: Service["reject"] = (unsafeInput) => { - const effect = Effect.gen(function*() { - const input = yield* validateInput(RejectRequest, unsafeInput) - return yield* terminalTransition( - TerminalRequest.make({ - channel: input.channel, - relayMessageId: input.relayMessageId, - claimToken: input.claimToken, - messageHash: input.messageHash, - sessionGeneration: input.sessionGeneration, - recipient: input.recipient - }), - "DeadLettered", - input.reason - ) - }) - return PeerRpcObservability.observeRelay({ - effect, - operation: "RelayAcknowledge", - direction: "Receive", - facts: (exit) => ({ - items: Exit.isSuccess(exit) && exit.value.status !== "Stale" ? 1 : 0 - }), - result: (exit) => - Exit.isSuccess(exit) - ? exit.value.status === "Changed" - ? "DeadLettered" - : exit.value.status - : relayFailureResult(exit) - }) - } - - const release: Service["release"] = (unsafeInput) => { - const effect = Effect.gen(function*() { - const input = yield* validateInput(ReleaseRequest, unsafeInput) - return yield* write(Effect.gen(function*() { - const now = (yield* nowQuery(sql)).now - const rowOption = yield* findTerminal({ - channel: input.channel, - relayMessageId: input.relayMessageId - }) - if (Option.isNone(rowOption)) { - return { - status: "Stale", - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" - } as const - } - const row = rowOption.value - if ( - row.state !== "Claimed" || - row.claimToken !== input.claimToken || - row.claimSessionGeneration !== input.sessionGeneration - ) { - return { - status: "Stale", - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" - } as const - } - const retry = yield* SqlSchema.findOne({ - Request: Schema.Void, - Result: Schema.Struct({ retryCount: PositiveInt }), - execute: () => - sql`SELECT retry_count + 1 AS "retryCount" - FROM effect_local_relay_messages - WHERE message_id = ${row.messageId}` - })(undefined) - if (retry.retryCount >= limits.maximumDeliveryAttempts) { - yield* sql`UPDATE effect_local_relay_messages - SET retry_count = ${retry.retryCount} - WHERE message_id = ${row.messageId} - AND state = 'Claimed' - AND claim_token = ${input.claimToken} - AND claim_session_generation = ${input.sessionGeneration}` - yield* terminalize(row.messageId, "DeadLettered", now, { - reason: "MaximumDeliveryAttempts" - }) - return { - status: "Changed", - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" - } as const - } - const random = yield* Random.next - const maximum = Math.min( - limits.retryMaximumDelayMillis, - limits.retryBaseDelayMillis * 2 ** Math.min(retry.retryCount - 1, 30) - ) - const delay = Math.max(1, Math.floor(maximum / 2 + random * maximum / 2)) - const nextEligibleAt = now + delay - yield* sql`UPDATE effect_local_relay_messages - SET state = 'Pending', - retry_count = ${retry.retryCount}, - next_eligible_at = ${nextEligibleAt}, - claim_token = NULL, - claim_session_generation = NULL, - claim_deadline = NULL - WHERE message_id = ${row.messageId} - AND state = 'Claimed' - AND claim_token = ${input.claimToken} - AND claim_session_generation = ${input.sessionGeneration}` - yield* sql`UPDATE effect_local_relay_channels - SET claimed_message_id = NULL, - claim_session_generation = NULL, - claim_token = NULL, - claim_deadline = NULL - WHERE channel_id = ${row.channelId} - AND claimed_message_id = ${row.messageId} - AND claim_token = ${input.claimToken} - AND claim_session_generation = ${input.sessionGeneration}` - return { - status: "Changed", - ready: false, - nextEligibleAt: Option.some(nextEligibleAt), - lane: "Retry" - } as const - })) - }).pipe(Effect.catchCause((cause) => - Effect.failCause(Cause.map(cause, (error) => { - switch (error._tag) { - case "SqlError": - case "NestedPeerRelayTransactionError": - return new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageUnavailable({ cause: error }) - }) - case "SchemaError": - case "NoSuchElementError": - return new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ cause: error }) - }) - default: - return error - } - })) - )) - return PeerRpcObservability.observeRelay({ - effect, - operation: "RelayRelease", - direction: "Receive", - facts: (exit) => ({ - items: Exit.isSuccess(exit) && exit.value.status === "Changed" ? 1 : 0 - }), - result: (exit) => - Exit.isSuccess(exit) - ? exit.value.status === "Changed" ? "Released" : exit.value.status - : relayFailureResult(exit) - }) - } - - const maintenanceResult = ( - ids: ReadonlyArray, - requestedBatchSize: number - ): MaintenanceResult => { - const processedIds = ids.slice(0, requestedBatchSize) - const cursor = processedIds.at(-1) - return cursor === undefined - ? { processed: 0, hasMore: false } - : { - cursor, - processed: processedIds.length, - hasMore: ids.length > requestedBatchSize - } - } - - const recover: Service["recover"] = (unsafeInput) => - Effect.suspend(() => { - let deadLettered = 0 - const effect = Effect.gen(function*() { - const input = yield* validateInput(MaintenanceRequest, unsafeInput) - const effectiveBatch = Math.min(input.batchSize, limits.claimRecoveryBatchSize) - return yield* write(Effect.gen(function*() { - const now = (yield* nowQuery(sql)).now - const rows = yield* SqlSchema.findAll({ - Request: Schema.Void, - Result: Schema.Struct({ - messageId: PositiveInt, - retryCount: NonNegativeInt - }), - execute: () => - sql`SELECT - message_id AS "messageId", - retry_count AS "retryCount" - FROM effect_local_relay_messages - WHERE state = 'Claimed' - AND claim_deadline <= ${now} - ORDER BY claim_deadline, message_id - LIMIT ${sql.literal(String(effectiveBatch + 1))}` - })(undefined) - for (const row of rows.slice(0, effectiveBatch)) { - const retryCount = row.retryCount + 1 - if (retryCount >= limits.maximumDeliveryAttempts) { - deadLettered += 1 - yield* sql`UPDATE effect_local_relay_messages - SET retry_count = ${retryCount} - WHERE message_id = ${row.messageId} - AND state = 'Claimed' - AND claim_deadline <= ${now}` - yield* terminalize(row.messageId, "DeadLettered", now, { - reason: "MaximumDeliveryAttempts" - }) - continue - } - const random = yield* Random.next - const maximum = Math.min( - limits.retryMaximumDelayMillis, - limits.retryBaseDelayMillis * 2 ** Math.min(retryCount - 1, 30) - ) - const delay = Math.max(1, Math.floor(maximum / 2 + random * maximum / 2)) - yield* sql`UPDATE effect_local_relay_messages - SET state = 'Pending', - retry_count = ${retryCount}, - next_eligible_at = ${now + delay}, - claim_token = NULL, - claim_session_generation = NULL, - claim_deadline = NULL - WHERE message_id = ${row.messageId} - AND state = 'Claimed' - AND claim_deadline <= ${now}` - yield* sql`UPDATE effect_local_relay_channels - SET claimed_message_id = NULL, - claim_session_generation = NULL, - claim_token = NULL, - claim_deadline = NULL - WHERE claimed_message_id = ${row.messageId} - AND claim_deadline <= ${now}` - } - return maintenanceResult(rows.map((row) => row.messageId), effectiveBatch) - })) - }).pipe(Effect.catchCause((cause) => - Effect.failCause(Cause.map(cause, (error) => { - switch (error._tag) { - case "SqlError": - case "NestedPeerRelayTransactionError": - return new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageUnavailable({ cause: error }) - }) - case "SchemaError": - case "NoSuchElementError": - return new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ cause: error }) - }) - default: - return error - } - })) - )) - return PeerRpcObservability.observeRelay({ - effect, - operation: "RelayMaintenance", - direction: "Receive", - stage: "Recover", - facts: (exit) => ({ - items: Exit.isSuccess(exit) ? exit.value.processed : 0 - }), - result: (exit) => - Exit.isSuccess(exit) - ? deadLettered > 0 ? "DeadLettered" : "Released" - : relayFailureResult(exit) - }) - }) - - const expire: Service["expire"] = (unsafeInput) => { - const effect = Effect.gen(function*() { - const input = yield* validateInput(MaintenanceRequest, unsafeInput) - const effectiveBatch = Math.min(input.batchSize, limits.expiryBatchSize) - return yield* write(Effect.gen(function*() { - const now = (yield* nowQuery(sql)).now - const rows = yield* SqlSchema.findAll({ - Request: Schema.Void, - Result: KeyRow, - execute: () => - sql`SELECT message_id AS "messageId" - FROM effect_local_relay_messages - WHERE state IN ('Pending', 'Claimed') - AND expires_at <= ${now} - ORDER BY expires_at, message_id - LIMIT ${sql.literal(String(effectiveBatch + 1))}` - })(undefined) - for (const row of rows.slice(0, effectiveBatch)) { - yield* terminalize(row.messageId, "Expired", now, { reason: "Expired" }) - } - return maintenanceResult(rows.map((row) => row.messageId), effectiveBatch) - })) - }).pipe(Effect.catchCause((cause) => - Effect.failCause(Cause.map(cause, (error) => { - switch (error._tag) { - case "SqlError": - case "NestedPeerRelayTransactionError": - return new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageUnavailable({ cause: error }) - }) - case "SchemaError": - case "NoSuchElementError": - return new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ cause: error }) - }) - default: - return error - } - })) - )) - return PeerRpcObservability.observeRelay({ - effect, - operation: "RelayMaintenance", - direction: "Receive", - stage: "Expire", - facts: (exit) => ({ - items: Exit.isSuccess(exit) ? exit.value.processed : 0 - }), - result: (exit) => Exit.isSuccess(exit) ? "Expired" : relayFailureResult(exit) - }) - } - - const IntegrityRow = Schema.Struct({ - messageId: PositiveInt, - state: Schema.String, - messageTenantId: Schema.NonEmptyString, - channelTenantId: Schema.NullOr(Schema.NonEmptyString), - messageSenderSubjectId: Schema.NonEmptyString, - channelSenderSubjectId: Schema.NullOr(Schema.NonEmptyString), - messageSenderPeerId: Schema.String, - channelSenderPeerId: Schema.NullOr(Schema.String), - messageRecipientSubjectId: Schema.NullOr(Schema.String), - channelRecipientSubjectId: Schema.NullOr(Schema.String), - messageRecipientPeerId: Schema.NullOr(Schema.String), - channelRecipientPeerId: Schema.NullOr(Schema.String), - payloadLength: NonNegativeInt, - actualLength: Schema.NullOr(NonNegativeInt), - reservationPresent: Schema.Literals([0, 1]), - activeConsumed: Schema.NullOr(Schema.Literals([0, 1])), - retainedConsumed: Schema.NullOr(Schema.Literals([0, 1])) - }) - - const repair: Service["repair"] = (unsafeInput) => { - const effect = Effect.gen(function*() { - const input = yield* validateInput(MaintenanceRequest, unsafeInput) - const effectiveBatch = Math.min(input.batchSize, limits.integrityBatchSize) - return yield* write(Effect.gen(function*() { - const now = (yield* nowQuery(sql)).now - const rows = yield* SqlSchema.findAll({ - Request: Schema.Void, - Result: IntegrityRow, - execute: () => - sql`SELECT - m.message_id AS "messageId", - m.state, - m.tenant_id AS "messageTenantId", - c.tenant_id AS "channelTenantId", - m.sender_subject_id AS "messageSenderSubjectId", - c.sender_subject_id AS "channelSenderSubjectId", - m.sender_peer_id AS "messageSenderPeerId", - c.sender_peer_id AS "channelSenderPeerId", - m.recipient_subject_id AS "messageRecipientSubjectId", - c.recipient_subject_id AS "channelRecipientSubjectId", - m.recipient_peer_id AS "messageRecipientPeerId", - c.recipient_peer_id AS "channelRecipientPeerId", - m.payload_length AS "payloadLength", - length(m.payload) AS "actualLength", - CASE WHEN r.message_id IS NULL THEN 0 ELSE 1 END AS "reservationPresent", - r.active_consumed AS "activeConsumed", - r.retained_consumed AS "retainedConsumed" - FROM effect_local_relay_messages m - LEFT JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id - LEFT JOIN effect_local_relay_reservations r ON r.message_id = m.message_id - WHERE m.message_id > ${input.cursor ?? 0} - ORDER BY m.message_id - LIMIT ${sql.literal(String(effectiveBatch + 1))}` - })(undefined) - for (const row of rows.slice(0, effectiveBatch)) { - if ( - row.reservationPresent === 0 || - row.activeConsumed === null || - row.retainedConsumed === null - ) { - return yield* new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ - cause: new Error("Relay reservation reconciliation required") - }) - }) - } - const active = row.state === "Pending" || row.state === "Claimed" - const terminal = row.state === "Acknowledged" || - row.state === "DeadLettered" || - row.state === "Expired" - if ( - (active && row.activeConsumed !== 0) || - (terminal && row.activeConsumed !== 1) - ) { - return yield* new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ - cause: new Error("Corrupt relay reservation entitlement") - }) - }) - } - const corrupt = (!active && !terminal) || - row.messageTenantId !== row.channelTenantId || - row.messageSenderSubjectId !== row.channelSenderSubjectId || - row.messageSenderPeerId !== row.channelSenderPeerId || - row.messageRecipientSubjectId !== row.channelRecipientSubjectId || - row.messageRecipientPeerId !== row.channelRecipientPeerId || - (active && ( - row.actualLength === null || - row.actualLength !== row.payloadLength - )) || - (terminal && ( - row.actualLength !== null || - row.payloadLength !== 0 - )) - if (corrupt && active) { - yield* terminalize(row.messageId, "DeadLettered", now, { reason: "Corrupt" }) - } else if (corrupt) { - return yield* new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ - cause: new Error("Corrupt terminal relay row") - }) - }) - } - } - return maintenanceResult(rows.map((row) => row.messageId), effectiveBatch) - })) - }).pipe(Effect.catchCause((cause) => - Effect.failCause(Cause.map(cause, (error) => { - switch (error._tag) { - case "SqlError": - case "NestedPeerRelayTransactionError": - return new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageUnavailable({ cause: error }) - }) - case "SchemaError": - case "NoSuchElementError": - return new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ cause: error }) - }) - default: - return error - } - })) - )) - return PeerRpcObservability.observeRelay({ - effect, - operation: "RelayMaintenance", - direction: "Receive", - stage: "Repair", - facts: (exit) => ({ - items: Exit.isSuccess(exit) ? exit.value.processed : 0 - }), - result: (exit) => Exit.isSuccess(exit) ? "DeadLettered" : relayFailureResult(exit) - }) - } - - const reconcile: Service["reconcile"] = (unsafeInput) => { - const effect = Effect.gen(function*() { - const input = yield* validateInput(MaintenanceRequest, unsafeInput) - const effectiveBatch = Math.min(input.batchSize, limits.reconciliationBatchSize) - return yield* write(Effect.gen(function*() { - const messageRows = yield* SqlSchema.findAll({ - Request: Schema.Void, - Result: KeyRow, - execute: () => - sql`SELECT message_id AS "messageId" - FROM effect_local_relay_messages - WHERE message_id > ${input.cursor ?? 0} - ORDER BY message_id - LIMIT ${sql.literal(String(effectiveBatch + 1))}` - })(undefined) - const reservationRows = yield* SqlSchema.findAll({ - Request: Schema.Void, - Result: KeyRow, - execute: () => - sql`SELECT message_id AS "messageId" - FROM effect_local_relay_reservations - WHERE message_id > ${input.cursor ?? 0} - ORDER BY message_id - LIMIT ${sql.literal(String(effectiveBatch + 1))}` - })(undefined) - if ( - messageRows.length !== reservationRows.length || - messageRows.some((row, index) => row.messageId !== reservationRows[index]?.messageId) - ) { - return yield* new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ - cause: new Error("Relay reservation bijection is corrupt") - }) - }) - } - return maintenanceResult(messageRows.map((row) => row.messageId), effectiveBatch) - })) - }).pipe(Effect.catchCause((cause) => - Effect.failCause(Cause.map(cause, (error) => { - switch (error._tag) { - case "SqlError": - case "NestedPeerRelayTransactionError": - return new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageUnavailable({ cause: error }) - }) - case "SchemaError": - return new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ cause: error }) - }) - default: - return error - } - })) - )) - return PeerRpcObservability.observeRelay({ - effect, - operation: "RelayMaintenance", - direction: "Receive", - stage: "Reconcile", - facts: (exit) => ({ - items: Exit.isSuccess(exit) ? exit.value.processed : 0 - }), - result: (exit) => Exit.isSuccess(exit) ? "Success" : relayFailureResult(exit) - }) - } - - const collect: Service["collect"] = (unsafeInput) => { - const effect = Effect.gen(function*() { - const input = yield* validateInput(MaintenanceRequest, unsafeInput) - const effectiveBatch = Math.min(input.batchSize, limits.terminalCollectionBatchSize) - return yield* write(Effect.gen(function*() { - const now = (yield* nowQuery(sql)).now - const rows = yield* SqlSchema.findAll({ - Request: Schema.Void, - Result: Schema.Struct({ - messageId: PositiveInt, - channelId: PositiveInt - }), - execute: () => - sql`SELECT - message_id AS "messageId", - channel_id AS "channelId" - FROM effect_local_relay_messages - WHERE state IN ('Acknowledged', 'DeadLettered', 'Expired') - AND deduplicate_until <= ${now} - ORDER BY deduplicate_until, message_id - LIMIT ${sql.literal(String(effectiveBatch + 1))}` - })(undefined) - for (const row of rows.slice(0, effectiveBatch)) { - yield* releaseRetainedUsage(row.messageId) - const reservationDeleted = yield* mutationCount( - sql`DELETE FROM effect_local_relay_reservations - WHERE message_id = ${row.messageId} - AND active_consumed = 1 - AND retained_consumed = 1`.raw, - SqlSchema.findAll({ - Request: Schema.Void, - Result: UnitRow, - execute: () => - sql`DELETE FROM effect_local_relay_reservations - WHERE message_id = ${row.messageId} - AND active_consumed = 1 - AND retained_consumed = 1 - RETURNING 1 AS value` - })(undefined) - ) - if (reservationDeleted !== 1) { - return yield* new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ - cause: new Error("Invalid collected relay quota reservation") - }) - }) - } - yield* sql`DELETE FROM effect_local_relay_messages - WHERE message_id = ${row.messageId} - AND state IN ('Acknowledged', 'DeadLettered', 'Expired') - AND deduplicate_until <= ${now}` - yield* sql`DELETE FROM effect_local_relay_channels - WHERE channel_id = ${row.channelId} - AND claimed_message_id IS NULL - AND NOT EXISTS ( - SELECT 1 - FROM effect_local_relay_messages m - WHERE m.channel_id = ${row.channelId} - )` - } - return maintenanceResult(rows.map((row) => row.messageId), effectiveBatch) - })) - }).pipe(Effect.catchCause((cause) => - Effect.failCause(Cause.map(cause, (error) => { - switch (error._tag) { - case "SqlError": - case "NestedPeerRelayTransactionError": - return new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageUnavailable({ cause: error }) - }) - case "SchemaError": - case "NoSuchElementError": - return new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ cause: error }) - }) - default: - return error - } - })) - )) - return PeerRpcObservability.observeRelay({ - effect, - operation: "RelayMaintenance", - direction: "Receive", - stage: "Collect", - facts: (exit) => ({ - items: Exit.isSuccess(exit) ? exit.value.processed : 0 - }), - result: (exit) => Exit.isSuccess(exit) ? "Success" : relayFailureResult(exit) - }) - } - - const usage: Service["usage"] = (unsafeInput) => { - const exactShard = unsafeInput === undefined - const effect = Effect.gen(function*() { - const input = unsafeInput === undefined - ? UsageRequest.make({ scopeKind: "Shard", scopeKey: encodeKey("local") }) - : yield* validateInput(UsageRequest, unsafeInput) - const row = yield* SqlSchema.findOneOption({ - Request: Schema.Void, - Result: UsageRow, - execute: () => - sql`SELECT - active_count AS "activeCount", - active_bytes AS "activeBytes", - retained_count AS "retainedCount", - retained_bytes AS "retainedBytes" - FROM effect_local_relay_usage - WHERE scope_kind = ${input.scopeKind} - AND scope_key = ${input.scopeKey}` - })(undefined) - return Option.getOrElse(row, (): Usage => ({ - activeCount: 0, - activeBytes: 0, - retainedCount: 0, - retainedBytes: 0 - })) - }).pipe(Effect.catchTags({ - SqlError: (cause) => - Effect.fail( - new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageUnavailable({ cause }) - }) - ), - SchemaError: (cause) => - Effect.fail( - new ReplicaError.ReplicaError({ - reason: new ReplicaError.StorageCorrupt({ cause }) - }) - ) - })) - return PeerRpcObservability.observeRelay({ - effect, - operation: "RelayMaintenance", - direction: "Receive", - stage: "Usage", - facts: (exit) => ({ - items: Exit.isSuccess(exit) ? exit.value.activeCount : 0, - ...(Exit.isSuccess(exit) ? { bytes: exit.value.activeBytes, version: 1 } : {}) - }), - result: (exit) => Exit.isSuccess(exit) ? "Success" : relayFailureResult(exit) - }).pipe( - Effect.tap((value) => - exactShard - ? PeerRpcObservability.setRelayPending( - value.activeCount, - value.activeBytes - ).pipe(Effect.ignoreCause) - : Effect.void - ) - ) - } - - return PeerRelayStore.of({ - admit, - claim, - loadClaimedPayload, - acknowledge, - reject, - release, - recover, - expire, - repair, - reconcile, - collect, - usage - }) -}) - -export const layer: Layer.Layer< - PeerRelayStore, - | Migrator.MigrationError - | SqlError.SqlError - | Schema.SchemaError - | ReplicaError.ReplicaError, - SqlClient.SqlClient | Crypto.Crypto | PeerRelayLimits.PeerRelayLimits -> = Layer.effect(PeerRelayStore, make) diff --git a/packages/local-rpc/src/index.ts b/packages/local-rpc/src/index.ts index 8c5c4a9..f9ba971 100644 --- a/packages/local-rpc/src/index.ts +++ b/packages/local-rpc/src/index.ts @@ -2,16 +2,12 @@ export * as PeerAuthentication from "./PeerAuthentication.js" export * as PeerAuthenticator from "./PeerAuthenticator.js" export * as PeerCredentials from "./PeerCredentials.js" export * as PeerRelayAuthorization from "./PeerRelayAuthorization.js" -export * as PeerRelayIngress from "./PeerRelayIngress.js" export * as PeerRelayLimits from "./PeerRelayLimits.js" -export * as PeerRelayStore from "./PeerRelayStore.js" export * as PeerRpc from "./PeerRpc.js" export * as PeerRpcError from "./PeerRpcError.js" -export * as PeerRpcServer from "./PeerRpcServer.js" export * as RelayInbox from "./RelayInbox.js" export * as RelayInboxMaintenance from "./RelayInboxMaintenance.js" export * as RelayInboxStore from "./RelayInboxStore.js" export * as RelayServer from "./RelayServer.js" export * as RpcPeerTransport from "./RpcPeerTransport.js" -export * as SqlPeerRelayStore from "./SqlPeerRelayStore.js" export * as SqlRelayInboxStore from "./SqlRelayInboxStore.js" diff --git a/packages/local-rpc/src/internal/peerRelayMigrations.ts b/packages/local-rpc/src/internal/peerRelayMigrations.ts deleted file mode 100644 index 828a813..0000000 --- a/packages/local-rpc/src/internal/peerRelayMigrations.ts +++ /dev/null @@ -1,872 +0,0 @@ -import * as Effect from "effect/Effect" -import * as Schema from "effect/Schema" -import * as Migrator from "effect/unstable/sql/Migrator" -import * as SqlClient from "effect/unstable/sql/SqlClient" - -const executeStatements = ( - sql: SqlClient.SqlClient, - source: string -) => - Effect.forEach( - source.split(";").map((statement) => statement.trim()).filter((statement) => statement.length > 0), - (statement) => sql.unsafe(statement), - { discard: true } - ) - -const mysqlIndexes = [ - { - table: "effect_local_relay_messages", - name: "effect_local_relay_messages_channel_head", - columns: ["channel_id", "channel_sequence", "state", "next_eligible_at"], - statement: `CREATE INDEX effect_local_relay_messages_channel_head - ON effect_local_relay_messages(channel_id, channel_sequence, state, next_eligible_at)` - }, - { - table: "effect_local_relay_channels", - name: "effect_local_relay_channels_discovery", - columns: ["recipient_peer_id", "sender_peer_id", "channel_id"], - statement: `CREATE INDEX effect_local_relay_channels_discovery - ON effect_local_relay_channels(recipient_peer_id, sender_peer_id, channel_id)` - }, - { - table: "effect_local_relay_messages", - name: "effect_local_relay_messages_admission_order", - columns: ["created_at", "message_id", "channel_id", "channel_sequence"], - statement: `CREATE INDEX effect_local_relay_messages_admission_order - ON effect_local_relay_messages(created_at, message_id, channel_id, channel_sequence)` - }, - { - table: "effect_local_relay_messages", - name: "effect_local_relay_messages_recovery", - columns: ["state", "claim_deadline", "message_id"], - statement: `CREATE INDEX effect_local_relay_messages_recovery - ON effect_local_relay_messages(state, claim_deadline, message_id)` - }, - { - table: "effect_local_relay_messages", - name: "effect_local_relay_messages_expiry", - columns: ["state", "expires_at", "message_id"], - statement: `CREATE INDEX effect_local_relay_messages_expiry - ON effect_local_relay_messages(state, expires_at, message_id)` - }, - { - table: "effect_local_relay_messages", - name: "effect_local_relay_messages_collection", - columns: ["state", "deduplicate_until", "message_id"], - statement: `CREATE INDEX effect_local_relay_messages_collection - ON effect_local_relay_messages(state, deduplicate_until, message_id)` - }, - { - table: "effect_local_relay_channels", - name: "effect_local_relay_channels_claim", - columns: ["claimed_message_id", "channel_id"], - statement: `CREATE INDEX effect_local_relay_channels_claim - ON effect_local_relay_channels(claimed_message_id, channel_id)` - }, - { - table: "effect_local_relay_messages", - name: "effect_local_relay_messages_claim_admission", - columns: ["state", "tenant_id", "sender_peer_id", "recipient_peer_id", "created_at", "message_id"], - statement: `CREATE INDEX effect_local_relay_messages_claim_admission - ON effect_local_relay_messages( - state, - tenant_id, - sender_peer_id, - recipient_peer_id, - created_at, - message_id - )` - } -] as const - -const migrationFailure = (message: string, cause?: unknown) => - new Migrator.MigrationError({ - kind: "Failed", - message, - ...(cause === undefined ? {} : { cause }) - }) - -const DatabaseInt = Schema.Union([Schema.Int, Schema.NumberFromString]).check(Schema.isInt()) - -const decodeMysqlRows = ( - schema: S, - rows: unknown, - message: string -) => - Schema.decodeUnknownEffect(schema)(rows).pipe( - Effect.mapError((cause) => migrationFailure(message, cause)) - ) - -const createMysqlIndexes = (sql: SqlClient.SqlClient) => - Effect.gen(function*() { - const rows = yield* sql.unsafe(` - SELECT table_name AS tableName, index_name AS indexName - FROM information_schema.statistics - WHERE table_schema = DATABASE() - `) - const existing = yield* decodeMysqlRows( - Schema.Array(Schema.Struct({ - tableName: Schema.String, - indexName: Schema.String - })), - rows, - "Could not inspect MySQL relay indexes" - ) - const names = new Set(existing.map((row) => `${row.tableName}.${row.indexName}`)) - for (const index of mysqlIndexes) { - if (!names.has(`${index.table}.${index.name}`)) { - yield* sql.unsafe(index.statement) - } - } - }) - -const createRelayTables = Effect.gen(function*() { - const sql = (yield* SqlClient.SqlClient).withoutTransforms() - - yield* sql.onDialectOrElse({ - pg: () => - executeStatements( - sql, - ` - CREATE TABLE effect_local_relay_write_lock ( - lock_id INTEGER PRIMARY KEY CHECK (lock_id = 1) - ); - INSERT INTO effect_local_relay_write_lock (lock_id) VALUES (1); - - CREATE TABLE effect_local_relay_channels ( - channel_id BIGSERIAL PRIMARY KEY, - tenant_id VARCHAR(256) NOT NULL, - sender_subject_id VARCHAR(256) NOT NULL, - sender_peer_id VARCHAR(64) NOT NULL, - sender_replica_incarnation BIGINT NOT NULL, - recipient_subject_id VARCHAR(256) NOT NULL, - recipient_peer_id VARCHAR(64) NOT NULL, - next_sequence BIGINT NOT NULL DEFAULT 0 CHECK (next_sequence >= 0), - claimed_message_id BIGINT, - claim_session_generation BIGINT, - claim_token VARCHAR(256), - claim_deadline BIGINT, - UNIQUE ( - tenant_id, - sender_subject_id, - sender_peer_id, - sender_replica_incarnation, - recipient_subject_id, - recipient_peer_id - ) - ); - - CREATE TABLE effect_local_relay_messages ( - message_id BIGSERIAL PRIMARY KEY, - channel_id BIGINT NOT NULL REFERENCES effect_local_relay_channels(channel_id) ON DELETE CASCADE, - channel_sequence BIGINT NOT NULL CHECK (channel_sequence >= 0), - tenant_id VARCHAR(256) NOT NULL, - sender_subject_id VARCHAR(256) NOT NULL, - sender_peer_id VARCHAR(64) NOT NULL, - recipient_subject_id VARCHAR(256) NOT NULL, - recipient_peer_id VARCHAR(64) NOT NULL, - relay_message_id VARCHAR(64) NOT NULL, - relay_peer_id VARCHAR(64) NOT NULL, - sender_connection_epoch VARCHAR(256) NOT NULL, - sender_sequence BIGINT NOT NULL CHECK (sender_sequence >= 0), - document_ids TEXT NOT NULL, - payload_version INTEGER NOT NULL, - message_hash VARCHAR(256) NOT NULL, - outer_envelope_digest VARCHAR(256) NOT NULL, - payload BYTEA, - payload_length BIGINT NOT NULL CHECK (payload_length >= 0), - state VARCHAR(32) NOT NULL CHECK ( - state IN ('Pending', 'Claimed', 'Acknowledged', 'DeadLettered', 'Expired') - ), - created_at BIGINT NOT NULL, - expires_at BIGINT NOT NULL, - deduplicate_until BIGINT NOT NULL, - next_eligible_at BIGINT NOT NULL, - retry_count INTEGER NOT NULL DEFAULT 0 CHECK (retry_count >= 0), - claim_token VARCHAR(256), - claim_session_generation BIGINT, - claim_deadline BIGINT, - terminal_at BIGINT, - terminal_claim_token VARCHAR(256), - terminal_session_generation BIGINT, - terminal_reason VARCHAR(256), - UNIQUE(channel_id, channel_sequence), - UNIQUE(tenant_id, sender_subject_id, sender_peer_id, relay_message_id) - ); - - CREATE TABLE effect_local_relay_usage ( - scope_kind VARCHAR(32) NOT NULL, - scope_key VARCHAR(1024) NOT NULL, - active_count BIGINT NOT NULL CHECK (active_count >= 0), - active_bytes BIGINT NOT NULL CHECK (active_bytes >= 0), - retained_count BIGINT NOT NULL CHECK (retained_count >= 0), - retained_bytes BIGINT NOT NULL CHECK (retained_bytes >= 0), - PRIMARY KEY(scope_kind, scope_key) - ); - - CREATE TABLE effect_local_relay_reservations ( - message_id BIGINT PRIMARY KEY REFERENCES effect_local_relay_messages(message_id) ON DELETE CASCADE, - sender_peer_usage_key VARCHAR(1024) NOT NULL, - recipient_peer_usage_key VARCHAR(1024) NOT NULL, - recipient_subject_usage_key VARCHAR(1024) NOT NULL, - tenant_usage_key VARCHAR(1024) NOT NULL, - shard_usage_key VARCHAR(1024) NOT NULL, - active_count_delta INTEGER NOT NULL CHECK (active_count_delta = 1), - active_bytes_delta BIGINT NOT NULL CHECK (active_bytes_delta >= 0), - retained_count_delta INTEGER NOT NULL CHECK (retained_count_delta = 1), - retained_bytes_delta BIGINT NOT NULL CHECK (retained_bytes_delta >= 0), - active_consumed INTEGER NOT NULL DEFAULT 0 CHECK (active_consumed IN (0, 1)), - retained_consumed INTEGER NOT NULL DEFAULT 0 CHECK (retained_consumed IN (0, 1)) - ) - ` - ), - mysql: () => - executeStatements( - sql, - ` - CREATE TABLE IF NOT EXISTS effect_local_relay_write_lock ( - lock_id INT PRIMARY KEY, - CHECK (lock_id = 1) - ); - INSERT IGNORE INTO effect_local_relay_write_lock (lock_id) VALUES (1); - - CREATE TABLE IF NOT EXISTS effect_local_relay_channels ( - channel_id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, - tenant_id VARCHAR(256) COLLATE utf8mb4_bin NOT NULL, - sender_subject_id VARCHAR(256) COLLATE utf8mb4_bin NOT NULL, - sender_peer_id VARCHAR(64) COLLATE utf8mb4_bin NOT NULL, - sender_replica_incarnation BIGINT NOT NULL, - recipient_subject_id VARCHAR(256) COLLATE utf8mb4_bin NOT NULL, - recipient_peer_id VARCHAR(64) COLLATE utf8mb4_bin NOT NULL, - channel_identity CHAR(64) COLLATE utf8mb4_bin GENERATED ALWAYS AS ( - SHA2(JSON_ARRAY( - tenant_id, - sender_subject_id, - sender_peer_id, - sender_replica_incarnation, - recipient_subject_id, - recipient_peer_id - ), 256) - ) STORED, - next_sequence BIGINT NOT NULL DEFAULT 0, - claimed_message_id BIGINT, - claim_session_generation BIGINT, - claim_token VARCHAR(256) COLLATE utf8mb4_bin, - claim_deadline BIGINT, - UNIQUE KEY effect_local_relay_channels_identity (channel_identity), - CHECK (next_sequence >= 0) - ); - - CREATE TABLE IF NOT EXISTS effect_local_relay_messages ( - message_id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, - channel_id BIGINT NOT NULL, - channel_sequence BIGINT NOT NULL, - tenant_id VARCHAR(256) COLLATE utf8mb4_bin NOT NULL, - sender_subject_id VARCHAR(256) COLLATE utf8mb4_bin NOT NULL, - sender_peer_id VARCHAR(64) COLLATE utf8mb4_bin NOT NULL, - recipient_subject_id VARCHAR(256) COLLATE utf8mb4_bin NOT NULL, - recipient_peer_id VARCHAR(64) COLLATE utf8mb4_bin NOT NULL, - relay_message_id VARCHAR(64) COLLATE utf8mb4_bin NOT NULL, - relay_peer_id VARCHAR(64) COLLATE utf8mb4_bin NOT NULL, - sender_connection_epoch VARCHAR(256) COLLATE utf8mb4_bin NOT NULL, - sender_sequence BIGINT NOT NULL, - sender_message_identity CHAR(64) COLLATE utf8mb4_bin GENERATED ALWAYS AS ( - SHA2(JSON_ARRAY( - tenant_id, - sender_subject_id, - sender_peer_id, - relay_message_id - ), 256) - ) STORED, - document_ids LONGTEXT NOT NULL, - payload_version INT NOT NULL, - message_hash VARCHAR(256) COLLATE utf8mb4_bin NOT NULL, - outer_envelope_digest VARCHAR(256) COLLATE utf8mb4_bin NOT NULL, - payload LONGBLOB, - payload_length BIGINT NOT NULL, - state VARCHAR(32) COLLATE utf8mb4_bin NOT NULL, - created_at BIGINT NOT NULL, - expires_at BIGINT NOT NULL, - deduplicate_until BIGINT NOT NULL, - next_eligible_at BIGINT NOT NULL, - retry_count INT NOT NULL DEFAULT 0, - claim_token VARCHAR(256) COLLATE utf8mb4_bin, - claim_session_generation BIGINT, - claim_deadline BIGINT, - terminal_at BIGINT, - terminal_claim_token VARCHAR(256) COLLATE utf8mb4_bin, - terminal_session_generation BIGINT, - terminal_reason VARCHAR(256) COLLATE utf8mb4_bin, - UNIQUE KEY effect_local_relay_messages_channel_sequence(channel_id, channel_sequence), - UNIQUE KEY effect_local_relay_messages_sender_identity(sender_message_identity), - CONSTRAINT effect_local_relay_messages_channel_fk - FOREIGN KEY(channel_id) REFERENCES effect_local_relay_channels(channel_id) ON DELETE CASCADE, - CHECK (channel_sequence >= 0), - CHECK (sender_sequence >= 0), - CHECK (payload_length >= 0), - CHECK (retry_count >= 0), - CHECK (state IN ('Pending', 'Claimed', 'Acknowledged', 'DeadLettered', 'Expired')) - ); - - CREATE TABLE IF NOT EXISTS effect_local_relay_usage ( - scope_kind VARCHAR(32) COLLATE utf8mb4_bin NOT NULL, - scope_key VARCHAR(700) COLLATE utf8mb4_bin NOT NULL, - active_count BIGINT NOT NULL, - active_bytes BIGINT NOT NULL, - retained_count BIGINT NOT NULL, - retained_bytes BIGINT NOT NULL, - PRIMARY KEY(scope_kind, scope_key), - CHECK (active_count >= 0), - CHECK (active_bytes >= 0), - CHECK (retained_count >= 0), - CHECK (retained_bytes >= 0) - ); - - CREATE TABLE IF NOT EXISTS effect_local_relay_reservations ( - message_id BIGINT PRIMARY KEY, - sender_peer_usage_key TEXT NOT NULL, - recipient_peer_usage_key TEXT NOT NULL, - recipient_subject_usage_key TEXT NOT NULL, - tenant_usage_key TEXT NOT NULL, - shard_usage_key TEXT NOT NULL, - active_count_delta INT NOT NULL, - active_bytes_delta BIGINT NOT NULL, - retained_count_delta INT NOT NULL, - retained_bytes_delta BIGINT NOT NULL, - active_consumed INT NOT NULL DEFAULT 0, - retained_consumed INT NOT NULL DEFAULT 0, - CONSTRAINT effect_local_relay_reservations_message_fk - FOREIGN KEY(message_id) REFERENCES effect_local_relay_messages(message_id) ON DELETE CASCADE, - CHECK (active_count_delta = 1), - CHECK (active_bytes_delta >= 0), - CHECK (retained_count_delta = 1), - CHECK (retained_bytes_delta >= 0), - CHECK (active_consumed IN (0, 1)), - CHECK (retained_consumed IN (0, 1)) - ) - ` - ), - orElse: () => - executeStatements( - sql, - ` - CREATE TABLE effect_local_relay_write_lock ( - lock_id INTEGER PRIMARY KEY CHECK (lock_id = 1) - ); - INSERT INTO effect_local_relay_write_lock (lock_id) VALUES (1); - - CREATE TABLE effect_local_relay_channels ( - channel_id INTEGER PRIMARY KEY AUTOINCREMENT, - tenant_id TEXT NOT NULL, - sender_subject_id TEXT NOT NULL, - sender_peer_id TEXT NOT NULL, - sender_replica_incarnation INTEGER NOT NULL, - recipient_subject_id TEXT NOT NULL, - recipient_peer_id TEXT NOT NULL, - next_sequence INTEGER NOT NULL DEFAULT 0 CHECK (next_sequence >= 0), - claimed_message_id INTEGER, - claim_session_generation INTEGER, - claim_token TEXT, - claim_deadline INTEGER, - UNIQUE ( - tenant_id, - sender_subject_id, - sender_peer_id, - sender_replica_incarnation, - recipient_subject_id, - recipient_peer_id - ) - ); - - CREATE TABLE effect_local_relay_messages ( - message_id INTEGER PRIMARY KEY AUTOINCREMENT, - channel_id INTEGER NOT NULL REFERENCES effect_local_relay_channels(channel_id) ON DELETE CASCADE, - channel_sequence INTEGER NOT NULL CHECK (channel_sequence >= 0), - tenant_id TEXT NOT NULL, - sender_subject_id TEXT NOT NULL, - sender_peer_id TEXT NOT NULL, - recipient_subject_id TEXT NOT NULL, - recipient_peer_id TEXT NOT NULL, - relay_message_id TEXT NOT NULL, - relay_peer_id TEXT NOT NULL, - sender_connection_epoch TEXT NOT NULL, - sender_sequence INTEGER NOT NULL CHECK (sender_sequence >= 0), - document_ids TEXT NOT NULL, - payload_version INTEGER NOT NULL, - message_hash TEXT NOT NULL, - outer_envelope_digest TEXT NOT NULL, - payload BLOB, - payload_length INTEGER NOT NULL CHECK (payload_length >= 0), - state TEXT NOT NULL CHECK ( - state IN ('Pending', 'Claimed', 'Acknowledged', 'DeadLettered', 'Expired') - ), - created_at INTEGER NOT NULL, - expires_at INTEGER NOT NULL, - deduplicate_until INTEGER NOT NULL, - next_eligible_at INTEGER NOT NULL, - retry_count INTEGER NOT NULL DEFAULT 0 CHECK (retry_count >= 0), - claim_token TEXT, - claim_session_generation INTEGER, - claim_deadline INTEGER, - terminal_at INTEGER, - terminal_claim_token TEXT, - terminal_session_generation INTEGER, - terminal_reason TEXT, - UNIQUE(channel_id, channel_sequence), - UNIQUE(tenant_id, sender_subject_id, sender_peer_id, relay_message_id) - ); - - CREATE TABLE effect_local_relay_usage ( - scope_kind TEXT NOT NULL, - scope_key TEXT NOT NULL, - active_count INTEGER NOT NULL CHECK (active_count >= 0), - active_bytes INTEGER NOT NULL CHECK (active_bytes >= 0), - retained_count INTEGER NOT NULL CHECK (retained_count >= 0), - retained_bytes INTEGER NOT NULL CHECK (retained_bytes >= 0), - PRIMARY KEY(scope_kind, scope_key) - ); - - CREATE TABLE effect_local_relay_reservations ( - message_id INTEGER PRIMARY KEY REFERENCES effect_local_relay_messages(message_id) ON DELETE CASCADE, - sender_peer_usage_key TEXT NOT NULL, - recipient_peer_usage_key TEXT NOT NULL, - recipient_subject_usage_key TEXT NOT NULL, - tenant_usage_key TEXT NOT NULL, - shard_usage_key TEXT NOT NULL, - active_count_delta INTEGER NOT NULL CHECK (active_count_delta = 1), - active_bytes_delta INTEGER NOT NULL CHECK (active_bytes_delta >= 0), - retained_count_delta INTEGER NOT NULL CHECK (retained_count_delta = 1), - retained_bytes_delta INTEGER NOT NULL CHECK (retained_bytes_delta >= 0), - active_consumed INTEGER NOT NULL DEFAULT 0 CHECK (active_consumed IN (0, 1)), - retained_consumed INTEGER NOT NULL DEFAULT 0 CHECK (retained_consumed IN (0, 1)) - ) - ` - ) - }) - - yield* sql.onDialectOrElse({ - mysql: () => createMysqlIndexes(sql), - orElse: () => - executeStatements( - sql, - ` - CREATE INDEX effect_local_relay_messages_channel_head - ON effect_local_relay_messages(channel_id, channel_sequence, state, next_eligible_at); - CREATE INDEX effect_local_relay_channels_discovery - ON effect_local_relay_channels( - tenant_id, - recipient_subject_id, - recipient_peer_id, - sender_subject_id, - sender_peer_id, - channel_id - ); - CREATE INDEX effect_local_relay_messages_admission_order - ON effect_local_relay_messages(created_at, message_id, channel_id, channel_sequence); - CREATE INDEX effect_local_relay_messages_recovery - ON effect_local_relay_messages(state, claim_deadline, message_id); - CREATE INDEX effect_local_relay_messages_expiry - ON effect_local_relay_messages(expires_at, message_id); - CREATE INDEX effect_local_relay_messages_collection - ON effect_local_relay_messages(deduplicate_until, message_id); - CREATE INDEX effect_local_relay_channels_claim - ON effect_local_relay_channels(claimed_message_id, channel_id); - CREATE INDEX effect_local_relay_messages_claim_admission - ON effect_local_relay_messages( - tenant_id, - sender_subject_id, - sender_peer_id, - recipient_subject_id, - recipient_peer_id, - created_at, - message_id - ) - ` - ) - }) -}) - -const migrations = Migrator.fromRecord({ - "0001_create_relay_tables": createRelayTables -}) - -const mysqlExpectedColumns = { - effect_local_relay_write_lock: [ - "lock_id:int:NO" - ], - effect_local_relay_channels: [ - "channel_id:bigint:NO", - "tenant_id:varchar:NO", - "sender_subject_id:varchar:NO", - "sender_peer_id:varchar:NO", - "sender_replica_incarnation:bigint:NO", - "recipient_subject_id:varchar:NO", - "recipient_peer_id:varchar:NO", - "channel_identity:char:YES", - "next_sequence:bigint:NO", - "claimed_message_id:bigint:YES", - "claim_session_generation:bigint:YES", - "claim_token:varchar:YES", - "claim_deadline:bigint:YES" - ], - effect_local_relay_messages: [ - "message_id:bigint:NO", - "channel_id:bigint:NO", - "channel_sequence:bigint:NO", - "tenant_id:varchar:NO", - "sender_subject_id:varchar:NO", - "sender_peer_id:varchar:NO", - "recipient_subject_id:varchar:NO", - "recipient_peer_id:varchar:NO", - "relay_message_id:varchar:NO", - "relay_peer_id:varchar:NO", - "sender_connection_epoch:varchar:NO", - "sender_sequence:bigint:NO", - "sender_message_identity:char:YES", - "document_ids:longtext:NO", - "payload_version:int:NO", - "message_hash:varchar:NO", - "outer_envelope_digest:varchar:NO", - "payload:longblob:YES", - "payload_length:bigint:NO", - "state:varchar:NO", - "created_at:bigint:NO", - "expires_at:bigint:NO", - "deduplicate_until:bigint:NO", - "next_eligible_at:bigint:NO", - "retry_count:int:NO", - "claim_token:varchar:YES", - "claim_session_generation:bigint:YES", - "claim_deadline:bigint:YES", - "terminal_at:bigint:YES", - "terminal_claim_token:varchar:YES", - "terminal_session_generation:bigint:YES", - "terminal_reason:varchar:YES" - ], - effect_local_relay_usage: [ - "scope_kind:varchar:NO", - "scope_key:varchar:NO", - "active_count:bigint:NO", - "active_bytes:bigint:NO", - "retained_count:bigint:NO", - "retained_bytes:bigint:NO" - ], - effect_local_relay_reservations: [ - "message_id:bigint:NO", - "sender_peer_usage_key:text:NO", - "recipient_peer_usage_key:text:NO", - "recipient_subject_usage_key:text:NO", - "tenant_usage_key:text:NO", - "shard_usage_key:text:NO", - "active_count_delta:int:NO", - "active_bytes_delta:bigint:NO", - "retained_count_delta:int:NO", - "retained_bytes_delta:bigint:NO", - "active_consumed:int:NO", - "retained_consumed:int:NO" - ] -} as const - -const mysqlBinaryColumns = new Set([ - "effect_local_relay_channels.tenant_id", - "effect_local_relay_channels.sender_subject_id", - "effect_local_relay_channels.sender_peer_id", - "effect_local_relay_channels.recipient_subject_id", - "effect_local_relay_channels.recipient_peer_id", - "effect_local_relay_channels.channel_identity", - "effect_local_relay_channels.claim_token", - "effect_local_relay_messages.tenant_id", - "effect_local_relay_messages.sender_subject_id", - "effect_local_relay_messages.sender_peer_id", - "effect_local_relay_messages.recipient_subject_id", - "effect_local_relay_messages.recipient_peer_id", - "effect_local_relay_messages.relay_message_id", - "effect_local_relay_messages.relay_peer_id", - "effect_local_relay_messages.sender_connection_epoch", - "effect_local_relay_messages.sender_message_identity", - "effect_local_relay_messages.message_hash", - "effect_local_relay_messages.outer_envelope_digest", - "effect_local_relay_messages.state", - "effect_local_relay_messages.claim_token", - "effect_local_relay_messages.terminal_claim_token", - "effect_local_relay_messages.terminal_reason", - "effect_local_relay_usage.scope_kind", - "effect_local_relay_usage.scope_key" -]) - -const mysqlRequiredIndexes = [ - ...mysqlIndexes, - { - table: "effect_local_relay_write_lock", - name: "PRIMARY", - columns: ["lock_id"] - }, - { - table: "effect_local_relay_channels", - name: "PRIMARY", - columns: ["channel_id"] - }, - { - table: "effect_local_relay_channels", - name: "effect_local_relay_channels_identity", - columns: ["channel_identity"] - }, - { - table: "effect_local_relay_messages", - name: "PRIMARY", - columns: ["message_id"] - }, - { - table: "effect_local_relay_messages", - name: "effect_local_relay_messages_channel_sequence", - columns: ["channel_id", "channel_sequence"] - }, - { - table: "effect_local_relay_messages", - name: "effect_local_relay_messages_sender_identity", - columns: ["sender_message_identity"] - }, - { - table: "effect_local_relay_usage", - name: "PRIMARY", - columns: ["scope_kind", "scope_key"] - }, - { - table: "effect_local_relay_reservations", - name: "PRIMARY", - columns: ["message_id"] - } -] as const - -const verifyMysqlSchema = (sql: SqlClient.SqlClient) => - Effect.gen(function*() { - const columnRows = yield* sql.unsafe(` - SELECT - table_name AS tableName, - column_name AS columnName, - data_type AS dataType, - is_nullable AS isNullable, - collation_name AS collationName - FROM information_schema.columns - WHERE table_schema = DATABASE() - AND table_name IN ( - 'effect_local_relay_write_lock', - 'effect_local_relay_channels', - 'effect_local_relay_messages', - 'effect_local_relay_usage', - 'effect_local_relay_reservations' - ) - ORDER BY table_name, ordinal_position - `) - const columns = yield* decodeMysqlRows( - Schema.Array(Schema.Struct({ - tableName: Schema.String, - columnName: Schema.String, - dataType: Schema.String, - isNullable: Schema.String, - collationName: Schema.NullOr(Schema.String) - })), - columnRows, - "Could not inspect MySQL relay columns" - ) - for (const [table, expected] of Object.entries(mysqlExpectedColumns)) { - const actual = columns - .filter((column) => column.tableName === table) - .map((column) => `${column.columnName}:${column.dataType}:${column.isNullable}`) - if (actual.length !== expected.length || actual.some((value, index) => value !== expected[index])) { - return yield* migrationFailure(`MySQL relay table "${table}" does not match migration 1`) - } - } - for (const column of columns) { - if ( - mysqlBinaryColumns.has(`${column.tableName}.${column.columnName}`) && - column.collationName !== "utf8mb4_bin" - ) { - return yield* migrationFailure( - `MySQL relay column "${column.tableName}.${column.columnName}" must use utf8mb4_bin` - ) - } - } - - const indexRows = yield* sql.unsafe(` - SELECT - table_name AS tableName, - index_name AS indexName, - column_name AS columnName, - seq_in_index AS sequence - FROM information_schema.statistics - WHERE table_schema = DATABASE() - ORDER BY table_name, index_name, seq_in_index - `) - const indexes = yield* decodeMysqlRows( - Schema.Array(Schema.Struct({ - tableName: Schema.String, - indexName: Schema.String, - columnName: Schema.String, - sequence: DatabaseInt - })), - indexRows, - "Could not inspect MySQL relay indexes" - ) - for (const expected of mysqlRequiredIndexes) { - const actual = indexes - .filter((index) => index.tableName === expected.table && index.indexName === expected.name) - .toSorted((left, right) => left.sequence - right.sequence) - .map((index) => index.columnName) - if ( - actual.length !== expected.columns.length || - actual.some((column, index) => column !== expected.columns[index]) - ) { - return yield* migrationFailure(`MySQL relay index "${expected.name}" is missing or invalid`) - } - } - - const foreignKeyRows = yield* sql.unsafe(` - SELECT - usage_table.table_name AS tableName, - usage_table.column_name AS columnName, - usage_table.referenced_table_name AS referencedTableName, - usage_table.referenced_column_name AS referencedColumnName, - rules.delete_rule AS deleteRule - FROM information_schema.key_column_usage usage_table - JOIN information_schema.referential_constraints rules - ON rules.constraint_schema = usage_table.constraint_schema - AND rules.constraint_name = usage_table.constraint_name - WHERE usage_table.constraint_schema = DATABASE() - AND usage_table.referenced_table_name IS NOT NULL - AND usage_table.table_name IN ( - 'effect_local_relay_messages', - 'effect_local_relay_reservations' - ) - ORDER BY usage_table.table_name - `) - const foreignKeys = yield* decodeMysqlRows( - Schema.Array(Schema.Struct({ - tableName: Schema.String, - columnName: Schema.String, - referencedTableName: Schema.String, - referencedColumnName: Schema.String, - deleteRule: Schema.String - })), - foreignKeyRows, - "Could not inspect MySQL relay foreign keys" - ) - const actualForeignKeys = foreignKeys.map((foreignKey) => - [ - foreignKey.tableName, - foreignKey.columnName, - foreignKey.referencedTableName, - foreignKey.referencedColumnName, - foreignKey.deleteRule - ].join(":") - ) - const expectedForeignKeys = [ - "effect_local_relay_messages:channel_id:effect_local_relay_channels:channel_id:CASCADE", - "effect_local_relay_reservations:message_id:effect_local_relay_messages:message_id:CASCADE" - ] - if ( - actualForeignKeys.length !== expectedForeignKeys.length || - actualForeignKeys.some((foreignKey, index) => foreignKey !== expectedForeignKeys[index]) - ) { - return yield* migrationFailure("MySQL relay foreign keys do not match migration 1") - } - - const checkRows = yield* sql.unsafe(` - SELECT constraint_name AS constraintName - FROM information_schema.table_constraints - WHERE constraint_schema = DATABASE() - AND constraint_type = 'CHECK' - AND table_name IN ( - 'effect_local_relay_write_lock', - 'effect_local_relay_channels', - 'effect_local_relay_messages', - 'effect_local_relay_usage', - 'effect_local_relay_reservations' - ) - `) - const checks = yield* decodeMysqlRows( - Schema.Array(Schema.Struct({ constraintName: Schema.String })), - checkRows, - "Could not inspect MySQL relay check constraints" - ) - if (checks.length !== 17) { - return yield* migrationFailure("MySQL relay check constraints do not match migration 1") - } - }) - -const genericRun = Migrator.make({})({ - loader: migrations, - table: "effect_local_relay_migrations" -}) - -const runMysql = (sql: SqlClient.SqlClient) => - sql.withTransaction( - Effect.acquireUseRelease( - Effect.gen(function*() { - const lockRows = yield* sql`SELECT GET_LOCK( - 'effect_local_relay_migrations', - 30 - ) AS acquired` - const lock = yield* decodeMysqlRows( - Schema.Array(Schema.Struct({ acquired: Schema.NullOr(DatabaseInt) })), - lockRows, - "Could not decode the MySQL relay migration lock result" - ) - if (lock.length !== 1 || lock[0]!.acquired !== 1) { - return yield* new Migrator.MigrationError({ - kind: "Locked", - message: "Could not acquire the MySQL relay migration lock" - }) - } - }), - () => - Effect.gen(function*() { - yield* sql` - CREATE TABLE IF NOT EXISTS effect_local_relay_migrations ( - migration_id INTEGER UNSIGNED NOT NULL, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - name VARCHAR(255) NOT NULL, - PRIMARY KEY (migration_id) - ) - ` - const markerRows = yield* sql` - SELECT migration_id AS migrationId - FROM effect_local_relay_migrations - WHERE migration_id = 1 - ` - const markers = yield* decodeMysqlRows( - Schema.Array(Schema.Struct({ migrationId: DatabaseInt })), - markerRows, - "Could not decode MySQL relay migration state" - ) - if (markers.length > 1) { - return yield* migrationFailure("MySQL relay migration state contains duplicate markers") - } - if (markers.length === 0) { - yield* createRelayTables - } - yield* verifyMysqlSchema(sql) - if (markers.length === 0) { - yield* sql` - INSERT INTO effect_local_relay_migrations (migration_id, name) - VALUES (1, 'create_relay_tables') - ` - } - return markers.length === 0 - ? [[1, "create_relay_tables"] as const] - : [] - }), - () => - sql`SELECT RELEASE_LOCK('effect_local_relay_migrations')`.pipe( - Effect.asVoid - ) - ) - ) - -export const run = Effect.gen(function*() { - const sql = (yield* SqlClient.SqlClient).withoutTransforms() - return yield* sql.onDialectOrElse({ - mysql: () => runMysql(sql), - orElse: () => genericRun - }) -}) diff --git a/packages/local-rpc/src/internal/peerRelaySqlTransaction.ts b/packages/local-rpc/src/internal/peerRelaySqlTransaction.ts deleted file mode 100644 index a55ab55..0000000 --- a/packages/local-rpc/src/internal/peerRelaySqlTransaction.ts +++ /dev/null @@ -1,238 +0,0 @@ -import * as Cause from "effect/Cause" -import * as Effect from "effect/Effect" -import * as Exit from "effect/Exit" -import * as Option from "effect/Option" -import * as Schema from "effect/Schema" -import * as Scope from "effect/Scope" -import * as SqlClient from "effect/unstable/sql/SqlClient" -import type * as SqlConnection from "effect/unstable/sql/SqlConnection" -import type * as SqlError from "effect/unstable/sql/SqlError" - -export class NestedPeerRelayTransactionError extends Schema.TaggedErrorClass( - "@lucas-barake/effect-local-rpc/internal/peerRelaySqlTransaction/NestedPeerRelayTransactionError" -)("NestedPeerRelayTransactionError", {}) {} - -export interface Options { - readonly maxAcquireAttempts: number - readonly acquireRetryBaseDelayMillis: number - readonly acquireRetryMaximumDelayMillis: number -} - -const execute = (connection: SqlConnection.Connection, statement: string) => - connection.executeUnprepared(statement, [], undefined).pipe(Effect.asVoid) - -export const cleanupAcquisition = ( - original: Cause.Cause, - scope: Scope.Closeable | undefined, - connection: SqlConnection.Connection | undefined, - began: boolean -): Effect.Effect => - Effect.uninterruptible( - Effect.gen(function*() { - let combined: Cause.Cause = original - if (began && connection !== undefined) { - const rollback = yield* execute(connection, "ROLLBACK").pipe(Effect.exit) - if (Exit.isFailure(rollback)) { - combined = Cause.combine(combined, rollback.cause) - } - } - if (scope !== undefined) { - const closed = yield* Scope.close(scope, Exit.failCause(combined)).pipe(Effect.exit) - if (Exit.isFailure(closed)) { - combined = Cause.combine(combined, closed.cause) - } - } - return yield* Effect.failCause(combined) - }) - ) - -const acquireImmediate = ( - sql: SqlClient.SqlClient, - options: Options -): Effect.Effect< - readonly [Scope.Closeable, SqlConnection.Connection], - SqlError.SqlError -> => - Effect.interruptibleMask((restore) => { - const attempt = ( - remaining: number, - attemptNumber: number - ): Effect.Effect< - readonly [Scope.Closeable, SqlConnection.Connection], - SqlError.SqlError - > => - Effect.suspend(() => { - let began = false - let scope: Scope.Closeable | undefined - let connection: SqlConnection.Connection | undefined - const acquire = Effect.gen(function*() { - scope = yield* Scope.make("sequential") - const reserved = yield* sql.reserve.pipe( - Effect.provideService(Scope.Scope, scope) - ) - connection = reserved - return yield* restore( - execute(reserved, "BEGIN IMMEDIATE").pipe( - Effect.tap(() => - Effect.sync(() => { - began = true - }) - ), - Effect.as([scope, reserved] as const) - ) - ) - }).pipe( - Effect.catchCause((cause) => cleanupAcquisition(cause, scope, connection, began)) - ) - return acquire.pipe( - Effect.catchCause((cause) => { - const only = cause.reasons.length === 1 ? cause.reasons[0] : undefined - const lockTimeout = only !== undefined && - Cause.isFailReason(only) && - only.error._tag === "SqlError" && - only.error.reason._tag === "LockTimeoutError" - if (!lockTimeout || remaining <= 1) { - return Effect.failCause(cause) - } - return Effect.sleep(Math.min( - options.acquireRetryMaximumDelayMillis, - options.acquireRetryBaseDelayMillis * 2 ** attemptNumber - )).pipe( - Effect.andThen(attempt(remaining - 1, attemptNumber + 1)) - ) - }) - ) - }) - return attempt(options.maxAcquireAttempts, 0) - }) - -export const makeSqlite = ( - sql: SqlClient.SqlClient, - options: Options -) => { - return ( - effect: Effect.Effect - ): Effect.Effect< - A, - E | SqlError.SqlError | NestedPeerRelayTransactionError, - R - > => - Effect.gen(function*() { - const active = yield* Effect.serviceOption(sql.transactionService) - if (Option.isSome(active)) { - return yield* new NestedPeerRelayTransactionError() - } - return yield* Effect.suspend(() => { - let bodyCause: Cause.Cause | undefined - let commitCause: Cause.Cause | undefined - let rollbackCause: Cause.Cause | undefined - const commitTelemetryMarker = new Error("Peer relay transaction commit failed") - const withTransaction = SqlClient.makeWithTransaction({ - transactionService: sql.transactionService, - spanAttributes: [["db.system", "sqlite"]], - acquireConnection: acquireImmediate(sql, options), - begin: () => Effect.void, - savepoint: () => Effect.die(new Error("Nested relay transactions are forbidden")), - commit: (connection) => - execute(connection, "COMMIT").pipe( - Effect.exit, - Effect.flatMap((commitExit) => { - if (Exit.isSuccess(commitExit)) { - return Effect.void - } - return execute(connection, "ROLLBACK").pipe( - Effect.exit, - Effect.flatMap((rollbackExit) => { - const combined = Exit.isFailure(rollbackExit) - ? Cause.combine(commitExit.cause, rollbackExit.cause) - : commitExit.cause - return Effect.sync(() => { - commitCause = combined - }).pipe(Effect.andThen(Effect.die(commitTelemetryMarker))) - }) - ) - }) - ), - rollback: (connection) => - execute(connection, "ROLLBACK").pipe( - Effect.exit, - Effect.tap((exit) => - Effect.sync(() => { - if (Exit.isFailure(exit)) { - rollbackCause = exit.cause - } - }) - ), - Effect.asVoid - ), - rollbackSavepoint: () => Effect.die(new Error("Nested relay transactions are forbidden")) - }) - const observed = effect.pipe( - Effect.catchCause((cause) => - Effect.sync(() => { - bodyCause = cause - }).pipe(Effect.andThen(Effect.failCause(cause))) - ) - ) - return withTransaction(observed).pipe( - Effect.exit, - Effect.flatMap((exit) => { - if (Exit.isSuccess(exit)) { - return commitCause === undefined - ? Effect.succeed(exit.value) - : Effect.failCause(commitCause) - } - const returnedCause = commitCause === undefined - ? exit.cause - : Cause.fromReasons( - exit.cause.reasons.filter((reason) => - !(Cause.isDieReason(reason) && reason.defect === commitTelemetryMarker) - ) - ) - let combined = bodyCause === undefined - ? returnedCause - : Cause.combine(bodyCause, returnedCause) - if (commitCause !== undefined) { - combined = Cause.combine(combined, commitCause) - } - if (rollbackCause !== undefined) { - combined = Cause.combine(combined, rollbackCause) - } - return Effect.failCause(combined) - }) - ) - }) - }) -} - -const makeServer = (sql: SqlClient.SqlClient) => -( - effect: Effect.Effect -): Effect.Effect< - A, - E | SqlError.SqlError | NestedPeerRelayTransactionError, - R -> => - Effect.gen(function*() { - const active = yield* Effect.serviceOption(sql.transactionService) - if (Option.isSome(active)) { - return yield* new NestedPeerRelayTransactionError() - } - return yield* sql.withTransaction( - sql`SELECT lock_id - FROM effect_local_relay_write_lock - WHERE lock_id = 1 - FOR UPDATE`.pipe(Effect.andThen(effect)) - ) - }) - -export const make = ( - sql: SqlClient.SqlClient, - options: Options -) => - sql.onDialectOrElse({ - sqlite: () => makeSqlite(sql, options), - pg: () => makeServer(sql), - mysql: () => makeServer(sql), - orElse: () => makeServer(sql) - }) diff --git a/packages/local-rpc/test/PeerRelayIngress.test.ts b/packages/local-rpc/test/PeerRelayIngress.test.ts deleted file mode 100644 index b00fa8e..0000000 --- a/packages/local-rpc/test/PeerRelayIngress.test.ts +++ /dev/null @@ -1,737 +0,0 @@ -import { NodeSocket, NodeSocketServer } from "@effect/platform-node" -import { assert, describe, it } from "@effect/vitest" -import { Context, Deferred, Effect, Exit, Fiber, Layer, Scheduler, Scope } from "effect" -import { TestClock } from "effect/testing" -import type * as RpcMessage from "effect/unstable/rpc/RpcMessage" -import * as RpcServer from "effect/unstable/rpc/RpcServer" -import { Socket } from "effect/unstable/socket" -import type { SocketServer } from "effect/unstable/socket" -import * as PeerRelayIngress from "../src/PeerRelayIngress.ts" -import * as PeerRelayLimits from "../src/PeerRelayLimits.ts" - -const frame = (value: unknown) => { - const body = new TextEncoder().encode(JSON.stringify(value)) - const result = new Uint8Array(body.byteLength + 4) - new DataView(result.buffer).setUint32(0, body.byteLength, false) - result.set(body, 4) - return result -} - -const request = (id: number, padding = ""): RpcMessage.RequestEncoded => ({ - _tag: "Request", - id, - tag: "Test", - payload: { padding }, - headers: [] -}) - -const testLimits = ( - overrides: Partial = {} -): PeerRelayLimits.Values => ({ - ...PeerRelayLimits.defaults, - maxRelayConnections: 4, - maximumRawChunkBytes: 512, - maximumDeclaredFrameBytes: 4 * 1_024 * 1_024, - maximumIncompleteFrameBytes: 4 * 1_024 * 1_024 + 4, - incompleteFrameTimeoutMillis: 1_000, - maximumByteReservationWaiters: 4, - maxSessionsPerSubject: 4, - maxInFlightOpen: 4, - ...overrides -}) - -const buildServer = Effect.fnUntraced(function*( - values: PeerRelayLimits.Values, - port = 0 -) { - const scope = yield* Scope.make("sequential") - const context = yield* Layer.buildWithScope( - PeerRelayIngress.layerProtocolSocketServer(NodeSocketServer.layer({ port })).pipe( - Layer.provide(PeerRelayLimits.layer(values)) - ), - scope - ) - const ingress = Context.get(context, PeerRelayIngress.PeerRelayIngress) - const protocol = Context.get(context, RpcServer.Protocol) - return { ingress, protocol, scope } -}) - -const connect = Effect.fnUntraced(function*( - port: number, - scope: Scope.Closeable, - read = true -) { - const context = yield* Layer.buildWithScope(NodeSocket.layerNet({ port }), scope) - const socket = Context.get(context, Socket.Socket) - const readFiber = read - ? yield* Effect.forkIn(socket.runRaw(() => Effect.void), scope) - : undefined - const write = yield* Scope.provide(socket.writer, scope) - return { readFiber, socket, write } -}) - -const tcpPort = (address: SocketServer.Address) => { - assert.strictEqual(address._tag, "TcpAddress") - return (address as SocketServer.TcpAddress).port -} - -const awaitConnections = (ingress: PeerRelayIngress.PeerRelayIngress["Service"], expected: number) => - Effect.gen(function*() { - for (let attempt = 0; attempt < 10_000; attempt++) { - const usage = yield* ingress.usage - if (usage.connections === expected) return usage - yield* Effect.yieldNow - } - return yield* Effect.die(new Error("Connection count did not converge")) - }) - -const awaitReservedBytes = (ingress: PeerRelayIngress.PeerRelayIngress["Service"]) => - Effect.gen(function*() { - for (let attempt = 0; attempt < 10_000; attempt++) { - const usage = yield* ingress.usage - if (usage.reservedBytes > 0) return usage - yield* Effect.yieldNow - } - return yield* Effect.die(new Error("Byte reservation did not become observable")) - }) - -const awaitByteReservationWaiters = ( - ingress: PeerRelayIngress.PeerRelayIngress["Service"], - expected: number -) => - Effect.gen(function*() { - for (let attempt = 0; attempt < 10_000; attempt++) { - const usage = yield* ingress.usage - if (usage.byteReservationWaiters === expected) return usage - yield* Effect.yieldNow - } - return yield* Effect.die(new Error("Byte reservation waiter count did not converge")) - }) - -describe("PeerRelayIngress", () => { - it.effect("rejects an oversized declared frame before dispatch", () => - Effect.scoped(Effect.gen(function*() { - const received = Deferred.makeUnsafe() - const values = testLimits() - const server = yield* buildServer(values) - yield* Effect.forkIn( - server.protocol.run(() => Deferred.succeed(received, undefined)), - server.scope - ) - const clientScope = yield* Scope.make() - const client = yield* connect(tcpPort(server.ingress.address), clientScope) - - const header = new Uint8Array(4) - new DataView(header.buffer).setUint32(0, values.maximumDeclaredFrameBytes + 1, false) - yield* client.write(header) - const clientExit = yield* Fiber.await(client.readFiber!) - - assert.match(clientExit._tag, /^(Failure|Success)$/) - assert.strictEqual(Deferred.isDoneUnsafe(received), false) - assert.deepStrictEqual(yield* server.ingress.usage, { - connections: 0, - reservedBytes: 0, - byteReservationWaiters: 0 - }) - yield* Scope.close(clientScope, Exit.void) - yield* Scope.close(server.scope, Exit.void) - }))) - - it.effect("rejects an oversized raw chunk before JSON decode", () => - Effect.scoped(Effect.gen(function*() { - const received = Deferred.makeUnsafe() - const server = yield* buildServer(testLimits({ - maximumRawChunkBytes: 8 - })) - yield* Effect.forkIn( - server.protocol.run(() => Deferred.succeed(received, undefined)), - server.scope - ) - const clientScope = yield* Scope.make() - const client = yield* connect(tcpPort(server.ingress.address), clientScope) - - yield* client.write(frame(request(1))) - const clientExit = yield* Fiber.await(client.readFiber!) - - assert.match(clientExit._tag, /^(Failure|Success)$/) - assert.strictEqual(Deferred.isDoneUnsafe(received), false) - assert.strictEqual((yield* server.ingress.usage).reservedBytes, 0) - yield* Scope.close(clientScope, Exit.void) - yield* Scope.close(server.scope, Exit.void) - }))) - - it.effect("accepts a frame fragmented across every header and body byte", () => - Effect.scoped(Effect.gen(function*() { - const received = Deferred.makeUnsafe() - const server = yield* buildServer(testLimits()) - yield* Effect.forkIn( - server.protocol.run((clientId, message) => Deferred.succeed(received, [clientId, message] as const)), - server.scope - ) - const clientScope = yield* Scope.make() - const client = yield* connect(tcpPort(server.ingress.address), clientScope) - const encoded = frame(request(7, "fragmented")) - - for (const byte of encoded) { - yield* client.write(Uint8Array.of(byte)) - } - const [clientId, message] = yield* Deferred.await(received) - - assert.strictEqual(clientId, 0) - assert.strictEqual(message._tag, "Request") - assert.strictEqual( - message._tag === "Request" ? message.id : undefined, - 7 - ) - yield* Scope.close(clientScope, Exit.void) - yield* Scope.close(server.scope, Exit.void) - assert.strictEqual((yield* server.ingress.usage).reservedBytes, 0) - }))) - - it.effect("times out a slow incomplete frame and releases its connection", () => - Effect.scoped(Effect.gen(function*() { - const server = yield* buildServer(testLimits({ incompleteFrameTimeoutMillis: 500 })) - yield* Effect.forkIn(server.protocol.run(() => Effect.void), server.scope) - const clientScope = yield* Scope.make() - const client = yield* connect(tcpPort(server.ingress.address), clientScope) - - const partial = new Uint8Array(5) - new DataView(partial.buffer).setUint32(0, 10, false) - partial[4] = 123 - yield* client.write(partial) - yield* awaitConnections(server.ingress, 1) - yield* awaitReservedBytes(server.ingress) - yield* TestClock.adjust(500) - const clientExit = yield* Fiber.await(client.readFiber!) - - assert.match(clientExit._tag, /^(Failure|Success)$/) - assert.deepStrictEqual(yield* server.ingress.usage, { - connections: 0, - reservedBytes: 0, - byteReservationWaiters: 0 - }) - yield* Scope.close(clientScope, Exit.void) - yield* Scope.close(server.scope, Exit.void) - }))) - - it.effect("decodes many maximum sized fragmented frames without exceeding the shared budget", () => - Effect.scoped(Effect.gen(function*() { - const count = 8 - const maximumFrameBytes = 4 * 1_024 * 1_024 - const received = yield* Deferred.make() - let seen = 0 - const server = yield* buildServer(testLimits({ - maximumDeclaredFrameBytes: maximumFrameBytes, - maximumRawChunkBytes: maximumFrameBytes + 4, - maximumIncompleteFrameBytes: maximumFrameBytes + 4 - })) - yield* Effect.forkIn( - server.protocol.run(() => - Effect.sync(() => { - seen++ - if (seen === count) Deferred.doneUnsafe(received, Effect.succeed(seen)) - }) - ), - server.scope - ) - const clientScope = yield* Scope.make() - const client = yield* connect(tcpPort(server.ingress.address), clientScope) - - for (let id = 0; id < count; id++) { - const empty = frame(request(id)) - const encoded = frame(request(id, "x".repeat(maximumFrameBytes + 4 - empty.byteLength))) - assert.strictEqual(encoded.byteLength, maximumFrameBytes + 4) - yield* client.write(encoded.subarray(0, 4)) - yield* client.write(encoded.subarray(4)) - } - assert.strictEqual(yield* Deferred.await(received), count) - const usage = yield* server.ingress.usage - assert.isAtMost(usage.reservedBytes, PeerRelayLimits.defaults.maximumSharedPayloadBytes) - assert.strictEqual(usage.byteReservationWaiters, 0) - - yield* Scope.close(clientScope, Exit.void) - yield* Scope.close(server.scope, Exit.void) - assert.strictEqual((yield* server.ingress.usage).reservedBytes, 0) - }))) - - it.effect("caps connections and closes saturated clients", () => - Effect.scoped(Effect.gen(function*() { - const values = testLimits({ - maxRelayConnections: 1, - maximumByteReservationWaiters: 1, - maxSessionsPerSubject: 1, - maxInFlightOpen: 1, - maxInFlightOpenPerSubject: 1 - }) - const server = yield* buildServer(values) - yield* Effect.forkIn(server.protocol.run(() => Effect.void), server.scope) - const firstScope = yield* Scope.make() - const first = yield* connect(tcpPort(server.ingress.address), firstScope) - yield* first.write(Uint8Array.of(0)) - yield* awaitConnections(server.ingress, 1) - - const secondScope = yield* Scope.make() - const second = yield* connect(tcpPort(server.ingress.address), secondScope) - const secondExit = yield* Fiber.await(second.readFiber!) - - assert.match(secondExit._tag, /^(Failure|Success)$/) - assert.strictEqual((yield* server.ingress.usage).connections, 1) - yield* Scope.close(secondScope, Exit.void) - yield* Scope.close(firstScope, Exit.void) - yield* Scope.close(server.scope, Exit.void) - }))) - - it.effect("bounds stalled outbound reservations and releases every waiter on cleanup", () => - Effect.scoped(Effect.gen(function*() { - const server = yield* buildServer(testLimits()) - const reservation = yield* server.ingress.reserveOutbound( - PeerRelayLimits.defaults.maximumSharedPayloadBytes - ) - const waiting = yield* Effect.forkChild( - server.ingress.reserveOutbound(1) - ) - yield* Effect.yieldNow - - assert.deepStrictEqual(yield* server.ingress.usage, { - connections: 0, - reservedBytes: PeerRelayLimits.defaults.maximumSharedPayloadBytes, - byteReservationWaiters: 1 - }) - yield* Fiber.interrupt(waiting) - yield* reservation.release - assert.deepStrictEqual(yield* server.ingress.usage, { - connections: 0, - reservedBytes: 0, - byteReservationWaiters: 0 - }) - yield* Scope.close(server.scope, Exit.void) - }))) - - it.effect("releases an interrupted reservation during ownership handoff", () => - Effect.scoped(Effect.gen(function*() { - const server = yield* buildServer(testLimits()) - const reservation = yield* Effect.forkDetach( - server.ingress.reserveOutbound(1).pipe( - Effect.provideService(Scheduler.MaxOpsBeforeYield, 8) - ) - ) - let observedHandoff = false - for (let attempt = 0; attempt < 10_000; attempt++) { - const usage = yield* server.ingress.usage - if (usage.reservedBytes === 1 && reservation.pollUnsafe() === undefined) { - observedHandoff = true - break - } - yield* Effect.yieldNow - } - assert.strictEqual(observedHandoff, true) - - yield* Fiber.interrupt(reservation) - assert.deepStrictEqual(yield* server.ingress.usage, { - connections: 0, - reservedBytes: 0, - byteReservationWaiters: 0 - }) - yield* Scope.close(server.scope, Exit.void) - }))) - - it.effect("retains active request capacity through a defect and rejects unknown client tags", () => - Effect.scoped(Effect.gen(function*() { - const received = Deferred.makeUnsafe() - const defectSent = Deferred.makeUnsafe() - const messages: Array = [] - const server = yield* buildServer(testLimits({ - maximumRawChunkBytes: 2 * 1_024 - })) - yield* Effect.forkIn( - server.protocol.run((clientId, message) => - Effect.gen(function*() { - messages.push(message) - if (message._tag !== "Request") return - if (message.id === 1) { - Deferred.doneUnsafe(received, Effect.void) - return - } - if (message.id === 2) { - yield* server.protocol.send(clientId, { - _tag: "Defect", - defect: "request-defect" - }) - Deferred.doneUnsafe(defectSent, Effect.void) - } - }) - ), - server.scope - ) - const clientScope = yield* Scope.make() - const client = yield* connect(tcpPort(server.ingress.address), clientScope) - const encodedRequest = frame(request(1, "x".repeat(1_024))) - - yield* client.write(encodedRequest) - yield* Deferred.await(received) - assert.strictEqual( - (yield* server.ingress.usage).reservedBytes, - encodedRequest.byteLength - 4 - ) - - yield* client.write(frame(request(2))) - yield* Deferred.await(defectSent) - assert.strictEqual( - (yield* server.ingress.usage).reservedBytes, - encodedRequest.byteLength - 4 - ) - - yield* client.write(frame({ _tag: "Bogus" })) - const clientExit = yield* Fiber.await(client.readFiber!) - - assert.match(clientExit._tag, /^(Failure|Success)$/) - assert.deepStrictEqual(messages.map((message) => message._tag), ["Request", "Request"]) - assert.deepStrictEqual(yield* server.ingress.usage, { - connections: 0, - reservedBytes: 0, - byteReservationWaiters: 0 - }) - yield* Scope.close(clientScope, Exit.void) - yield* Scope.close(server.scope, Exit.void) - }))) - - it.effect("reserves outbound capacity before serializing a response", () => - Effect.scoped(Effect.gen(function*() { - const values = testLimits() - const server = yield* buildServer(values) - yield* Effect.forkIn(server.protocol.run(() => Effect.void), server.scope) - const clientScope = yield* Scope.make() - yield* connect(tcpPort(server.ingress.address), clientScope) - yield* awaitConnections(server.ingress, 1) - const [clientId] = yield* server.protocol.clientIds - assert.isDefined(clientId) - const blocker = yield* server.ingress.reserveOutbound( - values.maximumSharedPayloadBytes - ) - const serialized = Deferred.makeUnsafe() - const response: RpcMessage.FromServerEncoded = { - _tag: "Defect", - defect: { - toJSON() { - Deferred.doneUnsafe(serialized, Effect.void) - return "serialized" - } - } - } - - const send = yield* Effect.forkChild(server.protocol.send(clientId!, response)) - yield* awaitByteReservationWaiters(server.ingress, 1) - - assert.strictEqual(Deferred.isDoneUnsafe(serialized), false) - yield* Fiber.interrupt(send) - yield* blocker.release - assert.deepStrictEqual(yield* server.ingress.usage, { - connections: 1, - reservedBytes: 0, - byteReservationWaiters: 0 - }) - yield* Scope.close(clientScope, Exit.void) - yield* Scope.close(server.scope, Exit.void) - }))) - - it.effect("rejects overlapping raw chunks instead of accumulating parser fibers", () => - Effect.scoped(Effect.gen(function*() { - const values = testLimits() - const server = yield* buildServer(values) - yield* Effect.forkIn(server.protocol.run(() => Effect.void), server.scope) - const blocker = yield* server.ingress.reserveOutbound( - values.maximumSharedPayloadBytes - ) - const clientScope = yield* Scope.make() - const client = yield* connect(tcpPort(server.ingress.address), clientScope) - const header = new Uint8Array(4) - new DataView(header.buffer).setUint32(0, 10, false) - - yield* client.write(header) - yield* awaitByteReservationWaiters(server.ingress, 1) - yield* client.write(Uint8Array.of(123)).pipe(Effect.ignore) - yield* Fiber.await(client.readFiber!) - yield* awaitConnections(server.ingress, 0) - - assert.deepStrictEqual(yield* server.ingress.usage, { - connections: 0, - reservedBytes: values.maximumSharedPayloadBytes, - byteReservationWaiters: 0 - }) - yield* blocker.release - assert.deepStrictEqual(yield* server.ingress.usage, { - connections: 0, - reservedBytes: 0, - byteReservationWaiters: 0 - }) - yield* Scope.close(clientScope, Exit.void) - yield* Scope.close(server.scope, Exit.void) - }))) - - it.effect("releases a transferred reservation when an outbound write wait is interrupted", () => - Effect.scoped(Effect.gen(function*() { - const values = testLimits() - const server = yield* buildServer(values) - const ready = Deferred.makeUnsafe<{ - readonly blocker: PeerRelayIngress.Reservation - readonly clientId: number - }>() - const input = request(1) - const inboundBytes = frame(input).byteLength - 4 - yield* Effect.forkIn( - server.protocol.run((clientId) => - Effect.gen(function*() { - const transferred = yield* server.ingress.reserveOutbound(1) - yield* transferred.transferToCurrentRequest - const blocker = yield* server.ingress.reserveOutbound( - values.maximumSharedPayloadBytes - inboundBytes - 1 - ) - yield* Deferred.succeed(ready, { blocker, clientId }) - }).pipe(Effect.orDie) - ), - server.scope - ) - const clientScope = yield* Scope.make() - const client = yield* connect(tcpPort(server.ingress.address), clientScope) - yield* client.write(frame(input)) - const { blocker, clientId } = yield* Deferred.await(ready) - - const send = yield* Effect.forkChild( - server.protocol.send(clientId, { - _tag: "Chunk", - requestId: 1, - values: ["requires-more-than-one-byte"] - }) - ) - yield* Effect.yieldNow - assert.strictEqual( - (yield* server.ingress.usage).reservedBytes, - values.maximumSharedPayloadBytes - ) - yield* Fiber.interrupt(send) - yield* blocker.release - yield* Scope.close(clientScope, Exit.void) - yield* awaitConnections(server.ingress, 0) - assert.deepStrictEqual(yield* server.ingress.usage, { - connections: 0, - reservedBytes: 0, - byteReservationWaiters: 0 - }) - yield* Scope.close(server.scope, Exit.void) - }))) - - it.effect("rejects reservation transfer after client disconnect cleanup", () => - Effect.scoped(Effect.gen(function*() { - const server = yield* buildServer(testLimits()) - const ready = Deferred.makeUnsafe() - const transfer = Deferred.makeUnsafe() - const transferSucceeded = Deferred.makeUnsafe() - yield* Effect.forkIn( - server.protocol.run((clientId) => - Effect.gen(function*() { - const reservation = yield* server.ingress.reserveOutbound(1) - yield* Deferred.succeed(ready, clientId) - yield* Effect.uninterruptible( - Effect.gen(function*() { - yield* Deferred.await(transfer) - const succeeded = yield* reservation.transferToCurrentRequest.pipe( - Effect.match({ - onFailure: () => false, - onSuccess: () => true - }) - ) - if (!succeeded) yield* reservation.release - yield* Deferred.succeed(transferSucceeded, succeeded) - }) - ) - }).pipe(Effect.orDie) - ), - server.scope - ) - const clientScope = yield* Scope.make() - const client = yield* connect(tcpPort(server.ingress.address), clientScope) - yield* client.write(frame(request(1))) - const clientId = yield* Deferred.await(ready) - - yield* server.protocol.end(clientId) - yield* Deferred.succeed(transfer, undefined) - assert.strictEqual(yield* Deferred.await(transferSucceeded), false) - yield* Fiber.await(client.readFiber!) - yield* awaitConnections(server.ingress, 0) - - assert.deepStrictEqual(yield* server.ingress.usage, { - connections: 0, - reservedBytes: 0, - byteReservationWaiters: 0 - }) - yield* Scope.close(clientScope, Exit.void) - yield* Scope.close(server.scope, Exit.void) - }))) - - it.effect("keeps accepting connections when a waiting protocol runner takes ownership", () => - Effect.scoped(Effect.gen(function*() { - const firstReceived = Deferred.makeUnsafe() - const secondReceived = Deferred.makeUnsafe() - const server = yield* buildServer(testLimits()) - const firstRunner = yield* Effect.forkIn( - server.protocol.run(() => Deferred.succeed(firstReceived, undefined)), - server.scope - ) - const firstClientScope = yield* Scope.make() - const firstClient = yield* connect(tcpPort(server.ingress.address), firstClientScope) - yield* firstClient.write(frame({ _tag: "Ping" })) - yield* Deferred.await(firstReceived) - - yield* Effect.forkIn( - server.protocol.run(() => Deferred.succeed(secondReceived, undefined)), - server.scope - ) - yield* Effect.yieldNow - yield* Fiber.interrupt(firstRunner) - - const secondClientScope = yield* Scope.make() - const secondClient = yield* connect(tcpPort(server.ingress.address), secondClientScope) - yield* secondClient.write(frame({ _tag: "Ping" })) - yield* Deferred.await(secondReceived) - - assert.strictEqual((yield* server.ingress.usage).connections, 2) - yield* Scope.close(secondClientScope, Exit.void) - yield* Scope.close(firstClientScope, Exit.void) - yield* Scope.close(server.scope, Exit.void) - }))) - - it.effect("reserves disconnect queue capacity before accepting a connection", () => - Effect.scoped(Effect.gen(function*() { - const server = yield* buildServer(testLimits({ - maxRelayConnections: 1, - maximumByteReservationWaiters: 1, - maxSessionsPerSubject: 1, - maxInFlightOpen: 1, - maxInFlightOpenPerSubject: 1 - })) - yield* Effect.forkIn(server.protocol.run(() => Effect.void), server.scope) - - const firstScope = yield* Scope.make() - yield* connect(tcpPort(server.ingress.address), firstScope) - yield* awaitConnections(server.ingress, 1) - yield* Scope.close(firstScope, Exit.void) - yield* awaitConnections(server.ingress, 0) - - const secondScope = yield* Scope.make() - const second = yield* connect(tcpPort(server.ingress.address), secondScope) - const secondExit = yield* Fiber.await(second.readFiber!) - - assert.match(secondExit._tag, /^(Failure|Success)$/) - assert.strictEqual((yield* server.ingress.usage).connections, 0) - yield* Scope.close(secondScope, Exit.void) - yield* Scope.close(server.scope, Exit.void) - }))) - - it.effect("rejects pre-run connection churn without filling the disconnect queue", () => - Effect.scoped(Effect.gen(function*() { - const values = testLimits({ - maxRelayConnections: 1, - maximumByteReservationWaiters: 1, - maxSessionsPerSubject: 1, - maxInFlightOpen: 1, - maxInFlightOpenPerSubject: 1 - }) - const server = yield* buildServer(values) - for (let attempt = 0; attempt < 8; attempt++) { - const rejectedScope = yield* Scope.make() - const rejected = yield* connect(tcpPort(server.ingress.address), rejectedScope) - yield* Fiber.await(rejected.readFiber!) - yield* Scope.close(rejectedScope, Exit.void) - } - assert.strictEqual((yield* server.ingress.usage).connections, 0) - - yield* Effect.forkIn(server.protocol.run(() => Effect.void), server.scope) - const acceptedScope = yield* Scope.make() - const accepted = yield* connect(tcpPort(server.ingress.address), acceptedScope) - yield* accepted.write(Uint8Array.of(0)) - yield* awaitConnections(server.ingress, 1) - yield* Scope.close(acceptedScope, Exit.void) - yield* Scope.close(server.scope, Exit.void) - }))) - - it.effect("reports normal client EOF and fails later sends promptly", () => - Effect.scoped(Effect.gen(function*() { - const values = testLimits() - const server = yield* buildServer(values) - yield* Effect.forkIn(server.protocol.run(() => Effect.void), server.scope) - const clientScope = yield* Scope.make() - const socketContext = yield* Layer.buildWithScope( - NodeSocket.layerNet({ port: tcpPort(server.ingress.address) }), - clientScope - ) - const socket = Context.get(socketContext, Socket.Socket) - const clientProtocol = yield* Scope.provide( - PeerRelayIngress.makeProtocolSocket.pipe( - Effect.provideService(Socket.Socket, socket), - Effect.provideService(PeerRelayLimits.PeerRelayLimits, values) - ), - clientScope - ) - const protocolFailure = Deferred.makeUnsafe() - yield* Effect.forkIn( - clientProtocol.run(0, (message) => - message._tag === "ClientProtocolError" - ? Deferred.succeed(protocolFailure, message) - : Effect.void), - clientScope - ) - yield* awaitConnections(server.ingress, 1) - yield* Scope.close(server.scope, Exit.void) - - const failure = yield* Deferred.await(protocolFailure) - assert.strictEqual(failure._tag, "ClientProtocolError") - const sendExit = yield* Effect.exit( - clientProtocol.send(0, { _tag: "Ping" }) - ) - assert.strictEqual(sendExit._tag, "Failure") - yield* Scope.close(clientScope, Exit.void) - }))) - - it.effect("closes a partially built socket layer and can bind the same address again", () => - Effect.scoped(Effect.gen(function*() { - const probeScope = yield* Scope.make() - const probe = yield* Scope.provide(NodeSocketServer.make({ port: 0 }), probeScope) - const port = tcpPort(probe.address) - yield* Scope.close(probeScope, Exit.void) - - const partial = Layer.effectContext( - Effect.gen(function*() { - yield* NodeSocketServer.make({ port }) - return yield* Effect.fail("partial-build") - }) - ) as Layer.Layer - const failedScope = yield* Scope.make() - const failed = yield* Effect.exit( - Layer.buildWithScope( - PeerRelayIngress.layerProtocolSocketServer(partial).pipe( - Layer.provide(PeerRelayLimits.layer(testLimits())) - ), - failedScope - ) - ) - assert.strictEqual(failed._tag, "Failure") - yield* Scope.close(failedScope, failed) - - const restarted = yield* buildServer(testLimits(), port) - assert.strictEqual(tcpPort(restarted.ingress.address), port) - yield* Scope.close(restarted.scope, Exit.void) - }))) - - it.effect("closes the child listener scope and restarts on the retained port", () => - Effect.scoped(Effect.gen(function*() { - const first = yield* buildServer(testLimits()) - const port = tcpPort(first.ingress.address) - yield* Scope.close(first.scope, Exit.void) - const firstExit = yield* Effect.exit(first.ingress.await) - assert.strictEqual(firstExit._tag, "Failure") - - const second = yield* buildServer(testLimits(), port) - assert.strictEqual(tcpPort(second.ingress.address), port) - yield* Scope.close(second.scope, Exit.void) - }))) -}) diff --git a/packages/local-rpc/test/PeerRelaySqliteTransaction.test.ts b/packages/local-rpc/test/PeerRelaySqliteTransaction.test.ts deleted file mode 100644 index 8de51b4..0000000 --- a/packages/local-rpc/test/PeerRelaySqliteTransaction.test.ts +++ /dev/null @@ -1,278 +0,0 @@ -import { SqliteClient } from "@effect/sql-sqlite-node" -import { assert, describe, it } from "@effect/vitest" -import * as Cause from "effect/Cause" -import * as Effect from "effect/Effect" -import * as Exit from "effect/Exit" -import * as Result from "effect/Result" -import * as Scope from "effect/Scope" -import * as SqlClient from "effect/unstable/sql/SqlClient" -import type * as SqlConnection from "effect/unstable/sql/SqlConnection" -import * as SqlError from "effect/unstable/sql/SqlError" -import { cleanupAcquisition, makeSqlite as make } from "../src/internal/peerRelaySqlTransaction.js" - -describe("peerRelaySqliteTransaction", () => { - it.effect("preserves interruption together with rollback failure during acquisition", () => - Effect.gen(function*() { - const rollbackDefect = new Error("rollback failed") - const connection = { - executeUnprepared: (statement: string) => { - if (statement === "ROLLBACK") { - return Effect.die(rollbackDefect) - } - return Effect.succeed([]) - } - } as unknown as SqlConnection.Connection - const scope = yield* Scope.make("sequential") - const exit = yield* cleanupAcquisition( - Cause.interrupt(99_023), - scope, - connection, - true - ).pipe(Effect.exit) - assert.strictEqual(Exit.isFailure(exit), true) - if (Exit.isFailure(exit)) { - assert.strictEqual(Cause.hasInterrupts(exit.cause), true) - assert.strictEqual(Result.getOrThrow(Cause.findDefect(exit.cause)), rollbackDefect) - } - })) - - it.effect("does not retry LockTimeoutError when Scope release also defects", () => - Effect.gen(function*() { - const closeDefect = new Error("scope close failed") - const lock = new SqlError.SqlError({ - reason: new SqlError.LockTimeoutError({ cause: new Error("busy") }) - }) - let begins = 0 - const connection = { - executeUnprepared: (statement: string) => { - if (statement === "BEGIN IMMEDIATE") { - begins++ - return Effect.fail(lock) - } - return Effect.succeed([]) - } - } as unknown as SqlConnection.Connection - const reserve = Effect.gen(function*() { - const scope = yield* Scope.Scope - yield* Scope.addFinalizer(scope, Effect.die(closeDefect)) - return connection - }) - const sql = { - reserve, - transactionService: SqlClient.TransactionConnection(999_024) - } as unknown as SqlClient.SqlClient - const exit = yield* make(sql, { - maxAcquireAttempts: 2, - acquireRetryBaseDelayMillis: 0, - acquireRetryMaximumDelayMillis: 0 - })(Effect.void).pipe(Effect.exit) - assert.strictEqual(begins, 1) - assert.strictEqual(Exit.isFailure(exit), true) - if (Exit.isFailure(exit)) { - assert.strictEqual(Result.getOrThrow(Cause.findDefect(exit.cause)), closeDefect) - const failure = Result.getOrThrow(Cause.findError(exit.cause)) - assert.strictEqual(failure, lock) - } - })) - - it.effect("preserves a body failure together with rollback failure", () => - Effect.gen(function*() { - const bodyFailure = new Error("body failed") - const rollbackDefect = new Error("rollback failed") - const connection = { - executeUnprepared: (statement: string) => - statement === "ROLLBACK" - ? Effect.die(rollbackDefect) - : Effect.succeed([]) - } as unknown as SqlConnection.Connection - const sql = { - reserve: Effect.succeed(connection), - transactionService: SqlClient.TransactionConnection(999_025) - } as unknown as SqlClient.SqlClient - const exit = yield* make(sql, { - maxAcquireAttempts: 1, - acquireRetryBaseDelayMillis: 0, - acquireRetryMaximumDelayMillis: 0 - })(Effect.fail(bodyFailure)).pipe(Effect.exit) - assert.strictEqual(Exit.isFailure(exit), true) - if (Exit.isFailure(exit)) { - assert.strictEqual(Result.getOrThrow(Cause.findError(exit.cause)), bodyFailure) - assert.strictEqual(Result.getOrThrow(Cause.findDefect(exit.cause)), rollbackDefect) - } - })) - - it.effect("preserves a body interruption together with rollback failure", () => - Effect.gen(function*() { - const rollbackDefect = new Error("rollback failed") - const connection = { - executeUnprepared: (statement: string) => - statement === "ROLLBACK" - ? Effect.die(rollbackDefect) - : Effect.succeed([]) - } as unknown as SqlConnection.Connection - const sql = { - reserve: Effect.succeed(connection), - transactionService: SqlClient.TransactionConnection(999_026) - } as unknown as SqlClient.SqlClient - const exit = yield* make(sql, { - maxAcquireAttempts: 1, - acquireRetryBaseDelayMillis: 0, - acquireRetryMaximumDelayMillis: 0 - })(Effect.interrupt).pipe(Effect.exit) - assert.strictEqual(Exit.isFailure(exit), true) - if (Exit.isFailure(exit)) { - assert.strictEqual(Cause.hasInterrupts(exit.cause), true) - assert.strictEqual(Result.getOrThrow(Cause.findDefect(exit.cause)), rollbackDefect) - } - })) - - it.effect("preserves body, rollback, and transaction scope close failures", () => - Effect.gen(function*() { - const bodyFailure = new Error("body failed") - const rollbackDefect = new Error("rollback failed") - const closeDefect = new Error("scope close failed") - const connection = { - executeUnprepared: (statement: string) => - statement === "ROLLBACK" - ? Effect.die(rollbackDefect) - : Effect.succeed([]) - } as unknown as SqlConnection.Connection - const reserve = Effect.gen(function*() { - const scope = yield* Scope.Scope - yield* Scope.addFinalizer(scope, Effect.die(closeDefect)) - return connection - }) - const sql = { - reserve, - transactionService: SqlClient.TransactionConnection(999_027) - } as unknown as SqlClient.SqlClient - const exit = yield* make(sql, { - maxAcquireAttempts: 1, - acquireRetryBaseDelayMillis: 0, - acquireRetryMaximumDelayMillis: 0 - })(Effect.fail(bodyFailure)).pipe(Effect.exit) - assert.strictEqual(Exit.isFailure(exit), true) - if (Exit.isFailure(exit)) { - assert.strictEqual(Result.getOrThrow(Cause.findError(exit.cause)), bodyFailure) - const defects = new Set( - exit.cause.reasons - .filter(Cause.isDieReason) - .map((reason) => reason.defect) - ) - assert.strictEqual(defects.has(rollbackDefect), true) - assert.strictEqual(defects.has(closeDefect), true) - } - })) - - it.effect("rolls back a deferred constraint commit failure before reusing the connection", () => - Effect.gen(function*() { - const sql = yield* SqlClient.SqlClient - const transaction = make(sql, { - maxAcquireAttempts: 1, - acquireRetryBaseDelayMillis: 0, - acquireRetryMaximumDelayMillis: 0 - }) - yield* sql`PRAGMA foreign_keys = ON` - yield* sql`CREATE TABLE parents ( - parent_id INTEGER PRIMARY KEY - )` - yield* sql`CREATE TABLE children ( - child_id INTEGER PRIMARY KEY, - parent_id INTEGER NOT NULL, - FOREIGN KEY (parent_id) REFERENCES parents(parent_id) - DEFERRABLE INITIALLY DEFERRED - )` - - const failed = yield* transaction( - sql`INSERT INTO children (child_id, parent_id) VALUES (1, 99)` - ).pipe(Effect.exit) - assert.strictEqual(Exit.isFailure(failed), true) - - const succeeded = yield* transaction( - sql`INSERT INTO parents (parent_id) VALUES (1)` - ).pipe(Effect.exit) - assert.strictEqual(Exit.isSuccess(succeeded), true) - - const children = yield* sql<{ readonly count: number }>` - SELECT COUNT(*) AS count FROM children - ` - const parents = yield* sql<{ readonly count: number }>` - SELECT COUNT(*) AS count FROM parents - ` - assert.strictEqual(children[0]?.count, 0) - assert.strictEqual(parents[0]?.count, 1) - }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:", disableWAL: true })))) - - it.effect("preserves commit, rollback, and transaction scope close failures", () => - Effect.gen(function*() { - const commitDefect = new Error("commit failed") - const rollbackDefect = new Error("rollback failed") - const closeDefect = new Error("scope close failed") - const connection = { - executeUnprepared: (statement: string) => { - if (statement === "COMMIT") { - return Effect.die(commitDefect) - } - if (statement === "ROLLBACK") { - return Effect.die(rollbackDefect) - } - return Effect.succeed([]) - } - } as unknown as SqlConnection.Connection - const reserve = Effect.gen(function*() { - const scope = yield* Scope.Scope - yield* Scope.addFinalizer(scope, Effect.die(closeDefect)) - return connection - }) - const sql = { - reserve, - transactionService: SqlClient.TransactionConnection(999_028) - } as unknown as SqlClient.SqlClient - const exit = yield* make(sql, { - maxAcquireAttempts: 1, - acquireRetryBaseDelayMillis: 0, - acquireRetryMaximumDelayMillis: 0 - })(Effect.void).pipe(Effect.exit) - assert.strictEqual(Exit.isFailure(exit), true) - if (Exit.isFailure(exit)) { - const defects = new Set( - exit.cause.reasons - .filter(Cause.isDieReason) - .map((reason) => reason.defect) - ) - assert.strictEqual(defects.has(commitDefect), true) - assert.strictEqual(defects.has(rollbackDefect), true) - assert.strictEqual(defects.has(closeDefect), true) - } - })) - - it.effect("does not duplicate a typed commit failure as a defect", () => - Effect.gen(function*() { - const commitFailure = new SqlError.SqlError({ - reason: new SqlError.UnknownError({ cause: new Error("commit failed") }) - }) - const connection = { - executeUnprepared: (statement: string) => - statement === "COMMIT" - ? Effect.fail(commitFailure) - : Effect.succeed([]) - } as unknown as SqlConnection.Connection - const sql = { - reserve: Effect.succeed(connection), - transactionService: SqlClient.TransactionConnection(999_029) - } as unknown as SqlClient.SqlClient - const exit = yield* make(sql, { - maxAcquireAttempts: 1, - acquireRetryBaseDelayMillis: 0, - acquireRetryMaximumDelayMillis: 0 - })(Effect.void).pipe(Effect.exit) - assert.strictEqual(Exit.isFailure(exit), true) - if (Exit.isFailure(exit)) { - assert.strictEqual(Result.getOrThrow(Cause.findError(exit.cause)), commitFailure) - assert.strictEqual( - exit.cause.reasons.some((reason) => Cause.isDieReason(reason) && reason.defect === commitFailure), - false - ) - } - })) -}) diff --git a/packages/local-rpc/test/PeerRelayStore.test.ts b/packages/local-rpc/test/PeerRelayStore.test.ts deleted file mode 100644 index 993004d..0000000 --- a/packages/local-rpc/test/PeerRelayStore.test.ts +++ /dev/null @@ -1,1189 +0,0 @@ -import { NodeCrypto } from "@effect/platform-node" -import { SqliteClient } from "@effect/sql-sqlite-node" -import { assert, describe, it } from "@effect/vitest" -import * as Identity from "@lucas-barake/effect-local/Identity" -import * as Effect from "effect/Effect" -import * as Exit from "effect/Exit" -import * as Layer from "effect/Layer" -import * as Metric from "effect/Metric" -import * as Option from "effect/Option" -import * as SqlClient from "effect/unstable/sql/SqlClient" -import { rmSync } from "node:fs" -import { tmpdir } from "node:os" -import { join } from "node:path" -import * as PeerRpcObservability from "../src/internal/peerRpcObservability.js" -import * as PeerRelayLimits from "../src/PeerRelayLimits.js" -import * as PeerRelayStore from "../src/PeerRelayStore.js" -import * as PeerRpc from "../src/PeerRpc.js" -import * as SqlPeerRelayStore from "../src/SqlPeerRelayStore.js" - -const peer = (value: string) => Identity.PeerId.make(`peer_00000000-0000-4000-8000-${value}`) -const relayId = (value: string) => Identity.RelayMessageId.make(`rly_00000000-0000-4000-8000-${value}`) -const documentId = (value: string) => Identity.DocumentId.make(`doc_00000000-0000-4000-8000-${value}`) - -const makeLayer = ( - filename: string, - limits: PeerRelayLimits.Values = PeerRelayLimits.defaults -) => { - const base = Layer.mergeAll( - SqliteClient.layer({ filename }), - NodeCrypto.layer, - PeerRelayLimits.layer(limits) - ) - const store = SqlPeerRelayStore.layer.pipe(Layer.provide(base)) - return Layer.merge(base, store) -} - -const withStore = ( - effect: Effect.Effect< - A, - E, - PeerRelayStore.PeerRelayStore | SqlClient.SqlClient - >, - limits: PeerRelayLimits.Values = PeerRelayLimits.defaults -) => - Effect.gen(function*() { - const filename = join(tmpdir(), `effect-local-relay-${globalThis.crypto.randomUUID()}.sqlite`) - yield* Effect.addFinalizer(() => - Effect.sync(() => { - rmSync(filename, { force: true }) - rmSync(`${filename}-shm`, { force: true }) - rmSync(`${filename}-wal`, { force: true }) - }) - ) - return yield* Effect.scoped(effect.pipe(Effect.provide(makeLayer(filename, limits)))) - }) - -describe("PeerRelayStore", () => { - it.effect("records fixed relay outcomes, acknowledgement latency, and exact pending gauges", () => { - const registry = new Map() - const limits = PeerRelayLimits.Values.make({ - ...PeerRelayLimits.defaults, - maxActiveMessagesPerSenderPeer: 1, - maxRetainedRowsPerSenderPeer: 1 - }) - const metricValue = (metric: Metric.Metric) => - Metric.value(metric).pipe( - Effect.provideService(Metric.CurrentMetricAttributes, {}) - ) - return withStore( - Effect.gen(function*() { - const store = yield* PeerRelayStore.PeerRelayStore - const sql = yield* SqlClient.SqlClient - const channel = PeerRelayStore.ChannelKey.make({ - tenantId: "tenant-observe", - senderSubjectId: "sender-observe", - senderPeerId: peer("000000000051"), - senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), - recipientSubjectId: "recipient-observe", - recipientPeerId: peer("000000000052") - }) - const admission = PeerRelayStore.Admission.make({ - channel, - relayMessageId: relayId("000000000051"), - relayPeerId: peer("000000000053"), - documentIds: [documentId("000000000051")], - senderConnectionEpoch: "epoch-observe", - senderSequence: 0, - payloadVersion: 1, - messageHash: "message-hash-observe", - outerEnvelopeDigest: PeerRpc.RelayDigest.make("5".repeat(64)), - payload: new Uint8Array([5]), - messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, - senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, - minimumTerminalRetentionMillis: PeerRelayLimits.defaults.minimumTerminalRetentionMillis - }) - yield* store.admit(admission) - const claimed = yield* store.claim({ - recipient: { - tenantId: channel.tenantId, - subjectId: channel.recipientSubjectId, - peerId: channel.recipientPeerId - }, - sender: { - subjectId: channel.senderSubjectId, - peerId: channel.senderPeerId - }, - sessionGeneration: 1, - authorizedDocumentIds: admission.documentIds - }) - assert.strictEqual(Option.isSome(claimed.message), true) - if (Option.isNone(claimed.message)) return - const message = claimed.message.value - yield* sql`UPDATE effect_local_relay_messages - SET created_at = 0 - WHERE message_id = ${message.rowId}` - yield* store.acknowledge({ - channel, - relayMessageId: message.relayMessageId, - claimToken: message.claimToken, - messageHash: message.messageHash, - sessionGeneration: message.sessionGeneration, - recipient: { - tenantId: channel.tenantId, - subjectId: channel.recipientSubjectId, - peerId: channel.recipientPeerId - } - }) - yield* store.usage() - const rejected = yield* store.admit(PeerRelayStore.Admission.make({ - ...admission, - relayMessageId: relayId("000000000054"), - outerEnvelopeDigest: PeerRpc.RelayDigest.make("4".repeat(64)) - })).pipe(Effect.exit) - assert.strictEqual(Exit.isFailure(rejected), true) - assert.strictEqual( - (yield* metricValue( - PeerRpcObservability.relayOutcomes("RelayAdmit", "Send", "Accepted") - )).count, - 1 - ) - assert.strictEqual( - (yield* metricValue( - PeerRpcObservability.relayOutcomes("RelayClaim", "Receive", "Claimed") - )).count, - 1 - ) - assert.strictEqual( - (yield* metricValue( - PeerRpcObservability.relayOutcomes( - "RelayAcknowledge", - "Receive", - "Acknowledged" - ) - )).count, - 1 - ) - assert.strictEqual( - (yield* metricValue( - PeerRpcObservability.relayLatencyMillis( - "RelayAcknowledge", - "Receive", - "Acknowledged" - ) - )).count, - 1 - ) - assert.strictEqual( - (yield* metricValue(PeerRpcObservability.relayPendingItems())).value, - 0 - ) - assert.strictEqual( - (yield* metricValue(PeerRpcObservability.relayPendingBytes())).value, - 0 - ) - assert.strictEqual( - (yield* metricValue( - PeerRpcObservability.relayQuotaRejections("SenderPeer") - )).count, - 1 - ) - }), - limits - ).pipe(Effect.provideService(Metric.MetricRegistry, registry)) - }) - - it.effect("migrates a real WAL database and fences claim payload loading and terminal duplicates", () => - Effect.gen(function*() { - const filename = join(tmpdir(), `effect-local-relay-${globalThis.crypto.randomUUID()}.sqlite`) - yield* Effect.addFinalizer(() => - Effect.sync(() => { - rmSync(filename, { force: true }) - rmSync(`${filename}-shm`, { force: true }) - rmSync(`${filename}-wal`, { force: true }) - }) - ) - yield* Effect.scoped( - Effect.gen(function*() { - const store = yield* PeerRelayStore.PeerRelayStore - const sql = yield* SqlClient.SqlClient - const channel = PeerRelayStore.ChannelKey.make({ - tenantId: "tenant", - senderSubjectId: "sender", - senderPeerId: peer("000000000001"), - senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), - recipientSubjectId: "recipient", - recipientPeerId: peer("000000000002") - }) - const payload = new Uint8Array([1, 2, 3, 4]) - const admission = PeerRelayStore.Admission.make({ - channel, - relayMessageId: relayId("000000000001"), - relayPeerId: peer("000000000003"), - documentIds: [documentId("000000000001")], - senderConnectionEpoch: "epoch-1", - senderSequence: 0, - payloadVersion: 1, - messageHash: "message-hash", - outerEnvelopeDigest: PeerRpc.RelayDigest.make("a".repeat(64)), - payload, - messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, - senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, - minimumTerminalRetentionMillis: PeerRelayLimits.defaults.minimumTerminalRetentionMillis - }) - const accepted = yield* store.admit(admission) - assert.strictEqual(accepted.status, "Accepted") - assert.deepStrictEqual(yield* store.usage(), { - activeCount: 1, - activeBytes: payload.byteLength, - retainedCount: 1, - retainedBytes: payload.byteLength - }) - const duplicate = yield* store.admit(admission) - assert.strictEqual(duplicate.status, "Duplicate") - assert.strictEqual((yield* store.usage()).activeCount, 1) - - const claimed = yield* store.claim({ - recipient: { - tenantId: "tenant", - subjectId: "recipient", - peerId: channel.recipientPeerId - }, - sender: { - subjectId: "sender", - peerId: channel.senderPeerId - }, - sessionGeneration: 7, - authorizedDocumentIds: admission.documentIds - }) - assert.strictEqual(Option.isSome(claimed.message), true) - if (Option.isNone(claimed.message)) return - const message = claimed.message.value - assert.strictEqual(Object.hasOwn(message, "payload"), false) - assert.strictEqual(message.payloadBytes, payload.byteLength) - const routeColumns = [ - ["tenant_id", "corrupt-tenant", channel.tenantId], - ["sender_subject_id", "corrupt-sender", channel.senderSubjectId], - ["sender_peer_id", "corrupt-peer", channel.senderPeerId] - ] as const - for (const [column, corrupt, original] of routeColumns) { - yield* sql.unsafe( - `UPDATE effect_local_relay_messages SET ${column} = ? WHERE message_id = ?`, - [corrupt, message.rowId] - ) - assert.strictEqual( - (yield* Effect.exit(store.loadClaimedPayload({ - rowId: message.rowId, - channel, - relayMessageId: message.relayMessageId, - claimToken: message.claimToken, - sessionGeneration: message.sessionGeneration, - payloadBytes: message.payloadBytes - })))._tag, - "Failure" - ) - yield* sql.unsafe( - `UPDATE effect_local_relay_messages SET ${column} = ? WHERE message_id = ?`, - [original, message.rowId] - ) - } - assert.deepStrictEqual( - yield* store.loadClaimedPayload({ - rowId: message.rowId, - channel, - relayMessageId: message.relayMessageId, - claimToken: message.claimToken, - sessionGeneration: message.sessionGeneration, - payloadBytes: message.payloadBytes - }), - payload - ) - const stalePayload = yield* Effect.exit(store.loadClaimedPayload({ - rowId: message.rowId, - channel, - relayMessageId: message.relayMessageId, - claimToken: message.claimToken, - sessionGeneration: message.sessionGeneration + 1, - payloadBytes: message.payloadBytes - })) - assert.strictEqual(stalePayload._tag, "Failure") - const terminal = { - channel, - relayMessageId: message.relayMessageId, - claimToken: message.claimToken, - messageHash: message.messageHash, - sessionGeneration: message.sessionGeneration, - recipient: { - tenantId: "tenant", - subjectId: "recipient", - peerId: channel.recipientPeerId - } - } as const - assert.strictEqual((yield* store.acknowledge(terminal)).status, "Changed") - assert.strictEqual((yield* store.acknowledge(terminal)).status, "Duplicate") - assert.strictEqual((yield* store.usage()).activeCount, 0) - assert.strictEqual((yield* store.usage()).retainedCount, 1) - const stored = yield* sql<{ readonly payload: Uint8Array | null }>` - SELECT payload FROM effect_local_relay_messages - ` - assert.strictEqual(stored[0]?.payload, null) - - const expiringAdmission = PeerRelayStore.Admission.make({ - ...admission, - relayMessageId: relayId("000000000002"), - outerEnvelopeDigest: PeerRpc.RelayDigest.make("b".repeat(64)) - }) - yield* store.admit(expiringAdmission) - const expiringClaim = yield* store.claim({ - recipient: terminal.recipient, - sender: { - subjectId: channel.senderSubjectId, - peerId: channel.senderPeerId - }, - sessionGeneration: 8, - authorizedDocumentIds: expiringAdmission.documentIds - }) - assert.strictEqual(Option.isSome(expiringClaim.message), true) - if (Option.isNone(expiringClaim.message)) return - const expiring = expiringClaim.message.value - const loadExpiring = () => - store.loadClaimedPayload({ - rowId: expiring.rowId, - channel, - relayMessageId: expiring.relayMessageId, - claimToken: expiring.claimToken, - sessionGeneration: expiring.sessionGeneration, - payloadBytes: expiring.payloadBytes - }) - const claimColumns = [ - ["claimed_message_id", String(expiring.rowId + 1)], - ["claim_token", "'clm_00000000-0000-4000-8000-000000000009'"], - ["claim_session_generation", String(expiring.sessionGeneration + 1)], - ["claim_deadline", String(expiring.claimDeadline + 1)] - ] as const - for (const [column, corrupt] of claimColumns) { - yield* sql.unsafe( - `UPDATE effect_local_relay_channels SET ${column} = ${corrupt} WHERE channel_id = ?`, - [1] - ) - assert.strictEqual((yield* Effect.exit(loadExpiring()))._tag, "Failure") - yield* sql`UPDATE effect_local_relay_channels - SET claimed_message_id = ${expiring.rowId}, - claim_token = ${expiring.claimToken}, - claim_session_generation = ${expiring.sessionGeneration}, - claim_deadline = ${expiring.claimDeadline} - WHERE channel_id = 1` - } - yield* sql`UPDATE effect_local_relay_messages - SET expires_at = CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) - 1 - WHERE message_id = ${expiring.rowId}` - assert.strictEqual((yield* Effect.exit(loadExpiring()))._tag, "Failure") - assert.strictEqual( - (yield* store.acknowledge({ - ...terminal, - relayMessageId: expiring.relayMessageId, - claimToken: expiring.claimToken, - sessionGeneration: expiring.sessionGeneration - })).status, - "Stale" - ) - yield* store.expire({ batchSize: 10 }) - const expired = yield* sql<{ readonly state: string }>` - SELECT state FROM effect_local_relay_messages WHERE message_id = ${expiring.rowId} - ` - assert.strictEqual(expired[0]?.state, "Expired") - - const corruptAdmission = PeerRelayStore.Admission.make({ - ...admission, - relayMessageId: relayId("000000000003"), - outerEnvelopeDigest: PeerRpc.RelayDigest.make("c".repeat(64)) - }) - yield* store.admit(corruptAdmission) - yield* sql`UPDATE effect_local_relay_messages - SET tenant_id = 'other' - WHERE relay_message_id = ${corruptAdmission.relayMessageId}` - const undisclosed = yield* store.claim({ - recipient: terminal.recipient, - sender: { - subjectId: channel.senderSubjectId, - peerId: channel.senderPeerId - }, - sessionGeneration: 9, - authorizedDocumentIds: corruptAdmission.documentIds - }) - assert.strictEqual(Option.isNone(undisclosed.message), true) - yield* store.repair({ batchSize: 10 }) - const repaired = yield* sql<{ readonly state: string }>` - SELECT state FROM effect_local_relay_messages - WHERE relay_message_id = ${corruptAdmission.relayMessageId} - ` - assert.strictEqual(repaired[0]?.state, "DeadLettered") - - const restartedChannel = PeerRelayStore.ChannelKey.make({ - ...channel, - senderReplicaIncarnation: Identity.ReplicaIncarnation.make(2) - }) - const restartedAdmission = PeerRelayStore.Admission.make({ - ...admission, - channel: restartedChannel, - relayMessageId: relayId("000000000004"), - outerEnvelopeDigest: PeerRpc.RelayDigest.make("d".repeat(64)) - }) - yield* store.admit(restartedAdmission) - const discovered = yield* store.claim({ - recipient: terminal.recipient, - sender: { - subjectId: restartedChannel.senderSubjectId, - peerId: restartedChannel.senderPeerId - }, - sessionGeneration: 10, - authorizedDocumentIds: restartedAdmission.documentIds - }) - assert.strictEqual(Option.isSome(discovered.message), true) - if (Option.isSome(discovered.message)) { - assert.strictEqual(discovered.message.value.channel.senderReplicaIncarnation, 2) - } - }).pipe(Effect.provide(makeLayer(filename))) - ) - })) - - it.effect("collects reservations when SQLite foreign keys are disabled", () => - Effect.gen(function*() { - const filename = join(tmpdir(), `effect-local-relay-${globalThis.crypto.randomUUID()}.sqlite`) - yield* Effect.addFinalizer(() => - Effect.sync(() => { - rmSync(filename, { force: true }) - rmSync(`${filename}-shm`, { force: true }) - rmSync(`${filename}-wal`, { force: true }) - }) - ) - yield* Effect.scoped( - Effect.gen(function*() { - const store = yield* PeerRelayStore.PeerRelayStore - const sql = yield* SqlClient.SqlClient - yield* sql`PRAGMA foreign_keys = OFF` - const foreignKeys = yield* sql<{ readonly foreign_keys: number }>`PRAGMA foreign_keys` - assert.strictEqual(foreignKeys[0]?.foreign_keys, 0) - const channel = PeerRelayStore.ChannelKey.make({ - tenantId: "tenant", - senderSubjectId: "sender", - senderPeerId: peer("000000000011"), - senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), - recipientSubjectId: "recipient", - recipientPeerId: peer("000000000012") - }) - const admission = PeerRelayStore.Admission.make({ - channel, - relayMessageId: relayId("000000000011"), - relayPeerId: peer("000000000013"), - documentIds: [documentId("000000000011")], - senderConnectionEpoch: "epoch-1", - senderSequence: 0, - payloadVersion: 1, - messageHash: "message-hash", - outerEnvelopeDigest: PeerRpc.RelayDigest.make("e".repeat(64)), - payload: new Uint8Array([1]), - messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, - senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, - minimumTerminalRetentionMillis: PeerRelayLimits.defaults.minimumTerminalRetentionMillis - }) - yield* store.admit(admission) - const claimed = yield* store.claim({ - recipient: { - tenantId: channel.tenantId, - subjectId: channel.recipientSubjectId, - peerId: channel.recipientPeerId - }, - sender: { - subjectId: channel.senderSubjectId, - peerId: channel.senderPeerId - }, - sessionGeneration: 1, - authorizedDocumentIds: admission.documentIds - }) - assert.strictEqual(Option.isSome(claimed.message), true) - if (Option.isNone(claimed.message)) return - const message = claimed.message.value - yield* store.acknowledge({ - channel, - relayMessageId: message.relayMessageId, - claimToken: message.claimToken, - messageHash: message.messageHash, - sessionGeneration: message.sessionGeneration, - recipient: { - tenantId: channel.tenantId, - subjectId: channel.recipientSubjectId, - peerId: channel.recipientPeerId - } - }) - yield* sql`UPDATE effect_local_relay_messages - SET deduplicate_until = 0 - WHERE message_id = ${message.rowId}` - yield* sql`INSERT INTO effect_local_relay_usage ( - scope_kind, - scope_key, - active_count, - active_bytes, - retained_count, - retained_bytes - ) VALUES ('Tenant', 'unrelated-zero', 0, 0, 0, 0)` - yield* store.collect({ batchSize: 10 }) - const reservations = yield* sql<{ readonly count: number }>` - SELECT COUNT(*) AS count - FROM effect_local_relay_reservations - WHERE message_id = ${message.rowId} - ` - assert.strictEqual(reservations[0]?.count, 0) - const unrelated = yield* sql<{ readonly count: number }>` - SELECT COUNT(*) AS count - FROM effect_local_relay_usage - WHERE scope_kind = 'Tenant' - AND scope_key = 'unrelated-zero' - ` - assert.strictEqual(unrelated[0]?.count, 1) - }).pipe(Effect.provide(makeLayer(filename))) - ) - })) - - it.effect("repairs an active message whose channel is missing", () => - withStore(Effect.gen(function*() { - const store = yield* PeerRelayStore.PeerRelayStore - const sql = yield* SqlClient.SqlClient - const channel = PeerRelayStore.ChannelKey.make({ - tenantId: "tenant-orphan", - senderSubjectId: "sender-orphan", - senderPeerId: peer("000000000081"), - senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), - recipientSubjectId: "recipient-orphan", - recipientPeerId: peer("000000000082") - }) - yield* store.admit(PeerRelayStore.Admission.make({ - channel, - relayMessageId: relayId("000000000081"), - relayPeerId: peer("000000000083"), - documentIds: [documentId("000000000081")], - senderConnectionEpoch: "epoch-orphan", - senderSequence: 0, - payloadVersion: 1, - messageHash: "message-hash-orphan", - outerEnvelopeDigest: PeerRpc.RelayDigest.make("8".repeat(64)), - payload: new Uint8Array([8]), - messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, - senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, - minimumTerminalRetentionMillis: PeerRelayLimits.defaults.minimumTerminalRetentionMillis - })) - yield* sql`PRAGMA foreign_keys = OFF` - yield* sql`DELETE FROM effect_local_relay_channels` - yield* store.repair({ batchSize: 1 }) - const rows = yield* sql<{ - readonly state: string - readonly payload: Uint8Array | null - }>`SELECT state, payload FROM effect_local_relay_messages` - assert.deepStrictEqual(rows, [{ - state: "DeadLettered", - payload: null - }]) - assert.deepStrictEqual(yield* store.usage(), { - activeCount: 0, - activeBytes: 0, - retainedCount: 1, - retainedBytes: 1 - }) - }))) - - it.effect("dead letters a released claim at the delivery attempt cap and preserves it across restart", () => - Effect.gen(function*() { - const filename = join(tmpdir(), `effect-local-relay-${globalThis.crypto.randomUUID()}.sqlite`) - const limits = PeerRelayLimits.Values.make({ - ...PeerRelayLimits.defaults, - maximumDeliveryAttempts: 1 - }) - yield* Effect.addFinalizer(() => - Effect.sync(() => { - rmSync(filename, { force: true }) - rmSync(`${filename}-shm`, { force: true }) - rmSync(`${filename}-wal`, { force: true }) - }) - ) - const channel = PeerRelayStore.ChannelKey.make({ - tenantId: "tenant-release-cap", - senderSubjectId: "sender-release-cap", - senderPeerId: peer("000000000071"), - senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), - recipientSubjectId: "recipient-release-cap", - recipientPeerId: peer("000000000072") - }) - const admission = PeerRelayStore.Admission.make({ - channel, - relayMessageId: relayId("000000000071"), - relayPeerId: peer("000000000073"), - documentIds: [documentId("000000000071")], - senderConnectionEpoch: "epoch-release-cap", - senderSequence: 0, - payloadVersion: 1, - messageHash: "message-hash-release-cap", - outerEnvelopeDigest: PeerRpc.RelayDigest.make("7".repeat(64)), - payload: new Uint8Array([7]), - messageTtlMillis: limits.messageTtlMillis, - senderRetryHorizonMillis: limits.maximumSenderRetryHorizonMillis, - minimumTerminalRetentionMillis: limits.minimumTerminalRetentionMillis - }) - const claimRequest = { - recipient: { - tenantId: channel.tenantId, - subjectId: channel.recipientSubjectId, - peerId: channel.recipientPeerId - }, - sender: { - subjectId: channel.senderSubjectId, - peerId: channel.senderPeerId - }, - sessionGeneration: 1, - authorizedDocumentIds: admission.documentIds - } as const - yield* Effect.scoped( - Effect.gen(function*() { - const store = yield* PeerRelayStore.PeerRelayStore - const sql = yield* SqlClient.SqlClient - yield* store.admit(admission) - const claimed = yield* store.claim(claimRequest) - assert.strictEqual(Option.isSome(claimed.message), true) - if (Option.isNone(claimed.message)) return - const message = claimed.message.value - const releaseRequest = PeerRelayStore.ReleaseRequest.make({ - channel, - relayMessageId: message.relayMessageId, - claimToken: message.claimToken, - sessionGeneration: message.sessionGeneration - }) - assert.deepStrictEqual(yield* store.release(releaseRequest), { - status: "Changed", - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" - }) - const rows = yield* sql<{ - readonly state: string - readonly retryCount: number - readonly payload: Uint8Array | null - readonly payloadLength: number - readonly claimToken: string | null - readonly terminalReason: string | null - }>`SELECT - state, - retry_count AS retryCount, - payload, - payload_length AS payloadLength, - claim_token AS claimToken, - terminal_reason AS terminalReason - FROM effect_local_relay_messages` - assert.deepStrictEqual(rows, [{ - state: "DeadLettered", - retryCount: 1, - payload: null, - payloadLength: 0, - claimToken: null, - terminalReason: "MaximumDeliveryAttempts" - }]) - const fences = yield* sql<{ - readonly claimedMessageId: number | null - readonly claimToken: string | null - }>`SELECT - claimed_message_id AS claimedMessageId, - claim_token AS claimToken - FROM effect_local_relay_channels` - assert.deepStrictEqual(fences, [{ - claimedMessageId: null, - claimToken: null - }]) - const reservations = yield* sql<{ - readonly activeConsumed: number - readonly retainedConsumed: number - }>`SELECT - active_consumed AS activeConsumed, - retained_consumed AS retainedConsumed - FROM effect_local_relay_reservations` - assert.deepStrictEqual(reservations, [{ - activeConsumed: 1, - retainedConsumed: 0 - }]) - assert.deepStrictEqual(yield* store.usage(), { - activeCount: 0, - activeBytes: 0, - retainedCount: 1, - retainedBytes: 1 - }) - assert.strictEqual(Option.isNone((yield* store.claim(claimRequest)).message), true) - assert.strictEqual((yield* store.release(releaseRequest)).status, "Stale") - }).pipe(Effect.provide(makeLayer(filename, limits))) - ) - yield* Effect.scoped( - Effect.gen(function*() { - const store = yield* PeerRelayStore.PeerRelayStore - const sql = yield* SqlClient.SqlClient - const rows = yield* sql<{ - readonly state: string - readonly retryCount: number - readonly payload: Uint8Array | null - }>`SELECT - state, - retry_count AS retryCount, - payload - FROM effect_local_relay_messages` - assert.deepStrictEqual(rows, [{ - state: "DeadLettered", - retryCount: 1, - payload: null - }]) - assert.deepStrictEqual(yield* store.usage(), { - activeCount: 0, - activeBytes: 0, - retainedCount: 1, - retainedBytes: 1 - }) - assert.strictEqual(Option.isNone((yield* store.claim(claimRequest)).message), true) - }).pipe(Effect.provide(makeLayer(filename, limits))) - ) - })) - - it.effect("dead letters an abandoned claim at the delivery attempt cap exactly once", () => { - const registry = new Map() - const metricValue = (metric: Metric.Metric) => - Metric.value(metric).pipe( - Effect.provideService(Metric.CurrentMetricAttributes, {}) - ) - return withStore( - Effect.gen(function*() { - const store = yield* PeerRelayStore.PeerRelayStore - const sql = yield* SqlClient.SqlClient - const channel = PeerRelayStore.ChannelKey.make({ - tenantId: "tenant-recover-cap", - senderSubjectId: "sender-recover-cap", - senderPeerId: peer("000000000061"), - senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), - recipientSubjectId: "recipient-recover-cap", - recipientPeerId: peer("000000000062") - }) - const admission = PeerRelayStore.Admission.make({ - channel, - relayMessageId: relayId("000000000061"), - relayPeerId: peer("000000000063"), - documentIds: [documentId("000000000061")], - senderConnectionEpoch: "epoch-recover-cap", - senderSequence: 0, - payloadVersion: 1, - messageHash: "message-hash-recover-cap", - outerEnvelopeDigest: PeerRpc.RelayDigest.make("6".repeat(64)), - payload: new Uint8Array([6]), - messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, - senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, - minimumTerminalRetentionMillis: PeerRelayLimits.defaults.minimumTerminalRetentionMillis - }) - yield* store.admit(admission) - const claimRequest = { - recipient: { - tenantId: channel.tenantId, - subjectId: channel.recipientSubjectId, - peerId: channel.recipientPeerId - }, - sender: { - subjectId: channel.senderSubjectId, - peerId: channel.senderPeerId - }, - sessionGeneration: 1, - authorizedDocumentIds: admission.documentIds - } as const - const claimed = yield* store.claim(claimRequest) - assert.strictEqual(Option.isSome(claimed.message), true) - if (Option.isNone(claimed.message)) return - yield* sql`UPDATE effect_local_relay_messages - SET claim_deadline = 0 - WHERE message_id = ${claimed.message.value.rowId}` - yield* sql`UPDATE effect_local_relay_channels - SET claim_deadline = 0 - WHERE claimed_message_id = ${claimed.message.value.rowId}` - const recovery = store.recover({ batchSize: 1 }) - assert.strictEqual((yield* recovery).processed, 1) - const rows = yield* sql<{ - readonly state: string - readonly retryCount: number - readonly payload: Uint8Array | null - readonly terminalReason: string | null - }>`SELECT - state, - retry_count AS retryCount, - payload, - terminal_reason AS terminalReason - FROM effect_local_relay_messages` - assert.deepStrictEqual(rows, [{ - state: "DeadLettered", - retryCount: 1, - payload: null, - terminalReason: "MaximumDeliveryAttempts" - }]) - assert.deepStrictEqual(yield* store.usage(), { - activeCount: 0, - activeBytes: 0, - retainedCount: 1, - retainedBytes: 1 - }) - assert.strictEqual((yield* recovery).processed, 0) - assert.strictEqual( - (yield* metricValue( - PeerRpcObservability.relayOutcomes( - "RelayMaintenance", - "Receive", - "DeadLettered", - "Recover" - ) - )).count, - 1 - ) - assert.strictEqual( - (yield* metricValue( - PeerRpcObservability.relayOutcomes( - "RelayMaintenance", - "Receive", - "Released", - "Recover" - ) - )).count, - 1 - ) - assert.strictEqual(Option.isNone((yield* store.claim(claimRequest)).message), true) - }), - PeerRelayLimits.Values.make({ - ...PeerRelayLimits.defaults, - maximumDeliveryAttempts: 1 - }) - ).pipe(Effect.provideService(Metric.MetricRegistry, registry)) - }) - - it.effect("drains maintenance by deadline even when the row cursor is ahead", () => - withStore(Effect.gen(function*() { - const store = yield* PeerRelayStore.PeerRelayStore - const sql = yield* SqlClient.SqlClient - const channel = PeerRelayStore.ChannelKey.make({ - tenantId: "tenant", - senderSubjectId: "sender", - senderPeerId: peer("000000000021"), - senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), - recipientSubjectId: "recipient", - recipientPeerId: peer("000000000022") - }) - const admission = PeerRelayStore.Admission.make({ - channel, - relayMessageId: relayId("000000000021"), - relayPeerId: peer("000000000023"), - documentIds: [documentId("000000000021")], - senderConnectionEpoch: "epoch-1", - senderSequence: 0, - payloadVersion: 1, - messageHash: "message-hash", - outerEnvelopeDigest: PeerRpc.RelayDigest.make("f".repeat(64)), - payload: new Uint8Array([1]), - messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, - senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, - minimumTerminalRetentionMillis: PeerRelayLimits.defaults.minimumTerminalRetentionMillis - }) - yield* store.admit(admission) - const claim = yield* store.claim({ - recipient: { - tenantId: channel.tenantId, - subjectId: channel.recipientSubjectId, - peerId: channel.recipientPeerId - }, - sender: { - subjectId: channel.senderSubjectId, - peerId: channel.senderPeerId - }, - sessionGeneration: 1, - authorizedDocumentIds: admission.documentIds - }) - assert.strictEqual(Option.isSome(claim.message), true) - if (Option.isNone(claim.message)) return - const message = claim.message.value - yield* sql`UPDATE effect_local_relay_messages - SET claim_deadline = 0 - WHERE message_id = ${message.rowId}` - yield* sql`UPDATE effect_local_relay_channels - SET claim_deadline = 0 - WHERE claimed_message_id = ${message.rowId}` - const recovered = yield* store.recover({ - cursor: message.rowId, - batchSize: 1 - }) - assert.strictEqual(recovered.processed, 1) - - yield* sql`UPDATE effect_local_relay_messages - SET expires_at = 0 - WHERE message_id = ${message.rowId}` - const expired = yield* store.expire({ - cursor: message.rowId, - batchSize: 1 - }) - assert.strictEqual(expired.processed, 1) - - yield* sql`UPDATE effect_local_relay_messages - SET deduplicate_until = 0 - WHERE message_id = ${message.rowId}` - const collected = yield* store.collect({ - cursor: message.rowId, - batchSize: 1 - }) - assert.strictEqual(collected.processed, 1) - const remaining = yield* sql<{ readonly count: number }>` - SELECT COUNT(*) AS count - FROM effect_local_relay_messages - WHERE message_id = ${message.rowId} - ` - assert.strictEqual(remaining[0]?.count, 0) - }))) - - it.effect("reconciles one bounded structural page without rebuilding usage", () => - withStore(Effect.gen(function*() { - const store = yield* PeerRelayStore.PeerRelayStore - const sql = yield* SqlClient.SqlClient - const channel = PeerRelayStore.ChannelKey.make({ - tenantId: "tenant", - senderSubjectId: "sender", - senderPeerId: peer("000000000031"), - senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), - recipientSubjectId: "recipient", - recipientPeerId: peer("000000000032") - }) - for (let index = 1; index <= 3; index++) { - yield* store.admit(PeerRelayStore.Admission.make({ - channel, - relayMessageId: relayId(`00000000003${index}`), - relayPeerId: peer("000000000033"), - documentIds: [documentId("000000000031")], - senderConnectionEpoch: "epoch-1", - senderSequence: index, - payloadVersion: 1, - messageHash: `message-hash-${index}`, - outerEnvelopeDigest: PeerRpc.RelayDigest.make(String(index).repeat(64)), - payload: new Uint8Array([index]), - messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, - senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, - minimumTerminalRetentionMillis: PeerRelayLimits.defaults.minimumTerminalRetentionMillis - })) - } - const ids = yield* sql<{ readonly messageId: number }>` - SELECT message_id AS messageId - FROM effect_local_relay_messages - ORDER BY message_id - ` - const first = yield* store.reconcile({ batchSize: 1 }) - assert.deepStrictEqual(first, { - cursor: ids[0]!.messageId, - processed: 1, - hasMore: true - }) - const firstCursor = first.cursor - assert.isDefined(firstCursor) - const second = yield* store.reconcile({ - cursor: firstCursor, - batchSize: 1 - }) - assert.deepStrictEqual(second, { - cursor: ids[1]!.messageId, - processed: 1, - hasMore: true - }) - const secondCursor = second.cursor - assert.isDefined(secondCursor) - const third = yield* store.reconcile({ - cursor: secondCursor, - batchSize: 1 - }) - assert.deepStrictEqual(third, { - cursor: ids[2]!.messageId, - processed: 1, - hasMore: false - }) - }))) - - it.effect("plans cross incarnation claims without a temporary ordering tree", () => - withStore(Effect.gen(function*() { - const sql = yield* SqlClient.SqlClient - yield* sql`WITH RECURSIVE incarnations(value) AS ( - VALUES(1) - UNION ALL - SELECT value + 1 FROM incarnations WHERE value < 10000 - ) - INSERT INTO effect_local_relay_channels ( - tenant_id, - sender_subject_id, - sender_peer_id, - sender_replica_incarnation, - recipient_subject_id, - recipient_peer_id, - next_sequence - ) - SELECT - 'tenant-plan', - 'sender-plan', - 'peer_00000000-0000-4000-8000-000000000041', - value, - 'recipient-plan', - 'peer_00000000-0000-4000-8000-000000000042', - 1 - FROM incarnations` - yield* sql`INSERT INTO effect_local_relay_messages ( - channel_id, - channel_sequence, - tenant_id, - sender_subject_id, - sender_peer_id, - recipient_subject_id, - recipient_peer_id, - relay_message_id, - relay_peer_id, - sender_connection_epoch, - sender_sequence, - document_ids, - payload_version, - message_hash, - outer_envelope_digest, - payload, - payload_length, - state, - created_at, - expires_at, - deduplicate_until, - next_eligible_at - ) - SELECT - channel_id, - 0, - tenant_id, - sender_subject_id, - sender_peer_id, - recipient_subject_id, - recipient_peer_id, - 'relay-' || channel_id, - 'peer_00000000-0000-4000-8000-000000000043', - 'epoch-1', - 0, - '["doc_00000000-0000-4000-8000-000000000041"]', - 1, - 'hash-' || channel_id, - ${"a".repeat(64)}, - x'01', - 1, - 'Pending', - channel_id, - 9999999999999, - 9999999999999, - 0 - FROM effect_local_relay_channels` - yield* sql`INSERT INTO effect_local_relay_reservations ( - message_id, - sender_peer_usage_key, - recipient_peer_usage_key, - recipient_subject_usage_key, - tenant_usage_key, - shard_usage_key, - active_count_delta, - active_bytes_delta, - retained_count_delta, - retained_bytes_delta - ) - SELECT - message_id, - 'sender', - 'recipient-peer', - 'recipient-subject', - 'tenant', - 'shard', - 1, - 1, - 1, - 1 - FROM effect_local_relay_messages` - yield* sql`ANALYZE` - const plan = yield* sql<{ readonly detail: string }>` - EXPLAIN QUERY PLAN - SELECT m.message_id - FROM effect_local_relay_messages m - INDEXED BY effect_local_relay_messages_claim_admission - JOIN effect_local_relay_channels c ON c.channel_id = m.channel_id - JOIN effect_local_relay_reservations r ON r.message_id = m.message_id - WHERE m.tenant_id = 'tenant-plan' - AND m.sender_subject_id = 'sender-plan' - AND m.sender_peer_id = 'peer_00000000-0000-4000-8000-000000000041' - AND m.recipient_subject_id = 'recipient-plan' - AND m.recipient_peer_id = 'peer_00000000-0000-4000-8000-000000000042' - AND c.tenant_id = 'tenant-plan' - AND c.recipient_subject_id = 'recipient-plan' - AND c.recipient_peer_id = 'peer_00000000-0000-4000-8000-000000000042' - AND c.sender_subject_id = 'sender-plan' - AND c.sender_peer_id = 'peer_00000000-0000-4000-8000-000000000041' - AND m.tenant_id = c.tenant_id - AND m.sender_subject_id = c.sender_subject_id - AND m.sender_peer_id = c.sender_peer_id - AND c.claimed_message_id IS NULL - AND m.state = 'Pending' - AND m.next_eligible_at <= 1 - AND m.expires_at > 1 - AND m.payload IS NOT NULL - AND m.payload_length = length(m.payload) - AND r.active_consumed = 0 - AND NOT EXISTS ( - SELECT 1 - FROM effect_local_relay_messages earlier - WHERE earlier.channel_id = m.channel_id - AND earlier.channel_sequence < m.channel_sequence - AND earlier.state IN ('Pending', 'Claimed') - ) - AND NOT EXISTS ( - SELECT 1 - FROM json_each(m.document_ids) document - WHERE document.value NOT IN ( - SELECT value - FROM json_each('["doc_00000000-0000-4000-8000-000000000041"]') - ) - ) - ORDER BY m.created_at, m.message_id - LIMIT 1` - assert.strictEqual( - plan.some((row) => row.detail.includes("effect_local_relay_messages_claim_admission")), - true - ) - assert.strictEqual( - plan.some((row) => row.detail.includes("USE TEMP B-TREE")), - false - ) - const recoveryPlan = yield* sql<{ readonly detail: string }>` - EXPLAIN QUERY PLAN - SELECT message_id - FROM effect_local_relay_messages - WHERE state = 'Claimed' - AND claim_deadline <= 1 - ORDER BY claim_deadline, message_id - LIMIT 101` - const expiryPlan = yield* sql<{ readonly detail: string }>` - EXPLAIN QUERY PLAN - SELECT message_id - FROM effect_local_relay_messages - WHERE state IN ('Pending', 'Claimed') - AND expires_at <= 1 - ORDER BY expires_at, message_id - LIMIT 101` - const collectionPlan = yield* sql<{ readonly detail: string }>` - EXPLAIN QUERY PLAN - SELECT message_id - FROM effect_local_relay_messages - WHERE state IN ('Acknowledged', 'DeadLettered', 'Expired') - AND deduplicate_until <= 1 - ORDER BY deduplicate_until, message_id - LIMIT 101` - for ( - const [maintenancePlan, index] of [ - [recoveryPlan, "effect_local_relay_messages_recovery"], - [expiryPlan, "effect_local_relay_messages_expiry"], - [collectionPlan, "effect_local_relay_messages_collection"] - ] as const - ) { - assert.strictEqual( - maintenancePlan.some((row) => row.detail.includes(index)), - true - ) - assert.strictEqual( - maintenancePlan.some((row) => row.detail.includes("USE TEMP B-TREE")), - false - ) - } - }))) -}) diff --git a/packages/local-rpc/test/PeerRelayStoreContract.ts b/packages/local-rpc/test/PeerRelayStoreContract.ts deleted file mode 100644 index 61a46e5..0000000 --- a/packages/local-rpc/test/PeerRelayStoreContract.ts +++ /dev/null @@ -1,239 +0,0 @@ -import { assert } from "@effect/vitest" -import * as Identity from "@lucas-barake/effect-local/Identity" -import * as Effect from "effect/Effect" -import * as Exit from "effect/Exit" -import * as Option from "effect/Option" -import * as SqlClient from "effect/unstable/sql/SqlClient" -import * as PeerRelayLimits from "../src/PeerRelayLimits.js" -import * as PeerRelayStore from "../src/PeerRelayStore.js" -import * as PeerRpc from "../src/PeerRpc.js" -import * as SqlPeerRelayStore from "../src/SqlPeerRelayStore.js" - -const peerId = (suffix: string) => Identity.PeerId.make(`peer_00000000-0000-4000-8000-${suffix}`) - -const relayMessageId = (suffix: string) => Identity.RelayMessageId.make(`rly_00000000-0000-4000-8000-${suffix}`) - -const documentId = (suffix: string) => Identity.DocumentId.make(`doc_00000000-0000-4000-8000-${suffix}`) - -const makeFixture = (index: number, overrides: Partial = {}) => { - const suffix = String(index).padStart(12, "0") - const channel = PeerRelayStore.ChannelKey.make({ - tenantId: `tenant-contract-${index}`, - senderSubjectId: `sender-contract-${index}`, - senderPeerId: peerId(suffix), - senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), - recipientSubjectId: `recipient-contract-${index}`, - recipientPeerId: peerId(String(index + 100).padStart(12, "0")) - }) - const admission = PeerRelayStore.Admission.make({ - channel, - relayMessageId: relayMessageId(suffix), - relayPeerId: peerId(String(index + 200).padStart(12, "0")), - documentIds: [documentId(suffix)], - senderConnectionEpoch: `epoch-contract-${index}`, - senderSequence: 0, - payloadVersion: 1, - messageHash: `message-hash-contract-${index}`, - outerEnvelopeDigest: PeerRpc.RelayDigest.make( - (index % 16).toString(16).repeat(64) - ), - payload: new Uint8Array([index]), - messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, - senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, - minimumTerminalRetentionMillis: PeerRelayLimits.defaults.minimumTerminalRetentionMillis, - ...overrides - }) - const claim = (sessionGeneration: number): PeerRelayStore.ClaimRequest => - PeerRelayStore.ClaimRequest.make({ - recipient: { - tenantId: admission.channel.tenantId, - subjectId: admission.channel.recipientSubjectId, - peerId: admission.channel.recipientPeerId - }, - sender: { - subjectId: admission.channel.senderSubjectId, - peerId: admission.channel.senderPeerId - }, - sessionGeneration, - authorizedDocumentIds: admission.documentIds - }) - return { admission, channel: admission.channel, claim } -} - -const terminalRequest = ( - message: PeerRelayStore.ClaimedMessage -): PeerRelayStore.TerminalRequest => - PeerRelayStore.TerminalRequest.make({ - channel: message.channel, - relayMessageId: message.relayMessageId, - claimToken: message.claimToken, - messageHash: message.messageHash, - sessionGeneration: message.sessionGeneration, - recipient: { - tenantId: message.channel.tenantId, - subjectId: message.channel.recipientSubjectId, - peerId: message.channel.recipientPeerId - } - }) - -const claimedMessage = ( - result: PeerRelayStore.ClaimResult -): PeerRelayStore.ClaimedMessage => { - assert.strictEqual(Option.isSome(result.message), true) - if (Option.isNone(result.message)) { - throw new Error("Expected the relay message to be claimed") - } - return result.message.value -} - -export const runPeerRelayStoreContract = Effect.gen(function*() { - const store = yield* PeerRelayStore.PeerRelayStore - const sql = yield* SqlClient.SqlClient - - assert.deepStrictEqual(yield* store.usage(), { - activeCount: 0, - activeBytes: 0, - retainedCount: 0, - retainedBytes: 0 - }) - - const lifecycle = makeFixture(1) - const accepted = yield* store.admit(lifecycle.admission) - assert.strictEqual(accepted.status, "Accepted") - assert.strictEqual((yield* store.admit(lifecycle.admission)).status, "Duplicate") - - const overQuota = PeerRelayStore.Admission.make({ - ...lifecycle.admission, - relayMessageId: relayMessageId("000000000002"), - senderSequence: 1, - messageHash: "message-hash-contract-quota", - outerEnvelopeDigest: PeerRpc.RelayDigest.make("f".repeat(64)) - }) - const quotaFailure = yield* Effect.flip(store.admit(overQuota)) - assert.strictEqual(quotaFailure.reason._tag, "QuotaExceeded") - assert.deepStrictEqual(yield* store.usage(), { - activeCount: 1, - activeBytes: 1, - retainedCount: 1, - retainedBytes: 1 - }) - - const firstClaim = claimedMessage(yield* store.claim(lifecycle.claim(1))) - assert.deepStrictEqual( - yield* store.loadClaimedPayload(PeerRelayStore.LoadClaimedPayloadRequest.make({ - rowId: firstClaim.rowId, - channel: firstClaim.channel, - relayMessageId: firstClaim.relayMessageId, - claimToken: firstClaim.claimToken, - sessionGeneration: firstClaim.sessionGeneration, - payloadBytes: firstClaim.payloadBytes - })), - lifecycle.admission.payload - ) - const firstTerminal = terminalRequest(firstClaim) - const released = yield* store.release(PeerRelayStore.ReleaseRequest.make({ - channel: firstClaim.channel, - relayMessageId: firstClaim.relayMessageId, - claimToken: firstClaim.claimToken, - sessionGeneration: firstClaim.sessionGeneration - })) - assert.strictEqual(released.status, "Changed") - assert.strictEqual(released.lane, "Retry") - yield* sql`UPDATE effect_local_relay_messages - SET next_eligible_at = 0 - WHERE message_id = ${firstClaim.rowId}` - - const retryResult = yield* store.claim(lifecycle.claim(2)) - assert.strictEqual(retryResult.lane, "Retry") - const retryClaim = claimedMessage(retryResult) - assert.strictEqual((yield* store.acknowledge(firstTerminal)).status, "Stale") - const retryTerminal = terminalRequest(retryClaim) - assert.strictEqual((yield* store.acknowledge(retryTerminal)).status, "Changed") - assert.strictEqual((yield* store.acknowledge(retryTerminal)).status, "Duplicate") - - const reopened = yield* SqlPeerRelayStore.make - assert.deepStrictEqual(yield* reopened.usage(), { - activeCount: 0, - activeBytes: 0, - retainedCount: 1, - retainedBytes: 1 - }) - - const rejectedFixture = makeFixture(3) - yield* store.admit(rejectedFixture.admission) - const rejectedClaim = claimedMessage(yield* store.claim(rejectedFixture.claim(1))) - const rejection = PeerRelayStore.RejectRequest.make({ - ...terminalRequest(rejectedClaim), - reason: "ApplicationRejected" - }) - assert.strictEqual((yield* store.reject(rejection)).status, "Changed") - assert.strictEqual((yield* store.reject(rejection)).status, "Duplicate") - const reopenedAfterReject = yield* SqlPeerRelayStore.make - assert.deepStrictEqual(yield* reopenedAfterReject.usage(), { - activeCount: 0, - activeBytes: 0, - retainedCount: 2, - retainedBytes: 2 - }) - - const recoveredFixture = makeFixture(4) - yield* store.admit(recoveredFixture.admission) - const abandoned = claimedMessage(yield* store.claim(recoveredFixture.claim(1))) - yield* sql`UPDATE effect_local_relay_messages - SET claim_deadline = 0 - WHERE message_id = ${abandoned.rowId}` - yield* sql`UPDATE effect_local_relay_channels - SET claim_deadline = 0 - WHERE claimed_message_id = ${abandoned.rowId}` - assert.strictEqual((yield* store.recover({ batchSize: 1 })).processed, 1) - yield* sql`UPDATE effect_local_relay_messages - SET next_eligible_at = 0 - WHERE message_id = ${abandoned.rowId}` - const recoveredResult = yield* store.claim(recoveredFixture.claim(2)) - assert.strictEqual(recoveredResult.lane, "Retry") - const recoveredClaim = claimedMessage(recoveredResult) - assert.strictEqual((yield* store.acknowledge(terminalRequest(recoveredClaim))).status, "Changed") - - const expiringFixture = makeFixture(5) - yield* store.admit(expiringFixture.admission) - const expiringClaim = claimedMessage(yield* store.claim(expiringFixture.claim(1))) - yield* sql`UPDATE effect_local_relay_messages - SET expires_at = 0 - WHERE message_id = ${expiringClaim.rowId}` - assert.strictEqual((yield* store.expire({ batchSize: 1 })).processed, 1) - assert.strictEqual((yield* store.acknowledge(terminalRequest(expiringClaim))).status, "Stale") - - yield* sql`UPDATE effect_local_relay_messages - SET deduplicate_until = 0 - WHERE message_id = ${rejectedClaim.rowId}` - assert.strictEqual((yield* store.collect({ batchSize: 1 })).processed, 1) - assert.strictEqual( - Exit.isFailure( - yield* Effect.exit(store.loadClaimedPayload({ - rowId: rejectedClaim.rowId, - channel: rejectedClaim.channel, - relayMessageId: rejectedClaim.relayMessageId, - claimToken: rejectedClaim.claimToken, - sessionGeneration: rejectedClaim.sessionGeneration, - payloadBytes: rejectedClaim.payloadBytes - })) - ), - true - ) - - const corruptFixture = makeFixture(6) - yield* store.admit(corruptFixture.admission) - yield* sql`UPDATE effect_local_relay_messages - SET tenant_id = ${"corrupt-contract-tenant"} - WHERE relay_message_id = ${corruptFixture.admission.relayMessageId}` - const repaired = yield* store.repair({ batchSize: 100 }) - assert.strictEqual(repaired.processed > 0, true) - assert.strictEqual( - Option.isNone((yield* store.claim(corruptFixture.claim(1))).message), - true - ) - - const reconciled = yield* store.reconcile({ batchSize: 1 }) - assert.strictEqual(reconciled.processed, 1) - assert.strictEqual(reconciled.hasMore, true) -}) diff --git a/packages/local-rpc/test/PeerRpcIntegration.test.ts b/packages/local-rpc/test/PeerRpcIntegration.test.ts deleted file mode 100644 index 3c5b0c8..0000000 --- a/packages/local-rpc/test/PeerRpcIntegration.test.ts +++ /dev/null @@ -1,296 +0,0 @@ -import { NodeCrypto, NodeFileSystem } from "@effect/platform-node" -import { SqliteClient } from "@effect/sql-sqlite-node" -import { assert, describe, it } from "@effect/vitest" -import * as PeerSyncEnvelope from "@lucas-barake/effect-local-sql/PeerSyncEnvelope" -import * as Canonical from "@lucas-barake/effect-local/Canonical" -import * as Document from "@lucas-barake/effect-local/Document" -import * as Identity from "@lucas-barake/effect-local/Identity" -import * as Context from "effect/Context" -import * as Crypto from "effect/Crypto" -import * as Effect from "effect/Effect" -import * as Fiber from "effect/Fiber" -import * as FileSystem from "effect/FileSystem" -import * as Layer from "effect/Layer" -import * as Queue from "effect/Queue" -import * as Redacted from "effect/Redacted" -import * as Ref from "effect/Ref" -import * as Schema from "effect/Schema" -import * as Stream from "effect/Stream" -import * as Reactivity from "effect/unstable/reactivity/Reactivity" -import * as RpcTest from "effect/unstable/rpc/RpcTest" -import * as SqlClient from "effect/unstable/sql/SqlClient" -import * as PeerAuthentication from "../src/PeerAuthentication.js" -import * as PeerAuthenticator from "../src/PeerAuthenticator.js" -import * as PeerCredentials from "../src/PeerCredentials.js" -import * as PeerRelayAuthorization from "../src/PeerRelayAuthorization.js" -import * as PeerRelayIngress from "../src/PeerRelayIngress.js" -import * as PeerRelayLimits from "../src/PeerRelayLimits.js" -import * as PeerRelayStore from "../src/PeerRelayStore.js" -import * as PeerRpc from "../src/PeerRpc.js" -import * as PeerRpcServer from "../src/PeerRpcServer.js" -import * as SqlPeerRelayStore from "../src/SqlPeerRelayStore.js" - -const Task = Document.make("Task", { - schema: Schema.Struct({ title: Schema.String }), - version: 1 -}) -const documentId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000001") -const relayPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000001") -const senderPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000002") -const recipientPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000003") -const relayMessageId = Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000001") - -const sender = PeerAuthentication.PeerPrincipal.make({ - tenantId: "tenant", - subjectId: "sender", - peerId: senderPeerId -}) -const recipient = PeerAuthentication.PeerPrincipal.make({ - tenantId: "tenant", - subjectId: "recipient", - peerId: recipientPeerId -}) - -const openRequest = ( - principal: PeerAuthentication.PeerPrincipal, - remote: PeerAuthentication.PeerPrincipal -) => - PeerRpc.OpenRpc.payloadSchema.make({ - protocolVersion: PeerRpc.protocolVersion, - expectedRelayPeerId: relayPeerId, - expectedLocal: principal, - senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), - remote: { - subjectId: remote.subjectId, - peerId: remote.peerId - }, - documents: [{ documentType: Task.name, documentId }], - receiptRetentionMillis: PeerRelayLimits.defaults.maximumReceiptRetentionMillis, - senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis - }) - -const authorize: PeerRelayAuthorization.Authorize = (request) => - Effect.succeed({ - remote: { - tenantId: request.principal.tenantId, - subjectId: request.remote.subjectId, - peerId: request.remote.peerId - }, - documents: request.documents.map((requested) => ({ - document: Task, - documentId: requested.documentId - })), - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.never - }) - -const authorizeUnsafe: PeerRelayAuthorization.AuthorizeUnsafeUnboundedAutomerge3Decode = ( - request -) => - Effect.succeed({ - _tag: "UnsafeUnboundedAutomerge3DecodeGrant", - risk: PeerRelayAuthorization.unsafeUnboundedAutomerge3DecodeRisk, - principal: request.principal, - remote: { - tenantId: request.principal.tenantId, - subjectId: request.remote.subjectId, - peerId: request.remote.peerId - }, - direction: request.direction, - documents: request.documents, - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.never - }) - -const ingress = PeerRelayIngress.PeerRelayIngress.of({ - address: { _tag: "UnixAddress", path: "relay-rpc-integration" }, - reserveOutbound: (bytes) => - Effect.succeed({ - bytes, - release: Effect.void, - transferToCurrentRequest: Effect.void - }), - usage: Effect.succeed({ - connections: 0, - reservedBytes: 0, - byteReservationWaiters: 0 - }), - await: Effect.never -}) - -const RelaySyncEnvelopeJson = Schema.fromJsonString( - Schema.toCodecJson(PeerSyncEnvelope.SyncEnvelope) -) - -describe("PeerRpc production composition", () => { - it.effect("round trips durable custody and owns session cleanup through RpcTest", () => - Effect.scoped(Effect.gen(function*() { - const limits = PeerRelayLimits.defaults - const crypto = yield* Crypto.Crypto.pipe(Effect.provide(NodeCrypto.layer)) - const fs = yield* FileSystem.FileSystem.pipe(Effect.provide(NodeFileSystem.layer)) - const directory = yield* fs.makeTempDirectoryScoped() - const sql = yield* SqliteClient.make({ filename: `${directory}/relay.sqlite` }).pipe( - Effect.provide(Reactivity.layer) - ) - const store = yield* SqlPeerRelayStore.make.pipe( - Effect.provideService(SqlClient.SqlClient, sql), - Effect.provideService(Crypto.Crypto, crypto), - Effect.provideService(PeerRelayLimits.PeerRelayLimits, limits) - ) - const authorization = yield* PeerRelayAuthorization.PeerRelayAuthorization.pipe( - Effect.provide(PeerRelayAuthorization.layer(authorize, authorizeUnsafe)) - ) - const handlers = yield* Layer.build( - PeerRpcServer.layerHandlers({ - tenantId: "tenant", - peerId: relayPeerId - }).pipe( - Layer.provide(Layer.mergeAll( - Layer.succeed(Crypto.Crypto, crypto), - Layer.succeed(PeerRelayLimits.PeerRelayLimits, limits), - Layer.succeed(PeerRelayAuthorization.PeerRelayAuthorization, authorization), - Layer.succeed(PeerRelayStore.PeerRelayStore, store), - Layer.succeed(PeerRelayIngress.PeerRelayIngress, ingress) - )) - ) - ) - const runtime = Context.get(handlers, PeerRpcServer.PeerRpcServerRuntime) - const credential = yield* Ref.make("sender") - const principals = new Map([ - ["sender", sender], - ["recipient", recipient] - ]) - const authentication = yield* PeerAuthentication.PeerAuthentication.pipe( - Effect.provide(PeerAuthentication.layerServer), - Effect.provideService(PeerAuthenticator.PeerAuthenticator, { - authenticate: (secret) => { - const principal = principals.get(Redacted.value(secret)) - return principal === undefined - ? Effect.die("Unknown integration credential") - : Effect.succeed({ - principal, - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.never - }) - } - }), - Effect.provideService(PeerRelayLimits.PeerRelayLimits, limits) - ) - const client = yield* RpcTest.makeClient(PeerRpc.Rpcs).pipe( - Effect.provideContext( - Context.add( - handlers, - PeerAuthentication.PeerAuthentication, - authentication - ) - ), - Effect.provide(PeerAuthentication.layerClient), - Effect.provideService(PeerCredentials.PeerCredentials, { - get: Ref.get(credential).pipe(Effect.map(Redacted.make)) - }) - ) - - const invalidVersion = yield* client.Open({ - ...openRequest(sender, recipient), - protocolVersion: 2 - }).pipe(Stream.runDrain, Effect.flip) - assert.strictEqual(invalidVersion._tag, "UnsupportedVersion") - assert.strictEqual((yield* runtime.usage).sessions, 0) - - assert.throws(() => - Schema.decodeUnknownSync(PeerRpc.OpenEvent)({ - _tag: "Opened", - protocolVersion: 2, - sessionId: "ses_00000000-0000-4000-8000-000000000001", - remotePeerId: recipientPeerId, - authenticatedLocal: sender - }) - ) - assert.strictEqual((yield* runtime.usage).sessions, 0) - - const senderEvents = yield* Queue.unbounded() - yield* Ref.set(credential, "sender") - const senderOpen = yield* client.Open(openRequest(sender, recipient)).pipe( - Stream.runForEach((event) => Queue.offer(senderEvents, event)), - Effect.forkScoped - ) - const senderOpened = yield* Queue.take(senderEvents) - assert.strictEqual(senderOpened._tag, "Opened") - if (senderOpened._tag !== "Opened") return assert.fail("Expected sender Opened event") - - const recipientEvents = yield* Queue.unbounded() - yield* Ref.set(credential, "recipient") - const recipientOpen = yield* client.Open(openRequest(recipient, sender)).pipe( - Stream.runForEach((event) => Queue.offer(recipientEvents, event)), - Effect.forkScoped - ) - const recipientOpened = yield* Queue.take(recipientEvents) - assert.strictEqual(recipientOpened._tag, "Opened") - if (recipientOpened._tag !== "Opened") return assert.fail("Expected recipient Opened event") - - const message = Uint8Array.of(1, 2, 3) - const messageHash = yield* Canonical.digest(message).pipe( - Effect.provideService(Crypto.Crypto, crypto) - ) - const payload = new TextEncoder().encode( - yield* Schema.encodeEffect(RelaySyncEnvelopeJson)({ - connectionEpoch: "epoch", - sequence: 0, - documentId, - documentType: Task.name, - messageHash, - message, - lineage: Identity.genesisLineage, - writerProvenance: [] - }) - ) - - yield* Ref.set(credential, "sender") - yield* client.Push({ - sessionId: senderOpened.sessionId, - relayMessageId, - payload - }) - - const stored = yield* Queue.take(recipientEvents) - assert.strictEqual(stored._tag, "StoredMessage") - if (stored._tag !== "StoredMessage") return assert.fail("Expected durable delivery") - assert.deepStrictEqual(stored.payload, payload) - assert.strictEqual((yield* store.usage()).activeCount, 1) - - yield* Ref.set(credential, "recipient") - yield* client.Acknowledge({ - sessionId: recipientOpened.sessionId, - relayMessageId: stored.relayMessageId, - claimToken: stored.claimToken, - messageHash: stored.messageHash - }) - const afterAcknowledge = yield* store.usage() - assert.strictEqual(afterAcknowledge.activeCount, 0) - assert.strictEqual(afterAcknowledge.retainedCount, 1) - - yield* Fiber.interrupt(senderOpen) - yield* Fiber.interrupt(recipientOpen) - assert.strictEqual((yield* runtime.usage).sessions, 0) - - const incumbentEvents = yield* Queue.unbounded() - yield* Ref.set(credential, "sender") - const incumbent = yield* client.Open(openRequest(sender, recipient)).pipe( - Stream.runForEach((event) => Queue.offer(incumbentEvents, event)), - Effect.forkScoped - ) - assert.strictEqual((yield* Queue.take(incumbentEvents))._tag, "Opened") - - const replacementEvents = yield* Queue.unbounded() - const replacement = yield* client.Open(openRequest(sender, recipient)).pipe( - Stream.runForEach((event) => Queue.offer(replacementEvents, event)), - Effect.forkScoped - ) - assert.strictEqual((yield* Queue.take(replacementEvents))._tag, "Opened") - yield* Fiber.await(incumbent) - assert.strictEqual((yield* runtime.usage).sessions, 1) - - yield* Fiber.interrupt(replacement) - assert.strictEqual((yield* runtime.usage).sessions, 0) - }))) -}) diff --git a/packages/local-rpc/test/PeerRpcServer.test.ts b/packages/local-rpc/test/PeerRpcServer.test.ts deleted file mode 100644 index 48f14ab..0000000 --- a/packages/local-rpc/test/PeerRpcServer.test.ts +++ /dev/null @@ -1,2564 +0,0 @@ -import { assert, describe, it, vi } from "@effect/vitest" -import * as PeerSyncEnvelope from "@lucas-barake/effect-local-sql/PeerSyncEnvelope" -import * as Canonical from "@lucas-barake/effect-local/Canonical" -import * as Document from "@lucas-barake/effect-local/Document" -import * as Identity from "@lucas-barake/effect-local/Identity" -import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" -import * as Cause from "effect/Cause" -import * as Clock from "effect/Clock" -import * as Context from "effect/Context" -import * as Crypto from "effect/Crypto" -import * as Deferred from "effect/Deferred" -import * as Effect from "effect/Effect" -import * as Exit from "effect/Exit" -import * as Fiber from "effect/Fiber" -import * as Layer from "effect/Layer" -import * as Metric from "effect/Metric" -import * as Option from "effect/Option" -import * as Queue from "effect/Queue" -import * as Ref from "effect/Ref" -import * as Schema from "effect/Schema" -import * as Stream from "effect/Stream" -import * as TestClock from "effect/testing/TestClock" -import type * as Rpc from "effect/unstable/rpc/Rpc" -import * as RpcServer from "effect/unstable/rpc/RpcServer" -import { createHash, randomBytes } from "node:crypto" -import * as PeerRpcObservability from "../src/internal/peerRpcObservability.js" -import * as PeerAuthentication from "../src/PeerAuthentication.js" -import * as PeerRelayAuthorization from "../src/PeerRelayAuthorization.js" -import * as PeerRelayIngress from "../src/PeerRelayIngress.js" -import * as PeerRelayLimits from "../src/PeerRelayLimits.js" -import * as PeerRelayStore from "../src/PeerRelayStore.js" -import * as PeerRpc from "../src/PeerRpc.js" -import * as PeerRpcError from "../src/PeerRpcError.js" -import * as PeerRpcServer from "../src/PeerRpcServer.js" - -const Task = Document.make("Task", { schema: Schema.Struct({ title: Schema.String }), version: 1 }) -const Note = Document.make("Note", { schema: Schema.Struct({ body: Schema.String }), version: 1 }) -const taskId = Identity.DocumentId.make("doc_00000000-0000-4000-8000-000000000001") -const serverPeerId = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000001") -const remotePeerA = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000002") -const remotePeerB = Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000003") -const crypto = Crypto.make({ - randomBytes: (size) => randomBytes(size), - digest: (algorithm, bytes) => - Effect.sync(() => new Uint8Array(createHash(algorithm.replace("-", "").toLowerCase()).update(bytes).digest())) -}) - -const relayTransition: PeerRelayStore.TransitionResult = { - status: "Changed", - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" -} - -const makeRelayStore = ( - overrides: Partial = {} -): PeerRelayStore.Service => ({ - admit: (input) => - Effect.succeed({ - status: "Accepted", - channel: input.channel, - ready: false, - nextEligibleAt: 0, - lane: "New" - }), - claim: () => - Effect.succeed({ - message: Option.none(), - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" - }), - loadClaimedPayload: () => Effect.succeed(new Uint8Array()), - acknowledge: () => Effect.succeed(relayTransition), - reject: () => Effect.succeed(relayTransition), - release: () => Effect.succeed(relayTransition), - recover: ({ cursor }) => - Effect.succeed({ - ...(cursor === undefined ? {} : { cursor }), - processed: 0, - hasMore: false - }), - expire: ({ cursor }) => - Effect.succeed({ - ...(cursor === undefined ? {} : { cursor }), - processed: 0, - hasMore: false - }), - repair: ({ cursor }) => - Effect.succeed({ - ...(cursor === undefined ? {} : { cursor }), - processed: 0, - hasMore: false - }), - reconcile: ({ cursor }) => - Effect.succeed({ - ...(cursor === undefined ? {} : { cursor }), - processed: 0, - hasMore: false - }), - collect: ({ cursor }) => - Effect.succeed({ - ...(cursor === undefined ? {} : { cursor }), - processed: 0, - hasMore: false - }), - usage: () => - Effect.succeed({ - activeCount: 0, - activeBytes: 0, - retainedCount: 0, - retainedBytes: 0 - }), - ...overrides -}) - -const relayPrincipal: PeerAuthentication.PeerPrincipal = { - tenantId: "tenant", - subjectId: "sender", - peerId: remotePeerA -} - -const relayOpenRequest = { - protocolVersion: PeerRpc.protocolVersion, - expectedRelayPeerId: serverPeerId, - expectedLocal: relayPrincipal, - senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), - remote: { - subjectId: "recipient", - peerId: remotePeerB - }, - documents: [{ documentType: Task.name, documentId: taskId }], - receiptRetentionMillis: PeerRelayLimits.defaults.maximumReceiptRetentionMillis, - senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis -} satisfies typeof PeerRpc.OpenRpc.payloadSchema.Type - -const relayAuthenticated = { - principal: relayPrincipal, - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.never -} - -const authorizeUnsafeRelayDecode: PeerRelayAuthorization.AuthorizeUnsafeUnboundedAutomerge3Decode = ( - request -) => - Effect.succeed({ - _tag: "UnsafeUnboundedAutomerge3DecodeGrant", - risk: PeerRelayAuthorization.unsafeUnboundedAutomerge3DecodeRisk, - principal: request.principal, - remote: { - tenantId: request.principal.tenantId, - subjectId: request.remote.subjectId, - peerId: request.remote.peerId - }, - direction: request.direction, - documents: request.documents, - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.never - }) - -const authorizeRelay: PeerRelayAuthorization.Authorize = (request) => - Effect.succeed({ - remote: { - tenantId: request.principal.tenantId, - subjectId: request.remote.subjectId, - peerId: request.remote.peerId - }, - documents: request.documents.map((document) => ({ - document: Task, - documentId: document.documentId - })), - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.never - }) - -const RelaySyncEnvelopeJson = Schema.fromJsonString( - Schema.toCodecJson(PeerSyncEnvelope.SyncEnvelope) -) - -const makeRelayPayload = Effect.gen(function*() { - const message = Uint8Array.of(1, 2, 3) - const messageHash = yield* Canonical.digest(message).pipe( - Effect.provideService(Crypto.Crypto, crypto) - ) - const payload = new TextEncoder().encode( - yield* Schema.encodeEffect(RelaySyncEnvelopeJson)({ - connectionEpoch: "epoch", - sequence: 0, - documentId: taskId, - documentType: Task.name, - messageHash, - message, - lineage: Identity.genesisLineage, - writerProvenance: [] - }) - ) - return { messageHash, payload } -}) - -const makeRelayClaim = ( - relayMessageId: Identity.RelayMessageId, - claimToken: PeerRpc.ClaimToken, - messageHash: string, - outerEnvelopeDigest: PeerRpc.RelayDigest, - payloadBytes: number, - sessionGeneration: number -) => - PeerRelayStore.ClaimedMessage.make({ - rowId: 1, - channel: { - tenantId: "tenant", - senderSubjectId: relayPrincipal.subjectId, - senderPeerId: relayPrincipal.peerId, - senderReplicaIncarnation: relayOpenRequest.senderReplicaIncarnation, - recipientSubjectId: relayOpenRequest.remote.subjectId, - recipientPeerId: relayOpenRequest.remote.peerId - }, - relayMessageId, - relayPeerId: serverPeerId, - senderConnectionEpoch: "epoch", - senderSequence: 0, - documentIds: [taskId], - payloadVersion: 1, - messageHash, - outerEnvelopeDigest, - payloadBytes, - claimToken, - claimDeadline: Number.MAX_SAFE_INTEGER, - sessionGeneration - }) - -const makeRelayOuterEnvelopeDigest = ( - relayMessageId: Identity.RelayMessageId, - messageHash: string, - payload: Uint8Array -) => - PeerSyncEnvelope.digestRelayOuterEnvelope({ - domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, - version: PeerSyncEnvelope.relayOuterEnvelopeVersion, - expectedLocal: relayPrincipal, - remote: { - tenantId: relayPrincipal.tenantId, - subjectId: relayOpenRequest.remote.subjectId, - peerId: relayOpenRequest.remote.peerId - }, - relayPeerId: serverPeerId, - relayMessageId, - protocolVersion: PeerRpc.protocolVersion, - payloadVersion: 1, - senderReplicaIncarnation: relayOpenRequest.senderReplicaIncarnation, - senderConnectionEpoch: "epoch", - senderSequence: 0, - document: { - documentId: taskId, - documentType: Task.name - }, - lineage: Identity.genesisLineage, - writerProvenance: [], - messageHash, - payload - }).pipe(Effect.provideService(Crypto.Crypto, crypto)) - -const makeRelayHandlerContext = ( - authorization: Context.Service.Shape, - store: PeerRelayStore.Service, - limits: PeerRelayLimits.Values = PeerRelayLimits.defaults, - ingress: Context.Service.Shape = { - address: { _tag: "UnixAddress", path: "relay-test" }, - reserveOutbound: () => Effect.fail(new PeerRpcError.RequestCapacityExceeded()), - usage: Effect.succeed({ - connections: 0, - reservedBytes: 0, - byteReservationWaiters: 0 - }), - await: Effect.never - } -) => - Layer.build( - PeerRpcServer.layerHandlers({ - tenantId: "tenant", - peerId: serverPeerId - }).pipe( - Layer.provide(Layer.mergeAll( - Layer.succeed(Crypto.Crypto, crypto), - Layer.succeed(PeerRelayLimits.PeerRelayLimits, limits), - Layer.succeed(PeerRelayAuthorization.PeerRelayAuthorization, authorization), - Layer.succeed(PeerRelayStore.PeerRelayStore, store), - Layer.succeed(PeerRelayIngress.PeerRelayIngress, ingress) - )) - ) - ) - -const relayHandlerEffect = ( - handler: Rpc.Handler, - effect: Effect.Effect -) => - effect.pipe( - Effect.provideContext(Context.add( - handler.context, - PeerAuthentication.AuthenticatedPeer, - relayAuthenticated - )) - ) - -const makeTerminalRelayContext = ( - relayMessageId: Identity.RelayMessageId, - claimToken: PeerRpc.ClaimToken, - authorization: Context.Service.Shape, - acknowledge: PeerRelayStore.Service["acknowledge"] -) => - Effect.gen(function*() { - const { messageHash, payload } = yield* makeRelayPayload - const outerEnvelopeDigest = yield* makeRelayOuterEnvelopeDigest( - relayMessageId, - messageHash, - payload - ) - let claimed = false - const store = makeRelayStore({ - claim: (request) => - Effect.sync(() => { - if (claimed) { - return { - message: Option.none(), - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" as const - } - } - claimed = true - return { - message: Option.some(makeRelayClaim( - relayMessageId, - claimToken, - messageHash, - outerEnvelopeDigest, - payload.byteLength, - request.sessionGeneration - )), - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" as const - } - }), - loadClaimedPayload: () => Effect.succeed(payload), - acknowledge - }) - const ingress = PeerRelayIngress.PeerRelayIngress.of({ - address: { _tag: "UnixAddress", path: "relay-test" }, - reserveOutbound: (bytes) => - Effect.succeed({ - bytes, - release: Effect.void, - transferToCurrentRequest: Effect.void - }), - usage: Effect.succeed({ - connections: 0, - reservedBytes: 0, - byteReservationWaiters: 0 - }), - await: Effect.never - }) - return yield* makeRelayHandlerContext( - authorization, - store, - PeerRelayLimits.defaults, - ingress - ) - }) - -describe("PeerRpcServer", () => { - it.effect("denies operation admission after an absolute grant deadline", () => - Effect.scoped(Effect.gen(function*() { - let now = 0 - const clock: Clock.Clock = { - currentTimeMillisUnsafe: () => now, - currentTimeMillis: Effect.sync(() => now), - currentTimeNanosUnsafe: () => BigInt(now) * 1_000_000n, - currentTimeNanos: Effect.sync(() => BigInt(now) * 1_000_000n), - sleep: () => Effect.never - } - let sendAuthorizations = 0 - const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ - authorize: (request) => - authorizeRelay(request).pipe( - Effect.map((result) => { - if (request.direction === "Send") sendAuthorizations += 1 - return request.direction === "Send" && sendAuthorizations === 2 - ? { - ...result, - validUntil: 1_000, - invalidated: Effect.sync(() => { - now = 1_001 - }).pipe(Effect.andThen(Effect.never)) - } - : result - }) - ), - authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode - }) - const admitCalls = yield* Ref.make(0) - const store = makeRelayStore({ - admit: (input) => - Ref.update(admitCalls, (count) => count + 1).pipe( - Effect.as({ - status: "Accepted", - channel: input.channel, - ready: false, - nextEligibleAt: 0, - lane: "New" - }) - ) - }) - const context = yield* makeRelayHandlerContext(authorization, store) - const openHandler = context.mapUnsafe.get( - PeerRpc.OpenRpc.key - ) as Rpc.Handler<"Open"> - const pushHandler = context.mapUnsafe.get( - PeerRpc.PushRpc.key - ) as Rpc.Handler<"Push"> - const provide = ( - handler: Rpc.Handler, - effect: Effect.Effect - ) => - effect.pipe( - Effect.provideContext( - Context.add( - Context.add( - handler.context, - PeerAuthentication.AuthenticatedPeer, - relayAuthenticated - ), - Clock.Clock, - clock - ) - ) - ) - const events = yield* Queue.unbounded() - const openFiber = yield* Stream.runForEach( - openHandler.handler( - relayOpenRequest, - {} as never - ) as Stream.Stream, - (event) => Queue.offer(events, event) - ).pipe( - (effect) => provide(openHandler, effect), - Effect.forkScoped - ) - const opened = yield* Queue.take(events) - assert.strictEqual(opened._tag, "Opened") - const { payload } = yield* makeRelayPayload - const denied = yield* ( - pushHandler.handler({ - sessionId: (opened as PeerRpc.Opened).sessionId, - relayMessageId: Identity.RelayMessageId.make( - "rly_00000000-0000-4000-8000-000000000052" - ), - payload - }, {} as never) as Effect.Effect - ).pipe( - (effect) => provide(pushHandler, effect), - Effect.flip - ) - assert.instanceOf(denied, PeerRpcError.AccessDenied) - assert.strictEqual(yield* Ref.get(admitCalls), 0) - yield* Fiber.interrupt(openFiber) - }))) - - it.effect("rejects Open when authentication expires during relay authorization", () => - Effect.scoped(Effect.gen(function*() { - const authorizationStarted = yield* Deferred.make() - const authorizationRelease = yield* Deferred.make() - let authorizationCalls = 0 - const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ - authorize: (request) => - Effect.gen(function*() { - authorizationCalls++ - if (authorizationCalls === 1) { - yield* Deferred.succeed(authorizationStarted, undefined) - yield* Deferred.await(authorizationRelease) - } - return yield* authorizeRelay(request) - }), - authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode - }) - const testClock = yield* TestClock.testClockWith(Effect.succeed) - const context = yield* makeRelayHandlerContext( - authorization, - makeRelayStore() - ).pipe(Effect.provideService(Clock.Clock, testClock)) - const handler = context.mapUnsafe.get( - PeerRpc.OpenRpc.key - ) as Rpc.Handler<"Open"> - const now = yield* Clock.currentTimeMillis - const fiber = yield* Stream.runHead( - handler.handler( - relayOpenRequest, - {} as never - ) as Stream.Stream - ).pipe( - Effect.provideContext( - Context.add( - Context.add( - handler.context, - PeerAuthentication.AuthenticatedPeer, - { - principal: relayPrincipal, - validUntil: now + 1_000, - invalidated: Effect.never - } - ), - Clock.Clock, - testClock - ) - ), - Effect.exit, - Effect.forkScoped - ) - yield* Deferred.await(authorizationStarted) - yield* TestClock.adjust(1_001) - yield* Deferred.succeed(authorizationRelease, undefined) - const exit = yield* Fiber.join(fiber) - assert.isTrue(Exit.isFailure(exit)) - if (Exit.isFailure(exit)) { - assert.instanceOf(Cause.squash(exit.cause), PeerRpcError.AuthenticationFailure) - } - }))) - - it.effect("requires a live unsafe decode grant before relay Push admission", () => - Effect.scoped(Effect.gen(function*() { - let sendMode: "Deny" | "Stale" = "Deny" - const unsafeRequests = yield* Ref.make< - Array - >([]) - const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ - authorize: authorizeRelay, - authorizeUnsafeUnboundedAutomerge3Decode: (request) => - Ref.update(unsafeRequests, (requests) => [...requests, request]).pipe( - Effect.andThen( - request.direction === "Receive" - ? authorizeUnsafeRelayDecode(request) - : sendMode === "Deny" - ? Effect.fail(new PeerRpcError.AccessDenied()) - : authorizeUnsafeRelayDecode(request).pipe( - Effect.map((grant) => ({ - ...grant, - invalidated: Effect.void - })) - ) - ) - ) - }) - const admitCalls = yield* Ref.make(0) - const store = makeRelayStore({ - admit: (input) => - Ref.update(admitCalls, (count) => count + 1).pipe( - Effect.as({ - status: "Accepted" as const, - channel: input.channel, - ready: false, - nextEligibleAt: 0, - lane: "New" as const - }) - ) - }) - const context = yield* makeRelayHandlerContext(authorization, store) - const openHandler = context.mapUnsafe.get( - PeerRpc.OpenRpc.key - ) as Rpc.Handler<"Open"> - const pushHandler = context.mapUnsafe.get( - PeerRpc.PushRpc.key - ) as Rpc.Handler<"Push"> - const events = yield* Queue.unbounded() - const openFiber = yield* Stream.runForEach( - openHandler.handler( - relayOpenRequest, - {} as never - ) as Stream.Stream, - (event) => Queue.offer(events, event) - ).pipe( - (effect) => relayHandlerEffect(openHandler, effect), - Effect.forkScoped - ) - const opened = yield* Queue.take(events) - assert.strictEqual(opened._tag, "Opened") - const sessionId = (opened as PeerRpc.Opened).sessionId - const { payload } = yield* makeRelayPayload - const push = (relayMessageId: Identity.RelayMessageId) => - pushHandler.handler({ - sessionId, - relayMessageId, - payload - }, {} as never) as Effect.Effect - - const denied = yield* push( - Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000041") - ).pipe( - (effect) => relayHandlerEffect(pushHandler, effect), - Effect.flip - ) - assert.instanceOf(denied, PeerRpcError.AccessDenied) - assert.strictEqual(yield* Ref.get(admitCalls), 0) - - sendMode = "Stale" - const stale = yield* push( - Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000042") - ).pipe( - (effect) => relayHandlerEffect(pushHandler, effect), - Effect.flip - ) - assert.instanceOf(stale, PeerRpcError.AccessDenied) - assert.strictEqual(yield* Ref.get(admitCalls), 0) - - const sendRequests = (yield* Ref.get(unsafeRequests)).filter( - (request) => request.direction === "Send" - ) - assert.deepStrictEqual(sendRequests, [ - { - risk: PeerRelayAuthorization.unsafeUnboundedAutomerge3DecodeRisk, - direction: "Send", - principal: relayPrincipal, - remote: relayOpenRequest.remote, - documents: [{ documentType: Task.name, documentId: taskId }] - }, - { - risk: PeerRelayAuthorization.unsafeUnboundedAutomerge3DecodeRisk, - direction: "Send", - principal: relayPrincipal, - remote: relayOpenRequest.remote, - documents: [{ documentType: Task.name, documentId: taskId }] - } - ]) - yield* Fiber.interrupt(openFiber) - }))) - - it.effect("requires unsafe Receive trust before relay claim or payload read", () => - Effect.scoped(Effect.gen(function*() { - const receiveDenied = yield* Deferred.make() - const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ - authorize: authorizeRelay, - authorizeUnsafeUnboundedAutomerge3Decode: (request) => - request.direction === "Send" - ? authorizeUnsafeRelayDecode(request) - : Deferred.succeed(receiveDenied, undefined).pipe( - Effect.andThen(Effect.fail(new PeerRpcError.AccessDenied())) - ) - }) - const claimCalls = yield* Ref.make(0) - const payloadReads = yield* Ref.make(0) - const store = makeRelayStore({ - claim: () => - Ref.update(claimCalls, (count) => count + 1).pipe( - Effect.as({ - message: Option.none(), - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" as const - }) - ), - loadClaimedPayload: () => - Ref.update(payloadReads, (count) => count + 1).pipe( - Effect.as(new Uint8Array()) - ) - }) - const context = yield* makeRelayHandlerContext(authorization, store) - const openHandler = context.mapUnsafe.get( - PeerRpc.OpenRpc.key - ) as Rpc.Handler<"Open"> - const events = yield* Queue.unbounded() - const openFiber = yield* Stream.runForEach( - openHandler.handler( - relayOpenRequest, - {} as never - ) as Stream.Stream, - (event) => Queue.offer(events, event) - ).pipe( - (effect) => relayHandlerEffect(openHandler, effect), - Effect.forkScoped - ) - assert.strictEqual((yield* Queue.take(events))._tag, "Opened") - yield* Deferred.await(receiveDenied) - yield* Effect.yieldNow - assert.strictEqual(yield* Ref.get(claimCalls), 0) - assert.strictEqual(yield* Ref.get(payloadReads), 0) - assert.isTrue(Option.isNone(yield* Queue.poll(events))) - yield* Fiber.interrupt(openFiber) - }))) - - it.effect("drains admitted Push while revocation denies later operations", () => - Effect.scoped(Effect.gen(function*() { - const sendInvalidated = yield* Deferred.make() - let pushing = false - const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ - authorize: (request) => - authorizeRelay(request).pipe( - Effect.map((result) => ({ - ...result, - invalidated: pushing && request.direction === "Send" - ? Deferred.await(sendInvalidated) - : Effect.never - })) - ), - authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode - }) - const admitStarted = yield* Deferred.make() - const allowCommit = yield* Deferred.make() - const commits = yield* Ref.make(0) - const store = makeRelayStore({ - admit: (input) => - Effect.uninterruptible( - Deferred.succeed(admitStarted, undefined).pipe( - Effect.andThen(Deferred.await(allowCommit)), - Effect.andThen(Ref.update(commits, (count) => count + 1)), - Effect.as({ - status: "Accepted", - channel: input.channel, - ready: false, - nextEligibleAt: 0, - lane: "New" - }) - ) - ) - }) - const context = yield* makeRelayHandlerContext(authorization, store) - const openHandler = context.mapUnsafe.get( - PeerRpc.OpenRpc.key - ) as Rpc.Handler<"Open"> - const pushHandler = context.mapUnsafe.get( - PeerRpc.PushRpc.key - ) as Rpc.Handler<"Push"> - const events = yield* Queue.unbounded() - const openFiber = yield* Stream.runForEach( - openHandler.handler( - relayOpenRequest, - {} as never - ) as Stream.Stream, - (event) => Queue.offer(events, event) - ).pipe( - (effect) => relayHandlerEffect(openHandler, effect), - Effect.forkScoped - ) - const opened = yield* Queue.take(events) - assert.strictEqual(opened._tag, "Opened") - pushing = true - const { payload } = yield* makeRelayPayload - const pushFiber = yield* ( - pushHandler.handler({ - sessionId: (opened as PeerRpc.Opened).sessionId, - relayMessageId: Identity.RelayMessageId.make( - "rly_00000000-0000-4000-8000-000000000043" - ), - payload - }, {} as never) as Effect.Effect - ).pipe( - (effect) => relayHandlerEffect(pushHandler, effect), - Effect.exit, - Effect.forkScoped - ) - yield* Deferred.await(admitStarted) - yield* Deferred.succeed(sendInvalidated, undefined) - yield* Effect.yieldNow - yield* Deferred.succeed(allowCommit, undefined) - const pushExit = yield* Fiber.join(pushFiber) - assert.isTrue(Exit.isSuccess(pushExit)) - assert.strictEqual(yield* Ref.get(commits), 1) - const later = yield* ( - pushHandler.handler({ - sessionId: (opened as PeerRpc.Opened).sessionId, - relayMessageId: Identity.RelayMessageId.make( - "rly_00000000-0000-4000-8000-000000000046" - ), - payload - }, {} as never) as Effect.Effect - ).pipe( - (effect) => relayHandlerEffect(pushHandler, effect), - Effect.flip - ) - assert.instanceOf(later, PeerRpcError.AccessDenied) - assert.strictEqual(yield* Ref.get(commits), 1) - yield* Fiber.interrupt(openFiber) - }))) - - it.effect("denies terminal mutation when fresh Receive revocation wins admission", () => - Effect.scoped(Effect.gen(function*() { - let receiveAuthorizations = 0 - const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ - authorize: (request) => - authorizeRelay(request).pipe( - Effect.map((result) => { - if (request.direction === "Receive") receiveAuthorizations += 1 - return { - ...result, - invalidated: request.direction === "Receive" && receiveAuthorizations === 3 - ? Effect.void - : Effect.never - } - }) - ), - authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode - }) - const acknowledgeCalls = yield* Ref.make(0) - const context = yield* makeTerminalRelayContext( - Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000050"), - PeerRpc.ClaimToken.make("clm_00000000-0000-4000-8000-000000000050"), - authorization, - () => - Ref.update(acknowledgeCalls, (count) => count + 1).pipe( - Effect.as(relayTransition) - ) - ) - const openHandler = context.mapUnsafe.get( - PeerRpc.OpenRpc.key - ) as Rpc.Handler<"Open"> - const acknowledgeHandler = context.mapUnsafe.get( - PeerRpc.AcknowledgeRpc.key - ) as Rpc.Handler<"Acknowledge"> - const events = yield* Queue.unbounded() - const openFiber = yield* Stream.runForEach( - openHandler.handler( - relayOpenRequest, - {} as never - ) as Stream.Stream, - (event) => Queue.offer(events, event) - ).pipe( - (effect) => relayHandlerEffect(openHandler, effect), - Effect.forkScoped - ) - const opened = yield* Queue.take(events) - const stored = yield* Queue.take(events) - if (opened._tag !== "Opened" || stored._tag !== "StoredMessage") { - return assert.fail("Expected relay open and stored message events") - } - const denied = yield* ( - acknowledgeHandler.handler({ - sessionId: opened.sessionId, - relayMessageId: stored.relayMessageId, - claimToken: stored.claimToken, - messageHash: stored.messageHash - }, {} as never) as Effect.Effect - ).pipe( - (effect) => relayHandlerEffect(acknowledgeHandler, effect), - Effect.flip - ) - assert.instanceOf(denied, PeerRpcError.AccessDenied) - assert.strictEqual(yield* Ref.get(acknowledgeCalls), 0) - yield* Fiber.interrupt(openFiber) - }))) - - it.effect("drains an admitted terminal mutation through revocation", () => - Effect.scoped(Effect.gen(function*() { - const terminalInvalidated = yield* Deferred.make() - let receiveAuthorizations = 0 - const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ - authorize: (request) => - authorizeRelay(request).pipe( - Effect.map((result) => { - if (request.direction === "Receive") receiveAuthorizations += 1 - return { - ...result, - invalidated: request.direction === "Receive" && receiveAuthorizations === 3 - ? Deferred.await(terminalInvalidated) - : Effect.never - } - }) - ), - authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode - }) - const acknowledgeStarted = yield* Deferred.make() - const allowAcknowledge = yield* Deferred.make() - const acknowledgeCalls = yield* Ref.make(0) - const context = yield* makeTerminalRelayContext( - Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000051"), - PeerRpc.ClaimToken.make("clm_00000000-0000-4000-8000-000000000051"), - authorization, - () => - Effect.uninterruptible( - Deferred.succeed(acknowledgeStarted, undefined).pipe( - Effect.andThen(Deferred.await(allowAcknowledge)), - Effect.andThen(Ref.update(acknowledgeCalls, (count) => count + 1)), - Effect.as(relayTransition) - ) - ) - ) - const openHandler = context.mapUnsafe.get( - PeerRpc.OpenRpc.key - ) as Rpc.Handler<"Open"> - const acknowledgeHandler = context.mapUnsafe.get( - PeerRpc.AcknowledgeRpc.key - ) as Rpc.Handler<"Acknowledge"> - const runtime = Context.get(context, PeerRpcServer.PeerRpcServerRuntime) - const events = yield* Queue.unbounded() - const openFiber = yield* Stream.runForEach( - openHandler.handler( - relayOpenRequest, - {} as never - ) as Stream.Stream, - (event) => Queue.offer(events, event) - ).pipe( - (effect) => relayHandlerEffect(openHandler, effect), - Effect.forkScoped - ) - const opened = yield* Queue.take(events) - const stored = yield* Queue.take(events) - if (opened._tag !== "Opened" || stored._tag !== "StoredMessage") { - return assert.fail("Expected relay open and stored message events") - } - const terminalFiber = yield* ( - acknowledgeHandler.handler({ - sessionId: opened.sessionId, - relayMessageId: stored.relayMessageId, - claimToken: stored.claimToken, - messageHash: stored.messageHash - }, {} as never) as Effect.Effect - ).pipe( - (effect) => relayHandlerEffect(acknowledgeHandler, effect), - Effect.exit, - Effect.forkScoped - ) - yield* Deferred.await(acknowledgeStarted) - yield* Deferred.succeed(terminalInvalidated, undefined) - yield* Effect.yieldNow - yield* Deferred.succeed(allowAcknowledge, undefined) - assert.isTrue(Exit.isSuccess(yield* Fiber.join(terminalFiber))) - assert.strictEqual(yield* Ref.get(acknowledgeCalls), 1) - assert.strictEqual((yield* runtime.usage).activeClaims, 0) - yield* Fiber.interrupt(openFiber) - }))) - - it.effect("drains admitted delivery through outbound offer during revocation", () => - Effect.scoped(Effect.gen(function*() { - const relayMessageId = Identity.RelayMessageId.make( - "rly_00000000-0000-4000-8000-000000000047" - ) - const claimToken = PeerRpc.ClaimToken.make( - "clm_00000000-0000-4000-8000-000000000047" - ) - const { messageHash, payload } = yield* makeRelayPayload - const outerEnvelopeDigest = yield* makeRelayOuterEnvelopeDigest( - relayMessageId, - messageHash, - payload - ) - const receiveInvalidated = yield* Deferred.make() - let receiveAuthorizations = 0 - const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ - authorize: (request) => - authorizeRelay(request).pipe( - Effect.map((result) => { - if (request.direction === "Receive") receiveAuthorizations += 1 - return { - ...result, - invalidated: request.direction === "Receive" && receiveAuthorizations > 1 - ? Deferred.await(receiveInvalidated) - : Effect.never - } - }) - ), - authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode - }) - const claimStarted = yield* Deferred.make() - const allowClaimCommit = yield* Deferred.make() - const payloadReads = yield* Ref.make(0) - let claimed = false - const store = makeRelayStore({ - claim: (request) => - Effect.uninterruptible( - Deferred.succeed(claimStarted, undefined).pipe( - Effect.andThen(Deferred.await(allowClaimCommit)), - Effect.map(() => { - if (claimed) { - return { - message: Option.none(), - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" as const - } - } - claimed = true - return { - message: Option.some(makeRelayClaim( - relayMessageId, - claimToken, - messageHash, - outerEnvelopeDigest, - payload.byteLength, - request.sessionGeneration - )), - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" as const - } - }) - ) - ), - loadClaimedPayload: () => - Ref.update(payloadReads, (count) => count + 1).pipe( - Effect.as(payload) - ) - }) - const ingress = PeerRelayIngress.PeerRelayIngress.of({ - address: { _tag: "UnixAddress", path: "relay-test" }, - reserveOutbound: (bytes) => - Effect.succeed({ - bytes, - release: Effect.void, - transferToCurrentRequest: Effect.void - }), - usage: Effect.succeed({ - connections: 0, - reservedBytes: 0, - byteReservationWaiters: 0 - }), - await: Effect.never - }) - const context = yield* makeRelayHandlerContext( - authorization, - store, - PeerRelayLimits.defaults, - ingress - ) - const openHandler = context.mapUnsafe.get( - PeerRpc.OpenRpc.key - ) as Rpc.Handler<"Open"> - const runtime = Context.get(context, PeerRpcServer.PeerRpcServerRuntime) - const events = yield* Queue.unbounded() - const openFiber = yield* Stream.runForEach( - openHandler.handler( - relayOpenRequest, - {} as never - ) as Stream.Stream, - (event) => Queue.offer(events, event) - ).pipe( - (effect) => relayHandlerEffect(openHandler, effect), - Effect.forkScoped - ) - assert.strictEqual((yield* Queue.take(events))._tag, "Opened") - yield* Deferred.await(claimStarted) - yield* Deferred.succeed(receiveInvalidated, undefined) - yield* Effect.yieldNow - yield* Deferred.succeed(allowClaimCommit, undefined) - assert.strictEqual((yield* Queue.take(events))._tag, "StoredMessage") - assert.strictEqual(yield* Ref.get(payloadReads), 1) - assert.strictEqual((yield* runtime.usage).activeClaims, 1) - yield* Fiber.interrupt(openFiber) - }))) - - it.effect("does not emit a dequeued delivery after session revocation", () => - Effect.scoped(Effect.gen(function*() { - const relayMessageId = Identity.RelayMessageId.make( - "rly_00000000-0000-4000-8000-000000000048" - ) - const claimToken = PeerRpc.ClaimToken.make( - "clm_00000000-0000-4000-8000-000000000048" - ) - const { messageHash, payload } = yield* makeRelayPayload - const outerEnvelopeDigest = yield* makeRelayOuterEnvelopeDigest( - relayMessageId, - messageHash, - payload - ) - const sessionInvalidated = yield* Deferred.make() - let receiveAuthorizations = 0 - const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ - authorize: (request) => - authorizeRelay(request).pipe( - Effect.map((result) => { - if (request.direction === "Receive") receiveAuthorizations += 1 - return { - ...result, - invalidated: request.direction === "Receive" && receiveAuthorizations === 1 - ? Deferred.await(sessionInvalidated) - : Effect.never - } - }) - ), - authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode - }) - let claimed = false - const store = makeRelayStore({ - claim: (request) => - Effect.sync(() => { - if (claimed) { - return { - message: Option.none(), - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" as const - } - } - claimed = true - return { - message: Option.some(makeRelayClaim( - relayMessageId, - claimToken, - messageHash, - outerEnvelopeDigest, - payload.byteLength, - request.sessionGeneration - )), - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" as const - } - }), - loadClaimedPayload: () => Effect.succeed(payload) - }) - const transferStarted = yield* Deferred.make() - const transferGate = yield* Deferred.make() - const reservationReleases = yield* Ref.make(0) - const ingress = PeerRelayIngress.PeerRelayIngress.of({ - address: { _tag: "UnixAddress", path: "relay-test" }, - reserveOutbound: (bytes) => - Effect.succeed({ - bytes, - release: Ref.update(reservationReleases, (count) => count + 1), - transferToCurrentRequest: Deferred.succeed(transferStarted, undefined).pipe( - Effect.andThen(Deferred.await(transferGate)) - ) - }), - usage: Effect.succeed({ - connections: 0, - reservedBytes: 0, - byteReservationWaiters: 0 - }), - await: Effect.never - }) - const context = yield* makeRelayHandlerContext( - authorization, - store, - PeerRelayLimits.defaults, - ingress - ) - const handler = context.mapUnsafe.get( - PeerRpc.OpenRpc.key - ) as Rpc.Handler<"Open"> - const runtime = Context.get(context, PeerRpcServer.PeerRpcServerRuntime) - const events = yield* Queue.unbounded() - const openFiber = yield* Stream.runForEach( - handler.handler( - relayOpenRequest, - {} as never - ) as Stream.Stream, - (event) => Queue.offer(events, event) - ).pipe( - (effect) => relayHandlerEffect(handler, effect), - Effect.exit, - Effect.forkScoped - ) - assert.strictEqual((yield* Queue.take(events))._tag, "Opened") - yield* Deferred.await(transferStarted) - yield* Deferred.succeed(sessionInvalidated, undefined) - for (let index = 0; index < 20; index++) yield* Effect.yieldNow - assert.strictEqual((yield* runtime.usage).sessions, 0) - yield* Deferred.succeed(transferGate, undefined) - yield* Fiber.join(openFiber) - assert.isTrue(Option.isNone(yield* Queue.poll(events))) - assert.strictEqual(yield* Ref.get(reservationReleases), 1) - }))) - - it.effect("treats a stale claimed payload as a local delivery race", () => - Effect.scoped(Effect.gen(function*() { - const relayMessageId = Identity.RelayMessageId.make( - "rly_00000000-0000-4000-8000-000000000044" - ) - const claimToken = PeerRpc.ClaimToken.make( - "clm_00000000-0000-4000-8000-000000000044" - ) - const { messageHash, payload } = yield* makeRelayPayload - const outerEnvelopeDigest = yield* makeRelayOuterEnvelopeDigest( - relayMessageId, - messageHash, - payload - ) - let claimed = false - const released = yield* Deferred.make() - const store = makeRelayStore({ - claim: (request) => - Effect.sync(() => { - if (claimed) { - return { - message: Option.none(), - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" as const - } - } - claimed = true - return { - message: Option.some(makeRelayClaim( - relayMessageId, - claimToken, - messageHash, - outerEnvelopeDigest, - payload.byteLength, - request.sessionGeneration - )), - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" as const - } - }), - loadClaimedPayload: () => - Effect.fail( - new ReplicaError.ReplicaError({ - reason: new ReplicaError.ProtocolMismatch({ - expected: "active relay claim", - observed: "stale relay claim" - }) - }) - ), - release: () => - Deferred.succeed(released, undefined).pipe( - Effect.as(relayTransition) - ) - }) - const ingress = PeerRelayIngress.PeerRelayIngress.of({ - address: { _tag: "UnixAddress", path: "relay-test" }, - reserveOutbound: (bytes) => - Effect.succeed({ - bytes, - release: Effect.void, - transferToCurrentRequest: Effect.void - }), - usage: Effect.succeed({ - connections: 0, - reservedBytes: 0, - byteReservationWaiters: 0 - }), - await: Effect.never - }) - const context = yield* makeRelayHandlerContext( - PeerRelayAuthorization.PeerRelayAuthorization.of({ - authorize: authorizeRelay, - authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode - }), - store, - PeerRelayLimits.defaults, - ingress - ) - const openHandler = context.mapUnsafe.get( - PeerRpc.OpenRpc.key - ) as Rpc.Handler<"Open"> - const runtime = Context.get(context, PeerRpcServer.PeerRpcServerRuntime) - const events = yield* Queue.unbounded() - const openFiber = yield* Stream.runForEach( - openHandler.handler( - relayOpenRequest, - {} as never - ) as Stream.Stream, - (event) => Queue.offer(events, event) - ).pipe( - (effect) => relayHandlerEffect(openHandler, effect), - Effect.forkScoped - ) - assert.strictEqual((yield* Queue.take(events))._tag, "Opened") - yield* Deferred.await(released) - yield* Effect.yieldNow - yield* Effect.yieldNow - yield* runtime.health - assert.strictEqual((yield* runtime.usage).activeClaims, 0) - assert.isTrue(Option.isNone(yield* Queue.poll(events))) - yield* Fiber.interrupt(openFiber) - }))) - - it.effect("clears local claim ownership after a stale terminal transition", () => - Effect.scoped(Effect.gen(function*() { - const relayMessageId = Identity.RelayMessageId.make( - "rly_00000000-0000-4000-8000-000000000045" - ) - const claimToken = PeerRpc.ClaimToken.make( - "clm_00000000-0000-4000-8000-000000000045" - ) - const { messageHash, payload } = yield* makeRelayPayload - const outerEnvelopeDigest = yield* makeRelayOuterEnvelopeDigest( - relayMessageId, - messageHash, - payload - ) - let claimed = false - const store = makeRelayStore({ - claim: (request) => - Effect.sync(() => { - if (claimed) { - return { - message: Option.none(), - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" as const - } - } - claimed = true - return { - message: Option.some(makeRelayClaim( - relayMessageId, - claimToken, - messageHash, - outerEnvelopeDigest, - payload.byteLength, - request.sessionGeneration - )), - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" as const - } - }), - loadClaimedPayload: () => Effect.succeed(payload), - acknowledge: () => - Effect.succeed({ - status: "Stale", - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" - }) - }) - const ingress = PeerRelayIngress.PeerRelayIngress.of({ - address: { _tag: "UnixAddress", path: "relay-test" }, - reserveOutbound: (bytes) => - Effect.succeed({ - bytes, - release: Effect.void, - transferToCurrentRequest: Effect.void - }), - usage: Effect.succeed({ - connections: 0, - reservedBytes: 0, - byteReservationWaiters: 0 - }), - await: Effect.never - }) - const context = yield* makeRelayHandlerContext( - PeerRelayAuthorization.PeerRelayAuthorization.of({ - authorize: authorizeRelay, - authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode - }), - store, - PeerRelayLimits.defaults, - ingress - ) - const openHandler = context.mapUnsafe.get( - PeerRpc.OpenRpc.key - ) as Rpc.Handler<"Open"> - const acknowledgeHandler = context.mapUnsafe.get( - PeerRpc.AcknowledgeRpc.key - ) as Rpc.Handler<"Acknowledge"> - const runtime = Context.get(context, PeerRpcServer.PeerRpcServerRuntime) - const events = yield* Queue.unbounded() - const openFiber = yield* Stream.runForEach( - openHandler.handler( - relayOpenRequest, - {} as never - ) as Stream.Stream, - (event) => Queue.offer(events, event) - ).pipe( - (effect) => relayHandlerEffect(openHandler, effect), - Effect.forkScoped - ) - const opened = yield* Queue.take(events) - assert.strictEqual(opened._tag, "Opened") - const stored = yield* Queue.take(events) - assert.strictEqual(stored._tag, "StoredMessage") - if (opened._tag !== "Opened" || stored._tag !== "StoredMessage") { - return assert.fail("Expected relay open and stored message events") - } - const error = yield* ( - acknowledgeHandler.handler({ - sessionId: opened.sessionId, - relayMessageId: stored.relayMessageId, - claimToken: stored.claimToken, - messageHash: stored.messageHash - }, {} as never) as Effect.Effect - ).pipe( - (effect) => relayHandlerEffect(acknowledgeHandler, effect), - Effect.flip - ) - assert.instanceOf(error, PeerRpcError.SessionUnavailable) - yield* runtime.health - assert.deepStrictEqual(yield* runtime.usage, { - accepting: true, - sessions: 1, - subjects: 1, - activeClaims: 0, - queuedChannels: 0 - }) - yield* Fiber.interrupt(openFiber) - }))) - - it.effect("drains an admitted empty claim after session revocation", () => - Effect.scoped(Effect.gen(function*() { - const sessionInvalidated = yield* Deferred.make() - let receiveAuthorizations = 0 - const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ - authorize: (request) => - authorizeRelay(request).pipe( - Effect.map((result) => { - if (request.direction === "Receive") receiveAuthorizations += 1 - return { - ...result, - invalidated: request.direction === "Receive" && receiveAuthorizations === 1 - ? Deferred.await(sessionInvalidated) - : Effect.never - } - }) - ), - authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode - }) - const claimStarted = yield* Deferred.make() - const allowClaim = yield* Deferred.make() - const claimCompleted = yield* Deferred.make() - const store = makeRelayStore({ - claim: () => - Deferred.succeed(claimStarted, undefined).pipe( - Effect.andThen(Deferred.await(allowClaim)), - Effect.andThen(Deferred.succeed(claimCompleted, undefined)), - Effect.as({ - message: Option.none(), - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" - }) - ) - }) - const context = yield* makeRelayHandlerContext(authorization, store) - const openHandler = context.mapUnsafe.get( - PeerRpc.OpenRpc.key - ) as Rpc.Handler<"Open"> - const runtime = Context.get(context, PeerRpcServer.PeerRpcServerRuntime) - const events = yield* Queue.unbounded() - const openFiber = yield* Stream.runForEach( - openHandler.handler( - relayOpenRequest, - {} as never - ) as Stream.Stream, - (event) => Queue.offer(events, event) - ).pipe( - (effect) => relayHandlerEffect(openHandler, effect), - Effect.forkScoped - ) - assert.strictEqual((yield* Queue.take(events))._tag, "Opened") - yield* Deferred.await(claimStarted) - yield* Deferred.succeed(sessionInvalidated, undefined) - yield* Effect.yieldNow - assert.strictEqual((yield* runtime.usage).sessions, 0) - yield* Deferred.succeed(allowClaim, undefined) - yield* Deferred.await(claimCompleted) - yield* Effect.yieldNow - yield* runtime.health - assert.deepStrictEqual(yield* runtime.usage, { - accepting: true, - sessions: 0, - subjects: 0, - activeClaims: 0, - queuedChannels: 0 - }) - yield* Fiber.interrupt(openFiber) - }))) - - it.effect("reports the logical remote recipient separately from the relay endpoint", () => - Effect.scoped(Effect.gen(function*() { - const localPrincipal: PeerAuthentication.PeerPrincipal = { - tenantId: "tenant", - subjectId: "sender", - peerId: remotePeerA - } - let authorizationDefect: unknown | undefined - const relayAuthorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ - authorize: (request) => - authorizationDefect === undefined - ? Effect.succeed({ - remote: { - tenantId: request.principal.tenantId, - subjectId: request.remote.subjectId, - peerId: request.remote.peerId - }, - documents: request.documents.map((document) => ({ - document: Task, - documentId: document.documentId - })), - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.never - }) - : Effect.die(authorizationDefect), - authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode - }) - const transition: PeerRelayStore.TransitionResult = { - status: "Changed", - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" - } - let maintenanceDefect: unknown | undefined - const admitted = yield* Deferred.make() - const acknowledgeCalls = yield* Ref.make(0) - const relayStore = PeerRelayStore.PeerRelayStore.of({ - admit: (input) => - Deferred.succeed(admitted, input).pipe( - Effect.as({ - status: "Accepted", - channel: input.channel, - ready: false, - nextEligibleAt: 0, - lane: "New" - }) - ), - claim: () => - Effect.succeed({ - message: Option.none(), - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" - }), - loadClaimedPayload: () => Effect.succeed(new Uint8Array()), - acknowledge: () => Ref.update(acknowledgeCalls, (count) => count + 1).pipe(Effect.as(transition)), - reject: () => Effect.succeed(transition), - release: () => Effect.succeed(transition), - recover: ({ cursor }) => - maintenanceDefect === undefined - ? Effect.succeed({ - ...(cursor === undefined ? {} : { cursor }), - processed: 0, - hasMore: false - }) - : Effect.die(maintenanceDefect), - expire: ({ cursor }) => - Effect.succeed({ ...(cursor === undefined ? {} : { cursor }), processed: 0, hasMore: false }), - repair: ({ cursor }) => - Effect.succeed({ ...(cursor === undefined ? {} : { cursor }), processed: 0, hasMore: false }), - reconcile: ({ cursor }) => - Effect.succeed({ ...(cursor === undefined ? {} : { cursor }), processed: 0, hasMore: false }), - collect: ({ cursor }) => - Effect.succeed({ ...(cursor === undefined ? {} : { cursor }), processed: 0, hasMore: false }), - usage: () => - Effect.succeed({ - activeCount: 0, - activeBytes: 0, - retainedCount: 0, - retainedBytes: 0 - }) - }) - const relayIngress = PeerRelayIngress.PeerRelayIngress.of({ - address: { _tag: "UnixAddress", path: "relay-test" }, - reserveOutbound: () => Effect.fail(new PeerRpcError.RequestCapacityExceeded()), - usage: Effect.succeed({ connections: 0, reservedBytes: 0, byteReservationWaiters: 0 }), - await: Effect.never - }) - const handlers = PeerRpcServer.layerHandlers({ - tenantId: "tenant", - peerId: serverPeerId - }).pipe( - Layer.provide(Layer.mergeAll( - Layer.succeed(Crypto.Crypto, crypto), - Layer.succeed(PeerRelayLimits.PeerRelayLimits, PeerRelayLimits.defaults), - Layer.succeed(PeerRelayAuthorization.PeerRelayAuthorization, relayAuthorization), - Layer.succeed(PeerRelayStore.PeerRelayStore, relayStore), - Layer.succeed(PeerRelayIngress.PeerRelayIngress, relayIngress) - )) - ) - const context = yield* Layer.build(handlers) - const openHandler = context.mapUnsafe.get(PeerRpc.OpenRpc.key) as Rpc.Handler<"Open"> - const pushHandler = context.mapUnsafe.get(PeerRpc.PushRpc.key) as Rpc.Handler<"Push"> - const acknowledgeHandler = context.mapUnsafe.get( - PeerRpc.AcknowledgeRpc.key - ) as Rpc.Handler<"Acknowledge"> - const runtime = Context.get(context, PeerRpcServer.PeerRpcServerRuntime) - const protocolStopped = yield* Deferred.make() - const ingressStopped = yield* Deferred.make() - const baseProtocol = yield* RpcServer.Protocol.make(() => - Effect.succeed({ - disconnects: Queue.unbounded().pipe(Effect.runSync), - send: () => Effect.void, - end: () => Effect.void, - clientIds: Effect.succeed(new Set()), - initialMessage: Effect.succeed(Option.none()), - supportsAck: true, - supportsTransferables: false, - supportsSpanPropagation: true - }) - ) - const protocol: RpcServer.Protocol["Service"] = { - ...baseProtocol, - run: (handler) => - baseProtocol.run(handler).pipe( - Effect.ensuring(Deferred.succeed(protocolStopped, undefined)) - ) - } - const openRequest = { - protocolVersion: PeerRpc.protocolVersion, - expectedRelayPeerId: serverPeerId, - expectedLocal: localPrincipal, - senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), - remote: { - subjectId: "recipient", - peerId: remotePeerB - }, - documents: [{ documentType: Task.name, documentId: taskId }], - receiptRetentionMillis: PeerRelayLimits.defaults.maximumReceiptRetentionMillis, - senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis - } satisfies typeof PeerRpc.OpenRpc.payloadSchema.Type - const authenticated = { - principal: localPrincipal, - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.never - } - const stream = openHandler.handler( - openRequest, - {} as never - ) as Stream.Stream - const events = yield* Queue.unbounded() - const openFiber = yield* Stream.runForEach( - stream.pipe( - Stream.provideContext(Context.add( - openHandler.context, - PeerAuthentication.AuthenticatedPeer, - authenticated - )) - ), - (event) => Queue.offer(events, event) - ).pipe(Effect.forkScoped) - const opened = yield* Queue.take(events) - assert.strictEqual(opened._tag, "Opened") - assert.strictEqual( - (opened as PeerRpc.Opened).remotePeerId, - remotePeerB - ) - const relayMessageId = Identity.RelayMessageId.make("rly_00000000-0000-4000-8000-000000000001") - const message = Uint8Array.of(1, 2, 3) - const messageHash = yield* Canonical.digest(message).pipe( - Effect.provideService(Crypto.Crypto, crypto) - ) - const payload = new TextEncoder().encode( - yield* Schema.encodeEffect(RelaySyncEnvelopeJson)({ - connectionEpoch: "epoch", - sequence: 0, - documentId: taskId, - documentType: Task.name, - messageHash, - message, - lineage: Identity.genesisLineage, - writerProvenance: [] - }) - ) - const expectedDigest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope({ - domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, - version: PeerSyncEnvelope.relayOuterEnvelopeVersion, - expectedLocal: localPrincipal, - remote: { - tenantId: "tenant", - subjectId: "recipient", - peerId: remotePeerB - }, - relayPeerId: serverPeerId, - relayMessageId, - protocolVersion: PeerRpc.protocolVersion, - payloadVersion: 1, - senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), - senderConnectionEpoch: "epoch", - senderSequence: 0, - document: { - documentId: taskId, - documentType: Task.name - }, - lineage: Identity.genesisLineage, - writerProvenance: [], - messageHash, - payload - }).pipe(Effect.provideService(Crypto.Crypto, crypto)) - yield* (pushHandler.handler({ - sessionId: (opened as PeerRpc.Opened).sessionId, - relayMessageId, - payload - }, {} as never) as Effect.Effect).pipe( - Effect.provideContext(Context.add( - pushHandler.context, - PeerAuthentication.AuthenticatedPeer, - authenticated - )) - ) - assert.strictEqual((yield* Deferred.await(admitted)).outerEnvelopeDigest, expectedDigest) - const defect = new Error("relay authorization defect") - authorizationDefect = defect - const defectExit = yield* Stream.runHead( - (openHandler.handler( - openRequest, - {} as never - ) as Stream.Stream).pipe( - Stream.provideContext(Context.add( - openHandler.context, - PeerAuthentication.AuthenticatedPeer, - authenticated - )) - ) - ).pipe(Effect.exit) - assert.isTrue(Exit.isFailure(defectExit)) - if (Exit.isFailure(defectExit)) { - assert.strictEqual(Cause.squash(defectExit.cause), defect) - } - authorizationDefect = undefined - const ownerContext = yield* Layer.build(Layer.fresh(handlers)) - const ownerRuntime = Context.get(ownerContext, PeerRpcServer.PeerRpcServerRuntime) - const serverContext = Context.add( - Context.add( - Context.add( - ownerContext, - RpcServer.Protocol, - protocol - ), - PeerRelayIngress.PeerRelayIngress, - { - ...relayIngress, - await: Effect.never.pipe( - Effect.ensuring(Deferred.succeed(ingressStopped, undefined)) - ) - } - ), - PeerAuthentication.PeerAuthentication, - PeerAuthentication.PeerAuthentication.of((effect) => - Effect.provideService(effect, PeerAuthentication.AuthenticatedPeer, { - principal: localPrincipal, - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.never - }) - ) - ) - yield* Layer.build( - PeerRpcServer.layerServer.pipe( - Layer.provide(Layer.succeedContext(serverContext)) - ) - ) - const ownerDefect = new Error("relay maintenance defect") - maintenanceDefect = ownerDefect - yield* TestClock.adjust(PeerRelayLimits.defaults.maintenanceIntervalMillis) - const ownerExit = yield* ownerRuntime.owner.pipe(Effect.exit) - assert.isTrue(Exit.isFailure(ownerExit)) - if (Exit.isFailure(ownerExit)) { - assert.strictEqual(Cause.squash(ownerExit.cause), ownerDefect) - } - yield* Effect.yieldNow - yield* Effect.yieldNow - assert.isTrue(Option.isSome(yield* Deferred.poll(protocolStopped))) - assert.isTrue(Option.isSome(yield* Deferred.poll(ingressStopped))) - yield* runtime.shutdown - yield* runtime.shutdown - const stale = yield* (acknowledgeHandler.handler({ - sessionId: (opened as PeerRpc.Opened).sessionId, - relayMessageId, - claimToken: PeerRpc.ClaimToken.make("clm_00000000-0000-4000-8000-000000000001"), - messageHash - }, {} as never) as Effect.Effect).pipe( - Effect.provideContext(Context.add( - acknowledgeHandler.context, - PeerAuthentication.AuthenticatedPeer, - authenticated - )), - Effect.flip - ) - assert.strictEqual(stale._tag, "SessionUnavailable") - assert.strictEqual(yield* Ref.get(acknowledgeCalls), 0) - assert.deepStrictEqual(yield* runtime.usage, { - accepting: false, - sessions: 0, - subjects: 0, - activeClaims: 0, - queuedChannels: 0 - }) - yield* Fiber.interrupt(openFiber) - }))) - - it.effect("does not deliver a document type excluded by fresh Receive authorization", () => - Effect.scoped(Effect.gen(function*() { - const relayMessageId = Identity.RelayMessageId.make( - "rly_00000000-0000-4000-8000-000000000021" - ) - const claimToken = PeerRpc.ClaimToken.make( - "clm_00000000-0000-4000-8000-000000000021" - ) - const { messageHash, payload } = yield* makeRelayPayload - const outerEnvelopeDigest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope({ - domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, - version: PeerSyncEnvelope.relayOuterEnvelopeVersion, - expectedLocal: relayPrincipal, - remote: { - tenantId: "tenant", - subjectId: relayOpenRequest.remote.subjectId, - peerId: relayOpenRequest.remote.peerId - }, - relayPeerId: serverPeerId, - relayMessageId, - protocolVersion: PeerRpc.protocolVersion, - payloadVersion: 1, - senderReplicaIncarnation: relayOpenRequest.senderReplicaIncarnation, - senderConnectionEpoch: "epoch", - senderSequence: 0, - document: { - documentId: taskId, - documentType: Task.name - }, - lineage: Identity.genesisLineage, - writerProvenance: [], - messageHash, - payload - }).pipe(Effect.provideService(Crypto.Crypto, crypto)) - let receiveAuthorizations = 0 - const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ - authorize: (request) => - Effect.sync(() => { - if (request.direction === "Receive") receiveAuthorizations++ - return { - remote: { - tenantId: request.principal.tenantId, - subjectId: request.remote.subjectId, - peerId: request.remote.peerId - }, - documents: request.documents.map((document) => ({ - document: request.direction === "Receive" && receiveAuthorizations > 1 - ? Note - : Task, - documentId: document.documentId - })), - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.never - } - }), - authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode - }) - let claimed = false - const storeReleaseCalls = yield* Ref.make(0) - const reservationReleaseCalls = yield* Ref.make(0) - const reservationTransferCalls = yield* Ref.make(0) - const deliverySettled = yield* Deferred.make<"Released" | "Transferred">() - const store = makeRelayStore({ - claim: (request) => - Effect.sync(() => { - if (claimed) { - return { - message: Option.none(), - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" as const - } - } - claimed = true - return { - message: Option.some(PeerRelayStore.ClaimedMessage.make({ - rowId: 1, - channel: { - tenantId: "tenant", - senderSubjectId: relayPrincipal.subjectId, - senderPeerId: relayPrincipal.peerId, - senderReplicaIncarnation: relayOpenRequest.senderReplicaIncarnation, - recipientSubjectId: relayOpenRequest.remote.subjectId, - recipientPeerId: relayOpenRequest.remote.peerId - }, - relayMessageId, - relayPeerId: serverPeerId, - senderConnectionEpoch: "epoch", - senderSequence: 0, - documentIds: [taskId], - payloadVersion: 1, - messageHash, - outerEnvelopeDigest, - payloadBytes: payload.byteLength, - claimToken, - claimDeadline: 1, - sessionGeneration: request.sessionGeneration - })), - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" as const - } - }), - loadClaimedPayload: () => Effect.succeed(payload), - release: () => - Ref.update(storeReleaseCalls, (count) => count + 1).pipe( - Effect.andThen(Deferred.succeed(deliverySettled, "Released")), - Effect.as(relayTransition) - ) - }) - const ingress = PeerRelayIngress.PeerRelayIngress.of({ - address: { _tag: "UnixAddress", path: "relay-test" }, - reserveOutbound: (bytes) => - Effect.succeed({ - bytes, - release: Ref.update(reservationReleaseCalls, (count) => count + 1), - transferToCurrentRequest: Ref.update( - reservationTransferCalls, - (count) => count + 1 - ).pipe( - Effect.andThen(Deferred.succeed(deliverySettled, "Transferred")), - Effect.asVoid - ) - }), - usage: Effect.succeed({ - connections: 0, - reservedBytes: 0, - byteReservationWaiters: 0 - }), - await: Effect.never - }) - const context = yield* makeRelayHandlerContext( - authorization, - store, - PeerRelayLimits.defaults, - ingress - ) - const handler = context.mapUnsafe.get( - PeerRpc.OpenRpc.key - ) as Rpc.Handler<"Open"> - const events = yield* Queue.unbounded() - const openFiber = yield* Stream.runForEach( - handler.handler( - relayOpenRequest, - {} as never - ) as Stream.Stream, - (event) => Queue.offer(events, event) - ).pipe( - (effect) => relayHandlerEffect(handler, effect), - Effect.forkScoped - ) - const opened = yield* Queue.take(events) - assert.strictEqual(opened._tag, "Opened") - assert.strictEqual(yield* Deferred.await(deliverySettled), "Released") - assert.strictEqual(yield* Ref.get(storeReleaseCalls), 1) - assert.strictEqual(yield* Ref.get(reservationReleaseCalls), 1) - assert.strictEqual(yield* Ref.get(reservationTransferCalls), 0) - assert.isTrue(Option.isNone(yield* Queue.poll(events))) - yield* Fiber.interrupt(openFiber) - }))) - - it.effect("keeps the sole subject Open slot owned after another Open is rejected", () => - Effect.scoped(Effect.gen(function*() { - const authorizationStarted = yield* Deferred.make() - const releaseAuthorization = yield* Deferred.make() - const authorizationCalls = yield* Ref.make(0) - const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ - authorize: (request) => - Ref.updateAndGet(authorizationCalls, (count) => count + 1).pipe( - Effect.tap((count) => - count === 1 - ? Deferred.succeed(authorizationStarted, undefined) - : Effect.void - ), - Effect.andThen(Deferred.await(releaseAuthorization)), - Effect.as({ - remote: { - tenantId: request.principal.tenantId, - subjectId: request.remote.subjectId, - peerId: request.remote.peerId - }, - documents: request.documents.map((document) => ({ - document: Task, - documentId: document.documentId - })), - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.never - }) - ), - authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode - }) - const context = yield* makeRelayHandlerContext( - authorization, - makeRelayStore(), - PeerRelayLimits.Values.make({ - ...PeerRelayLimits.defaults, - maxInFlightOpen: 3, - maxInFlightOpenPerSubject: 1, - openRatePerSecond: 1_000, - openBurst: 100 - }) - ) - const handler = context.mapUnsafe.get( - PeerRpc.OpenRpc.key - ) as Rpc.Handler<"Open"> - const invoke = Stream.runHead( - handler.handler( - relayOpenRequest, - {} as never - ) as Stream.Stream - ).pipe( - (effect) => relayHandlerEffect(handler, effect), - Effect.asVoid - ) - const first = yield* invoke.pipe(Effect.forkScoped) - yield* Deferred.await(authorizationStarted) - const second = yield* invoke.pipe(Effect.flip) - assert.instanceOf(second, PeerRpcError.RequestCapacityExceeded) - const third = yield* invoke.pipe(Effect.flip) - assert.instanceOf(third, PeerRpcError.RequestCapacityExceeded) - assert.strictEqual(yield* Ref.get(authorizationCalls), 1) - yield* Deferred.succeed(releaseAuthorization, undefined) - yield* Fiber.join(first) - }))) - - it.effect("detaches the incumbent when an interrupted replacement acquires the registry lock", () => - Effect.scoped(Effect.gen(function*() { - const replacementAuthorized = yield* Deferred.make() - let trackReplacement = false - let replacementAuthorizationCalls = 0 - const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ - authorize: (request) => { - if (trackReplacement) { - replacementAuthorizationCalls += 1 - if (replacementAuthorizationCalls === 2) { - Deferred.doneUnsafe(replacementAuthorized, Effect.void) - } - } - return authorizeRelay(request) - }, - authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode - }) - const context = yield* makeRelayHandlerContext(authorization, makeRelayStore()) - const handler = context.mapUnsafe.get( - PeerRpc.OpenRpc.key - ) as Rpc.Handler<"Open"> - const runtime = Context.get(context, PeerRpcServer.PeerRpcServerRuntime) - const invoke = (request: typeof PeerRpc.OpenRpc.payloadSchema.Type) => - Stream.runForEach( - handler.handler( - request, - {} as never - ) as Stream.Stream, - () => Effect.void - ).pipe((effect) => relayHandlerEffect(handler, effect)) - const incumbent = yield* invoke(relayOpenRequest).pipe(Effect.forkScoped) - const blockerRequest = { - ...relayOpenRequest, - remote: { - subjectId: "registry-blocker", - peerId: remotePeerB - } - } - const blocker = yield* invoke(blockerRequest).pipe(Effect.forkScoped) - while ((yield* runtime.usage).sessions !== 2) { - yield* Effect.yieldNow - } - - const registryLocked = yield* Deferred.make() - const releaseRegistry = yield* Deferred.make() - const setRelayActiveClaims = PeerRpcObservability.setRelayActiveClaims - const metricSpy = vi.spyOn(PeerRpcObservability, "setRelayActiveClaims") - .mockImplementation((amount) => - Deferred.succeed(registryLocked, undefined).pipe( - Effect.andThen(Deferred.await(releaseRegistry)), - Effect.andThen(setRelayActiveClaims(amount)) - ) - ) - - yield* Effect.gen(function*() { - const interruptBlocker = yield* Fiber.interrupt(blocker).pipe(Effect.forkScoped) - yield* Deferred.await(registryLocked) - - trackReplacement = true - const replacement = yield* invoke(relayOpenRequest).pipe(Effect.forkScoped) - yield* Deferred.await(replacementAuthorized) - yield* Effect.forEach( - Array.from({ length: 10 }), - () => Effect.yieldNow, - { discard: true } - ) - const interruptReplacement = yield* Fiber.interrupt(replacement).pipe(Effect.forkScoped) - yield* Effect.forEach( - Array.from({ length: 10 }), - () => Effect.yieldNow, - { discard: true } - ) - assert.isUndefined(interruptReplacement.pollUnsafe()) - - yield* Deferred.succeed(releaseRegistry, undefined) - yield* Fiber.join(interruptBlocker) - yield* Fiber.join(interruptReplacement) - assert.strictEqual((yield* runtime.usage).sessions, 0) - yield* Fiber.interrupt(incumbent) - }).pipe( - Effect.ensuring( - Deferred.succeed(releaseRegistry, undefined).pipe( - Effect.andThen(Effect.sync(() => metricSpy.mockRestore())) - ) - ) - ) - }))) - - it.effect("removes a newly registered Open when replacement cleanup is interrupted", () => - Effect.scoped(Effect.gen(function*() { - const lateMonitors = yield* Ref.make(0) - const lateMonitorSignal = yield* Deferred.make() - let trackLateMonitors = false - let trackedAuthorizations = 0 - const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ - authorize: (request) => { - const tracked = trackLateMonitors - if (tracked) trackedAuthorizations += 1 - return Effect.succeed({ - remote: { - tenantId: request.principal.tenantId, - subjectId: request.remote.subjectId, - peerId: request.remote.peerId - }, - documents: request.documents.map((document) => ({ - document: Task, - documentId: document.documentId - })), - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: tracked - ? Deferred.await(lateMonitorSignal).pipe( - Effect.tap(() => Ref.update(lateMonitors, (count) => count + 1)) - ) - : Effect.never - }) - }, - authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode - }) - const relayMessageId = Identity.RelayMessageId.make( - "rly_00000000-0000-4000-8000-000000000031" - ) - const claimToken = PeerRpc.ClaimToken.make( - "clm_00000000-0000-4000-8000-000000000031" - ) - const { messageHash, payload } = yield* makeRelayPayload - const outerEnvelopeDigest = yield* PeerSyncEnvelope.digestRelayOuterEnvelope({ - domain: PeerSyncEnvelope.relayOuterEnvelopeDomain, - version: PeerSyncEnvelope.relayOuterEnvelopeVersion, - expectedLocal: relayPrincipal, - remote: { - tenantId: "tenant", - subjectId: relayOpenRequest.remote.subjectId, - peerId: relayOpenRequest.remote.peerId - }, - relayPeerId: serverPeerId, - relayMessageId, - protocolVersion: PeerRpc.protocolVersion, - payloadVersion: 1, - senderReplicaIncarnation: relayOpenRequest.senderReplicaIncarnation, - senderConnectionEpoch: "epoch", - senderSequence: 0, - document: { - documentId: taskId, - documentType: Task.name - }, - lineage: Identity.genesisLineage, - writerProvenance: [], - messageHash, - payload - }).pipe(Effect.provideService(Crypto.Crypto, crypto)) - let claimed = false - let releaseStarted = yield* Deferred.make() - let releaseGate = yield* Deferred.make() - const store = makeRelayStore({ - claim: (request) => - Effect.sync(() => { - if (claimed) { - return { - message: Option.none(), - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" as const - } - } - claimed = true - return { - message: Option.some(PeerRelayStore.ClaimedMessage.make({ - rowId: 1, - channel: { - tenantId: "tenant", - senderSubjectId: relayPrincipal.subjectId, - senderPeerId: relayPrincipal.peerId, - senderReplicaIncarnation: relayOpenRequest.senderReplicaIncarnation, - recipientSubjectId: relayOpenRequest.remote.subjectId, - recipientPeerId: relayOpenRequest.remote.peerId - }, - relayMessageId, - relayPeerId: serverPeerId, - senderConnectionEpoch: "epoch", - senderSequence: 0, - documentIds: [taskId], - payloadVersion: 1, - messageHash, - outerEnvelopeDigest, - payloadBytes: payload.byteLength, - claimToken, - claimDeadline: 1, - sessionGeneration: request.sessionGeneration - })), - ready: false, - nextEligibleAt: Option.none(), - lane: "Retry" as const - } - }), - loadClaimedPayload: () => Effect.succeed(payload), - release: () => - Deferred.succeed(releaseStarted, undefined).pipe( - Effect.andThen(Deferred.await(releaseGate)), - Effect.as(relayTransition) - ) - }) - const ingress = PeerRelayIngress.PeerRelayIngress.of({ - address: { _tag: "UnixAddress", path: "relay-test" }, - reserveOutbound: (bytes) => - Effect.succeed({ - bytes, - release: Effect.void, - transferToCurrentRequest: Effect.void - }), - usage: Effect.succeed({ - connections: 0, - reservedBytes: 0, - byteReservationWaiters: 0 - }), - await: Effect.never - }) - const context = yield* makeRelayHandlerContext( - authorization, - store, - PeerRelayLimits.defaults, - ingress - ) - const handler = context.mapUnsafe.get( - PeerRpc.OpenRpc.key - ) as Rpc.Handler<"Open"> - const runtime = Context.get(context, PeerRpcServer.PeerRpcServerRuntime) - const events = yield* Queue.unbounded() - const first = yield* Stream.runForEach( - handler.handler( - relayOpenRequest, - {} as never - ) as Stream.Stream, - (event) => Queue.offer(events, event) - ).pipe( - (effect) => relayHandlerEffect(handler, effect), - Effect.forkScoped - ) - assert.strictEqual((yield* Queue.take(events))._tag, "Opened") - assert.strictEqual((yield* Queue.take(events))._tag, "StoredMessage") - assert.strictEqual((yield* runtime.usage).activeClaims, 1) - assert.strictEqual((yield* Metric.value(PeerRpcObservability.relayActiveClaims())).value, 1) - assert.strictEqual( - (yield* Metric.value(PeerRpcObservability.relayWorkers())).value, - PeerRelayLimits.defaults.relayWorkerConcurrency - ) - assert.strictEqual( - (yield* Metric.value( - PeerRpcObservability.relayOutcomes("RelayClaim", "Receive", "Delivered") - )).count, - 1 - ) - assert.strictEqual( - (yield* Metric.value( - PeerRpcObservability.relayBytes("RelayClaim", "Receive", "Delivered", 1) - )).count, - 1 - ) - assert.strictEqual((yield* Metric.value(PeerRpcObservability.relayPendingItems())).value, 0) - assert.strictEqual((yield* Metric.value(PeerRpcObservability.relayPendingBytes())).value, 0) - const second = yield* Stream.runHead( - handler.handler( - relayOpenRequest, - {} as never - ) as Stream.Stream - ).pipe( - (effect) => relayHandlerEffect(handler, effect), - Effect.forkScoped - ) - yield* Deferred.await(releaseStarted) - const interrupted = yield* Fiber.interrupt(second).pipe(Effect.forkScoped) - yield* Effect.yieldNow - yield* Deferred.succeed(releaseGate, undefined) - yield* Fiber.join(interrupted) - assert.deepStrictEqual(yield* runtime.usage, { - accepting: true, - sessions: 0, - subjects: 0, - activeClaims: 0, - queuedChannels: 0 - }) - assert.strictEqual((yield* Metric.value(PeerRpcObservability.relayActiveClaims())).value, 0) - yield* Fiber.interrupt(first) - - claimed = false - releaseStarted = yield* Deferred.make() - releaseGate = yield* Deferred.make() - const nextEvents = yield* Queue.unbounded() - const nextIncumbent = yield* Stream.runForEach( - handler.handler( - relayOpenRequest, - {} as never - ) as Stream.Stream, - (event) => Queue.offer(nextEvents, event) - ).pipe( - (effect) => relayHandlerEffect(handler, effect), - Effect.forkScoped - ) - assert.strictEqual((yield* Queue.take(nextEvents))._tag, "Opened") - assert.strictEqual((yield* Queue.take(nextEvents))._tag, "StoredMessage") - trackLateMonitors = true - const staleOpen = yield* Stream.runHead( - handler.handler( - relayOpenRequest, - {} as never - ) as Stream.Stream - ).pipe( - (effect) => relayHandlerEffect(handler, effect), - Effect.forkScoped - ) - yield* Deferred.await(releaseStarted) - assert.strictEqual(trackedAuthorizations, 2) - trackLateMonitors = false - yield* Stream.runHead( - handler.handler( - relayOpenRequest, - {} as never - ) as Stream.Stream - ).pipe((effect) => relayHandlerEffect(handler, effect)) - yield* Deferred.succeed(releaseGate, undefined) - yield* Fiber.join(staleOpen) - yield* Effect.forEach( - Array.from({ length: 100 }), - () => Effect.yieldNow, - { discard: true } - ) - yield* Deferred.succeed(lateMonitorSignal, undefined) - yield* Effect.forEach( - Array.from({ length: 100 }), - () => Effect.yieldNow, - { discard: true } - ) - assert.strictEqual(yield* Ref.get(lateMonitors), 0) - yield* Fiber.interrupt(nextIncumbent) - - yield* runtime.shutdown - assert.strictEqual((yield* Metric.value(PeerRpcObservability.relayWorkers())).value, 0) - assert.strictEqual((yield* Metric.value(PeerRpcObservability.relayReadyQueueItems("New"))).value, 0) - assert.strictEqual((yield* Metric.value(PeerRpcObservability.relayReadyQueueItems("Retry"))).value, 0) - })).pipe( - Effect.provideService(Metric.MetricRegistry, new Map()) - )) - - it.effect("drains admitted Push after a session authorization monitor defects", () => - Effect.scoped(Effect.gen(function*() { - const authorizationDefect = yield* Deferred.make() - const blockingFinalizer = yield* Deferred.make() - let authorizationCalls = 0 - const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ - authorize: (request) => - Effect.sync(() => { - authorizationCalls++ - return { - remote: { - tenantId: request.principal.tenantId, - subjectId: request.remote.subjectId, - peerId: request.remote.peerId - }, - documents: request.documents.map((document) => ({ - document: Task, - documentId: document.documentId - })), - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: authorizationCalls === 1 - ? Deferred.await(authorizationDefect) - : authorizationCalls === 2 - ? Effect.never.pipe( - Effect.ensuring(Deferred.await(blockingFinalizer)) - ) - : Effect.never - } - }), - authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode - }) - const admitStarted = yield* Deferred.make() - const allowAdmit = yield* Deferred.make() - const admitCalls = yield* Ref.make(0) - const store = makeRelayStore({ - admit: (input) => - Deferred.succeed(admitStarted, undefined).pipe( - Effect.andThen(Deferred.await(allowAdmit)), - Effect.andThen(Ref.update(admitCalls, (count) => count + 1)), - Effect.as({ - status: "Accepted", - channel: input.channel, - ready: false, - nextEligibleAt: 0, - lane: "New" - }) - ) - }) - const context = yield* makeRelayHandlerContext(authorization, store) - const openHandler = context.mapUnsafe.get( - PeerRpc.OpenRpc.key - ) as Rpc.Handler<"Open"> - const pushHandler = context.mapUnsafe.get( - PeerRpc.PushRpc.key - ) as Rpc.Handler<"Push"> - const runtime = Context.get(context, PeerRpcServer.PeerRpcServerRuntime) - const events = yield* Queue.unbounded() - const openFiber = yield* Stream.runForEach( - openHandler.handler( - relayOpenRequest, - {} as never - ) as Stream.Stream, - (event) => Queue.offer(events, event) - ).pipe( - (effect) => relayHandlerEffect(openHandler, effect), - Effect.forkScoped - ) - const opened = yield* Queue.take(events) - assert.strictEqual(opened._tag, "Opened") - const { payload } = yield* makeRelayPayload - const pushFiber = yield* ( - pushHandler.handler({ - sessionId: (opened as PeerRpc.Opened).sessionId, - relayMessageId: Identity.RelayMessageId.make( - "rly_00000000-0000-4000-8000-000000000041" - ), - payload - }, {} as never) as Effect.Effect - ).pipe( - (effect) => relayHandlerEffect(pushHandler, effect), - Effect.exit, - Effect.forkScoped - ) - yield* Deferred.await(admitStarted) - const defect = new Error("authorization invalidation defect") - yield* Deferred.die(authorizationDefect, defect) - yield* Effect.yieldNow - assert.strictEqual((yield* runtime.usage).sessions, 0) - yield* Deferred.succeed(allowAdmit, undefined) - const pushExit = yield* Fiber.join(pushFiber) - assert.isTrue(Exit.isSuccess(pushExit)) - assert.strictEqual(yield* Ref.get(admitCalls), 1) - const later = yield* ( - pushHandler.handler({ - sessionId: (opened as PeerRpc.Opened).sessionId, - relayMessageId: Identity.RelayMessageId.make( - "rly_00000000-0000-4000-8000-000000000049" - ), - payload - }, {} as never) as Effect.Effect - ).pipe( - (effect) => relayHandlerEffect(pushHandler, effect), - Effect.flip - ) - assert.instanceOf(later, PeerRpcError.SessionUnavailable) - assert.strictEqual(yield* Ref.get(admitCalls), 1) - yield* Effect.yieldNow - assert.deepStrictEqual(yield* runtime.usage, { - accepting: true, - sessions: 0, - subjects: 0, - activeClaims: 0, - queuedChannels: 0 - }) - yield* Deferred.succeed(blockingFinalizer, undefined) - yield* Fiber.interrupt(openFiber) - }))) - - it.effect("completes queued and permit-waiting SQL submissions during shutdown", () => - Effect.scoped(Effect.gen(function*() { - const authorization = PeerRelayAuthorization.PeerRelayAuthorization.of({ - authorize: (request) => - Effect.succeed({ - remote: { - tenantId: request.principal.tenantId, - subjectId: request.remote.subjectId, - peerId: request.remote.peerId - }, - documents: request.documents.map((document) => ({ - document: Task, - documentId: document.documentId - })), - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.never - }), - authorizeUnsafeUnboundedAutomerge3Decode: authorizeUnsafeRelayDecode - }) - const admitStarted = yield* Deferred.make() - const admitCalls = yield* Ref.make(0) - const store = makeRelayStore({ - admit: (input) => - Ref.updateAndGet(admitCalls, (count) => count + 1).pipe( - Effect.tap(() => Deferred.succeed(admitStarted, undefined)), - Effect.andThen(Effect.never), - Effect.as({ - status: "Accepted", - channel: input.channel, - ready: false, - nextEligibleAt: 0, - lane: "New" - }) - ) - }) - const context = yield* makeRelayHandlerContext( - authorization, - store, - PeerRelayLimits.Values.make({ - ...PeerRelayLimits.defaults, - maxInFlightPush: 4, - maxInFlightPushPerSubject: 4, - admissionRatePerSecond: 1_000, - admissionBurst: 100, - maxInFlightSqlTransactions: 2, - maxInFlightSqlAdmission: 1, - sqlAdmissionQueueCapacity: 4 - }) - ) - const openHandler = context.mapUnsafe.get( - PeerRpc.OpenRpc.key - ) as Rpc.Handler<"Open"> - const pushHandler = context.mapUnsafe.get( - PeerRpc.PushRpc.key - ) as Rpc.Handler<"Push"> - const runtime = Context.get(context, PeerRpcServer.PeerRpcServerRuntime) - const events = yield* Queue.unbounded() - const openFiber = yield* Stream.runForEach( - openHandler.handler( - relayOpenRequest, - {} as never - ) as Stream.Stream, - (event) => Queue.offer(events, event) - ).pipe( - (effect) => relayHandlerEffect(openHandler, effect), - Effect.forkScoped - ) - const opened = yield* Queue.take(events) - assert.strictEqual(opened._tag, "Opened") - const sessionId = (opened as PeerRpc.Opened).sessionId - const { payload } = yield* makeRelayPayload - const invoke = (relayMessageId: Identity.RelayMessageId) => - pushHandler.handler({ - sessionId, - relayMessageId, - payload - }, {} as never) as Effect.Effect - const calls = yield* Effect.forEach( - [ - "rly_00000000-0000-4000-8000-000000000011", - "rly_00000000-0000-4000-8000-000000000012", - "rly_00000000-0000-4000-8000-000000000013" - ], - (id) => - Effect.gen(function*() { - const completed = yield* Deferred.make>() - const fiber = yield* relayHandlerEffect( - pushHandler, - invoke(Identity.RelayMessageId.make(id)) - ).pipe( - Effect.exit, - Effect.flatMap((exit) => Deferred.succeed(completed, exit)), - Effect.forkScoped - ) - return { completed, fiber } - }) - ) - yield* Effect.yieldNow - yield* Deferred.await(admitStarted) - yield* Effect.yieldNow - yield* Effect.yieldNow - assert.strictEqual(yield* Ref.get(admitCalls), 1) - yield* runtime.shutdown - yield* Effect.yieldNow - yield* Effect.yieldNow - const completed = yield* Effect.forEach(calls, (call) => Deferred.poll(call.completed)) - yield* Effect.forEach(calls, (call) => Fiber.interrupt(call.fiber), { - discard: true - }) - assert.isTrue(completed.every(Option.isSome)) - yield* Fiber.interrupt(openFiber) - }))) -}) diff --git a/packages/local-rpc/test/PublicApi.types.test.ts b/packages/local-rpc/test/PublicApi.types.test.ts index 3cc8a2a..d80b18d 100644 --- a/packages/local-rpc/test/PublicApi.types.test.ts +++ b/packages/local-rpc/test/PublicApi.types.test.ts @@ -7,18 +7,31 @@ describe("public API", () => { for ( const name of [ "PeerRelayAuthorization", - "PeerRelayIngress", "PeerRelayLimits", "PeerRpc", - "PeerRelayStore", - "PeerRpcServer", - "SqlPeerRelayStore" + "RelayInbox", + "RelayInboxMaintenance", + "RelayInboxStore", + "RelayServer", + "SqlRelayInboxStore" ] ) { assert.property(PublicApi, name) } - for (const name of ["PeerRelayRpc", "PeerAuthorization", "PeerRpcLimits"]) { + // The single-process relay is gone, replaced by the cluster entity above. Asserting their + // absence keeps a stray re-export from quietly restoring an API with the old custody semantics. + for ( + const name of [ + "PeerRelayRpc", + "PeerAuthorization", + "PeerRpcLimits", + "PeerRelayIngress", + "PeerRelayStore", + "PeerRpcServer", + "SqlPeerRelayStore" + ] + ) { assert.notProperty(PublicApi, name) } diff --git a/packages/local-rpc/test/RelayInboxStore.test.ts b/packages/local-rpc/test/RelayInboxStore.test.ts index fbcab1e..2e9b6fe 100644 --- a/packages/local-rpc/test/RelayInboxStore.test.ts +++ b/packages/local-rpc/test/RelayInboxStore.test.ts @@ -1,397 +1,104 @@ -import { NodeCrypto } from "@effect/platform-node" +import { NodeCrypto, NodeFileSystem } from "@effect/platform-node" +import { MysqlClient } from "@effect/sql-mysql2" +import { PgClient } from "@effect/sql-pg" import { SqliteClient } from "@effect/sql-sqlite-node" -import { assert, describe, it } from "@effect/vitest" -import * as Identity from "@lucas-barake/effect-local/Identity" +import { describe, it } from "@effect/vitest" +import { MySqlContainer, type StartedMySqlContainer } from "@testcontainers/mysql" +import { PostgreSqlContainer } from "@testcontainers/postgresql" +import * as Context from "effect/Context" +import * as Data from "effect/Data" import * as Effect from "effect/Effect" +import * as FileSystem from "effect/FileSystem" import * as Layer from "effect/Layer" -import * as RelayInboxStore from "../src/RelayInboxStore.js" +import * as Redacted from "effect/Redacted" +import type * as SqlClient from "effect/unstable/sql/SqlClient" import * as SqlRelayInboxStore from "../src/SqlRelayInboxStore.js" - -const peer = (value: string) => Identity.PeerId.make(`peer_00000000-0000-4000-8000-${value}`) -const relayId = (value: string) => Identity.RelayMessageId.make(`rly_00000000-0000-4000-8000-${value}`) -const documentId = (value: string) => Identity.DocumentId.make(`doc_00000000-0000-4000-8000-${value}`) - -const layer = SqlRelayInboxStore.layer.pipe( - Layer.provide(SqliteClient.layer({ filename: ":memory:", disableWAL: true })), - Layer.provide(NodeCrypto.layer), - Layer.orDie -) - -const quota = { maxPendingMessages: 100, maxPendingBytes: 10_000_000 } - -const channel = ( - options?: { readonly epoch?: string | undefined; readonly subject?: string | undefined } -) => ({ - tenantId: "tenant-a", - senderSubjectId: options?.subject ?? "sender-a", - senderPeerId: peer("00000000aaa1"), - senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), - senderConnectionEpoch: options?.epoch ?? "epoch-1" -}) - -const envelope = (options: { - readonly id: string - readonly sequence: number - readonly digest?: string - readonly epoch?: string - readonly subject?: string -}) => { - const source = channel({ epoch: options.epoch, subject: options.subject }) - return { - relayMessageId: relayId(options.id), - relayPeerId: peer("00000000ffff"), - sender: { - tenantId: source.tenantId, - subjectId: source.senderSubjectId, - peerId: source.senderPeerId, - replicaIncarnation: source.senderReplicaIncarnation, - connectionEpoch: source.senderConnectionEpoch, - sequence: options.sequence - }, - recipient: { - tenantId: "tenant-a", - subjectId: "recipient-a", - peerId: peer("00000000bbb1") - }, - payloadVersion: 1 as const, - document: { documentId: documentId("00000000dddd"), documentType: "note" }, - writerProvenance: [], - messageHash: "a".repeat(64), - outerEnvelopeDigest: (options.digest ?? "b").repeat(64).slice(0, 64), - payload: new Uint8Array([1, 2, 3]) +import { relayInboxStoreContract } from "./RelayInboxStoreContract.js" + +/** + * The store contract against every dialect the relay claims to support. + * + * Multi-dialect parity is a decision, not an accident: the previous relay store supported all three + * and dropping one here would be a silent regression. It is also the only way this suite can see a + * whole class of defect — the PostgreSQL driver hands back `BIGINT`, `COUNT` and `SUM` as strings, + * and MySQL compares identity columns case-insensitively without a binary collation. Both were real + * bugs in this store, and both are invisible to SQLite. + */ + +class ContainerError extends Data.TaggedError("ContainerError")<{ + readonly cause: unknown +}> {} + +class PgContainer extends Context.Service()( + "@lucas-barake/effect-local-rpc/test/PgContainer", + { + make: Effect.acquireRelease( + Effect.tryPromise({ + try: () => new PostgreSqlContainer("postgres:alpine").start(), + catch: (cause) => new ContainerError({ cause }) + }), + (container) => Effect.promise(() => container.stop()) + ) } -} - -const admission = (options: { - readonly inboxKey: string - readonly id: string - readonly sequence: number - readonly now: number - readonly ttl?: number - readonly horizon?: number - readonly digest?: string - readonly epoch?: string - readonly subject?: string - readonly quota?: RelayInboxStore.AdmissionQuota -}) => ({ - inboxKey: options.inboxKey, - channel: channel({ epoch: options.epoch, subject: options.subject }), - envelope: envelope(options), - now: options.now, - messageTtlMillis: options.ttl ?? 1_000, - senderRetryHorizonMillis: options.horizon ?? 1_000, - quota: options.quota ?? quota -}) - -describe("RelayInboxStore", () => { - it.effect("admits a message and reports it as the pending head", () => - Effect.gen(function*() { - const store = yield* RelayInboxStore.RelayInboxStore - const result = yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0 })) - assert.strictEqual(result._tag, "Admitted") - - const heads = yield* store.pendingHeads("a", { limit: 10 }) - assert.strictEqual(heads.length, 1) - assert.strictEqual(heads[0]!.relayMessageId, relayId("000000000001")) - }).pipe(Effect.provide(layer))) - - it.effect("reports a replay of the same identity and digest as a duplicate", () => - Effect.gen(function*() { - const store = yield* RelayInboxStore.RelayInboxStore - yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0 })) - const replay = yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 1 })) - - assert.strictEqual(replay._tag, "Duplicate") - assert.strictEqual(replay._tag === "Duplicate" ? replay.state : "", "Pending") - const heads = yield* store.pendingHeads("a", { limit: 10 }) - assert.strictEqual(heads.length, 1, "a replay must not create a second row") - }).pipe(Effect.provide(layer))) - - it.effect("rejects the same identity carrying a different envelope digest", () => - Effect.gen(function*() { - const store = yield* RelayInboxStore.RelayInboxStore - yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0 })) - const conflict = yield* store.admit( - admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 1, digest: "c" }) - ) - assert.strictEqual(conflict._tag, "Conflict") - }).pipe(Effect.provide(layer))) - - it.effect("refuses a message whose channel disagrees with its envelope", () => - Effect.gen(function*() { - const store = yield* RelayInboxStore.RelayInboxStore - const request = admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0 }) - const forged = { - ...request, - channel: { ...request.channel, senderConnectionEpoch: "another-epoch" } - } - const result = yield* store.admit(forged) - assert.strictEqual( - result._tag, - "Conflict", - "a fabricated channel would file the message under an ordering stream it does not belong to" - ) - }).pipe(Effect.provide(layer))) - - it.effect("expiring one inbox leaves another inbox's copy of the same identity untouched", () => - Effect.gen(function*() { - const store = yield* RelayInboxStore.RelayInboxStore - // The same relay message id addressed to two devices. The key is (inbox, id), so a sweep - // that filters on the id alone would reach across inboxes. - yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0, ttl: 1_000 })) - yield* store.admit( - admission({ inboxKey: "b", id: "000000000001", sequence: 0, now: 100_000, ttl: 1_000_000 }) - ) - - const expired = yield* store.expire({ now: 5_000, limit: 10, terminalRetentionMillis: 1_000 }) - assert.strictEqual(expired, 1) - - const stale = yield* store.pendingHeads("a", { limit: 10 }) - assert.strictEqual(stale.length, 0, "the overdue message is expired") - const live = yield* store.pendingHeads("b", { limit: 10 }) - assert.strictEqual(live.length, 1, "the other inbox still holds a message with time left") - }).pipe(Effect.provide(layer))) - - it.effect("delivers one head per channel and does not starve a channel with high sequences", () => - Effect.gen(function*() { - const store = yield* RelayInboxStore.RelayInboxStore - // `senderSequence` restarts per connection epoch, so it cannot order heads across channels. - yield* store.admit( - admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0, subject: "sender-a" }) - ) - yield* store.admit( - admission({ inboxKey: "a", id: "000000000002", sequence: 1, now: 1, subject: "sender-b" }) - ) - yield* store.admit( - admission({ inboxKey: "a", id: "000000000003", sequence: 500, now: 2, subject: "sender-c" }) - ) - - const heads = yield* store.pendingHeads("a", { limit: 2 }) - assert.strictEqual(heads.length, 2) - - // Oldest waiting head first, so every channel is reachable regardless of its own numbering. - const all = yield* store.pendingHeads("a", { limit: 10 }) - assert.deepStrictEqual( - all.map((head) => head.channel.senderSubjectId).toSorted(), - ["sender-a", "sender-b", "sender-c"] - ) - }).pipe(Effect.provide(layer))) - - it.effect("orders within a channel by sender sequence", () => +) { + static readonly layerClient = Layer.unwrap( Effect.gen(function*() { - const store = yield* RelayInboxStore.RelayInboxStore - yield* store.admit(admission({ inboxKey: "a", id: "000000000002", sequence: 5, now: 0 })) - yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 1, now: 1 })) - - const heads = yield* store.pendingHeads("a", { limit: 10 }) - assert.strictEqual(heads.length, 1, "one head per channel") - assert.strictEqual(heads[0]!.relayMessageId, relayId("000000000001")) - }).pipe(Effect.provide(layer))) - - it.effect("spends the full delivery budget before dead lettering", () => - Effect.gen(function*() { - const store = yield* RelayInboxStore.RelayInboxStore - yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0 })) - - const first = yield* store.recordDelivery("a", relayId("000000000001"), { - maxDeliveries: 1, - now: 1 - }) - assert.strictEqual( - first._tag, - "Recorded", - "the first delivery of a budget of one must be allowed to be settled" - ) - - const second = yield* store.recordDelivery("a", relayId("000000000001"), { - maxDeliveries: 1, - now: 2 - }) - assert.strictEqual(second._tag, "DeadLettered") - - const heads = yield* store.pendingHeads("a", { limit: 10 }) - assert.strictEqual(heads.length, 0, "a dead lettered message stops blocking its channel") - }).pipe(Effect.provide(layer))) - - it.effect("makes an abandoned message answerable after the fact", () => - Effect.gen(function*() { - const store = yield* RelayInboxStore.RelayInboxStore - yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0 })) - yield* store.recordDelivery("a", relayId("000000000001"), { maxDeliveries: 1, now: 1 }) - yield* store.recordDelivery("a", relayId("000000000001"), { maxDeliveries: 1, now: 2 }) - - const abandoned = yield* store.abandoned("a", { limit: 10 }) - assert.strictEqual(abandoned.length, 1) - assert.strictEqual(abandoned[0]!.state, "DeadLettered") - }).pipe(Effect.provide(layer))) - - it.effect("settles on acknowledgement and does not redeliver", () => - Effect.gen(function*() { - const store = yield* RelayInboxStore.RelayInboxStore - yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0 })) - - const settled = yield* store.settle("a", relayId("000000000001"), { - outcome: "Acknowledged", - messageHash: "a".repeat(64), - now: 10, - terminalRetentionMillis: 1_000 - }) - assert.strictEqual(settled, "Settled") - - const heads = yield* store.pendingHeads("a", { limit: 10 }) - assert.strictEqual(heads.length, 0) - }).pipe(Effect.provide(layer))) - - it.effect("refuses a settlement whose content does not match", () => - Effect.gen(function*() { - const store = yield* RelayInboxStore.RelayInboxStore - yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0 })) + const container = yield* PgContainer + return PgClient.layer({ url: Redacted.make(container.getConnectionUri()) }) + }) + ).pipe(Layer.provide(Layer.effect(this)(this.make))) +} - const settled = yield* store.settle("a", relayId("000000000001"), { - outcome: "Acknowledged", - messageHash: "f".repeat(64), - now: 10, - terminalRetentionMillis: 1_000 - }) - assert.strictEqual(settled, "HashMismatch") +class MysqlContainer extends Context.Service()( + "@lucas-barake/effect-local-rpc/test/MysqlContainer" +) { + static readonly layer = Layer.effect(this)( + Effect.acquireRelease( + Effect.tryPromise({ + try: () => new MySqlContainer("mysql:lts").start(), + catch: (cause) => new ContainerError({ cause }) + }), + (container) => Effect.promise(() => container.stop()) + ) + ) + + static readonly layerClient = Layer.unwrap( + Effect.gen(function*() { + const container = yield* MysqlContainer + return MysqlClient.layer({ url: Redacted.make(container.getConnectionUri()) }) + }) + ).pipe(Layer.provide(this.layer)) +} - const heads = yield* store.pendingHeads("a", { limit: 10 }) - assert.strictEqual(heads.length, 1, "the message stays deliverable") - }).pipe(Effect.provide(layer))) +// A file rather than `:memory:`, so the store is exercised against a real page cache and a real +// journal instead of the in-process shortcut. +const SqliteLayer = Effect.gen(function*() { + const fs = yield* FileSystem.FileSystem + const directory = yield* fs.makeTempDirectoryScoped() + return SqliteClient.layer({ filename: `${directory}/relay-inbox.sqlite` }) +}).pipe(Layer.unwrap, Layer.provide(NodeFileSystem.layer)) + +const storeFor = (client: Layer.Layer) => + SqlRelayInboxStore.layer.pipe( + Layer.provide(client), + Layer.provide(NodeCrypto.layer), + Layer.orDie + ) - it.effect("reports a settlement that no longer applies", () => - Effect.gen(function*() { - const store = yield* RelayInboxStore.RelayInboxStore - yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0 })) - const options = { - outcome: "Acknowledged" as const, - messageHash: "a".repeat(64), - now: 10, - terminalRetentionMillis: 1_000 +describe("RelayInboxStore", () => { + for ( + const dialect of [ + { name: "sqlite", client: SqliteLayer, timeout: 60_000 }, + { name: "postgres", client: PgContainer.layerClient, timeout: 180_000 }, + { name: "mysql", client: MysqlContainer.layerClient, timeout: 240_000 } + ] + ) { + it.layer(storeFor(dialect.client), { + timeout: dialect.timeout + })(dialect.name, (it) => { + for (const check of relayInboxStoreContract) { + it.effect(check.name, () => check.run) } - yield* store.settle("a", relayId("000000000001"), options) - const again = yield* store.settle("a", relayId("000000000001"), options) - assert.strictEqual(again, "NotPending") - }).pipe(Effect.provide(layer))) - - it.effect("still deduplicates a replay that arrives after settlement", () => - Effect.gen(function*() { - const store = yield* RelayInboxStore.RelayInboxStore - yield* store.admit( - admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0, horizon: 100_000 }) - ) - yield* store.settle("a", relayId("000000000001"), { - outcome: "Acknowledged", - messageHash: "a".repeat(64), - now: 10, - terminalRetentionMillis: 1_000 - }) - - const replay = yield* store.admit( - admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 20, horizon: 100_000 }) - ) - assert.strictEqual(replay._tag, "Duplicate") - const heads = yield* store.pendingHeads("a", { limit: 10 }) - assert.strictEqual(heads.length, 0, "an acknowledged message must not become deliverable again") - }).pipe(Effect.provide(layer))) - - it.effect("revives a message the inbox had given up on rather than telling the sender it landed", () => - Effect.gen(function*() { - const store = yield* RelayInboxStore.RelayInboxStore - yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0, ttl: 1_000 })) - yield* store.expire({ now: 5_000, limit: 10, terminalRetentionMillis: 1_000 }) - - // The sender still holds custody and is replaying. Reporting a duplicate would make it drop - // the last copy of a message this inbox can no longer deliver. - const replay = yield* store.admit( - admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 6_000, ttl: 1_000 }) - ) - assert.strictEqual(replay._tag, "Admitted") - - const heads = yield* store.pendingHeads("a", { limit: 10 }) - assert.strictEqual(heads.length, 1, "the revived message is deliverable again") - }).pipe(Effect.provide(layer))) - - it.effect("charges a revived message against the inbox quota", () => - Effect.gen(function*() { - const store = yield* RelayInboxStore.RelayInboxStore - yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0, ttl: 1_000 })) - yield* store.expire({ now: 5_000, limit: 10, terminalRetentionMillis: 1_000 }) - - const revived = yield* store.admit(admission({ - inboxKey: "a", - id: "000000000001", - sequence: 0, - now: 6_000, - ttl: 1_000, - quota: { maxPendingMessages: 0, maxPendingBytes: 0 } - })) - assert.strictEqual( - revived._tag, - "QuotaExceeded", - "reviving creates pending work, so it cannot bypass the cap a first admission obeys" - ) - }).pipe(Effect.provide(layer))) - - it.effect("keeps a revived message deduplicated for the sender's replay window", () => - Effect.gen(function*() { - const store = yield* RelayInboxStore.RelayInboxStore - yield* store.admit( - admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0, ttl: 1_000, horizon: 1_000 }) - ) - yield* store.expire({ now: 2_000, limit: 10, terminalRetentionMillis: 1_000 }) - yield* store.admit( - admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 2_000, ttl: 1_000, horizon: 10_000 }) - ) - yield* store.expire({ now: 3_100, limit: 10, terminalRetentionMillis: 1_000 }) - - // The horizon was extended by the revive, so collection at this point must not remove it. - const collected = yield* store.collect({ now: 3_200, limit: 10 }) - assert.strictEqual( - collected, - 0, - "collecting inside the sender's retry window would let a replay be applied twice" - ) - }).pipe(Effect.provide(layer))) - - it.effect("removes terminal rows only once their deduplication horizon lapses", () => - Effect.gen(function*() { - const store = yield* RelayInboxStore.RelayInboxStore - yield* store.admit( - admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0, horizon: 1_000 }) - ) - yield* store.settle("a", relayId("000000000001"), { - outcome: "Acknowledged", - messageHash: "a".repeat(64), - now: 10, - terminalRetentionMillis: 100 - }) - - assert.strictEqual(yield* store.collect({ now: 500, limit: 10 }), 0) - assert.strictEqual(yield* store.collect({ now: 5_000, limit: 10 }), 1) - }).pipe(Effect.provide(layer))) - - it.effect("refuses admission beyond the inbox quota", () => - Effect.gen(function*() { - const store = yield* RelayInboxStore.RelayInboxStore - yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0 })) - const refused = yield* store.admit(admission({ - inboxKey: "a", - id: "000000000002", - sequence: 1, - now: 1, - quota: { maxPendingMessages: 1, maxPendingBytes: 10_000_000 } - })) - assert.strictEqual(refused._tag, "QuotaExceeded") - }).pipe(Effect.provide(layer))) - - it.effect("counts a replay once against usage", () => - Effect.gen(function*() { - const store = yield* RelayInboxStore.RelayInboxStore - yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 0 })) - yield* store.admit(admission({ inboxKey: "a", id: "000000000001", sequence: 0, now: 1 })) - - const usage = yield* store.usage("a") - assert.strictEqual(usage.pendingCount, 1, "quota is derived from rows, so replays cannot inflate it") - }).pipe(Effect.provide(layer))) + }) + } }) diff --git a/packages/local-rpc/test/RelayInboxStoreContract.ts b/packages/local-rpc/test/RelayInboxStoreContract.ts new file mode 100644 index 0000000..30b39c8 --- /dev/null +++ b/packages/local-rpc/test/RelayInboxStoreContract.ts @@ -0,0 +1,462 @@ +import { assert } from "@effect/vitest" +import * as Identity from "@lucas-barake/effect-local/Identity" +import type * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" +import * as Effect from "effect/Effect" +import * as RelayInboxStore from "../src/RelayInboxStore.js" + +/** + * The `RelayInboxStore` contract, run unchanged against every dialect the relay supports. + * + * A single dialect is not enough to establish this contract. The PostgreSQL driver returns + * `BIGINT`, `COUNT` and `SUM` as strings with no default parser, which made every numeric column + * fail to decode and left the store non-functional on that dialect while SQLite stayed green; and + * MySQL compares identity columns case-insensitively unless they carry a binary collation, which + * caused real route crossover. Both are invisible to a SQLite-only suite. + * + * Every check owns a distinct `inboxKey`, because `expire` and `collect` are deployment-wide sweeps + * rather than per-inbox operations and the dialect containers are shared across the whole block. + * For the same reason no check asserts a sweep's returned count: that count spans other checks' + * leftovers. Each asserts the effect on its own inbox instead, which is the real guarantee anyway. + */ + +const peer = (value: string) => Identity.PeerId.make(`peer_00000000-0000-4000-8000-${value}`) +const relayId = (value: string) => Identity.RelayMessageId.make(`rly_00000000-0000-4000-8000-${value}`) +const documentId = (value: string) => Identity.DocumentId.make(`doc_00000000-0000-4000-8000-${value}`) + +const quota = { maxPendingMessages: 100, maxPendingBytes: 10_000_000 } + +const channel = ( + options?: { readonly epoch?: string | undefined; readonly subject?: string | undefined } +) => ({ + tenantId: "tenant-a", + senderSubjectId: options?.subject ?? "sender-a", + senderPeerId: peer("00000000aaa1"), + senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), + senderConnectionEpoch: options?.epoch ?? "epoch-1" +}) + +const envelope = (options: { + readonly id: string + readonly sequence: number + readonly digest?: string + readonly epoch?: string + readonly subject?: string +}) => { + const source = channel({ epoch: options.epoch, subject: options.subject }) + return { + relayMessageId: relayId(options.id), + relayPeerId: peer("00000000ffff"), + sender: { + tenantId: source.tenantId, + subjectId: source.senderSubjectId, + peerId: source.senderPeerId, + replicaIncarnation: source.senderReplicaIncarnation, + connectionEpoch: source.senderConnectionEpoch, + sequence: options.sequence + }, + recipient: { + tenantId: "tenant-a", + subjectId: "recipient-a", + peerId: peer("00000000bbb1") + }, + payloadVersion: 1 as const, + document: { documentId: documentId("00000000dddd"), documentType: "note" }, + writerProvenance: [], + messageHash: "a".repeat(64), + outerEnvelopeDigest: (options.digest ?? "b").repeat(64).slice(0, 64), + payload: new Uint8Array([1, 2, 3]) + } +} + +const admission = (options: { + readonly inboxKey: string + readonly id: string + readonly sequence: number + readonly now: number + readonly ttl?: number + readonly horizon?: number + readonly digest?: string + readonly epoch?: string + readonly subject?: string + readonly quota?: RelayInboxStore.AdmissionQuota +}) => ({ + inboxKey: options.inboxKey, + channel: channel({ epoch: options.epoch, subject: options.subject }), + envelope: envelope(options), + now: options.now, + messageTtlMillis: options.ttl ?? 1_000, + senderRetryHorizonMillis: options.horizon ?? 1_000, + quota: options.quota ?? quota +}) + +export interface ContractCheck { + readonly name: string + readonly run: Effect.Effect +} + +export const relayInboxStoreContract: ReadonlyArray = [ + { + name: "admits a message and reports it as the pending head", + run: Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + const result = yield* store.admit(admission({ inboxKey: "admit", id: "000000000001", sequence: 0, now: 0 })) + assert.strictEqual(result._tag, "Admitted") + + const heads = yield* store.pendingHeads("admit", { limit: 10 }) + assert.strictEqual(heads.length, 1) + assert.strictEqual(heads[0]!.relayMessageId, relayId("000000000001")) + }) + }, + { + name: "reports a replay of the same identity and digest as a duplicate", + run: Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit(admission({ inboxKey: "replay", id: "000000000001", sequence: 0, now: 0 })) + const replay = yield* store.admit(admission({ inboxKey: "replay", id: "000000000001", sequence: 0, now: 1 })) + + assert.strictEqual(replay._tag, "Duplicate") + assert.strictEqual(replay._tag === "Duplicate" ? replay.state : "", "Pending") + const heads = yield* store.pendingHeads("replay", { limit: 10 }) + assert.strictEqual(heads.length, 1, "a replay must not create a second row") + }) + }, + { + name: "rejects the same identity carrying a different envelope digest", + run: Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit(admission({ inboxKey: "conflict", id: "000000000001", sequence: 0, now: 0 })) + const conflict = yield* store.admit( + admission({ inboxKey: "conflict", id: "000000000001", sequence: 0, now: 1, digest: "c" }) + ) + assert.strictEqual(conflict._tag, "Conflict") + }) + }, + { + name: "refuses a message whose channel disagrees with its envelope", + run: Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + const request = admission({ inboxKey: "forged", id: "000000000001", sequence: 0, now: 0 }) + const result = yield* store.admit({ + ...request, + channel: { ...request.channel, senderConnectionEpoch: "another-epoch" } + }) + assert.strictEqual( + result._tag, + "Conflict", + "a fabricated channel would file the message under an ordering stream it does not belong to" + ) + }) + }, + { + name: "expiring one inbox leaves another inbox's copy of the same identity untouched", + run: Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + // The same relay message id addressed to two devices. The key is (inbox, id), so a sweep that + // filters on the id alone would reach across inboxes. + yield* store.admit( + admission({ inboxKey: "sweep-overdue", id: "000000000001", sequence: 0, now: 1_000_000, ttl: 1_000 }) + ) + yield* store.admit( + admission({ inboxKey: "sweep-live", id: "000000000001", sequence: 0, now: 1_000_000, ttl: 10_000_000 }) + ) + + yield* store.expire({ now: 1_005_000, limit: 1_000, terminalRetentionMillis: 1_000 }) + + assert.strictEqual( + (yield* store.pendingHeads("sweep-overdue", { limit: 10 })).length, + 0, + "the overdue message is expired" + ) + assert.strictEqual( + (yield* store.pendingHeads("sweep-live", { limit: 10 })).length, + 1, + "the other inbox still holds a message with time left" + ) + }) + }, + { + name: "delivers one head per channel and does not starve a channel with high sequences", + run: Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + // `senderSequence` restarts per connection epoch, so it cannot order heads across channels. + yield* store.admit( + admission({ inboxKey: "heads", id: "000000000001", sequence: 0, now: 0, subject: "sender-a" }) + ) + yield* store.admit( + admission({ inboxKey: "heads", id: "000000000002", sequence: 1, now: 1, subject: "sender-b" }) + ) + yield* store.admit( + admission({ inboxKey: "heads", id: "000000000003", sequence: 500, now: 2, subject: "sender-c" }) + ) + + assert.strictEqual((yield* store.pendingHeads("heads", { limit: 2 })).length, 2) + + const all = yield* store.pendingHeads("heads", { limit: 10 }) + assert.deepStrictEqual( + all.map((head) => head.channel.senderSubjectId).toSorted(), + ["sender-a", "sender-b", "sender-c"] + ) + }) + }, + { + name: "orders within a channel by sender sequence", + run: Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit(admission({ inboxKey: "order", id: "000000000002", sequence: 5, now: 0 })) + yield* store.admit(admission({ inboxKey: "order", id: "000000000001", sequence: 1, now: 1 })) + + const heads = yield* store.pendingHeads("order", { limit: 10 }) + assert.strictEqual(heads.length, 1, "one head per channel") + assert.strictEqual(heads[0]!.relayMessageId, relayId("000000000001")) + }) + }, + { + name: "spends the full delivery budget before dead lettering", + run: Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit(admission({ inboxKey: "budget", id: "000000000001", sequence: 0, now: 0 })) + + const first = yield* store.recordDelivery("budget", relayId("000000000001"), { + maxDeliveries: 1, + now: 1 + }) + assert.strictEqual( + first._tag, + "Recorded", + "the first delivery of a budget of one must be allowed to be settled" + ) + + const second = yield* store.recordDelivery("budget", relayId("000000000001"), { + maxDeliveries: 1, + now: 2 + }) + assert.strictEqual(second._tag, "DeadLettered") + + const heads = yield* store.pendingHeads("budget", { limit: 10 }) + assert.strictEqual(heads.length, 0, "a dead lettered message stops blocking its channel") + }) + }, + { + name: "makes an abandoned message answerable after the fact", + run: Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit(admission({ inboxKey: "abandoned", id: "000000000001", sequence: 0, now: 0 })) + yield* store.recordDelivery("abandoned", relayId("000000000001"), { maxDeliveries: 1, now: 1 }) + yield* store.recordDelivery("abandoned", relayId("000000000001"), { maxDeliveries: 1, now: 2 }) + + const abandoned = yield* store.abandoned("abandoned", { limit: 10 }) + assert.strictEqual(abandoned.length, 1) + assert.strictEqual(abandoned[0]!.state, "DeadLettered") + }) + }, + { + name: "settles on acknowledgement and does not redeliver", + run: Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit(admission({ inboxKey: "ack", id: "000000000001", sequence: 0, now: 0 })) + + const settled = yield* store.settle("ack", relayId("000000000001"), { + outcome: "Acknowledged", + messageHash: "a".repeat(64), + now: 10, + terminalRetentionMillis: 1_000 + }) + assert.strictEqual(settled, "Settled") + assert.strictEqual((yield* store.pendingHeads("ack", { limit: 10 })).length, 0) + }) + }, + { + name: "refuses a settlement whose content does not match", + run: Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit(admission({ inboxKey: "hash", id: "000000000001", sequence: 0, now: 0 })) + + const settled = yield* store.settle("hash", relayId("000000000001"), { + outcome: "Acknowledged", + messageHash: "f".repeat(64), + now: 10, + terminalRetentionMillis: 1_000 + }) + assert.strictEqual(settled, "HashMismatch") + assert.strictEqual( + (yield* store.pendingHeads("hash", { limit: 10 })).length, + 1, + "the message stays deliverable" + ) + }) + }, + { + name: "reports a settlement that no longer applies", + run: Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit(admission({ inboxKey: "resettle", id: "000000000001", sequence: 0, now: 0 })) + const options = { + outcome: "Acknowledged" as const, + messageHash: "a".repeat(64), + now: 10, + terminalRetentionMillis: 1_000 + } + yield* store.settle("resettle", relayId("000000000001"), options) + const again = yield* store.settle("resettle", relayId("000000000001"), options) + assert.strictEqual(again, "NotPending") + }) + }, + { + name: "still deduplicates a replay that arrives after settlement", + run: Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit( + admission({ inboxKey: "dedupe", id: "000000000001", sequence: 0, now: 0, horizon: 100_000 }) + ) + yield* store.settle("dedupe", relayId("000000000001"), { + outcome: "Acknowledged", + messageHash: "a".repeat(64), + now: 10, + terminalRetentionMillis: 1_000 + }) + + const replay = yield* store.admit( + admission({ inboxKey: "dedupe", id: "000000000001", sequence: 0, now: 20, horizon: 100_000 }) + ) + assert.strictEqual(replay._tag, "Duplicate") + assert.strictEqual( + (yield* store.pendingHeads("dedupe", { limit: 10 })).length, + 0, + "an acknowledged message must not become deliverable again" + ) + }) + }, + { + name: "revives a message the inbox had given up on rather than telling the sender it landed", + run: Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit( + admission({ inboxKey: "revive", id: "000000000001", sequence: 0, now: 2_000_000, ttl: 1_000 }) + ) + yield* store.expire({ now: 2_005_000, limit: 1_000, terminalRetentionMillis: 1_000 }) + + // The sender still holds custody and is replaying. Reporting a duplicate would make it drop + // the last copy of a message this inbox can no longer deliver. + const replay = yield* store.admit( + admission({ inboxKey: "revive", id: "000000000001", sequence: 0, now: 2_006_000, ttl: 1_000 }) + ) + assert.strictEqual(replay._tag, "Admitted") + assert.strictEqual( + (yield* store.pendingHeads("revive", { limit: 10 })).length, + 1, + "the revived message is deliverable again" + ) + }) + }, + { + name: "charges a revived message against the inbox quota", + run: Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit( + admission({ inboxKey: "revive-quota", id: "000000000001", sequence: 0, now: 3_000_000, ttl: 1_000 }) + ) + yield* store.expire({ now: 3_005_000, limit: 1_000, terminalRetentionMillis: 1_000 }) + + const revived = yield* store.admit(admission({ + inboxKey: "revive-quota", + id: "000000000001", + sequence: 0, + now: 3_006_000, + ttl: 1_000, + quota: { maxPendingMessages: 0, maxPendingBytes: 0 } + })) + assert.strictEqual( + revived._tag, + "QuotaExceeded", + "reviving creates pending work, so it cannot bypass the cap a first admission obeys" + ) + }) + }, + { + name: "keeps a revived message deduplicated for the sender's replay window", + run: Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit( + admission({ + inboxKey: "revive-horizon", + id: "000000000001", + sequence: 0, + now: 4_000_000, + ttl: 1_000, + horizon: 1_000 + }) + ) + yield* store.expire({ now: 4_002_000, limit: 1_000, terminalRetentionMillis: 1_000 }) + yield* store.admit( + admission({ + inboxKey: "revive-horizon", + id: "000000000001", + sequence: 0, + now: 4_002_000, + ttl: 1_000, + horizon: 10_000 + }) + ) + yield* store.expire({ now: 4_003_100, limit: 1_000, terminalRetentionMillis: 1_000 }) + + // The horizon was extended by the revive, so collection at this point must not remove it. + yield* store.collect({ now: 4_003_200, limit: 1_000 }) + assert.strictEqual( + (yield* store.usage("revive-horizon")).retainedCount, + 1, + "collecting inside the sender's retry window would let a replay be applied twice" + ) + }) + }, + { + name: "removes terminal rows only once their deduplication horizon lapses", + run: Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit( + admission({ inboxKey: "collect", id: "000000000001", sequence: 0, now: 5_000_000, horizon: 1_000 }) + ) + yield* store.settle("collect", relayId("000000000001"), { + outcome: "Acknowledged", + messageHash: "a".repeat(64), + now: 5_000_010, + terminalRetentionMillis: 100 + }) + + yield* store.collect({ now: 5_000_500, limit: 1_000 }) + assert.strictEqual( + (yield* store.usage("collect")).retainedCount, + 1, + "the identity is retained while its sender may still replay it" + ) + + yield* store.collect({ now: 5_005_000, limit: 1_000 }) + assert.strictEqual((yield* store.usage("collect")).retainedCount, 0) + }) + }, + { + name: "refuses admission beyond the inbox quota", + run: Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit(admission({ inboxKey: "quota", id: "000000000001", sequence: 0, now: 0 })) + const refused = yield* store.admit(admission({ + inboxKey: "quota", + id: "000000000002", + sequence: 1, + now: 1, + quota: { maxPendingMessages: 1, maxPendingBytes: 10_000_000 } + })) + assert.strictEqual(refused._tag, "QuotaExceeded") + }) + }, + { + name: "counts a replay once against usage", + run: Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit(admission({ inboxKey: "usage", id: "000000000001", sequence: 0, now: 0 })) + yield* store.admit(admission({ inboxKey: "usage", id: "000000000001", sequence: 0, now: 1 })) + + const usage = yield* store.usage("usage") + assert.strictEqual(usage.pendingCount, 1, "quota is derived from rows, so replays cannot inflate it") + }) + } +] diff --git a/packages/local-rpc/test/SqlPeerRelayStore.test.ts b/packages/local-rpc/test/SqlPeerRelayStore.test.ts deleted file mode 100644 index d6f6c14..0000000 --- a/packages/local-rpc/test/SqlPeerRelayStore.test.ts +++ /dev/null @@ -1,255 +0,0 @@ -import { NodeCrypto, NodeFileSystem } from "@effect/platform-node" -import { MysqlClient } from "@effect/sql-mysql2" -import { PgClient } from "@effect/sql-pg" -import { SqliteClient } from "@effect/sql-sqlite-node" -import { assert, describe, it } from "@effect/vitest" -import * as Identity from "@lucas-barake/effect-local/Identity" -import { MySqlContainer, type StartedMySqlContainer } from "@testcontainers/mysql" -import { PostgreSqlContainer } from "@testcontainers/postgresql" -import { Context, Data, Effect, Exit, FileSystem, Layer, Option, Redacted } from "effect" -import * as SqlClient from "effect/unstable/sql/SqlClient" -import * as PeerRelayLimits from "../src/PeerRelayLimits.js" -import * as PeerRelayStore from "../src/PeerRelayStore.js" -import * as PeerRpc from "../src/PeerRpc.js" -import * as SqlPeerRelayStore from "../src/SqlPeerRelayStore.js" -import { runPeerRelayStoreContract } from "./PeerRelayStoreContract.js" - -class ContainerError extends Data.TaggedError("ContainerError")<{ - readonly cause: unknown -}> {} - -class PgContainer extends Context.Service()( - "@lucas-barake/effect-local-rpc/test/PgContainer", - { - make: Effect.acquireRelease( - Effect.tryPromise({ - try: () => new PostgreSqlContainer("postgres:alpine").start(), - catch: (cause) => new ContainerError({ cause }) - }), - (container) => Effect.promise(() => container.stop()) - ) - } -) { - static readonly layer = Layer.effect(this)(this.make) - - static readonly layerClient = Layer.unwrap( - Effect.gen(function*() { - const container = yield* PgContainer - return PgClient.layer({ - url: Redacted.make(container.getConnectionUri()) - }) - }) - ).pipe(Layer.provide(this.layer)) -} - -class MysqlContainer extends Context.Service< - MysqlContainer, - StartedMySqlContainer ->()("@lucas-barake/effect-local-rpc/test/MysqlContainer") { - static readonly layer = Layer.effect(this)( - Effect.acquireRelease( - Effect.tryPromise({ - try: () => new MySqlContainer("mysql:lts").start(), - catch: (cause) => new ContainerError({ cause }) - }), - (container) => Effect.promise(() => container.stop()) - ) - ) - - static readonly layerClient = Layer.unwrap( - Effect.gen(function*() { - const container = yield* MysqlContainer - return MysqlClient.layer({ - url: Redacted.make(container.getConnectionUri()) - }) - }) - ).pipe(Layer.provide(this.layer)) -} - -const SqliteLayer = Effect.gen(function*() { - const fs = yield* FileSystem.FileSystem - const directory = yield* fs.makeTempDirectoryScoped() - return SqliteClient.layer({ - filename: `${directory}/relay.sqlite` - }) -}).pipe(Layer.unwrap, Layer.provide(NodeFileSystem.layer)) - -const StoreLive = SqlPeerRelayStore.layer.pipe( - Layer.provideMerge(NodeCrypto.layer), - Layer.provideMerge(PeerRelayLimits.layer(PeerRelayLimits.Values.make({ - ...PeerRelayLimits.defaults, - maxActiveMessagesPerSenderPeer: 1 - }))) -) - -const MysqlMigrationTestLive = Layer.mergeAll( - MysqlContainer.layerClient, - NodeCrypto.layer, - PeerRelayLimits.layerDefaults -) - -const peerId = (suffix: string) => Identity.PeerId.make(`peer_00000000-0000-4000-8000-${suffix}`) - -const relayMessageId = (suffix: string) => Identity.RelayMessageId.make(`rly_00000000-0000-4000-8000-${suffix}`) - -const documentId = (suffix: string) => Identity.DocumentId.make(`doc_00000000-0000-4000-8000-${suffix}`) - -describe("SqlPeerRelayStore", () => { - it.layer(MysqlMigrationTestLive, { - timeout: 120_000 - })("mysql migrations", (it) => { - it.effect("recovers after DDL commits a partial first migration", () => - Effect.gen(function*() { - const sql = (yield* SqlClient.SqlClient).withoutTransforms() - - yield* sql.unsafe(` - CREATE TABLE effect_local_relay_channels ( - malformed INT NOT NULL - ) - `) - - const firstAttempt = yield* Effect.exit(SqlPeerRelayStore.make) - assert.strictEqual(Exit.isFailure(firstAttempt), true) - - const recordedAfterFailure = yield* sql.unsafe( - "SELECT migration_id FROM effect_local_relay_migrations WHERE migration_id = 1" - ) - assert.strictEqual(recordedAfterFailure.length, 0) - - yield* sql.unsafe("DROP TABLE effect_local_relay_channels") - - yield* SqlPeerRelayStore.make - - const recordedAfterRecovery = yield* sql.unsafe( - "SELECT migration_id FROM effect_local_relay_migrations WHERE migration_id = 1" - ) - assert.strictEqual(recordedAfterRecovery.length, 1) - - const relayTables = yield* sql.unsafe(` - SELECT table_name - FROM information_schema.tables - WHERE table_schema = DATABASE() - AND table_name IN ( - 'effect_local_relay_write_lock', - 'effect_local_relay_channels', - 'effect_local_relay_messages', - 'effect_local_relay_usage', - 'effect_local_relay_reservations' - ) - `) - assert.strictEqual(relayTables.length, 5) - })) - }) - ;([ - ["pg", Layer.orDie(PgContainer.layerClient)], - ["mysql", Layer.orDie(MysqlContainer.layerClient)], - ["sqlite", Layer.orDie(SqliteLayer)] - ] as const).forEach(([label, layer]) => { - it.layer(StoreLive.pipe(Layer.provideMerge(layer)), { - timeout: 120_000 - })(label, (it) => { - it.effect("conforms to the relay custody contract", () => runPeerRelayStoreContract) - - it.effect("keeps case-distinct routing custody and usage isolated", () => - Effect.gen(function*() { - const store = yield* PeerRelayStore.PeerRelayStore - const senderPeerId = peerId("000000000011") - const recipientPeerId = peerId("000000000012") - const upperChannel = PeerRelayStore.ChannelKey.make({ - tenantId: "Tenant-Case-Distinct", - senderSubjectId: "Sender-Case-Distinct", - senderPeerId, - senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), - recipientSubjectId: "Recipient-Case-Distinct", - recipientPeerId - }) - const lowerChannel = PeerRelayStore.ChannelKey.make({ - tenantId: upperChannel.tenantId.toLowerCase(), - senderSubjectId: upperChannel.senderSubjectId.toLowerCase(), - senderPeerId, - senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), - recipientSubjectId: upperChannel.recipientSubjectId.toLowerCase(), - recipientPeerId - }) - const makeAdmission = ( - channel: PeerRelayStore.ChannelKey, - relayId: Identity.RelayMessageId, - digest: string - ) => - PeerRelayStore.Admission.make({ - channel, - relayMessageId: relayId, - relayPeerId: peerId("000000000013"), - documentIds: [documentId("000000000011")], - senderConnectionEpoch: "epoch-case-distinct", - senderSequence: 0, - payloadVersion: 1, - messageHash: `message-hash-${digest}`, - outerEnvelopeDigest: PeerRpc.RelayDigest.make(digest.repeat(64)), - payload: new Uint8Array([1]), - messageTtlMillis: PeerRelayLimits.defaults.messageTtlMillis, - senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis, - minimumTerminalRetentionMillis: PeerRelayLimits.defaults.minimumTerminalRetentionMillis - }) - - assert.strictEqual( - (yield* store.admit( - makeAdmission(upperChannel, relayMessageId("000000000011"), "b") - )).status, - "Accepted" - ) - assert.strictEqual( - (yield* store.admit( - makeAdmission(lowerChannel, relayMessageId("000000000012"), "c") - )).status, - "Accepted" - ) - - for (const channel of [upperChannel, lowerChannel]) { - assert.deepStrictEqual( - yield* store.usage(PeerRelayStore.UsageRequest.make({ - scopeKind: "Tenant", - scopeKey: JSON.stringify([channel.tenantId]) - })), - { - activeCount: 1, - activeBytes: 1, - retainedCount: 1, - retainedBytes: 1 - } - ) - assert.deepStrictEqual( - yield* store.usage(PeerRelayStore.UsageRequest.make({ - scopeKind: "RecipientSubject", - scopeKey: JSON.stringify([channel.tenantId, channel.recipientSubjectId]) - })), - { - activeCount: 1, - activeBytes: 1, - retainedCount: 1, - retainedBytes: 1 - } - ) - } - - const lowerClaim = yield* store.claim(PeerRelayStore.ClaimRequest.make({ - recipient: { - tenantId: lowerChannel.tenantId, - subjectId: lowerChannel.recipientSubjectId, - peerId: lowerChannel.recipientPeerId - }, - sender: { - subjectId: lowerChannel.senderSubjectId, - peerId: lowerChannel.senderPeerId - }, - sessionGeneration: 1, - authorizedDocumentIds: [documentId("000000000011")] - })) - assert.strictEqual(Option.isSome(lowerClaim.message), true) - if (Option.isSome(lowerClaim.message)) { - assert.deepStrictEqual(lowerClaim.message.value.channel, lowerChannel) - } - })) - }) - }) -}) From 1d63e96e1d0b3f317e858ce3ba5f17fc58067e6e Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Mon, 27 Jul 2026 09:46:39 -0500 Subject: [PATCH 21/29] Stop a withheld delivery from stalling the rest of the inbox Re-authorizing every delivery introduced a stall. The front door takes a message off the entity's rendezvous queue and only then decides the recipient may not have it. By that point the delivering fiber has returned from the offer, charged the delivery, and is parked waiting for a settlement that can never arrive, because the client never saw the message. It holds its channel the whole time. The damage is not limited to that channel. Enough withheld heads occupy every one of the session's delivery slots, at which point channels the recipient is fully entitled to are never delivered at all - a permanent stall bounded only by the session ending. A reviewer reproduced both halves against the real composition. The seam was the problem: Subscribe and Settle gave the front door no way to say "not deliverable to this session, release it". Release does that. It ends the attempt without settling, so the row stays Pending for a later session, frees the channel, and records the channel as withheld for this session only - skipping the whole channel rather than the message, because passing over a head would break the ordering the channel exists to preserve. A reconnect starts clean, so a grant that comes back is picked up. The dispatcher also had to widen its poll by the number of withheld channels. Heads come back oldest first, so once enough withheld ones accumulate the query returned nothing else and the channels behind them were never looked at. Both halves are load bearing: removing either fails the regression test. The previous withhold test could not have caught any of this. It asserted an absence one scheduler pass too early, so it passed whether the message was withheld, delivered, or failed the stream. It now waits on a real rendezvous. The risk-grant knob was also tied to the ordinary authorization knob, which hid whether either port was enforced on its own; they are now independent. --- packages/local-rpc/src/RelayInbox.ts | 94 ++++++++++++++++++++- packages/local-rpc/src/RelayServer.ts | 34 +++++++- packages/local-rpc/test/RelayServer.test.ts | 80 +++++++++++++++--- 3 files changed, 192 insertions(+), 16 deletions(-) diff --git a/packages/local-rpc/src/RelayInbox.ts b/packages/local-rpc/src/RelayInbox.ts index 19e3804..6f055ec 100644 --- a/packages/local-rpc/src/RelayInbox.ts +++ b/packages/local-rpc/src/RelayInbox.ts @@ -102,6 +102,27 @@ export class HeartbeatRpc extends Rpc.make("Heartbeat", { error: InboxError }) {} +/** + * Ends a delivery attempt without settling it. + * + * The front door re-authorizes every message it takes off this entity's stream, and a message it + * may not hand over has already left the queue by then: the delivering fiber is waiting for a + * settlement that can never arrive, and it holds its channel while it waits. Releasing ends that + * attempt, leaves the row `Pending` for a later session, and frees the channel so the rest of the + * inbox keeps moving — without this, one withheld message stops every channel sharing the + * session's delivery slots, including channels the recipient is fully entitled to. + * + * Off the public wire contract, like `Heartbeat`: it is the front door's own bookkeeping. + */ +export class ReleaseRpc extends Rpc.make("Release", { + payload: { + sessionId: Identity.SessionId, + relayMessageId: Identity.RelayMessageId, + claimToken: PeerRpc.ClaimToken + }, + error: InboxError +}) {} + export class EndSessionRpc extends Rpc.make("EndSession", { payload: { sessionId: Identity.SessionId }, error: InboxError @@ -111,6 +132,7 @@ export const RelayInbox = Entity.make("EffectLocalRelayInbox", [ DeliverRpc, SubscribeRpc, SettleRpc, + ReleaseRpc, HeartbeatRpc, EndSessionRpc ]) @@ -153,11 +175,23 @@ export interface Options { readonly maxIdleTimeMillis: number } +/** + * How a delivery attempt ended. + * + * `Released` is not a terminal state: the front door took the message off the queue and then + * decided the recipient may not have it, so the row stays `Pending` for a later session. It has to + * be distinguishable from a settlement, because the attempt still has to end — otherwise the + * delivering fiber waits forever for a settlement that can never come, holding its channel. + */ +type AttemptOutcome = RelayInboxStore.TerminalOutcome | "Released" + interface Settlement { readonly claimToken: PeerRpc.ClaimToken readonly messageHash: string - /** Completed by the recipient's `Settle` call. */ - readonly requested: Deferred.Deferred + /** The channel this attempt belongs to, so releasing it can free exactly that channel. */ + readonly channelId: string + /** Completed by the recipient's `Settle` or `Release` call. */ + readonly requested: Deferred.Deferred /** * Completed by the delivering fiber once the terminal transition is durable. * @@ -174,6 +208,15 @@ interface Session { readonly settlements: Map /** Channels with a delivery in flight. One message per channel at a time preserves order. */ readonly busyChannels: Set + /** + * Channels whose head this session may not receive. + * + * Held for the life of the session only. Without it the dispatcher would immediately re-offer the + * head the front door just withheld and spin on it; skipping the whole channel rather than the + * message keeps per-channel ordering intact, since the head cannot be passed over. A reconnect + * starts with an empty set, so a grant that comes back is picked up. + */ + readonly withheldChannels: Set /** Opened whenever there may be new work: a fresh admission, or a channel falling idle. */ readonly wake: Latch.Latch readonly scope: Scope.Closeable @@ -255,7 +298,7 @@ export const layer = (options: Options) => const deliverHead = (session: Session, head: RelayInboxStore.PendingMessage) => Effect.gen(function*() { const claimToken = yield* makeClaimToken - const requested = yield* Deferred.make() + const requested = yield* Deferred.make() const durable = yield* Deferred.make() // Registered before the message is offered. The recipient can settle as soon as it reads // the message, and it does so on a different fiber, so the settlement must already be @@ -263,6 +306,7 @@ export const layer = (options: Options) => session.settlements.set(head.relayMessageId, { claimToken, messageHash: head.envelope.messageHash, + channelId: channelId(head.channel), requested, durable }) @@ -298,6 +342,19 @@ export const layer = (options: Options) => } const outcome = yield* Deferred.await(requested) + if (outcome === "Released") { + // The front door took the message and then found the recipient may not have it. Nothing + // is written: the row stays `Pending` for a later session. Returning here is what frees + // the channel, which is the whole point — a released attempt that kept waiting for a + // settlement would hold its channel, and enough of them would hold every delivery slot + // and starve channels the recipient is perfectly entitled to. + yield* Effect.sync(() => session.withheldChannels.add(channelId(head.channel))) + yield* Effect.logDebug("Relay inbox delivery released without settling").pipe( + Effect.annotateLogs({ inboxKey, relayMessageId: head.relayMessageId }) + ) + return + } + const settledAt = yield* Clock.currentTimeMillis const settled = yield* store.settle(inboxKey, head.relayMessageId, { outcome, @@ -354,12 +411,20 @@ export const layer = (options: Options) => // Closed before polling so an admission that arrives mid poll is not lost: it reopens // the latch and the following await returns immediately. yield* session.wake.close + // Widened by the channels this session has already been told it may not receive. + // Heads come back oldest first, so asking only for as many as can be delivered would + // return nothing but withheld ones once enough of them accumulate, and the channels + // behind them — which the recipient is entitled to — would never even be looked at. const heads = yield* store.pendingHeads(inboxKey, { - limit: options.maxConcurrentChannels + limit: options.maxConcurrentChannels + session.withheldChannels.size }) for (const head of heads) { const channel = channelId(head.channel) if (session.busyChannels.has(channel)) continue + // Skipped for this session only. Re-offering a head the front door just withheld + // would spin, and passing over it to reach the next message in the same channel + // would break the ordering the channel exists to preserve. + if (session.withheldChannels.has(channel)) continue if (session.busyChannels.size >= options.maxConcurrentChannels) break session.busyChannels.add(channel) yield* Effect.forkIn(deliverHead(session, head), session.scope) @@ -512,6 +577,7 @@ export const layer = (options: Options) => outbound, settlements: new Map(), busyChannels: new Set(), + withheldChannels: new Set(), wake: yield* Latch.make(true), scope, deadlineAt @@ -563,6 +629,26 @@ export const layer = (options: Options) => }) ), + Release: ({ payload }) => + Effect.gen(function*() { + const session = yield* currentSession(payload.sessionId) + const settlement = session.settlements.get(payload.relayMessageId) + if (settlement === undefined || settlement.claimToken !== payload.claimToken) { + // Nothing to release: the attempt already ended, or this token belongs to an earlier + // one. Either way the channel is not held by it, so there is nothing to undo. + return + } + yield* extendDeadline(session) + // Marked here as well as on the delivering fiber, so the channel is skipped from the + // moment the release is accepted rather than whenever that fiber next runs. + yield* Effect.sync(() => session.withheldChannels.add(settlement.channelId)) + yield* Deferred.succeed(settlement.requested, "Released") + }).pipe( + Effect.withSpan("RelayInbox.Release", { + attributes: { inbox_key: inboxKey, relay_message_id: payload.relayMessageId } + }) + ), + Heartbeat: ({ payload }) => currentSession(payload.sessionId).pipe( Effect.flatMap(extendDeadline), diff --git a/packages/local-rpc/src/RelayServer.ts b/packages/local-rpc/src/RelayServer.ts index b9be35e..1b25311 100644 --- a/packages/local-rpc/src/RelayServer.ts +++ b/packages/local-rpc/src/RelayServer.ts @@ -116,6 +116,26 @@ export const layerHandlers = (options: Options) => return Effect.succeed(session) }) + /** + * Hands a withheld delivery back to the entity. + * + * Best effort: if it does not land, the channel stays held until the session ends, which is the + * behaviour this call exists to avoid but is not itself a reason to fail the recipient's whole + * stream over a message it was never going to see. + */ + const release = ( + inboxKey: string, + sessionId: Identity.SessionId, + message: PeerRpc.StoredMessage + ) => + Effect.ignore(bounded( + inboxClient(inboxKey).Release({ + sessionId, + relayMessageId: message.relayMessageId, + claimToken: message.claimToken + }) + )) + const settle = ( payload: { readonly sessionId: Identity.SessionId @@ -370,14 +390,24 @@ export const layerHandlers = (options: Options) => sessionId, documentId: message.document.documentId }), + // The message has already left the entity's queue, so its delivering fiber + // is waiting for a settlement that can never arrive and is holding its + // channel while it waits. Releasing ends that attempt without settling: the + // row stays durable for a later session, and the channel stops occupying + // one of this session's delivery slots. Without this one withheld message + // stalls its channel for the whole session, and enough of them starve + // channels the recipient is perfectly entitled to receive. + Effect.andThen(release(inboxKeySelf, sessionId, message)), Effect.as(false) )), - Effect.catchTag("ServerUnavailable", () => Effect.succeed(false)) + Effect.catchTag("ServerUnavailable", () => + release(inboxKeySelf, sessionId, message).pipe(Effect.as(false))) ) ), // Cluster level failures are not part of the public contract, so each is reported as // the wire error that tells the client what to do about it. - Stream.catchTag("MailboxFull", () => Stream.fail(new PeerRpcError.SessionOverloaded())), + Stream.catchTag("MailboxFull", () => + Stream.fail(new PeerRpcError.SessionOverloaded())), Stream.catchTag( "AlreadyProcessingMessage", () => Stream.fail(new PeerRpcError.SessionOverloaded()) diff --git a/packages/local-rpc/test/RelayServer.test.ts b/packages/local-rpc/test/RelayServer.test.ts index b8427f0..4ee641c 100644 --- a/packages/local-rpc/test/RelayServer.test.ts +++ b/packages/local-rpc/test/RelayServer.test.ts @@ -120,7 +120,7 @@ interface Knobs { * The separate acknowledgement that relaying a document commits its recipient to an * allocation-unbounded decode. It denies by default in production, so it is its own knob. */ - allowRisk: boolean + allowRisk: (request: PeerRelayAuthorization.UnsafeUnboundedAutomerge3DecodeRequest) => boolean authority: { validUntil: number; invalidated: Effect.Effect } } @@ -131,7 +131,7 @@ const harness = (options?: { Effect.gen(function*() { const knobs: Knobs = { allow: options?.knobs?.allow ?? (() => true), - allowRisk: options?.knobs?.allowRisk ?? true, + allowRisk: options?.knobs?.allowRisk ?? (() => true), authority: options?.knobs?.authority ?? { validUntil: Number.MAX_SAFE_INTEGER, invalidated: Effect.never @@ -158,13 +158,10 @@ const harness = (options?: { invalidated: Effect.never }) : Effect.fail(new PeerRpcError.AccessDenied()), + // Deliberately independent of `allow`. Tying the two together would make removing either + // port from production invisible, since revoking one knob would revoke both gates at once. (request) => - knobs.allowRisk && knobs.allow({ - direction: request.direction, - principal: request.principal, - remote: request.remote, - documents: request.documents - }) + knobs.allowRisk(request) ? Effect.succeed({ _tag: "UnsafeUnboundedAutomerge3DecodeGrant" as const, risk: PeerRelayAuthorization.unsafeUnboundedAutomerge3DecodeRisk, @@ -276,6 +273,8 @@ const encodePayload = (peer: Harness, options?: { readonly hash?: string readonly documentId?: Identity.DocumentId readonly sequence?: number + /** Part of the channel identity, so two epochs from one sender are two ordered streams. */ + readonly epoch?: string }) => Effect.gen(function*() { const message = options?.message ?? Uint8Array.of(1, 2, 3) @@ -284,7 +283,7 @@ const encodePayload = (peer: Harness, options?: { )) return new TextEncoder().encode( yield* Schema.encodeEffect(RelaySyncEnvelopeJson)({ - connectionEpoch: "epoch", + connectionEpoch: options?.epoch ?? "epoch", sequence: options?.sequence ?? 0, documentId: options?.documentId ?? documentId, documentType: Task.name, @@ -634,12 +633,73 @@ describe("RelayServer", () => { assert.strictEqual(failure._tag, "RequestCapacityExceeded") }))) + it.effect("keeps delivering other channels when one channel's head is withheld", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + const documents = [ + { documentType: Task.name, documentId }, + { documentType: Task.name, documentId: otherDocumentId } + ] + const senderSession = yield* open(peer, sender, recipient, documents) + const recipientSession = yield* open(peer, recipient, sender, documents) + + // Denies only the per-delivery Receive check for one document. Both handshakes ask for both + // documents at once and every Send check is untouched, so nothing else is affected. + peer.knobs.allow = (request) => + !( + request.direction === "Receive" && + request.documents.length === 1 && + request.documents[0]!.documentId === documentId + ) + + // One withheld channel per concurrent delivery slot, admitted first so they are the older + // heads, then one channel the recipient is plainly entitled to. A withheld head that keeps + // waiting for a settlement it can never get holds its slot, and enough of them hold every + // slot — at which point the authorized channel is never even looked at. + // (Within a single channel a withheld head does correctly block what follows it: passing over + // it would break the ordering the channel exists to preserve.) + for (let index = 0; index < inboxOptions.maxConcurrentChannels; index++) { + yield* push( + peer, + senderSession.opened.sessionId, + yield* encodePayload(peer, { epoch: `withheld-${index}` }), + `00000000000${index + 1}` + ) + // Heads are ordered by admission time, and under virtual time these would otherwise all + // share one timestamp and fall back to an arbitrary tiebreak — which would sometimes float + // the authorized channel into the first batch and hide the starvation. + yield* TestClock.adjust(10) + } + yield* push( + peer, + senderSession.opened.sessionId, + yield* encodePayload(peer, { documentId: otherDocumentId, epoch: "clean" }), + "000000000009" + ) + + // Blocks until the authorized message arrives. If the withheld head still held its channel + // and its delivery slot, this would wait forever. + const delivered = yield* Queue.take(recipientSession.events) + assert.strictEqual(delivered._tag, "StoredMessage") + if (delivered._tag !== "StoredMessage") return + assert.strictEqual(delivered.relayMessageId, relayId("000000000009")) + + const pending = yield* peer.store.pendingHeads( + yield* inboxKeyOf(peer, recipient), + { limit: 10 } + ) + assert.isTrue( + pending.some((message) => message.relayMessageId === relayId("000000000001")), + "the withheld message stays durable rather than being settled away" + ) + }))) + it.effect("refuses a push when the unbounded decode risk is not acknowledged", () => Effect.scoped(Effect.gen(function*() { // Ordinary Send authorization is granted; only the separate risk acknowledgement is withheld. // Relaying commits the recipient to an allocation-unbounded decode, so it is gated on its own // port that denies by default, and an ordinary grant must not be able to stand in for it. - const peer = yield* harness({ knobs: { allowRisk: false } }) + const peer = yield* harness({ knobs: { allowRisk: () => false } }) const senderSession = yield* open(peer, sender, recipient) const payload = yield* encodePayload(peer) From 215ff571c4450a38d4f816e5b2647e38bff96887 Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Mon, 27 Jul 2026 09:49:35 -0500 Subject: [PATCH 22/29] Pin the front door's identity checks Mutation testing found three guards that could be deleted outright with the whole suite still green. All three are the kind that only ever matter when someone is doing something they should not. sessionFor's ownership check was one of them. A session id is unguessable, but unguessable is not authenticated, and nothing else in the handler re-derives the caller from the session - so a peer holding a perfectly valid credential of its own could push and settle on another peer's session. No test had ever presented one principal's session id under another principal's credential. The tenant check was another: every principal in the harness lived in the same tenant, so refusing a peer authenticated into a tenant this relay does not serve was never exercised. The third was the claimed-local-identity comparison. The existing case differed in two fields at once, so the clauses masked each other and any one of them could be removed unnoticed - including the peerId clause, which is what stops a credential being reused for a sibling device of the same subject. Each field is now checked on its own. Each test was verified by neutralizing the guard it covers and confirming it fails. --- packages/local-rpc/test/RelayServer.test.ts | 75 ++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/packages/local-rpc/test/RelayServer.test.ts b/packages/local-rpc/test/RelayServer.test.ts index 4ee641c..14819b7 100644 --- a/packages/local-rpc/test/RelayServer.test.ts +++ b/packages/local-rpc/test/RelayServer.test.ts @@ -61,6 +61,13 @@ const recipient = PeerAuthentication.PeerPrincipal.make({ peerId: recipientPeerId }) +/** A real, authenticated peer that simply belongs to a tenant this relay does not serve. */ +const outsider = PeerAuthentication.PeerPrincipal.make({ + tenantId: "other", + subjectId: "outsider", + peerId: Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000004") +}) + const serverOptions: RelayServer.Options = { tenantId: "tenant", peerId: relayPeerId, @@ -210,7 +217,7 @@ const harness = (options?: { ) const credential = yield* Ref.make("sender") - const principals = new Map([["sender", sender], ["recipient", recipient]]) + const principals = new Map([["sender", sender], ["recipient", recipient], ["outsider", outsider]]) const authentication = yield* PeerAuthentication.PeerAuthentication.pipe( Effect.provide(PeerAuthentication.layerServer), Effect.provideService(PeerAuthenticator.PeerAuthenticator, { @@ -694,6 +701,72 @@ describe("RelayServer", () => { ) }))) + it.effect("refuses a session id presented under another principal's credential", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + const senderSession = yield* open(peer, sender, recipient) + yield* open(peer, recipient, sender) + + // A session id is unguessable, but unguessable is not the same as authenticated. The recipient + // holds a perfectly valid credential; it is simply not the credential this session was opened + // with, and nothing else in the handler re-derives the caller from the session. + const payload = yield* encodePayload(peer) + yield* Ref.set(peer.credential, "recipient") + const stolen = yield* peer.client.Push({ + sessionId: senderSession.opened.sessionId, + relayMessageId: relayId("000000000001"), + payload + }).pipe(Effect.flip) + assert.strictEqual(stolen._tag, "SessionUnavailable") + + const stolenAck = yield* peer.client.Acknowledge({ + sessionId: senderSession.opened.sessionId, + relayMessageId: relayId("000000000001"), + claimToken: PeerRpc.ClaimToken.make("clm_00000000-0000-4000-8000-000000000000"), + messageHash: "a".repeat(64) + }).pipe(Effect.flip) + assert.strictEqual(stolenAck._tag, "SessionUnavailable") + + const pending = yield* peer.store.pendingHeads( + yield* inboxKeyOf(peer, recipient), + { limit: 10 } + ) + assert.strictEqual(pending.length, 0, "a borrowed session admits nothing") + }))) + + it.effect("refuses a principal authenticated into another tenant", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + // Authentication succeeded — this is a real peer with a real credential. It simply belongs to + // a tenant this relay does not serve, which is an authorization decision, not a mismatch. + yield* Ref.set(peer.credential, "outsider") + const failure = yield* peer.client.Open(openRequest(outsider, recipient)).pipe( + Stream.runDrain, + Effect.flip + ) + assert.strictEqual(failure._tag, "AccessDenied") + }))) + + it.effect("checks every field of the claimed local identity independently", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + // One field at a time, so no clause can be masked by another failing alongside it. A + // credential reused for a sibling device of the same subject differs only in peerId. + for ( + const claimed of [ + { ...sender, subjectId: "recipient" }, + { ...sender, peerId: recipientPeerId }, + { ...sender, tenantId: "other" } + ] + ) { + const failure = yield* peer.client.Open({ + ...openRequest(sender, recipient), + expectedLocal: PeerAuthentication.PeerPrincipal.make(claimed) + }).pipe(Stream.runDrain, Effect.flip) + assert.strictEqual(failure._tag, "PeerMismatch", `expectedLocal ${JSON.stringify(claimed)}`) + } + }))) + it.effect("refuses a push when the unbounded decode risk is not acknowledged", () => Effect.scoped(Effect.gen(function*() { // Ordinary Send authorization is granted; only the separate risk acknowledgement is withheld. From 13f8841cc02e6b20c25695d548393125f15dc849 Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Mon, 27 Jul 2026 10:26:45 -0500 Subject: [PATCH 23/29] Pin the front door's authorization at every call site A test-reviewer ran 69 mutations against this suite and found that the handshake's two grants, the recheck that runs after authorization returns, the per-message re-authorization, the unbounded-decode risk port, the heartbeat, the entity-call timeout and the per-subject cap could all be deleted or weakened without turning it red. Authorization that nothing pins is authorization that will be refactored away. Each gap is now closed by a test that denies exactly one thing at exactly one call site, and every one was verified by applying the mutation to a copy of the repo and watching it fail. Reaching the post-authorization rechecks needed a new seam: the authorization port already refuses a grant that was expired when it answered, so the only way the relay's own recheck can matter is if time passes while authorization is in flight, which is what the `onAuthorize` knob simulates. The risk-port delivery test survived its own mutation on the first attempt because "assert the queue is empty" was racing the dispatcher rather than observing a withheld message. It now requires a younger, authorized message on another channel to arrive first, which is the only shape in this suite that can distinguish withheld from not-yet-dispatched. Four tests are deleted. Three were decided entirely by the authentication middleware or by store-level deduplication, and one asserted a rejection outcome that no store operation can read back. The protocol-version pin moves to PeerRpc.test.ts, where the schema it exercises lives. --- packages/local-rpc/test/PeerRpc.test.ts | 14 + packages/local-rpc/test/RelayServer.test.ts | 516 +++++++++++++++----- 2 files changed, 414 insertions(+), 116 deletions(-) diff --git a/packages/local-rpc/test/PeerRpc.test.ts b/packages/local-rpc/test/PeerRpc.test.ts index 9e87984..765ff0b 100644 --- a/packages/local-rpc/test/PeerRpc.test.ts +++ b/packages/local-rpc/test/PeerRpc.test.ts @@ -29,6 +29,20 @@ describe("PeerRpc", () => { assert.deepStrictEqual(Schema.decodeUnknownSync(PeerRpc.OpenEvent)(opened), opened) }) + it("pins the handshake protocol version so a foreign version fails to decode", () => { + // `Opened` carries a literal, not a number, so a client cannot be talked into continuing + // against a relay speaking a version it does not implement. + assert.throws(() => + Schema.decodeUnknownSync(PeerRpc.OpenEvent)({ + _tag: "Opened", + protocolVersion: PeerRpc.protocolVersion + 1, + sessionId, + remotePeerId, + authenticatedLocal: { tenantId: "tenant", subjectId: "subject", peerId: localPeerId } + }) + ) + }) + it("roundtrips every relay request and stored message field", () => { const open = PeerRpc.OpenRpc.payloadSchema.make({ protocolVersion: PeerRpc.protocolVersion, diff --git a/packages/local-rpc/test/RelayServer.test.ts b/packages/local-rpc/test/RelayServer.test.ts index 14819b7..2199b6c 100644 --- a/packages/local-rpc/test/RelayServer.test.ts +++ b/packages/local-rpc/test/RelayServer.test.ts @@ -6,6 +6,7 @@ import * as Canonical from "@lucas-barake/effect-local/Canonical" import * as Document from "@lucas-barake/effect-local/Document" import * as Identity from "@lucas-barake/effect-local/Identity" import * as Cause from "effect/Cause" +import * as Clock from "effect/Clock" import * as Context from "effect/Context" import * as Crypto from "effect/Crypto" import * as Effect from "effect/Effect" @@ -128,17 +129,45 @@ interface Knobs { * allocation-unbounded decode. It denies by default in production, so it is its own knob. */ allowRisk: (request: PeerRelayAuthorization.UnsafeUnboundedAutomerge3DecodeRequest) => boolean + /** + * How long a granted authorization stays good. + * + * Per request rather than a single value, because the handshake takes two grants and the thing + * worth pinning is that each is re-checked against the clock on its own. + */ + grantValidUntil: (request: PeerRelayAuthorization.Request) => number + /** + * Run inside the authorization port, before it answers. + * + * Authorizing is an I/O call to a policy backend, so time passes while it is in flight. This is + * the only way to reach the checks the relay makes *after* authorization returns, because the + * port itself refuses an already-expired grant and would otherwise answer first. + */ + onAuthorize: (request: PeerRelayAuthorization.Request) => Effect.Effect authority: { validUntil: number; invalidated: Effect.Effect } } const harness = (options?: { readonly limits?: PeerRelayLimits.Values readonly knobs?: Partial + /** + * Wraps the real store. + * + * Used only to make a durable operation hang, which is the one thing real SQLite will not do and + * the entity-call timeout exists for. + */ + readonly store?: ( + real: RelayInboxStore.RelayInboxStore["Service"] + ) => RelayInboxStore.RelayInboxStore["Service"] + /** Builds the exported `RelayServer.layer` instead of the handlers on their own. */ + readonly composed?: Partial }) => Effect.gen(function*() { const knobs: Knobs = { allow: options?.knobs?.allow ?? (() => true), allowRisk: options?.knobs?.allowRisk ?? (() => true), + grantValidUntil: options?.knobs?.grantValidUntil ?? (() => Number.MAX_SAFE_INTEGER), + onAuthorize: options?.knobs?.onAuthorize ?? (() => Effect.void), authority: options?.knobs?.authority ?? { validUntil: Number.MAX_SAFE_INTEGER, invalidated: Effect.never @@ -150,21 +179,23 @@ const harness = (options?: { const authorization = yield* PeerRelayAuthorization.PeerRelayAuthorization.pipe( Effect.provide(PeerRelayAuthorization.layer( (request) => - knobs.allow(request) - ? Effect.succeed({ - remote: { - tenantId: request.principal.tenantId, - subjectId: request.remote.subjectId, - peerId: request.remote.peerId - }, - documents: request.documents.map((requested) => ({ - document: Task, - documentId: requested.documentId - })), - validUntil: Number.MAX_SAFE_INTEGER, - invalidated: Effect.never - }) - : Effect.fail(new PeerRpcError.AccessDenied()), + knobs.onAuthorize(request).pipe(Effect.andThen( + knobs.allow(request) + ? Effect.succeed({ + remote: { + tenantId: request.principal.tenantId, + subjectId: request.remote.subjectId, + peerId: request.remote.peerId + }, + documents: request.documents.map((requested) => ({ + document: Task, + documentId: requested.documentId + })), + validUntil: knobs.grantValidUntil(request), + invalidated: Effect.never + }) + : Effect.fail(new PeerRpcError.AccessDenied()) + )), // Deliberately independent of `allow`. Tying the two together would make removing either // port from production invisible, since revoking one knob would revoke both gates at once. (request) => @@ -189,24 +220,49 @@ const harness = (options?: { // The real durable store behind the real entity, so the front door is exercised against the // custody it actually forwards to rather than a stand-in that always says yes. - const cluster = RelayInbox.layer(inboxOptions).pipe( - Layer.provideMerge(Sharding.layer), + const realStore = SqlRelayInboxStore.layer.pipe( + Layer.provide(SqliteClient.layer({ filename: ":memory:", disableWAL: true })), + Layer.provide(Layer.succeed(Crypto.Crypto)(crypto)), + Layer.orDie + ) + const decorate = options?.store + const storeLayer = decorate === undefined + ? realStore + : Layer.effect(RelayInboxStore.RelayInboxStore)( + Effect.gen(function*() { + return decorate(yield* RelayInboxStore.RelayInboxStore) + }) + ).pipe(Layer.provide(realStore)) + + const cluster = Sharding.layer.pipe( Layer.provide(Runners.layerNoop), Layer.provideMerge(MessageStorage.layerMemory), Layer.provide(RunnerStorage.layerMemory), Layer.provide(RunnerHealth.layerNoop), Layer.provide(TestShardingConfig), - Layer.provideMerge( - SqlRelayInboxStore.layer.pipe( - Layer.provide(SqliteClient.layer({ filename: ":memory:", disableWAL: true })), - Layer.provide(Layer.succeed(Crypto.Crypto)(crypto)), - Layer.orDie - ) - ) + Layer.provideMerge(storeLayer) ) + // Either the handlers alone, which most tests want because they can then drive the entity + // directly, or the exported layer a deployment actually builds. + const relay = options?.composed === undefined + ? RelayServer.layerHandlers(serverOptions).pipe( + Layer.provideMerge(RelayInbox.layer(inboxOptions)) + ) + : RelayServer.layer({ + ...serverOptions, + ...options.composed, + inbox: inboxOptions, + maintenance: { + intervalMillis: 60_000, + batchLimit: 100, + terminalRetentionMillis: inboxOptions.terminalRetentionMillis, + enabled: true + } + }) + const context = yield* Layer.build( - RelayServer.layerHandlers(serverOptions).pipe( + relay.pipe( Layer.provideMerge(cluster), Layer.provide(Layer.mergeAll( Layer.succeed(Crypto.Crypto)(crypto), @@ -255,6 +311,9 @@ const harness = (options?: { type Harness = Effect.Success> +/** The same harness, built from the exported layer a deployment wires up. */ +const composed = (overrides?: Partial) => harness({ composed: overrides ?? {} }) + /** Opens a session and drains its events into a queue, leaving the stream live. */ const open = ( peer: Harness, @@ -373,6 +432,9 @@ describe("RelayServer", () => { assert.strictEqual(stored._tag, "StoredMessage") if (stored._tag !== "StoredMessage") return assert.deepStrictEqual(stored.payload, payload) + // The identity the sender releases custody against, so delivering the right bytes under the + // wrong id would still lose the message. + assert.strictEqual(stored.relayMessageId, relayId("000000000001")) assert.strictEqual(stored.relayPeerId, relayPeerId) assert.strictEqual(stored.sender.peerId, senderPeerId) assert.deepStrictEqual(stored.recipient, recipient) @@ -427,34 +489,6 @@ describe("RelayServer", () => { assert.strictEqual(pending.length, 0, "an acknowledged message is terminal") }))) - it.effect("honours a rejection end to end", () => - Effect.scoped(Effect.gen(function*() { - const peer = yield* harness() - const senderSession = yield* open(peer, sender, recipient) - const recipientSession = yield* open(peer, recipient, sender) - - yield* push(peer, senderSession.opened.sessionId, yield* encodePayload(peer)) - const stored = yield* Queue.take(recipientSession.events) - if (stored._tag !== "StoredMessage") return assert.fail("expected a delivery") - - // A rejection is a durable decision too: the recipient looked at the message and refused it, - // so redelivering it would loop forever. - yield* Ref.set(peer.credential, "recipient") - yield* peer.client.Reject({ - sessionId: recipientSession.opened.sessionId, - relayMessageId: stored.relayMessageId, - claimToken: stored.claimToken, - messageHash: stored.messageHash, - reason: "ApplicationRejected" - }) - - const pending = yield* peer.store.pendingHeads( - yield* inboxKeyOf(peer, recipient), - { limit: 10 } - ) - assert.strictEqual(pending.length, 0, "a rejected message is terminal") - }))) - it.effect("replaces the incumbent session and refuses its session id afterwards", () => Effect.scoped(Effect.gen(function*() { const peer = yield* harness() @@ -478,17 +512,6 @@ describe("RelayServer", () => { assert.strictEqual(staleAck._tag, "SessionUnavailable") }))) - it.effect("fails Open with an authentication failure when the credential has expired", () => - Effect.scoped(Effect.gen(function*() { - const peer = yield* harness({ knobs: { authority: { validUntil: 0, invalidated: Effect.never } } }) - const failure = yield* peer.client.Open(openRequest(sender, recipient)).pipe( - Stream.runDrain, - Effect.flip - ) - // Expiry at the handshake is an authentication problem, not an authorization one. - assert.strictEqual(failure._tag, "AuthenticationFailure") - }))) - it.effect("refuses a push whose send grant has been withdrawn and admits nothing", () => Effect.scoped(Effect.gen(function*() { const peer = yield* harness() @@ -506,22 +529,6 @@ describe("RelayServer", () => { assert.strictEqual(pending.length, 0, "a denied push must not reach durable custody") }))) - it.effect("commits an admitted push exactly once even if the grant is withdrawn afterwards", () => - Effect.scoped(Effect.gen(function*() { - const peer = yield* harness() - const senderSession = yield* open(peer, sender, recipient) - - const payload = yield* encodePayload(peer) - yield* push(peer, senderSession.opened.sessionId, payload) - // Revoked after admission. The message is already the relay's responsibility; withdrawing the - // grant must not retroactively destroy custody the sender was told the relay had taken. - peer.knobs.allow = () => false - - const inbox = yield* inboxKeyOf(peer, recipient) - const usage = yield* peer.store.usage(inbox) - assert.strictEqual(usage.pendingCount, 1, "the admitted message survives the revocation") - }))) - it.effect("withholds a delivery the recipient may no longer receive without erroring the stream", () => Effect.scoped(Effect.gen(function*() { const peer = yield* harness() @@ -600,28 +607,31 @@ describe("RelayServer", () => { assert.strictEqual(yield* Queue.size(senderSession.events), 0) }))) - it.effect("refuses an unauthenticated push and admits nothing", () => + it.effect("caps sessions per subject rather than across all of them", () => Effect.scoped(Effect.gen(function*() { - const peer = yield* harness() - const senderSession = yield* open(peer, sender, recipient) + const peer = yield* harness({ + limits: PeerRelayLimits.Values.make({ + ...PeerRelayLimits.defaults, + maxSessionsPerSubject: 1 + }) + }) + yield* open(peer, sender, recipient) - const payload = yield* encodePayload(peer) - yield* Ref.set(peer.credential, "nobody") - const failure = yield* peer.client.Push({ - sessionId: senderSession.opened.sessionId, - relayMessageId: relayId("000000000001"), - payload - }).pipe(Effect.flip) - assert.strictEqual(failure._tag, "AuthenticationFailure") + // A different subject at its own cap of one. A counter that simply totalled live sessions + // would refuse this, which would let one busy subject lock every other client out. + yield* open(peer, recipient, sender) - const pending = yield* peer.store.pendingHeads( - yield* inboxKeyOf(peer, recipient), - { limit: 10 } + // Sessions are the front door's only unbounded resource and an authenticated client can open + // them in a loop, so the cap has to be refused rather than absorbed. + yield* Ref.set(peer.credential, "sender") + const failure = yield* peer.client.Open(openRequest(sender, recipient)).pipe( + Stream.runDrain, + Effect.flip ) - assert.strictEqual(pending.length, 0, "an unauthenticated push creates no inbox state") + assert.strictEqual(failure._tag, "RequestCapacityExceeded") }))) - it.effect("refuses to open more sessions than a subject is allowed", () => + it.effect("leaves no session behind when a handshake is refused", () => Effect.scoped(Effect.gen(function*() { const peer = yield* harness({ limits: PeerRelayLimits.Values.make({ @@ -629,15 +639,33 @@ describe("RelayServer", () => { maxSessionsPerSubject: 1 }) }) - yield* open(peer, sender, recipient) - // Sessions are the front door's only unbounded resource and an authenticated client can open - // them in a loop, so the cap has to be refused rather than absorbed. - const failure = yield* peer.client.Open(openRequest(sender, recipient)).pipe( + // Every way a handshake can be refused, against a cap of one. If any of them registered the + // session before deciding, the accepted handshake at the end would be refused for capacity — + // and in production a client could exhaust its own cap with requests that never succeeded. + const refusals: ReadonlyArray< + [string, Partial<(typeof PeerRpc.OpenRpc.payloadSchema)["Type"]>] + > = [ + ["version", { protocolVersion: 2 }], + ["relay", { expectedRelayPeerId: recipientPeerId }], + ["local", { expectedLocal: recipient }], + ["retention", { receiptRetentionMillis: 1_000 }] + ] + for (const [label, override] of refusals) { + const failure = yield* peer.client.Open({ ...openRequest(sender, recipient), ...override }) + .pipe(Stream.runDrain, Effect.flip) + assert.notStrictEqual(failure._tag, "RequestCapacityExceeded", label) + } + peer.knobs.allow = () => false + const denied = yield* peer.client.Open(openRequest(sender, recipient)).pipe( Stream.runDrain, Effect.flip ) - assert.strictEqual(failure._tag, "RequestCapacityExceeded") + assert.strictEqual(denied._tag, "AccessDenied") + + peer.knobs.allow = () => true + const accepted = yield* open(peer, sender, recipient) + assert.strictEqual(accepted.opened._tag, "Opened") }))) it.effect("keeps delivering other channels when one channel's head is withheld", () => @@ -871,21 +899,6 @@ describe("RelayServer", () => { assert.strictEqual(pending.length, 1, "an unsettled message survives the revocation") }))) - it.effect("pins the handshake protocol version so a foreign version fails to decode", () => - Effect.gen(function*() { - // `Opened` carries a literal, not a number, so a client cannot be talked into continuing - // against a relay speaking a version it does not implement. - assert.throws(() => - Schema.decodeUnknownSync(PeerRpc.OpenEvent)({ - _tag: "Opened", - protocolVersion: PeerRpc.protocolVersion + 1, - sessionId: "ses_00000000-0000-4000-8000-000000000001", - remotePeerId: recipientPeerId, - authenticatedLocal: sender - }) - ) - })) - it.effect("refuses a handshake whose retention window cannot cover its retry horizon", () => Effect.scoped(Effect.gen(function*() { const peer = yield* harness() @@ -897,4 +910,275 @@ describe("RelayServer", () => { }).pipe(Stream.runDrain, Effect.flip) assert.strictEqual(failure._tag, "InvalidRequest") }))) + + it.effect("requires both directions to be authorized at the handshake", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + + // A session is bidirectional the moment it opens: the client may push, and the relay attaches + // a delivery stream. Each direction therefore has to be granted on its own, and denying one + // must not be answered by the other having said yes. + for (const denied of ["Send", "Receive"] as const) { + peer.knobs.allow = (request) => request.direction !== denied + const failure = yield* peer.client.Open(openRequest(sender, recipient)).pipe( + Stream.runDrain, + Effect.flip + ) + assert.strictEqual(failure._tag, "AccessDenied", `${denied} denied`) + } + }))) + + it.effect("refuses a handshake whose grant lapsed while the other direction was authorized", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + const openedAt = yield* Clock.currentTimeMillis + + // Authorization is I/O, so the two grants are not obtained at the same instant and neither is + // still fresh by the time the session is minted. The authorization port itself only refuses a + // grant that was already expired when it answered, so the relay has to re-check both against + // the clock afterwards or it opens a session on authority that has since run out. + peer.knobs.grantValidUntil = (request) => + request.direction === "Send" ? openedAt + 1_000 : Number.MAX_SAFE_INTEGER + peer.knobs.onAuthorize = (request) => request.direction === "Receive" ? TestClock.adjust(2_000) : Effect.void + + const failure = yield* peer.client.Open(openRequest(sender, recipient)).pipe( + Stream.runDrain, + Effect.flip + ) + assert.strictEqual(failure._tag, "AccessDenied") + }))) + + it.effect("refuses a handshake whose credential lapsed while authorization was in flight", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + const openedAt = yield* Clock.currentTimeMillis + + // The middleware only proves the credential was live when the request arrived. Authorizing + // takes time, so the credential is re-checked before the session exists — and an expired + // credential is an authentication problem, not an authorization one, because the client has + // to renew rather than ask for different permissions. + peer.knobs.authority.validUntil = openedAt + 1_000 + peer.knobs.onAuthorize = (request) => request.direction === "Receive" ? TestClock.adjust(2_000) : Effect.void + + const failure = yield* peer.client.Open(openRequest(sender, recipient)).pipe( + Stream.runDrain, + Effect.flip + ) + assert.strictEqual(failure._tag, "AuthenticationFailure") + }))) + + it.effect("re-authorizes each delivery against the document that message carries", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + const documents = [ + { documentType: Task.name, documentId }, + { documentType: Task.name, documentId: otherDocumentId } + ] + const senderSession = yield* open(peer, sender, recipient, documents) + const recipientSession = yield* open(peer, recipient, sender, documents) + + // Denies Receive for any request mentioning the first document. The handshake asked for both + // at once, so a re-check that reused the handshake's document set would deny every delivery + // on this session — including the second document, which the recipient is plainly entitled + // to. Only a check keyed on the message's own document lets that one through. + peer.knobs.allow = (request) => + !( + request.direction === "Receive" && + request.documents.some((entry) => entry.documentId === documentId) + ) + + yield* push( + peer, + senderSession.opened.sessionId, + yield* encodePayload(peer, { epoch: "withheld" }), + "000000000001" + ) + // Distinct channels, so the withheld message cannot block the other by ordering alone. + yield* TestClock.adjust(10) + yield* push( + peer, + senderSession.opened.sessionId, + yield* encodePayload(peer, { documentId: otherDocumentId, epoch: "allowed" }), + "000000000002" + ) + + const delivered = yield* Queue.take(recipientSession.events) + assert.strictEqual(delivered._tag, "StoredMessage") + if (delivered._tag !== "StoredMessage") return + assert.strictEqual(delivered.relayMessageId, relayId("000000000002")) + assert.strictEqual(delivered.document.documentId, otherDocumentId) + + const pending = yield* peer.store.pendingHeads( + yield* inboxKeyOf(peer, recipient), + { limit: 10 } + ) + assert.isTrue( + pending.some((message) => message.relayMessageId === relayId("000000000001")), + "the withheld document stays durable" + ) + }))) + + it.effect("withholds a delivery whose unbounded decode risk is not acknowledged", () => + Effect.scoped(Effect.gen(function*() { + // Ordinary authorization is granted throughout; only the risk acknowledgement is withheld, + // and only for the receiving side of one document so the push itself still lands. Accepting a + // delivery commits this peer to the same allocation-unbounded decode the sender needed + // acknowledged, so the ordinary grant must not be able to stand in for it here either. + const peer = yield* harness({ + knobs: { + allowRisk: (request) => + !( + request.direction === "Receive" && + request.documents.some((entry) => entry.documentId === documentId) + ) + } + }) + const documents = [ + { documentType: Task.name, documentId }, + { documentType: Task.name, documentId: otherDocumentId } + ] + const senderSession = yield* open(peer, sender, recipient, documents) + const recipientSession = yield* open(peer, recipient, sender, documents) + + yield* push( + peer, + senderSession.opened.sessionId, + yield* encodePayload(peer, { epoch: "withheld" }), + "000000000001" + ) + // Older, and on its own channel. Requiring the second message to arrive first is what makes + // the withholding observable: an absence assertion here would pass on a delivery that simply + // had not been dispatched yet. + yield* TestClock.adjust(10) + yield* push( + peer, + senderSession.opened.sessionId, + yield* encodePayload(peer, { documentId: otherDocumentId, epoch: "allowed" }), + "000000000002" + ) + + const delivered = yield* Queue.take(recipientSession.events) + assert.strictEqual(delivered._tag, "StoredMessage") + if (delivered._tag !== "StoredMessage") return + assert.strictEqual(delivered.relayMessageId, relayId("000000000002")) + + const pending = yield* peer.store.pendingHeads( + yield* inboxKeyOf(peer, recipient), + { limit: 10 } + ) + assert.isTrue( + pending.some((message) => message.relayMessageId === relayId("000000000001")), + "the unacknowledged document stays durable" + ) + + // Withheld rather than dropped: acknowledging the risk and reconnecting produces it. + peer.knobs.allowRisk = () => true + const restored = yield* open(peer, recipient, sender, documents) + const released = yield* Queue.take(restored.events) + assert.strictEqual(released._tag, "StoredMessage") + if (released._tag !== "StoredMessage") return + assert.strictEqual(released.relayMessageId, relayId("000000000001")) + }))) + + it.effect("refuses a settlement whose unbounded decode risk is no longer acknowledged", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + const senderSession = yield* open(peer, sender, recipient) + const recipientSession = yield* open(peer, recipient, sender) + + yield* push(peer, senderSession.opened.sessionId, yield* encodePayload(peer)) + const stored = yield* Queue.take(recipientSession.events) + if (stored._tag !== "StoredMessage") return assert.fail("expected a delivery") + + // Settling is the durable decision that the recipient has taken the document, so it carries + // the same risk the delivery did. Withdrawing only the risk acknowledgement, with ordinary + // Receive authorization untouched, has to be enough to refuse it. + peer.knobs.allowRisk = () => false + yield* Ref.set(peer.credential, "recipient") + const denied = yield* peer.client.Acknowledge({ + sessionId: recipientSession.opened.sessionId, + relayMessageId: stored.relayMessageId, + claimToken: stored.claimToken, + messageHash: stored.messageHash + }).pipe(Effect.flip) + assert.strictEqual(denied._tag, "AccessDenied") + + const pending = yield* peer.store.pendingHeads( + yield* inboxKeyOf(peer, recipient), + { limit: 10 } + ) + assert.strictEqual(pending.length, 1, "the terminal transition never ran") + }))) + + it.effect("keeps a session past its liveness deadline for as long as the socket lives", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + const senderSession = yield* open(peer, sender, recipient) + const recipientSession = yield* open(peer, recipient, sender) + + // A cluster cannot rely on disconnects announcing themselves, so the entity reaps any session + // that stops proving it is alive. Nothing on the wire refreshes it — the client does not know + // the session exists — so the relay drives the heartbeat itself, and without it every session + // would go dark one deadline after opening no matter how healthy the connection is. + yield* TestClock.adjust(inboxOptions.sessionDeadlineMillis * 4) + + yield* push(peer, senderSession.opened.sessionId, yield* encodePayload(peer)) + const delivered = yield* Queue.take(recipientSession.events) + assert.strictEqual(delivered._tag, "StoredMessage") + }))) + + it.effect("reports an entity call that outlives its timeout as unavailability", () => + Effect.scoped(Effect.gen(function*() { + const admitting = yield* Latch.make(false) + const peer = yield* harness({ + // `Sharding` retries `EntityNotAssignedToRunner` and `RunnerUnavailable` forever, so an + // entity call during a rebalance never returns on its own. Held here at the durable write + // because that is the one place real SQLite will not stall. + store: (real) => ({ ...real, admit: (request) => admitting.await.pipe(Effect.andThen(real.admit(request))) }) + }) + const senderSession = yield* open(peer, sender, recipient) + + const pushing = yield* push(peer, senderSession.opened.sessionId, yield* encodePayload(peer)) + .pipe(Effect.flip, Effect.forkScoped) + yield* TestClock.adjust(serverOptions.entityCallTimeoutMillis) + + const failure = yield* Fiber.join(pushing) + // Retryable: the write may or may not have landed, and the sender still holds its outbox copy. + assert.strictEqual(failure._tag, "ServerUnavailable") + yield* admitting.open + }))) + + it.effect("serves a session through the composed relay layer", () => + Effect.scoped(Effect.gen(function*() { + // The exported layer, not the handlers alone: it is what a deployment actually builds, and it + // is the only thing that composes the retention singleton and checks that the heartbeat and + // the session deadline agree. + const peer = yield* composed() + const senderSession = yield* open(peer, sender, recipient) + const recipientSession = yield* open(peer, recipient, sender) + + yield* push(peer, senderSession.opened.sessionId, yield* encodePayload(peer)) + const delivered = yield* Queue.take(recipientSession.events) + assert.strictEqual(delivered._tag, "StoredMessage") + }))) + + it.effect("refuses to build a relay whose heartbeat its session deadline cannot survive", () => + Effect.gen(function*() { + // Two independently configured values that have to agree. If they do not, every session is + // reaped on a fixed cycle and no client can hold a delivery stream — a total outage visible + // only under one particular configuration, so it is refused where it is written down. + for ( + const broken of [ + { heartbeatIntervalMillis: inboxOptions.sessionDeadlineMillis }, + { entityCallTimeoutMillis: 0 } + ] + ) { + const exit = yield* Effect.scoped(composed(broken)).pipe(Effect.exit) + assert.isTrue(exit._tag === "Failure", JSON.stringify(broken)) + if (exit._tag !== "Failure") continue + // A defect, not a typed failure: nothing downstream can recover from a relay that was + // configured to reap its own sessions. + assert.isTrue(Cause.hasDies(exit.cause), JSON.stringify(broken)) + } + })) }) From b2241c64ddc3c0a225e0b407b7cb682df0188d7c Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Mon, 27 Jul 2026 10:41:31 -0500 Subject: [PATCH 24/29] Stop a dead dispatcher from silently swallowing an inbox The per-channel dispatcher runs on a forked fiber that nothing observes, so a defect in it made the fiber quietly disappear. Everything else went on looking healthy: the session stayed installed, the front door kept heartbeating it, Deliver kept reporting success to senders, and the device received nothing at all until it happened to reconnect. A forked fiber's defect reaches neither RpcServer's defect path nor the entity's own defect restart, so nothing in the framework noticed either. The dispatcher now hands its cause to the one thing the recipient is watching. Failing the queue with the raw defect does not work - Subscribe declares a closed error union, so a Die cause has nowhere to go across the entity RPC boundary and the recipient still waited forever. The defect goes to the log at fatal with its whole cause, and the recipient gets the one thing it can act on: a retryable ServerUnavailable, after which the front door's session finalizer ends the session and the client opens a fresh one. Three entity calls that ran without a deadline are now bounded. The session finalizer's EndSession is the serious one: finalizers run uninterruptibly and Sharding retries an unassigned entity forever, so during a rebalance that call never returned, the request fiber never exited, and the node hung on shutdown over a connection whose client was already gone. The entity finalizer's comment claimed it runs before the framework's termination handshake. It does not, and it cannot: the build effect only ever sees the ResourceRef's inner scope, while the handshake is registered on the outer entity scope afterwards, and scope finalizers are last-registered-first. The comment now states the real ordering and what it costs on a rebalance. --- packages/local-rpc/src/RelayInbox.ts | 49 ++++++++++++++++++---- packages/local-rpc/src/RelayServer.ts | 13 ++++-- packages/local-rpc/test/RelayInbox.test.ts | 33 +++++++++++++++ 3 files changed, 85 insertions(+), 10 deletions(-) diff --git a/packages/local-rpc/src/RelayInbox.ts b/packages/local-rpc/src/RelayInbox.ts index 6f055ec..1a4b11d 100644 --- a/packages/local-rpc/src/RelayInbox.ts +++ b/packages/local-rpc/src/RelayInbox.ts @@ -1,5 +1,5 @@ import * as Identity from "@lucas-barake/effect-local/Identity" -import type * as Cause from "effect/Cause" +import * as Cause from "effect/Cause" import * as Clock from "effect/Clock" import * as Deferred from "effect/Deferred" import * as Effect from "effect/Effect" @@ -204,7 +204,7 @@ interface Settlement { interface Session { readonly sessionId: Identity.SessionId - readonly outbound: Queue.Queue + readonly outbound: Queue.Queue readonly settlements: Map /** Channels with a delivery in flight. One message per channel at a time preserves order. */ readonly busyChannels: Set @@ -465,9 +465,18 @@ export const layer = (options: Options) => Effect.forkScoped ) - // Runs before the framework's own termination handshake, which otherwise waits the full - // `entityTerminationTimeout` for a forked streaming handler that never returns on its own, - // rejecting deliveries and settlements for the whole of that window on every rebalance. + // Closes whatever session is live when this entity is torn down. + // + // It does NOT run before the framework's termination handshake, and the ordering cannot be + // changed from here. This build effect's `Scope` is the `ResourceRef`'s inner scope + // (`entityManager.ts:162`), while the handshake that writes `Eof` and then waits on + // `endLatch` is registered on the outer entity scope afterwards (`entityManager.ts:372-381`). + // Scope finalizers run last-registered-first, so the handshake goes first and waits the full + // `entityTerminationTimeout` for the forked `Subscribe`, which never returns on its own — + // rejecting `Deliver` and `Settle` for that whole window. The behaviour is handed no + // reference to the outer scope, so this is a property of the deployment's + // `entityTerminationTimeout`, not something the entity can fix: size it knowing a rebalance + // costs each moving device up to that long before its next session can be served. yield* Effect.addFinalizer(() => Effect.orDie(closeCurrentSession)) const currentSession = (sessionId: Identity.SessionId) => @@ -569,7 +578,10 @@ export const layer = (options: Options) => const scope = yield* Scope.make() const install = Effect.gen(function*() { - const outbound = yield* Queue.bounded(0) + const outbound = yield* Queue.bounded< + PeerRpc.StoredMessage, + PeerRpcError.ServerUnavailable | Cause.Done + >(0) const now = yield* Clock.currentTimeMillis const deadlineAt = yield* Ref.make(now + options.sessionDeadlineMillis) const session: Session = { @@ -582,7 +594,30 @@ export const layer = (options: Options) => scope, deadlineAt } - yield* Effect.forkIn(dispatch(session), scope) + // Observed rather than left to run alone. `dispatch` never returns, so any + // exit but interruption is a defect in it — and a forked fiber's defect + // reaches nothing: not `RpcServer`'s defect path, not the entity's own defect + // restart. The session would stay installed and keep looking healthy, with + // heartbeats extending its deadline and `Deliver` still reporting success to + // senders, while the device received nothing at all until it happened to + // reconnect. Handing the cause to the outbound queue is what makes the + // recipient notice: its stream fails, the front door's session finalizer then + // ends the session from a fiber that does not own this scope, and the client + // opens a fresh one. The cause is preserved rather than translated, because a + // dead dispatcher is a bug in the relay and not a decision about this peer. + yield* dispatch(session).pipe( + Effect.onExit((exit) => + Exit.isFailure(exit) && Cause.hasDies(exit.cause) + ? Effect.logFatal("Relay inbox dispatcher died", exit.cause).pipe( + Effect.annotateLogs({ inboxKey, sessionId: payload.sessionId }), + Effect.andThen( + Queue.fail(outbound, new PeerRpcError.ServerUnavailable()) + ) + ) + : Effect.void + ), + Effect.forkIn(scope) + ) yield* Ref.set(sessionRef, Option.some(session)) return outbound }) diff --git a/packages/local-rpc/src/RelayServer.ts b/packages/local-rpc/src/RelayServer.ts index 1b25311..de2aba7 100644 --- a/packages/local-rpc/src/RelayServer.ts +++ b/packages/local-rpc/src/RelayServer.ts @@ -306,7 +306,12 @@ export const layerHandlers = (options: Options) => }).pipe( // Best effort: the entity's own liveness deadline is what actually bounds an // abandoned session, because this never runs when the node itself fails. - Effect.andThen(Effect.ignore(inboxClient(inboxKeySelf).EndSession({ sessionId }))) + // Bounded, and deliberately so. Finalizers run uninterruptibly, and `Sharding` + // retries `EntityNotAssignedToRunner` and `RunnerUnavailable` forever, so during a + // rebalance an unbounded call here never returns: the request fiber never exits, + // `RpcServer`'s shutdown latch never opens, and the whole node hangs on shutdown + // over a session that is already gone. + Effect.andThen(Effect.ignore(bounded(inboxClient(inboxKeySelf).EndSession({ sessionId })))) ) ) @@ -331,12 +336,14 @@ export const layerHandlers = (options: Options) => Effect.annotateLogs({ sessionId }) ) ), - Effect.andThen(Effect.ignore(inboxClient(inboxKeySelf).EndSession({ sessionId }))), + Effect.andThen(Effect.ignore(bounded(inboxClient(inboxKeySelf).EndSession({ sessionId })))), Effect.forkScoped ) // Proves this node and its socket are still alive. Stopping is the signal. - yield* Effect.ignore(inboxClient(inboxKeySelf).Heartbeat({ sessionId })).pipe( + // Bounded per beat rather than per session: a heartbeat that hung would stall the + // repeat itself, and the session it exists to keep alive would then lapse. + yield* Effect.ignore(bounded(inboxClient(inboxKeySelf).Heartbeat({ sessionId }))).pipe( Effect.repeat(Schedule.spaced(options.heartbeatIntervalMillis)), Effect.forkScoped ) diff --git a/packages/local-rpc/test/RelayInbox.test.ts b/packages/local-rpc/test/RelayInbox.test.ts index e718b4a..4f7ffce 100644 --- a/packages/local-rpc/test/RelayInbox.test.ts +++ b/packages/local-rpc/test/RelayInbox.test.ts @@ -3,9 +3,12 @@ import { SqliteClient } from "@effect/sql-sqlite-node" import { assert, describe, it } from "@effect/vitest" import * as Identity from "@lucas-barake/effect-local/Identity" import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" +import * as Cause from "effect/Cause" import * as Effect from "effect/Effect" +import * as Exit from "effect/Exit" import * as Fiber from "effect/Fiber" import * as Layer from "effect/Layer" +import * as Option from "effect/Option" import * as Stream from "effect/Stream" import { TestClock } from "effect/testing" import * as MessageStorage from "effect/unstable/cluster/MessageStorage" @@ -409,6 +412,36 @@ describe("RelayInbox", () => { ) }).pipe(Effect.provide(layer))) + it.effect("ends the delivery stream when its dispatcher dies", () => + Effect.gen(function*() { + const client = yield* inbox + yield* client.Deliver(deliver({ id: "000000000001", sequence: 0 })) + + // The dispatcher runs on a forked fiber nobody observes, so a defect in it simply makes the + // fiber disappear. Left unnoticed the session stays installed and looks perfectly healthy: + // the front door keeps heartbeating it, `Deliver` keeps reporting success to senders, and the + // device receives nothing at all until it happens to reconnect. The framework does not catch + // it either — a forked fiber's defect never reaches the entity's own defect restart — so the + // dispatcher has to hand its cause to the one thing the recipient is watching. + const exit = yield* client.Subscribe({ sessionId: sessionId("000000000001") }).pipe( + Stream.runDrain, + Effect.exit + ) + assert.isTrue(Exit.isFailure(exit), "the recipient is told, rather than waiting forever") + if (Exit.isSuccess(exit)) return + // Retryable, and never a clean end of stream. `Subscribe` declares a closed error union, so a + // raw defect on the queue reaches nobody; the defect itself goes to the log at fatal with its + // whole cause, and the recipient is told the one thing it can act on — come back. + assert.deepStrictEqual( + Cause.findErrorOption(exit.cause).pipe(Option.map((error) => error._tag)), + Option.some("ServerUnavailable") + ) + }).pipe(Effect.provide(relay({ + store: storeFailing(() => ({ + pendingHeads: () => Effect.die(new Error("dispatcher fault")) + })) + })))) + it.effect("releases a session whose liveness deadline lapses", () => Effect.gen(function*() { const client = yield* inbox From 43d78c7199451aee7f6a74abb2fc92a595de6776 Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Mon, 27 Jul 2026 10:48:33 -0500 Subject: [PATCH 25/29] Delete the relay configuration that no longer reaches any code PeerRelayLimits carried the tuning surface of a single-process relay that owned its own socket framing, claim leases, SQL lanes and cross-scope quota counters. None of that machinery survived the move to cluster entities, so 87 of its 97 values had no reader at all - 79 of them referenced only by cross-field validations that validated each other. Dead configuration is worse than absent. An operator who sets maxActiveMessagesPerTenant believes their deployment is quota bounded; nothing reads it, so it is not. The ten values that are actually read stay, along with the two relations that still mean something: the receipt retention nesting rule, which the front door's Open validation mirrors and therefore has to agree with, and the authentication burst floor. The test now pins the exact surviving key set rather than a count, so a value cannot come back without a reader. Two families are gone with no replacement, and both are accepted regressions rather than oversights. Per-scope quotas span many inboxes, so no entity is their sole writer and enforcing them would reintroduce the cross-process arbitration this design removes; admission is now bounded per inbox, inside the transaction that performs the write. Connection and frame accounting belonged to the length-prefixed framing that standard RPC over a socket replaces; a single payload is still bounded by PeerRpc's schema check, but concurrent connections are now the deployment's socket server to bound. The observability module loses nineteen exports for the same reason. Beyond the session and queue metrics, the whole setRelay family and the six gauges behind it had no production writer either - only their own test drove them, which is a test keeping dead code alive. A gauge that is always zero reads as "no backlog" rather than as "nobody is measuring". internal/peerRpcProtocol.ts stays. Its two constants look inlinable, but PeerRelayLimits reading them from PeerRpc would close a cycle through PeerAuthentication. The leaf module is what breaks it. --- README.md | 49 +-- docs/durability.md | 2 +- docs/store-and-forward.md | 2 +- packages/local-rpc/README.md | 2 +- packages/local-rpc/src/PeerRelayLimits.ts | 395 +++--------------- .../src/internal/peerRpcObservability.ts | 100 +---- .../local-rpc/test/PeerRelayLimits.test.ts | 212 ++-------- .../test/PeerRpcObservability.test.ts | 34 -- 8 files changed, 139 insertions(+), 657 deletions(-) diff --git a/README.md b/README.md index 90afa19..c91a6c4 100644 --- a/README.md +++ b/README.md @@ -1231,27 +1231,27 @@ validate their bounds in the Effect error channel with the tagged `InvalidOption Consistency guarantees: -| Boundary | Guarantee | -| ------------------------ | ---------------------------------------------------------------------------------------------------------------------------- | -| One document command | Serialized through its Cluster entity and committed with canonical state, projections, receipt, sequence, and stored reply | -| Command retry | Within one replica incarnation, the same command ID and canonical request returns the durable result | -| Different command input | Within one replica incarnation, reusing a command ID for different input fails | -| Query | Reads local projection state under the replica operation gate | -| Multi document invariant | Not transactional. Model one aggregate document or an explicit Workflow | -| Peer convergence | Replicas converge after receiving the same valid Automerge change set | -| Cross lineage sync | Refused, never merged. A rewritten document stops synchronizing with every peer that holds the superseded lineage | -| Restore | Exclusive, fenced, staged, schema checked, and projection rebuilding | -| Atom invalidation | Reactive cache refresh, not a durability acknowledgement | -| Presence | Expiring best effort state with no durability guarantee | -| Automerge observation | The peer's sync state reports all current local changes observed. It does not prove remote storage durability | -| RPC `Open` handshake | One authenticated, authorized, bounded durable session exists for exactly the selected whole documents | -| RPC `Push` success | The backend relay store committed the complete envelope and quota reservation before replying | -| Relay recipient ack | The recipient SQL sync workflow and sender scoped receipt committed before the fenced acknowledgement was attempted | -| Relay delivery | At least once within configured retry, expiry, retention, authorization, and capacity boundaries | -| Relay poison bound | Delivery attempts are durable. Reaching `maximumDeliveryAttempts`, default `16`, dead letters the row and erases its payload | -| Effect RPC stream ack | The response chunk was acknowledged by the RPC protocol. It is neither a byte credit nor an application receipt | -| WebSocket frame order | Frames are ordered within one live RFC 6455 connection. Reconnect, replay, authorization, and persistence are separate | -| `durableConfirmation` | Always `false` in the shipped peer transports | +| Boundary | Guarantee | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------ | +| One document command | Serialized through its Cluster entity and committed with canonical state, projections, receipt, sequence, and stored reply | +| Command retry | Within one replica incarnation, the same command ID and canonical request returns the durable result | +| Different command input | Within one replica incarnation, reusing a command ID for different input fails | +| Query | Reads local projection state under the replica operation gate | +| Multi document invariant | Not transactional. Model one aggregate document or an explicit Workflow | +| Peer convergence | Replicas converge after receiving the same valid Automerge change set | +| Cross lineage sync | Refused, never merged. A rewritten document stops synchronizing with every peer that holds the superseded lineage | +| Restore | Exclusive, fenced, staged, schema checked, and projection rebuilding | +| Atom invalidation | Reactive cache refresh, not a durability acknowledgement | +| Presence | Expiring best effort state with no durability guarantee | +| Automerge observation | The peer's sync state reports all current local changes observed. It does not prove remote storage durability | +| RPC `Open` handshake | One authenticated, authorized, bounded durable session exists for exactly the selected whole documents | +| RPC `Push` success | The backend relay store committed the complete envelope and quota reservation before replying | +| Relay recipient ack | The recipient SQL sync workflow and sender scoped receipt committed before the fenced acknowledgement was attempted | +| Relay delivery | At least once within configured retry, expiry, retention, authorization, and capacity boundaries | +| Relay poison bound | Delivery attempts are durable. Reaching `RelayInbox.Options.maxDeliveries` dead letters the row, which stops it blocking its channel | +| Effect RPC stream ack | The response chunk was acknowledged by the RPC protocol. It is neither a byte credit nor an application receipt | +| WebSocket frame order | Frames are ordered within one live RFC 6455 connection. Reconnect, replay, authorization, and persistence are separate | +| `durableConfirmation` | Always `false` in the shipped peer transports | ### Retry and ambiguity @@ -1272,9 +1272,10 @@ Consistency guarantees: - Retry relay admission with the same stable `RelayMessageId` and exact outer envelope. A lost custody response, recipient acknowledgement, reconnect, or expired claim can duplicate delivery. Sender scoped recipient receipts suppress repeated application during the negotiated retention window. -- Failed, interrupted, disconnected, and expired relay claims advance the durable delivery attempt count. Reaching - `PeerRelayLimits.maximumDeliveryAttempts` dead letters the message and erases its payload. Restart does not reset - the count. +- A relay delivery advances the durable attempt count once the message has provably reached the transport, so a + message prepared for a channel and then abandoned on a flaky connection costs nothing. Reaching + `RelayInbox.Options.maxDeliveries` dead letters the message so it stops blocking its channel. Restart does not + reset the count. - Durable commit events are retried until publication into the bounded process local stream. Subscriber delivery is best effort because the stream uses sliding capacity. A later sequence gap, `FullRefreshRequired`, or refresh generation change instructs the subscriber to reread canonical state. If no later event arrives, no notification diff --git a/docs/durability.md b/docs/durability.md index 7dc68c6..10fc118 100644 --- a/docs/durability.md +++ b/docs/durability.md @@ -161,7 +161,7 @@ idempotent during the negotiated window. Relay FIFO ordering is scoped to one exact directed peer channel. One claimed head blocks later rows in that channel. Unrelated channels do not share that ordering fence. -Recipient delivery attempts are durable and bounded by `PeerRelayLimits.maximumDeliveryAttempts`, which defaults to +Recipient delivery attempts are durable and bounded by `RelayInbox.Options.maxDeliveries`, which is set to `16`. Failed, interrupted, disconnected, and expired claims advance the count. Reaching the cap changes the message to `DeadLettered` and erases the payload. Restart cannot reset this poison bound. diff --git a/docs/store-and-forward.md b/docs/store-and-forward.md index d08b84c..0f05b53 100644 --- a/docs/store-and-forward.md +++ b/docs/store-and-forward.md @@ -142,7 +142,7 @@ Claims are finite fences. A stale worker can duplicate delivery after its claim acknowledge or reject the new claim. Retry uses bounded exponential delay with jitter. The sender outbox keeps pending admissions until custody succeeds or their configured retry horizon expires. -`PeerRelayLimits.maximumDeliveryAttempts` caps recipient delivery. The default is `16`. A failed, interrupted, +`RelayInbox.Options.maxDeliveries` caps recipient delivery. It is required per deployment. A failed, interrupted, disconnected, or expired claim durably advances the attempt count. When that count reaches the configured cap, the relay moves the message to `DeadLettered`, erases its payload, and retains only bounded terminal deduplication evidence. Process restart does not reset the count. Message expiry can terminalize the row before the attempt cap. diff --git a/packages/local-rpc/README.md b/packages/local-rpc/README.md index ec3033d..2f1124e 100644 --- a/packages/local-rpc/README.md +++ b/packages/local-rpc/README.md @@ -10,7 +10,7 @@ and recipient receipts remain SQLite. Automerge `3.3.2` has no allocation bounded semantic decode API, so safe relay composition explicitly passes `denyUnsafeUnboundedAutomerge3Decode`. A separate exact unsafe resource trust grant is required to proceed. Ordinary authentication and document authorization do not provide that grant. -Delivery attempts are durable and bounded. `PeerRelayLimits.maximumDeliveryAttempts` defaults to `16`, then dead +Delivery attempts are durable and bounded. `RelayInbox.Options.maxDeliveries` is required per deployment, then dead letters the message and erases its payload. Policy revocation and each bounded relay operation contend on a local gate. Revocation admitted first prevents SQL mutation or payload emission. An operation admitted first may finish. Revocation drains it and blocks later work. diff --git a/packages/local-rpc/src/PeerRelayLimits.ts b/packages/local-rpc/src/PeerRelayLimits.ts index 0cd8a53..5868d50 100644 --- a/packages/local-rpc/src/PeerRelayLimits.ts +++ b/packages/local-rpc/src/PeerRelayLimits.ts @@ -5,235 +5,80 @@ import * as Schema from "effect/Schema" import type * as SchemaIssue from "effect/SchemaIssue" import * as PeerRpcProtocol from "./internal/peerRpcProtocol.js" +/** + * The relay's deployment-tunable limits. + * + * Every value here is read by something. That is a deliberate property rather than an accident: + * this used to carry the tuning surface of a single-process relay that owned its own socket + * framing, claim leases, SQL lanes and cross-scope quota counters, and none of that machinery + * survives the move to cluster entities. Configuration that no longer reaches any code is worse + * than absent — an operator who sets a quota that is silently never enforced believes their + * deployment is bounded when it is not — so the orphaned values were removed rather than retained + * for a future that may never use them. + * + * Two families are gone with no replacement here, and both are accepted regressions rather than + * oversights. Per-scope quotas (per sender peer, recipient peer, recipient subject, tenant, shard) + * span many inboxes, so no entity is their sole writer and enforcing them would reintroduce exactly + * the cross-process arbitration this design removes; admission is now bounded **per inbox** by + * `RelayInbox.Options`, inside the same transaction as the write. Connection and frame accounting + * belonged to the bespoke length-prefixed framing, which standard Effect RPC over a socket replaces; + * a single relayed payload is still bounded, by `PeerRpc`'s schema check against + * `maximumRelayPayloadBytes`, but concurrent connections and in-flight frame bytes are now the + * deployment's socket server to bound. + */ + const PositiveInt = Schema.Int.check(Schema.isGreaterThan(0)) const PositiveNumber = Schema.Number.check(Schema.isFinite(), Schema.isGreaterThan(0)) const NegotiatedDuration = PositiveInt.check( Schema.isLessThanOrEqualTo(PeerRpcProtocol.maximumNegotiatedDurationMillis) ) -const Percentage = PositiveNumber.check(Schema.isLessThanOrEqualTo(100)) export const Values = Schema.Struct({ - maxActiveMessagesPerSenderPeer: PositiveInt, - maxActiveBytesPerSenderPeer: PositiveInt, - maxActiveMessagesPerRecipientPeer: PositiveInt, - maxActiveBytesPerRecipientPeer: PositiveInt, - maxActiveMessagesPerRecipientSubject: PositiveInt, - maxActiveBytesPerRecipientSubject: PositiveInt, - maxActiveMessagesPerTenant: PositiveInt, - maxActiveBytesPerTenant: PositiveInt, - maxActiveMessagesPerShard: PositiveInt, - maxActiveBytesPerShard: PositiveInt, - - maxRetainedRowsPerSenderPeer: PositiveInt, - maxRetainedBytesPerSenderPeer: PositiveInt, - maxRetainedRowsPerRecipientPeer: PositiveInt, - maxRetainedBytesPerRecipientPeer: PositiveInt, - maxRetainedRowsPerRecipientSubject: PositiveInt, - maxRetainedBytesPerRecipientSubject: PositiveInt, - maxRetainedRowsPerTenant: PositiveInt, - maxRetainedBytesPerTenant: PositiveInt, - maxRetainedRowsPerShard: PositiveInt, - maxRetainedBytesPerShard: PositiveInt, - + /** How long an undelivered message survives in an inbox before it is expired. */ messageTtlMillis: NegotiatedDuration, + /** The longest replay window a client may negotiate at `Open`. */ maximumSenderRetryHorizonMillis: NegotiatedDuration, + /** The slack a receipt must outlive its message and that message's replay window by. */ minimumTerminalRetentionMillis: NegotiatedDuration, + /** The longest receipt retention a client may negotiate at `Open`. */ maximumReceiptRetentionMillis: NegotiatedDuration, - claimLeaseMillis: PositiveInt, - maximumRecipientProcessingMillis: PositiveInt, - acknowledgementTimeoutMillis: PositiveInt, - claimSafetyMarginMillis: PositiveInt, - retryBaseDelayMillis: PositiveInt, - retryMaximumDelayMillis: PositiveInt, - maximumDeliveryAttempts: PositiveInt, - sqliteLockRetryBaseDelayMillis: PositiveInt, - sqliteLockRetryMaximumDelayMillis: PositiveInt, - sqliteLockRetryMaxAttempts: PositiveInt, - maxRelayConnections: PositiveInt, - maximumRawChunkBytes: PositiveInt, - maximumDeclaredFrameBytes: PositiveInt, - maximumIncompleteFrameBytes: PositiveInt, - incompleteFrameTimeoutMillis: PositiveInt, - maximumSharedPayloadBytes: PositiveInt, - maximumByteReservationWaiters: PositiveInt, + /** Credentials verified concurrently. Beyond this a caller is told to come back. */ maxInFlightAuthentication: PositiveInt, + /** Sustained authentication attempts allowed per client. */ authenticationRatePerSecond: PositiveNumber, + /** Authentication attempts a client may spend at once before the sustained rate applies. */ authenticationBurst: PositiveInt, - - maxSessionsPerSubject: PositiveInt, - maxInFlightOpen: PositiveInt, - maxInFlightOpenPerSubject: PositiveInt, - openRatePerSecond: PositiveNumber, - openBurst: PositiveInt, - maxInFlightPush: PositiveInt, - maxInFlightPushPerSubject: PositiveInt, - admissionRatePerSecond: PositiveNumber, - admissionBurst: PositiveInt, + /** Rate-limiter state retained for idle clients, so the table cannot grow without bound. */ maxRetainedRateLimitedConnections: PositiveInt, - maxRetainedRateLimitedSubjects: PositiveInt, + /** How long a client's rate-limiter state is kept after its last request. */ rateLimitIdleRetentionMillis: PositiveInt, - terminalResponseQueueCapacity: PositiveInt, - maxInFlightTerminalResponses: PositiveInt, - maxInFlightTerminalResponsesPerSubject: PositiveInt, - terminalResponseRatePerSecond: PositiveNumber, - terminalResponseBurst: PositiveInt, - maxRetainedTerminalResponseSubjects: PositiveInt, - terminalResponseSubjectIdleRetentionMillis: PositiveInt, - - newWorkQueueCapacity: PositiveInt, - retryQueueCapacity: PositiveInt, - relayWorkerConcurrency: PositiveInt, - newWorkWeight: PositiveInt, - retryWorkWeight: PositiveInt, - maxActiveChannels: PositiveInt, - compensationIntervalMillis: PositiveInt, - compensationBatchSize: PositiveInt, - - sqlAdmissionQueueCapacity: PositiveInt, - sqlTerminalQueueCapacity: PositiveInt, - sqlDeliveryQueueCapacity: PositiveInt, - sqlMaintenanceQueueCapacity: PositiveInt, - maxInFlightSqlTransactions: PositiveInt, - maxInFlightSqlAdmission: PositiveInt, - maxInFlightSqlTerminal: PositiveInt, - maxInFlightSqlDelivery: PositiveInt, - maxInFlightSqlMaintenance: PositiveInt, - - maintenanceIntervalMillis: PositiveInt, - claimRecoveryBatchSize: PositiveInt, - claimRecoveryRowsPerSecond: PositiveNumber, - expiryBatchSize: PositiveInt, - expiryRowsPerSecond: PositiveNumber, - integrityBatchSize: PositiveInt, - integrityRowsPerSecond: PositiveNumber, - reconciliationBatchSize: PositiveInt, - reconciliationRowsPerSecond: PositiveNumber, - terminalCollectionBatchSize: PositiveInt, - terminalCollectionRowsPerSecond: PositiveNumber, - orphanChannelCleanupBatchSize: PositiveInt, - orphanChannelCleanupRowsPerSecond: PositiveNumber, - sqliteTransactionCapacityPerSecond: PositiveNumber, - sqliteCapacityHeadroomPercent: Percentage, - - shutdownReleaseConcurrency: PositiveInt, - shutdownReleaseTimeoutMillis: PositiveInt + /** + * Live relay sessions one subject may hold. + * + * Sessions are the front door's only unbounded resource and an authenticated client can open them + * in a loop, so this is the cap that makes a refusal possible. + */ + maxSessionsPerSubject: PositiveInt }) export type Values = typeof Values.Type -const kibibyte = 1_024 -const mebibyte = 1_024 * kibibyte -const gibibyte = 1_024 * mebibyte const day = 24 * 60 * 60 * 1_000 export const defaults: Values = Values.make({ - maxActiveMessagesPerSenderPeer: 10_000, - maxActiveBytesPerSenderPeer: 256 * mebibyte, - maxActiveMessagesPerRecipientPeer: 10_000, - maxActiveBytesPerRecipientPeer: 256 * mebibyte, - maxActiveMessagesPerRecipientSubject: 40_000, - maxActiveBytesPerRecipientSubject: gibibyte, - maxActiveMessagesPerTenant: 100_000, - maxActiveBytesPerTenant: 4 * gibibyte, - maxActiveMessagesPerShard: 1_000_000, - maxActiveBytesPerShard: 16 * gibibyte, - - maxRetainedRowsPerSenderPeer: 10_000, - maxRetainedBytesPerSenderPeer: 64 * mebibyte, - maxRetainedRowsPerRecipientPeer: 10_000, - maxRetainedBytesPerRecipientPeer: 64 * mebibyte, - maxRetainedRowsPerRecipientSubject: 40_000, - maxRetainedBytesPerRecipientSubject: 256 * mebibyte, - maxRetainedRowsPerTenant: 100_000, - maxRetainedBytesPerTenant: gibibyte, - maxRetainedRowsPerShard: 1_000_000, - maxRetainedBytesPerShard: 8 * gibibyte, - messageTtlMillis: 7 * day, maximumSenderRetryHorizonMillis: 7 * day, minimumTerminalRetentionMillis: day, maximumReceiptRetentionMillis: 8 * day, - claimLeaseMillis: 60_000, - maximumRecipientProcessingMillis: 40_000, - acknowledgementTimeoutMillis: 10_000, - claimSafetyMarginMillis: 5_000, - retryBaseDelayMillis: 250, - retryMaximumDelayMillis: 30_000, - maximumDeliveryAttempts: 16, - sqliteLockRetryBaseDelayMillis: 10, - sqliteLockRetryMaximumDelayMillis: 250, - sqliteLockRetryMaxAttempts: 5, - maxRelayConnections: 1_024, - maximumRawChunkBytes: 8 * mebibyte + 4, - maximumDeclaredFrameBytes: 8 * mebibyte, - maximumIncompleteFrameBytes: 8 * mebibyte + 4, - incompleteFrameTimeoutMillis: 10_000, - maximumSharedPayloadBytes: 64 * mebibyte, - maximumByteReservationWaiters: 1_024, maxInFlightAuthentication: 64, authenticationRatePerSecond: 16, authenticationBurst: 64, - - maxSessionsPerSubject: 4, - maxInFlightOpen: 32, - maxInFlightOpenPerSubject: 2, - openRatePerSecond: 16, - openBurst: 32, - maxInFlightPush: 128, - maxInFlightPushPerSubject: 8, - admissionRatePerSecond: 100, - admissionBurst: 128, maxRetainedRateLimitedConnections: 10_000, - maxRetainedRateLimitedSubjects: 10_000, rateLimitIdleRetentionMillis: 10 * 60_000, - terminalResponseQueueCapacity: 256, - maxInFlightTerminalResponses: 64, - maxInFlightTerminalResponsesPerSubject: 4, - terminalResponseRatePerSecond: 100, - terminalResponseBurst: 128, - maxRetainedTerminalResponseSubjects: 10_000, - terminalResponseSubjectIdleRetentionMillis: 10 * 60_000, - - newWorkQueueCapacity: 1_024, - retryQueueCapacity: 1_024, - relayWorkerConcurrency: 8, - newWorkWeight: 3, - retryWorkWeight: 1, - maxActiveChannels: 10_000, - compensationIntervalMillis: 1_000, - compensationBatchSize: 256, - - sqlAdmissionQueueCapacity: 256, - sqlTerminalQueueCapacity: 256, - sqlDeliveryQueueCapacity: 256, - sqlMaintenanceQueueCapacity: 256, - maxInFlightSqlTransactions: 16, - maxInFlightSqlAdmission: 4, - maxInFlightSqlTerminal: 4, - maxInFlightSqlDelivery: 4, - maxInFlightSqlMaintenance: 4, - - maintenanceIntervalMillis: 1_000, - claimRecoveryBatchSize: 100, - claimRecoveryRowsPerSecond: 100, - expiryBatchSize: 100, - expiryRowsPerSecond: 100, - integrityBatchSize: 100, - integrityRowsPerSecond: 100, - reconciliationBatchSize: 100, - reconciliationRowsPerSecond: 100, - terminalCollectionBatchSize: 100, - terminalCollectionRowsPerSecond: 100, - orphanChannelCleanupBatchSize: 100, - orphanChannelCleanupRowsPerSecond: 100, - sqliteTransactionCapacityPerSecond: 200, - sqliteCapacityHeadroomPercent: 20, - - shutdownReleaseConcurrency: 16, - shutdownReleaseTimeoutMillis: 30_000 + maxSessionsPerSubject: 4 }) export class InvalidPeerRelayLimits extends Schema.TaggedErrorClass( @@ -263,166 +108,24 @@ const firstField = (issue: SchemaIssue.Issue): string | undefined => { const validate = (values: Values) => { const relations: ReadonlyArray = [ [ - "maxActiveMessagesPerRecipientSubject", - values.maxActiveMessagesPerRecipientSubject >= values.maxActiveMessagesPerRecipientPeer - ], - [ - "maxActiveBytesPerRecipientSubject", - values.maxActiveBytesPerRecipientSubject >= values.maxActiveBytesPerRecipientPeer - ], - [ - "maxActiveMessagesPerTenant", - values.maxActiveMessagesPerTenant >= values.maxActiveMessagesPerRecipientSubject && - values.maxActiveMessagesPerTenant >= values.maxActiveMessagesPerSenderPeer - ], - [ - "maxActiveBytesPerTenant", - values.maxActiveBytesPerTenant >= values.maxActiveBytesPerRecipientSubject && - values.maxActiveBytesPerTenant >= values.maxActiveBytesPerSenderPeer - ], - ["maxActiveMessagesPerShard", values.maxActiveMessagesPerShard >= values.maxActiveMessagesPerTenant], - ["maxActiveBytesPerShard", values.maxActiveBytesPerShard >= values.maxActiveBytesPerTenant], - [ - "maxRetainedRowsPerRecipientSubject", - values.maxRetainedRowsPerRecipientSubject >= values.maxRetainedRowsPerRecipientPeer - ], - [ - "maxRetainedBytesPerRecipientSubject", - values.maxRetainedBytesPerRecipientSubject >= values.maxRetainedBytesPerRecipientPeer - ], - [ - "maxRetainedRowsPerTenant", - values.maxRetainedRowsPerTenant >= values.maxRetainedRowsPerRecipientSubject && - values.maxRetainedRowsPerTenant >= values.maxRetainedRowsPerSenderPeer - ], - [ - "maxRetainedBytesPerTenant", - values.maxRetainedBytesPerTenant >= values.maxRetainedBytesPerRecipientSubject && - values.maxRetainedBytesPerTenant >= values.maxRetainedBytesPerSenderPeer - ], - ["maxRetainedRowsPerShard", values.maxRetainedRowsPerShard >= values.maxRetainedRowsPerTenant], - ["maxRetainedBytesPerShard", values.maxRetainedBytesPerShard >= values.maxRetainedBytesPerTenant], - [ + // The windows have to nest. A receipt that lapses while its sender may still replay the + // message lets that replay land after the deduplication record is gone, and the recipient + // applies it a second time. The front door refuses a handshake that would breach this, so the + // two checks have to agree — this is the one that makes the deployment's own numbers legal. "maximumReceiptRetentionMillis", values.maximumReceiptRetentionMillis >= Math.max(values.messageTtlMillis, values.maximumSenderRetryHorizonMillis) + values.minimumTerminalRetentionMillis ], [ - "claimLeaseMillis", - values.maximumRecipientProcessingMillis + - values.acknowledgementTimeoutMillis + - values.claimSafetyMarginMillis < - values.claimLeaseMillis - ], - ["claimLeaseMillis", values.claimLeaseMillis < values.messageTtlMillis], - ["retryMaximumDelayMillis", values.retryMaximumDelayMillis >= values.retryBaseDelayMillis], - ["retryMaximumDelayMillis", values.retryMaximumDelayMillis < values.claimLeaseMillis], - [ - "sqliteLockRetryMaximumDelayMillis", - values.sqliteLockRetryMaximumDelayMillis >= values.sqliteLockRetryBaseDelayMillis - ], - ["maximumRawChunkBytes", values.maximumRawChunkBytes <= values.maximumIncompleteFrameBytes], - ["maximumDeclaredFrameBytes", values.maximumDeclaredFrameBytes + 4 <= values.maximumIncompleteFrameBytes], - ["maximumDeclaredFrameBytes", values.maximumDeclaredFrameBytes >= PeerRpcProtocol.maximumRelayPayloadBytes], - ["maximumSharedPayloadBytes", values.maximumSharedPayloadBytes >= values.maximumIncompleteFrameBytes], - [ - "maximumSharedPayloadBytes", - values.relayWorkerConcurrency * PeerRpcProtocol.maximumRelayPayloadBytes <= - values.maximumSharedPayloadBytes - ], - ["maximumByteReservationWaiters", values.maximumByteReservationWaiters >= values.maxRelayConnections], - ["authenticationBurst", values.authenticationBurst >= values.maxInFlightAuthentication], - ["maxSessionsPerSubject", values.maxSessionsPerSubject <= values.maxRelayConnections], - ["maxInFlightOpen", values.maxInFlightOpen <= values.maxRelayConnections], - ["maxInFlightOpenPerSubject", values.maxInFlightOpenPerSubject <= values.maxInFlightOpen], - ["openBurst", values.openBurst >= values.maxInFlightOpen], - ["maxInFlightPushPerSubject", values.maxInFlightPushPerSubject <= values.maxInFlightPush], - ["admissionBurst", values.admissionBurst >= values.maxInFlightPush], - ["maxRetainedRateLimitedConnections", values.maxRetainedRateLimitedConnections >= values.maxRelayConnections], - ["maxRetainedRateLimitedSubjects", values.maxRetainedRateLimitedSubjects >= values.maxInFlightOpen], - [ - "maxInFlightTerminalResponsesPerSubject", - values.maxInFlightTerminalResponsesPerSubject <= values.maxInFlightTerminalResponses - ], - ["terminalResponseQueueCapacity", values.terminalResponseQueueCapacity >= values.maxInFlightTerminalResponses], - ["terminalResponseBurst", values.terminalResponseBurst >= values.maxInFlightTerminalResponses], - ["terminalResponseRatePerSecond", values.terminalResponseRatePerSecond >= values.admissionRatePerSecond], - [ - "maxRetainedTerminalResponseSubjects", - values.maxRetainedTerminalResponseSubjects >= values.maxInFlightTerminalResponses - ], - ["compensationBatchSize", values.compensationBatchSize <= values.maxActiveChannels], - ["sqlAdmissionQueueCapacity", values.sqlAdmissionQueueCapacity >= values.maxInFlightSqlAdmission], - ["sqlTerminalQueueCapacity", values.sqlTerminalQueueCapacity >= values.maxInFlightSqlTerminal], - ["sqlDeliveryQueueCapacity", values.sqlDeliveryQueueCapacity >= values.maxInFlightSqlDelivery], - ["sqlMaintenanceQueueCapacity", values.sqlMaintenanceQueueCapacity >= values.maxInFlightSqlMaintenance], - [ - "maxInFlightSqlTransactions", - values.maxInFlightSqlAdmission + - values.maxInFlightSqlTerminal + - values.maxInFlightSqlDelivery + - values.maxInFlightSqlMaintenance <= - values.maxInFlightSqlTransactions - ], - [ - "maintenanceIntervalMillis", - [ - values.claimRecoveryBatchSize, - values.expiryBatchSize, - values.integrityBatchSize, - values.reconciliationBatchSize, - values.terminalCollectionBatchSize - ].every((batchSize) => batchSize * 1_000 / values.maintenanceIntervalMillis >= values.admissionRatePerSecond) - ], - ["claimRecoveryRowsPerSecond", values.claimRecoveryRowsPerSecond >= values.admissionRatePerSecond], - [ - "claimRecoveryRowsPerSecond", - values.claimRecoveryRowsPerSecond >= - values.claimRecoveryBatchSize * 1_000 / values.maintenanceIntervalMillis - ], - ["expiryRowsPerSecond", values.expiryRowsPerSecond >= values.admissionRatePerSecond], - [ - "expiryRowsPerSecond", - values.expiryRowsPerSecond >= - values.expiryBatchSize * 1_000 / values.maintenanceIntervalMillis - ], - ["integrityRowsPerSecond", values.integrityRowsPerSecond >= values.admissionRatePerSecond], - [ - "integrityRowsPerSecond", - values.integrityRowsPerSecond >= - values.integrityBatchSize * 1_000 / values.maintenanceIntervalMillis - ], - ["reconciliationRowsPerSecond", values.reconciliationRowsPerSecond >= values.admissionRatePerSecond], - [ - "reconciliationRowsPerSecond", - values.reconciliationRowsPerSecond >= - values.reconciliationBatchSize * 1_000 / values.maintenanceIntervalMillis - ], - ["terminalCollectionRowsPerSecond", values.terminalCollectionRowsPerSecond >= values.admissionRatePerSecond], - [ - "terminalCollectionRowsPerSecond", - values.terminalCollectionRowsPerSecond >= - values.terminalCollectionBatchSize * 1_000 / values.maintenanceIntervalMillis - ], - ["orphanChannelCleanupRowsPerSecond", values.orphanChannelCleanupRowsPerSecond >= values.admissionRatePerSecond], - ["shutdownReleaseConcurrency", values.shutdownReleaseConcurrency <= values.maxInFlightSqlTransactions], - ["shutdownReleaseTimeoutMillis", values.shutdownReleaseTimeoutMillis < values.claimLeaseMillis] + // A burst smaller than the concurrency cap would refuse callers the concurrency limit was + // sized to admit, so the rate limiter would be the binding constraint rather than the pool. + "authenticationBurst", + values.authenticationBurst >= values.maxInFlightAuthentication + ] ] const firstInvalid = relations.find(([, valid]) => !valid) - if (firstInvalid !== undefined) return invalid(firstInvalid[0]) - - const requiredTransactionsPerSecond = values.admissionRatePerSecond + - values.claimRecoveryRowsPerSecond / values.claimRecoveryBatchSize + - values.expiryRowsPerSecond / values.expiryBatchSize + - values.integrityRowsPerSecond / values.integrityBatchSize + - values.reconciliationRowsPerSecond / values.reconciliationBatchSize + - values.terminalCollectionRowsPerSecond / values.terminalCollectionBatchSize + - values.orphanChannelCleanupRowsPerSecond / values.orphanChannelCleanupBatchSize - return values.sqliteTransactionCapacityPerSecond >= - requiredTransactionsPerSecond * (1 + values.sqliteCapacityHeadroomPercent / 100) - ? Effect.succeed(values) - : invalid("sqliteTransactionCapacityPerSecond") + return firstInvalid === undefined ? Effect.succeed(values) : invalid(firstInvalid[0]) } export const make = (values: Values) => diff --git a/packages/local-rpc/src/internal/peerRpcObservability.ts b/packages/local-rpc/src/internal/peerRpcObservability.ts index 2660f38..a8059ce 100644 --- a/packages/local-rpc/src/internal/peerRpcObservability.ts +++ b/packages/local-rpc/src/internal/peerRpcObservability.ts @@ -7,6 +7,21 @@ import * as Option from "effect/Option" import * as References from "effect/References" import * as Tracer from "effect/Tracer" +/** + * Spans and metrics for the peer RPC boundary. + * + * Trimmed with the single-process relay. The gauges it used to publish — active sessions, queue + * depth, pending items and bytes, active claims, worker count, ready-queue depth, per-scope quota + * rejections — were all written by `PeerRpcServer` and `PeerRelayIngress`, and nothing writes them + * now: durable inbox depth belongs to the entity's store, and worker and queue accounting belongs + * to the cluster. They were removed rather than left declared, because a gauge that is always zero + * reads to an operator as "there is no backlog" rather than as "nobody is measuring". + * + * What remains is what the surviving code actually drives: the authentication boundary counter, and + * the relay outcome, byte, item, attempt, duration and latency families that `observeRelay` records + * from the client transport. + */ + export type Operation = | "Authentication" | "Open" @@ -87,14 +102,6 @@ export type RelayMaintenanceStage = | "Collect" | "Usage" -export type RelayQuotaDomain = - | "Payload" - | "SenderPeer" - | "RecipientPeer" - | "RecipientSubject" - | "Tenant" - | "Shard" - export interface RelayFacts { readonly bytes?: number readonly items?: number @@ -204,73 +211,17 @@ export const relayLatencyMillis = ( boundaries: [0, 1, 5, 10, 25, 50, 100, 250, 500, 1_000, 5_000, 30_000, 300_000, 3_600_000, 86_400_000] }) -export const relayQuotaRejections = (domain: RelayQuotaDomain) => - Metric.counter("effect_local_rpc_relay_quota_rejection_total", { - incremental: true, - attributes: { domain } - }) - -export const relayPendingItems = () => Metric.gauge("effect_local_rpc_relay_pending_items") - -export const relayPendingBytes = () => Metric.gauge("effect_local_rpc_relay_pending_bytes") - -export const relayActiveClaims = () => Metric.gauge("effect_local_rpc_relay_active_claims") - -export const relayWorkers = () => Metric.gauge("effect_local_rpc_relay_workers") - -export const relayReadyQueueItems = (lane: "New" | "Retry") => - Metric.gauge("effect_local_rpc_relay_ready_queue_items", { - attributes: { lane } - }) - export const boundary = (operation: Operation, result: Result) => Metric.counter("effect_local_rpc_boundary_total", { incremental: true, attributes: { operation, result } }) -export const activeSessions = () => Metric.gauge("effect_local_rpc_active_sessions") - -export const queueItems = (operation: "Inbound" | "Outbound") => - Metric.gauge("effect_local_rpc_queue_items", { attributes: { operation } }) - -export const bytes = (operation: "Inbound" | "Outbound") => - Metric.histogram("effect_local_rpc_message_bytes", { - attributes: { operation }, - boundaries: [0, 64, 256, 1_024, 4_096, 16_384, 65_536, 262_144, 1_048_576] - }) - -export const selectedDocuments = () => - Metric.histogram("effect_local_rpc_selected_documents", { - attributes: { operation: "Open" }, - boundaries: [0, 1, 2, 4, 8, 16, 32, 64, 128] - }) - -export const record = (operation: Operation, result: Result, amount: number) => +const record = (operation: Operation, result: Result, amount: number) => Metric.update(boundary(operation, result), amount).pipe( Effect.provideService(Metric.CurrentMetricAttributes, {}) ) -export const modifyActiveSessions = (amount: number) => - Metric.modify(activeSessions(), amount).pipe( - Effect.provideService(Metric.CurrentMetricAttributes, {}) - ) - -export const modifyQueueItems = (operation: "Inbound" | "Outbound", amount: number) => - Metric.modify(queueItems(operation), amount).pipe( - Effect.provideService(Metric.CurrentMetricAttributes, {}) - ) - -export const recordBytes = (operation: "Inbound" | "Outbound", amount: number) => - Metric.update(bytes(operation), amount).pipe( - Effect.provideService(Metric.CurrentMetricAttributes, {}) - ) - -export const recordSelectedDocuments = (amount: number) => - Metric.update(selectedDocuments(), amount).pipe( - Effect.provideService(Metric.CurrentMetricAttributes, {}) - ) - const withoutMetricAttributes = (effect: Effect.Effect) => effect.pipe(Effect.provideService(Metric.CurrentMetricAttributes, {})) @@ -467,25 +418,6 @@ export const recordRelayOutcome = (options: { return Effect.all(updates, { discard: true }).pipe(withoutMetricAttributes) } -export const recordRelayQuotaRejection = (domain: RelayQuotaDomain) => - Metric.update(relayQuotaRejections(domain), 1).pipe(withoutMetricAttributes) - -export const setRelayPending = (items: number, amountBytes: number) => - Effect.all([ - Metric.update(relayPendingItems(), items), - Metric.update(relayPendingBytes(), amountBytes) - ], { discard: true }).pipe(withoutMetricAttributes) - -export const setRelayActiveClaims = (amount: number) => - Metric.update(relayActiveClaims(), amount).pipe(withoutMetricAttributes) - -export const setRelayWorkers = (amount: number) => Metric.update(relayWorkers(), amount).pipe(withoutMetricAttributes) - -export const setRelayReadyQueueItems = ( - lane: "New" | "Retry", - amount: number -) => Metric.update(relayReadyQueueItems(lane), amount).pipe(withoutMetricAttributes) - export const observe = (options: { readonly effect: Effect.Effect readonly operation: Operation diff --git a/packages/local-rpc/test/PeerRelayLimits.test.ts b/packages/local-rpc/test/PeerRelayLimits.test.ts index 5832a59..9a77c0d 100644 --- a/packages/local-rpc/test/PeerRelayLimits.test.ts +++ b/packages/local-rpc/test/PeerRelayLimits.test.ts @@ -1,39 +1,37 @@ import { assert, describe, it } from "@effect/vitest" import * as Effect from "effect/Effect" -import * as Layer from "effect/Layer" import * as PeerRelayLimits from "../src/PeerRelayLimits.js" import * as PeerRpc from "../src/PeerRpc.js" describe("PeerRelayLimits", () => { - it.effect("publishes the complete validated production defaults through its Layer", () => + it.effect("publishes the validated production defaults through its Layer", () => Effect.gen(function*() { const limits = yield* PeerRelayLimits.PeerRelayLimits assert.deepStrictEqual(limits, PeerRelayLimits.defaults) - assert.strictEqual(Object.keys(limits).length, 97) - assert.strictEqual(limits.maximumDeliveryAttempts, 16) + // Pinned so a value cannot be added back without a reader, and cannot quietly disappear from + // under a deployment that sets it. Every one of these is read by production code today. + assert.deepStrictEqual(Object.keys(limits).toSorted(), [ + "authenticationBurst", + "authenticationRatePerSecond", + "maxInFlightAuthentication", + "maxRetainedRateLimitedConnections", + "maxSessionsPerSubject", + "maximumReceiptRetentionMillis", + "maximumSenderRetryHorizonMillis", + "messageTtlMillis", + "minimumTerminalRetentionMillis", + "rateLimitIdleRetentionMillis" + ]) }).pipe(Effect.provide(PeerRelayLimits.layerDefaults))) it.effect.each( [ - ["maxActiveMessagesPerShard", 0, "Expected a value greater than 0, got 0"], - ["maxRetainedBytesPerTenant", -1, "Expected a value greater than 0, got -1"], - ["claimLeaseMillis", 1.5, "Expected an integer, got 1.5"], - ["maximumDeliveryAttempts", 0, "Expected a value greater than 0, got 0"], - ["openRatePerSecond", Number.NaN, "Expected a finite number, got NaN"], - [ - "terminalResponseRatePerSecond", - Number.POSITIVE_INFINITY, - "Expected a finite number, got Infinity" - ], - [ - "maximumReceiptRetentionMillis", - PeerRpc.maximumNegotiatedDurationMillis + 1, - `Expected a value less than or equal to ${PeerRpc.maximumNegotiatedDurationMillis}, got ${ - PeerRpc.maximumNegotiatedDurationMillis + 1 - }` - ], - ["sqliteCapacityHeadroomPercent", 101, "Expected a value less than or equal to 100, got 101"], - ["shutdownReleaseConcurrency", Number.MAX_SAFE_INTEGER + 1, "Expected an integer"] + ["maxSessionsPerSubject", 0], + ["messageTtlMillis", 1.5], + ["authenticationRatePerSecond", Number.NaN], + ["authenticationRatePerSecond", Number.POSITIVE_INFINITY], + ["maximumReceiptRetentionMillis", PeerRpc.maximumNegotiatedDurationMillis + 1], + ["rateLimitIdleRetentionMillis", Number.MAX_SAFE_INTEGER + 1] ] as const )("rejects scalar %s with the stable configuration error", ([field, value]) => Effect.gen(function*() { @@ -45,158 +43,40 @@ describe("PeerRelayLimits", () => { if (error._tag === "InvalidPeerRelayLimits") assert.strictEqual(error.field, field) })) - const crossFieldCases = [ - ["maxActiveMessagesPerRecipientSubject", { maxActiveMessagesPerRecipientSubject: 9_999 }], - ["maxActiveBytesPerRecipientSubject", { maxActiveBytesPerRecipientSubject: 1 }], - ["maxActiveMessagesPerTenant", { maxActiveMessagesPerTenant: 9_999 }], - ["maxActiveBytesPerTenant", { maxActiveBytesPerTenant: 1 }], - ["maxActiveMessagesPerShard", { maxActiveMessagesPerShard: 99_999 }], - ["maxActiveBytesPerShard", { maxActiveBytesPerShard: 1 }], - ["maxRetainedRowsPerRecipientSubject", { maxRetainedRowsPerRecipientSubject: 9_999 }], - ["maxRetainedBytesPerRecipientSubject", { maxRetainedBytesPerRecipientSubject: 1 }], - ["maxRetainedRowsPerTenant", { maxRetainedRowsPerTenant: 9_999 }], - ["maxRetainedBytesPerTenant", { maxRetainedBytesPerTenant: 1 }], - ["maxRetainedRowsPerShard", { maxRetainedRowsPerShard: 99_999 }], - ["maxRetainedBytesPerShard", { maxRetainedBytesPerShard: 1 }], - [ - "maximumReceiptRetentionMillis", - { maximumReceiptRetentionMillis: PeerRelayLimits.defaults.maximumReceiptRetentionMillis - 1 } - ], - ["claimLeaseMillis", { maximumRecipientProcessingMillis: 45_000 }], - ["claimLeaseMillis", { claimLeaseMillis: PeerRelayLimits.defaults.messageTtlMillis }], - ["retryMaximumDelayMillis", { retryMaximumDelayMillis: 249 }], - ["retryMaximumDelayMillis", { retryMaximumDelayMillis: 60_000 }], - ["sqliteLockRetryMaximumDelayMillis", { sqliteLockRetryMaximumDelayMillis: 9 }], - [ - "maximumRawChunkBytes", - { - maximumRawChunkBytes: PeerRelayLimits.defaults.maximumIncompleteFrameBytes + 1 - } - ], - [ - "maximumDeclaredFrameBytes", - { - maximumDeclaredFrameBytes: PeerRelayLimits.defaults.maximumIncompleteFrameBytes - 3 - } - ], - [ - "maximumDeclaredFrameBytes", - { maximumDeclaredFrameBytes: PeerRpc.maximumRelayPayloadBytes - 1 } - ], - [ - "maximumSharedPayloadBytes", - { - maximumSharedPayloadBytes: PeerRelayLimits.defaults.maximumIncompleteFrameBytes - 1 - } - ], - ["maximumSharedPayloadBytes", { relayWorkerConcurrency: 17 }], - ["maximumByteReservationWaiters", { maximumByteReservationWaiters: 1_023 }], - ["maxSessionsPerSubject", { maxSessionsPerSubject: 1_025 }], - ["maxInFlightOpen", { maxInFlightOpen: 1_025, openBurst: 1_025 }], - ["maxInFlightOpenPerSubject", { maxInFlightOpenPerSubject: 33 }], - ["openBurst", { openBurst: 31 }], - ["maxInFlightPushPerSubject", { maxInFlightPushPerSubject: 129 }], - ["admissionBurst", { admissionBurst: 127 }], - ["maxRetainedRateLimitedConnections", { maxRetainedRateLimitedConnections: 1_023 }], - ["maxRetainedRateLimitedSubjects", { maxRetainedRateLimitedSubjects: 31 }], + it.effect.each( [ - "maxInFlightTerminalResponsesPerSubject", - { maxInFlightTerminalResponsesPerSubject: 65 } - ], - ["terminalResponseQueueCapacity", { terminalResponseQueueCapacity: 63 }], - ["terminalResponseBurst", { terminalResponseBurst: 63 }], - ["terminalResponseRatePerSecond", { terminalResponseRatePerSecond: 99 }], - ["maxRetainedTerminalResponseSubjects", { maxRetainedTerminalResponseSubjects: 63 }], - ["compensationBatchSize", { compensationBatchSize: 10_001 }], - ["sqlAdmissionQueueCapacity", { sqlAdmissionQueueCapacity: 3 }], - ["sqlTerminalQueueCapacity", { sqlTerminalQueueCapacity: 3 }], - ["sqlDeliveryQueueCapacity", { sqlDeliveryQueueCapacity: 3 }], - ["sqlMaintenanceQueueCapacity", { sqlMaintenanceQueueCapacity: 3 }], - ["maxInFlightSqlTransactions", { maxInFlightSqlTransactions: 15 }], - ["claimRecoveryRowsPerSecond", { claimRecoveryRowsPerSecond: 99 }], - ["expiryRowsPerSecond", { expiryRowsPerSecond: 99 }], - ["integrityRowsPerSecond", { integrityRowsPerSecond: 99 }], - ["reconciliationRowsPerSecond", { reconciliationRowsPerSecond: 99 }], - ["terminalCollectionRowsPerSecond", { terminalCollectionRowsPerSecond: 99 }], - ["orphanChannelCleanupRowsPerSecond", { orphanChannelCleanupRowsPerSecond: 99 }], - ["shutdownReleaseConcurrency", { shutdownReleaseConcurrency: 17 }], - ["shutdownReleaseTimeoutMillis", { shutdownReleaseTimeoutMillis: 60_000 }], - ["sqliteTransactionCapacityPerSecond", { sqliteTransactionCapacityPerSecond: 127 }] - ] as const satisfies ReadonlyArray< - readonly [keyof PeerRelayLimits.Values, Partial] - > - - it.effect.each(crossFieldCases)( + // A receipt that lapses inside its sender's replay window lets the replay land after the + // deduplication record is gone, and the recipient applies the message twice. + ["maximumReceiptRetentionMillis", { minimumTerminalRetentionMillis: 7 * 24 * 60 * 60 * 1_000 }], + ["maximumReceiptRetentionMillis", { maximumSenderRetryHorizonMillis: 8 * 24 * 60 * 60 * 1_000 }], + // A burst under the concurrency cap makes the rate limiter, not the pool, the binding + // constraint — so the pool is sized for a concurrency it is never allowed to reach. + ["authenticationBurst", { authenticationBurst: 1 }] + ] as const + )( "rejects the %s cross field constraint through the production Layer", - ([field, patch]) => + ([field, overrides]) => Effect.gen(function*() { - const error = yield* Layer.build( - PeerRelayLimits.layer({ - ...PeerRelayLimits.defaults, - ...patch - }) - ).pipe(Effect.scoped, Effect.flip) + const error = yield* PeerRelayLimits.make({ + ...PeerRelayLimits.defaults, + ...overrides + }).pipe(Effect.flip) assert.strictEqual(error._tag, "InvalidPeerRelayLimits") if (error._tag === "InvalidPeerRelayLimits") assert.strictEqual(error.field, field) }) ) - it.effect("accepts an exact aggregate SQLite capacity boundary with configured headroom", () => + it.effect("accepts a retention window that exactly covers its retry horizon", () => Effect.gen(function*() { - const required = PeerRelayLimits.defaults.admissionRatePerSecond + - PeerRelayLimits.defaults.claimRecoveryRowsPerSecond / - PeerRelayLimits.defaults.claimRecoveryBatchSize + - PeerRelayLimits.defaults.expiryRowsPerSecond / - PeerRelayLimits.defaults.expiryBatchSize + - PeerRelayLimits.defaults.integrityRowsPerSecond / - PeerRelayLimits.defaults.integrityBatchSize + - PeerRelayLimits.defaults.reconciliationRowsPerSecond / - PeerRelayLimits.defaults.reconciliationBatchSize + - PeerRelayLimits.defaults.terminalCollectionRowsPerSecond / - PeerRelayLimits.defaults.terminalCollectionBatchSize + - PeerRelayLimits.defaults.orphanChannelCleanupRowsPerSecond / - PeerRelayLimits.defaults.orphanChannelCleanupBatchSize - const exact = required * (1 + PeerRelayLimits.defaults.sqliteCapacityHeadroomPercent / 100) - const limits = yield* PeerRelayLimits.PeerRelayLimits - assert.isAtLeast(limits.sqliteTransactionCapacityPerSecond, exact) - }).pipe( - Effect.provide( - PeerRelayLimits.layer({ - ...PeerRelayLimits.defaults, - sqliteTransactionCapacityPerSecond: 127.2 - }) - ) - )) - - it.effect("rejects a maintenance interval that exceeds the declared row rate", () => - Effect.gen(function*() { - const exit = yield* Effect.exit(PeerRelayLimits.make({ + // The boundary itself is legal: the relation is stated as `>=`, and a deployment that sizes + // retention to exactly cover the horizon plus the required slack is correctly configured. + const values = yield* PeerRelayLimits.make({ ...PeerRelayLimits.defaults, - maintenanceIntervalMillis: 1 - })) - assert.strictEqual(exit._tag, "Failure") - if (exit._tag === "Failure") { - const error = yield* Effect.failCause(exit.cause).pipe(Effect.flip) - assert.strictEqual(error._tag, "InvalidPeerRelayLimits") - if (error._tag === "InvalidPeerRelayLimits") { - assert.strictEqual(error.field, "claimRecoveryRowsPerSecond") - } - } - })) - - it.effect("rejects a maintenance interval that cannot keep up with admission", () => - Effect.gen(function*() { - const exit = yield* Effect.exit(PeerRelayLimits.make({ - ...PeerRelayLimits.defaults, - maintenanceIntervalMillis: 2_000 - })) - assert.strictEqual(exit._tag, "Failure") - if (exit._tag === "Failure") { - const error = yield* Effect.failCause(exit.cause).pipe(Effect.flip) - assert.strictEqual(error._tag, "InvalidPeerRelayLimits") - if (error._tag === "InvalidPeerRelayLimits") { - assert.strictEqual(error.field, "maintenanceIntervalMillis") - } - } + messageTtlMillis: 1_000, + maximumSenderRetryHorizonMillis: 2_000, + minimumTerminalRetentionMillis: 500, + maximumReceiptRetentionMillis: 2_500 + }) + assert.strictEqual(values.maximumReceiptRetentionMillis, 2_500) })) }) diff --git a/packages/local-rpc/test/PeerRpcObservability.test.ts b/packages/local-rpc/test/PeerRpcObservability.test.ts index 00c24bd..a8a3493 100644 --- a/packages/local-rpc/test/PeerRpcObservability.test.ts +++ b/packages/local-rpc/test/PeerRpcObservability.test.ts @@ -224,12 +224,6 @@ describe("PeerRpcObservability", () => { assert.isTrue(Exit.isFailure(exit)) if (Exit.isFailure(exit)) assert.strictEqual(exit.cause, cause) } - yield* PeerRpcObservability.recordRelayQuotaRejection("Shard") - yield* PeerRpcObservability.setRelayPending(2, 8) - yield* PeerRpcObservability.setRelayActiveClaims(1) - yield* PeerRpcObservability.setRelayWorkers(3) - yield* PeerRpcObservability.setRelayReadyQueueItems("New", 2) - yield* PeerRpcObservability.setRelayReadyQueueItems("Retry", 1) yield* PeerRpcObservability.recordRelayOutcome({ operation: "RelayAcknowledge", direction: "Receive", @@ -293,34 +287,6 @@ describe("PeerRpcObservability", () => { )).count, causes.length ) - assert.strictEqual( - (yield* metricValue(PeerRpcObservability.relayQuotaRejections("Shard"))).count, - 1 - ) - assert.strictEqual( - (yield* metricValue(PeerRpcObservability.relayPendingItems())).value, - 2 - ) - assert.strictEqual( - (yield* metricValue(PeerRpcObservability.relayPendingBytes())).value, - 8 - ) - assert.strictEqual( - (yield* metricValue(PeerRpcObservability.relayActiveClaims())).value, - 1 - ) - assert.strictEqual( - (yield* metricValue(PeerRpcObservability.relayWorkers())).value, - 3 - ) - assert.strictEqual( - (yield* metricValue(PeerRpcObservability.relayReadyQueueItems("New"))).value, - 2 - ) - assert.strictEqual( - (yield* metricValue(PeerRpcObservability.relayReadyQueueItems("Retry"))).value, - 1 - ) assert.strictEqual( (yield* metricValue( PeerRpcObservability.relayLatencyMillis( From 2c1e80d525a5016beae73fe5fcdf2b239420e687 Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Mon, 27 Jul 2026 10:57:12 -0500 Subject: [PATCH 26/29] Take Duration.Input for configured durations A configured duration named somethingMillis puts the unit in the identifier instead of the type, and makes every deployment do its own arithmetic to say "seven days". Every configuration surface this branch owns now takes Duration.Input, so a deployment can write Duration.days(7), "7 days", or a raw number, and each is converted once where the Layer is built rather than on every request. Two things deliberately keep their unit in the name, and RULES.md now says why: values that cross the wire contract, and values that feed a millisecond column. Duration has no stable serialized encoding, so a protocol or a column has to name its unit. The first attempt at this renamed the store's operation arguments along with the configuration and the typechecker caught it - when the boundary between configuration and protocol is a field name, a regex is the wrong tool. The two duration schemas are Schema.declare guards over Duration.fromInput, which is safe on arbitrary input, so InvalidPeerRelayLimits still names the offending field. --- RULES.md | 3 + packages/local-rpc/src/PeerAuthentication.ts | 8 ++- packages/local-rpc/src/PeerRelayLimits.ts | 70 ++++++++++++++----- packages/local-rpc/src/RelayInbox.ts | 36 ++++++---- .../local-rpc/src/RelayInboxMaintenance.ts | 18 +++-- packages/local-rpc/src/RelayServer.ts | 35 ++++++---- .../local-rpc/test/PeerAuthentication.test.ts | 2 +- .../local-rpc/test/PeerRelayLimits.test.ts | 55 ++++++++++----- packages/local-rpc/test/RelayInbox.test.ts | 23 +++--- .../test/RelayInboxMaintenance.test.ts | 5 +- packages/local-rpc/test/RelayServer.test.ts | 33 ++++----- .../local-rpc/test/RelayWireFormat.test.ts | 21 +++--- 12 files changed, 199 insertions(+), 110 deletions(-) diff --git a/RULES.md b/RULES.md index 3f88e78..4cfe21d 100644 --- a/RULES.md +++ b/RULES.md @@ -71,6 +71,9 @@ Read this file before any work. Treat these rules as required for every package. - Optimize public APIs for low setup friction and explicit consumer control. - For pipeable public combinators, use `Function.dual` when data first and data last forms materially improve use. +- Express a configurable duration as `Duration.Input`, never as a bare number named with a unit suffix such as `Millis`. `Duration.Input` lets a consumer write `"30 seconds"`, `Duration.minutes(5)`, or a raw number, and it states the unit in the type instead of in the identifier. +- Convert a configured duration once, with `Duration.toMillis`, at the point the Layer or service is constructed, and close over the result. Do not convert per call and do not thread `Duration.Input` into arithmetic. +- A duration that crosses a wire protocol or is persisted stays a plain number with its unit in the name. `Duration` has no stable serialized encoding, so a protocol or column must name its unit. - Use `camelCase` for values and functions. Use `PascalCase` for types, services, schemas, and error classes. - Add an `Error` suffix when it improves clarity. Preserve established class names and serialized `_tag` values for public or protocol errors. - Name Layer values and constructors so their implementation, configuration, or lifecycle is clear. diff --git a/packages/local-rpc/src/PeerAuthentication.ts b/packages/local-rpc/src/PeerAuthentication.ts index fe253b9..d35f006 100644 --- a/packages/local-rpc/src/PeerAuthentication.ts +++ b/packages/local-rpc/src/PeerAuthentication.ts @@ -1,6 +1,7 @@ import * as Cause from "effect/Cause" import * as Clock from "effect/Clock" import * as Context from "effect/Context" +import * as Duration from "effect/Duration" import * as Effect from "effect/Effect" import * as Exit from "effect/Exit" import * as Layer from "effect/Layer" @@ -41,6 +42,9 @@ export const layerServer = Layer.effect( const authenticator = yield* PeerAuthenticator.PeerAuthenticator const limits = yield* PeerRelayLimits.PeerRelayLimits const verifierPermits = yield* Semaphore.make(limits.maxInFlightAuthentication) + // Converted once here rather than on every request: the rate limiter's arithmetic is in + // milliseconds, while the configuration states a duration. + const rateLimitIdleRetentionMillis = Duration.toMillis(limits.rateLimitIdleRetention) const rateStateLock = yield* Semaphore.make(1) type RateState = { tokens: number @@ -54,7 +58,7 @@ export const layerServer = Layer.effect( const expireOldestInactive = (now: number) => { const oldest = inactiveRateState.entries().next().value if (oldest === undefined) return - if (now - oldest[1].lastUsedAt < limits.rateLimitIdleRetentionMillis) return + if (now - oldest[1].lastUsedAt < rateLimitIdleRetentionMillis) return inactiveRateState.delete(oldest[0]) rateState.delete(oldest[0]) } @@ -70,7 +74,7 @@ export const layerServer = Layer.effect( let entry = rateState.get(clientId) if (entry?.inFlight === 0) { inactiveRateState.delete(clientId) - if (now - entry.lastUsedAt >= limits.rateLimitIdleRetentionMillis) { + if (now - entry.lastUsedAt >= rateLimitIdleRetentionMillis) { rateState.delete(clientId) entry = undefined } diff --git a/packages/local-rpc/src/PeerRelayLimits.ts b/packages/local-rpc/src/PeerRelayLimits.ts index 5868d50..b0f62f3 100644 --- a/packages/local-rpc/src/PeerRelayLimits.ts +++ b/packages/local-rpc/src/PeerRelayLimits.ts @@ -1,6 +1,8 @@ import * as Context from "effect/Context" +import * as Duration from "effect/Duration" import * as Effect from "effect/Effect" import * as Layer from "effect/Layer" +import * as Option from "effect/Option" import * as Schema from "effect/Schema" import type * as SchemaIssue from "effect/SchemaIssue" import * as PeerRpcProtocol from "./internal/peerRpcProtocol.js" @@ -29,19 +31,51 @@ import * as PeerRpcProtocol from "./internal/peerRpcProtocol.js" const PositiveInt = Schema.Int.check(Schema.isGreaterThan(0)) const PositiveNumber = Schema.Number.check(Schema.isFinite(), Schema.isGreaterThan(0)) -const NegotiatedDuration = PositiveInt.check( - Schema.isLessThanOrEqualTo(PeerRpcProtocol.maximumNegotiatedDurationMillis) + +/** Milliseconds for a finite, positive duration input, or `undefined` if it is neither. */ +const finiteMillis = (input: unknown): number | undefined => + Option.match(Duration.fromInput(input as Duration.Input), { + onNone: () => undefined, + onSome: (duration) => Duration.isFinite(duration) ? Duration.toMillis(duration) : undefined + }) + +/** + * A configured duration. + * + * `Duration.Input` rather than a number of milliseconds, so the unit lives in the type instead of + * in the field name and a deployment can write whichever form reads best: `"7 days"`, + * `Duration.hours(1)`, or a plain number. + */ +const PositiveDuration = Schema.declare( + (input): input is Duration.Input => { + const millis = finiteMillis(input) + return millis !== undefined && millis > 0 + }, + { expected: "a finite positive duration" } +) + +/** A duration the wire contract also has to be able to carry, so it is bounded by the protocol. */ +const NegotiatedDuration = Schema.declare( + (input): input is Duration.Input => { + const millis = finiteMillis(input) + return millis !== undefined && millis > 0 && + millis <= PeerRpcProtocol.maximumNegotiatedDurationMillis + }, + { + expected: "a finite positive duration no longer than " + + `${PeerRpcProtocol.maximumNegotiatedDurationMillis}ms` + } ) export const Values = Schema.Struct({ /** How long an undelivered message survives in an inbox before it is expired. */ - messageTtlMillis: NegotiatedDuration, + messageTtl: NegotiatedDuration, /** The longest replay window a client may negotiate at `Open`. */ - maximumSenderRetryHorizonMillis: NegotiatedDuration, + maximumSenderRetryHorizon: NegotiatedDuration, /** The slack a receipt must outlive its message and that message's replay window by. */ - minimumTerminalRetentionMillis: NegotiatedDuration, + minimumTerminalRetention: NegotiatedDuration, /** The longest receipt retention a client may negotiate at `Open`. */ - maximumReceiptRetentionMillis: NegotiatedDuration, + maximumReceiptRetention: NegotiatedDuration, /** Credentials verified concurrently. Beyond this a caller is told to come back. */ maxInFlightAuthentication: PositiveInt, @@ -52,7 +86,7 @@ export const Values = Schema.Struct({ /** Rate-limiter state retained for idle clients, so the table cannot grow without bound. */ maxRetainedRateLimitedConnections: PositiveInt, /** How long a client's rate-limiter state is kept after its last request. */ - rateLimitIdleRetentionMillis: PositiveInt, + rateLimitIdleRetention: PositiveDuration, /** * Live relay sessions one subject may hold. @@ -64,19 +98,17 @@ export const Values = Schema.Struct({ }) export type Values = typeof Values.Type -const day = 24 * 60 * 60 * 1_000 - export const defaults: Values = Values.make({ - messageTtlMillis: 7 * day, - maximumSenderRetryHorizonMillis: 7 * day, - minimumTerminalRetentionMillis: day, - maximumReceiptRetentionMillis: 8 * day, + messageTtl: Duration.days(7), + maximumSenderRetryHorizon: Duration.days(7), + minimumTerminalRetention: Duration.days(1), + maximumReceiptRetention: Duration.days(8), maxInFlightAuthentication: 64, authenticationRatePerSecond: 16, authenticationBurst: 64, maxRetainedRateLimitedConnections: 10_000, - rateLimitIdleRetentionMillis: 10 * 60_000, + rateLimitIdleRetention: Duration.minutes(10), maxSessionsPerSubject: 4 }) @@ -112,10 +144,12 @@ const validate = (values: Values) => { // message lets that replay land after the deduplication record is gone, and the recipient // applies it a second time. The front door refuses a handshake that would breach this, so the // two checks have to agree — this is the one that makes the deployment's own numbers legal. - "maximumReceiptRetentionMillis", - values.maximumReceiptRetentionMillis >= - Math.max(values.messageTtlMillis, values.maximumSenderRetryHorizonMillis) + - values.minimumTerminalRetentionMillis + "maximumReceiptRetention", + Duration.toMillis(values.maximumReceiptRetention) >= + Math.max( + Duration.toMillis(values.messageTtl), + Duration.toMillis(values.maximumSenderRetryHorizon) + ) + Duration.toMillis(values.minimumTerminalRetention) ], [ // A burst smaller than the concurrency cap would refuse callers the concurrency limit was diff --git a/packages/local-rpc/src/RelayInbox.ts b/packages/local-rpc/src/RelayInbox.ts index 1a4b11d..ebea272 100644 --- a/packages/local-rpc/src/RelayInbox.ts +++ b/packages/local-rpc/src/RelayInbox.ts @@ -2,6 +2,7 @@ import * as Identity from "@lucas-barake/effect-local/Identity" import * as Cause from "effect/Cause" import * as Clock from "effect/Clock" import * as Deferred from "effect/Deferred" +import * as Duration from "effect/Duration" import * as Effect from "effect/Effect" import * as Exit from "effect/Exit" import * as Latch from "effect/Latch" @@ -147,20 +148,20 @@ export interface Options { */ readonly maxDeliveries: number /** How long an undelivered message survives before it is expired. */ - readonly messageTtlMillis: number + readonly messageTtl: Duration.Input /** * How long a settled message's identity is retained. Must cover the sender's retry horizon, or a * replay could outlive the deduplication record and be applied twice. */ - readonly terminalRetentionMillis: number + readonly terminalRetention: Duration.Input /** How long a session survives without a heartbeat. */ - readonly sessionDeadlineMillis: number + readonly sessionDeadline: Duration.Input /** How often the entity checks for a lapsed session deadline. */ - readonly sessionSweepMillis: number + readonly sessionSweep: Duration.Input /** Maximum channels delivered concurrently to one session. */ readonly maxConcurrentChannels: number /** Backoff after a store failure, so a failing database cannot spin the dispatcher. */ - readonly storeRetryMillis: number + readonly storeRetry: Duration.Input /** Per-inbox admission caps, enforced inside the admission transaction. */ readonly maxPendingMessages: number readonly maxPendingBytes: number @@ -172,7 +173,7 @@ export interface Options { */ readonly mailboxCapacity: number /** How long an idle entity survives before the cluster passivates it. */ - readonly maxIdleTimeMillis: number + readonly maxIdleTime: Duration.Input } /** @@ -249,6 +250,13 @@ export const layer = (options: Options) => const inboxKey = address.entityId const store = yield* RelayInboxStore.RelayInboxStore + // Converted once at construction. Everything downstream of these is arithmetic against + // `Clock.currentTimeMillis` or a millisecond column, so the durations are resolved here + // rather than threaded through as inputs. + const messageTtlMillis = Duration.toMillis(options.messageTtl) + const terminalRetentionMillis = Duration.toMillis(options.terminalRetention) + const sessionDeadlineMillis = Duration.toMillis(options.sessionDeadline) + const sessionRef = yield* Ref.make(Option.none()) // Serializes session installation so two concurrent reconnects cannot both install // themselves, which would leave two dispatchers draining one inbox. @@ -360,7 +368,7 @@ export const layer = (options: Options) => outcome, messageHash: head.envelope.messageHash, now: settledAt, - terminalRetentionMillis: options.terminalRetentionMillis + terminalRetentionMillis }) if (settled !== "Settled") { yield* Effect.logWarning("Relay inbox settlement did not apply").pipe( @@ -381,7 +389,7 @@ export const layer = (options: Options) => relayMessageId: head.relayMessageId, reason: error.reason._tag }), - Effect.andThen(Effect.sleep(options.storeRetryMillis)) + Effect.andThen(Effect.sleep(options.storeRetry)) )), // Fails any settle still waiting on this delivery so the recipient retries rather than // believing an acknowledgement landed. @@ -437,7 +445,7 @@ export const layer = (options: Options) => Effect.catchTag("ReplicaError", (error) => Effect.logWarning("Relay inbox dispatch failed; retrying").pipe( Effect.annotateLogs({ inboxKey, reason: error.reason._tag }), - Effect.andThen(Effect.sleep(options.storeRetryMillis)), + Effect.andThen(Effect.sleep(options.storeRetry)), Effect.andThen(dispatch(session)) )) ) as Effect.Effect @@ -461,7 +469,7 @@ export const layer = (options: Options) => }) ) }).pipe( - Effect.repeat(Schedule.spaced(options.sessionSweepMillis)), + Effect.repeat(Schedule.spaced(options.sessionSweep)), Effect.forkScoped ) @@ -492,7 +500,7 @@ export const layer = (options: Options) => const extendDeadline = (session: Session) => Clock.currentTimeMillis.pipe( - Effect.flatMap((now) => Ref.set(session.deadlineAt, now + options.sessionDeadlineMillis)) + Effect.flatMap((now) => Ref.set(session.deadlineAt, now + sessionDeadlineMillis)) ) return RelayInbox.of({ @@ -504,7 +512,7 @@ export const layer = (options: Options) => channel: payload.channel, envelope: payload.envelope, now, - messageTtlMillis: options.messageTtlMillis, + messageTtlMillis, senderRetryHorizonMillis: payload.senderRetryHorizonMillis, quota: { maxPendingMessages: options.maxPendingMessages, @@ -583,7 +591,7 @@ export const layer = (options: Options) => PeerRpcError.ServerUnavailable | Cause.Done >(0) const now = yield* Clock.currentTimeMillis - const deadlineAt = yield* Ref.make(now + options.sessionDeadlineMillis) + const deadlineAt = yield* Ref.make(now + sessionDeadlineMillis) const session: Session = { sessionId: payload.sessionId, outbound, @@ -708,6 +716,6 @@ export const layer = (options: Options) => // framework default that could change underneath these invariants. concurrency: 1, mailboxCapacity: options.mailboxCapacity, - maxIdleTime: options.maxIdleTimeMillis + maxIdleTime: options.maxIdleTime } ) diff --git a/packages/local-rpc/src/RelayInboxMaintenance.ts b/packages/local-rpc/src/RelayInboxMaintenance.ts index 88c7c1d..f8209d0 100644 --- a/packages/local-rpc/src/RelayInboxMaintenance.ts +++ b/packages/local-rpc/src/RelayInboxMaintenance.ts @@ -1,4 +1,5 @@ import * as Clock from "effect/Clock" +import * as Duration from "effect/Duration" import * as Effect from "effect/Effect" import * as Layer from "effect/Layer" import * as Schedule from "effect/Schedule" @@ -15,14 +16,14 @@ import * as RelayInboxStore from "./RelayInboxStore.js" * a cluster singleton — one runner owns it at a time, and ownership moves with the shard rather than * being pinned to a particular process. * - * Without it `messageTtlMillis` and `terminalRetentionMillis` are inert: undelivered messages are + * Without it `messageTtl` and `terminalRetention` are inert: undelivered messages are * never expired, terminal rows are never collected, the table grows without bound, and the retained * row count climbs until admission is permanently refused. */ export interface Options { /** How often the sweep runs. */ - readonly intervalMillis: number + readonly interval: Duration.Input /** * Rows touched per sweep, per operation. * @@ -36,7 +37,7 @@ export interface Options { * Expiry is a terminal transition like settlement, so the identity has to outlive the window in * which its sender may still replay it. The store only ever grows this horizon, never shrinks it. */ - readonly terminalRetentionMillis: number + readonly terminalRetention: Duration.Input /** * Whether this deployment runs retention at all. * @@ -48,14 +49,15 @@ export interface Options { const sweep = ( store: RelayInboxStore.RelayInboxStore["Service"], - options: Options + options: Options, + terminalRetentionMillis: number ) => Effect.gen(function*() { const now = yield* Clock.currentTimeMillis const expired = yield* store.expire({ now, limit: options.batchLimit, - terminalRetentionMillis: options.terminalRetentionMillis + terminalRetentionMillis }) const collected = yield* store.collect({ now, limit: options.batchLimit }) if (expired > 0 || collected > 0) { @@ -90,8 +92,10 @@ export const layer = ( Effect.gen(function*() { // Resolved once here rather than per sweep, so the loop carries no requirement of its own. const store = yield* RelayInboxStore.RelayInboxStore - yield* sweep(store, options).pipe( - Effect.repeat(Schedule.spaced(options.intervalMillis)) + // Converted once, alongside the store: the column it feeds is a millisecond timestamp. + const terminalRetentionMillis = Duration.toMillis(options.terminalRetention) + yield* sweep(store, options, terminalRetentionMillis).pipe( + Effect.repeat(Schedule.spaced(options.interval)) ) }) ) diff --git a/packages/local-rpc/src/RelayServer.ts b/packages/local-rpc/src/RelayServer.ts index de2aba7..8097cdd 100644 --- a/packages/local-rpc/src/RelayServer.ts +++ b/packages/local-rpc/src/RelayServer.ts @@ -3,6 +3,7 @@ import * as Canonical from "@lucas-barake/effect-local/Canonical" import * as Identity from "@lucas-barake/effect-local/Identity" import * as Clock from "effect/Clock" import * as Crypto from "effect/Crypto" +import * as Duration from "effect/Duration" import * as Effect from "effect/Effect" import * as Layer from "effect/Layer" import * as Schedule from "effect/Schedule" @@ -43,14 +44,14 @@ export interface Options { * Driven from here rather than by the client, because what has to be proven alive is this node * and its socket, not the peer at the other end. */ - readonly heartbeatIntervalMillis: number + readonly heartbeatInterval: Duration.Input /** * How long to wait on an entity call before giving up. * * Sharding retries `EntityNotAssignedToRunner` and `RunnerUnavailable` indefinitely, so without a * bound a request during a rebalance would hang forever and leak a fiber per in-flight call. */ - readonly entityCallTimeoutMillis: number + readonly entityCallTimeout: Duration.Input } /** Everything the handshake established, held for the life of one client connection. */ @@ -80,6 +81,13 @@ export const layerHandlers = (options: Options) => const sessions = new Map() const decodeEnvelope = Schema.decodeUnknownEffect(RelayEnvelopeJson) + // The wire negotiates its windows in milliseconds, so the configured durations are converted + // once here rather than on every handshake. + const maximumSenderRetryHorizonMillis = Duration.toMillis(limits.maximumSenderRetryHorizon) + const maximumReceiptRetentionMillis = Duration.toMillis(limits.maximumReceiptRetention) + const messageTtlMillis = Duration.toMillis(limits.messageTtl) + const minimumTerminalRetentionMillis = Duration.toMillis(limits.minimumTerminalRetention) + /** * Bounds an entity call and reports exhaustion as unavailability. * @@ -89,7 +97,7 @@ export const layerHandlers = (options: Options) => const bounded = (effect: Effect.Effect) => effect.pipe( Effect.timeoutOrElse({ - duration: options.entityCallTimeoutMillis, + duration: options.entityCallTimeout, orElse: () => Effect.fail(new PeerRpcError.ServerUnavailable()) }) ) @@ -227,11 +235,11 @@ export const layerHandlers = (options: Options) => // The negotiated windows have to nest: a receipt must outlive both the message and the // window in which its sender may replay it, or a redelivery could be applied twice. if ( - payload.senderRetryHorizonMillis > limits.maximumSenderRetryHorizonMillis || - payload.receiptRetentionMillis > limits.maximumReceiptRetentionMillis || + payload.senderRetryHorizonMillis > maximumSenderRetryHorizonMillis || + payload.receiptRetentionMillis > maximumReceiptRetentionMillis || payload.receiptRetentionMillis < - Math.max(limits.messageTtlMillis, payload.senderRetryHorizonMillis) + - limits.minimumTerminalRetentionMillis + Math.max(messageTtlMillis, payload.senderRetryHorizonMillis) + + minimumTerminalRetentionMillis ) { return yield* new PeerRpcError.InvalidRequest() } @@ -344,7 +352,7 @@ export const layerHandlers = (options: Options) => // Bounded per beat rather than per session: a heartbeat that hung would stall the // repeat itself, and the session it exists to keep alive would then lapse. yield* Effect.ignore(bounded(inboxClient(inboxKeySelf).Heartbeat({ sessionId }))).pipe( - Effect.repeat(Schedule.spaced(options.heartbeatIntervalMillis)), + Effect.repeat(Schedule.spaced(options.heartbeatInterval)), Effect.forkScoped ) @@ -571,7 +579,7 @@ export const layer = ( * Retention for every inbox in the deployment. * * Required rather than optional, and composed here rather than left to the consumer, because - * `messageTtlMillis` and `terminalRetentionMillis` are inert without it: a deployment that + * `messageTtl` and `terminalRetention` are inert without it: a deployment that * forgot to add the singleton separately would expire nothing and collect nothing, and would * only find out when the table had grown past the point where admission still succeeded. */ @@ -588,13 +596,14 @@ export const layer = ( // particular configuration, so it is refused at construction rather than discovered in // production. The factor of two leaves room for one lost heartbeat. Layer.provide(Layer.effectDiscard( - options.heartbeatIntervalMillis * 2 < options.inbox.sessionDeadlineMillis && - options.entityCallTimeoutMillis > 0 + Duration.toMillis(options.heartbeatInterval) * 2 < + Duration.toMillis(options.inbox.sessionDeadline) && + Duration.toMillis(options.entityCallTimeout) > 0 ? Effect.void : Effect.die( new Error( - "RelayServer: heartbeatIntervalMillis * 2 must be less than " + - "inbox.sessionDeadlineMillis, and entityCallTimeoutMillis must be positive" + "RelayServer: heartbeatInterval * 2 must be less than " + + "inbox.sessionDeadline, and entityCallTimeout must be positive" ) ) )) diff --git a/packages/local-rpc/test/PeerAuthentication.test.ts b/packages/local-rpc/test/PeerAuthentication.test.ts index e584ad7..55a3872 100644 --- a/packages/local-rpc/test/PeerAuthentication.test.ts +++ b/packages/local-rpc/test/PeerAuthentication.test.ts @@ -403,7 +403,7 @@ describe("PeerAuthentication", () => { ...PeerRelayLimits.defaults, authenticationRatePerSecond: Number.MIN_VALUE, authenticationBurst: 1, - rateLimitIdleRetentionMillis: 1_000, + rateLimitIdleRetention: 1_000, maxRetainedRateLimitedConnections: 8 }) ) diff --git a/packages/local-rpc/test/PeerRelayLimits.test.ts b/packages/local-rpc/test/PeerRelayLimits.test.ts index 9a77c0d..c697c66 100644 --- a/packages/local-rpc/test/PeerRelayLimits.test.ts +++ b/packages/local-rpc/test/PeerRelayLimits.test.ts @@ -1,7 +1,7 @@ import { assert, describe, it } from "@effect/vitest" +import * as Duration from "effect/Duration" import * as Effect from "effect/Effect" import * as PeerRelayLimits from "../src/PeerRelayLimits.js" -import * as PeerRpc from "../src/PeerRpc.js" describe("PeerRelayLimits", () => { it.effect("publishes the validated production defaults through its Layer", () => @@ -16,22 +16,25 @@ describe("PeerRelayLimits", () => { "maxInFlightAuthentication", "maxRetainedRateLimitedConnections", "maxSessionsPerSubject", - "maximumReceiptRetentionMillis", - "maximumSenderRetryHorizonMillis", - "messageTtlMillis", - "minimumTerminalRetentionMillis", - "rateLimitIdleRetentionMillis" + "maximumReceiptRetention", + "maximumSenderRetryHorizon", + "messageTtl", + "minimumTerminalRetention", + "rateLimitIdleRetention" ]) }).pipe(Effect.provide(PeerRelayLimits.layerDefaults))) it.effect.each( [ + // A duration has to be finite and positive to bound anything at all. + ["messageTtl", 0], + ["messageTtl", Duration.infinity], + ["rateLimitIdleRetention", -1], + ["rateLimitIdleRetention", "not a duration"], + // Negotiated windows also have to fit in what the wire contract can carry. + ["maximumReceiptRetention", Duration.days(91)], ["maxSessionsPerSubject", 0], - ["messageTtlMillis", 1.5], - ["authenticationRatePerSecond", Number.NaN], - ["authenticationRatePerSecond", Number.POSITIVE_INFINITY], - ["maximumReceiptRetentionMillis", PeerRpc.maximumNegotiatedDurationMillis + 1], - ["rateLimitIdleRetentionMillis", Number.MAX_SAFE_INTEGER + 1] + ["authenticationRatePerSecond", Number.NaN] ] as const )("rejects scalar %s with the stable configuration error", ([field, value]) => Effect.gen(function*() { @@ -47,8 +50,8 @@ describe("PeerRelayLimits", () => { [ // A receipt that lapses inside its sender's replay window lets the replay land after the // deduplication record is gone, and the recipient applies the message twice. - ["maximumReceiptRetentionMillis", { minimumTerminalRetentionMillis: 7 * 24 * 60 * 60 * 1_000 }], - ["maximumReceiptRetentionMillis", { maximumSenderRetryHorizonMillis: 8 * 24 * 60 * 60 * 1_000 }], + ["maximumReceiptRetention", { minimumTerminalRetention: Duration.days(7) }], + ["maximumReceiptRetention", { maximumSenderRetryHorizon: Duration.days(8) }], // A burst under the concurrency cap makes the rate limiter, not the pool, the binding // constraint — so the pool is sized for a concurrency it is never allowed to reach. ["authenticationBurst", { authenticationBurst: 1 }] @@ -72,11 +75,27 @@ describe("PeerRelayLimits", () => { // retention to exactly cover the horizon plus the required slack is correctly configured. const values = yield* PeerRelayLimits.make({ ...PeerRelayLimits.defaults, - messageTtlMillis: 1_000, - maximumSenderRetryHorizonMillis: 2_000, - minimumTerminalRetentionMillis: 500, - maximumReceiptRetentionMillis: 2_500 + messageTtl: Duration.seconds(1), + maximumSenderRetryHorizon: Duration.seconds(2), + minimumTerminalRetention: Duration.millis(500), + maximumReceiptRetention: Duration.millis(2_500) }) - assert.strictEqual(values.maximumReceiptRetentionMillis, 2_500) + assert.strictEqual(Duration.toMillis(values.maximumReceiptRetention), 2_500) + })) + + it.effect("accepts every shape Duration.Input allows for one value", () => + Effect.gen(function*() { + // The point of taking `Duration.Input` rather than a number of milliseconds: a deployment + // writes whichever form reads best, and none of them is a different amount of time. + for (const horizon of [Duration.days(7), "7 days", 7 * 24 * 60 * 60 * 1_000] as const) { + const values = yield* PeerRelayLimits.make({ + ...PeerRelayLimits.defaults, + maximumSenderRetryHorizon: horizon + }) + assert.strictEqual( + Duration.toMillis(values.maximumSenderRetryHorizon), + 7 * 24 * 60 * 60 * 1_000 + ) + } })) }) diff --git a/packages/local-rpc/test/RelayInbox.test.ts b/packages/local-rpc/test/RelayInbox.test.ts index 4f7ffce..20d7b93 100644 --- a/packages/local-rpc/test/RelayInbox.test.ts +++ b/packages/local-rpc/test/RelayInbox.test.ts @@ -4,6 +4,7 @@ import { assert, describe, it } from "@effect/vitest" import * as Identity from "@lucas-barake/effect-local/Identity" import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" import * as Cause from "effect/Cause" +import * as Duration from "effect/Duration" import * as Effect from "effect/Effect" import * as Exit from "effect/Exit" import * as Fiber from "effect/Fiber" @@ -31,16 +32,16 @@ const inboxKey = "inbox-a" const baseOptions: RelayInbox.Options = { maxDeliveries: 10, - messageTtlMillis: 600_000, - terminalRetentionMillis: 600_000, - sessionDeadlineMillis: 90_000, - sessionSweepMillis: 1_000, + messageTtl: Duration.minutes(10), + terminalRetention: Duration.minutes(10), + sessionDeadline: Duration.seconds(90), + sessionSweep: Duration.seconds(1), maxConcurrentChannels: 4, - storeRetryMillis: 0, + storeRetry: Duration.zero, maxPendingMessages: 100, maxPendingBytes: 10_000_000, mailboxCapacity: 16, - maxIdleTimeMillis: 3_600_000 + maxIdleTime: Duration.hours(1) } /** @@ -87,6 +88,10 @@ const relay = ( const layer = relay() +// The sweeper's own arithmetic is in milliseconds; the options state durations. +const sessionDeadlineMillis = Duration.toMillis(baseOptions.sessionDeadline) +const sessionSweepMillis = Duration.toMillis(baseOptions.sessionSweep) + /** * Replaces one store operation with a failure, leaving the rest of the real store intact. * @@ -317,7 +322,7 @@ describe("RelayInbox", () => { }).pipe(Effect.catch(() => Effect.void), Effect.exit) ) yield* TestClock.adjust(10) - yield* TestClock.adjust(baseOptions.sessionDeadlineMillis + baseOptions.sessionSweepMillis * 3) + yield* TestClock.adjust(sessionDeadlineMillis + sessionSweepMillis * 3) const outcome = yield* Fiber.join(settling) assert.strictEqual( @@ -454,7 +459,7 @@ describe("RelayInbox", () => { // Nothing heartbeats it, so the entity's own deadline is what bounds it. A cluster cannot // rely on disconnects announcing themselves. - yield* TestClock.adjust(baseOptions.sessionDeadlineMillis + baseOptions.sessionSweepMillis * 2) + yield* TestClock.adjust(sessionDeadlineMillis + sessionSweepMillis * 2) const collected = yield* Fiber.join(subscriber) assert.strictEqual(collected.length, 0, "the released session's stream ends") @@ -671,7 +676,7 @@ describe("RelayInbox", () => { outcome: "Acknowledged", messageHash: "000000000001".padStart(64, "a"), now: 1_000, - terminalRetentionMillis: baseOptions.terminalRetentionMillis + terminalRetentionMillis: Duration.toMillis(baseOptions.terminalRetention) }) const promoted = (yield* store.pendingHeads(inboxKey, { limit: 10 })) .find((message) => message.relayMessageId === relayId("000000000002")) diff --git a/packages/local-rpc/test/RelayInboxMaintenance.test.ts b/packages/local-rpc/test/RelayInboxMaintenance.test.ts index a754588..1da4bec 100644 --- a/packages/local-rpc/test/RelayInboxMaintenance.test.ts +++ b/packages/local-rpc/test/RelayInboxMaintenance.test.ts @@ -3,6 +3,7 @@ import { SqliteClient } from "@effect/sql-sqlite-node" import { assert, describe, it } from "@effect/vitest" import * as Identity from "@lucas-barake/effect-local/Identity" import * as Clock from "effect/Clock" +import * as Duration from "effect/Duration" import * as Effect from "effect/Effect" import * as Layer from "effect/Layer" import { TestClock } from "effect/testing" @@ -23,9 +24,9 @@ const documentId = (value: string) => Identity.DocumentId.make(`doc_00000000-000 const inboxKey = "inbox-a" const options: RelayInboxMaintenance.Options = { - intervalMillis: 1_000, + interval: Duration.seconds(1), batchLimit: 100, - terminalRetentionMillis: 10_000, + terminalRetention: Duration.seconds(10), enabled: true } diff --git a/packages/local-rpc/test/RelayServer.test.ts b/packages/local-rpc/test/RelayServer.test.ts index 2199b6c..67a90cf 100644 --- a/packages/local-rpc/test/RelayServer.test.ts +++ b/packages/local-rpc/test/RelayServer.test.ts @@ -9,6 +9,7 @@ import * as Cause from "effect/Cause" import * as Clock from "effect/Clock" import * as Context from "effect/Context" import * as Crypto from "effect/Crypto" +import * as Duration from "effect/Duration" import * as Effect from "effect/Effect" import * as Fiber from "effect/Fiber" import * as Latch from "effect/Latch" @@ -72,22 +73,22 @@ const outsider = PeerAuthentication.PeerPrincipal.make({ const serverOptions: RelayServer.Options = { tenantId: "tenant", peerId: relayPeerId, - heartbeatIntervalMillis: 30_000, - entityCallTimeoutMillis: 30_000 + heartbeatInterval: Duration.seconds(30), + entityCallTimeout: Duration.seconds(30) } const inboxOptions: RelayInbox.Options = { maxDeliveries: 10, - messageTtlMillis: 600_000, - terminalRetentionMillis: 600_000, - sessionDeadlineMillis: 90_000, - sessionSweepMillis: 1_000, + messageTtl: Duration.minutes(10), + terminalRetention: Duration.minutes(10), + sessionDeadline: Duration.seconds(90), + sessionSweep: Duration.seconds(1), maxConcurrentChannels: 4, - storeRetryMillis: 0, + storeRetry: Duration.zero, maxPendingMessages: 100, maxPendingBytes: 10_000_000, mailboxCapacity: 16, - maxIdleTimeMillis: 3_600_000 + maxIdleTime: Duration.hours(1) } const TestShardingConfig = ShardingConfig.layer({ @@ -112,8 +113,8 @@ const openRequest = ( senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), remote: { subjectId: remote.subjectId, peerId: remote.peerId }, documents, - receiptRetentionMillis: PeerRelayLimits.defaults.maximumReceiptRetentionMillis, - senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis + receiptRetentionMillis: Duration.toMillis(PeerRelayLimits.defaults.maximumReceiptRetention), + senderRetryHorizonMillis: Duration.toMillis(PeerRelayLimits.defaults.maximumSenderRetryHorizon) }) /** @@ -254,9 +255,9 @@ const harness = (options?: { ...options.composed, inbox: inboxOptions, maintenance: { - intervalMillis: 60_000, + interval: Duration.minutes(1), batchLimit: 100, - terminalRetentionMillis: inboxOptions.terminalRetentionMillis, + terminalRetention: inboxOptions.terminalRetention, enabled: true } }) @@ -1120,7 +1121,7 @@ describe("RelayServer", () => { // that stops proving it is alive. Nothing on the wire refreshes it — the client does not know // the session exists — so the relay drives the heartbeat itself, and without it every session // would go dark one deadline after opening no matter how healthy the connection is. - yield* TestClock.adjust(inboxOptions.sessionDeadlineMillis * 4) + yield* TestClock.adjust(Duration.toMillis(inboxOptions.sessionDeadline) * 4) yield* push(peer, senderSession.opened.sessionId, yield* encodePayload(peer)) const delivered = yield* Queue.take(recipientSession.events) @@ -1140,7 +1141,7 @@ describe("RelayServer", () => { const pushing = yield* push(peer, senderSession.opened.sessionId, yield* encodePayload(peer)) .pipe(Effect.flip, Effect.forkScoped) - yield* TestClock.adjust(serverOptions.entityCallTimeoutMillis) + yield* TestClock.adjust(serverOptions.entityCallTimeout) const failure = yield* Fiber.join(pushing) // Retryable: the write may or may not have landed, and the sender still holds its outbox copy. @@ -1169,8 +1170,8 @@ describe("RelayServer", () => { // only under one particular configuration, so it is refused where it is written down. for ( const broken of [ - { heartbeatIntervalMillis: inboxOptions.sessionDeadlineMillis }, - { entityCallTimeoutMillis: 0 } + { heartbeatInterval: inboxOptions.sessionDeadline }, + { entityCallTimeout: 0 } ] ) { const exit = yield* Effect.scoped(composed(broken)).pipe(Effect.exit) diff --git a/packages/local-rpc/test/RelayWireFormat.test.ts b/packages/local-rpc/test/RelayWireFormat.test.ts index c4ecd5a..7b3f17b 100644 --- a/packages/local-rpc/test/RelayWireFormat.test.ts +++ b/packages/local-rpc/test/RelayWireFormat.test.ts @@ -7,6 +7,7 @@ import * as Document from "@lucas-barake/effect-local/Document" import * as Identity from "@lucas-barake/effect-local/Identity" import * as Context from "effect/Context" import * as Crypto from "effect/Crypto" +import * as Duration from "effect/Duration" import * as Effect from "effect/Effect" import * as Layer from "effect/Layer" import * as Queue from "effect/Queue" @@ -70,16 +71,16 @@ const recipient = PeerAuthentication.PeerPrincipal.make({ const inboxOptions: RelayInbox.Options = { maxDeliveries: 10, - messageTtlMillis: 600_000, - terminalRetentionMillis: 600_000, - sessionDeadlineMillis: 90_000, - sessionSweepMillis: 1_000, + messageTtl: Duration.minutes(10), + terminalRetention: Duration.minutes(10), + sessionDeadline: Duration.seconds(90), + sessionSweep: Duration.seconds(1), maxConcurrentChannels: 4, - storeRetryMillis: 0, + storeRetry: Duration.zero, maxPendingMessages: 100, maxPendingBytes: 10_000_000, mailboxCapacity: 16, - maxIdleTimeMillis: 3_600_000 + maxIdleTime: Duration.hours(1) } const openRequest = ( @@ -93,8 +94,8 @@ const openRequest = ( senderReplicaIncarnation: Identity.ReplicaIncarnation.make(1), remote: { subjectId: remote.subjectId, peerId: remote.peerId }, documents: [{ documentType: Task.name, documentId }], - receiptRetentionMillis: PeerRelayLimits.defaults.maximumReceiptRetentionMillis, - senderRetryHorizonMillis: PeerRelayLimits.defaults.maximumSenderRetryHorizonMillis + receiptRetentionMillis: Duration.toMillis(PeerRelayLimits.defaults.maximumReceiptRetention), + senderRetryHorizonMillis: Duration.toMillis(PeerRelayLimits.defaults.maximumSenderRetryHorizon) }) const RelaySyncEnvelopeJson = Schema.fromJsonString( @@ -189,8 +190,8 @@ describe("relay wire format", () => { Layer.provide(RelayServer.layerHandlers({ tenantId: "tenant", peerId: relayPeerId, - heartbeatIntervalMillis: 30_000, - entityCallTimeoutMillis: 30_000 + heartbeatInterval: Duration.seconds(30), + entityCallTimeout: Duration.seconds(30) })), Layer.provide(PeerAuthentication.layerServer), Layer.provide(RpcServer.layerProtocolSocketServer), From d32975e4c7a1247157fcb7183a96d0f5c99f82c6 Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Mon, 27 Jul 2026 11:10:44 -0500 Subject: [PATCH 27/29] Describe the relay that exists rather than the one that was deleted The README and every doc still explained the single-process relay: claim tokens with lease deadlines, session generations, per-scope quota reservations, bounded frame accounting, and payload erasure at the terminal transition. Some of it was runnable-looking sample code importing modules that no longer exist, which is the worst kind of stale doc because a reader finds out by compiling. The composition examples now build RelayServer.layer over a Sharding the application supplies, and say plainly that the retention singleton is required because messageTtl and terminalRetention are inert without it. The deployment sections say active-active is supported and what enforces single ownership, rather than warning it is unsupported. Two behaviours were documented wrongly and are now stated as they are. A delivery is charged only after the message reaches the transport, not on every failed or abandoned attempt. And a terminal row keeps its stored envelope until retention deletes the whole row, rather than erasing the payload at the terminal transition - the envelope has to survive because re-admitting a dead lettered or expired identity revives that row in place, which is what lets a sender that still holds custody recover a message the relay gave up on. Also drops an unnecessary export from the migrations module, found by the same sweep: loader is only ever used one line below its own definition. --- .changeset/initial-release.md | 5 + README.md | 76 +++---- docs/architecture.md | 37 ++-- docs/durability.md | 22 +- docs/store-and-forward.md | 204 ++++++++++++------ docs/sync.md | 11 +- packages/local-rpc/README.md | 22 +- .../src/internal/relayInboxMigrations.ts | 2 +- 8 files changed, 234 insertions(+), 145 deletions(-) diff --git a/.changeset/initial-release.md b/.changeset/initial-release.md index a50bbf9..8844c45 100644 --- a/.changeset/initial-release.md +++ b/.changeset/initial-release.md @@ -7,3 +7,8 @@ --- Initial release of Effect Local and its core, SQL, RPC, browser, and testing packages. + +Local first document synchronization built on Automerge, with SQLite backed replicas on Node and in the browser, and +an authenticated store and forward peer relay over Effect RPC. The relay is a cluster of per device entities, so more +than one relay node can serve one database, and durable custody is injectable behind `RelayInboxStore` with a +`SqlClient` implementation for SQLite, PostgreSQL, and MySQL. diff --git a/README.md b/README.md index c91a6c4..ca52ea5 100644 --- a/README.md +++ b/README.md @@ -181,8 +181,8 @@ remote storage durability. Effect Local therefore keeps durable confirmation sep false. Effect RPC response acknowledgements bound streamed response chunks. They are not byte credits, durable receipts, or -proof that Automerge applied a message. `PeerRpcServer` adds item limits, byte reservations, quotas, rate limits, and -admission concurrency above the RPC stream. WebSocket ordering applies only within one live connection. +proof that Automerge applied a message. `RelayServer` adds per inbox admission quotas, authentication rate limits, +and a per subject session cap above the RPC stream. WebSocket ordering applies only within one live connection. The browser topology also assigns distinct roles: @@ -232,7 +232,7 @@ always recover by reading the replica again because Atom is not part of the pers | Retry a command after losing the response | The same command ID, then the matching lookup method | | Coordinate several durable steps | Effect Workflow | | Exchange changes with another replica | `PeerSession` and an application supplied transport | -| Host a live authenticated replica peer | `PeerRpcServer` plus application owned Effect RPC | +| Host a live authenticated replica peer | `RelayServer` plus a cluster and application owned Effect RPC | | Connect through the generated RPC client | `RpcPeerTransport.makeSession` | | Bound a high churn document's checkpoint | `ReplicaWorkflow.HistoryRewriteWorkflow` | | Add asynchronous relay custody | The opt in [store and forward](docs/store-and-forward.md) composition | @@ -344,10 +344,12 @@ The RPC package itself is platform neutral. A network deployment requires an Eff `RpcClient.Protocol`, the same `RpcSerialization` on both ends, a platform `Socket`, an HTTP server, TLS, WebSocket upgrade routing, Origin policy, ingress byte and connection limits, credential issuance, secret rotation, tenant routing, process supervision, and graceful shutdown. None of those responsibilities are inferred from -`PeerRpcServer.layerHandlers`. +`RelayServer.layerHandlers`. -The durable relay uses the bounded `PeerRelayIngress` protocol. The application owns TLS, routing, identity, policy, -process supervision, and the SQLite, PostgreSQL, or MySQL deployment that holds one relay shard. +The durable relay speaks standard Effect RPC over a socket with `RpcSerialization`. The application also owns the +cluster: `RelayServer` requires a `Sharding` and never builds one, so single process, one runner over SQL, and many +sharded runners are all its choice. The application owns TLS, routing, identity, policy, process supervision, and the +SQLite, PostgreSQL, or MySQL deployment behind the relay. ## Composition recipes @@ -994,13 +996,17 @@ reclaims expired entries. The RPC package exposes one protocol, `PeerRpc` version `1`. Its procedures are `Open`, `Push`, `Acknowledge`, and `Reject`. Every connection uses durable custody. There is no direct in memory RPC topology. -Applications provide `PeerAuthenticator`, `PeerRelayAuthorization`, `PeerRelayLimits`, -`PeerRelayIngress`, and `PeerRelayStore`. The semantic store is injectable. The built in -`SqlPeerRelayStore.layer` requires a generic `SqlClient` and supports SQLite, PostgreSQL, and MySQL. The frontend -replica, sender outbox, and recipient receipts remain SQLite. +Applications provide `PeerAuthenticator`, `PeerRelayAuthorization`, `PeerRelayLimits`, a socket, and a `Sharding`. +The semantic store is injectable through `RelayInboxStore`; the built in `SqlRelayInboxStore.layer` requires a +generic `SqlClient` and supports SQLite, PostgreSQL, and MySQL. The frontend replica, sender outbox, and recipient +receipts remain SQLite. -`PeerRpcServer.layerHandlers({ tenantId, peerId })` builds the handlers. `PeerRpcServer.layerServer` supervises the -RPC server and bounded ingress. A successful `Push` means the backend store committed custody before replying. +The relay is a cluster of `RelayInbox` entities, one per recipient device, each the sole writer for its device's +inbox. `RelayServer.layerHandlers(options)` builds the front door alone, which is what composes into an existing RPC +server. `RelayServer.layer(options)` is the whole thing: front door, entity behaviour, and the retention singleton +that makes `messageTtl` and `terminalRetention` actually run. Because ownership comes from sharding rather than from +process local state, several relay nodes can serve one database. A successful `Push` means the owning entity +committed custody before replying. ### 13. Connect an RPC peer @@ -1460,12 +1466,6 @@ Run commands from the repository root. | `pnpm docgen` | Compile documentation through the root TypeScript graph | | `pnpm clean` | Remove generated build artifacts | -`pnpm bench` excludes `PeerRpcServerPerformance.bench.ts`. That harness imports an instrumentation module created only -by `base-admission-instrumentation.patch` or `candidate-admission-instrumentation.patch` inside detached benchmark -worktrees. It must never run against or add hooks to the shipped source. After applying the matching patch, run it with -`pnpm exec vitest bench --run --config packages/local-rpc/bench/vitest.admission.config.ts`. Base and candidate must use -the same harness, dependency lock, sample count, and machine. - ## Public API inventory Every root package exports module namespaces. Every module is also available through its public subpath, such as @@ -1474,15 +1474,15 @@ Every root package exports module namespaces. Every module is also available thr The export surface supports several assembly levels. Most applications start with everyday domain and composition modules. Feature and advanced services remain public for products that need direct control. -| Level | Modules | -| --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Everyday domain | `Document`, `DocumentSet`, `Mutation`, `Projection`, `Query`, `ReplicaDefinition`, `Replica`, `CommandOutcome`, `Snapshot`, `Identity`, `ReplicaStatus`, `Backup` | -| Durable runtime | `SqlProjection`, `SqlReplica`, `BrowserReplica`, `BrowserSqlite` | -| Reactive and test adapters | `ReplicaAtom`, `TestReplica` | -| Optional features | `PeerTransport`, `PeerSession`, `Presence`, `ReplicaWorkflow`, `TestPeer`, `FaultInjection`, `PeerRpc`, `RpcPeerTransport` | -| RPC policy and server | `PeerCredentials`, `PeerAuthenticator`, `PeerAuthentication`, `PeerRelayAuthorization`, `PeerRelayLimits`, `PeerRelayIngress`, `PeerRelayStore`, `SqlPeerRelayStore`, `PeerRpcServer`, `PeerRpcError` | -| Advanced assembly | `CommitPublisher`, `Compaction`, `Recovery`, `PeerSync`, `DurableRuntime`, `ReplicaClient`, `ReplicaOwner`, `SessionManager` | -| Advanced framework assembly | `CommandExecutor`, `DocumentEntity`, `DocumentStore`, `EntityReplica`, `Migrations`, `ProjectionStore`, `QueryExecutor`, `ReplicaBootstrap`, `ReplicaGate`, `ReplicaRpc` | +| Level | Modules | +| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Everyday domain | `Document`, `DocumentSet`, `Mutation`, `Projection`, `Query`, `ReplicaDefinition`, `Replica`, `CommandOutcome`, `Snapshot`, `Identity`, `ReplicaStatus`, `Backup` | +| Durable runtime | `SqlProjection`, `SqlReplica`, `BrowserReplica`, `BrowserSqlite` | +| Reactive and test adapters | `ReplicaAtom`, `TestReplica` | +| Optional features | `PeerTransport`, `PeerSession`, `Presence`, `ReplicaWorkflow`, `TestPeer`, `FaultInjection`, `PeerRpc`, `RpcPeerTransport` | +| RPC policy and server | `PeerCredentials`, `PeerAuthenticator`, `PeerAuthentication`, `PeerRelayAuthorization`, `PeerRelayLimits`, `RelayInboxStore`, `SqlRelayInboxStore`, `RelayInbox`, `RelayInboxMaintenance`, `RelayServer`, `PeerRpcError` | +| Advanced assembly | `CommitPublisher`, `Compaction`, `Recovery`, `PeerSync`, `DurableRuntime`, `ReplicaClient`, `ReplicaOwner`, `SessionManager` | +| Advanced framework assembly | `CommandExecutor`, `DocumentEntity`, `DocumentStore`, `EntityReplica`, `Migrations`, `ProjectionStore`, `QueryExecutor`, `ReplicaBootstrap`, `ReplicaGate`, `ReplicaRpc` | These public framework assembly modules support custom runtimes and diagnostics. Paths under `internal/*` remain private and unsupported. The public assembly modules are not required for the normal application path shown in the @@ -1786,23 +1786,25 @@ planning: | `PeerAuthenticator` | Application credential verification service | | `PeerCredentials` | Client credential provider | | `PeerRelayAuthorization` | Exact send and receive authorization plus the explicit Automerge resource trust grant | -| `PeerRelayIngress` | Bounded server and client socket protocol | -| `PeerRelayLimits` | Authentication, ingress, custody, session, worker, quota, retry, and maintenance limits | +| `PeerRelayLimits` | Negotiation windows, authentication rate limits, and the per subject session cap | | `PeerRpc` | Durable protocol version `1`, `Open`, `Push`, `Acknowledge`, `Reject`, events, group, and generated client | -| `PeerRelayStore` | Injectable semantic custody contract | -| `SqlPeerRelayStore` | Generic `SqlClient` implementation with `make` and `layer` for SQLite, PostgreSQL, and MySQL | -| `PeerRpcServer` | `layerHandlers`, `layerServer`, and `PeerRpcServerRuntime` | +| `RelayInboxStore` | Injectable semantic custody contract for one device's inbox | +| `SqlRelayInboxStore` | Generic `SqlClient` implementation with `make` and `layer` for SQLite, PostgreSQL, and MySQL | +| `RelayInbox` | The cluster entity that owns one device's inbox, with `layer` and `Options` | +| `RelayInboxMaintenance` | Deployment wide expiry and collection, as a cluster singleton | +| `RelayServer` | `layerHandlers` for the front door alone, `layer` for the front door plus entity plus retention | | `RpcPeerTransport` | Durable `Options`, `layer`, `makeSession`, and `isRetryable` | The `Open` stream starts with `Opened` and then emits `StoredMessage` values. `Push` succeeds only after durable -custody. `Acknowledge` and `Reject` use the exact claim token and message hash. Every procedure can fail with a -fieldless `PeerRpcError`. +custody. `Acknowledge` and `Reject` use the claim token minted for that delivery attempt plus the message hash. Every +procedure can fail with a fieldless `PeerRpcError`. `PeerAuthentication.layerClient` requires `PeerCredentials`. `PeerAuthentication.layerServer` requires `PeerAuthenticator` and `PeerRelayLimits`. -`PeerRpcServer.layerHandlers` requires `Crypto`, `PeerRelayAuthorization`, `PeerRelayIngress`, -`PeerRelayLimits`, and `PeerRelayStore`. `SqlPeerRelayStore.layer` requires `SqlClient`, `Crypto`, and -`PeerRelayLimits`. `RpcPeerTransport.layer` additionally uses `PeerRelayClientRuntime`. +`RelayServer.layerHandlers` requires `Crypto`, `PeerRelayAuthorization`, `PeerRelayLimits`, and `Sharding`. +`RelayServer.layer` additionally registers the entity behaviour and the retention singleton, so it also requires +`RelayInboxStore`. `SqlRelayInboxStore.layer` requires `SqlClient` and `Crypto`. `RpcPeerTransport.layer` +additionally uses `PeerRelayClientRuntime`. ### Research basis diff --git a/docs/architecture.md b/docs/architecture.md index e15f933..5599a19 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -47,31 +47,36 @@ operations after process loss. Automerge remains responsible for causal history, ## Relay topology -Effect RPC uses one durable protocol. Version `1` uses `PeerRpc.Rpcs`, the bounded `PeerRelayIngress` socket -protocol, `PeerRpcServer.layerHandlers`, and `PeerRpcServer.layerServer`. +Effect RPC uses one durable protocol. Version `1` uses `PeerRpc.Rpcs` over a standard Effect RPC socket with +`RpcSerialization`, `RelayServer.layerHandlers` for the front door, and `RelayServer.layer` to compose the front door +with the entity behaviour and its retention singleton. ```mermaid flowchart LR Sender["Sender SQL replica"] --> Outbox["Stable sender relay outbox"] - Outbox --> RelayListener["Relay protocol v1 listener"] - RelayListener --> Custody["Injected backend SQL custody"] - Custody --> RecipientListener["Recipient relay session"] - RecipientListener --> Receipt["Recipient SQL sync and receipt"] - Receipt --> Ack["Fenced relay acknowledgement"] - Ack --> Custody + Outbox --> FrontDoor["RelayServer front door (any node)"] + FrontDoor --> Entity["RelayInbox entity (single owner per device)"] + Entity --> Custody["Injected backend SQL custody"] + Entity --> RecipientSession["Recipient delivery session"] + RecipientSession --> Receipt["Recipient SQL sync and receipt"] + Receipt --> Ack["Relay acknowledgement"] + Ack --> Entity ``` -The application must route every custody write and claim for one shard to the same logical SQL database. The supplied -`SqlClient` may target SQLite, PostgreSQL, or MySQL. This topology does not itself add leader election, quorum -replication, automatic failover, shard rebalance, or split brain recovery beyond the selected database. +Endpoint ownership, routing, and cross node wakeups come from cluster sharding rather than from process local state, +so several relay nodes may serve one database as active active. The entity that owns a device is the sole writer for +that device's inbox, which is what makes custody need no distributed protocol of its own. The supplied `SqlClient` may +target SQLite, PostgreSQL, or MySQL. The relay requires a `Sharding` and never builds one: single process in memory, +one runner over SQL, and many sharded runners are all the application's choice. -Relay ordering is FIFO within one exact directed peer channel. Distinct channels can progress concurrently. Claims -are finite fences. A stale claim may duplicate delivery, but its old token cannot retire a newer claim. See +Relay ordering is FIFO within one exact directed peer channel, where a channel includes the sender's connection +epoch. Distinct channels progress concurrently, so one stalled sender cannot block another. Delivery is stop and wait +within a channel: nothing is offered behind an unsettled message. See [Store and forward](store-and-forward.md) for delivery, capacity, security, and lifecycle details. -Relay infrastructure treats the Automerge message as opaque. It validates bounded framing, envelope Schema, endpoint -and document routing, hashes, outer digest, and writer provenance shape. Relay enabled `PeerSync` performs the one -semantic Automerge decode. +Relay infrastructure treats the Automerge message as opaque. It validates the envelope Schema, endpoint and document +routing, hashes, outer digest, and writer provenance shape. Relay enabled `PeerSync` performs the one semantic +Automerge decode. That boundary is explicit because Automerge `3.3.2` has no allocation bounded decode API. Ordinary authentication and document authorization do not imply resource trust. `PeerRelayAuthorization` requires a second exact diff --git a/docs/durability.md b/docs/durability.md index 10fc118..0d15249 100644 --- a/docs/durability.md +++ b/docs/durability.md @@ -154,20 +154,20 @@ share an atomic transaction. The extra grant exists because Automerge `3.3.2` do semantic decode. Authentication and document access alone do not satisfy it. These boundaries provide at least once transfer. They do not provide exactly once delivery. A response can be lost -after any commit. A claim can expire while an old recipient worker still runs. The durable identities and claim -tokens make retries and stale acknowledgements safe, while retained recipient receipts make duplicate application -idempotent during the negotiated window. +after any commit. A delivery attempt can be abandoned while its message is still `Pending`. The durable identities +and per attempt claim tokens make retries and stale acknowledgements safe, while retained recipient receipts make +duplicate application idempotent during the negotiated window. -Relay FIFO ordering is scoped to one exact directed peer channel. One claimed head blocks later rows in that channel. -Unrelated channels do not share that ordering fence. +Relay FIFO ordering is scoped to one exact directed peer channel, which includes the sender's connection epoch. One +in flight head blocks later rows in that channel. Unrelated channels do not share that ordering fence. -Recipient delivery attempts are durable and bounded by `RelayInbox.Options.maxDeliveries`, which is set to -`16`. Failed, interrupted, disconnected, and expired claims advance the count. Reaching the cap changes the message -to `DeadLettered` and erases the payload. Restart cannot reset this poison bound. +Recipient delivery attempts are durable and bounded by `RelayInbox.Options.maxDeliveries`. An attempt is charged only +once the message has reached the transport, so a delivery abandoned by a disconnect costs nothing. Reaching the cap +changes the message to `DeadLettered` so it stops blocking its channel. Restart cannot reset this poison bound. -Active payloads expire at `PeerRelayLimits.messageTtlMillis`. Acknowledged, rejected, and expired rows erase their -payload and retain bounded deduplication evidence until their retention deadline. Sender outbox rows expire at their -configured retry horizon. Recipient receipts expire according to `PeerRelayReceiptLimits`. +Undelivered messages expire at `RelayInbox.Options.messageTtl`. A terminal row keeps its stored envelope until +retention collects the whole row past its deduplication horizon, which is what lets a sender that still holds custody +revive a dead lettered or expired identity. Sender outbox rows expire at their configured retry horizon. Recipient receipts expire according to `PeerRelayReceiptLimits`. Relay custody rows and client relay outbox or receipt rows are not canonical document history and are not included in canonical backup archives. Restore fences the previous incarnation, removes stale relay client state, and reconciles diff --git a/docs/store-and-forward.md b/docs/store-and-forward.md index 0f05b53..5f8daee 100644 --- a/docs/store-and-forward.md +++ b/docs/store-and-forward.md @@ -16,45 +16,77 @@ its local writes. Automerge remains responsible for convergence after valid chan ## Composition -The application supplies relay policy, custody, limits, authentication, SQL, and a socket. The server uses -`PeerRpcServer.layerHandlers` with `PeerRpc.Rpcs`, plus `PeerRpcServer.layerServer` and `PeerRelayIngress` for the -bounded listener. +The application supplies relay policy, custody, limits, authentication, SQL, a socket, and a cluster. The server +composes `RelayServer.layer`, which is the front door handlers plus the `RelayInbox` entity behaviour plus the +retention singleton. The relay requires a `Sharding` and never builds one, so the deployment shape — single process +in memory, one runner over SQL, or many sharded runners — stays the application's choice. On the client, `SqlReplica.layerRelay` installs relay receipt support. `PeerRelayOutbox.layerSql` stores stable sender admissions. `PeerRelayClientRuntime.layer` supervises sender outbox and receipt maintenance. `RpcPeerTransport.makeSession` binds the generated relay client to the ordinary `PeerSession`. This example shows the server and client composition. The named application values provide SQL, Crypto, -authentication, socket, definition, projection, and mutation or query Layers. +authentication, socket, cluster, definition, projection, and mutation or query Layers. ```ts +import * as PeerAuthentication from "@lucas-barake/effect-local-rpc/PeerAuthentication" import * as PeerRelayAuthorization from "@lucas-barake/effect-local-rpc/PeerRelayAuthorization" import * as PeerRelayLimits from "@lucas-barake/effect-local-rpc/PeerRelayLimits" -import * as PeerRpcServer from "@lucas-barake/effect-local-rpc/PeerRpcServer" +import * as PeerRpc from "@lucas-barake/effect-local-rpc/PeerRpc" +import * as RelayServer from "@lucas-barake/effect-local-rpc/RelayServer" import * as RpcPeerTransport from "@lucas-barake/effect-local-rpc/RpcPeerTransport" -import * as SqlPeerRelayStore from "@lucas-barake/effect-local-rpc/SqlPeerRelayStore" +import * as SqlRelayInboxStore from "@lucas-barake/effect-local-rpc/SqlRelayInboxStore" import * as PeerRelayClientRuntime from "@lucas-barake/effect-local-sql/PeerRelayClientRuntime" import * as PeerRelayOutbox from "@lucas-barake/effect-local-sql/PeerRelayOutbox" import * as PeerRelayOutboxLimits from "@lucas-barake/effect-local-sql/PeerRelayOutboxLimits" import * as PeerRelayReceiptLimits from "@lucas-barake/effect-local-sql/PeerRelayReceiptLimits" import * as SqlReplica from "@lucas-barake/effect-local-sql/SqlReplica" -import { Effect, Layer } from "effect" +import { Duration, Effect, Layer } from "effect" +import { RpcSerialization, RpcServer } from "effect/unstable/rpc" -const RelayLimitsLive = PeerRelayLimits.layer(relayLimits) -const RelayStoreLive = SqlPeerRelayStore.layer.pipe( - Layer.provideMerge(RelayLimitsLive) -) -const RelayAuthorizationLive = PeerRelayAuthorization.layer( - authorizeRelay, - PeerRelayAuthorization.denyUnsafeUnboundedAutomerge3Decode -) -const RelayHandlersLive = PeerRpcServer.layerHandlers({ +const RelayLive = RelayServer.layer({ tenantId, - peerId: relayPeerId + peerId: relayPeerId, + heartbeatInterval: Duration.seconds(30), + entityCallTimeout: Duration.seconds(30), + inbox: { + maxDeliveries: 16, + messageTtl: Duration.days(7), + terminalRetention: Duration.days(8), + sessionDeadline: Duration.seconds(90), + sessionSweep: Duration.seconds(5), + maxConcurrentChannels: 8, + storeRetry: Duration.seconds(1), + maxPendingMessages: 10_000, + maxPendingBytes: 256 * 1_024 * 1_024, + mailboxCapacity: 64, + maxIdleTime: Duration.minutes(30) + }, + // Required, not optional. Without it messageTtl and terminalRetention are inert: nothing expires, + // nothing is collected, and the retained row count climbs until admission is refused. + maintenance: { + interval: Duration.minutes(1), + batchLimit: 500, + terminalRetention: Duration.days(8), + enabled: true + } }).pipe( - Layer.provideMerge(RelayLimitsLive), - Layer.provideMerge(RelayStoreLive), - Layer.provideMerge(RelayAuthorizationLive) + Layer.provide(SqlRelayInboxStore.layer), + Layer.provide(PeerRelayLimits.layer(relayLimits)), + Layer.provide(PeerRelayAuthorization.layer( + authorizeRelay, + PeerRelayAuthorization.denyUnsafeUnboundedAutomerge3Decode + )), + // Supplied by the application: SingleRunner for one node, SocketRunner for many. + Layer.provide(shardingLive) +) + +const ServerLive = RpcServer.layer(PeerRpc.Rpcs).pipe( + Layer.provide(RelayLive), + Layer.provide(PeerAuthentication.layerServer), + Layer.provide(RpcServer.layerProtocolSocketServer), + Layer.provide(RpcSerialization.layerJson), + Layer.provide(socketServerLive) ) const ReceiptLimitsLive = PeerRelayReceiptLimits.layer(receiptLimits) @@ -83,10 +115,19 @@ const session = RpcPeerTransport.makeSession(relayRpcClient, { ``` `relayRpcClient` must be a `PeerRpc.RpcClient` created with `PeerRpc.makeRpcClient` over -`PeerRelayIngress.layerProtocolSocket`. The server +`RpcClient.layerProtocolSocket()` with a matching `RpcSerialization` — the same codec the server uses. The server composition still requires the application SQL and Crypto Layers, `PeerAuthentication.layerServer`, the relay -authorization Layer, and a `PeerRelayLimits` Layer. The client composition still requires the application SQL, -Crypto, core replica limits, generated client authentication middleware, and a scoped relay socket. +authorization Layer, a `PeerRelayLimits` Layer, and a `Sharding`. The client composition still requires the +application SQL, Crypto, core replica limits, generated client authentication middleware, and a scoped relay socket. + +`RelayServer.layer` refuses to build when `heartbeatInterval * 2` is not less than `inbox.sessionDeadline`, or when +`entityCallTimeout` is not positive. Those two values have to agree or every session is reaped on a fixed cycle and +no client can hold a delivery stream, so the mistake is refused where it is written rather than discovered in +production. + +Every configured duration takes `Duration.Input`, so `"30 seconds"`, `Duration.seconds(30)` and `30_000` are +equivalent. Values that cross the wire contract, such as `receiptRetentionMillis` above, keep the unit in the name +because `Duration` has no stable serialized encoding. The example denies the separate unsafe Automerge decode grant. That is the required default. Ordinary authentication and `authorizeRelay` document authorization do not promote a caller into resource trust. @@ -111,8 +152,10 @@ and digest is safe. Conflicting reuse is rejected. ## Acknowledgement boundary -A relay claim carries an opaque claim token and a finite deadline. A recipient acknowledgement can terminalize only -the exact message, recipient principal, live session generation, token, and message hash. +Each delivery attempt mints an opaque claim token. A recipient acknowledgement can terminalize only the exact +message, under the caller's own inbox key, the live session, that attempt's token, and the matching message hash. A +token from an earlier attempt of the same message cannot settle a later one. There is no lease deadline to expire: +what is in flight lives only in memory, so an abandoned attempt simply leaves the row `Pending`. `PeerSession` sends `Acknowledge` only after all of these operations succeed: @@ -133,24 +176,33 @@ message for retry. ## Ordering, retry, expiry, and retention -Ordering is FIFO only within one exact directed channel. The channel key is the tenant, sender subject, sender peer, -sender replica incarnation, recipient subject, and recipient peer. The relay permits one claimed head for that -channel. Earlier pending or claimed rows block later rows in the same channel. Different channels can progress -independently. - -Claims are finite fences. A stale worker can duplicate delivery after its claim expires, but its old token cannot -acknowledge or reject the new claim. Retry uses bounded exponential delay with jitter. The sender outbox keeps pending -admissions until custody succeeds or their configured retry horizon expires. - -`RelayInbox.Options.maxDeliveries` caps recipient delivery. It is required per deployment. A failed, interrupted, -disconnected, or expired claim durably advances the attempt count. When that count reaches the configured cap, the -relay moves the message to `DeadLettered`, erases its payload, and retains only bounded terminal deduplication -evidence. Process restart does not reset the count. Message expiry can terminalize the row before the attempt cap. - -The relay message time to live is configured by `PeerRelayLimits.messageTtlMillis`. Expired active rows erase their -payload and become terminal. Acknowledged, rejected, and expired rows retain only bounded deduplication evidence until -their retention deadline. `PeerRelayLimits.maximumReceiptRetentionMillis` must cover the message time to live, the -maximum sender retry horizon, and minimum terminal retention. +Ordering is FIFO only within one exact directed channel. The recipient is the inbox itself, so the channel key is +the tenant, sender subject, sender peer, sender replica incarnation, and sender connection epoch. The epoch is part +of the key because `sender_sequence` restarts at zero on every sender reconnect; without it one channel would hold +sequences `{0,1,2,0,1}` and head selection would be arbitrary. Delivery is stop and wait within a channel: one head +is in flight at a time and later rows wait behind it. Different channels progress concurrently, bounded by +`maxConcurrentChannels`. + +A message the front door may not hand over — because receive authorization narrowed since it was admitted — is +`Release`d rather than settled. The row stays `Pending` for a later session, and the channel is skipped for the rest +of this session only, so withheld messages cannot occupy every delivery slot and starve channels the recipient is +entitled to. The sender outbox keeps pending admissions until custody succeeds or their configured retry horizon +expires. + +`RelayInbox.Options.maxDeliveries` caps recipient delivery. It is required per deployment. An attempt is charged +only once the message has provably reached the transport, so a delivery prepared for a channel and then abandoned +when the recipient disconnected — ordinary behaviour on a flaky connection — costs nothing. A withheld attempt does +spend one, which is what bounds how long an unauthorized message lingers. When the count reaches the cap the relay +moves the message to `DeadLettered` so it stops blocking its channel. Process restart does not reset the count. +Message expiry can terminalize the row before the attempt cap. + +The relay message time to live is `RelayInbox.Options.messageTtl`. A terminal row — acknowledged, rejected, dead +lettered, or expired — keeps its stored envelope until `RelayInboxMaintenance` collects it past its deduplication +horizon, at which point the whole row is deleted. The envelope is retained rather than erased at the terminal +transition because re-admitting a dead lettered or expired identity revives that row in place, which is what lets a +sender that still holds custody recover a message the relay gave up on. `PeerRelayLimits.maximumReceiptRetention` +must cover the message time to live, the maximum sender retry horizon, and the minimum terminal retention, and +`RelayServer` refuses a handshake that would breach it. The recipient keeps sender scoped receipts under its current replica incarnation. Receipt identity includes sender tenant, sender subject, sender peer, and relay message ID. `PeerRelayReceiptLimits` bounds their count, encoded bytes, @@ -161,14 +213,19 @@ Relay custody rows and client relay outbox or receipt rows are not part of the c ## Capacity and overload -`PeerRelayLimits` validates active and retained count and byte quotas for sender peers, recipient peers, recipient -subjects, tenants, and the shard. Admission reserves immutable quota entitlements in the same SQLite transaction as -the message. Accepted work is not evicted to admit new work. +Admission is bounded **per inbox** by `RelayInbox.Options.maxPendingMessages` and `maxPendingBytes`, and the cap is +re-checked inside the same transaction that performs the write. Accepted work is not evicted to admit new work. + +Cross inbox quotas — per sender peer, per recipient subject, per tenant, per shard — are not enforced. Those counters +span many inboxes, so no entity is their sole writer, and enforcing them would reintroduce the cross process +arbitration this topology exists to remove. A deployment that needs tenant wide caps needs a mechanism with its own +single writer story. -The same limits bound relay connections, raw chunks, declared frames, incomplete frames, shared payload bytes, byte -reservation waiters, per subject sessions, Open and Push concurrency and rates, terminal response work, channel -queues, relay workers, maximum delivery attempts, SQL work classes, maintenance batches and rates, and shutdown claim -release. +Connection and frame accounting is likewise not enforced here. It bounded the bespoke length prefixed framing that +standard Effect RPC over a socket replaces. A single relayed payload is still bounded, by `PeerRpc`'s schema check +against `maximumRelayPayloadBytes`; concurrent connections and in flight bytes are the deployment's socket server to +bound. `PeerRelayLimits` retains the negotiation windows, the authentication rate limits, and +`maxSessionsPerSubject`. `PeerRelayOutboxLimits` independently bounds pending sender rows and encoded bytes per remote and per replica. It also bounds retry horizon, pruning batch, pruning rate, and maintenance interval. @@ -266,30 +323,43 @@ existence. ## Deployment boundary and limitations -One logical SQL database is the custody authority for one relay shard. The application must route every write and -claim for that shard to it. `SqlPeerRelayStore.layer` supports SQLite, PostgreSQL, and MySQL. Replication and failover -are properties of the selected database deployment. The relay does not add leader election, shard rebalance, cross -region routing, or split brain recovery. +Several relay nodes may run against one logical SQL database as active active. Single owner per recipient is +enforced by cluster sharding rather than by deployment convention: exactly one runner owns a device's entity at a +time, and ownership moves with the shard. `SqlRelayInboxStore.layer` supports SQLite, PostgreSQL, and MySQL. +Replication and failover remain properties of the selected database deployment. -When that SQL authority is unavailable, new custody stops and delivery pauses. Durable pending rows remain -discoverable after recovery. Claims recover after their deadlines. The server also runs bounded -compensation and maintenance so a missed process notification does not permanently strand committed work. +Because the owning entity is the sole writer for its inbox, there is no claim ledger, lease deadline, or session +generation to recover. Nothing in flight is persisted: a lost runner loses only memory, and the next owner finds the +same `Pending` rows. -This package supplies a bounded single authority relay building block. It does not claim WhatsApp scale, global -availability, managed operations, peer discovery, end to end encryption, account management, or tenant routing. +When the SQL authority is unavailable, new custody stops and delivery pauses. Durable pending rows remain +discoverable after recovery. Retention runs as a cluster singleton so expiry and collection have exactly one owner +deployment wide. + +On a rebalance, an entity's termination handshake waits up to the cluster's `entityTerminationTimeout` for its forked +subscription before the session is closed, and rejects `Deliver` and `Settle` for that device during the window. Size +that value accordingly. + +Single owner per key across multiple runners is not verified by this package's test suite, which is single process. +This package supplies a relay building block. It does not claim global availability, managed operations, peer +discovery, end to end encryption, account management, or tenant routing. ## Observability The relay exposes Effect services and spans. It does not install a metrics exporter. -| Source | Exposed values | -| ---------------------------- | ----------------------------------------------------------------------------------- | -| `PeerRpcServerRuntime.usage` | `accepting`, `sessions`, `subjects`, `activeClaims`, `queuedChannels` | -| `PeerRelayIngress.usage` | `connections`, `reservedBytes`, `byteReservationWaiters` | -| `PeerRelayStore.usage` | `activeCount`, `activeBytes`, `retainedCount`, `retainedBytes` for a selected scope | -| `PeerRelayOutbox.usage` | Remote and replica `messageCount` and `encodedBytes` | +| Source | Exposed values | +| --------------------------- | ----------------------------------------------------------------- | +| `RelayInboxStore.usage` | `pendingCount`, `pendingBytes`, `retainedCount` for one inbox | +| `RelayInboxStore.abandoned` | Dead lettered and expired rows for one inbox, with attempt counts | +| `PeerRelayOutbox.usage` | Remote and replica `messageCount` and `encodedBytes` | -Store operations use spans named `PeerRelayStore.admit`, `claim`, `loadClaimedPayload`, `acknowledge`, `reject`, -`release`, `recover`, `expire`, `repair`, `reconcile`, `collect`, and `usage`. Client relay spans are +Store operations use spans named `SqlRelayInboxStore.admit`, `pendingHeads`, `recordDelivery`, `settle`, `usage`, +`abandoned`, `expire`, and `collect`. Entity spans are `RelayInbox.Deliver`, `Subscribe`, `Settle`, `Release`, +`Heartbeat`, and `EndSession`. Front door spans are `RelayServer.Open`, `RelayServer.Push`, and `RelayServer.Settle`. +Retention sweeps use `RelayInboxMaintenance.sweep`. Client relay spans are `effect_local_rpc.adapter.relay_open` with `rpc.selected_documents` and `effect_local_rpc.adapter.relay_push` with `rpc.payload_bytes`. + +Attributes are identifiers only. The inbox key is a digest and the message and relay identifiers are opaque; the +payload, the credential, and full principals are never span attributes. diff --git a/docs/sync.md b/docs/sync.md index 8847484..d022ac4 100644 --- a/docs/sync.md +++ b/docs/sync.md @@ -42,10 +42,10 @@ Effect RPC uses one durable protocol at version `1`. The sender first writes a s outbox. Relay acceptance means that the configured backend SQL store committed custody. The relay then delivers the oldest eligible message for one exact directed channel when the recipient reconnects. The recipient acknowledges only after its production sync workflow and sender scoped receipt commit. Lost -acknowledgements and expired claims can duplicate delivery, so the guarantee is at least once. +acknowledgements and abandoned delivery attempts can duplicate delivery, so the guarantee is at least once. -Ordering is FIFO within the tenant, sender subject, sender peer, sender replica incarnation, recipient subject, and -recipient peer channel. Other channels can progress independently. Message expiry, terminal retention, receipt +Ordering is FIFO within the tenant, sender subject, sender peer, sender replica incarnation, and sender connection +epoch channel; the recipient is the inbox itself. Other channels can progress independently. Message expiry, terminal retention, receipt retention, sender retry horizon, storage quotas, connection bounds, worker bounds, maximum delivery attempts, and maintenance rates are explicit configuration. The default maximum is `16` delivery attempts. Reaching it durably dead letters the message and erases its payload. @@ -67,8 +67,9 @@ policy authority do not share an atomic transaction. The durable protocol requires the separate unsafe grant. A future allocation bounded Automerge decode API should remove that grant. -Relay custody uses the injected `PeerRelayStore`. `SqlPeerRelayStore.layer` supports SQLite, PostgreSQL, and MySQL -through a generic `SqlClient`. It does not add cross region routing or peer discovery. See +Relay custody uses the injected `RelayInboxStore`, owned by one cluster entity per recipient device. +`SqlRelayInboxStore.layer` supports SQLite, PostgreSQL, and MySQL through a generic `SqlClient`. Several relay nodes +may serve one database, but the relay does not add cross region routing or peer discovery. See [Store and forward](store-and-forward.md) for the exact contract and composition. Automerge provides merge infrastructure, not collaboration UX. The public snapshot exposes the decoded value and head diff --git a/packages/local-rpc/README.md b/packages/local-rpc/README.md index 2f1124e..f640613 100644 --- a/packages/local-rpc/README.md +++ b/packages/local-rpc/README.md @@ -4,17 +4,23 @@ Platform neutral Effect RPC building blocks for authenticated, durable peer sync replica. Protocol version `1` uses one bounded relay topology with durable custody, reconnect delivery, and fenced acknowledgements. -Backend custody is injected through `PeerRelayStore`. `SqlPeerRelayStore.layer` follows Effect Workflow's SQL storage -shape and works with a supplied `SqlClient` for SQLite, PostgreSQL, or MySQL. Frontend replica state, sender outbox, -and recipient receipts remain SQLite. +The relay is a cluster of `RelayInbox` entities, one per recipient device, so more than one relay node can serve one +database. `RelayServer` is the stateless front door; each entity is the sole writer for its device's inbox. Backend +custody is injected through `RelayInboxStore`, and `SqlRelayInboxStore.layer` implements it against a supplied +`SqlClient` for SQLite, PostgreSQL, or MySQL. The relay requires a `Sharding` but never builds one, so the deployment +shape stays the application's choice. Frontend replica state, sender outbox, and recipient receipts remain SQLite. + Automerge `3.3.2` has no allocation bounded semantic decode API, so safe relay composition explicitly passes `denyUnsafeUnboundedAutomerge3Decode`. A separate exact unsafe resource trust grant is required to proceed. Ordinary authentication and document authorization do not provide that grant. -Delivery attempts are durable and bounded. `RelayInbox.Options.maxDeliveries` is required per deployment, then dead -letters the message and erases its payload. -Policy revocation and each bounded relay operation contend on a local gate. Revocation admitted first prevents SQL -mutation or payload emission. An operation admitted first may finish. Revocation drains it and blocks later work. -External policy and SQLite do not share an atomic transaction. + +Delivery attempts are durable and bounded. `RelayInbox.Options.maxDeliveries` is required per deployment; a message +that exhausts it is dead lettered so it stops blocking its channel. An attempt is charged only once the message has +reached the transport, so a delivery prepared and then abandoned on a flaky connection costs nothing. + +Authorization is re-checked on every operation rather than once at the handshake, because a grant can be narrowed or +revoked while a session is live. A session whose credential or grants lapse is ended by the relay, which is the only +party that observes them. See the [Effect Local documentation](https://github.com/lucas-barake/effect-local#readme) for synchronization architecture, deployment boundaries, and API reference. The relay contract, composition, limits, security, diff --git a/packages/local-rpc/src/internal/relayInboxMigrations.ts b/packages/local-rpc/src/internal/relayInboxMigrations.ts index 136a8bb..f578e84 100644 --- a/packages/local-rpc/src/internal/relayInboxMigrations.ts +++ b/packages/local-rpc/src/internal/relayInboxMigrations.ts @@ -183,7 +183,7 @@ const createTable = Effect.gen(function*() { }) }) -export const loader = Migrator.fromRecord({ +const loader = Migrator.fromRecord({ "1_create_relay_inbox": createTable }) From 320847889a5d850593223be1ed810b452b686e25 Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Mon, 27 Jul 2026 11:28:36 -0500 Subject: [PATCH 28/29] Authorize a delivery against the peer that actually sent it A device's inbox is keyed by that device alone, so every peer holding a Send grant to it writes into the same inbox and one session drains all of them. The per-message re-authorization knew that - its own comment says so - and then asked whether the *handshake's* counterparty was allowed to send the document. It answered a question about a different peer. The effect is a cross-peer authorization bypass. A recipient that reconnects to any still-authorized counterparty is handed queued messages from peers whose grants have been revoked entirely, on the authority of a grant naming someone else. Where Send and Receive policies are not perfectly symmetric per subject and document, no revocation step is needed at all. Settlement had the same root cause and survived fixing the delivery gate: it authorized against the handshake counterparty and the whole handshake document set, so a recipient could terminally discard a message it held no current authority over. Acknowledge and Reject carry only identifiers, so the front door now records what it handed over, and on whose authority, when the delivery gate allows it - and settles against that. The record is dropped once the transition is durable, so a transient failure is still retryable, and it is capped so a mistake in the stop-and-wait reasoning cannot grow it for the life of a connection. Both directions of the cap are safe: a refused settlement leaves the row Pending for a later session. Also from this round, in the store: the revive path charged a message against the pending-count cap but never the byte cap, so replaying identities the inbox had given up on bypassed it entirely; five read paths collapsed multi-tag failure channels into StorageUnavailable, reporting undecodable rows and permanent migration faults as transient; the abandoned-message query asserted a branded id on a raw column instead of decoding it; and pendingHeads validated four redundant columns but not the five channel columns that everything downstream orders and routes by. The entity now mints claim tokens from the deployment's Crypto rather than the ambient global - they are the anti-replay binding between an attempt and its settlement, so no deployment could substitute them and no test could make them deterministic. relayInboxKey loses a dead helper and a justification citing the PersistedQueue this design replaced. --- packages/local-rpc/src/RelayInbox.ts | 11 +- packages/local-rpc/src/RelayServer.ts | 137 ++++++++++++++---- packages/local-rpc/src/SqlRelayInboxStore.ts | 133 +++++++++++++---- .../src/internal/peerRpcObservability.ts | 25 +++- .../local-rpc/src/internal/relayInboxKey.ts | 16 +- packages/local-rpc/test/RelayInbox.test.ts | 4 +- .../local-rpc/test/RelayInboxStoreContract.ts | 24 +++ packages/local-rpc/test/RelayServer.test.ts | 116 ++++++++++++++- 8 files changed, 385 insertions(+), 81 deletions(-) diff --git a/packages/local-rpc/src/RelayInbox.ts b/packages/local-rpc/src/RelayInbox.ts index ebea272..9fe7b75 100644 --- a/packages/local-rpc/src/RelayInbox.ts +++ b/packages/local-rpc/src/RelayInbox.ts @@ -1,6 +1,7 @@ import * as Identity from "@lucas-barake/effect-local/Identity" import * as Cause from "effect/Cause" import * as Clock from "effect/Clock" +import * as Crypto from "effect/Crypto" import * as Deferred from "effect/Deferred" import * as Duration from "effect/Duration" import * as Effect from "effect/Effect" @@ -224,8 +225,6 @@ interface Session { readonly deadlineAt: Ref.Ref } -const makeClaimToken = Effect.sync(() => `clm_${crypto.randomUUID()}` as PeerRpc.ClaimToken) - /** * The in-flight key for a channel. * @@ -249,6 +248,14 @@ export const layer = (options: Options) => const address = yield* Entity.CurrentAddress const inboxKey = address.entityId const store = yield* RelayInboxStore.RelayInboxStore + // Resolved once here so the handlers carry no requirement of their own. A claim token is the + // anti-replay binding between one delivery attempt and its settlement, so it is minted from + // the deployment's own `Crypto` like every other identity in this package rather than from + // the ambient global, which no deployment can substitute and no test can make deterministic. + const crypto = yield* Crypto.Crypto + const makeClaimToken = Crypto.Crypto.use((platform) => + platform.randomUUIDv4.pipe(Effect.map((uuid) => PeerRpc.ClaimToken.make(`clm_${uuid}`))) + ).pipe(Effect.provideService(Crypto.Crypto, crypto)) // Converted once at construction. Everything downstream of these is arithmetic against // `Clock.currentTimeMillis` or a millisecond column, so the durations are resolved here diff --git a/packages/local-rpc/src/RelayServer.ts b/packages/local-rpc/src/RelayServer.ts index 8097cdd..15e2e86 100644 --- a/packages/local-rpc/src/RelayServer.ts +++ b/packages/local-rpc/src/RelayServer.ts @@ -54,6 +54,31 @@ export interface Options { readonly entityCallTimeout: Duration.Input } +/** + * What the front door handed over, and on whose authority. + * + * `Acknowledge` and `Reject` carry only identifiers, so without this the settlement would have to + * be authorized against the handshake's counterparty — which is not who sent the message. A device's + * inbox is keyed by that device alone, so one session drains messages from every peer that holds a + * Send grant to it. + */ +interface Delivered { + readonly remote: PeerRelayAuthorization.RemotePeer + readonly document: PeerRpc.RequestedDocument +} + +/** + * Safety valve on the provenance map, not a tuning knob. + * + * Delivery is stop and wait per channel, so the number of delivered-but-unsettled messages is + * already bounded by the entity's concurrent channel limit; an entry is dropped as soon as its + * settlement is durable. This cap exists only so a mistake in that reasoning cannot grow a map for + * the life of a connection. Eviction is safe in the direction that matters: a settlement for an + * evicted message is refused, and the row stays `Pending` for a later session rather than being + * discarded. + */ +const maxRememberedDeliveries = 1_024 + /** Everything the handshake established, held for the life of one client connection. */ interface Session { readonly sessionId: Identity.SessionId @@ -64,6 +89,7 @@ interface Session { readonly senderRetryHorizonMillis: number readonly inboxKeySelf: string readonly inboxKeyRemote: string + readonly delivered: Map } const RelayEnvelopeJson = Schema.fromJsonString( @@ -157,16 +183,26 @@ export const layerHandlers = (options: Options) => const authenticated = yield* PeerAuthentication.AuthenticatedPeer const session = yield* sessionFor(payload.sessionId, authenticated.principal) - // Re-authorized here, not merely at the handshake. Settling is a durable mutation of the - // relay's state, and a principal whose Receive authority has been withdrawn must not be - // able to perform one. The session monitor also ends a session whose grants lapse, but that - // is asynchronous, so on its own it leaves a window in which a revoked peer can still - // settle — and it reports the wrong thing, `SessionUnavailable` rather than `AccessDenied`. + // Authorized against what this session was actually handed, not against the handshake. The + // request carries only identifiers, and the handshake's counterparty is not necessarily the + // peer whose message is being settled — one session drains every sender that writes into + // this device's inbox — so settling on the handshake's grant would let a recipient + // terminally discard a message it holds no current authority over. + const delivered = session.delivered.get(payload.relayMessageId) + if (delivered === undefined) { + return yield* new PeerRpcError.SessionUnavailable() + } + + // Re-authorized here, not merely at delivery. Settling is a durable mutation of the relay's + // state, and a principal whose Receive authority has been withdrawn since must not be able + // to perform one. The session monitor also ends a session whose grants lapse, but that is + // asynchronous, so on its own it leaves a window in which a revoked peer can still settle — + // and it reports the wrong thing, `SessionUnavailable` rather than `AccessDenied`. const receive = yield* authorization.authorize({ direction: "Receive", principal: session.principal, - remote: session.remote, - documents: session.documents + remote: delivered.remote, + documents: [delivered.document] }) if (receive.documents.length === 0) { return yield* new PeerRpcError.AccessDenied() @@ -175,8 +211,8 @@ export const layerHandlers = (options: Options) => risk: PeerRelayAuthorization.unsafeUnboundedAutomerge3DecodeRisk, direction: "Receive", principal: session.principal, - remote: session.remote, - documents: session.documents + remote: delivered.remote, + documents: [delivered.document] }) // Keyed by the caller's own inbox, so settling another device's message is not expressible. @@ -193,6 +229,12 @@ export const layerHandlers = (options: Options) => Effect.catchTag("AlreadyProcessingMessage", () => new PeerRpcError.SessionOverloaded()), Effect.catchTag("PersistenceError", () => new PeerRpcError.ServerUnavailable()) ) + + // Dropped only once the transition is durable, so a settlement that failed transiently can + // still be retried on this session. + yield* Effect.sync(() => { + session.delivered.delete(payload.relayMessageId) + }) }).pipe( // Identifiers only. The session id is unguessable but not secret, and the message hash is // already a digest; the payload is never an attribute. @@ -201,6 +243,10 @@ export const layerHandlers = (options: Options) => }) ) + // The handlers below carry `AuthenticatedPeer` and, at `Open`, the request `Scope` in their + // requirement channels. Both are genuinely per call — the middleware provides the first per + // request, and the second is the rpc request scope the caller owns — so they are the only + // requirements a handler here is allowed to leak. Everything else is resolved above. return PeerRpc.Rpcs.of({ Open: (payload) => // Spans the handshake, not the session: the stream it returns outlives this effect. @@ -304,7 +350,8 @@ export const layerHandlers = (options: Options) => senderReplicaIncarnation: payload.senderReplicaIncarnation, senderRetryHorizonMillis: payload.senderRetryHorizonMillis, inboxKeySelf, - inboxKeyRemote + inboxKeyRemote, + delivered: new Map() } sessions.set(sessionId, session) @@ -375,16 +422,33 @@ export const layerHandlers = (options: Options) => // into its inbox, so the handshake's Receive decision cannot stand in for the // recipient's right to see a particular document. A message that fails here is left // unsettled and simply not emitted, so it stays durable for a later session. - Stream.filterEffect((message) => - authorization.authorize({ - direction: "Receive", - principal: session.principal, - remote: session.remote, - documents: [{ - documentType: message.document.documentType, - documentId: message.document.documentId - }] - }).pipe( + Stream.filterEffect((message) => { + // The counterparty is taken from the message, never from the handshake. This + // inbox is keyed by its owner alone, so every peer holding a Send grant to this + // device writes into it and one session drains all of them. Asking whether the + // handshake's counterparty may send this document answers a question about a + // different peer, and would release one sender's message on another sender's + // grant. + const remote = { + subjectId: message.sender.subjectId, + peerId: message.sender.peerId + } + const documents = [{ + documentType: message.document.documentType, + documentId: message.document.documentId + }] + return Effect.suspend(() => + // A grant can only ever name a remote inside the caller's own tenant, so a row + // claiming a foreign one is refused outright rather than put to the policy port. + message.sender.tenantId !== session.principal.tenantId + ? Effect.fail(new PeerRpcError.AccessDenied()) + : authorization.authorize({ + direction: "Receive", + principal: session.principal, + remote, + documents + }) + ).pipe( // Delivering commits this peer to decoding the document, so the same risk // acknowledgement the sender needed is required again on the receiving side, per // message rather than once at the handshake. @@ -392,11 +456,19 @@ export const layerHandlers = (options: Options) => risk: PeerRelayAuthorization.unsafeUnboundedAutomerge3DecodeRisk, direction: "Receive", principal: session.principal, - remote: session.remote, - documents: [{ - documentType: message.document.documentType, - documentId: message.document.documentId - }] + remote, + documents + })), + // Recorded before the message reaches the wire, because the recipient settles on + // a different fiber and can do so the instant it reads. `Settle` carries only + // identifiers, so this is the only place the settlement's authority can be + // pinned to the peer and document that were actually authorized here. + Effect.andThen(Effect.sync(() => { + if (session.delivered.size >= maxRememberedDeliveries) { + const oldest = session.delivered.keys().next() + if (!oldest.done) session.delivered.delete(oldest.value) + } + session.delivered.set(message.relayMessageId, { remote, document: documents[0]! }) })), Effect.as(true), Effect.catchTag("AccessDenied", () => @@ -415,10 +487,12 @@ export const layerHandlers = (options: Options) => Effect.andThen(release(inboxKeySelf, sessionId, message)), Effect.as(false) )), - Effect.catchTag("ServerUnavailable", () => - release(inboxKeySelf, sessionId, message).pipe(Effect.as(false))) + Effect.catchTag( + "ServerUnavailable", + () => release(inboxKeySelf, sessionId, message).pipe(Effect.as(false)) + ) ) - ), + }), // Cluster level failures are not part of the public contract, so each is reported as // the wire error that tells the client what to do about it. Stream.catchTag("MailboxFull", () => @@ -595,7 +669,10 @@ export const layer = ( // cycle and no client can hold a delivery stream — a total outage that only appears under a // particular configuration, so it is refused at construction rather than discovered in // production. The factor of two leaves room for one lost heartbeat. - Layer.provide(Layer.effectDiscard( + Layer.provide(Layer.effectDiscard(Effect.suspend(() => + // Suspended, because `Duration.toMillis` throws on an input that is not a duration at all. + // Evaluated eagerly it would throw out of `RelayServer.layer(...)` itself, before there is a + // layer to fail, which is not where a consumer looks for a configuration error. Duration.toMillis(options.heartbeatInterval) * 2 < Duration.toMillis(options.inbox.sessionDeadline) && Duration.toMillis(options.entityCallTimeout) > 0 @@ -606,5 +683,5 @@ export const layer = ( "inbox.sessionDeadline, and entityCallTimeout must be positive" ) ) - )) + ))) ) diff --git a/packages/local-rpc/src/SqlRelayInboxStore.ts b/packages/local-rpc/src/SqlRelayInboxStore.ts index a655f52..c0279bc 100644 --- a/packages/local-rpc/src/SqlRelayInboxStore.ts +++ b/packages/local-rpc/src/SqlRelayInboxStore.ts @@ -1,4 +1,5 @@ import * as Canonical from "@lucas-barake/effect-local/Canonical" +import * as Identity from "@lucas-barake/effect-local/Identity" import * as ReplicaError from "@lucas-barake/effect-local/ReplicaError" import * as Crypto from "effect/Crypto" import * as Effect from "effect/Effect" @@ -47,7 +48,9 @@ const ExistingRow = Schema.Struct({ state: Schema.String, terminal_at: Schema.NullOr(DatabaseInt), outer_envelope_digest: Schema.String, - message_hash: Schema.String + message_hash: Schema.String, + // Read so a revive can be charged the bytes it restores to pending, not just the row count. + byte_size: DatabaseInt }) const CountRow = Schema.Struct({ @@ -58,7 +61,9 @@ const CountRow = Schema.Struct({ const RetainedRow = Schema.Struct({ retained_count: DatabaseInt }) const AbandonedRow = Schema.Struct({ - relay_message_id: Schema.String, + // Decoded with its domain schema rather than asserted: the database is an external boundary, and + // a branded id that never passed its own pattern check is not the identity it claims to be. + relay_message_id: Identity.RelayMessageId, state: Schema.String, deliveries: DatabaseInt, terminal_at: Schema.NullOr(DatabaseInt) @@ -92,9 +97,19 @@ export const make = Effect.gen(function*() { const crypto = yield* Crypto.Crypto yield* Migrations.run.pipe( - Effect.mapError((cause) => - new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) }) - ) + // Discriminated rather than collapsed. A `MigrationError` is a permanent lineage fault — a + // duplicate id, an unloadable module — and reporting it as unavailability would have every + // caller above retry a schema that will never build. + Effect.catchTag( + "SqlError", + (cause) => Effect.fail(new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) })) + ), + Effect.catchTag("MigrationError", (cause) => + Effect.fail( + new ReplicaError.ReplicaError({ + reason: new ReplicaError.MigrationFailed({ migration: Migrations.tableName, cause }) + }) + )) ) const decodeEnvelope = Schema.decodeUnknownEffect(EnvelopeJson) @@ -105,7 +120,7 @@ export const make = Effect.gen(function*() { Result: ExistingRow, execute: (request) => sql` - SELECT state, terminal_at, outer_envelope_digest, message_hash FROM ${sql(table)} + SELECT state, terminal_at, outer_envelope_digest, message_hash, byte_size FROM ${sql(table)} WHERE inbox_key = ${request.inboxKey} AND relay_message_id = ${request.relayMessageId} ` }) @@ -173,7 +188,10 @@ export const make = Effect.gen(function*() { // A revive creates a `Pending` row, so it is charged against the same caps as a first // admission; otherwise replaying abandoned identities would restore unbounded pending work. const revivedUsage = yield* findUsage(request.inboxKey) - if (revivedUsage.pending_count + 1 > request.quota.maxPendingMessages) { + if ( + revivedUsage.pending_count + 1 > request.quota.maxPendingMessages || + revivedUsage.pending_bytes + row.byte_size > request.quota.maxPendingBytes + ) { return { _tag: "QuotaExceeded" } as const } yield* sql` @@ -270,8 +288,16 @@ export const make = Effect.gen(function*() { const pendingHeads = (inboxKey: string, options: { readonly limit: number }) => findHeads({ inboxKey, limit: options.limit }).pipe( - Effect.mapError((cause) => - new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) }) + // A `SchemaError` here is a row this store cannot decode, which is corruption rather than + // downtime. Collapsing the two would have callers retry a row that will never decode. + Effect.catchTag( + "SqlError", + (cause) => + Effect.fail(new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) })) + ), + Effect.catchTag( + "SchemaError", + (cause) => Effect.fail(new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageCorrupt({ cause }) })) ), Effect.flatMap(Effect.forEach((row) => Effect.gen(function*() { @@ -321,6 +347,25 @@ export const make = Effect.gen(function*() { }) ) ) + // The five channel columns are redundant metadata exactly like the four above, and they + // are what everything downstream orders and routes by: the SQL head-selection key, the + // entity's `channelId`, and its busy and withheld channel bookkeeping. A row whose + // columns disagree with its own envelope would be replayed under an ordering stream that + // contradicts its content. `admit` enforces this at write time; a row that reaches here + // violating it did not come from `admit`. + if ( + channel.tenantId !== envelope.sender.tenantId || + channel.senderSubjectId !== envelope.sender.subjectId || + channel.senderPeerId !== envelope.sender.peerId || + channel.senderReplicaIncarnation !== envelope.sender.replicaIncarnation || + channel.senderConnectionEpoch !== envelope.sender.connectionEpoch + ) { + return yield* new ReplicaError.ReplicaError({ + reason: new ReplicaError.StorageCorrupt({ + cause: `channel columns disagree with the envelope sender for ${row.relay_message_id}` + }) + }) + } return { relayMessageId: envelope.relayMessageId, channel, @@ -444,9 +489,15 @@ export const make = Effect.gen(function*() { const usage = (inboxKey: string) => Effect.all([findUsage(inboxKey), findRetained(inboxKey)]).pipe( - Effect.mapError((cause) => - new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) }) - ), + // A `SchemaError` here is a row this store cannot decode, which is corruption rather than + // downtime. Collapsing the two would have callers retry a row that will never decode. An + // aggregate returning no row at all is likewise not something a retry fixes. + Effect.catchTag("SqlError", (cause) => + Effect.fail(new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) }))), + Effect.catchTag("SchemaError", (cause) => + Effect.fail(new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageCorrupt({ cause }) }))), + Effect.catchTag("NoSuchElementError", (cause) => + Effect.fail(new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageCorrupt({ cause }) }))), Effect.map(([pending, retained]) => ({ pendingCount: pending.pending_count, pendingBytes: pending.pending_bytes, @@ -468,8 +519,16 @@ export const make = Effect.gen(function*() { const abandoned = (inboxKey: string, options: { readonly limit: number }) => findAbandoned({ inboxKey, limit: options.limit }).pipe( - Effect.mapError((cause) => - new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) }) + // A `SchemaError` here is a row this store cannot decode, which is corruption rather than + // downtime. Collapsing the two would have callers retry a row that will never decode. + Effect.catchTag( + "SqlError", + (cause) => + Effect.fail(new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) })) + ), + Effect.catchTag( + "SchemaError", + (cause) => Effect.fail(new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageCorrupt({ cause }) })) ), Effect.flatMap(Effect.forEach((row) => Schema.decodeUnknownEffect(RelayInboxStore.InboxState)(row.state).pipe( @@ -479,7 +538,7 @@ export const make = Effect.gen(function*() { }) ), Effect.map((state) => ({ - relayMessageId: row.relay_message_id as RelayInboxStore.AbandonedMessage["relayMessageId"], + relayMessageId: row.relay_message_id, state, deliveries: row.deliveries, terminalAt: row.terminal_at ?? 0 @@ -509,11 +568,14 @@ export const make = Effect.gen(function*() { ) => Effect.gen(function*() { const rows = yield* selectExpired({ now: options.now, limit: options.limit }).pipe( - Effect.mapError((cause) => - new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) }) - ) + Effect.catchTag("SqlError", (cause) => + Effect.fail(new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) }))), + Effect.catchTag("SchemaError", (cause) => + Effect.fail(new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageCorrupt({ cause }) }))) ) - if (rows.length === 0) return 0 + if (rows.length === 0) { + return 0 + } // Re-applies every predicate from the selection. Filtering on `relay_message_id` alone would // reach rows in other inboxes, because the key is `(inbox_key, relay_message_id)` and the // same identity can legitimately be addressed to more than one device. @@ -525,11 +587,14 @@ export const make = Effect.gen(function*() { ) WHERE state = 'Pending' AND expires_at <= ${options.now} AND ${ - sql.or(rows.map((row) => sql`(inbox_key = ${row.inbox_key} AND relay_message_id = ${row.relay_message_id})`)) + sql.or(rows.map((row) => + sql`(inbox_key = ${row.inbox_key} AND relay_message_id = ${row.relay_message_id})` + )) } - `.pipe(Effect.mapError((cause) => - new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) }) - )) + `.pipe( + Effect.catchTag("SqlError", (cause) => + Effect.fail(new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) }))) + ) yield* Effect.logWarning("Relay inbox expired undelivered messages").pipe( Effect.annotateLogs({ expired: rows.length }) ) @@ -550,20 +615,26 @@ export const make = Effect.gen(function*() { const collect = (options: { readonly now: number; readonly limit: number }) => Effect.gen(function*() { const rows = yield* selectCollectable({ now: options.now, limit: options.limit }).pipe( - Effect.mapError((cause) => - new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) }) - ) + Effect.catchTag("SqlError", (cause) => + Effect.fail(new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) }))), + Effect.catchTag("SchemaError", (cause) => + Effect.fail(new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageCorrupt({ cause }) }))) ) - if (rows.length === 0) return 0 + if (rows.length === 0) { + return 0 + } yield* sql` DELETE FROM ${sql(table)} WHERE state <> 'Pending' AND deduplicate_until <= ${options.now} AND ${ - sql.or(rows.map((row) => sql`(inbox_key = ${row.inbox_key} AND relay_message_id = ${row.relay_message_id})`)) + sql.or(rows.map((row) => + sql`(inbox_key = ${row.inbox_key} AND relay_message_id = ${row.relay_message_id})` + )) } - `.pipe(Effect.mapError((cause) => - new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) }) - )) + `.pipe( + Effect.catchTag("SqlError", (cause) => + Effect.fail(new ReplicaError.ReplicaError({ reason: new ReplicaError.StorageUnavailable({ cause }) }))) + ) return rows.length }) diff --git a/packages/local-rpc/src/internal/peerRpcObservability.ts b/packages/local-rpc/src/internal/peerRpcObservability.ts index a8059ce..2d918da 100644 --- a/packages/local-rpc/src/internal/peerRpcObservability.ts +++ b/packages/local-rpc/src/internal/peerRpcObservability.ts @@ -452,7 +452,16 @@ export const observe = (options: { ), Effect.tap((exit) => Effect.sync(() => options.result(exit)).pipe( - Effect.catchCause(() => Effect.succeed("Failure" as const)), + // Only a defect thrown by the caller's own classifier is recovered — the wrapped + // effect is `Effect.sync`, so nothing else can arrive here — and it is logged + // rather than swallowed. Recovering the whole `Cause` would relabel every + // observed operation as a failure with nothing anywhere saying why, and would + // silently absorb interruption alongside it. + Effect.catchDefect((defect) => + Effect.logWarning("Peer rpc observability classifier died", defect).pipe( + Effect.as("Failure" as const) + ) + ), Effect.flatMap((result) => Effect.sync(() => span.attribute("rpc.result", result)).pipe( Effect.andThen(record(options.operation, result, 1)) @@ -518,11 +527,21 @@ export const observeRelay = (options: { Effect.tap(([duration, exit]) => { const durationMillis = Duration.toMillis(duration) return Effect.gen(function*() { + // As above: a defect in the caller's classifier is recovered and reported; + // interruption is left alone. const result = yield* Effect.sync(() => options.result(exit)).pipe( - Effect.catchCause(() => Effect.succeed("Failure" as const)) + Effect.catchDefect((defect) => + Effect.logWarning("Relay observability classifier died", defect).pipe( + Effect.as("Failure" as const) + ) + ) ) const facts = yield* Effect.sync(() => options.facts(exit)).pipe( - Effect.catchCause(() => Effect.succeed({})) + Effect.catchDefect((defect) => + Effect.logWarning("Relay observability facts died", defect).pipe( + Effect.as({}) + ) + ) ) yield* Effect.sync(() => { span.attribute("rpc.result", result) diff --git a/packages/local-rpc/src/internal/relayInboxKey.ts b/packages/local-rpc/src/internal/relayInboxKey.ts index bb3d556..066a248 100644 --- a/packages/local-rpc/src/internal/relayInboxKey.ts +++ b/packages/local-rpc/src/internal/relayInboxKey.ts @@ -14,8 +14,9 @@ import type { PeerPrincipal } from "./peerPrincipal.js" * device's inbox and merges their deduplication namespaces, so it is a confidentiality boundary * rather than a routing convenience. * - * Bounded length. The backing `PersistedQueue` stores its queue name in a `VARCHAR(100)` column, - * and a principal alone can exceed that, so the encoding must be fixed width. + * Bounded length. This value is both the cluster `EntityId` and the `inbox_key` column, which is a + * fixed 64 character identity column across all three dialects, so the encoding must be fixed + * width regardless of how long a principal is. * * Canonical digesting satisfies both. The principal is serialized as a structured value, which * removes delimiter ambiguity entirely, and the digest is a fixed 64 character hex string. @@ -35,14 +36,3 @@ export const encodeInboxKey = ( * happens to cover the same fields. */ export const inboxKeyDomain = "effect-local/relay-inbox-key" - -/** - * Deduplication identity for one message inside one inbox. - * - * `PersistedQueue` enforces uniqueness on the element id alone, not on `(queue_name, id)`, so ids - * from different inboxes share one namespace. Without the inbox key prefix, the same - * `relayMessageId` addressed to two devices would be treated as a duplicate and the second device - * would silently never receive its copy. - */ -export const encodeInboxElementId = (inboxKey: string, relayMessageId: string): string => - `${inboxKey}:${relayMessageId}` diff --git a/packages/local-rpc/test/RelayInbox.test.ts b/packages/local-rpc/test/RelayInbox.test.ts index 20d7b93..e1d819c 100644 --- a/packages/local-rpc/test/RelayInbox.test.ts +++ b/packages/local-rpc/test/RelayInbox.test.ts @@ -83,7 +83,9 @@ const relay = ( Layer.provide(TestShardingConfig), // Merged rather than provided so a test can assert against the same durable rows the entity // wrote. Delivery budgets and terminal states are durable guarantees, not internals. - Layer.provideMerge(options?.store ?? sqliteStore) + Layer.provideMerge(options?.store ?? sqliteStore), + // The entity mints claim tokens from the deployment's own randomness, not the ambient global. + Layer.provide(NodeCrypto.layer) ) const layer = relay() diff --git a/packages/local-rpc/test/RelayInboxStoreContract.ts b/packages/local-rpc/test/RelayInboxStoreContract.ts index 30b39c8..ddbefc4 100644 --- a/packages/local-rpc/test/RelayInboxStoreContract.ts +++ b/packages/local-rpc/test/RelayInboxStoreContract.ts @@ -372,6 +372,30 @@ export const relayInboxStoreContract: ReadonlyArray = [ ) }) }, + { + name: "charges a revived message against the inbox byte quota", + run: Effect.gen(function*() { + const store = yield* RelayInboxStore.RelayInboxStore + yield* store.admit( + admission({ inboxKey: "revive-bytes", id: "000000000001", sequence: 0, now: 3_100_000, ttl: 1_000 }) + ) + yield* store.expire({ now: 3_105_000, limit: 1_000, terminalRetentionMillis: 1_000 }) + + // The message cap alone is generous, so only the byte cap can refuse this. A revive restores + // the row's stored bytes to pending, so a sender replaying identities the inbox already gave + // up on would otherwise be bounded only by the message count times the maximum payload — + // which is the number the byte cap exists to be lower than. + const revived = yield* store.admit(admission({ + inboxKey: "revive-bytes", + id: "000000000001", + sequence: 0, + now: 3_106_000, + ttl: 1_000, + quota: { maxPendingMessages: 100, maxPendingBytes: 1 } + })) + assert.strictEqual(revived._tag, "QuotaExceeded") + }) + }, { name: "keeps a revived message deduplicated for the sender's replay window", run: Effect.gen(function*() { diff --git a/packages/local-rpc/test/RelayServer.test.ts b/packages/local-rpc/test/RelayServer.test.ts index 67a90cf..11d6083 100644 --- a/packages/local-rpc/test/RelayServer.test.ts +++ b/packages/local-rpc/test/RelayServer.test.ts @@ -63,6 +63,20 @@ const recipient = PeerAuthentication.PeerPrincipal.make({ peerId: recipientPeerId }) +/** + * A third device in the same tenant. + * + * The recipient's inbox is keyed by the recipient alone, so every peer holding a Send grant to it + * writes into that one inbox. A session opened against one counterparty therefore drains messages + * from all of them, which is what makes "who sent this" a question the front door has to ask per + * message rather than once at the handshake. + */ +const intruder = PeerAuthentication.PeerPrincipal.make({ + tenantId: "tenant", + subjectId: "intruder", + peerId: Identity.PeerId.make("peer_00000000-0000-4000-8000-000000000005") +}) + /** A real, authenticated peer that simply belongs to a tenant this relay does not serve. */ const outsider = PeerAuthentication.PeerPrincipal.make({ tenantId: "other", @@ -274,7 +288,12 @@ const harness = (options?: { ) const credential = yield* Ref.make("sender") - const principals = new Map([["sender", sender], ["recipient", recipient], ["outsider", outsider]]) + const principals = new Map([ + ["sender", sender], + ["recipient", recipient], + ["outsider", outsider], + ["intruder", intruder] + ]) const authentication = yield* PeerAuthentication.PeerAuthentication.pipe( Effect.provide(PeerAuthentication.layerServer), Effect.provideService(PeerAuthenticator.PeerAuthenticator, { @@ -1019,6 +1038,101 @@ describe("RelayServer", () => { ) }))) + it.effect("re-authorizes each delivery against the peer that actually sent it", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + + // A third device writes into the recipient's inbox while it is fully entitled to. The inbox + // is keyed by the recipient alone, so this message now sits alongside every other sender's. + const intruderSession = yield* open(peer, intruder, recipient) + yield* Ref.set(peer.credential, "intruder") + yield* peer.client.Push({ + sessionId: intruderSession.opened.sessionId, + relayMessageId: relayId("000000000001"), + payload: yield* encodePayload(peer, { epoch: "intruder" }) + }) + yield* TestClock.adjust(10) + + // Every grant naming the intruder is withdrawn, both directions and both ports. The recipient + // is no longer entitled to anything at all from that device. + const withoutIntruder = (subject: string, remote: string) => subject !== "intruder" && remote !== "intruder" + peer.knobs.allow = (request) => withoutIntruder(request.principal.subjectId, request.remote.subjectId) + peer.knobs.allowRisk = (request) => withoutIntruder(request.principal.subjectId, request.remote.subjectId) + + // The recipient reconnects to a different, still-authorized counterparty, and a message from + // that counterparty arrives. Taking it proves the stream is live and has been dispatched + // past the intruder's older message, so the intruder's absence below is observed rather than + // merely not-yet-delivered. + const recipientSession = yield* open(peer, recipient, sender) + const senderSession = yield* open(peer, sender, recipient) + yield* push( + peer, + senderSession.opened.sessionId, + yield* encodePayload(peer, { epoch: "sender" }), + "000000000002" + ) + const delivered = yield* Queue.take(recipientSession.events) + assert.strictEqual(delivered._tag, "StoredMessage") + if (delivered._tag !== "StoredMessage") return + assert.strictEqual( + delivered.relayMessageId, + relayId("000000000002"), + "the authorized counterparty's message, not the intruder's older one" + ) + + const pending = yield* peer.store.pendingHeads( + yield* inboxKeyOf(peer, recipient), + { limit: 10 } + ) + assert.isTrue( + pending.some((message) => message.relayMessageId === relayId("000000000001")), + "the intruder's message stays durable rather than being handed over" + ) + }))) + + it.effect("re-authorizes a settlement against the peer whose message it settles", () => + Effect.scoped(Effect.gen(function*() { + const peer = yield* harness() + + // Delivered while the intruder was authorized, over a session whose counterparty is somebody + // else entirely. That is the ordinary shape of a busy device, not a contrived one. + const intruderSession = yield* open(peer, intruder, recipient) + yield* Ref.set(peer.credential, "intruder") + yield* peer.client.Push({ + sessionId: intruderSession.opened.sessionId, + relayMessageId: relayId("000000000001"), + payload: yield* encodePayload(peer, { epoch: "intruder" }) + }) + + const recipientSession = yield* open(peer, recipient, sender) + const stored = yield* Queue.take(recipientSession.events) + if (stored._tag !== "StoredMessage") return assert.fail("expected a delivery") + assert.strictEqual(stored.sender.peerId, intruder.peerId) + + // Settling is a durable mutation performed on the authority of a Receive grant. The recipient + // now holds none for this message's sender — only for the unrelated counterparty this socket + // was opened against, which must not stand in for it. + peer.knobs.allow = (request) => + request.principal.subjectId !== "intruder" && request.remote.subjectId !== "intruder" + peer.knobs.allowRisk = (request) => + request.principal.subjectId !== "intruder" && request.remote.subjectId !== "intruder" + + yield* Ref.set(peer.credential, "recipient") + const denied = yield* peer.client.Acknowledge({ + sessionId: recipientSession.opened.sessionId, + relayMessageId: stored.relayMessageId, + claimToken: stored.claimToken, + messageHash: stored.messageHash + }).pipe(Effect.flip) + assert.strictEqual(denied._tag, "AccessDenied") + + const pending = yield* peer.store.pendingHeads( + yield* inboxKeyOf(peer, recipient), + { limit: 10 } + ) + assert.strictEqual(pending.length, 1, "the terminal transition never ran") + }))) + it.effect("withholds a delivery whose unbounded decode risk is not acknowledged", () => Effect.scoped(Effect.gen(function*() { // Ordinary authorization is granted throughout; only the risk acknowledgement is withheld, From aa312684bfa41c191cfb55f326c511dbcbd9c591 Mon Sep 17 00:00:00 2001 From: lucas-barake Date: Mon, 27 Jul 2026 11:49:15 -0500 Subject: [PATCH 29/29] Make closing a replaced session atomic under interruption closeSession takes the session out of sessionRef first, and that ref is the only handle anything holds on it - the session scope is detached and its dispatcher and delivery fibers are forked into it as daemons. It then blocks in Scope.close waiting for those fibers, which can sit inside a durable write: sql.withTransaction runs connection acquisition, BEGIN and COMMIT inside an uninterruptible mask, so the wait is real rather than a microsecond window. That wait was interruptible, and every caller can be interrupted during it. Subscribe deliberately restores interruption around the call, the rpc lane is interrupted by a client interrupt or by entity shutdown, and the sweeper is interrupted when the entity scope closes. An interrupt landing there is permanent. Scope.close marks the scope closed before running its finalizers and runs them sequentially, so the ones it had not reached are never run by anyone: the dispatcher keeps polling this inbox, the outbound queue is never ended, and the recipient's stream neither ends nor errors. The front door's Open scope therefore never closes, its session entry is never deleted, and the device sits on a stream that will never deliver anything again and never tells it to reconnect. Two dispatchers then drain one inbox, which is the exact guarantee the session lock exists to provide. Making closeSession uninterruptible fixes all three callers at once and costs a wait already bounded by the same fiber interruption Scope.close performs. --- packages/local-rpc/src/RelayInbox.ts | 18 ++++++- packages/local-rpc/test/RelayInbox.test.ts | 56 ++++++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/packages/local-rpc/src/RelayInbox.ts b/packages/local-rpc/src/RelayInbox.ts index 9fe7b75..9d65ed8 100644 --- a/packages/local-rpc/src/RelayInbox.ts +++ b/packages/local-rpc/src/RelayInbox.ts @@ -297,7 +297,23 @@ export const layer = (options: Options) => session.settlements.clear() yield* Scope.close(session.scope, Exit.void) yield* Queue.end(session.outbound) - }) + }).pipe( + // Atomic once it starts. The first step takes the session out of `sessionRef`, which is + // the only handle anything holds on it — the session scope is detached and its fibers are + // forked as daemons into it — so an interrupt landing before `Scope.close` and + // `Queue.end` finish orphans the dispatcher, the delivery fibers, the scope and the queue + // with nothing left able to close them. `Scope.close` marks the scope closed *before* + // running its finalizers and runs them sequentially, so the ones it had not reached are + // gone for good: the dispatcher keeps draining this inbox and the recipient's stream + // never ends, which is the one thing that would have told it to reconnect. + // + // Stated here rather than at each call site because every caller can be interrupted while + // it runs: `Subscribe` deliberately restores interruption around it, the rpc lane is + // interrupted by a client interrupt or by entity shutdown, and the sweeper is interrupted + // when the entity scope closes. The wait this costs is already bounded by the same fiber + // interruption `Scope.close` performs. + Effect.uninterruptible + ) const closeCurrentSession = Ref.get(sessionRef).pipe( Effect.flatMap(Option.match({ onNone: () => Effect.void, onSome: closeSession })) diff --git a/packages/local-rpc/test/RelayInbox.test.ts b/packages/local-rpc/test/RelayInbox.test.ts index e1d819c..e330e68 100644 --- a/packages/local-rpc/test/RelayInbox.test.ts +++ b/packages/local-rpc/test/RelayInbox.test.ts @@ -10,6 +10,7 @@ import * as Exit from "effect/Exit" import * as Fiber from "effect/Fiber" import * as Layer from "effect/Layer" import * as Option from "effect/Option" +import * as Queue from "effect/Queue" import * as Stream from "effect/Stream" import { TestClock } from "effect/testing" import * as MessageStorage from "effect/unstable/cluster/MessageStorage" @@ -395,6 +396,61 @@ describe("RelayInbox", () => { ) }).pipe(Effect.provide(layer))) + it.effect("closes a replaced session even when the replacing subscribe is interrupted", () => + Effect.gen(function*() { + const client = yield* inbox + yield* client.Deliver(deliver({ id: "000000000001", sequence: 0 })) + + const incumbent = sessionId("00000000000a") + const delivered = yield* Queue.unbounded() + const stream = yield* Effect.forkChild( + Stream.runForEach( + client.Subscribe({ sessionId: incumbent }), + (message) => Queue.offer(delivered, message) + ) + ) + const head = yield* Queue.take(delivered) + + // Parks the incumbent's delivering fiber inside a durable write that interruption cannot cut + // short. This is faithful rather than convenient: `sql.withTransaction` runs connection + // acquisition, BEGIN and COMMIT inside an uninterruptible mask, so a real delivering fiber + // genuinely sits in a region that `Scope.close` has to wait out. + yield* Effect.forkChild( + client.Settle({ + sessionId: incumbent, + relayMessageId: head.relayMessageId, + claimToken: head.claimToken, + messageHash: head.messageHash, + outcome: "Acknowledged" + }).pipe(Effect.exit) + ) + yield* TestClock.adjust(1) + + // The replacement takes the incumbent out of `sessionRef` — the only handle anything holds on + // it — and then blocks closing its scope. A front door that loses its socket at that instant + // interrupts this call, and everything the close had not reached yet is orphaned: the + // dispatcher keeps draining this inbox and the outbound queue is never ended, so the + // recipient's stream never terminates and it is never told to reconnect. + const replacement = yield* Effect.forkChild( + Stream.runDrain(client.Subscribe({ sessionId: sessionId("00000000000b") })) + ) + yield* TestClock.adjust(1) + yield* Fiber.interrupt(replacement) + yield* TestClock.adjust(Duration.toMillis(Duration.hours(4))) + + assert.isTrue( + stream.pollUnsafe() !== undefined, + "a replaced session's stream must end; a half closed session leaves nothing able to close it" + ) + }).pipe(Effect.provide(relay({ + store: storeFailing((real) => ({ + settle: (key, relayMessageId, options) => + Effect.uninterruptible(Effect.sleep(Duration.hours(1))).pipe( + Effect.andThen(real.settle(key, relayMessageId, options)) + ) + })) + })))) + it.effect("leaves no dispatcher behind when a subscribe is interrupted", () => Effect.gen(function*() { const client = yield* inbox