From 20d0e9c7a2a72c6984c745879861b1419e23d44e Mon Sep 17 00:00:00 2001 From: David Mendez Date: Sat, 25 Jul 2026 13:59:03 -0500 Subject: [PATCH 1/6] fix(brain): remediate hive privacy findings --- packages/franken-brain/src/brain-registry.ts | 8 +- packages/franken-brain/src/hive-mind-store.ts | 28 +- packages/franken-brain/src/sqlite-brain.ts | 74 ++++-- .../tests/unit/hive-mind-store.test.ts | 241 +++++++++++++++++- ...ive-pr3775-post-merge-findings-progress.md | 14 + 5 files changed, 344 insertions(+), 21 deletions(-) create mode 100644 tasks/hive-pr3775-post-merge-findings-progress.md diff --git a/packages/franken-brain/src/brain-registry.ts b/packages/franken-brain/src/brain-registry.ts index c60e5b400..0472c3aec 100644 --- a/packages/franken-brain/src/brain-registry.ts +++ b/packages/franken-brain/src/brain-registry.ts @@ -63,7 +63,7 @@ export class BrainRegistry { constructor( private readonly brainsDir = join('.fbeast', 'brains'), private readonly hiveDbPath = join(dirname(brainsDir), 'hive', 'hive.db'), - private readonly publisherId: string = randomUUID(), + private readonly publisherId?: string, ) {} forAgentType(agentTypeId: string, dbPath?: string): SqliteBrain { @@ -88,6 +88,10 @@ export class BrainRegistry { const requestedDbPath = dbPath ?? join(this.brainsDir, `${agentTypeId}.db`); const resolvedDbPath = requestedDbPath === ':memory:' ? requestedDbPath : resolve(requestedDbPath); + const publisherId = this.publisherId + ?? (resolvedDbPath === ':memory:' + ? randomUUID() + : createHash('sha256').update(resolvedDbPath).digest('hex')); const existing = agentBrains?.get(resolvedDbPath); if (existing) { if (dbPath !== undefined) this.preferredDbPaths.set(registryKey, resolvedDbPath); @@ -101,7 +105,7 @@ export class BrainRegistry { hiveMind: { dbPath: resolvedDbPath === ':memory:' ? ':memory:' : this.hiveDbPath, namespace: hiveMindAgentTypeNamespace(agentTypeId), - publisherId: this.publisherId, + publisherId, }, }); const paths = agentBrains ?? new Map(); diff --git a/packages/franken-brain/src/hive-mind-store.ts b/packages/franken-brain/src/hive-mind-store.ts index 24d6afb2f..feaed7695 100644 --- a/packages/franken-brain/src/hive-mind-store.ts +++ b/packages/franken-brain/src/hive-mind-store.ts @@ -24,6 +24,8 @@ const WINDOWS_RESERVED_AGENT_TYPE_ID = export interface HiveMindLessonPublishEntry { readonly kind: 'lesson'; + /** Stable review-candidate identity used for precise revision revocation. */ + readonly candidateId?: string; readonly key: string; readonly status: 'pending' | 'approved'; readonly lesson: ConsolidatedLesson; @@ -139,6 +141,7 @@ function parseRow(row: HiveMindRow): HiveMindEntry { const lessonPayload = payload as HiveMindLessonPublishEntry; if ( typeof lessonPayload.key !== 'string' + || (lessonPayload.candidateId !== undefined && typeof lessonPayload.candidateId !== 'string') || (lessonPayload.status !== 'pending' && lessonPayload.status !== 'approved') || !lessonPayload.lesson || lessonPayload.lesson.kind !== 'consolidated-lesson' @@ -171,7 +174,10 @@ export class HiveMindStore { private readonly db: Database.Database; private readonly maxEntriesPerNamespace: number; - constructor(dbPath = '.fbeast/hive/hive.db', options: HiveMindStoreOptions = {}) { + constructor( + private readonly dbPath = '.fbeast/hive/hive.db', + options: HiveMindStoreOptions = {}, + ) { this.maxEntriesPerNamespace = options.maxEntriesPerNamespace ?? DEFAULT_MAX_ENTRIES_PER_NAMESPACE; if ( !Number.isSafeInteger(this.maxEntriesPerNamespace) @@ -185,6 +191,7 @@ export class HiveMindStore { if (dbPath !== ':memory:') mkdirSync(dirname(dbPath), { recursive: true }); this.db = new Database(dbPath); this.db.pragma('busy_timeout = 5000'); + if (dbPath !== ':memory:') this.db.pragma('secure_delete = ON'); this.db.pragma('journal_mode = WAL'); this.db.pragma('busy_timeout = 5000'); this.db.exec(` @@ -302,6 +309,21 @@ export class HiveMindStore { ); } + deleteLessonPublication( + namespace: HiveMindNamespace, + publisherId: string, + candidateId: string, + key: string, + ): number { + return this.deletePublishedWhere( + namespace, + publisherId, + entry => entry.kind === 'lesson' + && (entry.candidateId === candidateId + || (entry.candidateId === undefined && entry.key === key && entry.status === 'pending')), + ); + } + deletePublishedWhere( namespace: HiveMindNamespace, publisherId: string, @@ -323,6 +345,10 @@ export class HiveMindStore { for (const id of ids) statement.run(id); }); remove.immediate(); + if (this.dbPath !== ':memory:') { + this.db.pragma('wal_checkpoint(TRUNCATE)'); + this.db.exec('VACUUM'); + } return ids.length; } diff --git a/packages/franken-brain/src/sqlite-brain.ts b/packages/franken-brain/src/sqlite-brain.ts index df991a02d..9686f4063 100644 --- a/packages/franken-brain/src/sqlite-brain.ts +++ b/packages/franken-brain/src/sqlite-brain.ts @@ -3941,7 +3941,11 @@ export class SqliteMemoryReviewQueue { private encryption?: MemoryCipher, private audit?: MemoryAccessAuditRecorder, private expireWorkingKeys?: (keys: readonly string[]) => void, - private reviewDecision?: (candidate: MemoryCandidate, wasConsolidatedLesson: boolean) => void, + private reviewDecision?: ( + candidate: MemoryCandidate, + wasConsolidatedLesson: boolean, + retiredLessonKeys: readonly string[], + ) => void, ) { this.backfillCandidateKinds(); } @@ -4370,7 +4374,7 @@ export class SqliteMemoryReviewQueue { this.expireWorkingKeys?.(retiredLessonKeys); } const result = approvedCandidate ?? this.requireCandidate(id); - this.reviewDecision?.(result, wasConsolidatedLesson); + this.reviewDecision?.(result, wasConsolidatedLesson, retiredLessonKeys); return result; } @@ -4444,7 +4448,7 @@ export class SqliteMemoryReviewQueue { purgeDeletedSqliteContent(this.db, this.dbPath); } const result = rejectedCandidate ?? this.requireCandidate(id, 'rejected'); - this.reviewDecision?.(result, wasConsolidatedLesson); + this.reviewDecision?.(result, wasConsolidatedLesson, []); return result; } @@ -4517,7 +4521,7 @@ export class SqliteMemoryReviewQueue { finalizeWorkingFlush?.(); purgeDeletedSqliteContent(this.db, this.dbPath); const result = neverStoredCandidate ?? this.requireCandidate(id, 'never_store'); - this.reviewDecision?.(result, wasConsolidatedLesson); + this.reviewDecision?.(result, wasConsolidatedLesson, []); return result; } @@ -6535,6 +6539,7 @@ export class SqliteBrain implements IBrain { private readonly hiveMindStore: HiveMindStore | undefined; private readonly hiveMindNamespace: HiveMindNamespace | undefined; private readonly hiveMindPublisherId: string | undefined; + private readonly hiveMindPublishingEnabled: boolean; private retentionEpisodicScanCursor: MemoryRetentionScanCursor | undefined; private retentionCheckpointScanCursor: MemoryRetentionScanCursor | undefined; private retentionCheckpointFloorSearchCursorId: number | undefined; @@ -6634,7 +6639,8 @@ export class SqliteBrain implements IBrain { encryption, (event) => this.auditRecorder(event), (keys) => SqliteBrain.expireLivePrunedWorkingKeys(this.dbPath, keys), - (candidate, wasConsolidatedLesson) => this.handleHiveReviewDecision(candidate, wasConsolidatedLesson), + (candidate, wasConsolidatedLesson, retiredLessonKeys) => + this.handleHiveReviewDecision(candidate, wasConsolidatedLesson, retiredLessonKeys), ); if (options.conversationWorkspaceId !== null) { try { @@ -6647,9 +6653,10 @@ export class SqliteBrain implements IBrain { throw error; } } - const hiveMind = encryption ? undefined : options.hiveMind; + const hiveMind = options.hiveMind; this.hiveMindNamespace = hiveMind?.namespace; this.hiveMindPublisherId = hiveMind?.publisherId; + this.hiveMindPublishingEnabled = !encryption; let hiveMindStore: HiveMindStore | undefined; try { hiveMindStore = hiveMind ? new HiveMindStore(hiveMind.dbPath) : undefined; @@ -7034,6 +7041,7 @@ export class SqliteBrain implements IBrain { if (lesson.value.confidence < HIVE_MIN_LESSON_CONFIDENCE) continue; this.publishHiveEntry({ kind: 'lesson', + candidateId: lesson.id, key: lesson.key, status: lesson.status, lesson: lesson.value, @@ -7142,6 +7150,7 @@ export class SqliteBrain implements IBrain { private handleHiveReviewDecision( candidate: MemoryCandidate, wasConsolidatedLesson: boolean, + retiredLessonKeys: readonly string[], ): void { if ( !wasConsolidatedLesson @@ -7150,14 +7159,36 @@ export class SqliteBrain implements IBrain { || !this.hiveMindPublisherId ) return; try { - this.hiveMindStore.deleteLesson( - this.hiveMindNamespace, - this.hiveMindPublisherId, - candidate.key, - ); - if (candidate.status === 'approved' && isConsolidatedLesson(candidate.value)) { + if (candidate.status === 'approved') { + this.hiveMindStore.deleteLesson( + this.hiveMindNamespace, + this.hiveMindPublisherId, + candidate.key, + ); + for (const key of retiredLessonKeys) { + this.hiveMindStore.deleteLesson( + this.hiveMindNamespace, + this.hiveMindPublisherId, + key, + ); + } + } else { + this.hiveMindStore.deleteLessonPublication( + this.hiveMindNamespace, + this.hiveMindPublisherId, + candidate.id, + candidate.key, + ); + } + if ( + candidate.status === 'approved' + && isConsolidatedLesson(candidate.value) + && candidate.value.confidence >= HIVE_MIN_LESSON_CONFIDENCE + && this.hiveMindPublishingEnabled + ) { this.hiveMindStore.publish(this.hiveMindNamespace, this.hiveMindPublisherId, { kind: 'lesson', + candidateId: candidate.id, key: candidate.key, status: 'approved', lesson: candidate.value, @@ -7169,7 +7200,12 @@ export class SqliteBrain implements IBrain { } private publishHiveEntry(entry: Parameters[2]): void { - if (!this.hiveMindStore || !this.hiveMindNamespace || !this.hiveMindPublisherId) return; + if ( + !this.hiveMindPublishingEnabled + || !this.hiveMindStore + || !this.hiveMindNamespace + || !this.hiveMindPublisherId + ) return; try { this.hiveMindStore.publish(this.hiveMindNamespace, this.hiveMindPublisherId, entry); } catch { @@ -7180,12 +7216,13 @@ export class SqliteBrain implements IBrain { private deleteHiveMindMatches( selector: NormalizedRightToForgetSelector, memoryType: RightToForgetMemoryType, + dependentLessonKeys: ReadonlySet = new Set(), ): number { if (!this.hiveMindStore || !this.hiveMindNamespace || !this.hiveMindPublisherId) return 0; return this.hiveMindStore.deletePublishedWhere( this.hiveMindNamespace, this.hiveMindPublisherId, - entry => hiveMindEntryMatchesSelector(entry, selector, memoryType), + entry => hiveMindEntryMatchesSelector(entry, selector, memoryType, dependentLessonKeys), ); } @@ -8141,6 +8178,7 @@ export class SqliteBrain implements IBrain { let runtimeWorkingKeysToDelete = new Set(); const dependentWorkingKeysToDelete = new Set(); const dependentWorkingKeysToRefresh = new Set(); + const dependentHiveLessonKeys = new Set(); let episodicMatchCount = 0; let checkpointMatchCount = 0; let reviewMatchCount = 0; @@ -8233,6 +8271,7 @@ export class SqliteBrain implements IBrain { ? [] : this.matchingReviewPayloads(normalizedSelector); const dependentLessons = this.lessonCandidatesDependingOn(episodicMatches); + for (const candidate of dependentLessons) dependentHiveLessonKeys.add(candidate.key); const dependentReviewRows = this.lessonReviewRowsDependingOn(dependentLessons); for (const key of this.reviewWorkingKeysToDelete(reviewMatches)) { persistedWorkingMatches.add(key); @@ -8330,7 +8369,7 @@ export class SqliteBrain implements IBrain { return Number(result.lastInsertRowid); }); const auditEventId = tx() as number; - this.deleteHiveMindMatches(normalizedSelector, memoryType); + this.deleteHiveMindMatches(normalizedSelector, memoryType, dependentHiveLessonKeys); if (deletedWorkingKeys.size > 0 || episodicMatchCount > 0 || checkpointMatchCount > 0 || reviewMatchCount > 0) { finalizePersistedWorkingDelete?.(); this.working.deleteRuntimeKeys(Array.from(runtimeWorkingKeysToDelete)); @@ -9890,10 +9929,11 @@ function hiveMindEntryMatchesSelector( entry: HiveMindEntry, selector: NormalizedRightToForgetSelector, memoryType: RightToForgetMemoryType, + dependentLessonKeys: ReadonlySet, ): boolean { if (entry.kind === 'lesson') { - return memoryType !== 'episodic' - && workingEntryMatchesSelector(entry.key, entry.lesson, selector); + return dependentLessonKeys.has(entry.key) + || (memoryType !== 'episodic' && workingEntryMatchesSelector(entry.key, entry.lesson, selector)); } return memoryType !== 'working' && episodicRowMatchesSelector( diff --git a/packages/franken-brain/tests/unit/hive-mind-store.test.ts b/packages/franken-brain/tests/unit/hive-mind-store.test.ts index fb30054ac..49888c77f 100644 --- a/packages/franken-brain/tests/unit/hive-mind-store.test.ts +++ b/packages/franken-brain/tests/unit/hive-mind-store.test.ts @@ -1,4 +1,4 @@ -import { mkdirSync, mkdtempSync, rmSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -122,6 +122,38 @@ describe('HiveMindStore', () => { } }); + it('purges deleted payload bytes from the database and WAL', () => { + const root = mkdtempSync(join(tmpdir(), 'franken-hive-secure-delete-')); + const dbPath = join(root, 'hive.db'); + const namespace = hiveMindAgentTypeNamespace('coder'); + const secret = `hive-secret-${'sensitive-payload-'.repeat(128)}`; + const store = new HiveMindStore(dbPath); + try { + store.publish(namespace, 'publisher-a', { + kind: 'episode', + event: { + type: 'failure', + summary: secret, + createdAt: '2026-07-25T10:00:00.000Z', + }, + }); + expect(store.deletePublishedWhere(namespace, 'publisher-a', () => true)).toBe(1); + } finally { + store.close(); + } + + try { + const sqliteBytes = [dbPath, `${dbPath}-wal`] + .filter(existsSync) + .map(path => readFileSync(path)) + .map(buffer => buffer.toString('utf8')) + .join(''); + expect(sqliteBytes).not.toContain(secret); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('filters by kind before applying the newest-entry bound', () => { const root = mkdtempSync(join(tmpdir(), 'franken-hive-kind-bound-')); const store = new HiveMindStore(join(root, 'hive.db')); @@ -185,6 +217,122 @@ describe('HiveMindStore', () => { } }); + it('does not publish an approved lesson below the hive confidence floor', () => { + const root = mkdtempSync(join(tmpdir(), 'franken-hive-confidence-floor-')); + const hiveDbPath = join(root, 'hive.db'); + const registry = new BrainRegistry(join(root, 'brains'), hiveDbPath, 'publisher-a'); + const brain = registry.forAgentType('coder'); + try { + brain.episodic.record({ + type: 'failure', + summary: 'Low confidence build timeout one', + createdAt: '2026-07-25T10:00:00.000Z', + }); + brain.episodic.record({ + type: 'failure', + summary: 'Low confidence build timeout two', + createdAt: '2026-07-25T10:01:00.000Z', + }); + const [candidate] = brain.learning.consolidate({ threshold: 2 }); + expect(candidate?.value.confidence).toBeLessThan(0.65); + brain.memoryReview.approve(candidate!.id); + + const observer = new HiveMindStore(hiveDbPath); + try { + expect(observer.poll(hiveMindAgentTypeNamespace('coder'), { kind: 'lesson' })).toEqual([]); + } finally { + observer.close(); + } + } finally { + registry.close(); + rmSync(root, { recursive: true, force: true }); + } + }); + + it('preserves an approved hive lesson when its pending revision is rejected', () => { + const root = mkdtempSync(join(tmpdir(), 'franken-hive-reject-revision-')); + const hiveDbPath = join(root, 'hive.db'); + const namespace = hiveMindAgentTypeNamespace('coder'); + const registry = new BrainRegistry(join(root, 'brains'), hiveDbPath, 'publisher-a'); + const brain = registry.forAgentType('coder'); + try { + recordCluster(brain, 'RevisionMarker build timeout'); + const [baseline] = brain.learning.consolidate({ threshold: 3 }); + brain.memoryReview.approve(baseline!.id); + brain.episodic.record({ + type: 'failure', + summary: 'RevisionMarker build timeout failure 3', + createdAt: '2026-07-25T10:00:03.000Z', + }); + const [revision] = brain.learning.consolidate({ threshold: 3 }); + expect(revision?.key).toBe(baseline?.key); + + brain.memoryReview.reject(revision!.id); + + const observer = new HiveMindStore(hiveDbPath); + try { + expect(observer.poll(namespace, { kind: 'lesson' })).toEqual([ + expect.objectContaining({ key: baseline!.key, status: 'approved' }), + ]); + } finally { + observer.close(); + } + } finally { + registry.close(); + rmSync(root, { recursive: true, force: true }); + } + }); + + it('revokes every approved hive lesson absorbed by a bridging revision', () => { + const root = mkdtempSync(join(tmpdir(), 'franken-hive-bridge-revision-')); + const hiveDbPath = join(root, 'hive.db'); + const namespace = hiveMindAgentTypeNamespace('coder'); + const registry = new BrainRegistry(join(root, 'brains'), hiveDbPath, 'publisher-a'); + const brain = registry.forAgentType('coder'); + try { + for (const [summary, createdAt] of [ + ['AlphaOnlyMarker parser timeout crash', '2026-07-25T10:00:00.000Z'], + ['AlphaOnlyMarker parser timeout crash', '2026-07-25T10:01:00.000Z'], + ['AlphaOnlyMarker parser timeout crash', '2026-07-25T10:02:00.000Z'], + ['OmegaOnlyMarker cache mismatch overflow', '2026-07-25T10:03:00.000Z'], + ['OmegaOnlyMarker cache mismatch overflow', '2026-07-25T10:04:00.000Z'], + ['OmegaOnlyMarker cache mismatch overflow', '2026-07-25T10:05:00.000Z'], + ] as const) { + brain.episodic.record({ type: 'failure', summary, createdAt }); + } + const initial = brain.learning.consolidate({ + threshold: 3, + similarityThreshold: 0.5, + }); + expect(initial).toHaveLength(2); + for (const candidate of initial) brain.memoryReview.approve(candidate.id); + + brain.episodic.record({ + type: 'failure', + summary: 'AlphaOnlyMarker parser timeout crash OmegaOnlyMarker cache mismatch overflow', + createdAt: '2026-07-25T10:06:00.000Z', + }); + const [revision] = brain.learning.consolidate({ + threshold: 3, + similarityThreshold: 0.5, + }); + expect(revision?.replaces).toHaveLength(1); + brain.memoryReview.resolveConflict(revision!.id, { resolution: 'replace_existing' }); + + const observer = new HiveMindStore(hiveDbPath); + try { + expect(observer.poll(namespace, { kind: 'lesson' })).toEqual([ + expect.objectContaining({ candidateId: revision!.id, status: 'approved' }), + ]); + } finally { + observer.close(); + } + } finally { + registry.close(); + rmSync(root, { recursive: true, force: true }); + } + }); + it('publishes significant failure episodes from a durable agent brain', () => { const root = mkdtempSync(join(tmpdir(), 'franken-hive-significant-episode-')); const brainsDir = join(root, '.fbeast', 'brains'); @@ -268,6 +416,60 @@ describe('HiveMindStore', () => { } }); + it('revokes hive lessons derived from forgotten episodic evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'franken-hive-forget-derived-')); + const hiveDbPath = join(root, 'hive.db'); + const registry = new BrainRegistry(join(root, 'brains'), hiveDbPath, 'publisher-a'); + const brain = registry.forAgentType('coder'); + try { + recordCluster(brain, 'DerivedPrivateMarker build timeout'); + const [candidate] = brain.learning.consolidate({ threshold: 3 }); + brain.memoryReview.approve(candidate!.id); + + brain.rightToForget({ type: 'episodic', query: 'DerivedPrivateMarker' }); + + const observer = new HiveMindStore(hiveDbPath); + try { + expect(observer.poll(hiveMindAgentTypeNamespace('coder'), { kind: 'lesson' })).toEqual([]); + } finally { + observer.close(); + } + } finally { + registry.close(); + rmSync(root, { recursive: true, force: true }); + } + }); + + it('preserves durable publisher ownership across registry restarts', () => { + const root = mkdtempSync(join(tmpdir(), 'franken-hive-publisher-restart-')); + const brainsDir = join(root, 'brains'); + const hiveDbPath = join(root, 'hive', 'hive.db'); + try { + const firstRegistry = new BrainRegistry(brainsDir, hiveDbPath); + const firstBrain = firstRegistry.forAgentType('coder'); + firstBrain.episodic.record({ + type: 'failure', + summary: 'restart-private-token request failed', + createdAt: '2026-07-25T10:00:00.000Z', + }); + firstRegistry.close(); + + const secondRegistry = new BrainRegistry(brainsDir, hiveDbPath); + const secondBrain = secondRegistry.forAgentType('coder'); + secondBrain.rightToForget({ query: 'restart-private-token' }); + secondRegistry.close(); + + const observer = new HiveMindStore(hiveDbPath); + try { + expect(observer.poll(hiveMindAgentTypeNamespace('coder'))).toEqual([]); + } finally { + observer.close(); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('keeps encrypted and hive-unavailable local brains operational without publishing plaintext', () => { const root = mkdtempSync(join(tmpdir(), 'franken-hive-additive-')); const namespace = hiveMindAgentTypeNamespace('coder'); @@ -309,6 +511,43 @@ describe('HiveMindStore', () => { rmSync(root, { recursive: true, force: true }); } }); + + it('allows encrypted brains to read peer lessons without publishing local plaintext', () => { + const root = mkdtempSync(join(tmpdir(), 'franken-hive-encrypted-peer-')); + const namespace = hiveMindAgentTypeNamespace('coder'); + const hiveDbPath = join(root, 'hive.db'); + const publisher = new HiveMindStore(hiveDbPath); + publisher.publish(namespace, 'peer-run', { + kind: 'lesson', + key: 'lesson:peer-timeout', + status: 'approved', + lesson: lessonValue('peer build timeout recovery'), + }); + publisher.close(); + + const encrypted = new SqliteBrain(join(root, 'encrypted.db'), undefined, { + encryption: { enabled: true, key: 'hive-encrypted-peer-test-key' }, + hiveMind: { dbPath: hiveDbPath, namespace, publisherId: 'encrypted-run' }, + }); + try { + expect(encrypted.learning.relevantLessons('peer build timeout')).toEqual([ + expect.objectContaining({ key: 'lesson:peer-timeout', source: 'peer' }), + ]); + recordCluster(encrypted, 'encrypted local secret'); + encrypted.learning.consolidate({ threshold: 3 }); + + const observer = new HiveMindStore(hiveDbPath); + try { + expect(observer.poll(namespace).filter(({ publisherId }) => publisherId === 'encrypted-run')) + .toEqual([]); + } finally { + observer.close(); + } + } finally { + encrypted.close(); + rmSync(root, { recursive: true, force: true }); + } + }); }); function lessonValue(pattern: string) { diff --git a/tasks/hive-pr3775-post-merge-findings-progress.md b/tasks/hive-pr3775-post-merge-findings-progress.md new file mode 100644 index 000000000..dbff8b39d --- /dev/null +++ b/tasks/hive-pr3775-post-merge-findings-progress.md @@ -0,0 +1,14 @@ +# Hive PR #3775 post-merge findings progress + +- [x] Verify Kanban assignment, clean isolated worktree, exact detached `origin/main` head, GitHub auth, and Git identity. +- [x] Create the singular follow-up branch from verified `origin/main`. +- [x] Fetch and classify all seven unresolved Codex findings from merged PR #3775. +- [x] Trace affected definitions, usages, and existing tests. +- [x] Reproduce each valid contract-critical finding with a failing focused test. +- [x] Implement minimal fixes and keep focused tests green. +- [x] Run package tests, typecheck, build, and lint. (`@franken/brain`: 486/486 tests, build, and typecheck pass; root lint passes. Root test has three unrelated orchestrator failures; root typecheck/build remain blocked by pre-existing `franken-web` errors.) +- [x] Commit with the required Git identity. +- [ ] Route push and follow-up PR creation through the dedicated Hive Approval Cop. +- [ ] Run at most two batched Codex review rounds; resolve all original and follow-up threads through Approval Cop. +- [ ] Verify exact-head green CI, zero paginated unresolved Codex threads, and approval-routed merge/closeout. +- [ ] Record remediation evidence and terminalize Kanban card `t_fd5ece5d`. From ec1367b5bf5aef079fb034bf9e521aa93ee95668 Mon Sep 17 00:00:00 2001 From: David Mendez Date: Sat, 25 Jul 2026 16:49:46 -0500 Subject: [PATCH 2/6] fix(brain): address hive privacy review findings --- packages/franken-brain/src/brain-registry.ts | 35 +++++- packages/franken-brain/src/hive-mind-store.ts | 45 +++++++- packages/franken-brain/src/sqlite-brain.ts | 23 ++-- .../tests/unit/hive-mind-store.test.ts | 107 +++++++++++++++++- ...ive-pr3775-post-merge-findings-progress.md | 6 +- 5 files changed, 198 insertions(+), 18 deletions(-) diff --git a/packages/franken-brain/src/brain-registry.ts b/packages/franken-brain/src/brain-registry.ts index 0472c3aec..c8d5e231f 100644 --- a/packages/franken-brain/src/brain-registry.ts +++ b/packages/franken-brain/src/brain-registry.ts @@ -3,6 +3,8 @@ import { createHash, randomUUID } from 'node:crypto'; import { chmodSync, existsSync, mkdirSync } from 'node:fs'; import { dirname, join, resolve } from 'node:path'; +import Database from 'better-sqlite3'; + import { hiveMindAgentTypeNamespace } from './hive-mind-store.js'; import { SqliteBrain } from './sqlite-brain.js'; @@ -11,6 +13,31 @@ const MAX_DEFAULT_BRAIN_FILENAME_AGENT_TYPE_ID_BYTES = 244; const UNSAFE_AGENT_TYPE_ID_CHARACTERS = /[<>:"/\\|?*\u0000-\u001f\u007f]/u; const WINDOWS_RESERVED_AGENT_TYPE_ID = /^(?:con|prn|aux|nul|clock\$|com[1-9]|lpt[1-9])(?:\.|$)/iu; +const HIVE_PUBLISHER_ID_METADATA_KEY = 'hive-publisher-id'; + +function durablePublisherId(dbPath: string): string { + const db = new Database(dbPath); + try { + db.pragma('busy_timeout = 5000'); + db.exec(` + CREATE TABLE IF NOT EXISTS brain_registry_metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ) + `); + db.prepare(` + INSERT OR IGNORE INTO brain_registry_metadata (key, value) + VALUES (?, ?) + `).run(HIVE_PUBLISHER_ID_METADATA_KEY, randomUUID()); + const row = db.prepare(` + SELECT value FROM brain_registry_metadata WHERE key = ? + `).get(HIVE_PUBLISHER_ID_METADATA_KEY) as { value: string } | undefined; + if (!row) throw new Error('Durable brain publisher identity could not be initialized'); + return row.value; + } finally { + db.close(); + } +} function assertSafeAgentTypeId(agentTypeId: string): void { if ( @@ -88,18 +115,18 @@ export class BrainRegistry { const requestedDbPath = dbPath ?? join(this.brainsDir, `${agentTypeId}.db`); const resolvedDbPath = requestedDbPath === ':memory:' ? requestedDbPath : resolve(requestedDbPath); + if (resolvedDbPath !== ':memory:') { + mkdirSync(dirname(resolvedDbPath), { recursive: true }); + } const publisherId = this.publisherId ?? (resolvedDbPath === ':memory:' ? randomUUID() - : createHash('sha256').update(resolvedDbPath).digest('hex')); + : durablePublisherId(resolvedDbPath)); const existing = agentBrains?.get(resolvedDbPath); if (existing) { if (dbPath !== undefined) this.preferredDbPaths.set(registryKey, resolvedDbPath); return existing; } - if (dbPath === undefined) { - mkdirSync(this.brainsDir, { recursive: true }); - } const brain = new SqliteBrain(resolvedDbPath, undefined, { conversationWorkspaceId: null, hiveMind: { diff --git a/packages/franken-brain/src/hive-mind-store.ts b/packages/franken-brain/src/hive-mind-store.ts index feaed7695..97f90c184 100644 --- a/packages/franken-brain/src/hive-mind-store.ts +++ b/packages/franken-brain/src/hive-mind-store.ts @@ -18,6 +18,7 @@ const MAX_ENTRY_BYTES = 64 * 1024; const MAX_POLL_LIMIT = 1_000; const DEFAULT_MAX_ENTRIES_PER_NAMESPACE = 10_000; const MAX_CONFIGURED_ENTRIES_PER_NAMESPACE = 1_000_000; +const SECURE_DELETE_PENDING_KEY = 'secure-delete-pending'; const UNSAFE_AGENT_TYPE_ID_CHARACTERS = /[<>:"/\\|?*\u0000-\u001f\u007f]/u; const WINDOWS_RESERVED_AGENT_TYPE_ID = /^(?:con|prn|aux|nul|clock\$|com[1-9]|lpt[1-9])(?:\.|$)/iu; @@ -72,6 +73,19 @@ interface HiveMindRow { publishedAt: string; } +interface WalCheckpointResult { + busy: number; + log: number; + checkpointed: number; +} + +function truncateWalOrThrow(db: Database.Database): void { + const [result] = db.pragma('wal_checkpoint(TRUNCATE)') as WalCheckpointResult[]; + if (!result || result.busy !== 0) { + throw new Error('Secure deletion could not truncate the Hive WAL because a reader is active'); + } +} + function assertSafeAgentTypeId(agentTypeId: string): void { if ( typeof agentTypeId !== 'string' @@ -205,6 +219,10 @@ export class HiveMindStore { ); CREATE INDEX IF NOT EXISTS idx_hive_mind_poll ON hive_mind_entries(namespace, id); + CREATE TABLE IF NOT EXISTS hive_mind_metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); `); } @@ -339,19 +357,40 @@ export class HiveMindStore { ORDER BY id ASC `).all(namespace, publisherId) as HiveMindRow[]; const ids = rows.map(parseRow).filter(predicate).map(entry => entry.id); - if (ids.length === 0) return 0; + if (ids.length === 0) { + if (this.dbPath !== ':memory:' && this.hasPendingSecureDelete()) { + this.purgeDeletedContent(); + } + return 0; + } const remove = this.db.transaction(() => { const statement = this.db.prepare('DELETE FROM hive_mind_entries WHERE id = ?'); for (const id of ids) statement.run(id); + if (this.dbPath !== ':memory:') { + this.db.prepare(` + INSERT OR REPLACE INTO hive_mind_metadata (key, value) VALUES (?, '1') + `).run(SECURE_DELETE_PENDING_KEY); + } }); remove.immediate(); if (this.dbPath !== ':memory:') { - this.db.pragma('wal_checkpoint(TRUNCATE)'); - this.db.exec('VACUUM'); + this.purgeDeletedContent(); } return ids.length; } + private hasPendingSecureDelete(): boolean { + return this.db.prepare(` + SELECT 1 FROM hive_mind_metadata WHERE key = ? + `).get(SECURE_DELETE_PENDING_KEY) !== undefined; + } + + private purgeDeletedContent(): void { + truncateWalOrThrow(this.db); + this.db.exec('VACUUM'); + this.db.prepare('DELETE FROM hive_mind_metadata WHERE key = ?').run(SECURE_DELETE_PENDING_KEY); + } + close(): void { this.db.close(); } diff --git a/packages/franken-brain/src/sqlite-brain.ts b/packages/franken-brain/src/sqlite-brain.ts index 9686f4063..519269d5d 100644 --- a/packages/franken-brain/src/sqlite-brain.ts +++ b/packages/franken-brain/src/sqlite-brain.ts @@ -7216,13 +7216,18 @@ export class SqliteBrain implements IBrain { private deleteHiveMindMatches( selector: NormalizedRightToForgetSelector, memoryType: RightToForgetMemoryType, - dependentLessonKeys: ReadonlySet = new Set(), + dependentLessonCandidateIds: ReadonlySet = new Set(), ): number { if (!this.hiveMindStore || !this.hiveMindNamespace || !this.hiveMindPublisherId) return 0; return this.hiveMindStore.deletePublishedWhere( this.hiveMindNamespace, this.hiveMindPublisherId, - entry => hiveMindEntryMatchesSelector(entry, selector, memoryType, dependentLessonKeys), + entry => hiveMindEntryMatchesSelector( + entry, + selector, + memoryType, + dependentLessonCandidateIds, + ), ); } @@ -8178,7 +8183,7 @@ export class SqliteBrain implements IBrain { let runtimeWorkingKeysToDelete = new Set(); const dependentWorkingKeysToDelete = new Set(); const dependentWorkingKeysToRefresh = new Set(); - const dependentHiveLessonKeys = new Set(); + const dependentHiveLessonCandidateIds = new Set(); let episodicMatchCount = 0; let checkpointMatchCount = 0; let reviewMatchCount = 0; @@ -8271,7 +8276,7 @@ export class SqliteBrain implements IBrain { ? [] : this.matchingReviewPayloads(normalizedSelector); const dependentLessons = this.lessonCandidatesDependingOn(episodicMatches); - for (const candidate of dependentLessons) dependentHiveLessonKeys.add(candidate.key); + for (const candidate of dependentLessons) dependentHiveLessonCandidateIds.add(candidate.id); const dependentReviewRows = this.lessonReviewRowsDependingOn(dependentLessons); for (const key of this.reviewWorkingKeysToDelete(reviewMatches)) { persistedWorkingMatches.add(key); @@ -8369,7 +8374,11 @@ export class SqliteBrain implements IBrain { return Number(result.lastInsertRowid); }); const auditEventId = tx() as number; - this.deleteHiveMindMatches(normalizedSelector, memoryType, dependentHiveLessonKeys); + this.deleteHiveMindMatches( + normalizedSelector, + memoryType, + dependentHiveLessonCandidateIds, + ); if (deletedWorkingKeys.size > 0 || episodicMatchCount > 0 || checkpointMatchCount > 0 || reviewMatchCount > 0) { finalizePersistedWorkingDelete?.(); this.working.deleteRuntimeKeys(Array.from(runtimeWorkingKeysToDelete)); @@ -9929,10 +9938,10 @@ function hiveMindEntryMatchesSelector( entry: HiveMindEntry, selector: NormalizedRightToForgetSelector, memoryType: RightToForgetMemoryType, - dependentLessonKeys: ReadonlySet, + dependentLessonCandidateIds: ReadonlySet, ): boolean { if (entry.kind === 'lesson') { - return dependentLessonKeys.has(entry.key) + return (entry.candidateId !== undefined && dependentLessonCandidateIds.has(entry.candidateId)) || (memoryType !== 'episodic' && workingEntryMatchesSelector(entry.key, entry.lesson, selector)); } return memoryType !== 'working' diff --git a/packages/franken-brain/tests/unit/hive-mind-store.test.ts b/packages/franken-brain/tests/unit/hive-mind-store.test.ts index 49888c77f..9b1f8172a 100644 --- a/packages/franken-brain/tests/unit/hive-mind-store.test.ts +++ b/packages/franken-brain/tests/unit/hive-mind-store.test.ts @@ -1,7 +1,8 @@ -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import Database from 'better-sqlite3'; import { describe, expect, it } from 'vitest'; import { @@ -154,6 +155,42 @@ describe('HiveMindStore', () => { } }); + it('fails secure deletion when an active reader prevents WAL truncation', () => { + const root = mkdtempSync(join(tmpdir(), 'franken-hive-secure-delete-busy-')); + const dbPath = join(root, 'hive.db'); + const namespace = hiveMindAgentTypeNamespace('coder'); + const secret = `reader-held-secret-${'private-payload-'.repeat(128)}`; + const store = new HiveMindStore(dbPath); + const reader = new Database(dbPath); + try { + store.publish(namespace, 'publisher-a', { + kind: 'episode', + event: { + type: 'failure', + summary: secret, + createdAt: '2026-07-25T10:00:00.000Z', + }, + }); + reader.exec('BEGIN'); + reader.prepare('SELECT payload FROM hive_mind_entries').all(); + + expect(() => store.deletePublishedWhere(namespace, 'publisher-a', () => true)) + .toThrow('Secure deletion could not truncate the Hive WAL'); + reader.exec('ROLLBACK'); + expect(store.deletePublishedWhere(namespace, 'publisher-a', () => true)).toBe(0); + const sqliteBytes = [dbPath, `${dbPath}-wal`] + .filter(existsSync) + .map(path => readFileSync(path).toString('utf8')) + .join(''); + expect(sqliteBytes).not.toContain(secret); + } finally { + if (reader.inTransaction) reader.exec('ROLLBACK'); + reader.close(); + store.close(); + rmSync(root, { recursive: true, force: true }); + } + }, 10_000); + it('filters by kind before applying the newest-entry bound', () => { const root = mkdtempSync(join(tmpdir(), 'franken-hive-kind-bound-')); const store = new HiveMindStore(join(root, 'hive.db')); @@ -283,6 +320,40 @@ describe('HiveMindStore', () => { } }); + it('revokes only the dependent hive revision when an approved baseline shares its key', () => { + const root = mkdtempSync(join(tmpdir(), 'franken-hive-forget-revision-')); + const hiveDbPath = join(root, 'hive.db'); + const namespace = hiveMindAgentTypeNamespace('coder'); + const registry = new BrainRegistry(join(root, 'brains'), hiveDbPath, 'publisher-a'); + const brain = registry.forAgentType('coder'); + try { + recordCluster(brain, 'RevisionIdentityMarker build timeout'); + const [baseline] = brain.learning.consolidate({ threshold: 3 }); + brain.memoryReview.approve(baseline!.id); + brain.episodic.record({ + type: 'failure', + summary: 'RevisionIdentityMarker build timeout PrivateRevisionEvidence', + createdAt: '2026-07-25T10:00:03.000Z', + }); + const [revision] = brain.learning.consolidate({ threshold: 3 }); + expect(revision?.key).toBe(baseline?.key); + + brain.rightToForget({ type: 'episodic', query: 'PrivateRevisionEvidence' }); + + const observer = new HiveMindStore(hiveDbPath); + try { + expect(observer.poll(namespace, { kind: 'lesson' })).toEqual([ + expect.objectContaining({ candidateId: baseline!.id, status: 'approved' }), + ]); + } finally { + observer.close(); + } + } finally { + registry.close(); + rmSync(root, { recursive: true, force: true }); + } + }); + it('revokes every approved hive lesson absorbed by a bridging revision', () => { const root = mkdtempSync(join(tmpdir(), 'franken-hive-bridge-revision-')); const hiveDbPath = join(root, 'hive.db'); @@ -470,6 +541,40 @@ describe('HiveMindStore', () => { } }); + it('preserves durable publisher ownership when the same brain is reopened through a symlink', () => { + const root = mkdtempSync(join(tmpdir(), 'franken-hive-publisher-symlink-')); + const brainsDir = join(root, 'brains'); + const hiveDbPath = join(root, 'hive', 'hive.db'); + const durableDbPath = join(brainsDir, 'durable.db'); + const linkedDbPath = join(root, 'linked-durable.db'); + try { + mkdirSync(brainsDir, { recursive: true }); + const firstRegistry = new BrainRegistry(brainsDir, hiveDbPath); + const firstBrain = firstRegistry.forAgentType('coder', durableDbPath); + firstBrain.episodic.record({ + type: 'failure', + summary: 'symlink-private-token request failed', + createdAt: '2026-07-25T10:00:00.000Z', + }); + firstRegistry.close(); + symlinkSync(durableDbPath, linkedDbPath); + + const secondRegistry = new BrainRegistry(brainsDir, hiveDbPath); + const secondBrain = secondRegistry.forAgentType('coder', linkedDbPath); + secondBrain.rightToForget({ query: 'symlink-private-token' }); + secondRegistry.close(); + + const observer = new HiveMindStore(hiveDbPath); + try { + expect(observer.poll(hiveMindAgentTypeNamespace('coder'))).toEqual([]); + } finally { + observer.close(); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('keeps encrypted and hive-unavailable local brains operational without publishing plaintext', () => { const root = mkdtempSync(join(tmpdir(), 'franken-hive-additive-')); const namespace = hiveMindAgentTypeNamespace('coder'); diff --git a/tasks/hive-pr3775-post-merge-findings-progress.md b/tasks/hive-pr3775-post-merge-findings-progress.md index dbff8b39d..c7fb99902 100644 --- a/tasks/hive-pr3775-post-merge-findings-progress.md +++ b/tasks/hive-pr3775-post-merge-findings-progress.md @@ -6,9 +6,9 @@ - [x] Trace affected definitions, usages, and existing tests. - [x] Reproduce each valid contract-critical finding with a failing focused test. - [x] Implement minimal fixes and keep focused tests green. -- [x] Run package tests, typecheck, build, and lint. (`@franken/brain`: 486/486 tests, build, and typecheck pass; root lint passes. Root test has three unrelated orchestrator failures; root typecheck/build remain blocked by pre-existing `franken-web` errors.) +- [x] Run package tests, typecheck, build, and lint. (`@franken/brain`: 489/489 tests, typecheck, lint, and build pass after the first follow-up Codex fixes; root lint previously passed. Root test has three unrelated orchestrator failures; root typecheck/build remain blocked by pre-existing `franken-web` errors.) - [x] Commit with the required Git identity. -- [ ] Route push and follow-up PR creation through the dedicated Hive Approval Cop. -- [ ] Run at most two batched Codex review rounds; resolve all original and follow-up threads through Approval Cop. +- [x] Route push and follow-up PR creation through the dedicated Hive Approval Cop. (PR #3788 created at immutable head `20d0e9c7a2a72c6984c745879861b1419e23d44e`.) +- [ ] Run at most two batched Codex review rounds; resolve all original and follow-up threads through Approval Cop. (Automatic round 1/2 produced three findings; fixes are locally verified and awaiting approval-routed publication/thread closeout.) - [ ] Verify exact-head green CI, zero paginated unresolved Codex threads, and approval-routed merge/closeout. - [ ] Record remediation evidence and terminalize Kanban card `t_fd5ece5d`. From 807bda917432c6e1365fcab82d6c35a98a8c4bdc Mon Sep 17 00:00:00 2001 From: David Mendez Date: Sat, 25 Jul 2026 17:07:54 -0500 Subject: [PATCH 3/6] fix(brain): close final hive review findings --- packages/franken-brain/src/brain-registry.ts | 15 +- packages/franken-brain/src/hive-mind-store.ts | 12 +- packages/franken-brain/src/sqlite-brain.ts | 14 +- .../tests/unit/hive-mind-store.test.ts | 131 +++++++++++++++++- ...ive-pr3775-post-merge-findings-progress.md | 8 +- 5 files changed, 168 insertions(+), 12 deletions(-) diff --git a/packages/franken-brain/src/brain-registry.ts b/packages/franken-brain/src/brain-registry.ts index c8d5e231f..fcf63586f 100644 --- a/packages/franken-brain/src/brain-registry.ts +++ b/packages/franken-brain/src/brain-registry.ts @@ -28,7 +28,10 @@ function durablePublisherId(dbPath: string): string { db.prepare(` INSERT OR IGNORE INTO brain_registry_metadata (key, value) VALUES (?, ?) - `).run(HIVE_PUBLISHER_ID_METADATA_KEY, randomUUID()); + `).run( + HIVE_PUBLISHER_ID_METADATA_KEY, + createHash('sha256').update(dbPath).digest('hex'), + ); const row = db.prepare(` SELECT value FROM brain_registry_metadata WHERE key = ? `).get(HIVE_PUBLISHER_ID_METADATA_KEY) as { value: string } | undefined; @@ -115,6 +118,11 @@ export class BrainRegistry { const requestedDbPath = dbPath ?? join(this.brainsDir, `${agentTypeId}.db`); const resolvedDbPath = requestedDbPath === ':memory:' ? requestedDbPath : resolve(requestedDbPath); + const existing = agentBrains?.get(resolvedDbPath); + if (existing) { + if (dbPath !== undefined) this.preferredDbPaths.set(registryKey, resolvedDbPath); + return existing; + } if (resolvedDbPath !== ':memory:') { mkdirSync(dirname(resolvedDbPath), { recursive: true }); } @@ -122,11 +130,6 @@ export class BrainRegistry { ?? (resolvedDbPath === ':memory:' ? randomUUID() : durablePublisherId(resolvedDbPath)); - const existing = agentBrains?.get(resolvedDbPath); - if (existing) { - if (dbPath !== undefined) this.preferredDbPaths.set(registryKey, resolvedDbPath); - return existing; - } const brain = new SqliteBrain(resolvedDbPath, undefined, { conversationWorkspaceId: null, hiveMind: { diff --git a/packages/franken-brain/src/hive-mind-store.ts b/packages/franken-brain/src/hive-mind-store.ts index 97f90c184..82140f0b4 100644 --- a/packages/franken-brain/src/hive-mind-store.ts +++ b/packages/franken-brain/src/hive-mind-store.ts @@ -224,6 +224,7 @@ export class HiveMindStore { value TEXT NOT NULL ); `); + this.retryPendingSecureDelete(); } publish( @@ -233,6 +234,7 @@ export class HiveMindStore { ): HiveMindEntry { assertNamespace(namespace); assertPublisherId(publisherId); + this.retryPendingSecureDelete(); if (entry.kind !== 'lesson' && entry.kind !== 'episode') { throw new TypeError('Hive mind entry kind must be lesson or episode'); } @@ -270,6 +272,7 @@ export class HiveMindStore { poll(namespace: HiveMindNamespace, options: HiveMindPollOptions = {}): HiveMindEntry[] { assertNamespace(namespace); + this.retryPendingSecureDelete(); const { sinceId, limit } = assertPollOptions(options); const clauses = ['namespace = ?', 'id > ?']; const parameters: Array = [namespace, sinceId]; @@ -296,6 +299,7 @@ export class HiveMindStore { /** Return the newest bounded window, ordered newest first. */ recent(namespace: HiveMindNamespace, options: HiveMindRecentOptions = {}): HiveMindEntry[] { assertNamespace(namespace); + this.retryPendingSecureDelete(); const { limit } = assertPollOptions(options); const clauses = ['namespace = ?']; const parameters: Array = [namespace]; @@ -385,13 +389,19 @@ export class HiveMindStore { `).get(SECURE_DELETE_PENDING_KEY) !== undefined; } + private retryPendingSecureDelete(): void { + if (this.dbPath !== ':memory:' && this.hasPendingSecureDelete()) { + this.purgeDeletedContent(); + } + } + private purgeDeletedContent(): void { truncateWalOrThrow(this.db); - this.db.exec('VACUUM'); this.db.prepare('DELETE FROM hive_mind_metadata WHERE key = ?').run(SECURE_DELETE_PENDING_KEY); } close(): void { + this.retryPendingSecureDelete(); this.db.close(); } } diff --git a/packages/franken-brain/src/sqlite-brain.ts b/packages/franken-brain/src/sqlite-brain.ts index 519269d5d..434f116b9 100644 --- a/packages/franken-brain/src/sqlite-brain.ts +++ b/packages/franken-brain/src/sqlite-brain.ts @@ -7217,6 +7217,7 @@ export class SqliteBrain implements IBrain { selector: NormalizedRightToForgetSelector, memoryType: RightToForgetMemoryType, dependentLessonCandidateIds: ReadonlySet = new Set(), + dependentLessonKeys: ReadonlySet = new Set(), ): number { if (!this.hiveMindStore || !this.hiveMindNamespace || !this.hiveMindPublisherId) return 0; return this.hiveMindStore.deletePublishedWhere( @@ -7227,6 +7228,7 @@ export class SqliteBrain implements IBrain { selector, memoryType, dependentLessonCandidateIds, + dependentLessonKeys, ), ); } @@ -8184,6 +8186,7 @@ export class SqliteBrain implements IBrain { const dependentWorkingKeysToDelete = new Set(); const dependentWorkingKeysToRefresh = new Set(); const dependentHiveLessonCandidateIds = new Set(); + const dependentHiveLessonKeys = new Set(); let episodicMatchCount = 0; let checkpointMatchCount = 0; let reviewMatchCount = 0; @@ -8276,7 +8279,10 @@ export class SqliteBrain implements IBrain { ? [] : this.matchingReviewPayloads(normalizedSelector); const dependentLessons = this.lessonCandidatesDependingOn(episodicMatches); - for (const candidate of dependentLessons) dependentHiveLessonCandidateIds.add(candidate.id); + for (const candidate of dependentLessons) { + dependentHiveLessonCandidateIds.add(candidate.id); + dependentHiveLessonKeys.add(candidate.key); + } const dependentReviewRows = this.lessonReviewRowsDependingOn(dependentLessons); for (const key of this.reviewWorkingKeysToDelete(reviewMatches)) { persistedWorkingMatches.add(key); @@ -8378,6 +8384,7 @@ export class SqliteBrain implements IBrain { normalizedSelector, memoryType, dependentHiveLessonCandidateIds, + dependentHiveLessonKeys, ); if (deletedWorkingKeys.size > 0 || episodicMatchCount > 0 || checkpointMatchCount > 0 || reviewMatchCount > 0) { finalizePersistedWorkingDelete?.(); @@ -9939,9 +9946,12 @@ function hiveMindEntryMatchesSelector( selector: NormalizedRightToForgetSelector, memoryType: RightToForgetMemoryType, dependentLessonCandidateIds: ReadonlySet, + dependentLessonKeys: ReadonlySet, ): boolean { if (entry.kind === 'lesson') { - return (entry.candidateId !== undefined && dependentLessonCandidateIds.has(entry.candidateId)) + return (entry.candidateId === undefined + ? dependentLessonKeys.has(entry.key) + : dependentLessonCandidateIds.has(entry.candidateId)) || (memoryType !== 'episodic' && workingEntryMatchesSelector(entry.key, entry.lesson, selector)); } return memoryType !== 'working' diff --git a/packages/franken-brain/tests/unit/hive-mind-store.test.ts b/packages/franken-brain/tests/unit/hive-mind-store.test.ts index 9b1f8172a..27f38d25e 100644 --- a/packages/franken-brain/tests/unit/hive-mind-store.test.ts +++ b/packages/franken-brain/tests/unit/hive-mind-store.test.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -177,7 +178,7 @@ describe('HiveMindStore', () => { expect(() => store.deletePublishedWhere(namespace, 'publisher-a', () => true)) .toThrow('Secure deletion could not truncate the Hive WAL'); reader.exec('ROLLBACK'); - expect(store.deletePublishedWhere(namespace, 'publisher-a', () => true)).toBe(0); + expect(store.poll(namespace)).toEqual([]); const sqliteBytes = [dbPath, `${dbPath}-wal`] .filter(existsSync) .map(path => readFileSync(path).toString('utf8')) @@ -191,6 +192,37 @@ describe('HiveMindStore', () => { } }, 10_000); + it('leaves securely erased pages available for reuse instead of vacuuming per deletion', () => { + const root = mkdtempSync(join(tmpdir(), 'franken-hive-secure-delete-reuse-')); + const dbPath = join(root, 'hive.db'); + const namespace = hiveMindAgentTypeNamespace('coder'); + const store = new HiveMindStore(dbPath); + const observer = new Database(dbPath); + try { + for (let index = 0; index < 64; index += 1) { + store.publish(namespace, 'publisher-a', { + kind: 'episode', + event: { + type: 'failure', + summary: `reusable-page-${index}-${'payload-'.repeat(1_024)}`, + createdAt: '2026-07-25T10:00:00.000Z', + }, + }); + } + observer.pragma('wal_checkpoint(TRUNCATE)'); + const pageCount = observer.pragma('page_count', { simple: true }) as number; + + expect(store.deletePublishedWhere(namespace, 'publisher-a', () => true)).toBe(64); + + expect(observer.pragma('page_count', { simple: true })).toBe(pageCount); + expect(observer.pragma('freelist_count', { simple: true }) as number).toBeGreaterThan(0); + } finally { + observer.close(); + store.close(); + rmSync(root, { recursive: true, force: true }); + } + }); + it('filters by kind before applying the newest-entry bound', () => { const root = mkdtempSync(join(tmpdir(), 'franken-hive-kind-bound-')); const store = new HiveMindStore(join(root, 'hive.db')); @@ -354,6 +386,42 @@ describe('HiveMindStore', () => { } }); + it('conservatively revokes a dependent legacy hive lesson without a candidate identity', () => { + const root = mkdtempSync(join(tmpdir(), 'franken-hive-forget-legacy-lesson-')); + const hiveDbPath = join(root, 'hive.db'); + const namespace = hiveMindAgentTypeNamespace('coder'); + const registry = new BrainRegistry(join(root, 'brains'), hiveDbPath, 'publisher-a'); + const brain = registry.forAgentType('coder'); + try { + recordCluster(brain, 'LegacyCandidateMarker build timeout'); + const [candidate] = brain.learning.consolidate({ threshold: 3 }); + const publisher = new HiveMindStore(hiveDbPath); + try { + publisher.deleteLessonPublication(namespace, 'publisher-a', candidate!.id, candidate!.key); + publisher.publish(namespace, 'publisher-a', { + kind: 'lesson', + key: candidate!.key, + status: 'pending', + lesson: candidate!.value as ReturnType, + }); + } finally { + publisher.close(); + } + + brain.rightToForget({ type: 'episodic', query: 'LegacyCandidateMarker' }); + + const observer = new HiveMindStore(hiveDbPath); + try { + expect(observer.poll(namespace, { kind: 'lesson' })).toEqual([]); + } finally { + observer.close(); + } + } finally { + registry.close(); + rmSync(root, { recursive: true, force: true }); + } + }); + it('revokes every approved hive lesson absorbed by a bridging revision', () => { const root = mkdtempSync(join(tmpdir(), 'franken-hive-bridge-revision-')); const hiveDbPath = join(root, 'hive.db'); @@ -541,6 +609,67 @@ describe('HiveMindStore', () => { } }); + it('migrates publications owned by the preceding path-derived publisher identity', () => { + const root = mkdtempSync(join(tmpdir(), 'franken-hive-publisher-migration-')); + const brainsDir = join(root, 'brains'); + const hiveDbPath = join(root, 'hive', 'hive.db'); + const durableDbPath = join(brainsDir, 'coder.db'); + const namespace = hiveMindAgentTypeNamespace('coder'); + const legacyPublisherId = createHash('sha256').update(durableDbPath).digest('hex'); + try { + mkdirSync(brainsDir, { recursive: true }); + const legacyBrain = new SqliteBrain(durableDbPath); + legacyBrain.episodic.record({ + type: 'failure', + summary: 'legacy-path-private-token request failed', + createdAt: '2026-07-25T10:00:00.000Z', + }); + legacyBrain.close(); + const publisher = new HiveMindStore(hiveDbPath); + publisher.publish(namespace, legacyPublisherId, { + kind: 'episode', + event: { + type: 'failure', + summary: 'legacy-path-private-token request failed', + createdAt: '2026-07-25T10:00:00.000Z', + }, + }); + publisher.close(); + + const registry = new BrainRegistry(brainsDir, hiveDbPath); + registry.forAgentType('coder', durableDbPath) + .rightToForget({ query: 'legacy-path-private-token' }); + registry.close(); + + const observer = new HiveMindStore(hiveDbPath); + try { + expect(observer.poll(namespace)).toEqual([]); + } finally { + observer.close(); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('returns a cached durable brain without reopening its locked database', () => { + const root = mkdtempSync(join(tmpdir(), 'franken-hive-publisher-cache-')); + const brainsDir = join(root, 'brains'); + const durableDbPath = join(brainsDir, 'coder.db'); + const registry = new BrainRegistry(brainsDir, join(root, 'hive.db')); + const brain = registry.forAgentType('coder', durableDbPath); + const blocker = new Database(durableDbPath); + try { + blocker.exec('BEGIN IMMEDIATE'); + expect(registry.forAgentType('coder', durableDbPath)).toBe(brain); + } finally { + if (blocker.inTransaction) blocker.exec('ROLLBACK'); + blocker.close(); + registry.close(); + rmSync(root, { recursive: true, force: true }); + } + }, 10_000); + it('preserves durable publisher ownership when the same brain is reopened through a symlink', () => { const root = mkdtempSync(join(tmpdir(), 'franken-hive-publisher-symlink-')); const brainsDir = join(root, 'brains'); diff --git a/tasks/hive-pr3775-post-merge-findings-progress.md b/tasks/hive-pr3775-post-merge-findings-progress.md index c7fb99902..71d524ab4 100644 --- a/tasks/hive-pr3775-post-merge-findings-progress.md +++ b/tasks/hive-pr3775-post-merge-findings-progress.md @@ -9,6 +9,10 @@ - [x] Run package tests, typecheck, build, and lint. (`@franken/brain`: 489/489 tests, typecheck, lint, and build pass after the first follow-up Codex fixes; root lint previously passed. Root test has three unrelated orchestrator failures; root typecheck/build remain blocked by pre-existing `franken-web` errors.) - [x] Commit with the required Git identity. - [x] Route push and follow-up PR creation through the dedicated Hive Approval Cop. (PR #3788 created at immutable head `20d0e9c7a2a72c6984c745879861b1419e23d44e`.) -- [ ] Run at most two batched Codex review rounds; resolve all original and follow-up threads through Approval Cop. (Automatic round 1/2 produced three findings; fixes are locally verified and awaiting approval-routed publication/thread closeout.) +- [x] Run at most two batched Codex review rounds. (Round 2/2 identified five current-head findings: `3651129009`, `3651129011`, `3651129012`, `3651129013`, `3651129015`; no third invocation is permitted.) +- [x] Reproduce all five final-round findings with focused failing tests on `ec1367b5bf5aef079fb034bf9e521aa93ee95668`. +- [x] Implement one cap-compliant final batch: migrate path-derived publisher ownership, retry pending WAL purges on store activity/close, return cached brains before DB initialization, conservatively revoke legacy lessons without candidate IDs, and defer page reuse without per-deletion `VACUUM`. +- [x] Re-run focused and package gates. (`HiveMindStore`: 22/22; `@franken/brain`: 493/493 tests, lint, typecheck, and build.) +- [ ] Commit the final batch with required identity and route push plus five thread replies/resolutions through Approval Cop. - [ ] Verify exact-head green CI, zero paginated unresolved Codex threads, and approval-routed merge/closeout. -- [ ] Record remediation evidence and terminalize Kanban card `t_fd5ece5d`. +- [ ] Record remediation evidence and terminalize replacement Kanban card `t_6bb27e14`. From 9bd207dbf44a02e801d4ffda2e61fab73e860986 Mon Sep 17 00:00:00 2001 From: David Mendez Date: Sat, 25 Jul 2026 17:27:20 -0500 Subject: [PATCH 4/6] fix(brain): purge legacy hive ownership --- packages/franken-brain/src/brain-registry.ts | 52 +++++++++++++------ packages/franken-brain/src/hive-mind-store.ts | 20 +++++++ .../tests/unit/hive-mind-store.test.ts | 17 ++++-- ...ive-pr3775-post-merge-findings-progress.md | 5 +- 4 files changed, 75 insertions(+), 19 deletions(-) diff --git a/packages/franken-brain/src/brain-registry.ts b/packages/franken-brain/src/brain-registry.ts index fcf63586f..d6a2eabb9 100644 --- a/packages/franken-brain/src/brain-registry.ts +++ b/packages/franken-brain/src/brain-registry.ts @@ -5,7 +5,7 @@ import { dirname, join, resolve } from 'node:path'; import Database from 'better-sqlite3'; -import { hiveMindAgentTypeNamespace } from './hive-mind-store.js'; +import { HiveMindStore, hiveMindAgentTypeNamespace, type HiveMindNamespace } from './hive-mind-store.js'; import { SqliteBrain } from './sqlite-brain.js'; const MAX_AGENT_TYPE_ID_BYTES = 255; @@ -15,7 +15,11 @@ const WINDOWS_RESERVED_AGENT_TYPE_ID = /^(?:con|prn|aux|nul|clock\$|com[1-9]|lpt[1-9])(?:\.|$)/iu; const HIVE_PUBLISHER_ID_METADATA_KEY = 'hive-publisher-id'; -function durablePublisherId(dbPath: string): string { +function durablePublisherId( + dbPath: string, + hiveDbPath: string, + namespace: HiveMindNamespace, +): string { const db = new Database(dbPath); try { db.pragma('busy_timeout = 5000'); @@ -25,18 +29,35 @@ function durablePublisherId(dbPath: string): string { value TEXT NOT NULL ) `); - db.prepare(` - INSERT OR IGNORE INTO brain_registry_metadata (key, value) - VALUES (?, ?) - `).run( - HIVE_PUBLISHER_ID_METADATA_KEY, - createHash('sha256').update(dbPath).digest('hex'), - ); - const row = db.prepare(` + const selectPublisherId = db.prepare(` SELECT value FROM brain_registry_metadata WHERE key = ? - `).get(HIVE_PUBLISHER_ID_METADATA_KEY) as { value: string } | undefined; - if (!row) throw new Error('Durable brain publisher identity could not be initialized'); - return row.value; + `); + const existing = selectPublisherId.get(HIVE_PUBLISHER_ID_METADATA_KEY) as + { value: string } | undefined; + if (existing) return existing.value; + + const initializePublisherId = db.transaction(() => { + const concurrent = selectPublisherId.get(HIVE_PUBLISHER_ID_METADATA_KEY) as + { value: string } | undefined; + if (concurrent) return concurrent.value; + + const hiveStore = new HiveMindStore(hiveDbPath); + try { + // Previous releases used process-random publisher IDs, so ownership cannot + // be reconstructed safely. Purge only this agent-type namespace before + // adopting a durable identity; otherwise right-to-forget cannot reach it. + hiveStore.deleteNamespace(namespace); + } finally { + hiveStore.close(); + } + + const publisherId = randomUUID(); + db.prepare(` + INSERT INTO brain_registry_metadata (key, value) VALUES (?, ?) + `).run(HIVE_PUBLISHER_ID_METADATA_KEY, publisherId); + return publisherId; + }); + return initializePublisherId.immediate(); } finally { db.close(); } @@ -126,15 +147,16 @@ export class BrainRegistry { if (resolvedDbPath !== ':memory:') { mkdirSync(dirname(resolvedDbPath), { recursive: true }); } + const namespace = hiveMindAgentTypeNamespace(agentTypeId); const publisherId = this.publisherId ?? (resolvedDbPath === ':memory:' ? randomUUID() - : durablePublisherId(resolvedDbPath)); + : durablePublisherId(resolvedDbPath, this.hiveDbPath, namespace)); const brain = new SqliteBrain(resolvedDbPath, undefined, { conversationWorkspaceId: null, hiveMind: { dbPath: resolvedDbPath === ':memory:' ? ':memory:' : this.hiveDbPath, - namespace: hiveMindAgentTypeNamespace(agentTypeId), + namespace, publisherId, }, }); diff --git a/packages/franken-brain/src/hive-mind-store.ts b/packages/franken-brain/src/hive-mind-store.ts index 82140f0b4..242992457 100644 --- a/packages/franken-brain/src/hive-mind-store.ts +++ b/packages/franken-brain/src/hive-mind-store.ts @@ -346,6 +346,26 @@ export class HiveMindStore { ); } + /** Purge every legacy publication in one namespace before durable ownership is adopted. */ + deleteNamespace(namespace: HiveMindNamespace): number { + assertNamespace(namespace); + this.retryPendingSecureDelete(); + const remove = this.db.transaction(() => { + const result = this.db.prepare(` + DELETE FROM hive_mind_entries WHERE namespace = ? + `).run(namespace); + if (result.changes > 0 && this.dbPath !== ':memory:') { + this.db.prepare(` + INSERT OR REPLACE INTO hive_mind_metadata (key, value) VALUES (?, '1') + `).run(SECURE_DELETE_PENDING_KEY); + } + return result.changes; + }); + const removed = remove.immediate(); + if (removed > 0 && this.dbPath !== ':memory:') this.purgeDeletedContent(); + return removed; + } + deletePublishedWhere( namespace: HiveMindNamespace, publisherId: string, diff --git a/packages/franken-brain/tests/unit/hive-mind-store.test.ts b/packages/franken-brain/tests/unit/hive-mind-store.test.ts index 27f38d25e..7f02a5731 100644 --- a/packages/franken-brain/tests/unit/hive-mind-store.test.ts +++ b/packages/franken-brain/tests/unit/hive-mind-store.test.ts @@ -1,4 +1,3 @@ -import { createHash } from 'node:crypto'; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -609,13 +608,14 @@ describe('HiveMindStore', () => { } }); - it('migrates publications owned by the preceding path-derived publisher identity', () => { + it('purges publications owned by the preceding process-random publisher identity', () => { const root = mkdtempSync(join(tmpdir(), 'franken-hive-publisher-migration-')); const brainsDir = join(root, 'brains'); const hiveDbPath = join(root, 'hive', 'hive.db'); const durableDbPath = join(brainsDir, 'coder.db'); const namespace = hiveMindAgentTypeNamespace('coder'); - const legacyPublisherId = createHash('sha256').update(durableDbPath).digest('hex'); + const unrelatedNamespace = hiveMindAgentTypeNamespace('reviewer'); + const legacyPublisherId = 'legacy-process-random-publisher'; try { mkdirSync(brainsDir, { recursive: true }); const legacyBrain = new SqliteBrain(durableDbPath); @@ -634,6 +634,14 @@ describe('HiveMindStore', () => { createdAt: '2026-07-25T10:00:00.000Z', }, }); + publisher.publish(unrelatedNamespace, 'unrelated-publisher', { + kind: 'episode', + event: { + type: 'failure', + summary: 'unrelated publication remains available', + createdAt: '2026-07-25T10:00:00.000Z', + }, + }); publisher.close(); const registry = new BrainRegistry(brainsDir, hiveDbPath); @@ -644,6 +652,9 @@ describe('HiveMindStore', () => { const observer = new HiveMindStore(hiveDbPath); try { expect(observer.poll(namespace)).toEqual([]); + expect(observer.poll(unrelatedNamespace)).toEqual([ + expect.objectContaining({ publisherId: 'unrelated-publisher' }), + ]); } finally { observer.close(); } diff --git a/tasks/hive-pr3775-post-merge-findings-progress.md b/tasks/hive-pr3775-post-merge-findings-progress.md index 71d524ab4..634546061 100644 --- a/tasks/hive-pr3775-post-merge-findings-progress.md +++ b/tasks/hive-pr3775-post-merge-findings-progress.md @@ -13,6 +13,9 @@ - [x] Reproduce all five final-round findings with focused failing tests on `ec1367b5bf5aef079fb034bf9e521aa93ee95668`. - [x] Implement one cap-compliant final batch: migrate path-derived publisher ownership, retry pending WAL purges on store activity/close, return cached brains before DB initialization, conservatively revoke legacy lessons without candidate IDs, and defer page reuse without per-deletion `VACUUM`. - [x] Re-run focused and package gates. (`HiveMindStore`: 22/22; `@franken/brain`: 493/493 tests, lint, typecheck, and build.) -- [ ] Commit the final batch with required identity and route push plus five thread replies/resolutions through Approval Cop. +- [x] Commit the final batch with required identity and route its first push through Approval Cop. (`807bda917432c6e1365fcab82d6c35a98a8c4bdc`.) +- [x] Close the independent-audit P1 gap by purging unidentifiable process-random legacy publications before adopting a durable publisher identity; reproduced with a focused failing test first. +- [x] Re-run focused and package gates. (`HiveMindStore`: 22/22; `@franken/brain`: 493/493 tests, lint, typecheck, build; `git diff --check`.) +- [ ] Commit with the required identity and route the corrective push plus five thread replies/resolutions through Approval Cop. - [ ] Verify exact-head green CI, zero paginated unresolved Codex threads, and approval-routed merge/closeout. - [ ] Record remediation evidence and terminalize replacement Kanban card `t_6bb27e14`. From 2d1f18af9c7bb7de9ffe039b7935a66c9e3c400f Mon Sep 17 00:00:00 2001 From: David Mendez Date: Sat, 25 Jul 2026 18:16:16 -0500 Subject: [PATCH 5/6] fix(brain): serialize publisher migration ownership --- packages/franken-brain/src/brain-registry.ts | 20 +- packages/franken-brain/src/hive-mind-store.ts | 61 ++++- .../tests/unit/hive-mind-store.test.ts | 214 ++++++++++++++++++ .../pr3788-multi-brain-migration-progress.md | 12 + 4 files changed, 302 insertions(+), 5 deletions(-) create mode 100644 tasks/pr3788-multi-brain-migration-progress.md diff --git a/packages/franken-brain/src/brain-registry.ts b/packages/franken-brain/src/brain-registry.ts index d6a2eabb9..fe846295a 100644 --- a/packages/franken-brain/src/brain-registry.ts +++ b/packages/franken-brain/src/brain-registry.ts @@ -34,7 +34,19 @@ function durablePublisherId( `); const existing = selectPublisherId.get(HIVE_PUBLISHER_ID_METADATA_KEY) as { value: string } | undefined; - if (existing) return existing.value; + if (existing) { + try { + const hiveStore = new HiveMindStore(hiveDbPath); + try { + hiveStore.completeLegacyPublisherMigration(namespace, false); + } finally { + hiveStore.close(); + } + } catch { + // Hive sharing is additive; an outage must not block an existing local brain. + } + return existing.value; + } const initializePublisherId = db.transaction(() => { const concurrent = selectPublisherId.get(HIVE_PUBLISHER_ID_METADATA_KEY) as @@ -44,9 +56,9 @@ function durablePublisherId( const hiveStore = new HiveMindStore(hiveDbPath); try { // Previous releases used process-random publisher IDs, so ownership cannot - // be reconstructed safely. Purge only this agent-type namespace before - // adopting a durable identity; otherwise right-to-forget cannot reach it. - hiveStore.deleteNamespace(namespace); + // be reconstructed safely. The shared marker makes this purge atomic and + // once-only across distinct durable brain databases for the same type. + hiveStore.completeLegacyPublisherMigration(namespace, true); } finally { hiveStore.close(); } diff --git a/packages/franken-brain/src/hive-mind-store.ts b/packages/franken-brain/src/hive-mind-store.ts index 242992457..24aa06bbe 100644 --- a/packages/franken-brain/src/hive-mind-store.ts +++ b/packages/franken-brain/src/hive-mind-store.ts @@ -19,6 +19,10 @@ const MAX_POLL_LIMIT = 1_000; const DEFAULT_MAX_ENTRIES_PER_NAMESPACE = 10_000; const MAX_CONFIGURED_ENTRIES_PER_NAMESPACE = 1_000_000; const SECURE_DELETE_PENDING_KEY = 'secure-delete-pending'; +const MIGRATION_SECURE_DELETE_PENDING_VALUE = 'migration'; +const LEGACY_PUBLISHER_MIGRATION_KEY_PREFIX = 'legacy-publisher-migration:'; +const MIGRATION_CHECKPOINT_RETRY_MS = 10; +const MIGRATION_CHECKPOINT_TIMEOUT_MS = 5_000; const UNSAFE_AGENT_TYPE_ID_CHARACTERS = /[<>:"/\\|?*\u0000-\u001f\u007f]/u; const WINDOWS_RESERVED_AGENT_TYPE_ID = /^(?:con|prn|aux|nul|clock\$|com[1-9]|lpt[1-9])(?:\.|$)/iu; @@ -86,6 +90,17 @@ function truncateWalOrThrow(db: Database.Database): void { } } +function truncateWalAfterConcurrentMigrationOrThrow(db: Database.Database): void { + const deadline = Date.now() + MIGRATION_CHECKPOINT_TIMEOUT_MS; + const waitBuffer = new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT)); + do { + const [result] = db.pragma('wal_checkpoint(TRUNCATE)') as WalCheckpointResult[]; + if (result?.busy === 0) return; + Atomics.wait(waitBuffer, 0, 0, MIGRATION_CHECKPOINT_RETRY_MS); + } while (Date.now() < deadline); + throw new Error('Secure deletion could not truncate the Hive WAL because a reader is active'); +} + function assertSafeAgentTypeId(agentTypeId: string): void { if ( typeof agentTypeId !== 'string' @@ -366,6 +381,43 @@ export class HiveMindStore { return removed; } + /** + * Record durable publisher adoption once per namespace, optionally purging + * publications from the preceding process-random ownership model. + */ + completeLegacyPublisherMigration( + namespace: HiveMindNamespace, + purgeLegacyPublications: boolean, + ): boolean { + assertNamespace(namespace); + this.retryPendingSecureDelete(); + const migrationKey = `${LEGACY_PUBLISHER_MIGRATION_KEY_PREFIX}${namespace}`; + const migrate = this.db.transaction(() => { + const migrated = this.db.prepare(` + SELECT 1 FROM hive_mind_metadata WHERE key = ? + `).get(migrationKey); + if (migrated) return { completed: false, removed: 0 }; + + const removed = purgeLegacyPublications + ? this.db.prepare('DELETE FROM hive_mind_entries WHERE namespace = ?').run(namespace).changes + : 0; + this.db.prepare(` + INSERT INTO hive_mind_metadata (key, value) VALUES (?, '1') + `).run(migrationKey); + if (removed > 0 && this.dbPath !== ':memory:') { + this.db.prepare(` + INSERT OR REPLACE INTO hive_mind_metadata (key, value) VALUES (?, ?) + `).run(SECURE_DELETE_PENDING_KEY, MIGRATION_SECURE_DELETE_PENDING_VALUE); + } + return { completed: true, removed }; + }); + const result = migrate.immediate(); + if (result.removed > 0 && this.dbPath !== ':memory:') { + this.purgeDeletedContent(); + } + return result.completed; + } + deletePublishedWhere( namespace: HiveMindNamespace, publisherId: string, @@ -416,7 +468,14 @@ export class HiveMindStore { } private purgeDeletedContent(): void { - truncateWalOrThrow(this.db); + const pending = this.db.prepare(` + SELECT value FROM hive_mind_metadata WHERE key = ? + `).get(SECURE_DELETE_PENDING_KEY) as { value: string } | undefined; + if (pending?.value === MIGRATION_SECURE_DELETE_PENDING_VALUE) { + truncateWalAfterConcurrentMigrationOrThrow(this.db); + } else { + truncateWalOrThrow(this.db); + } this.db.prepare('DELETE FROM hive_mind_metadata WHERE key = ?').run(SECURE_DELETE_PENDING_KEY); } diff --git a/packages/franken-brain/tests/unit/hive-mind-store.test.ts b/packages/franken-brain/tests/unit/hive-mind-store.test.ts index 7f02a5731..c57520c46 100644 --- a/packages/franken-brain/tests/unit/hive-mind-store.test.ts +++ b/packages/franken-brain/tests/unit/hive-mind-store.test.ts @@ -1,6 +1,8 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { Worker } from 'node:worker_threads'; import Database from 'better-sqlite3'; import { describe, expect, it } from 'vitest'; @@ -608,6 +610,95 @@ describe('HiveMindStore', () => { } }); + it('reopens an existing durable brain when its additive Hive store is unavailable', () => { + const root = mkdtempSync(join(tmpdir(), 'franken-hive-publisher-restart-outage-')); + const brainsDir = join(root, 'brains'); + const hiveDbPath = join(root, 'hive', 'hive.db'); + try { + const firstRegistry = new BrainRegistry(brainsDir, hiveDbPath); + firstRegistry.forAgentType('coder').episodic.record({ + type: 'observation', + summary: 'local state survives hive outage', + createdAt: '2026-07-25T10:00:00.000Z', + }); + firstRegistry.close(); + rmSync(join(root, 'hive'), { recursive: true, force: true }); + mkdirSync(hiveDbPath, { recursive: true }); + + const restartedRegistry = new BrainRegistry(brainsDir, hiveDbPath); + try { + expect(restartedRegistry.forAgentType('coder').episodic.recent()[0]?.summary) + .toBe('local state survives hive outage'); + } finally { + restartedRegistry.close(); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it('keeps same-type publications when a second durable brain adopts its own publisher identity', () => { + const root = mkdtempSync(join(tmpdir(), 'franken-hive-multi-brain-publisher-')); + const brainsDir = join(root, 'brains'); + const hiveDbPath = join(root, 'hive', 'hive.db'); + const namespace = hiveMindAgentTypeNamespace('coder'); + const firstRegistry = new BrainRegistry(brainsDir, hiveDbPath); + const secondRegistry = new BrainRegistry(brainsDir, hiveDbPath); + try { + firstRegistry.forAgentType('coder', join(brainsDir, 'coder-a.db')).episodic.record({ + type: 'failure', + summary: 'first durable brain publication survives peer initialization', + createdAt: '2026-07-25T10:00:00.000Z', + }); + const before = new HiveMindStore(hiveDbPath); + try { + expect(before.poll(namespace)).toHaveLength(1); + } finally { + before.close(); + } + + secondRegistry.forAgentType('coder', join(brainsDir, 'coder-b.db')).episodic.record({ + type: 'failure', + summary: 'second durable brain has distinct ownership', + createdAt: '2026-07-25T10:01:00.000Z', + }); + + const after = new HiveMindStore(hiveDbPath); + try { + const publications = after.poll(namespace); + expect(publications).toHaveLength(2); + expect(new Set(publications.map(({ publisherId }) => publisherId)).size).toBe(2); + } finally { + after.close(); + } + + firstRegistry.close(); + secondRegistry.close(); + const restartedFirst = new BrainRegistry(brainsDir, hiveDbPath); + const restartedSecond = new BrainRegistry(brainsDir, hiveDbPath); + restartedFirst.forAgentType('coder', join(brainsDir, 'coder-a.db')) + .rightToForget({ query: 'first durable brain publication' }); + restartedSecond.forAgentType('coder', join(brainsDir, 'coder-b.db')); + restartedFirst.close(); + restartedSecond.close(); + + const afterRestart = new HiveMindStore(hiveDbPath); + try { + expect(afterRestart.poll(namespace)).toEqual([ + expect.objectContaining({ + event: expect.objectContaining({ summary: 'second durable brain has distinct ownership' }), + }), + ]); + } finally { + afterRestart.close(); + } + } finally { + firstRegistry.close(); + secondRegistry.close(); + rmSync(root, { recursive: true, force: true }); + } + }); + it('purges publications owned by the preceding process-random publisher identity', () => { const root = mkdtempSync(join(tmpdir(), 'franken-hive-publisher-migration-')); const brainsDir = join(root, 'brains'); @@ -663,6 +754,129 @@ describe('HiveMindStore', () => { } }); + it('serializes legacy migration across concurrent durable brain initialization', async () => { + const root = mkdtempSync(join(tmpdir(), 'franken-hive-publisher-concurrent-migration-')); + const brainsDir = join(root, 'brains'); + const hiveDbPath = join(root, 'hive', 'hive.db'); + const namespace = hiveMindAgentTypeNamespace('coder'); + const unrelatedNamespace = hiveMindAgentTypeNamespace('reviewer'); + const workers: Worker[] = []; + const sourcePath = fileURLToPath(new URL('../../src/brain-registry.ts', import.meta.url)); + const vitestConfigPath = fileURLToPath(new URL('../../vitest.config.ts', import.meta.url)); + const workerScript = String.raw` + const { parentPort, workerData } = require('node:worker_threads'); + void (async () => { + const { createServer } = await import('vite'); + const vite = await createServer({ + configFile: workerData.vitestConfigPath, + logLevel: 'silent', + server: { middlewareMode: true, hmr: false, watch: null }, + }); + const { BrainRegistry } = await vite.ssrLoadModule(workerData.sourcePath); + parentPort.postMessage({ type: 'ready' }); + parentPort.once('message', async (message) => { + if (message?.type !== 'start') return; + try { + const registry = new BrainRegistry(workerData.brainsDir, workerData.hiveDbPath); + registry.forAgentType('coder', workerData.brainDbPath); + registry.close(); + await vite.close(); + parentPort.postMessage({ type: 'done' }); + } catch (error) { + await vite.close(); + parentPort.postMessage({ + type: 'error', + message: error instanceof Error ? error.stack ?? error.message : String(error), + }); + } + }); + })().catch((error) => parentPort.postMessage({ + type: 'error', + message: error instanceof Error ? error.stack ?? error.message : String(error), + })); + `; + + try { + const publisher = new HiveMindStore(hiveDbPath); + publisher.publish(namespace, 'legacy-process-random-publisher', { + kind: 'episode', + event: { + type: 'failure', + summary: 'legacy publication is purged exactly once', + createdAt: '2026-07-25T10:00:00.000Z', + }, + }); + publisher.publish(unrelatedNamespace, 'unrelated-publisher', { + kind: 'episode', + event: { + type: 'observation', + summary: 'unrelated publication survives concurrent migration', + createdAt: '2026-07-25T10:01:00.000Z', + }, + }); + publisher.close(); + + const controls = Array.from({ length: 2 }, (_, index) => { + const worker = new Worker(workerScript, { + eval: true, + workerData: { + brainDbPath: join(brainsDir, `coder-${index}.db`), + brainsDir, + hiveDbPath, + sourcePath, + vitestConfigPath, + }, + }); + workers.push(worker); + let readyResolve: (() => void) | undefined; + let doneResolve: (() => void) | undefined; + let reject: ((error: Error) => void) | undefined; + const ready = new Promise((resolve) => { readyResolve = resolve; }); + const done = new Promise((resolve, fail) => { + doneResolve = resolve; + reject = fail; + }); + worker.on('message', (message: { type?: string; message?: string }) => { + if (message.type === 'ready') readyResolve?.(); + if (message.type === 'done') doneResolve?.(); + if (message.type === 'error') reject?.(new Error(message.message ?? 'worker failed')); + }); + worker.on('error', error => reject?.( + error instanceof Error ? error : new Error(String(error)), + )); + return { worker, ready, done }; + }); + + await Promise.all(controls.map(({ ready }) => ready)); + for (const { worker } of controls) worker.postMessage({ type: 'start' }); + await Promise.all(controls.map(({ done }) => done)); + + const observer = new HiveMindStore(hiveDbPath); + try { + expect(observer.poll(namespace)).toEqual([]); + expect(observer.poll(unrelatedNamespace)).toEqual([ + expect.objectContaining({ publisherId: 'unrelated-publisher' }), + ]); + } finally { + observer.close(); + } + const publisherIds = Array.from({ length: 2 }, (_, index) => { + const db = new Database(join(brainsDir, `coder-${index}.db`), { readonly: true }); + try { + return (db.prepare(` + SELECT value FROM brain_registry_metadata WHERE key = 'hive-publisher-id' + `).get() as { value: string }).value; + } finally { + db.close(); + } + }); + expect(new Set(publisherIds)).toHaveLength(2); + } finally { + await Promise.all(workers.map(worker => worker.terminate())); + rmSync(root, { recursive: true, force: true }); + } + }, 20_000); + it('returns a cached durable brain without reopening its locked database', () => { const root = mkdtempSync(join(tmpdir(), 'franken-hive-publisher-cache-')); const brainsDir = join(root, 'brains'); diff --git a/tasks/pr3788-multi-brain-migration-progress.md b/tasks/pr3788-multi-brain-migration-progress.md new file mode 100644 index 000000000..1d4a4bc5c --- /dev/null +++ b/tasks/pr3788-multi-brain-migration-progress.md @@ -0,0 +1,12 @@ +# PR #3788 multi-brain publisher migration repair progress + +- [x] Confirm canonical worktree and exact PR head `9bd207dbf44a02e801d4ffda2e61fab73e860986`. +- [x] Read card, parent verifier evidence, package README, and shared lessons. +- [x] Reproduce the same-type second-brain publication loss with a focused RED test. +- [x] Implement a shared-Hive, concurrency-safe once-per-namespace legacy migration marker while preserving distinct durable publisher identities. +- [x] Add/verify restart persistence, first-time legacy purge, unrelated namespace preservation, and concurrent initialization coverage. +- [x] Run focused Hive tests and full `@franken/brain` test/lint/typecheck/build plus `git diff --check` (496/496 tests; concurrent migration stress 10/10). +- [x] Independently audit the diff for blocking data-loss/concurrency/privacy regressions; fixed the checkpoint race and additive-Hive restart finding, then reran all gates. +- [x] Commit with David Mendez identity and record the immutable fast-forward push command for Hive Approval Cop. +- [ ] Route the authorized push through Hive Approval Cop; do not trigger Codex or merge. +- [ ] Verify exact-head 4/4 CI, resolve only current threads if any, and post evidence to verifier/root cards. From 924f116231f8dea35d5d8fbdec423803a4ee3742 Mon Sep 17 00:00:00 2001 From: David Mendez Date: Sat, 25 Jul 2026 18:38:21 -0500 Subject: [PATCH 6/6] fix(brain): purge pre-marker legacy publications --- packages/franken-brain/src/brain-registry.ts | 2 +- .../tests/unit/hive-mind-store.test.ts | 48 +++++++++++++++++++ .../pr3788-multi-brain-migration-progress.md | 4 +- 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/packages/franken-brain/src/brain-registry.ts b/packages/franken-brain/src/brain-registry.ts index fe846295a..957d9fbc9 100644 --- a/packages/franken-brain/src/brain-registry.ts +++ b/packages/franken-brain/src/brain-registry.ts @@ -38,7 +38,7 @@ function durablePublisherId( try { const hiveStore = new HiveMindStore(hiveDbPath); try { - hiveStore.completeLegacyPublisherMigration(namespace, false); + hiveStore.completeLegacyPublisherMigration(namespace, true); } finally { hiveStore.close(); } diff --git a/packages/franken-brain/tests/unit/hive-mind-store.test.ts b/packages/franken-brain/tests/unit/hive-mind-store.test.ts index c57520c46..f5fba3525 100644 --- a/packages/franken-brain/tests/unit/hive-mind-store.test.ts +++ b/packages/franken-brain/tests/unit/hive-mind-store.test.ts @@ -754,6 +754,54 @@ describe('HiveMindStore', () => { } }); + it('purges legacy publications when a durable publisher ID predates the shared marker', () => { + const root = mkdtempSync(join(tmpdir(), 'franken-hive-publisher-marker-upgrade-')); + const brainsDir = join(root, 'brains'); + const hiveDbPath = join(root, 'hive', 'hive.db'); + const durableDbPath = join(brainsDir, 'coder.db'); + const namespace = hiveMindAgentTypeNamespace('coder'); + try { + mkdirSync(brainsDir, { recursive: true }); + const legacyBrain = new SqliteBrain(durableDbPath); + legacyBrain.close(); + const brainDb = new Database(durableDbPath); + brainDb.exec(` + CREATE TABLE brain_registry_metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ) + `); + brainDb.prepare(` + INSERT INTO brain_registry_metadata (key, value) VALUES ('hive-publisher-id', ?) + `).run('durable-id-created-before-shared-marker'); + brainDb.close(); + + const publisher = new HiveMindStore(hiveDbPath); + publisher.publish(namespace, 'legacy-process-random-publisher', { + kind: 'episode', + event: { + type: 'failure', + summary: 'legacy publication predates the shared migration marker', + createdAt: '2026-07-25T10:00:00.000Z', + }, + }); + publisher.close(); + + const registry = new BrainRegistry(brainsDir, hiveDbPath); + registry.forAgentType('coder', durableDbPath); + registry.close(); + + const observer = new HiveMindStore(hiveDbPath); + try { + expect(observer.poll(namespace)).toEqual([]); + } finally { + observer.close(); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + it('serializes legacy migration across concurrent durable brain initialization', async () => { const root = mkdtempSync(join(tmpdir(), 'franken-hive-publisher-concurrent-migration-')); const brainsDir = join(root, 'brains'); diff --git a/tasks/pr3788-multi-brain-migration-progress.md b/tasks/pr3788-multi-brain-migration-progress.md index 1d4a4bc5c..e48c08d1b 100644 --- a/tasks/pr3788-multi-brain-migration-progress.md +++ b/tasks/pr3788-multi-brain-migration-progress.md @@ -5,8 +5,8 @@ - [x] Reproduce the same-type second-brain publication loss with a focused RED test. - [x] Implement a shared-Hive, concurrency-safe once-per-namespace legacy migration marker while preserving distinct durable publisher identities. - [x] Add/verify restart persistence, first-time legacy purge, unrelated namespace preservation, and concurrent initialization coverage. -- [x] Run focused Hive tests and full `@franken/brain` test/lint/typecheck/build plus `git diff --check` (496/496 tests; concurrent migration stress 10/10). -- [x] Independently audit the diff for blocking data-loss/concurrency/privacy regressions; fixed the checkpoint race and additive-Hive restart finding, then reran all gates. +- [x] Run focused Hive tests and full `@franken/brain` test/lint/typecheck/build plus `git diff --check` (497/497 tests; concurrent migration stress 10/10). +- [x] Independently audit the diff for blocking data-loss/concurrency/privacy regressions; fixed the checkpoint race, additive-Hive restart finding, and pre-marker durable-ID privacy gap, then reran all gates. - [x] Commit with David Mendez identity and record the immutable fast-forward push command for Hive Approval Cop. - [ ] Route the authorized push through Hive Approval Cop; do not trigger Codex or merge. - [ ] Verify exact-head 4/4 CI, resolve only current threads if any, and post evidence to verifier/root cards.