Repository navigation
Expand file tree
/
Copy pathevidence_cache.ts
More file actions
225 lines (205 loc) · 9.76 KB
/
Copy pathevidence_cache.ts
File metadata and controls
225 lines (205 loc) · 9.76 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
import crypto from 'node:crypto'
import fs from 'node:fs'
import path from 'node:path'
import { dataDir } from './constants.js'
import { displaySafePath, hostPathOfIndexKey, normalizePath } from './paths.js'
import { redactIfDotenv } from './dotenv_redact.js'
import { redactSecrets } from './secret_redact.js'
import { embedTexts, isAvailable } from './embeddings.js'
import { decodeSource, ensureDirSync, statSize } from './util.js'
const MAX_ENTRIES = 500
const MAX_TEXT_BYTES = 128 * 1024
// UTF-32, the least dense encoding decodeSource reads, still yields a UTF-8 byte for each four-byte unit after its four-byte mark and drops up to three trailing bytes, so no longer file can decode to text under the cap.
const MAX_FILE_BYTES = 4 * MAX_TEXT_BYTES + 7
const CACHE_FILE = 'workspace-evidence.json'
export type EvidenceRepresentation = 'file' | 'tool-output'
export interface EvidenceEntry {
id: string
projectRoot: string
source: string
representation: EvidenceRepresentation
contentHash: string
text: string
createdAt: number
embedding?: string
}
function cachePath(): string {
return path.join(dataDir(), CACHE_FILE)
}
function hash(text: string): string {
return crypto.createHash('sha256').update(text).digest('hex')
}
function load(): EvidenceEntry[] {
try {
const parsed: unknown = JSON.parse(fs.readFileSync(cachePath(), 'utf8'))
if (!Array.isArray(parsed)) return []
return parsed.filter((entry): entry is EvidenceEntry =>
typeof entry === 'object' && entry !== null &&
typeof (entry as EvidenceEntry).id === 'string' &&
typeof (entry as EvidenceEntry).projectRoot === 'string' &&
typeof (entry as EvidenceEntry).source === 'string' &&
((entry as EvidenceEntry).representation === 'file' || (entry as EvidenceEntry).representation === 'tool-output') &&
typeof (entry as EvidenceEntry).contentHash === 'string' &&
typeof (entry as EvidenceEntry).text === 'string' &&
typeof (entry as EvidenceEntry).createdAt === 'number' &&
((entry as EvidenceEntry).embedding === undefined || typeof (entry as EvidenceEntry).embedding === 'string'),
)
} catch {
return []
}
}
function save(entries: readonly EvidenceEntry[]): void {
try {
ensureDirSync(dataDir())
fs.writeFileSync(cachePath(), JSON.stringify(entries.slice(0, MAX_ENTRIES)), 'utf8')
} catch {
// Evidence caching is an optimization and must never block the caller.
}
}
/** A file's text as its evidence is hashed, recorded and verified: decoded by its byte-order mark the way the indexer reads it, or null for a file that cannot fit the cache, which a stat settles before any of a longer file is read. Every side of the cache reads through here, since a hash taken over one decoding never matches a hash taken over another. `filePath` is the index key the entry is recorded under, read at its host path. */
export function readEvidenceFileText(filePath: string): string | null {
const onDisk = hostPathOfIndexKey(filePath)
const size = statSize(onDisk)
if (size === null || size > MAX_FILE_BYTES) return null
const text = decodeSource(fs.readFileSync(onDisk))
return Buffer.byteLength(text, 'utf8') > MAX_TEXT_BYTES ? null : text
}
export function recordEvidence(input: {
projectRoot: string
source: string
representation: EvidenceRepresentation
text: string
}): EvidenceEntry | null {
if (Buffer.byteLength(input.text, 'utf8') > MAX_TEXT_BYTES) return null
const contentHash = hash(input.text)
// redactSecrets is keyword-driven, so on a dotenv file it catches `API_KEY=...` and misses `DEBUG=true` or any value whose key name does not look like a secret -- a partial redaction, which reads as handled and is not. Apply the file-type rule first: in a dotenv file every value is secret regardless of its key. See dotenv_redact.ts.
const text = redactSecrets(redactIfDotenv(input.source, input.text)).text
const projectRoot = normalizePath(input.projectRoot)
const source = input.representation === 'file' ? normalizePath(input.source) : input.source
const id = hash(`${projectRoot}\0${source}\0${input.representation}\0${contentHash}`).slice(0, 24)
const entry: EvidenceEntry = {
id,
projectRoot,
source,
representation: input.representation,
contentHash,
text,
createdAt: Date.now(),
}
const entries = load().filter((candidate) =>
candidate.projectRoot !== projectRoot ||
candidate.source !== source ||
candidate.representation !== input.representation,
)
entries.unshift(entry)
save(entries)
return entry
}
export function findVerifiedFileEvidence(
projectRoot: string,
source: string,
currentText: string | (() => string | null),
): EvidenceEntry | null {
const normalizedRoot = normalizePath(projectRoot)
const normalizedSource = normalizePath(source)
const candidates = load().filter((entry) =>
entry.projectRoot === normalizedRoot &&
entry.source === normalizedSource &&
entry.representation === 'file',
)
if (candidates.length === 0) return null
// The hook can defer opening the file until the cache proves there is a matching path to verify.
const text = typeof currentText === 'function' ? currentText() : currentText
if (text === null) return null
const currentHash = hash(text)
return candidates.find((entry) => entry.contentHash === currentHash) ?? null
}
export function searchEvidence(projectRoot: string, query: string, limit = 10): EvidenceEntry[] {
const tokens = query.toLowerCase().split(/\s+/).filter(Boolean)
if (tokens.length === 0) return []
const root = normalizePath(projectRoot)
return load()
.filter((entry) => entry.projectRoot === root)
.map((entry) => ({ entry, score: tokens.reduce((score, token) => score + Number(entry.text.toLowerCase().includes(token)), 0) }))
.filter(({ score }) => score > 0)
.sort((a, b) => b.score - a.score || b.entry.createdAt - a.entry.createdAt)
.slice(0, limit)
.map(({ entry }) => entry)
}
function decodeEmbedding(encoded: string | undefined): Float32Array | null {
if (encoded === undefined) return null
try {
const bytes = Buffer.from(encoded, 'base64')
if (bytes.length === 0 || bytes.length % Float32Array.BYTES_PER_ELEMENT !== 0) return null
return new Float32Array(bytes.buffer, bytes.byteOffset, bytes.length / Float32Array.BYTES_PER_ELEMENT)
} catch {
return null
}
}
function encodeEmbedding(vector: readonly number[]): string {
const values = Float32Array.from(vector)
return Buffer.from(values.buffer, values.byteOffset, values.byteLength).toString('base64')
}
function cosineSimilarity(left: Float32Array, right: Float32Array): number {
if (left.length !== right.length) return Number.NEGATIVE_INFINITY
let dot = 0
let leftMagnitude = 0
let rightMagnitude = 0
for (let index = 0; index < left.length; index += 1) {
const a = left[index] ?? 0
const b = right[index] ?? 0
dot += a * b
leftMagnitude += a * a
rightMagnitude += b * b
}
return leftMagnitude === 0 || rightMagnitude === 0
? Number.NEGATIVE_INFINITY
: dot / Math.sqrt(leftMagnitude * rightMagnitude)
}
export async function searchEvidenceSemantically(projectRoot: string, query: string, limit = 10): Promise<EvidenceEntry[]> {
if (query.trim() === '' || !isAvailable()) return []
const root = normalizePath(projectRoot)
// Score every entry the project holds (bounded by MAX_ENTRIES = 500 total across all projects, so this project's slice is cheap to run cosine similarity over) before capping to `limit`: a pre-score slice by recency would drop entries that outrank whatever survives it, silently reporting a miss for evidence the cache actually holds.
const entries = load().filter((entry) => entry.projectRoot === root)
if (entries.length === 0) return []
try {
const missing = entries.filter((entry) => decodeEmbedding(entry.embedding) === null)
const vectors = await embedTexts([query, ...missing.map((entry) => entry.text)])
const queryVector = vectors[0]
if (queryVector === undefined) return []
for (let index = 0; index < missing.length; index += 1) {
const vector = vectors[index + 1]
if (vector !== undefined) missing[index]!.embedding = encodeEmbedding(vector)
}
if (missing.length > 0) {
const updated = new Map(entries.map((entry) => [entry.id, entry]))
save(load().map((entry) => updated.get(entry.id) ?? entry))
}
const queryValues = Float32Array.from(queryVector)
return entries
.map((entry) => ({ entry, score: cosineSimilarity(queryValues, decodeEmbedding(entry.embedding) ?? new Float32Array()) }))
.filter(({ score }) => Number.isFinite(score))
.sort((left, right) => right.score - left.score || right.entry.createdAt - left.entry.createdAt)
.slice(0, limit)
.map(({ entry }) => entry)
} catch {
return []
}
}
export function buildDeltaCapsule(projectRoot: string, limit = 8): string | null {
const root = normalizePath(projectRoot)
const changed = load()
.filter((entry) => entry.projectRoot === root && entry.representation === 'file')
.filter((entry) => {
try {
const current = readEvidenceFileText(entry.source)
return current === null || hash(current) !== entry.contentHash
} catch {
return true
}
})
.slice(0, limit)
if (changed.length === 0) return null
// The path is repository-authored: it comes from whatever file a Read touched, and this capsule is delivered as a SessionStart context block token-goat speaks in its own voice. A checkout containing a file named with our own marker would otherwise put an instruction-shaped line into that block, so the path goes through the same display helper every other model-facing path in this codebase uses.
return `Cross-session evidence changed since it was cached:\n${changed.map((entry) => `- ${displaySafePath(entry.source)} (use a fresh surgical read)`).join('\n')}`
}