Repository navigation
Expand file tree
/
Copy pathsecret_redact.ts
More file actions
264 lines (233 loc) · 35.7 KB
/
Copy pathsecret_redact.ts
File metadata and controls
264 lines (233 loc) · 35.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
/** Defense-in-depth secret redaction for {@link file://./disk_cache.ts}'s `storeBlob()` choke point. token-goat persists a lot of tool output to disk (bash-output, web-output, and mcp-output caches, all funneling through `storeBlob()`). If a cached tool result happens to contain a real credential (an API key echoed by a misconfigured script, a token pasted into a bash command's output), it would otherwise sit in plaintext under `~/.token-goat` indefinitely. This module scans the JSON-serialized blob text for a small set of high-confidence secret patterns and replaces each match with a fixed-width placeholder before it ever reaches disk. Deliberately a small, high-confidence pattern set rather than an exhaustive one: broad heuristics (generic "api_key=..." key/value pairs, high-entropy hex/base64 blobs, bare AWS secret-access-key strings with no anchoring prefix) false-fire constantly on normal code, JSON, and log output, and a redaction pass that mangles ordinary content is worse for a caching layer than one that misses an unusual credential format. Each pattern here has a distinctive, low-collision prefix or block marker. `src/pack.ts` has its own `SECRET_PATTERNS` list, but it exists for a different purpose (scanning project files for `token-goat pack`'s report-only secret warning) and is deliberately noisier — it includes generic "api_key=", "password=", and database-URL patterns that are fine for a human-reviewed report but too false-positive-prone to blindly mangle cached tool output. That list is also module-private. This module keeps a separate, narrower set tuned for automatic in-place redaction. All patterns are single-pass and non-backtracking: fixed-width, bounded, or a single negated-class quantifier whose alternatives are disjoint, so nothing here has a nested or overlapping quantifier for the engine to explore two ways. Every quantifier inside a lookbehind is bounded, which is a stronger requirement than the rest of the pattern needs and is not decoration. A variable-length lookbehind is re-evaluated at each start position, so an unbounded run inside one turns the pass quadratic even though no single match backtracks: the generic assignment pattern below once held `\s*` around its separator, and the literal input `'password' + ' '.repeat(n) + '=!'` took 108 ms at n=20000 and 1726 ms at n=80000 -- four times the work for twice the input, on a path that runs over every command output before it reaches the model. Bounding the run made the same input 0.5 ms and 1.9 ms. Keep quantifiers inside a lookbehind bounded. */
import { loadConfig, type Config } from './config.js'
import { growsExponentially, hasNestedQuantifier } from './regex_guard.js'
const SECRET_PATTERNS: Array<[string, RegExp]> = [
// Anthropic keys share OpenAI's "sk-" prefix but are more specific ("sk-ant-"), so they're matched first — the generic OpenAI pattern's negative lookahead below is defense in depth, not the sole guard.
['anthropic_api_key', /sk-ant-[A-Za-z0-9_-]{20,}/g],
// Modern (post-2024, now the default) OpenAI project keys: "sk-proj-" followed by a long base64url-ish body that legitimately contains '-' and '_' -- the generic pattern below deliberately excludes those characters (hyphenated prose would false-fire), so this needs its own entry with the more specific prefix matched first, like sk-ant- above.
['openai_project_key', /sk-proj-[A-Za-z0-9_-]{20,}/g],
['openai_api_key', /sk-(?!ant-|proj-)[A-Za-z0-9]{20,}/g],
['aws_access_key', /AKIA[0-9A-Z]{16}/g],
// Fine-grained PATs ("github_pat_...") are matched before the classic gh[oprsu]_ pattern so the full token is always consumed as one match -- a fine-grained token's own body can contain '_' and could otherwise partially match the classic pattern.
['github_token', /github_pat_[A-Za-z0-9_]{22,}/g],
['github_token', /gh[oprsu]_[A-Za-z0-9]{36,}/g],
// xapp- is Slack's app-level token (Socket Mode), a bearer credential in its own right that the xox[baprs]- prefix does not cover. It gets its own entry rather than joining the alternation above because it needs the full segmented shape (xapp-<ver>-<app id>-<digits>- <hex>) to be safe: a bare /xapp-[A-Za-z0-9-]+/ redacted ordinary identifiers like "xapp-config" and the css class "xapp-container", mangling normal source. The xox* prefixes are distinctive enough on their own; "xapp-" is not.
['slack_token', /xox[baprs]-[A-Za-z0-9-]+/g],
['slack_token', /xapp-\d-[A-Za-z0-9]{6,}-\d{8,}-[A-Za-z0-9]{16,}/g],
// Matches the full block (BEGIN marker through its matching END marker), not just the header -- the actual secret material is the base64 body between them, so redacting only the header line would leave the key bytes themselves fully readable in the cached blob. The body is lazy and additionally refuses to cross a following BEGIN marker. Both halves matter. The laziness bounds a successful match at the very next END marker; the negative lookahead bounds a *failing* one, because a BEGIN with no END of its own would otherwise scan to end-of-input, and a blob full of such markers made the pass quadratic -- 6 ms, 20 ms and 78 ms for 2000, 4000 and 8000 of them, four times the work for twice the input, the same shape as the lookbehind incident described above. Private key blocks do not nest, so refusing to cross a BEGIN costs nothing in correctness. The algorithm and ` BLOCK` groups are backreferenced in the END marker rather than repeated, so a BEGIN only ever pairs with its own END spelling. Written as two independent optional groups, `BEGIN PGP PRIVATE KEY` would happily close on a distant `END RSA PRIVATE KEY BLOCK` and redact everything in between. A non-participating group backreferences as the empty string in JS, which is exactly what the unprefixed `BEGIN PRIVATE KEY` form needs. The algorithm list covers every armored private-key header openssl, ssh-keygen and gpg actually emit, not just the ones a first draft happened to think of. ENCRYPTED is the PKCS#8 passphrase-protected form (`openssl genpkey -aes256`, `ssh-keygen -m PKCS8`) and is the most common shape of all; DSA is legacy but still written verbatim; PGP carries the ` BLOCK` suffix, which is why that suffix is optional here. All three used to fall through to disk in full. An encrypted key is still key material: the passphrase can be attacked offline once the bytes are cached, so it is redacted like any other. PuTTY `.ppk` files are deliberately not matched. They have no END marker -- the private section is a `Private-Lines: N` count followed by exactly N base64 lines -- so bounding a match would take a stateful parse rather than a regex, on a path that runs over every command output. A regex guess at where the body ends is exactly the over-eager match this module's header warns against.
['private_key_block', /-----BEGIN (RSA |DSA |EC |OPENSSH |ENCRYPTED |PGP )?PRIVATE KEY( BLOCK)?-----(?:(?!-----BEGIN )[\s\S])*?-----END \1PRIVATE KEY\2-----/g],
// Redacts only the token itself, not the "Authorization: Bearer " prefix -- the lookbehind anchors on the header name and scheme so the surrounding request-log line stays readable, matching how AWS_ACCESS_KEY_ID=... above keeps its own prefix intact. The optional quotes on either side of the colon are what let this see a header carried in JSON rather than in raw wire format. Without them the lookbehind demanded the colon sit directly against the header name and the scheme directly against the space, so a body like {"Authorization": "Bearer <token>"} -- the shape any logged fetch or MCP result arrives in -- matched nothing and the token was cached verbatim. `token` is the second scheme spelling in wide use -- it is what curl and gh examples pass for GitHub and many other APIs -- and an opaque value behind it carries exactly the same authority as one behind `Bearer`. The trailing gap is `{1,8}` rather than a single space for the same reason every other gap in this lookbehind already is: a hand-aligned or reformatted header ("Authorization: Bearer <token>") is ordinary, and demanding exactly one space there made this the one position in the pattern that a second space defeated.
['auth_bearer_token', /(?<=Authorization["']?[ \t]{0,8}:[ \t]{0,8}["']?[ \t]{0,8}(?:Bearer|token)[ \t]{1,8})[A-Za-z0-9\-._~+/]{10,}=*/gi],
['auth_basic_token', /(?<=Authorization["']?[ \t]{0,8}:[ \t]{0,8}["']?[ \t]{0,8}Basic[ \t]{1,8})[A-Za-z0-9+/]{6,}=*/gi],
// JWTs have no distinctive prefix of their own, but the base64url encoding of the smallest realistic header ('{"alg":' or similar) always starts with "eyJ", so that's the practical anchor here -- each of the three dot-separated segments requires a minimum length to avoid matching a short, coincidentally dotted token. The trailing `={0,2}` on each segment is what makes a padded token match. Base64url as the JWT spec defines it drops the `=` padding, but producers that reach for a plain base64 encoder emit it anyway, and a `=` in the header or payload segment used to defeat the match outright: not a partial redaction, but none at all, so the entire token was printed. `=` cannot appear anywhere except the end of a segment, since it is not one of the characters the segment body allows, so accepting it here cannot widen the match onto anything else.
['jwt', /eyJ[A-Za-z0-9_-]{10,}={0,2}\.[A-Za-z0-9_-]{10,}={0,2}\.[A-Za-z0-9_-]{10,}={0,2}/g],
['npm_token', /npm_[A-Za-z0-9]{36}/g],
// rk_live_ (restricted keys) share the sk_live_/sk_test_ secret-key shape and risk level, so one pattern covers all three rather than adding a near-duplicate entry.
['stripe_key', /(?:sk_live_|sk_test_|rk_live_)[A-Za-z0-9]{20,}/g],
['google_api_key', /AIza[A-Za-z0-9_-]{35}/g],
// Presigned-url signatures: AWS SigV4 (X-Amz-Signature), Google Cloud Storage (X-Goog-Signature), and Azure blob SAS (sig). These are bearer credentials in query-string clothing -- anyone holding the whole url can read or write the object until it expires, and none of the prefix-anchored patterns above match them because the signature is a bare hex or base64 blob with no distinctive prefix of its own. The '[?&]' anchor and the 16-char floor are what make the short, generic 'sig' name safe to key on: a prose or code mention of "sig" never sits directly after a query separator followed by that much opaque token.
['presigned_signature', /(?<=[?&](?:X-Amz-Signature|X-Goog-Signature|sig)=)[A-Za-z0-9%+/=_-]{16,}/gi],
// A password inside a connection url. `postgres://user:hunter2@db.internal` carries the credential in the authority section, where there is no `key=value` separator for the generic pattern below to anchor on, so a DATABASE_URL echoed by a failing migration or a psql error went through untouched. The anchors are what keep this narrow: a scheme's `://`, a userinfo segment, the colon, and a following `@`. `http://host:8080/path` has no `@` and is left alone; so is any url without credentials. The password class admits `:` and `@` (RFC 3986 lets userinfo carry the first, and a sloppy producer leaves the second unencoded), so the greedy run backtracks to the LAST `@` before the host; it still stops at whitespace, `/`, quotes and angle brackets, which keeps a quoted JSON neighbour or a following email out of the match. The username may be empty, because redis-py, Celery and most broker clients write a password-only url as `redis://:password@host:6379/0`, and it may hold an unencoded `@`, because a username that is itself an email address arrives that way; it still excludes `:` so the first colon after the scheme is the one that starts the password.
['url_credentials', /(?<=:\/\/[^\s:/]{0,64}:)[^\s/"'`<>]{1,256}(?=@)/g],
// Azure storage account connection strings (`DefaultEndpointsProtocol=https;AccountName=...; AccountKey=<base64>==;EndpointSuffix=core.windows.net`) carry a full read/write key to the account in the AccountKey field, and none of the generic patterns above catch it: generic_secret_assignment's keyword list (password|passwd|secret|api[_-]?key| access[_-]?token|refresh[_-]?token|id[_-]?token) has nothing that matches "AccountKey", and an unanchored base64-shape pattern was deliberately rejected -- this module's header already warns that a bare high-entropy-blob heuristic false-fires on ordinary code, JSON and log output, and an 88-char base64 value is exactly the shape a hash, a compiled asset digest or a generated id can also take. Anchoring on the literal `AccountKey=` field name instead keeps the match specific to this one connection-string field. The value class is base64 proper (letters, digits, `+`, `/`, trailing `=` padding) and none of those characters include `;`, so the match terminates on its own at the `;` that starts the next `Name=` field -- unlike generic_secret_assignment's separator characters (`& ; # , :`), which double as ordinary credential characters and need a lookahead to tell the two roles apart, `;` is never valid base64 and needs no such lookahead here. The lookbehind keeps `AccountKey=` itself in the output, matching auth_bearer_token and presigned_signature above, so `;EndpointSuffix=core.windows.net` after it stays fully readable too. `SharedAccessKey` is the same credential one Azure service over: Service Bus, Event Hubs and Relay spell it that way (`Endpoint=sb://ns.servicebus.windows.net/;SharedAccessKeyName=Root; SharedAccessKey=<base64>`) and it carries the same authority over that namespace that AccountKey does over a storage account. `SharedAccessKeyName` is a plain identifier rather than a secret, and never matches: the separator in the lookbehind sits directly against the key name, so the `Name` in between stops it dead. The separator is spelled the way auth_bearer_token above spells its own, for the same reasons that comment records having learned the hard way. Optional quotes on either side of it, so the JSON and YAML forms a logged MCP result or api response actually arrives in are matched rather than stopped dead at the opening quote. A bounded gap rather than exactly one space, so a hand-aligned or reformatted `AccountKey = ...` in an appsettings file is not the single variant that defeats the whole pattern. Case-insensitive for the same reason presigned_signature is. The leading word boundary is what keeps the widened name from reaching into the middle of a longer identifier such as `myaccountkey=`.
['azure_storage_key', /(?<=\b(?:AccountKey|SharedAccessKey)["']?[ \t]{0,8}[:=][ \t]{0,8}["']?)[A-Za-z0-9+/]{40,}=*/gi],
// Generic key=value assignments in .env-file and connection-string/query-string shape. The lookbehind again redacts only the value, and the value's character class deliberately excludes whitespace, '&', ';', '#', quote characters, and '[' ']' ':' -- that exclusion is what stops this from swallowing the rest of the line (a trailing comment or the next key=value pair) or the remainder of a query string past the matched parameter, which is exactly the kind of over-eager match this module's own design note above warns broad heuristics produce. The '[' ']' ':' exclusion also matters because this pattern runs last: an earlier pattern's own "OPENAI_API_KEY=[REDACTED:openai_project_key]" replacement text contains "API_KEY=" too, and without excluding those characters this pattern would re-match and double-redact its own placeholder. The length has a lower bound only (no upper bound): capping it at 64 used to leave the tail of any longer secret unredacted in plain text, which is worse than no redaction because it looks handled. A single negated-class quantifier like this cannot backtrack catastrophically -- there is no nested or overlapping quantifier for the engine to explore multiple ways of matching, so removing the upper bound does not introduce a ReDoS risk. Quotes are permitted around the separator, but never inside the value class. A quoted value is the ordinary way secrets are written -- .env files, JSON, YAML, TOML all quote by default -- and the lookbehind used to stop dead at the opening quote, so `API_KEY="..."` passed through in full while the bare `API_KEY=...` was caught. The closing quote of the key name blocked it from the other side too, which is what kept every JSON body unredacted. Keeping quotes out of the value class is still what stops the match running past the closing quote. The keyword may be a prefix of a longer key name rather than the whole of it, so the trailing identifier class below is load-bearing: without it the lookbehind required the keyword to sit immediately before the separator, and AWS_SECRET_ACCESS_KEY=, SECRET_KEY=, and DB_PASSWORD_HASH= all passed through in full. That class matches identifier characters only, so prose that merely mentions a keyword still never reaches a separator and stays unredacted. api[_-]?key covers the apikey and api-key spellings too. `& ; # , :` play two incompatible roles. They separate one field from the next (a query string, a cookie header, an inline env list), and they are also perfectly ordinary credential characters. Rejecting them outright got the first role right and the second badly wrong: the match stopped at the first one and left everything after it in plain text, so `password=corr&horse&battery` redacted four characters and printed the rest, and `DB_PASSWORD=Aa1:xyz` matched nothing at all because the run before the `:` was under the four-character floor. A tail left sitting in the open is the outcome this module's header calls worse than no redaction, because it reads as handled. So the separator role is decided by what follows rather than assumed: one of these characters ends the value only when the next thing along is another `name=` / `name:` pair, which is what an actual field separator is always followed by. `,OTHER=public` and `; other=1` still end it; the `&` in the middle of a passphrase does not. Whitespace, quotes and brackets are unchanged -- they end a value unconditionally, which is also what keeps this pattern from re-matching the `[REDACTED:...]` placeholder it just wrote.
['generic_secret_assignment', /(?<=(?:password|passwd|secret|api[_-]?key|access[_-]?token|refresh[_-]?token|id[_-]?token)[a-z0-9_-]{0,64}["']?[ \t]{0,8}[:=][ \t]{0,8}["']?)(?:\\[^\n]|[^\s\\&;#,:'"[\]{}]|[&;#,:](?![ \t]*[A-Za-z_][A-Za-z0-9_.-]*[ \t]*[:=])){4,}/gi],
]
export interface RedactResult {
/** Text with every detected secret replaced by a `[REDACTED:<kind>]` placeholder. */
text: string
/** Number of secrets redacted (0 when the input had none). */
count: number
}
/** Count the `[REDACTED:<kind>]` placeholders present in `text`. `redactSecrets().count` answers "how many secrets were in the input"; this answers "how many are gone from THIS string" -- and for the `secret_redacted` stat the second is the honest number, because handlers rarely emit the whole redacted text. A poll-diff handler emits a suffix slice, the approved-plan handler emits a truncated prefix, and several branches replace the output with a notice entirely. In each of those, a secret outside the emitted region was removed by slicing, truncation, or replacement -- not by redaction -- so crediting the input count would report a protection that some other mechanism had already provided. Counting the emitted text also makes the number correct by construction for a branch nobody has written yet, which a hand-computed count at each emit site is not. Known false positive, in the over-reporting direction: text that already contained a literal `[REDACTED:foo]` before redaction ran counts as one. That is accepted rather than defended against -- distinguishing them would mean diffing against the pre-redaction string, which reintroduces exactly the slice-alignment problem this exists to avoid. */
export function countRedactionPlaceholders(text: string): number {
return text.match(/\[REDACTED:[a-z0-9_]+\]/g)?.length ?? 0
}
/** The most patterns a `redaction.custom_patterns` list may contain, and the longest any one of them may be. These are not security limits -- the patterns come from the machine's own config file, so somebody able to set them can already do worse. They bound the cost of a mistake: this runs over every command output, and one accidentally pasted 50 KB regex, or a list grown to thousands of entries by a generator, turns a redaction pass into a visible stall with no obvious cause. */
const MAX_CUSTOM_PATTERNS = 64
const MAX_CUSTOM_PATTERN_LENGTH = 512
export interface CustomPatternProblem {
readonly pattern: string
readonly reason: string
}
/** Texts a custom pattern is probed against: the empty string, and a short phrase with word boundaries, since `\b` and a bare lookahead match zero-width there but not against ''. */
const ZERO_WIDTH_PROBES = ['', 'a b']
/** Whether `pattern` matches an empty string anywhere on a probe text, which would make the replace insert a marker at every such position. */
function matchesZeroWidth(compiled: RegExp): boolean {
return ZERO_WIDTH_PROBES.some((probe) => Array.from(probe.matchAll(compiled)).some((m) => m[0].length === 0))
}
/** Compiled custom patterns plus whatever was rejected, memoised on the exact source list. */
let customCache: { key: string; patterns: RegExp[]; problems: CustomPatternProblem[] } | null = null
/** The compiled form of every usable custom pattern, plus what was wrong with the rest. A pattern is rejected, rather than used, when it cannot compile, when it is longer than {@link MAX_CUSTOM_PATTERN_LENGTH}, when it matches the empty string, or when it nests a quantifier inside a quantified group. The last two both come from an adversarial review: - `x*` matches the empty string at every position, so the replace inserts a marker between every character. `hello world` came back as twelve markers wrapped around eleven letters, with the redaction count reported as twelve. Output destroyed, and the counter -- the number an operator would read to decide the rule was working -- inflated to agree with it. - `(a+)+$` did not finish against twenty-two characters in sixty seconds. Both are what a trailing `*` or a copied snippet produces, so neither needs an attacker. Every rejection lands in `problems`, which `token-goat doctor` prints: a redaction rule an operator believes is running and which is not is worse than no rule at all, and that is only true if the silence is broken. */
export function compileCustomPatterns(sources: readonly string[]): {
patterns: RegExp[]
problems: CustomPatternProblem[]
} {
const key = JSON.stringify(sources)
if (customCache !== null && customCache.key === key) {
return { patterns: customCache.patterns, problems: customCache.problems }
}
const patterns: RegExp[] = []
const problems: CustomPatternProblem[] = []
// Blank entries are dropped before the cap rather than after it. The environment variable that carries these is split on line breaks, so a list written with a blank line between groups, or with a trailing newline, arrives with empty entries in it -- and counting those against the cap would silently discard real patterns at the end of the list while reporting nothing.
const written = sources.map((raw) => raw.trim()).filter((source) => source.length > 0)
for (const source of written.slice(0, MAX_CUSTOM_PATTERNS)) {
if (source.length > MAX_CUSTOM_PATTERN_LENGTH) {
problems.push({ pattern: source.slice(0, 60), reason: `longer than ${MAX_CUSTOM_PATTERN_LENGTH} characters` })
continue
}
if (hasNestedQuantifier(source)) {
problems.push({
pattern: source,
reason: 'repeats a group that already repeats, which can take exponential time to match',
})
continue
}
let compiled: RegExp
try {
compiled = new RegExp(source, 'g')
} catch (e) {
problems.push({ pattern: source, reason: e instanceof Error ? e.message : 'is not a valid regular expression' })
continue
}
if (matchesZeroWidth(compiled)) {
problems.push({
pattern: source,
reason: 'matches the empty string, so it would replace every position in the text',
})
continue
}
if (growsExponentially(compiled)) {
problems.push({
pattern: source,
reason: 'its running time doubles as the text grows, which can stall the process on a short input',
})
continue
}
patterns.push(compiled)
}
if (written.length > MAX_CUSTOM_PATTERNS) {
problems.push({
pattern: `(${written.length} entries)`,
reason: `only the first ${MAX_CUSTOM_PATTERNS} custom patterns are used`,
})
}
customCache = { key, patterns, problems }
return { patterns, problems }
}
/** A run long enough, and varied enough, to be worth testing for randomness. Deliberately not anchored to any credential format: strict mode exists for the credentials whose format we do not know. `=` is accepted only as trailing padding, never inside the run. Allowing it in the body let a match run backwards through `TOKEN=` and redact the label along with the value, which loses the one piece of context an operator reading the output actually needs. */
const STRICT_CANDIDATE = /[A-Za-z0-9+/_-]{24,}={0,2}/g
/** Shannon entropy in bits per character. */
function entropyBitsPerChar(s: string): number {
const counts = new Map<string, number>()
for (const ch of s) counts.set(ch, (counts.get(ch) ?? 0) + 1)
let bits = 0
for (const n of counts.values()) {
const p = n / s.length
bits -= p * Math.log2(p)
}
return bits
}
/** How many of {lowercase, uppercase, digit, symbol} a string draws on. */
function characterClasses(s: string): number {
let n = 0
if (/[a-z]/.test(s)) n++
if (/[A-Z]/.test(s)) n++
if (/[0-9]/.test(s)) n++
if (/[+/=_-]/.test(s)) n++
return n
}
/** Whether a candidate run looks like credential material rather than ordinary output. Three conditions, and the character-class count is the one doing most of the work. Entropy alone is a poor discriminator at this length: a 24-character string of distinct characters tops out near log2(24), so a long camelCase identifier scores close to a real token. Requiring three of the four classes separates them, and it spares the two things people most resent losing -- a git SHA and a hex digest are lowercase-plus-digits, two classes, so neither is touched. A base64 credential is almost always three or four. This is a heuristic and is documented as one. It runs only when `redaction.strict` is on. */
function looksLikeCredential(s: string): boolean {
return characterClasses(s) >= 3 && entropyBitsPerChar(s) >= 3.5
}
/** Redact every secret this build knows how to recognise. Three passes, in order, and the order matters. The built-in patterns run first so a credential with a known shape is always labelled with that shape rather than the generic `custom` or `high_entropy`. Custom patterns run second, so an organisation's own rule can still catch what the built-ins missed. Strict mode runs last and only over what the first two left behind, which is what keeps it from relabelling a token the built-ins already handled correctly. `config` is a parameter rather than a module-level read so a test can drive a specific policy without writing a config file; every production caller takes the default. */
/** A note on markers this function did not write. Untrusted text can contain the literal string `[REDACTED:aws_access_key]`, and nothing here removes it, so a marker in the output is not by itself proof that this function put it there. An adversarial review raised defusing incoming markers as a fix. It was tried and reverted: the only way to defuse them is to rewrite them, and that breaks idempotence -- redacting already-redacted text stops being a no-op, which several call sites and their tests rely on, and it puts a second odd-looking marker in front of the reader to solve a cosmetic problem. So the limit is documented instead of papered over: `count` is the number of replacements this call made, never the number of markers in the text, and a marker is not evidence of anything on its own. Nothing downstream should read either as a measure of what was handled. */
/** The one pattern above that is tuned for recall rather than precision, named here so a caller whose cost asymmetry runs the other way can stand down from it. `generic_secret_assignment` keys on a keyword followed by a separator, with no left word boundary (a leading `\b` would stop matching `DB_PASSWORD_HASH=` and `AWS_SECRET_ACCESS_KEY=`, since `_` is a word character) and a trailing identifier class that must stay for the same reason. Those two choices are correct where it runs: on the persistence paths, over-redacting a cached blob costs nothing and a missed credential costs everything. They also make it match ordinary source -- a `PasswordException:` in a doc comment, a `SECRET_PATTERNS: Array<...> =` type annotation, a `secret_redacted:` object key -- so anything that reads its verdict as "this file holds a credential" is wrong roughly one file in ten. */
const RECALL_TUNED_KIND = 'generic_secret_assignment'
/** Whether an already-compiled pattern matches anywhere in `text`. `lastIndex` is cleared on both sides because most of the patterns above are global and are shared module state: a `test` left partway through one caller's text would make the next scan start mid-string and miss a secret sitting before that offset. */
function matchesAnywhere(pattern: RegExp, text: string): boolean {
pattern.lastIndex = 0
const hit = pattern.test(text)
pattern.lastIndex = 0
return hit
}
// matchAll iterates a clone that starts from the original's lastIndex, so resetting it first is enough and the original is never advanced.
function matchesNonEmpty(compiled: RegExp, text: string): boolean {
compiled.lastIndex = 0
for (const match of text.matchAll(compiled)) {
if (match[0].length > 0) return true
}
return false
}
/** Whether `text` matches a secret pattern precise enough to act on when a false positive costs the caller something. Written for the Read-hook fold gates, whose asymmetry is the inverse of the persistence paths': they decline to fold on a match, and a declined fold passes the file through to the model UNREDACTED, so a false positive buys no protection at all and costs the whole fold. Measured against first-party source of 12 kB or more, {@link RECALL_TUNED_KIND} alone vetoed every fold -- skeleton, outline and body -- for 9.4% of files. Every other built-in pattern is anchored on a literal credential prefix, header name or key field, so it stays in. Custom patterns stay in too: they are declared by the machine's own operator, who is entitled to have them treated as precise. So does the strict-mode entropy heuristic, which is off by default and, when on, is a deliberate request to err toward treating opaque blobs as secrets. Reads the same list {@link redactSecrets} does, so a pattern added there is covered here without a second edit, and changes nothing about what gets written to disk. */
export function hasPreciseSecret(text: string, config: Config = loadConfig()): boolean {
for (const [kind, pattern] of SECRET_PATTERNS) {
if (kind === RECALL_TUNED_KIND) continue
if (matchesAnywhere(pattern, text)) return true
}
// A custom pattern can pass the compile-time zero-width probe and still match empty on real text, as `(?<=token:)` does; redactSecrets skips such a match, so counting it here would veto a fold over a file with nothing to redact.
for (const custom of compileCustomPatterns(config.redaction.custom_patterns).patterns) {
if (matchesNonEmpty(custom, text)) return true
}
if (config.redaction.strict) {
for (const candidate of text.match(STRICT_CANDIDATE) ?? []) {
if (looksLikeCredential(candidate)) return true
}
}
return false
}
/** Scan `text` for high-confidence secret patterns and replace each match with `[REDACTED:<kind>]`. Never partially reveals a matched secret. Pure and synchronous: callers decide how to handle a thrown error (regex engine failures are not expected in practice given the patterns above, but this function does not swallow them itself; see `storeBlob()` for the fail-safe wrapping applied at the actual disk-write choke point). */
export function redactSecrets(text: string, config: Config = loadConfig(), opts: { keepLineCount?: boolean } = {}): RedactResult {
let count = 0
let out = text
// With keepLineCount a match spanning lines (a PEM private-key block, a custom pattern) leaves one placeholder per line it covered and keeps each line break, so a caller that numbers lines (`bash-output --file -n`, `--lines A-B`) stays aligned with the original text after it; the default collapses the match to one placeholder.
const placeholder = (tag: string, match: string): string => (opts.keepLineCount === true ? match.split(/(\r?\n)/).map((part, i) => (i % 2 === 1 ? part : tag)).join('') : tag)
for (const [kind, pattern] of SECRET_PATTERNS) {
out = out.replace(pattern, (match) => {
count++
return placeholder(`[REDACTED:${kind}]`, match)
})
}
for (const pattern of compileCustomPatterns(config.redaction.custom_patterns).patterns) {
out = out.replace(pattern, (match) => {
// A zero-width match (a lookbehind-only pattern, say) would insert a marker at a point and hide nothing
if (match.length === 0) return match
count++
return placeholder('[REDACTED:custom]', match)
})
}
if (config.redaction.strict) {
out = out.replace(STRICT_CANDIDATE, (candidate) => {
if (!looksLikeCredential(candidate)) return candidate
count++
return '[REDACTED:high_entropy]'
})
}
return { text: out, count }
}
/** JSON.stringify's spelling of each control character, read off JSON.stringify itself. */
const CONTROL_ESCAPES: ReadonlyMap<string, string> = new Map(
Array.from({ length: 0x20 }, (_, code) => {
const ch = String.fromCharCode(code)
return [ch, JSON.stringify(ch).slice(1, -1)] as const
}),
)
const CONTROL_CHARACTER_OF_ESCAPE: ReadonlyMap<string, string> = new Map([...CONTROL_ESCAPES].map(([ch, escape]) => [escape, ch]))
/** One JSON escape sequence, matched whole, so the second half of an escaped backslash is never read as the start of an escape of its own. */
const JSON_ESCAPE = /\\(?:u[0-9a-fA-F]{4}|[^u])/g
/** The escape of a surrogate. JSON.stringify writes a surrogate as an escape only when it has no partner, and never beside the half that would pair with it, so the code unit it decodes to is still unpaired. */
const SURROGATE_ESCAPE = /^\\u[dD][89a-fA-F][0-9a-fA-F]{2}$/
/** Every character the decoding below can leave raw that JSON.stringify never writes raw: a control character, or a surrogate with no partner beside it. */
// eslint-disable-next-line no-control-regex -- the control characters are the point: JSON.stringify writes none of them raw, so each one here came from the decoding
const DECODED_CHARACTER = /[\u0000-\u001f]|[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/g
/** The character a JSON escape stands for, when it is one the patterns have to read as that character. A quote or a backslash stays escaped, since either one raw would end or break the string it sits in. */
function decodeEscape(escape: string): string {
return CONTROL_CHARACTER_OF_ESCAPE.get(escape) ?? (SURROGATE_ESCAPE.test(escape) ? String.fromCharCode(Number.parseInt(escape.slice(2), 16)) : escape)
}
/** Redact a compact JSON serialization (JSON.stringify output with no indentation) as the text its strings hold. The patterns are written for raw text, and JSON spells every control character as an escape, so a line break reads as the two characters `\` and `n`: an assignment value ran on across it and took the lines after it, a pattern anchored on a word boundary missed a match at the start of a line, and strict mode's candidate run took in the escape's letter and left a lone backslash that made the whole document unparseable. A surrogate with no partner is written as a `\u` escape as well, whose letter and hex digits gave strict mode's run the same start. Each of those escapes is decoded before the patterns run and escaped again after, byte for byte, which is safe because JSON.stringify writes none of those characters raw, so each one in the result came from that decoding. */
export function redactSerializedJson(json: string): RedactResult {
const result = redactSecrets(json.replace(JSON_ESCAPE, decodeEscape))
return { text: result.text.replace(DECODED_CHARACTER, (ch) => CONTROL_ESCAPES.get(ch) ?? JSON.stringify(ch).slice(1, -1)), count: result.count }
}