Skip to content

Commit af5da53

Browse files
committed
fix(canonical-skill): harden CRLF and UTF-8 byte counting in boundary validation
1 parent 9564517 commit af5da53

2 files changed

Lines changed: 56 additions & 16 deletions

File tree

‎scripts/generate-canonical-skill.mjs‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,13 +51,30 @@ function render(blob) {
5151
}
5252

5353
function validateCompactBoundary(source) {
54-
const lines = source.split(/\r?\n/)
55-
const markerIndexes = lines.flatMap((line, index) => line.trim() === COMPACT_END_MARKER ? [index] : [])
54+
// Match skill_cache.ts::contentLineEntries: a marker example inside a fenced
55+
// block is prose, not a boundary. Splitting only on LF also preserves CRLF
56+
// bytes inside the extracted compact slice.
57+
const lines = source.split('\n')
58+
const markerIndexes = []
59+
let inCodeBlock = false
60+
for (const [index, line] of lines.entries()) {
61+
const stripped = line.trim()
62+
if (stripped.startsWith('```') || stripped.startsWith('~~~')) {
63+
inCodeBlock = !inCodeBlock
64+
continue
65+
}
66+
if (!inCodeBlock && stripped === COMPACT_END_MARKER) markerIndexes.push(index)
67+
}
68+
5669
if (markerIndexes.length !== 1) {
57-
return `src/canonical_skill.md must contain exactly one standalone ${COMPACT_END_MARKER} marker (found ${markerIndexes.length}).`
70+
return `src/canonical_skill.md must contain exactly one standalone ${COMPACT_END_MARKER} marker outside fenced code blocks (found ${markerIndexes.length}).`
5871
}
5972

6073
const compact = lines.slice(0, markerIndexes[0]).join('\n').trim()
74+
if (!compact) {
75+
return `src/canonical_skill.md ${COMPACT_END_MARKER} marker must follow a non-empty compact slice.`
76+
}
77+
6178
const compactBytes = Buffer.byteLength(compact, 'utf8')
6279
if (compactBytes > MAX_COMPACT_PREFIX_BYTES) {
6380
return `src/canonical_skill.md compact slice exceeds the ${MAX_COMPACT_PREFIX_BYTES}-byte limit (${compactBytes} bytes before the marker).`

‎tests/guards/canonical_skill_source.test.ts‎

Lines changed: 36 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import { fileURLToPath } from 'node:url'
88
import { afterAll, describe, expect, it } from 'vitest'
99

1010
import { CANONICAL_SKILL_MD } from '../../src/canonical_skill.js'
11+
import { extractCompactFromMarker } from '../../src/skill_cache.js'
1112

1213
const REPO = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', '..')
1314
const SCRIPT = path.join('scripts', 'generate-canonical-skill.mjs')
@@ -32,15 +33,11 @@ describe('canonical skill source', () => {
3233
})
3334

3435
it('keeps one compact marker within the generator byte budget', () => {
35-
const lines = CANONICAL_SKILL_MD.split(/\r?\n/)
36-
const markerIndexes = lines.flatMap((line, index) => line.trim() === COMPACT_END_MARKER ? [index] : [])
37-
expect(markerIndexes).toHaveLength(1)
38-
39-
const markerIndex = markerIndexes[0]!
40-
const compact = lines.slice(0, markerIndex).join('\n').trim()
41-
expect(Buffer.byteLength(compact, 'utf8')).toBeLessThanOrEqual(MAX_COMPACT_PREFIX_BYTES)
36+
const compact = extractCompactFromMarker(CANONICAL_SKILL_MD)
37+
expect(compact).not.toBeNull()
38+
expect(Buffer.byteLength(compact ?? '', 'utf8')).toBeLessThanOrEqual(MAX_COMPACT_PREFIX_BYTES)
4239
expect(compact).toContain('Always, before a file read')
43-
expect(lines.slice(markerIndex + 1).join('\n')).toMatch(/^## token-goat/m)
40+
expect(CANONICAL_SKILL_MD.split(COMPACT_END_MARKER)[1]).toMatch(/^\s*## token-goat/m)
4441
})
4542

4643
it('has a blob the generator reports as up to date', () => {
@@ -77,7 +74,7 @@ describe('canonical skill source', () => {
7774
expect(decode.stdout).toBe(edited)
7875
})
7976

80-
it('rejects missing and oversized compact boundaries without overwriting the blob', () => {
77+
it('validates compact boundaries against markdown fences and exact UTF-8 bytes', () => {
8178
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'tg-canonical-skill-'))
8279
scratch.push(root)
8380
fs.mkdirSync(path.join(root, 'scripts'))
@@ -91,21 +88,47 @@ describe('canonical skill source', () => {
9188
const originalSource = fs.readFileSync(sourcePath, 'utf8')
9289
const originalOutput = fs.readFileSync(outputPath, 'utf8')
9390

91+
fs.writeFileSync(sourcePath, `${COMPACT_END_MARKER}\n`)
92+
const empty = runGenerator(root, [])
93+
expect(empty.status).toBe(1)
94+
expect(empty.stderr).toContain('non-empty compact slice')
95+
expect(fs.readFileSync(outputPath, 'utf8')).toBe(originalOutput)
96+
9497
fs.writeFileSync(sourcePath, originalSource.replace(/^<!-- COMPACT_END -->\r?\n/m, ''))
9598
const missing = runGenerator(root, [])
9699
expect(missing.status).toBe(1)
97100
expect(missing.stderr).toContain('exactly one standalone')
98101
expect(fs.readFileSync(outputPath, 'utf8')).toBe(originalOutput)
99102

100-
const oversizedSource = originalSource.replace(
103+
const onlyFenced = originalSource.replace(
101104
COMPACT_END_MARKER,
102-
`${'x'.repeat(MAX_COMPACT_PREFIX_BYTES + 1)}\n${COMPACT_END_MARKER}`,
105+
`~~~md\n${COMPACT_END_MARKER}\n~~~`,
103106
)
104-
fs.writeFileSync(sourcePath, oversizedSource)
107+
fs.writeFileSync(sourcePath, onlyFenced)
108+
const fencedOnly = runGenerator(root, [])
109+
expect(fencedOnly.status).toBe(1)
110+
expect(fencedOnly.stderr).toContain('found 0')
111+
expect(fs.readFileSync(outputPath, 'utf8')).toBe(originalOutput)
112+
113+
const fencedExamples = originalSource.replace(
114+
COMPACT_END_MARKER,
115+
`\`\`\`md\n${COMPACT_END_MARKER}\n\`\`\`\n~~~md\n${COMPACT_END_MARKER}\n~~~\n${COMPACT_END_MARKER}`,
116+
)
117+
fs.writeFileSync(sourcePath, fencedExamples)
118+
expect(runGenerator(root, []).status).toBe(0)
119+
expect(runGenerator(root, ['--check']).status).toBe(0)
120+
121+
const exactCrlfPrefix = `${'a'.repeat(1199)}\r\n${'b'.repeat(1199)}\r\n${COMPACT_END_MARKER}\r\n`
122+
fs.writeFileSync(sourcePath, exactCrlfPrefix)
123+
expect(runGenerator(root, []).status).toBe(0)
124+
const exactOutput = fs.readFileSync(outputPath, 'utf8')
125+
126+
const oversizedUtf8Prefix = `${'a'.repeat(1199)}\r\n${'b'.repeat(1198)}é\r\n${COMPACT_END_MARKER}\r\n`
127+
fs.writeFileSync(sourcePath, oversizedUtf8Prefix)
105128
const oversized = runGenerator(root, ['--check'])
106129
expect(oversized.status).toBe(1)
107130
expect(oversized.stderr).toContain('2400-byte limit')
108-
expect(fs.readFileSync(outputPath, 'utf8')).toBe(originalOutput)
131+
expect(fs.readFileSync(outputPath, 'utf8')).toBe(exactOutput)
109132
})
110133

111134
it('rejects an unknown argument without writing anything', () => {

0 commit comments

Comments
 (0)