diff --git a/.changeset/fix-stdio-utf8-chunk-boundary.md b/.changeset/fix-stdio-utf8-chunk-boundary.md new file mode 100644 index 0000000000..90d2bcbca9 --- /dev/null +++ b/.changeset/fix-stdio-utf8-chunk-boundary.md @@ -0,0 +1,6 @@ +--- +'@modelcontextprotocol/sdk': patch +--- + +Fix UTF-8 corruption in `ReadBuffer` when multi-byte sequences (em-dash, emoji) split across stdio chunk boundaries. `Buffer.toString('utf8', ...)` decodes slices eagerly and produces replacement characters when a multi-byte sequence is split between chunks. Replaced with +`TextDecoder` in streaming mode, which carries partial bytes forward until the sequence completes. diff --git a/src/shared/stdio.ts b/src/shared/stdio.ts index fe14612bda..7dbcdf3507 100644 --- a/src/shared/stdio.ts +++ b/src/shared/stdio.ts @@ -4,29 +4,31 @@ import { JSONRPCMessage, JSONRPCMessageSchema } from '../types.js'; * Buffers a continuous stdio stream into discrete JSON-RPC messages. */ export class ReadBuffer { - private _buffer?: Buffer; + private _decoder = new TextDecoder('utf-8'); + private _text = ''; append(chunk: Buffer): void { - this._buffer = this._buffer ? Buffer.concat([this._buffer, chunk]) : chunk; + this._text += this._decoder.decode(chunk, { stream: true }); } readMessage(): JSONRPCMessage | null { - if (!this._buffer) { + if (!this._text) { return null; } - const index = this._buffer.indexOf('\n'); + const index = this._text.indexOf('\n'); if (index === -1) { return null; } - const line = this._buffer.toString('utf8', 0, index).replace(/\r$/, ''); - this._buffer = this._buffer.subarray(index + 1); + const line = this._text.slice(0, index).replace(/\r$/, ''); + this._text = this._text.slice(index + 1); return deserializeMessage(line); } clear(): void { - this._buffer = undefined; + this._decoder = new TextDecoder('utf-8'); + this._text = ''; } } diff --git a/test/shared/stdio.test.ts b/test/shared/stdio.test.ts index e8cbb5245a..1caee035cb 100644 --- a/test/shared/stdio.test.ts +++ b/test/shared/stdio.test.ts @@ -33,3 +33,53 @@ test('should be reusable after clearing', () => { readBuffer.append(Buffer.from('\n')); expect(readBuffer.readMessage()).toEqual(testMessage); }); + +describe('multi-byte UTF-8 across chunk boundaries', () => { + test('should preserve em-dash split across two chunks', () => { + const readBuffer = new ReadBuffer(); + const message: JSONRPCMessage = { jsonrpc: '2.0', method: 'test', params: { text: 'a—b' } }; + const fullBuffer = Buffer.from(JSON.stringify(message) + '\n'); + const splitIndex = fullBuffer.indexOf(0xe2) + 1; + + readBuffer.append(fullBuffer.subarray(0, splitIndex)); + readBuffer.append(fullBuffer.subarray(splitIndex)); + + expect(readBuffer.readMessage()).toEqual(message); + }); + + test('should preserve emoji split across two chunks', () => { + const readBuffer = new ReadBuffer(); + const message: JSONRPCMessage = { jsonrpc: '2.0', method: 'test', params: { text: '✅' } }; + const fullBuffer = Buffer.from(JSON.stringify(message) + '\n'); + const splitIndex = fullBuffer.indexOf(0xe2) + 2; + + readBuffer.append(fullBuffer.subarray(0, splitIndex)); + readBuffer.append(fullBuffer.subarray(splitIndex)); + + expect(readBuffer.readMessage()).toEqual(message); + }); + + test('should preserve four-byte emoji split across chunks', () => { + const readBuffer = new ReadBuffer(); + const message: JSONRPCMessage = { jsonrpc: '2.0', method: 'test', params: { text: '🎉' } }; + const fullBuffer = Buffer.from(JSON.stringify(message) + '\n'); + const splitIndex = fullBuffer.indexOf(0xf0) + 2; + + readBuffer.append(fullBuffer.subarray(0, splitIndex)); + readBuffer.append(fullBuffer.subarray(splitIndex)); + + expect(readBuffer.readMessage()).toEqual(message); + }); + + test('should preserve multi-byte chars across many chunks of size 1', () => { + const readBuffer = new ReadBuffer(); + const message: JSONRPCMessage = { jsonrpc: '2.0', method: 'test', params: { text: 'em — dash and ✅ check' } }; + const fullBuffer = Buffer.from(JSON.stringify(message) + '\n'); + + for (let i = 0; i < fullBuffer.length; i++) { + readBuffer.append(fullBuffer.subarray(i, i + 1)); + } + + expect(readBuffer.readMessage()).toEqual(message); + }); +});