fix(stdio): preserve multi-byte UTF-8 across chunk boundaries

Author: fmunteanu

src/shared/stdio.ts

@@ -4,29 +4,31 @@ import { JSONRPCMessage, JSONRPCMessageSchema } from '../types.js';
  * Buffers a continuous stdio stream into discrete JSON-RPC messages.
  */
 export class ReadBuffer {
-    private _buffer?: Buffer;
+    private _decoder = new TextDecoder('utf-8');
+    private _text = '';
 
     append(chunk: Buffer): void {
-        this._buffer = this._buffer ? Buffer.concat([this._buffer, chunk]) : chunk;
+        this._text += this._decoder.decode(chunk, { stream: true });
     }
 
     readMessage(): JSONRPCMessage | null {
-        if (!this._buffer) {
+        if (!this._text) {
             return null;
         }
 
-        const index = this._buffer.indexOf('\n');
+        const index = this._text.indexOf('\n');
         if (index === -1) {
             return null;
         }
 
-        const line = this._buffer.toString('utf8', 0, index).replace(/\r$/, '');
-        this._buffer = this._buffer.subarray(index + 1);
+        const line = this._text.slice(0, index).replace(/\r$/, '');
+        this._text = this._text.slice(index + 1);
         return deserializeMessage(line);
     }
 
     clear(): void {
-        this._buffer = undefined;
+        this._decoder = new TextDecoder('utf-8');
+        this._text = '';
     }
 }
 

test/shared/stdio.test.ts

@@ -33,3 +33,53 @@ test('should be reusable after clearing', () => {
     readBuffer.append(Buffer.from('\n'));
     expect(readBuffer.readMessage()).toEqual(testMessage);
 });
+
+describe('multi-byte UTF-8 across chunk boundaries', () => {
+    test('should preserve em-dash split across two chunks', () => {
+        const readBuffer = new ReadBuffer();
+        const message: JSONRPCMessage = { jsonrpc: '2.0', method: 'test', params: { text: 'a—b' } };
+        const fullBuffer = Buffer.from(JSON.stringify(message) + '\n');
+        const splitIndex = fullBuffer.indexOf(0xe2) + 1;
+
+        readBuffer.append(fullBuffer.subarray(0, splitIndex));
+        readBuffer.append(fullBuffer.subarray(splitIndex));
+
+        expect(readBuffer.readMessage()).toEqual(message);
+    });
+
+    test('should preserve emoji split across two chunks', () => {
+        const readBuffer = new ReadBuffer();
+        const message: JSONRPCMessage = { jsonrpc: '2.0', method: 'test', params: { text: '✅' } };
+        const fullBuffer = Buffer.from(JSON.stringify(message) + '\n');
+        const splitIndex = fullBuffer.indexOf(0xe2) + 2;
+
+        readBuffer.append(fullBuffer.subarray(0, splitIndex));
+        readBuffer.append(fullBuffer.subarray(splitIndex));
+
+        expect(readBuffer.readMessage()).toEqual(message);
+    });
+
+    test('should preserve four-byte emoji split across chunks', () => {
+        const readBuffer = new ReadBuffer();
+        const message: JSONRPCMessage = { jsonrpc: '2.0', method: 'test', params: { text: '🎉' } };
+        const fullBuffer = Buffer.from(JSON.stringify(message) + '\n');
+        const splitIndex = fullBuffer.indexOf(0xf0) + 2;
+
+        readBuffer.append(fullBuffer.subarray(0, splitIndex));
+        readBuffer.append(fullBuffer.subarray(splitIndex));
+
+        expect(readBuffer.readMessage()).toEqual(message);
+    });
+
+    test('should preserve multi-byte chars across many chunks of size 1', () => {
+        const readBuffer = new ReadBuffer();
+        const message: JSONRPCMessage = { jsonrpc: '2.0', method: 'test', params: { text: 'em — dash and ✅ check' } };
+        const fullBuffer = Buffer.from(JSON.stringify(message) + '\n');
+
+        for (let i = 0; i < fullBuffer.length; i++) {
+            readBuffer.append(fullBuffer.subarray(i, i + 1));
+        }
+
+        expect(readBuffer.readMessage()).toEqual(message);
+    });
+});