From 4265e8945a216ce51d2c0c883f375af7dd0d9a28 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Mon, 15 Jun 2026 22:43:01 +0300 Subject: [PATCH 001/246] feat(mcp): add AirtableClient.createRecords (internal-API row create) --- packages/mcp-server/src/client.js | 46 +++++++++++++++ .../mcp-server/test/test-record-write.test.js | 57 +++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 packages/mcp-server/test/test-record-write.test.js diff --git a/packages/mcp-server/src/client.js b/packages/mcp-server/src/client.js index 01ec95e..5359a5e 100644 --- a/packages/mcp-server/src/client.js +++ b/packages/mcp-server/src/client.js @@ -2463,6 +2463,52 @@ export class AirtableClient { }; } + /** + * Create records (one row per item). The client generates each rowId locally + * (Airtable accepts a client-supplied rec ID in the URL), so the returned + * rowId is final — no response parsing needed for the ID map. + * Per-row isolation: a failing row is collected in `failed`, not thrown. + * + * @param {string} appId + * @param {string} tableId + * @param {{cellValuesByColumnId: object, sourceKey?: string}[]} rows + * @param {{viewId?: string}} [opts] + * @returns {Promise<{created: {rowId:string, sourceKey:any}[], failed: {sourceKey:any, error:string}[]}>} + */ + async createRecords(appId, tableId, rows, { viewId } = {}) { + assertAirtableId(appId, 'appId'); + assertAirtableId(tableId, 'tableId'); + if (!Array.isArray(rows) || rows.length === 0) { + throw new Error('rows must be a non-empty array'); + } + let activeViewId = viewId; + if (!activeViewId) { + const table = await this.resolveTable(appId, tableId); + activeViewId = (table.views || [])[0]?.id || null; + } + + const created = []; + const failed = []; + for (const row of rows) { + const rowId = 'rec' + this._genRandomId(); + const payload = { tableId, cellValuesByColumnId: row.cellValuesByColumnId || {} }; + if (activeViewId) payload.activeViewId = activeViewId; + const url = `https://airtable.com/v0.3/row/${rowId}/create`; + try { + const res = await this.auth.postForm(url, this._mutationParams(payload, appId), appId); + if (!res.ok) { + const body = await res.text().catch(() => ''); + failed.push({ sourceKey: row.sourceKey ?? null, error: `createRecord failed (${res.status}): ${body}` }); + continue; + } + created.push({ rowId, sourceKey: row.sourceKey ?? null }); + } catch (err) { + failed.push({ sourceKey: row.sourceKey ?? null, error: String(err?.message || err) }); + } + } + return { created, failed }; + } + _genRequestId() { return 'req' + this._genRandomId(); } diff --git a/packages/mcp-server/test/test-record-write.test.js b/packages/mcp-server/test/test-record-write.test.js new file mode 100644 index 0000000..d12a76a --- /dev/null +++ b/packages/mcp-server/test/test-record-write.test.js @@ -0,0 +1,57 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { AirtableClient } from '../src/client.js'; + +function createMockAuth(responses = {}) { + const calls = []; + const defaultResponse = { ok: true, status: 200, json: async () => ({ data: {} }), text: async () => '{}' }; + return { + calls, + getSecretSocketId: () => 'socTEST', + get(url, appId) { calls.push({ method: 'GET', url, appId }); return responses.get?.(url) || { ...defaultResponse }; }, + postForm(url, params, appId) { calls.push({ method: 'POST', url, params, appId }); return responses.postForm?.(url, params) || { ...defaultResponse }; }, + }; +} +// Decode the payload the client put into stringifiedObjectParams +function payloadOf(call) { return JSON.parse(call.params.stringifiedObjectParams); } + +describe('AirtableClient.createRecords', () => { + it('creates one row per record with client-generated rec IDs', async () => { + const auth = createMockAuth({}); + const client = new AirtableClient(auth); + const result = await client.createRecords('appT', 'tblT', [ + { cellValuesByColumnId: { fldA: 'x' }, sourceKey: 's1' }, + { cellValuesByColumnId: { fldA: 'y' }, sourceKey: 's2' }, + ], { viewId: 'viwT' }); + + assert.equal(result.created.length, 2); + assert.equal(result.failed.length, 0); + const createCalls = auth.calls.filter(c => /\/row\/rec[A-Za-z0-9]+\/create$/.test(c.url)); + assert.equal(createCalls.length, 2); + const p0 = payloadOf(createCalls[0]); + assert.equal(p0.tableId, 'tblT'); + assert.deepEqual(p0.cellValuesByColumnId, { fldA: 'x' }); + assert.equal(p0.activeViewId, 'viwT'); + assert.notEqual(result.created[0].rowId, result.created[1].rowId); + assert.equal(result.created[0].sourceKey, 's1'); + }); + + it('isolates a failing row without aborting the batch', async () => { + const auth = createMockAuth({ + postForm(url, params) { + const p = JSON.parse(params.stringifiedObjectParams); + if (p.cellValuesByColumnId?.fldA === 'BOOM') return { ok: false, status: 422, json: async () => ({}), text: async () => 'computed field' }; + return { ok: true, status: 200, json: async () => ({ data: {} }), text: async () => '{}' }; + }, + }); + const client = new AirtableClient(auth); + const result = await client.createRecords('appT', 'tblT', [ + { cellValuesByColumnId: { fldA: 'ok' }, sourceKey: 's1' }, + { cellValuesByColumnId: { fldA: 'BOOM' }, sourceKey: 's2' }, + ], { viewId: 'viwT' }); + assert.equal(result.created.length, 1); + assert.equal(result.failed.length, 1); + assert.equal(result.failed[0].sourceKey, 's2'); + assert.match(result.failed[0].error, /422/); + }); +}); From 3e063d923f8b07353b32bf0dc0a7f5f6db98f4d0 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Mon, 15 Jun 2026 22:47:56 +0300 Subject: [PATCH 002/246] refactor(mcp): validate optional viewId + clarify error in createRecords --- packages/mcp-server/src/client.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/mcp-server/src/client.js b/packages/mcp-server/src/client.js index 5359a5e..46d483f 100644 --- a/packages/mcp-server/src/client.js +++ b/packages/mcp-server/src/client.js @@ -2481,6 +2481,7 @@ export class AirtableClient { if (!Array.isArray(rows) || rows.length === 0) { throw new Error('rows must be a non-empty array'); } + if (viewId) assertAirtableId(viewId, 'viewId'); let activeViewId = viewId; if (!activeViewId) { const table = await this.resolveTable(appId, tableId); @@ -2498,7 +2499,7 @@ export class AirtableClient { const res = await this.auth.postForm(url, this._mutationParams(payload, appId), appId); if (!res.ok) { const body = await res.text().catch(() => ''); - failed.push({ sourceKey: row.sourceKey ?? null, error: `createRecord failed (${res.status}): ${body}` }); + failed.push({ sourceKey: row.sourceKey ?? null, error: `createRecords row failed (${res.status}): ${body}` }); continue; } created.push({ rowId, sourceKey: row.sourceKey ?? null }); From 8b4f1cdf7e588185bf0bc900a07d83afbedb31ee Mon Sep 17 00:00:00 2001 From: "A.R." Date: Mon, 15 Jun 2026 22:50:32 +0300 Subject: [PATCH 003/246] feat(mcp): add AirtableClient.updateRecords + deleteRecords --- packages/mcp-server/src/client.js | 74 +++++++++++++++++++ .../mcp-server/test/test-record-write.test.js | 30 ++++++++ 2 files changed, 104 insertions(+) diff --git a/packages/mcp-server/src/client.js b/packages/mcp-server/src/client.js index 46d483f..250eea0 100644 --- a/packages/mcp-server/src/client.js +++ b/packages/mcp-server/src/client.js @@ -2510,6 +2510,80 @@ export class AirtableClient { return { created, failed }; } + /** + * Update records' PRIMITIVE / single-select cells via updatePrimitiveCell. + * Array cells (multi-select / link / attachment) are NOT handled here — the + * engine sets those via pasteCells (Milestone 3) where column configs exist. + * + * @param {string} appId + * @param {string} tableId (kept for symmetry / future routing) + * @param {{rowId:string, cellValuesByColumnId:object}[]} updates + * @returns {Promise<{updated:{rowId:string}[], failed:{rowId:string, error:string}[]}>} + */ + async updateRecords(appId, tableId, updates) { + assertAirtableId(appId, 'appId'); + assertAirtableId(tableId, 'tableId'); + if (!Array.isArray(updates) || updates.length === 0) { + throw new Error('updates must be a non-empty array'); + } + const updated = []; + const failed = []; + for (const u of updates) { + assertAirtableId(u.rowId, 'rowId'); + const url = `https://airtable.com/v0.3/row/${u.rowId}/updatePrimitiveCell`; + let rowFailed = null; + for (const [columnId, cellValue] of Object.entries(u.cellValuesByColumnId || {})) { + try { + const res = await this.auth.postForm(url, this._mutationParams({ columnId, cellValue }, appId), appId); + if (!res.ok) { + const body = await res.text().catch(() => ''); + rowFailed = `updatePrimitiveCell ${columnId} failed (${res.status}): ${body}`; + break; + } + } catch (err) { + rowFailed = String(err?.message || err); + break; + } + } + if (rowFailed) failed.push({ rowId: u.rowId, error: rowFailed }); + else updated.push({ rowId: u.rowId }); + } + return { updated, failed }; + } + + /** + * Delete records in one native batch call (destroyMultipleRows). + * + * @param {string} appId + * @param {string} tableId + * @param {string[]} rowIds + * @param {{viewId?: string}} [opts] + * @returns {Promise<{deleted:number, actionId:string|null}>} + */ + async deleteRecords(appId, tableId, rowIds, { viewId } = {}) { + assertAirtableId(appId, 'appId'); + assertAirtableId(tableId, 'tableId'); + if (!Array.isArray(rowIds) || rowIds.length === 0) { + throw new Error('rowIds must be a non-empty array'); + } + if (viewId) assertAirtableId(viewId, 'viewId'); + let activeViewId = viewId; + if (!activeViewId) { + const table = await this.resolveTable(appId, tableId); + activeViewId = (table.views || [])[0]?.id || null; + } + const payload = { rowIds }; + if (activeViewId) payload.activeViewId = activeViewId; + const url = `https://airtable.com/v0.3/table/${tableId}/destroyMultipleRows`; + const res = await this.auth.postForm(url, this._mutationParams(payload, appId), appId); + if (!res.ok) { + const body = await res.text().catch(() => ''); + throw new Error(`destroyMultipleRows failed (${res.status}): ${body}`); + } + const data = (await res.json().catch(() => ({})))?.data || {}; + return { deleted: rowIds.length, actionId: data.actionId || null }; + } + _genRequestId() { return 'req' + this._genRandomId(); } diff --git a/packages/mcp-server/test/test-record-write.test.js b/packages/mcp-server/test/test-record-write.test.js index d12a76a..fb2c46f 100644 --- a/packages/mcp-server/test/test-record-write.test.js +++ b/packages/mcp-server/test/test-record-write.test.js @@ -55,3 +55,33 @@ describe('AirtableClient.createRecords', () => { assert.match(result.failed[0].error, /422/); }); }); + +describe('AirtableClient.updateRecords', () => { + it('sets each primitive cell via updatePrimitiveCell', async () => { + const auth = createMockAuth({}); + const client = new AirtableClient(auth); + const result = await client.updateRecords('appT', 'tblT', [ + { rowId: 'rec1', cellValuesByColumnId: { fldA: 'x', fldB: 'selZ' } }, + ]); + assert.equal(result.updated.length, 1); + assert.equal(result.failed.length, 0); + const cellCalls = auth.calls.filter(c => /\/row\/rec1\/updatePrimitiveCell$/.test(c.url)); + assert.equal(cellCalls.length, 2); + const cols = cellCalls.map(c => JSON.parse(c.params.stringifiedObjectParams).columnId).sort(); + assert.deepEqual(cols, ['fldA', 'fldB']); + }); +}); + +describe('AirtableClient.deleteRecords', () => { + it('deletes rows in one destroyMultipleRows call', async () => { + const auth = createMockAuth({ + postForm: () => ({ ok: true, status: 200, json: async () => ({ data: { actionId: 'actX' } }), text: async () => '{}' }), + }); + const client = new AirtableClient(auth); + const result = await client.deleteRecords('appT', 'tblT', ['rec1', 'rec2'], { viewId: 'viwT' }); + assert.equal(result.deleted, 2); + const calls = auth.calls.filter(c => /\/table\/tblT\/destroyMultipleRows$/.test(c.url)); + assert.equal(calls.length, 1); + assert.deepEqual(JSON.parse(calls[0].params.stringifiedObjectParams).rowIds, ['rec1', 'rec2']); + }); +}); From 0a0e85dc8985b3210296ad55fd56030c8b9742c6 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Mon, 15 Jun 2026 22:55:13 +0300 Subject: [PATCH 004/246] fix(mcp): updateRecords per-row isolation + flag empty cell updates Wrap each row in try/catch so assertAirtableId throws isolate to failed instead of aborting the batch. Empty cellValuesByColumnId now reports as failed instead of silently landing in updated with zero API calls made. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/client.js | 25 +++++++++++-------- .../mcp-server/test/test-record-write.test.js | 22 ++++++++++++++++ 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/packages/mcp-server/src/client.js b/packages/mcp-server/src/client.js index 250eea0..151e19f 100644 --- a/packages/mcp-server/src/client.js +++ b/packages/mcp-server/src/client.js @@ -2529,24 +2529,28 @@ export class AirtableClient { const updated = []; const failed = []; for (const u of updates) { - assertAirtableId(u.rowId, 'rowId'); - const url = `https://airtable.com/v0.3/row/${u.rowId}/updatePrimitiveCell`; - let rowFailed = null; - for (const [columnId, cellValue] of Object.entries(u.cellValuesByColumnId || {})) { - try { + try { + assertAirtableId(u.rowId, 'rowId'); + const entries = Object.entries(u.cellValuesByColumnId || {}); + if (entries.length === 0) { + failed.push({ rowId: u.rowId, error: 'cellValuesByColumnId is empty — nothing to update' }); + continue; + } + const url = `https://airtable.com/v0.3/row/${u.rowId}/updatePrimitiveCell`; + let rowFailed = null; + for (const [columnId, cellValue] of entries) { const res = await this.auth.postForm(url, this._mutationParams({ columnId, cellValue }, appId), appId); if (!res.ok) { const body = await res.text().catch(() => ''); rowFailed = `updatePrimitiveCell ${columnId} failed (${res.status}): ${body}`; break; } - } catch (err) { - rowFailed = String(err?.message || err); - break; } + if (rowFailed) failed.push({ rowId: u.rowId, error: rowFailed }); + else updated.push({ rowId: u.rowId }); + } catch (err) { + failed.push({ rowId: u.rowId ?? '(unknown)', error: String(err?.message || err) }); } - if (rowFailed) failed.push({ rowId: u.rowId, error: rowFailed }); - else updated.push({ rowId: u.rowId }); } return { updated, failed }; } @@ -2559,6 +2563,7 @@ export class AirtableClient { * @param {string[]} rowIds * @param {{viewId?: string}} [opts] * @returns {Promise<{deleted:number, actionId:string|null}>} + * @note deleted count is optimistic (rowIds.length); the server may skip already-deleted rows. */ async deleteRecords(appId, tableId, rowIds, { viewId } = {}) { assertAirtableId(appId, 'appId'); diff --git a/packages/mcp-server/test/test-record-write.test.js b/packages/mcp-server/test/test-record-write.test.js index fb2c46f..2fc2027 100644 --- a/packages/mcp-server/test/test-record-write.test.js +++ b/packages/mcp-server/test/test-record-write.test.js @@ -85,3 +85,25 @@ describe('AirtableClient.deleteRecords', () => { assert.deepEqual(JSON.parse(calls[0].params.stringifiedObjectParams).rowIds, ['rec1', 'rec2']); }); }); + +describe('AirtableClient.updateRecords isolation', () => { + it('isolates a failing row and flags empty updates', async () => { + const auth = createMockAuth({ + postForm(url, params) { + const p = JSON.parse(params.stringifiedObjectParams); + if (p.cellValue === 'BOOM') return { ok: false, status: 422, json: async () => ({}), text: async () => 'bad' }; + return { ok: true, status: 200, json: async () => ({ data: {} }), text: async () => '{}' }; + }, + }); + const client = new AirtableClient(auth); + const result = await client.updateRecords('appT', 'tblT', [ + { rowId: 'rec1', cellValuesByColumnId: { fldA: 'ok' } }, + { rowId: 'rec2', cellValuesByColumnId: { fldA: 'BOOM' } }, + { rowId: 'rec3', cellValuesByColumnId: {} }, + ]); + assert.equal(result.updated.length, 1); + assert.equal(result.updated[0].rowId, 'rec1'); + assert.equal(result.failed.length, 2); + assert.deepEqual(result.failed.map(f => f.rowId).sort(), ['rec2', 'rec3']); + }); +}); From f901277f91ea2475cc3a89ae6543661e7958e4ef Mon Sep 17 00:00:00 2001 From: "A.R." Date: Mon, 15 Jun 2026 22:58:54 +0300 Subject: [PATCH 005/246] feat(mcp): add AirtableClient.uploadAttachment (5-step multipart) Co-Authored-By: Claude Sonnet 4.6 --- packages/mcp-server/src/client.js | 71 ++++++++++++++++++- .../test/test-attachment-upload.test.js | 54 ++++++++++++++ 2 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 packages/mcp-server/test/test-attachment-upload.test.js diff --git a/packages/mcp-server/src/client.js b/packages/mcp-server/src/client.js index 151e19f..b389e83 100644 --- a/packages/mcp-server/src/client.js +++ b/packages/mcp-server/src/client.js @@ -1,5 +1,5 @@ import { SchemaCache } from './cache.js'; -import { randomBytes } from 'node:crypto'; +import { randomBytes, createHash } from 'node:crypto'; /** * Generate an Airtable-style filter ID: "flt" + 14 base62 characters. @@ -2589,6 +2589,75 @@ export class AirtableClient { return { deleted: rowIds.length, actionId: data.actionId || null }; } + /** Raw S3 presigned PUT of file bytes. Returns { ok, etag, error? }. Overridable in tests. */ + async _putBytes(presignedUrl, bytes, checksumB64) { + try { + const res = await fetch(presignedUrl, { + method: 'PUT', + body: bytes, + headers: { 'x-amz-checksum-sha256': checksumB64 }, + }); + if (!res.ok) return { ok: false, error: `S3 PUT failed (${res.status})` }; + return { ok: true, etag: res.headers.get('etag') || '' }; + } catch (err) { + return { ok: false, error: String(err?.message || err) }; + } + } + + /** + * Upload a file into an attachment cell via the 5-step multipart flow. + * Soft-fails (returns {ok:false,error}) instead of throwing, so one bad + * attachment doesn't abort a record sync. + * + * @param {string} appId + * @param {string} rowId + * @param {string} columnId + * @param {{bytes: Buffer|Uint8Array, filename: string, contentType: string}} file + * @returns {Promise<{ok:boolean, attachmentId?:string, url?:string, error?:string}>} + */ + async uploadAttachment(appId, rowId, columnId, { bytes, filename, contentType }) { + assertAirtableId(appId, 'appId'); + assertAirtableId(rowId, 'rowId'); + const checksumB64 = createHash('sha256').update(bytes).digest('base64'); + const post = async (verb, payload) => { + const url = `https://airtable.com/v0.3/application/${appId}/${verb}`; + const res = await this.auth.postForm(url, this._mutationParams(payload, appId), appId); + if (!res.ok) { + const body = await res.text().catch(() => ''); + throw new Error(`${verb} failed (${res.status}): ${body}`); + } + return (await res.json().catch(() => ({})))?.data || {}; + }; + try { + const create = await post('createMultipartUpload', { + uploadCandidate: { contentLength: bytes.length, filename, contentType, directUploadUserContentPurpose: 'directUploadAttachment' }, + }); + const getUrl = await post('getUrlMultipartUpload', { + uploadPartCandidate: { uploadId: create.uploadId, objectKey: create.objectKey, partNumber: 1, checksumSHA256: checksumB64, directUploadUserContentPurpose: 'directUploadAttachment' }, + }); + const put = await this._putBytes(getUrl.presignedUrl, bytes, checksumB64); + if (!put.ok) return { ok: false, error: put.error }; + const complete = await post('completeMultipartUpload', { + multipartUploadComplete: { + uploadId: create.uploadId, objectKey: create.objectKey, + parts: [{ etag: put.etag, partNumber: 1, checksumSHA256: checksumB64 }], + directUploadUserContentPurpose: 'directUploadAttachment', + }, + }); + const attachmentId = 'att' + this._genRandomId(); + const item = { id: attachmentId, ...complete.propsToAddToAttachmentObj, filename, expiringInitialPreviewUrl: complete.signedUrl }; + const attachUrl = `https://airtable.com/v0.3/row/${rowId}/updateArrayTypeCellByAddingItem`; + const res = await this.auth.postForm(attachUrl, this._mutationParams({ columnId, item }, appId), appId); + if (!res.ok) { + const body = await res.text().catch(() => ''); + return { ok: false, error: `attach failed (${res.status}): ${body}` }; + } + return { ok: true, attachmentId, url: complete.url }; + } catch (err) { + return { ok: false, error: String(err?.message || err) }; + } + } + _genRequestId() { return 'req' + this._genRandomId(); } diff --git a/packages/mcp-server/test/test-attachment-upload.test.js b/packages/mcp-server/test/test-attachment-upload.test.js new file mode 100644 index 0000000..f44071b --- /dev/null +++ b/packages/mcp-server/test/test-attachment-upload.test.js @@ -0,0 +1,54 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { AirtableClient } from '../src/client.js'; + +function mockAuthForUpload() { + const calls = []; + return { + calls, + getSecretSocketId: () => 'socTEST', + postForm(url, params) { + calls.push({ url, params }); + if (url.endsWith('/createMultipartUpload')) return { ok: true, status: 200, json: async () => ({ data: { uploadId: 'up1', objectKey: 'ok1', bucketName: 'b' } }), text: async () => '{}' }; + if (url.endsWith('/getUrlMultipartUpload')) return { ok: true, status: 200, json: async () => ({ data: { presignedUrl: 'https://s3.example/put?x=1' } }), text: async () => '{}' }; + if (url.endsWith('/completeMultipartUpload')) return { ok: true, status: 200, json: async () => ({ data: { url: 'https://dl.airtable.com/f', signedUrl: 'https://v5.airtableusercontent.com/s', propsToAddToAttachmentObj: { signedMetadata: { version: 1, data: 'b64.sig' }, size: 5, type: 'text/plain', url: 'https://dl.airtable.com/f' } } }), text: async () => '{}' }; + if (url.includes('/updateArrayTypeCellByAddingItem')) return { ok: true, status: 200, json: async () => ({ data: null }), text: async () => '{}' }; + return { ok: false, status: 500, json: async () => ({}), text: async () => 'unexpected' }; + }, + }; +} + +describe('AirtableClient.uploadAttachment', () => { + it('runs the 5-step flow and attaches the file', async () => { + const auth = mockAuthForUpload(); + const client = new AirtableClient(auth); + const putCalls = []; + client._putBytes = async (url, bytes, checksum) => { putCalls.push({ url, checksum }); return { ok: true, etag: '"etag1"' }; }; + + const result = await client.uploadAttachment('appT', 'recT', 'fldT', { + bytes: Buffer.from('hello'), filename: 'hi.txt', contentType: 'text/plain', + }); + + assert.equal(result.ok, true); + assert.ok(result.attachmentId.startsWith('att')); + assert.equal(putCalls.length, 1); + assert.equal(putCalls[0].url, 'https://s3.example/put?x=1'); + const seq = auth.calls.map(c => c.url.split('/').pop()); + assert.deepEqual(seq, ['createMultipartUpload', 'getUrlMultipartUpload', 'completeMultipartUpload', 'updateArrayTypeCellByAddingItem']); + const completePayload = JSON.parse(auth.calls[2].params.stringifiedObjectParams); + assert.equal(completePayload.multipartUploadComplete.parts[0].etag, '"etag1"'); + }); + + it('returns a soft failure (no throw) when a step errors', async () => { + const auth = mockAuthForUpload(); + const orig = auth.postForm; + auth.postForm = (url, params) => url.endsWith('/createMultipartUpload') + ? { ok: false, status: 500, json: async () => ({}), text: async () => 'boom' } + : orig(url, params); + const client = new AirtableClient(auth); + client._putBytes = async () => ({ ok: true, etag: '"e"' }); + const result = await client.uploadAttachment('appT', 'recT', 'fldT', { bytes: Buffer.from('x'), filename: 'x', contentType: 'text/plain' }); + assert.equal(result.ok, false); + assert.match(result.error, /createMultipartUpload/); + }); +}); From eaf149b38755ecbc82975300a2df89edf5adfc75 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Mon, 15 Jun 2026 23:04:49 +0300 Subject: [PATCH 006/246] refactor(mcp): guard columnId + presignedUrl in uploadAttachment; assert upload checksum --- packages/mcp-server/src/client.js | 5 +++++ packages/mcp-server/test/test-attachment-upload.test.js | 3 +++ 2 files changed, 8 insertions(+) diff --git a/packages/mcp-server/src/client.js b/packages/mcp-server/src/client.js index b389e83..6636232 100644 --- a/packages/mcp-server/src/client.js +++ b/packages/mcp-server/src/client.js @@ -2592,6 +2592,9 @@ export class AirtableClient { /** Raw S3 presigned PUT of file bytes. Returns { ok, etag, error? }. Overridable in tests. */ async _putBytes(presignedUrl, bytes, checksumB64) { try { + // Captured presigned URL signs only `host;x-amz-checksum-sha256` (X-Amz-SignedHeaders), + // so we send exactly that one header — adding others (e.g. Content-Type) is unnecessary + // and risks a signature mismatch if S3 ever signs them. const res = await fetch(presignedUrl, { method: 'PUT', body: bytes, @@ -2618,6 +2621,7 @@ export class AirtableClient { async uploadAttachment(appId, rowId, columnId, { bytes, filename, contentType }) { assertAirtableId(appId, 'appId'); assertAirtableId(rowId, 'rowId'); + assertAirtableId(columnId, 'columnId'); const checksumB64 = createHash('sha256').update(bytes).digest('base64'); const post = async (verb, payload) => { const url = `https://airtable.com/v0.3/application/${appId}/${verb}`; @@ -2635,6 +2639,7 @@ export class AirtableClient { const getUrl = await post('getUrlMultipartUpload', { uploadPartCandidate: { uploadId: create.uploadId, objectKey: create.objectKey, partNumber: 1, checksumSHA256: checksumB64, directUploadUserContentPurpose: 'directUploadAttachment' }, }); + if (!getUrl.presignedUrl) return { ok: false, error: 'getUrlMultipartUpload: no presignedUrl in response' }; const put = await this._putBytes(getUrl.presignedUrl, bytes, checksumB64); if (!put.ok) return { ok: false, error: put.error }; const complete = await post('completeMultipartUpload', { diff --git a/packages/mcp-server/test/test-attachment-upload.test.js b/packages/mcp-server/test/test-attachment-upload.test.js index f44071b..ff57bc3 100644 --- a/packages/mcp-server/test/test-attachment-upload.test.js +++ b/packages/mcp-server/test/test-attachment-upload.test.js @@ -1,5 +1,6 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; import { AirtableClient } from '../src/client.js'; function mockAuthForUpload() { @@ -33,6 +34,8 @@ describe('AirtableClient.uploadAttachment', () => { assert.ok(result.attachmentId.startsWith('att')); assert.equal(putCalls.length, 1); assert.equal(putCalls[0].url, 'https://s3.example/put?x=1'); + const expectedChecksum = createHash('sha256').update(Buffer.from('hello')).digest('base64'); + assert.equal(putCalls[0].checksum, expectedChecksum); const seq = auth.calls.map(c => c.url.split('/').pop()); assert.deepEqual(seq, ['createMultipartUpload', 'getUrlMultipartUpload', 'completeMultipartUpload', 'updateArrayTypeCellByAddingItem']); const completePayload = JSON.parse(auth.calls[2].params.stringifiedObjectParams); From 1834221bec71d2dfb3821c31c39dbac1c200a3b3 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Mon, 15 Jun 2026 23:08:41 +0300 Subject: [PATCH 007/246] feat(mcp): expose create_records / update_records / delete_records tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire the three existing client methods as MCP tools in the record-write category. Tool definitions, handlers, category entries, and test count updates (66→69) are all included. Co-Authored-By: Claude Sonnet 4.6 --- packages/mcp-server/src/index.js | 70 +++++++++++++++++++ packages/mcp-server/src/tool-config.js | 3 + .../mcp-server/test/test-tool-config.test.js | 22 ++++-- 3 files changed, 88 insertions(+), 7 deletions(-) diff --git a/packages/mcp-server/src/index.js b/packages/mcp-server/src/index.js index 52e3431..8b1c63f 100644 --- a/packages/mcp-server/src/index.js +++ b/packages/mcp-server/src/index.js @@ -1520,6 +1520,61 @@ Note: "form title" is the view name itself — use rename_view to change it. "Fi required: ['appId', 'tableId', 'viewId', 'sourceRowIds'], }, }, + { + name: 'create_records', + description: 'Create one or more records in a table. Each item supplies cellValuesByColumnId (computed fields are read-only and must be omitted). Returns created record IDs. Per-row isolation: a failing row is reported, not fatal.', + annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false }, + inputSchema: { + type: 'object', + properties: { + appId: { type: 'string', description: 'The Airtable base/application ID' }, + tableId: { type: 'string', description: 'The table ID to create records in' }, + viewId: { type: 'string', description: 'Optional view ID; defaults to the table\'s first view' }, + records: { + type: 'array', + description: 'Records to create, each { cellValuesByColumnId: { "": }, sourceKey?: }', + items: { type: 'object' }, + }, + debug: debugProp, + }, + required: ['appId', 'tableId', 'records'], + }, + }, + { + name: 'update_records', + description: 'Update primitive / single-select cells of existing records via cellValuesByColumnId. (Array cells — multi-select, links, attachments — are not set here.) Per-row isolation.', + annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + inputSchema: { + type: 'object', + properties: { + appId: { type: 'string', description: 'The Airtable base/application ID' }, + tableId: { type: 'string', description: 'The table ID containing the records' }, + updates: { + type: 'array', + description: 'Updates, each { rowId: "recXXX", cellValuesByColumnId: { "": } }', + items: { type: 'object' }, + }, + debug: debugProp, + }, + required: ['appId', 'tableId', 'updates'], + }, + }, + { + name: 'delete_records', + description: 'Delete one or more records from a table in a single batch call. Returns the count deleted.', + annotations: { readOnlyHint: false, destructiveHint: true, idempotentHint: false, openWorldHint: false }, + inputSchema: { + type: 'object', + properties: { + appId: { type: 'string', description: 'The Airtable base/application ID' }, + tableId: { type: 'string', description: 'The table ID containing the records' }, + rowIds: { type: 'array', description: 'Record IDs to delete (e.g. ["recXXX"])', items: { type: 'string' } }, + viewId: { type: 'string', description: 'Optional view ID; defaults to the table\'s first view' }, + debug: debugProp, + }, + required: ['appId', 'tableId', 'rowIds'], + }, + }, ]; // ─── Meta-Tool: manage_tools ───────────────────────────────── @@ -2299,6 +2354,21 @@ const handlers = { return ok(result, result, debug); }, + async create_records({ appId, tableId, viewId, records, debug }) { + const result = await client.createRecords(appId, tableId, records, { viewId }); + return ok(result, result, debug); + }, + + async update_records({ appId, tableId, updates, debug }) { + const result = await client.updateRecords(appId, tableId, updates); + return ok(result, result, debug); + }, + + async delete_records({ appId, tableId, rowIds, viewId, debug }) { + const result = await client.deleteRecords(appId, tableId, rowIds, { viewId }); + return ok(result, result, debug); + }, + // ── Meta: Tool Management ── async manage_tools({ action, profile, tool, category, enabled }) { diff --git a/packages/mcp-server/src/tool-config.js b/packages/mcp-server/src/tool-config.js index 4be83c4..a3f4e0c 100644 --- a/packages/mcp-server/src/tool-config.js +++ b/packages/mcp-server/src/tool-config.js @@ -108,6 +108,9 @@ export const TOOL_CATEGORIES = { // Record Write (duplicate via pasteCells) duplicate_records: 'record-write', + create_records: 'record-write', + update_records: 'record-write', + delete_records: 'record-write', }; /** Human-readable labels for categories */ diff --git a/packages/mcp-server/test/test-tool-config.test.js b/packages/mcp-server/test/test-tool-config.test.js index 1b034fe..4b00eee 100644 --- a/packages/mcp-server/test/test-tool-config.test.js +++ b/packages/mcp-server/test/test-tool-config.test.js @@ -17,7 +17,7 @@ import { describe('TOOL_CATEGORIES', () => { it('maps all tools to valid categories', () => { const tools = Object.keys(TOOL_CATEGORIES); - assert.equal(tools.length, 66, `Expected 66 tools, got ${tools.length}`); + assert.equal(tools.length, 69, `Expected 69 tools, got ${tools.length}`); for (const [tool, cat] of Object.entries(TOOL_CATEGORIES)) { assert.ok(CATEGORY_LABELS[cat], `Tool "${tool}" has unknown category "${cat}"`); } @@ -83,9 +83,9 @@ describe('ToolConfigManager', () => { assert.equal(mgr.activeProfile, 'full'); }); - it('enables all 66 tools on full profile', () => { + it('enables all 69 tools on full profile', () => { const enabled = mgr.enabledToolNames(); - assert.equal(enabled.size, 66); + assert.equal(enabled.size, 69); }); it('manage_tools is always enabled', () => { @@ -121,10 +121,10 @@ describe('ToolConfigManager', () => { assert.ok(!enabled.has('create_extension')); }); - it('full enables all 66 tools', async () => { + it('full enables all 69 tools', async () => { await mgr.switchProfile('full'); const enabled = mgr.enabledToolNames(); - assert.equal(enabled.size, 66); + assert.equal(enabled.size, 69); }); it('unknown profile fails closed to read-only', async () => { @@ -207,10 +207,10 @@ describe('ToolConfigManager', () => { }); describe('getToolStatus()', () => { - it('returns status for all 66 tools', async () => { + it('returns status for all 69 tools', async () => { await mgr.switchProfile('full'); const status = mgr.getToolStatus(); - assert.equal(status.length, 66); + assert.equal(status.length, 69); assert.ok(status.every(s => s.enabled === true)); }); @@ -236,3 +236,11 @@ describe('ToolConfigManager', () => { }); }); }); + +describe('record-write tools', () => { + it('maps create/update/delete_records to record-write', () => { + assert.equal(TOOL_CATEGORIES.create_records, 'record-write'); + assert.equal(TOOL_CATEGORIES.update_records, 'record-write'); + assert.equal(TOOL_CATEGORIES.delete_records, 'record-write'); + }); +}); From 52f1a8705d99a84d921d6f5b8f22996ecb51e60f Mon Sep 17 00:00:00 2001 From: "A.R." Date: Mon, 15 Jun 2026 23:16:27 +0300 Subject: [PATCH 008/246] refactor(mcp): correct update_records idempotency hint + clarify record-tool docs Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/index.js | 8 ++++---- packages/mcp-server/test/test-tool-config.test.js | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/mcp-server/src/index.js b/packages/mcp-server/src/index.js index 8b1c63f..b104ce3 100644 --- a/packages/mcp-server/src/index.js +++ b/packages/mcp-server/src/index.js @@ -1529,7 +1529,7 @@ Note: "form title" is the view name itself — use rename_view to change it. "Fi properties: { appId: { type: 'string', description: 'The Airtable base/application ID' }, tableId: { type: 'string', description: 'The table ID to create records in' }, - viewId: { type: 'string', description: 'Optional view ID; defaults to the table\'s first view' }, + viewId: { type: 'string', description: 'Optional view ID used to position new rows within the view\'s row order; defaults to the table\'s first view.' }, records: { type: 'array', description: 'Records to create, each { cellValuesByColumnId: { "": }, sourceKey?: }', @@ -1543,7 +1543,7 @@ Note: "form title" is the view name itself — use rename_view to change it. "Fi { name: 'update_records', description: 'Update primitive / single-select cells of existing records via cellValuesByColumnId. (Array cells — multi-select, links, attachments — are not set here.) Per-row isolation.', - annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false }, inputSchema: { type: 'object', properties: { @@ -1561,7 +1561,7 @@ Note: "form title" is the view name itself — use rename_view to change it. "Fi }, { name: 'delete_records', - description: 'Delete one or more records from a table in a single batch call. Returns the count deleted.', + description: 'Delete one or more records from a table in a single batch call. The returned deleted count equals rowIds.length (optimistic) — already-deleted rows are silently skipped by the server.', annotations: { readOnlyHint: false, destructiveHint: true, idempotentHint: false, openWorldHint: false }, inputSchema: { type: 'object', @@ -1569,7 +1569,7 @@ Note: "form title" is the view name itself — use rename_view to change it. "Fi appId: { type: 'string', description: 'The Airtable base/application ID' }, tableId: { type: 'string', description: 'The table ID containing the records' }, rowIds: { type: 'array', description: 'Record IDs to delete (e.g. ["recXXX"])', items: { type: 'string' } }, - viewId: { type: 'string', description: 'Optional view ID; defaults to the table\'s first view' }, + viewId: { type: 'string', description: 'Optional view ID passed as UI context; does NOT filter which records are deleted — rowIds is the sole selector.' }, debug: debugProp, }, required: ['appId', 'tableId', 'rowIds'], diff --git a/packages/mcp-server/test/test-tool-config.test.js b/packages/mcp-server/test/test-tool-config.test.js index 4b00eee..f62e735 100644 --- a/packages/mcp-server/test/test-tool-config.test.js +++ b/packages/mcp-server/test/test-tool-config.test.js @@ -119,6 +119,9 @@ describe('ToolConfigManager', () => { assert.ok(!enabled.has('delete_field')); assert.ok(!enabled.has('delete_view')); assert.ok(!enabled.has('create_extension')); + assert.ok(enabled.has('create_records')); + assert.ok(enabled.has('update_records')); + assert.ok(enabled.has('delete_records')); }); it('full enables all 69 tools', async () => { From 3ac79e5884d61cf2f126484a5da5179628399ffd Mon Sep 17 00:00:00 2001 From: "A.R." Date: Mon, 15 Jun 2026 23:19:39 +0300 Subject: [PATCH 009/246] chore(tool-sync): register record-write tools across mirror + counts --- packages/extension/package.json | 6 +++--- packages/extension/src/mcp/tool-profile.ts | 5 ++++- packages/webview/src/store.ts | 4 ++-- packages/webview/src/test/store.test.ts | 2 +- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/extension/package.json b/packages/extension/package.json index 7004942..d4cdb9a 100644 --- a/packages/extension/package.json +++ b/packages/extension/package.json @@ -491,8 +491,8 @@ ], "enumDescriptions": [ "Schema inspection, formula validation, and record reading only (12 tools)", - "Read + record read/write + create/update tables, fields, views, sidebar sections, and record templates, no deletes, no form metadata (51 tools)", - "All tools enabled including destructive ops, form metadata, and extensions (66 tools)", + "Read + record read/write + create/update tables, fields, views, sidebar sections, and record templates, no deletes, no form metadata (54 tools)", + "All tools enabled including destructive ops, form metadata, and extensions (69 tools)", "User-defined per-tool selection" ], "description": "Active MCP tool profile. Controls which tools are exposed to AI agents through the bundled Airtable MCP server." @@ -669,7 +669,7 @@ { "id": "AirtableFormula.server", "label": "Airtable User MCP", - "description": "Manage Airtable bases via 66 MCP tools: schema read, table CRUD, field CRUD (including formula/rollup/lookup/download), view configuration (filters / sorts / groups / visibility / column ordering / freezing — with merge-safe append mode), sidebar section grouping, view-type metadata (Kanban/Gallery/Calendar covers, colors, calendar date ranges), legacy-form metadata, formula validation, formula download, bulk base formula download, record template management (create / rename / pre-fill cells / duplicate / apply / delete), and extension management. Operates over stdio using JSON-RPC 2.0." + "description": "Manage Airtable bases via 69 MCP tools: schema read, table CRUD, field CRUD (including formula/rollup/lookup/download), view configuration (filters / sorts / groups / visibility / column ordering / freezing — with merge-safe append mode), sidebar section grouping, view-type metadata (Kanban/Gallery/Calendar covers, colors, calendar date ranges), legacy-form metadata, formula validation, formula download, bulk base formula download, record template management (create / rename / pre-fill cells / duplicate / apply / delete), record CRUD (create / update / delete / duplicate), and extension management. Operates over stdio using JSON-RPC 2.0." } ] }, diff --git a/packages/extension/src/mcp/tool-profile.ts b/packages/extension/src/mcp/tool-profile.ts index 8a44cbc..f2f624e 100644 --- a/packages/extension/src/mcp/tool-profile.ts +++ b/packages/extension/src/mcp/tool-profile.ts @@ -107,8 +107,11 @@ export const TOOL_CATEGORIES: Record = { diff --git a/packages/webview/src/store.ts b/packages/webview/src/store.ts index bafc63a..7431adb 100644 --- a/packages/webview/src/store.ts +++ b/packages/webview/src/store.ts @@ -62,8 +62,8 @@ const defaultSettings: SettingsSnapshot = { notifyOnUpdates: true, toolProfile: { profile: 'safe-write', - enabledCount: 51, - totalCount: 66, + enabledCount: 54, + totalCount: 69, categories: { read: true, recordRead: true, tableWrite: true, tableDestructive: true, fieldWrite: true, fieldDestructive: true, diff --git a/packages/webview/src/test/store.test.ts b/packages/webview/src/test/store.test.ts index ef4260d..793f4f7 100644 --- a/packages/webview/src/test/store.test.ts +++ b/packages/webview/src/test/store.test.ts @@ -13,7 +13,7 @@ beforeEach(() => { ideStatuses: [], versions: { extension: '—', mcpServerBundled: '—' }, aiFilesCount: 0, loading: true, activeTab: 'overview', pendingActions: new Set(), settings: { - mcp: { autoConfigureOnInstall: true, notifyOnUpdates: true, toolProfile: { profile: 'safe-write', enabledCount: 51, totalCount: 66, categories: { read: true, recordRead: true, tableWrite: true, tableDestructive: true, fieldWrite: true, fieldDestructive: true, viewWrite: true, viewDestructive: true, viewSection: true, viewSectionDestructive: true, formWrite: true, extension: true, recordWrite: true } }, serverSource: 'bundled' }, + mcp: { autoConfigureOnInstall: true, notifyOnUpdates: true, toolProfile: { profile: 'safe-write', enabledCount: 54, totalCount: 69, categories: { read: true, recordRead: true, tableWrite: true, tableDestructive: true, fieldWrite: true, fieldDestructive: true, viewWrite: true, viewDestructive: true, viewSection: true, viewSectionDestructive: true, formWrite: true, extension: true, recordWrite: true } }, serverSource: 'bundled' }, ai: { autoInstallFiles: true, includeAgents: false }, formula: { formatterVersion: 'v2' }, script: { beautifyStyle: 'default', minifyLevel: 'standard' }, From 8d2b2b9bcabd47d4148603fe5a7905db20f347c9 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 09:46:12 +0300 Subject: [PATCH 010/246] feat(mcp): gate record deletes behind new record-destructive category (full-only) Co-Authored-By: Claude Opus 4.8 --- packages/extension/package.json | 9 +++++++-- packages/extension/src/mcp/tool-profile.ts | 12 ++++++++---- packages/mcp-server/src/tool-config.js | 7 +++++-- packages/mcp-server/test/test-tool-config.test.js | 7 ++++--- packages/shared/src/types.ts | 1 + packages/webview/src/store.ts | 4 ++-- packages/webview/src/tabs/Settings.tsx | 3 ++- packages/webview/src/test/store.test.ts | 2 +- 8 files changed, 30 insertions(+), 15 deletions(-) diff --git a/packages/extension/package.json b/packages/extension/package.json index d4cdb9a..28ee3aa 100644 --- a/packages/extension/package.json +++ b/packages/extension/package.json @@ -491,7 +491,7 @@ ], "enumDescriptions": [ "Schema inspection, formula validation, and record reading only (12 tools)", - "Read + record read/write + create/update tables, fields, views, sidebar sections, and record templates, no deletes, no form metadata (54 tools)", + "Read + record read/write + create/update tables, fields, views, sidebar sections, and record templates, no deletes, no form metadata (53 tools)", "All tools enabled including destructive ops, form metadata, and extensions (69 tools)", "User-defined per-tool selection" ], @@ -560,7 +560,12 @@ "airtableFormula.mcp.categories.recordWrite": { "type": "boolean", "default": true, - "description": "Record Write tool: duplicate_records (bulk record duplication via pasteCells)." + "description": "Record Write tools: create_records, update_records, duplicate_records (bulk record create/update/duplication)." + }, + "airtableFormula.mcp.categories.recordDestructive": { + "type": "boolean", + "default": true, + "description": "Record Destructive tools: delete_records (batch record deletion)." }, "airtableFormula.mcp.tools": { "type": "object", diff --git a/packages/extension/src/mcp/tool-profile.ts b/packages/extension/src/mcp/tool-profile.ts index f2f624e..8723713 100644 --- a/packages/extension/src/mcp/tool-profile.ts +++ b/packages/extension/src/mcp/tool-profile.ts @@ -28,7 +28,7 @@ type ExtraCategoryKey = | 'view-write' | 'view-destructive' | 'view-section' | 'view-section-destructive' | 'form-write' - | 'record-read' | 'record-write'; + | 'record-read' | 'record-write' | 'record-destructive'; export const TOOL_CATEGORIES: Record = { // Read-only / inspection get_base_schema: 'read', @@ -111,7 +111,8 @@ export const TOOL_CATEGORIES: Record = { @@ -128,6 +129,7 @@ export const CATEGORY_LABELS: Record = { 'form-write': 'Form Metadata', 'extension': 'Extension Management', 'record-write': 'Record Write', + 'record-destructive': 'Record Destructive', }; interface ProfileDef { @@ -142,7 +144,7 @@ export const BUILTIN_PROFILES: Record<'read-only' | 'safe-write' | 'full', Profi categories: ['read', 'record-read', 'record-write', 'table-write', 'field-write', 'view-write', 'view-section'] }, full: { description: 'All tools enabled including destructive ops, form metadata, and extensions', categories: [ - 'read', 'record-read', 'record-write', + 'read', 'record-read', 'record-write', 'record-destructive', 'table-write', 'table-destructive', 'field-write', 'field-destructive', 'view-write', 'view-destructive', @@ -159,6 +161,7 @@ const SETTINGS_TO_CATEGORY: Record = { tableDestructive: 'table-destructive', fieldWrite: 'field-write', fieldDestructive: 'field-destructive', + recordDestructive: 'record-destructive', viewWrite: 'view-write', viewDestructive: 'view-destructive', viewSection: 'view-section', @@ -261,6 +264,7 @@ export class ToolProfileManager implements vscode.Disposable { tableDestructive: cfg.get('mcp.categories.tableDestructive', true), fieldWrite: cfg.get('mcp.categories.fieldWrite', true), fieldDestructive: cfg.get('mcp.categories.fieldDestructive', true), + recordDestructive: cfg.get('mcp.categories.recordDestructive', true), viewWrite: cfg.get('mcp.categories.viewWrite', true), viewDestructive: cfg.get('mcp.categories.viewDestructive', true), viewSection: cfg.get('mcp.categories.viewSection', true), @@ -322,7 +326,7 @@ export class ToolProfileManager implements vscode.Disposable { 'view-section', 'view-section-destructive', 'form-write', 'extension', - 'record-write', + 'record-write', 'record-destructive', ]; for (const cat of categoryOrder) { const label = CATEGORY_LABELS[cat] ?? cat; diff --git a/packages/mcp-server/src/tool-config.js b/packages/mcp-server/src/tool-config.js index a3f4e0c..3c8d190 100644 --- a/packages/mcp-server/src/tool-config.js +++ b/packages/mcp-server/src/tool-config.js @@ -110,7 +110,9 @@ export const TOOL_CATEGORIES = { duplicate_records: 'record-write', create_records: 'record-write', update_records: 'record-write', - delete_records: 'record-write', + + // Record Destructive (batch record deletion) + delete_records: 'record-destructive', }; /** Human-readable labels for categories */ @@ -128,6 +130,7 @@ export const CATEGORY_LABELS = { 'form-write': 'Form Metadata', 'extension': 'Extension Management', 'record-write': 'Record Write', + 'record-destructive': 'Record Destructive', }; // ─── Built-in Profiles ─────────────────────────────────────── @@ -144,7 +147,7 @@ export const BUILTIN_PROFILES = { full: { description: 'All tools enabled including destructive ops, form metadata, and extensions', categories: [ - 'read', 'record-read', 'record-write', + 'read', 'record-read', 'record-write', 'record-destructive', 'table-write', 'table-destructive', 'field-write', 'field-destructive', 'view-write', 'view-destructive', diff --git a/packages/mcp-server/test/test-tool-config.test.js b/packages/mcp-server/test/test-tool-config.test.js index f62e735..2743e87 100644 --- a/packages/mcp-server/test/test-tool-config.test.js +++ b/packages/mcp-server/test/test-tool-config.test.js @@ -58,6 +58,7 @@ describe('BUILTIN_PROFILES', () => { it('safe-write excludes destructive categories', () => { const cats = BUILTIN_PROFILES['safe-write'].categories; assert.ok(!cats.includes('field-destructive')); + assert.ok(!cats.includes('record-destructive')); assert.ok(!cats.includes('view-destructive')); assert.ok(!cats.includes('table-destructive')); assert.ok(!cats.includes('extension')); @@ -121,7 +122,7 @@ describe('ToolConfigManager', () => { assert.ok(!enabled.has('create_extension')); assert.ok(enabled.has('create_records')); assert.ok(enabled.has('update_records')); - assert.ok(enabled.has('delete_records')); + assert.ok(!enabled.has('delete_records')); }); it('full enables all 69 tools', async () => { @@ -241,9 +242,9 @@ describe('ToolConfigManager', () => { }); describe('record-write tools', () => { - it('maps create/update/delete_records to record-write', () => { + it('maps create/update_records to record-write and delete_records to record-destructive', () => { assert.equal(TOOL_CATEGORIES.create_records, 'record-write'); assert.equal(TOOL_CATEGORIES.update_records, 'record-write'); - assert.equal(TOOL_CATEGORIES.delete_records, 'record-write'); + assert.equal(TOOL_CATEGORIES.delete_records, 'record-destructive'); }); }); diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index 9e3920b..806c3e2 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -99,6 +99,7 @@ export interface ToolCategories { tableDestructive: boolean; fieldWrite: boolean; fieldDestructive: boolean; + recordDestructive: boolean; viewWrite: boolean; viewDestructive: boolean; viewSection: boolean; diff --git a/packages/webview/src/store.ts b/packages/webview/src/store.ts index 7431adb..498d292 100644 --- a/packages/webview/src/store.ts +++ b/packages/webview/src/store.ts @@ -62,11 +62,11 @@ const defaultSettings: SettingsSnapshot = { notifyOnUpdates: true, toolProfile: { profile: 'safe-write', - enabledCount: 54, + enabledCount: 53, totalCount: 69, categories: { read: true, recordRead: true, tableWrite: true, tableDestructive: true, - fieldWrite: true, fieldDestructive: true, + fieldWrite: true, fieldDestructive: true, recordDestructive: true, viewWrite: true, viewDestructive: true, viewSection: true, viewSectionDestructive: true, formWrite: true, extension: true, recordWrite: true, diff --git a/packages/webview/src/tabs/Settings.tsx b/packages/webview/src/tabs/Settings.tsx index d1c9fdc..ac2ae98 100644 --- a/packages/webview/src/tabs/Settings.tsx +++ b/packages/webview/src/tabs/Settings.tsx @@ -610,7 +610,8 @@ export function Settings() { - + + )} diff --git a/packages/webview/src/test/store.test.ts b/packages/webview/src/test/store.test.ts index 793f4f7..378d3a2 100644 --- a/packages/webview/src/test/store.test.ts +++ b/packages/webview/src/test/store.test.ts @@ -13,7 +13,7 @@ beforeEach(() => { ideStatuses: [], versions: { extension: '—', mcpServerBundled: '—' }, aiFilesCount: 0, loading: true, activeTab: 'overview', pendingActions: new Set(), settings: { - mcp: { autoConfigureOnInstall: true, notifyOnUpdates: true, toolProfile: { profile: 'safe-write', enabledCount: 54, totalCount: 69, categories: { read: true, recordRead: true, tableWrite: true, tableDestructive: true, fieldWrite: true, fieldDestructive: true, viewWrite: true, viewDestructive: true, viewSection: true, viewSectionDestructive: true, formWrite: true, extension: true, recordWrite: true } }, serverSource: 'bundled' }, + mcp: { autoConfigureOnInstall: true, notifyOnUpdates: true, toolProfile: { profile: 'safe-write', enabledCount: 53, totalCount: 69, categories: { read: true, recordRead: true, tableWrite: true, tableDestructive: true, fieldWrite: true, fieldDestructive: true, recordDestructive: true, viewWrite: true, viewDestructive: true, viewSection: true, viewSectionDestructive: true, formWrite: true, extension: true, recordWrite: true } }, serverSource: 'bundled' }, ai: { autoInstallFiles: true, includeAgents: false }, formula: { formatterVersion: 'v2' }, script: { beautifyStyle: 'default', minifyLevel: 'standard' }, From 66268f8aa96894b98b3ae32054c89eddb70c0715 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 09:48:57 +0300 Subject: [PATCH 011/246] test(mcp): cover viewId fallback, id guards, and attachment soft-fail paths Co-Authored-By: Claude Sonnet 4.6 --- .../test/test-attachment-upload.test.js | 28 +++++++++++++++++++ .../mcp-server/test/test-record-write.test.js | 24 ++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/packages/mcp-server/test/test-attachment-upload.test.js b/packages/mcp-server/test/test-attachment-upload.test.js index ff57bc3..0f706b5 100644 --- a/packages/mcp-server/test/test-attachment-upload.test.js +++ b/packages/mcp-server/test/test-attachment-upload.test.js @@ -55,3 +55,31 @@ describe('AirtableClient.uploadAttachment', () => { assert.match(result.error, /createMultipartUpload/); }); }); + +describe('AirtableClient.uploadAttachment guards', () => { + it('soft-fails when getUrlMultipartUpload returns no presignedUrl', async () => { + const auth = mockAuthForUpload(); + const orig = auth.postForm; + auth.postForm = (url, params) => url.endsWith('/getUrlMultipartUpload') + ? { ok: true, status: 200, json: async () => ({ data: {} }), text: async () => '{}' } + : orig(url, params); + const client = new AirtableClient(auth); + client._putBytes = async () => ({ ok: true, etag: '"e"' }); + const result = await client.uploadAttachment('appT', 'recT', 'fldT', { bytes: Buffer.from('x'), filename: 'x', contentType: 'text/plain' }); + assert.equal(result.ok, false); + assert.match(result.error, /presignedUrl/); + }); + + it('soft-fails when the final attach call errors', async () => { + const auth = mockAuthForUpload(); + const orig = auth.postForm; + auth.postForm = (url, params) => url.includes('/updateArrayTypeCellByAddingItem') + ? { ok: false, status: 422, json: async () => ({}), text: async () => 'bad attach' } + : orig(url, params); + const client = new AirtableClient(auth); + client._putBytes = async () => ({ ok: true, etag: '"e"' }); + const result = await client.uploadAttachment('appT', 'recT', 'fldT', { bytes: Buffer.from('x'), filename: 'x', contentType: 'text/plain' }); + assert.equal(result.ok, false); + assert.match(result.error, /attach failed/); + }); +}); diff --git a/packages/mcp-server/test/test-record-write.test.js b/packages/mcp-server/test/test-record-write.test.js index 2fc2027..280ecd4 100644 --- a/packages/mcp-server/test/test-record-write.test.js +++ b/packages/mcp-server/test/test-record-write.test.js @@ -107,3 +107,27 @@ describe('AirtableClient.updateRecords isolation', () => { assert.deepEqual(result.failed.map(f => f.rowId).sort(), ['rec2', 'rec3']); }); }); + +describe('AirtableClient.createRecords viewId fallback', () => { + it('resolves the first view when no viewId is given', async () => { + const auth = createMockAuth({ + get: () => ({ ok: true, status: 200, json: async () => ({ data: { tableSchemas: [{ id: 'tblT', columns: [], views: [{ id: 'viwFIRST' }, { id: 'viwSECOND' }] }] } }), text: async () => '{}' }), + }); + const client = new AirtableClient(auth); + const result = await client.createRecords('appT', 'tblT', [{ cellValuesByColumnId: { fldA: 'x' } }]); + assert.equal(result.created.length, 1); + const createCall = auth.calls.find(c => /\/row\/rec[A-Za-z0-9]+\/create$/.test(c.url)); + assert.equal(JSON.parse(createCall.params.stringifiedObjectParams).activeViewId, 'viwFIRST'); + }); +}); + +describe('AirtableClient record-write id guards', () => { + it('createRecords rejects an invalid appId', async () => { + const client = new AirtableClient(createMockAuth({})); + await assert.rejects(() => client.createRecords('BAD!', 'tblT', [{ cellValuesByColumnId: {} }], { viewId: 'viwT' })); + }); + it('deleteRecords rejects an invalid tableId', async () => { + const client = new AirtableClient(createMockAuth({})); + await assert.rejects(() => client.deleteRecords('appT', 'BAD!', ['rec1'], { viewId: 'viwT' })); + }); +}); From eec14e87d55c85032a4f1922584ff8a5465b38c4 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 10:31:39 +0300 Subject: [PATCH 012/246] feat(sync): add schema snapshot/normalize for plan engine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces packages/mcp-server/src/sync/snapshot.js — the first module of the M2a schema-plan engine. Exports normalizeSchema() (handles both .tableSchemas/.columns and .tables/.fields shapes, resolves primaryColumnId → primaryFieldId → first-col fallback), isComputedType() (full set of non-writable field types), and snapshotBase() (async helper over AirtableClient.getApplicationData). Pure logic, no mutations. Three node:test assertions pass covering isComputedType flags and both API shape variants. Co-Authored-By: Claude Opus 4.8 --- packages/mcp-server/src/sync/snapshot.js | 65 +++++++++++++++++++ .../test/sync/test-snapshot.test.js | 40 ++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 packages/mcp-server/src/sync/snapshot.js create mode 100644 packages/mcp-server/test/sync/test-snapshot.test.js diff --git a/packages/mcp-server/src/sync/snapshot.js b/packages/mcp-server/src/sync/snapshot.js new file mode 100644 index 0000000..f6a867a --- /dev/null +++ b/packages/mcp-server/src/sync/snapshot.js @@ -0,0 +1,65 @@ +// Schema snapshot: normalize getApplicationData into a comparable model. +// Supports both .tableSchemas/.columns (real internal API) and .tables/.fields (public API shape). + +const COMPUTED_TYPES = new Set([ + 'formula', 'rollup', 'lookup', 'multipleLookupValues', 'count', + 'autoNumber', 'autonumber', 'createdTime', 'lastModifiedTime', + 'createdBy', 'lastModifiedBy', 'button', 'aiText', 'asyncText', 'externalSyncSource', +]); + +/** + * Returns true if the field type is computed (not directly writable by records API). + * @param {string} type + * @returns {boolean} + */ +export function isComputedType(type) { + return COMPUTED_TYPES.has(type); +} + +/** + * Normalizes the raw response from client.getApplicationData() into a + * stable, comparable snapshot model. + * + * @param {object} rawData - Full API response from getApplicationData + * @returns {{ tables: Array }} + */ +export function normalizeSchema(rawData) { + const tables = rawData?.data?.tableSchemas ?? rawData?.data?.tables ?? []; + return { + tables: tables.map((t) => { + const cols = t.columns ?? t.fields ?? []; + return { + id: t.id, + name: t.name, + // Real internal API uses primaryColumnId; public API uses primaryFieldId. + // Fall back to the first column id if neither is present. + primaryFieldId: t.primaryColumnId ?? t.primaryFieldId ?? cols[0]?.id ?? null, + fields: cols.map((c) => ({ + id: c.id, + name: c.name, + type: c.type, + typeOptions: c.typeOptions ?? null, + description: c.description ?? null, + isComputed: isComputedType(c.type), + })), + }; + }), + }; +} + +/** + * Fetches and normalizes the schema for a base. + * + * @param {import('../client.js').AirtableClient} client + * @param {string} appId - Base ID (e.g. 'appXXXXXXXXXXXXXX') + * @returns {Promise<{ baseId: string, tables: Array }>} + */ +export async function snapshotBase(client, appId) { + const raw = await client.getApplicationData(appId); + return { baseId: appId, ...normalizeSchema(raw) }; +} + +/** + * @typedef {{ id: string, name: string, type: string, typeOptions: object|null, description: string|null, isComputed: boolean }} NormalizedField + * @typedef {{ id: string, name: string, primaryFieldId: string|null, fields: NormalizedField[] }} NormalizedTable + */ diff --git a/packages/mcp-server/test/sync/test-snapshot.test.js b/packages/mcp-server/test/sync/test-snapshot.test.js new file mode 100644 index 0000000..7f7045c --- /dev/null +++ b/packages/mcp-server/test/sync/test-snapshot.test.js @@ -0,0 +1,40 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { normalizeSchema, isComputedType } from '../../src/sync/snapshot.js'; + +describe('snapshot.isComputedType', () => { + it('flags computed types and not writable ones', () => { + assert.equal(isComputedType('formula'), true); + assert.equal(isComputedType('rollup'), true); + assert.equal(isComputedType('lookup'), true); + assert.equal(isComputedType('count'), true); + assert.equal(isComputedType('button'), true); + assert.equal(isComputedType('text'), false); + assert.equal(isComputedType('multiSelect'), false); + }); +}); + +describe('snapshot.normalizeSchema', () => { + const raw = { data: { tableSchemas: [ + { id: 'tbl1', name: 'Offers', primaryColumnId: 'fldA', columns: [ + { id: 'fldA', name: 'Name', type: 'text', typeOptions: null, description: null }, + { id: 'fldB', name: 'Total', type: 'formula', typeOptions: { formulaTextParsed: 'SUM(1)' }, description: 'd' }, + ] }, + ] } }; + it('flattens tables + fields with computed flags and primary id', () => { + const snap = normalizeSchema(raw); + assert.equal(snap.tables.length, 1); + const t = snap.tables[0]; + assert.equal(t.id, 'tbl1'); + assert.equal(t.name, 'Offers'); + assert.equal(t.primaryFieldId, 'fldA'); + assert.equal(t.fields.length, 2); + assert.equal(t.fields[0].isComputed, false); + assert.equal(t.fields[1].isComputed, true); + assert.equal(t.fields[1].typeOptions.formulaTextParsed, 'SUM(1)'); + }); + it('falls back to .tables and first column for primary', () => { + const snap = normalizeSchema({ data: { tables: [{ id: 'tbl2', name: 'X', fields: [{ id: 'fldZ', name: 'P', type: 'text' }] }] } }); + assert.equal(snap.tables[0].primaryFieldId, 'fldZ'); + }); +}); From 71900d53a9c037b7c9697c2be3546751ef84b240 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 10:36:30 +0300 Subject: [PATCH 013/246] feat(sync): add idmap name-matching + sync-state persistence Pure matchByName() builds tables/fields/choices maps from two snapshots. saveIdmap/loadIdmap/savePlan/saveState persist per-pair state under ~/.airtable-user-mcp/sync/__/ via safeAtomicWriteFileSync. Two node:test suites pass (matching + round-trip I/O). Co-Authored-By: Claude Opus 4.8 --- packages/mcp-server/src/sync/idmap.js | 139 ++++++++++++++++++ .../mcp-server/test/sync/test-idmap.test.js | 43 ++++++ 2 files changed, 182 insertions(+) create mode 100644 packages/mcp-server/src/sync/idmap.js create mode 100644 packages/mcp-server/test/sync/test-idmap.test.js diff --git a/packages/mcp-server/src/sync/idmap.js b/packages/mcp-server/src/sync/idmap.js new file mode 100644 index 0000000..dbfbe51 --- /dev/null +++ b/packages/mcp-server/src/sync/idmap.js @@ -0,0 +1,139 @@ +import { join } from 'node:path'; +import { existsSync, readFileSync, mkdirSync } from 'node:fs'; +import { getHomeDir } from '../paths.js'; +import { safeAtomicWriteFileSync } from '../safe-write.js'; + +/** + * Build a name→item Map from an array of objects with a `.name` property. + * First occurrence wins; duplicates are silently dropped (diff phase warns). + * @param {Array<{name: string}>} items + * @returns {Map} + */ +function indexByName(items) { + const m = new Map(); + for (const it of items) { + if (!m.has(it.name)) m.set(it.name, it); + } + return m; +} + +/** + * Match choice IDs from a source field to dest field by choice name. + * Returns `{}` if either field has no typeOptions.choices. + * @param {{ typeOptions?: { choices?: object } }} srcField + * @param {{ typeOptions?: { choices?: object } }} destField + * @returns {Record} srcChoiceId → destChoiceId + */ +function matchChoices(srcField, destField) { + const sc = srcField.typeOptions?.choices; + const dc = destField.typeOptions?.choices; + if (!sc || !dc) return {}; + const destByName = new Map(Object.values(dc).map((c) => [c.name, c.id])); + const out = {}; + for (const c of Object.values(sc)) { + const destId = destByName.get(c.name); + if (destId) out[c.id] = destId; + } + return out; +} + +/** + * Produce an ID-map by matching tables, fields, and select choices by name. + * + * @param {{ tables: Array<{id:string, name:string, fields:Array<{id:string,name:string,typeOptions?:object}>}> }} srcSnap + * @param {{ tables: Array<{id:string, name:string, fields:Array<{id:string,name:string,typeOptions?:object}>}> }} destSnap + * @returns {{ + * tables: Record, + * fields: Record }> + * }} + */ +export function matchByName(srcSnap, destSnap) { + const destTables = indexByName(destSnap.tables); + const tables = {}; + const fields = {}; + + for (const st of srcSnap.tables) { + const dt = destTables.get(st.name); + if (!dt) continue; + tables[st.id] = dt.id; + + const destFields = indexByName(dt.fields); + for (const sf of st.fields) { + const df = destFields.get(sf.name); + if (!df) continue; + fields[sf.id] = { destFld: df.id, choices: matchChoices(sf, df) }; + } + } + + return { tables, fields }; +} + +// ── State I/O ────────────────────────────────────────────────────────────── + +/** + * Return the per-pair sync directory path. + * Does NOT create the directory. + * @param {string} sourceBaseId + * @param {string} destBaseId + * @returns {string} + */ +export function syncDir(sourceBaseId, destBaseId) { + return join(getHomeDir(), 'sync', `${sourceBaseId}__${destBaseId}`); +} + +/** + * @param {string} dir Parent directory (created if absent). + * @param {string} file File name within dir. + * @param {object} obj Value to JSON-serialise. + */ +function writeJson(dir, file, obj) { + mkdirSync(dir, { recursive: true }); + safeAtomicWriteFileSync(join(dir, file), JSON.stringify(obj, null, 2)); +} + +/** + * Persist an ID-map for a source→dest base pair. + * @param {string} sourceBaseId + * @param {string} destBaseId + * @param {object} idmap + */ +export function saveIdmap(sourceBaseId, destBaseId, idmap) { + writeJson(syncDir(sourceBaseId, destBaseId), 'idmap.json', idmap); +} + +/** + * Load a previously saved ID-map. Returns `{ tables: {}, fields: {} }` when + * the file is absent or unparseable. + * @param {string} sourceBaseId + * @param {string} destBaseId + * @returns {{ tables: Record, fields: Record }} + */ +export function loadIdmap(sourceBaseId, destBaseId) { + const p = join(syncDir(sourceBaseId, destBaseId), 'idmap.json'); + if (!existsSync(p)) return { tables: {}, fields: {} }; + try { + return JSON.parse(readFileSync(p, 'utf8')); + } catch { + return { tables: {}, fields: {} }; + } +} + +/** + * Persist a sync plan (schema diff output) for a base pair. + * @param {string} sourceBaseId + * @param {string} destBaseId + * @param {{ planId: string }} plan + */ +export function savePlan(sourceBaseId, destBaseId, plan) { + writeJson(syncDir(sourceBaseId, destBaseId), `plan-${plan.planId}.json`, plan); +} + +/** + * Persist sync execution state for a base pair. + * @param {string} sourceBaseId + * @param {string} destBaseId + * @param {object} state + */ +export function saveState(sourceBaseId, destBaseId, state) { + writeJson(syncDir(sourceBaseId, destBaseId), 'state.json', state); +} diff --git a/packages/mcp-server/test/sync/test-idmap.test.js b/packages/mcp-server/test/sync/test-idmap.test.js new file mode 100644 index 0000000..627b862 --- /dev/null +++ b/packages/mcp-server/test/sync/test-idmap.test.js @@ -0,0 +1,43 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { join } from 'node:path'; +import { mkdtempSync, existsSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { matchByName, saveIdmap, loadIdmap, syncDir } from '../../src/sync/idmap.js'; + +const src = { tables: [ + { id: 'tS', name: 'Offers', fields: [ + { id: 'fS1', name: 'Name', type: 'text' }, + { id: 'fS2', name: 'Status', type: 'singleSelect', typeOptions: { choices: { selS: { id: 'selS', name: 'Open' } } } }, + { id: 'fS3', name: 'OnlyInSource', type: 'text' }, + ] }, + { id: 'tSonly', name: 'Ghost', fields: [] }, +] }; +const dest = { tables: [ + { id: 'tD', name: 'Offers', fields: [ + { id: 'fD1', name: 'Name', type: 'text' }, + { id: 'fD2', name: 'Status', type: 'singleSelect', typeOptions: { choices: { selD: { id: 'selD', name: 'Open' } } } }, + ] }, +] }; + +describe('idmap.matchByName', () => { + it('maps tables, fields, and choices by name; skips unmatched', () => { + const m = matchByName(src, dest); + assert.equal(m.tables.tS, 'tD'); + assert.equal(m.tables.tSonly, undefined); + assert.equal(m.fields.fS1.destFld, 'fD1'); + assert.equal(m.fields.fS2.destFld, 'fD2'); + assert.equal(m.fields.fS2.choices.selS, 'selD'); + assert.equal(m.fields.fS3, undefined); + }); +}); + +describe('idmap state I/O', () => { + it('round-trips idmap through the sync dir', () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-test-')); + const m = { tables: { tS: 'tD' }, fields: {} }; + saveIdmap('appAAAAAAAAAAAAAA', 'appBBBBBBBBBBBBBB', m); + assert.ok(existsSync(join(syncDir('appAAAAAAAAAAAAAA', 'appBBBBBBBBBBBBBB'), 'idmap.json'))); + assert.deepEqual(loadIdmap('appAAAAAAAAAAAAAA', 'appBBBBBBBBBBBBBB'), m); + }); +}); From cf0109682896dd395ebc0b8bc887b74efad03d57 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 10:40:17 +0300 Subject: [PATCH 014/246] feat(sync): add computed-field canonicalization for ref-stable diffing Co-Authored-By: Claude Sonnet 4.6 --- packages/mcp-server/src/sync/remap.js | 24 ++++++++++++++++ .../mcp-server/test/sync/test-remap.test.js | 28 +++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 packages/mcp-server/src/sync/remap.js create mode 100644 packages/mcp-server/test/sync/test-remap.test.js diff --git a/packages/mcp-server/src/sync/remap.js b/packages/mcp-server/src/sync/remap.js new file mode 100644 index 0000000..9f6b6b1 --- /dev/null +++ b/packages/mcp-server/src/sync/remap.js @@ -0,0 +1,24 @@ +// Canonicalize computed-field configs so cross-base comparisons ignore field-ID churn: +// every fld... reference is replaced with the referenced field's name. +const FLD_TOKEN = /fld[A-Za-z0-9]+/g; + +function subIds(str, fldIdToName) { + if (typeof str !== 'string') return str; + return str.replace(FLD_TOKEN, (id) => `{{${fldIdToName[id] ?? id}}}`); +} + +export function canonicalizeComputed(type, typeOptions, fldIdToName) { + const opts = typeOptions || {}; + // Pull the formula expression under whichever key the API used. + const formula = opts.formulaTextParsed ?? opts.formulaText ?? opts.formula ?? ''; + const relation = opts.relationColumnId ?? opts.recordLinkFieldId ?? ''; + const target = opts.foreignTableRollupColumnId ?? opts.fieldIdInLinkedTable ?? ''; + return JSON.stringify({ + type, + formula: subIds(formula, fldIdToName), + relation: fldIdToName[relation] ?? relation, + target: fldIdToName[target] ?? target, + // result aggregation type (rollup) is value-stable across bases + result: opts.result?.type ?? null, + }); +} diff --git a/packages/mcp-server/test/sync/test-remap.test.js b/packages/mcp-server/test/sync/test-remap.test.js new file mode 100644 index 0000000..1eee279 --- /dev/null +++ b/packages/mcp-server/test/sync/test-remap.test.js @@ -0,0 +1,28 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { canonicalizeComputed } from '../../src/sync/remap.js'; + +describe('remap.canonicalizeComputed', () => { + const srcNames = { fldA: 'Price', fldB: 'Qty' }; + const destNames = { fldX: 'Price', fldY: 'Qty' }; + + it('makes two formulas with different field IDs but same names compare equal', () => { + const s = canonicalizeComputed('formula', { formulaTextParsed: '{fldA}*{fldB}' }, srcNames); + const d = canonicalizeComputed('formula', { formulaTextParsed: '{fldX}*{fldY}' }, destNames); + assert.equal(s, d); + }); + it('detects a real formula difference', () => { + const s = canonicalizeComputed('formula', { formulaTextParsed: '{fldA}+{fldB}' }, srcNames); + const d = canonicalizeComputed('formula', { formulaTextParsed: '{fldX}*{fldY}' }, destNames); + assert.notEqual(s, d); + }); + it('canonicalizes rollup relation/target columns by name', () => { + const s = canonicalizeComputed('rollup', { relationColumnId: 'fldA', foreignTableRollupColumnId: 'fldB', formulaText: 'SUM(values)' }, srcNames); + const d = canonicalizeComputed('rollup', { relationColumnId: 'fldX', foreignTableRollupColumnId: 'fldY', formulaText: 'SUM(values)' }, destNames); + assert.equal(s, d); + }); + it('treats an unknown ref id as itself (no crash)', () => { + const s = canonicalizeComputed('formula', { formulaTextParsed: '{fldUNKNOWN}+1' }, {}); + assert.match(s, /fldUNKNOWN/); + }); +}); From 0f4ff23126d92984a901317f0470d81411c73534 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 10:42:33 +0300 Subject: [PATCH 015/246] chore(mcp): recurse test glob to include test/sync suites --- packages/mcp-server/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/mcp-server/package.json b/packages/mcp-server/package.json index c0153a4..bf42565 100644 --- a/packages/mcp-server/package.json +++ b/packages/mcp-server/package.json @@ -63,7 +63,7 @@ "scripts": { "start": "node src/index.js", "login": "node src/login.js", - "test": "node --test \"test/*.test.js\"" + "test": "node --test \"test/**/*.test.js\"" }, "dependencies": { "@modelcontextprotocol/sdk": "^1.0.0", From 019e24cc6071112e4b40a8299ca78b8580ff3082 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 10:46:09 +0300 Subject: [PATCH 016/246] feat(sync): add schema diff -> ordered plan (create/update/orphan, ref-stable) Co-Authored-By: Claude Opus 4.8 --- packages/mcp-server/src/sync/diff.js | 281 ++++++++++++++++++ .../mcp-server/test/sync/test-diff.test.js | 53 ++++ 2 files changed, 334 insertions(+) create mode 100644 packages/mcp-server/src/sync/diff.js create mode 100644 packages/mcp-server/test/sync/test-diff.test.js diff --git a/packages/mcp-server/src/sync/diff.js b/packages/mcp-server/src/sync/diff.js new file mode 100644 index 0000000..7b1eb78 --- /dev/null +++ b/packages/mcp-server/src/sync/diff.js @@ -0,0 +1,281 @@ +import { canonicalizeComputed } from './remap.js'; + +/** + * Build a flat map of { fieldId → fieldName } across all tables in a snapshot. + * @param {{ tables: Array<{fields: Array<{id:string, name:string}>}> }} snap + * @returns {Record} + */ +function fldNameMap(snap) { + const m = {}; + for (const t of snap.tables) for (const f of t.fields) m[f.id] = f.name; + return m; +} + +/** + * Build a regex that matches any field ID key in the given map, wrapped in + * curly braces (the Airtable formula token syntax: `{fldXYZ}`). Returns null + * if the map is empty. + * @param {Record} idToName + * @returns {RegExp|null} + */ +function buildIdRegex(idToName) { + const ids = Object.keys(idToName); + if (ids.length === 0) return null; + // Escape each ID and sort longest-first to avoid partial matches. + const escaped = ids + .slice() + .sort((a, b) => b.length - a.length) + .map((id) => id.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')); + return new RegExp(escaped.join('|'), 'g'); +} + +/** + * Replace every occurrence of a field ID found in `idToName` within `str` + * with `{{}}`. Handles both bare IDs (in formulaText) and + * curly-brace-wrapped tokens (`{fldXYZ}` in formulaTextParsed). + * @param {string} str + * @param {Record} idToName + * @returns {string} + */ +function subAllIds(str, idToName) { + if (typeof str !== 'string' || str.length === 0) return str; + const re = buildIdRegex(idToName); + if (!re) return str; + return str.replace(re, (id) => `{{${idToName[id] ?? id}}}`); +} + +/** + * Stable, key-sorted JSON so equal options compare equal regardless of key insertion order. + * @param {unknown} obj + * @returns {string} + */ +function stableStringify(obj) { + if (obj === null || typeof obj !== 'object') return JSON.stringify(obj); + if (Array.isArray(obj)) return `[${obj.map(stableStringify).join(',')}]`; + return `{${Object.keys(obj).sort().map((k) => JSON.stringify(k) + ':' + stableStringify(obj[k])).join(',')}}`; +} + +/** + * Comparable signature for a field. + * - Computed fields canonicalize field-ID references to field names so + * cross-base ID churn is invisible (two formulas that refer to the same-named field + * produce identical signatures even if the underlying field IDs differ). + * We first run our own ID→name substitution over the formula text (which handles + * any ID format, not just `fld`-prefixed ones), then call canonicalizeComputed for + * the remaining structured fields (relation, target, result type). + * - Scalar fields compare type + stable-serialised typeOptions + description. + * + * @param {{ type:string, typeOptions:object|null, description:string|null, isComputed:boolean }} field + * @param {Record} fldNames fieldId → name for the relevant base + * @returns {string} + */ +function fieldSignature(field, fldNames) { + if (field.isComputed) { + // Normalize typeOptions: replace all field-ID tokens with names so the + // signature is base-agnostic (works for both `fldXYZ` and short test IDs). + const opts = field.typeOptions || {}; + const normalizedOpts = { + ...opts, + formulaTextParsed: subAllIds(opts.formulaTextParsed ?? '', fldNames), + formulaText: subAllIds(opts.formulaText ?? '', fldNames), + formula: subAllIds(opts.formula ?? '', fldNames), + relationColumnId: fldNames[opts.relationColumnId ?? ''] ?? opts.relationColumnId ?? '', + recordLinkFieldId: fldNames[opts.recordLinkFieldId ?? ''] ?? opts.recordLinkFieldId ?? '', + foreignTableRollupColumnId: fldNames[opts.foreignTableRollupColumnId ?? ''] ?? opts.foreignTableRollupColumnId ?? '', + fieldIdInLinkedTable: fldNames[opts.fieldIdInLinkedTable ?? ''] ?? opts.fieldIdInLinkedTable ?? '', + }; + // canonicalizeComputed handles structured extraction; pass empty idToName + // since we already resolved all IDs above. + return 'C|' + field.type + '|' + canonicalizeComputed(field.type, normalizedOpts, {}) + '|' + (field.description ?? ''); + } + return 'S|' + field.type + '|' + stableStringify(field.typeOptions ?? null) + '|' + (field.description ?? ''); +} + +/** Link-type fields must be created after plain scalar fields. Computed last. */ +const LINK_TYPES = new Set(['multipleRecordLinks', 'foreignKey']); +function fieldOrder(f) { return f.isComputed ? 2 : (LINK_TYPES.has(f.type) ? 1 : 0); } + +/** + * Collect all field IDs referenced by a field (formula tokens, link columns, etc.) + * so the executor can topologically sort creation order. + * @param {{ typeOptions: object|null }} field + * @returns {string[]} + */ +function referencedFieldIds(field) { + const o = field.typeOptions || {}; + const ids = new Set(); + const deps = o.dependencies?.referencedColumnIdsForValue; + if (Array.isArray(deps)) deps.forEach((id) => ids.add(id)); + for (const k of ['relationColumnId', 'recordLinkFieldId', 'foreignTableRollupColumnId', 'fieldIdInLinkedTable']) { + if (o[k]) ids.add(o[k]); + } + const formula = o.formulaTextParsed ?? o.formulaText ?? ''; + if (typeof formula === 'string') (formula.match(/fld[A-Za-z0-9]+/g) || []).forEach((id) => ids.add(id)); + return [...ids]; +} + +/** + * Build a createField action for a source field. + * @param {{ id:string }} srcTable + * @param {{ id:string, name:string, type:string, typeOptions:object|null, isComputed:boolean }} f + * @returns {object} + */ +function makeCreateField(srcTable, f) { + return { + kind: 'createField', + sourceTableId: srcTable.id, + sourceFieldId: f.id, + name: f.name, + type: f.type, + typeOptions: f.typeOptions, + computed: f.isComputed, + dependsOn: referencedFieldIds(f), + }; +} + +/** + * Compute a schema-sync plan by diffing source and dest snapshots. + * + * The plan contains: + * - `actions` — ordered list of schema mutations needed to bring dest up to date with src. + * Action kinds (in emission order): + * - `createTable` — dest has no table by this name; create it. + * - `reconcilePrimary` — emitted immediately after createTable (or when the primary + * field name/type diverges on an existing table). + * - `createField` — field exists in src but not in dest (for this table). + * - `updateField` — field exists in both but has diverged (type / typeOptions / + * description differ); carries a `changes` map of what to update. + * - `orphans` — dest tables/fields with no counterpart in src (reported only, never mutated). + * - `warnings` — non-fatal issues such as duplicate field names in src or approaching + * Airtable's 500-field-per-table limit. + * + * @param {{ baseId:string, tables: Array<{id:string, name:string, primaryFieldId:string, fields:Array<{id:string,name:string,type:string,typeOptions:object|null,description:string|null,isComputed:boolean}>}> }} srcSnap + * @param {{ baseId:string, tables: Array<{id:string, name:string, primaryFieldId:string, fields:Array<{id:string,name:string,type:string,typeOptions:object|null,description:string|null,isComputed:boolean}>}> }} destSnap + * @param {{ tables: Record, fields: Record}> }} idmap + * @returns {{ sourceBaseId:string, destBaseId:string, idmap:object, actions:object[], orphans:object[], warnings:object[] }} + */ +export function computePlan(srcSnap, destSnap, idmap) { + const actions = []; + const orphans = []; + const warnings = []; + + const srcNames = fldNameMap(srcSnap); + const destNames = fldNameMap(destSnap); + const destTablesById = new Map(destSnap.tables.map((t) => [t.id, t])); + + // ── Duplicate-name detection in src (warn; first-occurrence wins in idmap) ── + for (const st of srcSnap.tables) { + const seen = new Set(); + for (const f of st.fields) { + if (seen.has(f.name)) { + warnings.push({ code: 'DUPLICATE_NAME', message: `Table "${st.name}" has duplicate field name "${f.name}"` }); + } + seen.add(f.name); + } + } + + // ── Per-table diff ──────────────────────────────────────────────────────── + for (const st of srcSnap.tables) { + const destTableId = idmap.tables[st.id]; + const destTable = destTableId ? destTablesById.get(destTableId) : null; + + if (!destTable) { + // Table doesn't exist in dest → create it + all its fields. + actions.push({ kind: 'createTable', sourceTableId: st.id, name: st.name }); + + const primary = st.fields.find((f) => f.id === st.primaryFieldId) || st.fields[0]; + if (primary) { + actions.push({ + kind: 'reconcilePrimary', + sourceTableId: st.id, + toName: primary.name, + toType: primary.type, + toTypeOptions: primary.typeOptions, + }); + } + + // Non-primary fields sorted: scalars → links → computed. + const rest = st.fields.filter((f) => f !== primary); + for (const f of [...rest].sort((a, b) => fieldOrder(a) - fieldOrder(b))) { + actions.push(makeCreateField(st, f)); + } + continue; + } + + // Table exists — reconcile primary if name or type diverged. + const srcPrimary = st.fields.find((f) => f.id === st.primaryFieldId); + const destPrimary = destTable.fields.find((f) => f.id === destTable.primaryFieldId); + if ( + srcPrimary && destPrimary && + (srcPrimary.name !== destPrimary.name || srcPrimary.type !== destPrimary.type) + ) { + actions.push({ + kind: 'reconcilePrimary', + sourceTableId: st.id, + toName: srcPrimary.name, + toType: srcPrimary.type, + toTypeOptions: srcPrimary.typeOptions, + }); + } + + // Index dest fields by name for O(1) lookup. + const destFieldsByName = new Map(destTable.fields.map((f) => [f.name, f])); + + const newFields = []; + for (const sf of st.fields) { + if (sf.id === st.primaryFieldId) continue; // primary handled above + + const df = destFieldsByName.get(sf.name); + if (!df) { + newFields.push(sf); + continue; + } + + // Field exists in both — compare signatures. + if (fieldSignature(sf, srcNames) !== fieldSignature(df, destNames)) { + const changes = {}; + if (sf.type !== df.type) changes.type = sf.type; + if (stableStringify(sf.typeOptions ?? null) !== stableStringify(df.typeOptions ?? null)) { + changes.typeOptions = sf.typeOptions; + } + if ((sf.description ?? null) !== (df.description ?? null)) { + changes.description = sf.description; + } + if (Object.keys(changes).length > 0) { + actions.push({ kind: 'updateField', sourceFieldId: sf.id, destFld: df.id, changes }); + } + } + } + + // Emit new fields in dependency-safe order. + for (const f of newFields.sort((a, b) => fieldOrder(a) - fieldOrder(b))) { + actions.push(makeCreateField(st, f)); + } + + // Warn if we'd push dest past Airtable's 500-field cap. + if (destTable.fields.length + newFields.length > 500) { + warnings.push({ code: 'FIELD_CAP', message: `Table "${st.name}" would exceed 500 fields` }); + } + } + + // ── Orphan detection: dest tables/fields not matched by any src counterpart ── + const srcTableByDestId = new Map(Object.entries(idmap.tables).map(([s, d]) => [d, s])); + const srcTablesById = new Map(srcSnap.tables.map((t) => [t.id, t])); + + for (const dt of destSnap.tables) { + const srcTableId = srcTableByDestId.get(dt.id); + if (!srcTableId) { + orphans.push({ kind: 'table', destId: dt.id, name: dt.name }); + continue; + } + const srcTable = srcTablesById.get(srcTableId); + const srcNamesSet = new Set(srcTable.fields.map((f) => f.name)); + for (const df of dt.fields) { + if (!srcNamesSet.has(df.name)) { + orphans.push({ kind: 'field', destId: df.id, name: df.name, tableName: dt.name }); + } + } + } + + return { sourceBaseId: srcSnap.baseId, destBaseId: destSnap.baseId, idmap, actions, orphans, warnings }; +} diff --git a/packages/mcp-server/test/sync/test-diff.test.js b/packages/mcp-server/test/sync/test-diff.test.js new file mode 100644 index 0000000..ead4c2c --- /dev/null +++ b/packages/mcp-server/test/sync/test-diff.test.js @@ -0,0 +1,53 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { computePlan } from '../../src/sync/diff.js'; +import { matchByName } from '../../src/sync/idmap.js'; + +function field(id, name, type, extra = {}) { return { id, name, type, typeOptions: extra.typeOptions ?? null, description: extra.description ?? null, isComputed: !!extra.isComputed }; } + +describe('diff.computePlan', () => { + it('emits createTable + fields for a source-only table', () => { + const src = { baseId: 'appS', tables: [{ id: 'tS', name: 'New', primaryFieldId: 'fS1', fields: [field('fS1','Name','text'), field('fS2','Note','multilineText')] }] }; + const dest = { baseId: 'appD', tables: [] }; + const plan = computePlan(src, dest, matchByName(src, dest)); + const kinds = plan.actions.map(a => a.kind); + assert.deepEqual(kinds, ['createTable', 'reconcilePrimary', 'createField']); + assert.equal(plan.actions[0].name, 'New'); + assert.equal(plan.actions[2].name, 'Note'); + }); + + it('skips an identical existing field (idempotent) and updates a diverged one', () => { + const src = { baseId: 'appS', tables: [{ id: 'tS', name: 'T', primaryFieldId: 'fS1', fields: [ + field('fS1','Name','text'), field('fS2','Count','number',{ typeOptions:{ precision: 0 } }) ] }] }; + const dest = { baseId: 'appD', tables: [{ id: 'tD', name: 'T', primaryFieldId: 'fD1', fields: [ + field('fD1','Name','text'), field('fD2','Count','text') ] }] }; + const plan = computePlan(src, dest, matchByName(src, dest)); + const updates = plan.actions.filter(a => a.kind === 'updateField'); + assert.equal(updates.length, 1); + assert.equal(updates[0].destFld, 'fD2'); + assert.equal(updates[0].changes.type, 'number'); + assert.ok(!plan.actions.some(a => a.kind === 'createField')); + }); + + it('does NOT flag a computed field whose formula differs only by field IDs', () => { + const src = { baseId: 'appS', tables: [{ id: 'tS', name: 'T', primaryFieldId: 'fS1', fields: [ + field('fS1','Name','text'), + field('fS2','Total','formula',{ isComputed:true, typeOptions:{ formulaTextParsed:'{fS1}' } }) ] }] }; + const dest = { baseId: 'appD', tables: [{ id: 'tD', name: 'T', primaryFieldId: 'fD1', fields: [ + field('fD1','Name','text'), + field('fD2','Total','formula',{ isComputed:true, typeOptions:{ formulaTextParsed:'{fD1}' } }) ] }] }; + const plan = computePlan(src, dest, matchByName(src, dest)); + assert.equal(plan.actions.length, 0, 'identical-by-name formula must not produce an action'); + }); + + it('reports dest-only tables/fields as orphans, never as actions', () => { + const src = { baseId: 'appS', tables: [{ id: 'tS', name: 'T', primaryFieldId: 'fS1', fields: [field('fS1','Name','text')] }] }; + const dest = { baseId: 'appD', tables: [ + { id: 'tD', name: 'T', primaryFieldId: 'fD1', fields: [field('fD1','Name','text'), field('fDx','Extra','text')] }, + { id: 'tDghost', name: 'Ghost', primaryFieldId: 'fg', fields: [field('fg','x','text')] } ] }; + const plan = computePlan(src, dest, matchByName(src, dest)); + const orphanNames = plan.orphans.map(o => o.name).sort(); + assert.deepEqual(orphanNames, ['Extra', 'Ghost']); + assert.equal(plan.actions.length, 0); + }); +}); From 0a9444bd72842510fc691c7d763810ab15d380fb Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 10:57:49 +0300 Subject: [PATCH 017/246] fix(sync): canonical typeOptions diff for computed fields + ordering test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract computedSig() helper (description-free, ID-stable) so updateField only includes changes.typeOptions for computed fields when the formula itself genuinely differs — not when field IDs merely differ across bases. Add two new tests: creation-ordering (scalars < links < computed) and the computed-description-only diff guard. Co-Authored-By: Claude Opus 4.8 --- packages/mcp-server/src/sync/diff.js | 53 +++++++++++++------ .../mcp-server/test/sync/test-diff.test.js | 30 +++++++++++ 2 files changed, 66 insertions(+), 17 deletions(-) diff --git a/packages/mcp-server/src/sync/diff.js b/packages/mcp-server/src/sync/diff.js index 7b1eb78..86057ce 100644 --- a/packages/mcp-server/src/sync/diff.js +++ b/packages/mcp-server/src/sync/diff.js @@ -55,6 +55,33 @@ function stableStringify(obj) { return `{${Object.keys(obj).sort().map((k) => JSON.stringify(k) + ':' + stableStringify(obj[k])).join(',')}}`; } +/** + * Description-free canonical signature for a computed field's typeOptions. + * Normalises all field-ID references to their names so cross-base ID churn is + * invisible. Used both by `fieldSignature` (which appends description) and by + * the `updateField` diff to decide whether typeOptions genuinely changed. + * + * @param {{ type:string, typeOptions:object|null }} field + * @param {Record} fldNames fieldId → name for the relevant base + * @returns {string} + */ +function computedSig(field, fldNames) { + const opts = field.typeOptions || {}; + const normalizedOpts = { + ...opts, + formulaTextParsed: subAllIds(opts.formulaTextParsed ?? '', fldNames), + formulaText: subAllIds(opts.formulaText ?? '', fldNames), + formula: subAllIds(opts.formula ?? '', fldNames), + relationColumnId: fldNames[opts.relationColumnId ?? ''] ?? opts.relationColumnId ?? '', + recordLinkFieldId: fldNames[opts.recordLinkFieldId ?? ''] ?? opts.recordLinkFieldId ?? '', + foreignTableRollupColumnId: fldNames[opts.foreignTableRollupColumnId ?? ''] ?? opts.foreignTableRollupColumnId ?? '', + fieldIdInLinkedTable: fldNames[opts.fieldIdInLinkedTable ?? ''] ?? opts.fieldIdInLinkedTable ?? '', + }; + // canonicalizeComputed handles structured extraction; pass empty idToName + // since we already resolved all IDs above. + return 'C|' + field.type + '|' + canonicalizeComputed(field.type, normalizedOpts, {}); +} + /** * Comparable signature for a field. * - Computed fields canonicalize field-ID references to field names so @@ -71,22 +98,7 @@ function stableStringify(obj) { */ function fieldSignature(field, fldNames) { if (field.isComputed) { - // Normalize typeOptions: replace all field-ID tokens with names so the - // signature is base-agnostic (works for both `fldXYZ` and short test IDs). - const opts = field.typeOptions || {}; - const normalizedOpts = { - ...opts, - formulaTextParsed: subAllIds(opts.formulaTextParsed ?? '', fldNames), - formulaText: subAllIds(opts.formulaText ?? '', fldNames), - formula: subAllIds(opts.formula ?? '', fldNames), - relationColumnId: fldNames[opts.relationColumnId ?? ''] ?? opts.relationColumnId ?? '', - recordLinkFieldId: fldNames[opts.recordLinkFieldId ?? ''] ?? opts.recordLinkFieldId ?? '', - foreignTableRollupColumnId: fldNames[opts.foreignTableRollupColumnId ?? ''] ?? opts.foreignTableRollupColumnId ?? '', - fieldIdInLinkedTable: fldNames[opts.fieldIdInLinkedTable ?? ''] ?? opts.fieldIdInLinkedTable ?? '', - }; - // canonicalizeComputed handles structured extraction; pass empty idToName - // since we already resolved all IDs above. - return 'C|' + field.type + '|' + canonicalizeComputed(field.type, normalizedOpts, {}) + '|' + (field.description ?? ''); + return computedSig(field, fldNames) + '|' + (field.description ?? ''); } return 'S|' + field.type + '|' + stableStringify(field.typeOptions ?? null) + '|' + (field.description ?? ''); } @@ -235,7 +247,14 @@ export function computePlan(srcSnap, destSnap, idmap) { if (fieldSignature(sf, srcNames) !== fieldSignature(df, destNames)) { const changes = {}; if (sf.type !== df.type) changes.type = sf.type; - if (stableStringify(sf.typeOptions ?? null) !== stableStringify(df.typeOptions ?? null)) { + // For computed fields, only flag typeOptions when the canonical (ID-stable) + // options differ — not when field IDs merely differ across bases. + // For scalar fields, use a raw stable-stringify comparison. + if (sf.isComputed) { + if (computedSig(sf, srcNames) !== computedSig(df, destNames)) { + changes.typeOptions = sf.typeOptions; + } + } else if (stableStringify(sf.typeOptions ?? null) !== stableStringify(df.typeOptions ?? null)) { changes.typeOptions = sf.typeOptions; } if ((sf.description ?? null) !== (df.description ?? null)) { diff --git a/packages/mcp-server/test/sync/test-diff.test.js b/packages/mcp-server/test/sync/test-diff.test.js index ead4c2c..85c85b2 100644 --- a/packages/mcp-server/test/sync/test-diff.test.js +++ b/packages/mcp-server/test/sync/test-diff.test.js @@ -50,4 +50,34 @@ describe('diff.computePlan', () => { assert.deepEqual(orphanNames, ['Extra', 'Ghost']); assert.equal(plan.actions.length, 0); }); + + it('orders new-table fields: scalars before links before computed', () => { + const src = { baseId: 'appS', tables: [{ id: 'tS', name: 'T', primaryFieldId: 'fS1', fields: [ + field('fS1', 'Name', 'text'), + field('fS4', 'Formula', 'formula', { isComputed: true, typeOptions: { formulaTextParsed: '1' } }), + field('fS3', 'Link', 'multipleRecordLinks'), + field('fS2', 'Score', 'number'), + ] }] }; + const dest = { baseId: 'appD', tables: [] }; + const plan = computePlan(src, dest, matchByName(src, dest)); + const createNames = plan.actions.filter(a => a.kind === 'createField').map(a => a.name); + assert.ok(createNames.indexOf('Score') < createNames.indexOf('Link'), 'scalar before link'); + assert.ok(createNames.indexOf('Link') < createNames.indexOf('Formula'), 'link before computed'); + }); + + it('computed field differing only by description -> changes has description, not typeOptions', () => { + const src = { baseId: 'appS', tables: [{ id: 'tS', name: 'T', primaryFieldId: 'fS1', fields: [ + field('fS1', 'Name', 'text'), + { id: 'fS2', name: 'Total', type: 'formula', typeOptions: { formulaTextParsed: '{fS1}' }, description: 'new', isComputed: true }, + ] }] }; + const dest = { baseId: 'appD', tables: [{ id: 'tD', name: 'T', primaryFieldId: 'fD1', fields: [ + field('fD1', 'Name', 'text'), + { id: 'fD2', name: 'Total', type: 'formula', typeOptions: { formulaTextParsed: '{fD1}' }, description: 'old', isComputed: true }, + ] }] }; + const plan = computePlan(src, dest, matchByName(src, dest)); + const upd = plan.actions.filter(a => a.kind === 'updateField'); + assert.equal(upd.length, 1); + assert.equal(upd[0].changes.description, 'new'); + assert.equal(upd[0].changes.typeOptions, undefined, 'no spurious typeOptions for ID-only formula difference'); + }); }); From 168ce62b29a47cacd86b47f5e7acd2b571e9f33d Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 11:00:39 +0300 Subject: [PATCH 018/246] feat(sync): add plan report + orchestrator with dest fingerprint Co-Authored-By: Claude Opus 4.8 --- packages/mcp-server/src/sync/index.js | 51 +++++++++++++++++++ packages/mcp-server/src/sync/report.js | 24 +++++++++ .../mcp-server/test/sync/test-index.test.js | 23 +++++++++ .../mcp-server/test/sync/test-report.test.js | 18 +++++++ 4 files changed, 116 insertions(+) create mode 100644 packages/mcp-server/src/sync/index.js create mode 100644 packages/mcp-server/src/sync/report.js create mode 100644 packages/mcp-server/test/sync/test-index.test.js create mode 100644 packages/mcp-server/test/sync/test-report.test.js diff --git a/packages/mcp-server/src/sync/index.js b/packages/mcp-server/src/sync/index.js new file mode 100644 index 0000000..3ae8622 --- /dev/null +++ b/packages/mcp-server/src/sync/index.js @@ -0,0 +1,51 @@ +import { createHash } from 'node:crypto'; +import { snapshotBase } from './snapshot.js'; +import { matchByName, saveIdmap, savePlan, saveState } from './idmap.js'; +import { computePlan } from './diff.js'; +import { renderPlan } from './report.js'; + +const ENGINE_VERSION = '2a'; + +/** + * Produce a deterministic SHA-256 fingerprint of a schema snapshot. + * Order-independent: tables are sorted before hashing. + * + * @param {{ tables: Array<{id:string, name:string, fields:Array<{id:string,name:string,type:string}>}> }} snap + * @returns {string} hex digest + */ +export function fingerprintSchema(snap) { + const basis = snap.tables + .map((t) => `${t.id}:${t.name}:` + t.fields.map((f) => `${f.id}=${f.name}=${f.type}`).join(',')) + .sort() + .join('|'); + return createHash('sha256').update(basis).digest('hex'); +} + +/** + * Compute a schema plan. `planId` is supplied by the caller (tool handler) so + * the engine stays deterministic/testable. + * + * @param {{ client: object, sourceBaseId: string, destBaseId: string, planId: string }} opts + * @returns {Promise<{ human: string, machine: object }>} + */ +export async function plan({ client, sourceBaseId, destBaseId, planId }) { + const src = await snapshotBase(client, sourceBaseId); + const dest = await snapshotBase(client, destBaseId); + const idmap = matchByName(src, dest); + const base = computePlan(src, dest, idmap); + const fullPlan = { + planId, + engineVersion: ENGINE_VERSION, + destFingerprint: fingerprintSchema(dest), + ...base, + }; + saveIdmap(sourceBaseId, destBaseId, idmap); + savePlan(sourceBaseId, destBaseId, fullPlan); + saveState(sourceBaseId, destBaseId, { + sourceBaseId, + destBaseId, + engineVersion: ENGINE_VERSION, + lastPlanId: planId, + }); + return renderPlan(fullPlan); +} diff --git a/packages/mcp-server/src/sync/report.js b/packages/mcp-server/src/sync/report.js new file mode 100644 index 0000000..54044d2 --- /dev/null +++ b/packages/mcp-server/src/sync/report.js @@ -0,0 +1,24 @@ +/** + * Human-readable and machine-readable rendering of a schema sync plan. + */ + +/** + * Render a plan into a human-readable summary and a machine-readable copy. + * + * @param {{ actions: object[], orphans: object[], warnings: object[] }} plan + * @returns {{ human: string, machine: object }} + */ +export function renderPlan(plan) { + const counts = {}; + for (const a of plan.actions) counts[a.kind] = (counts[a.kind] || 0) + 1; + const lines = ['Schema plan:']; + for (const k of ['createTable', 'reconcilePrimary', 'createField', 'updateField']) { + if (counts[k]) lines.push(` ${k}: ${counts[k]}`); + } + lines.push(` orphans: ${plan.orphans.length} (reported, not changed)`); + if (plan.warnings.length) { + lines.push(' warnings:'); + for (const w of plan.warnings) lines.push(` - ${w.code}: ${w.message}`); + } + return { human: lines.join('\n'), machine: plan }; +} diff --git a/packages/mcp-server/test/sync/test-index.test.js b/packages/mcp-server/test/sync/test-index.test.js new file mode 100644 index 0000000..9c8e4e5 --- /dev/null +++ b/packages/mcp-server/test/sync/test-index.test.js @@ -0,0 +1,23 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { plan as computeSyncPlan, fingerprintSchema } from '../../src/sync/index.js'; +import { mkdtempSync, existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { syncDir } from '../../src/sync/idmap.js'; + +describe('sync index.plan', () => { + it('produces a plan summary and persists plan-.json', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-idx-')); + const mk = (tableName, fieldName) => ({ data: { tableSchemas: [{ id: 'tbl1', name: tableName, primaryColumnId: 'fld1', columns: [{ id: 'fld1', name: fieldName, type: 'text' }] }] } }); + const client = { getApplicationData: async (appId) => (appId === 'appSSSSSSSSSSSSSS' ? mk('T', 'Name') : { data: { tableSchemas: [] } }) }; + const out = await computeSyncPlan({ client, sourceBaseId: 'appSSSSSSSSSSSSSS', destBaseId: 'appDDDDDDDDDDDDDD', planId: 'plnTEST' }); + assert.match(out.human, /createTable: 1/); + assert.ok(existsSync(join(syncDir('appSSSSSSSSSSSSSS', 'appDDDDDDDDDDDDDD'), 'plan-plnTEST.json'))); + }); + it('fingerprintSchema is stable + order-independent', () => { + const a = { tables: [{ id: 't1', name: 'A', fields: [{ id: 'f1', name: 'x', type: 'text' }] }, { id: 't2', name: 'B', fields: [] }] }; + const b = { tables: [{ id: 't2', name: 'B', fields: [] }, { id: 't1', name: 'A', fields: [{ id: 'f1', name: 'x', type: 'text' }] }] }; + assert.equal(fingerprintSchema(a), fingerprintSchema(b)); + }); +}); diff --git a/packages/mcp-server/test/sync/test-report.test.js b/packages/mcp-server/test/sync/test-report.test.js new file mode 100644 index 0000000..6c260f1 --- /dev/null +++ b/packages/mcp-server/test/sync/test-report.test.js @@ -0,0 +1,18 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { renderPlan } from '../../src/sync/report.js'; + +describe('report.renderPlan', () => { + it('summarizes action/orphan/warning counts', () => { + const plan = { actions: [ + { kind: 'createTable', name: 'T' }, { kind: 'createField', name: 'a' }, + { kind: 'updateField', destFld: 'fD' } ], orphans: [{ kind: 'field', name: 'x' }], warnings: [{ code: 'FIELD_CAP', message: 'm' }] }; + const out = renderPlan(plan); + assert.equal(out.machine, plan); + assert.match(out.human, /createTable: 1/); + assert.match(out.human, /createField: 1/); + assert.match(out.human, /updateField: 1/); + assert.match(out.human, /orphans: 1/); + assert.match(out.human, /FIELD_CAP/); + }); +}); From 679868b4b00a98c5c95b66d9d4331ad2012aa138 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 11:05:05 +0300 Subject: [PATCH 019/246] feat(mcp): add sync_base tool (mode=plan) in new sync category Co-Authored-By: Claude Sonnet 4.6 --- packages/mcp-server/src/index.js | 26 +++++++++++++++++++ packages/mcp-server/src/tool-config.js | 8 ++++-- .../mcp-server/test/test-tool-config.test.js | 20 +++++++++----- 3 files changed, 45 insertions(+), 9 deletions(-) diff --git a/packages/mcp-server/src/index.js b/packages/mcp-server/src/index.js index b104ce3..2ae8468 100644 --- a/packages/mcp-server/src/index.js +++ b/packages/mcp-server/src/index.js @@ -1575,6 +1575,22 @@ Note: "form title" is the view name itself — use rename_view to change it. "Fi required: ['appId', 'tableId', 'rowIds'], }, }, + // ── Sync Tools ── + { + name: 'sync_base', + description: 'Plan a base-to-base schema sync (READ-ONLY): compares source and destination schema by name and returns an ordered plan of tables/fields to create or update, plus reported orphans and warnings. Does NOT mutate the destination. (mode "apply" arrives in a later release.)', + annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + inputSchema: { + type: 'object', + properties: { + mode: { type: 'string', enum: ['plan'], description: 'Only "plan" is supported in this release (read-only).' }, + sourceAppId: { type: 'string', description: 'Source base/application ID to copy schema FROM' }, + destAppId: { type: 'string', description: 'Destination base/application ID to copy schema TO' }, + debug: debugProp, + }, + required: ['mode', 'sourceAppId', 'destAppId'], + }, + }, ]; // ─── Meta-Tool: manage_tools ───────────────────────────────── @@ -2369,6 +2385,16 @@ const handlers = { return ok(result, result, debug); }, + // ── Sync ── + + async sync_base({ mode, sourceAppId, destAppId, debug }) { + if (mode !== 'plan') return err(`Unsupported mode "${mode}". Only "plan" is available in this release.`); + const { plan } = await import('./sync/index.js'); + const planId = 'pln' + client._genRandomId(); + const out = await plan({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, planId }); + return ok({ planId, summary: out.human }, out.machine, debug); + }, + // ── Meta: Tool Management ── async manage_tools({ action, profile, tool, category, enabled }) { diff --git a/packages/mcp-server/src/tool-config.js b/packages/mcp-server/src/tool-config.js index 3c8d190..4cb3b02 100644 --- a/packages/mcp-server/src/tool-config.js +++ b/packages/mcp-server/src/tool-config.js @@ -113,6 +113,9 @@ export const TOOL_CATEGORIES = { // Record Destructive (batch record deletion) delete_records: 'record-destructive', + + // Sync (base-to-base schema sync — read-only plan mode) + sync_base: 'sync', }; /** Human-readable labels for categories */ @@ -131,6 +134,7 @@ export const CATEGORY_LABELS = { 'extension': 'Extension Management', 'record-write': 'Record Write', 'record-destructive': 'Record Destructive', + 'sync': 'Sync', }; // ─── Built-in Profiles ─────────────────────────────────────── @@ -142,7 +146,7 @@ export const BUILTIN_PROFILES = { }, 'safe-write': { description: 'Read + record read/write + create/update tables, fields, views, and sidebar sections (no deletes, no form metadata)', - categories: ['read', 'record-read', 'record-write', 'table-write', 'field-write', 'view-write', 'view-section'], + categories: ['read', 'record-read', 'record-write', 'table-write', 'field-write', 'view-write', 'view-section', 'sync'], }, full: { description: 'All tools enabled including destructive ops, form metadata, and extensions', @@ -152,7 +156,7 @@ export const BUILTIN_PROFILES = { 'field-write', 'field-destructive', 'view-write', 'view-destructive', 'view-section', 'view-section-destructive', - 'form-write', 'extension', + 'form-write', 'extension', 'sync', ], }, }; diff --git a/packages/mcp-server/test/test-tool-config.test.js b/packages/mcp-server/test/test-tool-config.test.js index 2743e87..24c79d8 100644 --- a/packages/mcp-server/test/test-tool-config.test.js +++ b/packages/mcp-server/test/test-tool-config.test.js @@ -17,7 +17,7 @@ import { describe('TOOL_CATEGORIES', () => { it('maps all tools to valid categories', () => { const tools = Object.keys(TOOL_CATEGORIES); - assert.equal(tools.length, 69, `Expected 69 tools, got ${tools.length}`); + assert.equal(tools.length, 70, `Expected 70 tools, got ${tools.length}`); for (const [tool, cat] of Object.entries(TOOL_CATEGORIES)) { assert.ok(CATEGORY_LABELS[cat], `Tool "${tool}" has unknown category "${cat}"`); } @@ -84,9 +84,9 @@ describe('ToolConfigManager', () => { assert.equal(mgr.activeProfile, 'full'); }); - it('enables all 69 tools on full profile', () => { + it('enables all 70 tools on full profile', () => { const enabled = mgr.enabledToolNames(); - assert.equal(enabled.size, 69); + assert.equal(enabled.size, 70); }); it('manage_tools is always enabled', () => { @@ -125,10 +125,10 @@ describe('ToolConfigManager', () => { assert.ok(!enabled.has('delete_records')); }); - it('full enables all 69 tools', async () => { + it('full enables all 70 tools', async () => { await mgr.switchProfile('full'); const enabled = mgr.enabledToolNames(); - assert.equal(enabled.size, 69); + assert.equal(enabled.size, 70); }); it('unknown profile fails closed to read-only', async () => { @@ -211,10 +211,10 @@ describe('ToolConfigManager', () => { }); describe('getToolStatus()', () => { - it('returns status for all 69 tools', async () => { + it('returns status for all 70 tools', async () => { await mgr.switchProfile('full'); const status = mgr.getToolStatus(); - assert.equal(status.length, 69); + assert.equal(status.length, 70); assert.ok(status.every(s => s.enabled === true)); }); @@ -248,3 +248,9 @@ describe('record-write tools', () => { assert.equal(TOOL_CATEGORIES.delete_records, 'record-destructive'); }); }); + +describe('sync tools', () => { + it('maps sync_base to the sync category', () => { + assert.equal(TOOL_CATEGORIES.sync_base, 'sync'); + }); +}); From b645f2cc11db66b465014e8687ea001380867a61 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 11:10:00 +0300 Subject: [PATCH 020/246] chore(tool-sync): register sync category + sync_base across mirror + counts Co-Authored-By: Claude Sonnet 4.6 --- packages/extension/package.json | 13 +++++++++---- packages/extension/src/mcp/tool-profile.ts | 13 ++++++++++--- packages/shared/src/types.ts | 1 + packages/webview/src/store.ts | 5 +++-- packages/webview/src/tabs/Settings.tsx | 1 + packages/webview/src/test/store.test.ts | 2 +- 6 files changed, 25 insertions(+), 10 deletions(-) diff --git a/packages/extension/package.json b/packages/extension/package.json index 28ee3aa..80036f9 100644 --- a/packages/extension/package.json +++ b/packages/extension/package.json @@ -4,7 +4,7 @@ "version": "2.0.94", "publisher": "Nskha", "displayName": "Airtable Formulas, Scripts, Automation, MCP & LSP", - "description": "Airtable formula, script & automation editor with MCP server (66 tools), language server, and AI skills for VS Code.", + "description": "Airtable formula, script & automation editor with MCP server (70 tools), language server, and AI skills for VS Code.", "icon": "images/icon.png", "license": "MIT", "author": { @@ -491,8 +491,8 @@ ], "enumDescriptions": [ "Schema inspection, formula validation, and record reading only (12 tools)", - "Read + record read/write + create/update tables, fields, views, sidebar sections, and record templates, no deletes, no form metadata (53 tools)", - "All tools enabled including destructive ops, form metadata, and extensions (69 tools)", + "Read + record read/write + create/update tables, fields, views, sidebar sections, and record templates, no deletes, no form metadata (54 tools)", + "All tools enabled including destructive ops, form metadata, and extensions (70 tools)", "User-defined per-tool selection" ], "description": "Active MCP tool profile. Controls which tools are exposed to AI agents through the bundled Airtable MCP server." @@ -567,6 +567,11 @@ "default": true, "description": "Record Destructive tools: delete_records (batch record deletion)." }, + "airtableFormula.mcp.categories.sync": { + "type": "boolean", + "default": true, + "description": "Sync tools: sync_base (base-to-base schema plan — read-only plan mode)." + }, "airtableFormula.mcp.tools": { "type": "object", "default": {}, @@ -674,7 +679,7 @@ { "id": "AirtableFormula.server", "label": "Airtable User MCP", - "description": "Manage Airtable bases via 69 MCP tools: schema read, table CRUD, field CRUD (including formula/rollup/lookup/download), view configuration (filters / sorts / groups / visibility / column ordering / freezing — with merge-safe append mode), sidebar section grouping, view-type metadata (Kanban/Gallery/Calendar covers, colors, calendar date ranges), legacy-form metadata, formula validation, formula download, bulk base formula download, record template management (create / rename / pre-fill cells / duplicate / apply / delete), record CRUD (create / update / delete / duplicate), and extension management. Operates over stdio using JSON-RPC 2.0." + "description": "Manage Airtable bases via 70 MCP tools: schema read, table CRUD, field CRUD (including formula/rollup/lookup/download), view configuration (filters / sorts / groups / visibility / column ordering / freezing — with merge-safe append mode), sidebar section grouping, view-type metadata (Kanban/Gallery/Calendar covers, colors, calendar date ranges), legacy-form metadata, formula validation, formula download, bulk base formula download, record template management (create / rename / pre-fill cells / duplicate / apply / delete), record CRUD (create / update / delete / duplicate), and extension management. Operates over stdio using JSON-RPC 2.0." } ] }, diff --git a/packages/extension/src/mcp/tool-profile.ts b/packages/extension/src/mcp/tool-profile.ts index 8723713..03c39b2 100644 --- a/packages/extension/src/mcp/tool-profile.ts +++ b/packages/extension/src/mcp/tool-profile.ts @@ -28,7 +28,8 @@ type ExtraCategoryKey = | 'view-write' | 'view-destructive' | 'view-section' | 'view-section-destructive' | 'form-write' - | 'record-read' | 'record-write' | 'record-destructive'; + | 'record-read' | 'record-write' | 'record-destructive' + | 'sync'; export const TOOL_CATEGORIES: Record = { // Read-only / inspection get_base_schema: 'read', @@ -113,6 +114,8 @@ export const TOOL_CATEGORIES: Record = { @@ -130,6 +133,7 @@ export const CATEGORY_LABELS: Record = { 'extension': 'Extension Management', 'record-write': 'Record Write', 'record-destructive': 'Record Destructive', + 'sync': 'Sync', }; interface ProfileDef { @@ -141,7 +145,7 @@ interface ProfileDef { export const BUILTIN_PROFILES: Record<'read-only' | 'safe-write' | 'full', ProfileDef> = { 'read-only': { description: 'Schema inspection, formula validation, and record reading only', categories: ['read', 'record-read'] }, 'safe-write':{ description: 'Read + record read/write + create/update tables, fields, views, sidebar sections, and record templates (no deletes, no form metadata)', - categories: ['read', 'record-read', 'record-write', 'table-write', 'field-write', 'view-write', 'view-section'] }, + categories: ['read', 'record-read', 'record-write', 'table-write', 'field-write', 'view-write', 'view-section', 'sync'] }, full: { description: 'All tools enabled including destructive ops, form metadata, and extensions', categories: [ 'read', 'record-read', 'record-write', 'record-destructive', @@ -149,7 +153,7 @@ export const BUILTIN_PROFILES: Record<'read-only' | 'safe-write' | 'full', Profi 'field-write', 'field-destructive', 'view-write', 'view-destructive', 'view-section', 'view-section-destructive', - 'form-write', 'extension', + 'form-write', 'extension', 'sync', ] }, }; @@ -169,6 +173,7 @@ const SETTINGS_TO_CATEGORY: Record = { formWrite: 'form-write', extension: 'extension', recordWrite: 'record-write', + sync: 'sync', }; // Inverse: file-format category key → settings key suffix @@ -272,6 +277,7 @@ export class ToolProfileManager implements vscode.Disposable { formWrite: cfg.get('mcp.categories.formWrite', true), extension: cfg.get('mcp.categories.extension', true), recordWrite: cfg.get('mcp.categories.recordWrite', true), + sync: cfg.get('mcp.categories.sync', true), }; return { profile, @@ -327,6 +333,7 @@ export class ToolProfileManager implements vscode.Disposable { 'form-write', 'extension', 'record-write', 'record-destructive', + 'sync', ]; for (const cat of categoryOrder) { const label = CATEGORY_LABELS[cat] ?? cat; diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index 806c3e2..1afa4b9 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -107,6 +107,7 @@ export interface ToolCategories { formWrite: boolean; extension: boolean; recordWrite: boolean; + sync: boolean; } export interface ToolProfileSnapshot { diff --git a/packages/webview/src/store.ts b/packages/webview/src/store.ts index 498d292..aa998b3 100644 --- a/packages/webview/src/store.ts +++ b/packages/webview/src/store.ts @@ -62,14 +62,15 @@ const defaultSettings: SettingsSnapshot = { notifyOnUpdates: true, toolProfile: { profile: 'safe-write', - enabledCount: 53, - totalCount: 69, + enabledCount: 54, + totalCount: 70, categories: { read: true, recordRead: true, tableWrite: true, tableDestructive: true, fieldWrite: true, fieldDestructive: true, recordDestructive: true, viewWrite: true, viewDestructive: true, viewSection: true, viewSectionDestructive: true, formWrite: true, extension: true, recordWrite: true, + sync: true, }, }, serverSource: 'bundled' as const, diff --git a/packages/webview/src/tabs/Settings.tsx b/packages/webview/src/tabs/Settings.tsx index ac2ae98..6e257e1 100644 --- a/packages/webview/src/tabs/Settings.tsx +++ b/packages/webview/src/tabs/Settings.tsx @@ -612,6 +612,7 @@ export function Settings() { + )} diff --git a/packages/webview/src/test/store.test.ts b/packages/webview/src/test/store.test.ts index 378d3a2..762c963 100644 --- a/packages/webview/src/test/store.test.ts +++ b/packages/webview/src/test/store.test.ts @@ -13,7 +13,7 @@ beforeEach(() => { ideStatuses: [], versions: { extension: '—', mcpServerBundled: '—' }, aiFilesCount: 0, loading: true, activeTab: 'overview', pendingActions: new Set(), settings: { - mcp: { autoConfigureOnInstall: true, notifyOnUpdates: true, toolProfile: { profile: 'safe-write', enabledCount: 53, totalCount: 69, categories: { read: true, recordRead: true, tableWrite: true, tableDestructive: true, fieldWrite: true, fieldDestructive: true, recordDestructive: true, viewWrite: true, viewDestructive: true, viewSection: true, viewSectionDestructive: true, formWrite: true, extension: true, recordWrite: true } }, serverSource: 'bundled' }, + mcp: { autoConfigureOnInstall: true, notifyOnUpdates: true, toolProfile: { profile: 'safe-write', enabledCount: 54, totalCount: 70, categories: { read: true, recordRead: true, tableWrite: true, tableDestructive: true, fieldWrite: true, fieldDestructive: true, recordDestructive: true, viewWrite: true, viewDestructive: true, viewSection: true, viewSectionDestructive: true, formWrite: true, extension: true, recordWrite: true, sync: true } }, serverSource: 'bundled' }, ai: { autoInstallFiles: true, includeAgents: false }, formula: { formatterVersion: 'v2' }, script: { beautifyStyle: 'default', minifyLevel: 'standard' }, From 0532e5b8d6dcf15c1747dc0b66409e35e97e1de6 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 11:16:59 +0300 Subject: [PATCH 021/246] feat(sync): capture link foreign-table deps + document source-id contract --- packages/mcp-server/src/sync/diff.js | 21 +++++++++++ .../mcp-server/test/sync/test-diff.test.js | 35 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/packages/mcp-server/src/sync/diff.js b/packages/mcp-server/src/sync/diff.js index 86057ce..5df86a4 100644 --- a/packages/mcp-server/src/sync/diff.js +++ b/packages/mcp-server/src/sync/diff.js @@ -126,6 +126,26 @@ function referencedFieldIds(field) { return [...ids]; } +/** + * Collect table IDs that a link field's typeOptions references. + * Reuses LINK_TYPES which is already authoritative for ordering. + * @param {{ type:string, typeOptions:object|null }} field + * @returns {string[]} + */ +function referencedTableIds(field) { + const o = field.typeOptions || {}; + const ids = []; + if (LINK_TYPES.has(field.type) && o.foreignTableId) ids.push(o.foreignTableId); + return ids; +} + +// NOTE (contract for the apply engine): every `typeOptions` / formula text in +// the emitted actions (createField, updateField.changes, reconcilePrimary.toTypeOptions) +// is in SOURCE id-space — it contains source `fld…`/`tbl…` IDs. The apply engine MUST +// remap these to destination IDs via `plan.idmap` (and defer refs whose target field +// is created in the same run) before sending them to Airtable. remap.js is the intended +// single source of truth for that rewrite. + /** * Build a createField action for a source field. * @param {{ id:string }} srcTable @@ -142,6 +162,7 @@ function makeCreateField(srcTable, f) { typeOptions: f.typeOptions, computed: f.isComputed, dependsOn: referencedFieldIds(f), + dependsOnTables: referencedTableIds(f), }; } diff --git a/packages/mcp-server/test/sync/test-diff.test.js b/packages/mcp-server/test/sync/test-diff.test.js index 85c85b2..fc8e12d 100644 --- a/packages/mcp-server/test/sync/test-diff.test.js +++ b/packages/mcp-server/test/sync/test-diff.test.js @@ -80,4 +80,39 @@ describe('diff.computePlan', () => { assert.equal(upd[0].changes.description, 'new'); assert.equal(upd[0].changes.typeOptions, undefined, 'no spurious typeOptions for ID-only formula difference'); }); + + it('captures foreign table dependency on a link field createField', () => { + const src = { baseId: 'appS', tables: [{ id: 'tS', name: 'T', primaryFieldId: 'fS1', fields: [ + field('fS1','Name','text'), + field('fS2','Link','multipleRecordLinks',{ typeOptions:{ foreignTableId: 'tblFOREIGN' } }) ] }] }; + const dest = { baseId: 'appD', tables: [] }; + const plan = computePlan(src, dest, matchByName(src, dest)); + const link = plan.actions.find(a => a.kind === 'createField' && a.name === 'Link'); + assert.ok(link, 'link createField present'); + assert.deepEqual(link.dependsOnTables, ['tblFOREIGN']); + }); + + it('computed createField dependsOn includes referenced source field ids', () => { + const src = { baseId: 'appS', tables: [{ id: 'tS', name: 'T', primaryFieldId: 'fS1', fields: [ + field('fS1','Name','text'), + field('fS2','Total','formula',{ isComputed:true, typeOptions:{ formulaTextParsed:'{fldREF1}+{fldREF2}', dependencies:{ referencedColumnIdsForValue:['fldREF1','fldREF2'] } } }) ] }] }; + const dest = { baseId: 'appD', tables: [] }; + const plan = computePlan(src, dest, matchByName(src, dest)); + const f = plan.actions.find(a => a.kind === 'createField' && a.name === 'Total'); + assert.ok(f.dependsOn.includes('fldREF1') && f.dependsOn.includes('fldREF2')); + }); + + it('emits updateField when an existing select field gained a source choice', () => { + const sel = (choices) => ({ typeOptions: { choices } }); + const src = { baseId: 'appS', tables: [{ id: 'tS', name: 'T', primaryFieldId: 'fS1', fields: [ + field('fS1','Name','text'), + field('fS2','Status','singleSelect', sel({ s1:{id:'s1',name:'Open'}, s2:{id:'s2',name:'Closed'} })) ] }] }; + const dest = { baseId: 'appD', tables: [{ id: 'tD', name: 'T', primaryFieldId: 'fD1', fields: [ + field('fD1','Name','text'), + field('fD2','Status','singleSelect', sel({ d1:{id:'d1',name:'Open'} })) ] }] }; + const plan = computePlan(src, dest, matchByName(src, dest)); + const upd = plan.actions.filter(a => a.kind === 'updateField' && a.destFld === 'fD2'); + assert.equal(upd.length, 1, 'a select field that gained a choice should produce an updateField'); + assert.ok(upd[0].changes.typeOptions, 'carries typeOptions (source choice set; apply engine remaps/merges)'); + }); }); From a72b4ac9cfc22367ae324c84210bca113041d9db Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 18:49:41 +0300 Subject: [PATCH 022/246] feat(sync): carry sourcePrimaryFieldId + field description in plan actions for apply Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/diff.js | 3 +++ packages/mcp-server/test/sync/test-diff.test.js | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/packages/mcp-server/src/sync/diff.js b/packages/mcp-server/src/sync/diff.js index 5df86a4..43c393d 100644 --- a/packages/mcp-server/src/sync/diff.js +++ b/packages/mcp-server/src/sync/diff.js @@ -160,6 +160,7 @@ function makeCreateField(srcTable, f) { name: f.name, type: f.type, typeOptions: f.typeOptions, + description: f.description, computed: f.isComputed, dependsOn: referencedFieldIds(f), dependsOnTables: referencedTableIds(f), @@ -221,6 +222,7 @@ export function computePlan(srcSnap, destSnap, idmap) { actions.push({ kind: 'reconcilePrimary', sourceTableId: st.id, + sourcePrimaryFieldId: primary.id, toName: primary.name, toType: primary.type, toTypeOptions: primary.typeOptions, @@ -245,6 +247,7 @@ export function computePlan(srcSnap, destSnap, idmap) { actions.push({ kind: 'reconcilePrimary', sourceTableId: st.id, + sourcePrimaryFieldId: srcPrimary.id, toName: srcPrimary.name, toType: srcPrimary.type, toTypeOptions: srcPrimary.typeOptions, diff --git a/packages/mcp-server/test/sync/test-diff.test.js b/packages/mcp-server/test/sync/test-diff.test.js index fc8e12d..958f590 100644 --- a/packages/mcp-server/test/sync/test-diff.test.js +++ b/packages/mcp-server/test/sync/test-diff.test.js @@ -116,3 +116,18 @@ describe('diff.computePlan', () => { assert.ok(upd[0].changes.typeOptions, 'carries typeOptions (source choice set; apply engine remaps/merges)'); }); }); + +describe('diff plan contract (M2b extensions)', () => { + it('reconcilePrimary carries sourcePrimaryFieldId; createField carries description', () => { + const src = { baseId: 'appS', tables: [{ id: 'tS', name: 'New', primaryFieldId: 'fS1', fields: [ + field('fS1', 'Name', 'text'), + field('fS2', 'Note', 'multilineText', { description: 'hello' }), + ] }] }; + const dest = { baseId: 'appD', tables: [] }; + const plan = computePlan(src, dest, matchByName(src, dest)); + const prim = plan.actions.find((a) => a.kind === 'reconcilePrimary'); + assert.equal(prim.sourcePrimaryFieldId, 'fS1'); + const note = plan.actions.find((a) => a.kind === 'createField' && a.name === 'Note'); + assert.equal(note.description, 'hello'); + }); +}); From 9b2faa96b2235d76db45a40c542f0121f04d4f61 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 18:54:18 +0300 Subject: [PATCH 023/246] feat(sync): add remapRefs source->dest rewrite + writable computed options Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/remap.js | 68 ++++++++++++++++++ .../test/sync/test-remap-refs.test.js | 71 +++++++++++++++++++ 2 files changed, 139 insertions(+) create mode 100644 packages/mcp-server/test/sync/test-remap-refs.test.js diff --git a/packages/mcp-server/src/sync/remap.js b/packages/mcp-server/src/sync/remap.js index 9f6b6b1..dfbbde7 100644 --- a/packages/mcp-server/src/sync/remap.js +++ b/packages/mcp-server/src/sync/remap.js @@ -22,3 +22,71 @@ export function canonicalizeComputed(type, typeOptions, fldIdToName) { result: opts.result?.type ?? null, }); } + +// ── Source→dest reference rewrite (the apply path's single source of truth) ── +function jsonClone(v) { return v == null ? v : JSON.parse(JSON.stringify(v)); } + +function fldLookup(idmap) { + const m = {}; + for (const [src, v] of Object.entries(idmap.fields || {})) if (v && v.destFld) m[src] = v.destFld; + return m; +} +function selLookup(idmap) { + const m = {}; + for (const v of Object.values(idmap.fields || {})) for (const [s, d] of Object.entries((v && v.choices) || {})) m[s] = d; + return m; +} + +// Replace {column_value_fldXXX} and {fldXXX} tokens inside curly braces only. +// Bare `fldXXX` text outside braces is left untouched — the input form Airtable +// accepts is `{fldXXX}`, NOT the stored `{column_value_fldXXX}`, and bare ids +// outside braces are not formula references. +function subFormulaTokens(str, lookup) { + if (typeof str !== 'string') return str; + return str.replace(/\{(?:column_value_)?((?:fld|sel)[A-Za-z0-9]+)\}/g, (_m, id) => `{${lookup[id] ?? id}}`); +} + +const SINGLE_ID_REF_KEYS = ['relationColumnId', 'recordLinkFieldId', 'foreignTableRollupColumnId', 'fieldIdInLinkedTable']; +const FORMULA_KEYS = ['formulaText', 'formulaTextParsed', 'formula']; + +export function remapRefs(typeOptions, idmap) { + if (typeOptions == null || typeof typeOptions !== 'object') return typeOptions; + const flds = fldLookup(idmap); + const refTokens = { ...flds, ...selLookup(idmap) }; + const tbls = idmap.tables || {}; + const out = jsonClone(typeOptions); + + for (const k of FORMULA_KEYS) if (typeof out[k] === 'string') out[k] = subFormulaTokens(out[k], refTokens); + for (const k of SINGLE_ID_REF_KEYS) if (out[k] && flds[out[k]]) out[k] = flds[out[k]]; + if (out.foreignTableId && tbls[out.foreignTableId]) out.foreignTableId = tbls[out.foreignTableId]; + const dep = out.dependencies && out.dependencies.referencedColumnIdsForValue; + if (Array.isArray(dep)) out.dependencies.referencedColumnIdsForValue = dep.map((id) => flds[id] ?? id); + + return out; +} + +// ── Writable computed payload (strip read-only keys; emit input-form formulaText) ── +const COMPUTED_FORMAT_KEYS = [ + 'format', 'precision', 'symbol', 'negative', 'percentV2', 'separatorFormat', 'shouldShowThousandsSeparator', + 'dateFormat', 'timeFormat', 'timeZone', 'isDateTime', 'shouldDisplayTimeZone', 'displayType', +]; +function formulaExpr(opts) { return opts.formulaText ?? opts.formulaTextParsed ?? opts.formula ?? ''; } + +export function toWritableComputedOptions(type, opts) { + const o = opts || {}; + const fmt = {}; + for (const k of COMPUTED_FORMAT_KEYS) if (k in o) fmt[k] = o[k]; + switch (type) { + case 'formula': + return { formulaText: formulaExpr(o), ...fmt }; + case 'rollup': + return { relationColumnId: o.relationColumnId, foreignTableRollupColumnId: o.foreignTableRollupColumnId, formulaText: formulaExpr(o), ...fmt }; + case 'lookup': + case 'multipleLookupValues': + return { relationColumnId: o.relationColumnId, foreignTableRollupColumnId: o.foreignTableRollupColumnId }; + case 'count': + return { recordLinkFieldId: o.recordLinkFieldId }; + default: + return o; + } +} diff --git a/packages/mcp-server/test/sync/test-remap-refs.test.js b/packages/mcp-server/test/sync/test-remap-refs.test.js new file mode 100644 index 0000000..0f9379a --- /dev/null +++ b/packages/mcp-server/test/sync/test-remap-refs.test.js @@ -0,0 +1,71 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { remapRefs, toWritableComputedOptions } from '../../src/sync/remap.js'; + +const idmap = { + tables: { tblSRC: 'tblDST' }, + fields: { + fldA: { destFld: 'fldX', choices: {} }, + fldB: { destFld: 'fldY', choices: {} }, + }, +}; + +describe('remap.remapRefs', () => { + it('converts {column_value_fldX} + bare tokens to BARE dest ids (input form)', () => { + const out = remapRefs({ formulaTextParsed: '{column_value_fldA}+{column_value_fldB}', formulaText: '{fldA}+fldB' }, idmap); + assert.equal(out.formulaTextParsed, '{fldX}+{fldY}'); + assert.equal(out.formulaText, '{fldX}+fldB'); + }); + it('rewrites single-id ref keys and foreignTableId', () => { + const out = remapRefs({ relationColumnId: 'fldA', foreignTableRollupColumnId: 'fldB', foreignTableId: 'tblSRC' }, idmap); + assert.equal(out.relationColumnId, 'fldX'); + assert.equal(out.foreignTableRollupColumnId, 'fldY'); + assert.equal(out.foreignTableId, 'tblDST'); + }); + it('rewrites the dependencies array and leaves unknown ids intact', () => { + const out = remapRefs({ dependencies: { referencedColumnIdsForValue: ['fldA', 'fldUNKNOWN'] } }, idmap); + assert.deepEqual(out.dependencies.referencedColumnIdsForValue, ['fldX', 'fldUNKNOWN']); + }); + it('deep-clones (does not mutate input)', () => { + const input = { relationColumnId: 'fldA' }; + const out = remapRefs(input, idmap); + assert.equal(input.relationColumnId, 'fldA'); + assert.notEqual(out, input); + }); + it('passes null/undefined through', () => { + assert.equal(remapRefs(null, idmap), null); + assert.equal(remapRefs(undefined, idmap), undefined); + }); +}); + +describe('remap.toWritableComputedOptions', () => { + it('formula: emits formulaText, drops read-only keys', () => { + const remapped = remapRefs({ formulaTextParsed: '{column_value_fldA}*2', dependencies: { referencedColumnIdsForValue: ['fldA'] }, resultType: 'number', resultIsArray: false }, idmap); + const w = toWritableComputedOptions('formula', remapped); + assert.equal(w.formulaText, '{fldX}*2'); + assert.equal(w.formulaTextParsed, undefined); + assert.equal(w.dependencies, undefined); + assert.equal(w.resultType, undefined); + assert.equal(w.resultIsArray, undefined); + }); + it('formula: preserves whitelisted format keys (date-typed formula)', () => { + const w = toWritableComputedOptions('formula', { formulaTextParsed: 'CREATED_TIME()', isDateTime: true, dateFormat: 'Local', timeZone: 'client', resultType: 'date' }); + assert.equal(w.formulaText, 'CREATED_TIME()'); + assert.equal(w.isDateTime, true); + assert.equal(w.dateFormat, 'Local'); + }); + it('rollup: emits relation + target + formulaText', () => { + const remapped = remapRefs({ relationColumnId: 'fldA', foreignTableRollupColumnId: 'fldB', formulaTextParsed: 'SUM(values)', resultType: 'number' }, idmap); + const w = toWritableComputedOptions('rollup', remapped); + assert.equal(w.relationColumnId, 'fldX'); + assert.equal(w.foreignTableRollupColumnId, 'fldY'); + assert.equal(w.formulaText, 'SUM(values)'); + assert.equal(w.resultType, undefined); + }); + it('count: emits recordLinkFieldId only', () => { + const remapped = remapRefs({ recordLinkFieldId: 'fldA', resultType: 'number' }, idmap); + const w = toWritableComputedOptions('count', remapped); + assert.equal(w.recordLinkFieldId, 'fldX'); + assert.equal(w.resultType, undefined); + }); +}); From dc141ef8af5ebbc70b93baece71bc80ae103467e Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 18:56:34 +0300 Subject: [PATCH 024/246] test(sync): lock brace-aware remapRefs (string-literal safety) Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/test/sync/test-remap-refs.test.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/mcp-server/test/sync/test-remap-refs.test.js b/packages/mcp-server/test/sync/test-remap-refs.test.js index 0f9379a..613ff09 100644 --- a/packages/mcp-server/test/sync/test-remap-refs.test.js +++ b/packages/mcp-server/test/sync/test-remap-refs.test.js @@ -16,6 +16,11 @@ describe('remap.remapRefs', () => { assert.equal(out.formulaTextParsed, '{fldX}+{fldY}'); assert.equal(out.formulaText, '{fldX}+fldB'); }); + it('is brace-aware: a mapped id inside a string literal is NOT remapped (no corruption)', () => { + // fldB maps to fldY, but here it appears in a string literal, not a {ref}. + const out = remapRefs({ formulaText: 'IF({column_value_fldA}, "fldB is a label", "")' }, idmap); + assert.equal(out.formulaText, 'IF({fldX}, "fldB is a label", "")'); + }); it('rewrites single-id ref keys and foreignTableId', () => { const out = remapRefs({ relationColumnId: 'fldA', foreignTableRollupColumnId: 'fldB', foreignTableId: 'tblSRC' }, idmap); assert.equal(out.relationColumnId, 'fldX'); From dad6371853a3b080b48827d1320fc82f1cac5da3 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 18:58:57 +0300 Subject: [PATCH 025/246] feat(sync): add resumable apply journal + loadPlan Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/idmap.js | 14 ++++++++ packages/mcp-server/src/sync/journal.js | 33 +++++++++++++++++++ .../mcp-server/test/sync/test-journal.test.js | 28 ++++++++++++++++ 3 files changed, 75 insertions(+) create mode 100644 packages/mcp-server/src/sync/journal.js create mode 100644 packages/mcp-server/test/sync/test-journal.test.js diff --git a/packages/mcp-server/src/sync/idmap.js b/packages/mcp-server/src/sync/idmap.js index dbfbe51..0dfa27e 100644 --- a/packages/mcp-server/src/sync/idmap.js +++ b/packages/mcp-server/src/sync/idmap.js @@ -137,3 +137,17 @@ export function savePlan(sourceBaseId, destBaseId, plan) { export function saveState(sourceBaseId, destBaseId, state) { writeJson(syncDir(sourceBaseId, destBaseId), 'state.json', state); } + +/** + * Load a previously saved sync plan. Returns `null` when the file is absent + * or unparseable. + * @param {string} sourceBaseId + * @param {string} destBaseId + * @param {string} planId + * @returns {object|null} + */ +export function loadPlan(sourceBaseId, destBaseId, planId) { + const p = join(syncDir(sourceBaseId, destBaseId), `plan-${planId}.json`); + if (!existsSync(p)) return null; + try { return JSON.parse(readFileSync(p, 'utf8')); } catch { return null; } +} diff --git a/packages/mcp-server/src/sync/journal.js b/packages/mcp-server/src/sync/journal.js new file mode 100644 index 0000000..eed3754 --- /dev/null +++ b/packages/mcp-server/src/sync/journal.js @@ -0,0 +1,33 @@ +import { join } from 'node:path'; +import { existsSync, readFileSync, mkdirSync } from 'node:fs'; +import { syncDir } from './idmap.js'; +import { safeAtomicWriteFileSync } from '../safe-write.js'; + +function journalPath(sourceBaseId, destBaseId, planId) { + return join(syncDir(sourceBaseId, destBaseId), `journal-${planId}.json`); +} +export function newJournal(planId, startedAt) { + return { planId, startedAt, actions: [] }; +} +export function isDone(journal, idx) { + return journal.actions.some((a) => a.idx === idx && a.status === 'done'); +} +function upsert(journal, entry) { + const i = journal.actions.findIndex((a) => a.idx === entry.idx); + if (i >= 0) journal.actions[i] = entry; else journal.actions.push(entry); +} +export function recordDone(journal, idx, kind, destId) { + upsert(journal, { idx, kind, status: 'done', destId }); +} +export function recordFailed(journal, idx, kind, error) { + upsert(journal, { idx, kind, status: 'failed', error }); +} +export function saveJournal(sourceBaseId, destBaseId, journal) { + mkdirSync(syncDir(sourceBaseId, destBaseId), { recursive: true }); + safeAtomicWriteFileSync(journalPath(sourceBaseId, destBaseId, journal.planId), JSON.stringify(journal, null, 2)); +} +export function loadJournal(sourceBaseId, destBaseId, planId) { + const p = journalPath(sourceBaseId, destBaseId, planId); + if (!existsSync(p)) return null; + try { return JSON.parse(readFileSync(p, 'utf8')); } catch { return null; } +} diff --git a/packages/mcp-server/test/sync/test-journal.test.js b/packages/mcp-server/test/sync/test-journal.test.js new file mode 100644 index 0000000..a54cc79 --- /dev/null +++ b/packages/mcp-server/test/sync/test-journal.test.js @@ -0,0 +1,28 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { join } from 'node:path'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { newJournal, isDone, recordDone, recordFailed, saveJournal, loadJournal } from '../../src/sync/journal.js'; + +describe('journal', () => { + it('records done/failed and reports done by idx', () => { + const j = newJournal('plnX', '2026-06-16T00:00:00Z'); + assert.equal(isDone(j, 0), false); + recordDone(j, 0, 'createTable', 'tblD'); + recordFailed(j, 1, 'createField', 'boom'); + assert.equal(isDone(j, 0), true); + assert.equal(isDone(j, 1), false); // failed is retryable, not done + assert.equal(j.actions.find((a) => a.idx === 0).destId, 'tblD'); + assert.equal(j.actions.find((a) => a.idx === 1).error, 'boom'); + }); + it('round-trips through the sync dir', () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'jrnl-')); + const j = newJournal('plnY', 'ts'); + recordDone(j, 0, 'createTable', 'tblD'); + saveJournal('appAAAAAAAAAAAAAA', 'appBBBBBBBBBBBBBB', j); + const back = loadJournal('appAAAAAAAAAAAAAA', 'appBBBBBBBBBBBBBB', 'plnY'); + assert.equal(back.actions[0].destId, 'tblD'); + assert.equal(loadJournal('appAAAAAAAAAAAAAA', 'appBBBBBBBBBBBBBB', 'nope'), null); + }); +}); From e4b9388f3e6be725144b4d3c961054fabbf1680f Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 19:01:32 +0300 Subject: [PATCH 026/246] test(sync): add deterministic stateful mock client for apply tests Co-Authored-By: Claude Opus 4.8 (1M context) --- .../test/sync/helpers/mock-client.js | 64 +++++++++++++++++++ .../test/sync/test-mock-client.test.js | 22 +++++++ 2 files changed, 86 insertions(+) create mode 100644 packages/mcp-server/test/sync/helpers/mock-client.js create mode 100644 packages/mcp-server/test/sync/test-mock-client.test.js diff --git a/packages/mcp-server/test/sync/helpers/mock-client.js b/packages/mcp-server/test/sync/helpers/mock-client.js new file mode 100644 index 0000000..0581ad8 --- /dev/null +++ b/packages/mcp-server/test/sync/helpers/mock-client.js @@ -0,0 +1,64 @@ +// Deterministic in-memory fake of the AirtableClient schema mutators for apply tests. +// createTable mirrors the live internal API: a new table is born with these 6 fields +// (verified 2026-06-16); the first (Name/text) is the primary, the rest are scaffolding. +const DEFAULT_FIELDS = [ + { name: 'Name', type: 'text' }, // primary + { name: 'Notes', type: 'multilineText' }, + { name: 'Assignee', type: 'collaborator', typeOptions: { shouldNotify: true } }, + { name: 'Status', type: 'select', typeOptions: { choices: {} } }, + { name: 'Attachments', type: 'multipleAttachment', typeOptions: { unreversed: true } }, + { name: 'Attachment Summary', type: 'asyncText', typeOptions: { sourceType: 'ai' } }, +]; + +export class MockClient { + constructor(opts = {}) { + this.tables = []; // [{ id, name, primaryColumnId, columns:[{id,name,type,typeOptions,description}] }] + this.calls = []; // call log: 'kind:arg' + this._seq = 0; + this.formulaValid = opts.formulaValid ?? true; + } + _id(prefix) { this._seq += 1; return prefix + String(this._seq).padStart(6, '0'); } + async getApplicationData() { return { data: { tableSchemas: JSON.parse(JSON.stringify(this.tables)) } }; } + async createTable(appId, name) { + const tableId = this._id('tbl'); + const columns = DEFAULT_FIELDS.map((f) => ({ id: this._id('fld'), name: f.name, type: f.type, typeOptions: f.typeOptions ?? null, description: null })); + this.tables.push({ id: tableId, name, primaryColumnId: columns[0].id, columns }); + this.calls.push(`createTable:${name}`); + return { tableId }; + } + _table(id) { const t = this.tables.find((x) => x.id === id); if (!t) throw new Error('no table ' + id); return t; } + _field(colId) { for (const t of this.tables) { const f = t.columns.find((c) => c.id === colId); if (f) return f; } throw new Error('no field ' + colId); } + async deleteField(appId, fieldId, expectedName) { + for (const t of this.tables) { + const i = t.columns.findIndex((c) => c.id === fieldId); + if (i >= 0) { + if (expectedName != null && t.columns[i].name !== expectedName) throw new Error(`deleteField name mismatch: ${t.columns[i].name} !== ${expectedName}`); + t.columns.splice(i, 1); + this.calls.push(`deleteField:${fieldId}`); + return { deleted: true }; + } + } + throw new Error('no field ' + fieldId); + } + async createField(appId, tableId, cfg) { + const columnId = this._id('fld'); + this._table(tableId).columns.push({ id: columnId, name: cfg.name, type: cfg.type, typeOptions: cfg.typeOptions ?? null, description: cfg.description ?? null }); + this.calls.push(`createField:${cfg.name}:${cfg.type}`); + return { columnId }; + } + async updateFieldConfig(appId, columnId, cfg) { + const f = this._field(columnId); f.type = cfg.type; if (cfg.typeOptions !== undefined) f.typeOptions = cfg.typeOptions; + this.calls.push(`updateFieldConfig:${columnId}:${cfg.type}`); + return { ok: true }; + } + async renameField(appId, columnId, name) { + this._field(columnId).name = name; this.calls.push(`renameField:${columnId}:${name}`); return { ok: true }; + } + async updateFieldDescription(appId, columnId, description) { + this._field(columnId).description = description; this.calls.push(`updateFieldDescription:${columnId}`); return { ok: true }; + } + async validateFormula(appId, tableId, formulaText) { + this.calls.push(`validateFormula:${formulaText}`); + return { valid: this.formulaValid, resultType: 'text' }; + } +} diff --git a/packages/mcp-server/test/sync/test-mock-client.test.js b/packages/mcp-server/test/sync/test-mock-client.test.js new file mode 100644 index 0000000..c578c82 --- /dev/null +++ b/packages/mcp-server/test/sync/test-mock-client.test.js @@ -0,0 +1,22 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { MockClient } from './helpers/mock-client.js'; + +describe('MockClient', () => { + it('createTable spawns the 6 default fields (Name primary first); createField appends; deleteField removes', async () => { + const c = new MockClient(); + const { tableId } = await c.createTable('appD', 'Offers'); + const raw = await c.getApplicationData('appD'); + const t = raw.data.tableSchemas.find((x) => x.id === tableId); + assert.equal(t.columns.length, 6); + assert.equal(t.columns[0].id, t.primaryColumnId); + assert.equal(t.columns[0].name, 'Name'); + assert.equal(t.columns[0].type, 'text'); + const { columnId } = await c.createField('appD', tableId, { name: 'Score', type: 'number' }); + assert.equal((await c.getApplicationData('appD')).data.tableSchemas[0].columns.length, 7); + const notes = t.columns.find((x) => x.name === 'Notes'); + await c.deleteField('appD', notes.id, 'Notes'); + assert.equal((await c.getApplicationData('appD')).data.tableSchemas[0].columns.some((x) => x.name === 'Notes'), false); + assert.ok(c.calls.some((k) => k.startsWith('createField'))); + }); +}); From 17686d05df3f7cd8e05848ded92ee245b2477987 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 19:08:00 +0300 Subject: [PATCH 027/246] feat(sync): apply skeleton -- createTable (scaffolding-delete) + primary reconcile Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/apply.js | 116 ++++++++++++++++++ .../mcp-server/test/sync/test-apply.test.js | 75 +++++++++++ 2 files changed, 191 insertions(+) create mode 100644 packages/mcp-server/src/sync/apply.js create mode 100644 packages/mcp-server/test/sync/test-apply.test.js diff --git a/packages/mcp-server/src/sync/apply.js b/packages/mcp-server/src/sync/apply.js new file mode 100644 index 0000000..4774e97 --- /dev/null +++ b/packages/mcp-server/src/sync/apply.js @@ -0,0 +1,116 @@ +// Schema apply engine: execute an M2a Plan against a live destination base. +// Live-first + growing idmap + journal (resume) + existence-check (idempotency). +// POST-SPIKE: primary defaults to Name/text; createTable spawns 6 default fields → +// delete the 5 non-primary scaffolding fields (D1); links are foreignKey (later task). +import { remapRefs } from './remap.js'; +import { isDone, recordDone, recordFailed } from './journal.js'; + +// Types Airtable refuses as a primary field — keep a placeholder + warn instead of +// retyping. The try/catch around the retype is the ultimate guard (set is an optimization). +const ILLEGAL_PRIMARY_TYPES = new Set([ + 'select', 'singleSelect', 'multiSelect', 'multipleSelects', + 'multipleRecordLinks', 'foreignKey', 'multipleAttachment', 'multipleAttachments', + 'collaborator', 'button', 'rollup', 'lookup', 'multipleLookupValues', 'count', + 'asyncText', 'aiText', 'checkbox', +]); + +function buildIndex(snap) { + const tablesById = new Map(); + const tablesByName = new Map(); + for (const t of snap.tables) { + const entry = { + id: t.id, name: t.name, primaryFieldId: t.primaryFieldId, + fieldsByName: new Map(t.fields.map((f) => [f.name, { id: f.id, name: f.name, type: f.type, typeOptions: f.typeOptions }])), + }; + tablesById.set(t.id, entry); + tablesByName.set(t.name, entry); + } + return { tablesById, tablesByName }; +} + +async function readTable(client, appId, tableId) { + const raw = await client.getApplicationData(appId); + const tables = raw?.data?.tableSchemas ?? raw?.data?.tables ?? []; + const t = tables.find((x) => x.id === tableId); + if (!t) throw new Error(`table ${tableId} not found after create`); + const cols = t.columns ?? t.fields ?? []; + const primaryId = t.primaryColumnId ?? t.primaryFieldId ?? cols[0]?.id; + return { primaryId, primary: cols.find((c) => c.id === primaryId), cols }; +} + +export async function applyPlan({ client, plan, destAppId, destSnapshot, idmap, journal, persist }) { + const index = buildIndex(destSnapshot); + const state = { createdLinks: new Map(), adoptedReverse: new Set() }; + const result = { planId: plan.planId, aborted: false, created: 0, updated: 0, skipped: 0, failed: 0, warnings: [], idmap }; + + for (let idx = 0; idx < plan.actions.length; idx++) { + const a = plan.actions[idx]; + if (isDone(journal, idx)) { result.skipped++; continue; } + try { + await applyAction({ client, destAppId, a, idmap, index, state, result }); + recordDone(journal, idx, a.kind, idmap.tables[a.sourceTableId] ?? (a.sourceFieldId && idmap.fields[a.sourceFieldId]?.destFld)); + persist(idmap, journal); + } catch (e) { + recordFailed(journal, idx, a.kind, String(e && e.message ? e.message : e)); + result.failed++; + result.warnings.push({ code: 'ACTION_FAILED', message: `${a.kind} "${a.name ?? a.sourceFieldId ?? a.sourceTableId}": ${e.message ?? e}` }); + persist(idmap, journal); + break; // halt forward progress; resume re-runs from here + } + } + return result; +} + +async function applyAction({ client, destAppId, a, idmap, index, state, result }) { + switch (a.kind) { + case 'createTable': { + const existing = index.tablesByName.get(a.name); + if (existing) { idmap.tables[a.sourceTableId] = existing.id; result.skipped++; return; } + const { tableId } = await client.createTable(destAppId, a.name); + idmap.tables[a.sourceTableId] = tableId; + // D1: delete the auto-created non-primary scaffolding fields for a clean mirror. + const { primaryId, primary, cols } = await readTable(client, destAppId, tableId); + for (const c of cols) { + if (c.id === primaryId) continue; + await client.deleteField(destAppId, c.id, c.name); + } + const entry = { + id: tableId, name: a.name, primaryFieldId: primaryId, + fieldsByName: new Map([[primary.name, { id: primaryId, name: primary.name, type: primary.type, typeOptions: primary.typeOptions ?? null }]]), + }; + index.tablesById.set(tableId, entry); + index.tablesByName.set(a.name, entry); + result.created++; + return; + } + + case 'reconcilePrimary': { + const destTableId = idmap.tables[a.sourceTableId]; + const entry = index.tablesById.get(destTableId); + if (!entry || !entry.primaryFieldId) throw new Error(`reconcilePrimary: no primary for ${destTableId}`); + const primaryId = entry.primaryFieldId; + let primary = null; + for (const f of entry.fieldsByName.values()) if (f.id === primaryId) primary = f; + let renamed = false, retyped = false; + if (primary && primary.name !== a.toName) { await client.renameField(destAppId, primaryId, a.toName); renamed = true; } + if (a.toType && primary && primary.type !== a.toType) { + if (ILLEGAL_PRIMARY_TYPES.has(a.toType)) { + result.warnings.push({ code: 'PRIMARY_TYPE_INCOMPATIBLE', message: `Primary "${a.toName}" wants ${a.toType}; kept ${primary.type} placeholder` }); + } else { + try { await client.updateFieldConfig(destAppId, primaryId, { type: a.toType, typeOptions: remapRefs(a.toTypeOptions, idmap) }); retyped = true; } + catch (e) { result.warnings.push({ code: 'PRIMARY_TYPE_INCOMPATIBLE', message: `Primary "${a.toName}" retype to ${a.toType} rejected: ${e.message ?? e}` }); } + } + } + idmap.fields[a.sourcePrimaryFieldId] = { destFld: primaryId, choices: {} }; + if (primary) { + if (renamed) { entry.fieldsByName.delete(primary.name); primary.name = a.toName; entry.fieldsByName.set(a.toName, primary); } + if (retyped) primary.type = a.toType; + } + if (renamed || retyped) result.updated++; else result.skipped++; + return; + } + + default: + throw new Error(`unhandled action kind: ${a.kind}`); + } +} diff --git a/packages/mcp-server/test/sync/test-apply.test.js b/packages/mcp-server/test/sync/test-apply.test.js new file mode 100644 index 0000000..7cfae55 --- /dev/null +++ b/packages/mcp-server/test/sync/test-apply.test.js @@ -0,0 +1,75 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { MockClient } from './helpers/mock-client.js'; +import { applyPlan } from '../../src/sync/apply.js'; +import { newJournal } from '../../src/sync/journal.js'; +import { snapshotBase } from '../../src/sync/snapshot.js'; + +function run(client, plan, destSnapshot) { + return applyPlan({ + client, plan, destAppId: plan.destBaseId, destSnapshot, + idmap: JSON.parse(JSON.stringify(plan.idmap)), + journal: newJournal(plan.planId, 'ts'), + persist: () => {}, + }); +} + +describe('apply: createTable + scaffolding-delete + reconcilePrimary', () => { + it('creates a table, deletes the 5 non-primary defaults, renames+retypes the primary, maps it', async () => { + const client = new MockClient(); + const destSnapshot = await snapshotBase(client, 'appD'); // empty base + const plan = { + planId: 'pln1', sourceBaseId: 'appS', destBaseId: 'appD', + idmap: { tables: {}, fields: {} }, + actions: [ + { kind: 'createTable', sourceTableId: 'tS', name: 'Offers' }, + { kind: 'reconcilePrimary', sourceTableId: 'tS', sourcePrimaryFieldId: 'fS1', toName: 'Title', toType: 'number', toTypeOptions: { format: 'integer' } }, + ], + orphans: [], warnings: [], + }; + const res = await run(client, plan, destSnapshot); + assert.equal(res.failed, 0); + const t = (await client.getApplicationData('appD')).data.tableSchemas[0]; + assert.equal(t.name, 'Offers'); + assert.equal(t.columns.length, 1, 'scaffolding deleted, only primary remains'); + assert.equal(t.columns[0].name, 'Title'); + assert.equal(t.columns[0].type, 'number'); + assert.ok(res.idmap.tables.tS); + assert.equal(res.idmap.fields.fS1.destFld, t.columns[0].id); + }); + + it('warns PRIMARY_TYPE_INCOMPATIBLE on rejected retype, keeps placeholder (still renamed)', async () => { + const client = new MockClient(); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { + planId: 'pln2', sourceBaseId: 'appS', destBaseId: 'appD', + idmap: { tables: {}, fields: {} }, + actions: [ + { kind: 'createTable', sourceTableId: 'tS', name: 'T' }, + { kind: 'reconcilePrimary', sourceTableId: 'tS', sourcePrimaryFieldId: 'fS1', toName: 'Key', toType: 'singleSelect', toTypeOptions: { choices: {} } }, + ], + orphans: [], warnings: [], + }; + const res = await run(client, plan, destSnapshot); + assert.equal(res.failed, 0); + assert.ok(res.warnings.some((w) => w.code === 'PRIMARY_TYPE_INCOMPATIBLE')); + const t = (await client.getApplicationData('appD')).data.tableSchemas[0]; + assert.equal(t.columns[0].name, 'Key'); + }); + + it('adopts an existing table by name instead of recreating', async () => { + const client = new MockClient(); + await client.createTable('appD', 'Offers'); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { + planId: 'pln3', sourceBaseId: 'appS', destBaseId: 'appD', + idmap: { tables: {}, fields: {} }, + actions: [{ kind: 'createTable', sourceTableId: 'tS', name: 'Offers' }], + orphans: [], warnings: [], + }; + const res = await run(client, plan, destSnapshot); + assert.equal(res.created, 0); + assert.equal(res.skipped, 1); + assert.equal((await client.getApplicationData('appD')).data.tableSchemas.length, 1); + }); +}); From 8e69e3f7855ae373f094c575cb0d5cdc9db7b724 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 19:10:49 +0300 Subject: [PATCH 028/246] feat(sync): apply scalar createField with existence-skip idempotency Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/apply.js | 17 +++++++++++++ .../mcp-server/test/sync/test-apply.test.js | 25 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/packages/mcp-server/src/sync/apply.js b/packages/mcp-server/src/sync/apply.js index 4774e97..5c53940 100644 --- a/packages/mcp-server/src/sync/apply.js +++ b/packages/mcp-server/src/sync/apply.js @@ -110,6 +110,23 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result } return; } + case 'createField': { + const destTableId = idmap.tables[a.sourceTableId]; + const entry = index.tablesById.get(destTableId); + const existing = entry && entry.fieldsByName.get(a.name); + if (existing) { + idmap.fields[a.sourceFieldId] = { destFld: existing.id, choices: {} }; + result.skipped++; + return; + } + const typeOptions = remapRefs(a.typeOptions, idmap); + const { columnId } = await client.createField(destAppId, destTableId, { name: a.name, type: a.type, typeOptions, description: a.description ?? undefined }); + idmap.fields[a.sourceFieldId] = { destFld: columnId, choices: {} }; + if (entry) entry.fieldsByName.set(a.name, { id: columnId, name: a.name, type: a.type, typeOptions }); + result.created++; + return; + } + default: throw new Error(`unhandled action kind: ${a.kind}`); } diff --git a/packages/mcp-server/test/sync/test-apply.test.js b/packages/mcp-server/test/sync/test-apply.test.js index 7cfae55..55d9ae2 100644 --- a/packages/mcp-server/test/sync/test-apply.test.js +++ b/packages/mcp-server/test/sync/test-apply.test.js @@ -73,3 +73,28 @@ describe('apply: createTable + scaffolding-delete + reconcilePrimary', () => { assert.equal((await client.getApplicationData('appD')).data.tableSchemas.length, 1); }); }); + +describe('apply: createField (scalar)', () => { + it('creates a scalar field and maps it; skips one that already exists by name', async () => { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'Offers'); + await client.createField('appD', tableId, { name: 'Existing', type: 'text' }); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { + planId: 'plnF', sourceBaseId: 'appS', destBaseId: 'appD', + idmap: { tables: { tS: tableId }, fields: {} }, + actions: [ + { kind: 'createField', sourceTableId: 'tS', sourceFieldId: 'fNew', name: 'Score', type: 'number', typeOptions: { precision: 0 }, description: 'pts', computed: false, dependsOn: [], dependsOnTables: [] }, + { kind: 'createField', sourceTableId: 'tS', sourceFieldId: 'fExist', name: 'Existing', type: 'text', typeOptions: null, description: null, computed: false, dependsOn: [], dependsOnTables: [] }, + ], + orphans: [], warnings: [], + }; + const res = await applyPlan({ client, plan, destAppId: 'appD', destSnapshot, idmap: JSON.parse(JSON.stringify(plan.idmap)), journal: newJournal('plnF', 'ts'), persist: () => {} }); + assert.equal(res.created, 1); // Score + assert.equal(res.skipped, 1); // Existing adopted + assert.equal(res.idmap.fields.fNew.destFld.startsWith('fld'), true); + assert.ok(res.idmap.fields.fExist.destFld); + const score = (await client.getApplicationData('appD')).data.tableSchemas[0].columns.find((c) => c.name === 'Score'); + assert.equal(score.description, 'pts'); + }); +}); From b64fec88533ab81dcbd760901e48dc0ebdefc219 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 19:14:30 +0300 Subject: [PATCH 029/246] feat(sync): apply computed createField -- writable shape, validate, skip unsupported Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/apply.js | 21 ++++++- .../mcp-server/test/sync/test-apply.test.js | 60 +++++++++++++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/packages/mcp-server/src/sync/apply.js b/packages/mcp-server/src/sync/apply.js index 5c53940..f6bf56c 100644 --- a/packages/mcp-server/src/sync/apply.js +++ b/packages/mcp-server/src/sync/apply.js @@ -2,9 +2,12 @@ // Live-first + growing idmap + journal (resume) + existence-check (idempotency). // POST-SPIKE: primary defaults to Name/text; createTable spawns 6 default fields → // delete the 5 non-primary scaffolding fields (D1); links are foreignKey (later task). -import { remapRefs } from './remap.js'; +import { remapRefs, toWritableComputedOptions } from './remap.js'; import { isDone, recordDone, recordFailed } from './journal.js'; +const UNSUPPORTED_TYPES = new Set(['button', 'asyncText', 'aiText', 'externalSyncSource']); +const COMPUTED_TYPES = new Set(['formula', 'rollup', 'lookup', 'multipleLookupValues', 'count']); + // Types Airtable refuses as a primary field — keep a placeholder + warn instead of // retyping. The try/catch around the retype is the ultimate guard (set is an optimization). const ILLEGAL_PRIMARY_TYPES = new Set([ @@ -113,13 +116,27 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result } case 'createField': { const destTableId = idmap.tables[a.sourceTableId]; const entry = index.tablesById.get(destTableId); + // D2: skip types we can't faithfully recreate cross-base (do NOT map them). + if (UNSUPPORTED_TYPES.has(a.type)) { + result.warnings.push({ code: 'SKIPPED_UNSUPPORTED', message: `Field "${a.name}" (${a.type}) skipped — unsupported by the apply engine` }); + result.skipped++; + return; + } const existing = entry && entry.fieldsByName.get(a.name); if (existing) { idmap.fields[a.sourceFieldId] = { destFld: existing.id, choices: {} }; result.skipped++; return; } - const typeOptions = remapRefs(a.typeOptions, idmap); + const remapped = remapRefs(a.typeOptions, idmap); + let typeOptions = remapped; + if (COMPUTED_TYPES.has(a.type)) { + typeOptions = toWritableComputedOptions(a.type, remapped); + if (a.type === 'formula') { + const v = await client.validateFormula(destAppId, destTableId, typeOptions.formulaText ?? ''); + if (!v.valid) throw new Error(`formula invalid: ${v.message ?? v.error ?? 'rejected'}`); + } + } const { columnId } = await client.createField(destAppId, destTableId, { name: a.name, type: a.type, typeOptions, description: a.description ?? undefined }); idmap.fields[a.sourceFieldId] = { destFld: columnId, choices: {} }; if (entry) entry.fieldsByName.set(a.name, { id: columnId, name: a.name, type: a.type, typeOptions }); diff --git a/packages/mcp-server/test/sync/test-apply.test.js b/packages/mcp-server/test/sync/test-apply.test.js index 55d9ae2..4cd3a6c 100644 --- a/packages/mcp-server/test/sync/test-apply.test.js +++ b/packages/mcp-server/test/sync/test-apply.test.js @@ -98,3 +98,63 @@ describe('apply: createField (scalar)', () => { assert.equal(score.description, 'pts'); }); }); + +describe('apply: createField (computed + unsupported)', () => { + it('remaps a formula to dest ids, validates, creates it, strips read-only keys', async () => { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'T'); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { + planId: 'plnC', sourceBaseId: 'appS', destBaseId: 'appD', + idmap: { tables: { tS: tableId }, fields: {} }, + actions: [ + { kind: 'createField', sourceTableId: 'tS', sourceFieldId: 'fldPrice', name: 'Price', type: 'number', typeOptions: { precision: 2 }, description: null, computed: false, dependsOn: [], dependsOnTables: [] }, + { kind: 'createField', sourceTableId: 'tS', sourceFieldId: 'fldTotal', name: 'Total', type: 'formula', typeOptions: { formulaTextParsed: '{column_value_fldPrice}*2', dependencies: { referencedColumnIdsForValue: ['fldPrice'] }, resultType: 'number' }, description: null, computed: true, dependsOn: ['fldPrice'], dependsOnTables: [] }, + ], + orphans: [], warnings: [], + }; + const res = await applyPlan({ client, plan, destAppId: 'appD', destSnapshot, idmap: JSON.parse(JSON.stringify(plan.idmap)), journal: newJournal('plnC', 'ts'), persist: () => {} }); + assert.equal(res.created, 2); + assert.equal(res.failed, 0); + const destPriceId = res.idmap.fields.fldPrice.destFld; + const total = (await client.getApplicationData('appD')).data.tableSchemas[0].columns.find((c) => c.name === 'Total'); + assert.equal(total.typeOptions.formulaText, `{${destPriceId}}*2`); + assert.equal(total.typeOptions.formulaTextParsed, undefined); // read-only key stripped + assert.ok(client.calls.some((k) => k === `validateFormula:{${destPriceId}}*2`)); + }); + + it('fails the action when formula validation fails (no create)', async () => { + const client = new MockClient({ formulaValid: false }); + const { tableId } = await client.createTable('appD', 'T'); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { + planId: 'plnBad', sourceBaseId: 'appS', destBaseId: 'appD', + idmap: { tables: { tS: tableId }, fields: {} }, + actions: [{ kind: 'createField', sourceTableId: 'tS', sourceFieldId: 'fldX', name: 'Bad', type: 'formula', typeOptions: { formulaTextParsed: 'NONSENSE(' }, description: null, computed: true, dependsOn: [], dependsOnTables: [] }], + orphans: [], warnings: [], + }; + const res = await applyPlan({ client, plan, destAppId: 'appD', destSnapshot, idmap: JSON.parse(JSON.stringify(plan.idmap)), journal: newJournal('plnBad', 'ts'), persist: () => {} }); + assert.equal(res.created, 0); + assert.equal(res.failed, 1); + assert.ok(res.warnings.some((w) => w.code === 'ACTION_FAILED')); + }); + + it('skips an unsupported type (button) with SKIPPED_UNSUPPORTED and no create', async () => { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'T'); + const destSnapshot = await snapshotBase(client, 'appD'); + const before = (await client.getApplicationData('appD')).data.tableSchemas[0].columns.length; + const plan = { + planId: 'plnBtn', sourceBaseId: 'appS', destBaseId: 'appD', + idmap: { tables: { tS: tableId }, fields: {} }, + actions: [{ kind: 'createField', sourceTableId: 'tS', sourceFieldId: 'fldBtn', name: 'OpenIt', type: 'button', typeOptions: { label: {} }, description: null, computed: true, dependsOn: [], dependsOnTables: [] }], + orphans: [], warnings: [], + }; + const res = await applyPlan({ client, plan, destAppId: 'appD', destSnapshot, idmap: JSON.parse(JSON.stringify(plan.idmap)), journal: newJournal('plnBtn', 'ts'), persist: () => {} }); + assert.equal(res.created, 0); + assert.equal(res.skipped, 1); + assert.ok(res.warnings.some((w) => w.code === 'SKIPPED_UNSUPPORTED')); + assert.equal(res.idmap.fields.fldBtn, undefined); + assert.equal((await client.getApplicationData('appD')).data.tableSchemas[0].columns.length, before); + }); +}); From 1d0dd908773551c22fdd60ee5d1921f068ae08fa Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 19:18:35 +0300 Subject: [PATCH 030/246] feat(sync): apply foreignKey link createField with reciprocal-once adoption Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/apply.js | 40 ++++++++++++++++++- .../test/sync/helpers/mock-client.js | 10 ++++- .../mcp-server/test/sync/test-apply.test.js | 27 +++++++++++++ 3 files changed, 74 insertions(+), 3 deletions(-) diff --git a/packages/mcp-server/src/sync/apply.js b/packages/mcp-server/src/sync/apply.js index f6bf56c..46daff7 100644 --- a/packages/mcp-server/src/sync/apply.js +++ b/packages/mcp-server/src/sync/apply.js @@ -7,6 +7,7 @@ import { isDone, recordDone, recordFailed } from './journal.js'; const UNSUPPORTED_TYPES = new Set(['button', 'asyncText', 'aiText', 'externalSyncSource']); const COMPUTED_TYPES = new Set(['formula', 'rollup', 'lookup', 'multipleLookupValues', 'count']); +const LINK_TYPES = new Set(['foreignKey', 'multipleRecordLinks']); // Types Airtable refuses as a primary field — keep a placeholder + warn instead of // retyping. The try/catch around the retype is the ultimate guard (set is an optimization). @@ -41,6 +42,36 @@ async function readTable(client, appId, tableId) { return { primaryId, primary: cols.find((c) => c.id === primaryId), cols }; } +function rememberLink(state, forwardFieldId, destTableId) { + state.createdLinks.set(forwardFieldId, { destTableId }); +} + +async function adoptReverseLink({ client, destAppId, a, idmap, index, state, result }) { + const destTableId = idmap.tables[a.sourceTableId]; + const { cols } = await readTable(client, destAppId, destTableId); + for (const c of cols) { + if (!LINK_TYPES.has(c.type)) continue; + const sym = c.typeOptions && c.typeOptions.symmetricColumnId; + if (sym && state.createdLinks.has(sym) && !state.adoptedReverse.has(c.id)) { + if (c.name !== a.name) await client.renameField(destAppId, c.id, a.name); + idmap.fields[a.sourceFieldId] = { destFld: c.id, choices: {} }; + const entry = index.tablesById.get(destTableId); + if (entry) entry.fieldsByName.set(a.name, { id: c.id, name: a.name, type: c.type, typeOptions: c.typeOptions }); + state.adoptedReverse.add(c.id); + result.skipped++; + return true; + } + } + return false; +} + +function writableLinkOptions(remappedTypeOptions) { + const o = remappedTypeOptions || {}; + const out = { foreignTableId: o.foreignTableId }; + if (o.relationship) out.relationship = o.relationship; + return out; // drop symmetricColumnId/unreversed (auto-managed by Airtable) +} + export async function applyPlan({ client, plan, destAppId, destSnapshot, idmap, journal, persist }) { const index = buildIndex(destSnapshot); const state = { createdLinks: new Map(), adoptedReverse: new Set() }; @@ -116,7 +147,6 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result } case 'createField': { const destTableId = idmap.tables[a.sourceTableId]; const entry = index.tablesById.get(destTableId); - // D2: skip types we can't faithfully recreate cross-base (do NOT map them). if (UNSUPPORTED_TYPES.has(a.type)) { result.warnings.push({ code: 'SKIPPED_UNSUPPORTED', message: `Field "${a.name}" (${a.type}) skipped — unsupported by the apply engine` }); result.skipped++; @@ -128,9 +158,14 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result } result.skipped++; return; } + // Reciprocal-once: adopt the auto-created reverse of a link made earlier this run. + if (LINK_TYPES.has(a.type) && await adoptReverseLink({ client, destAppId, a, idmap, index, state, result })) return; + const remapped = remapRefs(a.typeOptions, idmap); let typeOptions = remapped; - if (COMPUTED_TYPES.has(a.type)) { + if (LINK_TYPES.has(a.type)) { + typeOptions = writableLinkOptions(remapped); + } else if (COMPUTED_TYPES.has(a.type)) { typeOptions = toWritableComputedOptions(a.type, remapped); if (a.type === 'formula') { const v = await client.validateFormula(destAppId, destTableId, typeOptions.formulaText ?? ''); @@ -140,6 +175,7 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result } const { columnId } = await client.createField(destAppId, destTableId, { name: a.name, type: a.type, typeOptions, description: a.description ?? undefined }); idmap.fields[a.sourceFieldId] = { destFld: columnId, choices: {} }; if (entry) entry.fieldsByName.set(a.name, { id: columnId, name: a.name, type: a.type, typeOptions }); + if (LINK_TYPES.has(a.type)) rememberLink(state, columnId, destTableId); result.created++; return; } diff --git a/packages/mcp-server/test/sync/helpers/mock-client.js b/packages/mcp-server/test/sync/helpers/mock-client.js index 0581ad8..842cfdd 100644 --- a/packages/mcp-server/test/sync/helpers/mock-client.js +++ b/packages/mcp-server/test/sync/helpers/mock-client.js @@ -42,8 +42,16 @@ export class MockClient { } async createField(appId, tableId, cfg) { const columnId = this._id('fld'); - this._table(tableId).columns.push({ id: columnId, name: cfg.name, type: cfg.type, typeOptions: cfg.typeOptions ?? null, description: cfg.description ?? null }); + const field = { id: columnId, name: cfg.name, type: cfg.type, typeOptions: cfg.typeOptions ?? null, description: cfg.description ?? null }; + this._table(tableId).columns.push(field); this.calls.push(`createField:${cfg.name}:${cfg.type}`); + if (cfg.type === 'foreignKey' && cfg.typeOptions && cfg.typeOptions.foreignTableId) { + const foreign = this._table(cfg.typeOptions.foreignTableId); + const reverseId = this._id('fld'); + foreign.columns.push({ id: reverseId, name: this._table(tableId).name, type: 'foreignKey', + typeOptions: { foreignTableId: tableId, symmetricColumnId: columnId, relationship: cfg.typeOptions.relationship === 'one' ? 'many' : 'one', unreversed: true }, description: null }); + field.typeOptions = { ...field.typeOptions, symmetricColumnId: reverseId }; + } return { columnId }; } async updateFieldConfig(appId, columnId, cfg) { diff --git a/packages/mcp-server/test/sync/test-apply.test.js b/packages/mcp-server/test/sync/test-apply.test.js index 4cd3a6c..1749e55 100644 --- a/packages/mcp-server/test/sync/test-apply.test.js +++ b/packages/mcp-server/test/sync/test-apply.test.js @@ -158,3 +158,30 @@ describe('apply: createField (computed + unsupported)', () => { assert.equal((await client.getApplicationData('appD')).data.tableSchemas[0].columns.length, before); }); }); + +describe('apply: createField (link foreignKey) reciprocal-once', () => { + it('creates the forward link and adopts the auto-created reverse instead of duplicating', async () => { + const client = new MockClient(); + const a = await client.createTable('appD', 'A'); + const b = await client.createTable('appD', 'B'); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { + planId: 'plnL', sourceBaseId: 'appS', destBaseId: 'appD', + idmap: { tables: { tA: a.tableId, tB: b.tableId }, fields: {} }, + actions: [ + { kind: 'createField', sourceTableId: 'tA', sourceFieldId: 'fldAlink', name: 'Bs', type: 'foreignKey', typeOptions: { foreignTableId: 'tB', relationship: 'many' }, description: null, computed: false, dependsOn: [], dependsOnTables: ['tB'] }, + { kind: 'createField', sourceTableId: 'tB', sourceFieldId: 'fldBlink', name: 'As', type: 'foreignKey', typeOptions: { foreignTableId: 'tA', relationship: 'many' }, description: null, computed: false, dependsOn: [], dependsOnTables: ['tA'] }, + ], + orphans: [], warnings: [], + }; + const res = await applyPlan({ client, plan, destAppId: 'appD', destSnapshot, idmap: JSON.parse(JSON.stringify(plan.idmap)), journal: newJournal('plnL', 'ts'), persist: () => {} }); + assert.equal(res.created, 1); // only the forward link issues a create + assert.equal(res.skipped, 1); // reverse adopted + const tables = (await client.getApplicationData('appD')).data.tableSchemas; + const tableB = tables.find((t) => t.name === 'B'); + const links = tableB.columns.filter((c) => c.type === 'foreignKey'); + assert.equal(links.length, 1, 'reverse link must not be duplicated'); + assert.equal(links[0].name, 'As'); // adopted reverse renamed to source name + assert.ok(res.idmap.fields.fldBlink.destFld); + }); +}); From 87fccdb51725f075e482a53b51b3a0a9cdd9a21c Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 19:22:10 +0300 Subject: [PATCH 031/246] feat(sync): apply updateField -- safe changes, select-merge, retype deferred Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/apply.js | 41 +++++++++++++++++++ .../mcp-server/test/sync/test-apply.test.js | 40 ++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/packages/mcp-server/src/sync/apply.js b/packages/mcp-server/src/sync/apply.js index 46daff7..7c19cca 100644 --- a/packages/mcp-server/src/sync/apply.js +++ b/packages/mcp-server/src/sync/apply.js @@ -72,6 +72,28 @@ function writableLinkOptions(remappedTypeOptions) { return out; // drop symmetricColumnId/unreversed (auto-managed by Airtable) } +function findDestField(index, destFieldId) { + for (const entry of index.tablesById.values()) { + for (const f of entry.fieldsByName.values()) if (f.id === destFieldId) return f; + } + return null; +} +function findDestFieldType(index, destFieldId) { + const f = findDestField(index, destFieldId); + return f ? f.type : undefined; +} +// Merge source choices into dest choices by NAME (never drop a dest choice). Dest choices +// keep their ids; new source choices are added without ids (Airtable assigns). +function mergeChoices(destField, srcTypeOptions) { + if (!srcTypeOptions || !srcTypeOptions.choices) return srcTypeOptions; + const destChoices = (destField && destField.typeOptions && destField.typeOptions.choices) || {}; + const byName = new Map(Object.values(destChoices).map((c) => [c.name, c])); + for (const c of Object.values(srcTypeOptions.choices)) if (!byName.has(c.name)) byName.set(c.name, { name: c.name, color: c.color }); + const merged = {}; + for (const c of byName.values()) { const id = c.id || c.name; merged[id] = c; } + return { ...srcTypeOptions, choices: merged }; +} + export async function applyPlan({ client, plan, destAppId, destSnapshot, idmap, journal, persist }) { const index = buildIndex(destSnapshot); const state = { createdLinks: new Map(), adoptedReverse: new Set() }; @@ -180,6 +202,25 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result } return; } + case 'updateField': { + const changes = a.changes || {}; + if (changes.type !== undefined) { + result.warnings.push({ code: 'RETYPE_DEFERRED', message: `Field ${a.destFld}: type change to "${changes.type}" deferred (M4)` }); + return; // never retype an existing field in M2b + } + let mutated = false; + if (changes.name !== undefined) { await client.renameField(destAppId, a.destFld, changes.name); mutated = true; } + if (changes.typeOptions !== undefined) { + const remapped = remapRefs(changes.typeOptions, idmap); + const merged = mergeChoices(findDestField(index, a.destFld), remapped); + await client.updateFieldConfig(destAppId, a.destFld, { type: findDestFieldType(index, a.destFld), typeOptions: merged }); + mutated = true; + } + if (changes.description !== undefined) { await client.updateFieldDescription(destAppId, a.destFld, changes.description); mutated = true; } + if (mutated) result.updated++; else result.skipped++; + return; + } + default: throw new Error(`unhandled action kind: ${a.kind}`); } diff --git a/packages/mcp-server/test/sync/test-apply.test.js b/packages/mcp-server/test/sync/test-apply.test.js index 1749e55..8ccae77 100644 --- a/packages/mcp-server/test/sync/test-apply.test.js +++ b/packages/mcp-server/test/sync/test-apply.test.js @@ -185,3 +185,43 @@ describe('apply: createField (link foreignKey) reciprocal-once', () => { assert.ok(res.idmap.fields.fldBlink.destFld); }); }); + +describe('apply: updateField', () => { + async function destWith(fieldType, typeOptions) { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'T'); + const { columnId } = await client.createField('appD', tableId, { name: 'F', type: fieldType, typeOptions }); + return { client, columnId, destSnapshot: await snapshotBase(client, 'appD') }; + } + function planUpdate(destFld, changes) { + return { planId: 'plnU', sourceBaseId: 'appS', destBaseId: 'appD', idmap: { tables: {}, fields: {} }, + actions: [{ kind: 'updateField', sourceFieldId: 'fS', destFld, changes }], orphans: [], warnings: [] }; + } + const run = (client, plan, destSnapshot) => applyPlan({ client, plan, destAppId: 'appD', destSnapshot, idmap: { tables: {}, fields: {} }, journal: newJournal(plan.planId, 'ts'), persist: () => {} }); + + it('applies a description change', async () => { + const { client, columnId, destSnapshot } = await destWith('text', null); + const res = await run(client, planUpdate(columnId, { description: 'new' }), destSnapshot); + assert.equal(res.updated, 1); + assert.equal(client._field(columnId).description, 'new'); + }); + + it('skips + reports a type change (RETYPE_DEFERRED), making no client mutation', async () => { + const { client, columnId, destSnapshot } = await destWith('text', null); + const before = client.calls.length; + const res = await run(client, planUpdate(columnId, { type: 'number', typeOptions: { precision: 0 } }), destSnapshot); + assert.equal(res.updated, 0); + assert.ok(res.warnings.some((w) => w.code === 'RETYPE_DEFERRED')); + assert.equal(client.calls.length, before); // no updateFieldConfig issued + }); + + it('merges select choices (keeps dest-only, adds source-only) without dropping any', async () => { + const destChoices = { selD1: { id: 'selD1', name: 'Open' }, selD2: { id: 'selD2', name: 'DestOnly' } }; + const { client, columnId, destSnapshot } = await destWith('select', { choices: destChoices }); + const srcChoices = { selS1: { id: 'selS1', name: 'Open' }, selS3: { id: 'selS3', name: 'Closed' } }; + const res = await run(client, planUpdate(columnId, { typeOptions: { choices: srcChoices } }), destSnapshot); + assert.equal(res.updated, 1); + const names = Object.values(client._field(columnId).typeOptions.choices).map((c) => c.name).sort(); + assert.deepEqual(names, ['Closed', 'DestOnly', 'Open']); + }); +}); From a91a538cbd956ff5a6f4e6a8c4581efae6234c5a Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 19:25:04 +0300 Subject: [PATCH 032/246] test(sync): prove apply journal-resume + run-twice idempotency Co-Authored-By: Claude Opus 4.8 (1M context) --- .../mcp-server/test/sync/test-apply.test.js | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/packages/mcp-server/test/sync/test-apply.test.js b/packages/mcp-server/test/sync/test-apply.test.js index 8ccae77..d7976fc 100644 --- a/packages/mcp-server/test/sync/test-apply.test.js +++ b/packages/mcp-server/test/sync/test-apply.test.js @@ -225,3 +225,41 @@ describe('apply: updateField', () => { assert.deepEqual(names, ['Closed', 'DestOnly', 'Open']); }); }); + +describe('apply: resume + idempotency', () => { + function fullPlan() { + return { + planId: 'plnR', sourceBaseId: 'appS', destBaseId: 'appD', + idmap: { tables: {}, fields: {} }, + actions: [ + { kind: 'createTable', sourceTableId: 'tS', name: 'T' }, + { kind: 'reconcilePrimary', sourceTableId: 'tS', sourcePrimaryFieldId: 'fS1', toName: 'Title', toType: 'text', toTypeOptions: null }, + { kind: 'createField', sourceTableId: 'tS', sourceFieldId: 'fS2', name: 'Score', type: 'number', typeOptions: { precision: 0 }, description: null, computed: false, dependsOn: [], dependsOnTables: [] }, + ], + orphans: [], warnings: [], + }; + } + + it('skips journal-done actions on resume (does not recreate the table)', async () => { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'T'); // simulate a prior run's createTable + const before = client.calls.length; + const journal = newJournal('plnR', 'ts'); + journal.actions.push({ idx: 0, kind: 'createTable', status: 'done', destId: tableId }); + const idmap = { tables: { tS: tableId }, fields: {} }; // grown idmap a prior run persisted + const destSnapshot = await snapshotBase(client, 'appD'); + const res = await applyPlan({ client, plan: fullPlan(), destAppId: 'appD', destSnapshot, idmap, journal, persist: () => {} }); + assert.equal(res.failed, 0); + assert.ok(res.skipped >= 1); + assert.ok(!client.calls.slice(before).some((k) => k === 'createTable:T')); // table NOT recreated + }); + + it('a second apply over the populated dest is a no-op (existence-skip self-heal)', async () => { + const client = new MockClient(); + const r1 = await applyPlan({ client, plan: fullPlan(), destAppId: 'appD', destSnapshot: await snapshotBase(client, 'appD'), idmap: { tables: {}, fields: {} }, journal: newJournal('plnR', 'ts'), persist: () => {} }); + assert.equal(r1.failed, 0); + const r2 = await applyPlan({ client, plan: fullPlan(), destAppId: 'appD', destSnapshot: await snapshotBase(client, 'appD'), idmap: { tables: {}, fields: {} }, journal: newJournal('plnR', 'ts'), persist: () => {} }); + assert.equal(r2.created, 0); + assert.equal(r2.failed, 0); + }); +}); From f3f364c35e49228f60bd5a0c1f34cc123254c362 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 19:28:19 +0300 Subject: [PATCH 033/246] feat(sync): apply orchestrator with drift guard + journal/idmap persistence + apply report Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/index.js | 35 +++++++++++++++-- packages/mcp-server/src/sync/report.js | 19 +++++++++ .../test/sync/test-apply-index.test.js | 39 +++++++++++++++++++ 3 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 packages/mcp-server/test/sync/test-apply-index.test.js diff --git a/packages/mcp-server/src/sync/index.js b/packages/mcp-server/src/sync/index.js index 3ae8622..49c98be 100644 --- a/packages/mcp-server/src/sync/index.js +++ b/packages/mcp-server/src/sync/index.js @@ -1,10 +1,12 @@ import { createHash } from 'node:crypto'; import { snapshotBase } from './snapshot.js'; -import { matchByName, saveIdmap, savePlan, saveState } from './idmap.js'; +import { matchByName, saveIdmap, savePlan, saveState, loadPlan, loadIdmap } from './idmap.js'; import { computePlan } from './diff.js'; -import { renderPlan } from './report.js'; +import { renderPlan, renderApplyResult } from './report.js'; +import { applyPlan } from './apply.js'; +import { newJournal, loadJournal, saveJournal } from './journal.js'; -const ENGINE_VERSION = '2a'; +const ENGINE_VERSION = '2b'; /** * Produce a deterministic SHA-256 fingerprint of a schema snapshot. @@ -49,3 +51,30 @@ export async function plan({ client, sourceBaseId, destBaseId, planId }) { }); return renderPlan(fullPlan); } + +export async function apply({ client, sourceBaseId, destBaseId, planId, runStartedAt }) { + const fullPlan = loadPlan(sourceBaseId, destBaseId, planId); + if (!fullPlan) throw new Error(`No saved plan "${planId}" for ${sourceBaseId} -> ${destBaseId}. Run mode=plan first.`); + + const destSnapshot = await snapshotBase(client, destBaseId); + if (fingerprintSchema(destSnapshot) !== fullPlan.destFingerprint) { + return renderApplyResult({ planId, aborted: true, reason: 'DRIFT', warnings: [{ code: 'DRIFT', message: `Destination changed since plan ${planId}. Re-run mode=plan.` }] }); + } + + const journal = loadJournal(sourceBaseId, destBaseId, planId) ?? newJournal(planId, runStartedAt); + const idmap = journal.actions.length > 0 ? mergeIdmaps(sourceBaseId, destBaseId, fullPlan) : JSON.parse(JSON.stringify(fullPlan.idmap)); + + const result = await applyPlan({ + client, plan: fullPlan, destAppId: destBaseId, destSnapshot, idmap, journal, + persist: (m, j) => { saveIdmap(sourceBaseId, destBaseId, m); saveJournal(sourceBaseId, destBaseId, j); }, + }); + saveState(sourceBaseId, destBaseId, { sourceBaseId, destBaseId, engineVersion: ENGINE_VERSION, lastPlanId: planId, lastApplyAt: runStartedAt }); + return renderApplyResult(result); +} + +// On resume, merge the persisted (grown) idmap over the plan's base matches so this-run +// creations from a prior crashed run survive. +function mergeIdmaps(sourceBaseId, destBaseId, fullPlan) { + const m = loadIdmap(sourceBaseId, destBaseId); + return { tables: { ...fullPlan.idmap.tables, ...m.tables }, fields: { ...fullPlan.idmap.fields, ...m.fields } }; +} diff --git a/packages/mcp-server/src/sync/report.js b/packages/mcp-server/src/sync/report.js index 54044d2..15fa76d 100644 --- a/packages/mcp-server/src/sync/report.js +++ b/packages/mcp-server/src/sync/report.js @@ -22,3 +22,22 @@ export function renderPlan(plan) { } return { human: lines.join('\n'), machine: plan }; } + +export function renderApplyResult(result) { + if (result.aborted) { + const msg = (result.warnings && result.warnings[0] && result.warnings[0].message) || 'aborted'; + return { human: `Apply aborted (${result.reason}): ${msg}`, machine: result }; + } + const lines = [ + `Apply ${result.planId}:`, + ` created: ${result.created}`, + ` updated: ${result.updated}`, + ` skipped: ${result.skipped}`, + ` failed: ${result.failed}`, + ]; + if (result.warnings && result.warnings.length) { + lines.push(' warnings:'); + for (const w of result.warnings) lines.push(` - ${w.code}: ${w.message}`); + } + return { human: lines.join('\n'), machine: result }; +} diff --git a/packages/mcp-server/test/sync/test-apply-index.test.js b/packages/mcp-server/test/sync/test-apply-index.test.js new file mode 100644 index 0000000..e38ed09 --- /dev/null +++ b/packages/mcp-server/test/sync/test-apply-index.test.js @@ -0,0 +1,39 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { join } from 'node:path'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { MockClient } from './helpers/mock-client.js'; +import { apply, fingerprintSchema } from '../../src/sync/index.js'; +import { snapshotBase } from '../../src/sync/snapshot.js'; +import { savePlan } from '../../src/sync/idmap.js'; +import { renderApplyResult } from '../../src/sync/report.js'; + +describe('report.renderApplyResult', () => { + it('summarizes counts and drift abort', () => { + assert.match(renderApplyResult({ planId: 'p', aborted: false, created: 2, updated: 1, skipped: 3, failed: 0, warnings: [] }).human, /created: 2/); + assert.match(renderApplyResult({ planId: 'p', aborted: true, reason: 'DRIFT', warnings: [{ code: 'DRIFT', message: 'm' }] }).human, /DRIFT/); + }); +}); + +describe('index.apply', () => { + it('aborts on dest drift (no mutation)', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'apply-')); + const client = new MockClient(); + const plan = { planId: 'plnD', engineVersion: '2b', destFingerprint: 'STALE', sourceBaseId: 'appSSSSSSSSSSSSSS', destBaseId: 'appDDDDDDDDDDDDDD', idmap: { tables: {}, fields: {} }, actions: [{ kind: 'createTable', sourceTableId: 'tS', name: 'T' }], orphans: [], warnings: [] }; + savePlan('appSSSSSSSSSSSSSS', 'appDDDDDDDDDDDDDD', plan); + const out = await apply({ client, sourceBaseId: 'appSSSSSSSSSSSSSS', destBaseId: 'appDDDDDDDDDDDDDD', planId: 'plnD', runStartedAt: 'ts' }); + assert.match(out.human, /DRIFT/); + assert.equal((await client.getApplicationData('appDDDDDDDDDDDDDD')).data.tableSchemas.length, 0); + }); + + it('applies when the fingerprint matches', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'apply2-')); + const client = new MockClient(); + const dest = await snapshotBase(client, 'appDDDDDDDDDDDDDD'); + const plan = { planId: 'plnOK', engineVersion: '2b', destFingerprint: fingerprintSchema(dest), sourceBaseId: 'appSSSSSSSSSSSSSS', destBaseId: 'appDDDDDDDDDDDDDD', idmap: { tables: {}, fields: {} }, actions: [{ kind: 'createTable', sourceTableId: 'tS', name: 'T' }], orphans: [], warnings: [] }; + savePlan('appSSSSSSSSSSSSSS', 'appDDDDDDDDDDDDDD', plan); + const out = await apply({ client, sourceBaseId: 'appSSSSSSSSSSSSSS', destBaseId: 'appDDDDDDDDDDDDDD', planId: 'plnOK', runStartedAt: 'ts' }); + assert.match(out.human, /created: 1/); + }); +}); From 384b0e6b49cf775dd809124e1a459bab691064c9 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 19:30:43 +0300 Subject: [PATCH 034/246] feat(mcp): sync_base mode=apply (execute saved plan, drift-guarded) Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/index.js | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/packages/mcp-server/src/index.js b/packages/mcp-server/src/index.js index 2ae8468..1e27847 100644 --- a/packages/mcp-server/src/index.js +++ b/packages/mcp-server/src/index.js @@ -1578,14 +1578,15 @@ Note: "form title" is the view name itself — use rename_view to change it. "Fi // ── Sync Tools ── { name: 'sync_base', - description: 'Plan a base-to-base schema sync (READ-ONLY): compares source and destination schema by name and returns an ordered plan of tables/fields to create or update, plus reported orphans and warnings. Does NOT mutate the destination. (mode "apply" arrives in a later release.)', - annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false }, + description: 'Base-to-base schema sync. mode="plan" (read-only): compare source/dest schema by name and return an ordered plan of tables/fields to create or update, plus orphans + warnings; does NOT mutate. mode="apply": execute a saved plan against the destination — create tables, reconcile the primary, create scalar/link/computed fields (with source->dest reference remapping + formula validation), and apply non-destructive field updates. apply requires planId and aborts if the destination drifted since the plan. Type-changing retypes, deletions, records and views are out of scope for this release.', + annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, inputSchema: { type: 'object', properties: { - mode: { type: 'string', enum: ['plan'], description: 'Only "plan" is supported in this release (read-only).' }, + mode: { type: 'string', enum: ['plan', 'apply'], description: '"plan" (read-only preview) or "apply" (execute a saved plan).' }, sourceAppId: { type: 'string', description: 'Source base/application ID to copy schema FROM' }, destAppId: { type: 'string', description: 'Destination base/application ID to copy schema TO' }, + planId: { type: 'string', description: 'Required for mode="apply": the planId returned by a prior mode="plan" run.' }, debug: debugProp, }, required: ['mode', 'sourceAppId', 'destAppId'], @@ -2387,12 +2388,20 @@ const handlers = { // ── Sync ── - async sync_base({ mode, sourceAppId, destAppId, debug }) { - if (mode !== 'plan') return err(`Unsupported mode "${mode}". Only "plan" is available in this release.`); - const { plan } = await import('./sync/index.js'); - const planId = 'pln' + client._genRandomId(); - const out = await plan({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, planId }); - return ok({ planId, summary: out.human }, out.machine, debug); + async sync_base({ mode, sourceAppId, destAppId, planId, debug }) { + const sync = await import('./sync/index.js'); + if (mode === 'plan') { + const id = 'pln' + client._genRandomId(); + const out = await sync.plan({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, planId: id }); + return ok({ planId: id, summary: out.human }, out.machine, debug); + } + if (mode === 'apply') { + if (!planId) return err('mode="apply" requires planId (from a prior mode="plan" run).'); + const runStartedAt = new Date().toISOString(); + const out = await sync.apply({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, planId, runStartedAt }); + return ok({ planId, summary: out.human }, out.machine, debug); + } + return err(`Unsupported mode "${mode}". Use "plan" or "apply".`); }, // ── Meta: Tool Management ── From f9e666abb987c041bbc0bf4be75ed72d74837277 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 19:34:09 +0300 Subject: [PATCH 035/246] docs(sync): document sync_base mode=apply across CLAUDE.md, CHANGELOG, READMEs Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 5 +++++ CLAUDE.md | 1 + README.md | 1 + packages/mcp-server/README.md | 6 ++++++ 4 files changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff8474a..cc341c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how ## [Unreleased] +### MCP server — sync_base mode=apply (2026-06-16) + +#### Added +- `sync_base mode=apply`: execute a saved base-to-base schema plan against the destination — creates tables, reconciles the primary, creates scalar/link/computed fields (source→dest reference remapping + dest-space formula validation), and applies non-destructive field updates. Drift-guarded (aborts if the destination changed since the plan) and resumable via an on-disk journal. Type-changing retypes, deletions, records, and views are out of scope for this release. + ### LSP server — fix runtime startup + CI bin-link warning (2026-06-12) - **`airtable-user-lsp` was runtime-broken when executed with Node directly** diff --git a/CLAUDE.md b/CLAUDE.md index 6692ee3..bd24696 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,6 +65,7 @@ Key files: - `dev-tools/` — reverse-engineering helpers (capture.js, analyze.js, debug-*.js), **gitignored** — kept locally for future reference when Airtable changes their internal API - `src/daemon/` — daemon subsystem (see daemon subsection below) - `src/safe-write.js` — atomic JSON write helper (used by lockfile, token, and settings) +- `src/sync/` — base-to-base schema sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: plan | apply`) in the `sync` category. Out of scope: retypes, deletions, records, views. #### packages/mcp-server — Daemon subsystem diff --git a/README.md b/README.md index c36c4e7..44d3683 100644 --- a/README.md +++ b/README.md @@ -208,6 +208,7 @@ Manage Airtable bases with capabilities **not available through the official RES | **Form Metadata** | 2 | Description, redirect URL, attribution, copy-to-respondent, branding (legacy form views) | | **Extension Management** | 7 | Create, install, enable/disable, rename, duplicate, remove extensions | | **Tool Management** | 1 | List profiles, switch profile, toggle tools/categories (meta-tool, always enabled) | +| **Base Sync** | 1 | `sync_base` — copy a base's schema to another base. `mode=plan` produces a diff report; `mode=apply` executes it (creates tables/fields with ref remapping, drift-guarded, resumable via journal). Retypes, deletions, records, and views are out of scope. | See the full tool reference in [`packages/mcp-server/README.md`](packages/mcp-server/README.md). diff --git a/packages/mcp-server/README.md b/packages/mcp-server/README.md index 390e3cf..005fe4d 100644 --- a/packages/mcp-server/README.md +++ b/packages/mcp-server/README.md @@ -519,6 +519,12 @@ Saved row scaffolds Airtable surfaces under "+ Add record" and the row-create ex | `duplicate_extension` | Clone an installed extension | | `remove_extension` | Remove an extension from a dashboard | +### Base Sync (1) + +| Tool | Description | +|:-----|:------------| +| `sync_base` | Copy a base's schema to another base. `mode=plan` snapshots both bases and produces a diff report (no writes). `mode=apply` executes a saved plan — creates tables (removing Airtable's auto-scaffolding fields), reconciles the primary field, creates scalar/link/computed fields with source→dest reference remapping and dest-space formula validation, and applies non-destructive field updates. Drift-guarded (aborts if the destination changed since the plan was generated) and resumable via an on-disk journal. Out of scope: type-changing retypes, deletions, records, views. | + --- ## Usage Examples From a6ba2a44ba5558b494326bd6d5208ad3cd31fdc8 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 19:44:30 +0300 Subject: [PATCH 036/246] fix(sync): skip computed field with unresolvable refs instead of halting apply Review finding C1: a formula referencing a skipped (unsupported) field left an unresolved token that halted the run. Now skip + UNRESOLVABLE_REF warning, continue. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/apply.js | 6 +++++ .../mcp-server/test/sync/test-apply.test.js | 26 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/packages/mcp-server/src/sync/apply.js b/packages/mcp-server/src/sync/apply.js index 7c19cca..6ecc1bb 100644 --- a/packages/mcp-server/src/sync/apply.js +++ b/packages/mcp-server/src/sync/apply.js @@ -188,6 +188,12 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result } if (LINK_TYPES.has(a.type)) { typeOptions = writableLinkOptions(remapped); } else if (COMPUTED_TYPES.has(a.type)) { + const unresolved = (a.dependsOn || []).filter((d) => !(idmap.fields[d] && idmap.fields[d].destFld)); + if (unresolved.length) { + result.warnings.push({ code: 'UNRESOLVABLE_REF', message: `Field "${a.name}" references unresolved field(s) [${unresolved.join(', ')}] (skipped or unmapped) — not created` }); + result.skipped++; + return; + } typeOptions = toWritableComputedOptions(a.type, remapped); if (a.type === 'formula') { const v = await client.validateFormula(destAppId, destTableId, typeOptions.formulaText ?? ''); diff --git a/packages/mcp-server/test/sync/test-apply.test.js b/packages/mcp-server/test/sync/test-apply.test.js index d7976fc..5fd7d67 100644 --- a/packages/mcp-server/test/sync/test-apply.test.js +++ b/packages/mcp-server/test/sync/test-apply.test.js @@ -263,3 +263,29 @@ describe('apply: resume + idempotency', () => { assert.equal(r2.failed, 0); }); }); + +describe('apply: unresolvable computed ref (no halt)', () => { + it('skips a computed field referencing a skipped/unmapped field, and keeps going', async () => { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'T'); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { + planId: 'plnU2', sourceBaseId: 'appS', destBaseId: 'appD', + idmap: { tables: { tS: tableId }, fields: {} }, + actions: [ + { kind: 'createField', sourceTableId: 'tS', sourceFieldId: 'fldBtn', name: 'Btn', type: 'button', typeOptions: {}, description: null, computed: true, dependsOn: [], dependsOnTables: [] }, + { kind: 'createField', sourceTableId: 'tS', sourceFieldId: 'fldF', name: 'F', type: 'formula', typeOptions: { formulaTextParsed: '{column_value_fldBtn}' }, description: null, computed: true, dependsOn: ['fldBtn'], dependsOnTables: [] }, + { kind: 'createField', sourceTableId: 'tS', sourceFieldId: 'fldOk', name: 'OK', type: 'number', typeOptions: { precision: 0 }, description: null, computed: false, dependsOn: [], dependsOnTables: [] }, + ], + orphans: [], warnings: [], + }; + const res = await applyPlan({ client, plan, destAppId: 'appD', destSnapshot, idmap: JSON.parse(JSON.stringify(plan.idmap)), journal: newJournal('plnU2', 'ts'), persist: () => {} }); + assert.equal(res.failed, 0); // did NOT halt + assert.ok(res.warnings.some((w) => w.code === 'SKIPPED_UNSUPPORTED')); // the button + assert.ok(res.warnings.some((w) => w.code === 'UNRESOLVABLE_REF')); // the formula + assert.equal(res.created, 1); // only the scalar 'OK' + const cols = (await client.getApplicationData('appD')).data.tableSchemas[0].columns; + assert.ok(cols.some((c) => c.name === 'OK')); + assert.ok(!cols.some((c) => c.name === 'F')); // formula NOT created + }); +}); From 4fac8167cdd05471c9d386fcff1210c633f83530 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 22:47:08 +0300 Subject: [PATCH 037/246] fix(sync): apply updateField computed writable shape + continue-on-failure Live smoke found 3 real bugs the mock could not (it never 422s): - updateField on a computed field sent read-only keys (formulaTextParsed/ dependencies/resultType) -> 422; now uses toWritableComputedOptions. - one failed action halted the whole run (494 creates blocked behind a bad update); now records failure and continues (dependents guarded by UNRESOLVABLE_REF; re-run retries non-done actions). - a computed update referencing a not-yet-created field is deferred with UNRESOLVABLE_REF (updates run before creates; a re-plan after creates converges). Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/apply.js | 24 ++++++-- .../mcp-server/test/sync/test-apply.test.js | 56 +++++++++++++++++++ 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/packages/mcp-server/src/sync/apply.js b/packages/mcp-server/src/sync/apply.js index 6ecc1bb..fb2f714 100644 --- a/packages/mcp-server/src/sync/apply.js +++ b/packages/mcp-server/src/sync/apply.js @@ -111,7 +111,9 @@ export async function applyPlan({ client, plan, destAppId, destSnapshot, idmap, result.failed++; result.warnings.push({ code: 'ACTION_FAILED', message: `${a.kind} "${a.name ?? a.sourceFieldId ?? a.sourceTableId}": ${e.message ?? e}` }); persist(idmap, journal); - break; // halt forward progress; resume re-runs from here + // ponytail: continue, don't halt. A failed create's dependents are guarded by the + // UNRESOLVABLE_REF check; re-run retries non-done actions. Halting on one bad field + // blocked the entire sync (494 creates never ran behind one failing update). } } return result; @@ -217,10 +219,24 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result } let mutated = false; if (changes.name !== undefined) { await client.renameField(destAppId, a.destFld, changes.name); mutated = true; } if (changes.typeOptions !== undefined) { + const destType = findDestFieldType(index, a.destFld); const remapped = remapRefs(changes.typeOptions, idmap); - const merged = mergeChoices(findDestField(index, a.destFld), remapped); - await client.updateFieldConfig(destAppId, a.destFld, { type: findDestFieldType(index, a.destFld), typeOptions: merged }); - mutated = true; + if (COMPUTED_TYPES.has(destType)) { + // Computed updates need the writable shape (drop read-only formulaTextParsed/ + // dependencies/resultType — the API 422s on them). Defer if a referenced field + // isn't created yet (updates run before creates); a re-plan after creates converges. + const refs = (changes.typeOptions.dependencies && changes.typeOptions.dependencies.referencedColumnIdsForValue) || []; + const unresolved = refs.filter((d) => !(idmap.fields[d] && idmap.fields[d].destFld)); + if (unresolved.length) { + result.warnings.push({ code: 'UNRESOLVABLE_REF', message: `Update of "${a.destFld}" typeOptions deferred — refs [${unresolved.join(', ')}] not yet created (re-plan after creates)` }); + } else { + await client.updateFieldConfig(destAppId, a.destFld, { type: destType, typeOptions: toWritableComputedOptions(destType, remapped) }); + mutated = true; + } + } else { + await client.updateFieldConfig(destAppId, a.destFld, { type: destType, typeOptions: mergeChoices(findDestField(index, a.destFld), remapped) }); + mutated = true; + } } if (changes.description !== undefined) { await client.updateFieldDescription(destAppId, a.destFld, changes.description); mutated = true; } if (mutated) result.updated++; else result.skipped++; diff --git a/packages/mcp-server/test/sync/test-apply.test.js b/packages/mcp-server/test/sync/test-apply.test.js index 5fd7d67..d078ce2 100644 --- a/packages/mcp-server/test/sync/test-apply.test.js +++ b/packages/mcp-server/test/sync/test-apply.test.js @@ -289,3 +289,59 @@ describe('apply: unresolvable computed ref (no halt)', () => { assert.ok(!cols.some((c) => c.name === 'F')); // formula NOT created }); }); + +describe('apply: updateField computed + continue-on-failure (live-smoke fixes)', () => { + it('computed updateField sends the writable shape (drops read-only keys) when refs resolve', async () => { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'T'); + const price = await client.createField('appD', tableId, { name: 'Price', type: 'number' }); + const total = await client.createField('appD', tableId, { name: 'Total', type: 'formula', typeOptions: { formulaTextParsed: '{column_value_OLD}' } }); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { + planId: 'plnUC', sourceBaseId: 'appS', destBaseId: 'appD', + idmap: { tables: {}, fields: { [price.columnId]: { destFld: price.columnId, choices: {} } } }, + actions: [{ kind: 'updateField', sourceFieldId: total.columnId, destFld: total.columnId, + changes: { typeOptions: { formulaTextParsed: `{column_value_${price.columnId}}*2`, dependencies: { referencedColumnIdsForValue: [price.columnId] }, resultType: 'number', resultIsArray: false } } }], + orphans: [], warnings: [], + }; + const res = await applyPlan({ client, plan, destAppId: 'appD', destSnapshot, idmap: JSON.parse(JSON.stringify(plan.idmap)), journal: newJournal('plnUC', 'ts'), persist: () => {} }); + assert.equal(res.updated, 1); + const f = client._field(total.columnId); + assert.equal(f.typeOptions.formulaText, `{${price.columnId}}*2`); + assert.equal(f.typeOptions.formulaTextParsed, undefined); + assert.equal(f.typeOptions.dependencies, undefined); + }); + + it('defers a computed update whose ref is not yet created (UNRESOLVABLE_REF, no fail)', async () => { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'T'); + const total = await client.createField('appD', tableId, { name: 'Total', type: 'formula', typeOptions: { formulaTextParsed: 'x' } }); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { + planId: 'plnUD', sourceBaseId: 'appS', destBaseId: 'appD', idmap: { tables: {}, fields: {} }, + actions: [{ kind: 'updateField', sourceFieldId: total.columnId, destFld: total.columnId, + changes: { typeOptions: { formulaTextParsed: '{column_value_fldNOTYET}', dependencies: { referencedColumnIdsForValue: ['fldNOTYET'] } } } }], + orphans: [], warnings: [], + }; + const res = await applyPlan({ client, plan, destAppId: 'appD', destSnapshot, idmap: { tables: {}, fields: {} }, journal: newJournal('plnUD', 'ts'), persist: () => {} }); + assert.equal(res.failed, 0); + assert.ok(res.warnings.some((w) => w.code === 'UNRESOLVABLE_REF')); + }); + + it('does not halt the rest of the plan when one action fails', async () => { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'T'); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { + planId: 'plnCont', sourceBaseId: 'appS', destBaseId: 'appD', idmap: { tables: { tS: tableId }, fields: {} }, + actions: [ + { kind: 'updateField', sourceFieldId: 'fX', destFld: 'fldGHOST', changes: { description: 'x' } }, + { kind: 'createField', sourceTableId: 'tS', sourceFieldId: 'fNew', name: 'Score', type: 'number', typeOptions: { precision: 0 }, description: null, computed: false, dependsOn: [], dependsOnTables: [] }, + ], + orphans: [], warnings: [], + }; + const res = await applyPlan({ client, plan, destAppId: 'appD', destSnapshot, idmap: { tables: { tS: tableId }, fields: {} }, journal: newJournal('plnCont', 'ts'), persist: () => {} }); + assert.equal(res.failed, 1); // ghost update failed + assert.equal(res.created, 1); // Score still created — no halt + }); +}); From cc7e7d98dc9bd872e9ab136a376741a3345ca6f1 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 23:13:45 +0300 Subject: [PATCH 038/246] =?UTF-8?q?fix(sync):=20converge=20plan=20?= =?UTF-8?q?=E2=80=94=20select=20choices=20are=20superset-equal,=20skip=20e?= =?UTF-8?q?mpty=20typeOptions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live smoke left 2 phantom updateFields that re-emitted every plan: the diff compared select choices strictly while apply merges additively, and a field with empty source typeOptions produced a no-op {} update. Align the diff with apply: select needs update only when source has a choice dest lacks; skip updates whose source typeOptions is empty. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/diff.js | 29 ++++++++++++++++++- .../mcp-server/test/sync/test-diff.test.js | 27 +++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/packages/mcp-server/src/sync/diff.js b/packages/mcp-server/src/sync/diff.js index 43c393d..912b95f 100644 --- a/packages/mcp-server/src/sync/diff.js +++ b/packages/mcp-server/src/sync/diff.js @@ -107,6 +107,33 @@ function fieldSignature(field, fldNames) { const LINK_TYPES = new Set(['multipleRecordLinks', 'foreignKey']); function fieldOrder(f) { return f.isComputed ? 2 : (LINK_TYPES.has(f.type) ? 1 : 0); } +/** Set of choice names for a select-like field, or null if not a select. */ +function choiceNames(typeOptions) { + const ch = typeOptions && typeOptions.choices; + return ch ? new Set(Object.values(ch).map((c) => c.name)) : null; +} + +/** + * Whether a non-computed field's typeOptions genuinely needs an update, aligned with what + * apply actually does (so plan converges to zero instead of re-emitting phantom updates): + * - select/multiSelect: apply MERGES choices additively (never drops, invariant 7), so an + * update is needed only when SOURCE has a choice name DEST lacks. dest ⊇ src ⇒ equal. + * (Existing-choice colour/order are kept by apply's merge, so they're not flagged here.) + * - other types: a real typeOptions diff, but skip when source options are empty — apply + * can't clear options non-destructively and sending {} is a no-op that never converges. + */ +function scalarTypeOptionsChanged(sf, df) { + const srcChoices = choiceNames(sf.typeOptions); + if (srcChoices) { + const destChoices = choiceNames(df.typeOptions) || new Set(); + for (const n of srcChoices) if (!destChoices.has(n)) return true; // a source choice missing in dest + return false; // dest already has every source choice + } + if (stableStringify(sf.typeOptions ?? null) === stableStringify(df.typeOptions ?? null)) return false; + // ponytail: source has no options to push and we don't strip dest options (destructive, M4) → skip + return !!(sf.typeOptions && Object.keys(sf.typeOptions).length > 0); +} + /** * Collect all field IDs referenced by a field (formula tokens, link columns, etc.) * so the executor can topologically sort creation order. @@ -278,7 +305,7 @@ export function computePlan(srcSnap, destSnap, idmap) { if (computedSig(sf, srcNames) !== computedSig(df, destNames)) { changes.typeOptions = sf.typeOptions; } - } else if (stableStringify(sf.typeOptions ?? null) !== stableStringify(df.typeOptions ?? null)) { + } else if (scalarTypeOptionsChanged(sf, df)) { changes.typeOptions = sf.typeOptions; } if ((sf.description ?? null) !== (df.description ?? null)) { diff --git a/packages/mcp-server/test/sync/test-diff.test.js b/packages/mcp-server/test/sync/test-diff.test.js index 958f590..e30f391 100644 --- a/packages/mcp-server/test/sync/test-diff.test.js +++ b/packages/mcp-server/test/sync/test-diff.test.js @@ -131,3 +131,30 @@ describe('diff plan contract (M2b extensions)', () => { assert.equal(note.description, 'hello'); }); }); + +describe('diff convergence (apply-aligned typeOptions)', () => { + const sel = (id, name, names) => field(id, name, 'singleSelect', { typeOptions: { choices: Object.fromEntries(names.map((n, i) => [`sel${id}${i}`, { id: `sel${id}${i}`, name: n }])) } }); + + it('no updateField when dest select is a superset of source choices', () => { + const src = { baseId: 'appS', tables: [{ id: 'tS', name: 'T', primaryFieldId: 'fS1', fields: [field('fS1', 'Name', 'text'), sel('fS2', 'Status', ['Open', 'Closed'])] }] }; + const dest = { baseId: 'appD', tables: [{ id: 'tD', name: 'T', primaryFieldId: 'fD1', fields: [field('fD1', 'Name', 'text'), sel('fD2', 'Status', ['Open', 'Closed', 'DestOnly'])] }] }; + const plan = computePlan(src, dest, matchByName(src, dest)); + assert.equal(plan.actions.filter((a) => a.kind === 'updateField').length, 0); + }); + + it('updateField when source select has a choice dest lacks', () => { + const src = { baseId: 'appS', tables: [{ id: 'tS', name: 'T', primaryFieldId: 'fS1', fields: [field('fS1', 'Name', 'text'), sel('fS2', 'Status', ['Open', 'Closed', 'New'])] }] }; + const dest = { baseId: 'appD', tables: [{ id: 'tD', name: 'T', primaryFieldId: 'fD1', fields: [field('fD1', 'Name', 'text'), sel('fD2', 'Status', ['Open', 'Closed'])] }] }; + const plan = computePlan(src, dest, matchByName(src, dest)); + const u = plan.actions.filter((a) => a.kind === 'updateField'); + assert.equal(u.length, 1); + assert.equal(u[0].destFld, 'fD2'); + }); + + it('no updateField when source typeOptions is empty but dest has options (no phantom {})', () => { + const src = { baseId: 'appS', tables: [{ id: 'tS', name: 'T', primaryFieldId: 'fS1', fields: [field('fS1', 'Name', 'text'), field('fS2', 'Note', 'text')] }] }; + const dest = { baseId: 'appD', tables: [{ id: 'tD', name: 'T', primaryFieldId: 'fD1', fields: [field('fD1', 'Name', 'text'), field('fD2', 'Note', 'text', { typeOptions: { validatorName: 'url' } })] }] }; + const plan = computePlan(src, dest, matchByName(src, dest)); + assert.equal(plan.actions.filter((a) => a.kind === 'updateField').length, 0); + }); +}); From 0c3ed54f1281ae2d83a6b8264c7f053d98b40e5d Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 23:38:33 +0300 Subject: [PATCH 039/246] feat(sync): add remapViewConfig + canonicalizeViewConfig for view sync Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/remap.js | 51 ++++++++++++++++ .../test/sync/test-remap-view.test.js | 58 +++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 packages/mcp-server/test/sync/test-remap-view.test.js diff --git a/packages/mcp-server/src/sync/remap.js b/packages/mcp-server/src/sync/remap.js index dfbbde7..dcfdd8c 100644 --- a/packages/mcp-server/src/sync/remap.js +++ b/packages/mcp-server/src/sync/remap.js @@ -90,3 +90,54 @@ export function toWritableComputedOptions(type, opts) { return o; } } + +// ── View-config source→dest rewrite (analogue of remapRefs, for views) ── +function destFldId(idmap, src) { const v = (idmap.fields || {})[src]; return v && v.destFld ? v.destFld : src; } +function destSelId(idmap, src) { + for (const v of Object.values(idmap.fields || {})) { const d = ((v && v.choices) || {})[src]; if (d) return d; } + return src; +} +function remapFilterSet(set, idmap) { + return set.map((f) => { + if (f.filterSet) return { ...(f.type ? { type: f.type } : {}), conjunction: f.conjunction, filterSet: remapFilterSet(f.filterSet, idmap) }; + const out = { columnId: destFldId(idmap, f.columnId), operator: f.operator, value: f.value }; + if (typeof f.value === 'string') out.value = destSelId(idmap, f.value); + else if (Array.isArray(f.value)) out.value = f.value.map((v) => (typeof v === 'string' ? destSelId(idmap, v) : v)); + return out; + }); +} +export function remapViewConfig(config, idmap) { + if (config == null || typeof config !== 'object') return config; + const c = JSON.parse(JSON.stringify(config)); + if (c.filters && Array.isArray(c.filters.filterSet)) c.filters = { conjunction: c.filters.conjunction, filterSet: remapFilterSet(c.filters.filterSet, idmap) }; + if (Array.isArray(c.sorts)) c.sorts = c.sorts.map((s) => ({ columnId: destFldId(idmap, s.columnId), ascending: s.ascending })); + if (Array.isArray(c.groupLevels)) c.groupLevels = c.groupLevels.map((g) => ({ columnId: destFldId(idmap, g.columnId), order: g.order, emptyGroupState: g.emptyGroupState })); + if (Array.isArray(c.columnOrder)) c.columnOrder = c.columnOrder.map((co) => ({ columnId: destFldId(idmap, co.columnId), visibility: co.visibility })); + if (c.colorConfig && c.colorConfig.selectColumnId) c.colorConfig = { ...c.colorConfig, selectColumnId: destFldId(idmap, c.colorConfig.selectColumnId) }; + if (c.cover && c.cover.coverColumnId) c.cover = { ...c.cover, coverColumnId: destFldId(idmap, c.cover.coverColumnId) }; + if (c.calendar && Array.isArray(c.calendar.dateColumnRanges)) c.calendar = { dateColumnRanges: c.calendar.dateColumnRanges.map((r) => ({ startColumnId: destFldId(idmap, r.startColumnId), ...(r.endColumnId ? { endColumnId: destFldId(idmap, r.endColumnId) } : {}) })) }; + return c; +} + +// Canonical, id-free, name-based string for a convergent diff compare (ids→names; auto-ids/width dropped). +function viewNameOf(map, id) { return map[id] ?? id; } +function canonFilterSet(set, fldNames, selNames) { + return set.map((f) => f.filterSet + ? { c: f.conjunction, n: canonFilterSet(f.filterSet, fldNames, selNames) } + : { col: viewNameOf(fldNames, f.columnId), op: f.operator, val: typeof f.value === 'string' ? viewNameOf(selNames, f.value) : f.value }); +} +export function canonicalizeViewConfig(config, fldNames, selNames) { + const c = config || {}; + return JSON.stringify({ + filters: c.filters ? { conj: c.filters.conjunction, set: canonFilterSet(c.filters.filterSet || [], fldNames, selNames) } : null, + sorts: (c.sorts || []).map((s) => ({ col: viewNameOf(fldNames, s.columnId), asc: s.ascending })), + groups: (c.groupLevels || []).map((g) => ({ col: viewNameOf(fldNames, g.columnId), order: g.order })), + columns: (c.columnOrder || []).map((co) => ({ col: viewNameOf(fldNames, co.columnId), vis: co.visibility })), + frozen: c.frozenColumnCount ?? null, + color: c.colorConfig ? { type: c.colorConfig.type, col: viewNameOf(fldNames, c.colorConfig.selectColumnId) } : null, + cover: c.cover ? { col: viewNameOf(fldNames, c.cover.coverColumnId), fit: c.cover.coverFitType } : null, + calendar: c.calendar ? (c.calendar.dateColumnRanges || []).map((r) => ({ s: viewNameOf(fldNames, r.startColumnId), e: r.endColumnId ? viewNameOf(fldNames, r.endColumnId) : null })) : null, + rowHeight: c.rowHeight ?? null, + form: c.form ?? null, + }); +} diff --git a/packages/mcp-server/test/sync/test-remap-view.test.js b/packages/mcp-server/test/sync/test-remap-view.test.js new file mode 100644 index 0000000..29897eb --- /dev/null +++ b/packages/mcp-server/test/sync/test-remap-view.test.js @@ -0,0 +1,58 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { remapViewConfig, canonicalizeViewConfig } from '../../src/sync/remap.js'; + +const idmap = { tables: {}, views: {}, fields: { + fldA: { destFld: 'fldX', choices: { selA: 'selX' } }, + fldB: { destFld: 'fldY', choices: {} }, +} }; + +describe('remap.remapViewConfig', () => { + it('remaps fld ids across filters (nested), sorts, groups, columnOrder, colorConfig, cover, calendar', () => { + const cfg = { + filters: { conjunction: 'and', filterSet: [ + { id: 'fltAAA', columnId: 'fldA', operator: '=', value: 'selA' }, + { type: 'nested', conjunction: 'or', filterSet: [{ id: 'fltBBB', columnId: 'fldB', operator: '=', value: 'x' }] }, + ] }, + sorts: [{ columnId: 'fldB', ascending: true }], + groupLevels: [{ columnId: 'fldA', order: 'ascending', emptyGroupState: 'hidden' }], + columnOrder: [{ columnId: 'fldA', visibility: true, width: 120 }, { columnId: 'fldB', visibility: false }], + colorConfig: { type: 'selectColumn', selectColumnId: 'fldA' }, + cover: { coverColumnId: 'fldB', coverFitType: 'fit' }, + calendar: { dateColumnRanges: [{ startColumnId: 'fldA', endColumnId: 'fldB' }] }, + }; + const out = remapViewConfig(cfg, idmap); + assert.equal(out.filters.filterSet[0].columnId, 'fldX'); + assert.equal(out.filters.filterSet[0].value, 'selX'); // select value choice-id remapped + assert.equal(out.filters.filterSet[0].id, undefined); // auto flt id stripped + assert.equal(out.filters.filterSet[1].filterSet[0].columnId, 'fldY'); // nested remapped + assert.equal(out.sorts[0].columnId, 'fldY'); + assert.equal(out.groupLevels[0].columnId, 'fldX'); + assert.equal(out.columnOrder[0].columnId, 'fldX'); + assert.equal(out.columnOrder[0].width, undefined); // width stripped + assert.equal(out.colorConfig.selectColumnId, 'fldX'); + assert.equal(out.cover.coverColumnId, 'fldY'); + assert.equal(out.calendar.dateColumnRanges[0].startColumnId, 'fldX'); + assert.equal(out.calendar.dateColumnRanges[0].endColumnId, 'fldY'); + }); + it('does not mutate input + passes null through', () => { + const input = { sorts: [{ columnId: 'fldA', ascending: true }] }; + const out = remapViewConfig(input, idmap); + assert.equal(input.sorts[0].columnId, 'fldA'); + assert.notEqual(out, input); + assert.equal(remapViewConfig(null, idmap), null); + }); +}); + +describe('remap.canonicalizeViewConfig', () => { + it('identical-by-name configs are canonical-equal despite different ids/auto-ids/width (convergence)', () => { + const a = canonicalizeViewConfig({ sorts: [{ columnId: 'fldA', ascending: true }], columnOrder: [{ columnId: 'fldB', visibility: true, width: 99 }] }, { fldA: 'Price', fldB: 'Qty' }, {}); + const b = canonicalizeViewConfig({ sorts: [{ columnId: 'fldX', ascending: true }], columnOrder: [{ columnId: 'fldY', visibility: true, width: 12 }] }, { fldX: 'Price', fldY: 'Qty' }, {}); + assert.equal(a, b); + }); + it('a real difference (different sort field) is NOT canonical-equal', () => { + const a = canonicalizeViewConfig({ sorts: [{ columnId: 'fldA', ascending: true }] }, { fldA: 'Price' }, {}); + const b = canonicalizeViewConfig({ sorts: [{ columnId: 'fldB', ascending: true }] }, { fldB: 'Qty' }, {}); + assert.notEqual(a, b); + }); +}); From e9bd5b2c16ae094f218734022f81a9f13fd338ed Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 23:41:41 +0300 Subject: [PATCH 040/246] feat(sync): snapshot collaborative views + live config (sortSet unwrap, metadata lift) Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/snapshot.js | 41 ++++++++++++++++++- .../test/sync/test-snapshot.test.js | 30 +++++++++++++- 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/packages/mcp-server/src/sync/snapshot.js b/packages/mcp-server/src/sync/snapshot.js index f6a867a..a3970a9 100644 --- a/packages/mcp-server/src/sync/snapshot.js +++ b/packages/mcp-server/src/sync/snapshot.js @@ -42,11 +42,48 @@ export function normalizeSchema(rawData) { description: c.description ?? null, isComputed: isComputedType(c.type), })), + views: (t.views || []).map((v) => ({ + id: v.id, name: v.name, type: v.type, + description: v.description ?? null, + personalForUserId: v.personalForUserId ?? null, + })), }; }), }; } +// Map a raw getView result into the flat view-config snapshot shape (null facets omitted). +// PROBE-VERIFIED (spec §11): sorts is {sortSet:[...]} → flatten to array; type-specific config +// lives under metadata. (calendar.dateColumnRanges, gallery.coverColumnId), NOT top-level. +export function normalizeViewConfig(v) { + const cfg = {}; + if (v.filters) cfg.filters = v.filters; + if (v.sorts && Array.isArray(v.sorts.sortSet)) cfg.sorts = v.sorts.sortSet.map((s) => ({ columnId: s.columnId, ascending: s.ascending })); + if (v.groupLevels) cfg.groupLevels = v.groupLevels.map((g) => ({ columnId: g.columnId, order: g.order, emptyGroupState: g.emptyGroupState })); + if (v.columnOrder) cfg.columnOrder = v.columnOrder.map((c) => ({ columnId: c.columnId, visibility: c.visibility })); + if (typeof v.frozenColumnCount === 'number') cfg.frozenColumnCount = v.frozenColumnCount; + if (v.colorConfig) cfg.colorConfig = v.colorConfig; + const md = v.metadata || {}; + if (md.gallery && md.gallery.coverColumnId) cfg.cover = { coverColumnId: md.gallery.coverColumnId, coverFitType: md.gallery.coverFitType }; + if (md.calendar && Array.isArray(md.calendar.dateColumnRanges)) cfg.calendar = { dateColumnRanges: md.calendar.dateColumnRanges.map((r) => ({ startColumnId: r.startColumnId, ...(r.endColumnId ? { endColumnId: r.endColumnId } : {}) })) }; + if (md.form) cfg.form = md.form; + if (v.rowHeight) cfg.rowHeight = v.rowHeight; + return cfg; +} + +/** Attach live config to each COLLABORATIVE view (personal views skipped). Mutates + returns snap. */ +export async function snapshotViews(client, appId, snap) { + if (typeof client.getView !== 'function') return snap; + for (const t of snap.tables) { + for (const v of t.views || []) { + if (v.personalForUserId) continue; + const live = await client.getView(appId, v.id); + v.config = normalizeViewConfig(live); + } + } + return snap; +} + /** * Fetches and normalizes the schema for a base. * @@ -56,7 +93,9 @@ export function normalizeSchema(rawData) { */ export async function snapshotBase(client, appId) { const raw = await client.getApplicationData(appId); - return { baseId: appId, ...normalizeSchema(raw) }; + const snap = { baseId: appId, ...normalizeSchema(raw) }; + await snapshotViews(client, appId, snap); + return snap; } /** diff --git a/packages/mcp-server/test/sync/test-snapshot.test.js b/packages/mcp-server/test/sync/test-snapshot.test.js index 7f7045c..2a786e6 100644 --- a/packages/mcp-server/test/sync/test-snapshot.test.js +++ b/packages/mcp-server/test/sync/test-snapshot.test.js @@ -1,6 +1,6 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { normalizeSchema, isComputedType } from '../../src/sync/snapshot.js'; +import { normalizeSchema, isComputedType, snapshotViews } from '../../src/sync/snapshot.js'; describe('snapshot.isComputedType', () => { it('flags computed types and not writable ones', () => { @@ -38,3 +38,31 @@ describe('snapshot.normalizeSchema', () => { assert.equal(snap.tables[0].primaryFieldId, 'fldZ'); }); }); + +describe('snapshot views', () => { + it('normalizeSchema attaches static views with personal flag', () => { + const raw = { data: { tableSchemas: [{ id: 'tbl1', name: 'T', primaryColumnId: 'f1', columns: [{ id: 'f1', name: 'Name', type: 'text' }], + views: [{ id: 'viwA', name: 'Grid view', type: 'grid', description: null, personalForUserId: null }, + { id: 'viwP', name: 'Mine', type: 'grid', personalForUserId: 'usr1' }] }] } }; + const snap = normalizeSchema(raw); + assert.equal(snap.tables[0].views.length, 2); + assert.equal(snap.tables[0].views[0].id, 'viwA'); + assert.equal(snap.tables[0].views[1].personalForUserId, 'usr1'); + }); + it('snapshotViews attaches live config (sorts unwrapped, calendar lifted from metadata); skips personal', async () => { + const snap = { baseId: 'appD', tables: [{ id: 'tbl1', name: 'T', views: [ + { id: 'viwA', name: 'Grid view', type: 'grid', personalForUserId: null }, + { id: 'viwP', name: 'Mine', type: 'grid', personalForUserId: 'usr1' } ] }] }; + const client = { getView: async () => ({ filters: { conjunction: 'and', filterSet: [] }, + sorts: { sortSet: [{ id: 'srt1', columnId: 'f1', ascending: true }], shouldAutoSort: true }, + groupLevels: null, columnOrder: null, frozenColumnCount: 1, colorConfig: null, + metadata: { calendar: { dateColumnRanges: [{ startColumnId: 'fDate' }] } }, rowHeight: 'small', description: null }) }; + await snapshotViews(client, 'appD', snap); + const cfg = snap.tables[0].views[0].config; + assert.ok(cfg); + assert.equal(cfg.frozenColumnCount, 1); + assert.deepEqual(cfg.sorts, [{ columnId: 'f1', ascending: true }]); // sortSet unwrapped + assert.deepEqual(cfg.calendar, { dateColumnRanges: [{ startColumnId: 'fDate' }] }); // lifted from metadata + assert.equal(snap.tables[0].views[1].config, undefined); // personal skipped + }); +}); From 2596d416f8445c0211f0b6c18e442ad3069c1173 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 23:43:31 +0300 Subject: [PATCH 041/246] feat(sync): match collaborative views by name in idmap Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/idmap.js | 10 +++++++- .../mcp-server/test/sync/test-idmap.test.js | 24 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/packages/mcp-server/src/sync/idmap.js b/packages/mcp-server/src/sync/idmap.js index 0dfa27e..2d0e3ff 100644 --- a/packages/mcp-server/src/sync/idmap.js +++ b/packages/mcp-server/src/sync/idmap.js @@ -51,6 +51,7 @@ export function matchByName(srcSnap, destSnap) { const destTables = indexByName(destSnap.tables); const tables = {}; const fields = {}; + const views = {}; for (const st of srcSnap.tables) { const dt = destTables.get(st.name); @@ -63,9 +64,16 @@ export function matchByName(srcSnap, destSnap) { if (!df) continue; fields[sf.id] = { destFld: df.id, choices: matchChoices(sf, df) }; } + + const destViews = indexByName((dt.views || []).filter((v) => !v.personalForUserId)); + for (const sv of (st.views || [])) { + if (sv.personalForUserId) continue; + const dv = destViews.get(sv.name); + if (dv) views[sv.id] = dv.id; + } } - return { tables, fields }; + return { tables, fields, views }; } // ── State I/O ────────────────────────────────────────────────────────────── diff --git a/packages/mcp-server/test/sync/test-idmap.test.js b/packages/mcp-server/test/sync/test-idmap.test.js index 627b862..4b17226 100644 --- a/packages/mcp-server/test/sync/test-idmap.test.js +++ b/packages/mcp-server/test/sync/test-idmap.test.js @@ -32,6 +32,30 @@ describe('idmap.matchByName', () => { }); }); +describe('idmap view matching', () => { + it('matches collaborative views by name within a table; skips personal + unmatched', () => { + const src = { tables: [{ id: 'tS', name: 'T', fields: [{ id: 'fS', name: 'Name' }], views: [ + { id: 'vS1', name: 'Grid view', personalForUserId: null }, + { id: 'vS2', name: 'OnlySrc', personalForUserId: null }, + { id: 'vS3', name: 'Mine', personalForUserId: 'usr1' } ] }] }; + const dest = { tables: [{ id: 'tD', name: 'T', fields: [{ id: 'fD', name: 'Name' }], views: [ + { id: 'vD1', name: 'Grid view', personalForUserId: null }, + { id: 'vDm', name: 'Mine', personalForUserId: 'usr2' } ] }] }; + const m = matchByName(src, dest); + assert.equal(m.views.vS1, 'vD1'); // matched by name + assert.equal(m.views.vS2, undefined); // no dest match + assert.equal(m.views.vS3, undefined); // source personal skipped + }); + it('still returns tables + fields maps (additive)', () => { + const src = { tables: [{ id: 'tS', name: 'T', fields: [{ id: 'fS', name: 'Name' }], views: [] }] }; + const dest = { tables: [{ id: 'tD', name: 'T', fields: [{ id: 'fD', name: 'Name' }], views: [] }] }; + const m = matchByName(src, dest); + assert.equal(m.tables.tS, 'tD'); + assert.equal(m.fields.fS.destFld, 'fD'); + assert.deepEqual(m.views, {}); + }); +}); + describe('idmap state I/O', () => { it('round-trips idmap through the sync dir', () => { process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-test-')); From c420b3bb4862395c8edab3e3aa35d1473d3dec76 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 23:47:50 +0300 Subject: [PATCH 042/246] feat(sync): diff emits createView + applyViewConfig (convergent, after fields) Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/diff.js | 49 ++++++++++++++++++- .../mcp-server/test/sync/test-diff.test.js | 32 ++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/packages/mcp-server/src/sync/diff.js b/packages/mcp-server/src/sync/diff.js index 912b95f..010a8e9 100644 --- a/packages/mcp-server/src/sync/diff.js +++ b/packages/mcp-server/src/sync/diff.js @@ -1,4 +1,4 @@ -import { canonicalizeComputed } from './remap.js'; +import { canonicalizeComputed, canonicalizeViewConfig } from './remap.js'; /** * Build a flat map of { fieldId → fieldName } across all tables in a snapshot. @@ -11,6 +11,23 @@ function fldNameMap(snap) { return m; } +/** + * Build a flat map of { choiceId → choiceName } across all select-like fields in a snapshot. + * @param {{ tables: Array<{fields: Array<{typeOptions: object|null}>}> }} snap + * @returns {Record} + */ +function selNameMap(snap) { + const m = {}; + for (const t of snap.tables) for (const f of t.fields) { + const ch = f.typeOptions && f.typeOptions.choices; + if (ch) for (const c of Object.values(ch)) m[c.id] = c.name; + } + return m; +} + +/** Return only collaborative (non-personal) views from a table. */ +function collabViews(table) { return (table.views || []).filter((v) => !v.personalForUserId); } + /** * Build a regex that matches any field ID key in the given map, wrapped in * curly braces (the Airtable formula token syntax: `{fldXYZ}`). Returns null @@ -347,5 +364,35 @@ export function computePlan(srcSnap, destSnap, idmap) { } } + // ── View diff (collaborative only; appended AFTER all field/table actions) ── + const srcSelNames = selNameMap(srcSnap); + const destSelNames = selNameMap(destSnap); + const viewActions = []; + for (const st of srcSnap.tables) { + const destTableId = idmap.tables[st.id]; + const destTable = destTableId ? destTablesById.get(destTableId) : null; + const destViewsByName = destTable ? new Map(collabViews(destTable).map((v) => [v.name, v])) : new Map(); + for (const sv of (st.views || [])) { + if (sv.personalForUserId) { warnings.push({ code: 'VIEW_PERSONAL_SKIPPED', message: `Personal view "${sv.name}" in "${st.name}" skipped` }); continue; } + const dv = destViewsByName.get(sv.name); + if (!dv) { + viewActions.push({ kind: 'createView', sourceTableId: st.id, sourceViewId: sv.id, name: sv.name, type: sv.type }); + viewActions.push({ kind: 'applyViewConfig', sourceTableId: st.id, sourceViewId: sv.id, type: sv.type, config: sv.config || {} }); + } else if (canonicalizeViewConfig(sv.config || {}, srcNames, srcSelNames) !== canonicalizeViewConfig(dv.config || {}, destNames, destSelNames)) { + viewActions.push({ kind: 'applyViewConfig', sourceTableId: st.id, sourceViewId: sv.id, type: sv.type, config: sv.config || {} }); + } + } + } + actions.push(...viewActions); + + // View orphans: dest-only collaborative views in a matched table. + for (const dt of destSnap.tables) { + const srcTableId = srcTableByDestId.get(dt.id); + if (!srcTableId) continue; // whole table is already a table orphan + const srcTable = srcTablesById.get(srcTableId); + const srcViewNames = new Set(collabViews(srcTable).map((v) => v.name)); + for (const dv of collabViews(dt)) if (!srcViewNames.has(dv.name)) orphans.push({ kind: 'view', destId: dv.id, name: dv.name, tableName: dt.name }); + } + return { sourceBaseId: srcSnap.baseId, destBaseId: destSnap.baseId, idmap, actions, orphans, warnings }; } diff --git a/packages/mcp-server/test/sync/test-diff.test.js b/packages/mcp-server/test/sync/test-diff.test.js index e30f391..73023c2 100644 --- a/packages/mcp-server/test/sync/test-diff.test.js +++ b/packages/mcp-server/test/sync/test-diff.test.js @@ -158,3 +158,35 @@ describe('diff convergence (apply-aligned typeOptions)', () => { assert.equal(plan.actions.filter((a) => a.kind === 'updateField').length, 0); }); }); + +function vw(id, name, type, config, extra = {}) { return { id, name, type, config: config ?? {}, personalForUserId: extra.personal ?? null }; } + +describe('diff view actions', () => { + it('createView + applyViewConfig for a source-only view; skip canonical-equal matched view; views after fields', () => { + const src = { baseId: 'appS', tables: [{ id: 'tS', name: 'T', primaryFieldId: 'fS1', fields: [field('fS1', 'Name', 'text')], views: [ + vw('vS1', 'Grid view', 'grid', { sorts: [{ columnId: 'fS1', ascending: true }] }), + vw('vS2', 'New View', 'grid', { frozenColumnCount: 1 }) ] }] }; + const dest = { baseId: 'appD', tables: [{ id: 'tD', name: 'T', primaryFieldId: 'fD1', fields: [field('fD1', 'Name', 'text')], views: [ + vw('vD1', 'Grid view', 'grid', { sorts: [{ columnId: 'fD1', ascending: true }] }) ] }] }; + const plan = computePlan(src, dest, matchByName(src, dest)); + const cv = plan.actions.find((a) => a.kind === 'createView'); + assert.ok(cv && cv.name === 'New View'); + // 'Grid view' matched + canonical-equal (sort on 'Name' both sides) → NO applyViewConfig for vS1 + assert.equal(plan.actions.filter((a) => a.kind === 'applyViewConfig' && a.sourceViewId === 'vS1').length, 0); + // view actions come after every field/table action + const idxs = plan.actions.map((a, i) => ({ a, i })); + const lastField = Math.max(-1, ...idxs.filter(({ a }) => a.kind === 'createTable' || a.kind === 'reconcilePrimary' || a.kind === 'createField' || a.kind === 'updateField').map(({ i }) => i)); + const firstView = plan.actions.findIndex((a) => a.kind === 'createView' || a.kind === 'applyViewConfig'); + assert.ok(firstView === -1 || firstView > lastField); + }); + + it('reports dest-only view as orphan; flags source personal view', () => { + const src = { baseId: 'appS', tables: [{ id: 'tS', name: 'T', primaryFieldId: 'fS1', fields: [field('fS1', 'Name', 'text')], views: [ + vw('vS1', 'Grid view', 'grid', {}), vw('vSp', 'Mine', 'grid', {}, { personal: 'usr1' }) ] }] }; + const dest = { baseId: 'appD', tables: [{ id: 'tD', name: 'T', primaryFieldId: 'fD1', fields: [field('fD1', 'Name', 'text')], views: [ + vw('vD1', 'Grid view', 'grid', {}), vw('vDx', 'DestOnly', 'grid', {}) ] }] }; + const plan = computePlan(src, dest, matchByName(src, dest)); + assert.ok(plan.orphans.some((o) => o.kind === 'view' && o.name === 'DestOnly')); + assert.ok(plan.warnings.some((w) => w.code === 'VIEW_PERSONAL_SKIPPED')); + }); +}); From 9f2b909bcfa5cd12fd088682f79d2c296ae23732 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 23:51:47 +0300 Subject: [PATCH 043/246] feat(sync): apply createView (adopt-by-name) + mock view state Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/apply.js | 15 ++++++++++++ .../test/sync/helpers/mock-client.js | 23 ++++++++++++++++++- .../mcp-server/test/sync/test-apply.test.js | 19 +++++++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/packages/mcp-server/src/sync/apply.js b/packages/mcp-server/src/sync/apply.js index fb2f714..c56bda8 100644 --- a/packages/mcp-server/src/sync/apply.js +++ b/packages/mcp-server/src/sync/apply.js @@ -25,6 +25,7 @@ function buildIndex(snap) { const entry = { id: t.id, name: t.name, primaryFieldId: t.primaryFieldId, fieldsByName: new Map(t.fields.map((f) => [f.name, { id: f.id, name: f.name, type: f.type, typeOptions: f.typeOptions }])), + viewsByName: new Map((t.views || []).map((v) => [v.name, { id: v.id, type: v.type }])), }; tablesById.set(t.id, entry); tablesByName.set(t.name, entry); @@ -97,6 +98,7 @@ function mergeChoices(destField, srcTypeOptions) { export async function applyPlan({ client, plan, destAppId, destSnapshot, idmap, journal, persist }) { const index = buildIndex(destSnapshot); const state = { createdLinks: new Map(), adoptedReverse: new Set() }; + if (!idmap.views) idmap.views = {}; const result = { planId: plan.planId, aborted: false, created: 0, updated: 0, skipped: 0, failed: 0, warnings: [], idmap }; for (let idx = 0; idx < plan.actions.length; idx++) { @@ -243,6 +245,19 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result } return; } + case 'createView': { + const destTableId = idmap.tables[a.sourceTableId]; + const entry = index.tablesById.get(destTableId); + const existing = entry && entry.viewsByName.get(a.name); + if (existing) { idmap.views[a.sourceViewId] = existing.id; result.skipped++; return; } + const template = entry && [...entry.viewsByName.values()][0]; // dest table always has ≥1 view + const { viewId } = await client.createView(destAppId, destTableId, { name: a.name, type: a.type, copyFromViewId: template ? template.id : undefined }); + idmap.views[a.sourceViewId] = viewId; + if (entry) entry.viewsByName.set(a.name, { id: viewId, type: a.type }); + result.created++; + return; + } + default: throw new Error(`unhandled action kind: ${a.kind}`); } diff --git a/packages/mcp-server/test/sync/helpers/mock-client.js b/packages/mcp-server/test/sync/helpers/mock-client.js index 842cfdd..1eaad9c 100644 --- a/packages/mcp-server/test/sync/helpers/mock-client.js +++ b/packages/mcp-server/test/sync/helpers/mock-client.js @@ -22,7 +22,8 @@ export class MockClient { async createTable(appId, name) { const tableId = this._id('tbl'); const columns = DEFAULT_FIELDS.map((f) => ({ id: this._id('fld'), name: f.name, type: f.type, typeOptions: f.typeOptions ?? null, description: null })); - this.tables.push({ id: tableId, name, primaryColumnId: columns[0].id, columns }); + this.tables.push({ id: tableId, name, primaryColumnId: columns[0].id, columns, + views: [{ id: this._id('viw'), name: 'Grid view', type: 'grid', personalForUserId: null, config: {} }] }); this.calls.push(`createTable:${name}`); return { tableId }; } @@ -65,6 +66,26 @@ export class MockClient { async updateFieldDescription(appId, columnId, description) { this._field(columnId).description = description; this.calls.push(`updateFieldDescription:${columnId}`); return { ok: true }; } + _view(viewId) { for (const t of this.tables) { const v = (t.views || []).find((x) => x.id === viewId); if (v) return v; } throw new Error('no view ' + viewId); } + async createView(appId, tableId, cfg) { + const id = this._id('viw'); + this._table(tableId).views.push({ id, name: cfg.name, type: cfg.type || 'grid', personalForUserId: null, config: {} }); + this.calls.push(`createView:${cfg.name}:${cfg.type}`); + return { viewId: id }; + } + async getView(appId, viewId) { return { ...(this._view(viewId).config || {}) }; } + async renameView(appId, viewId, name) { this._view(viewId).name = name; this.calls.push(`renameView:${viewId}:${name}`); return { ok: true }; } + async updateViewFilters(appId, viewId, filters) { this._view(viewId).config.filters = filters; this.calls.push(`updateViewFilters:${viewId}`); return { ok: true }; } + async applySorts(appId, viewId, sortObjs) { this._view(viewId).config.sorts = sortObjs; this.calls.push(`applySorts:${viewId}`); return { ok: true }; } + async updateGroupLevels(appId, viewId, groupLevels) { this._view(viewId).config.groupLevels = groupLevels; this.calls.push(`updateGroupLevels:${viewId}`); return { ok: true }; } + async setViewColumns(appId, viewId, opts) { this._view(viewId).config.columns = opts; this.calls.push(`setViewColumns:${viewId}`); return { ok: true }; } + async updateFrozenColumnCount(appId, viewId, n) { this._view(viewId).config.frozenColumnCount = n; this.calls.push(`updateFrozenColumnCount:${viewId}`); return { ok: true }; } + async setViewColorConfig(appId, viewId, cc) { this._view(viewId).config.colorConfig = cc; this.calls.push(`setViewColorConfig:${viewId}`); return { ok: true }; } + async setViewCover(appId, viewId, cover) { this._view(viewId).config.cover = cover; this.calls.push(`setViewCover:${viewId}`); return { ok: true }; } + async setCalendarDateColumns(appId, viewId, ranges) { this._view(viewId).config.calendar = { dateColumnRanges: ranges }; this.calls.push(`setCalendarDateColumns:${viewId}`); return { ok: true }; } + async setFormMetadata(appId, viewId, form) { this._view(viewId).config.form = form; this.calls.push(`setFormMetadata:${viewId}`); return { ok: true }; } + async updateRowHeight(appId, viewId, rh) { this._view(viewId).config.rowHeight = rh; this.calls.push(`updateRowHeight:${viewId}`); return { ok: true }; } + async updateViewDescription(appId, viewId, d) { this._view(viewId).config.description = d; this.calls.push(`updateViewDescription:${viewId}`); return { ok: true }; } async validateFormula(appId, tableId, formulaText) { this.calls.push(`validateFormula:${formulaText}`); return { valid: this.formulaValid, resultType: 'text' }; diff --git a/packages/mcp-server/test/sync/test-apply.test.js b/packages/mcp-server/test/sync/test-apply.test.js index d078ce2..fa198f3 100644 --- a/packages/mcp-server/test/sync/test-apply.test.js +++ b/packages/mcp-server/test/sync/test-apply.test.js @@ -345,3 +345,22 @@ describe('apply: updateField computed + continue-on-failure (live-smoke fixes)', assert.equal(res.created, 1); // Score still created — no halt }); }); + +describe('apply: createView', () => { + it('creates a missing view (copyFromViewId=default), maps it; adopts an existing one by name', async () => { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'T'); // has default Grid view + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { planId: 'plnV', sourceBaseId: 'appS', destBaseId: 'appD', idmap: { tables: { tS: tableId }, fields: {}, views: {} }, + actions: [ + { kind: 'createView', sourceTableId: 'tS', sourceViewId: 'vGrid', name: 'Grid view', type: 'grid' }, + { kind: 'createView', sourceTableId: 'tS', sourceViewId: 'vNew', name: 'New View', type: 'grid' }, + ], orphans: [], warnings: [] }; + const res = await applyPlan({ client, plan, destAppId: 'appD', destSnapshot, idmap: JSON.parse(JSON.stringify(plan.idmap)), journal: newJournal('plnV', 'ts'), persist: () => {} }); + assert.ok(res.idmap.views.vGrid); // adopted the default Grid view + assert.ok(res.idmap.views.vNew); // created + const views = (await client.getApplicationData('appD')).data.tableSchemas[0].views; + assert.equal(views.filter((v) => v.name === 'New View').length, 1); + assert.equal(views.filter((v) => v.name === 'Grid view').length, 1); // not duplicated + }); +}); From 9a857997ca6ec974109f385b11503db63be6a2bf Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 23:53:43 +0300 Subject: [PATCH 044/246] fix(sync): createTable index entry includes viewsByName so same-run createView adopts the default view T6 review caught: buildIndex set viewsByName but the createTable case built its entry manually without it -> a createView for a same-run-created table would crash or duplicate the auto Grid view. readTable now returns views. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/apply.js | 6 ++++-- .../mcp-server/test/sync/test-apply.test.js | 20 +++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/packages/mcp-server/src/sync/apply.js b/packages/mcp-server/src/sync/apply.js index c56bda8..e513a44 100644 --- a/packages/mcp-server/src/sync/apply.js +++ b/packages/mcp-server/src/sync/apply.js @@ -40,7 +40,7 @@ async function readTable(client, appId, tableId) { if (!t) throw new Error(`table ${tableId} not found after create`); const cols = t.columns ?? t.fields ?? []; const primaryId = t.primaryColumnId ?? t.primaryFieldId ?? cols[0]?.id; - return { primaryId, primary: cols.find((c) => c.id === primaryId), cols }; + return { primaryId, primary: cols.find((c) => c.id === primaryId), cols, views: t.views ?? [] }; } function rememberLink(state, forwardFieldId, destTableId) { @@ -129,7 +129,7 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result } const { tableId } = await client.createTable(destAppId, a.name); idmap.tables[a.sourceTableId] = tableId; // D1: delete the auto-created non-primary scaffolding fields for a clean mirror. - const { primaryId, primary, cols } = await readTable(client, destAppId, tableId); + const { primaryId, primary, cols, views } = await readTable(client, destAppId, tableId); for (const c of cols) { if (c.id === primaryId) continue; await client.deleteField(destAppId, c.id, c.name); @@ -137,6 +137,8 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result } const entry = { id: tableId, name: a.name, primaryFieldId: primaryId, fieldsByName: new Map([[primary.name, { id: primaryId, name: primary.name, type: primary.type, typeOptions: primary.typeOptions ?? null }]]), + // include the auto-created default view(s) so a same-run createView adopts (not duplicates) them + viewsByName: new Map((views || []).map((v) => [v.name, { id: v.id, type: v.type }])), }; index.tablesById.set(tableId, entry); index.tablesByName.set(a.name, entry); diff --git a/packages/mcp-server/test/sync/test-apply.test.js b/packages/mcp-server/test/sync/test-apply.test.js index fa198f3..a62ade2 100644 --- a/packages/mcp-server/test/sync/test-apply.test.js +++ b/packages/mcp-server/test/sync/test-apply.test.js @@ -364,3 +364,23 @@ describe('apply: createView', () => { assert.equal(views.filter((v) => v.name === 'Grid view').length, 1); // not duplicated }); }); + +describe('apply: createTable then createView in one run (adopt default view)', () => { + it('adopts the auto-created Grid view (no crash, no duplicate) and creates new views', async () => { + const client = new MockClient(); + const destSnapshot = await snapshotBase(client, 'appD'); // empty base + const plan = { planId: 'plnTV', sourceBaseId: 'appS', destBaseId: 'appD', idmap: { tables: {}, fields: {}, views: {} }, + actions: [ + { kind: 'createTable', sourceTableId: 'tS', name: 'T' }, + { kind: 'reconcilePrimary', sourceTableId: 'tS', sourcePrimaryFieldId: 'fS1', toName: 'Name', toType: 'text', toTypeOptions: null }, + { kind: 'createView', sourceTableId: 'tS', sourceViewId: 'vGrid', name: 'Grid view', type: 'grid' }, + { kind: 'createView', sourceTableId: 'tS', sourceViewId: 'vNew', name: 'Board', type: 'grid' }, + ], orphans: [], warnings: [] }; + const res = await applyPlan({ client, plan, destAppId: 'appD', destSnapshot, idmap: { tables: {}, fields: {}, views: {} }, journal: newJournal('plnTV', 'ts'), persist: () => {} }); + assert.equal(res.failed, 0); + const t = (await client.getApplicationData('appD')).data.tableSchemas[0]; + assert.equal(t.views.filter((v) => v.name === 'Grid view').length, 1); // default adopted, not duplicated + assert.ok(res.idmap.views.vGrid); + assert.ok(t.views.some((v) => v.name === 'Board')); + }); +}); From 4d17b2d4b8c21f4fc8715733c93ac18c87e5983d Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 23:57:20 +0300 Subject: [PATCH 045/246] feat(sync): apply applyViewConfig -- facets, anchor validation (kanban=groupLevels), grid fallback Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/apply.js | 40 ++++++++++++++++++- .../mcp-server/test/sync/test-apply.test.js | 35 ++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/packages/mcp-server/src/sync/apply.js b/packages/mcp-server/src/sync/apply.js index e513a44..c41c744 100644 --- a/packages/mcp-server/src/sync/apply.js +++ b/packages/mcp-server/src/sync/apply.js @@ -2,10 +2,11 @@ // Live-first + growing idmap + journal (resume) + existence-check (idempotency). // POST-SPIKE: primary defaults to Name/text; createTable spawns 6 default fields → // delete the 5 non-primary scaffolding fields (D1); links are foreignKey (later task). -import { remapRefs, toWritableComputedOptions } from './remap.js'; +import { remapRefs, toWritableComputedOptions, remapViewConfig } from './remap.js'; import { isDone, recordDone, recordFailed } from './journal.js'; const UNSUPPORTED_TYPES = new Set(['button', 'asyncText', 'aiText', 'externalSyncSource']); +const VIEW_GROUP_ANCHOR = new Set(['select', 'singleSelect', 'multiSelect', 'multipleSelects', 'collaborator']); const COMPUTED_TYPES = new Set(['formula', 'rollup', 'lookup', 'multipleLookupValues', 'count']); const LINK_TYPES = new Set(['foreignKey', 'multipleRecordLinks']); @@ -260,6 +261,43 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result } return; } + case 'applyViewConfig': { + const destViewId = idmap.views[a.sourceViewId]; + if (!destViewId) { result.skipped++; return; } // view not created (e.g. createView failed) + const cfg = remapViewConfig(a.config || {}, idmap); + const refOk = (id) => !!(id && findDestField(index, id)); + const warnRef = (facet) => result.warnings.push({ code: 'VIEW_UNRESOLVABLE_REF', message: `View ${destViewId} ${facet}: unresolved field ref dropped` }); + + // Anchor validation (PROBE-VERIFIED): kanban stack = groupLevels[0].columnId; calendar = dateColumnRanges startColumnId. + let anchorOk = true; + if (a.type === 'kanban') { const stack = cfg.groupLevels && cfg.groupLevels[0] && cfg.groupLevels[0].columnId; anchorOk = !!(stack && refOk(stack) && VIEW_GROUP_ANCHOR.has(findDestFieldType(index, stack))); } + else if (a.type === 'calendar') anchorOk = !!(cfg.calendar && cfg.calendar.dateColumnRanges && cfg.calendar.dateColumnRanges.every((r) => refOk(r.startColumnId))); + if (!anchorOk) result.warnings.push({ code: 'VIEW_ANCHOR_FALLBACK', message: `View ${destViewId} (${a.type}) missing/incompatible anchor → grid-safe config only` }); + + const tryFacet = async (name, fn) => { try { await fn(); } catch (e) { result.warnings.push({ code: 'VIEW_FACET_FAILED', message: `View ${destViewId} ${name}: ${e.message ?? e}` }); } }; + + // Grid-safe facets (always; drop unresolved-ref ones). + if (cfg.filters) await tryFacet('filters', () => client.updateViewFilters(destAppId, destViewId, cfg.filters)); + if (cfg.sorts && cfg.sorts.length) { if (cfg.sorts.every((s) => refOk(s.columnId))) await tryFacet('sorts', () => client.applySorts(destAppId, destViewId, cfg.sorts)); else warnRef('sorts'); } + if (cfg.groupLevels && cfg.groupLevels.length) { if (cfg.groupLevels.every((g) => refOk(g.columnId))) await tryFacet('groups', () => client.updateGroupLevels(destAppId, destViewId, cfg.groupLevels)); else warnRef('groups'); } + if (cfg.columnOrder && cfg.columnOrder.length) { + const visible = cfg.columnOrder.filter((c) => c.visibility && refOk(c.columnId)).map((c) => c.columnId); + await tryFacet('columns', () => client.setViewColumns(destAppId, destViewId, { visibleColumnIds: visible, frozenColumnCount: cfg.frozenColumnCount })); + } else if (typeof cfg.frozenColumnCount === 'number') { + await tryFacet('frozen', () => client.updateFrozenColumnCount(destAppId, destViewId, cfg.frozenColumnCount)); + } + if (cfg.rowHeight) await tryFacet('rowHeight', () => client.updateRowHeight(destAppId, destViewId, cfg.rowHeight)); + + // Type-specific facets — only when the anchor validated. + if (anchorOk && cfg.colorConfig && refOk(cfg.colorConfig.selectColumnId)) await tryFacet('color', () => client.setViewColorConfig(destAppId, destViewId, cfg.colorConfig)); + if (anchorOk && cfg.cover && refOk(cfg.cover.coverColumnId)) await tryFacet('cover', () => client.setViewCover(destAppId, destViewId, cfg.cover)); + if (anchorOk && cfg.calendar && cfg.calendar.dateColumnRanges && cfg.calendar.dateColumnRanges.every((r) => refOk(r.startColumnId))) await tryFacet('calendar', () => client.setCalendarDateColumns(destAppId, destViewId, cfg.calendar.dateColumnRanges)); + if (cfg.form) await tryFacet('form', () => client.setFormMetadata(destAppId, destViewId, cfg.form)); + + result.updated++; + return; + } + default: throw new Error(`unhandled action kind: ${a.kind}`); } diff --git a/packages/mcp-server/test/sync/test-apply.test.js b/packages/mcp-server/test/sync/test-apply.test.js index a62ade2..19e36b6 100644 --- a/packages/mcp-server/test/sync/test-apply.test.js +++ b/packages/mcp-server/test/sync/test-apply.test.js @@ -384,3 +384,38 @@ describe('apply: createTable then createView in one run (adopt default view)', ( assert.ok(t.views.some((v) => v.name === 'Board')); }); }); + +describe('apply: applyViewConfig', () => { + async function destWithView(type) { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'T'); + const price = await client.createField('appD', tableId, { name: 'Price', type: 'number' }); + const { viewId } = await client.createView('appD', tableId, { name: 'V', type }); + const destSnapshot = await snapshotBase(client, 'appD'); + return { client, tableId, viewId, price: price.columnId, destSnapshot }; + } + const run = (client, plan, destSnapshot) => applyPlan({ client, plan, destAppId: 'appD', destSnapshot, idmap: JSON.parse(JSON.stringify(plan.idmap)), journal: newJournal(plan.planId, 'ts'), persist: () => {} }); + + it('pushes remapped sorts + frozen for a grid view', async () => { + const { client, viewId, price, destSnapshot } = await destWithView('grid'); + const plan = { planId: 'pV1', sourceBaseId: 'appS', destBaseId: 'appD', + idmap: { tables: {}, fields: { fSrcPrice: { destFld: price, choices: {} } }, views: { vSrc: viewId } }, + actions: [{ kind: 'applyViewConfig', sourceTableId: 'tS', sourceViewId: 'vSrc', type: 'grid', + config: { sorts: [{ columnId: 'fSrcPrice', ascending: true }], frozenColumnCount: 1 } }], orphans: [], warnings: [] }; + const res = await run(client, plan, destSnapshot); + assert.equal(res.failed, 0); + assert.ok(client.calls.some((k) => k === `applySorts:${viewId}`)); + assert.deepEqual(client._view(viewId).config.sorts, [{ columnId: price, ascending: true }]); // remapped + }); + + it('kanban whose stack (groupLevels) field is missing in dest → fallback warning, no failure', async () => { + const { client, viewId, destSnapshot } = await destWithView('kanban'); + const plan = { planId: 'pV2', sourceBaseId: 'appS', destBaseId: 'appD', + idmap: { tables: {}, fields: {}, views: { vSrc: viewId } }, + actions: [{ kind: 'applyViewConfig', sourceTableId: 'tS', sourceViewId: 'vSrc', type: 'kanban', + config: { groupLevels: [{ columnId: 'fGONE', order: 'ascending', emptyGroupState: 'hidden' }] } }], orphans: [], warnings: [] }; + const res = await run(client, plan, destSnapshot); + assert.equal(res.failed, 0); + assert.ok(res.warnings.some((w) => w.code === 'VIEW_ANCHOR_FALLBACK' || w.code === 'VIEW_UNRESOLVABLE_REF')); + }); +}); From a5bbc7366edb08db74cfe30ee6f0501af954820b Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 16 Jun 2026 23:59:51 +0300 Subject: [PATCH 046/246] feat(sync): view-aware fingerprint + count view actions in plan report Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/index.js | 3 ++- packages/mcp-server/src/sync/report.js | 2 +- .../test/sync/test-apply-index.test.js | 25 +++++++++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/packages/mcp-server/src/sync/index.js b/packages/mcp-server/src/sync/index.js index 49c98be..4939203 100644 --- a/packages/mcp-server/src/sync/index.js +++ b/packages/mcp-server/src/sync/index.js @@ -17,7 +17,8 @@ const ENGINE_VERSION = '2b'; */ export function fingerprintSchema(snap) { const basis = snap.tables - .map((t) => `${t.id}:${t.name}:` + t.fields.map((f) => `${f.id}=${f.name}=${f.type}`).join(',')) + .map((t) => `${t.id}:${t.name}:` + t.fields.map((f) => `${f.id}=${f.name}=${f.type}`).join(',') + + ';V:' + (t.views || []).map((v) => `${v.id}=${v.name}=${v.type}`).sort().join(',')) .sort() .join('|'); return createHash('sha256').update(basis).digest('hex'); diff --git a/packages/mcp-server/src/sync/report.js b/packages/mcp-server/src/sync/report.js index 15fa76d..ea338e4 100644 --- a/packages/mcp-server/src/sync/report.js +++ b/packages/mcp-server/src/sync/report.js @@ -12,7 +12,7 @@ export function renderPlan(plan) { const counts = {}; for (const a of plan.actions) counts[a.kind] = (counts[a.kind] || 0) + 1; const lines = ['Schema plan:']; - for (const k of ['createTable', 'reconcilePrimary', 'createField', 'updateField']) { + for (const k of ['createTable', 'reconcilePrimary', 'createField', 'updateField', 'createView', 'applyViewConfig']) { if (counts[k]) lines.push(` ${k}: ${counts[k]}`); } lines.push(` orphans: ${plan.orphans.length} (reported, not changed)`); diff --git a/packages/mcp-server/test/sync/test-apply-index.test.js b/packages/mcp-server/test/sync/test-apply-index.test.js index e38ed09..d82eabb 100644 --- a/packages/mcp-server/test/sync/test-apply-index.test.js +++ b/packages/mcp-server/test/sync/test-apply-index.test.js @@ -37,3 +37,28 @@ describe('index.apply', () => { assert.match(out.human, /created: 1/); }); }); + +describe('index: views', () => { + it('fingerprintSchema changes when a table gains a view', () => { + const a = { tables: [{ id: 't1', name: 'T', fields: [{ id: 'f1', name: 'Name', type: 'text' }], views: [{ id: 'v1', name: 'Grid view', type: 'grid' }] }] }; + const b = { tables: [{ id: 't1', name: 'T', fields: [{ id: 'f1', name: 'Name', type: 'text' }], views: [{ id: 'v1', name: 'Grid view', type: 'grid' }, { id: 'v2', name: 'Board', type: 'kanban' }] }] }; + assert.notEqual(fingerprintSchema(a), fingerprintSchema(b)); + }); + + it('apply creates a source-only view end to end (views applied after fields)', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'apply-v-')); + const client = new MockClient(); + const dest = await snapshotBase(client, 'appDDDDDDDDDDDDDD'); // empty base + const plan = { planId: 'plnVI', engineVersion: '2b', destFingerprint: fingerprintSchema(dest), sourceBaseId: 'appSSSSSSSSSSSSSS', destBaseId: 'appDDDDDDDDDDDDDD', idmap: { tables: {}, fields: {}, views: {} }, + actions: [ + { kind: 'createTable', sourceTableId: 'tS', name: 'T' }, + { kind: 'reconcilePrimary', sourceTableId: 'tS', sourcePrimaryFieldId: 'fS1', toName: 'Name', toType: 'text', toTypeOptions: null }, + { kind: 'createView', sourceTableId: 'tS', sourceViewId: 'vNew', name: 'Board', type: 'grid' }, + ], orphans: [], warnings: [] }; + savePlan('appSSSSSSSSSSSSSS', 'appDDDDDDDDDDDDDD', plan); + const out = await apply({ client, sourceBaseId: 'appSSSSSSSSSSSSSS', destBaseId: 'appDDDDDDDDDDDDDD', planId: 'plnVI', runStartedAt: 'ts' }); + assert.match(out.human, /created/); + const views = (await client.getApplicationData('appDDDDDDDDDDDDDD')).data.tableSchemas[0].views; + assert.ok(views.some((v) => v.name === 'Board')); + }); +}); From ba4179a87c50a8617cc14b0651405bdafc6cce78 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 00:02:20 +0300 Subject: [PATCH 047/246] docs(sync): document collaborative view sync in sync_base Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 5 +++++ CLAUDE.md | 2 +- README.md | 2 +- packages/mcp-server/README.md | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc341c9..2fb5bf0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how ## [Unreleased] +### MCP server — sync_base collaborative view sync (2026-06-17) + +#### Added +- `sync_base`: full collaborative-view sync — create views + mirror filters, sorts, group levels, field visibility + column order, frozen columns, color config, cover, calendar date columns, form metadata, and row height, with source→dest field+choice ID remapping, per-view-type anchor validation + grid fallback, applied after fields. Idempotent (convergent canonical compare). Personal views skipped; orphan views reported (not deleted). + ### MCP server — sync_base mode=apply (2026-06-16) #### Added diff --git a/CLAUDE.md b/CLAUDE.md index bd24696..46d5779 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,7 +65,7 @@ Key files: - `dev-tools/` — reverse-engineering helpers (capture.js, analyze.js, debug-*.js), **gitignored** — kept locally for future reference when Airtable changes their internal API - `src/daemon/` — daemon subsystem (see daemon subsection below) - `src/safe-write.js` — atomic JSON write helper (used by lockfile, token, and settings) -- `src/sync/` — base-to-base schema sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: plan | apply`) in the `sync` category. Out of scope: retypes, deletions, records, views. +- `src/sync/` — base-to-base schema sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: plan | apply`) in the `sync` category. Out of scope: retypes, deletions, records. View sync (M3): `snapshot` also captures collaborative views + live config; `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported). #### packages/mcp-server — Daemon subsystem diff --git a/README.md b/README.md index 44d3683..4267fc4 100644 --- a/README.md +++ b/README.md @@ -208,7 +208,7 @@ Manage Airtable bases with capabilities **not available through the official RES | **Form Metadata** | 2 | Description, redirect URL, attribution, copy-to-respondent, branding (legacy form views) | | **Extension Management** | 7 | Create, install, enable/disable, rename, duplicate, remove extensions | | **Tool Management** | 1 | List profiles, switch profile, toggle tools/categories (meta-tool, always enabled) | -| **Base Sync** | 1 | `sync_base` — copy a base's schema to another base. `mode=plan` produces a diff report; `mode=apply` executes it (creates tables/fields with ref remapping, drift-guarded, resumable via journal). Retypes, deletions, records, and views are out of scope. | +| **Base Sync** | 1 | `sync_base` — copy a base's schema to another base. `mode=plan` produces a diff report; `mode=apply` executes it (creates tables/fields with ref remapping, drift-guarded, resumable via journal) and also syncs collaborative views (filters, sorts, groups, column order, frozen, color, cover, calendar, form, row height). Retypes, deletions, and records are out of scope. | See the full tool reference in [`packages/mcp-server/README.md`](packages/mcp-server/README.md). diff --git a/packages/mcp-server/README.md b/packages/mcp-server/README.md index 005fe4d..4205eca 100644 --- a/packages/mcp-server/README.md +++ b/packages/mcp-server/README.md @@ -523,7 +523,7 @@ Saved row scaffolds Airtable surfaces under "+ Add record" and the row-create ex | Tool | Description | |:-----|:------------| -| `sync_base` | Copy a base's schema to another base. `mode=plan` snapshots both bases and produces a diff report (no writes). `mode=apply` executes a saved plan — creates tables (removing Airtable's auto-scaffolding fields), reconciles the primary field, creates scalar/link/computed fields with source→dest reference remapping and dest-space formula validation, and applies non-destructive field updates. Drift-guarded (aborts if the destination changed since the plan was generated) and resumable via an on-disk journal. Out of scope: type-changing retypes, deletions, records, views. | +| `sync_base` | Copy a base's schema to another base. `mode=plan` snapshots both bases and produces a diff report (no writes). `mode=apply` executes a saved plan — creates tables (removing Airtable's auto-scaffolding fields), reconciles the primary field, creates scalar/link/computed fields with source→dest reference remapping and dest-space formula validation, applies non-destructive field updates, and syncs collaborative views (creates missing views + mirrors filters, sorts, group levels, field visibility + column order, frozen columns, color config, cover, calendar date columns, form metadata, and row height, with source→dest field+choice ID remapping). View sync is idempotent (convergent canonical compare); personal views are skipped and orphan destination views are reported (not deleted). Drift-guarded and resumable via an on-disk journal. Out of scope: type-changing retypes, deletions, records. | --- From 814e61ad1c285c1c33642013e6547368d6f304c0 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 00:49:10 +0300 Subject: [PATCH 048/246] fix(client): setViewColumns verify-readback + retry (bulk under-apply of column visibility) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live M3-views smoke: column visibility never converged — a one-shot showOrHideColumns under sustained bulk load returns 200 but leaves columns hidden until the backend settles, so views ended with ~2 visible of 51. setViewColumns now re-reads and retries the un-shown set until confirmed. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/client.js | 28 +++++++++++++++++-- .../test/test-view-columns-retry.test.js | 25 +++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 packages/mcp-server/test/test-view-columns-retry.test.js diff --git a/packages/mcp-server/src/client.js b/packages/mcp-server/src/client.js index 6636232..bd22b57 100644 --- a/packages/mcp-server/src/client.js +++ b/packages/mcp-server/src/client.js @@ -1766,8 +1766,11 @@ export class AirtableClient { // 1. Hide everything. await this.showOrHideAllColumns(appId, viewId, false); - // 2. Show the requested set in one batched call. - await this.showOrHideColumns(appId, viewId, visibleColumnIds, true); + // 2. Show the requested set, verifying via read-back and retrying any columns + // that didn't take. Under a bulk run (100s of views configured back-to-back) + // a single showOrHideColumns can return 200 yet leave columns hidden until the + // backend settles — a one-shot call then silently under-applies. Retry-until-confirmed. + await this._showColumnsWithRetry(appId, viewId, visibleColumnIds); // 3. Identify the primary column (always index 0, immovable) so we can // exclude it from the move call — including it causes FAILED_STATE_CHECK. const view = await this.getView(appId, viewId); @@ -1785,6 +1788,27 @@ export class AirtableClient { return { updated: true, viewId, visibleColumnIds, frozenColumnCount: frozenColumnCount ?? null }; } + /** + * Show the given columns, re-reading the view and retrying any that didn't take. + * Works around the internal API silently under-applying a large showOrHideColumns + * call under sustained bulk load (returns 200 but leaves columns hidden until the + * backend settles). Returns the number of columns confirmed visible. + */ + async _showColumnsWithRetry(appId, viewId, columnIds, maxAttempts = 5) { + const wanted = Array.from(new Set(columnIds)); + let remaining = wanted; + for (let attempt = 0; attempt < maxAttempts; attempt++) { + await this.showOrHideColumns(appId, viewId, remaining, true); + const view = await this.getView(appId, viewId); + const visible = new Set((view.columnOrder || []).filter((c) => c.visibility).map((c) => c.columnId)); + remaining = wanted.filter((id) => !visible.has(id)); + if (remaining.length === 0) return wanted.length; + // Let eventual consistency settle before retrying the columns that didn't take. + await new Promise((resolve) => setTimeout(resolve, 400)); + } + return wanted.length - remaining.length; // best-effort + } + // ─── View Presentation (cover image, color rules, cell wrap) ── // Endpoints captured 2026-04-30 (Recording 3 — Kanban + Gallery). diff --git a/packages/mcp-server/test/test-view-columns-retry.test.js b/packages/mcp-server/test/test-view-columns-retry.test.js new file mode 100644 index 0000000..76b6e24 --- /dev/null +++ b/packages/mcp-server/test/test-view-columns-retry.test.js @@ -0,0 +1,25 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { AirtableClient } from '../src/client.js'; + +describe('setViewColumns: _showColumnsWithRetry', () => { + it('retries columns that did not take on the first show, until read-back confirms all', async () => { + const c = new AirtableClient({}); // dummy auth; helper only uses showOrHideColumns + getView + const shown = new Set(); + let calls = 0; + // Simulate bulk under-apply: first show only takes 1 column; retries take the rest. + c.showOrHideColumns = async (_a, _v, ids) => { calls++; if (calls === 1) shown.add(ids[0]); else ids.forEach((id) => shown.add(id)); return {}; }; + c.getView = async () => ({ columnOrder: ['fA', 'fB', 'fC'].map((id) => ({ columnId: id, visibility: shown.has(id) })) }); + const n = await c._showColumnsWithRetry('app', 'viw', ['fA', 'fB', 'fC']); + assert.equal(n, 3); // all confirmed visible + assert.ok(calls >= 2); // it retried the un-shown set + }); + + it('returns best-effort count if some columns never take (no infinite loop)', async () => { + const c = new AirtableClient({}); + c.showOrHideColumns = async () => ({}); + c.getView = async () => ({ columnOrder: [{ columnId: 'fA', visibility: true }, { columnId: 'fB', visibility: false }] }); // fB never shows + const n = await c._showColumnsWithRetry('app', 'viw', ['fA', 'fB'], 2); + assert.equal(n, 1); // only fA confirmed; bounded by maxAttempts + }); +}); From f4346209e271cab35211c6a2dce49d584be5e602 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 08:14:25 +0300 Subject: [PATCH 049/246] fix(sync): converge view columns -- canonical compares visible order + hidden as a set Apply controls the visible columns (set + order via setViewColumns) but Airtable appends hidden columns in an order we do not control. Comparing the full columnOrder order re-flagged applyViewConfig every plan. Canonicalize columns as { visible: ordered, hidden: sorted set } to match what apply achieves. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/remap.js | 9 ++++++++- packages/mcp-server/test/sync/test-remap-view.test.js | 11 +++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/mcp-server/src/sync/remap.js b/packages/mcp-server/src/sync/remap.js index dcfdd8c..c3d3d09 100644 --- a/packages/mcp-server/src/sync/remap.js +++ b/packages/mcp-server/src/sync/remap.js @@ -132,7 +132,14 @@ export function canonicalizeViewConfig(config, fldNames, selNames) { filters: c.filters ? { conj: c.filters.conjunction, set: canonFilterSet(c.filters.filterSet || [], fldNames, selNames) } : null, sorts: (c.sorts || []).map((s) => ({ col: viewNameOf(fldNames, s.columnId), asc: s.ascending })), groups: (c.groupLevels || []).map((g) => ({ col: viewNameOf(fldNames, g.columnId), order: g.order })), - columns: (c.columnOrder || []).map((co) => ({ col: viewNameOf(fldNames, co.columnId), vis: co.visibility })), + // Compare what apply actually controls: the VISIBLE columns in order, plus which + // columns are hidden as an order-agnostic set. Apply sets the visible set + order + // (setViewColumns), but Airtable appends hidden columns in an order we don't control — + // so comparing the full columnOrder order would re-flag forever (never converge). + columns: { + visible: (c.columnOrder || []).filter((co) => co.visibility).map((co) => viewNameOf(fldNames, co.columnId)), + hidden: (c.columnOrder || []).filter((co) => !co.visibility).map((co) => viewNameOf(fldNames, co.columnId)).sort(), + }, frozen: c.frozenColumnCount ?? null, color: c.colorConfig ? { type: c.colorConfig.type, col: viewNameOf(fldNames, c.colorConfig.selectColumnId) } : null, cover: c.cover ? { col: viewNameOf(fldNames, c.cover.coverColumnId), fit: c.cover.coverFitType } : null, diff --git a/packages/mcp-server/test/sync/test-remap-view.test.js b/packages/mcp-server/test/sync/test-remap-view.test.js index 29897eb..8f681bb 100644 --- a/packages/mcp-server/test/sync/test-remap-view.test.js +++ b/packages/mcp-server/test/sync/test-remap-view.test.js @@ -56,3 +56,14 @@ describe('remap.canonicalizeViewConfig', () => { assert.notEqual(a, b); }); }); + +describe('remap.canonicalizeViewConfig columns convergence', () => { + const names = { fA: 'A', fB: 'B', fC: 'C', fD: 'D' }; + it('ignores hidden-column ORDER (apply cannot control it) but keeps visible order', () => { + const x = canonicalizeViewConfig({ columnOrder: [{ columnId: 'fA', visibility: true }, { columnId: 'fB', visibility: true }, { columnId: 'fC', visibility: false }, { columnId: 'fD', visibility: false }] }, names, {}); + const y = canonicalizeViewConfig({ columnOrder: [{ columnId: 'fA', visibility: true }, { columnId: 'fB', visibility: true }, { columnId: 'fD', visibility: false }, { columnId: 'fC', visibility: false }] }, names, {}); + assert.equal(x, y); // hidden order differs → still equal → converges + const z = canonicalizeViewConfig({ columnOrder: [{ columnId: 'fB', visibility: true }, { columnId: 'fA', visibility: true }, { columnId: 'fC', visibility: false }, { columnId: 'fD', visibility: false }] }, names, {}); + assert.notEqual(x, z); // different VISIBLE order → not equal + }); +}); From fe9b032b4dcee97bc7255b4b65852ca0fa6cd04a Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 08:28:05 +0300 Subject: [PATCH 050/246] fix(sync): converge view columns on visible/hidden SET, not order (order best-effort) The internal API reorder (moveVisibleColumns) does not reliably reproduce the source left-to-right order under bulk, so comparing exact order re-flagged 66 views forever. Visibility IS reliably applied (setViewColumns verify-retry); compare which columns are shown/hidden as sets. Column order is best-effort. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/remap.js | 10 +++++----- packages/mcp-server/test/sync/test-remap-view.test.js | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/mcp-server/src/sync/remap.js b/packages/mcp-server/src/sync/remap.js index c3d3d09..fdda236 100644 --- a/packages/mcp-server/src/sync/remap.js +++ b/packages/mcp-server/src/sync/remap.js @@ -132,12 +132,12 @@ export function canonicalizeViewConfig(config, fldNames, selNames) { filters: c.filters ? { conj: c.filters.conjunction, set: canonFilterSet(c.filters.filterSet || [], fldNames, selNames) } : null, sorts: (c.sorts || []).map((s) => ({ col: viewNameOf(fldNames, s.columnId), asc: s.ascending })), groups: (c.groupLevels || []).map((g) => ({ col: viewNameOf(fldNames, g.columnId), order: g.order })), - // Compare what apply actually controls: the VISIBLE columns in order, plus which - // columns are hidden as an order-agnostic set. Apply sets the visible set + order - // (setViewColumns), but Airtable appends hidden columns in an order we don't control — - // so comparing the full columnOrder order would re-flag forever (never converge). + // Compare WHICH columns are visible vs hidden (each as an order-agnostic set), not their + // left-to-right order. Apply reliably sets visibility (setViewColumns verify-retry) but the + // internal API's reorder is unreliable under bulk, so comparing exact order would re-flag + // forever. Column ORDER is therefore best-effort (applied, not gated on convergence). columns: { - visible: (c.columnOrder || []).filter((co) => co.visibility).map((co) => viewNameOf(fldNames, co.columnId)), + visible: (c.columnOrder || []).filter((co) => co.visibility).map((co) => viewNameOf(fldNames, co.columnId)).sort(), hidden: (c.columnOrder || []).filter((co) => !co.visibility).map((co) => viewNameOf(fldNames, co.columnId)).sort(), }, frozen: c.frozenColumnCount ?? null, diff --git a/packages/mcp-server/test/sync/test-remap-view.test.js b/packages/mcp-server/test/sync/test-remap-view.test.js index 8f681bb..23ce5f6 100644 --- a/packages/mcp-server/test/sync/test-remap-view.test.js +++ b/packages/mcp-server/test/sync/test-remap-view.test.js @@ -59,11 +59,11 @@ describe('remap.canonicalizeViewConfig', () => { describe('remap.canonicalizeViewConfig columns convergence', () => { const names = { fA: 'A', fB: 'B', fC: 'C', fD: 'D' }; - it('ignores hidden-column ORDER (apply cannot control it) but keeps visible order', () => { + it('ignores column ORDER (visible + hidden); only WHICH columns are shown/hidden gates convergence', () => { const x = canonicalizeViewConfig({ columnOrder: [{ columnId: 'fA', visibility: true }, { columnId: 'fB', visibility: true }, { columnId: 'fC', visibility: false }, { columnId: 'fD', visibility: false }] }, names, {}); - const y = canonicalizeViewConfig({ columnOrder: [{ columnId: 'fA', visibility: true }, { columnId: 'fB', visibility: true }, { columnId: 'fD', visibility: false }, { columnId: 'fC', visibility: false }] }, names, {}); - assert.equal(x, y); // hidden order differs → still equal → converges - const z = canonicalizeViewConfig({ columnOrder: [{ columnId: 'fB', visibility: true }, { columnId: 'fA', visibility: true }, { columnId: 'fC', visibility: false }, { columnId: 'fD', visibility: false }] }, names, {}); - assert.notEqual(x, z); // different VISIBLE order → not equal + const shuffled = canonicalizeViewConfig({ columnOrder: [{ columnId: 'fB', visibility: true }, { columnId: 'fA', visibility: true }, { columnId: 'fD', visibility: false }, { columnId: 'fC', visibility: false }] }, names, {}); + assert.equal(x, shuffled); // same visible/hidden SETS, different order → equal → converges + const differentVisible = canonicalizeViewConfig({ columnOrder: [{ columnId: 'fA', visibility: true }, { columnId: 'fC', visibility: true }, { columnId: 'fB', visibility: false }, { columnId: 'fD', visibility: false }] }, names, {}); + assert.notEqual(x, differentVisible); // C shown instead of B → different visible set → not equal }); }); From a0172af0e4fa65584a35cbc3f3400410f9f1d7d6 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 08:36:28 +0300 Subject: [PATCH 051/246] fix(sync): exclude record-based colorConfig (colorDefinitions) from view convergence Last stuck view: a colorConfig of type colorDefinitions whose filters reference specific record ids (records are out of scope). apply correctly skips it, but the canonical compare still flagged it forever. Only the select-driven colour rule is syncable; exclude colorDefinitions from the compare. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/remap.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/mcp-server/src/sync/remap.js b/packages/mcp-server/src/sync/remap.js index fdda236..4247a3f 100644 --- a/packages/mcp-server/src/sync/remap.js +++ b/packages/mcp-server/src/sync/remap.js @@ -141,7 +141,9 @@ export function canonicalizeViewConfig(config, fldNames, selNames) { hidden: (c.columnOrder || []).filter((co) => !co.visibility).map((co) => viewNameOf(fldNames, co.columnId)).sort(), }, frozen: c.frozenColumnCount ?? null, - color: c.colorConfig ? { type: c.colorConfig.type, col: viewNameOf(fldNames, c.colorConfig.selectColumnId) } : null, + // Only the select-driven colour rule is syncable; 'colorDefinitions' (conditional rules + // whose filters reference specific RECORD ids) are out of scope — exclude so they don't re-flag. + color: (c.colorConfig && c.colorConfig.type === 'selectColumn') ? { col: viewNameOf(fldNames, c.colorConfig.selectColumnId) } : null, cover: c.cover ? { col: viewNameOf(fldNames, c.cover.coverColumnId), fit: c.cover.coverFitType } : null, calendar: c.calendar ? (c.calendar.dateColumnRanges || []).map((r) => ({ s: viewNameOf(fldNames, r.startColumnId), e: r.endColumnId ? viewNameOf(fldNames, r.endColumnId) : null })) : null, rowHeight: c.rowHeight ?? null, From 5295e4948a24d32c515aca2a616f5d5f9ea7d349 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 08:56:16 +0300 Subject: [PATCH 052/246] fix(sync): clear stray dest view facets source lacks (filters/sorts/groups convergence) applyViewConfig only pushed filters/sorts/groups when the SOURCE had them, so a pre-existing dest filter/sort/group the source lacks was never cleared -> the view re-flagged on every re-plan (last non-converging view in the 117-view live base). Always push these facets to match source, clearing the dest when source is empty, and treat an empty filterSet as "no filter" in canonicalizeViewConfig so a cleared dest compares equal to a source with none. Live base now converges to 0 view actions. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/apply.js | 8 +++++--- packages/mcp-server/src/sync/remap.js | 2 +- packages/mcp-server/test/sync/test-remap-view.test.js | 7 +++++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/mcp-server/src/sync/apply.js b/packages/mcp-server/src/sync/apply.js index c41c744..882b2ff 100644 --- a/packages/mcp-server/src/sync/apply.js +++ b/packages/mcp-server/src/sync/apply.js @@ -277,9 +277,11 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result } const tryFacet = async (name, fn) => { try { await fn(); } catch (e) { result.warnings.push({ code: 'VIEW_FACET_FAILED', message: `View ${destViewId} ${name}: ${e.message ?? e}` }); } }; // Grid-safe facets (always; drop unresolved-ref ones). - if (cfg.filters) await tryFacet('filters', () => client.updateViewFilters(destAppId, destViewId, cfg.filters)); - if (cfg.sorts && cfg.sorts.length) { if (cfg.sorts.every((s) => refOk(s.columnId))) await tryFacet('sorts', () => client.applySorts(destAppId, destViewId, cfg.sorts)); else warnRef('sorts'); } - if (cfg.groupLevels && cfg.groupLevels.length) { if (cfg.groupLevels.every((g) => refOk(g.columnId))) await tryFacet('groups', () => client.updateGroupLevels(destAppId, destViewId, cfg.groupLevels)); else warnRef('groups'); } + // Always push filters/sorts/groups to MATCH source — clearing the dest when source has + // none (a stray dest filter/sort/group source lacks would otherwise never converge). + await tryFacet('filters', () => client.updateViewFilters(destAppId, destViewId, cfg.filters || { filterSet: [], conjunction: 'and' })); + if (!cfg.sorts || cfg.sorts.every((s) => refOk(s.columnId))) await tryFacet('sorts', () => client.applySorts(destAppId, destViewId, cfg.sorts || [])); else warnRef('sorts'); + if (!cfg.groupLevels || cfg.groupLevels.every((g) => refOk(g.columnId))) await tryFacet('groups', () => client.updateGroupLevels(destAppId, destViewId, cfg.groupLevels || [])); else warnRef('groups'); if (cfg.columnOrder && cfg.columnOrder.length) { const visible = cfg.columnOrder.filter((c) => c.visibility && refOk(c.columnId)).map((c) => c.columnId); await tryFacet('columns', () => client.setViewColumns(destAppId, destViewId, { visibleColumnIds: visible, frozenColumnCount: cfg.frozenColumnCount })); diff --git a/packages/mcp-server/src/sync/remap.js b/packages/mcp-server/src/sync/remap.js index 4247a3f..4cf2fee 100644 --- a/packages/mcp-server/src/sync/remap.js +++ b/packages/mcp-server/src/sync/remap.js @@ -129,7 +129,7 @@ function canonFilterSet(set, fldNames, selNames) { export function canonicalizeViewConfig(config, fldNames, selNames) { const c = config || {}; return JSON.stringify({ - filters: c.filters ? { conj: c.filters.conjunction, set: canonFilterSet(c.filters.filterSet || [], fldNames, selNames) } : null, + filters: (c.filters && Array.isArray(c.filters.filterSet) && c.filters.filterSet.length) ? { conj: c.filters.conjunction, set: canonFilterSet(c.filters.filterSet, fldNames, selNames) } : null, sorts: (c.sorts || []).map((s) => ({ col: viewNameOf(fldNames, s.columnId), asc: s.ascending })), groups: (c.groupLevels || []).map((g) => ({ col: viewNameOf(fldNames, g.columnId), order: g.order })), // Compare WHICH columns are visible vs hidden (each as an order-agnostic set), not their diff --git a/packages/mcp-server/test/sync/test-remap-view.test.js b/packages/mcp-server/test/sync/test-remap-view.test.js index 23ce5f6..a306cd2 100644 --- a/packages/mcp-server/test/sync/test-remap-view.test.js +++ b/packages/mcp-server/test/sync/test-remap-view.test.js @@ -55,6 +55,13 @@ describe('remap.canonicalizeViewConfig', () => { const b = canonicalizeViewConfig({ sorts: [{ columnId: 'fldB', ascending: true }] }, { fldB: 'Qty' }, {}); assert.notEqual(a, b); }); + it('an empty filterSet canonicalizes equal to no filters (apply clears stray dest filters → convergence)', () => { + const noFilter = canonicalizeViewConfig({ filters: null }, {}, {}); + const emptyFilter = canonicalizeViewConfig({ filters: { conjunction: 'and', filterSet: [] } }, {}, {}); + assert.equal(noFilter, emptyFilter); + const realFilter = canonicalizeViewConfig({ filters: { conjunction: 'and', filterSet: [{ columnId: 'fldA', operator: 'contains', value: null }] } }, { fldA: 'ID' }, {}); + assert.notEqual(noFilter, realFilter); // a genuine filter still differs from none + }); }); describe('remap.canonicalizeViewConfig columns convergence', () => { From c96e19215e35e02567ca2424116e8dedb398eed1 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 10:06:10 +0300 Subject: [PATCH 053/246] fix(sync): strip+report record-referencing view filters (no dangling refs, converges) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit View filters on link/collaborator fields carry raw rec…/usr… ids as their value. remapFilterSet only rewrote sel… choice ids, so source record ids were written verbatim into the dest filter, where no such record exists -> dangling ref / error, and it never converged (canonical couldn't name-resolve a record id). The ~40 per-game "🎮 " views in the test base each hit this (Game = [recXXX]). Strip record-ref filter clauses before write (Option A): detect by VALUE SHAPE (/^(rec|usr).../ ids + structured {tableId,columnId,rowId} dynamic values), which needs no source field-type plumbing (those types aren't carried into apply). Recursive strip + bottom-up prune of emptied groups; portable collaborator "me" sentinel is kept. Emit one VIEW_UNRESOLVABLE_RECORD_REF warning per affected view (direction-neutral: row set will differ from source). canonFilterSet mirrors the strip and unwraps singleton groups (collapse-agnostic). colorConfig.colorDefinitions (record-referencing conditional colour rules) defensively dropped from the pushed config. Asymmetric canonical (source stripped, dest raw) makes a dest still holding a dangling rec filter from a prior buggy sync diverge -> one cleanup apply -> then converge. Live: 68 views cleaned (0 failed, 68 warnings), re-plan = 0 view actions. Written map-aware so M3-records can later remap (instead of strip) once records sync. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/apply.js | 4 +- packages/mcp-server/src/sync/diff.js | 2 +- packages/mcp-server/src/sync/remap.js | 93 ++++++++++++++++--- .../test/sync/test-remap-view.test.js | 66 ++++++++++++- 4 files changed, 148 insertions(+), 17 deletions(-) diff --git a/packages/mcp-server/src/sync/apply.js b/packages/mcp-server/src/sync/apply.js index 882b2ff..a329f29 100644 --- a/packages/mcp-server/src/sync/apply.js +++ b/packages/mcp-server/src/sync/apply.js @@ -2,7 +2,7 @@ // Live-first + growing idmap + journal (resume) + existence-check (idempotency). // POST-SPIKE: primary defaults to Name/text; createTable spawns 6 default fields → // delete the 5 non-primary scaffolding fields (D1); links are foreignKey (later task). -import { remapRefs, toWritableComputedOptions, remapViewConfig } from './remap.js'; +import { remapRefs, toWritableComputedOptions, remapViewConfig, collectFilterRecordRefs } from './remap.js'; import { isDone, recordDone, recordFailed } from './journal.js'; const UNSUPPORTED_TYPES = new Set(['button', 'asyncText', 'aiText', 'externalSyncSource']); @@ -265,6 +265,8 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result } const destViewId = idmap.views[a.sourceViewId]; if (!destViewId) { result.skipped++; return; } // view not created (e.g. createView failed) const cfg = remapViewConfig(a.config || {}, idmap); + const strippedRefs = collectFilterRecordRefs(a.config || {}); + if (strippedRefs.length) result.warnings.push({ code: 'VIEW_UNRESOLVABLE_RECORD_REF', message: `View ${destViewId} filters: ${strippedRefs.length} record/collaborator-ref clause(s) dropped (records not synced; row set will differ from source)` }); const refOk = (id) => !!(id && findDestField(index, id)); const warnRef = (facet) => result.warnings.push({ code: 'VIEW_UNRESOLVABLE_REF', message: `View ${destViewId} ${facet}: unresolved field ref dropped` }); diff --git a/packages/mcp-server/src/sync/diff.js b/packages/mcp-server/src/sync/diff.js index 010a8e9..a544436 100644 --- a/packages/mcp-server/src/sync/diff.js +++ b/packages/mcp-server/src/sync/diff.js @@ -378,7 +378,7 @@ export function computePlan(srcSnap, destSnap, idmap) { if (!dv) { viewActions.push({ kind: 'createView', sourceTableId: st.id, sourceViewId: sv.id, name: sv.name, type: sv.type }); viewActions.push({ kind: 'applyViewConfig', sourceTableId: st.id, sourceViewId: sv.id, type: sv.type, config: sv.config || {} }); - } else if (canonicalizeViewConfig(sv.config || {}, srcNames, srcSelNames) !== canonicalizeViewConfig(dv.config || {}, destNames, destSelNames)) { + } else if (canonicalizeViewConfig(sv.config || {}, srcNames, srcSelNames, true) !== canonicalizeViewConfig(dv.config || {}, destNames, destSelNames, false)) { viewActions.push({ kind: 'applyViewConfig', sourceTableId: st.id, sourceViewId: sv.id, type: sv.type, config: sv.config || {} }); } } diff --git a/packages/mcp-server/src/sync/remap.js b/packages/mcp-server/src/sync/remap.js index 4cf2fee..71ff94c 100644 --- a/packages/mcp-server/src/sync/remap.js +++ b/packages/mcp-server/src/sync/remap.js @@ -97,14 +97,53 @@ function destSelId(idmap, src) { for (const v of Object.values(idmap.fields || {})) { const d = ((v && v.choices) || {})[src]; if (d) return d; } return src; } +// A filter LEAF whose value references a record/collaborator id — or a structured/dynamic +// value carrying SOURCE field/table/row ids — cannot be resolved in the dest: records and +// users are not synced (and ids differ across bases regardless). Detect by value SHAPE, which +// needs NO source field-type context (those types aren't carried into apply). Portable +// sentinels (current-user "me", null, booleans) are not id-shaped → kept. Forward-compat: once +// records sync (M3), a populated idmap.records lets these REMAP instead of strip — see canon note. +const RECORD_REF_ID = /^(rec|usr)[A-Za-z0-9]{14,}$/; +function isRecordRefValue(value) { + if (typeof value === 'string') return RECORD_REF_ID.test(value); + if (Array.isArray(value)) return value.some((v) => typeof v === 'string' && RECORD_REF_ID.test(v)); + if (value && typeof value === 'object') return 'columnId' in value || 'rowId' in value || 'tableId' in value; + return false; +} +function collectRecordRefIds(set) { + const ids = []; + for (const f of set || []) { + if (f.filterSet) { ids.push(...collectRecordRefIds(f.filterSet)); continue; } + if (!isRecordRefValue(f.value)) continue; + if (typeof f.value === 'string') ids.push(f.value); + else if (Array.isArray(f.value)) ids.push(...f.value.filter((v) => typeof v === 'string' && RECORD_REF_ID.test(v))); + else ids.push(''); + } + return ids; +} +// Source record/collaborator ids that view-filter strip will drop (for the apply-side warning). +export function collectFilterRecordRefs(config) { + return (config && config.filters && Array.isArray(config.filters.filterSet)) ? collectRecordRefIds(config.filters.filterSet) : []; +} +// Remap resolvable refs (fld/choice ids) and STRIP unresolvable record/collaborator-ref leaves, +// pruning groups emptied by stripping (bottom-up). Mirrored exactly by canonFilterSet so a +// stripped source filter canonicalizes identically to the stripped dest readback → converges. function remapFilterSet(set, idmap) { - return set.map((f) => { - if (f.filterSet) return { ...(f.type ? { type: f.type } : {}), conjunction: f.conjunction, filterSet: remapFilterSet(f.filterSet, idmap) }; - const out = { columnId: destFldId(idmap, f.columnId), operator: f.operator, value: f.value }; - if (typeof f.value === 'string') out.value = destSelId(idmap, f.value); - else if (Array.isArray(f.value)) out.value = f.value.map((v) => (typeof v === 'string' ? destSelId(idmap, v) : v)); - return out; - }); + const out = []; + for (const f of set) { + if (f.filterSet) { + const inner = remapFilterSet(f.filterSet, idmap); + if (inner.length === 0) continue; + out.push({ ...(f.type ? { type: f.type } : {}), conjunction: f.conjunction, filterSet: inner }); + continue; + } + if (isRecordRefValue(f.value)) continue; + const o = { columnId: destFldId(idmap, f.columnId), operator: f.operator, value: f.value }; + if (typeof f.value === 'string') o.value = destSelId(idmap, f.value); + else if (Array.isArray(f.value)) o.value = f.value.map((v) => (typeof v === 'string' ? destSelId(idmap, v) : v)); + out.push(o); + } + return out; } export function remapViewConfig(config, idmap) { if (config == null || typeof config !== 'object') return config; @@ -113,7 +152,13 @@ export function remapViewConfig(config, idmap) { if (Array.isArray(c.sorts)) c.sorts = c.sorts.map((s) => ({ columnId: destFldId(idmap, s.columnId), ascending: s.ascending })); if (Array.isArray(c.groupLevels)) c.groupLevels = c.groupLevels.map((g) => ({ columnId: destFldId(idmap, g.columnId), order: g.order, emptyGroupState: g.emptyGroupState })); if (Array.isArray(c.columnOrder)) c.columnOrder = c.columnOrder.map((co) => ({ columnId: destFldId(idmap, co.columnId), visibility: co.visibility })); - if (c.colorConfig && c.colorConfig.selectColumnId) c.colorConfig = { ...c.colorConfig, selectColumnId: destFldId(idmap, c.colorConfig.selectColumnId) }; + if (c.colorConfig) { + const cc = { ...c.colorConfig }; + if (cc.selectColumnId) cc.selectColumnId = destFldId(idmap, cc.selectColumnId); + // Conditional colour rules filter on specific RECORD ids — can't remap, would leak/error. Drop defensively. + delete cc.colorDefinitions; + c.colorConfig = cc; + } if (c.cover && c.cover.coverColumnId) c.cover = { ...c.cover, coverColumnId: destFldId(idmap, c.cover.coverColumnId) }; if (c.calendar && Array.isArray(c.calendar.dateColumnRanges)) c.calendar = { dateColumnRanges: c.calendar.dateColumnRanges.map((r) => ({ startColumnId: destFldId(idmap, r.startColumnId), ...(r.endColumnId ? { endColumnId: destFldId(idmap, r.endColumnId) } : {}) })) }; return c; @@ -121,15 +166,35 @@ export function remapViewConfig(config, idmap) { // Canonical, id-free, name-based string for a convergent diff compare (ids→names; auto-ids/width dropped). function viewNameOf(map, id) { return map[id] ?? id; } -function canonFilterSet(set, fldNames, selNames) { - return set.map((f) => f.filterSet - ? { c: f.conjunction, n: canonFilterSet(f.filterSet, fldNames, selNames) } - : { col: viewNameOf(fldNames, f.columnId), op: f.operator, val: typeof f.value === 'string' ? viewNameOf(selNames, f.value) : f.value }); +// Mirror remapFilterSet: strip record/collaborator-ref leaves and prune emptied groups so a +// source filter canonicalizes identically to the stripped dest readback. Also UNWRAP singleton +// groups (collapse-agnostic): Airtable may store a 1-element nested group as a bare leaf on +// readback — unwrapping on BOTH sides keeps it convergent either way. (Records aren't synced, so +// rec-ref leaves drop on both sides; once they do sync, this is where name-resolution lands — M3.) +function canonFilterSet(set, fldNames, selNames, strip) { + const out = []; + for (const f of set) { + if (f.filterSet) { + const inner = canonFilterSet(f.filterSet, fldNames, selNames, strip); + if (inner.length === 0) continue; + if (inner.length === 1) { out.push(inner[0]); continue; } + out.push({ c: f.conjunction, n: inner }); + continue; + } + if (strip && isRecordRefValue(f.value)) continue; + out.push({ col: viewNameOf(fldNames, f.columnId), op: f.operator, val: typeof f.value === 'string' ? viewNameOf(selNames, f.value) : f.value }); + } + return out; } -export function canonicalizeViewConfig(config, fldNames, selNames) { +// stripRecordRefs: TRUE for the SOURCE side (canonicalize as what apply will WRITE — rec/collab +// leaves dropped), FALSE for the DEST side (its raw actual filter). Asymmetry is deliberate: it +// makes a dest that still holds a dangling rec filter (from a prior buggy sync) diverge from the +// stripped source → one cleanup apply → then both read empty/equal → converges. +export function canonicalizeViewConfig(config, fldNames, selNames, stripRecordRefs = true) { const c = config || {}; + const fset = (c.filters && Array.isArray(c.filters.filterSet)) ? canonFilterSet(c.filters.filterSet, fldNames, selNames, stripRecordRefs) : []; return JSON.stringify({ - filters: (c.filters && Array.isArray(c.filters.filterSet) && c.filters.filterSet.length) ? { conj: c.filters.conjunction, set: canonFilterSet(c.filters.filterSet, fldNames, selNames) } : null, + filters: fset.length ? { conj: c.filters.conjunction, set: fset } : null, sorts: (c.sorts || []).map((s) => ({ col: viewNameOf(fldNames, s.columnId), asc: s.ascending })), groups: (c.groupLevels || []).map((g) => ({ col: viewNameOf(fldNames, g.columnId), order: g.order })), // Compare WHICH columns are visible vs hidden (each as an order-agnostic set), not their diff --git a/packages/mcp-server/test/sync/test-remap-view.test.js b/packages/mcp-server/test/sync/test-remap-view.test.js index a306cd2..96bbedb 100644 --- a/packages/mcp-server/test/sync/test-remap-view.test.js +++ b/packages/mcp-server/test/sync/test-remap-view.test.js @@ -1,6 +1,6 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { remapViewConfig, canonicalizeViewConfig } from '../../src/sync/remap.js'; +import { remapViewConfig, canonicalizeViewConfig, collectFilterRecordRefs } from '../../src/sync/remap.js'; const idmap = { tables: {}, views: {}, fields: { fldA: { destFld: 'fldX', choices: { selA: 'selX' } }, @@ -64,6 +64,70 @@ describe('remap.canonicalizeViewConfig', () => { }); }); +describe('remap — record-referencing view filters (strip + report + converge)', () => { + const REC = 'recAAAAAAAAAAAAAA', REC2 = 'recBBBBBBBBBBBBBB', USR = 'usrCCCCCCCCCCCCCC'; + it('strips a link-field rec leaf and reports it', () => { + const cfg = { filters: { conjunction: 'and', filterSet: [{ id: 'flt1', columnId: 'fldA', operator: '=', value: [REC] }] } }; + const out = remapViewConfig(cfg, idmap); + assert.equal(out.filters.filterSet.length, 0); // rec leaf dropped before write + assert.deepEqual(collectFilterRecordRefs(cfg), [REC]); // surfaced for the warning + }); + it('keeps resolvable (choice) leaves while stripping rec leaves in the same set', () => { + const cfg = { filters: { conjunction: 'and', filterSet: [ + { columnId: 'fldA', operator: '=', value: [REC] }, + { columnId: 'fldB', operator: '=', value: 'selA' }, + ] } }; + const out = remapViewConfig(cfg, idmap); + assert.equal(out.filters.filterSet.length, 1); + assert.equal(out.filters.filterSet[0].columnId, 'fldY'); // fldB remapped + assert.equal(out.filters.filterSet[0].value, 'selX'); // choice remapped, leaf kept + }); + it('prunes a nested group emptied by stripping; keeps siblings', () => { + const cfg = { filters: { conjunction: 'and', filterSet: [ + { type: 'nested', conjunction: 'or', filterSet: [{ columnId: 'fldA', operator: '=', value: [REC2] }] }, + { columnId: 'fldB', operator: '=', value: 'x' }, + ] } }; + const out = remapViewConfig(cfg, idmap); + assert.equal(out.filters.filterSet.length, 1); // emptied nested group pruned + assert.equal(out.filters.filterSet[0].columnId, 'fldY'); + }); + it('strips collaborator usr ids but keeps the portable "me" sentinel', () => { + const cfg = { filters: { conjunction: 'and', filterSet: [ + { columnId: 'fldB', operator: '=', value: USR }, + { columnId: 'fldB', operator: '=', value: 'me' }, + ] } }; + const out = remapViewConfig(cfg, idmap); + assert.equal(out.filters.filterSet.length, 1); + assert.equal(out.filters.filterSet[0].value, 'me'); + }); + it('strips structured/dynamic filter values that carry source ids', () => { + const cfg = { filters: { conjunction: 'and', filterSet: [{ columnId: 'fldA', operator: '|', value: { tableId: 'tblZ', columnId: 'fldZ', rowId: null } }] } }; + assert.equal(remapViewConfig(cfg, idmap).filters.filterSet.length, 0); + }); + it('a source filter of only rec leaves canonicalizes equal to no filter (converges)', () => { + const src = canonicalizeViewConfig({ filters: { conjunction: 'and', filterSet: [{ columnId: 'fldA', operator: '=', value: [REC] }] } }, { fldA: 'Game' }, {}); + const none = canonicalizeViewConfig({ filters: null }, {}, {}); + assert.equal(src, none); + }); + it('drops record-referencing colorDefinitions defensively (never written)', () => { + const out = remapViewConfig({ colorConfig: { type: 'colorDefinitions', colorDefinitions: [{ filterSet: [], color: 'blue' }], defaultColor: 'gray' } }, idmap); + assert.equal(out.colorConfig.colorDefinitions, undefined); + }); + it('dest still holding a dangling rec filter diverges from stripped source (forces cleanup), then converges once cleared', () => { + const srcCfg = { filters: { conjunction: 'and', filterSet: [{ columnId: 'fldA', operator: '=', value: [REC] }] } }; + const srcCanon = canonicalizeViewConfig(srcCfg, { fldA: 'Game' }, {}, true); // source: stripped → no filter + const destDangling = canonicalizeViewConfig(srcCfg, { fldA: 'Game' }, {}, false); // dest raw: keeps the dangling rec + assert.notEqual(srcCanon, destDangling); // → emits applyViewConfig (cleanup) + const destCleared = canonicalizeViewConfig({ filters: null }, {}, {}, false); // after apply clears it + assert.equal(srcCanon, destCleared); // → converged, no re-flag + }); + it('unwraps singleton nested groups in canonical (collapse-agnostic convergence)', () => { + const grouped = canonicalizeViewConfig({ filters: { conjunction: 'and', filterSet: [{ type: 'nested', conjunction: 'and', filterSet: [{ columnId: 'fldA', operator: '=', value: 'selA' }] }] } }, { fldA: 'X' }, { selA: 'A' }); + const flat = canonicalizeViewConfig({ filters: { conjunction: 'and', filterSet: [{ columnId: 'fldA', operator: '=', value: 'selA' }] } }, { fldA: 'X' }, { selA: 'A' }); + assert.equal(grouped, flat); + }); +}); + describe('remap.canonicalizeViewConfig columns convergence', () => { const names = { fA: 'A', fB: 'B', fC: 'C', fD: 'D' }; it('ignores column ORDER (visible + hidden); only WHICH columns are shown/hidden gates convergence', () => { From 7bb49745de678a23272f54202699b0117a4b19e0 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 12:25:45 +0300 Subject: [PATCH 054/246] feat(sync): record snapshot via queryRecords summary.rows Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/snapshot.js | 12 ++++++++++ .../test/sync/test-records-snapshot.test.js | 24 +++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 packages/mcp-server/test/sync/test-records-snapshot.test.js diff --git a/packages/mcp-server/src/sync/snapshot.js b/packages/mcp-server/src/sync/snapshot.js index a3970a9..6c0ee4c 100644 --- a/packages/mcp-server/src/sync/snapshot.js +++ b/packages/mcp-server/src/sync/snapshot.js @@ -98,6 +98,18 @@ export async function snapshotBase(client, appId) { return snap; } +/** + * Pull a table's records via its first collaborative view (single call, ≤1000 rows; + * the internal readQueries endpoint has no cursor — Task-11 pre-flight warns on >1000). + * @returns {Promise>} + */ +export async function snapshotTableRecords(client, appId, table) { + const view = (table.views || []).find((v) => !v.personalForUserId) || (table.views || [])[0]; + if (!view) return []; + const res = await client.queryRecords(appId, table.id, view.id, { limit: 1000 }); + return (res?.summary?.rows || []).map((r) => ({ id: r.id, cellValuesByColumnId: r.fields || {} })); +} + /** * @typedef {{ id: string, name: string, type: string, typeOptions: object|null, description: string|null, isComputed: boolean }} NormalizedField * @typedef {{ id: string, name: string, primaryFieldId: string|null, fields: NormalizedField[] }} NormalizedTable diff --git a/packages/mcp-server/test/sync/test-records-snapshot.test.js b/packages/mcp-server/test/sync/test-records-snapshot.test.js new file mode 100644 index 0000000..29ca08a --- /dev/null +++ b/packages/mcp-server/test/sync/test-records-snapshot.test.js @@ -0,0 +1,24 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { snapshotTableRecords } from '../../src/sync/snapshot.js'; + +describe('snapshot.snapshotTableRecords', () => { + it('maps summary.rows {id, fields} → {id, cellValuesByColumnId} via the first collaborative view', async () => { + let seenView = null; + const client = { + queryRecords: async (appId, tableId, viewId, opts) => { + seenView = viewId; + assert.equal(opts.limit, 1000); + return { summary: { rows: [{ id: 'rec1', fields: { fldA: 'x' } }, { id: 'rec2', fields: { fldA: 'y' } }], count: 2 } }; + }, + }; + const table = { id: 'tbl1', views: [{ id: 'viwP', personalForUserId: 'usr1' }, { id: 'viw1', name: 'Grid' }] }; + const out = await snapshotTableRecords(client, 'app1', table); + assert.equal(seenView, 'viw1'); // skipped the personal view + assert.deepEqual(out, [{ id: 'rec1', cellValuesByColumnId: { fldA: 'x' } }, { id: 'rec2', cellValuesByColumnId: { fldA: 'y' } }]); + }); + it('returns [] when the table has no view', async () => { + const out = await snapshotTableRecords({ queryRecords: async () => ({ summary: { rows: [] } }) }, 'app1', { id: 't', views: [] }); + assert.deepEqual(out, []); + }); +}); From 7977d00a2e84cbb44804c2f3714ce60ecfdc9690 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 12:31:07 +0300 Subject: [PATCH 055/246] feat(sync): persist records map in idmap Add records and views slots to idmap default (loadIdmap returns { tables: {}, fields: {}, records: {}, views: {} }). Export mergeIdmapsForTest so on-resume idmap merging is testable. Persisted map wins on conflict (has grown with prior run results). Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/idmap.js | 8 +- packages/mcp-server/src/sync/index.js | 24 +++++- .../test/sync/test-records-idmap.test.js | 82 +++++++++++++++++++ 3 files changed, 106 insertions(+), 8 deletions(-) create mode 100644 packages/mcp-server/test/sync/test-records-idmap.test.js diff --git a/packages/mcp-server/src/sync/idmap.js b/packages/mcp-server/src/sync/idmap.js index 2d0e3ff..4bdf74b 100644 --- a/packages/mcp-server/src/sync/idmap.js +++ b/packages/mcp-server/src/sync/idmap.js @@ -110,19 +110,19 @@ export function saveIdmap(sourceBaseId, destBaseId, idmap) { } /** - * Load a previously saved ID-map. Returns `{ tables: {}, fields: {} }` when + * Load a previously saved ID-map. Returns `{ tables: {}, fields: {}, records: {} }` when * the file is absent or unparseable. * @param {string} sourceBaseId * @param {string} destBaseId - * @returns {{ tables: Record, fields: Record }} + * @returns {{ tables: Record, fields: Record, records: Record }} */ export function loadIdmap(sourceBaseId, destBaseId) { const p = join(syncDir(sourceBaseId, destBaseId), 'idmap.json'); - if (!existsSync(p)) return { tables: {}, fields: {} }; + if (!existsSync(p)) return { tables: {}, fields: {}, records: {} }; try { return JSON.parse(readFileSync(p, 'utf8')); } catch { - return { tables: {}, fields: {} }; + return { tables: {}, fields: {}, records: {} }; } } diff --git a/packages/mcp-server/src/sync/index.js b/packages/mcp-server/src/sync/index.js index 4939203..37ea7bc 100644 --- a/packages/mcp-server/src/sync/index.js +++ b/packages/mcp-server/src/sync/index.js @@ -63,7 +63,14 @@ export async function apply({ client, sourceBaseId, destBaseId, planId, runStart } const journal = loadJournal(sourceBaseId, destBaseId, planId) ?? newJournal(planId, runStartedAt); - const idmap = journal.actions.length > 0 ? mergeIdmaps(sourceBaseId, destBaseId, fullPlan) : JSON.parse(JSON.stringify(fullPlan.idmap)); + let idmap; + if (journal.actions.length > 0) { + idmap = mergeIdmaps(sourceBaseId, destBaseId, fullPlan); + } else { + idmap = JSON.parse(JSON.stringify(fullPlan.idmap)); + idmap.records ??= {}; + idmap.views ??= {}; + } const result = await applyPlan({ client, plan: fullPlan, destAppId: destBaseId, destSnapshot, idmap, journal, @@ -74,8 +81,17 @@ export async function apply({ client, sourceBaseId, destBaseId, planId, runStart } // On resume, merge the persisted (grown) idmap over the plan's base matches so this-run -// creations from a prior crashed run survive. -function mergeIdmaps(sourceBaseId, destBaseId, fullPlan) { +// creations from a prior crashed run survive. Persisted wins on key conflict (has grown). +export function mergeIdmapsForTest(sourceBaseId, destBaseId, fullPlan) { const m = loadIdmap(sourceBaseId, destBaseId); - return { tables: { ...fullPlan.idmap.tables, ...m.tables }, fields: { ...fullPlan.idmap.fields, ...m.fields } }; + return { + tables: { ...fullPlan.idmap.tables, ...m.tables }, + fields: { ...fullPlan.idmap.fields, ...m.fields }, + records: { ...fullPlan.idmap.records, ...m.records }, + views: { ...fullPlan.idmap.views, ...m.views }, + }; +} + +function mergeIdmaps(sourceBaseId, destBaseId, fullPlan) { + return mergeIdmapsForTest(sourceBaseId, destBaseId, fullPlan); } diff --git a/packages/mcp-server/test/sync/test-records-idmap.test.js b/packages/mcp-server/test/sync/test-records-idmap.test.js new file mode 100644 index 0000000..d3310ad --- /dev/null +++ b/packages/mcp-server/test/sync/test-records-idmap.test.js @@ -0,0 +1,82 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { join } from 'node:path'; +import { mkdtempSync, mkdirSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { loadIdmap, saveIdmap } from '../../src/sync/idmap.js'; +import { mergeIdmapsForTest } from '../../src/sync/index.js'; + +describe('idmap records slot', () => { + it('loadIdmap defaults to { tables: {}, fields: {}, records: {} } when file missing', () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-test-')); + const m = loadIdmap('appAAAA', 'appBBBB'); + assert.deepEqual(m, { tables: {}, fields: {}, records: {} }); + }); + + it('loadIdmap defaults to { tables: {}, fields: {}, records: {} } on parse error', () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-test-')); + const homeDir = process.env.AIRTABLE_USER_MCP_HOME; + const syncPath = join(homeDir, 'sync', 'appAAAA__appBBBB'); + // Save broken JSON + mkdirSync(syncPath, { recursive: true }); + writeFileSync(join(syncPath, 'idmap.json'), 'not valid json {'); + const m = loadIdmap('appAAAA', 'appBBBB'); + assert.deepEqual(m, { tables: {}, fields: {}, records: {} }); + }); + + it('mergeIdmaps merges records with persisted winning on conflict', () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-test-')); + const homeDir = process.env.AIRTABLE_USER_MCP_HOME; + const syncPath = join(homeDir, 'sync', 'appAAAA__appBBBB'); + + // Write persisted idmap with records + mkdirSync(syncPath, { recursive: true }); + const persistedMap = { + tables: {}, + fields: {}, + records: { a: 'X', b: '2' } + }; + writeFileSync(join(syncPath, 'idmap.json'), JSON.stringify(persistedMap)); + + // Plan has different records + const planIdmap = { + tables: {}, + fields: {}, + records: { a: '1', c: '3' } + }; + const fullPlan = { idmap: planIdmap }; + + const merged = mergeIdmapsForTest('appAAAA', 'appBBBB', fullPlan); + + // Persisted should win on 'a', keep 'b', and plan contributes 'c' + assert.deepEqual(merged.records, { a: 'X', b: '2', c: '3' }); + }); + + it('mergeIdmaps also merges views with persisted winning', () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-test-')); + const homeDir = process.env.AIRTABLE_USER_MCP_HOME; + const syncPath = join(homeDir, 'sync', 'appAAAA__appBBBB'); + + mkdirSync(syncPath, { recursive: true }); + const persistedMap = { + tables: {}, + fields: {}, + records: {}, + views: { vS1: 'vD_old', vS2: 'vD2' } + }; + writeFileSync(join(syncPath, 'idmap.json'), JSON.stringify(persistedMap)); + + const planIdmap = { + tables: {}, + fields: {}, + records: {}, + views: { vS1: 'vD_new', vS3: 'vD3' } + }; + const fullPlan = { idmap: planIdmap }; + + const merged = mergeIdmapsForTest('appAAAA', 'appBBBB', fullPlan); + + // Persisted wins on vS1, keeps vS2, plan contributes vS3 + assert.deepEqual(merged.views, { vS1: 'vD_old', vS2: 'vD2', vS3: 'vD3' }); + }); +}); From 695e0413be76d8a6f1af84c32671fdabd623ec0f Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 12:34:15 +0300 Subject: [PATCH 056/246] feat(sync): cells.js pass-1 scalar/select coercion + writable gate Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/cells.js | 25 ++++++++++++++ .../mcp-server/test/sync/test-cells.test.js | 33 +++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 packages/mcp-server/src/sync/cells.js create mode 100644 packages/mcp-server/test/sync/test-cells.test.js diff --git a/packages/mcp-server/src/sync/cells.js b/packages/mcp-server/src/sync/cells.js new file mode 100644 index 0000000..990390e --- /dev/null +++ b/packages/mcp-server/src/sync/cells.js @@ -0,0 +1,25 @@ +import { isComputedType } from './snapshot.js'; + +const ARRAY_DEFERRED = new Set(['multipleRecordLinks', 'multipleAttachments']); + +export function isWritableForRecords(field) { + return !isComputedType(field.type) && !ARRAY_DEFERRED.has(field.type); +} + +function choiceMap(field, idmap) { return (idmap.fields?.[field.id]?.choices) || {}; } + +export function coercePass1Cell(field, srcValue, idmap) { + if (!isWritableForRecords(field)) return { write: false }; + if (srcValue == null) return { write: true, value: srcValue }; + if (field.type === 'select') { + const d = choiceMap(field, idmap)[srcValue]; + return d ? { write: true, value: d } : { write: false }; + } + if (field.type === 'multiSelect') { + const cm = choiceMap(field, idmap); + const mapped = (Array.isArray(srcValue) ? srcValue : []).map((s) => cm[s]).filter(Boolean); + if (mapped.length !== (srcValue?.length ?? 0)) return { write: false }; // partial choice map → skip + report upstream + return { write: true, value: mapped }; + } + return { write: true, value: srcValue }; // text/number/currency/date/checkbox/... +} diff --git a/packages/mcp-server/test/sync/test-cells.test.js b/packages/mcp-server/test/sync/test-cells.test.js new file mode 100644 index 0000000..e14d48a --- /dev/null +++ b/packages/mcp-server/test/sync/test-cells.test.js @@ -0,0 +1,33 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { isWritableForRecords, coercePass1Cell } from '../../src/sync/cells.js'; + +const idmap = { fields: { fldSel: { destFld: 'fldSelD', choices: { selA: 'selAD', selB: 'selBD' } } } }; + +describe('cells.isWritableForRecords', () => { + it('false for computed + link + attachment, true for scalar/select', () => { + assert.equal(isWritableForRecords({ type: 'formula' }), false); + assert.equal(isWritableForRecords({ type: 'multipleRecordLinks' }), false); + assert.equal(isWritableForRecords({ type: 'multipleAttachments' }), false); + assert.equal(isWritableForRecords({ type: 'text' }), true); + assert.equal(isWritableForRecords({ type: 'select' }), true); + }); +}); +describe('cells.coercePass1Cell', () => { + it('passes scalars through', () => { + assert.deepEqual(coercePass1Cell({ id: 'fldT', type: 'text' }, 'hi', idmap), { write: true, value: 'hi' }); + }); + it('remaps a single-select choice id', () => { + assert.deepEqual(coercePass1Cell({ id: 'fldSel', type: 'select' }, 'selA', idmap), { write: true, value: 'selAD' }); + }); + it('remaps a multi-select choice array', () => { + assert.deepEqual(coercePass1Cell({ id: 'fldSel', type: 'multiSelect' }, ['selA', 'selB'], idmap), { write: true, value: ['selAD', 'selBD'] }); + }); + it('does not write link / attachment / computed cells in pass 1', () => { + assert.equal(coercePass1Cell({ id: 'fldL', type: 'multipleRecordLinks' }, ['rec1'], idmap).write, false); + assert.equal(coercePass1Cell({ id: 'fldF', type: 'formula' }, 5, idmap).write, false); + }); + it('drops an unmappable choice id (reports via write:false)', () => { + assert.equal(coercePass1Cell({ id: 'fldSel', type: 'select' }, 'selUNKNOWN', idmap).write, false); + }); +}); From 9d75144e1486c1b774cb0193e19e163e47084d56 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 12:37:40 +0300 Subject: [PATCH 057/246] feat(sync): cells.js link partition (resolved/unresolved) Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/cells.js | 9 +++++++++ packages/mcp-server/test/sync/test-cells.test.js | 11 ++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/mcp-server/src/sync/cells.js b/packages/mcp-server/src/sync/cells.js index 990390e..a4a23df 100644 --- a/packages/mcp-server/src/sync/cells.js +++ b/packages/mcp-server/src/sync/cells.js @@ -23,3 +23,12 @@ export function coercePass1Cell(field, srcValue, idmap) { } return { write: true, value: srcValue }; // text/number/currency/date/checkbox/... } + +export function partitionLinkValue(srcValue, idmap) { + const recs = idmap.records || {}; + const resolved = [], unresolved = []; + for (const s of (Array.isArray(srcValue) ? srcValue : [])) { + if (recs[s]) resolved.push(recs[s]); else unresolved.push(s); + } + return { resolved, unresolved }; +} diff --git a/packages/mcp-server/test/sync/test-cells.test.js b/packages/mcp-server/test/sync/test-cells.test.js index e14d48a..c164714 100644 --- a/packages/mcp-server/test/sync/test-cells.test.js +++ b/packages/mcp-server/test/sync/test-cells.test.js @@ -1,6 +1,6 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { isWritableForRecords, coercePass1Cell } from '../../src/sync/cells.js'; +import { isWritableForRecords, coercePass1Cell, partitionLinkValue } from '../../src/sync/cells.js'; const idmap = { fields: { fldSel: { destFld: 'fldSelD', choices: { selA: 'selAD', selB: 'selBD' } } } }; @@ -31,3 +31,12 @@ describe('cells.coercePass1Cell', () => { assert.equal(coercePass1Cell({ id: 'fldSel', type: 'select' }, 'selUNKNOWN', idmap).write, false); }); }); +describe('cells.partitionLinkValue', () => { + const m = { records: { recS1: 'recD1', recS2: 'recD2' } }; + it('splits resolved vs unresolved by the record map', () => { + assert.deepEqual(partitionLinkValue(['recS1', 'recS2', 'recX'], m), { resolved: ['recD1', 'recD2'], unresolved: ['recX'] }); + }); + it('null/empty → empty partition', () => { + assert.deepEqual(partitionLinkValue(null, m), { resolved: [], unresolved: [] }); + }); +}); From a0f14fa37b2ffc9b433075e106bd11224d625e96 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 12:41:08 +0300 Subject: [PATCH 058/246] feat(sync): ratelimit token-bucket + transient retry Injectable clock/rand keeps the live path on real Date.now/Math.random while tests inject sleep/rand/now stubs and rps:1000 for zero real delay. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/ratelimit.js | 94 +++++++++ .../test/sync/test-ratelimit.test.js | 188 ++++++++++++++++++ 2 files changed, 282 insertions(+) create mode 100644 packages/mcp-server/src/sync/ratelimit.js create mode 100644 packages/mcp-server/test/sync/test-ratelimit.test.js diff --git a/packages/mcp-server/src/sync/ratelimit.js b/packages/mcp-server/src/sync/ratelimit.js new file mode 100644 index 0000000..8652e65 --- /dev/null +++ b/packages/mcp-server/src/sync/ratelimit.js @@ -0,0 +1,94 @@ +/** + * ratelimit.js — token-bucket serializer + transient retry with exponential backoff + * + * Clock and randomness are injectable so the live path uses real Date.now/Math.random + * while tests inject deterministic stubs and pass rps:1000 to eliminate real delays. + */ + +const realSleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +/** + * Returns true for errors that are safe to retry: + * - HTTP 429 (rate limit) + * - HTTP 5xx (server errors) + * - Network-level failures (connection refused, timeout, fetch failure, etc.) + */ +export function defaultIsTransient(err) { + const s = err && err.status; + if (s === 429 || (s >= 500 && s < 600)) return true; + return /network|fetch|ECONN|ETIMEDOUT|socket/i.test(String(err && err.message)); +} + +/** + * Retry `fn` on transient errors with exponential backoff + jitter. + * + * @param {() => Promise} fn - The operation to retry + * @param {object} opts + * @param {number} opts.retries - Max retries (default 4); after which the last error is rethrown + * @param {(ms:number)=>Promise} opts.sleep - Sleep fn (default: real setTimeout) + * @param {()=>number} opts.rand - Random source for jitter (default: Math.random) + * @param {(err:any)=>boolean} opts.isTransient - Predicate (default: defaultIsTransient) + * @param {number} opts.baseMs - Backoff base in ms (default 500) + * @param {number} opts.penaltyMs - Floor for 429 when no retryAfterMs (default 30000) + * @returns {Promise} + */ +export async function withRetry( + fn, + { + retries = 4, + sleep = realSleep, + rand = Math.random, + isTransient = defaultIsTransient, + baseMs = 500, + penaltyMs = 30000, + } = {}, +) { + let attempt = 0; + for (;;) { + try { + return await fn(); + } catch (err) { + if (attempt >= retries || !isTransient(err)) throw err; + // Honor explicit retryAfterMs; fall back to 429 penalty or plain backoff + const retryAfterMs = err.retryAfterMs ?? (err.status === 429 ? penaltyMs : 0); + const backoff = baseMs * 2 ** attempt + Math.floor(rand() * baseMs); + await sleep(Math.max(retryAfterMs, backoff)); + attempt++; + } + } +} + +/** + * Creates a rate-limiting queue that serializes tasks to at most `rps` calls/second. + * + * Tasks are chained on a single promise so they always run one-at-a-time in submission + * order. A rejected task leaves the chain intact so subsequent tasks still run. + * + * @param {object} opts + * @param {number} opts.rps - Max requests per second (default 5) + * @param {(ms:number)=>Promise} opts.sleep - Sleep fn (default: real setTimeout) + * @param {()=>number} opts.now - Monotonic clock in ms (default: Date.now) + * @returns {{ run: (fn: () => Promise) => Promise }} + */ +export function createLimiter({ rps = 5, sleep = realSleep, now = () => Date.now() } = {}) { + const minGap = 1000 / rps; // minimum ms between task starts + let chain = Promise.resolve(); + let lastStart = 0; // timestamp of the most recent task start + + const run = (fn) => { + // Capture the result promise separately from the chain so: + // - `run()` callers get the real result (or rejection) from `fn` + // - `chain` swallows rejections to keep the queue alive + const p = chain.then(async () => { + const wait = minGap - (now() - lastStart); + if (wait > 0) await sleep(wait); + lastStart = now(); + return fn(); + }); + // Keep chain alive even when a task rejects + chain = p.catch(() => {}); + return p; + }; + + return { run }; +} diff --git a/packages/mcp-server/test/sync/test-ratelimit.test.js b/packages/mcp-server/test/sync/test-ratelimit.test.js new file mode 100644 index 0000000..29a3b5d --- /dev/null +++ b/packages/mcp-server/test/sync/test-ratelimit.test.js @@ -0,0 +1,188 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { withRetry, createLimiter, defaultIsTransient } from '../../src/sync/ratelimit.js'; + +describe('ratelimit.withRetry', () => { + it('retries a transient 429 then succeeds', async () => { + let n = 0; + const slept = []; + const out = await withRetry( + async () => { + if (n++ < 2) { + const e = new Error('rate limited'); + e.status = 429; + throw e; + } + return 'ok'; + }, + { retries: 4, sleep: async (ms) => slept.push(ms), rand: () => 0, baseMs: 100, penaltyMs: 200 }, + ); + assert.equal(out, 'ok'); + assert.equal(n, 3); // called 3 times: fail, fail, succeed + assert.equal(slept.length, 2); // slept once per retry (2 retries) + }); + + it('retries a transient 503 then succeeds', async () => { + let n = 0; + const slept = []; + const out = await withRetry( + async () => { + if (n++ < 1) { + const e = new Error('server error'); + e.status = 503; + throw e; + } + return 'data'; + }, + { retries: 4, sleep: async (ms) => slept.push(ms), rand: () => 0, baseMs: 100 }, + ); + assert.equal(out, 'data'); + assert.equal(n, 2); + assert.equal(slept.length, 1); + }); + + it('rethrows a non-transient error immediately (no sleep)', async () => { + let n = 0; + const slept = []; + await assert.rejects( + () => + withRetry( + async () => { + n++; + const e = new Error('bad request'); + e.status = 422; + throw e; + }, + { retries: 4, sleep: async (ms) => slept.push(ms), rand: () => 0 }, + ), + /bad request/, + ); + assert.equal(n, 1); // only called once + assert.equal(slept.length, 0); // no sleep + }); + + it('rethrows after exhausting retries', async () => { + let n = 0; + await assert.rejects( + () => + withRetry( + async () => { + n++; + const e = new Error('always fails'); + e.status = 503; + throw e; + }, + { retries: 2, sleep: async () => {}, rand: () => 0, baseMs: 10 }, + ), + /always fails/, + ); + assert.equal(n, 3); // initial + 2 retries + }); + + it('honors err.retryAfterMs over backoff', async () => { + let n = 0; + const slept = []; + await withRetry( + async () => { + if (n++ < 1) { + const e = new Error('rate'); + e.status = 429; + e.retryAfterMs = 9999; + throw e; + } + return 'ok'; + }, + { retries: 2, sleep: async (ms) => slept.push(ms), rand: () => 0, baseMs: 10, penaltyMs: 1 }, + ); + // retryAfterMs=9999 should win over backoff(10*1+0=10) and penaltyMs(1) + assert.equal(slept[0], 9999); + }); + + it('returns value immediately when fn succeeds on first try', async () => { + const slept = []; + const out = await withRetry(async () => 42, { + retries: 3, + sleep: async (ms) => slept.push(ms), + rand: () => 0, + }); + assert.equal(out, 42); + assert.equal(slept.length, 0); + }); +}); + +describe('ratelimit.defaultIsTransient', () => { + it('returns true for 429', () => { + assert.equal(defaultIsTransient({ status: 429 }), true); + }); + it('returns true for 5xx', () => { + assert.equal(defaultIsTransient({ status: 500 }), true); + assert.equal(defaultIsTransient({ status: 503 }), true); + assert.equal(defaultIsTransient({ status: 599 }), true); + }); + it('returns false for 4xx non-429', () => { + assert.equal(defaultIsTransient({ status: 422 }), false); + assert.equal(defaultIsTransient({ status: 400 }), false); + assert.equal(defaultIsTransient({ status: 404 }), false); + }); + it('returns true for network errors by message', () => { + assert.equal(defaultIsTransient({ message: 'network error' }), true); + assert.equal(defaultIsTransient({ message: 'ECONNREFUSED' }), true); + assert.equal(defaultIsTransient({ message: 'ETIMEDOUT' }), true); + assert.equal(defaultIsTransient({ message: 'fetch failed' }), true); + assert.equal(defaultIsTransient({ message: 'socket hang up' }), true); + }); + it('returns false for non-network non-5xx errors', () => { + assert.equal(defaultIsTransient({ message: 'bad input', status: 400 }), false); + }); +}); + +describe('ratelimit.createLimiter', () => { + it('runs all tasks and preserves results/order', async () => { + const lim = createLimiter({ rps: 1000, sleep: async () => {} }); + const out = await Promise.all([1, 2, 3].map((x) => lim.run(async () => x * 2))); + assert.deepEqual(out, [2, 4, 6]); + }); + + it('a rejected task does not break the queue for subsequent tasks', async () => { + const lim = createLimiter({ rps: 1000, sleep: async () => {} }); + const results = []; + await Promise.allSettled([ + lim.run(async () => { throw new Error('task1 fails'); }), + lim.run(async () => { results.push('task2'); return 'task2'; }), + lim.run(async () => { results.push('task3'); return 'task3'; }), + ]); + assert.deepEqual(results, ['task2', 'task3']); + }); + + it('serializes tasks (each awaits the previous)', async () => { + const order = []; + // Use a fake sleep that records calls so we can verify ordering + const fakeSleep = async (ms) => { order.push(`sleep:${ms}`); }; + // High rps so minGap ~0, tasks run sequentially but don't actually wait + const lim = createLimiter({ rps: 1000, sleep: fakeSleep, now: (() => { let t = 0; return () => t += 10; })() }); + const r = await Promise.all([ + lim.run(async () => { order.push('a'); return 1; }), + lim.run(async () => { order.push('b'); return 2; }), + lim.run(async () => { order.push('c'); return 3; }), + ]); + assert.deepEqual(r, [1, 2, 3]); + // Tasks must run in order a→b→c + const tasks = order.filter((x) => !x.startsWith('sleep:')); + assert.deepEqual(tasks, ['a', 'b', 'c']); + }); + + it('spacing: enforces minGap between tasks when now advances slowly', async () => { + const slept = []; + // now() returns same value each call → wait = minGap - 0 = minGap each time + const lim = createLimiter({ + rps: 2, // minGap = 500ms + sleep: async (ms) => slept.push(ms), + now: () => 0, // always returns 0 → gap is always 0 → wait = minGap - 0 = 500 + }); + await Promise.all([1, 2].map((x) => lim.run(async () => x))); + // First task: last=0, now=0, wait=500-0=500, but first run has last=0 too + // Actually first task: wait = 500 - (0 - 0) = 500. Second: same. + // Both tasks sleep 500ms + assert.ok(slept.every((ms) => ms >= 499), `expected >= 499ms sleeps, got ${JSON.stringify(slept)}`); + }); +}); From 0ec4768bb3ac83f35b8e6f04fce0bd8f04880cba Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 12:44:20 +0300 Subject: [PATCH 059/246] feat(sync): records journal file (distinct from schema journal) Add saveRecordsJournal/loadRecordsJournal with separate records-journal-{planId}.json file, mirroring the existing saveJournal/loadJournal pattern. Reuses pure helpers (newJournal, recordDone, recordFailed, isDone) to avoid duplication. Records phase can now maintain its own journal independently from schema sync without file collision. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/journal.js | 12 ++++++++++++ .../test/sync/test-records-journal.test.js | 18 ++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 packages/mcp-server/test/sync/test-records-journal.test.js diff --git a/packages/mcp-server/src/sync/journal.js b/packages/mcp-server/src/sync/journal.js index eed3754..5d7f3d8 100644 --- a/packages/mcp-server/src/sync/journal.js +++ b/packages/mcp-server/src/sync/journal.js @@ -6,6 +6,9 @@ import { safeAtomicWriteFileSync } from '../safe-write.js'; function journalPath(sourceBaseId, destBaseId, planId) { return join(syncDir(sourceBaseId, destBaseId), `journal-${planId}.json`); } +function recordsJournalPath(sourceBaseId, destBaseId, planId) { + return join(syncDir(sourceBaseId, destBaseId), `records-journal-${planId}.json`); +} export function newJournal(planId, startedAt) { return { planId, startedAt, actions: [] }; } @@ -31,3 +34,12 @@ export function loadJournal(sourceBaseId, destBaseId, planId) { if (!existsSync(p)) return null; try { return JSON.parse(readFileSync(p, 'utf8')); } catch { return null; } } +export function saveRecordsJournal(sourceBaseId, destBaseId, journal) { + mkdirSync(syncDir(sourceBaseId, destBaseId), { recursive: true }); + safeAtomicWriteFileSync(recordsJournalPath(sourceBaseId, destBaseId, journal.planId), JSON.stringify(journal, null, 2)); +} +export function loadRecordsJournal(sourceBaseId, destBaseId, planId) { + const p = recordsJournalPath(sourceBaseId, destBaseId, planId); + if (!existsSync(p)) return null; + try { return JSON.parse(readFileSync(p, 'utf8')); } catch { return null; } +} diff --git a/packages/mcp-server/test/sync/test-records-journal.test.js b/packages/mcp-server/test/sync/test-records-journal.test.js new file mode 100644 index 0000000..092b3df --- /dev/null +++ b/packages/mcp-server/test/sync/test-records-journal.test.js @@ -0,0 +1,18 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { join } from 'node:path'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { newJournal, recordDone, saveRecordsJournal, loadRecordsJournal } from '../../src/sync/journal.js'; + +describe('records-journal', () => { + it('saveRecordsJournal writes records-journal-{planId}.json (distinct from journal)', () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'rjrnl-')); + const j = newJournal('plnZ', '2026-06-16T00:00:00Z'); + recordDone(j, 0, 'createRecord', 'recX'); + saveRecordsJournal('appAAAAAAAAAAAAAA', 'appBBBBBBBBBBBBBB', j); + const back = loadRecordsJournal('appAAAAAAAAAAAAAA', 'appBBBBBBBBBBBBBB', 'plnZ'); + assert.equal(back.actions[0].destId, 'recX'); + assert.equal(loadRecordsJournal('appAAAAAAAAAAAAAA', 'appBBBBBBBBBBBBBB', 'nope'), null); + }); +}); From d1d9f2bcaf002b5f8099994ecbce5b1332cf4d8d Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 12:48:25 +0300 Subject: [PATCH 060/246] =?UTF-8?q?fix(test):=20expand=20records-journal?= =?UTF-8?q?=20test=20=E2=80=94=20distinct=20filename,=20env=20cleanup,=20f?= =?UTF-8?q?ull=20round-trip?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add assertions for distinct filename (records-journal-{planId}.json exists, journal-{planId}.json does NOT), restore AIRTABLE_USER_MCP_HOME env var in after() hook, and expand round-trip to verify planId, startedAt, and full actions array. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../test/sync/test-records-journal.test.js | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/packages/mcp-server/test/sync/test-records-journal.test.js b/packages/mcp-server/test/sync/test-records-journal.test.js index 092b3df..26d8ab9 100644 --- a/packages/mcp-server/test/sync/test-records-journal.test.js +++ b/packages/mcp-server/test/sync/test-records-journal.test.js @@ -1,18 +1,40 @@ -import { describe, it } from 'node:test'; +import { describe, it, after } from 'node:test'; import assert from 'node:assert/strict'; import { join } from 'node:path'; -import { mkdtempSync } from 'node:fs'; +import { mkdtempSync, existsSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { newJournal, recordDone, saveRecordsJournal, loadRecordsJournal } from '../../src/sync/journal.js'; +import { syncDir } from '../../src/sync/idmap.js'; describe('records-journal', () => { + const originalHome = process.env.AIRTABLE_USER_MCP_HOME; + + after(() => { + process.env.AIRTABLE_USER_MCP_HOME = originalHome; + }); + it('saveRecordsJournal writes records-journal-{planId}.json (distinct from journal)', () => { process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'rjrnl-')); const j = newJournal('plnZ', '2026-06-16T00:00:00Z'); recordDone(j, 0, 'createRecord', 'recX'); saveRecordsJournal('appAAAAAAAAAAAAAA', 'appBBBBBBBBBBBBBB', j); + + // Assert distinct filename: records-journal exists + const recordsJournalPath = join(syncDir('appAAAAAAAAAAAAAA', 'appBBBBBBBBBBBBBB'), 'records-journal-plnZ.json'); + assert.ok(existsSync(recordsJournalPath), 'records-journal-plnZ.json should exist'); + + // Assert distinct filename: schema journal does NOT exist + const schemaJournalPath = join(syncDir('appAAAAAAAAAAAAAA', 'appBBBBBBBBBBBBBB'), 'journal-plnZ.json'); + assert.equal(existsSync(schemaJournalPath), false, 'journal-plnZ.json should NOT exist for records journal'); + + // Round-trip: verify full journal data const back = loadRecordsJournal('appAAAAAAAAAAAAAA', 'appBBBBBBBBBBBBBB', 'plnZ'); + assert.equal(back.planId, 'plnZ'); + assert.equal(back.startedAt, '2026-06-16T00:00:00Z'); + assert.deepEqual(back.actions, j.actions); assert.equal(back.actions[0].destId, 'recX'); + + // Assert null on missing journal assert.equal(loadRecordsJournal('appAAAAAAAAAAAAAA', 'appBBBBBBBBBBBBBB', 'nope'), null); }); }); From 3283b806c3ac9faedd4dfe716bea9283a0253205 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 12:55:09 +0300 Subject: [PATCH 061/246] feat(sync): records Pass 1 (scalar/select upsert + fill record map) Implements applyRecordsPass1 in packages/mcp-server/src/sync/records.js. CREATE path: builds cellValuesByColumnId for all writable cells (scalar, select, multiSelect) via coercePass1Cell, calls createRecords, fills idmap.records[srcId]=rowId. UPDATE path: skips multiSelect arrays (updateRecords limitation) with RECORD_ARRAY_UPDATE_DEFERRED warning; sends scalar + single-select only. All calls go through limiter.run(withRetry). Journaled per batch; continue-on-failure throughout. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/records.js | 200 ++++++++++++++++ .../mcp-server/test/sync/test-records.test.js | 214 ++++++++++++++++++ 2 files changed, 414 insertions(+) create mode 100644 packages/mcp-server/src/sync/records.js create mode 100644 packages/mcp-server/test/sync/test-records.test.js diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js new file mode 100644 index 0000000..4359843 --- /dev/null +++ b/packages/mcp-server/src/sync/records.js @@ -0,0 +1,200 @@ +/** + * records.js — Pass 1: sync scalar + select/multiSelect cells. + * + * For each matched table (those with idmap.tables[srcTableId]): + * CREATE path — source record not yet in idmap.records → createRecords, fill idmap.records. + * UPDATE path — source record already mapped → updateRecords (primitive + single-select ONLY; + * multiSelect arrays are deferred with a RECORD_ARRAY_UPDATE_DEFERRED warning). + * + * All client calls go through `limiter.run(() => withRetry(() => ...))`. + * Continue-on-failure: per-table errors are caught and pushed as warnings. + * `persist(idmap, journal)` is called after each batch. + */ + +import { coercePass1Cell } from './cells.js'; +import { withRetry } from './ratelimit.js'; + +/** + * Returns true if the field type is a multiSelect (arrays not accepted by updateRecords). + * @param {string} type + * @returns {boolean} + */ +function isMultiSelect(type) { + return type === 'multiSelect' || type === 'multipleSelects'; +} + +/** + * Build the per-record cell map for the CREATE path. + * Includes all writable cells (scalar, select, multiSelect). + * Pushes RECORD_CELL_SKIPPED warnings for unmappable choices. + * + * @param {Array} srcFields - source table field descriptors + * @param {object} srcCells - source record cellValuesByColumnId + * @param {object} idmap - id-map (fields + choices) + * @param {Array} warnings - result.warnings array (mutated) + * @param {string} srcRecId - source record id (for warning context) + * @returns {object} - dest cellValuesByColumnId + */ +function buildCreateCells(srcFields, srcCells, idmap, warnings, srcRecId) { + const cells = {}; + for (const field of srcFields) { + const srcVal = srcCells[field.id]; + if (srcVal === undefined) continue; + const mapping = idmap.fields[field.id]; + if (!mapping) continue; // field not mapped → skip + const coerced = coercePass1Cell(field, srcVal, idmap); + if (!coerced.write) { + // Only warn for select/multiSelect that failed choice mapping (not for structural skip) + if (field.type === 'select' || field.type === 'multiSelect' || field.type === 'multipleSelects') { + warnings.push({ + code: 'RECORD_CELL_SKIPPED', + message: `Record ${srcRecId}: field ${field.id} (${field.type}) choice could not be mapped to dest — cell skipped`, + }); + } + continue; + } + cells[mapping.destFld] = coerced.value; + } + return cells; +} + +/** + * Build the per-record cell map for the UPDATE path. + * Excludes multiSelect arrays (not supported by updateRecords). + * Pushes RECORD_ARRAY_UPDATE_DEFERRED warnings for skipped multiSelect cells. + * + * @param {Array} srcFields - source table field descriptors + * @param {object} srcCells - source record cellValuesByColumnId + * @param {object} idmap - id-map (fields + choices) + * @param {Array} warnings - result.warnings array (mutated) + * @param {string} srcRecId - source record id (for warning context) + * @returns {object} - dest cellValuesByColumnId (primitives + single-select only) + */ +function buildUpdateCells(srcFields, srcCells, idmap, warnings, srcRecId) { + const cells = {}; + for (const field of srcFields) { + const srcVal = srcCells[field.id]; + if (srcVal === undefined) continue; + const mapping = idmap.fields[field.id]; + if (!mapping) continue; // field not mapped → skip + // multiSelect arrays are not accepted by updateRecords → defer + if (isMultiSelect(field.type)) { + warnings.push({ + code: 'RECORD_ARRAY_UPDATE_DEFERRED', + message: `Record ${srcRecId}: field ${field.id} is multiSelect — array update deferred (updateRecords does not support arrays)`, + }); + continue; + } + const coerced = coercePass1Cell(field, srcVal, idmap); + if (!coerced.write) { + if (field.type === 'select') { + warnings.push({ + code: 'RECORD_CELL_SKIPPED', + message: `Record ${srcRecId}: field ${field.id} (select) choice could not be mapped to dest — cell skipped`, + }); + } + continue; + } + cells[mapping.destFld] = coerced.value; + } + return cells; +} + +/** + * Pass 1 record sync: scalar/select upsert + fill the global record map. + * + * @param {object} opts + * @param {object} opts.client - AirtableClient instance + * @param {object} opts.srcSnapshot - source base snapshot (tables with records attached) + * @param {object} opts.destSnapshot - dest base snapshot + * @param {object} opts.idmap - live id-map (mutated: idmap.records filled) + * @param {object} opts.limiter - rate limiter from createLimiter() + * @param {object} opts.journal - journal from newJournal() + * @param {Function} opts.persist - persist(idmap, journal) called after each batch + * @param {object} opts.result - result accumulator { created, updated, skipped, failed, warnings } + * @returns {Promise} + */ +export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result }) { + if (!idmap.records) idmap.records = {}; + + // destAppId comes from the snapshot (set by snapshotBase); fall back to '' for tests that omit it. + const destAppId = destSnapshot.baseId || ''; + + for (const srcTable of (srcSnapshot.tables || [])) { + const destTableId = idmap.tables[srcTable.id]; + if (!destTableId) continue; // table not matched → skip + + const records = srcTable.records || []; + if (records.length === 0) continue; + + // Partition into CREATE vs UPDATE batches + const createRows = []; + const updateRows = []; + + for (const rec of records) { + const srcCells = rec.cellValuesByColumnId || {}; + const destRecId = idmap.records[rec.id]; + + if (!destRecId) { + // CREATE path — multiSelect arrays ARE accepted by createRecords + const cells = buildCreateCells(srcTable.fields, srcCells, idmap, result.warnings, rec.id); + createRows.push({ cellValuesByColumnId: cells, sourceKey: rec.id }); + } else { + // UPDATE path — multiSelect arrays NOT accepted by updateRecords (deferred with warning) + const cells = buildUpdateCells(srcTable.fields, srcCells, idmap, result.warnings, rec.id); + updateRows.push({ rowId: destRecId, cellValuesByColumnId: cells }); + } + } + + // ── CREATE batch ────────────────────────────────────────────────────── + if (createRows.length > 0) { + try { + const res = await limiter.run(() => + withRetry(() => client.createRecords(destAppId, destTableId, createRows, {})), + ); + for (const created of (res.created || [])) { + idmap.records[created.sourceKey] = created.rowId; + result.created++; + } + for (const failed of (res.failed || [])) { + result.failed++; + result.warnings.push({ + code: 'RECORD_CREATE_FAILED', + message: `Table ${destTableId}: failed to create record (sourceKey=${failed.sourceKey}): ${failed.error}`, + }); + } + } catch (err) { + result.failed += createRows.length; + result.warnings.push({ + code: 'RECORD_CREATE_FAILED', + message: `Table ${destTableId}: createRecords threw: ${err.message}`, + }); + } + persist(idmap, journal); + } + + // ── UPDATE batch ────────────────────────────────────────────────────── + if (updateRows.length > 0) { + try { + const res = await limiter.run(() => + withRetry(() => client.updateRecords(destAppId, destTableId, updateRows)), + ); + result.updated += (res.updated || []).length; + for (const failed of (res.failed || [])) { + result.failed++; + result.warnings.push({ + code: 'RECORD_UPDATE_FAILED', + message: `Table ${destTableId}: failed to update record (rowId=${failed.rowId}): ${failed.error}`, + }); + } + } catch (err) { + result.failed += updateRows.length; + result.warnings.push({ + code: 'RECORD_UPDATE_FAILED', + message: `Table ${destTableId}: updateRecords threw: ${err.message}`, + }); + } + persist(idmap, journal); + } + } +} diff --git a/packages/mcp-server/test/sync/test-records.test.js b/packages/mcp-server/test/sync/test-records.test.js new file mode 100644 index 0000000..ac9cfd4 --- /dev/null +++ b/packages/mcp-server/test/sync/test-records.test.js @@ -0,0 +1,214 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { applyRecordsPass1 } from '../../src/sync/records.js'; +import { newJournal } from '../../src/sync/journal.js'; +import { createLimiter } from '../../src/sync/ratelimit.js'; + +function mockClient(captured) { + return { + createRecords: async (appId, tableId, rows) => { + const created = rows.map((r, i) => { + const rowId = 'recD' + i; + captured.push({ tableId, rowId, cells: r.cellValuesByColumnId, sourceKey: r.sourceKey }); + return { rowId, sourceKey: r.sourceKey }; + }); + return { created, failed: [] }; + }, + updateRecords: async () => ({ updated: [], failed: [] }), + }; +} + +describe('records.applyRecordsPass1', () => { + it('creates new records with coerced scalar/select cells and fills the record map', async () => { + const idmap = { + tables: { tblS: 'tblD' }, + fields: { + fldT: { destFld: 'fldTD', choices: {} }, + fldSel: { destFld: 'fldSelD', choices: { selA: 'selAD' } }, + }, + records: {}, + }; + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', + fields: [ + { id: 'fldT', type: 'text' }, + { id: 'fldSel', type: 'select' }, + { id: 'fldL', type: 'multipleRecordLinks' }, + ], + records: [{ id: 'recS1', cellValuesByColumnId: { fldT: 'hi', fldSel: 'selA', fldL: ['recS9'] } }], + }], + }; + const destSnapshot = { tables: [{ id: 'tblD', name: 'T', fields: [] }] }; + const captured = []; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + await applyRecordsPass1({ + client: mockClient(captured), + srcSnapshot, + destSnapshot, + idmap, + limiter: createLimiter({ rps: 1000, sleep: async () => {} }), + journal: newJournal('p1', 't'), + persist: () => {}, + result, + }); + assert.equal(captured.length, 1); + assert.equal(captured[0].tableId, 'tblD'); + assert.deepEqual(captured[0].cells, { fldTD: 'hi', fldSelD: 'selAD' }); // link cell NOT written in pass 1 + assert.equal(idmap.records.recS1, 'recD0'); // map filled + assert.equal(result.created, 1); + }); + + it('skips unmatched table (not in idmap.tables)', async () => { + const idmap = { tables: {}, fields: {}, records: {} }; + const srcSnapshot = { + tables: [{ + id: 'tblX', name: 'Orphan', + fields: [{ id: 'fldT', type: 'text' }], + records: [{ id: 'recX1', cellValuesByColumnId: { fldT: 'val' } }], + }], + }; + const destSnapshot = { tables: [] }; + const captured = []; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + await applyRecordsPass1({ + client: mockClient(captured), + srcSnapshot, destSnapshot, idmap, + limiter: createLimiter({ rps: 1000, sleep: async () => {} }), + journal: newJournal('p2', 't'), + persist: () => {}, + result, + }); + assert.equal(captured.length, 0); + assert.equal(result.created, 0); + }); + + it('updates existing records for scalar+single-select and skips multiSelect with RECORD_ARRAY_UPDATE_DEFERRED warning', async () => { + const updateCalls = []; + const client = { + createRecords: async () => ({ created: [], failed: [] }), + updateRecords: async (appId, tableId, rows) => { + updateCalls.push({ tableId, rows }); + return { updated: rows.map((r) => r.rowId), failed: [] }; + }, + }; + const idmap = { + tables: { tblS: 'tblD' }, + fields: { + fldT: { destFld: 'fldTD', choices: {} }, + fldSel: { destFld: 'fldSelD', choices: { selA: 'selAD' } }, + fldMS: { destFld: 'fldMSD', choices: { c1: 'c1D', c2: 'c2D' } }, + }, + records: { recS1: 'recD1' }, // already mapped → UPDATE path + }; + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', + fields: [ + { id: 'fldT', type: 'text' }, + { id: 'fldSel', type: 'select' }, + { id: 'fldMS', type: 'multiSelect' }, + ], + records: [{ + id: 'recS1', + cellValuesByColumnId: { fldT: 'hello', fldSel: 'selA', fldMS: ['c1', 'c2'] }, + }], + }], + }; + const destSnapshot = { tables: [{ id: 'tblD', name: 'T', fields: [] }] }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + await applyRecordsPass1({ + client, + srcSnapshot, destSnapshot, idmap, + limiter: createLimiter({ rps: 1000, sleep: async () => {} }), + journal: newJournal('p3', 't'), + persist: () => {}, + result, + }); + assert.equal(result.updated, 1); + // multiSelect should NOT appear in the updateRecords call (arrays not supported) + assert.equal(updateCalls.length, 1); + const sentCells = updateCalls[0].rows[0].cellValuesByColumnId; + assert.ok('fldTD' in sentCells, 'scalar text should be sent'); + assert.ok('fldSelD' in sentCells, 'single-select should be sent'); + assert.ok(!('fldMSD' in sentCells), 'multiSelect must NOT be sent to updateRecords'); + // Warning emitted for deferred multiSelect + assert.ok( + result.warnings.some((w) => w.code === 'RECORD_ARRAY_UPDATE_DEFERRED'), + 'expected RECORD_ARRAY_UPDATE_DEFERRED warning', + ); + }); + + it('pushes RECORD_CELL_SKIPPED warning when a select choice cannot be mapped', async () => { + const captured = []; + const idmap = { + tables: { tblS: 'tblD' }, + fields: { + fldSel: { destFld: 'fldSelD', choices: { selA: 'selAD' } }, // selB not in map + }, + records: {}, + }; + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', + fields: [{ id: 'fldSel', type: 'select' }], + records: [{ id: 'recS1', cellValuesByColumnId: { fldSel: 'selB' } }], + }], + }; + const destSnapshot = { tables: [{ id: 'tblD', name: 'T', fields: [] }] }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + await applyRecordsPass1({ + client: mockClient(captured), + srcSnapshot, destSnapshot, idmap, + limiter: createLimiter({ rps: 1000, sleep: async () => {} }), + journal: newJournal('p4', 't'), + persist: () => {}, + result, + }); + assert.ok( + result.warnings.some((w) => w.code === 'RECORD_CELL_SKIPPED'), + 'expected RECORD_CELL_SKIPPED warning for unmappable choice', + ); + // Record is still created (best-effort), just without the unmappable cell + assert.equal(result.created, 1); + }); + + it('continues on createRecords failure and emits RECORD_CREATE_FAILED warning', async () => { + const client = { + createRecords: async () => { + const e = new Error('bad request — schema mismatch'); + e.status = 422; // non-transient: withRetry will not retry, avoids real sleep delays + throw e; + }, + updateRecords: async () => ({ updated: [], failed: [] }), + }; + const idmap = { + tables: { tblS: 'tblD' }, + fields: { fldT: { destFld: 'fldTD', choices: {} } }, + records: {}, + }; + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', + fields: [{ id: 'fldT', type: 'text' }], + records: [{ id: 'recS1', cellValuesByColumnId: { fldT: 'x' } }], + }], + }; + const destSnapshot = { tables: [{ id: 'tblD', name: 'T', fields: [] }] }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + // Should not throw + await applyRecordsPass1({ + client, + srcSnapshot, destSnapshot, idmap, + limiter: createLimiter({ rps: 1000, sleep: async () => {} }), + journal: newJournal('p5', 't'), + persist: () => {}, + result, + }); + assert.equal(result.failed, 1); + assert.ok( + result.warnings.some((w) => w.code === 'RECORD_CREATE_FAILED'), + 'expected RECORD_CREATE_FAILED warning', + ); + }); +}); From 90009f44c0debadecc270780caa3115d7439a17d Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 13:01:33 +0300 Subject: [PATCH 062/246] feat(sync): records Pass 2 link cells via addLinkItems (idempotent) - client.addLinkItems: POSTs updateArrayTypeCellByAddingItem per item, soft-fails - applyRecordsPass2: resolves multipleRecordLinks cells, deduplicates against existing dest links, emits RECORD_LINK_UNRESOLVED warnings for unmapped src ids, continues on per-field failure with RECORD_LINK_FAILED warning Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/client.js | 30 +++ packages/mcp-server/src/sync/records.js | 117 ++++++++- .../mcp-server/test/sync/test-records.test.js | 246 +++++++++++++++++- 3 files changed, 391 insertions(+), 2 deletions(-) diff --git a/packages/mcp-server/src/client.js b/packages/mcp-server/src/client.js index bd22b57..292e21b 100644 --- a/packages/mcp-server/src/client.js +++ b/packages/mcp-server/src/client.js @@ -2687,6 +2687,36 @@ export class AirtableClient { } } + /** + * Add linked-record items to a link cell via `updateArrayTypeCellByAddingItem`. + * Called once per item (Airtable has no SET endpoint for link cells — only APPEND). + * Soft-fails (returns {ok:false,error}) instead of throwing, so one bad link + * doesn't abort a record sync. + * + * @param {string} appId + * @param {string} rowId + * @param {string} columnId + * @param {Array<{foreignRowId:string, foreignRowDisplayName:string}>} items + * @returns {Promise<{ok:boolean, added:number, error?:string}>} + */ + async addLinkItems(appId, rowId, columnId, items) { + const url = `https://airtable.com/v0.3/row/${rowId}/updateArrayTypeCellByAddingItem`; + let added = 0; + try { + for (const item of items) { + const res = await this.auth.postForm(url, this._mutationParams({ columnId, item }, appId), appId); + if (!res.ok) { + const body = await res.text().catch(() => ''); + return { ok: false, added, error: `addLinkItem failed (${res.status}): ${body}` }; + } + added++; + } + return { ok: true, added }; + } catch (err) { + return { ok: false, added, error: String(err?.message || err) }; + } + } + _genRequestId() { return 'req' + this._genRandomId(); } diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index 4359843..f9c2401 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -11,7 +11,7 @@ * `persist(idmap, journal)` is called after each batch. */ -import { coercePass1Cell } from './cells.js'; +import { coercePass1Cell, partitionLinkValue } from './cells.js'; import { withRetry } from './ratelimit.js'; /** @@ -198,3 +198,118 @@ export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idm } } } + +/** + * Pass 2 record sync: write link cells (multipleRecordLinks / foreignKey) that + * were deferred from Pass 1. + * + * For each matched table, for each source field of type `multipleRecordLinks` + * or `foreignKey` that has a dest field mapping, for each source record with a + * non-empty link cell: + * - Map src row → dest row (skip if the src row has no dest mapping). + * - `partitionLinkValue(srcCell, idmap)` → resolved (dest rec ids) + unresolved (src rec ids). + * - Dedup against current dest links: only add ids NOT already present. + * - Call `client.addLinkItems` via the limiter for the new items. + * - Push ONE `RECORD_LINK_UNRESOLVED` warning (count + context) for any unresolved ids. + * - Continue-on-failure: per-field errors are caught and pushed as `RECORD_LINK_FAILED` warnings. + * + * @param {object} opts + * @param {object} opts.client - AirtableClient instance (must have addLinkItems) + * @param {object} opts.srcSnapshot - source base snapshot (tables with records attached) + * @param {object} opts.destSnapshot - dest base snapshot (tables with records attached; used for dedup) + * @param {object} opts.idmap - live id-map (.tables, .fields, .records) + * @param {Map} opts.destDisplayNames - Map for foreignRowDisplayName + * @param {object} opts.limiter - rate limiter from createLimiter() + * @param {object} opts.journal - journal from newJournal() + * @param {Function} opts.persist - persist(idmap, journal) called after each table + * @param {object} opts.result - result accumulator { created, updated, skipped, failed, warnings } + * @returns {Promise} + */ +export async function applyRecordsPass2({ client, srcSnapshot, destSnapshot, idmap, destDisplayNames, limiter, journal, persist, result }) { + const destAppId = destSnapshot.baseId || ''; + + // Build a fast lookup: destTableId → Map> + // so we can check which links already exist on the dest row. + const destLinkIndex = new Map(); // destTableId → Map>> + for (const destTable of (destSnapshot.tables || [])) { + const recMap = new Map(); + for (const rec of (destTable.records || [])) { + recMap.set(rec.id, rec.cellValuesByColumnId || {}); + } + destLinkIndex.set(destTable.id, recMap); + } + + for (const srcTable of (srcSnapshot.tables || [])) { + const destTableId = idmap.tables[srcTable.id]; + if (!destTableId) continue; + + // Find all link fields in this source table that have a dest mapping + const linkFields = (srcTable.fields || []).filter( + (f) => (f.type === 'multipleRecordLinks' || f.type === 'foreignKey') && idmap.fields[f.id], + ); + if (linkFields.length === 0) continue; + + const destRecMap = destLinkIndex.get(destTableId) || new Map(); + + for (const rec of (srcTable.records || [])) { + const destRowId = idmap.records[rec.id]; + if (!destRowId) continue; // src row not yet mapped → skip + + const srcCells = rec.cellValuesByColumnId || {}; + const destCells = destRecMap.get(destRowId) || {}; + + for (const field of linkFields) { + const srcVal = srcCells[field.id]; + if (!srcVal || (Array.isArray(srcVal) && srcVal.length === 0)) continue; + + const mapping = idmap.fields[field.id]; + const destFldId = mapping.destFld; + + // Partition: resolved → dest rec ids; unresolved → src rec ids with no mapping + const { resolved, unresolved } = partitionLinkValue(srcVal, idmap); + + // Warn for unresolved (one warning per field+record, count in message) + if (unresolved.length > 0) { + result.warnings.push({ + code: 'RECORD_LINK_UNRESOLVED', + message: `Record ${rec.id} field ${field.id}: ${unresolved.length} linked record(s) could not be resolved (no dest mapping): ${unresolved.join(', ')}`, + }); + } + + if (resolved.length === 0) continue; + + // Dedup against current dest links + const currentLinks = destCells[destFldId]; + const alreadyPresent = new Set(Array.isArray(currentLinks) ? currentLinks : []); + const toAdd = resolved.filter((id) => !alreadyPresent.has(id)); + if (toAdd.length === 0) continue; + + const items = toAdd.map((destRec) => ({ + foreignRowId: destRec, + foreignRowDisplayName: destDisplayNames.get(destRec) ?? '', + })); + + try { + const res = await limiter.run(() => + withRetry(() => client.addLinkItems(destAppId, destRowId, destFldId, items)), + ); + if (!res.ok) { + result.warnings.push({ + code: 'RECORD_LINK_FAILED', + message: `Record ${rec.id} field ${field.id} → dest row ${destRowId}: addLinkItems failed: ${res.error}`, + }); + } else { + result.updated++; + } + } catch (err) { + result.warnings.push({ + code: 'RECORD_LINK_FAILED', + message: `Record ${rec.id} field ${field.id} → dest row ${destRowId}: addLinkItems threw: ${err.message}`, + }); + } + } + } + + persist(idmap, journal); + } +} diff --git a/packages/mcp-server/test/sync/test-records.test.js b/packages/mcp-server/test/sync/test-records.test.js index ac9cfd4..138cbc0 100644 --- a/packages/mcp-server/test/sync/test-records.test.js +++ b/packages/mcp-server/test/sync/test-records.test.js @@ -1,6 +1,6 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { applyRecordsPass1 } from '../../src/sync/records.js'; +import { applyRecordsPass1, applyRecordsPass2 } from '../../src/sync/records.js'; import { newJournal } from '../../src/sync/journal.js'; import { createLimiter } from '../../src/sync/ratelimit.js'; @@ -212,3 +212,247 @@ describe('records.applyRecordsPass1', () => { ); }); }); + +// ────────────────────────────────────────────────────────────────────────────── +// applyRecordsPass2 +// ────────────────────────────────────────────────────────────────────────────── + +describe('records.applyRecordsPass2', () => { + function makeLimiter() { + return createLimiter({ rps: 1000, sleep: async () => {} }); + } + + it('adds resolved link items with correct foreignRowId and displayName', async () => { + const addCalls = []; + const client = { + addLinkItems: async (appId, rowId, columnId, items) => { + addCalls.push({ appId, rowId, columnId, items }); + return { ok: true, added: items.length }; + }, + }; + + const idmap = { + tables: { tblS: 'tblD' }, + fields: { fldL: { destFld: 'fldLD' } }, + records: { recS1: 'recD1', recSTarget: 'recDTarget' }, + }; + + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', + fields: [{ id: 'fldL', type: 'multipleRecordLinks' }], + records: [{ id: 'recS1', cellValuesByColumnId: { fldL: ['recSTarget'] } }], + }], + }; + + // dest row's link cell is currently empty → nothing to dedup against + const destSnapshot = { + baseId: 'appDest', + tables: [{ + id: 'tblD', name: 'T', + fields: [{ id: 'fldLD', type: 'multipleRecordLinks' }], + records: [{ id: 'recD1', cellValuesByColumnId: { fldLD: [] } }], + }], + }; + + const destDisplayNames = new Map([['recDTarget', 'Target Row']]); + const result = { created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + + await applyRecordsPass2({ + client, srcSnapshot, destSnapshot, idmap, destDisplayNames, + limiter: makeLimiter(), + journal: newJournal('p2-1', 't'), + persist: () => {}, + result, + }); + + assert.equal(addCalls.length, 1, 'addLinkItems should be called once'); + assert.equal(addCalls[0].rowId, 'recD1'); + assert.equal(addCalls[0].columnId, 'fldLD'); + assert.deepEqual(addCalls[0].items, [{ foreignRowId: 'recDTarget', foreignRowDisplayName: 'Target Row' }]); + assert.equal(result.warnings.length, 0, 'no warnings for fully-resolved links'); + }); + + it('emits RECORD_LINK_UNRESOLVED warning for unresolved src ids, does not abort record', async () => { + const addCalls = []; + const client = { + addLinkItems: async (appId, rowId, columnId, items) => { + addCalls.push({ rowId, columnId, items }); + return { ok: true, added: items.length }; + }, + }; + + const idmap = { + tables: { tblS: 'tblD' }, + fields: { fldL: { destFld: 'fldLD' } }, + records: { recS1: 'recD1', recSTarget: 'recDTarget' }, // recX has no mapping + }; + + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', + fields: [{ id: 'fldL', type: 'multipleRecordLinks' }], + records: [{ id: 'recS1', cellValuesByColumnId: { fldL: ['recSTarget', 'recX'] } }], + }], + }; + + const destSnapshot = { + baseId: 'appDest', + tables: [{ + id: 'tblD', name: 'T', + fields: [{ id: 'fldLD', type: 'multipleRecordLinks' }], + records: [{ id: 'recD1', cellValuesByColumnId: {} }], + }], + }; + + const destDisplayNames = new Map([['recDTarget', 'Target']]); + const result = { created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + + await applyRecordsPass2({ + client, srcSnapshot, destSnapshot, idmap, destDisplayNames, + limiter: makeLimiter(), + journal: newJournal('p2-2', 't'), + persist: () => {}, + result, + }); + + // The resolved link should still be added (record not aborted) + assert.equal(addCalls.length, 1, 'addLinkItems still called for resolved items'); + assert.deepEqual(addCalls[0].items, [{ foreignRowId: 'recDTarget', foreignRowDisplayName: 'Target' }]); + + // One RECORD_LINK_UNRESOLVED warning for the unresolved recX + const unresolvedWarns = result.warnings.filter((w) => w.code === 'RECORD_LINK_UNRESOLVED'); + assert.equal(unresolvedWarns.length, 1, 'expected exactly one RECORD_LINK_UNRESOLVED warning'); + assert.ok(unresolvedWarns[0].message.includes('recX') || unresolvedWarns[0].message.includes('1'), 'warning should mention unresolved id or count'); + }); + + it('does not re-add links already present in destSnapshot (idempotency)', async () => { + const addCalls = []; + const client = { + addLinkItems: async (appId, rowId, columnId, items) => { + addCalls.push({ rowId, columnId, items }); + return { ok: true, added: items.length }; + }, + }; + + const idmap = { + tables: { tblS: 'tblD' }, + fields: { fldL: { destFld: 'fldLD' } }, + records: { recS1: 'recD1', recSTarget: 'recDTarget' }, + }; + + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', + fields: [{ id: 'fldL', type: 'multipleRecordLinks' }], + records: [{ id: 'recS1', cellValuesByColumnId: { fldL: ['recSTarget'] } }], + }], + }; + + // dest row already has recDTarget linked → must NOT re-add + const destSnapshot = { + baseId: 'appDest', + tables: [{ + id: 'tblD', name: 'T', + fields: [{ id: 'fldLD', type: 'multipleRecordLinks' }], + records: [{ id: 'recD1', cellValuesByColumnId: { fldLD: ['recDTarget'] } }], + }], + }; + + const destDisplayNames = new Map([['recDTarget', 'Target']]); + const result = { created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + + await applyRecordsPass2({ + client, srcSnapshot, destSnapshot, idmap, destDisplayNames, + limiter: makeLimiter(), + journal: newJournal('p2-3', 't'), + persist: () => {}, + result, + }); + + // addLinkItems must NOT be called (nothing new to add) + assert.equal(addCalls.length, 0, 'addLinkItems must not be called when link already present'); + assert.equal(result.warnings.length, 0); + }); + + it('skips src records not in idmap.records', async () => { + const addCalls = []; + const client = { + addLinkItems: async (appId, rowId, columnId, items) => { + addCalls.push({ rowId, items }); + return { ok: true, added: items.length }; + }, + }; + + const idmap = { + tables: { tblS: 'tblD' }, + fields: { fldL: { destFld: 'fldLD' } }, + records: {}, // recS1 has no dest mapping + }; + + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', + fields: [{ id: 'fldL', type: 'multipleRecordLinks' }], + records: [{ id: 'recS1', cellValuesByColumnId: { fldL: ['recS2'] } }], + }], + }; + + const destSnapshot = { baseId: 'appDest', tables: [{ id: 'tblD', records: [] }] }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + + await applyRecordsPass2({ + client, srcSnapshot, destSnapshot, idmap, destDisplayNames: new Map(), + limiter: makeLimiter(), + journal: newJournal('p2-4', 't'), + persist: () => {}, + result, + }); + + assert.equal(addCalls.length, 0, 'no addLinkItems when src row not mapped'); + }); + + it('continues on addLinkItems failure and emits RECORD_LINK_FAILED warning', async () => { + const client = { + addLinkItems: async () => ({ ok: false, error: 'network error' }), + }; + + const idmap = { + tables: { tblS: 'tblD' }, + fields: { fldL: { destFld: 'fldLD' } }, + records: { recS1: 'recD1', recSTarget: 'recDTarget' }, + }; + + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', + fields: [{ id: 'fldL', type: 'multipleRecordLinks' }], + records: [{ id: 'recS1', cellValuesByColumnId: { fldL: ['recSTarget'] } }], + }], + }; + + const destSnapshot = { + baseId: 'appDest', + tables: [{ + id: 'tblD', name: 'T', + fields: [{ id: 'fldLD', type: 'multipleRecordLinks' }], + records: [{ id: 'recD1', cellValuesByColumnId: {} }], + }], + }; + + const destDisplayNames = new Map([['recDTarget', 'Target']]); + const result = { created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + + // Must not throw + await applyRecordsPass2({ + client, srcSnapshot, destSnapshot, idmap, destDisplayNames, + limiter: makeLimiter(), + journal: newJournal('p2-5', 't'), + persist: () => {}, + result, + }); + + const failWarns = result.warnings.filter((w) => w.code === 'RECORD_LINK_FAILED'); + assert.equal(failWarns.length, 1, 'expected RECORD_LINK_FAILED warning'); + }); +}); From 0290dc0dbdba587f0fd0f0c7af6060dd088be2e5 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 13:07:56 +0300 Subject: [PATCH 063/246] fix(sync): harden link-cell shape handling in records Pass 1 & 2 Add linkRecId() pure helper to extract rec-id from link-cell elements (string or object shape). Update partitionLinkValue() to map elements through linkRecId before idmap lookup, so source cells work whether formatted as 'recXxx' or {foreignRowId:'recXxx'}/{id:'recXxx'}. Harden Pass 2 dedup guard to map dest link cells through linkRecId before building the present Set, fixing idempotency when API returns object shapes. Both code paths now robust to API shape variance. New tests: linkRecId unit suite (6 cases), partitionLinkValue mixed-shape case, Pass 2 object-shaped dest cell case. All 133 sync tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/cells.js | 21 +++++++- packages/mcp-server/src/sync/records.js | 6 ++- .../mcp-server/test/sync/test-cells.test.js | 31 +++++++++++- .../mcp-server/test/sync/test-records.test.js | 50 +++++++++++++++++++ 4 files changed, 103 insertions(+), 5 deletions(-) diff --git a/packages/mcp-server/src/sync/cells.js b/packages/mcp-server/src/sync/cells.js index a4a23df..cbe46ca 100644 --- a/packages/mcp-server/src/sync/cells.js +++ b/packages/mcp-server/src/sync/cells.js @@ -2,6 +2,21 @@ import { isComputedType } from './snapshot.js'; const ARRAY_DEFERRED = new Set(['multipleRecordLinks', 'multipleAttachments']); +/** + * Extract the record ID from a link-cell element. + * Link-cell elements can be plain strings ('recXxx') or objects ({ foreignRowId: 'recXxx' } or { id: 'recXxx' }). + * + * @param {string | object} x - link-cell element + * @returns {string | null} - the rec-id string, or null if not found + */ +export function linkRecId(x) { + if (typeof x === 'string') return x; + if (x && typeof x === 'object') { + return x.foreignRowId ?? x.id ?? null; + } + return null; +} + export function isWritableForRecords(field) { return !isComputedType(field.type) && !ARRAY_DEFERRED.has(field.type); } @@ -27,8 +42,10 @@ export function coercePass1Cell(field, srcValue, idmap) { export function partitionLinkValue(srcValue, idmap) { const recs = idmap.records || {}; const resolved = [], unresolved = []; - for (const s of (Array.isArray(srcValue) ? srcValue : [])) { - if (recs[s]) resolved.push(recs[s]); else unresolved.push(s); + for (const elem of (Array.isArray(srcValue) ? srcValue : [])) { + const srcRecId = linkRecId(elem); + if (srcRecId === null) continue; // skip garbage elements + if (recs[srcRecId]) resolved.push(recs[srcRecId]); else unresolved.push(srcRecId); } return { resolved, unresolved }; } diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index f9c2401..59e54f8 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -11,7 +11,7 @@ * `persist(idmap, journal)` is called after each batch. */ -import { coercePass1Cell, partitionLinkValue } from './cells.js'; +import { coercePass1Cell, partitionLinkValue, linkRecId } from './cells.js'; import { withRetry } from './ratelimit.js'; /** @@ -279,8 +279,10 @@ export async function applyRecordsPass2({ client, srcSnapshot, destSnapshot, idm if (resolved.length === 0) continue; // Dedup against current dest links + // Map link-cell elements through linkRecId to handle both string and object shapes const currentLinks = destCells[destFldId]; - const alreadyPresent = new Set(Array.isArray(currentLinks) ? currentLinks : []); + const currentLinkIds = (Array.isArray(currentLinks) ? currentLinks : []).map(linkRecId).filter(Boolean); + const alreadyPresent = new Set(currentLinkIds); const toAdd = resolved.filter((id) => !alreadyPresent.has(id)); if (toAdd.length === 0) continue; diff --git a/packages/mcp-server/test/sync/test-cells.test.js b/packages/mcp-server/test/sync/test-cells.test.js index c164714..7d1ba5e 100644 --- a/packages/mcp-server/test/sync/test-cells.test.js +++ b/packages/mcp-server/test/sync/test-cells.test.js @@ -1,6 +1,6 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { isWritableForRecords, coercePass1Cell, partitionLinkValue } from '../../src/sync/cells.js'; +import { isWritableForRecords, coercePass1Cell, partitionLinkValue, linkRecId } from '../../src/sync/cells.js'; const idmap = { fields: { fldSel: { destFld: 'fldSelD', choices: { selA: 'selAD', selB: 'selBD' } } } }; @@ -31,11 +31,40 @@ describe('cells.coercePass1Cell', () => { assert.equal(coercePass1Cell({ id: 'fldSel', type: 'select' }, 'selUNKNOWN', idmap).write, false); }); }); +describe('cells.linkRecId', () => { + it('string → string', () => { + assert.equal(linkRecId('recXyz'), 'recXyz'); + }); + it('object with foreignRowId → foreignRowId', () => { + assert.equal(linkRecId({ foreignRowId: 'recAbc' }), 'recAbc'); + }); + it('object with id → id', () => { + assert.equal(linkRecId({ id: 'recDef' }), 'recDef'); + }); + it('object with both foreignRowId and id → foreignRowId preferred', () => { + assert.equal(linkRecId({ foreignRowId: 'recA', id: 'recB' }), 'recA'); + }); + it('garbage object → null', () => { + assert.equal(linkRecId({}), null); + assert.equal(linkRecId({ other: 'val' }), null); + }); + it('null/undefined → null', () => { + assert.equal(linkRecId(null), null); + assert.equal(linkRecId(undefined), null); + }); +}); + describe('cells.partitionLinkValue', () => { const m = { records: { recS1: 'recD1', recS2: 'recD2' } }; it('splits resolved vs unresolved by the record map', () => { assert.deepEqual(partitionLinkValue(['recS1', 'recS2', 'recX'], m), { resolved: ['recD1', 'recD2'], unresolved: ['recX'] }); }); + it('handles mixed string and object shapes in source link cell', () => { + assert.deepEqual( + partitionLinkValue([{ foreignRowId: 'recS1' }, { id: 'recS2' }, 'recS3'], { records: { recS1: 'recD1', recS2: 'recD2' } }), + { resolved: ['recD1', 'recD2'], unresolved: ['recS3'] } + ); + }); it('null/empty → empty partition', () => { assert.deepEqual(partitionLinkValue(null, m), { resolved: [], unresolved: [] }); }); diff --git a/packages/mcp-server/test/sync/test-records.test.js b/packages/mcp-server/test/sync/test-records.test.js index 138cbc0..de7de12 100644 --- a/packages/mcp-server/test/sync/test-records.test.js +++ b/packages/mcp-server/test/sync/test-records.test.js @@ -375,6 +375,56 @@ describe('records.applyRecordsPass2', () => { assert.equal(result.warnings.length, 0); }); + it('handles object-shaped dest link cell elements (dedup on foreignRowId)', async () => { + const addCalls = []; + const client = { + addLinkItems: async (appId, rowId, columnId, items) => { + addCalls.push({ rowId, columnId, items }); + return { ok: true, added: items.length }; + }, + }; + + const idmap = { + tables: { tblS: 'tblD' }, + fields: { fldL: { destFld: 'fldLD' } }, + records: { recS1: 'recD1', recSTarget: 'recDTarget' }, + }; + + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', + fields: [{ id: 'fldL', type: 'multipleRecordLinks' }], + records: [{ id: 'recS1', cellValuesByColumnId: { fldL: ['recSTarget'] } }], + }], + }; + + // dest row's link cell is OBJECT-shaped (API returned objects, not strings) + // Current link is already { foreignRowId: 'recDTarget' } → must NOT re-add + const destSnapshot = { + baseId: 'appDest', + tables: [{ + id: 'tblD', name: 'T', + fields: [{ id: 'fldLD', type: 'multipleRecordLinks' }], + records: [{ id: 'recD1', cellValuesByColumnId: { fldLD: [{ foreignRowId: 'recDTarget' }] } }], + }], + }; + + const destDisplayNames = new Map([['recDTarget', 'Target']]); + const result = { created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + + await applyRecordsPass2({ + client, srcSnapshot, destSnapshot, idmap, destDisplayNames, + limiter: makeLimiter(), + journal: newJournal('p2-3b', 't'), + persist: () => {}, + result, + }); + + // addLinkItems must NOT be called (link already present in object form) + assert.equal(addCalls.length, 0, 'addLinkItems must not be called when link already present (object shape)'); + assert.equal(result.warnings.length, 0); + }); + it('skips src records not in idmap.records', async () => { const addCalls = []; const client = { From 02ce8f18e8a54d9137ff4e74f96e8600d1d1179d Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 13:14:26 +0300 Subject: [PATCH 064/246] feat(sync): record attachments (download/upload/verify/dedupe, non-blocking) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds applyAttachments() to records.js: for each multipleAttachments field in matched tables, fetches source URLs (signed, expire ~2h) during the run, uploads via client.uploadAttachment with rate-limit + retry. Dedupes by filename|size key in idmap.attachments (persists across calls for idempotency). {ok:false} or fetch errors push ATTACHMENT_FAILED warning and continue — never abort the record. No per-attachment read-back re-query: Task-0 spike confirmed {ok:true} guarantees dest hosting on dl.airtable.com. fetchBytes is injected for tests (no real network). 5 new tests covering upload, dedupe, ok:false, fetch-error, and unmapped-field paths. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/records.js | 144 ++++++++++- .../mcp-server/test/sync/test-records.test.js | 236 +++++++++++++++++- 2 files changed, 378 insertions(+), 2 deletions(-) diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index 59e54f8..a2176d0 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -1,5 +1,6 @@ /** - * records.js — Pass 1: sync scalar + select/multiSelect cells. + * records.js — Pass 1/2: sync scalar + select/multiSelect cells + link cells. + * Pass 3 (applyAttachments): download source attachments → upload to dest. * * For each matched table (those with idmap.tables[srcTableId]): * CREATE path — source record not yet in idmap.records → createRecords, fill idmap.records. @@ -315,3 +316,144 @@ export async function applyRecordsPass2({ client, srcSnapshot, destSnapshot, idm persist(idmap, journal); } } + +// ────────────────────────────────────────────────────────────────────────────── +// Default fetchBytes: real fetch-based implementation. +// Injected in tests so no real network is hit. +// ────────────────────────────────────────────────────────────────────────────── +async function defaultFetchBytes(url) { + const r = await fetch(url); + return { bytes: Buffer.from(await r.arrayBuffer()), contentType: r.headers.get('content-type') }; +} + +/** + * Attachment field type names accepted by Airtable (public API names). + */ +const ATTACHMENT_TYPES = new Set(['multipleAttachments', 'multipleAttachment']); + +/** + * Pass 3 record sync: download source attachments → upload to dest base. + * + * For each matched table, for each source field of type `multipleAttachments` + * that has a dest field mapping, for each source record with a non-empty + * attachment cell whose src row maps (idmap.records[srcRecId]): + * - For each attachment { url, filename, size, type }: + * - Dedupe key = `${filename}|${size}`. Skip if already uploaded. + * - Fetch bytes from source URL (URLs expire ~2h — must download during run). + * - Upload to dest via client.uploadAttachment (rate-limited + retried). + * - On {ok:true}: record dedupe key, bump attachmentsUploaded counter. + * - On {ok:false} OR thrown fetch error: push ATTACHMENT_FAILED warning, CONTINUE. + * - persist(idmap, journal) after each source table. + * + * NOTE: {ok:true} from uploadAttachment IS the hosting verification per Task-0 spike. + * The spike confirmed dest attachment URLs are on dl.airtable.com after a successful upload. + * No separate per-attachment read-back re-query is performed (that would be an extra network + * call per attachment; the spike already proved {ok:true} guarantees hosting). + * + * @param {object} opts + * @param {object} opts.client - AirtableClient instance (must have uploadAttachment) + * @param {object} opts.srcSnapshot - source base snapshot (tables with records attached) + * @param {object} opts.destSnapshot - dest base snapshot (provides destAppId via baseId) + * @param {object} opts.idmap - live id-map (.tables, .fields, .records, .attachments) + * @param {object} opts.limiter - rate limiter from createLimiter() + * @param {object} opts.journal - journal from newJournal() + * @param {Function} opts.persist - persist(idmap, journal) called after each table + * @param {object} opts.result - result accumulator (must include .warnings array and .attachmentsUploaded counter) + * @param {Function} [opts.fetchBytes] - async (url) => { bytes, contentType? } (default: real fetch) + * @returns {Promise} + */ +export async function applyAttachments({ + client, + srcSnapshot, + destSnapshot, + idmap, + limiter, + journal, + persist, + result, + fetchBytes = defaultFetchBytes, +}) { + // Initialize dedupe map if not already present (survives across calls for idempotency) + idmap.attachments ??= {}; + + const destAppId = destSnapshot.baseId || ''; + + for (const srcTable of (srcSnapshot.tables || [])) { + const destTableId = idmap.tables[srcTable.id]; + if (!destTableId) continue; // table not matched → skip + + // Find attachment fields that have a dest mapping + const attFields = (srcTable.fields || []).filter( + (f) => ATTACHMENT_TYPES.has(f.type) && idmap.fields[f.id], + ); + if (attFields.length === 0) continue; + + for (const rec of (srcTable.records || [])) { + const destRowId = idmap.records[rec.id]; + if (!destRowId) continue; // src record not yet mapped → skip + + const srcCells = rec.cellValuesByColumnId || {}; + + for (const field of attFields) { + const cellValue = srcCells[field.id]; + if (!cellValue || !Array.isArray(cellValue) || cellValue.length === 0) continue; + + const destFldId = idmap.fields[field.id].destFld; + + for (const attachment of cellValue) { + const { url, filename, size, type } = attachment; + if (!url || !filename) continue; + + const dedupeKey = `${filename}|${size}`; + if (idmap.attachments[dedupeKey]) continue; // already uploaded → skip + + // Fetch bytes from source (URLs expire ~2h) + let bytes; + let contentType = type; + try { + const fetched = await fetchBytes(url); + bytes = fetched.bytes; + if (!contentType && fetched.contentType) contentType = fetched.contentType; + } catch (fetchErr) { + result.warnings.push({ + code: 'ATTACHMENT_FAILED', + message: `Attachment ${filename}: fetch failed — ${fetchErr.message}`, + }); + continue; // next attachment, don't abort the record + } + + // Upload to dest via limiter + retry + try { + const res = await limiter.run(() => + withRetry(() => + client.uploadAttachment(destAppId, destRowId, destFldId, { + bytes, + filename, + contentType, + }), + ), + ); + + if (res.ok) { + idmap.attachments[dedupeKey] = true; + result.attachmentsUploaded = (result.attachmentsUploaded || 0) + 1; + } else { + result.warnings.push({ + code: 'ATTACHMENT_FAILED', + message: `Attachment ${filename}: upload failed — ${res.error || 'unknown error'}`, + }); + // Don't set dedupe key on failure so a retry run can re-attempt + } + } catch (uploadErr) { + result.warnings.push({ + code: 'ATTACHMENT_FAILED', + message: `Attachment ${filename}: upload threw — ${uploadErr.message}`, + }); + } + } + } + } + + persist(idmap, journal); + } +} diff --git a/packages/mcp-server/test/sync/test-records.test.js b/packages/mcp-server/test/sync/test-records.test.js index de7de12..091bb3a 100644 --- a/packages/mcp-server/test/sync/test-records.test.js +++ b/packages/mcp-server/test/sync/test-records.test.js @@ -1,6 +1,6 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { applyRecordsPass1, applyRecordsPass2 } from '../../src/sync/records.js'; +import { applyRecordsPass1, applyRecordsPass2, applyAttachments } from '../../src/sync/records.js'; import { newJournal } from '../../src/sync/journal.js'; import { createLimiter } from '../../src/sync/ratelimit.js'; @@ -506,3 +506,237 @@ describe('records.applyRecordsPass2', () => { assert.equal(failWarns.length, 1, 'expected RECORD_LINK_FAILED warning'); }); }); + +// ────────────────────────────────────────────────────────────────────────────── +// applyAttachments +// ────────────────────────────────────────────────────────────────────────────── + +describe('records.applyAttachments', () => { + function makeLimiter() { + return createLimiter({ rps: 1000, sleep: async () => {} }); + } + + function makeFixtures({ uploadResult = { ok: true, attachmentId: 'att1' } } = {}) { + const uploadCalls = []; + const client = { + uploadAttachment: async (appId, rowId, columnId, payload) => { + uploadCalls.push({ appId, rowId, columnId, payload }); + return uploadResult; + }, + }; + + const fetchCalls = []; + const fetchBytes = async (url) => { + fetchCalls.push(url); + return { bytes: Buffer.from('fake-image-data'), contentType: 'image/png' }; + }; + + const idmap = { + tables: { tblS: 'tblD' }, + fields: { + fldAtt: { destFld: 'fldAttD' }, + }, + records: { recS1: 'recD1' }, + // attachments intentionally omitted to test initialization + }; + + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', + fields: [{ id: 'fldAtt', type: 'multipleAttachments' }], + records: [{ + id: 'recS1', + cellValuesByColumnId: { + fldAtt: [{ id: 'attSrc1', url: 'https://example.com/image.png', filename: 'image.png', size: 1024, type: 'image/png' }], + }, + }], + }], + }; + + const destSnapshot = { baseId: 'appDest' }; + + const result = { created: 0, updated: 0, skipped: 0, failed: 0, attachmentsUploaded: 0, warnings: [] }; + + return { client, fetchBytes, fetchCalls, uploadCalls, idmap, srcSnapshot, destSnapshot, result }; + } + + it('downloads and uploads an attachment with correct destRow/destFld/filename', async () => { + const { client, fetchBytes, fetchCalls, uploadCalls, idmap, srcSnapshot, destSnapshot, result } = makeFixtures(); + + await applyAttachments({ + client, srcSnapshot, destSnapshot, idmap, + limiter: makeLimiter(), + journal: newJournal('att-1', 't'), + persist: () => {}, + result, + fetchBytes, + }); + + assert.equal(fetchCalls.length, 1, 'fetchBytes called once for the attachment URL'); + assert.equal(fetchCalls[0], 'https://example.com/image.png'); + + assert.equal(uploadCalls.length, 1, 'uploadAttachment called once'); + assert.equal(uploadCalls[0].appId, 'appDest'); + assert.equal(uploadCalls[0].rowId, 'recD1'); + assert.equal(uploadCalls[0].columnId, 'fldAttD'); + assert.equal(uploadCalls[0].payload.filename, 'image.png'); + assert.ok(uploadCalls[0].payload.bytes, 'bytes must be provided'); + + // dedupe key should be set after success + assert.equal(idmap.attachments['image.png|1024'], true); + }); + + it('deduplicates: same filename|size not re-uploaded on second applyAttachments call', async () => { + const { client, fetchBytes, uploadCalls, idmap, srcSnapshot, destSnapshot, result } = makeFixtures(); + + // First call + await applyAttachments({ + client, srcSnapshot, destSnapshot, idmap, + limiter: makeLimiter(), + journal: newJournal('att-2a', 't'), + persist: () => {}, + result, + fetchBytes, + }); + + assert.equal(uploadCalls.length, 1, 'first call uploads once'); + + // Second call — idmap.attachments is preserved, same attachment must be skipped + const result2 = { created: 0, updated: 0, skipped: 0, failed: 0, attachmentsUploaded: 0, warnings: [] }; + await applyAttachments({ + client, srcSnapshot, destSnapshot, idmap, + limiter: makeLimiter(), + journal: newJournal('att-2b', 't'), + persist: () => {}, + result: result2, + fetchBytes, + }); + + assert.equal(uploadCalls.length, 1, 'second call must NOT re-upload (dedupe by filename|size)'); + }); + + it('ok:false upload → ATTACHMENT_FAILED warning, no throw, record continues', async () => { + const { client, fetchBytes, uploadCalls, idmap, srcSnapshot, destSnapshot, result } = makeFixtures({ + uploadResult: { ok: false, error: 'upload quota exceeded' }, + }); + + // Must not throw + await applyAttachments({ + client, srcSnapshot, destSnapshot, idmap, + limiter: makeLimiter(), + journal: newJournal('att-3', 't'), + persist: () => {}, + result, + fetchBytes, + }); + + assert.equal(uploadCalls.length, 1, 'uploadAttachment was still called'); + + const failWarns = result.warnings.filter((w) => w.code === 'ATTACHMENT_FAILED'); + assert.equal(failWarns.length, 1, 'expected exactly one ATTACHMENT_FAILED warning'); + assert.ok(failWarns[0].message.includes('image.png'), 'warning should mention the filename'); + + // dedupe key must NOT be set on failure + assert.equal(idmap.attachments['image.png|1024'], undefined); + }); + + it('fetchBytes error → ATTACHMENT_FAILED warning, no throw, continues to next attachment', async () => { + const uploadCalls = []; + const client = { + uploadAttachment: async (appId, rowId, columnId, payload) => { + uploadCalls.push(payload.filename); + return { ok: true, attachmentId: 'att99' }; + }, + }; + + // fetchBytes throws on first URL, succeeds on second + let fetchCount = 0; + const fetchBytes = async (url) => { + fetchCount++; + if (fetchCount === 1) throw new Error('network timeout'); + return { bytes: Buffer.from('ok'), contentType: 'image/jpeg' }; + }; + + const idmap = { + tables: { tblS: 'tblD' }, + fields: { fldAtt: { destFld: 'fldAttD' } }, + records: { recS1: 'recD1' }, + }; + + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', + fields: [{ id: 'fldAtt', type: 'multipleAttachments' }], + records: [{ + id: 'recS1', + cellValuesByColumnId: { + fldAtt: [ + { id: 'a1', url: 'https://src/a.png', filename: 'a.png', size: 100, type: 'image/png' }, + { id: 'a2', url: 'https://src/b.jpg', filename: 'b.jpg', size: 200, type: 'image/jpeg' }, + ], + }, + }], + }], + }; + + const destSnapshot = { baseId: 'appDest' }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, attachmentsUploaded: 0, warnings: [] }; + + await applyAttachments({ + client, srcSnapshot, destSnapshot, idmap, + limiter: makeLimiter(), + journal: newJournal('att-4', 't'), + persist: () => {}, + result, + fetchBytes, + }); + + // First attachment failed (fetch threw), second should still be uploaded + const failWarns = result.warnings.filter((w) => w.code === 'ATTACHMENT_FAILED'); + assert.equal(failWarns.length, 1, 'one warning for the failed fetch'); + assert.ok(failWarns[0].message.includes('a.png'), 'warning should mention a.png'); + assert.equal(uploadCalls.length, 1, 'second attachment (b.jpg) should still be uploaded'); + assert.equal(uploadCalls[0], 'b.jpg'); + }); + + it('skips unmatched tables and fields without a destFld mapping', async () => { + const uploadCalls = []; + const client = { + uploadAttachment: async () => { uploadCalls.push(true); return { ok: true }; }, + }; + const fetchBytes = async () => ({ bytes: Buffer.from('x'), contentType: 'text/plain' }); + + const idmap = { + tables: { tblS: 'tblD' }, + fields: {}, // fldAtt not mapped + records: { recS1: 'recD1' }, + }; + + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', + fields: [{ id: 'fldAtt', type: 'multipleAttachments' }], + records: [{ + id: 'recS1', + cellValuesByColumnId: { + fldAtt: [{ id: 'a1', url: 'https://src/x.txt', filename: 'x.txt', size: 5, type: 'text/plain' }], + }, + }], + }], + }; + + const destSnapshot = { baseId: 'appDest' }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, attachmentsUploaded: 0, warnings: [] }; + + await applyAttachments({ + client, srcSnapshot, destSnapshot, idmap, + limiter: makeLimiter(), + journal: newJournal('att-5', 't'), + persist: () => {}, + result, + fetchBytes, + }); + + assert.equal(uploadCalls.length, 0, 'no upload when field is not mapped'); + }); +}); From ecab2dbb89e8f82d77d88fa9ca7e6b600a761b19 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 13:25:16 +0300 Subject: [PATCH 065/246] =?UTF-8?q?feat(sync):=20reapplyViewFilters=20?= =?UTF-8?q?=E2=80=94=20remap=20record-ref=20filters=20once=20records=20exi?= =?UTF-8?q?st?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - remap.js: add remapRecRefValue helper (strip vs remap via idmap.records); update remapFilterSet to remap resolvable rec ids instead of unconditionally stripping; update canonFilterSet + canonicalizeViewConfig to accept idmap and canonicalize via remapped dest rec ids for convergence - diff.js: pass idmap to both canonicalizeViewConfig calls in view diff - records.js: add reapplyViewFilters — for each matched collaborative view whose source filters contain now-resolvable record refs, calls client.updateViewFilters with remapped filter set via limiter+withRetry; continue-on-failure - test-reapply-filters.test.js: 13 new tests (pure remap flip, canonicalize convergence, mock-client reapply) Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/diff.js | 2 +- packages/mcp-server/src/sync/records.js | 64 +++ packages/mcp-server/src/sync/remap.js | 40 +- .../test/sync/test-reapply-filters.test.js | 380 ++++++++++++++++++ 4 files changed, 479 insertions(+), 7 deletions(-) create mode 100644 packages/mcp-server/test/sync/test-reapply-filters.test.js diff --git a/packages/mcp-server/src/sync/diff.js b/packages/mcp-server/src/sync/diff.js index a544436..0f97cf7 100644 --- a/packages/mcp-server/src/sync/diff.js +++ b/packages/mcp-server/src/sync/diff.js @@ -378,7 +378,7 @@ export function computePlan(srcSnap, destSnap, idmap) { if (!dv) { viewActions.push({ kind: 'createView', sourceTableId: st.id, sourceViewId: sv.id, name: sv.name, type: sv.type }); viewActions.push({ kind: 'applyViewConfig', sourceTableId: st.id, sourceViewId: sv.id, type: sv.type, config: sv.config || {} }); - } else if (canonicalizeViewConfig(sv.config || {}, srcNames, srcSelNames, true) !== canonicalizeViewConfig(dv.config || {}, destNames, destSelNames, false)) { + } else if (canonicalizeViewConfig(sv.config || {}, srcNames, srcSelNames, true, idmap) !== canonicalizeViewConfig(dv.config || {}, destNames, destSelNames, false, idmap)) { viewActions.push({ kind: 'applyViewConfig', sourceTableId: st.id, sourceViewId: sv.id, type: sv.type, config: sv.config || {} }); } } diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index a2176d0..702c58f 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -14,6 +14,7 @@ import { coercePass1Cell, partitionLinkValue, linkRecId } from './cells.js'; import { withRetry } from './ratelimit.js'; +import { remapViewConfig, collectFilterRecordRefs } from './remap.js'; /** * Returns true if the field type is a multiSelect (arrays not accepted by updateRecords). @@ -457,3 +458,66 @@ export async function applyAttachments({ persist(idmap, journal); } } + +/** + * After records exist, rewrite collaborative view filters whose record refs now resolve. + * + * For each source table matched in idmap.tables, for each collaborative view (non-personal) + * whose config.filters contain a record-ref leaf that resolves in idmap.records: + * - Compute remapViewConfig(view.config, idmap).filters + * - Call client.updateViewFilters(destApp, destViewId, filters) via limiter + withRetry + * - Map source view id -> dest view id via idmap.views[srcViewId] + * - Skip views with no resolvable record refs + * - Continue-on-failure (VIEW_FILTER_REAPPLY_FAILED warning) + * - Bump result.viewFiltersReapplied counter on success + * + * @param {object} opts + * @param {object} opts.client - AirtableClient instance + * @param {object} opts.srcSnapshot - source base snapshot + * @param {object} opts.destSnapshot - dest base snapshot + * @param {object} opts.idmap - live id-map (.tables, .fields, .records, .views) + * @param {object} opts.limiter - rate limiter from createLimiter() + * @param {object} opts.journal - journal from newJournal() + * @param {Function} opts.persist - persist(idmap, journal) called after each table + * @param {object} opts.result - result accumulator { warnings, viewFiltersReapplied } + * @returns {Promise} + */ +export async function reapplyViewFilters({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result }) { + const destAppId = destSnapshot.baseId || ''; + const recs = (idmap && idmap.records) || {}; + + for (const srcTable of (srcSnapshot.tables || [])) { + const destTableId = idmap.tables[srcTable.id]; + if (!destTableId) continue; // table not matched -> skip + + for (const sv of (srcTable.views || [])) { + if (sv.personalForUserId) continue; // skip personal views + + // Check if this view has any record-ref leaves that resolve in idmap.records + const refIds = collectFilterRecordRefs(sv.config); + if (refIds.length === 0) continue; // no record refs -> skip + + const hasResolvable = refIds.some((id) => recs[id] !== undefined); + if (!hasResolvable) continue; // none resolve -> skip + + const destViewId = (idmap.views || {})[sv.id]; + if (!destViewId) continue; // no dest view mapping -> skip + + // Remap the filters (remapRecRefValue with stripUnresolved=true) + const remapped = remapViewConfig(sv.config || {}, idmap); + const filters = remapped.filters || { filterSet: [], conjunction: 'and' }; + + try { + await limiter.run(() => withRetry(() => client.updateViewFilters(destAppId, destViewId, filters))); + result.viewFiltersReapplied = (result.viewFiltersReapplied || 0) + 1; + } catch (err) { + result.warnings.push({ + code: 'VIEW_FILTER_REAPPLY_FAILED', + message: `View ${sv.id} (dest ${destViewId}): reapplyViewFilters threw: ${err.message}`, + }); + } + } + + persist(idmap, journal); + } +} diff --git a/packages/mcp-server/src/sync/remap.js b/packages/mcp-server/src/sync/remap.js index 71ff94c..829ba85 100644 --- a/packages/mcp-server/src/sync/remap.js +++ b/packages/mcp-server/src/sync/remap.js @@ -104,6 +104,24 @@ function destSelId(idmap, src) { // sentinels (current-user "me", null, booleans) are not id-shaped → kept. Forward-compat: once // records sync (M3), a populated idmap.records lets these REMAP instead of strip — see canon note. const RECORD_REF_ID = /^(rec|usr)[A-Za-z0-9]{14,}$/; +// Remap a record-ref leaf VALUE via idmap.records. +// stripUnresolved=true (source/write path): src rec ids -> dest ids; ids with no mapping are DROPPED; +// returns undefined if nothing resolves (caller drops the whole leaf). +// stripUnresolved=false (dest canonical path): keep values as-is when unmapped (dest already holds dest ids). +function remapRecRefValue(value, idmap, stripUnresolved) { + const recs = (idmap && idmap.records) || {}; + // Structured/dynamic objects (columnId/rowId/tableId) carry source-side ids that can't be + // remapped from a records map — treat as unresolvable: drop on write path, keep on dest path. + if (value && typeof value === 'object' && !Array.isArray(value)) { + return stripUnresolved ? undefined : value; + } + const mapOne = (v) => (typeof v === 'string' ? (recs[v] ?? (stripUnresolved ? undefined : v)) : v); + if (Array.isArray(value)) { + const out = value.map(mapOne).filter((v) => v !== undefined); + return out.length ? out : undefined; + } + return mapOne(value); +} function isRecordRefValue(value) { if (typeof value === 'string') return RECORD_REF_ID.test(value); if (Array.isArray(value)) return value.some((v) => typeof v === 'string' && RECORD_REF_ID.test(v)); @@ -137,7 +155,12 @@ function remapFilterSet(set, idmap) { out.push({ ...(f.type ? { type: f.type } : {}), conjunction: f.conjunction, filterSet: inner }); continue; } - if (isRecordRefValue(f.value)) continue; + if (isRecordRefValue(f.value)) { + const v = remapRecRefValue(f.value, idmap, true); // write path always strips unresolved + if (v === undefined) continue; // nothing resolved -> drop leaf + out.push({ columnId: destFldId(idmap, f.columnId), operator: f.operator, value: v }); + continue; + } const o = { columnId: destFldId(idmap, f.columnId), operator: f.operator, value: f.value }; if (typeof f.value === 'string') o.value = destSelId(idmap, f.value); else if (Array.isArray(f.value)) o.value = f.value.map((v) => (typeof v === 'string' ? destSelId(idmap, v) : v)); @@ -171,17 +194,22 @@ function viewNameOf(map, id) { return map[id] ?? id; } // groups (collapse-agnostic): Airtable may store a 1-element nested group as a bare leaf on // readback — unwrapping on BOTH sides keeps it convergent either way. (Records aren't synced, so // rec-ref leaves drop on both sides; once they do sync, this is where name-resolution lands — M3.) -function canonFilterSet(set, fldNames, selNames, strip) { +function canonFilterSet(set, fldNames, selNames, strip, idmap) { const out = []; for (const f of set) { if (f.filterSet) { - const inner = canonFilterSet(f.filterSet, fldNames, selNames, strip); + const inner = canonFilterSet(f.filterSet, fldNames, selNames, strip, idmap); if (inner.length === 0) continue; if (inner.length === 1) { out.push(inner[0]); continue; } out.push({ c: f.conjunction, n: inner }); continue; } - if (strip && isRecordRefValue(f.value)) continue; + if (isRecordRefValue(f.value)) { + const v = remapRecRefValue(f.value, idmap, strip); // strip param drives strip-vs-keep of unresolved + if (v === undefined) continue; // dropped + out.push({ col: viewNameOf(fldNames, f.columnId), op: f.operator, val: typeof v === 'string' ? viewNameOf(selNames, v) : v }); + continue; + } out.push({ col: viewNameOf(fldNames, f.columnId), op: f.operator, val: typeof f.value === 'string' ? viewNameOf(selNames, f.value) : f.value }); } return out; @@ -190,9 +218,9 @@ function canonFilterSet(set, fldNames, selNames, strip) { // leaves dropped), FALSE for the DEST side (its raw actual filter). Asymmetry is deliberate: it // makes a dest that still holds a dangling rec filter (from a prior buggy sync) diverge from the // stripped source → one cleanup apply → then both read empty/equal → converges. -export function canonicalizeViewConfig(config, fldNames, selNames, stripRecordRefs = true) { +export function canonicalizeViewConfig(config, fldNames, selNames, stripRecordRefs = true, idmap) { const c = config || {}; - const fset = (c.filters && Array.isArray(c.filters.filterSet)) ? canonFilterSet(c.filters.filterSet, fldNames, selNames, stripRecordRefs) : []; + const fset = (c.filters && Array.isArray(c.filters.filterSet)) ? canonFilterSet(c.filters.filterSet, fldNames, selNames, stripRecordRefs, idmap) : []; return JSON.stringify({ filters: fset.length ? { conj: c.filters.conjunction, set: fset } : null, sorts: (c.sorts || []).map((s) => ({ col: viewNameOf(fldNames, s.columnId), asc: s.ascending })), diff --git a/packages/mcp-server/test/sync/test-reapply-filters.test.js b/packages/mcp-server/test/sync/test-reapply-filters.test.js new file mode 100644 index 0000000..d90e1d4 --- /dev/null +++ b/packages/mcp-server/test/sync/test-reapply-filters.test.js @@ -0,0 +1,380 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { remapViewConfig, canonicalizeViewConfig } from '../../src/sync/remap.js'; +import { reapplyViewFilters } from '../../src/sync/records.js'; + +// Realistic rec ids: "rec" + 14 alphanumeric chars = 17 chars total (matches RECORD_REF_ID regex) +const recSrc = 'recSRCSRCSRCSRCSS'; // source record id (17 chars) +const recDst = 'recDSTDSTDSTDSTOO'; // dest record id (17 chars) +const recMiss = 'recMISSINGXXXXXXX'; // unresolvable source record id (18 chars) + +// ── Pure remap flip tests ───────────────────────────────────────────────────── + +describe('remap — record-ref filter REMAP when records exist', () => { + const idmap = { + tables: {}, + views: {}, + fields: { fldG: { destFld: 'fldGD', choices: {} } }, + records: { [recSrc]: recDst }, + }; + + it('remaps a resolvable rec filter instead of stripping it', () => { + const cfg = { + filters: { + conjunction: 'and', + filterSet: [{ columnId: 'fldG', operator: '=', value: [recSrc] }], + }, + }; + const out = remapViewConfig(cfg, idmap); + assert.equal(out.filters.filterSet.length, 1); + assert.deepEqual(out.filters.filterSet[0].value, [recDst]); + assert.equal(out.filters.filterSet[0].columnId, 'fldGD'); + }); + + it('still strips an UNresolvable rec filter', () => { + const cfg = { + filters: { + conjunction: 'and', + filterSet: [{ columnId: 'fldG', operator: '=', value: [recMiss] }], + }, + }; + assert.equal(remapViewConfig(cfg, idmap).filters.filterSet.length, 0); + }); + + it('drops unresolvable ids from a mixed array (partial remap)', () => { + const cfg = { + filters: { + conjunction: 'and', + filterSet: [{ columnId: 'fldG', operator: '=', value: [recSrc, recMiss] }], + }, + }; + const out = remapViewConfig(cfg, idmap); + // recSrc -> recDst; recMiss dropped; leaf kept with resolved subset + assert.equal(out.filters.filterSet.length, 1); + assert.deepEqual(out.filters.filterSet[0].value, [recDst]); + }); + + it('drops entire leaf when all array elements are unresolvable', () => { + const recMiss2 = 'recALSOGONEXXXXX'; + const cfg = { + filters: { + conjunction: 'and', + filterSet: [{ columnId: 'fldG', operator: '=', value: [recMiss, recMiss2] }], + }, + }; + assert.equal(remapViewConfig(cfg, idmap).filters.filterSet.length, 0); + }); + + it('collaborator usr ids are still stripped (not in idmap.records)', () => { + const USR = 'usrCCCCCCCCCCCCCC'; + const cfg = { + filters: { + conjunction: 'and', + filterSet: [{ columnId: 'fldG', operator: '=', value: USR }], + }, + }; + assert.equal(remapViewConfig(cfg, idmap).filters.filterSet.length, 0); + }); + + it('empty idmap.records -> strip -> leaf dropped (backward compat)', () => { + const emptyIdmap = { tables: {}, views: {}, fields: { fldG: { destFld: 'fldGD', choices: {} } }, records: {} }; + const cfg = { + filters: { + conjunction: 'and', + filterSet: [{ columnId: 'fldG', operator: '=', value: [recSrc] }], + }, + }; + assert.equal(remapViewConfig(cfg, emptyIdmap).filters.filterSet.length, 0); + }); +}); + +// ── canonicalizeViewConfig convergence with idmap.records ──────────────────── + +describe('canonicalizeViewConfig — record-ref convergence with idmap', () => { + const idmap = { + tables: {}, + views: {}, + fields: { fldG: { destFld: 'fldGD', choices: {} } }, + records: { [recSrc]: recDst }, + }; + const srcFldNames = { fldG: 'Game' }; + const destFldNames = { fldGD: 'Game' }; + + it('source (strip=true+idmap, recSrc->recDst) and dest (strip=false+idmap, recDst) canonicalize equal', () => { + const srcCfg = { + filters: { + conjunction: 'and', + filterSet: [{ columnId: 'fldG', operator: '=', value: [recSrc] }], + }, + }; + const destCfg = { + filters: { + conjunction: 'and', + filterSet: [{ columnId: 'fldGD', operator: '=', value: [recDst] }], + }, + }; + const srcCanon = canonicalizeViewConfig(srcCfg, srcFldNames, {}, true, idmap); + const destCanon = canonicalizeViewConfig(destCfg, destFldNames, {}, false, idmap); + assert.equal(srcCanon, destCanon); + }); + + it('empty idmap.records -> strip -> source canonic equals no-filter (backward compat)', () => { + const emptyIdmap = { tables: {}, views: {}, fields: { fldG: { destFld: 'fldGD', choices: {} } }, records: {} }; + const srcCfg = { + filters: { + conjunction: 'and', + filterSet: [{ columnId: 'fldG', operator: '=', value: [recSrc] }], + }, + }; + const srcCanon = canonicalizeViewConfig(srcCfg, srcFldNames, {}, true, emptyIdmap); + const noneCanon = canonicalizeViewConfig({ filters: null }, {}, {}, true, emptyIdmap); + assert.equal(srcCanon, noneCanon); + }); + + it('dest with unresolved (dangling) rec diverges from stripped source when no idmap.records', () => { + const emptyIdmap = { tables: {}, views: {}, fields: { fldGD: { destFld: 'fldGD', choices: {} } }, records: {} }; + const cfg = { + filters: { + conjunction: 'and', + filterSet: [{ columnId: 'fldGD', operator: '=', value: [recDst] }], + }, + }; + // Source: strip=true -> drops the rec leaf -> no filter + const srcCanon = canonicalizeViewConfig(cfg, destFldNames, {}, true, emptyIdmap); + // Dest: strip=false, no records in idmap -> keeps the dangling rec -> filter present + const destCanon = canonicalizeViewConfig(cfg, destFldNames, {}, false, emptyIdmap); + // They must diverge (forces cleanup apply) + assert.notEqual(srcCanon, destCanon); + }); +}); + +// ── reapplyViewFilters mock-client test ────────────────────────────────────── + +describe('reapplyViewFilters — mock client', () => { + function makeLimiter() { return { run: (fn) => fn() }; } + function makeJournal() { return {}; } + function makeNoop() { return () => {}; } + + it('calls updateViewFilters with remapped recDst filter for a resolvable view', async () => { + const calls = []; + const client = { + updateViewFilters: async (appId, viewId, filters) => { + calls.push({ appId, viewId, filters }); + return { ok: true }; + }, + }; + + const srcSnapshot = { + baseId: 'appSRC', + tables: [{ + id: 'tblSRCSRCSRCSRCS', + fields: [], + records: [], + views: [{ + id: 'viwSRCSRCSRCSRCS', + name: 'By Game', + type: 'grid', + personalForUserId: null, + config: { + filters: { + conjunction: 'and', + filterSet: [{ columnId: 'fldG', operator: '=', value: [recSrc] }], + }, + }, + }], + }], + }; + + const destSnapshot = { baseId: 'appDST', tables: [] }; + + const idmap = { + tables: { tblSRCSRCSRCSRCS: 'tblDSTDSTDSTDSTO' }, + views: { viwSRCSRCSRCSRCS: 'viwDSTDSTDSTDSTO' }, + fields: { fldG: { destFld: 'fldGD', choices: {} } }, + records: { [recSrc]: recDst }, + }; + + const result = { warnings: [], viewFiltersReapplied: 0 }; + + await reapplyViewFilters({ + client, + srcSnapshot, + destSnapshot, + idmap, + limiter: makeLimiter(), + journal: makeJournal(), + persist: makeNoop(), + result, + }); + + assert.equal(calls.length, 1); + assert.equal(calls[0].appId, 'appDST'); + assert.equal(calls[0].viewId, 'viwDSTDSTDSTDSTO'); + assert.deepEqual(calls[0].filters.filterSet[0].value, [recDst]); + assert.equal(calls[0].filters.filterSet[0].columnId, 'fldGD'); + assert.equal(result.viewFiltersReapplied, 1); + assert.equal(result.warnings.length, 0); + }); + + it('skips views with no resolvable record refs (all unresolved)', async () => { + const calls = []; + const client = { + updateViewFilters: async (appId, viewId, filters) => { calls.push({ appId, viewId, filters }); return { ok: true }; }, + }; + + const srcSnapshot = { + baseId: 'appSRC', + tables: [{ + id: 'tblSRCSRCSRCSRCS', + fields: [], + records: [], + views: [{ + id: 'viwSRCSRCSRCSRCS', + name: 'Orphan filter', + type: 'grid', + personalForUserId: null, + config: { + filters: { + conjunction: 'and', + filterSet: [{ columnId: 'fldG', operator: '=', value: [recMiss] }], + }, + }, + }], + }], + }; + + const destSnapshot = { baseId: 'appDST', tables: [] }; + + const idmap = { + tables: { tblSRCSRCSRCSRCS: 'tblDSTDSTDSTDSTO' }, + views: { viwSRCSRCSRCSRCS: 'viwDSTDSTDSTDSTO' }, + fields: { fldG: { destFld: 'fldGD', choices: {} } }, + records: {}, // recMiss NOT in records + }; + + const result = { warnings: [], viewFiltersReapplied: 0 }; + + await reapplyViewFilters({ + client, + srcSnapshot, + destSnapshot, + idmap, + limiter: makeLimiter(), + journal: makeJournal(), + persist: makeNoop(), + result, + }); + + assert.equal(calls.length, 0); // should not call updateViewFilters + assert.equal(result.viewFiltersReapplied, 0); + }); + + it('skips personal views', async () => { + const calls = []; + const client = { + updateViewFilters: async (appId, viewId, filters) => { calls.push({ viewId }); return { ok: true }; }, + }; + + const srcSnapshot = { + baseId: 'appSRC', + tables: [{ + id: 'tblSRCSRCSRCSRCS', + fields: [], + records: [], + views: [{ + id: 'viwPersonalXXXXXX', + name: 'My View', + type: 'grid', + personalForUserId: 'usrSOMEUSER12345', + config: { + filters: { + conjunction: 'and', + filterSet: [{ columnId: 'fldG', operator: '=', value: [recSrc] }], + }, + }, + }], + }], + }; + + const destSnapshot = { baseId: 'appDST', tables: [] }; + + const idmap = { + tables: { tblSRCSRCSRCSRCS: 'tblDSTDSTDSTDSTO' }, + views: { viwPersonalXXXXXX: 'viwDPersonalXXXXX' }, + fields: { fldG: { destFld: 'fldGD', choices: {} } }, + records: { [recSrc]: recDst }, + }; + + const result = { warnings: [], viewFiltersReapplied: 0 }; + + await reapplyViewFilters({ + client, + srcSnapshot, + destSnapshot, + idmap, + limiter: makeLimiter(), + journal: makeJournal(), + persist: makeNoop(), + result, + }); + + assert.equal(calls.length, 0); + assert.equal(result.viewFiltersReapplied, 0); + }); + + it('continues on failure and pushes VIEW_FILTER_REAPPLY_FAILED warning', async () => { + const client = { + updateViewFilters: async () => { throw new Error('network error'); }, + }; + + const srcSnapshot = { + baseId: 'appSRC', + tables: [{ + id: 'tblSRCSRCSRCSRCS', + fields: [], + records: [], + views: [{ + id: 'viwSRCSRCSRCSRCS', + name: 'Test View', + type: 'grid', + personalForUserId: null, + config: { + filters: { + conjunction: 'and', + filterSet: [{ columnId: 'fldG', operator: '=', value: [recSrc] }], + }, + }, + }], + }], + }; + + const destSnapshot = { baseId: 'appDST', tables: [] }; + + const idmap = { + tables: { tblSRCSRCSRCSRCS: 'tblDSTDSTDSTDSTO' }, + views: { viwSRCSRCSRCSRCS: 'viwDSTDSTDSTDSTO' }, + fields: { fldG: { destFld: 'fldGD', choices: {} } }, + records: { [recSrc]: recDst }, + }; + + const result = { warnings: [], viewFiltersReapplied: 0 }; + + // Should not throw + await reapplyViewFilters({ + client, + srcSnapshot, + destSnapshot, + idmap, + limiter: makeLimiter(), + journal: makeJournal(), + persist: makeNoop(), + result, + }); + + assert.equal(result.viewFiltersReapplied, 0); + assert.equal(result.warnings.length, 1); + assert.equal(result.warnings[0].code, 'VIEW_FILTER_REAPPLY_FAILED'); + assert.ok(result.warnings[0].message.includes('viwSRCSRCSRCSRCS')); + assert.ok(result.warnings[0].message.includes('viwDSTDSTDSTDSTO')); + }); +}); From 950db47e50926c96619a3cc01551f35801b90680 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 13:34:44 +0300 Subject: [PATCH 066/246] feat(sync): wire records phase into apply + reconcile mode + pre-flight MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - records.js: add applyRecords() orchestrator — loads converged idmap, snapshots src+dest schema, attaches records via snapshotTableRecords, runs Pass1 → destDisplayNames rebuild → Pass2 → attachments → reapplyViewFilters with persist after each phase; emits RECORD_COUNT warning when exactly 1000 rows returned (cap hit / possible truncation) - records.js: add reconcile() — existence-prune stale idmap.records entries (dest record absent from snapshot); naturalKeys stub emits warning - index.js: apply() calls applyRecords after applyPlan (!aborted); merges counts into result.records + concatenates warnings; records-phase throw → RECORDS_PHASE_FAILED warning (non-fatal) - index.js: export reconcile() delegating to records.js reconcile() - test-records-apply.test.js: 5 end-to-end mock-client tests (2-table link round-trip, result shape, RECORD_COUNT pre-flight, reconcile prune+keep); 156 tests total, 0 failures Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/index.js | 41 ++ packages/mcp-server/src/sync/records.js | 186 +++++++- .../test/sync/test-records-apply.test.js | 399 ++++++++++++++++++ 3 files changed, 625 insertions(+), 1 deletion(-) create mode 100644 packages/mcp-server/test/sync/test-records-apply.test.js diff --git a/packages/mcp-server/src/sync/index.js b/packages/mcp-server/src/sync/index.js index 37ea7bc..1fd4f4b 100644 --- a/packages/mcp-server/src/sync/index.js +++ b/packages/mcp-server/src/sync/index.js @@ -5,6 +5,7 @@ import { computePlan } from './diff.js'; import { renderPlan, renderApplyResult } from './report.js'; import { applyPlan } from './apply.js'; import { newJournal, loadJournal, saveJournal } from './journal.js'; +import { applyRecords as applyRecordsImpl, reconcile as reconcileImpl } from './records.js'; const ENGINE_VERSION = '2b'; @@ -76,10 +77,50 @@ export async function apply({ client, sourceBaseId, destBaseId, planId, runStart client, plan: fullPlan, destAppId: destBaseId, destSnapshot, idmap, journal, persist: (m, j) => { saveIdmap(sourceBaseId, destBaseId, m); saveJournal(sourceBaseId, destBaseId, j); }, }); + + // Records phase: runs after schema apply, only if not aborted + if (!result.aborted) { + try { + const recResult = await applyRecordsImpl({ client, sourceBaseId, destBaseId, planId, runStartedAt }); + // Merge records phase counts + warnings into the schema result + result.records = { + created: recResult.created, + updated: recResult.updated, + failed: recResult.failed, + attachmentsUploaded: recResult.attachmentsUploaded || 0, + viewFiltersReapplied: recResult.viewFiltersReapplied || 0, + }; + if (recResult.warnings && recResult.warnings.length) { + result.warnings = (result.warnings || []).concat(recResult.warnings); + } + } catch (err) { + // Records phase failure is non-fatal for the schema result + result.warnings = (result.warnings || []).concat([{ + code: 'RECORDS_PHASE_FAILED', + message: `Records phase threw: ${err.message}`, + }]); + } + } + saveState(sourceBaseId, destBaseId, { sourceBaseId, destBaseId, engineVersion: ENGINE_VERSION, lastPlanId: planId, lastApplyAt: runStartedAt }); return renderApplyResult(result); } +/** + * Reconcile mode: prune stale idmap.records entries after manual deletions in dest. + * Delegates to records.js reconcile(). + * + * @param {object} opts + * @param {object} opts.client - AirtableClient instance + * @param {string} opts.sourceBaseId + * @param {string} opts.destBaseId + * @param {object} [opts.naturalKeys] - { [tableName]: fieldName } for re-match (stub) + * @returns {Promise} - { created, updated, skipped, failed, warnings, idmap } + */ +export async function reconcile({ client, sourceBaseId, destBaseId, naturalKeys = {} }) { + return reconcileImpl({ client, sourceBaseId, destBaseId, naturalKeys }); +} + // On resume, merge the persisted (grown) idmap over the plan's base matches so this-run // creations from a prior crashed run survive. Persisted wins on key conflict (has grown). export function mergeIdmapsForTest(sourceBaseId, destBaseId, fullPlan) { diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index 702c58f..68eb03a 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -13,8 +13,11 @@ */ import { coercePass1Cell, partitionLinkValue, linkRecId } from './cells.js'; -import { withRetry } from './ratelimit.js'; +import { withRetry, createLimiter } from './ratelimit.js'; import { remapViewConfig, collectFilterRecordRefs } from './remap.js'; +import { snapshotBase, snapshotTableRecords } from './snapshot.js'; +import { loadIdmap, saveIdmap } from './idmap.js'; +import { newJournal, loadRecordsJournal, saveRecordsJournal } from './journal.js'; /** * Returns true if the field type is a multiSelect (arrays not accepted by updateRecords). @@ -521,3 +524,184 @@ export async function reapplyViewFilters({ client, srcSnapshot, destSnapshot, id persist(idmap, journal); } } + +// ────────────────────────────────────────────────────────────────────────────── +// applyRecords — top-level orchestrator +// ────────────────────────────────────────────────────────────────────────────── + +/** + * Top-level records phase orchestrator. + * + * 1. Loads the converged schema idmap (written by the schema apply phase). + * 2. Snapshots src + dest schema (no records yet). + * 3. Attaches records to each table snapshot via snapshotTableRecords. + * Emits RECORD_COUNT warning when exactly 1000 rows are returned (possibly truncated). + * 4. Builds a rate limiter and records journal (resumable). + * 5. Runs: Pass 1 (scalar/select upsert) → rebuild destDisplayNames → Pass 2 (links) + * → Pass 3 (attachments) → reapplyViewFilters. + * 6. Persists idmap + journal after each phase. + * 7. Returns a result accumulator. + * + * @param {object} opts + * @param {object} opts.client - AirtableClient instance + * @param {string} opts.sourceBaseId - source base app ID + * @param {string} opts.destBaseId - dest base app ID + * @param {string} opts.planId - plan ID (used for journal file name) + * @param {string} opts.runStartedAt - ISO timestamp of this run start + * @returns {Promise} - result accumulator + */ +export async function applyRecords({ client, sourceBaseId, destBaseId, planId, runStartedAt }) { + // 1. Load the converged idmap (produced by the schema apply phase) + const idmap = loadIdmap(sourceBaseId, destBaseId); + idmap.records ??= {}; + idmap.attachments ??= {}; + + // 2. Snapshot schemas + const [srcSnapshot, destSnapshot] = await Promise.all([ + snapshotBase(client, sourceBaseId), + snapshotBase(client, destBaseId), + ]); + + // 3. Attach records to each table in both snapshots + for (const table of srcSnapshot.tables) { + table.records = await snapshotTableRecords(client, sourceBaseId, table); + } + for (const table of destSnapshot.tables) { + table.records = await snapshotTableRecords(client, destBaseId, table); + } + + // 4. Infra: rate limiter + journal + persist + result + const limiter = createLimiter({ rps: 5 }); + const journal = loadRecordsJournal(sourceBaseId, destBaseId, planId) ?? newJournal(planId, runStartedAt); + const persist = (m, j) => { + saveIdmap(sourceBaseId, destBaseId, m); + saveRecordsJournal(sourceBaseId, destBaseId, j); + }; + const result = { + planId, + aborted: false, + created: 0, + updated: 0, + skipped: 0, + failed: 0, + warnings: [], + idmap, + }; + + // Pre-flight: emit RECORD_COUNT warning for tables that returned exactly 1000 rows + // (indicates possible truncation — pagination is a follow-up feature). + for (const table of srcSnapshot.tables) { + const count = (table.records || []).length; + if (count === 1000) { + result.warnings.push({ + code: 'RECORD_COUNT', + message: `Table "${table.name}" (${table.id}) has ${count} records; record sync pulls up to 1000 per table — results may be truncated (pagination is a follow-up)`, + }); + } + } + + // 5a. Pass 1: scalar / select upsert — fills idmap.records + await applyRecordsPass1({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result }); + persist(idmap, journal); + + // 5b. Rebuild destDisplayNames from idmap.records now populated by Pass 1. + // Key = dest record id, value = source primary cell value (string). + // The source primary string is the correct display name because dest content mirrors source. + const destDisplayNames = new Map(); + for (const srcTable of srcSnapshot.tables) { + const primaryFieldId = srcTable.primaryFieldId; + for (const srcRec of (srcTable.records || [])) { + const destRecId = idmap.records[srcRec.id]; + if (!destRecId) continue; + const primaryVal = primaryFieldId ? (srcRec.cellValuesByColumnId || {})[primaryFieldId] : undefined; + destDisplayNames.set(destRecId, primaryVal !== undefined ? String(primaryVal) : ''); + } + } + + // 5c. Pass 2: link cells + await applyRecordsPass2({ client, srcSnapshot, destSnapshot, idmap, destDisplayNames, limiter, journal, persist, result }); + persist(idmap, journal); + + // 5d. Pass 3: attachments + await applyAttachments({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result }); + persist(idmap, journal); + + // 5e. Reapply view filters whose record refs now resolve + await reapplyViewFilters({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result }); + persist(idmap, journal); + + return result; +} + +// ────────────────────────────────────────────────────────────────────────────── +// reconcile — existence-prune stale idmap.records entries +// ────────────────────────────────────────────────────────────────────────────── + +/** + * Reconcile mode: prune stale idmap.records entries. + * + * 1. Loads the persisted idmap. + * 2. Snapshots the DEST base schema only, then pulls dest records for each matched table. + * 3. Builds a Set of all live dest record IDs across all tables. + * 4. Drops any idmap.records[src]=dest entry whose dest ID is not present in the snapshot. + * 5. naturalKeys re-match is stubbed (emits warning when provided — extend when needed). + * 6. Saves the pruned idmap. + * 7. Returns a result shaped like { created:0, updated:0, skipped, failed:0, warnings, idmap }. + * `skipped` = number of pruned entries. + * + * @param {object} opts + * @param {object} opts.client - AirtableClient instance + * @param {string} opts.sourceBaseId - source base app ID + * @param {string} opts.destBaseId - dest base app ID + * @param {object} [opts.naturalKeys] - { [tableName]: fieldName } for re-match (optional) + * @returns {Promise} - { created, updated, skipped, failed, warnings, idmap } + */ +export async function reconcile({ client, sourceBaseId, destBaseId, naturalKeys = {} }) { + const idmap = loadIdmap(sourceBaseId, destBaseId); + idmap.records ??= {}; + + const result = { + created: 0, + updated: 0, + skipped: 0, + failed: 0, + warnings: [], + idmap, + }; + + // Snapshot dest schema + attach records + const destSnapshot = await snapshotBase(client, destBaseId); + for (const table of destSnapshot.tables) { + table.records = await snapshotTableRecords(client, destBaseId, table); + } + + // Build set of all live dest record IDs across all tables + const allLiveDestIds = new Set(); + for (const dt of destSnapshot.tables) { + for (const r of (dt.records || [])) { + allLiveDestIds.add(r.id); + } + } + + // Existence-prune + for (const [srcRecId, destRecId] of Object.entries(idmap.records)) { + if (!allLiveDestIds.has(destRecId)) { + delete idmap.records[srcRecId]; + result.skipped++; + } + } + + // naturalKeys re-match (stub: emit warning for each non-empty key) + for (const [tableName, fieldName] of Object.entries(naturalKeys)) { + if (fieldName) { + result.warnings.push({ + code: 'RECONCILE_NATURAL_KEY_NOT_IMPLEMENTED', + message: `naturalKeys re-match for table "${tableName}" (field "${fieldName}") is not yet implemented`, + }); + } + } + + saveIdmap(sourceBaseId, destBaseId, idmap); + + return result; +} diff --git a/packages/mcp-server/test/sync/test-records-apply.test.js b/packages/mcp-server/test/sync/test-records-apply.test.js new file mode 100644 index 0000000..9c9109c --- /dev/null +++ b/packages/mcp-server/test/sync/test-records-apply.test.js @@ -0,0 +1,399 @@ +/** + * test-records-apply.test.js + * + * Integration tests for applyRecords() orchestrator and reconcile(). + * Uses mock clients; no real network. + */ +import { describe, it, before, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { applyRecords, reconcile } from '../../src/sync/records.js'; +import { saveIdmap, loadIdmap } from '../../src/sync/idmap.js'; + +// ── helpers ───────────────────────────────────────────────────────────────── + +function makeInMemoryClient(srcData, destData) { + // srcData / destData: { tables: [{id, name, primaryColumnId, columns, views, records}] } + // `records` here is the raw rows array expected from queryRecords (not stored on table schema side). + + const clients = { + src: { + getApplicationData: async (appId) => { + return { data: { tableSchemas: JSON.parse(JSON.stringify(srcData.tables)) } }; + }, + queryRecords: async (appId, tableId, viewId, opts) => { + const tbl = srcData.tables.find((t) => t.id === tableId); + const rows = (tbl && tbl.records) ? tbl.records : []; + // Return up to 1000 rows (the cap tested by pre-flight) + return { summary: { rows: rows.slice(0, 1000).map((r) => ({ id: r.id, fields: r.cellValuesByColumnId || {} })) } }; + }, + getView: async () => ({}), + }, + dest: { + getApplicationData: async (appId) => { + return { data: { tableSchemas: JSON.parse(JSON.stringify(destData.tables)) } }; + }, + queryRecords: async (appId, tableId, viewId, opts) => { + const tbl = destData.tables.find((t) => t.id === tableId); + const rows = (tbl && tbl.records) ? tbl.records : []; + return { summary: { rows: rows.slice(0, 1000).map((r) => ({ id: r.id, fields: r.cellValuesByColumnId || {} })) } }; + }, + getView: async () => ({}), + }, + }; + return clients; +} + +/** + * Build a unified mock client whose behavior depends on the appId passed. + * sourceBaseId → srcData, destBaseId → destData. + */ +function makeCombinedClient(sourceBaseId, destBaseId, srcData, destData, extra = {}) { + const { clients } = { clients: makeInMemoryClient(srcData, destData) }; + + const createCalls = []; + const addLinkCalls = []; + + const client = { + getApplicationData: async (appId) => { + if (appId === sourceBaseId) return clients.src.getApplicationData(appId); + return clients.dest.getApplicationData(appId); + }, + queryRecords: async (appId, tableId, viewId, opts) => { + if (appId === sourceBaseId) return clients.src.queryRecords(appId, tableId, viewId, opts); + return clients.dest.queryRecords(appId, tableId, viewId, opts); + }, + getView: async (appId, viewId) => { + if (appId === sourceBaseId) return clients.src.getView(appId, viewId); + return clients.dest.getView(appId, viewId); + }, + createRecords: async (appId, tableId, rows) => { + const created = rows.map((r, i) => { + const rowId = 'recDest' + tableId.slice(-3) + String(i); + createCalls.push({ appId, tableId, rowId, sourceKey: r.sourceKey, cells: r.cellValuesByColumnId }); + return { rowId, sourceKey: r.sourceKey }; + }); + return { created, failed: [] }; + }, + updateRecords: async () => ({ updated: [], failed: [] }), + addLinkItems: async (appId, rowId, columnId, items) => { + addLinkCalls.push({ appId, rowId, columnId, items }); + return { ok: true, added: items.length }; + }, + uploadAttachment: async () => ({ ok: false, error: 'no-op in test' }), + updateViewFilters: async () => ({ ok: true }), + ...extra, + }; + + return { client, createCalls, addLinkCalls }; +} + +// ── fixtures ───────────────────────────────────────────────────────────────── + +const SRC_APP = 'appSrcSrcSrcSrcSS'; +const DEST_APP = 'appDstDstDstDstDD'; + +/** + * Two-table source: tableA (with a link field to tableB) and tableB. + */ +function makeTwoTableSrc() { + return { + tables: [ + { + id: 'tblSA', name: 'TableA', + primaryColumnId: 'fldSAName', + columns: [ + { id: 'fldSAName', name: 'Name', type: 'text', typeOptions: null, description: null }, + { id: 'fldSALink', name: 'LinkedB', type: 'foreignKey', + typeOptions: { foreignTableId: 'tblSB', relationship: 'many', symmetricColumnId: 'fldSBLink' }, + description: null }, + ], + views: [{ id: 'viwSA', name: 'Grid view', type: 'grid', personalForUserId: null }], + records: [ + { id: 'recSA1', cellValuesByColumnId: { fldSAName: 'Alpha', fldSALink: ['recSB1'] } }, + ], + }, + { + id: 'tblSB', name: 'TableB', + primaryColumnId: 'fldSBName', + columns: [ + { id: 'fldSBName', name: 'Name', type: 'text', typeOptions: null, description: null }, + { id: 'fldSBLink', name: 'TableA', type: 'foreignKey', + typeOptions: { foreignTableId: 'tblSA', relationship: 'one', symmetricColumnId: 'fldSALink' }, + description: null }, + ], + views: [{ id: 'viwSB', name: 'Grid view', type: 'grid', personalForUserId: null }], + records: [ + { id: 'recSB1', cellValuesByColumnId: { fldSBName: 'Beta' } }, + ], + }, + ], + }; +} + +/** + * Matching (empty) dest tables for the two-table source. + */ +function makeTwoTableDest() { + return { + tables: [ + { + id: 'tblDA', name: 'TableA', + primaryColumnId: 'fldDAName', + columns: [ + { id: 'fldDAName', name: 'Name', type: 'text', typeOptions: null, description: null }, + { id: 'fldDALink', name: 'LinkedB', type: 'foreignKey', + typeOptions: { foreignTableId: 'tblDB', relationship: 'many', symmetricColumnId: 'fldDBLink' }, + description: null }, + ], + views: [{ id: 'viwDA', name: 'Grid view', type: 'grid', personalForUserId: null }], + records: [], // empty dest + }, + { + id: 'tblDB', name: 'TableB', + primaryColumnId: 'fldDBName', + columns: [ + { id: 'fldDBName', name: 'Name', type: 'text', typeOptions: null, description: null }, + { id: 'fldDBLink', name: 'TableA', type: 'foreignKey', + typeOptions: { foreignTableId: 'tblDA', relationship: 'one', symmetricColumnId: 'fldDALink' }, + description: null }, + ], + views: [{ id: 'viwDB', name: 'Grid view', type: 'grid', personalForUserId: null }], + records: [], + }, + ], + }; +} + +// ── applyRecords end-to-end ─────────────────────────────────────────────────── + +describe('applyRecords — end-to-end orchestration (mock client)', () => { + let tmpDir; + + before(() => { + tmpDir = mkdtempSync(join(tmpdir(), 'apply-recs-')); + process.env.AIRTABLE_USER_MCP_HOME = tmpDir; + }); + + after(() => { + try { rmSync(tmpDir, { recursive: true, force: true }); } catch {} + }); + + it('creates records in both tables and populates idmap.records', async () => { + const srcData = makeTwoTableSrc(); + const destData = makeTwoTableDest(); + + // Pre-populate the converged schema idmap (what applyPlan would have written): + const preIdmap = { + tables: { tblSA: 'tblDA', tblSB: 'tblDB' }, + fields: { + fldSAName: { destFld: 'fldDAName', choices: {} }, + fldSALink: { destFld: 'fldDALink', choices: {} }, + fldSBName: { destFld: 'fldDBName', choices: {} }, + fldSBLink: { destFld: 'fldDBLink', choices: {} }, + }, + records: {}, + views: { viwSA: 'viwDA', viwSB: 'viwDB' }, + }; + saveIdmap(SRC_APP, DEST_APP, preIdmap); + + const { client, createCalls, addLinkCalls } = makeCombinedClient(SRC_APP, DEST_APP, srcData, destData); + + const result = await applyRecords({ + client, + sourceBaseId: SRC_APP, + destBaseId: DEST_APP, + planId: 'test-plan-1', + runStartedAt: new Date().toISOString(), + }); + + // Both source records created + assert.equal(result.created, 2, `expected 2 created, got ${result.created}`); + assert.equal(result.failed, 0, `expected 0 failed, got ${result.failed}`); + + // idmap.records populated for both source records + const savedIdmap = loadIdmap(SRC_APP, DEST_APP); + assert.ok(savedIdmap.records['recSA1'], 'recSA1 must be in idmap.records'); + assert.ok(savedIdmap.records['recSB1'], 'recSB1 must be in idmap.records'); + + // Pass 2: addLinkItems called for the link in TableA + assert.ok(addLinkCalls.length >= 1, 'addLinkItems should have been called for the link field'); + const linkCall = addLinkCalls[0]; + assert.equal(linkCall.columnId, 'fldDALink', 'link items should target the dest link field'); + assert.equal(linkCall.items.length, 1, 'exactly one link item for recSB1→recDB1'); + // The foreignRowId should be the dest record for recSB1 + assert.equal(linkCall.items[0].foreignRowId, savedIdmap.records['recSB1']); + }); + + it('returns result with expected shape (created, updated, skipped, failed, warnings)', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'apply-shape-')); + + const srcData = { tables: [{ id: 'tblS1', name: 'Simple', primaryColumnId: 'fldS1', columns: [{ id: 'fldS1', name: 'Name', type: 'text', typeOptions: null, description: null }], views: [{ id: 'vS1', name: 'Grid view', type: 'grid', personalForUserId: null }], records: [{ id: 'recSimple1', cellValuesByColumnId: { fldS1: 'Hello' } }] }] }; + const destData = { tables: [{ id: 'tblD1', name: 'Simple', primaryColumnId: 'fldD1', columns: [{ id: 'fldD1', name: 'Name', type: 'text', typeOptions: null, description: null }], views: [{ id: 'vD1', name: 'Grid view', type: 'grid', personalForUserId: null }], records: [] }] }; + + const srcApp = 'appSrcShapeXXXXXX'; + const destApp = 'appDstShapeXXXXXX'; + + saveIdmap(srcApp, destApp, { + tables: { tblS1: 'tblD1' }, + fields: { fldS1: { destFld: 'fldD1', choices: {} } }, + records: {}, + views: {}, + }); + + const { client } = makeCombinedClient(srcApp, destApp, srcData, destData); + const result = await applyRecords({ client, sourceBaseId: srcApp, destBaseId: destApp, planId: 'pShape', runStartedAt: 'ts' }); + + // Required shape + assert.ok(typeof result.created === 'number', 'result.created must be number'); + assert.ok(typeof result.updated === 'number', 'result.updated must be number'); + assert.ok(typeof result.skipped === 'number', 'result.skipped must be number'); + assert.ok(typeof result.failed === 'number', 'result.failed must be number'); + assert.ok(Array.isArray(result.warnings), 'result.warnings must be array'); + assert.equal(result.created, 1, 'one record should be created'); + }); + + it('emits RECORD_COUNT warning when snapshotTableRecords returns exactly 1000 rows', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'apply-cap-')); + + const srcApp = 'appSrcCapXXXXXXXX'; + const destApp = 'appDstCapXXXXXXXX'; + + // Build 1000 rows + const rows = Array.from({ length: 1000 }, (_, i) => ({ id: `recCap${i}`, cellValuesByColumnId: { fldC1: `val${i}` } })); + const srcData = { tables: [{ id: 'tblC1', name: 'Big', primaryColumnId: 'fldC1', columns: [{ id: 'fldC1', name: 'Name', type: 'text', typeOptions: null, description: null }], views: [{ id: 'vC1', name: 'Grid view', type: 'grid', personalForUserId: null }], records: rows }] }; + const destData = { tables: [{ id: 'tblCD1', name: 'Big', primaryColumnId: 'fldCD1', columns: [{ id: 'fldCD1', name: 'Name', type: 'text', typeOptions: null, description: null }], views: [{ id: 'vCD1', name: 'Grid view', type: 'grid', personalForUserId: null }], records: [] }] }; + + saveIdmap(srcApp, destApp, { + tables: { tblC1: 'tblCD1' }, + fields: { fldC1: { destFld: 'fldCD1', choices: {} } }, + records: {}, + views: {}, + }); + + const { client } = makeCombinedClient(srcApp, destApp, srcData, destData); + const result = await applyRecords({ client, sourceBaseId: srcApp, destBaseId: destApp, planId: 'pCap', runStartedAt: 'ts' }); + + const capWarn = result.warnings.find((w) => w.code === 'RECORD_COUNT'); + assert.ok(capWarn, 'expected RECORD_COUNT warning when table returns exactly 1000 rows'); + assert.ok(capWarn.message.includes('1000'), 'warning should mention the count'); + }); +}); + +// ── reconcile ───────────────────────────────────────────────────────────────── + +describe('reconcile — prunes stale idmap.records entries', () => { + let tmpDir; + + before(() => { + tmpDir = mkdtempSync(join(tmpdir(), 'reconcile-')); + process.env.AIRTABLE_USER_MCP_HOME = tmpDir; + }); + + after(() => { + try { rmSync(tmpDir, { recursive: true, force: true }); } catch {} + }); + + it('removes a stale idmap.records entry whose dest record is absent from snapshot', async () => { + const srcApp = 'appSrcRecXXXXXXXX'; + const destApp = 'appDstRecXXXXXXXX'; + + // idmap has recA1→recDestA1 (stale) and recA2→recDestA2 (live) + const staleIdmap = { + tables: { tblSA: 'tblDA' }, + fields: { fldSAName: { destFld: 'fldDAName', choices: {} } }, + records: { + recA1: 'recDestA1', // stale — not in dest snapshot + recA2: 'recDestA2', // live — still in dest snapshot + }, + views: {}, + }; + saveIdmap(srcApp, destApp, staleIdmap); + + // Dest snapshot has only recDestA2 (recDestA1 was deleted) + const destData = { + tables: [{ + id: 'tblDA', name: 'TableA', + primaryColumnId: 'fldDAName', + columns: [{ id: 'fldDAName', name: 'Name', type: 'text', typeOptions: null, description: null }], + views: [{ id: 'vDA', name: 'Grid view', type: 'grid', personalForUserId: null }], + records: [ + { id: 'recDestA2', cellValuesByColumnId: { fldDAName: 'Live Row' } }, + ], + }], + }; + + const client = { + getApplicationData: async () => ({ data: { tableSchemas: JSON.parse(JSON.stringify(destData.tables)) } }), + queryRecords: async (appId, tableId, viewId) => { + const tbl = destData.tables.find((t) => t.id === tableId); + const rows = (tbl && tbl.records) ? tbl.records : []; + return { summary: { rows: rows.map((r) => ({ id: r.id, fields: r.cellValuesByColumnId || {} })) } }; + }, + getView: async () => ({}), + }; + + const result = await reconcile({ + client, + sourceBaseId: srcApp, + destBaseId: destApp, + naturalKeys: {}, + }); + + const after = loadIdmap(srcApp, destApp); + + // stale entry pruned + assert.equal(after.records['recA1'], undefined, 'stale entry recA1→recDestA1 should be pruned'); + // live entry kept + assert.equal(after.records['recA2'], 'recDestA2', 'live entry recA2→recDestA2 should be kept'); + + // skipped count reflects pruned entries + assert.equal(result.skipped, 1, 'one entry pruned = skipped:1'); + assert.equal(result.failed, 0); + }); + + it('keeps entries whose dest record is present in the snapshot', async () => { + const srcApp = 'appSrcRecYYYYYYYY'; + const destApp = 'appDstRecYYYYYYYY'; + + const existingIdmap = { + tables: { tblSX: 'tblDX' }, + fields: {}, + records: { recSX1: 'recDX1', recSX2: 'recDX2' }, + views: {}, + }; + saveIdmap(srcApp, destApp, existingIdmap); + + const destData = { + tables: [{ + id: 'tblDX', name: 'X', + primaryColumnId: 'fldDX1', + columns: [{ id: 'fldDX1', name: 'Name', type: 'text', typeOptions: null, description: null }], + views: [{ id: 'vDX', name: 'Grid view', type: 'grid', personalForUserId: null }], + records: [ + { id: 'recDX1', cellValuesByColumnId: {} }, + { id: 'recDX2', cellValuesByColumnId: {} }, + ], + }], + }; + + const client = { + getApplicationData: async () => ({ data: { tableSchemas: JSON.parse(JSON.stringify(destData.tables)) } }), + queryRecords: async (appId, tableId) => { + const tbl = destData.tables.find((t) => t.id === tableId); + return { summary: { rows: ((tbl && tbl.records) || []).map((r) => ({ id: r.id, fields: {} })) } }; + }, + getView: async () => ({}), + }; + + const result = await reconcile({ client, sourceBaseId: srcApp, destBaseId: destApp, naturalKeys: {} }); + + const after = loadIdmap(srcApp, destApp); + assert.equal(after.records['recSX1'], 'recDX1', 'entry 1 should be kept'); + assert.equal(after.records['recSX2'], 'recDX2', 'entry 2 should be kept'); + assert.equal(result.skipped, 0, 'no pruning expected'); + }); +}); From 96d21f90b1067092bda13196542cf19fd8d1dea0 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 13:41:57 +0300 Subject: [PATCH 067/246] feat(sync): sync_base mode=reconcile; apply now syncs records; records counts in human output - Add `reconcile` to sync_base mode enum + naturalKeys optional param - Route mode=reconcile --> sync.reconcile() rendered via renderApplyResult - renderApplyResult: append records: created N / updated N / failed N / attachments N / viewFilters N line when result.records present - Confirm mode=apply already calls records phase (no signature change) - 4 new renderApplyResult tests (records present/absent, warnings, aborted) - tool-sync unchanged: sync_base already in sync category, no new tools/categories Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/index.js | 15 +++++-- packages/mcp-server/src/sync/report.js | 6 +++ .../mcp-server/test/sync/test-report.test.js | 41 ++++++++++++++++++- 3 files changed, 57 insertions(+), 5 deletions(-) diff --git a/packages/mcp-server/src/index.js b/packages/mcp-server/src/index.js index 1e27847..8cfd7b8 100644 --- a/packages/mcp-server/src/index.js +++ b/packages/mcp-server/src/index.js @@ -1578,15 +1578,16 @@ Note: "form title" is the view name itself — use rename_view to change it. "Fi // ── Sync Tools ── { name: 'sync_base', - description: 'Base-to-base schema sync. mode="plan" (read-only): compare source/dest schema by name and return an ordered plan of tables/fields to create or update, plus orphans + warnings; does NOT mutate. mode="apply": execute a saved plan against the destination — create tables, reconcile the primary, create scalar/link/computed fields (with source->dest reference remapping + formula validation), and apply non-destructive field updates. apply requires planId and aborts if the destination drifted since the plan. Type-changing retypes, deletions, records and views are out of scope for this release.', + description: 'Base-to-base schema sync. mode="plan" (read-only): compare source/dest schema by name and return an ordered plan of tables/fields to create or update, plus orphans + warnings; does NOT mutate. mode="apply": execute a saved plan against the destination — create tables, reconcile the primary, create scalar/link/computed fields (with source->dest reference remapping + formula validation), apply non-destructive field updates, and sync records. apply requires planId and aborts if the destination drifted since the plan. mode="reconcile": rebuild/repair the record map — existence-prune dead entries from the idmap, with optional natural-key re-match per table.', annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, inputSchema: { type: 'object', properties: { - mode: { type: 'string', enum: ['plan', 'apply'], description: '"plan" (read-only preview) or "apply" (execute a saved plan).' }, + mode: { type: 'string', enum: ['plan', 'apply', 'reconcile'], description: '"plan" (read-only preview), "apply" (execute a saved plan, includes records sync), or "reconcile" (rebuild/repair the record map: existence-prune dead entries; optional natural-key re-match).' }, sourceAppId: { type: 'string', description: 'Source base/application ID to copy schema FROM' }, destAppId: { type: 'string', description: 'Destination base/application ID to copy schema TO' }, planId: { type: 'string', description: 'Required for mode="apply": the planId returned by a prior mode="plan" run.' }, + naturalKeys: { type: 'object', description: 'Used only by mode="reconcile": map of tableName → fieldName to use as a natural key for re-matching records (e.g. { "Projects": "Name" }). Omit to run existence-prune only.', additionalProperties: { type: 'string' } }, debug: debugProp, }, required: ['mode', 'sourceAppId', 'destAppId'], @@ -2388,7 +2389,7 @@ const handlers = { // ── Sync ── - async sync_base({ mode, sourceAppId, destAppId, planId, debug }) { + async sync_base({ mode, sourceAppId, destAppId, planId, naturalKeys, debug }) { const sync = await import('./sync/index.js'); if (mode === 'plan') { const id = 'pln' + client._genRandomId(); @@ -2401,7 +2402,13 @@ const handlers = { const out = await sync.apply({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, planId, runStartedAt }); return ok({ planId, summary: out.human }, out.machine, debug); } - return err(`Unsupported mode "${mode}". Use "plan" or "apply".`); + if (mode === 'reconcile') { + const raw = await sync.reconcile({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, naturalKeys: naturalKeys || {} }); + const { renderApplyResult } = await import('./sync/report.js'); + const out = renderApplyResult({ ...raw, planId: 'reconcile' }); + return ok({ summary: out.human }, out.machine, debug); + } + return err(`Unsupported mode "${mode}". Use "plan", "apply", or "reconcile".`); }, // ── Meta: Tool Management ── diff --git a/packages/mcp-server/src/sync/report.js b/packages/mcp-server/src/sync/report.js index ea338e4..a764fe8 100644 --- a/packages/mcp-server/src/sync/report.js +++ b/packages/mcp-server/src/sync/report.js @@ -35,6 +35,12 @@ export function renderApplyResult(result) { ` skipped: ${result.skipped}`, ` failed: ${result.failed}`, ]; + if (result.records) { + const r = result.records; + lines.push( + ` records: created ${r.created} / updated ${r.updated} / failed ${r.failed} / attachments ${r.attachmentsUploaded} / viewFilters ${r.viewFiltersReapplied}` + ); + } if (result.warnings && result.warnings.length) { lines.push(' warnings:'); for (const w of result.warnings) lines.push(` - ${w.code}: ${w.message}`); diff --git a/packages/mcp-server/test/sync/test-report.test.js b/packages/mcp-server/test/sync/test-report.test.js index 6c260f1..613ab04 100644 --- a/packages/mcp-server/test/sync/test-report.test.js +++ b/packages/mcp-server/test/sync/test-report.test.js @@ -1,6 +1,6 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { renderPlan } from '../../src/sync/report.js'; +import { renderPlan, renderApplyResult } from '../../src/sync/report.js'; describe('report.renderPlan', () => { it('summarizes action/orphan/warning counts', () => { @@ -16,3 +16,42 @@ describe('report.renderPlan', () => { assert.match(out.human, /FIELD_CAP/); }); }); + +describe('report.renderApplyResult', () => { + const baseResult = { planId: 'plnTEST', created: 2, updated: 1, skipped: 0, failed: 0 }; + + it('omits records line when result.records is absent', () => { + const out = renderApplyResult(baseResult); + assert.match(out.human, /Apply plnTEST/); + assert.match(out.human, /created: 2/); + assert.doesNotMatch(out.human, /records:/); + assert.equal(out.machine, baseResult); + }); + + it('includes records line when result.records is present', () => { + const result = { + ...baseResult, + records: { created: 10, updated: 3, failed: 1, attachmentsUploaded: 2, viewFiltersReapplied: 0 }, + }; + const out = renderApplyResult(result); + assert.match(out.human, /records: created 10 \/ updated 3 \/ failed 1 \/ attachments 2 \/ viewFilters 0/); + }); + + it('includes warnings after records line', () => { + const result = { + ...baseResult, + records: { created: 5, updated: 0, failed: 0, attachmentsUploaded: 0, viewFiltersReapplied: 1 }, + warnings: [{ code: 'RECORDS_PHASE_FAILED', message: 'oops' }], + }; + const out = renderApplyResult(result); + assert.match(out.human, /records: created 5/); + assert.match(out.human, /RECORDS_PHASE_FAILED/); + }); + + it('renders aborted result without records line', () => { + const result = { aborted: true, reason: 'DRIFT', warnings: [{ code: 'DRIFT', message: 'Destination changed' }] }; + const out = renderApplyResult(result); + assert.match(out.human, /Apply aborted/); + assert.doesNotMatch(out.human, /records:/); + }); +}); From b7d5788413b8bc3c3d510163c8ca1ddb8b8cfe0d Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 17:14:11 +0300 Subject: [PATCH 068/246] fix(auth): bound session init/recovery with a timeout (records apply hang) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _recoverSession() -> _doInit() -> launchPersistentContext has no timeout. The per-request path has a 30s budget, but it's only checked BETWEEN attempts, so a hung recovery (browser relaunch blocked on a locked/contended .chrome-profile — the chrome exit-21 contention failure mode) blocks the serialized request queue indefinitely. The records apply (first code path to issue many sequential requests) tripped a recovery mid-run and froze after creating 1 record for hours. Add _doInitBounded(): races _doInit() against AIRTABLE_INIT_TIMEOUT_MS (default 90s) and rejects on timeout so the caller's retry/budget proceeds; a late-arriving context is closed to avoid leaking a Chromium onto the contended profile. Used by both init() and _recoverSession(). Regression test asserts both reject within the bound and that _recovering resets (queue not permanently blocked). Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/auth.js | 59 ++++++++++++++++++- .../test/test-auth-init-timeout.test.js | 39 ++++++++++++ 2 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 packages/mcp-server/test/test-auth-init-timeout.test.js diff --git a/packages/mcp-server/src/auth.js b/packages/mcp-server/src/auth.js index aa47a60..706baed 100644 --- a/packages/mcp-server/src/auth.js +++ b/packages/mcp-server/src/auth.js @@ -83,7 +83,7 @@ export class AirtableAuth { if (this._initPromise) return this._initPromise; if (this.context && this.isLoggedIn) return; - this._initPromise = this._doInit(); + this._initPromise = this._doInitBounded(); try { await this._initPromise; } finally { @@ -91,6 +91,33 @@ export class AirtableAuth { } } + // Bound the browser (re)launch so a locked/contended .chrome-profile can't block + // the serialized request queue forever. `_doInit` -> `launchPersistentContext` has no + // built-in timeout; without this watchdog a single hung relaunch (profile contention, + // chrome exit 21) stalls every queued request indefinitely. On timeout we reject so the + // caller's retry/budget can proceed; a late-arriving context is closed to avoid leaking + // a Chromium onto the contended profile. + async _doInitBounded() { + const ms = Number(process.env.AIRTABLE_INIT_TIMEOUT_MS) || 90_000; + let timer; + let timedOut = false; + const inner = this._doInit(); + inner.then(() => { if (timedOut) this.context?.close?.().catch(() => {}); }, () => {}); + try { + return await Promise.race([ + inner, + new Promise((_, reject) => { + timer = setTimeout(() => { + timedOut = true; + reject(new Error(`session init/recovery exceeded ${ms / 1000}s — the browser profile may be locked by another process (profile contention)`)); + }, ms); + }), + ]); + } finally { + clearTimeout(timer); + } + } + async _doInit() { // Close existing context if recovering. Detach the prior network listener // from the old page so we don't leak a handler per recovery cycle. @@ -309,7 +336,8 @@ export class AirtableAuth { this.isLoggedIn = false; // Atomic assignment — init() observes this as already-in-progress and // awaits the same promise, preventing a parallel _doInit(). - const recovery = this._doInit(); + // Bounded so a hung relaunch (locked profile) can't block the queue forever. + const recovery = this._doInitBounded(); this._initPromise = recovery; try { await recovery; @@ -528,6 +556,33 @@ export class AirtableAuth { } } + /** + * Lightweight session check that reuses the already-open browser context — + * it never launches a second Chrome. The daemon exposes this via + * /daemon/session-health so the extension can verify the session through the + * daemon's single shared browser instead of forking a competing health-check + * (two Chromes on one persistent profile collide → Chrome exit code 21). + * + * @returns {Promise<{valid: boolean, userId?: string|null, status?: number, error?: string}>} + */ + async checkSessionHealth() { + try { + await this.ensureLoggedIn(); + const res = await this.get('/v0.3/getUserProperties'); + if (res.ok) { + let userId = this.userId; + try { + const data = await res.json(); + userId = data?.data?.userId ?? userId; + } catch { /* keep cached userId */ } + return { valid: true, userId: userId ?? null }; + } + return { valid: false, status: res.status }; + } catch (err) { + return { valid: false, error: err instanceof Error ? err.message : String(err) }; + } + } + async get(url, appId, { evalTimeoutMs = 15_000 } = {}) { const pattern = url.replace(/.*v0\.3\//, '').replace(/(app|tbl|viw|fld|rec|usr|wsp|sel|flt|blk|ext|col)[A-Za-z0-9]{10,}/g, '$1*'); trace('http', 'http:request', { method: 'GET', endpoint_pattern: pattern, has_payload: false }); diff --git a/packages/mcp-server/test/test-auth-init-timeout.test.js b/packages/mcp-server/test/test-auth-init-timeout.test.js new file mode 100644 index 0000000..fbbe70d --- /dev/null +++ b/packages/mcp-server/test/test-auth-init-timeout.test.js @@ -0,0 +1,39 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { AirtableAuth } from '../src/auth.js'; + +// Regression: a hung browser (re)launch (launchPersistentContext on a locked/contended +// .chrome-profile) must NOT block forever. Before _doInitBounded, init()/_recoverSession +// awaited an unbounded _doInit(), so one hung relaunch froze the serialized request queue +// indefinitely (records apply stuck after 1 record for hours). The bounded wrapper rejects. +describe('AirtableAuth — bounded init/recovery (no infinite hang on a locked profile)', () => { + function withShortTimeout(fn) { + const prev = process.env.AIRTABLE_INIT_TIMEOUT_MS; + process.env.AIRTABLE_INIT_TIMEOUT_MS = '50'; + return Promise.resolve(fn()).finally(() => { + if (prev === undefined) delete process.env.AIRTABLE_INIT_TIMEOUT_MS; + else process.env.AIRTABLE_INIT_TIMEOUT_MS = prev; + }); + } + + it('init() rejects within the timeout when _doInit hangs', () => withShortTimeout(async () => { + const auth = new AirtableAuth(); + auth._doInit = () => new Promise(() => {}); // simulate a hung launchPersistentContext + await assert.rejects(() => auth.init(), /exceeded .* the browser profile may be locked/); + })); + + it('_recoverSession() rejects within the timeout and resets _recovering (queue not permanently blocked)', () => withShortTimeout(async () => { + const auth = new AirtableAuth(); + auth._doInit = () => new Promise(() => {}); + await assert.rejects(() => auth._recoverSession(), /exceeded/); + assert.equal(auth._recovering, false); // flag reset so subsequent calls can proceed + })); + + it('init() still succeeds normally when _doInit resolves promptly', () => withShortTimeout(async () => { + const auth = new AirtableAuth(); + let called = 0; + auth._doInit = async () => { called++; }; + await auth.init(); + assert.equal(called, 1); + })); +}); From 78a254dd7da710cbb65487768177bce0f55287ef Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 19:09:51 +0300 Subject: [PATCH 069/246] perf(sync): records phase uses schema-only snapshot (eliminate getView read storm) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit applyRecords (and reconcile) called snapshotBase, which runs snapshotViews — one getView (a ~1s table/readData call) PER collaborative view, on BOTH bases. On a view-heavy base (this one has hundreds of views) that is hundreds of slow sequential reads BEFORE any record work, which read as an hours-long "hang" (records stuck at 1; the process was blocked in the read storm, not looping). Instrumentation confirmed: 239 requests in 90s, almost all table/readData. The records phase needs schema (tables/fields) + records, NOT view configs. Add snapshotSchemaOnly() (getApplicationData -> normalizeSchema, no view reads) and use it in applyRecords + reconcile. reapplyViewFilters reads SOURCE view configs lazily (source-only, once, at the end) since it genuinely needs filter configs. Verified: the upfront 2x view storm is gone and createRecords now flows. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/records.js | 24 ++++++++++++++++++------ packages/mcp-server/src/sync/snapshot.js | 13 +++++++++++++ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index 68eb03a..a4786cf 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -15,7 +15,7 @@ import { coercePass1Cell, partitionLinkValue, linkRecId } from './cells.js'; import { withRetry, createLimiter } from './ratelimit.js'; import { remapViewConfig, collectFilterRecordRefs } from './remap.js'; -import { snapshotBase, snapshotTableRecords } from './snapshot.js'; +import { snapshotSchemaOnly, snapshotViews, snapshotTableRecords } from './snapshot.js'; import { loadIdmap, saveIdmap } from './idmap.js'; import { newJournal, loadRecordsJournal, saveRecordsJournal } from './journal.js'; @@ -489,6 +489,14 @@ export async function reapplyViewFilters({ client, srcSnapshot, destSnapshot, id const destAppId = destSnapshot.baseId || ''; const recs = (idmap && idmap.records) || {}; + // applyRecords takes a SCHEMA-ONLY snapshot (no view configs) to avoid a getView storm. + // Populate SOURCE view live-configs now (source-only, once, at the end) so we can find and + // remap the record-ref filters. If configs are already present this is a no-op-ish refresh. + if (typeof client.getView === 'function') { + const needsConfig = (srcSnapshot.tables || []).some((t) => (t.views || []).some((v) => !v.personalForUserId && !v.config)); + if (needsConfig) await snapshotViews(client, srcSnapshot.baseId || '', srcSnapshot); + } + for (const srcTable of (srcSnapshot.tables || [])) { const destTableId = idmap.tables[srcTable.id]; if (!destTableId) continue; // table not matched -> skip @@ -556,10 +564,13 @@ export async function applyRecords({ client, sourceBaseId, destBaseId, planId, r idmap.records ??= {}; idmap.attachments ??= {}; - // 2. Snapshot schemas + // 2. Snapshot schemas ONLY (no per-view live-config reads — those are a getView/readData + // storm: ~1s per view, hundreds of views on a view-heavy base, on BOTH bases. The records + // phase needs schema + records, not view configs. reapplyViewFilters reads source view + // configs lazily, source-only, at the end.) const [srcSnapshot, destSnapshot] = await Promise.all([ - snapshotBase(client, sourceBaseId), - snapshotBase(client, destBaseId), + snapshotSchemaOnly(client, sourceBaseId), + snapshotSchemaOnly(client, destBaseId), ]); // 3. Attach records to each table in both snapshots @@ -669,8 +680,9 @@ export async function reconcile({ client, sourceBaseId, destBaseId, naturalKeys idmap, }; - // Snapshot dest schema + attach records - const destSnapshot = await snapshotBase(client, destBaseId); + // Snapshot dest schema + attach records (schema-only — reconcile needs record existence, + // not view configs; avoids the getView/readData storm). + const destSnapshot = await snapshotSchemaOnly(client, destBaseId); for (const table of destSnapshot.tables) { table.records = await snapshotTableRecords(client, destBaseId, table); } diff --git a/packages/mcp-server/src/sync/snapshot.js b/packages/mcp-server/src/sync/snapshot.js index 6c0ee4c..652040f 100644 --- a/packages/mcp-server/src/sync/snapshot.js +++ b/packages/mcp-server/src/sync/snapshot.js @@ -98,6 +98,19 @@ export async function snapshotBase(client, appId) { return snap; } +/** + * Schema-only snapshot — like snapshotBase but WITHOUT the per-view live-config reads. + * snapshotViews() issues one getView (a table/readData call, ~1s) PER collaborative view — + * hundreds of slow sequential requests on a view-heavy base. The records phase only needs + * schema (tables/fields) + records, so it uses this; callers needing view configs (e.g. + * reapplyViewFilters) run snapshotViews() lazily and only on the base they need. + * @returns {Promise<{ baseId: string, tables: Array }>} + */ +export async function snapshotSchemaOnly(client, appId) { + const raw = await client.getApplicationData(appId); + return { baseId: appId, ...normalizeSchema(raw) }; +} + /** * Pull a table's records via its first collaborative view (single call, ≤1000 rows; * the internal readQueries endpoint has no cursor — Task-11 pre-flight warns on >1000). From 9a46f2733f4fca61afe885150e0a0a2e6e28f9d9 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 19:29:26 +0300 Subject: [PATCH 070/246] feat(sync): run the records phase as a background job (non-blocking apply + status poll) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A full record sync is minutes-long for large bases, so a single blocking sync_base apply call looks hung / times out. Make apply() launch the records phase fire-and-forget and return immediately with a jobId; the phase persists progress (idmap + records journal) and a status file (records-job-.json). - records.js: writeRecordsJobStatus / readRecordsJobStatus (status file per planId). - sync/index.js: apply() drift-check now uses snapshotSchemaOnly (fingerprint needs only view metadata, not the per-view getView/readData storm); records phase launched in background (.then/.catch write done/failed status); new recordsStatus() reader (status file + live idmap.records count). - report.js: renderApplyResult shows the "started in background (jobId=…)" line. - index.js: sync_base gains mode="status" (poll by planId); apply description updated. 461 mcp tests green; tool-sync green (no new tool/category). Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/index.js | 20 +++++-- packages/mcp-server/src/sync/index.js | 60 +++++++++++-------- packages/mcp-server/src/sync/records.js | 22 ++++++- packages/mcp-server/src/sync/report.js | 10 +++- .../test/sync/test-records-job.test.js | 54 +++++++++++++++++ 5 files changed, 134 insertions(+), 32 deletions(-) create mode 100644 packages/mcp-server/test/sync/test-records-job.test.js diff --git a/packages/mcp-server/src/index.js b/packages/mcp-server/src/index.js index 8cfd7b8..a4f474f 100644 --- a/packages/mcp-server/src/index.js +++ b/packages/mcp-server/src/index.js @@ -1578,15 +1578,15 @@ Note: "form title" is the view name itself — use rename_view to change it. "Fi // ── Sync Tools ── { name: 'sync_base', - description: 'Base-to-base schema sync. mode="plan" (read-only): compare source/dest schema by name and return an ordered plan of tables/fields to create or update, plus orphans + warnings; does NOT mutate. mode="apply": execute a saved plan against the destination — create tables, reconcile the primary, create scalar/link/computed fields (with source->dest reference remapping + formula validation), apply non-destructive field updates, and sync records. apply requires planId and aborts if the destination drifted since the plan. mode="reconcile": rebuild/repair the record map — existence-prune dead entries from the idmap, with optional natural-key re-match per table.', + description: 'Base-to-base schema + record sync. mode="plan" (read-only): compare source/dest schema by name and return an ordered plan of tables/fields to create or update, plus orphans + warnings; does NOT mutate. mode="apply": execute a saved plan against the destination — create tables, reconcile the primary, create scalar/link/computed fields (with source->dest reference remapping + formula validation), apply non-destructive field updates; then start the RECORD sync (two-pass cells + links, attachments, view-filter restore) in the BACKGROUND and return immediately with a jobId. apply requires planId and aborts if the destination drifted since the plan. mode="status": poll the background record job (pass the apply planId as planId) for progress/completion. mode="reconcile": rebuild/repair the record map — existence-prune dead entries from the idmap, with optional natural-key re-match per table.', annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, inputSchema: { type: 'object', properties: { - mode: { type: 'string', enum: ['plan', 'apply', 'reconcile'], description: '"plan" (read-only preview), "apply" (execute a saved plan, includes records sync), or "reconcile" (rebuild/repair the record map: existence-prune dead entries; optional natural-key re-match).' }, + mode: { type: 'string', enum: ['plan', 'apply', 'reconcile', 'status'], description: '"plan" (read-only preview), "apply" (execute a saved plan; starts the record sync in the background and returns a jobId), "status" (poll the background record job by planId), or "reconcile" (rebuild/repair the record map: existence-prune dead entries; optional natural-key re-match).' }, sourceAppId: { type: 'string', description: 'Source base/application ID to copy schema FROM' }, destAppId: { type: 'string', description: 'Destination base/application ID to copy schema TO' }, - planId: { type: 'string', description: 'Required for mode="apply": the planId returned by a prior mode="plan" run.' }, + planId: { type: 'string', description: 'Required for mode="apply" (the planId from a prior mode="plan" run) and mode="status" (the same planId, which is the background record jobId).' }, naturalKeys: { type: 'object', description: 'Used only by mode="reconcile": map of tableName → fieldName to use as a natural key for re-matching records (e.g. { "Projects": "Name" }). Omit to run existence-prune only.', additionalProperties: { type: 'string' } }, debug: debugProp, }, @@ -2408,7 +2408,19 @@ const handlers = { const out = renderApplyResult({ ...raw, planId: 'reconcile' }); return ok({ summary: out.human }, out.machine, debug); } - return err(`Unsupported mode "${mode}". Use "plan", "apply", or "reconcile".`); + if (mode === 'status') { + if (!planId) return err('mode="status" requires planId (the jobId returned by mode="apply").'); + const raw = sync.recordsStatus({ sourceBaseId: sourceAppId, destBaseId: destAppId, planId }); + const summary = raw.status === 'running' + ? `Records job ${planId}: running — ${raw.recordsMapped} records mapped so far (started ${raw.startedAt})` + : raw.status === 'done' + ? `Records job ${planId}: done — ${JSON.stringify(raw.result)} (${raw.recordsMapped} records mapped)` + : raw.status === 'failed' + ? `Records job ${planId}: FAILED — ${raw.error}` + : `Records job ${planId}: ${raw.status}${raw.message ? ' — ' + raw.message : ''}`; + return ok({ planId, status: raw.status, summary }, raw, debug); + } + return err(`Unsupported mode "${mode}". Use "plan", "apply", "reconcile", or "status".`); }, // ── Meta: Tool Management ── diff --git a/packages/mcp-server/src/sync/index.js b/packages/mcp-server/src/sync/index.js index 1fd4f4b..35ca4a6 100644 --- a/packages/mcp-server/src/sync/index.js +++ b/packages/mcp-server/src/sync/index.js @@ -1,11 +1,11 @@ import { createHash } from 'node:crypto'; -import { snapshotBase } from './snapshot.js'; +import { snapshotBase, snapshotSchemaOnly } from './snapshot.js'; import { matchByName, saveIdmap, savePlan, saveState, loadPlan, loadIdmap } from './idmap.js'; import { computePlan } from './diff.js'; import { renderPlan, renderApplyResult } from './report.js'; import { applyPlan } from './apply.js'; import { newJournal, loadJournal, saveJournal } from './journal.js'; -import { applyRecords as applyRecordsImpl, reconcile as reconcileImpl } from './records.js'; +import { applyRecords as applyRecordsImpl, reconcile as reconcileImpl, writeRecordsJobStatus, readRecordsJobStatus } from './records.js'; const ENGINE_VERSION = '2b'; @@ -58,7 +58,9 @@ export async function apply({ client, sourceBaseId, destBaseId, planId, runStart const fullPlan = loadPlan(sourceBaseId, destBaseId, planId); if (!fullPlan) throw new Error(`No saved plan "${planId}" for ${sourceBaseId} -> ${destBaseId}. Run mode=plan first.`); - const destSnapshot = await snapshotBase(client, destBaseId); + // Schema-only: fingerprintSchema + buildIndex use table/field/view METADATA only, not per-view + // live config — so skip the getView/readData storm (one ~1s read per view) on the dest here. + const destSnapshot = await snapshotSchemaOnly(client, destBaseId); if (fingerprintSchema(destSnapshot) !== fullPlan.destFingerprint) { return renderApplyResult({ planId, aborted: true, reason: 'DRIFT', warnings: [{ code: 'DRIFT', message: `Destination changed since plan ${planId}. Re-run mode=plan.` }] }); } @@ -78,34 +80,44 @@ export async function apply({ client, sourceBaseId, destBaseId, planId, runStart persist: (m, j) => { saveIdmap(sourceBaseId, destBaseId, m); saveJournal(sourceBaseId, destBaseId, j); }, }); - // Records phase: runs after schema apply, only if not aborted + // Records phase: runs after schema apply, only if not aborted. It is minutes-long for large + // bases, so we launch it in the BACKGROUND (fire-and-forget) and return immediately — a single + // blocking apply call would look hung / time out. The phase persists progress (idmap + records + // journal) and writes a status file; poll via `sync_base mode=status`. if (!result.aborted) { - try { - const recResult = await applyRecordsImpl({ client, sourceBaseId, destBaseId, planId, runStartedAt }); - // Merge records phase counts + warnings into the schema result - result.records = { - created: recResult.created, - updated: recResult.updated, - failed: recResult.failed, - attachmentsUploaded: recResult.attachmentsUploaded || 0, - viewFiltersReapplied: recResult.viewFiltersReapplied || 0, - }; - if (recResult.warnings && recResult.warnings.length) { - result.warnings = (result.warnings || []).concat(recResult.warnings); - } - } catch (err) { - // Records phase failure is non-fatal for the schema result - result.warnings = (result.warnings || []).concat([{ - code: 'RECORDS_PHASE_FAILED', - message: `Records phase threw: ${err.message}`, - }]); - } + writeRecordsJobStatus(sourceBaseId, destBaseId, planId, { status: 'running', startedAt: runStartedAt }); + applyRecordsImpl({ client, sourceBaseId, destBaseId, planId, runStartedAt }) + .then((r) => writeRecordsJobStatus(sourceBaseId, destBaseId, planId, { + status: 'done', startedAt: runStartedAt, finishedAt: new Date().toISOString(), + result: { + created: r.created, updated: r.updated, failed: r.failed, + attachmentsUploaded: r.attachmentsUploaded || 0, viewFiltersReapplied: r.viewFiltersReapplied || 0, + warnings: (r.warnings || []).length, + }, + })) + .catch((err) => writeRecordsJobStatus(sourceBaseId, destBaseId, planId, { + status: 'failed', startedAt: runStartedAt, finishedAt: new Date().toISOString(), + error: String(err && err.message ? err.message : err), + })); + result.records = { status: 'running', jobId: planId }; } saveState(sourceBaseId, destBaseId, { sourceBaseId, destBaseId, engineVersion: ENGINE_VERSION, lastPlanId: planId, lastApplyAt: runStartedAt }); return renderApplyResult(result); } +/** + * Poll the background records job started by apply(). Reads the persisted status file and + * derives live progress (records mapped so far) from idmap.records. + * @returns {{ planId:string, status:string, recordsMapped:number, startedAt?:string, finishedAt?:string, result?:object, error?:string, message?:string }} + */ +export function recordsStatus({ sourceBaseId, destBaseId, planId }) { + const job = readRecordsJobStatus(sourceBaseId, destBaseId, planId) + || { planId, status: 'unknown', message: 'No records job found for this planId (run mode=apply first).' }; + const idmap = loadIdmap(sourceBaseId, destBaseId); + return { ...job, recordsMapped: Object.keys((idmap && idmap.records) || {}).length }; +} + /** * Reconcile mode: prune stale idmap.records entries after manual deletions in dest. * Delegates to records.js reconcile(). diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index a4786cf..aafeacc 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -12,12 +12,32 @@ * `persist(idmap, journal)` is called after each batch. */ +import { existsSync, readFileSync, mkdirSync } from 'node:fs'; +import { join } from 'node:path'; import { coercePass1Cell, partitionLinkValue, linkRecId } from './cells.js'; import { withRetry, createLimiter } from './ratelimit.js'; import { remapViewConfig, collectFilterRecordRefs } from './remap.js'; import { snapshotSchemaOnly, snapshotViews, snapshotTableRecords } from './snapshot.js'; -import { loadIdmap, saveIdmap } from './idmap.js'; +import { loadIdmap, saveIdmap, syncDir } from './idmap.js'; import { newJournal, loadRecordsJournal, saveRecordsJournal } from './journal.js'; +import { safeAtomicWriteFileSync } from '../safe-write.js'; + +// ── Records-job status file (records-job-.json) ── +// The records phase is minutes-long for large bases, so apply() launches it in the BACKGROUND +// and returns immediately. Status is persisted to disk (survives process restarts) and polled +// via sync_base mode=status. Progress (records mapped) is derived live from idmap.records. +function recordsJobPath(sourceBaseId, destBaseId, planId) { + return join(syncDir(sourceBaseId, destBaseId), `records-job-${planId}.json`); +} +export function writeRecordsJobStatus(sourceBaseId, destBaseId, planId, status) { + mkdirSync(syncDir(sourceBaseId, destBaseId), { recursive: true }); + safeAtomicWriteFileSync(recordsJobPath(sourceBaseId, destBaseId, planId), JSON.stringify({ planId, ...status }, null, 2)); +} +export function readRecordsJobStatus(sourceBaseId, destBaseId, planId) { + const p = recordsJobPath(sourceBaseId, destBaseId, planId); + if (!existsSync(p)) return null; + try { return JSON.parse(readFileSync(p, 'utf8')); } catch { return null; } +} /** * Returns true if the field type is a multiSelect (arrays not accepted by updateRecords). diff --git a/packages/mcp-server/src/sync/report.js b/packages/mcp-server/src/sync/report.js index a764fe8..5948112 100644 --- a/packages/mcp-server/src/sync/report.js +++ b/packages/mcp-server/src/sync/report.js @@ -37,9 +37,13 @@ export function renderApplyResult(result) { ]; if (result.records) { const r = result.records; - lines.push( - ` records: created ${r.created} / updated ${r.updated} / failed ${r.failed} / attachments ${r.attachmentsUploaded} / viewFilters ${r.viewFiltersReapplied}` - ); + if (r.status === 'running') { + lines.push(` records: started in background (jobId=${r.jobId}) — poll with sync_base mode=status, planId="${r.jobId}"`); + } else { + lines.push( + ` records: created ${r.created} / updated ${r.updated} / failed ${r.failed} / attachments ${r.attachmentsUploaded} / viewFilters ${r.viewFiltersReapplied}` + ); + } } if (result.warnings && result.warnings.length) { lines.push(' warnings:'); diff --git a/packages/mcp-server/test/sync/test-records-job.test.js b/packages/mcp-server/test/sync/test-records-job.test.js new file mode 100644 index 0000000..3b6c220 --- /dev/null +++ b/packages/mcp-server/test/sync/test-records-job.test.js @@ -0,0 +1,54 @@ +import { describe, it, beforeEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { writeRecordsJobStatus, readRecordsJobStatus } from '../../src/sync/records.js'; +import { recordsStatus } from '../../src/sync/index.js'; +import { saveIdmap } from '../../src/sync/idmap.js'; +import { renderApplyResult } from '../../src/sync/report.js'; + +const SRC = 'appSRC0000000000', DEST = 'appDST0000000000'; + +describe('records background-job status', () => { + beforeEach(() => { process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-job-')); }); + + it('writeRecordsJobStatus / readRecordsJobStatus round-trips, keyed per planId', () => { + writeRecordsJobStatus(SRC, DEST, 'pln1', { status: 'running', startedAt: 't0' }); + const j = readRecordsJobStatus(SRC, DEST, 'pln1'); + assert.equal(j.planId, 'pln1'); + assert.equal(j.status, 'running'); + assert.equal(j.startedAt, 't0'); + assert.equal(readRecordsJobStatus(SRC, DEST, 'other'), null); + }); + + it('recordsStatus merges the job file with the LIVE idmap.records count', () => { + saveIdmap(SRC, DEST, { tables: {}, fields: {}, records: { recA: 'recX', recB: 'recY' } }); + writeRecordsJobStatus(SRC, DEST, 'pln2', { status: 'running', startedAt: 't0' }); + const s = recordsStatus({ sourceBaseId: SRC, destBaseId: DEST, planId: 'pln2' }); + assert.equal(s.status, 'running'); + assert.equal(s.recordsMapped, 2); // live progress derived from idmap + }); + + it('recordsStatus returns status="unknown" when no job exists', () => { + const s = recordsStatus({ sourceBaseId: SRC, destBaseId: DEST, planId: 'plnNONE' }); + assert.equal(s.status, 'unknown'); + assert.equal(s.recordsMapped, 0); + }); + + it('done status carries the result summary', () => { + writeRecordsJobStatus(SRC, DEST, 'pln3', { status: 'done', startedAt: 't0', finishedAt: 't1', result: { created: 5, updated: 0, failed: 0, attachmentsUploaded: 1, viewFiltersReapplied: 2, warnings: 0 } }); + const s = recordsStatus({ sourceBaseId: SRC, destBaseId: DEST, planId: 'pln3' }); + assert.equal(s.status, 'done'); + assert.equal(s.result.created, 5); + assert.equal(s.result.viewFiltersReapplied, 2); + }); +}); + +describe('renderApplyResult — background records line', () => { + it('shows the running/jobId/poll line when records.status==="running"', () => { + const out = renderApplyResult({ planId: 'plnX', created: 0, updated: 0, skipped: 0, failed: 0, records: { status: 'running', jobId: 'plnX' } }); + assert.match(out.human, /records: started in background \(jobId=plnX\)/); + assert.match(out.human, /mode=status/); + }); +}); From 0038d4ae3a536370669b1557169694c29c6e37eb Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 19:50:42 +0300 Subject: [PATCH 071/246] docs(sync): document M3-records (background record sync, mode=status, reconcile) CLAUDE.md sync section, CHANGELOG, and mcp-server README updated: sync_base now syncs records (two-pass cells+links, attachments, view-filter restore) as a background job (apply->jobId, poll mode=status), plus mode=reconcile. Records removed from the engine's out-of-scope list (retypes/deletions remain, M4). Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 7 +++++++ CLAUDE.md | 2 +- packages/mcp-server/README.md | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fb5bf0..6159a31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how ## [Unreleased] +### MCP server — sync_base record sync (2026-06-17) + +#### Added +- `sync_base mode=apply`: after schema + view sync, launches a **background** record sync and returns a `jobId` immediately. Two-pass: Pass 1 creates/updates scalar + single/multi-select cells (computed fields + computed primary are never written) and fills a persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported). Attachments are then downloaded from source and re-uploaded to dest (deduped by filename+size). Record-referencing view filters that were stripped during view sync are restored once records exist. Rate-limited (~5 req/s), resumable (records journal), and continue-on-failure (per-record errors are warnings, not aborts). +- `sync_base mode=status`: poll a background record job by planId — returns running/done/failed plus live records-mapped count. +- `sync_base mode=reconcile`: rebuild/repair the record map — prunes dead entries and optionally re-matches by per-table natural key. + ### MCP server — sync_base collaborative view sync (2026-06-17) #### Added diff --git a/CLAUDE.md b/CLAUDE.md index 46d5779..f307d44 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,7 +65,7 @@ Key files: - `dev-tools/` — reverse-engineering helpers (capture.js, analyze.js, debug-*.js), **gitignored** — kept locally for future reference when Airtable changes their internal API - `src/daemon/` — daemon subsystem (see daemon subsection below) - `src/safe-write.js` — atomic JSON write helper (used by lockfile, token, and settings) -- `src/sync/` — base-to-base schema sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: plan | apply`) in the `sync` category. Out of scope: retypes, deletions, records. View sync (M3): `snapshot` also captures collaborative views + live config; `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported). +- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: plan | apply | status | reconcile`) in the `sync` category. Out of scope: retypes, deletions (M4). View sync: `snapshot` also captures collaborative views + live config; `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe by filename+size); then restores record-referencing view filters stripped during view sync. Rate-limited (~5 req/s), resumable (records journal), continue-on-failure (per-record errors are warnings). `mode=status` polls a background job by planId — returns running/done/failed + live records-mapped count. `mode=reconcile` rebuilds/repairs the record map (existence-prune + optional per-table natural-key re-match). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. #### packages/mcp-server — Daemon subsystem diff --git a/packages/mcp-server/README.md b/packages/mcp-server/README.md index 4205eca..f6cefe7 100644 --- a/packages/mcp-server/README.md +++ b/packages/mcp-server/README.md @@ -523,7 +523,7 @@ Saved row scaffolds Airtable surfaces under "+ Add record" and the row-create ex | Tool | Description | |:-----|:------------| -| `sync_base` | Copy a base's schema to another base. `mode=plan` snapshots both bases and produces a diff report (no writes). `mode=apply` executes a saved plan — creates tables (removing Airtable's auto-scaffolding fields), reconciles the primary field, creates scalar/link/computed fields with source→dest reference remapping and dest-space formula validation, applies non-destructive field updates, and syncs collaborative views (creates missing views + mirrors filters, sorts, group levels, field visibility + column order, frozen columns, color config, cover, calendar date columns, form metadata, and row height, with source→dest field+choice ID remapping). View sync is idempotent (convergent canonical compare); personal views are skipped and orphan destination views are reported (not deleted). Drift-guarded and resumable via an on-disk journal. Out of scope: type-changing retypes, deletions, records. | +| `sync_base` | Copy a base's schema, views, and records to another base. `mode=plan` snapshots both bases and produces a diff report (no writes). `mode=apply` executes a saved plan — creates tables, reconciles the primary field, creates scalar/link/computed fields with source→dest reference remapping and formula validation, applies non-destructive field updates, syncs collaborative views (idempotent; personal views skipped; orphan views reported), then launches **background** record sync and returns a `jobId` immediately. Record sync: Pass 1 writes scalar + select cells (computed fields never written) and builds a persisted rec→rec id map; Pass 2 writes linked-record cells (unresolved targets reported); then attachments (download→re-upload, deduped by filename+size); then restores record-referencing view filters. Rate-limited, resumable, continue-on-failure. `mode=status` polls the background job by planId (running/done/failed + live count). `mode=reconcile` repairs the record id map (existence-prune + optional natural-key re-match). Drift-guarded and resumable via an on-disk journal. Out of scope: type-changing retypes, deletions. | --- From 6c191e14f8756269ebe0433ba071706c5de8f015 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 20:00:11 +0300 Subject: [PATCH 072/246] =?UTF-8?q?fix(sync):=20final-review=20fixes=20?= =?UTF-8?q?=E2=80=94=20preserve=20records=20map=20across=20applies=20(C1),?= =?UTF-8?q?=20chunked=20persist=20(I2),=20doc=20accuracy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Final whole-branch review (opus) findings: - C1 (Critical, idempotency): a fresh re-apply (new planId) cloned the plan idmap (no records) and the schema phase persisted records={}, WIPING the rec->rec identity map — so the records phase re-created every record (duplicates) on every re-apply. Extract buildRunIdmap(): the fresh path now SEEDS records/attachments from the persisted idmap (resume path already merged them). Regression tests added; mergeIdmaps now merges attachments too. - I2 (Important): Pass 1 created a whole table in one createRecords call and persisted only after — a crash mid-table re-created the whole table and status showed 0 progress for minutes. Now chunk creates (50/chunk) and persist per chunk: bounded re-create on crash + live status progress. - I1/I3 (docs): the per-call limiter does not gate createRecords/updateRecords/addLinkItems internal per-row POSTs (throttle protection is the auth queue's 429 backoff); reconcile prunes only (natural-key re-match not implemented; does not de-duplicate). CLAUDE.md + CHANGELOG + README reworded to not overstate. 463 mcp tests green; tool-sync green. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 4 +-- CLAUDE.md | 2 +- packages/mcp-server/README.md | 2 +- packages/mcp-server/src/sync/index.js | 31 +++++++++++++------ packages/mcp-server/src/sync/records.js | 13 +++++--- .../test/sync/test-records-idmap.test.js | 24 +++++++++++++- 6 files changed, 57 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6159a31..072097d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,9 @@ Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how ### MCP server — sync_base record sync (2026-06-17) #### Added -- `sync_base mode=apply`: after schema + view sync, launches a **background** record sync and returns a `jobId` immediately. Two-pass: Pass 1 creates/updates scalar + single/multi-select cells (computed fields + computed primary are never written) and fills a persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported). Attachments are then downloaded from source and re-uploaded to dest (deduped by filename+size). Record-referencing view filters that were stripped during view sync are restored once records exist. Rate-limited (~5 req/s), resumable (records journal), and continue-on-failure (per-record errors are warnings, not aborts). +- `sync_base mode=apply`: after schema + view sync, launches a **background** record sync and returns a `jobId` immediately. Two-pass: Pass 1 creates/updates scalar + single/multi-select cells (computed fields + computed primary are never written) and fills a persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported). Attachments are then downloaded from source and re-uploaded to dest (deduped by filename+size). Record-referencing view filters that were stripped during view sync are restored once records exist. Throttling is handled by per-request 429 backoff in the auth queue; resumable (records journal, per-chunk persist) and continue-on-failure (per-record errors are warnings, not aborts). - `sync_base mode=status`: poll a background record job by planId — returns running/done/failed plus live records-mapped count. -- `sync_base mode=reconcile`: rebuild/repair the record map — prunes dead entries and optionally re-matches by per-table natural key. +- `sync_base mode=reconcile`: **prune** dead record-map entries (existence-prune). Per-table natural-key re-match is planned (not yet implemented); reconcile does not de-duplicate records. ### MCP server — sync_base collaborative view sync (2026-06-17) diff --git a/CLAUDE.md b/CLAUDE.md index f307d44..37b71db 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,7 +65,7 @@ Key files: - `dev-tools/` — reverse-engineering helpers (capture.js, analyze.js, debug-*.js), **gitignored** — kept locally for future reference when Airtable changes their internal API - `src/daemon/` — daemon subsystem (see daemon subsection below) - `src/safe-write.js` — atomic JSON write helper (used by lockfile, token, and settings) -- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: plan | apply | status | reconcile`) in the `sync` category. Out of scope: retypes, deletions (M4). View sync: `snapshot` also captures collaborative views + live config; `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe by filename+size); then restores record-referencing view filters stripped during view sync. Rate-limited (~5 req/s), resumable (records journal), continue-on-failure (per-record errors are warnings). `mode=status` polls a background job by planId — returns running/done/failed + live records-mapped count. `mode=reconcile` rebuilds/repairs the record map (existence-prune + optional per-table natural-key re-match). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. +- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: plan | apply | status | reconcile`) in the `sync` category. Out of scope: retypes, deletions (M4). View sync: `snapshot` also captures collaborative views + live config; `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe by filename+size); then restores record-referencing view filters stripped during view sync. Throttle handling = per-request 429 backoff in the auth queue (the per-call limiter does NOT gate the internal per-row POSTs inside createRecords/updateRecords/addLinkItems); resumable (records journal, per-chunk persist of ~50 creates); continue-on-failure (per-record errors are warnings). `mode=status` polls a background job by planId — returns running/done/failed + live records-mapped count. `mode=reconcile` **prunes** dead record-map entries (existence-prune; per-table natural-key re-match NOT yet implemented; reconcile does NOT de-duplicate records). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. #### packages/mcp-server — Daemon subsystem diff --git a/packages/mcp-server/README.md b/packages/mcp-server/README.md index f6cefe7..a336edf 100644 --- a/packages/mcp-server/README.md +++ b/packages/mcp-server/README.md @@ -523,7 +523,7 @@ Saved row scaffolds Airtable surfaces under "+ Add record" and the row-create ex | Tool | Description | |:-----|:------------| -| `sync_base` | Copy a base's schema, views, and records to another base. `mode=plan` snapshots both bases and produces a diff report (no writes). `mode=apply` executes a saved plan — creates tables, reconciles the primary field, creates scalar/link/computed fields with source→dest reference remapping and formula validation, applies non-destructive field updates, syncs collaborative views (idempotent; personal views skipped; orphan views reported), then launches **background** record sync and returns a `jobId` immediately. Record sync: Pass 1 writes scalar + select cells (computed fields never written) and builds a persisted rec→rec id map; Pass 2 writes linked-record cells (unresolved targets reported); then attachments (download→re-upload, deduped by filename+size); then restores record-referencing view filters. Rate-limited, resumable, continue-on-failure. `mode=status` polls the background job by planId (running/done/failed + live count). `mode=reconcile` repairs the record id map (existence-prune + optional natural-key re-match). Drift-guarded and resumable via an on-disk journal. Out of scope: type-changing retypes, deletions. | +| `sync_base` | Copy a base's schema, views, and records to another base. `mode=plan` snapshots both bases and produces a diff report (no writes). `mode=apply` executes a saved plan — creates tables, reconciles the primary field, creates scalar/link/computed fields with source→dest reference remapping and formula validation, applies non-destructive field updates, syncs collaborative views (idempotent; personal views skipped; orphan views reported), then launches **background** record sync and returns a `jobId` immediately. Record sync: Pass 1 writes scalar + select cells (computed fields never written) and builds a persisted rec→rec id map; Pass 2 writes linked-record cells (unresolved targets reported); then attachments (download→re-upload, deduped by filename+size); then restores record-referencing view filters. Throttling via per-request 429 backoff; resumable (records journal, per-chunk persist); continue-on-failure. `mode=status` polls the background job by planId (running/done/failed + live count). `mode=reconcile` **prunes** the record id map (existence-prune; natural-key re-match planned; does not de-duplicate). Drift-guarded and resumable via an on-disk journal. Out of scope: type-changing retypes, deletions. | --- diff --git a/packages/mcp-server/src/sync/index.js b/packages/mcp-server/src/sync/index.js index 35ca4a6..9c5a069 100644 --- a/packages/mcp-server/src/sync/index.js +++ b/packages/mcp-server/src/sync/index.js @@ -66,14 +66,8 @@ export async function apply({ client, sourceBaseId, destBaseId, planId, runStart } const journal = loadJournal(sourceBaseId, destBaseId, planId) ?? newJournal(planId, runStartedAt); - let idmap; - if (journal.actions.length > 0) { - idmap = mergeIdmaps(sourceBaseId, destBaseId, fullPlan); - } else { - idmap = JSON.parse(JSON.stringify(fullPlan.idmap)); - idmap.records ??= {}; - idmap.views ??= {}; - } + // C1: preserve the persisted records/attachments identity map across applies (see buildRunIdmap). + const idmap = buildRunIdmap(sourceBaseId, destBaseId, fullPlan, journal); const result = await applyPlan({ client, plan: fullPlan, destAppId: destBaseId, destSnapshot, idmap, journal, @@ -140,11 +134,28 @@ export function mergeIdmapsForTest(sourceBaseId, destBaseId, fullPlan) { return { tables: { ...fullPlan.idmap.tables, ...m.tables }, fields: { ...fullPlan.idmap.fields, ...m.fields }, - records: { ...fullPlan.idmap.records, ...m.records }, - views: { ...fullPlan.idmap.views, ...m.views }, + records: { ...(fullPlan.idmap.records || {}), ...(m.records || {}) }, + attachments: { ...(fullPlan.idmap.attachments || {}), ...(m.attachments || {}) }, + views: { ...(fullPlan.idmap.views || {}), ...(m.views || {}) }, }; } function mergeIdmaps(sourceBaseId, destBaseId, fullPlan) { return mergeIdmapsForTest(sourceBaseId, destBaseId, fullPlan); } + +// Build the idmap for an apply run. +// CRITICAL (C1 — record-duplication guard): the persisted records/attachments maps are the rec->rec +// identity and MUST survive across applies. A fresh plan's idmap has NO records, so on the fresh path +// we seed records/attachments from the ON-DISK idmap. Without this, the schema phase persists an empty +// records map and the records phase re-creates every record (duplicates) on every re-apply with a new +// planId. The resume path (journal has actions) goes through mergeIdmaps, which already preserves them. +export function buildRunIdmap(sourceBaseId, destBaseId, fullPlan, journal) { + if (journal.actions.length > 0) return mergeIdmaps(sourceBaseId, destBaseId, fullPlan); + const persisted = loadIdmap(sourceBaseId, destBaseId); + const idmap = JSON.parse(JSON.stringify(fullPlan.idmap)); + idmap.records = { ...(persisted.records || {}) }; + idmap.attachments = { ...(persisted.attachments || {}) }; + idmap.views ??= {}; + return idmap; +} diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index aafeacc..1149d2a 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -171,11 +171,16 @@ export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idm } } - // ── CREATE batch ────────────────────────────────────────────────────── - if (createRows.length > 0) { + // ── CREATE batches (chunked) ────────────────────────────────────────── + // Chunk + persist per chunk so progress is durable/visible: a crash loses at most one chunk + // (not the whole table → bounded duplication on resume), and idmap.records grows during the + // run so `sync_base mode=status` reports live progress instead of 0 until a huge table finishes. + const CREATE_CHUNK = 50; + for (let i = 0; i < createRows.length; i += CREATE_CHUNK) { + const chunk = createRows.slice(i, i + CREATE_CHUNK); try { const res = await limiter.run(() => - withRetry(() => client.createRecords(destAppId, destTableId, createRows, {})), + withRetry(() => client.createRecords(destAppId, destTableId, chunk, {})), ); for (const created of (res.created || [])) { idmap.records[created.sourceKey] = created.rowId; @@ -189,7 +194,7 @@ export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idm }); } } catch (err) { - result.failed += createRows.length; + result.failed += chunk.length; result.warnings.push({ code: 'RECORD_CREATE_FAILED', message: `Table ${destTableId}: createRecords threw: ${err.message}`, diff --git a/packages/mcp-server/test/sync/test-records-idmap.test.js b/packages/mcp-server/test/sync/test-records-idmap.test.js index d3310ad..6870628 100644 --- a/packages/mcp-server/test/sync/test-records-idmap.test.js +++ b/packages/mcp-server/test/sync/test-records-idmap.test.js @@ -4,7 +4,29 @@ import { join } from 'node:path'; import { mkdtempSync, mkdirSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { loadIdmap, saveIdmap } from '../../src/sync/idmap.js'; -import { mergeIdmapsForTest } from '../../src/sync/index.js'; +import { mergeIdmapsForTest, buildRunIdmap } from '../../src/sync/index.js'; + +describe('buildRunIdmap — records identity survives across applies (C1 duplication guard)', () => { + it('FRESH apply (empty journal) SEEDS records + attachments from the persisted idmap, not the plan', () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-c1-')); + // Persisted idmap holds the grown rec->rec identity from a prior apply. + saveIdmap('appAAAA', 'appBBBB', { tables: {}, fields: {}, records: { recA: 'recX' }, attachments: { 'photo.png|123': true } }); + // A fresh plan's idmap has NO records (plan is schema-only). + const fullPlan = { idmap: { tables: { tblS: 'tblD' }, fields: {}, views: {} } }; + const idmap = buildRunIdmap('appAAAA', 'appBBBB', fullPlan, { actions: [] }); + assert.deepEqual(idmap.records, { recA: 'recX' }); // preserved — NOT wiped (the bug) + assert.deepEqual(idmap.attachments, { 'photo.png|123': true }); // attachment dedupe map preserved + assert.equal(idmap.tables.tblS, 'tblD'); // plan's schema matches still present + }); + + it('RESUME (journal has actions) merges persisted records over the plan', () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-c1r-')); + saveIdmap('appAAAA', 'appBBBB', { tables: {}, fields: {}, records: { recA: 'recX' } }); + const fullPlan = { idmap: { tables: {}, fields: {}, records: {}, views: {} } }; + const idmap = buildRunIdmap('appAAAA', 'appBBBB', fullPlan, { actions: [{ idx: 0, status: 'done' }] }); + assert.equal(idmap.records.recA, 'recX'); + }); +}); describe('idmap records slot', () => { it('loadIdmap defaults to { tables: {}, fields: {}, records: {} } when file missing', () => { From 88a0c19bcb08d59768db3393d2015404f46c4880 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Wed, 17 Jun 2026 23:35:20 +0300 Subject: [PATCH 073/246] feat(sync): server-side cross-base attachment copy (fast-path) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces per-attachment download→upload loop with a 2-request server-side transfer: createDataTransferPolicy (step 1) obtains a signed policy from Airtable; pasteAttachmentsCrossBase (step 2) triggers server-to-server copy via pasteCells, which is idempotent (sets, not appends). Two new AirtableClient methods: createDataTransferPolicy and pasteAttachmentsCrossBase. applyAttachments reworked to use the fast-path per table, with uploadAttachmentFallback invoked for skipped attachments or on fast-path errors. All per-table errors are caught (no throw out of loop). 5 new unit tests in test-attachments-fastpath.test.js; full sync suite 172/172 passing. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/client.js | 82 ++++ packages/mcp-server/src/sync/records.js | 321 +++++++++++---- .../sync/test-attachments-fastpath.test.js | 384 ++++++++++++++++++ 3 files changed, 713 insertions(+), 74 deletions(-) create mode 100644 packages/mcp-server/test/sync/test-attachments-fastpath.test.js diff --git a/packages/mcp-server/src/client.js b/packages/mcp-server/src/client.js index 292e21b..dff0dfd 100644 --- a/packages/mcp-server/src/client.js +++ b/packages/mcp-server/src/client.js @@ -2487,6 +2487,88 @@ export class AirtableClient { }; } + /** + * Create a cross-base data transfer policy (step 1 of server-side attachment copy). + * + * POSTs to `application/{sourceAppId}/createDataTransferPolicyV2` and returns the + * `signedDataTransferPolicy` object needed by `pasteAttachmentsCrossBase`. + * + * @param {string} sourceAppId + * @param {string} tableId - SOURCE table ID + * @param {{rowId:string, columnId:string, attachmentId:string}[]} attachmentIdsWithCellLocation + * @returns {Promise} - signedDataTransferPolicy { version, data } + */ + async createDataTransferPolicy(sourceAppId, tableId, attachmentIdsWithCellLocation) { + const payload = { attachmentIdsWithCellLocation, tableId }; + const url = `https://airtable.com/v0.3/application/${sourceAppId}/createDataTransferPolicyV2`; + const res = await this.auth.postForm(url, this._mutationParams(payload, sourceAppId), sourceAppId); + + if (!res.ok) { + const errText = await res.text().catch(() => ''); + throw new Error(`createDataTransferPolicy failed (${res.status}): ${errText}`); + } + + const data = await res.json().catch(() => ({})); + return data?.data?.signedDataTransferPolicy; + } + + /** + * Paste attachments cross-base via server-side transfer (step 2 of cross-base attachment copy). + * + * POSTs to `table/{destTableId}/pasteCells` using a signed data transfer policy acquired from + * `createDataTransferPolicy`. The operation SETS (replaces) the target cells, so re-running is + * idempotent — no duplicate attachments accumulate. + * + * @param {string} destAppId + * @param {string} destTableId + * @param {object} opts + * @param {string} opts.viewId + * @param {string} opts.sourceAppId + * @param {string} opts.sourceTableId + * @param {Array} opts.sourceColumnConfigs - [{id, name, type, typeOptions}] + * @param {Array} opts.sourceCellValues2dArray - [rowIdx][colIdx] = attachment array + * @param {string[]} opts.targetRowIds + * @param {string[]} opts.targetColumnIds + * @param {object} opts.signedDataTransferPolicy + * @param {string[]} opts.sourceRowIds + * @returns {Promise} - { numUpdatedCells, pastedRowIds, pastedColumnIds, skippedAttachments, ... } + */ + async pasteAttachmentsCrossBase(destAppId, destTableId, { + viewId, + sourceAppId, + sourceTableId, + sourceColumnConfigs, + sourceCellValues2dArray, + targetRowIds, + targetColumnIds, + signedDataTransferPolicy, + sourceRowIds, + }) { + const payload = { + viewId, + sourceApplicationId: sourceAppId, + sourceTableId, + sourceColumnConfigs, + sourceCellValues2dArray, + targetRowIds, + targetColumnIds, + signedDataTransferPolicy, + isCut: false, + sourceRowIds, + }; + + const url = `https://airtable.com/v0.3/table/${destTableId}/pasteCells`; + const res = await this.auth.postForm(url, this._mutationParams(payload, destAppId), destAppId); + + if (!res.ok) { + const errText = await res.text().catch(() => ''); + throw new Error(`pasteAttachmentsCrossBase failed (${res.status}): ${errText}`); + } + + const data = await res.json().catch(() => ({})); + return data?.data || {}; + } + /** * Create records (one row per item). The client generates each rowId locally * (Airtable accepts a client-supplied rec ID in the URL), so the returned diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index 1149d2a..6409295 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -361,27 +361,85 @@ async function defaultFetchBytes(url) { const ATTACHMENT_TYPES = new Set(['multipleAttachments', 'multipleAttachment']); /** - * Pass 3 record sync: download source attachments → upload to dest base. + * Fallback: upload a single attachment via download→upload for cases where the + * server-side fast-path skipped or failed it. * - * For each matched table, for each source field of type `multipleAttachments` - * that has a dest field mapping, for each source record with a non-empty - * attachment cell whose src row maps (idmap.records[srcRecId]): - * - For each attachment { url, filename, size, type }: - * - Dedupe key = `${filename}|${size}`. Skip if already uploaded. - * - Fetch bytes from source URL (URLs expire ~2h — must download during run). - * - Upload to dest via client.uploadAttachment (rate-limited + retried). - * - On {ok:true}: record dedupe key, bump attachmentsUploaded counter. - * - On {ok:false} OR thrown fetch error: push ATTACHMENT_FAILED warning, CONTINUE. - * - persist(idmap, journal) after each source table. + * @param {object} opts + * @param {object} opts.client + * @param {string} opts.destAppId + * @param {string} opts.destRowId + * @param {string} opts.destFldId + * @param {object} opts.attachment - { url, filename, size, type } + * @param {object} opts.idmap + * @param {object} opts.result + * @param {Function} opts.fetchBytes + * @param {object} opts.limiter + */ +async function uploadAttachmentFallback({ client, destAppId, destRowId, destFldId, attachment, idmap, result, fetchBytes, limiter }) { + const { url, filename, size, type } = attachment; + if (!url || !filename) return; + + const dedupeKey = `${filename}|${size}`; + if (idmap.attachments[dedupeKey]) return; // already uploaded + + let bytes; + let contentType = type; + try { + const fetched = await fetchBytes(url); + bytes = fetched.bytes; + if (!contentType && fetched.contentType) contentType = fetched.contentType; + } catch (fetchErr) { + result.warnings.push({ + code: 'ATTACHMENT_FAILED', + message: `Attachment ${filename}: fetch failed — ${fetchErr.message}`, + }); + return; + } + + try { + const res = await limiter.run(() => + withRetry(() => + client.uploadAttachment(destAppId, destRowId, destFldId, { bytes, filename, contentType }), + ), + ); + if (res.ok) { + idmap.attachments[dedupeKey] = true; + result.attachmentsUploaded = (result.attachmentsUploaded || 0) + 1; + } else { + result.warnings.push({ + code: 'ATTACHMENT_FAILED', + message: `Attachment ${filename}: upload failed — ${res.error || 'unknown error'}`, + }); + } + } catch (uploadErr) { + result.warnings.push({ + code: 'ATTACHMENT_FAILED', + message: `Attachment ${filename}: upload threw — ${uploadErr.message}`, + }); + } +} + +/** + * Pass 3 record sync: server-side cross-base attachment copy (fast-path), with + * per-attachment download→upload fallback for skipped/failed items. + * + * Fast-path (per table with ≥1 mapped attachment field): + * 1. Collect included source records (mapped in idmap.records, have ≥1 attachment cell). + * 2. Build `attachmentIdsWithCellLocation` from all attachment objects in included cells. + * 3. Call `createDataTransferPolicy` (POST createDataTransferPolicyV2) to get the signed policy. + * 4. Call `pasteAttachmentsCrossBase` (POST pasteCells) which SETS the target cells (idempotent). + * 5. Bump result.attachmentsUploaded by pastedRowIds.length (or numUpdatedCells). * - * NOTE: {ok:true} from uploadAttachment IS the hosting verification per Task-0 spike. - * The spike confirmed dest attachment URLs are on dl.airtable.com after a successful upload. - * No separate per-attachment read-back re-query is performed (that would be an extra network - * call per attachment; the spike already proved {ok:true} guarantees hosting). + * Fallback (per skipped attachment or on fast-path error): + * - For res.skippedAttachments (non-empty): call uploadAttachmentFallback for each skipped att. + * - On thrown error from fast-path: warn ATTACHMENT_FALLBACK, iterate all included attachments + * through uploadAttachmentFallback. + * - Continue-on-failure: per-table errors never throw out of the loop. + * - persist(idmap, journal) after each table. * * @param {object} opts - * @param {object} opts.client - AirtableClient instance (must have uploadAttachment) - * @param {object} opts.srcSnapshot - source base snapshot (tables with records attached) + * @param {object} opts.client - AirtableClient instance (must have createDataTransferPolicy, pasteAttachmentsCrossBase, uploadAttachment) + * @param {object} opts.srcSnapshot - source base snapshot (tables with records attached; must have baseId) * @param {object} opts.destSnapshot - dest base snapshot (provides destAppId via baseId) * @param {object} opts.idmap - live id-map (.tables, .fields, .records, .attachments) * @param {object} opts.limiter - rate limiter from createLimiter() @@ -405,82 +463,197 @@ export async function applyAttachments({ // Initialize dedupe map if not already present (survives across calls for idempotency) idmap.attachments ??= {}; + const srcAppId = srcSnapshot.baseId || ''; const destAppId = destSnapshot.baseId || ''; for (const srcTable of (srcSnapshot.tables || [])) { const destTableId = idmap.tables[srcTable.id]; if (!destTableId) continue; // table not matched → skip - // Find attachment fields that have a dest mapping - const attFields = (srcTable.fields || []).filter( - (f) => ATTACHMENT_TYPES.has(f.type) && idmap.fields[f.id], - ); - if (attFields.length === 0) continue; + try { + // Find attachment fields that have a dest mapping + const attFields = (srcTable.fields || []).filter( + (f) => ATTACHMENT_TYPES.has(f.type) && idmap.fields[f.id], + ); + if (attFields.length === 0) continue; + + // Determine a dest viewId for pasteCells (requires a view context) + // Use the first collaborative (non-personal) view; fall back to first view of any type. + const destTable = (destSnapshot.tables || []).find((t) => t.id === destTableId); + const destViews = destTable?.views || []; + const destViewId = destViews.find((v) => !v.personalForUserId)?.id ?? destViews[0]?.id; + if (!destViewId) { + result.warnings.push({ + code: 'ATTACHMENT_FALLBACK', + message: `Table ${srcTable.id}: no dest view found for pasteCells — falling back to per-attachment upload`, + }); + // Fall through to fallback for all records/fields in this table + for (const rec of (srcTable.records || [])) { + const destRowId = idmap.records[rec.id]; + if (!destRowId) continue; + const srcCells = rec.cellValuesByColumnId || {}; + for (const field of attFields) { + const cellValue = srcCells[field.id]; + if (!cellValue || !Array.isArray(cellValue) || cellValue.length === 0) continue; + const destFldId = idmap.fields[field.id].destFld; + for (const att of cellValue) { + await uploadAttachmentFallback({ client, destAppId, destRowId, destFldId, attachment: att, idmap, result, fetchBytes, limiter }); + } + } + } + persist(idmap, journal); + continue; + } - for (const rec of (srcTable.records || [])) { - const destRowId = idmap.records[rec.id]; - if (!destRowId) continue; // src record not yet mapped → skip + // Collect included records: must be mapped AND have ≥1 attachment cell + const includedRecs = []; + for (const rec of (srcTable.records || [])) { + const destRowId = idmap.records[rec.id]; + if (!destRowId) continue; + const srcCells = rec.cellValuesByColumnId || {}; + const hasAtt = attFields.some((f) => { + const cv = srcCells[f.id]; + return Array.isArray(cv) && cv.length > 0; + }); + if (hasAtt) includedRecs.push({ rec, destRowId }); + } - const srcCells = rec.cellValuesByColumnId || {}; + if (includedRecs.length === 0) { + persist(idmap, journal); + continue; + } - for (const field of attFields) { - const cellValue = srcCells[field.id]; - if (!cellValue || !Array.isArray(cellValue) || cellValue.length === 0) continue; + // Build fast-path payload arrays + const sourceColumnConfigs = attFields.map((f) => ({ + id: f.id, + name: f.name, + type: f.type, + typeOptions: f.typeOptions || {}, + })); + const targetColumnIds = attFields.map((f) => idmap.fields[f.id].destFld); + const targetRowIds = includedRecs.map((r) => r.destRowId); + const sourceRowIds = includedRecs.map((r) => r.rec.id); + + // 2D grid: [rowIdx][colIdx] = attachment array (or []) + const sourceCellValues2dArray = includedRecs.map(({ rec }) => { + const srcCells = rec.cellValuesByColumnId || {}; + return attFields.map((f) => { + const cv = srcCells[f.id]; + return Array.isArray(cv) ? cv : []; + }); + }); - const destFldId = idmap.fields[field.id].destFld; + // Flat list of all attachment location refs for the policy request + const attachmentIdsWithCellLocation = []; + for (const { rec } of includedRecs) { + const srcCells = rec.cellValuesByColumnId || {}; + for (const field of attFields) { + const cv = srcCells[field.id]; + if (!Array.isArray(cv)) continue; + for (const att of cv) { + if (att.id) { + attachmentIdsWithCellLocation.push({ + rowId: rec.id, + columnId: field.id, + attachmentId: att.id, + }); + } + } + } + } + + if (attachmentIdsWithCellLocation.length === 0) { + persist(idmap, journal); + continue; + } - for (const attachment of cellValue) { - const { url, filename, size, type } = attachment; - if (!url || !filename) continue; + // Fast-path: createDataTransferPolicy → pasteAttachmentsCrossBase + let pasteRes; + try { + const policy = await limiter.run(() => + withRetry(() => + client.createDataTransferPolicy(srcAppId, srcTable.id, attachmentIdsWithCellLocation), + ), + ); - const dedupeKey = `${filename}|${size}`; - if (idmap.attachments[dedupeKey]) continue; // already uploaded → skip + pasteRes = await limiter.run(() => + withRetry(() => + client.pasteAttachmentsCrossBase(destAppId, destTableId, { + viewId: destViewId, + sourceAppId: srcAppId, + sourceTableId: srcTable.id, + sourceColumnConfigs, + sourceCellValues2dArray, + targetRowIds, + targetColumnIds, + signedDataTransferPolicy: policy, + sourceRowIds, + }), + ), + ); - // Fetch bytes from source (URLs expire ~2h) - let bytes; - let contentType = type; - try { - const fetched = await fetchBytes(url); - bytes = fetched.bytes; - if (!contentType && fetched.contentType) contentType = fetched.contentType; - } catch (fetchErr) { - result.warnings.push({ - code: 'ATTACHMENT_FAILED', - message: `Attachment ${filename}: fetch failed — ${fetchErr.message}`, - }); - continue; // next attachment, don't abort the record + // Count success by pastedRowIds (most reliable signal) + result.attachmentsUploaded = (result.attachmentsUploaded || 0) + + (pasteRes.pastedRowIds?.length ?? pasteRes.numUpdatedCells ?? 0); + + // Handle partial skips: fall back per skipped attachment + if (Array.isArray(pasteRes.skippedAttachments) && pasteRes.skippedAttachments.length > 0) { + result.warnings.push({ + code: 'ATTACHMENT_FALLBACK', + message: `Table ${srcTable.id}: ${pasteRes.skippedAttachments.length} attachment(s) skipped by pasteCells — falling back to per-attachment upload`, + }); + + // Build lookup: attachmentId → { rec, field } for fallback + const attLookup = new Map(); + for (const { rec } of includedRecs) { + const srcCells = rec.cellValuesByColumnId || {}; + for (const field of attFields) { + const cv = srcCells[field.id]; + if (!Array.isArray(cv)) continue; + for (const att of cv) { + if (att.id) attLookup.set(att.id, { rec, field, att }); + } + } } - // Upload to dest via limiter + retry - try { - const res = await limiter.run(() => - withRetry(() => - client.uploadAttachment(destAppId, destRowId, destFldId, { - bytes, - filename, - contentType, - }), - ), - ); - - if (res.ok) { - idmap.attachments[dedupeKey] = true; - result.attachmentsUploaded = (result.attachmentsUploaded || 0) + 1; - } else { - result.warnings.push({ - code: 'ATTACHMENT_FAILED', - message: `Attachment ${filename}: upload failed — ${res.error || 'unknown error'}`, - }); - // Don't set dedupe key on failure so a retry run can re-attempt + for (const skipped of pasteRes.skippedAttachments) { + const attId = skipped.attachmentId ?? skipped.id; + const entry = attLookup.get(attId); + if (!entry) continue; + const { rec, field, att } = entry; + const destRowId = idmap.records[rec.id]; + if (!destRowId) continue; + const destFldId = idmap.fields[field.id]?.destFld; + if (!destFldId) continue; + await uploadAttachmentFallback({ client, destAppId, destRowId, destFldId, attachment: att, idmap, result, fetchBytes, limiter }); + } + } + } catch (fastPathErr) { + // Fast-path threw — fall back to per-attachment upload for all included records + result.warnings.push({ + code: 'ATTACHMENT_FALLBACK', + message: `Table ${srcTable.id}: fast-path threw (${fastPathErr.message}) — falling back to per-attachment upload`, + }); + + for (const { rec, destRowId } of includedRecs) { + const srcCells = rec.cellValuesByColumnId || {}; + for (const field of attFields) { + const cv = srcCells[field.id]; + if (!Array.isArray(cv)) continue; + const destFldId = idmap.fields[field.id]?.destFld; + if (!destFldId) continue; + for (const att of cv) { + await uploadAttachmentFallback({ client, destAppId, destRowId, destFldId, attachment: att, idmap, result, fetchBytes, limiter }); } - } catch (uploadErr) { - result.warnings.push({ - code: 'ATTACHMENT_FAILED', - message: `Attachment ${filename}: upload threw — ${uploadErr.message}`, - }); } } } + } catch (tableErr) { + // Top-level per-table guard: never throw out of the loop + result.warnings.push({ + code: 'ATTACHMENT_FAILED', + message: `Table ${srcTable.id}: unexpected error — ${tableErr.message}`, + }); } persist(idmap, journal); diff --git a/packages/mcp-server/test/sync/test-attachments-fastpath.test.js b/packages/mcp-server/test/sync/test-attachments-fastpath.test.js new file mode 100644 index 0000000..4f03ad2 --- /dev/null +++ b/packages/mcp-server/test/sync/test-attachments-fastpath.test.js @@ -0,0 +1,384 @@ +/** + * test-attachments-fastpath.test.js + * + * Unit tests for the server-side cross-base attachment fast-path in applyAttachments. + * Covers: + * 1. Happy path — createDataTransferPolicy + pasteAttachmentsCrossBase called correctly. + * 2. Unmapped record exclusion — src records without an idmap.records entry are excluded. + * 3. Fallback path — when pasteAttachmentsCrossBase returns skippedAttachments, the + * per-attachment download+upload path is invoked for those items. + */ + +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { applyAttachments } from '../../src/sync/records.js'; +import { newJournal } from '../../src/sync/journal.js'; +import { createLimiter } from '../../src/sync/ratelimit.js'; + +function makeLimiter() { + return createLimiter({ rps: 1000, sleep: async () => {} }); +} + +// ────────────────────────────────────────────────────────────────────────────── +// Shared fixtures +// ────────────────────────────────────────────────────────────────────────────── + +/** + * Build a minimal test harness. Returns mutable `calls` for assertions and the + * core objects (client, idmap, srcSnapshot, destSnapshot, result, fetchBytes). + * + * @param {object} opts + * @param {object[]} [opts.skippedAttachments=[]] - returned by pasteAttachmentsCrossBase + * @param {boolean} [opts.fastPathThrows=false] - make pasteAttachmentsCrossBase throw + */ +function makeFixtures({ skippedAttachments = [], fastPathThrows = false } = {}) { + const calls = { + createDataTransferPolicy: [], + pasteAttachmentsCrossBase: [], + uploadAttachment: [], + fetchBytes: [], + }; + + const client = { + createDataTransferPolicy: async (sourceAppId, tableId, attachmentIdsWithCellLocation) => { + calls.createDataTransferPolicy.push({ sourceAppId, tableId, attachmentIdsWithCellLocation }); + return { version: 1, data: 'base64-policy-token' }; + }, + + pasteAttachmentsCrossBase: async (destAppId, destTableId, opts) => { + calls.pasteAttachmentsCrossBase.push({ destAppId, destTableId, opts }); + if (fastPathThrows) { + // Use status=422 and a non-transient message so withRetry does not retry + // (defaultIsTransient checks both err.status and err.message for "network"/"fetch"/etc.) + const err = new Error('pasteCells failed — schema mismatch (422)'); + err.status = 422; + throw err; + } + return { + numUpdatedCells: 2, + pastedRowIds: ['recD1', 'recD2'], + pastedColumnIds: ['fldAttD'], + skippedAttachments, + createdRowIdsDuringPaste: [], + createdForeignRowIdsDuringPaste: [], + }; + }, + + uploadAttachment: async (appId, rowId, columnId, payload) => { + calls.uploadAttachment.push({ appId, rowId, columnId, payload }); + return { ok: true, attachmentId: 'uploaded-' + payload.filename }; + }, + }; + + const fetchBytes = async (url) => { + calls.fetchBytes.push(url); + return { bytes: Buffer.from('fake-data'), contentType: 'image/png' }; + }; + + // Two source records, both mapped + const idmap = { + tables: { tblSrc: 'tblDest' }, + fields: { + fldAttSrc: { destFld: 'fldAttD' }, + }, + records: { + recS1: 'recD1', + recS2: 'recD2', + // recS_unmapped intentionally absent + }, + attachments: {}, + }; + + const srcSnapshot = { + baseId: 'appSrc', + tables: [{ + id: 'tblSrc', + name: 'Table A', + fields: [ + { id: 'fldAttSrc', name: 'Attachments', type: 'multipleAttachments', typeOptions: {} }, + ], + records: [ + { + id: 'recS1', + cellValuesByColumnId: { + fldAttSrc: [ + { id: 'att1', url: 'https://src/att1.png', filename: 'att1.png', size: 100, type: 'image/png' }, + ], + }, + }, + { + id: 'recS2', + cellValuesByColumnId: { + fldAttSrc: [ + { id: 'att2', url: 'https://src/att2.png', filename: 'att2.png', size: 200, type: 'image/png' }, + ], + }, + }, + { + // This record is NOT in idmap.records → must be excluded entirely + id: 'recS_unmapped', + cellValuesByColumnId: { + fldAttSrc: [ + { id: 'att99', url: 'https://src/att99.png', filename: 'att99.png', size: 999, type: 'image/png' }, + ], + }, + }, + ], + }], + }; + + const destSnapshot = { + baseId: 'appDest', + tables: [{ + id: 'tblDest', + name: 'Table A', + fields: [{ id: 'fldAttD', name: 'Attachments', type: 'multipleAttachments' }], + views: [ + { id: 'viwCollab1', name: 'Grid view', type: 'grid' }, // non-personal + ], + records: [], + }], + }; + + const result = { + created: 0, + updated: 0, + skipped: 0, + failed: 0, + attachmentsUploaded: 0, + warnings: [], + }; + + return { calls, client, fetchBytes, idmap, srcSnapshot, destSnapshot, result }; +} + +// ────────────────────────────────────────────────────────────────────────────── +// Test suite +// ────────────────────────────────────────────────────────────────────────────── + +describe('applyAttachments — fast-path (server-side cross-base copy)', () => { + it('happy path: createDataTransferPolicy called with correct src rowIds/colIds/attIds, pasteAttachmentsCrossBase called with correct payload, result.attachmentsUploaded incremented', async () => { + const { calls, client, fetchBytes, idmap, srcSnapshot, destSnapshot, result } = makeFixtures(); + + await applyAttachments({ + client, + srcSnapshot, + destSnapshot, + idmap, + limiter: makeLimiter(), + journal: newJournal('fp-1', 't'), + persist: () => {}, + result, + fetchBytes, + }); + + // createDataTransferPolicy must be called once + assert.equal(calls.createDataTransferPolicy.length, 1, 'createDataTransferPolicy called once'); + const policyCall = calls.createDataTransferPolicy[0]; + assert.equal(policyCall.sourceAppId, 'appSrc'); + assert.equal(policyCall.tableId, 'tblSrc'); + + // attachmentIdsWithCellLocation must contain entries for both mapped records only + const attLocs = policyCall.attachmentIdsWithCellLocation; + assert.equal(attLocs.length, 2, 'two attachments (unmapped record excluded)'); + + const attLocIds = attLocs.map((a) => a.attachmentId).sort(); + assert.deepEqual(attLocIds, ['att1', 'att2']); + + // rowIds in policy must be src rowIds (not dest) + const rowIds = attLocs.map((a) => a.rowId).sort(); + assert.deepEqual(rowIds, ['recS1', 'recS2']); + + // columnIds must be src column ids + const colIds = [...new Set(attLocs.map((a) => a.columnId))]; + assert.deepEqual(colIds, ['fldAttSrc']); + + // pasteAttachmentsCrossBase must be called once + assert.equal(calls.pasteAttachmentsCrossBase.length, 1, 'pasteAttachmentsCrossBase called once'); + const pasteCall = calls.pasteAttachmentsCrossBase[0]; + assert.equal(pasteCall.destAppId, 'appDest'); + assert.equal(pasteCall.destTableId, 'tblDest'); + + const pasteOpts = pasteCall.opts; + assert.equal(pasteOpts.sourceAppId, 'appSrc', 'sourceApplicationId = srcAppId'); + assert.equal(pasteOpts.sourceTableId, 'tblSrc'); + assert.deepEqual(pasteOpts.targetColumnIds, ['fldAttD'], 'dest column ids'); + + // targetRowIds must be dest row ids (recD1, recD2) + const targetRowIds = [...pasteOpts.targetRowIds].sort(); + assert.deepEqual(targetRowIds, ['recD1', 'recD2']); + + // sourceRowIds must be src row ids + const sourceRowIds = [...pasteOpts.sourceRowIds].sort(); + assert.deepEqual(sourceRowIds, ['recS1', 'recS2']); + + // signedDataTransferPolicy forwarded from createDataTransferPolicy return value + assert.deepEqual(pasteOpts.signedDataTransferPolicy, { version: 1, data: 'base64-policy-token' }); + + // sourceCellValues2dArray: 2 rows × 1 col, each cell is the source attachment array + assert.equal(pasteOpts.sourceCellValues2dArray.length, 2, '2 rows in 2d array'); + assert.equal(pasteOpts.sourceCellValues2dArray[0].length, 1, '1 col per row'); + + // result.attachmentsUploaded incremented (by pastedRowIds.length = 2) + assert.equal(result.attachmentsUploaded, 2, 'attachmentsUploaded = pastedRowIds.length'); + + // No fallback download calls (fast-path succeeded) + assert.equal(calls.fetchBytes.length, 0, 'no fetchBytes calls on fast-path success'); + assert.equal(calls.uploadAttachment.length, 0, 'no uploadAttachment calls on fast-path success'); + + // No warnings + assert.equal(result.warnings.length, 0, 'no warnings on happy path'); + }); + + it('unmapped record excluded: recS_unmapped absent from idmap.records is not in attachmentIdsWithCellLocation', async () => { + const { calls, client, fetchBytes, idmap, srcSnapshot, destSnapshot, result } = makeFixtures(); + + await applyAttachments({ + client, + srcSnapshot, + destSnapshot, + idmap, + limiter: makeLimiter(), + journal: newJournal('fp-2', 't'), + persist: () => {}, + result, + fetchBytes, + }); + + const policyCall = calls.createDataTransferPolicy[0]; + const attLocs = policyCall.attachmentIdsWithCellLocation; + + // att99 belongs to recS_unmapped — must be absent + const attIds = attLocs.map((a) => a.attachmentId); + assert.ok(!attIds.includes('att99'), 'att99 (unmapped record) must not appear in policy call'); + + // src row ids in policy must only be recS1 and recS2 + const rowIds = [...new Set(attLocs.map((a) => a.rowId))].sort(); + assert.deepEqual(rowIds, ['recS1', 'recS2'], 'only mapped src rowIds in policy call'); + + // targetRowIds in pasteCells must only be mapped dest rows + const pasteOpts = calls.pasteAttachmentsCrossBase[0].opts; + const targetRowIds = [...pasteOpts.targetRowIds].sort(); + assert.deepEqual(targetRowIds, ['recD1', 'recD2'], 'only mapped dest rowIds in pasteCells'); + }); + + it('fallback path: skippedAttachments triggers uploadAttachment + fetchBytes for the skipped item', async () => { + // att1 is returned as skipped by pasteAttachmentsCrossBase + const { calls, client, fetchBytes, idmap, srcSnapshot, destSnapshot, result } = makeFixtures({ + skippedAttachments: [{ attachmentId: 'att1', reason: 'expired' }], + }); + + await applyAttachments({ + client, + srcSnapshot, + destSnapshot, + idmap, + limiter: makeLimiter(), + journal: newJournal('fp-3', 't'), + persist: () => {}, + result, + fetchBytes, + }); + + // Fast-path still ran (both calls) + assert.equal(calls.createDataTransferPolicy.length, 1, 'createDataTransferPolicy still called'); + assert.equal(calls.pasteAttachmentsCrossBase.length, 1, 'pasteAttachmentsCrossBase still called'); + + // Fallback: fetchBytes called for the skipped attachment URL + assert.equal(calls.fetchBytes.length, 1, 'fetchBytes called once for skipped attachment'); + assert.ok(calls.fetchBytes[0].includes('att1.png'), 'fetchBytes URL matches skipped attachment'); + + // uploadAttachment called for the fallback + assert.equal(calls.uploadAttachment.length, 1, 'uploadAttachment called once for fallback'); + assert.equal(calls.uploadAttachment[0].appId, 'appDest'); + assert.equal(calls.uploadAttachment[0].rowId, 'recD1', 'fallback uploads to correct dest row'); + assert.equal(calls.uploadAttachment[0].columnId, 'fldAttD', 'fallback uploads to correct dest col'); + assert.equal(calls.uploadAttachment[0].payload.filename, 'att1.png'); + + // ATTACHMENT_FALLBACK warning emitted + const fallbackWarns = result.warnings.filter((w) => w.code === 'ATTACHMENT_FALLBACK'); + assert.equal(fallbackWarns.length, 1, 'one ATTACHMENT_FALLBACK warning for the skip'); + assert.ok(fallbackWarns[0].message.includes('1'), 'warning mentions skipped count'); + }); + + it('fast-path error: all included attachments fall back to per-attachment upload', async () => { + const { calls, client, fetchBytes, idmap, srcSnapshot, destSnapshot, result } = makeFixtures({ + fastPathThrows: true, + }); + + // Must not throw + await applyAttachments({ + client, + srcSnapshot, + destSnapshot, + idmap, + limiter: makeLimiter(), + journal: newJournal('fp-4', 't'), + persist: () => {}, + result, + fetchBytes, + }); + + // createDataTransferPolicy was called (fast-path started) + assert.equal(calls.createDataTransferPolicy.length, 1); + + // pasteAttachmentsCrossBase threw → fallback for 2 records × 1 field = 2 attachments + assert.equal(calls.fetchBytes.length, 2, 'fetchBytes called for both attachments in fallback'); + assert.equal(calls.uploadAttachment.length, 2, 'uploadAttachment called for both attachments'); + + // ATTACHMENT_FALLBACK warning emitted + const fallbackWarns = result.warnings.filter((w) => w.code === 'ATTACHMENT_FALLBACK'); + assert.ok(fallbackWarns.length >= 1, 'at least one ATTACHMENT_FALLBACK warning'); + assert.ok(fallbackWarns[0].message.includes('422'), 'warning includes error message'); + }); + + it('table with no attachment fields is skipped entirely (no client calls)', async () => { + const calls = { + createDataTransferPolicy: [], + pasteAttachmentsCrossBase: [], + }; + const client = { + createDataTransferPolicy: async (...args) => { calls.createDataTransferPolicy.push(args); return {}; }, + pasteAttachmentsCrossBase: async (...args) => { calls.pasteAttachmentsCrossBase.push(args); return { pastedRowIds: [] }; }, + uploadAttachment: async () => ({ ok: true }), + }; + + const idmap = { + tables: { tblSrc: 'tblDest' }, + fields: {}, // no attachment field mapping + records: { recS1: 'recD1' }, + attachments: {}, + }; + + const srcSnapshot = { + baseId: 'appSrc', + tables: [{ + id: 'tblSrc', + name: 'T', + fields: [{ id: 'fldText', name: 'Name', type: 'text' }], + records: [{ id: 'recS1', cellValuesByColumnId: { fldText: 'hello' } }], + }], + }; + + const destSnapshot = { + baseId: 'appDest', + tables: [{ id: 'tblDest', fields: [], views: [{ id: 'viw1', name: 'Grid' }], records: [] }], + }; + + const result = { attachmentsUploaded: 0, warnings: [] }; + + await applyAttachments({ + client, srcSnapshot, destSnapshot, idmap, + limiter: makeLimiter(), + journal: newJournal('fp-5', 't'), + persist: () => {}, + result, + fetchBytes: async () => ({ bytes: Buffer.from(''), contentType: '' }), + }); + + assert.equal(calls.createDataTransferPolicy.length, 0, 'no policy call when no attachment fields'); + assert.equal(calls.pasteAttachmentsCrossBase.length, 0, 'no paste call when no attachment fields'); + assert.equal(result.attachmentsUploaded, 0); + assert.equal(result.warnings.length, 0); + }); +}); From 8a55c9d8008ba5e7ef8f8d1ab9dbb78930005ebf Mon Sep 17 00:00:00 2001 From: "A.R." Date: Thu, 18 Jun 2026 00:16:54 +0300 Subject: [PATCH 074/246] =?UTF-8?q?fix(auth):=20circuit-breaker=20for=20a?= =?UTF-8?q?=20dead=20session=20(stop=20the=20403=E2=86=92recover=E2=86=924?= =?UTF-8?q?03=20loop)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the Airtable session is expired/blocked, every API call returns 403 → _recoverSession relaunches the browser → "verified" → next call 403s again, looping futilely (observed: a records run stuck 9+ min, 0 progress, hundreds of relaunches). _doInitBounded only bounds a HUNG recovery; here recovery SUCCEEDS but the session stays invalid. Add a circuit-breaker: count CONSECUTIVE recoveries (reset by any 2xx); past AIRTABLE_SESSION_DEAD_AFTER (default 6) consecutive auth/eval failures, mark the session dead and fail fast with a clear "SESSION_INVALID — re-authenticate (run `login`)" error. resetSessionHealth() clears it after re-login. Turns a multi- minute futile loop (and a background records job that looks hung) into an immediate, actionable error. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/auth.js | 30 ++++++++++++- .../test/test-auth-session-dead.test.js | 45 +++++++++++++++++++ 2 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 packages/mcp-server/test/test-auth-session-dead.test.js diff --git a/packages/mcp-server/src/auth.js b/packages/mcp-server/src/auth.js index 706baed..3188e81 100644 --- a/packages/mcp-server/src/auth.js +++ b/packages/mcp-server/src/auth.js @@ -74,6 +74,19 @@ export class AirtableAuth { // Session recovery state this._recovering = false; this._initPromise = null; + // Circuit-breaker: if the session keeps getting rejected (403/401, or page.evaluate keeps + // failing) even AFTER re-auth, stop looping recovery futilely (a dead/blocked/expired login — + // observed as endless "403 → recover → 403"). Counts CONSECUTIVE recoveries; any 2xx resets it. + // Past the threshold, mark the session dead and fail fast with a re-login message. + this._recoveryStreak = 0; + this._sessionDead = false; + this._sessionDeadAfter = Number(process.env.AIRTABLE_SESSION_DEAD_AFTER) || 6; + } + + /** Reset the dead-session circuit-breaker (e.g., after the user re-authenticates). */ + resetSessionHealth() { + this._sessionDead = false; + this._recoveryStreak = 0; } // ─── Initialization ────────────────────────────────────────── @@ -479,6 +492,9 @@ export class AirtableAuth { async _apiCall(method, urlPath, body = null, appId = null, contentType = 'form', evalTimeoutMs = 15_000) { return this._enqueue(async () => { + if (this._sessionDead) { + throw new Error('SESSION_INVALID: Airtable rejected repeated requests even after re-authentication — the login has expired or been blocked. Re-authenticate (run `login`) and retry.'); + } await this.ensureLoggedIn(); const MAX_RETRIES = 3; @@ -497,7 +513,11 @@ export class AirtableAuth { } catch (evalError) { // page.evaluate itself failed (browser crashed, context destroyed) if (!this._recovering && attempt < MAX_RETRIES) { - console.error(`[auth] page.evaluate failed: ${evalError.message}. Recovering...`); + if (++this._recoveryStreak >= this._sessionDeadAfter) { + this._sessionDead = true; + throw new Error(`SESSION_INVALID: ${this._recoveryStreak} consecutive failures across recoveries (page.evaluate) — re-authenticate (run \`login\`) and retry.`); + } + console.error(`[auth] page.evaluate failed: ${evalError.message}. Recovering... (streak ${this._recoveryStreak})`); await this._recoverSession(); attempt++; continue; @@ -517,12 +537,18 @@ export class AirtableAuth { // Session expired or network failure — attempt one recovery and retry const needsRecovery = result.status === 401 || result.status === 403 || result.error; if (needsRecovery && !this._recovering && attempt < MAX_RETRIES) { - console.error(`[auth] API call failed (status=${result.status}, error=${result.error || 'none'}). Recovering...`); + if (++this._recoveryStreak >= this._sessionDeadAfter) { + this._sessionDead = true; + throw new Error(`SESSION_INVALID: ${this._recoveryStreak} consecutive auth failures (status=${result.status}) across recoveries — the Airtable login has expired or been blocked. Re-authenticate (run \`login\`) and retry.`); + } + console.error(`[auth] API call failed (status=${result.status}, error=${result.error || 'none'}). Recovering... (streak ${this._recoveryStreak})`); await this._recoverSession(); attempt++; continue; } + // Healthy response → reset the dead-session circuit-breaker. + if (result.status >= 200 && result.status < 300) this._recoveryStreak = 0; return result; } }); diff --git a/packages/mcp-server/test/test-auth-session-dead.test.js b/packages/mcp-server/test/test-auth-session-dead.test.js new file mode 100644 index 0000000..fbf6ef4 --- /dev/null +++ b/packages/mcp-server/test/test-auth-session-dead.test.js @@ -0,0 +1,45 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { AirtableAuth } from '../src/auth.js'; + +// Regression: when the Airtable session is dead/blocked, EVERY API call 403s → recover → 403, +// which previously looped futilely for minutes (observed: 9+ min, 0 progress). The circuit-breaker +// counts consecutive recoveries (reset by any 2xx) and, past the threshold, marks the session dead +// and fails fast with a re-login message instead of looping. +function deadSessionAuth() { + const a = new AirtableAuth(); + a._sessionDeadAfter = 3; // trip within one call's MAX_RETRIES for the test + a.context = {}; a.isLoggedIn = true; // ensureLoggedIn becomes a no-op + a.page = { url: () => 'https://airtable.com/' }; + a._doInit = async () => { a.context = {}; a.isLoggedIn = true; }; // recovery "succeeds" + a._rawApiCall = async () => ({ status: 403, body: '' }); // every call 403s + return a; +} + +describe('AirtableAuth — dead-session circuit-breaker', () => { + it('aborts with SESSION_INVALID after N consecutive 403-recoveries (no infinite loop)', async () => { + const a = deadSessionAuth(); + await assert.rejects(() => a.get('/v0.3/x', 'app1'), /SESSION_INVALID/); + assert.equal(a._sessionDead, true); + // once dead, subsequent calls fail fast (no further browser relaunches) + await assert.rejects(() => a.get('/v0.3/y', 'app1'), /SESSION_INVALID/); + }); + + it('a 2xx resets the streak — transient blips do NOT trip the breaker', async () => { + const a = deadSessionAuth(); + let n = 0; + a._rawApiCall = async () => (++n <= 2 ? { status: 403, body: '' } : { status: 200, body: '{}' }); + const res = await a.get('/v0.3/x', 'app1'); // 403, 403, then 200 + assert.equal(res.status, 200); + assert.equal(a._recoveryStreak, 0); + assert.equal(a._sessionDead, false); + }); + + it('resetSessionHealth clears the dead flag (after a re-login)', () => { + const a = deadSessionAuth(); + a._sessionDead = true; a._recoveryStreak = 9; + a.resetSessionHealth(); + assert.equal(a._sessionDead, false); + assert.equal(a._recoveryStreak, 0); + }); +}); From 9dca92bcbb9371d78dde9b83386255ff7ba1bf50 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Thu, 18 Jun 2026 09:50:04 +0300 Subject: [PATCH 075/246] =?UTF-8?q?fix(auth):=20back=20off=20on=20403/429?= =?UTF-8?q?=20(don't=20relaunch)=20=E2=80=94=20stop=20amplifying=20the=20r?= =?UTF-8?q?ate=20limit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The internal API shares Airtable's ~5 req/s-per-base limit and answers a burst with 403. We treated 403 as session-expiry → _recoverSession (browser relaunch), but each relaunch fires its own burst of page-load requests that RE-TRIPS the throttle — the 403→relaunch→403 storm seen on record syncs. Treat 403/429/503 as a rate-limit signal: exponential backoff (toward the ~30s official penalty) and retry the SAME session, no relaunch; intentional waits extend the work budget so they don't trip it. Relaunch is reserved for genuine auth loss (401 / network error / page-evaluate crash). The dead-session circuit-breaker still bounds a throttle that never clears (default 8 consecutive, giving a real 30s window time to reset; backoff sleep injectable for tests). Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/auth.js | 35 +++++++++----- .../test/test-auth-session-dead.test.js | 47 +++++++++++-------- 2 files changed, 52 insertions(+), 30 deletions(-) diff --git a/packages/mcp-server/src/auth.js b/packages/mcp-server/src/auth.js index 3188e81..b7e328f 100644 --- a/packages/mcp-server/src/auth.js +++ b/packages/mcp-server/src/auth.js @@ -80,7 +80,8 @@ export class AirtableAuth { // Past the threshold, mark the session dead and fail fast with a re-login message. this._recoveryStreak = 0; this._sessionDead = false; - this._sessionDeadAfter = Number(process.env.AIRTABLE_SESSION_DEAD_AFTER) || 6; + this._sessionDeadAfter = Number(process.env.AIRTABLE_SESSION_DEAD_AFTER) || 8; + this._rlSleep = (ms) => new Promise((r) => setTimeout(r, ms)); // injectable for tests } /** Reset the dead-session circuit-breaker (e.g., after the user re-authenticates). */ @@ -499,8 +500,9 @@ export class AirtableAuth { const MAX_RETRIES = 3; const HARD_TIMEOUT_MS = 30_000; - const deadline = Date.now() + HARD_TIMEOUT_MS; - let attempt = 0; + let deadline = Date.now() + HARD_TIMEOUT_MS; // extended by intentional rate-limit backoff waits + let attempt = 0; // browser-recovery (relaunch) attempts + let rlAttempt = 0; // rate-limit/throttle backoff attempts (same session, no relaunch) while (true) { if (Date.now() > deadline) { // Bail out rather than retry past the caller's patience budget. @@ -525,17 +527,28 @@ export class AirtableAuth { throw evalError; } - // Rate limited — exponential backoff (429 = Too Many Requests, 503 = Service Unavailable) - if ((result.status === 429 || result.status === 503) && attempt < MAX_RETRIES) { - const delay = 500 * (2 ** attempt); - console.error(`[auth] Rate limited (${result.status}), retrying in ${delay}ms...`); - await new Promise(r => setTimeout(r, delay)); - attempt++; + // Rate-limit / throttle — 429 (Too Many Requests), 503 (Service Unavailable), and 403: + // the internal API shares Airtable's ~5 req/s-per-base limit and answers a burst with 403. + // BACK OFF and retry the SAME session — do NOT relaunch the browser. Relaunching on a + // throttle 403 only makes it worse: each _recoverSession fires a burst of page-load requests + // that re-trips the limit (observed as an endless 403→relaunch→403 storm). Exponential toward + // the ~30s official penalty; intentional waits don't count against the work budget; the + // dead-session circuit-breaker still bounds a throttle that never clears. + if (result.status === 429 || result.status === 503 || result.status === 403) { + if (++this._recoveryStreak >= this._sessionDeadAfter) { + this._sessionDead = true; + throw new Error(`SESSION_INVALID: ${this._recoveryStreak} consecutive rate-limit/throttle responses (status=${result.status}) that did not clear after backoff — the account is throttled or the login is blocked. Let the limit reset (wait), or re-authenticate (run \`login\`), then retry.`); + } + const delay = Math.min(30_000, 1000 * (2 ** rlAttempt)) + Math.floor(Math.random() * 500); + console.error(`[auth] ${result.status} (rate-limit/throttle) — backing off ${delay}ms on the same session, no relaunch (streak ${this._recoveryStreak})...`); + await this._rlSleep(delay); + deadline += delay; // an intentional backoff wait is not a stall — don't let it trip the budget + rlAttempt++; continue; } - // Session expired or network failure — attempt one recovery and retry - const needsRecovery = result.status === 401 || result.status === 403 || result.error; + // Genuine auth loss (401) or network failure — relaunch the session. + const needsRecovery = result.status === 401 || result.error; if (needsRecovery && !this._recovering && attempt < MAX_RETRIES) { if (++this._recoveryStreak >= this._sessionDeadAfter) { this._sessionDead = true; diff --git a/packages/mcp-server/test/test-auth-session-dead.test.js b/packages/mcp-server/test/test-auth-session-dead.test.js index fbf6ef4..434f1fa 100644 --- a/packages/mcp-server/test/test-auth-session-dead.test.js +++ b/packages/mcp-server/test/test-auth-session-dead.test.js @@ -2,41 +2,50 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { AirtableAuth } from '../src/auth.js'; -// Regression: when the Airtable session is dead/blocked, EVERY API call 403s → recover → 403, -// which previously looped futilely for minutes (observed: 9+ min, 0 progress). The circuit-breaker -// counts consecutive recoveries (reset by any 2xx) and, past the threshold, marks the session dead -// and fails fast with a re-login message instead of looping. -function deadSessionAuth() { +// The internal API shares Airtable's ~5 req/s-per-base limit; a burst answers 403/429. The auth +// layer must BACK OFF and retry the SAME session (relaunching on a throttle 403 only fires more +// page-load requests and amplifies the limit — the observed 403→relaunch→403 storm). Genuine auth +// loss (401/network) still relaunches. A throttle that never clears trips the dead-session breaker. +function fakeAuth() { const a = new AirtableAuth(); - a._sessionDeadAfter = 3; // trip within one call's MAX_RETRIES for the test - a.context = {}; a.isLoggedIn = true; // ensureLoggedIn becomes a no-op + a._sessionDeadAfter = 3; // trip quickly for the test + a._rlSleep = async () => {}; // no real backoff waits + a.context = {}; a.isLoggedIn = true; a.page = { url: () => 'https://airtable.com/' }; - a._doInit = async () => { a.context = {}; a.isLoggedIn = true; }; // recovery "succeeds" - a._rawApiCall = async () => ({ status: 403, body: '' }); // every call 403s + a._doInitCalls = 0; + a._doInit = async () => { a._doInitCalls++; a.context = {}; a.isLoggedIn = true; }; return a; } -describe('AirtableAuth — dead-session circuit-breaker', () => { - it('aborts with SESSION_INVALID after N consecutive 403-recoveries (no infinite loop)', async () => { - const a = deadSessionAuth(); +describe('AirtableAuth — throttle backoff vs. recovery', () => { + it('403 backs off on the SAME session (no relaunch) and trips the breaker if it never clears', async () => { + const a = fakeAuth(); + a._rawApiCall = async () => ({ status: 403, body: '' }); await assert.rejects(() => a.get('/v0.3/x', 'app1'), /SESSION_INVALID/); assert.equal(a._sessionDead, true); - // once dead, subsequent calls fail fast (no further browser relaunches) - await assert.rejects(() => a.get('/v0.3/y', 'app1'), /SESSION_INVALID/); + assert.equal(a._doInitCalls, 0); // KEY: a throttle 403 must NOT relaunch the browser }); - it('a 2xx resets the streak — transient blips do NOT trip the breaker', async () => { - const a = deadSessionAuth(); + it('a 2xx after some 403s resets the streak — no relaunch, no false trip', async () => { + const a = fakeAuth(); let n = 0; a._rawApiCall = async () => (++n <= 2 ? { status: 403, body: '' } : { status: 200, body: '{}' }); - const res = await a.get('/v0.3/x', 'app1'); // 403, 403, then 200 + const res = await a.get('/v0.3/x', 'app1'); assert.equal(res.status, 200); assert.equal(a._recoveryStreak, 0); - assert.equal(a._sessionDead, false); + assert.equal(a._doInitCalls, 0); + }); + + it('401 (genuine auth loss) DOES relaunch the session (recovery preserved)', async () => { + const a = fakeAuth(); + a._rawApiCall = async () => ({ status: 401, body: '' }); + await assert.rejects(() => a.get('/v0.3/x', 'app1'), /SESSION_INVALID/); + assert.equal(a._sessionDead, true); + assert.ok(a._doInitCalls >= 1); // 401 → re-auth via relaunch }); it('resetSessionHealth clears the dead flag (after a re-login)', () => { - const a = deadSessionAuth(); + const a = fakeAuth(); a._sessionDead = true; a._recoveryStreak = 9; a.resetSessionHealth(); assert.equal(a._sessionDead, false); From 44e5248406aa83bcd90b3a5a5b9e468d0fef6d44 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Thu, 18 Jun 2026 16:10:23 +0300 Subject: [PATCH 076/246] feat(sync): fold resolvable links into row/create (kills most of Pass 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records-sync request volume was ~3000 POSTs for ~1000 rows: ~1000 per-row creates + ~2000 per-item link appends (the dominant cost) + 2 attachment pastes. Capture evidence (06-15) proved row/create already accepts array cells (multi-select by id) in one POST, and the link item shape {foreignRowId,foreignRowDisplayName} is known — so links can ride in the create payload instead of a separate per-item Pass 2. - orderTablesByLinkDeps: topologically order tables so link TARGET tables are created first (cross-table links then resolve at create time); cycles + self-links broken deterministically and left to Pass 2. - Pass 1 folds resolvable (remapped) links into row/create's cellValuesByColumnId; mirrors them into the in-memory dest snapshot so Pass 2 dedups them out. - Safety circuit-breaker: if a folded link cell makes a create fail, retry that row WITHOUT links; if that rescues it, disable folding for the rest of the run (FOLD_LINKS_DISABLED) so Pass 2 writes links per-item. Bounds risk of the (high-confidence, not-yet-live-verified) create-with-links capability. - Fix latent bug: `foreignKey` (the INTERNAL link type) was missing from cells.js ARRAY_DEFERRED, so buildCreate/UpdateCells wrote RAW source rec-ids for link cells — only "worked" against an id-duplicate dest base. ~3000 → ~1200 POSTs, full remap fidelity, client-controlled rec ids. Pass 2 remains the correctness backstop for unresolved/forward/circular links. 479/479 mcp-server tests green (+ new test-records-fold-links). Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/cells.js | 7 +- packages/mcp-server/src/sync/records.js | 200 +++++++++++++++--- .../mcp-server/test/sync/test-cells.test.js | 1 + .../test/sync/test-records-fold-links.test.js | 182 ++++++++++++++++ 4 files changed, 364 insertions(+), 26 deletions(-) create mode 100644 packages/mcp-server/test/sync/test-records-fold-links.test.js diff --git a/packages/mcp-server/src/sync/cells.js b/packages/mcp-server/src/sync/cells.js index cbe46ca..0966f9a 100644 --- a/packages/mcp-server/src/sync/cells.js +++ b/packages/mcp-server/src/sync/cells.js @@ -1,6 +1,11 @@ import { isComputedType } from './snapshot.js'; -const ARRAY_DEFERRED = new Set(['multipleRecordLinks', 'multipleAttachments']); +// Array-type cells whose raw source value must NOT be written verbatim by buildCreate/UpdateCells: +// links carry source rec-ids that need src→dest remapping (handled by the link-fold path in Pass 1 +// + Pass 2), attachments need the cross-base transfer (Pass 3). `foreignKey` is the INTERNAL link +// type (snapshot passes type through verbatim) — omitting it here would write raw source rec-ids at +// create time (only "worked" against an id-duplicate dest base). +const ARRAY_DEFERRED = new Set(['multipleRecordLinks', 'foreignKey', 'multipleAttachments']); /** * Extract the record ID from a link-cell element. diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index 6409295..b7dd49e 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -139,13 +139,178 @@ function buildUpdateCells(srcFields, srcCells, idmap, warnings, srcRecId) { * @param {object} opts.result - result accumulator { created, updated, skipped, failed, warnings } * @returns {Promise} */ +/** + * Order source tables so a link target table is created BEFORE the tables that link to it, + * maximizing how many link cells can be folded into the create payload (their dest target + * already exists in idmap.records). Deterministic: ties / cycles preserve original order. + * + * An edge T → target exists when T has a MAPPED link field (foreignKey/multipleRecordLinks) + * whose `typeOptions.foreignTableId` is another known table. Self-links are ignored (the row's + * own dest id isn't known until after its create → handled by Pass 2). + * + * @param {Array} tables - source table descriptors + * @param {object} idmap - id-map (.fields used to skip unmapped link fields) + * @returns {Array} - tables reordered (targets first); cycles broken in original order + */ +export function orderTablesByLinkDeps(tables, idmap) { + const list = tables || []; + const byId = new Set(list.map((t) => t.id)); + const deps = new Map(); // tableId → Set(targetTableId) + for (const t of list) { + const set = new Set(); + for (const f of (t.fields || [])) { + if ((f.type === 'foreignKey' || f.type === 'multipleRecordLinks') && idmap.fields?.[f.id]) { + const target = f.typeOptions?.foreignTableId; + if (target && target !== t.id && byId.has(target)) set.add(target); + } + } + deps.set(t.id, set); + } + const ordered = []; + const emitted = new Set(); + const remaining = list.slice(); + while (remaining.length) { + // Emit the FIRST (original-order) table whose deps are all already emitted. + const idx = remaining.findIndex((t) => [...deps.get(t.id)].every((d) => emitted.has(d))); + const pick = idx === -1 ? 0 : idx; // no eligible table → cycle: break by taking the first remaining + const [t] = remaining.splice(pick, 1); + ordered.push(t); + emitted.add(t.id); + } + return ordered; +} + +/** + * Build the resolvable link cells for the CREATE payload: for each mapped link field whose + * source value references records already in idmap.records, emit + * `{ destFld: [{ foreignRowId: , foreignRowDisplayName: '' }] }`. + * Unresolvable targets (forward/circular refs) are left to Pass 2 (it re-adds anything missing). + * (Display name is cosmetic — '' is accepted; Airtable recomputes it.) + * + * @param {Array} srcFields + * @param {object} srcCells + * @param {object} idmap + * @returns {object} dest cellValuesByColumnId fragment containing only the resolvable link cells + */ +function buildResolvableLinkCells(srcFields, srcCells, idmap) { + const cells = {}; + for (const field of (srcFields || [])) { + if (field.type !== 'foreignKey' && field.type !== 'multipleRecordLinks') continue; + const mapping = idmap.fields[field.id]; + if (!mapping) continue; + const srcVal = srcCells[field.id]; + if (!srcVal || (Array.isArray(srcVal) && srcVal.length === 0)) continue; + const { resolved } = partitionLinkValue(srcVal, idmap); + if (resolved.length > 0) { + cells[mapping.destFld] = resolved.map((id) => ({ foreignRowId: id, foreignRowDisplayName: '' })); + } + } + return cells; +} + +/** + * Mirror links folded into a create into the in-memory dest snapshot, so Pass 2's dedup + * (built from destSnapshot) treats them as already-present and does NOT re-add them. + * (Created rows are absent from the pre-run dest snapshot; this keeps it consistent.) + */ +function mirrorFoldedLinks(destTableById, destTableId, destRowId, linkCells) { + if (!linkCells || Object.keys(linkCells).length === 0) return; + const dt = destTableById.get(destTableId); + if (!dt) return; + dt.records ??= []; + dt.records.push({ id: destRowId, cellValuesByColumnId: { ...linkCells } }); +} + +/** + * Create one chunk, with a SAFETY fallback: if folded link cells cause per-row create failures, + * retry just those rows WITHOUT the link cells. If stripping links rescues a row, the link cell + * is the culprit → disable link-folding for the remainder of the run (runState.foldLinks=false), + * so Pass 2 writes those links per-item. This bounds the blast radius if `row/create` ever rejects + * a folded link array to ~one chunk of retries (the create-with-links capability is high-confidence + * but not live-verified — this is the calibration knob). + */ +async function createChunkWithFallback({ client, destAppId, destTableId, chunk, idmap, result, limiter, runState, destTableById }) { + let res; + try { + res = await limiter.run(() => withRetry(() => client.createRecords(destAppId, destTableId, chunk, {}))); + } catch (err) { + result.failed += chunk.length; + result.warnings.push({ code: 'RECORD_CREATE_FAILED', message: `Table ${destTableId}: createRecords threw: ${err.message}` }); + return; + } + + for (const created of (res.created || [])) { + idmap.records[created.sourceKey] = created.rowId; + result.created++; + const row = chunk.find((r) => r.sourceKey === created.sourceKey); + mirrorFoldedLinks(destTableById, destTableId, created.rowId, row?.linkCells); + } + + const failed = res.failed || []; + if (failed.length === 0) return; + + // Split failures: those whose row carried folded links (retryable) vs genuine failures. + const failedKeys = new Set(failed.map((f) => f.sourceKey)); + const linkFailedRows = chunk.filter( + (r) => failedKeys.has(r.sourceKey) && r.linkCells && Object.keys(r.linkCells).length > 0, + ); + const linkFailedKeys = new Set(linkFailedRows.map((r) => r.sourceKey)); + for (const f of failed) { + if (linkFailedKeys.has(f.sourceKey)) continue; // handled by retry below + result.failed++; + result.warnings.push({ code: 'RECORD_CREATE_FAILED', message: `Table ${destTableId}: failed to create record (sourceKey=${f.sourceKey}): ${f.error}` }); + } + + if (linkFailedRows.length === 0) return; + + // Retry the link-bearing failures WITHOUT their link cells. + const stripped = linkFailedRows.map((r) => { + const cells = { ...r.cellValuesByColumnId }; + for (const k of Object.keys(r.linkCells)) delete cells[k]; + return { cellValuesByColumnId: cells, sourceKey: r.sourceKey, linkCells: {} }; + }); + let retryRes; + try { + retryRes = await limiter.run(() => withRetry(() => client.createRecords(destAppId, destTableId, stripped, {}))); + } catch (err) { + result.failed += stripped.length; + result.warnings.push({ code: 'RECORD_CREATE_FAILED', message: `Table ${destTableId}: retry-without-links threw: ${err.message}` }); + return; + } + let rescued = 0; + for (const created of (retryRes.created || [])) { + idmap.records[created.sourceKey] = created.rowId; // links NOT folded → Pass 2 adds them (no mirror) + result.created++; + rescued++; + } + for (const f of (retryRes.failed || [])) { + result.failed++; + result.warnings.push({ code: 'RECORD_CREATE_FAILED', message: `Table ${destTableId}: failed to create record without links (sourceKey=${f.sourceKey}): ${f.error}` }); + } + if (rescued > 0 && runState.foldLinks) { + runState.foldLinks = false; + result.warnings.push({ + code: 'FOLD_LINKS_DISABLED', + message: `Table ${destTableId}: create rejected folded link cells — disabled link-fold for the rest of this run (Pass 2 writes links per-item)`, + }); + } +} + export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result }) { if (!idmap.records) idmap.records = {}; // destAppId comes from the snapshot (set by snapshotBase); fall back to '' for tests that omit it. const destAppId = destSnapshot.baseId || ''; - for (const srcTable of (srcSnapshot.tables || [])) { + // Run-level circuit breaker for fold-into-create (see createChunkWithFallback). + const runState = { foldLinks: true }; + // Dest table lookup for mirroring folded links into the snapshot (so Pass 2 dedups them out). + const destTableById = new Map((destSnapshot.tables || []).map((t) => [t.id, t])); + + // Create link TARGET tables before their dependents so cross-table links resolve at create time. + const orderedTables = orderTablesByLinkDeps(srcSnapshot.tables || [], idmap); + + for (const srcTable of orderedTables) { const destTableId = idmap.tables[srcTable.id]; if (!destTableId) continue; // table not matched → skip @@ -161,9 +326,15 @@ export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idm const destRecId = idmap.records[rec.id]; if (!destRecId) { - // CREATE path — multiSelect arrays ARE accepted by createRecords + // CREATE path — scalar/select/multiSelect arrays ARE accepted by createRecords. const cells = buildCreateCells(srcTable.fields, srcCells, idmap, result.warnings, rec.id); - createRows.push({ cellValuesByColumnId: cells, sourceKey: rec.id }); + // Fold resolvable links (remapped) into the create payload (kills most of Pass 2). + let linkCells = {}; + if (runState.foldLinks) { + linkCells = buildResolvableLinkCells(srcTable.fields, srcCells, idmap); + Object.assign(cells, linkCells); + } + createRows.push({ cellValuesByColumnId: cells, sourceKey: rec.id, linkCells }); } else { // UPDATE path — multiSelect arrays NOT accepted by updateRecords (deferred with warning) const cells = buildUpdateCells(srcTable.fields, srcCells, idmap, result.warnings, rec.id); @@ -178,28 +349,7 @@ export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idm const CREATE_CHUNK = 50; for (let i = 0; i < createRows.length; i += CREATE_CHUNK) { const chunk = createRows.slice(i, i + CREATE_CHUNK); - try { - const res = await limiter.run(() => - withRetry(() => client.createRecords(destAppId, destTableId, chunk, {})), - ); - for (const created of (res.created || [])) { - idmap.records[created.sourceKey] = created.rowId; - result.created++; - } - for (const failed of (res.failed || [])) { - result.failed++; - result.warnings.push({ - code: 'RECORD_CREATE_FAILED', - message: `Table ${destTableId}: failed to create record (sourceKey=${failed.sourceKey}): ${failed.error}`, - }); - } - } catch (err) { - result.failed += chunk.length; - result.warnings.push({ - code: 'RECORD_CREATE_FAILED', - message: `Table ${destTableId}: createRecords threw: ${err.message}`, - }); - } + await createChunkWithFallback({ client, destAppId, destTableId, chunk, idmap, result, limiter, runState, destTableById }); persist(idmap, journal); } diff --git a/packages/mcp-server/test/sync/test-cells.test.js b/packages/mcp-server/test/sync/test-cells.test.js index 7d1ba5e..e7c19cf 100644 --- a/packages/mcp-server/test/sync/test-cells.test.js +++ b/packages/mcp-server/test/sync/test-cells.test.js @@ -8,6 +8,7 @@ describe('cells.isWritableForRecords', () => { it('false for computed + link + attachment, true for scalar/select', () => { assert.equal(isWritableForRecords({ type: 'formula' }), false); assert.equal(isWritableForRecords({ type: 'multipleRecordLinks' }), false); + assert.equal(isWritableForRecords({ type: 'foreignKey' }), false); // internal link type — must be remapped, not written raw assert.equal(isWritableForRecords({ type: 'multipleAttachments' }), false); assert.equal(isWritableForRecords({ type: 'text' }), true); assert.equal(isWritableForRecords({ type: 'select' }), true); diff --git a/packages/mcp-server/test/sync/test-records-fold-links.test.js b/packages/mcp-server/test/sync/test-records-fold-links.test.js new file mode 100644 index 0000000..b7a7998 --- /dev/null +++ b/packages/mcp-server/test/sync/test-records-fold-links.test.js @@ -0,0 +1,182 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { applyRecordsPass1, applyRecordsPass2, orderTablesByLinkDeps } from '../../src/sync/records.js'; +import { newJournal } from '../../src/sync/journal.js'; +import { createLimiter } from '../../src/sync/ratelimit.js'; + +const fastLimiter = () => createLimiter({ rps: 1000, sleep: async () => {} }); +const newResult = () => ({ created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }); + +// ── orderTablesByLinkDeps ──────────────────────────────────────────────────── +describe('orderTablesByLinkDeps', () => { + it('orders link TARGET tables before the tables that depend on them', () => { + // tblA links to tblB (foreignTableId=tblB) → tblB must come first + const tables = [ + { id: 'tblA', fields: [{ id: 'fA', type: 'foreignKey', typeOptions: { foreignTableId: 'tblB' } }] }, + { id: 'tblB', fields: [{ id: 'fB', type: 'text' }] }, + ]; + const idmap = { tables: { tblA: 'x', tblB: 'y' }, fields: { fA: { destFld: 'd' } } }; + const ordered = orderTablesByLinkDeps(tables, idmap).map((t) => t.id); + assert.deepEqual(ordered, ['tblB', 'tblA']); + }); + + it('breaks cycles deterministically (original order) without throwing or dropping tables', () => { + const tables = [ + { id: 'tblA', fields: [{ id: 'fA', type: 'foreignKey', typeOptions: { foreignTableId: 'tblB' } }] }, + { id: 'tblB', fields: [{ id: 'fB', type: 'foreignKey', typeOptions: { foreignTableId: 'tblA' } }] }, + ]; + const idmap = { tables: { tblA: 'x', tblB: 'y' }, fields: { fA: { destFld: 'd1' }, fB: { destFld: 'd2' } } }; + const ordered = orderTablesByLinkDeps(tables, idmap).map((t) => t.id); + assert.equal(ordered.length, 2); + assert.ok(ordered.includes('tblA') && ordered.includes('tblB')); + }); + + it('ignores self-links and unmapped link fields', () => { + const tables = [ + { id: 'tblA', fields: [{ id: 'fSelf', type: 'foreignKey', typeOptions: { foreignTableId: 'tblA' } }] }, + { id: 'tblB', fields: [{ id: 'fUnmapped', type: 'foreignKey', typeOptions: { foreignTableId: 'tblA' } }] }, + ]; + // fUnmapped has no idmap.fields entry → no dependency edge + const idmap = { tables: { tblA: 'x', tblB: 'y' }, fields: {} }; + const ordered = orderTablesByLinkDeps(tables, idmap).map((t) => t.id); + assert.deepEqual(ordered, ['tblA', 'tblB']); // stable original order + }); +}); + +// ── Pass 1 folds resolvable links into the create payload ──────────────────── +describe('applyRecordsPass1 — fold resolvable links into create', () => { + function captureClient(captured, { failOnLink = false } = {}) { + return { + createRecords: async (appId, tableId, rows) => { + const created = []; + const failed = []; + rows.forEach((r, i) => { + const hasLink = Object.values(r.cellValuesByColumnId || {}).some( + (v) => Array.isArray(v) && v.some((e) => e && typeof e === 'object' && e.foreignRowId), + ); + captured.push({ tableId, cells: r.cellValuesByColumnId, sourceKey: r.sourceKey }); + if (failOnLink && hasLink) { + failed.push({ sourceKey: r.sourceKey, error: 'simulated 422: link cell rejected' }); + } else { + created.push({ rowId: 'recD_' + r.sourceKey, sourceKey: r.sourceKey }); + } + }); + return { created, failed }; + }, + updateRecords: async () => ({ updated: [], failed: [] }), + addLinkItems: async () => ({ ok: true, added: 1 }), + }; + } + + it('writes a resolvable link as [{foreignRowId}] in the create payload', async () => { + const idmap = { + tables: { tblS: 'tblD' }, + fields: { fldT: { destFld: 'fldTD', choices: {} }, fldL: { destFld: 'fldLD' } }, + records: { recTarget: 'recTargetD' }, // target already created (earlier table) + }; + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', + fields: [{ id: 'fldT', type: 'text' }, { id: 'fldL', type: 'foreignKey', typeOptions: { foreignTableId: 'tblOther' } }], + records: [{ id: 'recS1', cellValuesByColumnId: { fldT: 'hi', fldL: ['recTarget'] } }], + }], + }; + const destSnapshot = { baseId: 'appD', tables: [{ id: 'tblD', name: 'T', fields: [], records: [] }] }; + const captured = []; + const result = newResult(); + await applyRecordsPass1({ + client: captureClient(captured), srcSnapshot, destSnapshot, idmap, + limiter: fastLimiter(), journal: newJournal('p', 't'), persist: () => {}, result, + }); + assert.equal(captured.length, 1); + assert.deepEqual(captured[0].cells.fldTD, 'hi'); + assert.deepEqual(captured[0].cells.fldLD, [{ foreignRowId: 'recTargetD', foreignRowDisplayName: '' }]); + assert.equal(result.created, 1); + }); + + it('does NOT fold an unresolvable link (target not yet mapped) — leaves it for Pass 2', async () => { + const idmap = { + tables: { tblS: 'tblD' }, + fields: { fldL: { destFld: 'fldLD' } }, + records: {}, // target NOT mapped + }; + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', + fields: [{ id: 'fldL', type: 'foreignKey', typeOptions: { foreignTableId: 'tblOther' } }], + records: [{ id: 'recS1', cellValuesByColumnId: { fldL: ['recMissing'] } }], + }], + }; + const destSnapshot = { baseId: 'appD', tables: [{ id: 'tblD', name: 'T', fields: [], records: [] }] }; + const captured = []; + const result = newResult(); + await applyRecordsPass1({ + client: captureClient(captured), srcSnapshot, destSnapshot, idmap, + limiter: fastLimiter(), journal: newJournal('p', 't'), persist: () => {}, result, + }); + assert.equal(captured.length, 1); + assert.equal(captured[0].cells.fldLD, undefined); // not folded + }); + + it('mirrors folded links into destSnapshot so Pass 2 does not re-add them', async () => { + const idmap = { + tables: { tblS: 'tblD' }, + fields: { fldL: { destFld: 'fldLD' } }, + records: { recTarget: 'recTargetD' }, + }; + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', + fields: [{ id: 'fldL', type: 'foreignKey', typeOptions: { foreignTableId: 'tblOther' } }], + records: [{ id: 'recS1', cellValuesByColumnId: { fldL: ['recTarget'] } }], + }], + }; + const destSnapshot = { baseId: 'appD', tables: [{ id: 'tblD', name: 'T', fields: [], records: [] }] }; + const captured = []; + const result = newResult(); + await applyRecordsPass1({ + client: captureClient(captured), srcSnapshot, destSnapshot, idmap, + limiter: fastLimiter(), journal: newJournal('p', 't'), persist: () => {}, result, + }); + // Pass 2 should now see the folded link in destSnapshot and add NOTHING + const addCalls = []; + const client2 = { + addLinkItems: async (appId, rowId, columnId, items) => { addCalls.push({ rowId, columnId, items }); return { ok: true, added: items.length }; }, + }; + await applyRecordsPass2({ + client: client2, srcSnapshot, destSnapshot, idmap, + destDisplayNames: new Map(), limiter: fastLimiter(), journal: newJournal('p', 't'), persist: () => {}, result, + }); + assert.equal(addCalls.length, 0, 'Pass 2 must not re-add a link already folded in Pass 1'); + }); + + it('SAFETY: if create rejects the link cell, retries without links, flips fold off, warns', async () => { + const idmap = { + tables: { tblS: 'tblD' }, + fields: { fldL: { destFld: 'fldLD' } }, + records: { recTarget: 'recTargetD' }, + }; + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', + fields: [{ id: 'fldL', type: 'foreignKey', typeOptions: { foreignTableId: 'tblOther' } }], + records: [{ id: 'recS1', cellValuesByColumnId: { fldL: ['recTarget'] } }], + }], + }; + const destSnapshot = { baseId: 'appD', tables: [{ id: 'tblD', name: 'T', fields: [], records: [] }] }; + const captured = []; + const result = newResult(); + await applyRecordsPass1({ + client: captureClient(captured, { failOnLink: true }), srcSnapshot, destSnapshot, idmap, + limiter: fastLimiter(), journal: newJournal('p', 't'), persist: () => {}, result, + }); + // First attempt (with link) failed, retry (without link) succeeded → row created + assert.equal(idmap.records.recS1, 'recD_recS1'); + assert.equal(result.created, 1); + assert.equal(result.failed, 0); + assert.ok(result.warnings.some((w) => w.code === 'FOLD_LINKS_DISABLED')); + // retry payload had NO link cell + const retry = captured[captured.length - 1]; + assert.equal(retry.cells.fldLD, undefined); + }); +}); From cd15864d704279ff3318e9888a1b984452cd5c3a Mon Sep 17 00:00:00 2001 From: "A.R." Date: Thu, 18 Jun 2026 16:13:29 +0300 Subject: [PATCH 077/246] feat(sync): proactively pace the records phase under ~5 req/s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The auth queue already serializes every browser-backed call and can space dequeued calls by `_rateDelayMs`, but it defaulted to 0 — so the records phase's per-row creates + per-item link appends burst past Airtable's ~5 req/s per-base limit and only got caught reactively by the 429/403 backoff. applyRecords now enables the delay (200ms ≈ 5/s, override via AIRTABLE_SYNC_RATE_DELAY_MS) for the duration of the phase and restores the prior value in a finally (so interactive MCP tool calls stay snappy). This paces the INNER POSTs the per-call limiter never gated. No-op without auth. 481/481 mcp-server tests green. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/records.js | 30 +++++++++++++++++++ .../test/sync/test-records-fold-links.test.js | 19 +++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index b7dd49e..858cefd 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -39,6 +39,29 @@ export function readRecordsJobStatus(sourceBaseId, destBaseId, planId) { try { return JSON.parse(readFileSync(p, 'utf8')); } catch { return null; } } +/** + * Enable per-request pacing for the records phase, then return a restore fn. + * + * Every browser-backed call funnels through the auth queue, which spaces dequeued calls by + * `auth._rateDelayMs` (default 0 = off). The records phase fires many POSTs (per-row creates + + * per-item link appends); without pacing they burst past Airtable's ~5 req/s per-base limit and + * trip the 429/403 backoff (reactive). Defaulting the delay ON (200ms ≈ 5/s, override via + * AIRTABLE_SYNC_RATE_DELAY_MS) paces the inner POSTs PROACTIVELY so we stay under the limit and + * don't provoke the abuse detector. Scoped to the phase: restored afterward so interactive MCP + * tool calls stay snappy. No-op when the client has no auth (tests). + * + * @param {object} client + * @returns {() => void} restore — resets the prior delay + */ +export function applySyncRateDelay(client) { + const auth = client?.auth; + if (!auth) return () => {}; + const prev = auth._rateDelayMs; + const ms = Number(process.env.AIRTABLE_SYNC_RATE_DELAY_MS); + auth._rateDelayMs = Number.isFinite(ms) && ms > 0 ? ms : 200; + return () => { auth._rateDelayMs = prev; }; +} + /** * Returns true if the field type is a multiSelect (arrays not accepted by updateRecords). * @param {string} type @@ -912,6 +935,10 @@ export async function applyRecords({ client, sourceBaseId, destBaseId, planId, r idmap.records ??= {}; idmap.attachments ??= {}; + // Pace every POST in this phase under ~5 req/s (proactive); restored in finally. + const restoreRateDelay = applySyncRateDelay(client); + try { + // 2. Snapshot schemas ONLY (no per-view live-config reads — those are a getView/readData // storm: ~1s per view, hundreds of views on a view-heavy base, on BOTH bases. The records // phase needs schema + records, not view configs. reapplyViewFilters reads source view @@ -990,6 +1017,9 @@ export async function applyRecords({ client, sourceBaseId, destBaseId, planId, r persist(idmap, journal); return result; + } finally { + restoreRateDelay(); + } } // ────────────────────────────────────────────────────────────────────────────── diff --git a/packages/mcp-server/test/sync/test-records-fold-links.test.js b/packages/mcp-server/test/sync/test-records-fold-links.test.js index b7a7998..c7b6ffd 100644 --- a/packages/mcp-server/test/sync/test-records-fold-links.test.js +++ b/packages/mcp-server/test/sync/test-records-fold-links.test.js @@ -1,6 +1,6 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { applyRecordsPass1, applyRecordsPass2, orderTablesByLinkDeps } from '../../src/sync/records.js'; +import { applyRecordsPass1, applyRecordsPass2, orderTablesByLinkDeps, applySyncRateDelay } from '../../src/sync/records.js'; import { newJournal } from '../../src/sync/journal.js'; import { createLimiter } from '../../src/sync/ratelimit.js'; @@ -43,6 +43,23 @@ describe('orderTablesByLinkDeps', () => { }); }); +// ── applySyncRateDelay (proactive pacing for the records phase) ────────────── +describe('applySyncRateDelay', () => { + it('enables a >0 inter-call delay and restores the prior value', () => { + const auth = { _rateDelayMs: 0 }; + const restore = applySyncRateDelay({ auth }); + assert.ok(auth._rateDelayMs >= 200, 'pacing delay enabled during the phase'); + restore(); + assert.equal(auth._rateDelayMs, 0, 'prior delay restored after the phase'); + }); + + it('is a safe no-op when the client has no auth (tests/mocks)', () => { + const restore = applySyncRateDelay({}); + assert.equal(typeof restore, 'function'); + assert.doesNotThrow(() => restore()); + }); +}); + // ── Pass 1 folds resolvable links into the create payload ──────────────────── describe('applyRecordsPass1 — fold resolvable links into create', () => { function captureClient(captured, { failOnLink = false } = {}) { From 8c81b55da0041630b912794055c9f7a33ac77c52 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Thu, 18 Jun 2026 16:23:13 +0300 Subject: [PATCH 078/246] fix(sync): pace records POSTs via a phase-local gate, not the shared auth queue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial review (HIGH): the previous pacing mutated the shared auth._rateDelayMs for the whole fire-and-forget records job, so every INTERACTIVE MCP tool call routed through the same auth queue was throttled to ~5 req/s for the entire multi-minute background run — the opposite of the "stay snappy" intent, and the non-refcounted save/restore corrupted the delay under overlapping runs. Replace it with a per-POST `gate` threaded into createRecords / updateRecords / addLinkItems (optional, defaults to pass-through). The records engine passes gate = (fn) => limiter.run(() => withRetry(fn)) using its OWN phase-local limiter, so only this phase's inner POSTs are paced (≈5 req/s) — interactive traffic on the shared auth queue is untouched. Also removes the double-pacing (engine limiter + auth delay) the review flagged. Correct the applyRecords docstring: resume is idmap + live-snapshot based (the records journal is advisory, not done-gated); documents the ~1-chunk duplicate window on a crash between create-ack and persist as a known limitation. 480/480 mcp-server tests green (gate-routing test asserts one limiter slot per inner create POST). Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/client.js | 18 +++-- packages/mcp-server/src/sync/records.js | 66 ++++++++----------- .../test/sync/test-records-fold-links.test.js | 48 +++++++++----- 3 files changed, 74 insertions(+), 58 deletions(-) diff --git a/packages/mcp-server/src/client.js b/packages/mcp-server/src/client.js index dff0dfd..f0fc6f6 100644 --- a/packages/mcp-server/src/client.js +++ b/packages/mcp-server/src/client.js @@ -2581,13 +2581,17 @@ export class AirtableClient { * @param {{viewId?: string}} [opts] * @returns {Promise<{created: {rowId:string, sourceKey:any}[], failed: {sourceKey:any, error:string}[]}>} */ - async createRecords(appId, tableId, rows, { viewId } = {}) { + async createRecords(appId, tableId, rows, { viewId, gate } = {}) { assertAirtableId(appId, 'appId'); assertAirtableId(tableId, 'tableId'); if (!Array.isArray(rows) || rows.length === 0) { throw new Error('rows must be a non-empty array'); } if (viewId) assertAirtableId(viewId, 'viewId'); + // `gate` wraps each per-row POST so a caller (the records sync) can pace/retry the INNER + // requests under a phase-local rate limiter — without throttling the shared auth queue that + // also serves interactive tool calls. Defaults to pass-through. + const g = gate || ((fn) => fn()); let activeViewId = viewId; if (!activeViewId) { const table = await this.resolveTable(appId, tableId); @@ -2602,7 +2606,7 @@ export class AirtableClient { if (activeViewId) payload.activeViewId = activeViewId; const url = `https://airtable.com/v0.3/row/${rowId}/create`; try { - const res = await this.auth.postForm(url, this._mutationParams(payload, appId), appId); + const res = await g(() => this.auth.postForm(url, this._mutationParams(payload, appId), appId)); if (!res.ok) { const body = await res.text().catch(() => ''); failed.push({ sourceKey: row.sourceKey ?? null, error: `createRecords row failed (${res.status}): ${body}` }); @@ -2626,12 +2630,13 @@ export class AirtableClient { * @param {{rowId:string, cellValuesByColumnId:object}[]} updates * @returns {Promise<{updated:{rowId:string}[], failed:{rowId:string, error:string}[]}>} */ - async updateRecords(appId, tableId, updates) { + async updateRecords(appId, tableId, updates, { gate } = {}) { assertAirtableId(appId, 'appId'); assertAirtableId(tableId, 'tableId'); if (!Array.isArray(updates) || updates.length === 0) { throw new Error('updates must be a non-empty array'); } + const g = gate || ((fn) => fn()); // see createRecords: paces inner per-cell POSTs when supplied const updated = []; const failed = []; for (const u of updates) { @@ -2645,7 +2650,7 @@ export class AirtableClient { const url = `https://airtable.com/v0.3/row/${u.rowId}/updatePrimitiveCell`; let rowFailed = null; for (const [columnId, cellValue] of entries) { - const res = await this.auth.postForm(url, this._mutationParams({ columnId, cellValue }, appId), appId); + const res = await g(() => this.auth.postForm(url, this._mutationParams({ columnId, cellValue }, appId), appId)); if (!res.ok) { const body = await res.text().catch(() => ''); rowFailed = `updatePrimitiveCell ${columnId} failed (${res.status}): ${body}`; @@ -2781,12 +2786,13 @@ export class AirtableClient { * @param {Array<{foreignRowId:string, foreignRowDisplayName:string}>} items * @returns {Promise<{ok:boolean, added:number, error?:string}>} */ - async addLinkItems(appId, rowId, columnId, items) { + async addLinkItems(appId, rowId, columnId, items, { gate } = {}) { const url = `https://airtable.com/v0.3/row/${rowId}/updateArrayTypeCellByAddingItem`; + const g = gate || ((fn) => fn()); // see createRecords: paces inner per-item POSTs when supplied let added = 0; try { for (const item of items) { - const res = await this.auth.postForm(url, this._mutationParams({ columnId, item }, appId), appId); + const res = await g(() => this.auth.postForm(url, this._mutationParams({ columnId, item }, appId), appId)); if (!res.ok) { const body = await res.text().catch(() => ''); return { ok: false, added, error: `addLinkItem failed (${res.status}): ${body}` }; diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index 858cefd..2ef855b 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -40,27 +40,20 @@ export function readRecordsJobStatus(sourceBaseId, destBaseId, planId) { } /** - * Enable per-request pacing for the records phase, then return a restore fn. + * Build a "gate" that wraps each INNER per-row/per-item POST of the records phase so it runs + * through the phase-local rate limiter (≈5 req/s) with transient retry. Passed into + * client.createRecords/updateRecords/addLinkItems via their `gate` option. * - * Every browser-backed call funnels through the auth queue, which spaces dequeued calls by - * `auth._rateDelayMs` (default 0 = off). The records phase fires many POSTs (per-row creates + - * per-item link appends); without pacing they burst past Airtable's ~5 req/s per-base limit and - * trip the 429/403 backoff (reactive). Defaulting the delay ON (200ms ≈ 5/s, override via - * AIRTABLE_SYNC_RATE_DELAY_MS) paces the inner POSTs PROACTIVELY so we stay under the limit and - * don't provoke the abuse detector. Scoped to the phase: restored afterward so interactive MCP - * tool calls stay snappy. No-op when the client has no auth (tests). + * Why a per-POST gate and not the shared auth queue's `_rateDelayMs`: the records phase runs as a + * non-awaited BACKGROUND job, and the daemon serves every interactive tool call through the SAME + * auth queue — delaying that queue would throttle all interactive traffic for the whole multi-minute + * run. This limiter is created per records run, so only THIS phase's POSTs are paced; interactive + * calls are untouched. * - * @param {object} client - * @returns {() => void} restore — resets the prior delay + * @param {{run:(fn:()=>Promise)=>Promise}} limiter + * @returns {(fn:()=>Promise)=>Promise} */ -export function applySyncRateDelay(client) { - const auth = client?.auth; - if (!auth) return () => {}; - const prev = auth._rateDelayMs; - const ms = Number(process.env.AIRTABLE_SYNC_RATE_DELAY_MS); - auth._rateDelayMs = Number.isFinite(ms) && ms > 0 ? ms : 200; - return () => { auth._rateDelayMs = prev; }; -} +const makeGate = (limiter) => (fn) => limiter.run(() => withRetry(fn)); /** * Returns true if the field type is a multiSelect (arrays not accepted by updateRecords). @@ -253,9 +246,10 @@ function mirrorFoldedLinks(destTableById, destTableId, destRowId, linkCells) { * but not live-verified — this is the calibration knob). */ async function createChunkWithFallback({ client, destAppId, destTableId, chunk, idmap, result, limiter, runState, destTableById }) { + const gate = makeGate(limiter); // paces each inner per-row create POST under the phase limiter let res; try { - res = await limiter.run(() => withRetry(() => client.createRecords(destAppId, destTableId, chunk, {}))); + res = await client.createRecords(destAppId, destTableId, chunk, { gate }); } catch (err) { result.failed += chunk.length; result.warnings.push({ code: 'RECORD_CREATE_FAILED', message: `Table ${destTableId}: createRecords threw: ${err.message}` }); @@ -294,7 +288,7 @@ async function createChunkWithFallback({ client, destAppId, destTableId, chunk, }); let retryRes; try { - retryRes = await limiter.run(() => withRetry(() => client.createRecords(destAppId, destTableId, stripped, {}))); + retryRes = await client.createRecords(destAppId, destTableId, stripped, { gate }); } catch (err) { result.failed += stripped.length; result.warnings.push({ code: 'RECORD_CREATE_FAILED', message: `Table ${destTableId}: retry-without-links threw: ${err.message}` }); @@ -379,9 +373,7 @@ export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idm // ── UPDATE batch ────────────────────────────────────────────────────── if (updateRows.length > 0) { try { - const res = await limiter.run(() => - withRetry(() => client.updateRecords(destAppId, destTableId, updateRows)), - ); + const res = await client.updateRecords(destAppId, destTableId, updateRows, { gate: makeGate(limiter) }); result.updated += (res.updated || []).length; for (const failed of (res.failed || [])) { result.failed++; @@ -495,9 +487,7 @@ export async function applyRecordsPass2({ client, srcSnapshot, destSnapshot, idm })); try { - const res = await limiter.run(() => - withRetry(() => client.addLinkItems(destAppId, destRowId, destFldId, items)), - ); + const res = await client.addLinkItems(destAppId, destRowId, destFldId, items, { gate: makeGate(limiter) }); if (!res.ok) { result.warnings.push({ code: 'RECORD_LINK_FAILED', @@ -915,12 +905,21 @@ export async function reapplyViewFilters({ client, srcSnapshot, destSnapshot, id * 2. Snapshots src + dest schema (no records yet). * 3. Attaches records to each table snapshot via snapshotTableRecords. * Emits RECORD_COUNT warning when exactly 1000 rows are returned (possibly truncated). - * 4. Builds a rate limiter and records journal (resumable). - * 5. Runs: Pass 1 (scalar/select upsert) → rebuild destDisplayNames → Pass 2 (links) - * → Pass 3 (attachments) → reapplyViewFilters. - * 6. Persists idmap + journal after each phase. + * 4. Builds a phase-local rate limiter (~5 req/s; gates inner POSTs) and a records journal. + * 5. Runs: Pass 1 (scalar/select upsert + fold resolvable links into create) → rebuild + * destDisplayNames → Pass 2 (remaining/forward links) → Pass 3 (attachments) + * → reapplyViewFilters. + * 6. Persists idmap (+ journal) after each chunk/phase. * 7. Returns a result accumulator. * + * RESUME MODEL: resume is driven by the persisted **idmap.records** + a fresh live dest snapshot — + * a created row is skipped on re-run because its source id is already mapped, and Pass 2 dedups + * links against the live dest. The records journal is persisted but currently advisory (no + * per-record done-gating); idmap + live re-snapshot are the source of truth. Known limitation: + * a crash between a create's server-ack and the per-chunk idmap persist can re-create up to one + * chunk (~50) of rows as duplicates on resume (reconcile only existence-prunes; natural-key + * re-match is a stub). + * * @param {object} opts * @param {object} opts.client - AirtableClient instance * @param {string} opts.sourceBaseId - source base app ID @@ -935,10 +934,6 @@ export async function applyRecords({ client, sourceBaseId, destBaseId, planId, r idmap.records ??= {}; idmap.attachments ??= {}; - // Pace every POST in this phase under ~5 req/s (proactive); restored in finally. - const restoreRateDelay = applySyncRateDelay(client); - try { - // 2. Snapshot schemas ONLY (no per-view live-config reads — those are a getView/readData // storm: ~1s per view, hundreds of views on a view-heavy base, on BOTH bases. The records // phase needs schema + records, not view configs. reapplyViewFilters reads source view @@ -1017,9 +1012,6 @@ export async function applyRecords({ client, sourceBaseId, destBaseId, planId, r persist(idmap, journal); return result; - } finally { - restoreRateDelay(); - } } // ────────────────────────────────────────────────────────────────────────────── diff --git a/packages/mcp-server/test/sync/test-records-fold-links.test.js b/packages/mcp-server/test/sync/test-records-fold-links.test.js index c7b6ffd..ee6b6ce 100644 --- a/packages/mcp-server/test/sync/test-records-fold-links.test.js +++ b/packages/mcp-server/test/sync/test-records-fold-links.test.js @@ -1,6 +1,6 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { applyRecordsPass1, applyRecordsPass2, orderTablesByLinkDeps, applySyncRateDelay } from '../../src/sync/records.js'; +import { applyRecordsPass1, applyRecordsPass2, orderTablesByLinkDeps } from '../../src/sync/records.js'; import { newJournal } from '../../src/sync/journal.js'; import { createLimiter } from '../../src/sync/ratelimit.js'; @@ -43,20 +43,38 @@ describe('orderTablesByLinkDeps', () => { }); }); -// ── applySyncRateDelay (proactive pacing for the records phase) ────────────── -describe('applySyncRateDelay', () => { - it('enables a >0 inter-call delay and restores the prior value', () => { - const auth = { _rateDelayMs: 0 }; - const restore = applySyncRateDelay({ auth }); - assert.ok(auth._rateDelayMs >= 200, 'pacing delay enabled during the phase'); - restore(); - assert.equal(auth._rateDelayMs, 0, 'prior delay restored after the phase'); - }); - - it('is a safe no-op when the client has no auth (tests/mocks)', () => { - const restore = applySyncRateDelay({}); - assert.equal(typeof restore, 'function'); - assert.doesNotThrow(() => restore()); +// ── inner-POST pacing: each create POST routes through the phase-local limiter (gate) ──────── +describe('records phase pacing (gate)', () => { + it('routes each inner create POST through the records limiter, not the shared auth queue', async () => { + let gateRuns = 0; + const limiterSpy = { run: (fn) => { gateRuns++; return fn(); } }; // stands in for createLimiter + const client = { + // simulate the real createRecords: one gated POST per row + createRecords: async (appId, tableId, rows, opts) => { + const created = []; + for (const r of rows) { + await opts.gate(() => Promise.resolve({ ok: true })); // inner per-row POST through the gate + created.push({ rowId: 'recD_' + r.sourceKey, sourceKey: r.sourceKey }); + } + return { created, failed: [] }; + }, + updateRecords: async () => ({ updated: [], failed: [] }), + }; + const idmap = { tables: { tblS: 'tblD' }, fields: { fldT: { destFld: 'fldTD', choices: {} } }, records: {} }; + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', fields: [{ id: 'fldT', type: 'text' }], + records: [1, 2, 3].map((n) => ({ id: 'recS' + n, cellValuesByColumnId: { fldT: 'v' + n } })), + }], + }; + const destSnapshot = { baseId: 'appD', tables: [{ id: 'tblD', name: 'T', fields: [], records: [] }] }; + const result = newResult(); + await applyRecordsPass1({ + client, srcSnapshot, destSnapshot, idmap, + limiter: limiterSpy, journal: newJournal('p', 't'), persist: () => {}, result, + }); + assert.equal(result.created, 3); + assert.equal(gateRuns, 3, 'one limiter slot per inner create POST (paced), not one per chunk'); }); }); From 8934a054a4b28d349d57227230af2fd240eaa1bd Mon Sep 17 00:00:00 2001 From: "A.R." Date: Thu, 18 Jun 2026 16:53:43 +0300 Subject: [PATCH 079/246] feat(sync): AIRTABLE_SYNC_RPS knob for the records-phase limiter A scoped live run validated fold-into-create (11 records, 0 failures, breaker untripped) but the dest account re-throttled after ~18 cumulative requests: Airtable enforces a per-window cap that per-second pacing can't dodge, and a recently-abused base stays throttle-sensitized. Expose the phase limiter rate (default 5) so a sensitized base can be dialed down (AIRTABLE_SYNC_RPS=2) without a code change. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/records.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index 2ef855b..0510876 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -952,7 +952,11 @@ export async function applyRecords({ client, sourceBaseId, destBaseId, planId, r } // 4. Infra: rate limiter + journal + persist + result - const limiter = createLimiter({ rps: 5 }); + // Phase-local pacing. Default ~5 req/s; lower it (AIRTABLE_SYNC_RPS=2) for an account that's + // throttle-sensitized — Airtable also enforces a per-WINDOW cap that per-second pacing can't dodge, + // so a recently-abused base may need a slower rate (or a full cooldown) to avoid re-tripping. + const rps = Number(process.env.AIRTABLE_SYNC_RPS) || 5; + const limiter = createLimiter({ rps }); const journal = loadRecordsJournal(sourceBaseId, destBaseId, planId) ?? newJournal(planId, runStartedAt); const persist = (m, j) => { saveIdmap(sourceBaseId, destBaseId, m); From dce82e2b027d43161df56851983745cfe4b1edd2 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Fri, 19 Jun 2026 16:52:02 +0300 Subject: [PATCH 080/246] feat(sync): capture view sections in normalizeSchema (no extra API call) Co-Authored-By: Claude Sonnet 4.6 --- packages/mcp-server/src/sync/snapshot.js | 18 +++++++++++++----- .../mcp-server/test/sync/test-snapshot.test.js | 17 +++++++++++++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/packages/mcp-server/src/sync/snapshot.js b/packages/mcp-server/src/sync/snapshot.js index 652040f..47aa464 100644 --- a/packages/mcp-server/src/sync/snapshot.js +++ b/packages/mcp-server/src/sync/snapshot.js @@ -28,6 +28,17 @@ export function normalizeSchema(rawData) { return { tables: tables.map((t) => { const cols = t.columns ?? t.fields ?? []; + const views = (t.views || []).map((v) => ({ + id: v.id, name: v.name, type: v.type, + description: v.description ?? null, + personalForUserId: v.personalForUserId ?? null, + })); + const viewNameById = new Map(views.map(v => [v.id, v.name])); + const sectionsObj = t.viewSectionsById || t.viewSections || {}; + const sections = Object.values(sectionsObj).map(s => ({ + name: s.name, + viewNames: (s.viewOrder || []).map(id => viewNameById.get(id)).filter(Boolean), + })); return { id: t.id, name: t.name, @@ -42,11 +53,8 @@ export function normalizeSchema(rawData) { description: c.description ?? null, isComputed: isComputedType(c.type), })), - views: (t.views || []).map((v) => ({ - id: v.id, name: v.name, type: v.type, - description: v.description ?? null, - personalForUserId: v.personalForUserId ?? null, - })), + views, + sections, }; }), }; diff --git a/packages/mcp-server/test/sync/test-snapshot.test.js b/packages/mcp-server/test/sync/test-snapshot.test.js index 2a786e6..b09a718 100644 --- a/packages/mcp-server/test/sync/test-snapshot.test.js +++ b/packages/mcp-server/test/sync/test-snapshot.test.js @@ -39,6 +39,23 @@ describe('snapshot.normalizeSchema', () => { }); }); +describe('snapshot sections', () => { + it('normalizeSchema attaches sections as {name, viewNames}', () => { + const raw = { data: { tableSchemas: [{ + id: 'tblA', name: 'T', primaryColumnId: 'fld1', + columns: [{ id: 'fld1', name: 'Name', type: 'text' }], + views: [{ id: 'viwX', name: 'Grid', type: 'grid' }, { id: 'viwY', name: 'Kanban', type: 'kanban' }], + viewSectionsById: { vsc1: { id: 'vsc1', name: 'Sales Views', viewOrder: ['viwY', 'viwX'] } }, + }] } }; + const snap = normalizeSchema(raw); + assert.deepEqual(snap.tables[0].sections, [{ name: 'Sales Views', viewNames: ['Kanban', 'Grid'] }]); + }); + it('normalizeSchema sections defaults to [] when absent', () => { + const raw = { data: { tableSchemas: [{ id: 'tblA', name: 'T', columns: [], views: [] }] } }; + assert.deepEqual(normalizeSchema(raw).tables[0].sections, []); + }); +}); + describe('snapshot views', () => { it('normalizeSchema attaches static views with personal flag', () => { const raw = { data: { tableSchemas: [{ id: 'tbl1', name: 'T', primaryColumnId: 'f1', columns: [{ id: 'f1', name: 'Name', type: 'text' }], From 11119504c09661ff6fffadbe08c0f59170c66b9e Mon Sep 17 00:00:00 2001 From: "A.R." Date: Fri, 19 Jun 2026 16:55:16 +0300 Subject: [PATCH 081/246] feat(sync): compare.js classification map Adds pure classOf(key) helper and DIFF_CLASS proxy for tagging schema diff attributes as 'drift' | 'best-effort' | 'not-synced'. Co-Authored-By: Claude Sonnet 4.6 --- packages/mcp-server/src/sync/compare.js | 20 +++++++++++++++++++ .../mcp-server/test/sync/test-compare.test.js | 14 +++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 packages/mcp-server/src/sync/compare.js create mode 100644 packages/mcp-server/test/sync/test-compare.test.js diff --git a/packages/mcp-server/src/sync/compare.js b/packages/mcp-server/src/sync/compare.js new file mode 100644 index 0000000..60e51ba --- /dev/null +++ b/packages/mcp-server/src/sync/compare.js @@ -0,0 +1,20 @@ +// compare.js — base-schema comparator helpers (pure: no fs, no network, no Date.now/Math.random) + +const BEST_EFFORT = new Set(['fieldOrder', 'viewOrder', 'columnOrder', 'sortOrder', 'groupOrder']); +const NOT_SYNCED = new Set(['sections']); + +/** @type {Record} */ +export const DIFF_CLASS = new Proxy({}, { + get(_target, key) { return classOf(key); }, +}); + +/** + * Classify an attribute key as 'drift', 'best-effort', or 'not-synced'. + * @param {string} key + * @returns {'drift'|'best-effort'|'not-synced'} + */ +export function classOf(key) { + if (BEST_EFFORT.has(key)) return 'best-effort'; + if (NOT_SYNCED.has(key)) return 'not-synced'; + return 'drift'; +} diff --git a/packages/mcp-server/test/sync/test-compare.test.js b/packages/mcp-server/test/sync/test-compare.test.js new file mode 100644 index 0000000..1302c93 --- /dev/null +++ b/packages/mcp-server/test/sync/test-compare.test.js @@ -0,0 +1,14 @@ +import { classOf } from '../../src/sync/compare.js'; +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +test('classOf maps order keys to best-effort, sections to not-synced, rest to drift', () => { + assert.equal(classOf('type'), 'drift'); + assert.equal(classOf('choices'), 'drift'); + assert.equal(classOf('fieldOrder'), 'best-effort'); + assert.equal(classOf('viewOrder'), 'best-effort'); + assert.equal(classOf('columnOrder'), 'best-effort'); + assert.equal(classOf('sortOrder'), 'best-effort'); + assert.equal(classOf('sections'), 'not-synced'); + assert.equal(classOf('somethingElse'), 'drift'); // default +}); From 0275d261cdbc08b75f1334fcf5e45fb855217a2d Mon Sep 17 00:00:00 2001 From: "A.R." Date: Fri, 19 Jun 2026 17:02:19 +0300 Subject: [PATCH 082/246] feat(sync): compareFields with drift + best-effort field order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Extract stableStringify, choiceNames, scalarTypeOptionsChanged, computedSig, fieldSignature into src/sync/field-compare.js (shared helpers) - Update diff.js to import from field-compare.js (behaviour byte-identical, all 15 diff tests still green) - Implement compareFields(srcTable, destTable, idmap) in compare.js: matches fields via idmap then by-name fallback; emits drift entries for type/typeOptions/choices/description; emits one best-effort fieldOrder entry when matched-name sequence differs; collects onlyInSource/onlyInDest - 12 TDD tests in test-compare.test.js (RED → GREEN); full suite 494/494 pass Co-Authored-By: Claude Sonnet 4.6 --- packages/mcp-server/src/sync/compare.js | 131 ++++++++++++++++ packages/mcp-server/src/sync/diff.js | 122 +-------------- packages/mcp-server/src/sync/field-compare.js | 126 ++++++++++++++++ .../mcp-server/test/sync/test-compare.test.js | 142 +++++++++++++++++- 4 files changed, 399 insertions(+), 122 deletions(-) create mode 100644 packages/mcp-server/src/sync/field-compare.js diff --git a/packages/mcp-server/src/sync/compare.js b/packages/mcp-server/src/sync/compare.js index 60e51ba..676525b 100644 --- a/packages/mcp-server/src/sync/compare.js +++ b/packages/mcp-server/src/sync/compare.js @@ -1,5 +1,7 @@ // compare.js — base-schema comparator helpers (pure: no fs, no network, no Date.now/Math.random) +import { stableStringify, choiceNames, scalarTypeOptionsChanged, computedSig } from './field-compare.js'; + const BEST_EFFORT = new Set(['fieldOrder', 'viewOrder', 'columnOrder', 'sortOrder', 'groupOrder']); const NOT_SYNCED = new Set(['sections']); @@ -18,3 +20,132 @@ export function classOf(key) { if (NOT_SYNCED.has(key)) return 'not-synced'; return 'drift'; } + +/** + * Build a flat map of { fieldId → fieldName } for the fields in a table. + * @param {{ fields: Array<{id:string, name:string}> }} table + * @returns {Record} + */ +function tableFieldNameMap(table) { + const m = {}; + for (const f of table.fields) m[f.id] = f.name; + return m; +} + +/** + * Compare fields between a source table and a destination table using the provided idmap. + * + * Matching strategy: + * 1. For each src field, look up destFld id via idmap.fields[srcField.id]. + * 2. If idmap has no entry for this src field, fall back to matching by field name in dest. + * + * For each matched pair, compare: + * - type (raw string) + * - typeOptions: for computed fields use computedSig (ID-stable); for scalars use scalarTypeOptionsChanged + * - choices: using scalarTypeOptionsChanged (which does set-membership for select types) + * - description (normalised to null) + * + * Also emits one { scope:'table', key:'fieldOrder', class:'best-effort', source, dest } entry + * when the matched-field name sequence differs between src and dest. + * + * @param {{ fields: Array<{id:string, name:string, type:string, typeOptions:object|null, description:string|null, isComputed:boolean}> }} srcTable + * @param {{ fields: Array<{id:string, name:string, type:string, typeOptions:object|null, description:string|null, isComputed:boolean}> }} destTable + * @param {{ fields: Record}> }} idmap + * @returns {{ entries: Array<{scope:string, key:string, source:unknown, dest:unknown, class:string}>, onlyInSource: string[], onlyInDest: string[] }} + */ +export function compareFields(srcTable, destTable, idmap) { + const entries = []; + const onlyInSource = []; + const onlyInDest = []; + + // Build name→field map for dest for by-name fallback and unmatched detection. + const destByName = new Map(destTable.fields.map((f) => [f.name, f])); + // Track which dest field names were matched so we can find dest-only fields. + const matchedDestNames = new Set(); + + // Build id→name maps for each side so computed sigs can normalise references. + const srcFldNames = tableFieldNameMap(srcTable); + const destFldNames = tableFieldNameMap(destTable); + + // srcMatchedOrder: matched field names in src iteration order (for fieldOrder check). + const srcMatchedOrder = []; + + for (const sf of srcTable.fields) { + // Resolve dest field: idmap first, then by-name fallback. + let df = null; + const idmapEntry = idmap.fields && idmap.fields[sf.id]; + if (idmapEntry) { + df = destTable.fields.find((f) => f.id === idmapEntry.destFld) ?? null; + } + if (!df) { + df = destByName.get(sf.name) ?? null; + } + + if (!df) { + onlyInSource.push(sf.name); + continue; + } + + matchedDestNames.add(df.name); + srcMatchedOrder.push(sf.name); + + const scope = `field:${sf.name}`; + + // Compare type. + if (sf.type !== df.type) { + entries.push({ scope, key: 'type', source: sf.type, dest: df.type, class: classOf('type') }); + } + + // Compare typeOptions / choices. + if (sf.isComputed) { + // Computed: use canonical, ID-stable sig for typeOptions. + if (computedSig(sf, srcFldNames) !== computedSig(df, destFldNames)) { + entries.push({ scope, key: 'typeOptions', source: sf.typeOptions, dest: df.typeOptions, class: classOf('typeOptions') }); + } + } else { + // Scalar: check choices separately (set-membership) and other typeOptions. + const srcChoices = choiceNames(sf.typeOptions); + if (srcChoices !== null) { + // It's a select-type — check via scalarTypeOptionsChanged (choice set membership). + if (scalarTypeOptionsChanged(sf, df)) { + entries.push({ scope, key: 'choices', source: sf.typeOptions, dest: df.typeOptions, class: classOf('choices') }); + } + } else if (scalarTypeOptionsChanged(sf, df)) { + entries.push({ scope, key: 'typeOptions', source: sf.typeOptions, dest: df.typeOptions, class: classOf('typeOptions') }); + } + } + + // Compare description (normalised to null). + const srcDesc = sf.description ?? null; + const destDesc = df.description ?? null; + if (srcDesc !== destDesc) { + entries.push({ scope, key: 'description', source: srcDesc, dest: destDesc, class: classOf('description') }); + } + } + + // Collect dest-only fields. + for (const df of destTable.fields) { + if (!matchedDestNames.has(df.name)) { + onlyInDest.push(df.name); + } + } + + // Check fieldOrder: compare the matched field name sequence in src iteration order + // vs the same names in their dest table iteration order. + const destFieldIndex = new Map(destTable.fields.map((f, i) => [f.name, i])); + const destOrderedMatchedNames = [...srcMatchedOrder].sort( + (a, b) => (destFieldIndex.get(a) ?? Infinity) - (destFieldIndex.get(b) ?? Infinity), + ); + + if (srcMatchedOrder.join('\0') !== destOrderedMatchedNames.join('\0')) { + entries.push({ + scope: 'table', + key: 'fieldOrder', + source: srcMatchedOrder, + dest: destOrderedMatchedNames, + class: 'best-effort', + }); + } + + return { entries, onlyInSource, onlyInDest }; +} diff --git a/packages/mcp-server/src/sync/diff.js b/packages/mcp-server/src/sync/diff.js index 0f97cf7..d6cd51b 100644 --- a/packages/mcp-server/src/sync/diff.js +++ b/packages/mcp-server/src/sync/diff.js @@ -1,4 +1,5 @@ -import { canonicalizeComputed, canonicalizeViewConfig } from './remap.js'; +import { canonicalizeViewConfig } from './remap.js'; +import { stableStringify, choiceNames, scalarTypeOptionsChanged, computedSig, fieldSignature } from './field-compare.js'; /** * Build a flat map of { fieldId → fieldName } across all tables in a snapshot. @@ -28,129 +29,10 @@ function selNameMap(snap) { /** Return only collaborative (non-personal) views from a table. */ function collabViews(table) { return (table.views || []).filter((v) => !v.personalForUserId); } -/** - * Build a regex that matches any field ID key in the given map, wrapped in - * curly braces (the Airtable formula token syntax: `{fldXYZ}`). Returns null - * if the map is empty. - * @param {Record} idToName - * @returns {RegExp|null} - */ -function buildIdRegex(idToName) { - const ids = Object.keys(idToName); - if (ids.length === 0) return null; - // Escape each ID and sort longest-first to avoid partial matches. - const escaped = ids - .slice() - .sort((a, b) => b.length - a.length) - .map((id) => id.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')); - return new RegExp(escaped.join('|'), 'g'); -} - -/** - * Replace every occurrence of a field ID found in `idToName` within `str` - * with `{{}}`. Handles both bare IDs (in formulaText) and - * curly-brace-wrapped tokens (`{fldXYZ}` in formulaTextParsed). - * @param {string} str - * @param {Record} idToName - * @returns {string} - */ -function subAllIds(str, idToName) { - if (typeof str !== 'string' || str.length === 0) return str; - const re = buildIdRegex(idToName); - if (!re) return str; - return str.replace(re, (id) => `{{${idToName[id] ?? id}}}`); -} - -/** - * Stable, key-sorted JSON so equal options compare equal regardless of key insertion order. - * @param {unknown} obj - * @returns {string} - */ -function stableStringify(obj) { - if (obj === null || typeof obj !== 'object') return JSON.stringify(obj); - if (Array.isArray(obj)) return `[${obj.map(stableStringify).join(',')}]`; - return `{${Object.keys(obj).sort().map((k) => JSON.stringify(k) + ':' + stableStringify(obj[k])).join(',')}}`; -} - -/** - * Description-free canonical signature for a computed field's typeOptions. - * Normalises all field-ID references to their names so cross-base ID churn is - * invisible. Used both by `fieldSignature` (which appends description) and by - * the `updateField` diff to decide whether typeOptions genuinely changed. - * - * @param {{ type:string, typeOptions:object|null }} field - * @param {Record} fldNames fieldId → name for the relevant base - * @returns {string} - */ -function computedSig(field, fldNames) { - const opts = field.typeOptions || {}; - const normalizedOpts = { - ...opts, - formulaTextParsed: subAllIds(opts.formulaTextParsed ?? '', fldNames), - formulaText: subAllIds(opts.formulaText ?? '', fldNames), - formula: subAllIds(opts.formula ?? '', fldNames), - relationColumnId: fldNames[opts.relationColumnId ?? ''] ?? opts.relationColumnId ?? '', - recordLinkFieldId: fldNames[opts.recordLinkFieldId ?? ''] ?? opts.recordLinkFieldId ?? '', - foreignTableRollupColumnId: fldNames[opts.foreignTableRollupColumnId ?? ''] ?? opts.foreignTableRollupColumnId ?? '', - fieldIdInLinkedTable: fldNames[opts.fieldIdInLinkedTable ?? ''] ?? opts.fieldIdInLinkedTable ?? '', - }; - // canonicalizeComputed handles structured extraction; pass empty idToName - // since we already resolved all IDs above. - return 'C|' + field.type + '|' + canonicalizeComputed(field.type, normalizedOpts, {}); -} - -/** - * Comparable signature for a field. - * - Computed fields canonicalize field-ID references to field names so - * cross-base ID churn is invisible (two formulas that refer to the same-named field - * produce identical signatures even if the underlying field IDs differ). - * We first run our own ID→name substitution over the formula text (which handles - * any ID format, not just `fld`-prefixed ones), then call canonicalizeComputed for - * the remaining structured fields (relation, target, result type). - * - Scalar fields compare type + stable-serialised typeOptions + description. - * - * @param {{ type:string, typeOptions:object|null, description:string|null, isComputed:boolean }} field - * @param {Record} fldNames fieldId → name for the relevant base - * @returns {string} - */ -function fieldSignature(field, fldNames) { - if (field.isComputed) { - return computedSig(field, fldNames) + '|' + (field.description ?? ''); - } - return 'S|' + field.type + '|' + stableStringify(field.typeOptions ?? null) + '|' + (field.description ?? ''); -} - /** Link-type fields must be created after plain scalar fields. Computed last. */ const LINK_TYPES = new Set(['multipleRecordLinks', 'foreignKey']); function fieldOrder(f) { return f.isComputed ? 2 : (LINK_TYPES.has(f.type) ? 1 : 0); } -/** Set of choice names for a select-like field, or null if not a select. */ -function choiceNames(typeOptions) { - const ch = typeOptions && typeOptions.choices; - return ch ? new Set(Object.values(ch).map((c) => c.name)) : null; -} - -/** - * Whether a non-computed field's typeOptions genuinely needs an update, aligned with what - * apply actually does (so plan converges to zero instead of re-emitting phantom updates): - * - select/multiSelect: apply MERGES choices additively (never drops, invariant 7), so an - * update is needed only when SOURCE has a choice name DEST lacks. dest ⊇ src ⇒ equal. - * (Existing-choice colour/order are kept by apply's merge, so they're not flagged here.) - * - other types: a real typeOptions diff, but skip when source options are empty — apply - * can't clear options non-destructively and sending {} is a no-op that never converges. - */ -function scalarTypeOptionsChanged(sf, df) { - const srcChoices = choiceNames(sf.typeOptions); - if (srcChoices) { - const destChoices = choiceNames(df.typeOptions) || new Set(); - for (const n of srcChoices) if (!destChoices.has(n)) return true; // a source choice missing in dest - return false; // dest already has every source choice - } - if (stableStringify(sf.typeOptions ?? null) === stableStringify(df.typeOptions ?? null)) return false; - // ponytail: source has no options to push and we don't strip dest options (destructive, M4) → skip - return !!(sf.typeOptions && Object.keys(sf.typeOptions).length > 0); -} - /** * Collect all field IDs referenced by a field (formula tokens, link columns, etc.) * so the executor can topologically sort creation order. diff --git a/packages/mcp-server/src/sync/field-compare.js b/packages/mcp-server/src/sync/field-compare.js new file mode 100644 index 0000000..356dafe --- /dev/null +++ b/packages/mcp-server/src/sync/field-compare.js @@ -0,0 +1,126 @@ +// field-compare.js — shared field-comparison helpers used by both diff.js and compare.js. +// Pure module: no fs, no network, no Date.now, no Math.random. + +import { canonicalizeComputed } from './remap.js'; + +// ── Internal helpers ───────────────────────────────────────────────────────── + +/** + * Build a regex that matches any field ID key in the given map. + * @param {Record} idToName + * @returns {RegExp|null} + */ +function buildIdRegex(idToName) { + const ids = Object.keys(idToName); + if (ids.length === 0) return null; + const escaped = ids + .slice() + .sort((a, b) => b.length - a.length) + .map((id) => id.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')); + return new RegExp(escaped.join('|'), 'g'); +} + +/** + * Replace every occurrence of a field ID found in `idToName` within `str` + * with `{{}}`. + * @param {string} str + * @param {Record} idToName + * @returns {string} + */ +function subAllIds(str, idToName) { + if (typeof str !== 'string' || str.length === 0) return str; + const re = buildIdRegex(idToName); + if (!re) return str; + return str.replace(re, (id) => `{{${idToName[id] ?? id}}}`); +} + +// ── Exported helpers ───────────────────────────────────────────────────────── + +/** + * Stable, key-sorted JSON so equal options compare equal regardless of key insertion order. + * @param {unknown} obj + * @returns {string} + */ +export function stableStringify(obj) { + if (obj === null || typeof obj !== 'object') return JSON.stringify(obj); + if (Array.isArray(obj)) return `[${obj.map(stableStringify).join(',')}]`; + return `{${Object.keys(obj).sort().map((k) => JSON.stringify(k) + ':' + stableStringify(obj[k])).join(',')}}`; +} + +/** + * Set of choice names for a select-like field, or null if not a select. + * @param {object|null} typeOptions + * @returns {Set|null} + */ +export function choiceNames(typeOptions) { + const ch = typeOptions && typeOptions.choices; + return ch ? new Set(Object.values(ch).map((c) => c.name)) : null; +} + +/** + * Whether a non-computed field's typeOptions genuinely needs an update, aligned with what + * apply actually does (so plan converges to zero instead of re-emitting phantom updates): + * - select/multiSelect: apply MERGES choices additively (never drops, invariant 7), so an + * update is needed only when SOURCE has a choice name DEST lacks. dest ⊇ src ⇒ equal. + * (Existing-choice colour/order are kept by apply's merge, so they're not flagged here.) + * - other types: a real typeOptions diff, but skip when source options are empty — apply + * can't clear options non-destructively and sending {} is a no-op that never converges. + * @param {{ typeOptions: object|null }} sf source field + * @param {{ typeOptions: object|null }} df dest field + * @returns {boolean} + */ +export function scalarTypeOptionsChanged(sf, df) { + const srcChoices = choiceNames(sf.typeOptions); + if (srcChoices) { + const destChoices = choiceNames(df.typeOptions) || new Set(); + for (const n of srcChoices) if (!destChoices.has(n)) return true; // a source choice missing in dest + return false; // dest already has every source choice + } + if (stableStringify(sf.typeOptions ?? null) === stableStringify(df.typeOptions ?? null)) return false; + // ponytail: source has no options to push and we don't strip dest options (destructive, M4) → skip + return !!(sf.typeOptions && Object.keys(sf.typeOptions).length > 0); +} + +/** + * Description-free canonical signature for a computed field's typeOptions. + * Normalises all field-ID references to their names so cross-base ID churn is + * invisible. Used both by `fieldSignature` (which appends description) and by + * the `updateField` diff to decide whether typeOptions genuinely changed. + * + * @param {{ type:string, typeOptions:object|null }} field + * @param {Record} fldNames fieldId → name for the relevant base + * @returns {string} + */ +export function computedSig(field, fldNames) { + const opts = field.typeOptions || {}; + const normalizedOpts = { + ...opts, + formulaTextParsed: subAllIds(opts.formulaTextParsed ?? '', fldNames), + formulaText: subAllIds(opts.formulaText ?? '', fldNames), + formula: subAllIds(opts.formula ?? '', fldNames), + relationColumnId: fldNames[opts.relationColumnId ?? ''] ?? opts.relationColumnId ?? '', + recordLinkFieldId: fldNames[opts.recordLinkFieldId ?? ''] ?? opts.recordLinkFieldId ?? '', + foreignTableRollupColumnId: fldNames[opts.foreignTableRollupColumnId ?? ''] ?? opts.foreignTableRollupColumnId ?? '', + fieldIdInLinkedTable: fldNames[opts.fieldIdInLinkedTable ?? ''] ?? opts.fieldIdInLinkedTable ?? '', + }; + // canonicalizeComputed handles structured extraction; pass empty idToName + // since we already resolved all IDs above. + return 'C|' + field.type + '|' + canonicalizeComputed(field.type, normalizedOpts, {}); +} + +/** + * Comparable signature for a field. + * - Computed fields canonicalize field-ID references to field names so + * cross-base ID churn is invisible. + * - Scalar fields compare type + stable-serialised typeOptions + description. + * + * @param {{ type:string, typeOptions:object|null, description:string|null, isComputed:boolean }} field + * @param {Record} fldNames fieldId → name for the relevant base + * @returns {string} + */ +export function fieldSignature(field, fldNames) { + if (field.isComputed) { + return computedSig(field, fldNames) + '|' + (field.description ?? ''); + } + return 'S|' + field.type + '|' + stableStringify(field.typeOptions ?? null) + '|' + (field.description ?? ''); +} diff --git a/packages/mcp-server/test/sync/test-compare.test.js b/packages/mcp-server/test/sync/test-compare.test.js index 1302c93..f246ae4 100644 --- a/packages/mcp-server/test/sync/test-compare.test.js +++ b/packages/mcp-server/test/sync/test-compare.test.js @@ -1,7 +1,9 @@ -import { classOf } from '../../src/sync/compare.js'; -import { test } from 'node:test'; +import { classOf, compareFields } from '../../src/sync/compare.js'; +import { test, describe } from 'node:test'; import assert from 'node:assert/strict'; +// ── classOf tests (Task 1, kept here for completeness) ────────────────────── + test('classOf maps order keys to best-effort, sections to not-synced, rest to drift', () => { assert.equal(classOf('type'), 'drift'); assert.equal(classOf('choices'), 'drift'); @@ -12,3 +14,139 @@ test('classOf maps order keys to best-effort, sections to not-synced, rest to dr assert.equal(classOf('sections'), 'not-synced'); assert.equal(classOf('somethingElse'), 'drift'); // default }); + +// ── compareFields tests (Task 2) ───────────────────────────────────────────── + +function fld(id, name, type, extra = {}) { + return { + id, + name, + type, + typeOptions: extra.typeOptions ?? null, + description: extra.description ?? null, + isComputed: !!extra.isComputed, + }; +} + +describe('compareFields', () => { + test('detects type drift', () => { + const src = { fields: [fld('s1', 'Price', 'number')] }; + const dest = { fields: [fld('d1', 'Price', 'text')] }; + const idmap = { fields: { s1: { destFld: 'd1', choices: {} } } }; + const { entries } = compareFields(src, dest, idmap); + assert.deepEqual( + entries.find((e) => e.key === 'type'), + { scope: 'field:Price', key: 'type', source: 'number', dest: 'text', class: 'drift' }, + ); + }); + + test('detects description drift', () => { + const src = { fields: [fld('s1', 'Notes', 'text', { description: 'new desc' })] }; + const dest = { fields: [fld('d1', 'Notes', 'text', { description: 'old desc' })] }; + const idmap = { fields: { s1: { destFld: 'd1', choices: {} } } }; + const { entries } = compareFields(src, dest, idmap); + assert.deepEqual( + entries.find((e) => e.key === 'description'), + { scope: 'field:Notes', key: 'description', source: 'new desc', dest: 'old desc', class: 'drift' }, + ); + }); + + test('detects choices drift when source has a choice dest lacks', () => { + const srcOpts = { choices: { s1: { id: 's1', name: 'Open' }, s2: { id: 's2', name: 'Closed' } } }; + const destOpts = { choices: { d1: { id: 'd1', name: 'Open' } } }; + const src = { fields: [fld('s1', 'Status', 'singleSelect', { typeOptions: srcOpts })] }; + const dest = { fields: [fld('d1', 'Status', 'singleSelect', { typeOptions: destOpts })] }; + const idmap = { fields: { s1: { destFld: 'd1', choices: {} } } }; + const { entries } = compareFields(src, dest, idmap); + const choicesEntry = entries.find((e) => e.key === 'choices'); + assert.ok(choicesEntry, 'should emit choices drift entry'); + assert.equal(choicesEntry.scope, 'field:Status'); + assert.equal(choicesEntry.class, 'drift'); + }); + + test('no choices drift when dest is a superset of src choices', () => { + const srcOpts = { choices: { s1: { id: 's1', name: 'Open' } } }; + const destOpts = { choices: { d1: { id: 'd1', name: 'Open' }, d2: { id: 'd2', name: 'Extra' } } }; + const src = { fields: [fld('s1', 'Status', 'singleSelect', { typeOptions: srcOpts })] }; + const dest = { fields: [fld('d1', 'Status', 'singleSelect', { typeOptions: destOpts })] }; + const idmap = { fields: { s1: { destFld: 'd1', choices: {} } } }; + const { entries } = compareFields(src, dest, idmap); + assert.ok(!entries.find((e) => e.key === 'choices'), 'dest superset of src → no choices drift'); + }); + + test('flags field order as best-effort when sequence differs', () => { + const mk = (names) => ({ fields: names.map((n) => fld(n, n, 'text')) }); + const idmap = { fields: { A: { destFld: 'A', choices: {} }, B: { destFld: 'B', choices: {} } } }; + const { entries } = compareFields(mk(['A', 'B']), mk(['B', 'A']), idmap); + const o = entries.find((e) => e.key === 'fieldOrder'); + assert.ok(o, 'fieldOrder entry should exist'); + assert.equal(o.class, 'best-effort'); + assert.deepEqual([o.source, o.dest], [['A', 'B'], ['B', 'A']]); + }); + + test('no fieldOrder entry when sequence is identical', () => { + const mk = (names) => ({ fields: names.map((n) => fld(n, n, 'text')) }); + const idmap = { fields: { A: { destFld: 'A', choices: {} }, B: { destFld: 'B', choices: {} } } }; + const { entries } = compareFields(mk(['A', 'B']), mk(['A', 'B']), idmap); + assert.ok(!entries.find((e) => e.key === 'fieldOrder'), 'same order → no fieldOrder entry'); + }); + + test('reports source-only field in onlyInSource, not as an entry', () => { + const src = { fields: [fld('s1', 'A', 'text'), fld('s2', 'OnlySrc', 'text')] }; + const dest = { fields: [fld('d1', 'A', 'text')] }; + const idmap = { fields: { s1: { destFld: 'd1', choices: {} } } }; + const { entries, onlyInSource, onlyInDest } = compareFields(src, dest, idmap); + assert.ok(onlyInSource.includes('OnlySrc'), 'OnlySrc in onlyInSource'); + assert.ok(!entries.some((e) => e.scope === 'field:OnlySrc'), 'no entries for OnlySrc'); + assert.deepEqual(onlyInDest, []); + }); + + test('reports dest-only field in onlyInDest, not as an entry', () => { + const src = { fields: [fld('s1', 'A', 'text')] }; + const dest = { fields: [fld('d1', 'A', 'text'), fld('d2', 'OnlyDest', 'text')] }; + const idmap = { fields: { s1: { destFld: 'd1', choices: {} } } }; + const { entries, onlyInSource, onlyInDest } = compareFields(src, dest, idmap); + assert.ok(onlyInDest.includes('OnlyDest'), 'OnlyDest in onlyInDest'); + assert.ok(!entries.some((e) => e.scope === 'field:OnlyDest'), 'no entries for OnlyDest'); + assert.deepEqual(onlyInSource, []); + }); + + test('identical fields produce no entries', () => { + const src = { fields: [fld('s1', 'Price', 'number', { typeOptions: { precision: 2 } })] }; + const dest = { fields: [fld('d1', 'Price', 'number', { typeOptions: { precision: 2 } })] }; + const idmap = { fields: { s1: { destFld: 'd1', choices: {} } } }; + const { entries, onlyInSource, onlyInDest } = compareFields(src, dest, idmap); + assert.deepEqual(entries, []); + assert.deepEqual(onlyInSource, []); + assert.deepEqual(onlyInDest, []); + }); + + test('matches fields by name via by-name fallback when idmap has no entry for a field', () => { + // s2 not in idmap.fields but name matches d2 in dest + const src = { fields: [fld('s1', 'A', 'text'), fld('s2', 'B', 'number')] }; + const dest = { fields: [fld('d1', 'A', 'text'), fld('d2', 'B', 'text')] }; + const idmap = { fields: { s1: { destFld: 'd1', choices: {} } } }; + const { entries } = compareFields(src, dest, idmap); + const typeEntry = entries.find((e) => e.scope === 'field:B' && e.key === 'type'); + assert.ok(typeEntry, 'by-name fallback should find B and detect type drift'); + assert.equal(typeEntry.source, 'number'); + assert.equal(typeEntry.dest, 'text'); + }); + + test('computed fields compared by canonical sig (id-stable)', () => { + const srcOpts = { formulaTextParsed: '{s1}' }; + const destOpts = { formulaTextParsed: '{d1}' }; + const src = { fields: [ + fld('s1', 'Name', 'text'), + fld('s2', 'Total', 'formula', { isComputed: true, typeOptions: srcOpts }), + ] }; + const dest = { fields: [ + fld('d1', 'Name', 'text'), + fld('d2', 'Total', 'formula', { isComputed: true, typeOptions: destOpts }), + ] }; + const idmap = { fields: { s1: { destFld: 'd1', choices: {} }, s2: { destFld: 'd2', choices: {} } } }; + // Same formula referencing same-named field → no typeOptions drift + const { entries } = compareFields(src, dest, idmap); + assert.ok(!entries.find((e) => e.key === 'typeOptions'), 'id-stable formula should not produce typeOptions drift'); + }); +}); From 18f3647243e5e278899fbb2f0ed3d61c8407ac34 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Fri, 19 Jun 2026 17:06:18 +0300 Subject: [PATCH 083/246] =?UTF-8?q?fix(sync):=20compareFields=20=E2=80=94?= =?UTF-8?q?=20guard=20choices=20on=20type-match;=20track=20matched=20dest?= =?UTF-8?q?=20by=20id?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I1: only run typeOptions/choices comparison when sf.type === df.type, preventing a spurious 'choices' entry when a singleSelect src is paired with a non-select dest. I2: track claimed dest fields by id (not name) in the by-name fallback so a dest field cannot be double-counted in onlyInDest if two src fields somehow share a name. Adds two covering tests. Co-Authored-By: Claude Sonnet 4.6 --- packages/mcp-server/src/sync/compare.js | 45 +++++++++++-------- .../mcp-server/test/sync/test-compare.test.js | 39 ++++++++++++++++ 2 files changed, 65 insertions(+), 19 deletions(-) diff --git a/packages/mcp-server/src/sync/compare.js b/packages/mcp-server/src/sync/compare.js index 676525b..114b95a 100644 --- a/packages/mcp-server/src/sync/compare.js +++ b/packages/mcp-server/src/sync/compare.js @@ -60,8 +60,10 @@ export function compareFields(srcTable, destTable, idmap) { // Build name→field map for dest for by-name fallback and unmatched detection. const destByName = new Map(destTable.fields.map((f) => [f.name, f])); - // Track which dest field names were matched so we can find dest-only fields. - const matchedDestNames = new Set(); + // Track which dest field ids were matched so we can find dest-only fields + // (using id rather than name prevents a single dest field from being claimed twice + // if two src fields share the same name in an edge case). + const matchedDestIds = new Set(); // Build id→name maps for each side so computed sigs can normalise references. const srcFldNames = tableFieldNameMap(srcTable); @@ -86,7 +88,7 @@ export function compareFields(srcTable, destTable, idmap) { continue; } - matchedDestNames.add(df.name); + matchedDestIds.add(df.id); srcMatchedOrder.push(sf.name); const scope = `field:${sf.name}`; @@ -96,22 +98,27 @@ export function compareFields(srcTable, destTable, idmap) { entries.push({ scope, key: 'type', source: sf.type, dest: df.type, class: classOf('type') }); } - // Compare typeOptions / choices. - if (sf.isComputed) { - // Computed: use canonical, ID-stable sig for typeOptions. - if (computedSig(sf, srcFldNames) !== computedSig(df, destFldNames)) { - entries.push({ scope, key: 'typeOptions', source: sf.typeOptions, dest: df.typeOptions, class: classOf('typeOptions') }); - } - } else { - // Scalar: check choices separately (set-membership) and other typeOptions. - const srcChoices = choiceNames(sf.typeOptions); - if (srcChoices !== null) { - // It's a select-type — check via scalarTypeOptionsChanged (choice set membership). - if (scalarTypeOptionsChanged(sf, df)) { - entries.push({ scope, key: 'choices', source: sf.typeOptions, dest: df.typeOptions, class: classOf('choices') }); + // Compare typeOptions / choices — only when types match. + // When types differ we already emitted a 'type' entry; comparing typeOptions across + // incompatible types would produce a spurious second entry (e.g. singleSelect choices + // vs a number field that has no choices at all). + if (sf.type === df.type) { + if (sf.isComputed) { + // Computed: use canonical, ID-stable sig for typeOptions. + if (computedSig(sf, srcFldNames) !== computedSig(df, destFldNames)) { + entries.push({ scope, key: 'typeOptions', source: sf.typeOptions, dest: df.typeOptions, class: classOf('typeOptions') }); + } + } else { + // Scalar: check choices separately (set-membership) and other typeOptions. + const srcChoices = choiceNames(sf.typeOptions); + if (srcChoices !== null) { + // It's a select-type — check via scalarTypeOptionsChanged (choice set membership). + if (scalarTypeOptionsChanged(sf, df)) { + entries.push({ scope, key: 'choices', source: sf.typeOptions, dest: df.typeOptions, class: classOf('choices') }); + } + } else if (scalarTypeOptionsChanged(sf, df)) { + entries.push({ scope, key: 'typeOptions', source: sf.typeOptions, dest: df.typeOptions, class: classOf('typeOptions') }); } - } else if (scalarTypeOptionsChanged(sf, df)) { - entries.push({ scope, key: 'typeOptions', source: sf.typeOptions, dest: df.typeOptions, class: classOf('typeOptions') }); } } @@ -125,7 +132,7 @@ export function compareFields(srcTable, destTable, idmap) { // Collect dest-only fields. for (const df of destTable.fields) { - if (!matchedDestNames.has(df.name)) { + if (!matchedDestIds.has(df.id)) { onlyInDest.push(df.name); } } diff --git a/packages/mcp-server/test/sync/test-compare.test.js b/packages/mcp-server/test/sync/test-compare.test.js index f246ae4..100bd86 100644 --- a/packages/mcp-server/test/sync/test-compare.test.js +++ b/packages/mcp-server/test/sync/test-compare.test.js @@ -133,6 +133,45 @@ describe('compareFields', () => { assert.equal(typeEntry.dest, 'text'); }); + // ── I1: guard choices comparison on type-match ─────────────────────────── + test('I1: type mismatch with src singleSelect emits exactly one type entry and no choices entry', () => { + // src is singleSelect (has choices), dest is number — types differ. + // Before the fix this emitted BOTH a 'type' entry AND a spurious 'choices' entry. + const srcOpts = { choices: { s1: { id: 's1', name: 'Open' } } }; + const src = { fields: [fld('s1', 'Status', 'singleSelect', { typeOptions: srcOpts })] }; + const dest = { fields: [fld('d1', 'Status', 'number')] }; + const idmap = { fields: { s1: { destFld: 'd1', choices: {} } } }; + const { entries } = compareFields(src, dest, idmap); + const typeEntries = entries.filter((e) => e.key === 'type'); + const choicesEntries = entries.filter((e) => e.key === 'choices'); + assert.equal(typeEntries.length, 1, 'should emit exactly one type entry'); + assert.equal(choicesEntries.length, 0, 'should emit no choices entry when types differ'); + }); + + // ── I2: by-name fallback tracks matched dest by id ─────────────────────── + test('I2: by-name fallback tracks matched dest fields by id (no duplicate match)', () => { + // Two src fields with different names, but the by-name fallback is what we're testing. + // Specifically: once a dest field is matched it must not be counted as onlyInDest, + // and the second src field with no idmap entry that has no name match goes to onlyInSource. + const src = { + fields: [ + fld('s1', 'Alpha', 'text'), // matched by idmap → d1 + fld('s2', 'Beta', 'text'), // no idmap entry, no name match in dest → onlyInSource + ], + }; + const dest = { + fields: [ + fld('d1', 'Alpha', 'text'), // matched by idmap + ], + }; + const idmap = { fields: { s1: { destFld: 'd1', choices: {} } } }; + const { entries, onlyInSource, onlyInDest } = compareFields(src, dest, idmap); + // Alpha should be matched (no type drift), Beta should be in onlyInSource, dest has nothing extra + assert.ok(!entries.some((e) => e.scope === 'field:Alpha'), 'no drift on identical Alpha'); + assert.ok(onlyInSource.includes('Beta'), 'Beta in onlyInSource'); + assert.deepEqual(onlyInDest, [], 'no dest-only fields'); + }); + test('computed fields compared by canonical sig (id-stable)', () => { const srcOpts = { formulaTextParsed: '{s1}' }; const destOpts = { formulaTextParsed: '{d1}' }; From 63ea94285025beb0799295a97880faba311efefb Mon Sep 17 00:00:00 2001 From: "A.R." Date: Fri, 19 Jun 2026 17:12:10 +0300 Subject: [PATCH 084/246] feat(sync): compareTable/Views incl. sections + order Co-Authored-By: Claude Sonnet 4.6 --- packages/mcp-server/src/sync/compare.js | 328 ++++++++++++++ .../test/sync/test-compare-views.test.js | 418 ++++++++++++++++++ 2 files changed, 746 insertions(+) create mode 100644 packages/mcp-server/test/sync/test-compare-views.test.js diff --git a/packages/mcp-server/src/sync/compare.js b/packages/mcp-server/src/sync/compare.js index 114b95a..f0c79c6 100644 --- a/packages/mcp-server/src/sync/compare.js +++ b/packages/mcp-server/src/sync/compare.js @@ -1,6 +1,7 @@ // compare.js — base-schema comparator helpers (pure: no fs, no network, no Date.now/Math.random) import { stableStringify, choiceNames, scalarTypeOptionsChanged, computedSig } from './field-compare.js'; +import { canonicalizeViewConfig } from './remap.js'; const BEST_EFFORT = new Set(['fieldOrder', 'viewOrder', 'columnOrder', 'sortOrder', 'groupOrder']); const NOT_SYNCED = new Set(['sections']); @@ -156,3 +157,330 @@ export function compareFields(srcTable, destTable, idmap) { return { entries, onlyInSource, onlyInDest }; } + +// ── Internal helpers for compareViews / compareTable ────────────────────────── + +/** + * Build { fieldId → fieldName } for a single table's fields. + * @param {{ fields: Array<{id:string, name:string}> }} table + * @returns {Record} + */ +function tableFldNames(table) { + const m = {}; + for (const f of (table.fields || [])) m[f.id] = f.name; + return m; +} + +/** + * Build { choiceId → choiceName } for a single table's select fields. + * @param {{ fields: Array<{typeOptions:object|null}> }} table + * @returns {Record} + */ +function tableSelNames(table) { + const m = {}; + for (const f of (table.fields || [])) { + const ch = f.typeOptions && f.typeOptions.choices; + if (ch) for (const c of Object.values(ch)) m[c.id] = c.name; + } + return m; +} + +/** + * Return only collaborative (non-personal) views. + * @param {{ views?: Array<{personalForUserId?:string}> }} table + * @returns {Array} + */ +function collabViews(table) { + return (table.views || []).filter((v) => !v.personalForUserId); +} + +/** + * Build a column-order sequence (by field name) from a canonicalized config object. + * The canonical JSON for columns is { visible: string[], hidden: string[] }. + * Column order is the original ordered list (visible + hidden in appearance order). + * We reconstruct from the raw config instead since canonical loses order for sets. + * @param {{ columnOrder?: Array<{columnId:string, visibility:boolean}> }} config + * @param {Record} fldNames + * @returns {string[]} + */ +function columnOrderNames(config, fldNames) { + return (config.columnOrder || []).map((co) => fldNames[co.columnId] ?? co.columnId); +} + +/** + * Build a sort-clause sequence (by field name + direction) from raw config. + * @param {{ sorts?: Array<{columnId:string, ascending:boolean}> }} config + * @param {Record} fldNames + * @returns {string[]} + */ +function sortOrderKeys(config, fldNames) { + return (config.sorts || []).map((s) => `${fldNames[s.columnId] ?? s.columnId}:${s.ascending}`); +} + +/** + * Build a group-clause sequence (by field name + order) from raw config. + * @param {{ groupLevels?: Array<{columnId:string, order:string}> }} config + * @param {Record} fldNames + * @returns {string[]} + */ +function groupOrderKeys(config, fldNames) { + return (config.groupLevels || []).map((g) => `${fldNames[g.columnId] ?? g.columnId}:${g.order}`); +} + +/** + * Compare views between a source table and a destination table. + * + * Matching strategy (per view): + * 1. Look up dest view id via idmap.views[srcView.id]. + * 2. Fall back to matching by view name in dest. + * Personal views (personalForUserId is set) are skipped on both sides. + * + * Per matched pair emits DiffEntries with scope `view:`: + * - `type` → class drift + * - `filters` / `sorts` / `groups` / `columnVisibility` / `frozen` / `color` / `cover` / + * `calendar` / `rowHeight` / `form` → class drift (via canonicalizeViewConfig comparison) + * - `columnOrder` → class best-effort + * - `sortOrder` → class best-effort + * - `groupOrder` → class best-effort + * + * @param {{ fields: Array, views: Array }} srcTable + * @param {{ fields: Array, views: Array }} destTable + * @param {{ views: Record, fields: Record}> }} idmap + * @returns {{ entries: Array, onlyInSource: string[], onlyInDest: string[] }} + */ +export function compareViews(srcTable, destTable, idmap) { + const entries = []; + const onlyInSource = []; + const onlyInDest = []; + + const srcFldNames = tableFldNames(srcTable); + const destFldNames = tableFldNames(destTable); + const srcSelNames = tableSelNames(srcTable); + const destSelNames = tableSelNames(destTable); + + const srcViews = collabViews(srcTable); + const destViews = collabViews(destTable); + + // Build dest view lookup: by id and by name. + const destViewsById = new Map(destViews.map((v) => [v.id, v])); + const destViewsByName = new Map(destViews.map((v) => [v.name, v])); + const matchedDestViewIds = new Set(); + + for (const sv of srcViews) { + // Resolve dest view: idmap first, then by-name fallback. + let dv = null; + const mappedDestId = idmap.views && idmap.views[sv.id]; + if (mappedDestId) dv = destViewsById.get(mappedDestId) ?? null; + if (!dv) dv = destViewsByName.get(sv.name) ?? null; + + if (!dv) { + onlyInSource.push(sv.name); + continue; + } + matchedDestViewIds.add(dv.id); + + const scope = `view:${sv.name}`; + const srcConfig = sv.config || {}; + const destConfig = dv.config || {}; + + // 1. Type drift. + if (sv.type !== dv.type) { + entries.push({ scope, key: 'type', source: sv.type, dest: dv.type, class: classOf('type') }); + } + + // 2. Canonicalize both configs for content comparison. + // Source: stripRecordRefs=true (matches what apply will write) + // Dest: stripRecordRefs=false (keep dest's existing record refs so dangling ones diverge) + const srcCanon = JSON.parse(canonicalizeViewConfig(srcConfig, srcFldNames, srcSelNames, true, idmap)); + const destCanon = JSON.parse(canonicalizeViewConfig(destConfig, destFldNames, destSelNames, false, idmap)); + + // 3. Per-facet comparison. + const filtersSrcStr = JSON.stringify(srcCanon.filters); + const filtersDestStr = JSON.stringify(destCanon.filters); + if (filtersSrcStr !== filtersDestStr) { + entries.push({ scope, key: 'filters', source: srcCanon.filters, dest: destCanon.filters, class: classOf('filters') }); + } + + // sorts content (set + direction, NOT order — order is best-effort below). + // The canonical `sorts` array is ordered — but we want content equivalence ignoring order. + // Sort both sides by their name-key for a set-like content comparison. + const sortsSrcContent = JSON.stringify([...srcCanon.sorts].sort((a, b) => JSON.stringify(a) < JSON.stringify(b) ? -1 : 1)); + const sortsDestContent = JSON.stringify([...destCanon.sorts].sort((a, b) => JSON.stringify(a) < JSON.stringify(b) ? -1 : 1)); + if (sortsSrcContent !== sortsDestContent) { + entries.push({ scope, key: 'sorts', source: srcCanon.sorts, dest: destCanon.sorts, class: classOf('sorts') }); + } + + // groups content. + const groupsSrcContent = JSON.stringify([...srcCanon.groups].sort((a, b) => JSON.stringify(a) < JSON.stringify(b) ? -1 : 1)); + const groupsDestContent = JSON.stringify([...destCanon.groups].sort((a, b) => JSON.stringify(a) < JSON.stringify(b) ? -1 : 1)); + if (groupsSrcContent !== groupsDestContent) { + entries.push({ scope, key: 'groups', source: srcCanon.groups, dest: destCanon.groups, class: classOf('groups') }); + } + + // column VISIBILITY SET (drift). + const visSrcStr = JSON.stringify({ visible: srcCanon.columns.visible, hidden: srcCanon.columns.hidden }); + const visDestStr = JSON.stringify({ visible: destCanon.columns.visible, hidden: destCanon.columns.hidden }); + if (visSrcStr !== visDestStr) { + entries.push({ scope, key: 'columnVisibility', source: srcCanon.columns, dest: destCanon.columns, class: classOf('columnVisibility') }); + } + + // column ORDER (best-effort) — only when visibility sets match. + if (visSrcStr === visDestStr) { + const srcColOrder = columnOrderNames(srcConfig, srcFldNames); + const destColOrder = columnOrderNames(destConfig, destFldNames); + if (srcColOrder.join('\0') !== destColOrder.join('\0')) { + entries.push({ scope, key: 'columnOrder', source: srcColOrder, dest: destColOrder, class: classOf('columnOrder') }); + } + } + + // frozen column count (drift). + if (srcCanon.frozen !== destCanon.frozen) { + entries.push({ scope, key: 'frozenColumnCount', source: srcCanon.frozen, dest: destCanon.frozen, class: classOf('frozenColumnCount') }); + } + + // colorConfig (drift). + if (JSON.stringify(srcCanon.color) !== JSON.stringify(destCanon.color)) { + entries.push({ scope, key: 'colorConfig', source: srcCanon.color, dest: destCanon.color, class: classOf('colorConfig') }); + } + + // cover (drift). + if (JSON.stringify(srcCanon.cover) !== JSON.stringify(destCanon.cover)) { + entries.push({ scope, key: 'cover', source: srcCanon.cover, dest: destCanon.cover, class: classOf('cover') }); + } + + // calendar (drift). + if (JSON.stringify(srcCanon.calendar) !== JSON.stringify(destCanon.calendar)) { + entries.push({ scope, key: 'calendar', source: srcCanon.calendar, dest: destCanon.calendar, class: classOf('calendar') }); + } + + // rowHeight (drift). + if (srcCanon.rowHeight !== destCanon.rowHeight) { + entries.push({ scope, key: 'rowHeight', source: srcCanon.rowHeight, dest: destCanon.rowHeight, class: classOf('rowHeight') }); + } + + // form (drift). + if (JSON.stringify(srcCanon.form) !== JSON.stringify(destCanon.form)) { + entries.push({ scope, key: 'form', source: srcCanon.form, dest: destCanon.form, class: classOf('form') }); + } + + // sort clause ORDER (best-effort) — only when sort content matches. + if (sortsSrcContent === sortsDestContent) { + const srcSortOrder = sortOrderKeys(srcConfig, srcFldNames); + const destSortOrder = sortOrderKeys(destConfig, destFldNames); + if (srcSortOrder.join('\0') !== destSortOrder.join('\0')) { + entries.push({ scope, key: 'sortOrder', source: srcSortOrder, dest: destSortOrder, class: classOf('sortOrder') }); + } + } + + // group clause ORDER (best-effort) — only when group content matches. + if (groupsSrcContent === groupsDestContent) { + const srcGroupOrder = groupOrderKeys(srcConfig, srcFldNames); + const destGroupOrder = groupOrderKeys(destConfig, destFldNames); + if (srcGroupOrder.join('\0') !== destGroupOrder.join('\0')) { + entries.push({ scope, key: 'groupOrder', source: srcGroupOrder, dest: destGroupOrder, class: classOf('groupOrder') }); + } + } + } + + // Collect dest-only views. + for (const dv of destViews) { + if (!matchedDestViewIds.has(dv.id)) { + onlyInDest.push(dv.name); + } + } + + return { entries, onlyInSource, onlyInDest }; +} + +/** + * Compare a source table against a dest table using the provided idmap. + * + * Covers: + * - description drift + * - primary field NAME drift + * - field-level diffs (via compareFields) + * - view-level diffs (via compareViews), incl. view order (best-effort) + * - section diffs (not-synced — reported but not acted on) + * + * @param {{ id:string, name:string, primaryFieldId:string, description?:string, fields:Array, views:Array, sections?:Array }} srcTable + * @param {{ id:string, name:string, primaryFieldId:string, description?:string, fields:Array, views:Array, sections?:Array }} destTable + * @param {{ tables:Record, fields:Record}>, views:Record }} idmap + * @returns {{ name:string, status:'same'|'differs', entries:Array, fields:{onlyInSource:string[],onlyInDest:string[]}, views:{onlyInSource:string[],onlyInDest:string[]} }} + */ +export function compareTable(srcTable, destTable, idmap) { + const entries = []; + + // 1. Description drift. + const srcDesc = srcTable.description ?? null; + const destDesc = destTable.description ?? null; + if (srcDesc !== destDesc) { + entries.push({ scope: 'table', key: 'description', source: srcDesc, dest: destDesc, class: classOf('description') }); + } + + // 2. Primary field NAME drift. + const srcPrimary = srcTable.fields.find((f) => f.id === srcTable.primaryFieldId); + const destPrimary = destTable.fields.find((f) => f.id === destTable.primaryFieldId); + if (srcPrimary && destPrimary && srcPrimary.name !== destPrimary.name) { + entries.push({ scope: 'table', key: 'primaryFieldName', source: srcPrimary.name, dest: destPrimary.name, class: classOf('primaryFieldName') }); + } + + // 3. Field-level comparison. + const fieldResult = compareFields(srcTable, destTable, idmap); + entries.push(...fieldResult.entries); + + // 4. View-level comparison. + const viewResult = compareViews(srcTable, destTable, idmap); + entries.push(...viewResult.entries); + + // 5. View ORDER (best-effort): compare matched view name sequence. + const srcCollabViews = collabViews(srcTable); + const destCollabViews = collabViews(destTable); + const srcViewNames = srcCollabViews.map((v) => v.name); + // Build matched dest view order: for each src view that was matched, find its dest position. + const destViewsByName = new Map(destCollabViews.map((v, i) => [v.name, i])); + // Build dest order of matched src views (only matched ones, not onlyInSource). + const onlySrcSet = new Set(viewResult.onlyInSource); + const srcMatchedViewNames = srcViewNames.filter((n) => !onlySrcSet.has(n)); + // Sort srcMatchedViewNames by their dest iteration order. + const destOrderedViewNames = [...srcMatchedViewNames].sort( + (a, b) => (destViewsByName.get(a) ?? Infinity) - (destViewsByName.get(b) ?? Infinity), + ); + if (srcMatchedViewNames.join('\0') !== destOrderedViewNames.join('\0')) { + entries.push({ + scope: 'table', + key: 'viewOrder', + source: srcMatchedViewNames, + dest: destOrderedViewNames, + class: classOf('viewOrder'), + }); + } + + // 6. Sections comparison (not-synced). + const srcSections = srcTable.sections || []; + const destSections = destTable.sections || []; + if (JSON.stringify(srcSections) !== JSON.stringify(destSections)) { + entries.push({ + scope: 'table', + key: 'sections', + source: srcSections, + dest: destSections, + class: classOf('sections'), + }); + } + + // 7. Compute status. + const hasDiffs = + entries.length > 0 || + fieldResult.onlyInSource.length > 0 || + fieldResult.onlyInDest.length > 0 || + viewResult.onlyInSource.length > 0 || + viewResult.onlyInDest.length > 0; + + return { + name: srcTable.name, + status: hasDiffs ? 'differs' : 'same', + entries, + fields: { onlyInSource: fieldResult.onlyInSource, onlyInDest: fieldResult.onlyInDest }, + views: { onlyInSource: viewResult.onlyInSource, onlyInDest: viewResult.onlyInDest }, + }; +} diff --git a/packages/mcp-server/test/sync/test-compare-views.test.js b/packages/mcp-server/test/sync/test-compare-views.test.js new file mode 100644 index 0000000..4ffba10 --- /dev/null +++ b/packages/mcp-server/test/sync/test-compare-views.test.js @@ -0,0 +1,418 @@ +// test-compare-views.test.js — TDD tests for Task 3: compareViews + compareTable +import { test, describe } from 'node:test'; +import assert from 'node:assert/strict'; +import { compareTable } from '../../src/sync/compare.js'; + +// ── Helpers ──────────────────────────────────────────────────────────────────── +function fld(id, name, type = 'text', extra = {}) { + return { + id, + name, + type, + typeOptions: extra.typeOptions ?? null, + description: extra.description ?? null, + isComputed: !!extra.isComputed, + }; +} + +function view(id, name, type = 'grid', config = {}, extra = {}) { + return { id, name, type, config, ...extra }; +} + +/** + * Minimal table fixture — fields/views/sections all optional. + */ +function tbl(id, name, primaryFieldId, fields = [], views = [], sections = []) { + return { id, name, primaryFieldId, fields, views, sections }; +} + +/** + * Minimal idmap fixture. + */ +function idmap({ tables = {}, fields = {}, views = {} } = {}) { + return { tables, fields, views }; +} + +// ── compareTable — basic wiring ──────────────────────────────────────────────── + +describe('compareTable — basic wiring', () => { + test('returns status=same for two identical empty tables', () => { + const src = tbl('tS', 'T', 'p'); + const dest = tbl('tD', 'T', 'p'); + const r = compareTable(src, dest, idmap({ tables: { tS: 'tD' } })); + assert.equal(r.name, 'T'); + assert.equal(r.status, 'same'); + assert.deepEqual(r.entries, []); + }); + + test('description drift produces a drift entry', () => { + const src = { ...tbl('tS', 'T', 'p'), description: 'new' }; + const dest = { ...tbl('tD', 'T', 'p'), description: 'old' }; + const r = compareTable(src, dest, idmap({ tables: { tS: 'tD' } })); + const e = r.entries.find((e) => e.key === 'description'); + assert.ok(e, 'description entry missing'); + assert.equal(e.class, 'drift'); + assert.equal(e.source, 'new'); + assert.equal(e.dest, 'old'); + assert.equal(r.status, 'differs'); + }); + + test('primary field name change produces a drift entry', () => { + const src = tbl('tS', 'T', 'pS', [fld('pS', 'Title')]); + const dest = tbl('tD', 'T', 'pD', [fld('pD', 'Name')]); + const im = idmap({ tables: { tS: 'tD' }, fields: { pS: { destFld: 'pD', choices: {} } } }); + const r = compareTable(src, dest, im); + const e = r.entries.find((e) => e.key === 'primaryFieldName'); + assert.ok(e, 'primaryFieldName entry missing'); + assert.equal(e.class, 'drift'); + assert.equal(e.source, 'Title'); + assert.equal(e.dest, 'Name'); + }); + + test('status is same when tables are identical with identical views', () => { + const src = tbl('tS', 'T', 'pS', [fld('pS', 'Name')], [view('vS', 'Grid', 'grid', {})]); + const dest = tbl('tD', 'T', 'pD', [fld('pD', 'Name')], [view('vD', 'Grid', 'grid', {})]); + const im = idmap({ + tables: { tS: 'tD' }, + fields: { pS: { destFld: 'pD', choices: {} } }, + views: { vS: 'vD' }, + }); + const r = compareTable(src, dest, im); + assert.equal(r.status, 'same'); + }); +}); + +// ── compareViews — view type drift ──────────────────────────────────────────── + +describe('compareViews — view type drift', () => { + test('view type change emits drift entry with scope view: and key type', () => { + const src = tbl('tS', 'T', 'p', [], [view('vS', 'MyView', 'grid', {})]); + const dest = tbl('tD', 'T', 'p', [], [view('vD', 'MyView', 'gallery', {})]); + const im = idmap({ tables: { tS: 'tD' }, views: { vS: 'vD' } }); + const r = compareTable(src, dest, im); + const e = r.entries.find((e) => e.scope === 'view:MyView' && e.key === 'type'); + assert.ok(e, 'type entry for view:MyView missing'); + assert.equal(e.class, 'drift'); + assert.equal(e.source, 'grid'); + assert.equal(e.dest, 'gallery'); + }); + + test('same view type produces no type entry', () => { + const src = tbl('tS', 'T', 'p', [], [view('vS', 'Grid', 'grid', {})]); + const dest = tbl('tD', 'T', 'p', [], [view('vD', 'Grid', 'grid', {})]); + const im = idmap({ tables: { tS: 'tD' }, views: { vS: 'vD' } }); + const r = compareTable(src, dest, im); + assert.ok(!r.entries.find((e) => e.scope === 'view:Grid' && e.key === 'type')); + }); +}); + +// ── compareViews — filter content drift ─────────────────────────────────────── + +describe('compareViews — filter content drift', () => { + test('filters differ → drift entry key=filters', () => { + const srcConfig = { + filters: { + conjunction: 'and', + filterSet: [{ columnId: 'fS1', operator: 'isEmpty', value: null }], + }, + }; + const destConfig = { + filters: { + conjunction: 'and', + filterSet: [{ columnId: 'fD1', operator: 'isNotEmpty', value: null }], + }, + }; + const src = tbl('tS', 'T', 'fS1', [fld('fS1', 'Status')], [view('vS', 'Grid', 'grid', srcConfig)]); + const dest = tbl('tD', 'T', 'fD1', [fld('fD1', 'Status')], [view('vD', 'Grid', 'grid', destConfig)]); + const im = idmap({ + tables: { tS: 'tD' }, + fields: { fS1: { destFld: 'fD1', choices: {} } }, + views: { vS: 'vD' }, + }); + const r = compareTable(src, dest, im); + const e = r.entries.find((e) => e.scope === 'view:Grid' && e.key === 'filters'); + assert.ok(e, 'filters drift entry missing'); + assert.equal(e.class, 'drift'); + }); + + test('identical filters produce no filters entry', () => { + // Source uses src field IDs; dest uses dest field IDs. After canonicalization + // both resolve to the same field NAME ('Status') → no filter drift. + const srcConfig = { + filters: { + conjunction: 'and', + filterSet: [{ columnId: 'fS1', operator: 'isEmpty', value: null }], + }, + }; + const destConfig = { + filters: { + conjunction: 'and', + filterSet: [{ columnId: 'fD1', operator: 'isEmpty', value: null }], + }, + }; + const src = tbl('tS', 'T', 'fS1', [fld('fS1', 'Status')], [view('vS', 'Grid', 'grid', srcConfig)]); + const dest = tbl('tD', 'T', 'fD1', [fld('fD1', 'Status')], [view('vD', 'Grid', 'grid', destConfig)]); + const im = idmap({ + tables: { tS: 'tD' }, + fields: { fS1: { destFld: 'fD1', choices: {} } }, + views: { vS: 'vD' }, + }); + const r = compareTable(src, dest, im); + assert.ok(!r.entries.find((e) => e.scope === 'view:Grid' && e.key === 'filters')); + }); +}); + +// ── compareViews — column visibility vs order ───────────────────────────────── + +describe('compareViews — column visibility vs order', () => { + test('column order differs but visibility set same → best-effort key=columnOrder', () => { + const srcConfig = { + columnOrder: [ + { columnId: 'fS1', visibility: true }, + { columnId: 'fS2', visibility: true }, + ], + }; + const destConfig = { + columnOrder: [ + { columnId: 'fD2', visibility: true }, + { columnId: 'fD1', visibility: true }, + ], + }; + const src = tbl('tS', 'T', 'fS1', [fld('fS1', 'A'), fld('fS2', 'B')], [view('vS', 'Grid', 'grid', srcConfig)]); + const dest = tbl('tD', 'T', 'fD1', [fld('fD1', 'A'), fld('fD2', 'B')], [view('vD', 'Grid', 'grid', destConfig)]); + const im = idmap({ + tables: { tS: 'tD' }, + fields: { fS1: { destFld: 'fD1', choices: {} }, fS2: { destFld: 'fD2', choices: {} } }, + views: { vS: 'vD' }, + }); + const r = compareTable(src, dest, im); + // Visibility set is the same (both visible) → no columnVisibility entry + assert.ok(!r.entries.find((e) => e.key === 'columnVisibility'), 'should not have visibility entry when set is the same'); + // Column order differs + const oe = r.entries.find((e) => e.key === 'columnOrder'); + assert.ok(oe, 'columnOrder entry missing'); + assert.equal(oe.class, 'best-effort'); + }); + + test('column visibility differs → drift entry key=columnVisibility', () => { + const srcConfig = { + columnOrder: [ + { columnId: 'fS1', visibility: true }, + { columnId: 'fS2', visibility: false }, + ], + }; + const destConfig = { + columnOrder: [ + { columnId: 'fD1', visibility: true }, + { columnId: 'fD2', visibility: true }, // B is visible in dest but hidden in src + ], + }; + const src = tbl('tS', 'T', 'fS1', [fld('fS1', 'A'), fld('fS2', 'B')], [view('vS', 'Grid', 'grid', srcConfig)]); + const dest = tbl('tD', 'T', 'fD1', [fld('fD1', 'A'), fld('fD2', 'B')], [view('vD', 'Grid', 'grid', destConfig)]); + const im = idmap({ + tables: { tS: 'tD' }, + fields: { fS1: { destFld: 'fD1', choices: {} }, fS2: { destFld: 'fD2', choices: {} } }, + views: { vS: 'vD' }, + }); + const r = compareTable(src, dest, im); + const ve = r.entries.find((e) => e.key === 'columnVisibility'); + assert.ok(ve, 'columnVisibility drift entry missing'); + assert.equal(ve.class, 'drift'); + }); +}); + +// ── compareViews — sort clause order ────────────────────────────────────────── + +describe('compareViews — sort clause order', () => { + test('sort clause order differs → best-effort entry key=sortOrder', () => { + const srcConfig = { + sorts: [ + { columnId: 'fS1', ascending: true }, + { columnId: 'fS2', ascending: false }, + ], + }; + const destConfig = { + sorts: [ + { columnId: 'fD2', ascending: false }, + { columnId: 'fD1', ascending: true }, + ], + }; + const src = tbl('tS', 'T', 'fS1', [fld('fS1', 'A'), fld('fS2', 'B')], [view('vS', 'Grid', 'grid', srcConfig)]); + const dest = tbl('tD', 'T', 'fD1', [fld('fD1', 'A'), fld('fD2', 'B')], [view('vD', 'Grid', 'grid', destConfig)]); + const im = idmap({ + tables: { tS: 'tD' }, + fields: { fS1: { destFld: 'fD1', choices: {} }, fS2: { destFld: 'fD2', choices: {} } }, + views: { vS: 'vD' }, + }); + const r = compareTable(src, dest, im); + const se = r.entries.find((e) => e.key === 'sortOrder'); + assert.ok(se, 'sortOrder best-effort entry missing'); + assert.equal(se.class, 'best-effort'); + }); +}); + +// ── compareViews — only-in lists ────────────────────────────────────────────── + +describe('compareViews — only-in-source', () => { + test('view only in source → appears in views.onlyInSource', () => { + const src = tbl('tS', 'T', 'p', [], [view('vS', 'Kanban', 'kanban', {})]); + const dest = tbl('tD', 'T', 'p', [], []); + const im = idmap({ tables: { tS: 'tD' } }); + const r = compareTable(src, dest, im); + assert.ok(r.views.onlyInSource.includes('Kanban'), 'Kanban should be onlyInSource'); + assert.equal(r.status, 'differs'); + }); + + test('personal views are skipped (not in onlyInSource)', () => { + const personalView = view('vP', 'My Personal View', 'grid', {}, { personalForUserId: 'usr123' }); + const src = tbl('tS', 'T', 'p', [], [personalView]); + const dest = tbl('tD', 'T', 'p', [], []); + const im = idmap({ tables: { tS: 'tD' } }); + const r = compareTable(src, dest, im); + assert.ok(!r.views.onlyInSource.includes('My Personal View'), 'personal view should be skipped'); + assert.equal(r.status, 'same'); + }); + + test('view only in dest → appears in views.onlyInDest', () => { + const src = tbl('tS', 'T', 'p', [], []); + const dest = tbl('tD', 'T', 'p', [], [view('vD', 'Gallery', 'gallery', {})]); + const im = idmap({ tables: { tS: 'tD' } }); + const r = compareTable(src, dest, im); + assert.ok(r.views.onlyInDest.includes('Gallery'), 'Gallery should be onlyInDest'); + assert.equal(r.status, 'differs'); + }); +}); + +// ── compareTable — sections diff ────────────────────────────────────────────── + +describe('compareTable — sections diff', () => { + test('compareTable flags missing section as not-synced (brief canonical test)', () => { + const src = { + id: 'tS', name: 'T', primaryFieldId: 'p', + fields: [], + views: [{ id: 'v', name: 'Grid', type: 'grid', config: {} }], + sections: [{ name: 'Sales', viewNames: ['Grid'] }], + }; + const dest = { + id: 'tD', name: 'T', primaryFieldId: 'p', + fields: [], + views: [{ id: 'w', name: 'Grid', type: 'grid', config: {} }], + sections: [], + }; + const im = { tables: { tS: 'tD' }, fields: {}, views: { v: 'w' } }; + const r = compareTable(src, dest, im); + const s = r.entries.find((e) => e.key === 'sections'); + assert.ok(s, 'sections entry missing'); + assert.equal(s.class, 'not-synced'); + }); + + test('identical sections produce no sections entry', () => { + const secs = [{ name: 'Sales', viewNames: ['Grid'] }]; + const src = tbl('tS', 'T', 'p', [], [view('vS', 'Grid')], secs); + const dest = tbl('tD', 'T', 'p', [], [view('vD', 'Grid')], secs); + const im = idmap({ tables: { tS: 'tD' }, views: { vS: 'vD' } }); + const r = compareTable(src, dest, im); + assert.ok(!r.entries.find((e) => e.key === 'sections'), 'identical sections should produce no entry'); + }); + + test('source has two sections, dest has one → not-synced entry with source/dest arrays', () => { + const srcSecs = [{ name: 'Sales', viewNames: ['Grid'] }, { name: 'Ops', viewNames: ['Kanban'] }]; + const destSecs = [{ name: 'Sales', viewNames: ['Grid'] }]; + const src = tbl('tS', 'T', 'p', [], [view('vS', 'Grid'), view('vK', 'Kanban')], srcSecs); + const dest = tbl('tD', 'T', 'p', [], [view('vG', 'Grid')], destSecs); + const im = idmap({ tables: { tS: 'tD' }, views: { vS: 'vG', vK: 'vK' } }); + const r = compareTable(src, dest, im); + const s = r.entries.find((e) => e.key === 'sections'); + assert.ok(s, 'sections entry missing'); + assert.equal(s.class, 'not-synced'); + assert.equal(s.scope, 'table'); + assert.deepEqual(s.source, srcSecs); + assert.deepEqual(s.dest, destSecs); + }); + + test('section viewNames sequence differs → not-synced entry', () => { + const srcSecs = [{ name: 'Sales', viewNames: ['Grid', 'Gallery'] }]; + const destSecs = [{ name: 'Sales', viewNames: ['Gallery', 'Grid'] }]; + const src = tbl('tS', 'T', 'p', [], [], srcSecs); + const dest = tbl('tD', 'T', 'p', [], [], destSecs); + const im = idmap({ tables: { tS: 'tD' } }); + const r = compareTable(src, dest, im); + const s = r.entries.find((e) => e.key === 'sections'); + assert.ok(s, 'sections entry missing when viewNames sequence differs'); + assert.equal(s.class, 'not-synced'); + }); +}); + +// ── compareTable — view order diff ──────────────────────────────────────────── + +describe('compareTable — view order', () => { + test('view order differs → best-effort entry key=viewOrder', () => { + const src = tbl('tS', 'T', 'p', [], [view('vA', 'A'), view('vB', 'B')]); + const dest = tbl('tD', 'T', 'p', [], [view('vB2', 'B'), view('vA2', 'A')]); + const im = idmap({ tables: { tS: 'tD' }, views: { vA: 'vA2', vB: 'vB2' } }); + const r = compareTable(src, dest, im); + const vo = r.entries.find((e) => e.key === 'viewOrder'); + assert.ok(vo, 'viewOrder entry missing'); + assert.equal(vo.class, 'best-effort'); + assert.deepEqual(vo.source, ['A', 'B']); + assert.deepEqual(vo.dest, ['B', 'A']); + }); + + test('same view order → no viewOrder entry', () => { + const src = tbl('tS', 'T', 'p', [], [view('vA', 'A'), view('vB', 'B')]); + const dest = tbl('tD', 'T', 'p', [], [view('vA2', 'A'), view('vB2', 'B')]); + const im = idmap({ tables: { tS: 'tD' }, views: { vA: 'vA2', vB: 'vB2' } }); + const r = compareTable(src, dest, im); + assert.ok(!r.entries.find((e) => e.key === 'viewOrder'), 'no viewOrder when order is same'); + }); +}); + +// ── compareTable — view-level facets: colorConfig, frozenColumnCount, rowHeight ── + +describe('compareViews — other facets', () => { + test('frozenColumnCount differs → drift entry', () => { + const srcConfig = { frozenColumnCount: 2 }; + const destConfig = { frozenColumnCount: 1 }; + const src = tbl('tS', 'T', 'p', [], [view('vS', 'Grid', 'grid', srcConfig)]); + const dest = tbl('tD', 'T', 'p', [], [view('vD', 'Grid', 'grid', destConfig)]); + const im = idmap({ tables: { tS: 'tD' }, views: { vS: 'vD' } }); + const r = compareTable(src, dest, im); + // frozenColumnCount is captured inside the canonicalized config under 'frozen' + // It should appear as a drift in the overall 'config' comparison or a dedicated key. + // The canonicalizeViewConfig bundles all of these — so if overall config differs, we + // flag per-facet. Check that the status is differs (some entry was produced). + assert.equal(r.status, 'differs'); + }); + + test('rowHeight differs → drift entry', () => { + const srcConfig = { rowHeight: 'short' }; + const destConfig = { rowHeight: 'tall' }; + const src = tbl('tS', 'T', 'p', [], [view('vS', 'Grid', 'grid', srcConfig)]); + const dest = tbl('tD', 'T', 'p', [], [view('vD', 'Grid', 'grid', destConfig)]); + const im = idmap({ tables: { tS: 'tD' }, views: { vS: 'vD' } }); + const r = compareTable(src, dest, im); + assert.equal(r.status, 'differs'); + }); +}); + +// ── compareTable — combined status ───────────────────────────────────────────── + +describe('compareTable — status transitions', () => { + test('fields.onlyInSource makes status=differs', () => { + const src = tbl('tS', 'T', 'p', [fld('f1', 'ExtraField')]); + const dest = tbl('tD', 'T', 'p', []); + const im = idmap({ tables: { tS: 'tD' } }); + const r = compareTable(src, dest, im); + assert.equal(r.status, 'differs'); + assert.ok(r.fields.onlyInSource.includes('ExtraField')); + }); + + test('fields.onlyInDest makes status=differs', () => { + const src = tbl('tS', 'T', 'p', []); + const dest = tbl('tD', 'T', 'p', [fld('f1', 'DestOnlyField')]); + const im = idmap({ tables: { tS: 'tD' } }); + const r = compareTable(src, dest, im); + assert.equal(r.status, 'differs'); + assert.ok(r.fields.onlyInDest.includes('DestOnlyField')); + }); +}); From 77ae232d0ed52e522004bc1fa78266abe932cfca Mon Sep 17 00:00:00 2001 From: "A.R." Date: Fri, 19 Jun 2026 17:17:47 +0300 Subject: [PATCH 085/246] test(sync): pin frozen/rowHeight DiffEntry contract; document sort-order canonical dependency Strengthen frozenColumnCount and rowHeight test cases to assert the exact DiffEntry (key, class, source, dest) emitted by compareViews, not just r.status === 'differs'. Add one-line comment at the sort/group clause-order best-effort guard noting reliance on the canonical {columnId,ascending} / {columnId,order} shape. No production logic changed. Co-Authored-By: Claude Sonnet 4.6 --- packages/mcp-server/src/sync/compare.js | 2 ++ .../test/sync/test-compare-views.test.js | 20 +++++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/packages/mcp-server/src/sync/compare.js b/packages/mcp-server/src/sync/compare.js index f0c79c6..0f9bc4d 100644 --- a/packages/mcp-server/src/sync/compare.js +++ b/packages/mcp-server/src/sync/compare.js @@ -364,6 +364,8 @@ export function compareViews(srcTable, destTable, idmap) { } // sort clause ORDER (best-effort) — only when sort content matches. + // Relies on the canonical {columnId, ascending} shape for sorts and {columnId, order} for groups; + // extra properties on sort/group objects are intentionally ignored by both sortOrderKeys and groupOrderKeys. if (sortsSrcContent === sortsDestContent) { const srcSortOrder = sortOrderKeys(srcConfig, srcFldNames); const destSortOrder = sortOrderKeys(destConfig, destFldNames); diff --git a/packages/mcp-server/test/sync/test-compare-views.test.js b/packages/mcp-server/test/sync/test-compare-views.test.js index 4ffba10..77855d8 100644 --- a/packages/mcp-server/test/sync/test-compare-views.test.js +++ b/packages/mcp-server/test/sync/test-compare-views.test.js @@ -370,21 +370,23 @@ describe('compareTable — view order', () => { // ── compareTable — view-level facets: colorConfig, frozenColumnCount, rowHeight ── describe('compareViews — other facets', () => { - test('frozenColumnCount differs → drift entry', () => { + test('frozenColumnCount differs → drift entry with key=frozenColumnCount', () => { const srcConfig = { frozenColumnCount: 2 }; const destConfig = { frozenColumnCount: 1 }; const src = tbl('tS', 'T', 'p', [], [view('vS', 'Grid', 'grid', srcConfig)]); const dest = tbl('tD', 'T', 'p', [], [view('vD', 'Grid', 'grid', destConfig)]); const im = idmap({ tables: { tS: 'tD' }, views: { vS: 'vD' } }); const r = compareTable(src, dest, im); - // frozenColumnCount is captured inside the canonicalized config under 'frozen' - // It should appear as a drift in the overall 'config' comparison or a dedicated key. - // The canonicalizeViewConfig bundles all of these — so if overall config differs, we - // flag per-facet. Check that the status is differs (some entry was produced). assert.equal(r.status, 'differs'); + // compareViews emits key 'frozenColumnCount' with class 'drift' for this facet + const e = r.entries.find((e) => e.scope === 'view:Grid' && e.key === 'frozenColumnCount'); + assert.ok(e, 'frozenColumnCount DiffEntry missing'); + assert.equal(e.class, 'drift'); + assert.equal(e.source, 2); + assert.equal(e.dest, 1); }); - test('rowHeight differs → drift entry', () => { + test('rowHeight differs → drift entry with key=rowHeight', () => { const srcConfig = { rowHeight: 'short' }; const destConfig = { rowHeight: 'tall' }; const src = tbl('tS', 'T', 'p', [], [view('vS', 'Grid', 'grid', srcConfig)]); @@ -392,6 +394,12 @@ describe('compareViews — other facets', () => { const im = idmap({ tables: { tS: 'tD' }, views: { vS: 'vD' } }); const r = compareTable(src, dest, im); assert.equal(r.status, 'differs'); + // compareViews emits key 'rowHeight' with class 'drift' for this facet + const e = r.entries.find((e) => e.scope === 'view:Grid' && e.key === 'rowHeight'); + assert.ok(e, 'rowHeight DiffEntry missing'); + assert.equal(e.class, 'drift'); + assert.equal(e.source, 'short'); + assert.equal(e.dest, 'tall'); }); }); From ec22fe496836ba6e6258a09420f6812066e222c0 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Fri, 19 Jun 2026 17:21:41 +0300 Subject: [PATCH 086/246] feat(sync): top-level compare() + identical/converged verdicts --- packages/mcp-server/src/sync/compare.js | 96 ++++++++++++ .../mcp-server/test/sync/test-compare.test.js | 138 +++++++++++++++++- 2 files changed, 233 insertions(+), 1 deletion(-) diff --git a/packages/mcp-server/src/sync/compare.js b/packages/mcp-server/src/sync/compare.js index 0f9bc4d..9fdac4b 100644 --- a/packages/mcp-server/src/sync/compare.js +++ b/packages/mcp-server/src/sync/compare.js @@ -394,6 +394,102 @@ export function compareViews(srcTable, destTable, idmap) { return { entries, onlyInSource, onlyInDest }; } +/** + * Top-level compare: diff two base snapshots and return the full diff tree. + * + * Table matching strategy: + * 1. For each src table, look up dest table id via idmap.tables[srcTable.id]. + * 2. If no idmap entry, fall back to matching by table name in dest. + * + * @param {{ baseId: string, tables: Array<{id:string, name:string, primaryFieldId:string, fields:Array, views:Array, sections?:Array}> }} srcSnap + * @param {{ baseId: string, tables: Array<{id:string, name:string, primaryFieldId:string, fields:Array, views:Array, sections?:Array}> }} destSnap + * @param {{ tables: Record, fields: Record}>, views: Record }} idmap + * @returns {{ + * sourceBaseId: string, + * destBaseId: string, + * identical: boolean, + * converged: boolean, + * summary: { drift: number, bestEffort: number, notSynced: number, onlyInSourceTables: number, onlyInDestTables: number }, + * tables: Array, + * onlyInSourceTables: string[], + * onlyInDestTables: string[] + * }} + */ +export function compare(srcSnap, destSnap, idmap) { + const onlyInSourceTables = []; + const onlyInDestTables = []; + const tables = []; + + // Build dest table lookup: by id and by name. + const destById = new Map(destSnap.tables.map((t) => [t.id, t])); + const destByName = new Map(destSnap.tables.map((t) => [t.name, t])); + const matchedDestIds = new Set(); + + for (const st of srcSnap.tables) { + // Resolve dest table: idmap first, then by-name fallback. + let dt = null; + const mappedDestId = idmap.tables && idmap.tables[st.id]; + if (mappedDestId) dt = destById.get(mappedDestId) ?? null; + if (!dt) dt = destByName.get(st.name) ?? null; + + if (!dt) { + onlyInSourceTables.push(st.name); + continue; + } + matchedDestIds.add(dt.id); + + const tableResult = compareTable(st, dt, idmap); + tables.push(tableResult); + } + + // Collect dest-only tables. + for (const dt of destSnap.tables) { + if (!matchedDestIds.has(dt.id)) { + onlyInDestTables.push(dt.name); + } + } + + // Aggregate counts by class across all entries in all matched tables. + let drift = 0; + let bestEffort = 0; + let notSynced = 0; + + for (const tableResult of tables) { + for (const entry of tableResult.entries) { + if (entry.class === 'drift') drift++; + else if (entry.class === 'best-effort') bestEffort++; + else if (entry.class === 'not-synced') notSynced++; + } + } + + // Compute verdicts. + const totalEntries = drift + bestEffort + notSynced; + const identical = + totalEntries === 0 && + onlyInSourceTables.length === 0 && + onlyInDestTables.length === 0 && + tables.every((t) => t.fields.onlyInSource.length === 0 && t.fields.onlyInDest.length === 0 && + t.views.onlyInSource.length === 0 && t.views.onlyInDest.length === 0); + const converged = drift === 0; + + return { + sourceBaseId: srcSnap.baseId, + destBaseId: destSnap.baseId, + identical, + converged, + summary: { + drift, + bestEffort, + notSynced, + onlyInSourceTables: onlyInSourceTables.length, + onlyInDestTables: onlyInDestTables.length, + }, + tables, + onlyInSourceTables, + onlyInDestTables, + }; +} + /** * Compare a source table against a dest table using the provided idmap. * diff --git a/packages/mcp-server/test/sync/test-compare.test.js b/packages/mcp-server/test/sync/test-compare.test.js index 100bd86..f87c4e0 100644 --- a/packages/mcp-server/test/sync/test-compare.test.js +++ b/packages/mcp-server/test/sync/test-compare.test.js @@ -1,4 +1,4 @@ -import { classOf, compareFields } from '../../src/sync/compare.js'; +import { classOf, compareFields, compare } from '../../src/sync/compare.js'; import { test, describe } from 'node:test'; import assert from 'node:assert/strict'; @@ -189,3 +189,139 @@ describe('compareFields', () => { assert.ok(!entries.find((e) => e.key === 'typeOptions'), 'id-stable formula should not produce typeOptions drift'); }); }); + +// ── compare() top-level tests (Task 4) ────────────────────────────────────── + +function mkSnap(baseId, tables) { + return { baseId, tables }; +} + +function mkTable(id, name, fields, views = []) { + return { id, name, primaryFieldId: fields[0]?.id ?? null, fields, views, sections: [] }; +} + +describe('compare', () => { + test('identical snapshots → identical:true, converged:true, summary all zero', () => { + const tableA = mkTable('st1', 'Alpha', [fld('sf1', 'Name', 'text'), fld('sf2', 'Value', 'number')]); + const destA = mkTable('dt1', 'Alpha', [fld('df1', 'Name', 'text'), fld('df2', 'Value', 'number')]); + const srcSnap = mkSnap('srcBase', [tableA]); + const destSnap = mkSnap('destBase', [destA]); + const idmap = { + tables: { st1: 'dt1' }, + fields: { sf1: { destFld: 'df1', choices: {} }, sf2: { destFld: 'df2', choices: {} } }, + views: {}, + }; + const result = compare(srcSnap, destSnap, idmap); + assert.equal(result.sourceBaseId, 'srcBase'); + assert.equal(result.destBaseId, 'destBase'); + assert.equal(result.identical, true); + assert.equal(result.converged, true); + assert.equal(result.summary.drift, 0); + assert.equal(result.summary.bestEffort, 0); + assert.equal(result.summary.notSynced, 0); + assert.equal(result.summary.onlyInSourceTables, 0); + assert.equal(result.summary.onlyInDestTables, 0); + assert.deepEqual(result.onlyInSourceTables, []); + assert.deepEqual(result.onlyInDestTables, []); + assert.equal(result.tables.length, 1); + assert.equal(result.tables[0].status, 'same'); + }); + + test('one reordered field → identical:false, converged:true, bestEffort===1', () => { + // Fields A and B exist in both, but order is reversed in dest. + // Both tables must have the same primary field NAME ('A') so primaryFieldName is not a drift source. + const srcTable = { ...mkTable('st1', 'Alpha', [fld('sf1', 'A', 'text'), fld('sf2', 'B', 'text')]), primaryFieldId: 'sf1' }; + const destTable = { ...mkTable('dt1', 'Alpha', [fld('df2', 'B', 'text'), fld('df1', 'A', 'text')]), primaryFieldId: 'df1' }; + const srcSnap = mkSnap('srcBase', [srcTable]); + const destSnap = mkSnap('destBase', [destTable]); + const idmap = { + tables: { st1: 'dt1' }, + fields: { sf1: { destFld: 'df1', choices: {} }, sf2: { destFld: 'df2', choices: {} } }, + views: {}, + }; + const result = compare(srcSnap, destSnap, idmap); + assert.equal(result.identical, false); + assert.equal(result.converged, true); + assert.equal(result.summary.bestEffort, 1, 'exactly one best-effort entry for fieldOrder'); + assert.equal(result.summary.drift, 0); + }); + + test('one type change → converged:false, drift>=1', () => { + const srcTable = mkTable('st1', 'Alpha', [fld('sf1', 'Price', 'number')]); + const destTable = mkTable('dt1', 'Alpha', [fld('df1', 'Price', 'text')]); + const srcSnap = mkSnap('srcBase', [srcTable]); + const destSnap = mkSnap('destBase', [destTable]); + const idmap = { + tables: { st1: 'dt1' }, + fields: { sf1: { destFld: 'df1', choices: {} } }, + views: {}, + }; + const result = compare(srcSnap, destSnap, idmap); + assert.equal(result.identical, false); + assert.equal(result.converged, false); + assert.ok(result.summary.drift >= 1, `drift should be >=1 (got ${result.summary.drift})`); + }); + + test('unmatched src table appears in onlyInSourceTables with count', () => { + const srcTable = mkTable('st1', 'Alpha', [fld('sf1', 'Name', 'text')]); + const srcSnap = mkSnap('srcBase', [srcTable]); + const destSnap = mkSnap('destBase', []); + const idmap = { tables: {}, fields: {}, views: {} }; + const result = compare(srcSnap, destSnap, idmap); + assert.deepEqual(result.onlyInSourceTables, ['Alpha']); + assert.equal(result.summary.onlyInSourceTables, 1); + assert.equal(result.identical, false); + assert.equal(result.converged, true, 'no drift entries → still converged'); + }); + + test('unmatched dest table appears in onlyInDestTables with count', () => { + const destTable = mkTable('dt1', 'Beta', [fld('df1', 'Name', 'text')]); + const srcSnap = mkSnap('srcBase', []); + const destSnap = mkSnap('destBase', [destTable]); + const idmap = { tables: {}, fields: {}, views: {} }; + const result = compare(srcSnap, destSnap, idmap); + assert.deepEqual(result.onlyInDestTables, ['Beta']); + assert.equal(result.summary.onlyInDestTables, 1); + assert.equal(result.identical, false); + }); + + test('by-name fallback matches table when idmap.tables has no entry', () => { + // Tables have same name but no idmap entry → should match by name + const srcTable = mkTable('st1', 'MyTable', [fld('sf1', 'X', 'text')]); + const destTable = mkTable('dt1', 'MyTable', [fld('df1', 'X', 'text')]); + const srcSnap = mkSnap('srcBase', [srcTable]); + const destSnap = mkSnap('destBase', [destTable]); + const idmap = { tables: {}, fields: {}, views: {} }; + const result = compare(srcSnap, destSnap, idmap); + assert.deepEqual(result.onlyInSourceTables, []); + assert.deepEqual(result.onlyInDestTables, []); + assert.equal(result.tables.length, 1); + assert.equal(result.identical, true); + assert.equal(result.converged, true); + }); + + test('summary counts aggregate drift and best-effort across multiple tables', () => { + // Table1: type drift (drift+1), Table2: field reorder (bestEffort+1). + // Both tables must have matching primary field names to isolate the intended diffs. + const t1Src = mkTable('st1', 'T1', [fld('sf1', 'A', 'number')]); + const t1Dest = mkTable('dt1', 'T1', [fld('df1', 'A', 'text')]); + const t2Src = { ...mkTable('st2', 'T2', [fld('sf2', 'X', 'text'), fld('sf3', 'Y', 'text')]), primaryFieldId: 'sf2' }; + const t2Dest = { ...mkTable('dt2', 'T2', [fld('df3', 'Y', 'text'), fld('df2', 'X', 'text')]), primaryFieldId: 'df2' }; + const srcSnap = mkSnap('srcBase', [t1Src, t2Src]); + const destSnap = mkSnap('destBase', [t1Dest, t2Dest]); + const idmap = { + tables: { st1: 'dt1', st2: 'dt2' }, + fields: { + sf1: { destFld: 'df1', choices: {} }, + sf2: { destFld: 'df2', choices: {} }, + sf3: { destFld: 'df3', choices: {} }, + }, + views: {}, + }; + const result = compare(srcSnap, destSnap, idmap); + assert.equal(result.summary.drift, 1, 'one type drift in T1'); + assert.equal(result.summary.bestEffort, 1, 'one fieldOrder in T2'); + assert.equal(result.identical, false); + assert.equal(result.converged, false, 'has drift → not converged'); + }); +}); From e69f2d3064867f597951456b1968787b04986825 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Fri, 19 Jun 2026 17:26:21 +0300 Subject: [PATCH 087/246] fix(sync): converged must account for onlyInSource (missing) items, not onlyInDest orphans MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit converged previously returned true whenever drift===0, even when the source had tables/fields/views absent in the dest — exactly the createTable/createField/createView work a sync would still need to do. Fix: converged = (drift===0) AND (onlyInSourceTables.length===0) AND (every matched table has fields.onlyInSource.length===0 && views.onlyInSource.length===0). onlyInDest* orphans are intentionally excluded — the sync reports but never deletes them, so they must not block the converged verdict. Tests added (4): src-only table → converged false; src-only field in matched table → converged false; dest-only orphan table+field → converged true, identical false; best-effort-only reorder → converged still true. Updated pre-existing 'unmatched src table' assertion from true→false. Co-Authored-By: Claude Sonnet 4.6 --- packages/mcp-server/src/sync/compare.js | 9 ++- .../mcp-server/test/sync/test-compare.test.js | 72 ++++++++++++++++++- 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/packages/mcp-server/src/sync/compare.js b/packages/mcp-server/src/sync/compare.js index 9fdac4b..a5de8e9 100644 --- a/packages/mcp-server/src/sync/compare.js +++ b/packages/mcp-server/src/sync/compare.js @@ -470,7 +470,14 @@ export function compare(srcSnap, destSnap, idmap) { onlyInDestTables.length === 0 && tables.every((t) => t.fields.onlyInSource.length === 0 && t.fields.onlyInDest.length === 0 && t.views.onlyInSource.length === 0 && t.views.onlyInDest.length === 0); - const converged = drift === 0; + // converged = the sync would produce no remaining changes. + // Missing items (onlyInSourceTables, per-table fields.onlyInSource, views.onlyInSource) represent + // createTable/createField/createView operations the sync WOULD make, so they break convergence. + // onlyInDest* are orphans — the sync reports but never deletes them, so they must NOT affect converged. + const converged = + drift === 0 && + onlyInSourceTables.length === 0 && + tables.every((t) => t.fields.onlyInSource.length === 0 && t.views.onlyInSource.length === 0); return { sourceBaseId: srcSnap.baseId, diff --git a/packages/mcp-server/test/sync/test-compare.test.js b/packages/mcp-server/test/sync/test-compare.test.js index f87c4e0..aad1b87 100644 --- a/packages/mcp-server/test/sync/test-compare.test.js +++ b/packages/mcp-server/test/sync/test-compare.test.js @@ -271,7 +271,7 @@ describe('compare', () => { assert.deepEqual(result.onlyInSourceTables, ['Alpha']); assert.equal(result.summary.onlyInSourceTables, 1); assert.equal(result.identical, false); - assert.equal(result.converged, true, 'no drift entries → still converged'); + assert.equal(result.converged, false, 'missing dest table counts as not-converged (I1 fix)'); }); test('unmatched dest table appears in onlyInDestTables with count', () => { @@ -300,6 +300,76 @@ describe('compare', () => { assert.equal(result.converged, true); }); + // ── I1 (converged fix) tests ───────────────────────────────────────────── + + test('I1: source has a table dest lacks → converged:false, identical:false', () => { + // onlyInSourceTables is non-empty; sync would need to createTable → not converged. + const srcTable = mkTable('st1', 'MissingInDest', [fld('sf1', 'Name', 'text')]); + const srcSnap = mkSnap('srcBase', [srcTable]); + const destSnap = mkSnap('destBase', []); + const idmap = { tables: {}, fields: {}, views: {} }; + const result = compare(srcSnap, destSnap, idmap); + assert.equal(result.converged, false, 'missing dest table → converged must be false'); + assert.equal(result.identical, false); + assert.deepEqual(result.onlyInSourceTables, ['MissingInDest']); + }); + + test('I1: source has a field dest lacks within a matched table → converged:false', () => { + // Matched table "Alpha" but src has an extra field "Extra" absent in dest. + // Sync would need to createField → not converged. + const srcTable = mkTable('st1', 'Alpha', [fld('sf1', 'Name', 'text'), fld('sf2', 'Extra', 'number')]); + const destTable = mkTable('dt1', 'Alpha', [fld('df1', 'Name', 'text')]); + const srcSnap = mkSnap('srcBase', [srcTable]); + const destSnap = mkSnap('destBase', [destTable]); + const idmap = { + tables: { st1: 'dt1' }, + fields: { sf1: { destFld: 'df1', choices: {} } }, + views: {}, + }; + const result = compare(srcSnap, destSnap, idmap); + assert.equal(result.converged, false, 'missing dest field → converged must be false'); + assert.equal(result.identical, false); + assert.ok(result.tables[0].fields.onlyInSource.includes('Extra')); + }); + + test('I1: dest has an extra table/field (onlyInDest only) → converged:true, identical:false', () => { + // Orphan dest table and orphan dest field — sync never deletes them. + // converged must remain true; identical must be false. + const srcTable = mkTable('st1', 'Alpha', [fld('sf1', 'Name', 'text')]); + const destTable = mkTable('dt1', 'Alpha', [fld('df1', 'Name', 'text'), fld('df2', 'OrphanField', 'text')]); + const destExtra = mkTable('dt2', 'OrphanTable', [fld('df3', 'X', 'text')]); + const srcSnap = mkSnap('srcBase', [srcTable]); + const destSnap = mkSnap('destBase', [destTable, destExtra]); + const idmap = { + tables: { st1: 'dt1' }, + fields: { sf1: { destFld: 'df1', choices: {} } }, + views: {}, + }; + const result = compare(srcSnap, destSnap, idmap); + assert.equal(result.converged, true, 'orphan dest items must not break convergence'); + assert.equal(result.identical, false, 'orphan items still break identity'); + assert.deepEqual(result.onlyInDestTables, ['OrphanTable']); + assert.ok(result.tables[0].fields.onlyInDest.includes('OrphanField')); + }); + + test('I1: best-effort-only diff (field reorder) still yields converged:true', () => { + // Confirm the pre-existing best-effort-only case is unaffected by the fix. + const srcTable = { ...mkTable('st1', 'Alpha', [fld('sf1', 'A', 'text'), fld('sf2', 'B', 'text')]), primaryFieldId: 'sf1' }; + const destTable = { ...mkTable('dt1', 'Alpha', [fld('df2', 'B', 'text'), fld('df1', 'A', 'text')]), primaryFieldId: 'df1' }; + const srcSnap = mkSnap('srcBase', [srcTable]); + const destSnap = mkSnap('destBase', [destTable]); + const idmap = { + tables: { st1: 'dt1' }, + fields: { sf1: { destFld: 'df1', choices: {} }, sf2: { destFld: 'df2', choices: {} } }, + views: {}, + }; + const result = compare(srcSnap, destSnap, idmap); + assert.equal(result.converged, true, 'best-effort-only diff must still yield converged:true'); + assert.equal(result.identical, false); + assert.equal(result.summary.bestEffort, 1); + assert.equal(result.summary.drift, 0); + }); + test('summary counts aggregate drift and best-effort across multiple tables', () => { // Table1: type drift (drift+1), Table2: field reorder (bestEffort+1). // Both tables must have matching primary field names to isolate the intended diffs. From 837f6a81acfd2e40476cdad55383cbdcfdf81332 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Fri, 19 Jun 2026 17:29:57 +0300 Subject: [PATCH 088/246] feat(sync): saveDiff/loadDiff/latestDiffId Add three exports to idmap.js for persisting and retrieving schema diffs alongside the existing savePlan/loadPlan pattern. latestDiffId picks the most recently modified diff-*.json file via statSync mtime. Co-Authored-By: Claude Sonnet 4.6 --- packages/mcp-server/src/sync/idmap.js | 51 ++++++++++++++++++- .../mcp-server/test/sync/test-idmap.test.js | 40 ++++++++++++++- 2 files changed, 89 insertions(+), 2 deletions(-) diff --git a/packages/mcp-server/src/sync/idmap.js b/packages/mcp-server/src/sync/idmap.js index 4bdf74b..fefddd4 100644 --- a/packages/mcp-server/src/sync/idmap.js +++ b/packages/mcp-server/src/sync/idmap.js @@ -1,5 +1,5 @@ import { join } from 'node:path'; -import { existsSync, readFileSync, mkdirSync } from 'node:fs'; +import { existsSync, readFileSync, mkdirSync, readdirSync, statSync } from 'node:fs'; import { getHomeDir } from '../paths.js'; import { safeAtomicWriteFileSync } from '../safe-write.js'; @@ -159,3 +159,52 @@ export function loadPlan(sourceBaseId, destBaseId, planId) { if (!existsSync(p)) return null; try { return JSON.parse(readFileSync(p, 'utf8')); } catch { return null; } } + +/** + * Persist a schema diff for a source→dest base pair. + * @param {string} sourceBaseId + * @param {string} destBaseId + * @param {{ diffId: string }} diff + */ +export function saveDiff(sourceBaseId, destBaseId, diff) { + writeJson(syncDir(sourceBaseId, destBaseId), `diff-${diff.diffId}.json`, diff); +} + +/** + * Load a previously saved schema diff. Returns `null` when the file is absent + * or unparseable. + * @param {string} sourceBaseId + * @param {string} destBaseId + * @param {string} diffId + * @returns {object|null} + */ +export function loadDiff(sourceBaseId, destBaseId, diffId) { + const p = join(syncDir(sourceBaseId, destBaseId), `diff-${diffId}.json`); + if (!existsSync(p)) return null; + try { return JSON.parse(readFileSync(p, 'utf8')); } catch { return null; } +} + +/** + * Return the diffId of the most recently written diff file for a base pair, + * or `null` if no diff files exist (or the sync dir does not exist yet). + * @param {string} sourceBaseId + * @param {string} destBaseId + * @returns {string|null} + */ +export function latestDiffId(sourceBaseId, destBaseId) { + const dir = syncDir(sourceBaseId, destBaseId); + if (!existsSync(dir)) return null; + const DIFF_RE = /^diff-(.+)\.json$/; + let best = null; + let bestMtime = -Infinity; + for (const name of readdirSync(dir)) { + const m = DIFF_RE.exec(name); + if (!m) continue; + const mtime = statSync(join(dir, name)).mtimeMs; + if (mtime > bestMtime) { + bestMtime = mtime; + best = m[1]; + } + } + return best; +} diff --git a/packages/mcp-server/test/sync/test-idmap.test.js b/packages/mcp-server/test/sync/test-idmap.test.js index 4b17226..0f6efe8 100644 --- a/packages/mcp-server/test/sync/test-idmap.test.js +++ b/packages/mcp-server/test/sync/test-idmap.test.js @@ -3,7 +3,7 @@ import assert from 'node:assert/strict'; import { join } from 'node:path'; import { mkdtempSync, existsSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import { matchByName, saveIdmap, loadIdmap, syncDir } from '../../src/sync/idmap.js'; +import { matchByName, saveIdmap, loadIdmap, syncDir, saveDiff, loadDiff, latestDiffId } from '../../src/sync/idmap.js'; const src = { tables: [ { id: 'tS', name: 'Offers', fields: [ @@ -65,3 +65,41 @@ describe('idmap state I/O', () => { assert.deepEqual(loadIdmap('appAAAAAAAAAAAAAA', 'appBBBBBBBBBBBBBB'), m); }); }); + +describe('idmap diff I/O', () => { + const SRC = 'appSSSSSSSSSSSSSS'; + const DEST = 'appDDDDDDDDDDDDDD'; + + it('saveDiff writes and loadDiff reads back the diff', () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-diff-test-')); + const diff = { diffId: 'd1', tables: [] }; + saveDiff(SRC, DEST, diff); + assert.ok(existsSync(join(syncDir(SRC, DEST), 'diff-d1.json'))); + assert.deepEqual(loadDiff(SRC, DEST, 'd1'), diff); + }); + + it('latestDiffId returns the id of the most recently saved diff', () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-diff-test-')); + saveDiff(SRC, DEST, { diffId: 'd1', tables: [] }); + assert.equal(latestDiffId(SRC, DEST), 'd1'); + // Save a second diff later — it should win + saveDiff(SRC, DEST, { diffId: 'd2', tables: ['x'] }); + assert.equal(latestDiffId(SRC, DEST), 'd2'); + }); + + it('loadDiff returns null for unknown diffId', () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-diff-test-')); + assert.equal(loadDiff(SRC, DEST, 'no-such-diff'), null); + }); + + it('latestDiffId returns null when no diff files exist', () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-diff-test-')); + assert.equal(latestDiffId(SRC, DEST), null); + }); + + it('latestDiffId returns null when sync dir does not exist', () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-diff-test-')); + // Use a pair that was never written to + assert.equal(latestDiffId('appNEVER1111111111', 'appNEVER2222222222'), null); + }); +}); From c768a6769f7afadf4176f2fb6700dd7a995569ff Mon Sep 17 00:00:00 2001 From: "A.R." Date: Fri, 19 Jun 2026 17:34:23 +0300 Subject: [PATCH 089/246] feat(sync): renderDiff digest + drill-in Add renderDiff(diff, {detail, offset, limit}) to report.js returning {human, machine}. Digest mode caps driftSample at DRIFT_SAMPLE_CAP=25 with a +N overflow note; per-table rollup exposes drift/bestEffort/ notSynced as counts. Detail mode returns a table's full entries slice plus fields/views only-in lists, or an {error} for unknown tables. 24 new tests, all passing (555 total, 0 failures). Co-Authored-By: Claude Sonnet 4.6 --- packages/mcp-server/src/sync/report.js | 137 ++++++++++- .../mcp-server/test/sync/test-report.test.js | 223 +++++++++++++++++- 2 files changed, 358 insertions(+), 2 deletions(-) diff --git a/packages/mcp-server/src/sync/report.js b/packages/mcp-server/src/sync/report.js index 5948112..af6e277 100644 --- a/packages/mcp-server/src/sync/report.js +++ b/packages/mcp-server/src/sync/report.js @@ -1,7 +1,10 @@ /** - * Human-readable and machine-readable rendering of a schema sync plan. + * Human-readable and machine-readable rendering of a schema sync plan and diff. */ +/** Maximum number of drift entries to include in a digest driftSample. */ +export const DRIFT_SAMPLE_CAP = 25; + /** * Render a plan into a human-readable summary and a machine-readable copy. * @@ -51,3 +54,135 @@ export function renderApplyResult(result) { } return { human: lines.join('\n'), machine: result }; } + +/** + * Render a diff (from compare()) into human-readable and machine-readable forms. + * + * Without `detail`: returns a digest — per-table count rollup and a capped drift sample. + * With `detail=`: returns that table's full entries (sliced by offset/limit). + * + * @param {{ diffId:string, sourceBaseId:string, destBaseId:string, identical:boolean, converged:boolean, summary:object, tables:Array, onlyInSourceTables:string[], onlyInDestTables:string[] }} diff + * @param {{ detail?:string, offset?:number, limit?:number }} [opts] + * @returns {{ human: string, machine: object }} + */ +export function renderDiff(diff, { detail, offset = 0, limit } = {}) { + if (detail !== undefined && detail !== null && detail !== '') { + // Detail mode: return a single table's full entries. + const tableEntry = diff.tables.find((t) => t.name === detail); + if (!tableEntry) { + const available = diff.tables.filter((t) => t.status === 'differs').map((t) => t.name).join(', '); + return { + human: `Table "${detail}" not found in diff. Available differing tables: ${available || '(none)'}`, + machine: { error: `Table "${detail}" not found in diff. Available: ${available || '(none)'}` }, + }; + } + + // Slice entries by offset/limit. + const sliced = limit !== undefined + ? tableEntry.entries.slice(offset, offset + limit) + : tableEntry.entries.slice(offset); + + const lines = [`Diff detail — ${detail} (${sliced.length} of ${tableEntry.entries.length} entries):`]; + for (const e of sliced) { + lines.push(` [${e.class}] ${e.scope} / ${e.key}: ${JSON.stringify(e.source)} → ${JSON.stringify(e.dest)}`); + } + if (tableEntry.fields.onlyInSource.length) { + lines.push(` fields only in source: ${tableEntry.fields.onlyInSource.join(', ')}`); + } + if (tableEntry.fields.onlyInDest.length) { + lines.push(` fields only in dest: ${tableEntry.fields.onlyInDest.join(', ')}`); + } + if (tableEntry.views.onlyInSource.length) { + lines.push(` views only in source: ${tableEntry.views.onlyInSource.join(', ')}`); + } + if (tableEntry.views.onlyInDest.length) { + lines.push(` views only in dest: ${tableEntry.views.onlyInDest.join(', ')}`); + } + + return { + human: lines.join('\n'), + machine: { + entries: sliced, + fields: tableEntry.fields, + views: tableEntry.views, + }, + }; + } + + // Digest mode: build per-table rollup counts and a capped drift sample. + const tableSummaries = []; + const driftSample = []; + + for (const t of diff.tables) { + let drift = 0; + let bestEffort = 0; + let notSynced = 0; + for (const e of t.entries) { + if (e.class === 'drift') drift++; + else if (e.class === 'best-effort') bestEffort++; + else if (e.class === 'not-synced') notSynced++; + } + + tableSummaries.push({ table: t.name, status: t.status, drift, bestEffort, notSynced }); + + // Collect drift entries for the sample (capped at DRIFT_SAMPLE_CAP across all tables). + if (driftSample.length < DRIFT_SAMPLE_CAP) { + for (const e of t.entries) { + if (e.class === 'drift' && driftSample.length < DRIFT_SAMPLE_CAP) { + driftSample.push({ table: t.name, scope: e.scope, key: e.key, source: e.source, dest: e.dest }); + } + } + } + } + + const totalDrift = diff.summary.drift; + const driftMore = totalDrift - driftSample.length; + + // Build detailHint (list differing tables, not "same" ones). + const differingTables = diff.tables.filter((t) => t.status === 'differs').map((t) => t.name); + const detailHint = differingTables.length > 0 + ? `Re-call with detail= to drill in. Tables with diffs: ${differingTables.join(', ')}` + : 'No differing tables.'; + + // Build human digest. + const verdictLine = diff.identical + ? 'Diff: identical (fully converged)' + : diff.converged + ? 'Diff: not identical but converged (no sync actions needed)' + : `Diff: ${totalDrift} drift, ${diff.summary.bestEffort} best-effort, ${diff.summary.notSynced} not-synced`; + const lines = [verdictLine]; + for (const ts of tableSummaries) { + lines.push(` ${ts.table} [${ts.status}]: drift=${ts.drift}, best-effort=${ts.bestEffort}, not-synced=${ts.notSynced}`); + } + if (diff.onlyInSourceTables.length) { + lines.push(` source-only tables: ${diff.onlyInSourceTables.join(', ')}`); + } + if (diff.onlyInDestTables.length) { + lines.push(` dest-only tables: ${diff.onlyInDestTables.join(', ')}`); + } + if (driftSample.length > 0) { + lines.push(`Drift sample (first ${driftSample.length} of ${totalDrift}):`); + for (const s of driftSample) { + lines.push(` [${s.table}] ${s.scope} / ${s.key}: ${JSON.stringify(s.source)} → ${JSON.stringify(s.dest)}`); + } + if (driftMore > 0) { + lines.push(` ... +${driftMore} more drift entries — use detail= to inspect.`); + } + } + lines.push(detailHint); + + return { + human: lines.join('\n'), + machine: { + diffId: diff.diffId, + sourceBaseId: diff.sourceBaseId, + destBaseId: diff.destBaseId, + identical: diff.identical, + converged: diff.converged, + summary: diff.summary, + tables: tableSummaries, + driftSample, + detailHint, + }, + }; +} diff --git a/packages/mcp-server/test/sync/test-report.test.js b/packages/mcp-server/test/sync/test-report.test.js index 613ab04..e114852 100644 --- a/packages/mcp-server/test/sync/test-report.test.js +++ b/packages/mcp-server/test/sync/test-report.test.js @@ -1,6 +1,6 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { renderPlan, renderApplyResult } from '../../src/sync/report.js'; +import { renderPlan, renderApplyResult, renderDiff, DRIFT_SAMPLE_CAP } from '../../src/sync/report.js'; describe('report.renderPlan', () => { it('summarizes action/orphan/warning counts', () => { @@ -55,3 +55,224 @@ describe('report.renderApplyResult', () => { assert.doesNotMatch(out.human, /records:/); }); }); + +// ── renderDiff tests (Task 6) ──────────────────────────────────────────────── + +/** + * Build a fake diff with `driftCount` drift entries spread across two tables. + * Table "Products" gets ceil(driftCount/2), "Offers" gets floor(driftCount/2). + * Also adds 2 best-effort + 1 not-synced entries in Products, and 3 in Offers. + */ +function buildDiff(driftCount = 30) { + const productsDrift = Math.ceil(driftCount / 2); // 15 + const offersDrift = driftCount - productsDrift; // 15 + + function driftEntries(tableName, count) { + return Array.from({ length: count }, (_, i) => ({ + scope: `field:F${i}`, + key: 'type', + source: 'text', + dest: 'number', + class: 'drift', + })); + } + + const productsEntries = [ + ...driftEntries('Products', productsDrift), + { scope: 'table', key: 'fieldOrder', source: ['A', 'B'], dest: ['B', 'A'], class: 'best-effort' }, + { scope: 'table', key: 'viewOrder', source: ['V1', 'V2'], dest: ['V2', 'V1'], class: 'best-effort' }, + { scope: 'table', key: 'sections', source: [], dest: [{ id: 's1' }], class: 'not-synced' }, + ]; + + const offersEntries = [ + ...driftEntries('Offers', offersDrift), + { scope: 'table', key: 'fieldOrder', source: ['X', 'Y'], dest: ['Y', 'X'], class: 'best-effort' }, + { scope: 'table', key: 'viewOrder', source: ['V3'], dest: ['V3'], class: 'best-effort' }, + { scope: 'table', key: 'sections', source: [], dest: [], class: 'not-synced' }, + ]; + + return { + diffId: 'diff-001', + sourceBaseId: 'srcBase', + destBaseId: 'destBase', + identical: false, + converged: false, + summary: { + drift: driftCount, + bestEffort: 4, + notSynced: 2, + onlyInSourceTables: 0, + onlyInDestTables: 0, + }, + tables: [ + { + name: 'Products', + status: 'differs', + entries: productsEntries, + fields: { onlyInSource: [], onlyInDest: [] }, + views: { onlyInSource: [], onlyInDest: [] }, + }, + { + name: 'Offers', + status: 'differs', + entries: offersEntries, + fields: { onlyInSource: ['MissingField'], onlyInDest: [] }, + views: { onlyInSource: ['ExtraView'], onlyInDest: [] }, + }, + ], + onlyInSourceTables: [], + onlyInDestTables: [], + }; +} + +describe('DRIFT_SAMPLE_CAP', () => { + it('is exported as 25', () => { + assert.equal(DRIFT_SAMPLE_CAP, 25); + }); +}); + +describe('renderDiff — digest (no detail)', () => { + const diff = buildDiff(30); + + it('returns { human, machine }', () => { + const out = renderDiff(diff); + assert.ok(typeof out.human === 'string', 'human should be a string'); + assert.ok(out.machine !== null && typeof out.machine === 'object', 'machine should be an object'); + }); + + it('machine has top-level identity fields', () => { + const { machine } = renderDiff(diff); + assert.equal(machine.diffId, 'diff-001'); + assert.equal(machine.sourceBaseId, 'srcBase'); + assert.equal(machine.destBaseId, 'destBase'); + assert.equal(machine.identical, false); + assert.equal(machine.converged, false); + }); + + it('machine.summary.drift === 30', () => { + const { machine } = renderDiff(diff); + assert.equal(machine.summary.drift, 30); + }); + + it('machine.summary bestEffort and notSynced are numbers', () => { + const { machine } = renderDiff(diff); + assert.equal(typeof machine.summary.bestEffort, 'number'); + assert.equal(typeof machine.summary.notSynced, 'number'); + assert.equal(machine.summary.bestEffort, 4); + assert.equal(machine.summary.notSynced, 2); + }); + + it('machine.driftSample.length === 25 (capped at DRIFT_SAMPLE_CAP)', () => { + const { machine } = renderDiff(diff); + assert.equal(machine.driftSample.length, 25); + }); + + it('driftSample entries each have table, scope, key, source, dest', () => { + const { machine } = renderDiff(diff); + for (const s of machine.driftSample) { + assert.ok('table' in s, 'entry should have table'); + assert.ok('scope' in s, 'entry should have scope'); + assert.ok('key' in s, 'entry should have key'); + assert.ok('source' in s, 'entry should have source'); + assert.ok('dest' in s, 'entry should have dest'); + } + }); + + it('machine.tables is per-table rollup with drift/bestEffort/notSynced counts', () => { + const { machine } = renderDiff(diff); + assert.equal(machine.tables.length, 2); + + const products = machine.tables.find((t) => t.table === 'Products'); + assert.ok(products, 'Products table in rollup'); + assert.equal(products.status, 'differs'); + assert.equal(products.drift, 15); + assert.equal(products.bestEffort, 2); + assert.equal(products.notSynced, 1); + + const offers = machine.tables.find((t) => t.table === 'Offers'); + assert.ok(offers, 'Offers table in rollup'); + assert.equal(offers.drift, 15); + assert.equal(offers.bestEffort, 2); + assert.equal(offers.notSynced, 1); + }); + + it('machine.tables bestEffort and notSynced are NUMBERS not arrays', () => { + const { machine } = renderDiff(diff); + for (const t of machine.tables) { + assert.equal(typeof t.bestEffort, 'number', `${t.table}.bestEffort should be number`); + assert.equal(typeof t.notSynced, 'number', `${t.table}.notSynced should be number`); + } + }); + + it('machine.detailHint is a string', () => { + const { machine } = renderDiff(diff); + assert.ok(typeof machine.detailHint === 'string', 'detailHint should be a string'); + }); + + it('human contains table names and drift count', () => { + const { human } = renderDiff(diff); + assert.match(human, /Products/); + assert.match(human, /Offers/); + assert.match(human, /drift/i); + }); + + it('human mentions the cap / overflow when there are more than 25 drift entries', () => { + const { human } = renderDiff(diff); + // Should say something about more drift entries (5 more after cap of 25) + assert.match(human, /\+5|5 more/i); + }); +}); + +describe('renderDiff — digest with 10 drift entries (no cap needed)', () => { + it('driftSample.length === 10 when total drift < DRIFT_SAMPLE_CAP', () => { + const diff = buildDiff(10); + // Fix the summary to match actual entry count (5 per table) + diff.summary.drift = 10; + const { machine } = renderDiff(diff); + assert.equal(machine.driftSample.length, 10); + }); +}); + +describe('renderDiff — detail mode', () => { + const diff = buildDiff(30); + + it('returns Offers entries when detail="Offers"', () => { + const { machine } = renderDiff(diff, { detail: 'Offers' }); + // Should have entries array (the raw entries for Offers table) + assert.ok(Array.isArray(machine.entries), 'machine.entries should be array'); + // Offers has 15 drift + 3 non-drift = 18 entries + assert.equal(machine.entries.length, 18); + }); + + it('includes fields and views only-in lists in detail mode', () => { + const { machine } = renderDiff(diff, { detail: 'Offers' }); + assert.ok('fields' in machine, 'machine should have fields'); + assert.ok('views' in machine, 'machine should have views'); + assert.deepEqual(machine.fields.onlyInSource, ['MissingField']); + assert.deepEqual(machine.views.onlyInSource, ['ExtraView']); + }); + + it('slices entries with offset and limit', () => { + const { machine } = renderDiff(diff, { detail: 'Offers', offset: 5, limit: 3 }); + assert.equal(machine.entries.length, 3); + }); + + it('returns error object for unknown table name', () => { + const { machine } = renderDiff(diff, { detail: 'NonExistent' }); + assert.ok('error' in machine, 'should return error for unknown table'); + assert.match(machine.error, /NonExistent|available/i); + }); + + it('human in detail mode describes Offers entries', () => { + const { human } = renderDiff(diff, { detail: 'Offers' }); + assert.ok(typeof human === 'string', 'human should be a string'); + assert.match(human, /Offers/); + }); + + it('detail mode returns Products entries when detail="Products"', () => { + const { machine } = renderDiff(diff, { detail: 'Products' }); + assert.ok(Array.isArray(machine.entries)); + // Products has 15 drift + 3 non-drift = 18 entries + assert.equal(machine.entries.length, 18); + }); +}); From 96a6b35f9eb2644fe2609c38beb265ff3ef6c6bb Mon Sep 17 00:00:00 2001 From: "A.R." Date: Fri, 19 Jun 2026 17:40:20 +0300 Subject: [PATCH 090/246] feat(sync): add diff() + diffDetail() engine functions - diff(): snapshots both bases, matchByName, compare, saveDiff, renderDiff digest - diffDetail(): loadDiff (or latestDiffId fallback) + renderDiff with detail/offset/limit - Adds compare, saveDiff/loadDiff/latestDiffId, renderDiff imports to index.js - 3 new tests; full suite 558/558 green Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/index.js | 41 +++++- .../test/sync/test-diff-engine.test.js | 121 ++++++++++++++++++ 2 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 packages/mcp-server/test/sync/test-diff-engine.test.js diff --git a/packages/mcp-server/src/sync/index.js b/packages/mcp-server/src/sync/index.js index 9c5a069..7b9b13c 100644 --- a/packages/mcp-server/src/sync/index.js +++ b/packages/mcp-server/src/sync/index.js @@ -1,8 +1,9 @@ import { createHash } from 'node:crypto'; import { snapshotBase, snapshotSchemaOnly } from './snapshot.js'; -import { matchByName, saveIdmap, savePlan, saveState, loadPlan, loadIdmap } from './idmap.js'; +import { matchByName, saveIdmap, savePlan, saveState, loadPlan, loadIdmap, saveDiff, loadDiff, latestDiffId } from './idmap.js'; import { computePlan } from './diff.js'; -import { renderPlan, renderApplyResult } from './report.js'; +import { compare } from './compare.js'; +import { renderPlan, renderApplyResult, renderDiff } from './report.js'; import { applyPlan } from './apply.js'; import { newJournal, loadJournal, saveJournal } from './journal.js'; import { applyRecords as applyRecordsImpl, reconcile as reconcileImpl, writeRecordsJobStatus, readRecordsJobStatus } from './records.js'; @@ -54,6 +55,42 @@ export async function plan({ client, sourceBaseId, destBaseId, planId }) { return renderPlan(fullPlan); } +/** + * Compute a schema diff between two bases. `diffId` is supplied by the caller so the engine + * stays deterministic/testable. Saves the full diff to disk and returns a rendered digest. + * + * @param {{ client: object, sourceBaseId: string, destBaseId: string, diffId: string }} opts + * @returns {Promise<{ human: string, machine: object }>} + */ +export async function diff({ client, sourceBaseId, destBaseId, diffId }) { + const src = await snapshotBase(client, sourceBaseId); + const dest = await snapshotBase(client, destBaseId); + const idmap = matchByName(src, dest); + const base = compare(src, dest, idmap); + const fullDiff = { diffId, ...base }; + saveDiff(sourceBaseId, destBaseId, fullDiff); + return renderDiff(fullDiff); +} + +/** + * Load a previously saved schema diff and return a detail view (or digest if no detail given). + * If `diffId` is falsy, resolves to the most recently written diff via `latestDiffId`. + * + * @param {{ sourceBaseId: string, destBaseId: string, diffId?: string, detail?: string, offset?: number, limit?: number }} opts + * @returns {{ human: string, machine: object }} + */ +export function diffDetail({ sourceBaseId, destBaseId, diffId, detail, offset, limit }) { + const resolvedDiffId = diffId || latestDiffId(sourceBaseId, destBaseId); + if (!resolvedDiffId) { + return { human: 'no diff found', machine: { error: 'no diff found — run mode=diff first' } }; + } + const savedDiff = loadDiff(sourceBaseId, destBaseId, resolvedDiffId); + if (!savedDiff) { + return { human: 'no diff found', machine: { error: 'no diff found — run mode=diff first' } }; + } + return renderDiff(savedDiff, { detail, offset, limit }); +} + export async function apply({ client, sourceBaseId, destBaseId, planId, runStartedAt }) { const fullPlan = loadPlan(sourceBaseId, destBaseId, planId); if (!fullPlan) throw new Error(`No saved plan "${planId}" for ${sourceBaseId} -> ${destBaseId}. Run mode=plan first.`); diff --git a/packages/mcp-server/test/sync/test-diff-engine.test.js b/packages/mcp-server/test/sync/test-diff-engine.test.js new file mode 100644 index 0000000..78828ea --- /dev/null +++ b/packages/mcp-server/test/sync/test-diff-engine.test.js @@ -0,0 +1,121 @@ +/** + * test-diff-engine.test.js + * + * Tests for diff() and diffDetail() engine functions in src/sync/index.js. + * Uses a mock client; no real network. + */ +import { describe, it, before } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; + +import { diff, diffDetail } from '../../src/sync/index.js'; +import { syncDir } from '../../src/sync/idmap.js'; + +// ── fixtures ───────────────────────────────────────────────────────────────── + +const SRC_APP = 'appSrcDiffSrcDiff'; +const DEST_APP = 'appDstDiffDstDiff'; +const TABLE_NAME = 'Tasks'; + +/** + * Source schema: Tasks table with a text field "Title". + */ +const SRC_DATA = { + tableSchemas: [ + { + id: 'tblSrc001', + name: TABLE_NAME, + primaryColumnId: 'fldSrc001', + columns: [ + { id: 'fldSrc001', name: 'Title', type: 'text' }, + { id: 'fldSrc002', name: 'Count', type: 'text' }, + ], + }, + ], +}; + +/** + * Destination schema: same table, same field names, but "Count" is a number type — creates drift. + */ +const DEST_DATA = { + tableSchemas: [ + { + id: 'tblDst001', + name: TABLE_NAME, + primaryColumnId: 'fldDst001', + columns: [ + { id: 'fldDst001', name: 'Title', type: 'text' }, + { id: 'fldDst002', name: 'Count', type: 'number' }, + ], + }, + ], +}; + +/** + * Build a unified mock client whose behavior depends on the appId passed. + */ +function makeMockClient(sourceBaseId, destBaseId) { + return { + getApplicationData: async (appId) => { + if (appId === sourceBaseId) return { data: SRC_DATA }; + return { data: DEST_DATA }; + }, + getView: async () => ({}), + }; +} + +// ── tests ───────────────────────────────────────────────────────────────────── + +describe('sync index.diff / diffDetail', () => { + before(() => { + // Redirect home dir so test writes don't pollute real user data. + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-diff-')); + }); + + it('diff() returns a digest with drift >= 1 and writes diff-.json', async () => { + const client = makeMockClient(SRC_APP, DEST_APP); + const result = await diff({ client, sourceBaseId: SRC_APP, destBaseId: DEST_APP, diffId: 'd1' }); + + // Should have a human-readable summary. + assert.ok(typeof result.human === 'string', 'human should be a string'); + assert.ok(result.human.length > 0, 'human should not be empty'); + + // machine.summary.drift should be >= 1 (type mismatch on "Count" field). + assert.ok(result.machine.summary.drift >= 1, `expected drift >= 1, got ${result.machine.summary.drift}`); + + // diffId should be reflected in the machine output. + assert.equal(result.machine.diffId, 'd1', 'machine.diffId should equal d1'); + + // The file should have been persisted. + const dir = syncDir(SRC_APP, DEST_APP); + const filePath = join(dir, 'diff-d1.json'); + assert.ok(existsSync(filePath), `diff-d1.json should exist at ${filePath}`); + }); + + it('diffDetail() with explicit diffId returns entries for the table', async () => { + // d1 was written by the previous test; reuse via diffDetail (no client needed). + const result = diffDetail({ sourceBaseId: SRC_APP, destBaseId: DEST_APP, diffId: 'd1', detail: TABLE_NAME }); + + assert.ok(typeof result.human === 'string', 'human should be a string'); + // In detail mode, machine contains entries for the requested table. + assert.ok(Array.isArray(result.machine.entries), 'machine.entries should be an array'); + // There should be at least one entry (the type drift on "Count"). + assert.ok(result.machine.entries.length >= 1, `expected >= 1 entry, got ${result.machine.entries.length}`); + // Each entry should have scope/key/source/dest/class. + const entry = result.machine.entries[0]; + assert.ok(entry.scope, 'entry.scope should be present'); + assert.ok(entry.key, 'entry.key should be present'); + assert.ok(entry.class, 'entry.class should be present'); + }); + + it('diffDetail() with no diffId uses latestDiffId() to find the file', async () => { + // Pass no diffId — should auto-resolve to the latest diff (d1). + const result = diffDetail({ sourceBaseId: SRC_APP, destBaseId: DEST_APP, detail: TABLE_NAME }); + + assert.ok(typeof result.human === 'string', 'human should be a string'); + assert.ok(Array.isArray(result.machine.entries), 'machine.entries should be an array'); + assert.ok(result.machine.entries.length >= 1, 'should resolve to the latest diff and return entries'); + }); +}); From 15559b0330e44199313fb14484ef1473fd9b33f7 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Fri, 19 Jun 2026 17:46:08 +0300 Subject: [PATCH 091/246] feat(sync): sync_base mode=diff (digest + detail drill-in) Co-Authored-By: Claude Sonnet 4.6 --- packages/mcp-server/src/index.js | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/packages/mcp-server/src/index.js b/packages/mcp-server/src/index.js index a4f474f..76592fe 100644 --- a/packages/mcp-server/src/index.js +++ b/packages/mcp-server/src/index.js @@ -1578,16 +1578,20 @@ Note: "form title" is the view name itself — use rename_view to change it. "Fi // ── Sync Tools ── { name: 'sync_base', - description: 'Base-to-base schema + record sync. mode="plan" (read-only): compare source/dest schema by name and return an ordered plan of tables/fields to create or update, plus orphans + warnings; does NOT mutate. mode="apply": execute a saved plan against the destination — create tables, reconcile the primary, create scalar/link/computed fields (with source->dest reference remapping + formula validation), apply non-destructive field updates; then start the RECORD sync (two-pass cells + links, attachments, view-filter restore) in the BACKGROUND and return immediately with a jobId. apply requires planId and aborts if the destination drifted since the plan. mode="status": poll the background record job (pass the apply planId as planId) for progress/completion. mode="reconcile": rebuild/repair the record map — existence-prune dead entries from the idmap, with optional natural-key re-match per table.', + description: 'Base-to-base schema + record sync. mode="plan" (read-only): compare source/dest schema by name and return an ordered plan of tables/fields to create or update, plus orphans + warnings; does NOT mutate. mode="apply": execute a saved plan against the destination — create tables, reconcile the primary, create scalar/link/computed fields (with source->dest reference remapping + formula validation), apply non-destructive field updates; then start the RECORD sync (two-pass cells + links, attachments, view-filter restore) in the BACKGROUND and return immediately with a jobId. apply requires planId and aborts if the destination drifted since the plan. mode="status": poll the background record job (pass the apply planId as planId) for progress/completion. mode="reconcile": rebuild/repair the record map — existence-prune dead entries from the idmap, with optional natural-key re-match per table. mode="diff": compute a schema digest comparing source and destination WITHOUT saving a plan; returns a diffId and human-readable summary; pass detail=
to drill into a specific section of a prior diff.', annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, inputSchema: { type: 'object', properties: { - mode: { type: 'string', enum: ['plan', 'apply', 'reconcile', 'status'], description: '"plan" (read-only preview), "apply" (execute a saved plan; starts the record sync in the background and returns a jobId), "status" (poll the background record job by planId), or "reconcile" (rebuild/repair the record map: existence-prune dead entries; optional natural-key re-match).' }, + mode: { type: 'string', enum: ['plan', 'apply', 'reconcile', 'status', 'diff'], description: '"plan" (read-only preview), "apply" (execute a saved plan; starts the record sync in the background and returns a jobId), "status" (poll the background record job by planId), "reconcile" (rebuild/repair the record map: existence-prune dead entries; optional natural-key re-match), or "diff" (schema digest comparing source and destination without saving a plan; optionally drill into a section with detail=
).' }, sourceAppId: { type: 'string', description: 'Source base/application ID to copy schema FROM' }, destAppId: { type: 'string', description: 'Destination base/application ID to copy schema TO' }, planId: { type: 'string', description: 'Required for mode="apply" (the planId from a prior mode="plan" run) and mode="status" (the same planId, which is the background record jobId).' }, naturalKeys: { type: 'object', description: 'Used only by mode="reconcile": map of tableName → fieldName to use as a natural key for re-matching records (e.g. { "Projects": "Name" }). Omit to run existence-prune only.', additionalProperties: { type: 'string' } }, + detail: { type: 'string', description: 'Used only by mode="diff": section name to drill into (e.g. a table name or action key). Requires diffId.' }, + diffId: { type: 'string', description: 'Used by mode="diff": ID of a prior diff to retrieve or drill into. If omitted on the initial call, one is generated automatically.' }, + offset: { type: 'number', description: 'Used by mode="diff" with detail: skip this many entries before returning results.' }, + limit: { type: 'number', description: 'Used by mode="diff" with detail: maximum number of entries to return.' }, debug: debugProp, }, required: ['mode', 'sourceAppId', 'destAppId'], @@ -2389,7 +2393,7 @@ const handlers = { // ── Sync ── - async sync_base({ mode, sourceAppId, destAppId, planId, naturalKeys, debug }) { + async sync_base({ mode, sourceAppId, destAppId, planId, naturalKeys, detail, diffId, offset, limit, debug }) { const sync = await import('./sync/index.js'); if (mode === 'plan') { const id = 'pln' + client._genRandomId(); @@ -2420,7 +2424,16 @@ const handlers = { : `Records job ${planId}: ${raw.status}${raw.message ? ' — ' + raw.message : ''}`; return ok({ planId, status: raw.status, summary }, raw, debug); } - return err(`Unsupported mode "${mode}". Use "plan", "apply", "reconcile", or "status".`); + if (mode === 'diff') { + if (detail) { + const out = sync.diffDetail({ sourceBaseId: sourceAppId, destBaseId: destAppId, diffId, detail, offset, limit }); + return ok({ diffId: diffId ?? null, summary: out.human }, out.machine, debug); + } + const id = diffId || ('dif' + client._genRandomId()); + const out = await sync.diff({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, diffId: id }); + return ok({ diffId: id, summary: out.human }, out.machine, debug); + } + return err(`Unsupported mode "${mode}". Use "plan", "apply", "reconcile", "status", or "diff".`); }, // ── Meta: Tool Management ── From 4b6edf28cceb165c10d9ea4eb336e6b2b8890f9e Mon Sep 17 00:00:00 2001 From: "A.R." Date: Fri, 19 Jun 2026 17:50:51 +0300 Subject: [PATCH 092/246] feat(sync): stable changeId + class + apply flag on plan actions Post-step in computePlan annotates every action with a name-based changeId (format: '||'), class:'drift', and apply:true. Resolves updateField (no sourceTableId) and applyViewConfig (no name) via lookup maps built from srcSnap. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/diff.js | 54 ++++++ .../test/sync/test-changeid.test.js | 155 ++++++++++++++++++ 2 files changed, 209 insertions(+) create mode 100644 packages/mcp-server/test/sync/test-changeid.test.js diff --git a/packages/mcp-server/src/sync/diff.js b/packages/mcp-server/src/sync/diff.js index d6cd51b..8c69778 100644 --- a/packages/mcp-server/src/sync/diff.js +++ b/packages/mcp-server/src/sync/diff.js @@ -276,5 +276,59 @@ export function computePlan(srcSnap, destSnap, idmap) { for (const dv of collabViews(dt)) if (!srcViewNames.has(dv.name)) orphans.push({ kind: 'view', destId: dv.id, name: dv.name, tableName: dt.name }); } + // ── Annotate every action with a stable changeId + class + apply ───────── + // Build resolution maps from srcSnap once (name-based, stable across bases). + const srcTableNameById = new Map(srcSnap.tables.map((t) => [t.id, t.name])); + const srcFieldTableName = new Map(); // fieldId → tableName + const srcFieldName = new Map(); // fieldId → fieldName + const srcViewName = new Map(); // viewId → viewName + for (const t of srcSnap.tables) { + for (const f of t.fields) { + srcFieldTableName.set(f.id, t.name); + srcFieldName.set(f.id, f.name); + } + for (const v of (t.views || [])) { + srcViewName.set(v.id, v.name); + } + } + + for (const a of actions) { + let tableName, targetName; + switch (a.kind) { + case 'createTable': + tableName = a.name; + targetName = a.name; + break; + case 'reconcilePrimary': + tableName = srcTableNameById.get(a.sourceTableId) ?? a.sourceTableId; + targetName = tableName; + break; + case 'createField': + tableName = srcTableNameById.get(a.sourceTableId) ?? a.sourceTableId; + targetName = a.name; + break; + case 'updateField': + // No sourceTableId — resolve via srcFieldId→tableName map. + tableName = srcFieldTableName.get(a.sourceFieldId) ?? a.sourceFieldId; + targetName = srcFieldName.get(a.sourceFieldId) ?? a.sourceFieldId; + break; + case 'createView': + tableName = srcTableNameById.get(a.sourceTableId) ?? a.sourceTableId; + targetName = a.name; + break; + case 'applyViewConfig': + // No name field — resolve view name from sourceViewId. + tableName = srcTableNameById.get(a.sourceTableId) ?? a.sourceTableId; + targetName = srcViewName.get(a.sourceViewId) ?? a.sourceViewId; + break; + default: + tableName = a.sourceTableId ?? ''; + targetName = a.name ?? ''; + } + a.changeId = `${a.kind}|${tableName}|${targetName}`; + a.class = 'drift'; + a.apply = true; + } + return { sourceBaseId: srcSnap.baseId, destBaseId: destSnap.baseId, idmap, actions, orphans, warnings }; } diff --git a/packages/mcp-server/test/sync/test-changeid.test.js b/packages/mcp-server/test/sync/test-changeid.test.js new file mode 100644 index 0000000..5b5ee47 --- /dev/null +++ b/packages/mcp-server/test/sync/test-changeid.test.js @@ -0,0 +1,155 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { computePlan } from '../../src/sync/diff.js'; + +// Minimal snapshot helpers +function makeSnap(baseId, tables) { + return { baseId, tables }; +} + +test('computePlan actions carry stable name-based changeId + class + apply', () => { + const src = makeSnap('a', [{ + id: 'tS', name: 'Offers', primaryFieldId: 'p', + fields: [ + { id: 'p', name: 'Name', type: 'text', typeOptions: null, description: null, isComputed: false }, + { id: 'f', name: 'Price', type: 'number', typeOptions: null, description: null, isComputed: false }, + ], + views: [], + }]); + const dest = makeSnap('b', []); + const idmap = { tables: {}, fields: {}, views: {} }; + + const p1 = computePlan(src, dest, idmap); + const cf = p1.actions.find(a => a.kind === 'createField' && a.name === 'Price'); + assert.ok(cf, 'createField for Price should exist'); + assert.equal(cf.changeId, 'createField|Offers|Price'); + assert.equal(cf.class, 'drift'); + assert.equal(cf.apply, true); + + // stability across a second identical run + const p2 = computePlan(src, dest, idmap); + assert.deepEqual(p1.actions.map(a => a.changeId), p2.actions.map(a => a.changeId)); +}); + +test('createTable action gets stable changeId', () => { + const src = makeSnap('a', [{ + id: 'tS', name: 'Projects', primaryFieldId: 'p', + fields: [{ id: 'p', name: 'Name', type: 'text', typeOptions: null, description: null, isComputed: false }], + views: [], + }]); + const dest = makeSnap('b', []); + const idmap = { tables: {}, fields: {}, views: {} }; + + const plan = computePlan(src, dest, idmap); + const ct = plan.actions.find(a => a.kind === 'createTable'); + assert.ok(ct, 'createTable action should exist'); + assert.equal(ct.changeId, 'createTable|Projects|Projects'); + assert.equal(ct.class, 'drift'); + assert.equal(ct.apply, true); +}); + +test('reconcilePrimary action gets stable changeId', () => { + const src = makeSnap('a', [{ + id: 'tS', name: 'Projects', primaryFieldId: 'p', + fields: [{ id: 'p', name: 'Title', type: 'text', typeOptions: null, description: null, isComputed: false }], + views: [], + }]); + const dest = makeSnap('b', []); + const idmap = { tables: {}, fields: {}, views: {} }; + + const plan = computePlan(src, dest, idmap); + const rp = plan.actions.find(a => a.kind === 'reconcilePrimary'); + assert.ok(rp, 'reconcilePrimary action should exist'); + assert.equal(rp.changeId, 'reconcilePrimary|Projects|Projects'); + assert.equal(rp.class, 'drift'); + assert.equal(rp.apply, true); +}); + +test('updateField action (no sourceTableId) gets table-qualified changeId via srcFieldId lookup', () => { + // src and dest have same table (matched via idmap), but field type differs + const src = makeSnap('a', [{ + id: 'tS', name: 'Orders', primaryFieldId: 'p', + fields: [ + { id: 'p', name: 'Name', type: 'text', typeOptions: null, description: null, isComputed: false }, + { id: 'fA', name: 'Amount', type: 'number', typeOptions: null, description: null, isComputed: false }, + ], + views: [], + }]); + const dest = makeSnap('b', [{ + id: 'tD', name: 'Orders', primaryFieldId: 'dp', + fields: [ + { id: 'dp', name: 'Name', type: 'text', typeOptions: null, description: null, isComputed: false }, + { id: 'dfA', name: 'Amount', type: 'text', typeOptions: null, description: null, isComputed: false }, // type differs + ], + views: [], + }]); + const idmap = { + tables: { tS: 'tD' }, + fields: { fA: { destFld: 'dfA', choices: {} } }, + views: {}, + }; + + const plan = computePlan(src, dest, idmap); + const uf = plan.actions.find(a => a.kind === 'updateField'); + assert.ok(uf, 'updateField action should exist'); + // updateField has no sourceTableId — must resolve table from sourceFieldId + assert.equal(uf.changeId, 'updateField|Orders|Amount'); + assert.equal(uf.class, 'drift'); + assert.equal(uf.apply, true); +}); + +test('applyViewConfig action (no name field) gets view-name-qualified changeId', () => { + const src = makeSnap('a', [{ + id: 'tS', name: 'Tasks', primaryFieldId: 'p', + fields: [{ id: 'p', name: 'Name', type: 'text', typeOptions: null, description: null, isComputed: false }], + views: [{ + id: 'vS', name: 'My Grid', type: 'grid', personalForUserId: null, + config: { filters: [], sorts: [], groups: [] }, + }], + }]); + const dest = makeSnap('b', [{ + id: 'tD', name: 'Tasks', primaryFieldId: 'dp', + fields: [{ id: 'dp', name: 'Name', type: 'text', typeOptions: null, description: null, isComputed: false }], + views: [], // no views — force createView + applyViewConfig + }]); + const idmap = { + tables: { tS: 'tD' }, + fields: {}, + views: {}, + }; + + const plan = computePlan(src, dest, idmap); + const avc = plan.actions.find(a => a.kind === 'applyViewConfig'); + assert.ok(avc, 'applyViewConfig action should exist'); + assert.equal(avc.changeId, 'applyViewConfig|Tasks|My Grid'); + assert.equal(avc.class, 'drift'); + assert.equal(avc.apply, true); +}); + +test('createView action gets correct changeId', () => { + const src = makeSnap('a', [{ + id: 'tS', name: 'Tasks', primaryFieldId: 'p', + fields: [{ id: 'p', name: 'Name', type: 'text', typeOptions: null, description: null, isComputed: false }], + views: [{ + id: 'vS', name: 'Kanban Board', type: 'kanban', personalForUserId: null, + config: {}, + }], + }]); + const dest = makeSnap('b', [{ + id: 'tD', name: 'Tasks', primaryFieldId: 'dp', + fields: [{ id: 'dp', name: 'Name', type: 'text', typeOptions: null, description: null, isComputed: false }], + views: [], + }]); + const idmap = { + tables: { tS: 'tD' }, + fields: {}, + views: {}, + }; + + const plan = computePlan(src, dest, idmap); + const cv = plan.actions.find(a => a.kind === 'createView'); + assert.ok(cv, 'createView action should exist'); + assert.equal(cv.changeId, 'createView|Tasks|Kanban Board'); + assert.equal(cv.class, 'drift'); + assert.equal(cv.apply, true); +}); From bd0cab566789d135c2763437639cf9bb5b53e9ed Mon Sep 17 00:00:00 2001 From: "A.R." Date: Fri, 19 Jun 2026 17:57:49 +0300 Subject: [PATCH 093/246] feat(sync): changeset digest + direction in plan() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - renderPlan: add machine.sample (≤25 {changeId,op,table,target} entries derived from annotated actions) and editHint in human output mentioning skip/apply:false to selectively exclude changes; reuses DRIFT_SAMPLE_CAP - plan(): add direction param ('to-dest'|'to-source', default 'to-dest'); 'to-source' swaps src/dest before snapshot/compute so changeset targets the source base; persisted state uses effective (swapped) IDs - sync_base tool: expose direction in inputSchema + pass through to plan() - Tests: 8 new renderPlan digest tests + 3 direction tests (all green); 575/575 mcp-server tests pass; tool-sync ✓ (no category change) Co-Authored-By: Claude Sonnet 4.6 --- packages/mcp-server/src/index.js | 5 +- packages/mcp-server/src/sync/index.js | 28 ++++-- packages/mcp-server/src/sync/report.js | 30 ++++++ .../mcp-server/test/sync/test-index.test.js | 49 ++++++++++ .../mcp-server/test/sync/test-report.test.js | 97 +++++++++++++++++++ 5 files changed, 198 insertions(+), 11 deletions(-) diff --git a/packages/mcp-server/src/index.js b/packages/mcp-server/src/index.js index 76592fe..4adf0a1 100644 --- a/packages/mcp-server/src/index.js +++ b/packages/mcp-server/src/index.js @@ -1592,6 +1592,7 @@ Note: "form title" is the view name itself — use rename_view to change it. "Fi diffId: { type: 'string', description: 'Used by mode="diff": ID of a prior diff to retrieve or drill into. If omitted on the initial call, one is generated automatically.' }, offset: { type: 'number', description: 'Used by mode="diff" with detail: skip this many entries before returning results.' }, limit: { type: 'number', description: 'Used by mode="diff" with detail: maximum number of entries to return.' }, + direction: { type: 'string', enum: ['to-dest', 'to-source'], description: 'Used only by mode="plan": direction of the changeset. "to-dest" (default) brings the destination base up to date with the source. "to-source" swaps the roles so the changeset targets the source base (makes the source match the destination).' }, debug: debugProp, }, required: ['mode', 'sourceAppId', 'destAppId'], @@ -2393,11 +2394,11 @@ const handlers = { // ── Sync ── - async sync_base({ mode, sourceAppId, destAppId, planId, naturalKeys, detail, diffId, offset, limit, debug }) { + async sync_base({ mode, sourceAppId, destAppId, planId, naturalKeys, detail, diffId, offset, limit, direction, debug }) { const sync = await import('./sync/index.js'); if (mode === 'plan') { const id = 'pln' + client._genRandomId(); - const out = await sync.plan({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, planId: id }); + const out = await sync.plan({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, planId: id, direction }); return ok({ planId: id, summary: out.human }, out.machine, debug); } if (mode === 'apply') { diff --git a/packages/mcp-server/src/sync/index.js b/packages/mcp-server/src/sync/index.js index 7b9b13c..a148175 100644 --- a/packages/mcp-server/src/sync/index.js +++ b/packages/mcp-server/src/sync/index.js @@ -30,12 +30,22 @@ export function fingerprintSchema(snap) { * Compute a schema plan. `planId` is supplied by the caller (tool handler) so * the engine stays deterministic/testable. * - * @param {{ client: object, sourceBaseId: string, destBaseId: string, planId: string }} opts + * `direction` controls which base is treated as the source of truth: + * - `'to-dest'` (default) — changeset targets the DEST base (dest is brought up to date with src). + * - `'to-source'` — swap src/dest before snapshotting so the changeset targets the SOURCE base + * (source is brought up to date with dest). Persisted state uses swapped IDs so load/apply + * work correctly on the swapped pair. + * + * @param {{ client: object, sourceBaseId: string, destBaseId: string, planId: string, direction?: 'to-dest'|'to-source' }} opts * @returns {Promise<{ human: string, machine: object }>} */ -export async function plan({ client, sourceBaseId, destBaseId, planId }) { - const src = await snapshotBase(client, sourceBaseId); - const dest = await snapshotBase(client, destBaseId); +export async function plan({ client, sourceBaseId, destBaseId, planId, direction = 'to-dest' }) { + // When direction='to-source', swap so the changeset targets the original SOURCE base. + const effectiveSrcId = direction === 'to-source' ? destBaseId : sourceBaseId; + const effectiveDestId = direction === 'to-source' ? sourceBaseId : destBaseId; + + const src = await snapshotBase(client, effectiveSrcId); + const dest = await snapshotBase(client, effectiveDestId); const idmap = matchByName(src, dest); const base = computePlan(src, dest, idmap); const fullPlan = { @@ -44,11 +54,11 @@ export async function plan({ client, sourceBaseId, destBaseId, planId }) { destFingerprint: fingerprintSchema(dest), ...base, }; - saveIdmap(sourceBaseId, destBaseId, idmap); - savePlan(sourceBaseId, destBaseId, fullPlan); - saveState(sourceBaseId, destBaseId, { - sourceBaseId, - destBaseId, + saveIdmap(effectiveSrcId, effectiveDestId, idmap); + savePlan(effectiveSrcId, effectiveDestId, fullPlan); + saveState(effectiveSrcId, effectiveDestId, { + sourceBaseId: effectiveSrcId, + destBaseId: effectiveDestId, engineVersion: ENGINE_VERSION, lastPlanId: planId, }); diff --git a/packages/mcp-server/src/sync/report.js b/packages/mcp-server/src/sync/report.js index af6e277..f1e4c0e 100644 --- a/packages/mcp-server/src/sync/report.js +++ b/packages/mcp-server/src/sync/report.js @@ -5,9 +5,18 @@ /** Maximum number of drift entries to include in a digest driftSample. */ export const DRIFT_SAMPLE_CAP = 25; +/** Maximum number of changeset entries to include in the plan sample digest. */ +export const CHANGESET_SAMPLE_CAP = DRIFT_SAMPLE_CAP; // reuse the same 25 limit + /** * Render a plan into a human-readable summary and a machine-readable copy. * + * When actions carry annotated `changeId` fields (added by Task 9), the returned + * `machine` object gains a `sample` array (capped at CHANGESET_SAMPLE_CAP) where + * each entry is `{ changeId, op, table, target }` derived by splitting the changeId + * on `"|"`. The `human` string also includes an `editHint` telling the caller how to + * selectively skip actions via `apply:false` or the `skip:[changeId]` param. + * * @param {{ actions: object[], orphans: object[], warnings: object[] }} plan * @returns {{ human: string, machine: object }} */ @@ -23,6 +32,27 @@ export function renderPlan(plan) { lines.push(' warnings:'); for (const w of plan.warnings) lines.push(` - ${w.code}: ${w.message}`); } + + // Build changeset sample from annotated actions (Task 10). + // Only include actions that carry a changeId (added by Task 9's computePlan annotation). + const sample = []; + for (const a of plan.actions) { + if (!a.changeId) continue; + if (sample.length >= CHANGESET_SAMPLE_CAP) break; + const [op = '', table = '', target = ''] = a.changeId.split('|'); + sample.push({ changeId: a.changeId, op, table, target }); + } + + // Attach sample to plan in-place so machine === plan still holds. + plan.sample = sample; + + // editHint: only emit when there are annotated actions. + const editHint = sample.length > 0 + ? `To skip individual changes, set apply:false on entries or pass skip:[changeId,...] to mode=apply.` + : null; + + if (editHint) lines.push(editHint); + return { human: lines.join('\n'), machine: plan }; } diff --git a/packages/mcp-server/test/sync/test-index.test.js b/packages/mcp-server/test/sync/test-index.test.js index 9c8e4e5..250e3ba 100644 --- a/packages/mcp-server/test/sync/test-index.test.js +++ b/packages/mcp-server/test/sync/test-index.test.js @@ -21,3 +21,52 @@ describe('sync index.plan', () => { assert.equal(fingerprintSchema(a), fingerprintSchema(b)); }); }); + +describe('sync index.plan — direction param (Task 10)', () => { + // When direction='to-source', the src/dest roles are SWAPPED: dest is snapshotted as + // "source" and source is snapshotted as "dest". So the changeset targets the SOURCE base. + // We verify this by checking which base the mock client was asked about for each role: + // With direction='to-source', appDDDDDDDDDDDDDD (the original dest) should be the "source" + // that has the table, and appSSSSSSSSSSSSSS (the original source) should be the empty "dest". + it('direction="to-dest" (default) — createTable targets destBase', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-dir-')); + const mk = (tableName) => ({ data: { tableSchemas: [{ id: 'tbl1', name: tableName, primaryColumnId: 'fld1', columns: [{ id: 'fld1', name: 'Name', type: 'text' }] }] } }); + // Source (appSSSS) has a table; dest (appDDDD) is empty. + const client = { getApplicationData: async (appId) => (appId === 'appSSSSSSSSSSSSSS' ? mk('TableInSrc') : { data: { tableSchemas: [] } }) }; + const out = await computeSyncPlan({ client, sourceBaseId: 'appSSSSSSSSSSSSSS', destBaseId: 'appDDDDDDDDDDDDDD', planId: 'plnDirDest', direction: 'to-dest' }); + // Plan should create the table (because src has it, dest doesn't) + assert.match(out.human, /createTable: 1/, 'direction=to-dest: createTable should appear for dest'); + // The sample entry should name the table from source + const sample = out.machine.sample || []; + const ct = sample.find((e) => e.op === 'createTable'); + assert.ok(ct, 'sample should have a createTable entry'); + assert.equal(ct.table, 'TableInSrc', 'sample entry should reference the source table name'); + }); + + it('direction="to-source" — swaps src/dest: createTable targets sourceBase', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-dir-src-')); + const mk = (tableName) => ({ data: { tableSchemas: [{ id: 'tbl1', name: tableName, primaryColumnId: 'fld1', columns: [{ id: 'fld1', name: 'Name', type: 'text' }] }] } }); + // Dest (appDDDD) has a table; source (appSSSS) is empty. + // With direction='to-source', roles swap: dest becomes "source", source becomes "dest". + // So the plan should create the table in appSSSSSSSSSSSSSS (the original source = swapped dest). + const client = { getApplicationData: async (appId) => (appId === 'appDDDDDDDDDDDDDD' ? mk('TableInDest') : { data: { tableSchemas: [] } }) }; + const out = await computeSyncPlan({ client, sourceBaseId: 'appSSSSSSSSSSSSSS', destBaseId: 'appDDDDDDDDDDDDDD', planId: 'plnDirSrc', direction: 'to-source' }); + // With the swap: dest (which has the table) is treated as source → plan creates it in original source. + assert.match(out.human, /createTable: 1/, 'direction=to-source: createTable should appear'); + // The sample entry should name the table from dest (swapped to be source) + const sample = out.machine.sample || []; + const ct = sample.find((e) => e.op === 'createTable'); + assert.ok(ct, 'sample should have a createTable entry'); + assert.equal(ct.table, 'TableInDest', 'sample entry should reference the table from swapped source (original dest)'); + // The plan file should be saved with swapped IDs (dest as src, src as dest) + assert.ok(existsSync(join(syncDir('appDDDDDDDDDDDDDD', 'appSSSSSSSSSSSSSS'), 'plan-plnDirSrc.json')), 'plan saved under swapped dir'); + }); + + it('direction defaults to "to-dest" when omitted', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-dir-def-')); + const mk = (tableName) => ({ data: { tableSchemas: [{ id: 'tbl1', name: tableName, primaryColumnId: 'fld1', columns: [{ id: 'fld1', name: 'Name', type: 'text' }] }] } }); + const client = { getApplicationData: async (appId) => (appId === 'appSSSSSSSSSSSSSS' ? mk('DefaultTable') : { data: { tableSchemas: [] } }) }; + const out = await computeSyncPlan({ client, sourceBaseId: 'appSSSSSSSSSSSSSS', destBaseId: 'appDDDDDDDDDDDDDD', planId: 'plnDefault' }); + assert.match(out.human, /createTable: 1/); + }); +}); diff --git a/packages/mcp-server/test/sync/test-report.test.js b/packages/mcp-server/test/sync/test-report.test.js index e114852..5c70f2d 100644 --- a/packages/mcp-server/test/sync/test-report.test.js +++ b/packages/mcp-server/test/sync/test-report.test.js @@ -276,3 +276,100 @@ describe('renderDiff — detail mode', () => { assert.equal(machine.entries.length, 18); }); }); + +// ── renderPlan — changeset digest (Task 10) ────────────────────────────────── + +/** + * Build a plan with annotated actions (changeId/class/apply — from Task 9). + * Uses more than 25 actions so we can verify the cap. + */ +function buildAnnotatedPlan(actionCount = 30) { + const actions = Array.from({ length: actionCount }, (_, i) => ({ + kind: 'createField', + sourceTableId: 'tSrc', + name: `Field${i}`, + changeId: `createField|MyTable|Field${i}`, + class: 'drift', + apply: true, + })); + return { + planId: 'plnTEST', + actions, + orphans: [], + warnings: [], + }; +} + +describe('renderPlan — changeset digest sample (Task 10)', () => { + it('machine has a sample array when actions carry changeId', () => { + const plan = buildAnnotatedPlan(10); + const { machine } = renderPlan(plan); + assert.ok(Array.isArray(machine.sample), 'machine.sample should be an array'); + }); + + it('sample length is capped at DRIFT_SAMPLE_CAP (25)', () => { + const plan = buildAnnotatedPlan(30); + const { machine } = renderPlan(plan); + assert.ok(machine.sample.length <= DRIFT_SAMPLE_CAP, `sample.length (${machine.sample.length}) must be <= 25`); + assert.equal(machine.sample.length, 25); + }); + + it('sample length equals action count when actions < cap', () => { + const plan = buildAnnotatedPlan(10); + const { machine } = renderPlan(plan); + assert.equal(machine.sample.length, 10); + }); + + it('each sample entry has changeId, op, table, target', () => { + const plan = buildAnnotatedPlan(5); + const { machine } = renderPlan(plan); + for (const entry of machine.sample) { + assert.ok('changeId' in entry, 'entry must have changeId'); + assert.ok('op' in entry, 'entry must have op'); + assert.ok('table' in entry, 'entry must have table'); + assert.ok('target' in entry, 'entry must have target'); + } + }); + + it('sample entries derive op/table/target from changeId', () => { + const plan = buildAnnotatedPlan(1); + const { machine } = renderPlan(plan); + const entry = machine.sample[0]; + assert.equal(entry.changeId, 'createField|MyTable|Field0'); + assert.equal(entry.op, 'createField'); + assert.equal(entry.table, 'MyTable'); + assert.equal(entry.target, 'Field0'); + }); + + it('human mentions skip or apply-flag hint', () => { + const plan = buildAnnotatedPlan(5); + const { human } = renderPlan(plan); + const mentionsSkip = /skip/i.test(human); + const mentionsApply = /apply.*false|apply:false/i.test(human); + assert.ok(mentionsSkip || mentionsApply, 'human should mention skip or apply:false hint'); + }); + + it('machine.sample is an empty array when no actions carry changeId', () => { + const plan = { + actions: [{ kind: 'createTable', name: 'T' }], + orphans: [], + warnings: [], + }; + const { machine } = renderPlan(plan); + assert.ok(Array.isArray(machine.sample), 'machine.sample should be an array even when empty'); + assert.equal(machine.sample.length, 0); + }); + + it('existing renderPlan machine reference stays intact', () => { + // The original test: machine === plan (same object reference). + // After Task 10, machine IS the plan object (mutated with .sample). + const plan = { + actions: [{ kind: 'createTable', name: 'T' }], + orphans: [], + warnings: [], + }; + const { machine } = renderPlan(plan); + // machine must be === plan (we add .sample in-place, not wrap) + assert.equal(machine, plan, 'machine should still be === plan'); + }); +}); From 997ab9045e90abe50306c69c5f8c65a1c6296e29 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Fri, 19 Jun 2026 18:02:00 +0300 Subject: [PATCH 094/246] feat(sync): journal-safe selective apply (skip-list + apply:false) --- packages/mcp-server/src/sync/apply.js | 4 +- .../test/sync/test-selective-apply.test.js | 78 +++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 packages/mcp-server/test/sync/test-selective-apply.test.js diff --git a/packages/mcp-server/src/sync/apply.js b/packages/mcp-server/src/sync/apply.js index a329f29..2a8f187 100644 --- a/packages/mcp-server/src/sync/apply.js +++ b/packages/mcp-server/src/sync/apply.js @@ -96,15 +96,17 @@ function mergeChoices(destField, srcTypeOptions) { return { ...srcTypeOptions, choices: merged }; } -export async function applyPlan({ client, plan, destAppId, destSnapshot, idmap, journal, persist }) { +export async function applyPlan({ client, plan, destAppId, destSnapshot, idmap, journal, persist, skip = [] }) { const index = buildIndex(destSnapshot); const state = { createdLinks: new Map(), adoptedReverse: new Set() }; if (!idmap.views) idmap.views = {}; const result = { planId: plan.planId, aborted: false, created: 0, updated: 0, skipped: 0, failed: 0, warnings: [], idmap }; + const skipSet = new Set(skip); for (let idx = 0; idx < plan.actions.length; idx++) { const a = plan.actions[idx]; if (isDone(journal, idx)) { result.skipped++; continue; } + if (a.apply === false || skipSet.has(a.changeId)) { result.skipped++; continue; } try { await applyAction({ client, destAppId, a, idmap, index, state, result }); recordDone(journal, idx, a.kind, idmap.tables[a.sourceTableId] ?? (a.sourceFieldId && idmap.fields[a.sourceFieldId]?.destFld)); diff --git a/packages/mcp-server/test/sync/test-selective-apply.test.js b/packages/mcp-server/test/sync/test-selective-apply.test.js new file mode 100644 index 0000000..4a41668 --- /dev/null +++ b/packages/mcp-server/test/sync/test-selective-apply.test.js @@ -0,0 +1,78 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { MockClient } from './helpers/mock-client.js'; +import { applyPlan } from '../../src/sync/apply.js'; +import { newJournal } from '../../src/sync/journal.js'; +import { snapshotBase } from '../../src/sync/snapshot.js'; + +describe('applyPlan: journal-safe selective-apply skip guard', () => { + it('skips actions by changeId (skip-list) and by apply:false, without filtering plan.actions', async () => { + const client = new MockClient(); + const destSnapshot = await snapshotBase(client, 'appD'); // empty base + + const plan = { + planId: 'p-sel', sourceBaseId: 'appS', destBaseId: 'appD', + idmap: { tables: {}, fields: {}, views: {} }, + actions: [ + // A: apply:true, not in skip → should be applied (createTable) + { kind: 'createTable', sourceTableId: 't1', name: 'A', changeId: 'createTable|A|A', apply: true }, + // B: apply:true, IN skip list → should be skipped + { kind: 'createTable', sourceTableId: 't2', name: 'B', changeId: 'createTable|B|B', apply: true }, + // C: apply:false → should be skipped + { kind: 'createTable', sourceTableId: 't3', name: 'C', changeId: 'createTable|C|C', apply: false }, + ], + orphans: [], warnings: [], + }; + const idmap = { tables: {}, fields: {}, views: {} }; + const res = await applyPlan({ + client, plan, destAppId: 'appD', destSnapshot, idmap, + journal: newJournal('p-sel', 'ts'), + persist: () => {}, + skip: ['createTable|B|B'], + }); + + // A applied, B skipped (skip-list), C skipped (apply:false) + assert.equal(res.skipped, 2, `expected 2 skipped, got ${res.skipped}`); + assert.equal(res.created, 1, `expected 1 created (only A), got ${res.created}`); + assert.equal(res.failed, 0, `expected 0 failed, got ${res.failed}`); + + // A's idmap entry must exist + assert.ok(res.idmap.tables.t1, 'A (t1) must be in idmap.tables'); + // B and C must NOT be in idmap (never applied) + assert.equal(res.idmap.tables.t2, undefined, 'B (t2) must NOT be in idmap.tables'); + assert.equal(res.idmap.tables.t3, undefined, 'C (t3) must NOT be in idmap.tables'); + + // Only table A should be created in the mock + const tables = (await client.getApplicationData('appD')).data.tableSchemas; + const names = tables.map((t) => t.name).sort(); + assert.deepEqual(names, ['A'], `expected only table A in dest, got ${JSON.stringify(names)}`); + }); + + it('default skip=[] and apply!==false means existing behavior is unchanged', async () => { + const client = new MockClient(); + const destSnapshot = await snapshotBase(client, 'appD'); + + const plan = { + planId: 'p-compat', sourceBaseId: 'appS', destBaseId: 'appD', + idmap: { tables: {}, fields: {}, views: {} }, + actions: [ + // No apply field, no changeId — mirrors old plan format + { kind: 'createTable', sourceTableId: 'tX', name: 'X' }, + { kind: 'createTable', sourceTableId: 'tY', name: 'Y' }, + ], + orphans: [], warnings: [], + }; + const idmap = { tables: {}, fields: {}, views: {} }; + // Call WITHOUT skip param — must still work + const res = await applyPlan({ + client, plan, destAppId: 'appD', destSnapshot, idmap, + journal: newJournal('p-compat', 'ts'), + persist: () => {}, + }); + + assert.equal(res.created, 2, `expected 2 created, got ${res.created}`); + assert.equal(res.failed, 0); + assert.ok(res.idmap.tables.tX); + assert.ok(res.idmap.tables.tY); + }); +}); From c7098fcf716c9c6e3e684b75af07078dfc1c0860 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Fri, 19 Jun 2026 18:06:45 +0300 Subject: [PATCH 095/246] feat(sync): thread skip-list through apply() + sync_base MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `apply({...,skip=[]})` in src/sync/index.js now accepts a skip list and forwards it to `applyPlan({...,skip})` so per-changeId skip-guarding works end-to-end from the engine level. - `sync_base` tool (src/index.js): adds `skip` array property to inputSchema; handler destructures it and passes `skip: skip || []` into `sync.apply()`. - New engine-level test in test-apply-index.test.js: apply() with skip:[changeId] produces created:1/skipped:1 and omits the skipped table from the dest; all 578 suite tests green; tool-sync ✓. Co-Authored-By: Claude Sonnet 4.6 --- packages/mcp-server/src/index.js | 5 ++- packages/mcp-server/src/sync/index.js | 3 +- .../test/sync/test-apply-index.test.js | 39 +++++++++++++++++++ 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/packages/mcp-server/src/index.js b/packages/mcp-server/src/index.js index 4adf0a1..07cdbbc 100644 --- a/packages/mcp-server/src/index.js +++ b/packages/mcp-server/src/index.js @@ -1593,6 +1593,7 @@ Note: "form title" is the view name itself — use rename_view to change it. "Fi offset: { type: 'number', description: 'Used by mode="diff" with detail: skip this many entries before returning results.' }, limit: { type: 'number', description: 'Used by mode="diff" with detail: maximum number of entries to return.' }, direction: { type: 'string', enum: ['to-dest', 'to-source'], description: 'Used only by mode="plan": direction of the changeset. "to-dest" (default) brings the destination base up to date with the source. "to-source" swaps the roles so the changeset targets the source base (makes the source match the destination).' }, + skip: { type: 'array', items: { type: 'string' }, description: 'Used only by mode="apply": list of changeIds to skip (actions with a matching changeId are counted as skipped but not applied). Use changeIds from the plan output.' }, debug: debugProp, }, required: ['mode', 'sourceAppId', 'destAppId'], @@ -2394,7 +2395,7 @@ const handlers = { // ── Sync ── - async sync_base({ mode, sourceAppId, destAppId, planId, naturalKeys, detail, diffId, offset, limit, direction, debug }) { + async sync_base({ mode, sourceAppId, destAppId, planId, naturalKeys, detail, diffId, offset, limit, direction, skip, debug }) { const sync = await import('./sync/index.js'); if (mode === 'plan') { const id = 'pln' + client._genRandomId(); @@ -2404,7 +2405,7 @@ const handlers = { if (mode === 'apply') { if (!planId) return err('mode="apply" requires planId (from a prior mode="plan" run).'); const runStartedAt = new Date().toISOString(); - const out = await sync.apply({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, planId, runStartedAt }); + const out = await sync.apply({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, planId, runStartedAt, skip: skip || [] }); return ok({ planId, summary: out.human }, out.machine, debug); } if (mode === 'reconcile') { diff --git a/packages/mcp-server/src/sync/index.js b/packages/mcp-server/src/sync/index.js index a148175..bc4b867 100644 --- a/packages/mcp-server/src/sync/index.js +++ b/packages/mcp-server/src/sync/index.js @@ -101,7 +101,7 @@ export function diffDetail({ sourceBaseId, destBaseId, diffId, detail, offset, l return renderDiff(savedDiff, { detail, offset, limit }); } -export async function apply({ client, sourceBaseId, destBaseId, planId, runStartedAt }) { +export async function apply({ client, sourceBaseId, destBaseId, planId, runStartedAt, skip = [] }) { const fullPlan = loadPlan(sourceBaseId, destBaseId, planId); if (!fullPlan) throw new Error(`No saved plan "${planId}" for ${sourceBaseId} -> ${destBaseId}. Run mode=plan first.`); @@ -119,6 +119,7 @@ export async function apply({ client, sourceBaseId, destBaseId, planId, runStart const result = await applyPlan({ client, plan: fullPlan, destAppId: destBaseId, destSnapshot, idmap, journal, persist: (m, j) => { saveIdmap(sourceBaseId, destBaseId, m); saveJournal(sourceBaseId, destBaseId, j); }, + skip, }); // Records phase: runs after schema apply, only if not aborted. It is minutes-long for large diff --git a/packages/mcp-server/test/sync/test-apply-index.test.js b/packages/mcp-server/test/sync/test-apply-index.test.js index d82eabb..70b0f3a 100644 --- a/packages/mcp-server/test/sync/test-apply-index.test.js +++ b/packages/mcp-server/test/sync/test-apply-index.test.js @@ -62,3 +62,42 @@ describe('index: views', () => { assert.ok(views.some((v) => v.name === 'Board')); }); }); + +describe('index.apply: skip forwarding', () => { + it('apply() with skip:[changeId] forwards to applyPlan so the skipped action is not applied', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'apply-skip-')); + const client = new MockClient(); + const dest = await snapshotBase(client, 'appDDDDDDDDDDDDDD'); + const plan = { + planId: 'plnSKIP', + engineVersion: '2b', + destFingerprint: fingerprintSchema(dest), + sourceBaseId: 'appSSSSSSSSSSSSSS', + destBaseId: 'appDDDDDDDDDDDDDD', + idmap: { tables: {}, fields: {}, views: {} }, + actions: [ + { kind: 'createTable', sourceTableId: 'tA', name: 'TableA', changeId: 'createTable|A|A', apply: true }, + { kind: 'createTable', sourceTableId: 'tB', name: 'TableB', changeId: 'createTable|B|B', apply: true }, + ], + orphans: [], + warnings: [], + }; + savePlan('appSSSSSSSSSSSSSS', 'appDDDDDDDDDDDDDD', plan); + // Pass skip: ['createTable|B|B'] — TableB should be skipped + const out = await apply({ + client, + sourceBaseId: 'appSSSSSSSSSSSSSS', + destBaseId: 'appDDDDDDDDDDDDDD', + planId: 'plnSKIP', + runStartedAt: 'ts', + skip: ['createTable|B|B'], + }); + // One created (A), one skipped (B) + assert.match(out.human, /created: 1/, `expected created:1 in: ${out.human}`); + assert.match(out.human, /skipped: 1/, `expected skipped:1 in: ${out.human}`); + // Only TableA should be in the dest + const tables = (await client.getApplicationData('appDDDDDDDDDDDDDD')).data.tableSchemas; + const names = tables.map((t) => t.name).sort(); + assert.deepEqual(names, ['TableA'], `expected only TableA in dest, got ${JSON.stringify(names)}`); + }); +}); From 06f970a4e024e9efd1d16ce519b7fa13141f6a59 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Fri, 19 Jun 2026 18:11:32 +0300 Subject: [PATCH 096/246] docs(sync): compare/changeset/selective-apply Document mode=diff (read-only classified compare, digest + detail drill-in, sections now captured as not-synced), the direction param on mode=plan (to-dest default | to-source), the curatable changeset (changeId/class/apply), and the skip:[changeId] selective-apply on mode=apply. Co-Authored-By: Claude Sonnet 4.6 --- CHANGELOG.md | 7 +++++++ CLAUDE.md | 2 +- packages/mcp-server/README.md | 7 ++++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 072097d..b95f614 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how ## [Unreleased] +### MCP server — sync_base compare, curatable changeset, selective apply (2026-06-19) + +#### Added +- `sync_base mode=diff` (read-only): comprehensive schema comparison of two bases via a new pure `src/sync/compare.js`. Every difference is classified as **drift** (sync enforces it — table/field/view existence, field type/options/choices, view filters/sorts/groups content, column visibility, colors/cover/frozen/rowHeight, primary, form), **best-effort** (sync applies but doesn't guarantee — field/view/column order, sort/group clause order), or **not-synced** (view sections, now captured by the snapshot). Two verdicts: `identical` (zero diffs) and `converged` (zero drift; best-effort/not-synced may remain). The full diff is persisted to `~/.airtable-user-mcp/sync/__/diff-.json`; the tool returns a token-budgeted DIGEST (verdicts + class counts + per-table count rollup + a capped driftSample, ≤25). Drill into one table via `mode=diff detail=""` (params: `detail`, `diffId`, `offset`, `limit`). +- `sync_base mode=plan` now emits a curatable **changeset**: each action carries a stable name-based `changeId` (`|
|`), a `class`, and an `apply:true` flag; the digest includes a `sample` + an `editHint`. New `direction` param (`to-dest` default | `to-source`) targets either base as the destination. +- `sync_base mode=apply` accepts a `skip:[changeId]` list (and honors `apply:false` on changeset entries) → applies everything except the removed changes. Journal-safe; skipped dependencies degrade gracefully to UNRESOLVABLE_REF. + ### MCP server — sync_base record sync (2026-06-17) #### Added diff --git a/CLAUDE.md b/CLAUDE.md index 37b71db..ea74d29 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,7 +65,7 @@ Key files: - `dev-tools/` — reverse-engineering helpers (capture.js, analyze.js, debug-*.js), **gitignored** — kept locally for future reference when Airtable changes their internal API - `src/daemon/` — daemon subsystem (see daemon subsection below) - `src/safe-write.js` — atomic JSON write helper (used by lockfile, token, and settings) -- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: plan | apply | status | reconcile`) in the `sync` category. Out of scope: retypes, deletions (M4). View sync: `snapshot` also captures collaborative views + live config; `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe by filename+size); then restores record-referencing view filters stripped during view sync. Throttle handling = per-request 429 backoff in the auth queue (the per-call limiter does NOT gate the internal per-row POSTs inside createRecords/updateRecords/addLinkItems); resumable (records journal, per-chunk persist of ~50 creates); continue-on-failure (per-record errors are warnings). `mode=status` polls a background job by planId — returns running/done/failed + live records-mapped count. `mode=reconcile` **prunes** dead record-map entries (existence-prune; per-table natural-key re-match NOT yet implemented; reconcile does NOT de-duplicate records). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. +- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: diff | plan | apply | status | reconcile`) in the `sync` category. Out of scope: retypes, deletions (M4). `mode=diff` (read-only): comprehensive schema comparison via pure `src/sync/compare.js` — every difference classified as **drift** (sync enforces: table/field/view existence, field type/options/choices, view filters/sorts/groups content, column visibility, colors/cover/frozen/rowHeight, primary, form), **best-effort** (sync applies but doesn't guarantee: field/view/column order, sort/group clause order), or **not-synced** (view sections — now captured by snapshot). Verdicts: `identical` (zero diffs) or `converged` (zero drift; best-effort/not-synced may remain). Persists full diff to `~/.airtable-user-mcp/sync/__/diff-.json`; returns a token-budgeted DIGEST (verdicts + class counts + per-table rollup + capped driftSample ≤25). Drill in with `detail="
"` (params: `detail`, `diffId`, `offset`, `limit`). `mode=plan` now emits a curatable **changeset**: each action carries a stable name-based `changeId` (`|
|`), a `class`, and an `apply:true` flag; digest includes a `sample` + `editHint`. New `direction` param (`to-dest` default | `to-source`) targets either base. View sync: `snapshot` also captures collaborative views + live config + **view sections** (not-synced class); `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported); accepts a `skip:[changeId]` list (and honors `apply:false` on changeset entries) — applies everything except removed changes (journal-safe; skipped dependencies degrade to UNRESOLVABLE_REF). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe by filename+size); then restores record-referencing view filters stripped during view sync. Throttle handling = per-request 429 backoff in the auth queue (the per-call limiter does NOT gate the internal per-row POSTs inside createRecords/updateRecords/addLinkItems); resumable (records journal, per-chunk persist of ~50 creates); continue-on-failure (per-record errors are warnings). `mode=status` polls a background job by planId — returns running/done/failed + live records-mapped count. `mode=reconcile` **prunes** dead record-map entries (existence-prune; per-table natural-key re-match NOT yet implemented; reconcile does NOT de-duplicate records). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json, diff-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile), `compare.js` (pure classified compare, no I/O); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. #### packages/mcp-server — Daemon subsystem diff --git a/packages/mcp-server/README.md b/packages/mcp-server/README.md index a336edf..6a2ee98 100644 --- a/packages/mcp-server/README.md +++ b/packages/mcp-server/README.md @@ -523,7 +523,12 @@ Saved row scaffolds Airtable surfaces under "+ Add record" and the row-create ex | Tool | Description | |:-----|:------------| -| `sync_base` | Copy a base's schema, views, and records to another base. `mode=plan` snapshots both bases and produces a diff report (no writes). `mode=apply` executes a saved plan — creates tables, reconciles the primary field, creates scalar/link/computed fields with source→dest reference remapping and formula validation, applies non-destructive field updates, syncs collaborative views (idempotent; personal views skipped; orphan views reported), then launches **background** record sync and returns a `jobId` immediately. Record sync: Pass 1 writes scalar + select cells (computed fields never written) and builds a persisted rec→rec id map; Pass 2 writes linked-record cells (unresolved targets reported); then attachments (download→re-upload, deduped by filename+size); then restores record-referencing view filters. Throttling via per-request 429 backoff; resumable (records journal, per-chunk persist); continue-on-failure. `mode=status` polls the background job by planId (running/done/failed + live count). `mode=reconcile` **prunes** the record id map (existence-prune; natural-key re-match planned; does not de-duplicate). Drift-guarded and resumable via an on-disk journal. Out of scope: type-changing retypes, deletions. | +| `sync_base` | Copy a base's schema, views, and records to another base. `mode=diff` (read-only) compares two bases and classifies every difference as **drift** (sync enforces), **best-effort** (sync applies, not guaranteed), or **not-synced** (view sections). Returns a token-budgeted digest (verdicts + class counts + driftSample); drill in with `detail="
"`. Verdicts: `identical` or `converged`. `mode=plan` snapshots both bases and produces a curatable **changeset** — each action has a stable `changeId` (`\|
\|`), a `class`, and an `apply:true` flag. New `direction` param (`to-dest` default \| `to-source`) selects which base is written. `mode=apply` executes a saved plan — creates tables, reconciles the primary field, creates scalar/link/computed fields with source→dest reference remapping and formula validation, applies non-destructive field updates, syncs collaborative views (idempotent; personal views skipped; orphan views reported), then launches **background** record sync and returns a `jobId` immediately. Accepts a `skip:[changeId]` list (or `apply:false` entries in the changeset) to exclude specific changes; skipped dependencies degrade to UNRESOLVABLE_REF. Record sync: Pass 1 writes scalar + select cells (computed fields never written) and builds a persisted rec→rec id map; Pass 2 writes linked-record cells (unresolved targets reported); then attachments (download→re-upload, deduped by filename+size); then restores record-referencing view filters. Throttling via per-request 429 backoff; resumable (records journal, per-chunk persist); continue-on-failure. `mode=status` polls the background job by planId (running/done/failed + live count). `mode=reconcile` **prunes** the record id map (existence-prune; natural-key re-match planned; does not de-duplicate). Drift-guarded and resumable via an on-disk journal. Out of scope: type-changing retypes, deletions. | + +**Typical diff → curate → apply workflow:** +1. `mode=diff` — compare bases, review the digest. Verdict `converged` means no drift; `identical` means nothing to do. +2. `mode=plan` — generate the full changeset. Edit `apply:false` on any `changeId` entries you want to skip, or collect their IDs. +3. `mode=apply skip=[...]` — execute the plan minus excluded changes. --- From b05f0ac907e2e738d41c19a29b31cc65f2ec8442 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Fri, 19 Jun 2026 18:20:01 +0300 Subject: [PATCH 097/246] chore(sync): drop dead import; document to-source apply base-swap Remove unused `stableStringify` from compare.js import (it's defined and used in field-compare.js and diff.js, not needed here). Add a sentence to README clarifying that a to-source plan is persisted under the swapped pair and must be applied with bases swapped accordingly. Co-Authored-By: Claude Sonnet 4.6 --- packages/mcp-server/README.md | 2 +- packages/mcp-server/src/sync/compare.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/mcp-server/README.md b/packages/mcp-server/README.md index 6a2ee98..fbcf44e 100644 --- a/packages/mcp-server/README.md +++ b/packages/mcp-server/README.md @@ -527,7 +527,7 @@ Saved row scaffolds Airtable surfaces under "+ Add record" and the row-create ex **Typical diff → curate → apply workflow:** 1. `mode=diff` — compare bases, review the digest. Verdict `converged` means no drift; `identical` means nothing to do. -2. `mode=plan` — generate the full changeset. Edit `apply:false` on any `changeId` entries you want to skip, or collect their IDs. +2. `mode=plan` — generate the full changeset. Edit `apply:false` on any `changeId` entries you want to skip, or collect their IDs. When `direction=to-source`, the plan is persisted under the swapped base pair, so applying it requires calling `mode=apply` with `sourceAppId` and `destAppId` swapped accordingly. 3. `mode=apply skip=[...]` — execute the plan minus excluded changes. --- diff --git a/packages/mcp-server/src/sync/compare.js b/packages/mcp-server/src/sync/compare.js index a5de8e9..e6708ac 100644 --- a/packages/mcp-server/src/sync/compare.js +++ b/packages/mcp-server/src/sync/compare.js @@ -1,6 +1,6 @@ // compare.js — base-schema comparator helpers (pure: no fs, no network, no Date.now/Math.random) -import { stableStringify, choiceNames, scalarTypeOptionsChanged, computedSig } from './field-compare.js'; +import { choiceNames, scalarTypeOptionsChanged, computedSig } from './field-compare.js'; import { canonicalizeViewConfig } from './remap.js'; const BEST_EFFORT = new Set(['fieldOrder', 'viewOrder', 'columnOrder', 'sortOrder', 'groupOrder']); From 7c96c75c4e884ddb7191d9386938c9bdc59e3db2 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Fri, 19 Jun 2026 19:22:27 +0300 Subject: [PATCH 098/246] fix(sync): compare canonicalizes cross-base computed/link refs (no false drift) --- packages/mcp-server/src/sync/compare.js | 68 +++- .../mcp-server/test/sync/test-compare.test.js | 304 ++++++++++++++++++ 2 files changed, 365 insertions(+), 7 deletions(-) diff --git a/packages/mcp-server/src/sync/compare.js b/packages/mcp-server/src/sync/compare.js index e6708ac..09a469b 100644 --- a/packages/mcp-server/src/sync/compare.js +++ b/packages/mcp-server/src/sync/compare.js @@ -33,6 +33,31 @@ function tableFieldNameMap(table) { return m; } +/** + * Build a flat map of { fieldId → fieldName } across ALL tables in a snapshot. + * Mirrors diff.js's fldNameMap so computed sigs for cross-table refs (e.g. rollup's + * foreignTableRollupColumnId) resolve correctly. + * @param {{ tables: Array<{fields: Array<{id:string, name:string}>}> }} snap + * @returns {Record} + */ +function snapFldNames(snap) { + const m = {}; + for (const t of snap.tables) for (const f of t.fields) m[f.id] = f.name; + return m; +} + +/** + * Build a flat map of { tableId → tableName } across all tables in a snapshot. + * Used to canonicalize link field foreignTableId references for cross-base comparison. + * @param {{ tables: Array<{id:string, name:string}> }} snap + * @returns {Record} + */ +function snapTblNames(snap) { + const m = {}; + for (const t of snap.tables) m[t.id] = t.name; + return m; +} + /** * Compare fields between a source table and a destination table using the provided idmap. * @@ -54,7 +79,7 @@ function tableFieldNameMap(table) { * @param {{ fields: Record}> }} idmap * @returns {{ entries: Array<{scope:string, key:string, source:unknown, dest:unknown, class:string}>, onlyInSource: string[], onlyInDest: string[] }} */ -export function compareFields(srcTable, destTable, idmap) { +export function compareFields(srcTable, destTable, idmap, srcGlobalFldNames, destGlobalFldNames, srcGlobalTblNames, destGlobalTblNames) { const entries = []; const onlyInSource = []; const onlyInDest = []; @@ -66,9 +91,13 @@ export function compareFields(srcTable, destTable, idmap) { // if two src fields share the same name in an edge case). const matchedDestIds = new Set(); - // Build id→name maps for each side so computed sigs can normalise references. - const srcFldNames = tableFieldNameMap(srcTable); - const destFldNames = tableFieldNameMap(destTable); + // Build id→name maps for computed sig resolution. + // Use snapshot-global maps (passed in from compare()/compareTable()) so that + // cross-table refs (e.g. rollup's foreignTableRollupColumnId targeting a field in + // another table) resolve to a name instead of remaining as a raw id. + // Fall back to table-local if not provided (e.g. direct compareFields() calls in tests). + const srcFldNames = srcGlobalFldNames ?? tableFieldNameMap(srcTable); + const destFldNames = destGlobalFldNames ?? tableFieldNameMap(destTable); // srcMatchedOrder: matched field names in src iteration order (for fieldOrder check). const srcMatchedOrder = []; @@ -109,6 +138,24 @@ export function compareFields(srcTable, destTable, idmap) { if (computedSig(sf, srcFldNames) !== computedSig(df, destFldNames)) { entries.push({ scope, key: 'typeOptions', source: sf.typeOptions, dest: df.typeOptions, class: classOf('typeOptions') }); } + } else if (sf.type === 'foreignKey' || sf.type === 'multipleRecordLinks') { + // Link fields: canonicalize before comparing. + // foreignTableId is base-scoped (always differs cross-base) → resolve to table NAME. + // symmetricColumnId is a base-scoped reciprocal field id → drop it entirely (no semantic + // content that sync controls; it's auto-created by Airtable when the link is made). + const canonLink = (opts, tblNames) => { + if (!opts) return null; + const { symmetricColumnId: _dropped, foreignTableId, ...rest } = opts; // eslint-disable-line no-unused-vars + return JSON.stringify({ + ...rest, + foreignTableName: (tblNames && tblNames[foreignTableId]) ?? foreignTableId ?? null, + }); + }; + const srcTblNames = srcGlobalTblNames ?? {}; + const destTblNames = destGlobalTblNames ?? {}; + if (canonLink(sf.typeOptions, srcTblNames) !== canonLink(df.typeOptions, destTblNames)) { + entries.push({ scope, key: 'typeOptions', source: sf.typeOptions, dest: df.typeOptions, class: classOf('typeOptions') }); + } } else { // Scalar: check choices separately (set-membership) and other typeOptions. const srcChoices = choiceNames(sf.typeOptions); @@ -425,6 +472,13 @@ export function compare(srcSnap, destSnap, idmap) { const destByName = new Map(destSnap.tables.map((t) => [t.name, t])); const matchedDestIds = new Set(); + // Build global field-id→name maps for computed sig resolution across all tables. + const srcGlobalFldNames = snapFldNames(srcSnap); + const destGlobalFldNames = snapFldNames(destSnap); + // Build global table-id→name maps for link field foreignTableId canonicalization. + const srcGlobalTblNames = snapTblNames(srcSnap); + const destGlobalTblNames = snapTblNames(destSnap); + for (const st of srcSnap.tables) { // Resolve dest table: idmap first, then by-name fallback. let dt = null; @@ -438,7 +492,7 @@ export function compare(srcSnap, destSnap, idmap) { } matchedDestIds.add(dt.id); - const tableResult = compareTable(st, dt, idmap); + const tableResult = compareTable(st, dt, idmap, srcGlobalFldNames, destGlobalFldNames, srcGlobalTblNames, destGlobalTblNames); tables.push(tableResult); } @@ -512,7 +566,7 @@ export function compare(srcSnap, destSnap, idmap) { * @param {{ tables:Record, fields:Record}>, views:Record }} idmap * @returns {{ name:string, status:'same'|'differs', entries:Array, fields:{onlyInSource:string[],onlyInDest:string[]}, views:{onlyInSource:string[],onlyInDest:string[]} }} */ -export function compareTable(srcTable, destTable, idmap) { +export function compareTable(srcTable, destTable, idmap, srcGlobalFldNames, destGlobalFldNames, srcGlobalTblNames, destGlobalTblNames) { const entries = []; // 1. Description drift. @@ -530,7 +584,7 @@ export function compareTable(srcTable, destTable, idmap) { } // 3. Field-level comparison. - const fieldResult = compareFields(srcTable, destTable, idmap); + const fieldResult = compareFields(srcTable, destTable, idmap, srcGlobalFldNames, destGlobalFldNames, srcGlobalTblNames, destGlobalTblNames); entries.push(...fieldResult.entries); // 4. View-level comparison. diff --git a/packages/mcp-server/test/sync/test-compare.test.js b/packages/mcp-server/test/sync/test-compare.test.js index aad1b87..e833ae7 100644 --- a/packages/mcp-server/test/sync/test-compare.test.js +++ b/packages/mcp-server/test/sync/test-compare.test.js @@ -395,3 +395,307 @@ describe('compare', () => { assert.equal(result.converged, false, 'has drift → not converged'); }); }); + +// ── Bug fix tests: cross-base computed/link ref canonicalization ────────────── + +describe('compareFields — cross-base ref canonicalization', () => { + // Bug 1: rollup with foreignTableRollupColumnId in ANOTHER table + test('Bug1: rollup referencing field in another table — different raw ids, same name → no typeOptions drift', () => { + // "Revenue" table has a rollup field "Total" that rolls up "Amount" from "Orders" table. + // Src: foreignTableRollupColumnId = 'fldSrcAmount' (field in Orders table, not Revenue) + // Dest: foreignTableRollupColumnId = 'fldDestAmount' (same logical field, different id) + // Both map to the name 'Amount'. With a global fldNames map, computedSig resolves both + // to '{{Amount}}' → sigs match → no drift. + const srcSnap = { + baseId: 'srcBase', + tables: [ + { + id: 'tSrcOrders', name: 'Orders', primaryFieldId: 'fSrcOrderName', + fields: [ + { id: 'fSrcOrderName', name: 'Order Name', type: 'text', typeOptions: null, description: null, isComputed: false }, + { id: 'fSrcAmount', name: 'Amount', type: 'number', typeOptions: { precision: 2 }, description: null, isComputed: false }, + ], + views: [], sections: [], + }, + { + id: 'tSrcRevenue', name: 'Revenue', primaryFieldId: 'fSrcRevName', + fields: [ + { id: 'fSrcRevName', name: 'Name', type: 'text', typeOptions: null, description: null, isComputed: false }, + { id: 'fSrcLink', name: 'Orders Link', type: 'foreignKey', typeOptions: { foreignTableId: 'tSrcOrders', symmetricColumnId: 'fSrcSym', relationship: 'many', unreversed: true }, description: null, isComputed: false }, + { + id: 'fSrcTotal', name: 'Total', type: 'rollup', + typeOptions: { + relationColumnId: 'fSrcLink', + foreignTableRollupColumnId: 'fSrcAmount', // field in ANOTHER table + formulaTextParsed: 'SUM(values)', + }, + description: null, isComputed: true, + }, + ], + views: [], sections: [], + }, + ], + }; + const destSnap = { + baseId: 'destBase', + tables: [ + { + id: 'tDestOrders', name: 'Orders', primaryFieldId: 'fDestOrderName', + fields: [ + { id: 'fDestOrderName', name: 'Order Name', type: 'text', typeOptions: null, description: null, isComputed: false }, + { id: 'fDestAmount', name: 'Amount', type: 'number', typeOptions: { precision: 2 }, description: null, isComputed: false }, + ], + views: [], sections: [], + }, + { + id: 'tDestRevenue', name: 'Revenue', primaryFieldId: 'fDestRevName', + fields: [ + { id: 'fDestRevName', name: 'Name', type: 'text', typeOptions: null, description: null, isComputed: false }, + { id: 'fDestLink', name: 'Orders Link', type: 'foreignKey', typeOptions: { foreignTableId: 'tDestOrders', symmetricColumnId: 'fDestSym', relationship: 'many', unreversed: true }, description: null, isComputed: false }, + { + id: 'fDestTotal', name: 'Total', type: 'rollup', + typeOptions: { + relationColumnId: 'fDestLink', + foreignTableRollupColumnId: 'fDestAmount', // different raw id, same name + formulaTextParsed: 'SUM(values)', + }, + description: null, isComputed: true, + }, + ], + views: [], sections: [], + }, + ], + }; + const idmap = { + tables: { tSrcOrders: 'tDestOrders', tSrcRevenue: 'tDestRevenue' }, + fields: { + fSrcOrderName: { destFld: 'fDestOrderName', choices: {} }, + fSrcAmount: { destFld: 'fDestAmount', choices: {} }, + fSrcRevName: { destFld: 'fDestRevName', choices: {} }, + fSrcLink: { destFld: 'fDestLink', choices: {} }, + fSrcTotal: { destFld: 'fDestTotal', choices: {} }, + }, + views: {}, + }; + // compare() is the top-level entry point that threads snapshots to compareFields + const result = compare(srcSnap, destSnap, idmap); + const typeOptsDrift = result.tables + .flatMap((t) => t.entries) + .filter((e) => e.key === 'typeOptions' && e.scope === 'field:Total'); + assert.deepEqual(typeOptsDrift, [], 'rollup with cross-table foreignTableRollupColumnId must not produce typeOptions drift when target field name is the same'); + }); + + // Bug 2a: link field — same target table name, different raw ids → NO drift + test('Bug2a: link field — same target table name, different foreignTableId → no typeOptions drift', () => { + const srcSnap = { + baseId: 'srcBase', + tables: [ + { + id: 'tSrcProjects', name: 'Projects', primaryFieldId: 'fSrcProjName', + fields: [ + { id: 'fSrcProjName', name: 'Name', type: 'text', typeOptions: null, description: null, isComputed: false }, + ], + views: [], sections: [], + }, + { + id: 'tSrcTasks', name: 'Tasks', primaryFieldId: 'fSrcTaskName', + fields: [ + { id: 'fSrcTaskName', name: 'Task', type: 'text', typeOptions: null, description: null, isComputed: false }, + { + id: 'fSrcLink', name: 'Project', type: 'foreignKey', + typeOptions: { + foreignTableId: 'tSrcProjects', // raw src table id + symmetricColumnId: 'fSrcSymLink', // raw src field id (reciprocal) + relationship: 'many', + unreversed: true, + }, + description: null, isComputed: false, + }, + ], + views: [], sections: [], + }, + ], + }; + const destSnap = { + baseId: 'destBase', + tables: [ + { + id: 'tDestProjects', name: 'Projects', primaryFieldId: 'fDestProjName', + fields: [ + { id: 'fDestProjName', name: 'Name', type: 'text', typeOptions: null, description: null, isComputed: false }, + ], + views: [], sections: [], + }, + { + id: 'tDestTasks', name: 'Tasks', primaryFieldId: 'fDestTaskName', + fields: [ + { id: 'fDestTaskName', name: 'Task', type: 'text', typeOptions: null, description: null, isComputed: false }, + { + id: 'fDestLink', name: 'Project', type: 'foreignKey', + typeOptions: { + foreignTableId: 'tDestProjects', // different raw id but same TABLE NAME + symmetricColumnId: 'fDestSymLink', // different raw id (but irrelevant — dropped) + relationship: 'many', + unreversed: true, + }, + description: null, isComputed: false, + }, + ], + views: [], sections: [], + }, + ], + }; + const idmap = { + tables: { tSrcProjects: 'tDestProjects', tSrcTasks: 'tDestTasks' }, + fields: { + fSrcProjName: { destFld: 'fDestProjName', choices: {} }, + fSrcTaskName: { destFld: 'fDestTaskName', choices: {} }, + fSrcLink: { destFld: 'fDestLink', choices: {} }, + }, + views: {}, + }; + const result = compare(srcSnap, destSnap, idmap); + const typeOptsDrift = result.tables + .flatMap((t) => t.entries) + .filter((e) => e.key === 'typeOptions' && e.scope === 'field:Project'); + assert.deepEqual(typeOptsDrift, [], 'link field pointing to same-named table must not produce typeOptions drift'); + assert.equal(result.converged, true, 'synced link field should be converged'); + }); + + // Bug 2b: link field — genuinely different target table → MUST drift + test('Bug2b: link field — different target table name → typeOptions drift IS emitted', () => { + const srcSnap = { + baseId: 'srcBase', + tables: [ + { + id: 'tSrcProjects', name: 'Projects', primaryFieldId: 'fSrcProjName', + fields: [ + { id: 'fSrcProjName', name: 'Name', type: 'text', typeOptions: null, description: null, isComputed: false }, + ], + views: [], sections: [], + }, + { + id: 'tSrcClients', name: 'Clients', primaryFieldId: 'fSrcClientName', + fields: [ + { id: 'fSrcClientName', name: 'Client', type: 'text', typeOptions: null, description: null, isComputed: false }, + ], + views: [], sections: [], + }, + { + id: 'tSrcTasks', name: 'Tasks', primaryFieldId: 'fSrcTaskName', + fields: [ + { id: 'fSrcTaskName', name: 'Task', type: 'text', typeOptions: null, description: null, isComputed: false }, + { + id: 'fSrcLink', name: 'Related', type: 'foreignKey', + typeOptions: { + foreignTableId: 'tSrcProjects', // links to Projects + symmetricColumnId: 'fSrcSym', + relationship: 'many', + unreversed: true, + }, + description: null, isComputed: false, + }, + ], + views: [], sections: [], + }, + ], + }; + const destSnap = { + baseId: 'destBase', + tables: [ + { + id: 'tDestProjects', name: 'Projects', primaryFieldId: 'fDestProjName', + fields: [ + { id: 'fDestProjName', name: 'Name', type: 'text', typeOptions: null, description: null, isComputed: false }, + ], + views: [], sections: [], + }, + { + id: 'tDestClients', name: 'Clients', primaryFieldId: 'fDestClientName', + fields: [ + { id: 'fDestClientName', name: 'Client', type: 'text', typeOptions: null, description: null, isComputed: false }, + ], + views: [], sections: [], + }, + { + id: 'tDestTasks', name: 'Tasks', primaryFieldId: 'fDestTaskName', + fields: [ + { id: 'fDestTaskName', name: 'Task', type: 'text', typeOptions: null, description: null, isComputed: false }, + { + id: 'fDestLink', name: 'Related', type: 'foreignKey', + typeOptions: { + foreignTableId: 'tDestClients', // links to Clients — DIFFERENT table! + symmetricColumnId: 'fDestSym', + relationship: 'many', + unreversed: true, + }, + description: null, isComputed: false, + }, + ], + views: [], sections: [], + }, + ], + }; + const idmap = { + tables: { tSrcProjects: 'tDestProjects', tSrcClients: 'tDestClients', tSrcTasks: 'tDestTasks' }, + fields: { + fSrcProjName: { destFld: 'fDestProjName', choices: {} }, + fSrcClientName: { destFld: 'fDestClientName', choices: {} }, + fSrcTaskName: { destFld: 'fDestTaskName', choices: {} }, + fSrcLink: { destFld: 'fDestLink', choices: {} }, + }, + views: {}, + }; + const result = compare(srcSnap, destSnap, idmap); + const typeOptsDrift = result.tables + .flatMap((t) => t.entries) + .filter((e) => e.key === 'typeOptions' && e.scope === 'field:Related'); + assert.equal(typeOptsDrift.length, 1, 'link to genuinely different-named table MUST produce typeOptions drift'); + assert.equal(typeOptsDrift[0].class, 'drift'); + }); + + // Regression: existing computed field test still works (table-local refs remain resolved) + test('Regression: intra-table computed field (formula referencing own-table field) still no drift', () => { + const srcSnap = { + baseId: 'srcBase', + tables: [{ + id: 'tSrc', name: 'T', primaryFieldId: 'fSrcName', + fields: [ + { id: 'fSrcName', name: 'Name', type: 'text', typeOptions: null, description: null, isComputed: false }, + { + id: 'fSrcFormula', name: 'ShortName', type: 'formula', + typeOptions: { formulaTextParsed: '{fSrcName}' }, + description: null, isComputed: true, + }, + ], + views: [], sections: [], + }], + }; + const destSnap = { + baseId: 'destBase', + tables: [{ + id: 'tDest', name: 'T', primaryFieldId: 'fDestName', + fields: [ + { id: 'fDestName', name: 'Name', type: 'text', typeOptions: null, description: null, isComputed: false }, + { + id: 'fDestFormula', name: 'ShortName', type: 'formula', + typeOptions: { formulaTextParsed: '{fDestName}' }, + description: null, isComputed: true, + }, + ], + views: [], sections: [], + }], + }; + const idmap = { + tables: { tSrc: 'tDest' }, + fields: { + fSrcName: { destFld: 'fDestName', choices: {} }, + fSrcFormula: { destFld: 'fDestFormula', choices: {} }, + }, + views: {}, + }; + const result = compare(srcSnap, destSnap, idmap); + const typeOptsDrift = result.tables.flatMap((t) => t.entries).filter((e) => e.key === 'typeOptions'); + assert.deepEqual(typeOptsDrift, [], 'intra-table formula must still produce no drift'); + }); +}); From 7754291a18f8bed7db551c04819644e4cdf6396f Mon Sep 17 00:00:00 2001 From: "A.R." Date: Fri, 19 Jun 2026 20:06:04 +0300 Subject: [PATCH 099/246] fix(sync): strip read-only maxUsedAutoNumber so autoNumber fields create autoNumber typeOptions carry a runtime counter (maxUsedAutoNumber) that the internal Airtable API rejects on field creation (422). Strip it in the createField handler before the client call; dest base manages its own auto-numbering sequence. Covers all 5 autoNumber fields that failed in the live sync repro. TDD: RED test confirmed the pass-through, GREEN after the guard. Co-Authored-By: Claude Sonnet 4.6 --- packages/mcp-server/src/sync/apply.js | 6 +++ .../mcp-server/test/sync/test-apply.test.js | 44 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/packages/mcp-server/src/sync/apply.js b/packages/mcp-server/src/sync/apply.js index 2a8f187..47f16cd 100644 --- a/packages/mcp-server/src/sync/apply.js +++ b/packages/mcp-server/src/sync/apply.js @@ -209,6 +209,12 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result } if (!v.valid) throw new Error(`formula invalid: ${v.message ?? v.error ?? 'rejected'}`); } } + if (a.type === 'autoNumber') { + // maxUsedAutoNumber is a read-only runtime counter — the internal API rejects it on create. + // The dest base starts its own auto-numbering; strip it so the field creates cleanly. + const { maxUsedAutoNumber, ...rest } = typeOptions || {}; + typeOptions = rest; + } const { columnId } = await client.createField(destAppId, destTableId, { name: a.name, type: a.type, typeOptions, description: a.description ?? undefined }); idmap.fields[a.sourceFieldId] = { destFld: columnId, choices: {} }; if (entry) entry.fieldsByName.set(a.name, { id: columnId, name: a.name, type: a.type, typeOptions }); diff --git a/packages/mcp-server/test/sync/test-apply.test.js b/packages/mcp-server/test/sync/test-apply.test.js index 19e36b6..a0f332a 100644 --- a/packages/mcp-server/test/sync/test-apply.test.js +++ b/packages/mcp-server/test/sync/test-apply.test.js @@ -419,3 +419,47 @@ describe('apply: applyViewConfig', () => { assert.ok(res.warnings.some((w) => w.code === 'VIEW_ANCHOR_FALLBACK' || w.code === 'VIEW_UNRESOLVABLE_REF')); }); }); + +describe('apply: createField (autoNumber — strip read-only maxUsedAutoNumber)', () => { + it('creates an autoNumber field without passing maxUsedAutoNumber to the client', async () => { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'T'); + const destSnapshot = await snapshotBase(client, 'appD'); + // Track typeOptions actually passed to createField + const capturedTypeOptions = []; + const origCreate = client.createField.bind(client); + client.createField = async (appId, tblId, cfg) => { + capturedTypeOptions.push({ name: cfg.name, typeOptions: cfg.typeOptions }); + return origCreate(appId, tblId, cfg); + }; + const plan = { + planId: 'plnAN', sourceBaseId: 'appS', destBaseId: 'appD', + idmap: { tables: { tS: tableId }, fields: {} }, + actions: [ + { + kind: 'createField', sourceTableId: 'tS', sourceFieldId: 'fldAN', + name: 'ID', type: 'autoNumber', + typeOptions: { maxUsedAutoNumber: 5 }, + description: null, computed: false, dependsOn: [], dependsOnTables: [], + }, + ], + orphans: [], warnings: [], + }; + const res = await applyPlan({ + client, plan, destAppId: 'appD', destSnapshot, + idmap: JSON.parse(JSON.stringify(plan.idmap)), + journal: newJournal('plnAN', 'ts'), + persist: () => {}, + }); + assert.equal(res.created, 1, 'autoNumber field must be created'); + assert.equal(res.failed, 0, 'must not fail'); + assert.ok(res.idmap.fields.fldAN, 'field must be mapped'); + const captured = capturedTypeOptions.find((c) => c.name === 'ID'); + assert.ok(captured, 'createField must have been called for ID'); + assert.equal( + captured.typeOptions && 'maxUsedAutoNumber' in captured.typeOptions, + false, + 'maxUsedAutoNumber must be stripped from typeOptions before createField', + ); + }); +}); From 3fe3de5d7ee1d5114e3f8644c20f0cb5f94fda28 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Fri, 19 Jun 2026 20:57:22 +0300 Subject: [PATCH 100/246] fix(sync): count fields emit internal relationColumnId (not public recordLinkFieldId) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit toWritableComputedOptions count branch read o.recordLinkFieldId and emitted that public-REST key. But internal-API snapshots store a count's link under relationColumnId (like rollup), and createField passes count typeOptions through untranslated — so every count field 422'd with "options not valid". Mirror rollup: read relationColumnId (fall back to recordLinkFieldId) and emit the internal key. Found via live convergence of appCn3FOO98v8kq1X (3 count fields failing). The existing test fed the public key as input, masking the bug. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/remap.js | 5 ++++- .../mcp-server/test/sync/test-remap-refs.test.js | 15 ++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/packages/mcp-server/src/sync/remap.js b/packages/mcp-server/src/sync/remap.js index 829ba85..17ed05b 100644 --- a/packages/mcp-server/src/sync/remap.js +++ b/packages/mcp-server/src/sync/remap.js @@ -85,7 +85,10 @@ export function toWritableComputedOptions(type, opts) { case 'multipleLookupValues': return { relationColumnId: o.relationColumnId, foreignTableRollupColumnId: o.foreignTableRollupColumnId }; case 'count': - return { recordLinkFieldId: o.recordLinkFieldId }; + // Internal API stores/expects the link under relationColumnId (like rollup); the public + // REST name recordLinkFieldId is never present in snapshot data and createField passes + // count typeOptions through untranslated → emit the internal key or it 422s "options not valid". + return { relationColumnId: o.relationColumnId ?? o.recordLinkFieldId }; default: return o; } diff --git a/packages/mcp-server/test/sync/test-remap-refs.test.js b/packages/mcp-server/test/sync/test-remap-refs.test.js index 613ff09..09354a3 100644 --- a/packages/mcp-server/test/sync/test-remap-refs.test.js +++ b/packages/mcp-server/test/sync/test-remap-refs.test.js @@ -67,10 +67,19 @@ describe('remap.toWritableComputedOptions', () => { assert.equal(w.formulaText, 'SUM(values)'); assert.equal(w.resultType, undefined); }); - it('count: emits recordLinkFieldId only', () => { - const remapped = remapRefs({ recordLinkFieldId: 'fldA', resultType: 'number' }, idmap); + it('count: emits internal relationColumnId (remapped) — internal-API key that createField passes through untranslated', () => { + // Real internal-API count fields store the link under relationColumnId (like rollup), + // NOT the public-REST name recordLinkFieldId. Emitting the public name made the internal + // API reject every count field with "options not valid". + const remapped = remapRefs({ relationColumnId: 'fldA', resultType: 'number' }, idmap); const w = toWritableComputedOptions('count', remapped); - assert.equal(w.recordLinkFieldId, 'fldX'); + assert.equal(w.relationColumnId, 'fldX'); + assert.equal(w.recordLinkFieldId, undefined); assert.equal(w.resultType, undefined); }); + it('count: also accepts the public recordLinkFieldId key but still emits relationColumnId', () => { + const remapped = remapRefs({ recordLinkFieldId: 'fldA' }, idmap); + const w = toWritableComputedOptions('count', remapped); + assert.equal(w.relationColumnId, 'fldX'); + }); }); From 818ba6cf66593bc3b3a8755711b42d161576a020 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Fri, 19 Jun 2026 21:00:34 +0300 Subject: [PATCH 101/246] =?UTF-8?q?docs(sync):=20changelog=20=E2=80=94=20a?= =?UTF-8?q?utoNumber=20+=20count=20field-create=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b95f614..0d6fa03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,10 @@ Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how - `sync_base mode=plan` now emits a curatable **changeset**: each action carries a stable name-based `changeId` (`|
|`), a `class`, and an `apply:true` flag; the digest includes a `sample` + an `editHint`. New `direction` param (`to-dest` default | `to-source`) targets either base as the destination. - `sync_base mode=apply` accepts a `skip:[changeId]` list (and honors `apply:false` on changeset entries) → applies everything except the removed changes. Journal-safe; skipped dependencies degrade gracefully to UNRESOLVABLE_REF. +#### Fixed +- `sync_base mode=apply` field creation — **autoNumber** fields now create. The read-only runtime counter `maxUsedAutoNumber` is stripped from `typeOptions` before `createField` (the internal API rejected it with 422 "options not valid"). +- `sync_base mode=apply` field creation — **count** fields now create. `toWritableComputedOptions` emitted the public-REST key `recordLinkFieldId`, but internal-API snapshots store a count's link under `relationColumnId` (like rollup) and the count create path passes `typeOptions` through untranslated — so every count field 422'd "options not valid". It now reads/emits the internal `relationColumnId`. (Both fixes validated live by converging a full base copy: 5 autoNumber + 3 count fields that previously failed now create cleanly.) + ### MCP server — sync_base record sync (2026-06-17) #### Added From c871bb4bc334ebc2879294fb996b1b523aed901d Mon Sep 17 00:00:00 2001 From: "A.R." Date: Fri, 19 Jun 2026 22:21:37 +0300 Subject: [PATCH 102/246] fix(sync): canonicalize empty-predicate filter operators (no false view drift) Airtable's internal API stores the same emptiness check two equivalent ways depending on field type/era: {isNotEmpty,null} == {!=,""} and {isEmpty,null} == {=,""}. A source base holds the named-operator form; the dest holds the =/!= form after the sync applied it -> canonFilterSet compared op/val verbatim -> false `filters` drift in mode=diff AND perpetual applyViewConfig re-emit. normEmptyPredicate collapses the =/!= forms to the named operator, but only when the value is genuinely empty (0/false preserved). Both compare.js and diff.js consume canonicalizeViewConfig, so one fix lands the diff report and the apply convergence path. Verified on the real "Marketplaces IDs" filter from the appCn3FOO98v8kq1X diff: the src isNotEmpty/null and dest !=/"" forms now canonicalize identically. 587 tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 1 + packages/mcp-server/src/sync/remap.js | 16 ++++++++++++++- .../test/sync/test-remap-view.test.js | 20 +++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d6fa03..d330721 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how #### Fixed - `sync_base mode=apply` field creation — **autoNumber** fields now create. The read-only runtime counter `maxUsedAutoNumber` is stripped from `typeOptions` before `createField` (the internal API rejected it with 422 "options not valid"). - `sync_base mode=apply` field creation — **count** fields now create. `toWritableComputedOptions` emitted the public-REST key `recordLinkFieldId`, but internal-API snapshots store a count's link under `relationColumnId` (like rollup) and the count create path passes `typeOptions` through untranslated — so every count field 422'd "options not valid". It now reads/emits the internal `relationColumnId`. (Both fixes validated live by converging a full base copy: 5 autoNumber + 3 count fields that previously failed now create cleanly.) +- `sync_base mode=diff` / view convergence — **empty-predicate filter representations no longer report false drift.** Airtable's internal API stores the same emptiness check two equivalent ways (`isNotEmpty`/null ≡ `!=`/"" and `isEmpty`/null ≡ `=`/""): a source base holds the named-operator form, the dest holds the `=`/`!=` form after the sync applied it. `canonFilterSet` now normalizes these (only when the value is genuinely empty — `0`/`false` are preserved), so semantically-equal filters compare equal in the diff report and stop re-emitting `applyViewConfig` every apply. ### MCP server — sync_base record sync (2026-06-17) diff --git a/packages/mcp-server/src/sync/remap.js b/packages/mcp-server/src/sync/remap.js index 17ed05b..b67fb2f 100644 --- a/packages/mcp-server/src/sync/remap.js +++ b/packages/mcp-server/src/sync/remap.js @@ -197,6 +197,18 @@ function viewNameOf(map, id) { return map[id] ?? id; } // groups (collapse-agnostic): Airtable may store a 1-element nested group as a bare leaf on // readback — unwrapping on BOTH sides keeps it convergent either way. (Records aren't synced, so // rec-ref leaves drop on both sides; once they do sync, this is where name-resolution lands — M3.) +// Airtable's internal API stores the same emptiness predicate two equivalent ways (varies by field +// type / era): {isNotEmpty, null} ≡ {!=, ""} and {isEmpty, null} ≡ {=, ""}. A source base holds the +// named-operator form; the dest holds the =/!= form after the sync applied it → false `filters` drift +// in mode=diff and perpetual applyViewConfig re-emit. Collapse the =/!= forms to the named operator, +// but ONLY when the value is genuinely empty — real values (incl. 0/false) are left untouched. +function normEmptyPredicate(op, val) { + const empty = val === '' || val === null || val === undefined; + if (empty && (op === 'isNotEmpty' || op === '!=')) return { op: 'isNotEmpty', val: null }; + if (empty && (op === 'isEmpty' || op === '=')) return { op: 'isEmpty', val: null }; + return { op, val }; +} + function canonFilterSet(set, fldNames, selNames, strip, idmap) { const out = []; for (const f of set) { @@ -213,7 +225,9 @@ function canonFilterSet(set, fldNames, selNames, strip, idmap) { out.push({ col: viewNameOf(fldNames, f.columnId), op: f.operator, val: typeof v === 'string' ? viewNameOf(selNames, v) : v }); continue; } - out.push({ col: viewNameOf(fldNames, f.columnId), op: f.operator, val: typeof f.value === 'string' ? viewNameOf(selNames, f.value) : f.value }); + const mappedVal = typeof f.value === 'string' ? viewNameOf(selNames, f.value) : f.value; + const n = normEmptyPredicate(f.operator, mappedVal); + out.push({ col: viewNameOf(fldNames, f.columnId), op: n.op, val: n.val }); } return out; } diff --git a/packages/mcp-server/test/sync/test-remap-view.test.js b/packages/mcp-server/test/sync/test-remap-view.test.js index 96bbedb..6603efc 100644 --- a/packages/mcp-server/test/sync/test-remap-view.test.js +++ b/packages/mcp-server/test/sync/test-remap-view.test.js @@ -62,6 +62,26 @@ describe('remap.canonicalizeViewConfig', () => { const realFilter = canonicalizeViewConfig({ filters: { conjunction: 'and', filterSet: [{ columnId: 'fldA', operator: 'contains', value: null }] } }, { fldA: 'ID' }, {}); assert.notEqual(noFilter, realFilter); // a genuine filter still differs from none }); + it('empty-predicate operator representations are canonical-equal (isNotEmpty/null ≡ !=/"")', () => { + // Airtable's internal API stores the same "is not empty" predicate two equivalent ways: + // the source base holds {isNotEmpty, null}; the dest holds {!=, ""} after the sync applied it. + const src = canonicalizeViewConfig({ filters: { conjunction: 'and', filterSet: [{ columnId: 'fldA', operator: 'isNotEmpty', value: null }] } }, { fldA: 'Sold.id' }, {}); + const dest = canonicalizeViewConfig({ filters: { conjunction: 'and', filterSet: [{ columnId: 'fldA', operator: '!=', value: '' }] } }, { fldA: 'Sold.id' }, {}); + assert.equal(src, dest); + }); + it('empty-predicate isEmpty/null ≡ =/"" are canonical-equal', () => { + const src = canonicalizeViewConfig({ filters: { conjunction: 'and', filterSet: [{ columnId: 'fldA', operator: 'isEmpty', value: null }] } }, { fldA: 'ID' }, {}); + const dest = canonicalizeViewConfig({ filters: { conjunction: 'and', filterSet: [{ columnId: 'fldA', operator: '=', value: '' }] } }, { fldA: 'ID' }, {}); + assert.equal(src, dest); + }); + it('a NON-empty value with = is NOT collapsed to an empty predicate (real values preserved)', () => { + const empty = canonicalizeViewConfig({ filters: { conjunction: 'and', filterSet: [{ columnId: 'fldA', operator: '=', value: '' }] } }, { fldA: 'ID' }, {}); + const real = canonicalizeViewConfig({ filters: { conjunction: 'and', filterSet: [{ columnId: 'fldA', operator: '=', value: '2' }] } }, { fldA: 'ID' }, {}); + assert.notEqual(empty, real); + // 0 / false are real values, not "empty" + const zero = canonicalizeViewConfig({ filters: { conjunction: 'and', filterSet: [{ columnId: 'fldA', operator: '=', value: 0 }] } }, { fldA: 'ID' }, {}); + assert.notEqual(empty, zero); + }); }); describe('remap — record-referencing view filters (strip + report + converge)', () => { From c52118e377b05cda59496bc992a190d7cae9575b Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 20 Jun 2026 09:16:29 +0300 Subject: [PATCH 103/246] fix(sync): delete createTable scaffolding rows for a clean mirror MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit createTable seeds ~3 blank rows (live-confirmed: 3 per table) the same way it seeds 6 default fields; apply deleted the scaffolding FIELDS (D1) but left the ROWS, so a synced base carried blank rows in every table. D1b now clears them right after create (table holds only scaffolding pre-records, so a full clear is safe), best-effort (SCAFFOLDING_ROWS_KEPT warning on failure, never aborts). The mock's queryRecords returned a bare array; the real client returns {summary:{rows}} — corrected the mock to the real contract (the unit test had been green against the wrong shape) and read .summary.rows in apply. Live probe: createTable -> 3 rows -> D1b -> 0. 588 tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/apply.js | 12 ++++++++++++ .../mcp-server/test/sync/helpers/mock-client.js | 17 ++++++++++++++++- .../mcp-server/test/sync/test-apply.test.js | 16 ++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/packages/mcp-server/src/sync/apply.js b/packages/mcp-server/src/sync/apply.js index 47f16cd..1bea81e 100644 --- a/packages/mcp-server/src/sync/apply.js +++ b/packages/mcp-server/src/sync/apply.js @@ -137,6 +137,18 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result } if (c.id === primaryId) continue; await client.deleteField(destAppId, c.id, c.name); } + // D1b: delete the auto-created blank scaffolding ROWS (Airtable seeds ~3 on create) — the table + // holds only scaffolding at this point (records sync runs later), so a full clear is safe and + // keeps the mirror clean. Best-effort: a failure here must not abort the table create. + try { + const viewId = (views || [])[0]?.id; + if (viewId) { + const rowIds = ((await client.queryRecords(destAppId, tableId, viewId)).summary?.rows || []).map((r) => r.id); + if (rowIds.length) await client.deleteRecords(destAppId, tableId, rowIds, { viewId }); + } + } catch (e) { + result.warnings.push({ code: 'SCAFFOLDING_ROWS_KEPT', message: `Table "${a.name}": could not remove default scaffolding rows: ${e.message ?? e}` }); + } const entry = { id: tableId, name: a.name, primaryFieldId: primaryId, fieldsByName: new Map([[primary.name, { id: primaryId, name: primary.name, type: primary.type, typeOptions: primary.typeOptions ?? null }]]), diff --git a/packages/mcp-server/test/sync/helpers/mock-client.js b/packages/mcp-server/test/sync/helpers/mock-client.js index 1eaad9c..330e4d5 100644 --- a/packages/mcp-server/test/sync/helpers/mock-client.js +++ b/packages/mcp-server/test/sync/helpers/mock-client.js @@ -23,10 +23,25 @@ export class MockClient { const tableId = this._id('tbl'); const columns = DEFAULT_FIELDS.map((f) => ({ id: this._id('fld'), name: f.name, type: f.type, typeOptions: f.typeOptions ?? null, description: null })); this.tables.push({ id: tableId, name, primaryColumnId: columns[0].id, columns, - views: [{ id: this._id('viw'), name: 'Grid view', type: 'grid', personalForUserId: null, config: {} }] }); + views: [{ id: this._id('viw'), name: 'Grid view', type: 'grid', personalForUserId: null, config: {} }], + // live createTable also seeds ~3 blank scaffolding rows — model them so apply's row-cleanup is testable + rows: [{ id: this._id('rec'), fields: {} }, { id: this._id('rec'), fields: {} }, { id: this._id('rec'), fields: {} }] }); this.calls.push(`createTable:${name}`); return { tableId }; } + async queryRecords(appId, tableId) { + this.calls.push(`queryRecords:${tableId}`); + // mirror the real client's return shape: { summary: { rows: [{id, fields}] }, raw } + const rows = (this._table(tableId).rows || []).map((r) => ({ id: r.id, fields: r.fields })); + return { summary: { tableId, count: rows.length, rows }, raw: null }; + } + async deleteRecords(appId, tableId, rowIds) { + const t = this._table(tableId); + const before = (t.rows || []).length; + t.rows = (t.rows || []).filter((r) => !rowIds.includes(r.id)); + this.calls.push(`deleteRecords:${tableId}:${before - t.rows.length}`); + return { deleted: before - t.rows.length }; + } _table(id) { const t = this.tables.find((x) => x.id === id); if (!t) throw new Error('no table ' + id); return t; } _field(colId) { for (const t of this.tables) { const f = t.columns.find((c) => c.id === colId); if (f) return f; } throw new Error('no field ' + colId); } async deleteField(appId, fieldId, expectedName) { diff --git a/packages/mcp-server/test/sync/test-apply.test.js b/packages/mcp-server/test/sync/test-apply.test.js index a0f332a..5b6d528 100644 --- a/packages/mcp-server/test/sync/test-apply.test.js +++ b/packages/mcp-server/test/sync/test-apply.test.js @@ -38,6 +38,22 @@ describe('apply: createTable + scaffolding-delete + reconcilePrimary', () => { assert.equal(res.idmap.fields.fS1.destFld, t.columns[0].id); }); + it('removes the auto-created scaffolding rows for a clean mirror', async () => { + const client = new MockClient(); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { + planId: 'plnRows', sourceBaseId: 'appS', destBaseId: 'appD', + idmap: { tables: {}, fields: {} }, + actions: [{ kind: 'createTable', sourceTableId: 'tS', name: 'Clean' }], + orphans: [], warnings: [], + }; + const res = await run(client, plan, destSnapshot); + assert.equal(res.failed, 0); + const t = client.tables.find((x) => x.name === 'Clean'); + assert.equal(t.rows.length, 0, 'scaffolding rows removed'); + assert.ok(client.calls.some((c) => c.startsWith('deleteRecords:' + t.id)), 'deleteRecords called for the new table'); + }); + it('warns PRIMARY_TYPE_INCOMPATIBLE on rejected retype, keeps placeholder (still renamed)', async () => { const client = new MockClient(); const destSnapshot = await snapshotBase(client, 'appD'); From 0360193f0543c9f678a1b1b5d7fdc89b9c28a069 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 20 Jun 2026 09:17:07 +0300 Subject: [PATCH 104/246] =?UTF-8?q?docs(sync):=20changelog=20=E2=80=94=20c?= =?UTF-8?q?reateTable=20scaffolding-row=20cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d330721..49ca5ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how #### Fixed - `sync_base mode=apply` field creation — **autoNumber** fields now create. The read-only runtime counter `maxUsedAutoNumber` is stripped from `typeOptions` before `createField` (the internal API rejected it with 422 "options not valid"). - `sync_base mode=apply` field creation — **count** fields now create. `toWritableComputedOptions` emitted the public-REST key `recordLinkFieldId`, but internal-API snapshots store a count's link under `relationColumnId` (like rollup) and the count create path passes `typeOptions` through untranslated — so every count field 422'd "options not valid". It now reads/emits the internal `relationColumnId`. (Both fixes validated live by converging a full base copy: 5 autoNumber + 3 count fields that previously failed now create cleanly.) +- `sync_base mode=apply` — **createTable scaffolding rows are now removed** (clean mirror). A new table is seeded with ~3 blank rows (like the 6 default fields); apply already deleted the scaffolding fields but left the rows, so every synced table carried blank rows. The rows are now cleared right after create (best-effort; `SCAFFOLDING_ROWS_KEPT` warning on failure). Live-confirmed: createTable → 3 rows → 0. - `sync_base mode=diff` / view convergence — **empty-predicate filter representations no longer report false drift.** Airtable's internal API stores the same emptiness check two equivalent ways (`isNotEmpty`/null ≡ `!=`/"" and `isEmpty`/null ≡ `=`/""): a source base holds the named-operator form, the dest holds the `=`/`!=` form after the sync applied it. `canonFilterSet` now normalizes these (only when the value is genuinely empty — `0`/`false` are preserved), so semantically-equal filters compare equal in the diff report and stop re-emitting `applyViewConfig` every apply. ### MCP server — sync_base record sync (2026-06-17) From 5790df379f132b87fe1dcba86020fe53898c97bf Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 20 Jun 2026 12:17:36 +0300 Subject: [PATCH 105/246] feat(sync): policy presets + resolvePolicy + isDeleting Implement the reconciliation policy module with three named presets (mirror, overlay, preserve) defining two policy axes: extras keep|remove and conflicts source-wins|dest-wins. Export resolvePolicy() for per-table policy resolution with per-table overrides support, and isDeleting() to detect whether any policy configuration will result in record deletion. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/policy.js | 18 +++++++++++ .../mcp-server/test/sync/test-policy.test.js | 30 +++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 packages/mcp-server/src/sync/policy.js create mode 100644 packages/mcp-server/test/sync/test-policy.test.js diff --git a/packages/mcp-server/src/sync/policy.js b/packages/mcp-server/src/sync/policy.js new file mode 100644 index 0000000..8176c8b --- /dev/null +++ b/packages/mcp-server/src/sync/policy.js @@ -0,0 +1,18 @@ +// Reconciliation policy: two axes (extras keep|remove, conflicts source-wins|dest-wins) +// expressed as three named presets. See docs spec 2026-06-20-records-reconciliation-policy. +export const PRESETS = { + mirror: { extras: 'remove', conflicts: 'source-wins' }, + overlay: { extras: 'keep', conflicts: 'source-wins' }, + preserve: { extras: 'keep', conflicts: 'dest-wins' }, +}; + +export function resolvePolicy(globalPreset, overrides, tableName) { + const name = (overrides && overrides[tableName]) || globalPreset || 'overlay'; + return PRESETS[name] || PRESETS.overlay; +} + +export function isDeleting(globalPreset, overrides) { + const removes = (p) => PRESETS[p]?.extras === 'remove'; + if (removes(globalPreset)) return true; + return Object.values(overrides || {}).some(removes); +} diff --git a/packages/mcp-server/test/sync/test-policy.test.js b/packages/mcp-server/test/sync/test-policy.test.js new file mode 100644 index 0000000..edb5ad6 --- /dev/null +++ b/packages/mcp-server/test/sync/test-policy.test.js @@ -0,0 +1,30 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { PRESETS, resolvePolicy, isDeleting } from '../../src/sync/policy.js'; + +describe('policy.resolvePolicy', () => { + it('defaults to overlay when nothing specified', () => { + assert.deepEqual(resolvePolicy(undefined, undefined, 'X'), { extras: 'keep', conflicts: 'source-wins' }); + }); + it('uses the global preset for non-overridden tables', () => { + assert.deepEqual(resolvePolicy('mirror', { Games: 'preserve' }, 'Offers'), { extras: 'remove', conflicts: 'source-wins' }); + }); + it('per-table override wins over global', () => { + assert.deepEqual(resolvePolicy('mirror', { Games: 'preserve' }, 'Games'), { extras: 'keep', conflicts: 'dest-wins' }); + }); + it('unknown preset falls back to overlay (safe)', () => { + assert.deepEqual(resolvePolicy('bogus', undefined, 'X'), PRESETS.overlay); + }); +}); + +describe('policy.isDeleting', () => { + it('false when global+overrides all keep', () => { + assert.equal(isDeleting('overlay', { Games: 'preserve' }), false); + }); + it('true when global mirror', () => { + assert.equal(isDeleting('mirror', undefined), true); + }); + it('true when an override mirrors even if global keeps', () => { + assert.equal(isDeleting('overlay', { Games: 'mirror' }), true); + }); +}); From ddf069aef05fe5294cc04252ceab2d6788fbcc8b Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 20 Jun 2026 12:21:13 +0300 Subject: [PATCH 106/246] feat(sync): validateFieldMappings (fail-fast, 6 codes) + resolve to field ids Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/policy.js | 38 +++++++++++++++ .../mcp-server/test/sync/test-policy.test.js | 46 +++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/packages/mcp-server/src/sync/policy.js b/packages/mcp-server/src/sync/policy.js index 8176c8b..0b942bb 100644 --- a/packages/mcp-server/src/sync/policy.js +++ b/packages/mcp-server/src/sync/policy.js @@ -1,3 +1,5 @@ +import { isComputedType } from './snapshot.js'; + // Reconciliation policy: two axes (extras keep|remove, conflicts source-wins|dest-wins) // expressed as three named presets. See docs spec 2026-06-20-records-reconciliation-policy. export const PRESETS = { @@ -16,3 +18,39 @@ export function isDeleting(globalPreset, overrides) { if (removes(globalPreset)) return true; return Object.values(overrides || {}).some(removes); } + +// Array-shaped source values can't be injected into a scalar dest field. +const ARRAY_SOURCE_TYPES = new Set(['multipleRecordLinks', 'foreignKey', 'multipleAttachments', 'multiSelect', 'multipleLookupValues']); + +export function validateFieldMappings(srcSnapshot, destSnapshot, fieldMappings) { + const errors = []; + const resolved = []; + const srcTables = new Map((srcSnapshot.tables || []).map((t) => [t.name, t])); + const dstTables = new Map((destSnapshot.tables || []).map((t) => [t.name, t])); + + for (const [table, mappings] of Object.entries(fieldMappings || {})) { + const st = srcTables.get(table); + const dt = dstTables.get(table); + if (!st || !dt) { errors.push({ code: 'FIELD_MAP_TABLE_MISSING', table }); continue; } + const srcByName = new Map(st.fields.map((f) => [f.name, f])); + const dstByName = new Map(dt.fields.map((f) => [f.name, f])); + const targetsSeen = new Map(); + + for (const [srcName, destName] of Object.entries(mappings || {})) { + const sf = srcByName.get(srcName); + const df = dstByName.get(destName); + if (!sf) { errors.push({ code: 'FIELD_MAP_SOURCE_MISSING', table, source: srcName }); continue; } + if (!df) { errors.push({ code: 'FIELD_MAP_TARGET_MISSING', table, target: destName }); continue; } + if (isComputedType(df.type)) { errors.push({ code: 'FIELD_MAP_TARGET_COMPUTED', table, target: destName }); continue; } + if (ARRAY_SOURCE_TYPES.has(sf.type) || ARRAY_SOURCE_TYPES.has(df.type)) { + errors.push({ code: 'FIELD_MAP_TYPE_INCOMPATIBLE', table, source: srcName, target: destName }); continue; + } + if (targetsSeen.has(destName)) { + errors.push({ code: 'FIELD_MAP_COLLISION', table, target: destName }); continue; + } + targetsSeen.set(destName, srcName); + resolved.push({ table, srcFieldId: sf.id, destFieldId: df.id, destType: df.type }); + } + } + return { errors, resolved }; +} diff --git a/packages/mcp-server/test/sync/test-policy.test.js b/packages/mcp-server/test/sync/test-policy.test.js index edb5ad6..3ad1e26 100644 --- a/packages/mcp-server/test/sync/test-policy.test.js +++ b/packages/mcp-server/test/sync/test-policy.test.js @@ -28,3 +28,49 @@ describe('policy.isDeleting', () => { assert.equal(isDeleting('overlay', { Games: 'mirror' }), true); }); }); + +import { validateFieldMappings } from '../../src/sync/policy.js'; + +describe('policy.validateFieldMappings', () => { + const src = { tables: [{ id: 'tS', name: 'Offers', fields: [ + { id: 'fCode', name: 'Code', type: 'autoNumber' }, + { id: 'fLinks', name: 'Links', type: 'multipleRecordLinks' }, + ] }] }; + const dst = { tables: [{ id: 'tD', name: 'Offers', fields: [ + { id: 'dInject', name: 'InjectID', type: 'number' }, + { id: 'dFormula', name: 'PK', type: 'formula' }, + ] }] }; + + it('resolves a valid computed-source → scalar-dest mapping to field ids', () => { + const { errors, resolved } = validateFieldMappings(src, dst, { Offers: { Code: 'InjectID' } }); + assert.deepEqual(errors, []); + assert.deepEqual(resolved, [{ table: 'Offers', srcFieldId: 'fCode', destFieldId: 'dInject', destType: 'number' }]); + }); + it('FIELD_MAP_TABLE_MISSING when the table is absent in a base', () => { + const { errors } = validateFieldMappings(src, dst, { Nope: { Code: 'InjectID' } }); + assert.equal(errors[0].code, 'FIELD_MAP_TABLE_MISSING'); + }); + it('FIELD_MAP_SOURCE_MISSING when source field absent', () => { + const { errors } = validateFieldMappings(src, dst, { Offers: { Ghost: 'InjectID' } }); + assert.equal(errors[0].code, 'FIELD_MAP_SOURCE_MISSING'); + }); + it('FIELD_MAP_TARGET_MISSING when dest field absent', () => { + const { errors } = validateFieldMappings(src, dst, { Offers: { Code: 'Ghost' } }); + assert.equal(errors[0].code, 'FIELD_MAP_TARGET_MISSING'); + }); + it('FIELD_MAP_TARGET_COMPUTED when dest field is a formula', () => { + const { errors } = validateFieldMappings(src, dst, { Offers: { Code: 'PK' } }); + assert.equal(errors[0].code, 'FIELD_MAP_TARGET_COMPUTED'); + }); + it('FIELD_MAP_TYPE_INCOMPATIBLE when source is an array/link type', () => { + const { errors } = validateFieldMappings(src, dst, { Offers: { Links: 'InjectID' } }); + assert.equal(errors[0].code, 'FIELD_MAP_TYPE_INCOMPATIBLE'); + }); + it('FIELD_MAP_COLLISION when two sources target the same dest field', () => { + const src2 = { tables: [{ id: 'tS', name: 'Offers', fields: [ + { id: 'fa', name: 'A', type: 'text' }, { id: 'fb', name: 'B', type: 'text' }, + ] }] }; + const { errors } = validateFieldMappings(src2, dst, { Offers: { A: 'InjectID', B: 'InjectID' } }); + assert.ok(errors.some((e) => e.code === 'FIELD_MAP_COLLISION')); + }); +}); From 7cf8eeb7bf90ff0997870c34f4707a7287f70cd3 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 20 Jun 2026 12:24:50 +0300 Subject: [PATCH 107/246] fix(sync): add multipleSelects to ARRAY_SOURCE_TYPES hardening Defensive fix to reject multi-value sources mapping to scalar dest fields regardless of whether snapshots carry the public-API name (multipleSelects). Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/policy.js | 2 +- packages/mcp-server/test/sync/test-policy.test.js | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/mcp-server/src/sync/policy.js b/packages/mcp-server/src/sync/policy.js index 0b942bb..86b1628 100644 --- a/packages/mcp-server/src/sync/policy.js +++ b/packages/mcp-server/src/sync/policy.js @@ -20,7 +20,7 @@ export function isDeleting(globalPreset, overrides) { } // Array-shaped source values can't be injected into a scalar dest field. -const ARRAY_SOURCE_TYPES = new Set(['multipleRecordLinks', 'foreignKey', 'multipleAttachments', 'multiSelect', 'multipleLookupValues']); +const ARRAY_SOURCE_TYPES = new Set(['multipleRecordLinks', 'foreignKey', 'multipleAttachments', 'multiSelect', 'multipleSelects', 'multipleLookupValues']); export function validateFieldMappings(srcSnapshot, destSnapshot, fieldMappings) { const errors = []; diff --git a/packages/mcp-server/test/sync/test-policy.test.js b/packages/mcp-server/test/sync/test-policy.test.js index 3ad1e26..0c76f13 100644 --- a/packages/mcp-server/test/sync/test-policy.test.js +++ b/packages/mcp-server/test/sync/test-policy.test.js @@ -66,6 +66,14 @@ describe('policy.validateFieldMappings', () => { const { errors } = validateFieldMappings(src, dst, { Offers: { Links: 'InjectID' } }); assert.equal(errors[0].code, 'FIELD_MAP_TYPE_INCOMPATIBLE'); }); + it('FIELD_MAP_TYPE_INCOMPATIBLE when source is multipleSelects', () => { + const srcMulti = { tables: [{ id: 'tS', name: 'Offers', fields: [ + { id: 'fCode', name: 'Code', type: 'autoNumber' }, + { id: 'fMulti', name: 'Tags', type: 'multipleSelects' }, + ] }] }; + const { errors } = validateFieldMappings(srcMulti, dst, { Offers: { Tags: 'InjectID' } }); + assert.equal(errors[0].code, 'FIELD_MAP_TYPE_INCOMPATIBLE'); + }); it('FIELD_MAP_COLLISION when two sources target the same dest field', () => { const src2 = { tables: [{ id: 'tS', name: 'Offers', fields: [ { id: 'fa', name: 'A', type: 'text' }, { id: 'fb', name: 'B', type: 'text' }, From 02a03212b7aeb3a9d947e0b129d81731bf00139b Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 20 Jun 2026 12:26:25 +0300 Subject: [PATCH 108/246] =?UTF-8?q?feat(sync):=20coerceMappedValue=20?= =?UTF-8?q?=E2=80=94=20scalar=20source->dest=20injection?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/cells.js | 9 +++++++++ .../mcp-server/test/sync/test-cells.test.js | 17 ++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/packages/mcp-server/src/sync/cells.js b/packages/mcp-server/src/sync/cells.js index 0966f9a..2aecd60 100644 --- a/packages/mcp-server/src/sync/cells.js +++ b/packages/mcp-server/src/sync/cells.js @@ -54,3 +54,12 @@ export function partitionLinkValue(srcValue, idmap) { } return { resolved, unresolved }; } + +const TEXT_DEST_TYPES = new Set(['text', 'multilineText', 'richText', 'phone', 'email', 'url', 'singleLineText']); + +export function coerceMappedValue(srcValue, destType) { + if (srcValue == null) return { write: true, value: srcValue }; + if (Array.isArray(srcValue) || typeof srcValue === 'object') return { write: false }; + if (TEXT_DEST_TYPES.has(destType)) return { write: true, value: String(srcValue) }; + return { write: true, value: srcValue }; // number/currency/percent/date/checkbox/duration/rating/... +} diff --git a/packages/mcp-server/test/sync/test-cells.test.js b/packages/mcp-server/test/sync/test-cells.test.js index e7c19cf..1913b70 100644 --- a/packages/mcp-server/test/sync/test-cells.test.js +++ b/packages/mcp-server/test/sync/test-cells.test.js @@ -1,6 +1,6 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { isWritableForRecords, coercePass1Cell, partitionLinkValue, linkRecId } from '../../src/sync/cells.js'; +import { isWritableForRecords, coercePass1Cell, partitionLinkValue, linkRecId, coerceMappedValue } from '../../src/sync/cells.js'; const idmap = { fields: { fldSel: { destFld: 'fldSelD', choices: { selA: 'selAD', selB: 'selBD' } } } }; @@ -70,3 +70,18 @@ describe('cells.partitionLinkValue', () => { assert.deepEqual(partitionLinkValue(null, m), { resolved: [], unresolved: [] }); }); }); + +describe('cells.coerceMappedValue', () => { + it('passes a number through to a number target', () => { + assert.deepEqual(coerceMappedValue(1042, 'number'), { write: true, value: 1042 }); + }); + it('stringifies for a text target', () => { + assert.deepEqual(coerceMappedValue(1042, 'text'), { write: true, value: '1042' }); + }); + it('passes null through (clears the cell)', () => { + assert.deepEqual(coerceMappedValue(null, 'number'), { write: true, value: null }); + }); + it('refuses an array source value (defensive)', () => { + assert.deepEqual(coerceMappedValue(['x'], 'text'), { write: false }); + }); +}); From 6e50a8fba2d7186bc0b1963aa91da90b2c93882f Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 20 Jun 2026 12:30:20 +0300 Subject: [PATCH 109/246] feat(sync): Pass1 honors conflict policy (dest-wins preserves dest edits) Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/records.js | 6 +- .../test/sync/test-records-policy.test.js | 91 +++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 packages/mcp-server/test/sync/test-records-policy.test.js diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index 0510876..6a94f49 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -15,6 +15,7 @@ import { existsSync, readFileSync, mkdirSync } from 'node:fs'; import { join } from 'node:path'; import { coercePass1Cell, partitionLinkValue, linkRecId } from './cells.js'; +import { resolvePolicy } from './policy.js'; import { withRetry, createLimiter } from './ratelimit.js'; import { remapViewConfig, collectFilterRecordRefs } from './remap.js'; import { snapshotSchemaOnly, snapshotViews, snapshotTableRecords } from './snapshot.js'; @@ -313,7 +314,7 @@ async function createChunkWithFallback({ client, destAppId, destTableId, chunk, } } -export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result }) { +export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result, policy, policyOverrides }) { if (!idmap.records) idmap.records = {}; // destAppId comes from the snapshot (set by snapshotBase); fall back to '' for tests that omit it. @@ -331,6 +332,8 @@ export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idm const destTableId = idmap.tables[srcTable.id]; if (!destTableId) continue; // table not matched → skip + const { conflicts } = resolvePolicy(policy, policyOverrides, srcTable.name); + const records = srcTable.records || []; if (records.length === 0) continue; @@ -354,6 +357,7 @@ export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idm createRows.push({ cellValuesByColumnId: cells, sourceKey: rec.id, linkCells }); } else { // UPDATE path — multiSelect arrays NOT accepted by updateRecords (deferred with warning) + if (conflicts === 'dest-wins') continue; // preserve dest edits: skip the overwrite const cells = buildUpdateCells(srcTable.fields, srcCells, idmap, result.warnings, rec.id); updateRows.push({ rowId: destRecId, cellValuesByColumnId: cells }); } diff --git a/packages/mcp-server/test/sync/test-records-policy.test.js b/packages/mcp-server/test/sync/test-records-policy.test.js new file mode 100644 index 0000000..e269a7f --- /dev/null +++ b/packages/mcp-server/test/sync/test-records-policy.test.js @@ -0,0 +1,91 @@ +/** + * test-records-policy.test.js + * + * Tests for Pass 1 conflict-policy integration in applyRecordsPass1. + * Fixtures (fakeClient, noLimiter, baseSnap, srcSnap) are kept reusable + * for Task 5 and Task 6 which append more tests to this file. + */ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { applyRecordsPass1 } from '../../src/sync/records.js'; + +export function fakeClient() { + const calls = { create: [], update: [] }; + return { + calls, + async createRecords(appId, tableId, rows) { + calls.create.push({ tableId, rows }); + return { records: rows.map((r, i) => ({ id: 'recD' + i })), created: rows.map((r, i) => ({ id: 'recD' + i })) }; + }, + async updateRecords(appId, tableId, rows) { + calls.update.push({ tableId, rows }); + return { updated: rows, failed: [] }; + }, + }; +} + +export const noLimiter = { run: (fn) => fn() }; + +export const baseSnap = () => ({ + baseId: 'appD', + tables: [{ id: 'tD', name: 'Games', fields: [{ id: 'dN', name: 'Name', type: 'text' }] }], +}); + +export function srcSnap() { + return { + baseId: 'appS', + tables: [{ + id: 'tS', + name: 'Games', + primaryFieldId: 'sN', + fields: [{ id: 'sN', name: 'Name', type: 'text' }], + records: [{ id: 'recS1', cellValuesByColumnId: { sN: 'Zelda' } }], + }], + }; +} + +describe('Pass1 conflict policy', () => { + it('dest-wins (preserve) skips updating an already-mapped record', async () => { + const client = fakeClient(); + const idmap = { + tables: { tS: 'tD' }, + fields: { sN: { destFld: 'dN', choices: {} } }, + records: { recS1: 'recD_existing' }, + }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + await applyRecordsPass1({ + client, + srcSnapshot: srcSnap(), + destSnapshot: baseSnap(), + idmap, + limiter: noLimiter, + journal: {}, + persist: () => {}, + result, + policy: 'preserve', + }); + assert.equal(client.calls.update.length, 0, 'no update under preserve'); + }); + + it('source-wins (overlay) updates the mapped record', async () => { + const client = fakeClient(); + const idmap = { + tables: { tS: 'tD' }, + fields: { sN: { destFld: 'dN', choices: {} } }, + records: { recS1: 'recD_existing' }, + }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + await applyRecordsPass1({ + client, + srcSnapshot: srcSnap(), + destSnapshot: baseSnap(), + idmap, + limiter: noLimiter, + journal: {}, + persist: () => {}, + result, + policy: 'overlay', + }); + assert.equal(client.calls.update.length, 1, 'one update under overlay'); + }); +}); From f57539cbd5bf8b2f621d6435359dec3277823f93 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 20 Jun 2026 12:35:02 +0300 Subject: [PATCH 110/246] feat(sync): Pass1 injects resolved field mappings (source value -> dest field) Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/records.js | 26 +++++++++++++++++-- .../test/sync/test-records-policy.test.js | 17 ++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index 6a94f49..a9261d9 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -14,7 +14,7 @@ import { existsSync, readFileSync, mkdirSync } from 'node:fs'; import { join } from 'node:path'; -import { coercePass1Cell, partitionLinkValue, linkRecId } from './cells.js'; +import { coercePass1Cell, partitionLinkValue, linkRecId, coerceMappedValue } from './cells.js'; import { resolvePolicy } from './policy.js'; import { withRetry, createLimiter } from './ratelimit.js'; import { remapViewConfig, collectFilterRecordRefs } from './remap.js'; @@ -23,6 +23,25 @@ import { loadIdmap, saveIdmap, syncDir } from './idmap.js'; import { newJournal, loadRecordsJournal, saveRecordsJournal } from './journal.js'; import { safeAtomicWriteFileSync } from '../safe-write.js'; +/** + * Inject resolved field mappings for one record into a cells payload (mutates + returns it). + * + * For each mapping whose destType coercion returns write:true, sets cells[destFieldId] = value. + * Skips mappings whose coerced result is write:false (e.g. object/array src values). + * + * @param {object} cells - dest cellValuesByColumnId being built (mutated) + * @param {Array} tableMappings - resolved mappings filtered to the current table + * @param {object} srcCells - source record cellValuesByColumnId + * @returns {object} - same `cells` reference (mutated) + */ +function injectFieldMappings(cells, tableMappings, srcCells) { + for (const m of tableMappings) { + const r = coerceMappedValue(srcCells[m.srcFieldId], m.destType); + if (r.write) cells[m.destFieldId] = r.value; + } + return cells; +} + // ── Records-job status file (records-job-.json) ── // The records phase is minutes-long for large bases, so apply() launches it in the BACKGROUND // and returns immediately. Status is persisted to disk (survives process restarts) and polled @@ -314,7 +333,7 @@ async function createChunkWithFallback({ client, destAppId, destTableId, chunk, } } -export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result, policy, policyOverrides }) { +export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result, policy, policyOverrides, fieldMappings = [] }) { if (!idmap.records) idmap.records = {}; // destAppId comes from the snapshot (set by snapshotBase); fall back to '' for tests that omit it. @@ -333,6 +352,7 @@ export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idm if (!destTableId) continue; // table not matched → skip const { conflicts } = resolvePolicy(policy, policyOverrides, srcTable.name); + const tableMappings = fieldMappings.filter((m) => m.table === srcTable.name); const records = srcTable.records || []; if (records.length === 0) continue; @@ -348,6 +368,7 @@ export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idm if (!destRecId) { // CREATE path — scalar/select/multiSelect arrays ARE accepted by createRecords. const cells = buildCreateCells(srcTable.fields, srcCells, idmap, result.warnings, rec.id); + injectFieldMappings(cells, tableMappings, srcCells); // Fold resolvable links (remapped) into the create payload (kills most of Pass 2). let linkCells = {}; if (runState.foldLinks) { @@ -359,6 +380,7 @@ export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idm // UPDATE path — multiSelect arrays NOT accepted by updateRecords (deferred with warning) if (conflicts === 'dest-wins') continue; // preserve dest edits: skip the overwrite const cells = buildUpdateCells(srcTable.fields, srcCells, idmap, result.warnings, rec.id); + injectFieldMappings(cells, tableMappings, srcCells); updateRows.push({ rowId: destRecId, cellValuesByColumnId: cells }); } } diff --git a/packages/mcp-server/test/sync/test-records-policy.test.js b/packages/mcp-server/test/sync/test-records-policy.test.js index e269a7f..5f5dc1a 100644 --- a/packages/mcp-server/test/sync/test-records-policy.test.js +++ b/packages/mcp-server/test/sync/test-records-policy.test.js @@ -44,6 +44,23 @@ export function srcSnap() { }; } +describe('Pass1 field mapping (injection)', () => { + it('writes a computed source value into a scalar dest field on CREATE', async () => { + const client = fakeClient(); + const src = { baseId: 'appS', tables: [{ id: 'tS', name: 'Offers', primaryFieldId: 'sN', + fields: [{ id: 'sN', name: 'Name', type: 'text' }, { id: 'sCode', name: 'Code', type: 'autoNumber' }], + records: [{ id: 'recS1', cellValuesByColumnId: { sN: 'A', sCode: 1042 } }] }] }; + const dst = { baseId: 'appD', tables: [{ id: 'tD', name: 'Offers', + fields: [{ id: 'dN', name: 'Name', type: 'text' }, { id: 'dInject', name: 'InjectID', type: 'number' }] }] }; + const idmap = { tables: { tS: 'tD' }, fields: { sN: { destFld: 'dN', choices: {} } }, records: {} }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + const fieldMappings = [{ table: 'Offers', srcFieldId: 'sCode', destFieldId: 'dInject', destType: 'number' }]; + await applyRecordsPass1({ client, srcSnapshot: src, destSnapshot: dst, idmap, limiter: noLimiter, journal: {}, persist: () => {}, result, fieldMappings }); + const created = client.calls.create[0].rows[0].cellValuesByColumnId; + assert.equal(created.dInject, 1042, 'source Code value injected into dest InjectID'); + }); +}); + describe('Pass1 conflict policy', () => { it('dest-wins (preserve) skips updating an already-mapped record', async () => { const client = fakeClient(); From dd203322735afd91bc1704a0c004b6b79c360ba5 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 20 Jun 2026 12:39:19 +0300 Subject: [PATCH 111/246] =?UTF-8?q?feat(sync):=20pruneRecords=20=E2=80=94?= =?UTF-8?q?=20delete=20dest-only=20orphans=20under=20mirror=20(gated)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/records.js | 42 +++++++++++++++++++ .../test/sync/test-records-policy.test.js | 41 +++++++++++++++++- 2 files changed, 82 insertions(+), 1 deletion(-) diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index a9261d9..2c6d56a 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -1044,6 +1044,48 @@ export async function applyRecords({ client, sourceBaseId, destBaseId, planId, r return result; } +// ────────────────────────────────────────────────────────────────────────────── +// pruneRecords — extras axis: delete dest-only orphan records under mirror (gated) +// ────────────────────────────────────────────────────────────────────────────── + +/** + * Extras axis: delete dest rows no source record maps to, for tables whose policy removes extras. + * Gated by confirmDeletions — without it, nothing is deleted and a DELETION_GATED warning carries + * the would-delete count. result.deleted accumulates actual deletions. + * + * @param {object} opts + * @param {object} opts.client - AirtableClient instance + * @param {object} opts.destSnapshot - dest base snapshot (tables with views) + * @param {object} opts.idmap - live id-map (.records map srcId → destId) + * @param {string} opts.policy - global preset name ('mirror' | 'overlay' | 'preserve') + * @param {object} [opts.policyOverrides]- per-table preset overrides { [tableName]: preset } + * @param {boolean} opts.confirmDeletions - if false, gate deletions (push DELETION_GATED warning) + * @param {object} opts.limiter - rate limiter { run: (fn) => fn() } + * @param {object} opts.result - result accumulator (mutated: result.deleted, result.warnings) + * @returns {Promise} + */ +export async function pruneRecords({ client, destSnapshot, idmap, policy, policyOverrides, confirmDeletions, limiter, result }) { + if (result.deleted == null) result.deleted = 0; + const mappedDestIds = new Set(Object.values(idmap.records || {})); + for (const t of (destSnapshot.tables || [])) { + const { extras } = resolvePolicy(policy, policyOverrides, t.name); + if (extras !== 'remove') continue; + const viewId = (t.views || [])[0]?.id; + if (!viewId) continue; + let rows; + try { rows = (await limiter.run(() => client.queryRecords(destSnapshot.baseId, t.id, viewId))).summary?.rows || []; } + catch (e) { result.warnings.push({ code: 'PRUNE_QUERY_FAILED', message: `Table "${t.name}": ${e.message ?? e}` }); continue; } + const orphans = rows.map((r) => r.id).filter((id) => !mappedDestIds.has(id)); + if (orphans.length === 0) continue; + if (!confirmDeletions) { + result.warnings.push({ code: 'DELETION_GATED', message: `Table "${t.name}": ${orphans.length} dest-only record(s) would be deleted under mirror — re-run with confirmDeletions:true` }); + continue; + } + try { await limiter.run(() => client.deleteRecords(destSnapshot.baseId, t.id, orphans, { viewId })); result.deleted += orphans.length; } + catch (e) { result.failed += orphans.length; result.warnings.push({ code: 'RECORD_DELETE_FAILED', message: `Table "${t.name}": ${e.message ?? e}` }); } + } +} + // ────────────────────────────────────────────────────────────────────────────── // reconcile — existence-prune stale idmap.records entries // ────────────────────────────────────────────────────────────────────────────── diff --git a/packages/mcp-server/test/sync/test-records-policy.test.js b/packages/mcp-server/test/sync/test-records-policy.test.js index 5f5dc1a..4fa47fb 100644 --- a/packages/mcp-server/test/sync/test-records-policy.test.js +++ b/packages/mcp-server/test/sync/test-records-policy.test.js @@ -7,7 +7,7 @@ */ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { applyRecordsPass1 } from '../../src/sync/records.js'; +import { applyRecordsPass1, pruneRecords } from '../../src/sync/records.js'; export function fakeClient() { const calls = { create: [], update: [] }; @@ -106,3 +106,42 @@ describe('Pass1 conflict policy', () => { assert.equal(client.calls.update.length, 1, 'one update under overlay'); }); }); + +// ── Task 6: pruneRecords (extras axis) ─────────────────────────────────────── + +function pruneClient(rowsByTable) { + const calls = { deleted: [] }; + return { + calls, + async queryRecords(appId, tableId) { return { summary: { rows: (rowsByTable[tableId] || []).map((id) => ({ id, fields: {} })) } }; }, + async deleteRecords(appId, tableId, rowIds) { calls.deleted.push({ tableId, rowIds }); return { deleted: rowIds.length }; }, + }; +} +const destSnap2 = () => ({ baseId: 'appD', tables: [{ id: 'tD', name: 'Games', views: [{ id: 'viwD' }], fields: [] }] }); + +describe('pruneRecords (extras axis)', () => { + it('mirror + confirmDeletions deletes orphan dest rows (not in idmap.records)', async () => { + const client = pruneClient({ tD: ['recKeep', 'recOrphan'] }); + const idmap = { tables: { tS: 'tD' }, fields: {}, records: { recS1: 'recKeep' } }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, deleted: 0, warnings: [] }; + await pruneRecords({ client, destSnapshot: destSnap2(), idmap, policy: 'mirror', confirmDeletions: true, limiter: { run: (fn) => fn() }, result }); + assert.deepEqual(client.calls.deleted, [{ tableId: 'tD', rowIds: ['recOrphan'] }]); + assert.equal(result.deleted, 1); + }); + it('mirror WITHOUT confirmDeletions deletes nothing + warns DELETION_GATED', async () => { + const client = pruneClient({ tD: ['recKeep', 'recOrphan'] }); + const idmap = { tables: { tS: 'tD' }, fields: {}, records: { recS1: 'recKeep' } }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, deleted: 0, warnings: [] }; + await pruneRecords({ client, destSnapshot: destSnap2(), idmap, policy: 'mirror', confirmDeletions: false, limiter: { run: (fn) => fn() }, result }); + assert.equal(client.calls.deleted.length, 0); + const w = result.warnings.find((x) => x.code === 'DELETION_GATED'); + assert.ok(w && /1/.test(w.message), 'gated warning with count 1'); + }); + it('keep/preserve table is never pruned', async () => { + const client = pruneClient({ tD: ['recKeep', 'recOrphan'] }); + const idmap = { tables: { tS: 'tD' }, fields: {}, records: { recS1: 'recKeep' } }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, deleted: 0, warnings: [] }; + await pruneRecords({ client, destSnapshot: destSnap2(), idmap, policy: 'overlay', confirmDeletions: true, limiter: { run: (fn) => fn() }, result }); + assert.equal(client.calls.deleted.length, 0); + }); +}); From a87c3374b9ea9cbf901bdfa6c39a5644f85600ce Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 20 Jun 2026 17:54:50 +0300 Subject: [PATCH 112/246] fix(sync): result accumulator guards in pruneRecords Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/records.js | 2 ++ .../test/sync/test-records-policy.test.js | 23 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index 2c6d56a..9836664 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -1066,6 +1066,8 @@ export async function applyRecords({ client, sourceBaseId, destBaseId, planId, r */ export async function pruneRecords({ client, destSnapshot, idmap, policy, policyOverrides, confirmDeletions, limiter, result }) { if (result.deleted == null) result.deleted = 0; + if (result.failed == null) result.failed = 0; + if (!result.warnings) result.warnings = []; const mappedDestIds = new Set(Object.values(idmap.records || {})); for (const t of (destSnapshot.tables || [])) { const { extras } = resolvePolicy(policy, policyOverrides, t.name); diff --git a/packages/mcp-server/test/sync/test-records-policy.test.js b/packages/mcp-server/test/sync/test-records-policy.test.js index 4fa47fb..104aec3 100644 --- a/packages/mcp-server/test/sync/test-records-policy.test.js +++ b/packages/mcp-server/test/sync/test-records-policy.test.js @@ -144,4 +144,27 @@ describe('pruneRecords (extras axis)', () => { await pruneRecords({ client, destSnapshot: destSnap2(), idmap, policy: 'overlay', confirmDeletions: true, limiter: { run: (fn) => fn() }, result }); assert.equal(client.calls.deleted.length, 0); }); + it('deletion failure with missing result.failed/warnings guards against NaN and throw', async () => { + const failingClient = { + calls: { deleted: [] }, + async queryRecords(appId, tableId) { return { summary: { rows: [{ id: 'recOrphan', fields: {} }] } }; }, + async deleteRecords() { throw new Error('delete failed'); }, + }; + const idmap = { tables: { tS: 'tD' }, fields: {}, records: {} }; // empty records → orphan exists + const result = { deleted: 0 }; // omit failed and warnings + await pruneRecords({ + client: failingClient, + destSnapshot: destSnap2(), + idmap, + policy: 'mirror', + confirmDeletions: true, + limiter: { run: (fn) => fn() }, + result, + }); + assert.equal(typeof result.failed, 'number', 'result.failed is a number'); + assert.ok(!Number.isNaN(result.failed), 'result.failed is not NaN'); + assert.ok(Array.isArray(result.warnings), 'result.warnings is an array'); + const deleteFailedWarning = result.warnings.find((w) => w.code === 'RECORD_DELETE_FAILED'); + assert.ok(deleteFailedWarning, 'RECORD_DELETE_FAILED warning present'); + }); }); From bb54df68e4d04c8a8ee8b06f4feaed12e0fd0170 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 20 Jun 2026 12:46:56 +0300 Subject: [PATCH 113/246] =?UTF-8?q?feat(sync):=20runRecords=20core=20?= =?UTF-8?q?=E2=80=94=20pre-flight=20mapping=20validation=20+=20prune;=20th?= =?UTF-8?q?read=20policy=20opts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract post-snapshot core into exported runRecords(). validateFieldMappings runs BEFORE Pass1; any error throws FIELD_MAP_INVALID with .mappingErrors. applyRecords delegates to runRecords after I/O setup; accepts policy/policyOverrides/confirmDeletions/fieldMappings and forwards them. index.js apply() threads the same opts to the background job launch. pruneRecords called at end of runRecords. result includes deleted:0. 5 new tests; 619/619. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/index.js | 4 +- packages/mcp-server/src/sync/records.js | 107 ++++++--- .../sync/test-records-orchestration.test.js | 203 ++++++++++++++++++ 3 files changed, 279 insertions(+), 35 deletions(-) create mode 100644 packages/mcp-server/test/sync/test-records-orchestration.test.js diff --git a/packages/mcp-server/src/sync/index.js b/packages/mcp-server/src/sync/index.js index bc4b867..7f84d99 100644 --- a/packages/mcp-server/src/sync/index.js +++ b/packages/mcp-server/src/sync/index.js @@ -101,7 +101,7 @@ export function diffDetail({ sourceBaseId, destBaseId, diffId, detail, offset, l return renderDiff(savedDiff, { detail, offset, limit }); } -export async function apply({ client, sourceBaseId, destBaseId, planId, runStartedAt, skip = [] }) { +export async function apply({ client, sourceBaseId, destBaseId, planId, runStartedAt, skip = [], policy, policyOverrides, confirmDeletions, fieldMappings }) { const fullPlan = loadPlan(sourceBaseId, destBaseId, planId); if (!fullPlan) throw new Error(`No saved plan "${planId}" for ${sourceBaseId} -> ${destBaseId}. Run mode=plan first.`); @@ -128,7 +128,7 @@ export async function apply({ client, sourceBaseId, destBaseId, planId, runStart // journal) and writes a status file; poll via `sync_base mode=status`. if (!result.aborted) { writeRecordsJobStatus(sourceBaseId, destBaseId, planId, { status: 'running', startedAt: runStartedAt }); - applyRecordsImpl({ client, sourceBaseId, destBaseId, planId, runStartedAt }) + applyRecordsImpl({ client, sourceBaseId, destBaseId, planId, runStartedAt, policy, policyOverrides, confirmDeletions, fieldMappings }) .then((r) => writeRecordsJobStatus(sourceBaseId, destBaseId, planId, { status: 'done', startedAt: runStartedAt, finishedAt: new Date().toISOString(), result: { diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index 9836664..8db49e1 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -15,7 +15,7 @@ import { existsSync, readFileSync, mkdirSync } from 'node:fs'; import { join } from 'node:path'; import { coercePass1Cell, partitionLinkValue, linkRecId, coerceMappedValue } from './cells.js'; -import { resolvePolicy } from './policy.js'; +import { resolvePolicy, validateFieldMappings } from './policy.js'; import { withRetry, createLimiter } from './ratelimit.js'; import { remapViewConfig, collectFilterRecordRefs } from './remap.js'; import { snapshotSchemaOnly, snapshotViews, snapshotTableRecords } from './snapshot.js'; @@ -954,7 +954,76 @@ export async function reapplyViewFilters({ client, srcSnapshot, destSnapshot, id * @param {string} opts.runStartedAt - ISO timestamp of this run start * @returns {Promise} - result accumulator */ -export async function applyRecords({ client, sourceBaseId, destBaseId, planId, runStartedAt }) { +/** + * Core records orchestrator — drives Pass1 → destDisplayNames → Pass2 → attachments + * → reapplyViewFilters → pruneRecords. All I/O infrastructure (limiter, journal, persist, + * result) is supplied by the caller so this function is fully testable without filesystem. + * + * Pre-flight: runs validateFieldMappings BEFORE any write. If there are mapping errors, + * throws an Error with .code = 'FIELD_MAP_INVALID' and .mappingErrors = errors[]. + * + * @param {object} opts + * @param {object} opts.client - AirtableClient instance + * @param {object} opts.srcSnapshot - source base snapshot (with records attached) + * @param {object} opts.destSnapshot - dest base snapshot (with records attached) + * @param {object} opts.idmap - live id-map (.tables, .fields, .records) + * @param {string} [opts.policy] - global preset ('mirror'|'overlay'|'preserve') + * @param {object} [opts.policyOverrides]- per-table preset overrides + * @param {boolean} [opts.confirmDeletions] - gate for pruneRecords deletions + * @param {object} [opts.fieldMappings] - raw field mapping config { [table]: { srcName: destName } } + * @param {object} opts.limiter - rate limiter { run: (fn) => fn() } + * @param {object} opts.journal - records journal object + * @param {Function} opts.persist - (idmap, journal) => void — called after each phase + * @param {object} opts.result - result accumulator (mutated in place, returned) + * @returns {Promise} - the result accumulator + */ +export async function runRecords({ client, srcSnapshot, destSnapshot, idmap, policy, policyOverrides, confirmDeletions, fieldMappings, limiter, journal, persist, result }) { + // Pre-flight: validate field mappings BEFORE any write. Abort on error. + const { errors, resolved } = validateFieldMappings(srcSnapshot, destSnapshot, fieldMappings || {}); + if (errors.length) { + const err = new Error(`Invalid field mappings: ${errors.map((e) => e.code).join(', ')}`); + err.code = 'FIELD_MAP_INVALID'; + err.mappingErrors = errors; + throw err; + } + + // Pass 1: scalar / select upsert — fills idmap.records + await applyRecordsPass1({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result, policy, policyOverrides, fieldMappings: resolved }); + persist(idmap, journal); + + // Rebuild destDisplayNames from idmap.records now populated by Pass 1. + // Key = dest record id, value = source primary cell value (string). + const destDisplayNames = new Map(); + for (const srcTable of srcSnapshot.tables) { + const primaryFieldId = srcTable.primaryFieldId; + for (const srcRec of (srcTable.records || [])) { + const destRecId = idmap.records[srcRec.id]; + if (!destRecId) continue; + const primaryVal = primaryFieldId ? (srcRec.cellValuesByColumnId || {})[primaryFieldId] : undefined; + destDisplayNames.set(destRecId, primaryVal !== undefined ? String(primaryVal) : ''); + } + } + + // Pass 2: link cells + await applyRecordsPass2({ client, srcSnapshot, destSnapshot, idmap, destDisplayNames, limiter, journal, persist, result }); + persist(idmap, journal); + + // Pass 3: attachments + await applyAttachments({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result }); + persist(idmap, journal); + + // Reapply view filters whose record refs now resolve + await reapplyViewFilters({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result }); + persist(idmap, journal); + + // Extras axis: prune dest-only orphan records under mirror policy + await pruneRecords({ client, destSnapshot, idmap, policy, policyOverrides, confirmDeletions, limiter, result }); + persist(idmap, journal); + + return result; +} + +export async function applyRecords({ client, sourceBaseId, destBaseId, planId, runStartedAt, policy, policyOverrides, confirmDeletions, fieldMappings }) { // 1. Load the converged idmap (produced by the schema apply phase) const idmap = loadIdmap(sourceBaseId, destBaseId); idmap.records ??= {}; @@ -995,6 +1064,7 @@ export async function applyRecords({ client, sourceBaseId, destBaseId, planId, r updated: 0, skipped: 0, failed: 0, + deleted: 0, warnings: [], idmap, }; @@ -1011,37 +1081,8 @@ export async function applyRecords({ client, sourceBaseId, destBaseId, planId, r } } - // 5a. Pass 1: scalar / select upsert — fills idmap.records - await applyRecordsPass1({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result }); - persist(idmap, journal); - - // 5b. Rebuild destDisplayNames from idmap.records now populated by Pass 1. - // Key = dest record id, value = source primary cell value (string). - // The source primary string is the correct display name because dest content mirrors source. - const destDisplayNames = new Map(); - for (const srcTable of srcSnapshot.tables) { - const primaryFieldId = srcTable.primaryFieldId; - for (const srcRec of (srcTable.records || [])) { - const destRecId = idmap.records[srcRec.id]; - if (!destRecId) continue; - const primaryVal = primaryFieldId ? (srcRec.cellValuesByColumnId || {})[primaryFieldId] : undefined; - destDisplayNames.set(destRecId, primaryVal !== undefined ? String(primaryVal) : ''); - } - } - - // 5c. Pass 2: link cells - await applyRecordsPass2({ client, srcSnapshot, destSnapshot, idmap, destDisplayNames, limiter, journal, persist, result }); - persist(idmap, journal); - - // 5d. Pass 3: attachments - await applyAttachments({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result }); - persist(idmap, journal); - - // 5e. Reapply view filters whose record refs now resolve - await reapplyViewFilters({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result }); - persist(idmap, journal); - - return result; + // Delegate the core phases to runRecords + return runRecords({ client, srcSnapshot, destSnapshot, idmap, policy, policyOverrides, confirmDeletions, fieldMappings, limiter, journal, persist, result }); } // ────────────────────────────────────────────────────────────────────────────── diff --git a/packages/mcp-server/test/sync/test-records-orchestration.test.js b/packages/mcp-server/test/sync/test-records-orchestration.test.js new file mode 100644 index 0000000..e2cb7ab --- /dev/null +++ b/packages/mcp-server/test/sync/test-records-orchestration.test.js @@ -0,0 +1,203 @@ +/** + * test-records-orchestration.test.js + * + * Unit tests for runRecords() — the extracted core orchestrator. + * Drives runRecords directly so we can supply all infrastructure + * (limiter, journal, persist, result) without touching the filesystem. + * + * Task 7: pre-flight field-mapping validation aborts before any write. + */ + +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { runRecords } from '../../src/sync/records.js'; + +// ────────────────────────────────────────────────────────────────────────────── +// Helpers +// ────────────────────────────────────────────────────────────────────────────── + +function makeClient() { + const calls = []; + return { + calls, + createRecords: async (..._a) => { calls.push('create'); return { records: [], created: [] }; }, + updateRecords: async () => { calls.push('update'); return { updated: [], failed: [] }; }, + deleteRecords: async () => { calls.push('delete'); return {}; }, + queryRecords: async () => ({ summary: { rows: [] } }), + // view methods — shouldn't be reached in abort path + updateViewFilters: async () => ({}), + updateViewGroupLevels: async () => ({}), + applyViewSorts: async () => ({}), + setViewColumns: async () => ({}), + }; +} + +function makeInfra() { + const journal = {}; + const persisted = []; + return { + limiter: { run: (f) => f() }, + journal, + persist: (...args) => persisted.push(args), + persisted, + }; +} + +function makeResult() { + return { created: 0, updated: 0, skipped: 0, failed: 0, deleted: 0, warnings: [] }; +} + +// ────────────────────────────────────────────────────────────────────────────── +// Snapshots +// ────────────────────────────────────────────────────────────────────────────── + +/** src has: Offers table, two fields: Name(text) and Code(autoNumber) */ +function makeSrcSnapshot() { + return { + baseId: 'appSRC0000000001', + tables: [{ + id: 'tSRC01', + name: 'Offers', + primaryFieldId: 'fSrcName', + fields: [ + { id: 'fSrcName', name: 'Name', type: 'text' }, + { id: 'fSrcCode', name: 'Code', type: 'autoNumber' }, + ], + views: [{ id: 'vSrc1', name: 'Grid view', type: 'grid' }], + records: [], + }], + }; +} + +/** dest has: Offers table, one field: PK(formula) — computed, so mapping to it is invalid */ +function makeDestSnapshotWithComputedPK() { + return { + baseId: 'appDST0000000001', + tables: [{ + id: 'tDST01', + name: 'Offers', + primaryFieldId: 'fDstPK', + fields: [ + { id: 'fDstPK', name: 'PK', type: 'formula' }, + ], + views: [{ id: 'vDst1', name: 'Grid view', type: 'grid' }], + records: [], + }], + }; +} + +/** dest has: Offers table, writable target field */ +function makeDestSnapshotWritable() { + return { + baseId: 'appDST0000000002', + tables: [{ + id: 'tDST02', + name: 'Offers', + primaryFieldId: 'fDstName', + fields: [ + { id: 'fDstName', name: 'Name', type: 'text' }, + { id: 'fDstLabel', name: 'Label', type: 'text' }, + ], + views: [{ id: 'vDst2', name: 'Grid view', type: 'grid' }], + records: [], + }], + }; +} + +// ────────────────────────────────────────────────────────────────────────────── +// Tests +// ────────────────────────────────────────────────────────────────────────────── + +describe('runRecords — pre-flight field-mapping validation', () => { + it('is exported from records.js', () => { + assert.equal(typeof runRecords, 'function', 'runRecords must be exported'); + }); + + it('aborts before any write when a fieldMapping targets a computed (formula) dest field', async () => { + const client = makeClient(); + const { limiter, journal, persist } = makeInfra(); + const result = makeResult(); + + const srcSnapshot = makeSrcSnapshot(); + const destSnapshot = makeDestSnapshotWithComputedPK(); + const idmap = { tables: { tSRC01: 'tDST01' }, fields: {}, records: {} }; + + await assert.rejects( + () => runRecords({ + client, srcSnapshot, destSnapshot, idmap, + fieldMappings: { Offers: { Code: 'PK' } }, // Code → PK(formula) — invalid + limiter, journal, persist, result, + }), + (err) => { + assert.equal(err.code, 'FIELD_MAP_INVALID', 'error code must be FIELD_MAP_INVALID'); + assert.ok(Array.isArray(err.mappingErrors), '.mappingErrors must be an array'); + assert.ok(err.mappingErrors.length > 0, '.mappingErrors must be non-empty'); + return true; + }, + 'runRecords must reject with FIELD_MAP_INVALID for computed dest target', + ); + + assert.deepEqual(client.calls, [], 'no client writes should occur before validation completes'); + }); + + it('aborts before any write when a fieldMapping references a missing source field', async () => { + const client = makeClient(); + const { limiter, journal, persist } = makeInfra(); + const result = makeResult(); + + const srcSnapshot = makeSrcSnapshot(); + const destSnapshot = makeDestSnapshotWritable(); + const idmap = { tables: { tSRC01: 'tDST02' }, fields: {}, records: {} }; + + await assert.rejects( + () => runRecords({ + client, srcSnapshot, destSnapshot, idmap, + fieldMappings: { Offers: { NonExistent: 'Label' } }, + limiter, journal, persist, result, + }), + (err) => { + assert.equal(err.code, 'FIELD_MAP_INVALID'); + assert.ok(err.mappingErrors.some((e) => e.code === 'FIELD_MAP_SOURCE_MISSING')); + return true; + }, + ); + + assert.deepEqual(client.calls, [], 'no writes should occur for missing source field'); + }); + + it('completes without error when fieldMappings is empty (no-op)', async () => { + const client = makeClient(); + const { limiter, journal, persist } = makeInfra(); + const result = makeResult(); + + const srcSnapshot = makeSrcSnapshot(); + const destSnapshot = makeDestSnapshotWritable(); + // No idmap table match means Pass1/Pass2 find nothing to do — completes cleanly + const idmap = { tables: {}, fields: {}, records: {} }; + + // Should not throw + const out = await runRecords({ + client, srcSnapshot, destSnapshot, idmap, + fieldMappings: {}, + limiter, journal, persist, result, + }); + assert.ok(out, 'result should be returned'); + }); + + it('completes without error when fieldMappings is omitted (undefined)', async () => { + const client = makeClient(); + const { limiter, journal, persist } = makeInfra(); + const result = makeResult(); + + const srcSnapshot = makeSrcSnapshot(); + const destSnapshot = makeDestSnapshotWritable(); + const idmap = { tables: {}, fields: {}, records: {} }; + + // fieldMappings omitted entirely + const out = await runRecords({ + client, srcSnapshot, destSnapshot, idmap, + limiter, journal, persist, result, + }); + assert.ok(out, 'result should be returned'); + }); +}); From e046ad461db1a5435dce8aa9cf5e5963465ccb00 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 20 Jun 2026 12:52:33 +0300 Subject: [PATCH 114/246] fix(sync): surface deleted in records job-status + relocate applyRecords JSDoc - Add deleted: r.deleted || 0 to the result object written by apply() so mode=status reports deletions after a mirror sync - Move orphaned applyRecords JSDoc (documenting sourceBaseId, destBaseId, planId, runStartedAt) from above runRecords to immediately above applyRecords where it belongs (no text changes, surgical relocation only) Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/index.js | 1 + packages/mcp-server/src/sync/records.js | 62 ++++++++++++------------- 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/packages/mcp-server/src/sync/index.js b/packages/mcp-server/src/sync/index.js index 7f84d99..69ad604 100644 --- a/packages/mcp-server/src/sync/index.js +++ b/packages/mcp-server/src/sync/index.js @@ -134,6 +134,7 @@ export async function apply({ client, sourceBaseId, destBaseId, planId, runStart result: { created: r.created, updated: r.updated, failed: r.failed, attachmentsUploaded: r.attachmentsUploaded || 0, viewFiltersReapplied: r.viewFiltersReapplied || 0, + deleted: r.deleted || 0, warnings: (r.warnings || []).length, }, })) diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index 8db49e1..f95d9f8 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -921,39 +921,9 @@ export async function reapplyViewFilters({ client, srcSnapshot, destSnapshot, id } // ────────────────────────────────────────────────────────────────────────────── -// applyRecords — top-level orchestrator +// runRecords — core records orchestrator // ────────────────────────────────────────────────────────────────────────────── -/** - * Top-level records phase orchestrator. - * - * 1. Loads the converged schema idmap (written by the schema apply phase). - * 2. Snapshots src + dest schema (no records yet). - * 3. Attaches records to each table snapshot via snapshotTableRecords. - * Emits RECORD_COUNT warning when exactly 1000 rows are returned (possibly truncated). - * 4. Builds a phase-local rate limiter (~5 req/s; gates inner POSTs) and a records journal. - * 5. Runs: Pass 1 (scalar/select upsert + fold resolvable links into create) → rebuild - * destDisplayNames → Pass 2 (remaining/forward links) → Pass 3 (attachments) - * → reapplyViewFilters. - * 6. Persists idmap (+ journal) after each chunk/phase. - * 7. Returns a result accumulator. - * - * RESUME MODEL: resume is driven by the persisted **idmap.records** + a fresh live dest snapshot — - * a created row is skipped on re-run because its source id is already mapped, and Pass 2 dedups - * links against the live dest. The records journal is persisted but currently advisory (no - * per-record done-gating); idmap + live re-snapshot are the source of truth. Known limitation: - * a crash between a create's server-ack and the per-chunk idmap persist can re-create up to one - * chunk (~50) of rows as duplicates on resume (reconcile only existence-prunes; natural-key - * re-match is a stub). - * - * @param {object} opts - * @param {object} opts.client - AirtableClient instance - * @param {string} opts.sourceBaseId - source base app ID - * @param {string} opts.destBaseId - dest base app ID - * @param {string} opts.planId - plan ID (used for journal file name) - * @param {string} opts.runStartedAt - ISO timestamp of this run start - * @returns {Promise} - result accumulator - */ /** * Core records orchestrator — drives Pass1 → destDisplayNames → Pass2 → attachments * → reapplyViewFilters → pruneRecords. All I/O infrastructure (limiter, journal, persist, @@ -1023,6 +993,36 @@ export async function runRecords({ client, srcSnapshot, destSnapshot, idmap, pol return result; } +/** + * Top-level records phase orchestrator. + * + * 1. Loads the converged schema idmap (written by the schema apply phase). + * 2. Snapshots src + dest schema (no records yet). + * 3. Attaches records to each table snapshot via snapshotTableRecords. + * Emits RECORD_COUNT warning when exactly 1000 rows are returned (possibly truncated). + * 4. Builds a phase-local rate limiter (~5 req/s; gates inner POSTs) and a records journal. + * 5. Runs: Pass 1 (scalar/select upsert + fold resolvable links into create) → rebuild + * destDisplayNames → Pass 2 (remaining/forward links) → Pass 3 (attachments) + * → reapplyViewFilters. + * 6. Persists idmap (+ journal) after each chunk/phase. + * 7. Returns a result accumulator. + * + * RESUME MODEL: resume is driven by the persisted **idmap.records** + a fresh live dest snapshot — + * a created row is skipped on re-run because its source id is already mapped, and Pass 2 dedups + * links against the live dest. The records journal is persisted but currently advisory (no + * per-record done-gating); idmap + live re-snapshot are the source of truth. Known limitation: + * a crash between a create's server-ack and the per-chunk idmap persist can re-create up to one + * chunk (~50) of rows as duplicates on resume (reconcile only existence-prunes; natural-key + * re-match is a stub). + * + * @param {object} opts + * @param {object} opts.client - AirtableClient instance + * @param {string} opts.sourceBaseId - source base app ID + * @param {string} opts.destBaseId - dest base app ID + * @param {string} opts.planId - plan ID (used for journal file name) + * @param {string} opts.runStartedAt - ISO timestamp of this run start + * @returns {Promise} - result accumulator + */ export async function applyRecords({ client, sourceBaseId, destBaseId, planId, runStartedAt, policy, policyOverrides, confirmDeletions, fieldMappings }) { // 1. Load the converged idmap (produced by the schema apply phase) const idmap = loadIdmap(sourceBaseId, destBaseId); From af8a7f1f87cca19c0556e1bf8833b10a4a6b8791 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 20 Jun 2026 12:57:17 +0300 Subject: [PATCH 115/246] feat(sync): sync_base policy/fieldMappings params + plan-mode mapping validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add four optional params to the sync_base MCP tool input schema: - policy (mirror|overlay|preserve) — record reconciliation preset - policyOverrides (object) — per-table preset overrides - confirmDeletions (boolean) — gate for mirror-mode deletions - fieldMappings (object) — source→dest field injection overrides plan() and diff() in sync/index.js now accept fieldMappings; after loading schema snapshots they call validateFieldMappings() (dry-run, no mutation) and attach fieldMappingErrors to the machine output. apply() forwards all four params to applyRecords. The apply handler catches FIELD_MAP_INVALID errors and surfaces DELETION_GATED warnings in the digest text. TDD: 4 new tests in test-index.test.js; confirmed failing before implementation, all pass after. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/index.js | 27 +++- packages/mcp-server/src/sync/index.js | 25 +++- .../mcp-server/test/sync/test-index.test.js | 134 +++++++++++++++++- 3 files changed, 173 insertions(+), 13 deletions(-) diff --git a/packages/mcp-server/src/index.js b/packages/mcp-server/src/index.js index 07cdbbc..5782892 100644 --- a/packages/mcp-server/src/index.js +++ b/packages/mcp-server/src/index.js @@ -1594,6 +1594,10 @@ Note: "form title" is the view name itself — use rename_view to change it. "Fi limit: { type: 'number', description: 'Used by mode="diff" with detail: maximum number of entries to return.' }, direction: { type: 'string', enum: ['to-dest', 'to-source'], description: 'Used only by mode="plan": direction of the changeset. "to-dest" (default) brings the destination base up to date with the source. "to-source" swaps the roles so the changeset targets the source base (makes the source match the destination).' }, skip: { type: 'array', items: { type: 'string' }, description: 'Used only by mode="apply": list of changeIds to skip (actions with a matching changeId are counted as skipped but not applied). Use changeIds from the plan output.' }, + policy: { type: 'string', enum: ['mirror', 'overlay', 'preserve'], description: 'Used by mode="apply" for the record reconciliation preset. "mirror" = make dest identical to source (delete dest-only records, overwrite dest edits); "overlay" = keep dest-only records, source updates win on conflicts (default); "preserve" = keep dest-only records and never overwrite dest edits. Requires confirmDeletions=true to actually delete in mirror mode.' }, + policyOverrides: { type: 'object', description: 'Used by mode="apply": per-table reconciliation preset overrides. Maps table name → preset (e.g. { "Games": "preserve" }). Overrides the global policy for the named tables.', additionalProperties: { type: 'string', enum: ['mirror', 'overlay', 'preserve'] } }, + confirmDeletions: { type: 'boolean', description: 'Used by mode="apply" with policy="mirror": must be set to true to actually delete dest-only records. Without it, mirror mode reports a DELETION_GATED count and deletes nothing.' }, + fieldMappings: { type: 'object', description: 'Field mapping overrides: maps table name → { sourceField: destField } to inject a source field\'s value into a different (writable scalar) dest field during record sync. Example: { "Games": { "Title": "Name" } }. In mode="plan" and mode="diff", fieldMappings are validated against the two schemas (dry-run, no mutation) and errors are returned in fieldMappingErrors.', additionalProperties: { type: 'object', additionalProperties: { type: 'string' } } }, debug: debugProp, }, required: ['mode', 'sourceAppId', 'destAppId'], @@ -2395,18 +2399,31 @@ const handlers = { // ── Sync ── - async sync_base({ mode, sourceAppId, destAppId, planId, naturalKeys, detail, diffId, offset, limit, direction, skip, debug }) { + async sync_base({ mode, sourceAppId, destAppId, planId, naturalKeys, detail, diffId, offset, limit, direction, skip, policy, policyOverrides, confirmDeletions, fieldMappings, debug }) { const sync = await import('./sync/index.js'); if (mode === 'plan') { const id = 'pln' + client._genRandomId(); - const out = await sync.plan({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, planId: id, direction }); + const out = await sync.plan({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, planId: id, direction, fieldMappings }); return ok({ planId: id, summary: out.human }, out.machine, debug); } if (mode === 'apply') { if (!planId) return err('mode="apply" requires planId (from a prior mode="plan" run).'); const runStartedAt = new Date().toISOString(); - const out = await sync.apply({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, planId, runStartedAt, skip: skip || [] }); - return ok({ planId, summary: out.human }, out.machine, debug); + let out; + try { + out = await sync.apply({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, planId, runStartedAt, skip: skip || [], policy, policyOverrides, confirmDeletions, fieldMappings }); + } catch (e) { + if (e && e.code === 'FIELD_MAP_INVALID') { + return err(`Field mapping validation failed:\n${(e.mappingErrors || []).map((me) => ` [${me.code}] table=${me.table || '?'}${me.source ? ' source=' + me.source : ''}${me.target ? ' target=' + me.target : ''}`).join('\n')}`); + } + throw e; + } + // Surface DELETION_GATED warnings in the summary + const gatedWarnings = (out.machine && out.machine.warnings || []).filter((w) => w && w.code === 'DELETION_GATED'); + const gatedSuffix = gatedWarnings.length > 0 + ? `\n⚠ DELETION_GATED: ${gatedWarnings.map((w) => w.message || JSON.stringify(w)).join('; ')} — set confirmDeletions=true to delete.` + : ''; + return ok({ planId, summary: out.human + gatedSuffix }, out.machine, debug); } if (mode === 'reconcile') { const raw = await sync.reconcile({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, naturalKeys: naturalKeys || {} }); @@ -2432,7 +2449,7 @@ const handlers = { return ok({ diffId: diffId ?? null, summary: out.human }, out.machine, debug); } const id = diffId || ('dif' + client._genRandomId()); - const out = await sync.diff({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, diffId: id }); + const out = await sync.diff({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, diffId: id, fieldMappings }); return ok({ diffId: id, summary: out.human }, out.machine, debug); } return err(`Unsupported mode "${mode}". Use "plan", "apply", "reconcile", "status", or "diff".`); diff --git a/packages/mcp-server/src/sync/index.js b/packages/mcp-server/src/sync/index.js index 69ad604..ed00995 100644 --- a/packages/mcp-server/src/sync/index.js +++ b/packages/mcp-server/src/sync/index.js @@ -7,6 +7,7 @@ import { renderPlan, renderApplyResult, renderDiff } from './report.js'; import { applyPlan } from './apply.js'; import { newJournal, loadJournal, saveJournal } from './journal.js'; import { applyRecords as applyRecordsImpl, reconcile as reconcileImpl, writeRecordsJobStatus, readRecordsJobStatus } from './records.js'; +import { validateFieldMappings } from './policy.js'; const ENGINE_VERSION = '2b'; @@ -36,10 +37,10 @@ export function fingerprintSchema(snap) { * (source is brought up to date with dest). Persisted state uses swapped IDs so load/apply * work correctly on the swapped pair. * - * @param {{ client: object, sourceBaseId: string, destBaseId: string, planId: string, direction?: 'to-dest'|'to-source' }} opts + * @param {{ client: object, sourceBaseId: string, destBaseId: string, planId: string, direction?: 'to-dest'|'to-source', fieldMappings?: object }} opts * @returns {Promise<{ human: string, machine: object }>} */ -export async function plan({ client, sourceBaseId, destBaseId, planId, direction = 'to-dest' }) { +export async function plan({ client, sourceBaseId, destBaseId, planId, direction = 'to-dest', fieldMappings }) { // When direction='to-source', swap so the changeset targets the original SOURCE base. const effectiveSrcId = direction === 'to-source' ? destBaseId : sourceBaseId; const effectiveDestId = direction === 'to-source' ? sourceBaseId : destBaseId; @@ -62,24 +63,36 @@ export async function plan({ client, sourceBaseId, destBaseId, planId, direction engineVersion: ENGINE_VERSION, lastPlanId: planId, }); - return renderPlan(fullPlan); + const rendered = renderPlan(fullPlan); + // Dry-run field-mapping validation: attach errors to machine output (no mutation). + if (fieldMappings) { + const { errors } = validateFieldMappings(src, dest, fieldMappings); + rendered.machine = { ...rendered.machine, fieldMappingErrors: errors }; + } + return rendered; } /** * Compute a schema diff between two bases. `diffId` is supplied by the caller so the engine * stays deterministic/testable. Saves the full diff to disk and returns a rendered digest. * - * @param {{ client: object, sourceBaseId: string, destBaseId: string, diffId: string }} opts + * @param {{ client: object, sourceBaseId: string, destBaseId: string, diffId: string, fieldMappings?: object }} opts * @returns {Promise<{ human: string, machine: object }>} */ -export async function diff({ client, sourceBaseId, destBaseId, diffId }) { +export async function diff({ client, sourceBaseId, destBaseId, diffId, fieldMappings }) { const src = await snapshotBase(client, sourceBaseId); const dest = await snapshotBase(client, destBaseId); const idmap = matchByName(src, dest); const base = compare(src, dest, idmap); const fullDiff = { diffId, ...base }; saveDiff(sourceBaseId, destBaseId, fullDiff); - return renderDiff(fullDiff); + const rendered = renderDiff(fullDiff); + // Dry-run field-mapping validation: attach errors to machine output (no mutation). + if (fieldMappings) { + const { errors } = validateFieldMappings(src, dest, fieldMappings); + rendered.machine = { ...rendered.machine, fieldMappingErrors: errors }; + } + return rendered; } /** diff --git a/packages/mcp-server/test/sync/test-index.test.js b/packages/mcp-server/test/sync/test-index.test.js index 250e3ba..1b7b26b 100644 --- a/packages/mcp-server/test/sync/test-index.test.js +++ b/packages/mcp-server/test/sync/test-index.test.js @@ -1,6 +1,6 @@ -import { describe, it } from 'node:test'; +import { describe, it, mock } from 'node:test'; import assert from 'node:assert/strict'; -import { plan as computeSyncPlan, fingerprintSchema } from '../../src/sync/index.js'; +import { plan as computeSyncPlan, diff as computeSyncDiff, fingerprintSchema } from '../../src/sync/index.js'; import { mkdtempSync, existsSync } from 'node:fs'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; @@ -70,3 +70,133 @@ describe('sync index.plan — direction param (Task 10)', () => { assert.match(out.human, /createTable: 1/); }); }); + +describe('sync index.plan — fieldMappings validation (Task 8)', () => { + // Both src and dest have the same table/fields. We inject a mapping with a COMPUTED + // dest target (formula field) — validateFieldMappings must return FIELD_MAP_TARGET_COMPUTED. + // plan() must attach fieldMappingErrors to the returned result and make NO client writes. + it('mode=plan with invalid fieldMappings (computed dest) returns fieldMappingErrors, no mutation', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-fmap-')); + + // src: table T with text field "Name" + const srcSchema = { data: { tableSchemas: [{ + id: 'tbl1', name: 'T', primaryColumnId: 'fld1', + columns: [ + { id: 'fld1', name: 'Name', type: 'text' }, + ], + }] } }; + // dest: table T with text field "Name" and a formula field "Computed" + const destSchema = { data: { tableSchemas: [{ + id: 'tblD1', name: 'T', primaryColumnId: 'fldD1', + columns: [ + { id: 'fldD1', name: 'Name', type: 'text' }, + { id: 'fldD2', name: 'Computed', type: 'formula' }, + ], + }] } }; + + let writeCallCount = 0; + const client = { + getApplicationData: async (appId) => { + if (appId === 'appSSSSSSSSSSSSSS') return srcSchema; + return destSchema; + }, + // Track any mutating calls — there should be none + createTable: async () => { writeCallCount++; return {}; }, + createField: async () => { writeCallCount++; return {}; }, + }; + + const fieldMappings = { T: { Name: 'Computed' } }; // invalid: Computed is formula (computed) + + const out = await computeSyncPlan({ + client, + sourceBaseId: 'appSSSSSSSSSSSSSS', + destBaseId: 'appDDDDDDDDDDDDDD', + planId: 'plnFmapTest', + fieldMappings, + }); + + // Must include fieldMappingErrors + assert.ok(out.machine.fieldMappingErrors, 'fieldMappingErrors must be present in machine output'); + assert.equal(out.machine.fieldMappingErrors.length, 1, 'should have exactly one error'); + assert.equal(out.machine.fieldMappingErrors[0].code, 'FIELD_MAP_TARGET_COMPUTED', 'error code should be FIELD_MAP_TARGET_COMPUTED'); + assert.equal(out.machine.fieldMappingErrors[0].table, 'T', 'error should reference table T'); + assert.equal(out.machine.fieldMappingErrors[0].target, 'Computed', 'error should reference target field Computed'); + + // No mutations must have occurred + assert.equal(writeCallCount, 0, 'no client write calls should have been made'); + }); + + it('mode=diff with invalid fieldMappings returns fieldMappingErrors', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-fmap-diff-')); + + const srcSchema = { data: { tableSchemas: [{ + id: 'tbl1', name: 'T', primaryColumnId: 'fld1', + columns: [{ id: 'fld1', name: 'Name', type: 'text' }], + }] } }; + const destSchema = { data: { tableSchemas: [{ + id: 'tblD1', name: 'T', primaryColumnId: 'fldD1', + columns: [ + { id: 'fldD1', name: 'Name', type: 'text' }, + { id: 'fldD2', name: 'Computed', type: 'formula' }, + ], + }] } }; + + const client = { getApplicationData: async (appId) => (appId === 'appSSSSSSSSSSSSSS' ? srcSchema : destSchema) }; + const fieldMappings = { T: { Name: 'Computed' } }; + + const out = await computeSyncDiff({ + client, + sourceBaseId: 'appSSSSSSSSSSSSSS', + destBaseId: 'appDDDDDDDDDDDDDD', + diffId: 'difFmapTest', + fieldMappings, + }); + + assert.ok(out.machine.fieldMappingErrors, 'fieldMappingErrors must be present in diff machine output'); + assert.equal(out.machine.fieldMappingErrors[0].code, 'FIELD_MAP_TARGET_COMPUTED'); + }); + + it('mode=plan with valid fieldMappings returns no fieldMappingErrors', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-fmap-valid-')); + + const srcSchema = { data: { tableSchemas: [{ + id: 'tbl1', name: 'T', primaryColumnId: 'fld1', + columns: [{ id: 'fld1', name: 'Name', type: 'text' }], + }] } }; + const destSchema = { data: { tableSchemas: [{ + id: 'tblD1', name: 'T', primaryColumnId: 'fldD1', + columns: [ + { id: 'fldD1', name: 'Name', type: 'text' }, + { id: 'fldD2', name: 'Notes', type: 'text' }, + ], + }] } }; + + const client = { getApplicationData: async (appId) => (appId === 'appSSSSSSSSSSSSSS' ? srcSchema : destSchema) }; + const fieldMappings = { T: { Name: 'Notes' } }; // valid: both text fields + + const out = await computeSyncPlan({ + client, + sourceBaseId: 'appSSSSSSSSSSSSSS', + destBaseId: 'appDDDDDDDDDDDDDD', + planId: 'plnFmapValid', + fieldMappings, + }); + + assert.ok(Array.isArray(out.machine.fieldMappingErrors), 'fieldMappingErrors should be an array'); + assert.equal(out.machine.fieldMappingErrors.length, 0, 'should have no errors for a valid mapping'); + }); + + it('mode=plan without fieldMappings does not include fieldMappingErrors', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-fmap-none-')); + const mk = () => ({ data: { tableSchemas: [{ id: 'tbl1', name: 'T', primaryColumnId: 'fld1', columns: [{ id: 'fld1', name: 'Name', type: 'text' }] }] } }); + const client = { getApplicationData: async () => mk() }; + const out = await computeSyncPlan({ + client, + sourceBaseId: 'appSSSSSSSSSSSSSS', + destBaseId: 'appDDDDDDDDDDDDDD', + planId: 'plnNoFmap', + }); + // Without fieldMappings, fieldMappingErrors should not appear + assert.equal(out.machine.fieldMappingErrors, undefined, 'fieldMappingErrors should be absent when no fieldMappings given'); + }); +}); From 450cef617230fa51d749b155ad1d27e2633168a1 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 20 Jun 2026 13:06:54 +0300 Subject: [PATCH 116/246] fix(sync): synchronous fieldMappings pre-flight in apply() + dead DELETION_GATED code removed - apply() (src/sync/index.js): validate fieldMappings synchronously via snapshotSchemaOnly before launching the background records job; throws FIELD_MAP_INVALID with mappingErrors so the existing catch in the tool handler is now reachable - isDeleting imported from policy.js for use in apply() preflight context - writeRecordsJobStatus now persists warnings[] (full array) + warningCount alongside deleted so mode=status surfaces DELETION_GATED and RECORD_DELETE_FAILED per-item - src/index.js mode=apply: remove dead gatedWarnings filter (out.machine.warnings is schema-apply result, never contains DELETION_GATED); replace with synchronous forward- looking advisory via isDeleting(policy, policyOverrides) when confirmDeletions is unset - test/sync/test-index.test.js: remove unused mock import; add syncApply import; new test asserts apply() rejects synchronously with FIELD_MAP_INVALID for computed dest field and that no schema mutations or background job file are written Tests: 624/624 pass (was 623); pnpm check:tool-sync green (70 tools) Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/index.js | 14 ++-- packages/mcp-server/src/sync/index.js | 19 ++++- .../mcp-server/test/sync/test-index.test.js | 84 ++++++++++++++++++- 3 files changed, 108 insertions(+), 9 deletions(-) diff --git a/packages/mcp-server/src/index.js b/packages/mcp-server/src/index.js index 5782892..ffb1374 100644 --- a/packages/mcp-server/src/index.js +++ b/packages/mcp-server/src/index.js @@ -2418,12 +2418,16 @@ const handlers = { } throw e; } - // Surface DELETION_GATED warnings in the summary - const gatedWarnings = (out.machine && out.machine.warnings || []).filter((w) => w && w.code === 'DELETION_GATED'); - const gatedSuffix = gatedWarnings.length > 0 - ? `\n⚠ DELETION_GATED: ${gatedWarnings.map((w) => w.message || JSON.stringify(w)).join('; ')} — set confirmDeletions=true to delete.` + // Forward-looking note: DELETION_GATED warnings are only emitted by the background records + // job (after apply returns), so they cannot appear in out.machine.warnings here. Instead, + // we emit a synchronous advisory when the effective policy would delete dest-only records + // but confirmDeletions was not set. The actual DELETION_GATED warnings + deleted count are + // surfaced by mode=status once the background job completes. + const { isDeleting: isDeletingPolicy } = await import('./sync/policy.js'); + const deletingSuffix = isDeletingPolicy(policy, policyOverrides) && !confirmDeletions + ? '\nMirror/remove policy set — dest-only records will be reported as DELETION_GATED (deleted nothing). Poll mode=status for counts, then re-run with confirmDeletions:true to apply deletions.' : ''; - return ok({ planId, summary: out.human + gatedSuffix }, out.machine, debug); + return ok({ planId, summary: out.human + deletingSuffix }, out.machine, debug); } if (mode === 'reconcile') { const raw = await sync.reconcile({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, naturalKeys: naturalKeys || {} }); diff --git a/packages/mcp-server/src/sync/index.js b/packages/mcp-server/src/sync/index.js index ed00995..bb5a3d8 100644 --- a/packages/mcp-server/src/sync/index.js +++ b/packages/mcp-server/src/sync/index.js @@ -7,7 +7,7 @@ import { renderPlan, renderApplyResult, renderDiff } from './report.js'; import { applyPlan } from './apply.js'; import { newJournal, loadJournal, saveJournal } from './journal.js'; import { applyRecords as applyRecordsImpl, reconcile as reconcileImpl, writeRecordsJobStatus, readRecordsJobStatus } from './records.js'; -import { validateFieldMappings } from './policy.js'; +import { validateFieldMappings, isDeleting } from './policy.js'; const ENGINE_VERSION = '2b'; @@ -125,6 +125,20 @@ export async function apply({ client, sourceBaseId, destBaseId, planId, runStart return renderApplyResult({ planId, aborted: true, reason: 'DRIFT', warnings: [{ code: 'DRIFT', message: `Destination changed since plan ${planId}. Re-run mode=plan.` }] }); } + // Pre-flight: validate fieldMappings synchronously before launching the background job. + // This makes the FIELD_MAP_INVALID catch in the tool handler reachable (the background job + // re-validates as a safety net, but by then the caller has already returned). + if (fieldMappings && Object.keys(fieldMappings).length > 0) { + const srcSnapshot = await snapshotSchemaOnly(client, sourceBaseId); + const { errors } = validateFieldMappings(srcSnapshot, destSnapshot, fieldMappings); + if (errors.length > 0) { + const err = new Error('Field mapping validation failed'); + err.code = 'FIELD_MAP_INVALID'; + err.mappingErrors = errors; + throw err; + } + } + const journal = loadJournal(sourceBaseId, destBaseId, planId) ?? newJournal(planId, runStartedAt); // C1: preserve the persisted records/attachments identity map across applies (see buildRunIdmap). const idmap = buildRunIdmap(sourceBaseId, destBaseId, fullPlan, journal); @@ -148,7 +162,8 @@ export async function apply({ client, sourceBaseId, destBaseId, planId, runStart created: r.created, updated: r.updated, failed: r.failed, attachmentsUploaded: r.attachmentsUploaded || 0, viewFiltersReapplied: r.viewFiltersReapplied || 0, deleted: r.deleted || 0, - warnings: (r.warnings || []).length, + warningCount: (r.warnings || []).length, + warnings: r.warnings || [], }, })) .catch((err) => writeRecordsJobStatus(sourceBaseId, destBaseId, planId, { diff --git a/packages/mcp-server/test/sync/test-index.test.js b/packages/mcp-server/test/sync/test-index.test.js index 1b7b26b..9fd6196 100644 --- a/packages/mcp-server/test/sync/test-index.test.js +++ b/packages/mcp-server/test/sync/test-index.test.js @@ -1,6 +1,6 @@ -import { describe, it, mock } from 'node:test'; +import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { plan as computeSyncPlan, diff as computeSyncDiff, fingerprintSchema } from '../../src/sync/index.js'; +import { plan as computeSyncPlan, diff as computeSyncDiff, fingerprintSchema, apply as syncApply } from '../../src/sync/index.js'; import { mkdtempSync, existsSync } from 'node:fs'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; @@ -200,3 +200,83 @@ describe('sync index.plan — fieldMappings validation (Task 8)', () => { assert.equal(out.machine.fieldMappingErrors, undefined, 'fieldMappingErrors should be absent when no fieldMappings given'); }); }); + +describe('sync index.apply — synchronous fieldMappings pre-flight (Task 8 Fix)', () => { + // apply() must throw FIELD_MAP_INVALID synchronously (before launching the background records + // job) when fieldMappings references a computed (formula) dest field. + // Verified by: (a) rejection with the correct code, (b) no schema mutations occurred. + it('apply() with invalid fieldMappings (computed dest) rejects synchronously with FIELD_MAP_INVALID, no mutations', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-apply-fmap-')); + + // src: table T with a single text field "Name" + const srcSchema = { data: { tableSchemas: [{ + id: 'tbl1', name: 'T', primaryColumnId: 'fld1', + columns: [{ id: 'fld1', name: 'Name', type: 'text' }], + }] } }; + // dest: table T with text "Name" + formula "Computed" + const destSchema = { data: { tableSchemas: [{ + id: 'tblD1', name: 'T', primaryColumnId: 'fldD1', + columns: [ + { id: 'fldD1', name: 'Name', type: 'text' }, + { id: 'fldD2', name: 'Computed', type: 'formula' }, + ], + }] } }; + + let schemaMutationCount = 0; + let backgroundJobLaunched = false; + + const client = { + getApplicationData: async (appId) => (appId === 'appSSSSSSSSSSSSSS' ? srcSchema : destSchema), + // Detect any schema mutations (should NOT be called) + createTable: async () => { schemaMutationCount++; return {}; }, + createField: async () => { schemaMutationCount++; return {}; }, + updateField: async () => { schemaMutationCount++; return {}; }, + }; + + // Step 1: Run plan() so apply() can find a saved plan. + const planOut = await computeSyncPlan({ + client, + sourceBaseId: 'appSSSSSSSSSSSSSS', + destBaseId: 'appDDDDDDDDDDDDDD', + planId: 'plnApplyFmapTest', + }); + assert.ok(planOut, 'plan must succeed before apply'); + + // Reset mutation counter (plan itself may have no mutations, but be safe) + schemaMutationCount = 0; + + // Step 2: Call apply() with an invalid fieldMappings (Computed is a formula field). + // It should reject BEFORE any mutation or background job launch. + const fieldMappings = { T: { Name: 'Computed' } }; + + let thrown = null; + try { + await syncApply({ + client, + sourceBaseId: 'appSSSSSSSSSSSSSS', + destBaseId: 'appDDDDDDDDDDDDDD', + planId: 'plnApplyFmapTest', + runStartedAt: new Date().toISOString(), + fieldMappings, + }); + } catch (e) { + thrown = e; + } + + // Must have thrown + assert.ok(thrown, 'apply() must throw for invalid fieldMappings'); + assert.equal(thrown.code, 'FIELD_MAP_INVALID', 'error code must be FIELD_MAP_INVALID'); + assert.ok(Array.isArray(thrown.mappingErrors), 'mappingErrors must be an array'); + assert.equal(thrown.mappingErrors.length, 1, 'should have exactly one mapping error'); + assert.equal(thrown.mappingErrors[0].code, 'FIELD_MAP_TARGET_COMPUTED', 'error code must be FIELD_MAP_TARGET_COMPUTED'); + + // No schema mutations must have occurred + assert.equal(schemaMutationCount, 0, 'no schema mutations should have occurred before validation fails'); + + // No background job should have been launched (verified indirectly: apply threw before the + // fire-and-forget block, so no records job status file was written) + const { syncDir: getSyncDir } = await import('../../src/sync/idmap.js'); + const jobFile = join(getSyncDir('appSSSSSSSSSSSSSS', 'appDDDDDDDDDDDDDD'), 'records-job-plnApplyFmapTest.json'); + assert.ok(!existsSync(jobFile), 'records job status file must NOT exist (background job was not launched)'); + }); +}); From 0fefde6d7b2c95f64b3145b9befcce028aa4fed5 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 20 Jun 2026 13:11:01 +0300 Subject: [PATCH 117/246] docs(sync): document reconciliation policy + custom field mappings Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 7 +++++++ CLAUDE.md | 2 +- packages/mcp-server/README.md | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49ca5ef..632833c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how ## [Unreleased] +### MCP server — sync_base reconciliation policy + custom field mappings (2026-06-20) + +#### Added +- `sync_base mode=apply` — **records reconciliation policy**: new `policy` param (`mirror` | `overlay` | `preserve`, default `overlay`) controls two independent axes — **extras** (keep or remove dest-only records) and **conflicts** (source-wins or dest-wins when both bases hold the record). Presets: `mirror` = {remove, source-wins} (make dest identical to source), `overlay` = {keep, source-wins} (default — today's existing behavior), `preserve` = {keep, dest-wins} (never overwrite dest edits). `policyOverrides` (`{ [tableName]: preset }`) overrides the global preset per table. +- `sync_base mode=apply` — **deletion gate** (`confirmDeletions`): under `mirror` policy (or any table override that removes extras), dest-only orphan records are not deleted unless `confirmDeletions: true`. Without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count and does nothing — poll `mode=status` to see counts, then re-run with `confirmDeletions:true` to apply. `pruneRecords` also clears the blank scaffolding rows seeded by `createTable` under mirror (same gate; `SCAFFOLDING_ROWS_KEPT` warning on failure). Schema/view/attachment deletion under mirror and natural-key record matching are deferred follow-ups. +- `sync_base` — **custom field mappings** (`fieldMappings: { [tableName]: { sourceFieldName: destFieldName } }`): inject a source field's value (including computed fields like `autoNumber` or formula) into a different writable scalar dest field during record sync. Classic use: inject a source `autoNumber` field (`Code`) into a writable dest text field (`InjectID`) so a dest primary-key formula can reconstruct original identity across bases. Fail-fast pre-flight validation aborts `mode=apply` synchronously on any error (`FIELD_MAP_INVALID`); `mode=plan` and `mode=diff` run the same validation as a dry check and return errors in `fieldMappingErrors` (machine output). Validation codes: `FIELD_MAP_TABLE_MISSING`, `FIELD_MAP_SOURCE_MISSING`, `FIELD_MAP_TARGET_MISSING`, `FIELD_MAP_TARGET_COMPUTED` (dest must be writable scalar), `FIELD_MAP_TYPE_INCOMPATIBLE` (array-typed source/dest rejected), `FIELD_MAP_COLLISION` (two sources targeting the same dest). + ### MCP server — sync_base compare, curatable changeset, selective apply (2026-06-19) #### Added diff --git a/CLAUDE.md b/CLAUDE.md index ea74d29..ee967c0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,7 +65,7 @@ Key files: - `dev-tools/` — reverse-engineering helpers (capture.js, analyze.js, debug-*.js), **gitignored** — kept locally for future reference when Airtable changes their internal API - `src/daemon/` — daemon subsystem (see daemon subsection below) - `src/safe-write.js` — atomic JSON write helper (used by lockfile, token, and settings) -- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: diff | plan | apply | status | reconcile`) in the `sync` category. Out of scope: retypes, deletions (M4). `mode=diff` (read-only): comprehensive schema comparison via pure `src/sync/compare.js` — every difference classified as **drift** (sync enforces: table/field/view existence, field type/options/choices, view filters/sorts/groups content, column visibility, colors/cover/frozen/rowHeight, primary, form), **best-effort** (sync applies but doesn't guarantee: field/view/column order, sort/group clause order), or **not-synced** (view sections — now captured by snapshot). Verdicts: `identical` (zero diffs) or `converged` (zero drift; best-effort/not-synced may remain). Persists full diff to `~/.airtable-user-mcp/sync/__/diff-.json`; returns a token-budgeted DIGEST (verdicts + class counts + per-table rollup + capped driftSample ≤25). Drill in with `detail="
"` (params: `detail`, `diffId`, `offset`, `limit`). `mode=plan` now emits a curatable **changeset**: each action carries a stable name-based `changeId` (`|
|`), a `class`, and an `apply:true` flag; digest includes a `sample` + `editHint`. New `direction` param (`to-dest` default | `to-source`) targets either base. View sync: `snapshot` also captures collaborative views + live config + **view sections** (not-synced class); `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported); accepts a `skip:[changeId]` list (and honors `apply:false` on changeset entries) — applies everything except removed changes (journal-safe; skipped dependencies degrade to UNRESOLVABLE_REF). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe by filename+size); then restores record-referencing view filters stripped during view sync. Throttle handling = per-request 429 backoff in the auth queue (the per-call limiter does NOT gate the internal per-row POSTs inside createRecords/updateRecords/addLinkItems); resumable (records journal, per-chunk persist of ~50 creates); continue-on-failure (per-record errors are warnings). `mode=status` polls a background job by planId — returns running/done/failed + live records-mapped count. `mode=reconcile` **prunes** dead record-map entries (existence-prune; per-table natural-key re-match NOT yet implemented; reconcile does NOT de-duplicate records). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json, diff-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile), `compare.js` (pure classified compare, no I/O); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. +- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: diff | plan | apply | status | reconcile`) in the `sync` category. Out of scope: retypes, deletions (M4). `mode=diff` (read-only): comprehensive schema comparison via pure `src/sync/compare.js` — every difference classified as **drift** (sync enforces: table/field/view existence, field type/options/choices, view filters/sorts/groups content, column visibility, colors/cover/frozen/rowHeight, primary, form), **best-effort** (sync applies but doesn't guarantee: field/view/column order, sort/group clause order), or **not-synced** (view sections — now captured by snapshot). Verdicts: `identical` (zero diffs) or `converged` (zero drift; best-effort/not-synced may remain). Persists full diff to `~/.airtable-user-mcp/sync/__/diff-.json`; returns a token-budgeted DIGEST (verdicts + class counts + per-table rollup + capped driftSample ≤25). Drill in with `detail="
"` (params: `detail`, `diffId`, `offset`, `limit`). `mode=plan` now emits a curatable **changeset**: each action carries a stable name-based `changeId` (`|
|`), a `class`, and an `apply:true` flag; digest includes a `sample` + `editHint`. New `direction` param (`to-dest` default | `to-source`) targets either base. View sync: `snapshot` also captures collaborative views + live config + **view sections** (not-synced class); `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported); accepts a `skip:[changeId]` list (and honors `apply:false` on changeset entries) — applies everything except removed changes (journal-safe; skipped dependencies degrade to UNRESOLVABLE_REF). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe by filename+size); then restores record-referencing view filters stripped during view sync. Throttle handling = per-request 429 backoff in the auth queue (the per-call limiter does NOT gate the internal per-row POSTs inside createRecords/updateRecords/addLinkItems); resumable (records journal, per-chunk persist of ~50 creates); continue-on-failure (per-record errors are warnings). `mode=status` polls a background job by planId — returns running/done/failed + live records-mapped count. `mode=reconcile` **prunes** dead record-map entries (existence-prune; per-table natural-key re-match NOT yet implemented; reconcile does NOT de-duplicate records). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json, diff-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile), `compare.js` (pure classified compare, no I/O); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. Reconciliation policy (`src/sync/policy.js`): `mode=apply` accepts a `policy` param (`mirror` | `overlay` | `preserve`, default `overlay`) that controls two axes — **extras** (what to do with dest-only records) and **conflicts** (whose value wins when both bases have the record). Presets: `mirror` = {remove, source-wins} (make dest identical to source); `overlay` = {keep, source-wins} (today's default: keep dest-only rows, source updates win); `preserve` = {keep, dest-wins} (keep dest-only rows and never overwrite dest edits). `policyOverrides` (`{ [tableName]: preset }`) overrides the global preset per table. `confirmDeletions` (boolean, default false) is a safety gate for `mirror`: without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count but deletes nothing; set `confirmDeletions:true` to actually delete. `pruneRecords` also clears the blank scaffolding rows that `createTable` seeds (same deletion gate). Schema/view/attachment deletion under mirror and natural-key record matching are deferred follow-ups. Custom field mappings: `fieldMappings` (`{ [tableName]: { sourceFieldName: destFieldName } }`) injects a source field's value into a different writable scalar dest field during record sync — useful when the source is computed (e.g., `autoNumber`/formula) and you want its value preserved in a writable dest field (classic use: inject source autoNumber `Code` into dest text field `InjectID` so a dest formula can reconstruct original identity). Fail-fast pre-flight validation in `apply` (and dry-run in `plan`/`diff` via `fieldMappingErrors` in the machine output) with codes: `FIELD_MAP_TABLE_MISSING`, `FIELD_MAP_SOURCE_MISSING`, `FIELD_MAP_TARGET_MISSING`, `FIELD_MAP_TARGET_COMPUTED` (dest must be writable scalar), `FIELD_MAP_TYPE_INCOMPATIBLE` (array-typed source/dest not supported), `FIELD_MAP_COLLISION` (two sources map to same dest); an invalid mapping aborts `apply` synchronously (`FIELD_MAP_INVALID` error code). #### packages/mcp-server — Daemon subsystem diff --git a/packages/mcp-server/README.md b/packages/mcp-server/README.md index fbcf44e..b8803cd 100644 --- a/packages/mcp-server/README.md +++ b/packages/mcp-server/README.md @@ -523,7 +523,7 @@ Saved row scaffolds Airtable surfaces under "+ Add record" and the row-create ex | Tool | Description | |:-----|:------------| -| `sync_base` | Copy a base's schema, views, and records to another base. `mode=diff` (read-only) compares two bases and classifies every difference as **drift** (sync enforces), **best-effort** (sync applies, not guaranteed), or **not-synced** (view sections). Returns a token-budgeted digest (verdicts + class counts + driftSample); drill in with `detail="
"`. Verdicts: `identical` or `converged`. `mode=plan` snapshots both bases and produces a curatable **changeset** — each action has a stable `changeId` (`\|
\|`), a `class`, and an `apply:true` flag. New `direction` param (`to-dest` default \| `to-source`) selects which base is written. `mode=apply` executes a saved plan — creates tables, reconciles the primary field, creates scalar/link/computed fields with source→dest reference remapping and formula validation, applies non-destructive field updates, syncs collaborative views (idempotent; personal views skipped; orphan views reported), then launches **background** record sync and returns a `jobId` immediately. Accepts a `skip:[changeId]` list (or `apply:false` entries in the changeset) to exclude specific changes; skipped dependencies degrade to UNRESOLVABLE_REF. Record sync: Pass 1 writes scalar + select cells (computed fields never written) and builds a persisted rec→rec id map; Pass 2 writes linked-record cells (unresolved targets reported); then attachments (download→re-upload, deduped by filename+size); then restores record-referencing view filters. Throttling via per-request 429 backoff; resumable (records journal, per-chunk persist); continue-on-failure. `mode=status` polls the background job by planId (running/done/failed + live count). `mode=reconcile` **prunes** the record id map (existence-prune; natural-key re-match planned; does not de-duplicate). Drift-guarded and resumable via an on-disk journal. Out of scope: type-changing retypes, deletions. | +| `sync_base` | Copy a base's schema, views, and records to another base. `mode=diff` (read-only) compares two bases and classifies every difference as **drift** (sync enforces), **best-effort** (sync applies, not guaranteed), or **not-synced** (view sections). Returns a token-budgeted digest (verdicts + class counts + driftSample); drill in with `detail="
"`. Verdicts: `identical` or `converged`. `mode=plan` snapshots both bases and produces a curatable **changeset** — each action has a stable `changeId` (`\|
\|`), a `class`, and an `apply:true` flag. New `direction` param (`to-dest` default \| `to-source`) selects which base is written. `mode=apply` executes a saved plan — creates tables, reconciles the primary field, creates scalar/link/computed fields with source→dest reference remapping and formula validation, applies non-destructive field updates, syncs collaborative views (idempotent; personal views skipped; orphan views reported), then launches **background** record sync and returns a `jobId` immediately. Accepts a `skip:[changeId]` list (or `apply:false` entries in the changeset) to exclude specific changes; skipped dependencies degrade to UNRESOLVABLE_REF. Record sync: Pass 1 writes scalar + select cells (computed fields never written) and builds a persisted rec→rec id map; Pass 2 writes linked-record cells (unresolved targets reported); then attachments (download→re-upload, deduped by filename+size); then restores record-referencing view filters. Throttling via per-request 429 backoff; resumable (records journal, per-chunk persist); continue-on-failure. `mode=status` polls the background job by planId (running/done/failed + live count). `mode=reconcile` **prunes** the record id map (existence-prune; natural-key re-match planned; does not de-duplicate). Drift-guarded and resumable via an on-disk journal. Out of scope: type-changing retypes, deletions. **New params:** `policy` (`mirror`\|`overlay`\|`preserve`, default `overlay`) — reconciliation preset: `mirror` deletes dest-only records and lets source win conflicts; `overlay` keeps dest-only records and lets source win (default); `preserve` keeps dest-only records and never overwrites dest edits. `policyOverrides` (`{ [tableName]: preset }`) overrides the global preset per table. `confirmDeletions` (boolean) — safety gate for `mirror`/remove-extras: without it, orphan deletions are reported as `DELETION_GATED` and nothing is deleted; set `true` to apply. `fieldMappings` (`{ [tableName]: { sourceField: destField } }`) — inject a source field's value (including computed sources like `autoNumber`) into a different writable scalar dest field; validated pre-flight with fail-fast abort on error (`FIELD_MAP_INVALID`); `mode=plan`/`mode=diff` run as dry-check and return `fieldMappingErrors`. Example: `{ "Games": { "Code": "InjectID" } }` injects a source autoNumber into a dest text field so a dest formula can reconstruct original identity. | **Typical diff → curate → apply workflow:** 1. `mode=diff` — compare bases, review the digest. Verdict `converged` means no drift; `identical` means nothing to do. From 03ed538e925cc3488af845c4462420f533c13afc Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 20 Jun 2026 13:15:34 +0300 Subject: [PATCH 118/246] fix(docs): clarify createTable scaffolding-row clearing is unconditional MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CHANGELOG.md and CLAUDE.md incorrectly tied SCAFFOLDING_ROWS_KEPT warning and confirmDeletions gate to pruneRecords. Corrected to clarify that: - createTable unconditionally clears ~3 blank scaffolding rows (best-effort, emits SCAFFOLDING_ROWS_KEPT warning on failure) — separate mechanism. - pruneRecords under mirror+confirmDeletions removes dest-only orphans (including any leftover blank rows) as ordinary orphan prune, independent of the scaffolding-clearing mechanism. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 2 +- CLAUDE.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 632833c..299bd60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how #### Added - `sync_base mode=apply` — **records reconciliation policy**: new `policy` param (`mirror` | `overlay` | `preserve`, default `overlay`) controls two independent axes — **extras** (keep or remove dest-only records) and **conflicts** (source-wins or dest-wins when both bases hold the record). Presets: `mirror` = {remove, source-wins} (make dest identical to source), `overlay` = {keep, source-wins} (default — today's existing behavior), `preserve` = {keep, dest-wins} (never overwrite dest edits). `policyOverrides` (`{ [tableName]: preset }`) overrides the global preset per table. -- `sync_base mode=apply` — **deletion gate** (`confirmDeletions`): under `mirror` policy (or any table override that removes extras), dest-only orphan records are not deleted unless `confirmDeletions: true`. Without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count and does nothing — poll `mode=status` to see counts, then re-run with `confirmDeletions:true` to apply. `pruneRecords` also clears the blank scaffolding rows seeded by `createTable` under mirror (same gate; `SCAFFOLDING_ROWS_KEPT` warning on failure). Schema/view/attachment deletion under mirror and natural-key record matching are deferred follow-ups. +- `sync_base mode=apply` — **deletion gate** (`confirmDeletions`): under `mirror` policy (or any table override that removes extras), dest-only orphan records are not deleted unless `confirmDeletions: true`. Without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count and does nothing — poll `mode=status` to see counts, then re-run with `confirmDeletions:true` to apply. Separately, `createTable` unconditionally clears the ~3 blank scaffolding rows it seeds (best-effort; `SCAFFOLDING_ROWS_KEPT` warning if the deletion attempt fails); `pruneRecords` under `mirror`+`confirmDeletions` additionally removes any leftover dest-only blank rows as part of the ordinary orphan prune. Schema/view/attachment deletion under mirror and natural-key record matching are deferred follow-ups. - `sync_base` — **custom field mappings** (`fieldMappings: { [tableName]: { sourceFieldName: destFieldName } }`): inject a source field's value (including computed fields like `autoNumber` or formula) into a different writable scalar dest field during record sync. Classic use: inject a source `autoNumber` field (`Code`) into a writable dest text field (`InjectID`) so a dest primary-key formula can reconstruct original identity across bases. Fail-fast pre-flight validation aborts `mode=apply` synchronously on any error (`FIELD_MAP_INVALID`); `mode=plan` and `mode=diff` run the same validation as a dry check and return errors in `fieldMappingErrors` (machine output). Validation codes: `FIELD_MAP_TABLE_MISSING`, `FIELD_MAP_SOURCE_MISSING`, `FIELD_MAP_TARGET_MISSING`, `FIELD_MAP_TARGET_COMPUTED` (dest must be writable scalar), `FIELD_MAP_TYPE_INCOMPATIBLE` (array-typed source/dest rejected), `FIELD_MAP_COLLISION` (two sources targeting the same dest). ### MCP server — sync_base compare, curatable changeset, selective apply (2026-06-19) diff --git a/CLAUDE.md b/CLAUDE.md index ee967c0..ce9e3b8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,7 +65,7 @@ Key files: - `dev-tools/` — reverse-engineering helpers (capture.js, analyze.js, debug-*.js), **gitignored** — kept locally for future reference when Airtable changes their internal API - `src/daemon/` — daemon subsystem (see daemon subsection below) - `src/safe-write.js` — atomic JSON write helper (used by lockfile, token, and settings) -- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: diff | plan | apply | status | reconcile`) in the `sync` category. Out of scope: retypes, deletions (M4). `mode=diff` (read-only): comprehensive schema comparison via pure `src/sync/compare.js` — every difference classified as **drift** (sync enforces: table/field/view existence, field type/options/choices, view filters/sorts/groups content, column visibility, colors/cover/frozen/rowHeight, primary, form), **best-effort** (sync applies but doesn't guarantee: field/view/column order, sort/group clause order), or **not-synced** (view sections — now captured by snapshot). Verdicts: `identical` (zero diffs) or `converged` (zero drift; best-effort/not-synced may remain). Persists full diff to `~/.airtable-user-mcp/sync/__/diff-.json`; returns a token-budgeted DIGEST (verdicts + class counts + per-table rollup + capped driftSample ≤25). Drill in with `detail="
"` (params: `detail`, `diffId`, `offset`, `limit`). `mode=plan` now emits a curatable **changeset**: each action carries a stable name-based `changeId` (`|
|`), a `class`, and an `apply:true` flag; digest includes a `sample` + `editHint`. New `direction` param (`to-dest` default | `to-source`) targets either base. View sync: `snapshot` also captures collaborative views + live config + **view sections** (not-synced class); `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported); accepts a `skip:[changeId]` list (and honors `apply:false` on changeset entries) — applies everything except removed changes (journal-safe; skipped dependencies degrade to UNRESOLVABLE_REF). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe by filename+size); then restores record-referencing view filters stripped during view sync. Throttle handling = per-request 429 backoff in the auth queue (the per-call limiter does NOT gate the internal per-row POSTs inside createRecords/updateRecords/addLinkItems); resumable (records journal, per-chunk persist of ~50 creates); continue-on-failure (per-record errors are warnings). `mode=status` polls a background job by planId — returns running/done/failed + live records-mapped count. `mode=reconcile` **prunes** dead record-map entries (existence-prune; per-table natural-key re-match NOT yet implemented; reconcile does NOT de-duplicate records). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json, diff-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile), `compare.js` (pure classified compare, no I/O); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. Reconciliation policy (`src/sync/policy.js`): `mode=apply` accepts a `policy` param (`mirror` | `overlay` | `preserve`, default `overlay`) that controls two axes — **extras** (what to do with dest-only records) and **conflicts** (whose value wins when both bases have the record). Presets: `mirror` = {remove, source-wins} (make dest identical to source); `overlay` = {keep, source-wins} (today's default: keep dest-only rows, source updates win); `preserve` = {keep, dest-wins} (keep dest-only rows and never overwrite dest edits). `policyOverrides` (`{ [tableName]: preset }`) overrides the global preset per table. `confirmDeletions` (boolean, default false) is a safety gate for `mirror`: without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count but deletes nothing; set `confirmDeletions:true` to actually delete. `pruneRecords` also clears the blank scaffolding rows that `createTable` seeds (same deletion gate). Schema/view/attachment deletion under mirror and natural-key record matching are deferred follow-ups. Custom field mappings: `fieldMappings` (`{ [tableName]: { sourceFieldName: destFieldName } }`) injects a source field's value into a different writable scalar dest field during record sync — useful when the source is computed (e.g., `autoNumber`/formula) and you want its value preserved in a writable dest field (classic use: inject source autoNumber `Code` into dest text field `InjectID` so a dest formula can reconstruct original identity). Fail-fast pre-flight validation in `apply` (and dry-run in `plan`/`diff` via `fieldMappingErrors` in the machine output) with codes: `FIELD_MAP_TABLE_MISSING`, `FIELD_MAP_SOURCE_MISSING`, `FIELD_MAP_TARGET_MISSING`, `FIELD_MAP_TARGET_COMPUTED` (dest must be writable scalar), `FIELD_MAP_TYPE_INCOMPATIBLE` (array-typed source/dest not supported), `FIELD_MAP_COLLISION` (two sources map to same dest); an invalid mapping aborts `apply` synchronously (`FIELD_MAP_INVALID` error code). +- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: diff | plan | apply | status | reconcile`) in the `sync` category. Out of scope: retypes, deletions (M4). `mode=diff` (read-only): comprehensive schema comparison via pure `src/sync/compare.js` — every difference classified as **drift** (sync enforces: table/field/view existence, field type/options/choices, view filters/sorts/groups content, column visibility, colors/cover/frozen/rowHeight, primary, form), **best-effort** (sync applies but doesn't guarantee: field/view/column order, sort/group clause order), or **not-synced** (view sections — now captured by snapshot). Verdicts: `identical` (zero diffs) or `converged` (zero drift; best-effort/not-synced may remain). Persists full diff to `~/.airtable-user-mcp/sync/__/diff-.json`; returns a token-budgeted DIGEST (verdicts + class counts + per-table rollup + capped driftSample ≤25). Drill in with `detail="
"` (params: `detail`, `diffId`, `offset`, `limit`). `mode=plan` now emits a curatable **changeset**: each action carries a stable name-based `changeId` (`|
|`), a `class`, and an `apply:true` flag; digest includes a `sample` + `editHint`. New `direction` param (`to-dest` default | `to-source`) targets either base. View sync: `snapshot` also captures collaborative views + live config + **view sections** (not-synced class); `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported); accepts a `skip:[changeId]` list (and honors `apply:false` on changeset entries) — applies everything except removed changes (journal-safe; skipped dependencies degrade to UNRESOLVABLE_REF). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe by filename+size); then restores record-referencing view filters stripped during view sync. Throttle handling = per-request 429 backoff in the auth queue (the per-call limiter does NOT gate the internal per-row POSTs inside createRecords/updateRecords/addLinkItems); resumable (records journal, per-chunk persist of ~50 creates); continue-on-failure (per-record errors are warnings). `mode=status` polls a background job by planId — returns running/done/failed + live records-mapped count. `mode=reconcile` **prunes** dead record-map entries (existence-prune; per-table natural-key re-match NOT yet implemented; reconcile does NOT de-duplicate records). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json, diff-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile), `compare.js` (pure classified compare, no I/O); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. Reconciliation policy (`src/sync/policy.js`): `mode=apply` accepts a `policy` param (`mirror` | `overlay` | `preserve`, default `overlay`) that controls two axes — **extras** (what to do with dest-only records) and **conflicts** (whose value wins when both bases have the record). Presets: `mirror` = {remove, source-wins} (make dest identical to source); `overlay` = {keep, source-wins} (today's default: keep dest-only rows, source updates win); `preserve` = {keep, dest-wins} (keep dest-only rows and never overwrite dest edits). `policyOverrides` (`{ [tableName]: preset }`) overrides the global preset per table. `confirmDeletions` (boolean, default false) is a safety gate for `mirror`: without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count but deletes nothing; set `confirmDeletions:true` to actually delete. Separately, `createTable` unconditionally clears the ~3 blank scaffolding rows it seeds (best-effort; `SCAFFOLDING_ROWS_KEPT` warning if the deletion attempt fails); `pruneRecords` under `mirror`+`confirmDeletions` additionally removes any leftover dest-only blank rows as part of the ordinary orphan prune. Schema/view/attachment deletion under mirror and natural-key record matching are deferred follow-ups. Custom field mappings: `fieldMappings` (`{ [tableName]: { sourceFieldName: destFieldName } }`) injects a source field's value into a different writable scalar dest field during record sync — useful when the source is computed (e.g., `autoNumber`/formula) and you want its value preserved in a writable dest field (classic use: inject source autoNumber `Code` into dest text field `InjectID` so a dest formula can reconstruct original identity). Fail-fast pre-flight validation in `apply` (and dry-run in `plan`/`diff` via `fieldMappingErrors` in the machine output) with codes: `FIELD_MAP_TABLE_MISSING`, `FIELD_MAP_SOURCE_MISSING`, `FIELD_MAP_TARGET_MISSING`, `FIELD_MAP_TARGET_COMPUTED` (dest must be writable scalar), `FIELD_MAP_TYPE_INCOMPATIBLE` (array-typed source/dest not supported), `FIELD_MAP_COLLISION` (two sources map to same dest); an invalid mapping aborts `apply` synchronously (`FIELD_MAP_INVALID` error code). #### packages/mcp-server — Daemon subsystem From ab1fd2f61ca9f8e8c81bc0478826d91e98a3c4f7 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 20 Jun 2026 13:25:44 +0300 Subject: [PATCH 119/246] =?UTF-8?q?fix(sync):=20pruneRecords=20=E2=80=94?= =?UTF-8?q?=20snapshot-derived=20orphan=20detection,=20M3=20scoping,=20del?= =?UTF-8?q?ete=20chunking?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I1: replace queryRecords (silent 100-row cap, possible personal view) with the already-attached destSnapshot.tables[].records rows (fetched at limit 1000 by snapshotTableRecords). Orphan list is now complete and consistent with the rest of the engine. PRUNE_QUERY_FAILED warning removed; viewId derivation prefers a non-personal collaborative view. M3: skip dest-only tables (id not in idmap.tables values) — deleting their rows while keeping the table shell is an inconsistent half-state; schema-orphan deletion is deferred. Chunking: orphan deletes sent in batches of 50 ids per deleteRecords call (continue-on-failure per chunk; confirmDeletions gate preserved). Tests: updated pruneClient + destSnap2 fixtures to supply rows via .records (no queryRecords mock); added M3 guard test + >50-orphan chunking test. Full suite: 626/626 pass. check-tool-sync: green. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/records.js | 51 +++++++++-- .../test/sync/test-records-policy.test.js | 87 +++++++++++++++++-- 2 files changed, 120 insertions(+), 18 deletions(-) diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index f95d9f8..190077e 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -1094,10 +1094,16 @@ export async function applyRecords({ client, sourceBaseId, destBaseId, planId, r * Gated by confirmDeletions — without it, nothing is deleted and a DELETION_GATED warning carries * the would-delete count. result.deleted accumulates actual deletions. * + * Orphan detection uses the already-attached snapshot records (`.records` populated by + * applyRecords → snapshotTableRecords at limit 1000) instead of re-querying. This avoids + * the silent 100-row cap from queryRecords and the potential for a personal/filtered view. + * NOTE: tables with >1000 rows are still bounded by the snapshot's limit — consistent with + * the engine's existing RECORD_COUNT behaviour (pagination is a follow-up). + * * @param {object} opts * @param {object} opts.client - AirtableClient instance - * @param {object} opts.destSnapshot - dest base snapshot (tables with views) - * @param {object} opts.idmap - live id-map (.records map srcId → destId) + * @param {object} opts.destSnapshot - dest base snapshot (tables with .records + views) + * @param {object} opts.idmap - live id-map (.tables src→dest, .records srcId→destId) * @param {string} opts.policy - global preset name ('mirror' | 'overlay' | 'preserve') * @param {object} [opts.policyOverrides]- per-table preset overrides { [tableName]: preset } * @param {boolean} opts.confirmDeletions - if false, gate deletions (push DELETION_GATED warning) @@ -1109,23 +1115,50 @@ export async function pruneRecords({ client, destSnapshot, idmap, policy, policy if (result.deleted == null) result.deleted = 0; if (result.failed == null) result.failed = 0; if (!result.warnings) result.warnings = []; + const mappedDestIds = new Set(Object.values(idmap.records || {})); + + // M3 guard: only prune tables that are the dest-side of a matched pair. + // A dest table whose id is NOT a value in idmap.tables is a dest-only (schema-orphan) table — + // deleting its rows while keeping the table shell is an inconsistent half-state; skip it. + const matchedDestTableIds = new Set(Object.values(idmap.tables || {})); + + const DELETE_CHUNK = 50; + for (const t of (destSnapshot.tables || [])) { + // M3: skip dest-only tables (not matched to any source table) + if (!matchedDestTableIds.has(t.id)) continue; + const { extras } = resolvePolicy(policy, policyOverrides, t.name); if (extras !== 'remove') continue; - const viewId = (t.views || [])[0]?.id; - if (!viewId) continue; - let rows; - try { rows = (await limiter.run(() => client.queryRecords(destSnapshot.baseId, t.id, viewId))).summary?.rows || []; } - catch (e) { result.warnings.push({ code: 'PRUNE_QUERY_FAILED', message: `Table "${t.name}": ${e.message ?? e}` }); continue; } + + // Prefer the first collaborative (non-personal) view; fall back to first view of any type. + const views = t.views || []; + const viewId = views.find((v) => !v.personalForUserId)?.id ?? views[0]?.id; + + // Derive orphans from the already-attached snapshot rows (fetched at limit 1000 by applyRecords). + // Newly-created records are in idmap.records so they are NOT orphans. + const rows = t.records || []; const orphans = rows.map((r) => r.id).filter((id) => !mappedDestIds.has(id)); if (orphans.length === 0) continue; + if (!confirmDeletions) { result.warnings.push({ code: 'DELETION_GATED', message: `Table "${t.name}": ${orphans.length} dest-only record(s) would be deleted under mirror — re-run with confirmDeletions:true` }); continue; } - try { await limiter.run(() => client.deleteRecords(destSnapshot.baseId, t.id, orphans, { viewId })); result.deleted += orphans.length; } - catch (e) { result.failed += orphans.length; result.warnings.push({ code: 'RECORD_DELETE_FAILED', message: `Table "${t.name}": ${e.message ?? e}` }); } + + // Chunk deletions (~50 ids per call) — now that the 100-row cap is lifted, orphan counts + // can be large; deleteRecords sends all ids in one destroyMultipleRows so we batch here. + for (let i = 0; i < orphans.length; i += DELETE_CHUNK) { + const chunk = orphans.slice(i, i + DELETE_CHUNK); + try { + await limiter.run(() => client.deleteRecords(destSnapshot.baseId, t.id, chunk, { viewId })); + result.deleted += chunk.length; + } catch (e) { + result.failed += chunk.length; + result.warnings.push({ code: 'RECORD_DELETE_FAILED', message: `Table "${t.name}": ${e.message ?? e}` }); + } + } } } diff --git a/packages/mcp-server/test/sync/test-records-policy.test.js b/packages/mcp-server/test/sync/test-records-policy.test.js index 104aec3..be671a5 100644 --- a/packages/mcp-server/test/sync/test-records-policy.test.js +++ b/packages/mcp-server/test/sync/test-records-policy.test.js @@ -109,19 +109,31 @@ describe('Pass1 conflict policy', () => { // ── Task 6: pruneRecords (extras axis) ─────────────────────────────────────── -function pruneClient(rowsByTable) { +// pruneClient: rows are now supplied on destSnapshot.tables[].records, NOT via queryRecords. +// deleteRecords is still mocked; queryRecords is omitted (pruneRecords no longer calls it). +function pruneClient() { const calls = { deleted: [] }; return { calls, - async queryRecords(appId, tableId) { return { summary: { rows: (rowsByTable[tableId] || []).map((id) => ({ id, fields: {} })) } }; }, async deleteRecords(appId, tableId, rowIds) { calls.deleted.push({ tableId, rowIds }); return { deleted: rowIds.length }; }, }; } -const destSnap2 = () => ({ baseId: 'appD', tables: [{ id: 'tD', name: 'Games', views: [{ id: 'viwD' }], fields: [] }] }); + +// destSnap2: table carries .records so pruneRecords can derive orphans without re-querying. +const destSnap2 = () => ({ + baseId: 'appD', + tables: [{ + id: 'tD', + name: 'Games', + views: [{ id: 'viwD' }], + fields: [], + records: [{ id: 'recKeep', cellValuesByColumnId: {} }, { id: 'recOrphan', cellValuesByColumnId: {} }], + }], +}); describe('pruneRecords (extras axis)', () => { it('mirror + confirmDeletions deletes orphan dest rows (not in idmap.records)', async () => { - const client = pruneClient({ tD: ['recKeep', 'recOrphan'] }); + const client = pruneClient(); const idmap = { tables: { tS: 'tD' }, fields: {}, records: { recS1: 'recKeep' } }; const result = { created: 0, updated: 0, skipped: 0, failed: 0, deleted: 0, warnings: [] }; await pruneRecords({ client, destSnapshot: destSnap2(), idmap, policy: 'mirror', confirmDeletions: true, limiter: { run: (fn) => fn() }, result }); @@ -129,7 +141,7 @@ describe('pruneRecords (extras axis)', () => { assert.equal(result.deleted, 1); }); it('mirror WITHOUT confirmDeletions deletes nothing + warns DELETION_GATED', async () => { - const client = pruneClient({ tD: ['recKeep', 'recOrphan'] }); + const client = pruneClient(); const idmap = { tables: { tS: 'tD' }, fields: {}, records: { recS1: 'recKeep' } }; const result = { created: 0, updated: 0, skipped: 0, failed: 0, deleted: 0, warnings: [] }; await pruneRecords({ client, destSnapshot: destSnap2(), idmap, policy: 'mirror', confirmDeletions: false, limiter: { run: (fn) => fn() }, result }); @@ -138,23 +150,33 @@ describe('pruneRecords (extras axis)', () => { assert.ok(w && /1/.test(w.message), 'gated warning with count 1'); }); it('keep/preserve table is never pruned', async () => { - const client = pruneClient({ tD: ['recKeep', 'recOrphan'] }); + const client = pruneClient(); const idmap = { tables: { tS: 'tD' }, fields: {}, records: { recS1: 'recKeep' } }; const result = { created: 0, updated: 0, skipped: 0, failed: 0, deleted: 0, warnings: [] }; await pruneRecords({ client, destSnapshot: destSnap2(), idmap, policy: 'overlay', confirmDeletions: true, limiter: { run: (fn) => fn() }, result }); assert.equal(client.calls.deleted.length, 0); }); it('deletion failure with missing result.failed/warnings guards against NaN and throw', async () => { + // Table has one orphan row in .records (not in idmap.records → orphan); deleteRecords throws. + const destSnapWithOrphan = { + baseId: 'appD', + tables: [{ + id: 'tD', + name: 'Games', + views: [{ id: 'viwD' }], + fields: [], + records: [{ id: 'recOrphan', cellValuesByColumnId: {} }], + }], + }; const failingClient = { calls: { deleted: [] }, - async queryRecords(appId, tableId) { return { summary: { rows: [{ id: 'recOrphan', fields: {} }] } }; }, async deleteRecords() { throw new Error('delete failed'); }, }; const idmap = { tables: { tS: 'tD' }, fields: {}, records: {} }; // empty records → orphan exists - const result = { deleted: 0 }; // omit failed and warnings + const result = { deleted: 0 }; // omit failed and warnings — guards against NaN/throw await pruneRecords({ client: failingClient, - destSnapshot: destSnap2(), + destSnapshot: destSnapWithOrphan, idmap, policy: 'mirror', confirmDeletions: true, @@ -167,4 +189,51 @@ describe('pruneRecords (extras axis)', () => { const deleteFailedWarning = result.warnings.find((w) => w.code === 'RECORD_DELETE_FAILED'); assert.ok(deleteFailedWarning, 'RECORD_DELETE_FAILED warning present'); }); + + // M3: a dest table whose id is NOT a value in idmap.tables is NOT pruned, even under mirror+confirmDeletions. + it('M3: dest-only table (not matched to any source table) is never pruned', async () => { + // tD2 is a dest-only table — its id does not appear in idmap.tables values. + const destSnapDestOnly = { + baseId: 'appD', + tables: [{ + id: 'tD2', + name: 'DestOnly', + views: [{ id: 'viwD2' }], + fields: [], + records: [{ id: 'recX', cellValuesByColumnId: {} }], + }], + }; + const client = pruneClient(); + // idmap.tables maps tS → tD (a different table), so tD2 is not matched. + const idmap = { tables: { tS: 'tD' }, fields: {}, records: {} }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, deleted: 0, warnings: [] }; + await pruneRecords({ client, destSnapshot: destSnapDestOnly, idmap, policy: 'mirror', confirmDeletions: true, limiter: { run: (fn) => fn() }, result }); + assert.equal(client.calls.deleted.length, 0, 'no deleteRecords call for a dest-only table'); + assert.equal(result.deleted, 0); + }); + + // Delete chunking: >50 orphans should produce multiple deleteRecords calls of ≤50 each. + it('>50 orphans produce multiple delete chunks of ≤50 ids each', async () => { + // 75 dest records; none in idmap.records → all are orphans. + const allRecords = Array.from({ length: 75 }, (_, i) => ({ id: `recOrphan${i}`, cellValuesByColumnId: {} })); + const destSnapLarge = { + baseId: 'appD', + tables: [{ + id: 'tD', + name: 'Games', + views: [{ id: 'viwD' }], + fields: [], + records: allRecords, + }], + }; + const client = pruneClient(); + const idmap = { tables: { tS: 'tD' }, fields: {}, records: {} }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, deleted: 0, warnings: [] }; + await pruneRecords({ client, destSnapshot: destSnapLarge, idmap, policy: 'mirror', confirmDeletions: true, limiter: { run: (fn) => fn() }, result }); + // 75 orphans → 2 chunks (50 + 25) + assert.equal(client.calls.deleted.length, 2, 'two deleteRecords chunks for 75 orphans'); + assert.equal(client.calls.deleted[0].rowIds.length, 50, 'first chunk has 50 ids'); + assert.equal(client.calls.deleted[1].rowIds.length, 25, 'second chunk has 25 ids'); + assert.equal(result.deleted, 75, 'all 75 orphans counted as deleted'); + }); }); From 772df216e179ca2be95d5a60e40762a654cd9aa9 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 20 Jun 2026 17:56:15 +0300 Subject: [PATCH 120/246] chore: WIP auth/daemon/webview changes (separated from records-policy) Pre-existing uncommitted work a fix subagent accidentally bundled into the records-policy feature commit; split out so the feature history is clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/extension/package.json | 2 +- packages/extension/src/extension.ts | 2 +- packages/extension/src/mcp/auth-manager.ts | 112 +- packages/extension/src/mcp/daemon-manager.ts | 73 +- .../extension/src/test/auth-manager.test.ts | 89 + .../extension/src/test/daemon-manager.test.ts | 46 + .../src/webview/DashboardProvider.ts | 14 +- packages/mcp-server/src/daemon/server.js | 34 + .../sync/fixtures/get-application-data.json | 5308 +++++++++++++++++ .../test/test-daemon-server.test.js | 53 + packages/webview/src/lib/friendlyError.ts | 20 +- .../webview/src/test/friendlyError.test.ts | 52 + 12 files changed, 5793 insertions(+), 12 deletions(-) create mode 100644 packages/extension/src/test/auth-manager.test.ts create mode 100644 packages/mcp-server/test/sync/fixtures/get-application-data.json create mode 100644 packages/webview/src/test/friendlyError.test.ts diff --git a/packages/extension/package.json b/packages/extension/package.json index 80036f9..e5e07d5 100644 --- a/packages/extension/package.json +++ b/packages/extension/package.json @@ -1,7 +1,7 @@ { "name": "airtable-formula", "private": true, - "version": "2.0.94", + "version": "2.1.6", "publisher": "Nskha", "displayName": "Airtable Formulas, Scripts, Automation, MCP & LSP", "description": "Airtable formula, script & automation editor with MCP server (70 tools), language server, and AI skills for VS Code.", diff --git a/packages/extension/src/extension.ts b/packages/extension/src/extension.ts index ebbab21..4b8bd62 100644 --- a/packages/extension/src/extension.ts +++ b/packages/extension/src/extension.ts @@ -562,7 +562,7 @@ export async function activate(context: vscode.ExtensionContext): Promise await toolProfileManager.openConfigFile(); }), vscode.commands.registerCommand('airtable-formula.stopDaemon', async () => { - const result = await daemonManager.stopDaemon(); + const result = await daemonManager.forceStop(); if (result.stopped) { vscode.window.showInformationMessage(`Airtable Formula: Daemon stopped.${result.reason ? ` (${result.reason})` : ''}`); } else { diff --git a/packages/extension/src/mcp/auth-manager.ts b/packages/extension/src/mcp/auth-manager.ts index 9a4243f..1ddb715 100644 --- a/packages/extension/src/mcp/auth-manager.ts +++ b/packages/extension/src/mcp/auth-manager.ts @@ -282,7 +282,15 @@ export class AuthManager implements vscode.Disposable { // ─── Health Check ──────────────────────────────────────────── async checkSession(): Promise { - // Preflight — no point spawning the child process if no browser exists + // Prefer the daemon's SINGLE shared browser. This avoids forking a local + // Chrome that would collide with the daemon over the persistent profile + // (Chrome exit code 21 → the "session dead / network error" failure), and + // it works even when no local browser is installed. + const viaDaemon = await this._checkViaDaemon(); + if (viaDaemon) return viaDaemon; + + // No daemon (stdio mode) or a daemon that predates the endpoint — fall back + // to forking our own health-check. Preflight: no browser, no point spawning. const probe = this.refreshBrowserDetection(); if (!probe.found) { this._updateState({ @@ -333,6 +341,102 @@ export class AuthManager implements vscode.Disposable { return this._state; } + /** + * Verify the session through the daemon's already-open browser. + * + * Returns the resulting AuthState when the daemon handled the check, or + * `null` to signal the caller to fall back to forking a local health-check + * (no daemon, daemon disabled, daemon unreachable, or a daemon too old to + * expose /daemon/session-health). + */ + private async _checkViaDaemon(): Promise { + const dm = this._daemonManager; + if (!dm || !getSettings().mcp.useDaemon) return null; + + let status: Awaited>; + try { + status = await dm.getDaemonStatus(); + } catch { + return null; + } + if (!status.running || !status.healthy || status.port == null || !status.bearerToken) { + return null; + } + + this._updateState({ status: 'checking' }); + + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 30_000); + try { + const resp = await fetch(`http://127.0.0.1:${status.port}/daemon/session-health`, { + headers: { Authorization: `Bearer ${status.bearerToken}` }, + signal: controller.signal, + }); + + // Old daemon without the endpoint — fall back to the fork path. + if (resp.status === 404) return null; + + const now = new Date().toISOString(); + if (!resp.ok) { + this._updateState({ status: 'expired', lastChecked: now, error: `HTTP ${resp.status}` }); + return this._state; + } + + const result = await resp.json().catch(() => null) as + { valid?: boolean; userId?: string | null; status?: number; error?: string } | null; + + if (result?.valid) { + this._updateState({ status: 'valid', userId: result.userId ?? undefined, lastChecked: now, error: undefined }); + } else { + this._updateState({ + status: 'expired', + lastChecked: now, + error: result?.error || (result?.status ? `HTTP ${result.status}` : 'Session invalid'), + }); + } + return this._state; + } catch { + // Daemon became unreachable mid-check — fall back rather than show an error. + return null; + } finally { + clearTimeout(timeout); + } + } + + /** + * Ask the daemon to close its shared browser so an interactive (headful) + * login can open the persistent profile exclusively. Best-effort: if it + * fails, the login runner's own exit-21 retry still covers the common case. + * The daemon re-opens the browser on its next session check / tool call, + * picking up the freshly-written session cookies. + */ + private async _releaseDaemonBrowser(): Promise { + const dm = this._daemonManager; + if (!dm || !getSettings().mcp.useDaemon) return; + + let status: Awaited>; + try { + status = await dm.getDaemonStatus(); + } catch { + return; + } + if (!status.running || !status.healthy || status.port == null || !status.bearerToken) return; + + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 5_000); + try { + await fetch(`http://127.0.0.1:${status.port}/daemon/release-browser`, { + method: 'POST', + headers: { Authorization: `Bearer ${status.bearerToken}` }, + signal: controller.signal, + }); + } catch { + // best-effort + } finally { + clearTimeout(timeout); + } + } + // ─── Login ─────────────────────────────────────────────────── async login(): Promise { @@ -354,6 +458,9 @@ export class AuthManager implements vscode.Disposable { return this._state; } this._updateState({ status: 'logging-in' }); + // Free the profile: the daemon's shared browser must let go before the + // login runner can open the same persistent profile. + await this._releaseDaemonBrowser(); try { // Credentials go over the IPC channel, never the child environment. const result = await this._spawnScript('login-runner.mjs', { @@ -387,6 +494,9 @@ export class AuthManager implements vscode.Disposable { } this._updateState({ status: 'logging-in' }); + // Free the profile: the daemon's shared browser must let go before the + // login runner can open the same persistent profile. + await this._releaseDaemonBrowser(); try { const result = await this._spawnScript( diff --git a/packages/extension/src/mcp/daemon-manager.ts b/packages/extension/src/mcp/daemon-manager.ts index a93a9ec..7b203e1 100644 --- a/packages/extension/src/mcp/daemon-manager.ts +++ b/packages/extension/src/mcp/daemon-manager.ts @@ -2,7 +2,7 @@ import * as vscode from 'vscode'; import * as path from 'path'; import * as fs from 'fs/promises'; import { existsSync, rmSync } from 'fs'; -import { spawn } from 'child_process'; +import { spawn, execFileSync } from 'child_process'; export interface DaemonStatus { running: boolean; @@ -241,6 +241,71 @@ export class DaemonManager implements vscode.Disposable { }; } + /** + * Stop the daemon AND kill anything the lockfile can't see: orphaned daemon + * processes (whose lock was lost/deleted while the process lived) and stray + * Chrome instances still holding the persistent profile. This is what the + * "Stop" button calls — graceful stop alone is a no-op when the lock is gone, + * which leaves a browser holding the profile (Chrome exit code 21 for + * everything after). + */ + async forceStop(): Promise { + const result = await this.stopDaemon(); + const killed = this._sweepOrphans(); + this._reclaimLockfile(); + if (killed > 0) { + return { + stopped: true, + forced: true, + reason: `${result.reason ? result.reason + ' ' : ''}Force-killed ${killed} stray process(es).`, + }; + } + return result; + } + + /** + * Kill orphaned airtable-mcp daemons and stray profile-holding Chromes by + * scanning command lines — independent of the lockfile. Best-effort. + * + * ponytail: matches by `.chrome-profile` / `index.mjs … daemon start` across + * the whole user session, so it also nukes daemons from other installs that + * share ~/.airtable-user-mcp. That's the intent of a "force stop". + */ + private _sweepOrphans(): number { + const pids = new Set(); + try { + if (process.platform === 'win32') { + for (const filter of ["CommandLine like '%.chrome-profile%'", "CommandLine like '%index.mjs%daemon%start%'"]) { + try { + const out = execFileSync('wmic', ['process', 'where', filter, 'get', 'ProcessId', '/format:list'], { encoding: 'utf8', windowsHide: true, timeout: 8000 }); + for (const pid of this._parsePids(out)) if (pid !== process.pid) pids.add(pid); + } catch { /* no matches → wmic exits non-zero */ } + } + for (const pid of pids) { + try { execFileSync('taskkill', ['/PID', String(pid), '/T', '/F'], { stdio: 'ignore', windowsHide: true, timeout: 5000 }); } catch { /* already gone */ } + } + return pids.size; + } + // POSIX: pkill kills directly; we can't easily count, so report 0. + for (const pat of ['\\.chrome-profile', 'index\\.mjs.*daemon.*start']) { + try { execFileSync('pkill', ['-9', '-f', pat], { stdio: 'ignore', timeout: 5000 }); } catch { /* none matched */ } + } + return 0; + } catch { + return pids.size; + } + } + + /** Extract unique positive PIDs from `wmic … get ProcessId /format:list` output. */ + private _parsePids(wmicOutput: string): number[] { + const out: number[] = []; + for (const m of wmicOutput.matchAll(/ProcessId=(\d+)/g)) { + const n = Number(m[1]); + if (Number.isInteger(n) && n > 0 && !out.includes(n)) out.push(n); + } + return out; + } + /** * Proof of identity for kill escalation: /daemon/health, authenticated with * the lockfile's bearer token, must echo the lockfile's uuid. @@ -311,6 +376,12 @@ export class DaemonManager implements vscode.Disposable { AIRTABLE_USER_MCP_HOME: this.configDir, AIRTABLE_HEADLESS_ONLY: '1', NODE_PATH: path.join(this.extensionPath, 'dist', 'node_modules'), + // We spawn `process.execPath` which, in the extension host, is the editor's + // Electron binary — not Node. Without this it would launch the editor (or + // do nothing useful) instead of running the daemon script, so the lockfile + // never appears and ensureDaemon() times out. Harmless when execPath is + // already real Node (the flag is ignored). + ELECTRON_RUN_AS_NODE: '1', }; if (credEnv) Object.assign(env, credEnv); return env; diff --git a/packages/extension/src/test/auth-manager.test.ts b/packages/extension/src/test/auth-manager.test.ts new file mode 100644 index 0000000..24186d9 --- /dev/null +++ b/packages/extension/src/test/auth-manager.test.ts @@ -0,0 +1,89 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import * as http from 'http'; + +// Mock vscode before importing the module under test. +vi.mock('vscode', () => ({ + EventEmitter: vi.fn(() => ({ event: vi.fn(), fire: vi.fn(), dispose: vi.fn() })), + Disposable: vi.fn(), + workspace: { getConfiguration: vi.fn(() => ({ get: (_k: string, d: unknown) => d, inspect: () => undefined })) }, + ConfigurationTarget: { Global: 1 }, + window: { showWarningMessage: vi.fn() }, +})); + +// getSettings is read inside checkSession — control mcp.useDaemon per test. +let useDaemon = true; +vi.mock('../settings.js', () => ({ + getSettings: () => ({ mcp: { useDaemon }, auth: { browserChoice: undefined } }), +})); + +import { AuthManager } from '../mcp/auth-manager.js'; + +type DaemonStatusStub = { + running: boolean; healthy: boolean; port: number | null; bearerToken: string | null; +}; + +const makeAuthManager = (statusStub: DaemonStatusStub) => { + const daemonManager = { getDaemonStatus: async () => statusStub } as any; + return new AuthManager({} as any, '/tmp/ext', daemonManager); +}; + +describe('AuthManager.checkSession routes through the daemon (single browser owner)', () => { + let server: http.Server | undefined; + let port = 0; + let handler: http.RequestListener = (_req, res) => { res.writeHead(404); res.end(); }; + + beforeEach(async () => { + useDaemon = true; + server = http.createServer((req, res) => handler(req, res)); + await new Promise(r => server!.listen(0, '127.0.0.1', r)); + port = (server!.address() as import('net').AddressInfo).port; + }); + + afterEach(async () => { + if (server) { await new Promise(r => server!.close(() => r())); server = undefined; } + }); + + it('reports "valid" from the daemon session-health endpoint without forking a browser', async () => { + handler = (req, res) => { + expect(req.url).toBe('/daemon/session-health'); + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ valid: true, userId: 'usr999' })); + }; + const am = makeAuthManager({ running: true, healthy: true, port, bearerToken: 'tok' }); + const state = await (am as any)._checkViaDaemon(); + expect(state).not.toBeNull(); + expect(state.status).toBe('valid'); + expect(state.userId).toBe('usr999'); + }); + + it('reports "expired" when the daemon says the session is invalid', async () => { + handler = (_req, res) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ valid: false, error: 'Session expired: redirected to /login' })); + }; + const am = makeAuthManager({ running: true, healthy: true, port, bearerToken: 'tok' }); + const state = await (am as any)._checkViaDaemon(); + expect(state.status).toBe('expired'); + expect(state.error).toMatch(/expired/i); + }); + + it('falls back (returns null) when the daemon predates the endpoint (404)', async () => { + handler = (_req, res) => { res.writeHead(404); res.end('Cannot GET /daemon/session-health'); }; + const am = makeAuthManager({ running: true, healthy: true, port, bearerToken: 'tok' }); + const state = await (am as any)._checkViaDaemon(); + expect(state).toBeNull(); + }); + + it('falls back (returns null) when no healthy daemon is running', async () => { + const am = makeAuthManager({ running: false, healthy: false, port: null, bearerToken: null }); + const state = await (am as any)._checkViaDaemon(); + expect(state).toBeNull(); + }); + + it('falls back (returns null) when mcp.useDaemon is disabled', async () => { + useDaemon = false; + const am = makeAuthManager({ running: true, healthy: true, port, bearerToken: 'tok' }); + const state = await (am as any)._checkViaDaemon(); + expect(state).toBeNull(); + }); +}); diff --git a/packages/extension/src/test/daemon-manager.test.ts b/packages/extension/src/test/daemon-manager.test.ts index ff3d000..cc01297 100644 --- a/packages/extension/src/test/daemon-manager.test.ts +++ b/packages/extension/src/test/daemon-manager.test.ts @@ -38,6 +38,14 @@ describe('DaemonManager.buildDaemonEnv', () => { expect(result.AIRTABLE_HEADLESS_ONLY).toBe('1'); }); + it('sets ELECTRON_RUN_AS_NODE=1 so the Electron host binary runs the daemon as Node', () => { + // process.execPath in the extension host is the editor's Electron binary; + // without this flag the spawned child launches the editor instead of the + // daemon → no lockfile → "Timed out waiting for daemon startup". + const result = dm.buildDaemonEnv(); + expect(result.ELECTRON_RUN_AS_NODE).toBe('1'); + }); + it('merges credEnv keys on top of base env', () => { const credEnv = { AIRTABLE_EMAIL: 'test@test.com' }; const result = dm.buildDaemonEnv(credEnv); @@ -244,6 +252,44 @@ describe('DaemonManager.stopDaemon', () => { }); }); +describe('DaemonManager.forceStop', () => { + it('sweeps orphaned processes after the lock-based stop and reports the count', async () => { + const dm = new DaemonManager('/tmp/x', '/tmp/y'); + (dm as any).stopDaemon = vi.fn(async () => ({ stopped: true, forced: false })); + (dm as any)._sweepOrphans = vi.fn(() => 3); + (dm as any)._reclaimLockfile = vi.fn(); + + const r = await dm.forceStop(); + expect((dm as any)._sweepOrphans).toHaveBeenCalled(); + expect(r.stopped).toBe(true); + expect(r.forced).toBe(true); + expect(r.reason).toMatch(/3/); + }); + + it('returns the lock-based result unchanged when nothing stray is found', async () => { + const dm = new DaemonManager('/tmp/x', '/tmp/y'); + (dm as any).stopDaemon = vi.fn(async () => ({ stopped: true, forced: false })); + (dm as any)._sweepOrphans = vi.fn(() => 0); + (dm as any)._reclaimLockfile = vi.fn(); + + const r = await dm.forceStop(); + expect(r.forced).toBe(false); + }); +}); + +describe('DaemonManager._parsePids', () => { + it('extracts unique positive pids from wmic /format:list output', () => { + const dm = new DaemonManager('/tmp/x', '/tmp/y'); + const out = 'ProcessId=1234\r\n\r\nProcessId=5678\r\n\r\nProcessId=1234\r\n'; + expect((dm as any)._parsePids(out)).toEqual([1234, 5678]); + }); + + it('returns empty array when there are no matches', () => { + const dm = new DaemonManager('/tmp/x', '/tmp/y'); + expect((dm as any)._parsePids('No Instance(s) Available.')).toEqual([]); + }); +}); + describe('DaemonManager.restartDaemon', () => { let tmpDir: string; let dm: InstanceType; diff --git a/packages/extension/src/webview/DashboardProvider.ts b/packages/extension/src/webview/DashboardProvider.ts index b2fa543..95e3c32 100644 --- a/packages/extension/src/webview/DashboardProvider.ts +++ b/packages/extension/src/webview/DashboardProvider.ts @@ -572,8 +572,15 @@ export class DashboardProvider implements vscode.WebviewViewProvider { } if (msg.type === 'daemon:stop') { + // Clear any in-flight (possibly stuck) start so the UI doesn't keep + // showing "Starting…/running" after a Stop — that's what makes Stop look + // like it did nothing / acted like a restart. + this._daemonStarting = false; try { - const result = await this._daemonManager?.stopDaemon(); + // forceStop: graceful stop + sweep of orphaned daemons / stray profile + // browsers the lockfile can't see (otherwise "Stop" is a no-op when the + // lock is missing and a browser keeps holding the profile). + const result = await this._daemonManager?.forceStop(); await this.pushState(); if (result && !result.stopped) { const reason = result.reason ?? 'Daemon did not exit.'; @@ -1039,7 +1046,10 @@ export class DashboardProvider implements vscode.WebviewViewProvider { if (this._daemonStarting) { return { running: false, healthy: false, port: null, port_lsp: null, tunnelUrl: null, uptime: null, starting: true }; } - return undefined; + // Explicit "stopped" object, NOT undefined: undefined is dropped during + // webview message serialization, so the store's spread keeps the stale + // "Running" card. A concrete value overwrites it → the UI shows Stopped. + return { running: false, healthy: false, port: null, port_lsp: null, tunnelUrl: null, uptime: null, starting: false }; } return { running: status.running, diff --git a/packages/mcp-server/src/daemon/server.js b/packages/mcp-server/src/daemon/server.js index 7aba285..e0ee85d 100644 --- a/packages/mcp-server/src/daemon/server.js +++ b/packages/mcp-server/src/daemon/server.js @@ -349,6 +349,40 @@ export async function startDaemonServer(options = {}) { res.json({ ok: true, clientId }); }); + // Session health — verified through the daemon's SINGLE shared browser. + // The extension calls this instead of forking its own health-check Chrome; + // two Chromes on one persistent profile collide (Chrome exit code 21), which + // is the root cause of the "session dead / network error" failure mode. + app.get('/daemon/session-health', requireBearer, async (_req, res) => { + try { + if (options.getSessionHealth) { + res.json(await options.getSessionHealth()); + return; + } + await getClient(); // ensure the shared browser is initialized + res.json(await auth.checkSessionHealth()); + } catch (error) { + // init/verify threw (e.g. session expired → login redirect) — report it + // as an invalid session rather than a 500 so the dashboard shows the + // right state and offers re-login. + res.json({ valid: false, error: error instanceof Error ? error.message : String(error) }); + } + }); + + // Release the shared browser so an interactive (headful) login can open the + // persistent profile exclusively. The next getClient()/session-health call + // re-initializes it, picking up the freshly-written session cookies. + app.post('/daemon/release-browser', requireBearer, async (_req, res) => { + try { + if (auth) await auth.close().catch(() => {}); + } finally { + auth = undefined; + client = undefined; + clientInitPromise = null; + } + res.json({ ok: true }); + }); + app.post('/daemon/rotate-token', requireBearer, async (_req, res, next) => { try { currentToken = rotateToken({ tokenPath }); diff --git a/packages/mcp-server/test/sync/fixtures/get-application-data.json b/packages/mcp-server/test/sync/fixtures/get-application-data.json new file mode 100644 index 0000000..0ce7666 --- /dev/null +++ b/packages/mcp-server/test/sync/fixtures/get-application-data.json @@ -0,0 +1,5308 @@ +{ + "msg": "SUCCESS", + "data": { + "appBlanket": { + "workspaceSyncSources": [ + "airtableSharedView" + ], + "packageConstructionInfo": null, + "sandboxInfo": null, + "userInfoById": { + "usrKu6HNqQKma2iWV": { + "id": "usrKu6HNqQKma2iWV", + "firstName": "Karek", + "lastName": "Rami", + "email": "airtable@admins.8shield.net", + "profilePicUrl": "https://dl.airtable.com/.directUploadProfilePics/d302807f0f3bde809cf5ef5c1434c2ee/ed082c02/071014.png", + "permissionLevel": "owner", + "pageBundlePermissionLevelByPageBundleId": {}, + "appBlanketUserState": "active" + } + }, + "userGroupInfoById": {}, + "externalAccountInfoById": {}, + "activeUserIdByAcceptedInviteId": {}, + "isWorkspaceLinkedToEnterpriseAccount": false, + "isGridRootEnterpriseAccount": false, + "enterpriseAccountId": null, + "enterpriseAttachmentRestrictions": { + "restrictionType": "unrestricted", + "attachmentTypeAllowlist": [] + }, + "enterpriseBillingPlanInfo": null, + "topEnterpriseCustomerNameForErrorReporting": null, + "peopleTableMetadataForEnterpriseAssociatedWithApplication": null, + "adminCustomizedHelpMessage": null, + "isAiAllowed": true, + "isInternetAccessEnabledForAiForEnterpriseAccount": false, + "isAiImageGenerationEnabled": true, + "isAiImageAnalysisEnabled": true, + "isAiVideoGenerationEnabled": true, + "isAiWebContentGenerationEnabled": true, + "aiPermissionsState": "enabled", + "aiAssistantPermissionsState": "enabled", + "allowedAiModelProviders": [ + "openAi", + "anthropic", + "amazon", + "selfHosted", + "google" + ], + "allowedAiModelCreatorsByAiModelProvider": { + "openAi": [ + "openAi" + ], + "amazonBedrock": [ + "anthropic", + "meta" + ], + "selfHosted": [ + "sentenceTransformers" + ], + "google": [ + "google" + ] + } + }, + "name": "Untitled Base", + "color": "greenDusty", + "enterpriseColorId": null, + "shouldColorBeUsedAsPageBackground": false, + "icon": null, + "description": null, + "isRevisionHistoryEnabled": true, + "sortTiebreakerKey": null, + "defaultViewMutability": null, + "maintenanceModeSettings": null, + "sharesById": {}, + "workflowSectionsById": { + "wscTuPsT8BfEOTXCy": { + "id": "wscTuPsT8BfEOTXCy", + "name": "More", + "createdByUserId": "usrKu6HNqQKma2iWV", + "isDefaultSection": true, + "workflowOrder": [], + "applicationId": "apphwgei0ETb2zCXz", + "fractionalIndex": null + } + }, + "workflowTriggerConnectionsById": {}, + "applicationTransactionNumber": 80, + "tableSchemas": [ + { + "id": "tblooKJLpdRd2SI3I", + "name": "Table 1", + "primaryColumnId": "fld6ISd4khYuH27l6", + "columns": [ + { + "id": "fld6ISd4khYuH27l6", + "name": "Name", + "type": "text", + "initialCreatedTime": "2026-04-17T15:22:57.000Z", + "initialCreatedByUserId": "usrKu6HNqQKma2iWV", + "lastModifiedTime": "2026-04-17T15:22:57.000Z", + "lastModifiedByUserId": "usrKu6HNqQKma2iWV" + }, + { + "id": "fld0REIF5QK5sdRW8", + "name": "Notes", + "type": "multilineText", + "initialCreatedTime": "2026-04-17T15:22:57.000Z", + "initialCreatedByUserId": "usrKu6HNqQKma2iWV", + "lastModifiedTime": "2026-06-15T19:23:43.000Z", + "lastModifiedByUserId": "usrKu6HNqQKma2iWV" + }, + { + "id": "fldzYJncU1NAXKuOg", + "name": "Assignee", + "type": "multilineText", + "initialCreatedTime": "2026-04-17T15:22:57.000Z", + "initialCreatedByUserId": "usrKu6HNqQKma2iWV", + "lastModifiedTime": "2026-06-15T19:23:43.000Z", + "lastModifiedByUserId": "usrKu6HNqQKma2iWV" + }, + { + "id": "flduztL3p6LJ1r3GR", + "name": "Status", + "type": "select", + "typeOptions": { + "choices": { + "selSOmr7hxDdQOF1X": { + "id": "selSOmr7hxDdQOF1X", + "name": "Todo", + "color": "red" + }, + "selI1uROgLXkHga5l": { + "id": "selI1uROgLXkHga5l", + "name": "In progress", + "color": "yellow" + }, + "selGFbfPNLBugiEN5": { + "id": "selGFbfPNLBugiEN5", + "name": "Done", + "color": "green" + }, + "seltLTY0BfBXhCQ8J": { + "id": "seltLTY0BfBXhCQ8J", + "name": "2test", + "color": "blue" + } + }, + "choiceOrder": [ + "selSOmr7hxDdQOF1X", + "selI1uROgLXkHga5l", + "selGFbfPNLBugiEN5", + "seltLTY0BfBXhCQ8J" + ] + }, + "initialCreatedTime": "2026-04-17T15:22:57.000Z", + "initialCreatedByUserId": "usrKu6HNqQKma2iWV", + "lastModifiedTime": "2026-06-15T19:23:43.000Z", + "lastModifiedByUserId": "usrKu6HNqQKma2iWV" + }, + { + "id": "fldCEPy54raOrxFoL", + "name": "Attachments", + "type": "multilineText", + "initialCreatedTime": "2026-04-17T15:22:57.000Z", + "initialCreatedByUserId": "usrKu6HNqQKma2iWV", + "lastModifiedTime": "2026-06-15T19:23:43.000Z", + "lastModifiedByUserId": "usrKu6HNqQKma2iWV" + }, + { + "id": "fld2iy0JmkuemZZAf", + "name": "Attachment Summary", + "type": "asyncText", + "description": "An AI generated summary of the Attachments field. Upload files to Attachments to generate a summary.", + "typeOptions": { + "sourceType": "ai", + "sourceParams": { + "prompt": [ + "You are a professional document analyst specializing in summarizing content from various file types. Your task is to extract key points and provide a concise summary of the content found in the attached files. Ensure that the summary captures the main ideas and any important details without including unnecessary information.\n\nAnalyze the content of the attached files to identify the main themes and significant details. Summarize these points clearly and concisely, ensuring that the summary is informative and relevant to the document's purpose.\n\nFormat your response as plain text, keeping it concise and to the point. If you cannot summarize the content, output \"Unable to summarize the content.\" Example: \"The document discusses the impact of climate change on polar bears, highlighting the loss of habitat and food sources.\" (Real examples should be longer and more detailed.)\n\nFiles:\n", + { + "field": { + "columnId": "fldCEPy54raOrxFoL" + } + } + ], + "columnIdDependencies": [ + "fldCEPy54raOrxFoL" + ], + "optionalColumnIdDependencies": [], + "model": "default", + "temperature": 0, + "generationMode": "onDemand", + "availableTools": [], + "userConfigurationForPrompt": { + "type": "fullyUserManaged" + }, + "attachmentProcessingStrategy": { + "type": "documentAgent" + }, + "integrationSettings": {} + } + }, + "initialCreatedTime": "2026-04-17T15:22:57.000Z", + "initialCreatedByUserId": "usrKu6HNqQKma2iWV", + "lastModifiedTime": "2026-04-17T15:22:57.000Z", + "lastModifiedByUserId": "usrKu6HNqQKma2iWV" + }, + { + "id": "fldlMHqqi4O6gqPqv", + "name": "This is test url field", + "type": "text", + "description": "This is desc of url field", + "typeOptions": { + "validatorName": "url" + }, + "initialCreatedTime": "2026-04-17T15:23:21.000Z", + "initialCreatedByUserId": "usrKu6HNqQKma2iWV", + "lastModifiedTime": "2026-04-17T15:23:21.000Z", + "lastModifiedByUserId": "usrKu6HNqQKma2iWV" + }, + { + "id": "fldBhukTKsuf9QzBO", + "name": "Date -updated", + "type": "date", + "description": "this is desc updated", + "default": { + "specialValue": "dynamicCurrentDateValue" + }, + "typeOptions": { + "isDateTime": true, + "dateFormat": "Local", + "timeFormat": "24hour", + "timeZone": "UTC", + "shouldDisplayTimeZone": true + }, + "initialCreatedTime": "2026-04-17T15:24:20.000Z", + "initialCreatedByUserId": "usrKu6HNqQKma2iWV", + "lastModifiedTime": "2026-04-17T15:24:43.000Z", + "lastModifiedByUserId": "usrKu6HNqQKma2iWV" + }, + { + "id": "fldC82CKp6sAOaRP7", + "name": "test single select", + "type": "select", + "description": "test desc", + "default": "selZqwkHj1rglLWVj", + "typeOptions": { + "choiceOrder": [ + "selZqwkHj1rglLWVj", + "selueCkiWd8Oo3e4K", + "sellGBnVyysTkEP6B", + "selGlRSVPeujNSFO4", + "selmQQMHVpbOkArE0", + "selwl7vx8a6d68saY", + "selcncTjjv8OR9QTf", + "selvApCpOn5g4zrSH" + ], + "choices": { + "selZqwkHj1rglLWVj": { + "id": "selZqwkHj1rglLWVj", + "color": "blue", + "name": "test 1" + }, + "selueCkiWd8Oo3e4K": { + "id": "selueCkiWd8Oo3e4K", + "color": "cyan", + "name": "test 2" + }, + "sellGBnVyysTkEP6B": { + "id": "sellGBnVyysTkEP6B", + "color": "teal", + "name": "test 3" + }, + "selGlRSVPeujNSFO4": { + "id": "selGlRSVPeujNSFO4", + "color": "green", + "name": "test 4" + }, + "selmQQMHVpbOkArE0": { + "id": "selmQQMHVpbOkArE0", + "color": "orange", + "name": "test 6" + }, + "selwl7vx8a6d68saY": { + "id": "selwl7vx8a6d68saY", + "color": "red", + "name": "test 7" + }, + "selcncTjjv8OR9QTf": { + "id": "selcncTjjv8OR9QTf", + "color": "pink", + "name": "test 8" + }, + "selvApCpOn5g4zrSH": { + "id": "selvApCpOn5g4zrSH", + "color": "purple", + "name": "test 9" + } + }, + "disableColors": false + }, + "initialCreatedTime": "2026-05-28T14:59:37.000Z", + "initialCreatedByUserId": "usrKu6HNqQKma2iWV", + "lastModifiedTime": "2026-05-28T15:00:05.000Z", + "lastModifiedByUserId": "usrKu6HNqQKma2iWV" + }, + { + "id": "fldR2CCi7SbLg6CcZ", + "name": "Tags", + "type": "multiSelect", + "default": [ + "selv3IDdBfe4N7Axy" + ], + "typeOptions": { + "choiceOrder": [ + "selbMhuvR7Au5Y2nd", + "selv3IDdBfe4N7Axy", + "selaHOcAIJljou72O", + "selZho3sXgQ8S3ry9" + ], + "choices": { + "selbMhuvR7Au5Y2nd": { + "id": "selbMhuvR7Au5Y2nd", + "color": "blue", + "name": "test 1" + }, + "selv3IDdBfe4N7Axy": { + "id": "selv3IDdBfe4N7Axy", + "color": "teal", + "name": "test 3" + }, + "selaHOcAIJljou72O": { + "id": "selaHOcAIJljou72O", + "color": "green", + "name": "test 4" + }, + "selZho3sXgQ8S3ry9": { + "id": "selZho3sXgQ8S3ry9", + "color": "yellow", + "name": "test 5" + } + }, + "disableColors": false + }, + "initialCreatedTime": "2026-05-28T15:00:21.000Z", + "initialCreatedByUserId": "usrKu6HNqQKma2iWV", + "lastModifiedTime": "2026-05-28T15:00:41.000Z", + "lastModifiedByUserId": "usrKu6HNqQKma2iWV" + }, + { + "id": "fldlIM9UIZ297nyJl", + "name": "Table 2", + "type": "foreignKey", + "typeOptions": { + "foreignTableId": "tblq7se6YHVutB7xW", + "symmetricColumnId": "fldnH8Hutzd9BT4tj", + "relationship": "many", + "unreversed": true + }, + "initialCreatedTime": "2026-06-15T19:22:27.000Z", + "initialCreatedByUserId": "usrKu6HNqQKma2iWV", + "lastModifiedTime": "2026-06-15T19:22:27.000Z", + "lastModifiedByUserId": "usrKu6HNqQKma2iWV" + } + ], + "meaningfulColumnOrder": [ + { + "columnId": "fld6ISd4khYuH27l6", + "visibility": true + }, + { + "columnId": "fld0REIF5QK5sdRW8", + "visibility": true + }, + { + "columnId": "fldzYJncU1NAXKuOg", + "visibility": true + }, + { + "columnId": "flduztL3p6LJ1r3GR", + "visibility": true + }, + { + "columnId": "fldCEPy54raOrxFoL", + "visibility": true + }, + { + "columnId": "fld2iy0JmkuemZZAf", + "visibility": true + }, + { + "columnId": "fldlMHqqi4O6gqPqv", + "visibility": true + }, + { + "columnId": "fldBhukTKsuf9QzBO", + "visibility": true + }, + { + "columnId": "fldC82CKp6sAOaRP7", + "visibility": true + }, + { + "columnId": "fldR2CCi7SbLg6CcZ", + "visibility": true + }, + { + "columnId": "fldlIM9UIZ297nyJl", + "visibility": true + } + ], + "views": [ + { + "id": "viwCR7VNVVDrFhj5k", + "name": "test 3", + "type": "grid", + "personalForUserId": null, + "description": null, + "createdByUserId": "usrKu6HNqQKma2iWV" + }, + { + "id": "viwDwGrkhMGoVUQyX", + "name": "Grid view", + "type": "grid", + "personalForUserId": null, + "description": null, + "createdByUserId": "usrKu6HNqQKma2iWV" + }, + { + "id": "viw7qK5EyOo3RJW6g", + "name": "test 45", + "type": "grid", + "personalForUserId": null, + "description": null, + "createdByUserId": "usrKu6HNqQKma2iWV" + } + ], + "viewOrder": [ + "viwCR7VNVVDrFhj5k", + "viwDwGrkhMGoVUQyX", + "viw7qK5EyOo3RJW6g" + ], + "viewsById": { + "viwCR7VNVVDrFhj5k": { + "id": "viwCR7VNVVDrFhj5k", + "name": "test 3", + "type": "grid", + "personalForUserId": null, + "description": null, + "createdByUserId": "usrKu6HNqQKma2iWV" + }, + "viwDwGrkhMGoVUQyX": { + "id": "viwDwGrkhMGoVUQyX", + "name": "Grid view", + "type": "grid", + "personalForUserId": null, + "description": null, + "createdByUserId": "usrKu6HNqQKma2iWV" + }, + "viw7qK5EyOo3RJW6g": { + "id": "viw7qK5EyOo3RJW6g", + "name": "test 45", + "type": "grid", + "personalForUserId": null, + "description": null, + "createdByUserId": "usrKu6HNqQKma2iWV" + } + }, + "viewSectionsById": {}, + "rowTemplatesById": {}, + "dateDependencySettings": null, + "rowUnit": null, + "isHiddenFromSwitcher": false, + "columnSetById": {} + }, + { + "id": "tblq7se6YHVutB7xW", + "name": "Table 2", + "primaryColumnId": "fldstX6p99IPfCbYC", + "columns": [ + { + "id": "fldstX6p99IPfCbYC", + "name": "Name", + "type": "text", + "initialCreatedTime": "2026-06-15T19:22:03.000Z", + "initialCreatedByUserId": "usrKu6HNqQKma2iWV", + "lastModifiedTime": "2026-06-15T19:22:03.000Z", + "lastModifiedByUserId": "usrKu6HNqQKma2iWV" + }, + { + "id": "fldlwRUgPHGrt7SdU", + "name": "Notes", + "type": "multilineText", + "initialCreatedTime": "2026-06-15T19:22:03.000Z", + "initialCreatedByUserId": "usrKu6HNqQKma2iWV", + "lastModifiedTime": "2026-06-15T19:22:03.000Z", + "lastModifiedByUserId": "usrKu6HNqQKma2iWV" + }, + { + "id": "fldbJpvyt3G4b2WvW", + "name": "Assignee", + "type": "collaborator", + "typeOptions": { + "shouldNotify": true, + "canAddNonBaseCollaborators": true + }, + "initialCreatedTime": "2026-06-15T19:22:03.000Z", + "initialCreatedByUserId": "usrKu6HNqQKma2iWV", + "lastModifiedTime": "2026-06-15T19:22:03.000Z", + "lastModifiedByUserId": "usrKu6HNqQKma2iWV" + }, + { + "id": "fldbGGSTSDoujij9H", + "name": "Status", + "type": "select", + "typeOptions": { + "choices": { + "selMia1ayapOSISP7": { + "id": "selMia1ayapOSISP7", + "name": "Todo", + "color": "red" + }, + "selRXq2RWWr6H540f": { + "id": "selRXq2RWWr6H540f", + "name": "In progress", + "color": "yellow" + }, + "selInjcoXK6SveIko": { + "id": "selInjcoXK6SveIko", + "name": "Done", + "color": "green" + } + }, + "choiceOrder": [ + "selMia1ayapOSISP7", + "selRXq2RWWr6H540f", + "selInjcoXK6SveIko" + ] + }, + "initialCreatedTime": "2026-06-15T19:22:03.000Z", + "initialCreatedByUserId": "usrKu6HNqQKma2iWV", + "lastModifiedTime": "2026-06-15T19:22:03.000Z", + "lastModifiedByUserId": "usrKu6HNqQKma2iWV" + }, + { + "id": "fldeFc1iO0WM9TsuH", + "name": "Attachments", + "type": "multipleAttachment", + "typeOptions": { + "unreversed": true + }, + "initialCreatedTime": "2026-06-15T19:22:03.000Z", + "initialCreatedByUserId": "usrKu6HNqQKma2iWV", + "lastModifiedTime": "2026-06-15T19:22:03.000Z", + "lastModifiedByUserId": "usrKu6HNqQKma2iWV" + }, + { + "id": "fldVFDbhgHlazczgJ", + "name": "Attachment Summary", + "type": "asyncText", + "description": "An AI generated summary of the Attachments field. Upload files to Attachments to generate a summary.", + "typeOptions": { + "sourceType": "ai", + "sourceParams": { + "prompt": [ + "You are a professional document analyst specializing in summarizing content from various file types. Your task is to extract key points and provide a concise summary of the content found in the attached files. Ensure that the summary captures the main ideas and any important details without including unnecessary information.\n\nAnalyze the content of the attached files to identify the main themes and significant details. Summarize these points clearly and concisely, ensuring that the summary is informative and relevant to the document's purpose.\n\nFormat your response as plain text, keeping it concise and to the point. If you cannot summarize the content, output \"Unable to summarize the content.\" Example: \"The document discusses the impact of climate change on polar bears, highlighting the loss of habitat and food sources.\" (Real examples should be longer and more detailed.)\n\nFiles:\n", + { + "field": { + "columnId": "fldeFc1iO0WM9TsuH" + } + } + ], + "columnIdDependencies": [ + "fldeFc1iO0WM9TsuH" + ], + "optionalColumnIdDependencies": [], + "model": "default", + "temperature": 0, + "generationMode": "onDemand", + "availableTools": [], + "userConfigurationForPrompt": { + "type": "fullyUserManaged" + }, + "attachmentProcessingStrategy": { + "type": "documentAgent" + }, + "integrationSettings": {} + } + }, + "initialCreatedTime": "2026-06-15T19:22:03.000Z", + "initialCreatedByUserId": "usrKu6HNqQKma2iWV", + "lastModifiedTime": "2026-06-15T19:22:03.000Z", + "lastModifiedByUserId": "usrKu6HNqQKma2iWV" + }, + { + "id": "fldnH8Hutzd9BT4tj", + "name": "test linked field", + "type": "foreignKey", + "description": "test desc", + "typeOptions": { + "foreignTableId": "tblooKJLpdRd2SI3I", + "relationship": "many", + "unreversed": true, + "symmetricColumnId": "fldlIM9UIZ297nyJl" + }, + "initialCreatedTime": "2026-06-15T19:22:27.000Z", + "initialCreatedByUserId": "usrKu6HNqQKma2iWV", + "lastModifiedTime": "2026-06-15T19:22:27.000Z", + "lastModifiedByUserId": "usrKu6HNqQKma2iWV" + }, + { + "id": "fldyEl33qtpaVToBH", + "name": "Status (from test linked field)", + "type": "lookup", + "typeOptions": { + "relationColumnId": "fldnH8Hutzd9BT4tj", + "foreignTableRollupColumnId": "flduztL3p6LJ1r3GR", + "canUseSelectResultType": true, + "foreignTableRollupColumnType": "select", + "choices": { + "selSOmr7hxDdQOF1X": { + "id": "selSOmr7hxDdQOF1X", + "name": "Todo", + "color": "red" + }, + "selI1uROgLXkHga5l": { + "id": "selI1uROgLXkHga5l", + "name": "In progress", + "color": "yellow" + }, + "selGFbfPNLBugiEN5": { + "id": "selGFbfPNLBugiEN5", + "name": "Done", + "color": "green" + }, + "seltLTY0BfBXhCQ8J": { + "id": "seltLTY0BfBXhCQ8J", + "name": "2test", + "color": "blue" + } + }, + "choiceOrder": [ + "selSOmr7hxDdQOF1X", + "selI1uROgLXkHga5l", + "selGFbfPNLBugiEN5", + "seltLTY0BfBXhCQ8J" + ], + "dependencies": { + "referencedColumnIdsForValue": [ + "fldnH8Hutzd9BT4tj", + "flduztL3p6LJ1r3GR" + ] + }, + "resultType": "select", + "resultIsArray": true + }, + "initialCreatedTime": "2026-06-15T19:22:32.000Z", + "initialCreatedByUserId": "usrKu6HNqQKma2iWV", + "lastModifiedTime": "2026-06-15T19:22:32.000Z", + "lastModifiedByUserId": "usrKu6HNqQKma2iWV" + }, + { + "id": "fldXmK9Z88Rz01ByE", + "name": "Tags (from test linked field)", + "type": "lookup", + "typeOptions": { + "relationColumnId": "fldnH8Hutzd9BT4tj", + "foreignTableRollupColumnId": "fldR2CCi7SbLg6CcZ", + "canUseSelectResultType": true, + "foreignTableRollupColumnType": "multiSelect", + "choices": { + "selbMhuvR7Au5Y2nd": { + "id": "selbMhuvR7Au5Y2nd", + "color": "blue", + "name": "test 1" + }, + "selv3IDdBfe4N7Axy": { + "id": "selv3IDdBfe4N7Axy", + "color": "teal", + "name": "test 3" + }, + "selaHOcAIJljou72O": { + "id": "selaHOcAIJljou72O", + "color": "green", + "name": "test 4" + }, + "selZho3sXgQ8S3ry9": { + "id": "selZho3sXgQ8S3ry9", + "color": "yellow", + "name": "test 5" + } + }, + "choiceOrder": [ + "selbMhuvR7Au5Y2nd", + "selv3IDdBfe4N7Axy", + "selaHOcAIJljou72O", + "selZho3sXgQ8S3ry9" + ], + "disableColors": false, + "dependencies": { + "referencedColumnIdsForValue": [ + "fldnH8Hutzd9BT4tj", + "fldR2CCi7SbLg6CcZ" + ] + }, + "resultType": "multiSelect", + "resultIsArray": true + }, + "initialCreatedTime": "2026-06-15T19:22:52.000Z", + "initialCreatedByUserId": "usrKu6HNqQKma2iWV", + "lastModifiedTime": "2026-06-15T19:22:52.000Z", + "lastModifiedByUserId": "usrKu6HNqQKma2iWV" + }, + { + "id": "fldYxuKRLugoaUDxR", + "name": "test32 edit", + "type": "formula", + "typeOptions": { + "formulaTextParsed": "'2test'", + "dependencies": { + "referencedColumnIdsForValue": [] + }, + "resultType": "text", + "resultIsArray": false + }, + "initialCreatedTime": "2026-06-15T19:23:20.000Z", + "initialCreatedByUserId": "usrKu6HNqQKma2iWV", + "lastModifiedTime": "2026-06-15T19:23:26.000Z", + "lastModifiedByUserId": "usrKu6HNqQKma2iWV" + }, + { + "id": "fldkV40yf91ZNmrHa", + "name": "Attachments 2", + "type": "multipleAttachment", + "typeOptions": { + "unreversed": true + }, + "initialCreatedTime": "2026-06-15T19:24:33.000Z", + "initialCreatedByUserId": "usrKu6HNqQKma2iWV", + "lastModifiedTime": "2026-06-15T19:24:33.000Z", + "lastModifiedByUserId": "usrKu6HNqQKma2iWV" + } + ], + "meaningfulColumnOrder": [ + { + "columnId": "fldstX6p99IPfCbYC", + "visibility": true + }, + { + "columnId": "fldlwRUgPHGrt7SdU", + "visibility": true + }, + { + "columnId": "fldbJpvyt3G4b2WvW", + "visibility": true + }, + { + "columnId": "fldbGGSTSDoujij9H", + "visibility": true + }, + { + "columnId": "fldeFc1iO0WM9TsuH", + "visibility": true + }, + { + "columnId": "fldVFDbhgHlazczgJ", + "visibility": true + }, + { + "columnId": "fldnH8Hutzd9BT4tj", + "visibility": true + }, + { + "columnId": "fldXmK9Z88Rz01ByE", + "visibility": true + }, + { + "columnId": "fldyEl33qtpaVToBH", + "visibility": true + }, + { + "columnId": "fldYxuKRLugoaUDxR", + "visibility": true + }, + { + "columnId": "fldkV40yf91ZNmrHa", + "visibility": true + } + ], + "views": [ + { + "id": "viw6SdJ50gtx4ujY4", + "name": "Grid view", + "type": "grid", + "personalForUserId": null, + "description": null, + "createdByUserId": "usrKu6HNqQKma2iWV" + } + ], + "viewOrder": [ + "viw6SdJ50gtx4ujY4" + ], + "viewsById": { + "viw6SdJ50gtx4ujY4": { + "id": "viw6SdJ50gtx4ujY4", + "name": "Grid view", + "type": "grid", + "personalForUserId": null, + "description": null, + "createdByUserId": "usrKu6HNqQKma2iWV" + } + }, + "viewSectionsById": {}, + "rowTemplatesById": {}, + "dateDependencySettings": null, + "rowUnit": null, + "isHiddenFromSwitcher": false, + "columnSetById": {} + } + ], + "tableDatas": [ + { + "id": "tblq7se6YHVutB7xW", + "rows": [ + { + "id": "recI5lXwz2wvEoR0W", + "createdTime": "2026-06-15T19:22:03.000Z", + "cellValuesByColumnId": { + "fldVFDbhgHlazczgJ": { + "state": "error", + "errorType": "emptyDependency", + "value": null, + "erroredColumnIds": [ + "fldeFc1iO0WM9TsuH" + ], + "doesNeedRegeneration": false + }, + "fldYxuKRLugoaUDxR": "2test" + } + }, + { + "id": "recuIhArOOZsrtSgi", + "createdTime": "2026-06-15T19:22:03.000Z", + "cellValuesByColumnId": { + "fldVFDbhgHlazczgJ": { + "state": "error", + "errorType": "emptyDependency", + "value": null, + "erroredColumnIds": [ + "fldeFc1iO0WM9TsuH" + ], + "doesNeedRegeneration": false + }, + "fldkV40yf91ZNmrHa": [ + { + "id": "attROb0RNhJ8CJO0B", + "url": "https://dl.airtable.com/.directUploadAttachment/9d2fdb053de09ae14bb17698bc6e8865/eedf094e/test.txt", + "filename": "test.txt", + "type": "text/plain", + "size": 48 + } + ], + "fldYxuKRLugoaUDxR": "2test" + } + }, + { + "id": "rec1JpEpJIfgv5nAL", + "createdTime": "2026-06-15T19:22:03.000Z", + "cellValuesByColumnId": { + "fldVFDbhgHlazczgJ": { + "state": "error", + "errorType": "emptyDependency", + "value": null, + "erroredColumnIds": [ + "fldeFc1iO0WM9TsuH" + ], + "doesNeedRegeneration": false + }, + "fldYxuKRLugoaUDxR": "2test" + } + } + ], + "viewDatas": [ + { + "id": "viw6SdJ50gtx4ujY4", + "frozenColumnCount": 1, + "columnOrder": [ + { + "columnId": "fldstX6p99IPfCbYC", + "visibility": true + }, + { + "columnId": "fldlwRUgPHGrt7SdU", + "visibility": true + }, + { + "columnId": "fldbJpvyt3G4b2WvW", + "visibility": true + }, + { + "columnId": "fldbGGSTSDoujij9H", + "visibility": true + }, + { + "columnId": "fldeFc1iO0WM9TsuH", + "visibility": true + }, + { + "columnId": "fldVFDbhgHlazczgJ", + "visibility": true + }, + { + "columnId": "fldnH8Hutzd9BT4tj", + "visibility": true + }, + { + "columnId": "fldXmK9Z88Rz01ByE", + "visibility": true + }, + { + "columnId": "fldyEl33qtpaVToBH", + "visibility": true + }, + { + "columnId": "fldYxuKRLugoaUDxR", + "visibility": true + }, + { + "columnId": "fldkV40yf91ZNmrHa", + "visibility": true + } + ], + "filters": null, + "lastSortsApplied": null, + "groupLevels": null, + "colorConfig": null, + "description": null, + "createdByUserId": "usrKu6HNqQKma2iWV", + "type": "grid", + "applicationTransactionNumber": 80, + "rowOrder": [ + { + "rowId": "recI5lXwz2wvEoR0W", + "visibility": true + }, + { + "rowId": "recuIhArOOZsrtSgi", + "visibility": true + }, + { + "rowId": "rec1JpEpJIfgv5nAL", + "visibility": true + } + ], + "signedUserContentUrls": {} + } + ], + "signedUserContentUrls": { + "https://dl.airtable.com/.directUploadAttachment/9d2fdb053de09ae14bb17698bc6e8865/eedf094e/test.txt": "https://v5.airtableusercontent.com/v3/u/54/54/1781604000000/m_A7xQlE3NiXcZUcmcLH0Q/yBqlPhXVka1SMUkPLn7ExVx0qwRW43Cw6XjWAi2dYhrrKt6ucxBBa48REj9nC3nK0cid479V7e4FD6paTnPOU-A_jwvHC05YVxUCmapeX_cCMPvbVwblYkfeSde0w3eZ5TzdH2Ujj5luvdDXMMVlQQ/u5KIqVWFEZ5Gc7ohm51_D5B4Xo-mDjZrGP07f7710FU" + }, + "hasOnlyIncludedRowAndCellDataForIncludedViews": false + } + ], + "hasBlockInstallations": false, + "packageInstallationsById": {}, + "featureKitInstallationById": {}, + "applicationAdminFlags": { + "UPDATE_PRIMITIVE_CELL_THROTTLE_MS": null, + "UPDATE_RICH_TEXT_CELL_THROTTLE_MS": null, + "MAX_WORKFLOWS_PER_APPLICATION": null, + "MAX_SYNC_SOURCES_PER_APPLICATION": null, + "MAX_SYNC_SOURCES_PER_TABLE": null, + "MAX_SYNCED_TABLES_PER_APPLICATION": null, + "CUSTOM_MAX_NUM_COLUMNS_PER_TABLE": null, + "CUSTOM_MAX_NUM_ROWS_PER_TABLE": null, + "DENYLISTED_PAGE_BUNDLES_FOR_PAGE_BUNDLE_COLLABORATORS": null, + "MAX_ROW_TEMPLATES_PER_APPLICATION": null, + "APPLICATION_AI_BULK_GENERATION_RECORD_COUNT_LIMIT": null, + "WORKFLOW_FIND_RECORDS_MAX_LIMIT": null, + "WORKFLOW_EXECUTION_READ_WORKFLOW_ACTION_EXECUTION_BATCH_SIZE": null, + "WORKFLOW_MAX_SUBSCRIBERS_PER_WORKFLOW": null, + "APPLICATION_AI_QA_IFRAME_URL": null, + "AUTO_REFRESH_TIMEOUT_MS": null, + "ENABLE_PRODUCT_CENTRAL": null, + "APPLICATION_DEPENDENCY_GRAPH_THROTTLE_MS": null, + "SEQUENTIAL_CRUD_REQUEST_CLIENT_SIDE_TIMEOUT_MS": null + }, + "pageLayoutSchemaVersion": 26, + "pageSections": [], + "pageBundles": [], + "unparentedPageMetadata": [], + "canUserModifyAllUnparentedPages": true, + "portal": null, + "isCurrentUserPortalPageBundleCollaborator": false, + "uploadedUserContentCdnSetting": { + "applicationScopedAuthMode": "private" + }, + "applicationV2TargetedFeatureFlagClientConfiguration": { + "dateSparklineDisplay": { + "trafficLevel": 0 + }, + "blockModelBridgeLongWaitUntilReadyBeforeWriteTimeout": { + "trafficLevel": 0 + }, + "aiFieldMatching": { + "trafficLevel": 100 + }, + "aiAutomaticFieldMatching": { + "trafficLevel": 0 + }, + "applicationChangeHookCreateSyncHooksWithSpecificationPayloadEnabled": { + "trafficLevel": 100 + }, + "externalTableSyncSkipRealTimeSyncForSyncWithFieldTypeCustomizationToLinkedRecord": { + "trafficLevel": 100 + }, + "syncLinkedRecordsCellUpdateOptimization": { + "trafficLevel": 100 + }, + "canCreateRelationFieldsSourcedByCollaboratorField": { + "trafficLevel": 0 + }, + "enableExternalTableSyncInfoManagerForAuthoritativeSourceWrites": { + "trafficLevel": 0 + }, + "externalTableSyncThreadedEnqueue": { + "trafficLevel": 100 + }, + "attachmentsAuthV3ShouldUseSignedAuthV5Urls": { + "trafficLevel": 0 + }, + "attachmentsAuthV3": { + "trafficLevel": 0 + }, + "eudrAttachmentBoundaryCrossings": { + "trafficLevel": 0 + }, + "externalTableSyncPingSourceBasesFromTarget": { + "trafficLevel": 100 + }, + "changeHooksOnUelLogActions": { + "trafficLevel": 0 + }, + "performChangeHookProcessingOnUelPath": { + "trafficLevel": 100 + }, + "applicationSearchMultiTokenBooleanSearch": { + "trafficLevel": 100 + }, + "applicationSearchMultiTokenBooleanSearchAllowFuzziness": { + "trafficLevel": 0 + }, + "twoWaySyncServerSideTransparentEditingNonBlocking": { + "trafficLevel": 100 + }, + "twoWaySyncPricingGating": { + "trafficLevel": 100 + }, + "externalTableSyncQuerySchemaInIncrementalSync": { + "trafficLevel": 100 + }, + "revisionHistoryErrorLogging": { + "trafficLevel": 0 + }, + "workflowsSlackShowPermalinkInExpressionBuilder": { + "trafficLevel": 0 + }, + "dateDependenciesRevisionHistory": { + "trafficLevel": 100 + }, + "disableUsageInsightsOneYearOption": { + "precalculatedVariant": { + "variant": "control", + "reason": "disabled" + } + }, + "tableDeleteOrphanCascadeForApplication": { + "trafficLevel": 0 + }, + "disabledWorkflowOnSchemaChangeSuggestion": { + "trafficLevel": 100 + }, + "syncFailureSuggestion": { + "trafficLevel": 100 + }, + "unusedViewsSuggestion": { + "trafficLevel": 100 + }, + "unusedSelectChoicesSuggestion": { + "trafficLevel": 0 + }, + "unifiedEventLog": { + "trafficLevel": 100 + }, + "unifiedEventLogPublishingBlockDbCommits": { + "trafficLevel": 100 + }, + "projectReadDataForRowCards": { + "trafficLevel": 0 + }, + "threadSafeStoreUseLargerMapSize": { + "trafficLevel": 0 + }, + "clientSideProfilingForApplication": { + "trafficLevel": 0 + } + }, + "applicationV2FeatureFlagClientContext": { + "applicationId": "apphwgei0ETb2zCXz", + "flagNameToPrecalculatedVariant": { + "dateSparklineDisplay": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "blockModelBridgeLongWaitUntilReadyBeforeWriteTimeout": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "aiFieldMatching": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiAutomaticFieldMatching": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "applicationChangeHookCreateSyncHooksWithSpecificationPayloadEnabled": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "externalTableSyncSkipRealTimeSyncForSyncWithFieldTypeCustomizationToLinkedRecord": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "syncLinkedRecordsCellUpdateOptimization": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "canCreateRelationFieldsSourcedByCollaboratorField": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "enableExternalTableSyncInfoManagerForAuthoritativeSourceWrites": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "externalTableSyncThreadedEnqueue": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "attachmentsAuthV3ShouldUseSignedAuthV5Urls": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "attachmentsAuthV3": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "eudrAttachmentBoundaryCrossings": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "externalTableSyncPingSourceBasesFromTarget": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "changeHooksOnUelLogActions": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "performChangeHookProcessingOnUelPath": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "applicationSearchMultiTokenBooleanSearch": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "applicationSearchMultiTokenBooleanSearchAllowFuzziness": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "twoWaySyncServerSideTransparentEditingNonBlocking": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "twoWaySyncPricingGating": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "externalTableSyncQuerySchemaInIncrementalSync": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "revisionHistoryErrorLogging": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "workflowsSlackShowPermalinkInExpressionBuilder": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "dateDependenciesRevisionHistory": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "disableUsageInsightsOneYearOption": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "control", + "reason": "disabled" + } + }, + "tableDeleteOrphanCascadeForApplication": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "disabledWorkflowOnSchemaChangeSuggestion": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "syncFailureSuggestion": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "unusedViewsSuggestion": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "unusedSelectChoicesSuggestion": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "unifiedEventLog": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "unifiedEventLogPublishingBlockDbCommits": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "projectReadDataForRowCards": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "threadSafeStoreUseLargerMapSize": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "clientSideProfilingForApplication": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationV2", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + } + } + }, + "applicationV2EnabledFeatureNames": [ + "aiFieldMatching", + "applicationChangeHookCreateSyncHooksWithSpecificationPayloadEnabled", + "externalTableSyncSkipRealTimeSyncForSyncWithFieldTypeCustomizationToLinkedRecord", + "syncLinkedRecordsCellUpdateOptimization", + "externalTableSyncThreadedEnqueue", + "externalTableSyncPingSourceBasesFromTarget", + "performChangeHookProcessingOnUelPath", + "applicationSearchMultiTokenBooleanSearch", + "twoWaySyncServerSideTransparentEditingNonBlocking", + "twoWaySyncPricingGating", + "externalTableSyncQuerySchemaInIncrementalSync", + "dateDependenciesRevisionHistory", + "disabledWorkflowOnSchemaChangeSuggestion", + "syncFailureSuggestion", + "unusedViewsSuggestion", + "unifiedEventLog", + "unifiedEventLogPublishingBlockDbCommits" + ], + "applicationUnifiedFeatureFlagClientContext": { + "applicationId": "apphwgei0ETb2zCXz", + "flagNameToPrecalculatedVariant": { + "iCalSharedViewBestEffortAllDayColumn": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "restrictPagesDataToAllEntryPagesAndReachableRowPagesInPageBundleForMobileClient": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "disableFilterSourceHintInCellEditorElement": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "logUnexpectedQueryAuthorizationPageError": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "shouldCheckRowUpdatedLeasesSecond": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "groupHeadingScreenReaderLabels": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "timelineShowExactStartTimeWithoutEndColumn": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "timelineViewGanttCriticalPath": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "timelineGanttNesting": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "interfaceDesignerCustomFormFieldValidation": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "interfaceDesignerLinkedRecordReordering": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "interfaceChartMultipleSeries": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "interfaceChartDualYAxes": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "interfaceDesignerIncreaseNumberOfLegendItemsForTopBottom": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "interfaceDesignerSupersizeRecordDetailPage": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "standaloneFormSection": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "allowHighContrastForms": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "standaloneFormReordering": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "interfaceDesignerStagedUpdateForms": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "interfaceDesignerStagedUpdateFormsBetaBadge": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "disabled" + } + }, + "shouldUseExistingCellValuesForVisibleColumnsDuringPermissionsCheck": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "escaPrefilledLinkedRecordSymmetricColumnFallback": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "interfaceDesignerGalleryExpandTruncatedProperties": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "viewsTopControlsMenuComponentsUpdate": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "iosOmniApplication": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "androidOmniApplication": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "productCentralApplicationOverride": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "forceSolutionThemeForBase": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "allowRenamingSolution": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "enableClaudeOpus4p8ForOmni": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "dashboardQueryContainerSearch": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "useLevelsMultiSelectComponentInBaseDetailView": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "interfaceDesignerDisableRuntimeOutputLiveMemo": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "dashboardGridContainer": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiEnableStreamingForCobuilder": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "aiOmniAppCreationEnableParallelBuilderAgents": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiPlayInterfaceLaunch": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiAppBuildingDocExtractionInBaseCreationModal": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiGenerateStructuredOutputNoRelevantInputDataStatus": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "customElementsAllowAiGenerationInPageCreationWizard": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "customElementCodeGenStreaming": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiCoverImageGenerationForStandaloneFormsApplicationScoped": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "cobuilderCustomizationAnthropicByDefault": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "disabled" + } + }, + "cobuilderCustomizationGetOrderedModelListForCustomizationPromptOverridesVariant": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "cobuilderCustomizationWorkflowIteration": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "aiFieldLogToBaseKillSwitch": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "omniUnifiedAgent": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "disabled" + } + }, + "aiServerQaAgent": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "omniV2InternalDogfoodingApplicationScoped": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "aiAssistantCreateTableWithAi": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiAssistantAttachmentsForPBC": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiAssistantMakeSingleGetContextForAgentRequest": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiAssistantGridArtifactStreaming": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "aiAssistantAdvisors": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "aiSpikeOmniAppDescriptions": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiSpikeAlwaysShowTableTemplates": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "omniReportGeneration": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "omniReportGenerationAdvancedFeatures": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "omniReportGenerationFeatureAwarenessCard": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "omniUseVercelForOpenAi": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "omniUseVercelForAnthropicBedrock": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "omniUseVercelForGemini": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiAssistantMessagePdfExport": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "aiAssistantGridArtifactLiveBaseData": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiAssistantShouldFallBackThroughDynamicModels": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "enableAiChatsManagementKillSwitch": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "disabled" + } + }, + "aiAssistantWebDataRetrieval": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "aiServerWebChatOwnershipCheck": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiServerAgentPersistEventsToDynamo": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "omniInterfaceLongTextEditor": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "omniRecordScopedContext": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "omniCellContextMenuAskOmni": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "omniRecordScopedContextRollout": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "omniContextualSearchEntryPoint": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "omniRecordAgentMarkdownSupport": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "enableAiAssistantMarkdownCobuilderTokens": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "omniRecordAgentFineGrainedToolStreaming": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "disableEditingOfEditableSharedViews": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "linkedRecordPickerMultiSelectInRecordDetail": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "cascadingEndUserFilters": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "computedFieldDynamicFiltersInForms": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "blocksUseSeparateTwoWaySyncedRichTextCellAction": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "packagePlatformVerboseLogging": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "packagesPlatformSendRealtimeLikePayloadAfterReleaseCreated": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "packagesPlatformDisableCustomizingLevelsPageElement": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "packagesPlatformDisableCustomizingQueryContainerFilters": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "packagesPlatformAllowCustomizingQueryContainerEndUserActions": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "packagesPlatformAllowCustomizingQueryContainerEndUserFilters": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "packagesPlatformAllowCustomizingQueryContainerCallToActionButton": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "packagesPlatformAllowCustomizingInboxPageElement": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "connectSingleApplicationRealtimeWebSocketToAppAirtableCom": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "disabled" + } + }, + "interfaceExtensionsAllowOnPubliclySharedInterfacePages": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "interfaceExtensionsHideTopBarOnPubliclySharedInterfacePages": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "interfaceExtensionsAllowBlockInstallationInQueryContainerPageElement": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "interfaceExtensionsEnableMultiTable": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "interfaceExtensionsSkipPreloadingQueries": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "interfaceExtensionsEnableAiGeneration": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "interfaceExtensionsEnableAiGenerationRolloutExcludingEnterprise": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "interfaceExtensionsEnableImagesAsContext": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "interfaceExtensionsPrependPromptWhenMenuItemSelected": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "sandboxEligibleApplications": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "sandboxPreventUndestroy": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "sandboxCopyTestOuputToSandbox": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "sandboxSkipPageDestroyIfMissing": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "sandboxCommitSchemaValidationOnRead": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "sandboxCompactColumnOrder": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "sandboxSelectivePublishing": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "sandboxSelectivePublishingFormulaChaining": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "sandboxAttemptToAutoResolveColumnDependencyCycles": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "sandboxPackages": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "sandboxV1DataExtensions": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "sandboxV1InterfaceCustomExtensions": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "sandboxGenerateActionSequenceApiWithDisplayInfo": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "sandboxOnlyAllowApplicationMetadataUpdatesFromProductionApplication": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "sandboxAllowDestroyingMainApplicationsWithoutDestroyingSandboxFirst": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "sandboxGenerateSignedActionSequence": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "sandboxGenerateStepsDueToSyncSourceChanges": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "sandboxRequireSignedActionSequence": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "sandboxPublishHistory": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "sandboxSkipActionSequenceStalenessCheckForApplication": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "sandboxPreviewAsInactiveCollaborators": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "openAio1ModelAvailabilityUnified": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "openAiChatGpt5P2ModelAvailability": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "openAiChatGpt5P4ModelAvailability": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "anthropicClaudeOpus4p6ModelAvailability": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "anthropicClaudeSonnet4p6ModelAvailability": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "anthropicClaudeOpus4p7ModelAvailability": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "anthropicClaudeOpus4p8ModelAvailability": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "gemini3ProModelAvailability": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "gemini3p1ProPreviewModelAvailability": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "gemini3FlashModelAvailability": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "geminiDocExtractionImageVision": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "anthropicDocExtractionImageVision": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "mistralAiModelAvailability": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "openAiImageGenerationModelAvailability": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "openAiGptImage1P5ModelAvailabilityUnified": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "openAiGptImage1MiniModelAvailabilityUnified": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "openAiGptImage2ModelAvailabilityUnified": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "gemini2P5FlashImageModelAvailabilityUnified": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "gemini3ProImageModelAvailabilityUnified": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "gemini3P1FlashImageModelAvailabilityUnified": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "gemini3ProImageResolution": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "imageFieldAgentFeedbackLinkForAiLabs": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "aiImageGenerationAutomation": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "aiImageGenerationAutomationBusinessEnterprise": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "aiImageGenerationFieldAgent": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "aiImageGenerationFieldAgentBusinessEnterprise": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "googleVeo3VideoGenerationModelAvailability": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "aiWebContentGenerationFieldAgent": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiFieldAgentConfigV2": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiLinkedRecordFieldAgent": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "aiEnableLinkedRecordAiFieldForEkmBases": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "aiLinkedRecordFieldAllowStaleEmbeddings": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "aiLinkedRecordFieldKeywordSearch": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiCreateLinkedRecordAiFieldsFromOmni": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "aiFieldAgentsLogToBraintrust": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "aiComposioFieldAgentsLogToBraintrust": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "aiFieldAgentGenerationTriggers": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "aiFieldAgentPerApplicationCapacityAdmissionControl": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "aiFieldAgentBlockAutomaticGenerationOnTodayDependency": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiPromptEditorAtAnchor": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "aiReduceTaskQueueEmbeddingCrudRequestPriority": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiFieldMatchingUseLlmForAutomaticMatching": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "aiFieldMatchingAdvancedConfiguration": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "aiAdoptionSuggestionsInFieldContextMenu": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiFieldAgentTokenMaxiExperimentProactiveSuggestionsOverride": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "aiAutomationsLibrary": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiAssistantCreateFieldAgentCommand": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "displayChartArtifactInAiAssistant": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "saveChartToPage": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiEnableSemanticSearch": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "aiDisablePbcSemanticSearchEmbeddingConfigCreation": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "aiGenerateWorkflowActionOutputSchema": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "aiAssistantForBillingPlansThatIncludeAiSku": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiAssistantSuggestionGenerationV2": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "aiAssistantSupportHandoffLlmQuery": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiAssistantAllowBlankPageQueries": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiAssistantEnableCreditCharges": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "commandKRecordSearch": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "aiVectorSearchFilteringHeuristics": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "enableGoogleDriveToolsInFieldAgents": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "enableGoogleDriveFolderScopingInFieldAgents": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "enableOneDriveToolsInFieldAgents": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "enableJiraToolsInFieldAgents": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "enableComposioToolsInFieldAgents": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "enableComposioToolsInFieldAgentsBusinessEnterprise": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "enableFieldAgentIntegrationRestrictions": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "useLargeDocExtractionPipeline": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "useLargeDocExtractionPipelineInAutomation": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "letAiFieldReadExcelFiles": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "letAiFieldReadWebpImageFiles": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "letAiFieldReadImagesWithVisionModels": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "letAiFieldReadImagesWithVisionModelsBusinessEnterprise": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "showOmniStopButton": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + } + }, + "allowEkmMilvusEmbeddingForInternalOnly": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "doubleWriteToMilvusAndLance": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "disabled" + } + }, + "writeNewEmbeddingConfigsToMilvus": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "readEmbeddingsFromMilvus": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "forceMilvusMigration": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiFinnhubAssistantAvailability": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "aiAssistantAirtableHelp": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "aiAssistantGenerateArtifactsForNonBaseData": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "aiAssistantVirtualTable": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiBedrockEmbeddings": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiEmbeddingsModelAutomaticCutover": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiEmbeddingConfigs": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiAssistantRouterPlanning": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "aiAssistantRouterUserClarification": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "disabled" + } + }, + "aiAssistantRouterUseAnthropic": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "aiAssistantWithSyncInfo": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiAdoptionAiFieldCatalog": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiFreeGeneration": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "enableCustomElementPregeneration": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "workflowsExecuteActionsInThread": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "workflowsStructuredValueInsertion": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "workflowsEmailTrigger": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "workflowsEmailTriggerAttachments": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "workflowsEmailTriggerSkipTextToHtmlParsing": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "workflowsRowCommentCreatedTrigger": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "isAutomationPerformanceStatsEnabled": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "workflowTriggerConnectionAvoidBaseMetadataApiCall": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "workflowsCustomScriptsSetNodeMaxOldSpaceSize": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "workflowExecutionCheckAvailableLeasesBeforeScheduling": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "workflowCronTriggerWeeklyNewComputeFirstTriggerTimeImplementation": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "enableExternalTableSyncWebhookRegistration": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "syncedLinkedRecordsSupportSelfLinkedRecords": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "levelsConfigAllowSyncedLinkedRecordColumns": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "externalTableSyncSyncLinkedRecordsColumnEditableForSourceWithSelectionCondition": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "reconnectSyncedLinkedRecordsColumnsWhenPossibleWhenRemovingSyncs": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "externalTableSyncFixMissingAuthoritativeColumn": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "sandboxPublishMergeSyncTargetConfig": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "externalTableSyncBatchOnNonEmptyCellsInCreatedRows": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "externalTableSyncBatchOnNumberOfComparedCells": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "externalTableSyncSkipDuplicateCreatedRows": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "externalTableSyncTimeOutUsingLastBatchStartTime": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "addExternalApiRequestBodyToDebugObj": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "relationshipMapOverrideComplexityGating": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "disableSendToMsTeamsWorkflowActionOneOnOneMessaging": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "syncSetUmappedColumnsToDestroyedInFixers": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "disabled" + } + }, + "processExternalTableSyncsGetEnqueueContext": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "externalTableSyncFullTableDiffInThreadedEnqueue": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "omniCollaboratorInfoExtraction": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "attachmentProcessingTaskUseBatchedProcessingForExperimentalUploadTypes": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "fileUploaderModalContainFocus": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "attachmentVideoHackilyTryToPlayAlways": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "numberBadgeOnAllAnnotations": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "attachmentAltText": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "attachmentAltTextEditor": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "singleSelectApprovalsVisualVariant": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "atlassianJiraTrimFieldMetadata": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "jiraV3IssueSearchValidation": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "optimizeGetIssueMetadataForJiraCloud": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "applyJiraV3IssueSearchAPIForJiraCloud": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "enableIntegrationsPlatformJiraServerTwoWaySync": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "azureDevOpsTwoWaySync": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "twoWaySyncIgnoreNoopArrayOperationsToSource": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "commentsNotificationRoutingForAttachments": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "revisionHistoryHideAttachmentPreviews": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "workflowsSlackEnableThreadedMessageSupport": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "workflowsSlackChannelsDistributedCache": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "workflowsOutlookEventUseHtmlForDescriptionField": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "workflowSubscribers": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "dateDependenciesDisplayDateDependencyMetadataOutsideTimelineArrows": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "dateDependenciesAutoScheduleOnAddPredecessor": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "dateDependenciesDefaultLinkMetadata": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "dateDependenciesForwardOnly": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "enableSharedApplicationOrBlockAppBlanketRedaction": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "unloadApplicationDataAndUnsubcribeFromRealtimeOnSetInactive": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "portalsShareTabApplicationScoped": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "coreDbInterfacesDemo": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "coreDbShadowValidatePrivilegesPolicyConstruction": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "disabled" + } + }, + "coreDbPrivilegesPolicyConstruction": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "coreDbShadowValidateQueryAuthorization": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "shouldEnforcePrivilegesPolicyQueryAuthorizationTimingThreshold": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "dedupeTextResultTypeLookupFilterValues": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "shouldUseNewExpandedForeignRowPolicyModel": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "delayLastPageRuntimeDestroy": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "skipPreloadingQueriesDuringPageNavigation": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "useImportRowsShadowboxOpForPaste": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "skipPublishingUnparentedPagesWithNoChanges": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "adminPanelSyncFtcForeignKeyResolveByPrimaryColumn": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "interfaceDesignerIncreasedMaxNumPagesPerPageBundle": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "interfaceDesignerEndUserHideEmptyParentsMultipleLevels": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "disableNewWorkflowTemplateInstalls": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "hideWorkflowTemplateSettings": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "truncateLongForeignKeyCellEditors": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "commentAttachments": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "proofingCommentAttachments": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "proofingCommentAttachmentsDragAndDrop": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "proofingCommentResolution": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "commentsQA": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "commentResolution": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "dlpCommentAttachments": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "updateRowsFromExternalTableSyncUnifiedEventLogBlockDbCommits": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "redactSyncCellHistory": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "disableSyncCellHistoryWrites": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "enforcePublicAPIBillingLimit": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "pageBundleInviteReminders": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "pageBundleInviteEmailRevamp2025": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "asyncInviteEmails": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "shareDialogSimplificationInterfaceEnterpriseRollout": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "enableCollaboratorCellTagNotificationsOnInterfaceForms": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "implicitSorts": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "sendResponseDataWithOtRevisionForRowUpdateRichTextCellByApplyingOperation": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "singleApplicationRealtimeServerVerboseLogging": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "singleApplicationRealtimeValidateMultiThreadSubscriptionBuffer": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "singleApplicationRealtimePromoteFromMultiThreadSubscriptionBuffer": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "sarCoreDbShadowValidation": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "memAppRecalculatorRecordedDependencyChangesMeasurements": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "replicatedApplicationHandoff": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "replicatedApplicationHandoffFallbackToCoordinatedHandoff": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "replicatedCoreDbLoadingSidecar": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "enableReadQueriesMsgpackExperiment": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "enableRealtimeUpdatesPausingForReadQueries": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "coreDbSecondaryModeShadowValidationExperiment": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "coreDbSecondaryModeShadowValidationExperimentDisableTimeouts": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "useDispatchArchitecture": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "dispatchArchitectureServing": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "logAllShadowboxOperationsPerformedInCrudRequestLogLine": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "ineligibleFromRule", + "ruleType": "enterpriseAccountIds" + } + }, + "shouldEnablePreviewModeForViewsInApplication": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "shouldEnablePreviewModeForExperimentalViewTypesInApplication": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "backendSearchForPreviewMode": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "enableSearchHighlighting": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "previewModeThreshold": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "fiveThousand", + "reason": "winningVariant" + } + }, + "previewModeRowLimit": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "oneThousand", + "reason": "winningVariant" + } + }, + "foreignRowCustomDisplayNames": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "twoWaySyncCollaboratorField": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "featureKitsEkmEncryptionForS3Data": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "searchAllVisibleColumnsInTableListRowsQuery": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "realTimeSyncTwoWaySyncPermissions": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "loadCellsByColumnInsteadOfByTable": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "realTimeSyncSourceTableLock": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "feedbackSentimentChart": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "solutionsPromptAdditionalContext": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "realTimeSyncConvertSourceDiffToTargetDiffWithNewSchema": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "ftcSelectColorReuseOnSync": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "googleCalendarSyncEnableEventTypeField": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "workflowExpressionBuilderUseCache": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "workflowFanoutDecisionSkipLookupCellConversion": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "workflowCustomScriptRetryOnUserScriptTimeout": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "workflowExecutionLoadPayloadInBatch": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "workflowExecutionJobLockingDuringScheduling": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "workflowExpressionEvaluationOnlyLoadBranchData": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "selfLinkedRecordBackLink": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "nestedSubtasksInLevelsView": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "enableCustomGridHeaderHeight": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "shouldShowDatasetsInLinkedRecordPicker": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "includeColumnIsSyncedInPublicApi": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "coreDbFirstLoading": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "disabled" + } + }, + "coreDbFirstLoadingSidecar": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "coreDbFirstParallelLoading": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "enableCoreDbSearchIndexing": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "enableCoreDbShadowboxMigrationWrapper": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "lazyLoadRowTemplateUserAccessStatusClient": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "lazyLoadRowTemplateUserAccessStatusServer": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "atlassianJiraAlwaysConstructSchemaFromGetFieldsEndpoint": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "editIndividualFieldPropertiesInManagedCurrencyField": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "snowflakeGetSqlQueryResultsAsStream": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "enablePublicApiRevertAction": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "useRecordCardsForSmallerArtifacts": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "allowGiantGroupsApplicationScoped": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "allowGiantGroupsHydrateUserGroupMemberProfiles": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "enableOmniToDeleteWorkflows": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "hideOmniMoleInInterfaces": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "detailViewFocusScopeContain": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "disabled" + } + }, + "enableCoreDbIsoTableDataCache": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "enableOmniAppCreationSharePromptPopover": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "winningVariant" + } + }, + "packagesPlatformEnableBlocks": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "enableListInterfaceReestructure": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "disabled" + } + }, + "enableHiveCsamModeration": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiBaseAgent": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "useLargeCsvImporter": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "onlyTriggerAutomationOnInitialAttachmentUpload": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "enableExternalTableSyncSnowflakeClientRefresher": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "hydrateGroupMembersFromRelatedEnterpriseAccounts": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "useFullReplacementForLargeRichTextDiffs": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "enableExternalTableSyncSnowflakeDataFetcherChunkedConcurrency": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "assignNumberedSubdomainForCdn": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "sqlSyncAwarenessTooltip": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "workerGroupTagOnWorkerLookupError": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "shipAiCapabilityPolicyToClient": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "includeWorkflowMetadataInMailgunUserVariables": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "publicInterfacePagesAttachmentDownloads": { + "targetedEntityId": "apphwgei0ETb2zCXz", + "targetedEntityType": "applicationUnified", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + } + } + }, + "rootBillingEntityFeatureFlagClientContext": { + "flagNameToPrecalculatedVariant": { + "enableEffectiveAiPolicyShadowEnforcement": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "enableAiCapabilityEngineEnforcement": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "gridAccessibilityImprovements4": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "tabDropdownImprovements": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "timelineSummaryConfigPreservedWhenInvalid": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "interfaceDesignerOmniButtonAction": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "aiFilterAssistance": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "aiInterfaceFilterAssistance": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "omniUnifiedAgentForAppCreationRootBillingEntityScoped": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "control", + "reason": "disabled" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "omniConsultativeBuildingRootBillingEntityScoped": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "overrideFromRule", + "ruleType": "objectTags", + "matchContext": { + "objectTags": [ + "aiAlpha" + ] + } + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "omniCellGridViewLongText": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "omniCellGridViewContextMenuAskOmniForMultipleRows": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "packagesPlatformSkipExternalTableSyncValidityChecksOnResetAndUpdate": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "enableAiPolicyJournal": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "allowSubThreeYearDataRetention": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "allowApplicationMoveFromNonDataResidentToDataResidentRegion": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "showEstimatedTaxInCheckout": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "adminPanelSyncConvertToForeignKeyColumns": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "aiCreditsForPortalAddOnCustomers": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "linkMandateConfirmationOnSubscriptionCreation": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "trackSubscriptionStatus": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "enableCreditExpiration": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "enableOneTimeCreditNotification": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "enableSecondBatchCreditExpirationNotification": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "enableShadowCreditExpiration": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "enableCreditExpirationMondayCadence": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "dataTableShouldEnableDataResidencyForS3Data": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "sendOrbBillingEvents": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "enableOrbCustomerCreation": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "enableOrbCreditLimitEnforcement": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "enableOrbSyncInStripeTasks": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "allowGiantGroups": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "enableExternalShareCollaboratorInviteRestriction": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "addRecordDialogFocusManagement": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "interfaceLayoutAccessibilityImprovements": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "checkoutPromoCode": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + }, + "requireAiWorkspaceAllowlistToEnableAi": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "rootBillingEntity", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + }, + "decisionCacheKey": "wsp1QdxLoq6W33Qpq" + } + } + }, + "workspaceFeatureFlagClientContext": { + "workspaceId": "wsp1QdxLoq6W33Qpq", + "flagNameToPrecalculatedVariant": { + "enableExternalAgentOnboardingXP": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "omniChatSidebarResizing": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "sandboxAllowMoveMainAndSandboxApplicationsOutOfWorkspace": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "aiModerationParallelizationForOpenAiEnabled": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "control", + "reason": "disabled" + } + }, + "aiFieldAgentTokenMaxiExperiment": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "displayTaxIdComponents": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "billingRefactoredBusinessUpgradePaths": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "snapshotRestoreUnifiedEventLogEvents": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "snapshotRestoreUnifiedEventLogBlockDbCommits": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "automaticUserContextInferenceFromEmailExperiment": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "control", + "reason": "disabled" + } + }, + "enableHardRecordEnforcementServerSide": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "enableAttachmentLimitEnforcement": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "enableUpdatedOverLimitBadge": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "workspaceSettingsGracefullyHandleMissingApplicationOrPageBundle": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "workspaceSettingsShow300kAnd400kAiCreditPacks": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "unpaidTeamTrialEmailExperiment": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "control", + "reason": "random" + } + }, + "workspaceInviteEmailRevamp2025": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "shareDialogInlineInviteLink": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "churnPreventionTeamPlanBillingCoupon2025Q4": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "control", + "reason": "disabled" + } + }, + "freeUserHighActivityDiscount": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "treatment", + "reason": "winningVariant" + } + }, + "highActivityOverL56dFreeWorkspaceBillingCoupon2025Q1": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "control", + "reason": "random" + } + }, + "highActivityOverL56DFreeWorkspaceBillingCouponGa": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "nearOrAtLimitsFreeWorkspaceBillingCoupon2025Q1": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "control", + "reason": "random" + } + }, + "checkoutAnnualFreeMonths": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "control", + "reason": "disabled" + } + }, + "checkoutModalStepperExperiment2026": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "deprecateHasMigratedToStripeDynamoDbCheck": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + }, + "templateMatchingFastStart": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "enableDecember2025TeamStarterPackPromotion": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "control", + "reason": "trafficLevel" + } + }, + "trialConversionAnnualDefault": { + "targetedEntityId": "wsp1QdxLoq6W33Qpq", + "targetedEntityType": "workspace", + "precalculatedVariant": { + "variant": "treatment", + "reason": "random" + } + } + } + }, + "hasApplicationEverInstalledSolutionsPackage": false, + "isUserLimitsHardEnforcementEnabledForParentWorkspace": false, + "ekmEnablementMetadata": { + "isEkmAttachmentUploadEnabled": false, + "isEkmEnabled": false, + "isFilestackFullyDisabled": true + }, + "syncedViewLimitMetadata": { + "maxSyncableSharedViews": 100, + "currentSyncableSharedViews": 0 + }, + "aiConsumptionInfo": null, + "dataResidentRegion": null, + "assistantsById": {}, + "hasUserSeenAntiPhishingWarningModalForApplication": false, + "areImplicitSortsEnabled": false, + "rowLimitPerViewOrPage": 1000, + "previewModeThreshold": 5000, + "customLoginPageProps": { + "loginCoverImageUrl": null, + "loginLogoImageUrl": null, + "signedLoginUserContentUrls": {} + }, + "emailWorkflowTriggerDomain": "automations.airtableemail.com", + "appAgentsById": {}, + "isDispatchArchitectureEnabled": false, + "isConstantPooledData": false + } +} \ No newline at end of file diff --git a/packages/mcp-server/test/test-daemon-server.test.js b/packages/mcp-server/test/test-daemon-server.test.js index f865cd8..589528d 100644 --- a/packages/mcp-server/test/test-daemon-server.test.js +++ b/packages/mcp-server/test/test-daemon-server.test.js @@ -55,6 +55,59 @@ describe('GET /daemon/health', () => { }); }); +describe('GET /daemon/session-health', () => { + let s2; + let tmp2; + + before(async () => { + tmp2 = join(tmpdir(), 'test-airtable-session-health-' + process.pid); + mkdirSync(tmp2, { recursive: true }); + // Inject the session check so the endpoint never launches a real browser. + s2 = await startDaemonServer({ + port: 0, + configDir: tmp2, + getSessionHealth: async () => ({ valid: true, userId: 'usr123' }), + }); + }); + + after(async () => { + if (s2) await s2.stop().catch(() => {}); + rmSync(tmp2, { recursive: true, force: true }); + }); + + it('returns 401 when Authorization header is missing', async () => { + const response = await fetch(`http://127.0.0.1:${s2.port}/daemon/session-health`); + assert.strictEqual(response.status, 401); + }); + + it('reuses the shared browser and returns the injected session health', async () => { + const response = await fetch(`http://127.0.0.1:${s2.port}/daemon/session-health`, { + headers: { 'Authorization': `Bearer ${s2.bearerToken}` }, + }); + assert.strictEqual(response.status, 200); + const body = await response.json(); + assert.strictEqual(body.valid, true); + assert.strictEqual(body.userId, 'usr123'); + }); +}); + +describe('POST /daemon/release-browser', () => { + it('returns 401 when Authorization header is missing', async () => { + const response = await fetch(`http://127.0.0.1:${server.port}/daemon/release-browser`, { method: 'POST' }); + assert.strictEqual(response.status, 401); + }); + + it('returns ok:true with a valid bearer token (idempotent when no browser is open)', async () => { + const response = await fetch(`http://127.0.0.1:${server.port}/daemon/release-browser`, { + method: 'POST', + headers: { 'Authorization': `Bearer ${server.bearerToken}` }, + }); + assert.strictEqual(response.status, 200); + const body = await response.json(); + assert.strictEqual(body.ok, true); + }); +}); + describe('GET /daemon/events', () => { it('returns 401 when bearer token is wrong', async () => { const controller = new AbortController(); diff --git a/packages/webview/src/lib/friendlyError.ts b/packages/webview/src/lib/friendlyError.ts index b25dcfe..fbca8bb 100644 --- a/packages/webview/src/lib/friendlyError.ts +++ b/packages/webview/src/lib/friendlyError.ts @@ -11,7 +11,20 @@ export interface FriendlyError { const PATTERNS: Array<{ test: RegExp; message: string; hint?: string }> = [ { - test: /ENOTFOUND|EAI_AGAIN|ECONNREFUSED|ECONNRESET|ETIMEDOUT|fetch failed|network/i, + // Browser-profile contention (Chrome exit code 21) — a second Chrome tried + // to open the shared persistent profile that another process already holds. + // MUST precede the network rule: the raw Chrome launch log contains the + // flag `--disable-background-networking`, which a bare /network/ match would + // mis-classify as a connectivity problem and send the user chasing VPNs. + test: /launchPersistentContext|exit ?code ?21|exitCode=21|Target page, context or browser has been closed|ProcessSingleton|user-data-dir/i, + message: 'Another program is using the Airtable browser profile.', + hint: 'Close other Airtable/MCP sessions (or wait a moment) and try again — the extension shares one browser profile.', + }, + { + // Genuine connectivity failures only: Node errno codes, undici "fetch + // failed", and Chromium net:: navigation errors. A bare "network" was + // removed because it false-matched Chrome command-line flags. + test: /ENOTFOUND|EAI_AGAIN|ECONNREFUSED|ECONNRESET|ECONNABORTED|ETIMEDOUT|ENETUNREACH|EHOSTUNREACH|fetch failed|net::ERR|NetworkError/i, message: 'Network error while contacting the service.', hint: 'Check your internet connection (or proxy/VPN) and try again.', }, @@ -30,11 +43,6 @@ const PATTERNS: Array<{ test: RegExp; message: string; hint?: string }> = [ message: 'Airtable is rate-limiting requests.', hint: 'Wait a minute and try again.', }, - { - test: /exit code 21|launchPersistentContext|Target page, context or browser has been closed/i, - message: 'The browser could not start (its profile may be locked).', - hint: 'Close any leftover Chrome windows from a previous login and retry.', - }, { test: /No supported browser|executable doesn't exist|chrome-missing/i, message: 'No usable browser was found.', diff --git a/packages/webview/src/test/friendlyError.test.ts b/packages/webview/src/test/friendlyError.test.ts new file mode 100644 index 0000000..399bfe2 --- /dev/null +++ b/packages/webview/src/test/friendlyError.test.ts @@ -0,0 +1,52 @@ +import { describe, it, expect } from 'vitest'; +import { friendlyError } from '../lib/friendlyError'; + +// The exact shape of the raw error the MCP health-check emits when a SECOND +// Chrome tries to open the already-in-use persistent profile (exit code 21). +// Note it contains the Chrome flag `--disable-background-networking`, which is +// what historically tripped the greedy /network/i rule into a bogus +// "Network error — check your VPN" message. +const PROFILE_IN_USE_RAW = + 'browserType.launchPersistentContext: Target page, context or browser has been closed\n' + + 'Browser logs:\n\n C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe ' + + '--disable-background-networking --disable-background-timer-throttling ' + + '--user-data-dir=C:\\Users\\admin\\.airtable-user-mcp\\.chrome-profile --remote-debugging-pipe about:blank\n' + + ' pid=43736\n - [pid=43736] \n'; + +describe('friendlyError — browser-profile contention is not a network error', () => { + it('classifies the exit-21 launchPersistentContext failure as a profile-in-use error', () => { + const result = friendlyError(PROFILE_IN_USE_RAW); + expect(result).not.toBeNull(); + expect(result!.message).toMatch(/profile/i); + expect(result!.message).not.toBe('Network error while contacting the service.'); + }); + + it('does not treat the bare Chrome flag --disable-background-networking as a network error', () => { + const result = friendlyError('chrome launched with --disable-background-networking and exited'); + expect(result!.message).not.toBe('Network error while contacting the service.'); + }); +}); + +describe('friendlyError — genuine network failures still map to the network message', () => { + it('maps an ECONNREFUSED errno to the network message', () => { + const result = friendlyError('connect ECONNREFUSED 127.0.0.1:443'); + expect(result!.message).toBe('Network error while contacting the service.'); + }); + + it('maps a Chromium net::ERR_* navigation failure to the network message', () => { + const result = friendlyError('page.goto: net::ERR_NAME_NOT_RESOLVED at https://airtable.com/'); + expect(result!.message).toBe('Network error while contacting the service.'); + }); + + it('maps an undici "fetch failed" to the network message', () => { + const result = friendlyError('TypeError: fetch failed'); + expect(result!.message).toBe('Network error while contacting the service.'); + }); +}); + +describe('friendlyError — unrelated classifications are unchanged', () => { + it('still maps a 401 to the authentication message', () => { + const result = friendlyError('Request failed with status 401 Unauthorized'); + expect(result!.message).toBe('Authentication was rejected.'); + }); +}); From 80aa32708157143ea4eea6b5d088628b3bb6d101 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sun, 21 Jun 2026 11:42:44 +0300 Subject: [PATCH 121/246] =?UTF-8?q?feat(sync):=20matchByNaturalKeys=20?= =?UTF-8?q?=E2=80=94=20add-only,=20ambiguity-safe=20record=20matching?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/records.js | 72 ++++++++++++++++ .../test/sync/test-natural-key.test.js | 84 +++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 packages/mcp-server/test/sync/test-natural-key.test.js diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index 190077e..562a10a 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -1162,6 +1162,78 @@ export async function pruneRecords({ client, destSnapshot, idmap, policy, policy } } +// ────────────────────────────────────────────────────────────────────────────── +// matchByNaturalKeys — pure natural-key record matching +// ────────────────────────────────────────────────────────────────────────────── + +/** + * Natural-key record matching: add idmap.records entries by matching source→dest records on a + * key field's VALUE (field id resolved per-base by name). ADD-ONLY + ambiguity-safe — never + * removes/overwrites a mapping, never claims an already-mapped dest, never guesses → can only + * protect records (prevent false-orphan deletes / duplicates), never mis-map. + */ +export function matchByNaturalKeys({ srcSnapshot, destSnapshot, naturalKeys, idmap, result }) { + if (result.matched == null) result.matched = 0; + if (!result.warnings) result.warnings = []; + idmap.records ??= {}; + const srcTables = new Map((srcSnapshot.tables || []).map((t) => [t.name, t])); + const dstTables = new Map((destSnapshot.tables || []).map((t) => [t.name, t])); + + for (const [tableName, keyFieldName] of Object.entries(naturalKeys || {})) { + if (!keyFieldName) continue; + const st = srcTables.get(tableName); + const dt = dstTables.get(tableName); + const srcKeyId = st?.fields.find((f) => f.name === keyFieldName)?.id; + const dstKeyId = dt?.fields.find((f) => f.name === keyFieldName)?.id; + if (!st || !dt || !srcKeyId || !dstKeyId) { + result.warnings.push({ code: 'NATURAL_KEY_FIELD_MISSING', message: `Table "${tableName}": key field "${keyFieldName}" not found on ${(!st || !srcKeyId) ? 'source' : 'dest'}` }); + continue; + } + + // dest records grouped by key value (skip empty) + const destByKey = new Map(); + for (const r of (dt.records || [])) { + const v = (r.cellValuesByColumnId || {})[dstKeyId]; + if (v == null || v === '') continue; + const k = String(v); + if (!destByKey.has(k)) destByKey.set(k, []); + destByKey.get(k).push(r.id); + } + + // count unmapped source key values (a value on >1 unmapped source = ambiguous) + const srcKeyCount = new Map(); + for (const r of (st.records || [])) { + if (idmap.records[r.id]) continue; + const v = (r.cellValuesByColumnId || {})[srcKeyId]; + if (v == null || v === '') continue; + const k = String(v); + srcKeyCount.set(k, (srcKeyCount.get(k) || 0) + 1); + } + + const claimed = new Set(Object.values(idmap.records)); // updated as we add + for (const r of (st.records || [])) { + if (idmap.records[r.id]) continue; // existing mapping wins + const v = (r.cellValuesByColumnId || {})[srcKeyId]; + if (v == null || v === '') continue; // empty key → skip + const k = String(v); + if (srcKeyCount.get(k) > 1) { + result.warnings.push({ code: 'NATURAL_KEY_AMBIGUOUS', message: `Table "${tableName}": key "${k}" on multiple source records — skipped` }); + continue; + } + const dests = destByKey.get(k) || []; + if (dests.length === 0) continue; // no dest match → new record (Pass 1 creates) + if (dests.length > 1) { + result.warnings.push({ code: 'NATURAL_KEY_AMBIGUOUS', message: `Table "${tableName}": key "${k}" matches multiple dest records — skipped` }); + continue; + } + if (claimed.has(dests[0])) continue; // single dest already mapped → don't steal + idmap.records[r.id] = dests[0]; + claimed.add(dests[0]); + result.matched++; + } + } +} + // ────────────────────────────────────────────────────────────────────────────── // reconcile — existence-prune stale idmap.records entries // ────────────────────────────────────────────────────────────────────────────── diff --git a/packages/mcp-server/test/sync/test-natural-key.test.js b/packages/mcp-server/test/sync/test-natural-key.test.js new file mode 100644 index 0000000..489ea93 --- /dev/null +++ b/packages/mcp-server/test/sync/test-natural-key.test.js @@ -0,0 +1,84 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { matchByNaturalKeys } from '../../src/sync/records.js'; + +// helper builders +const tbl = (name, fields, records) => ({ id: 't' + name, name, fields, records }); +const fld = (id, name, type = 'text') => ({ id, name, type }); +const rec = (id, cells) => ({ id, cellValuesByColumnId: cells }); +const fresh = () => ({ matched: 0, warnings: [], idmap: undefined }); + +function setup(srcRecs, dstRecs, idmapRecords = {}) { + const src = { tables: [tbl('Customers', [fld('sEmail', 'Email')], srcRecs)] }; + const dst = { tables: [tbl('Customers', [fld('dEmail', 'Email')], dstRecs)] }; + const idmap = { tables: {}, fields: {}, records: { ...idmapRecords } }; + const result = { matched: 0, warnings: [] }; + return { src, dst, idmap, result }; +} + +describe('matchByNaturalKeys', () => { + it('matches an unmapped src/dest pair sharing a key value', () => { + const { src, dst, idmap, result } = setup( + [rec('recS1', { sEmail: 'a@x.com' })], + [rec('recD1', { dEmail: 'a@x.com' })]); + matchByNaturalKeys({ srcSnapshot: src, destSnapshot: dst, naturalKeys: { Customers: 'Email' }, idmap, result }); + assert.equal(idmap.records.recS1, 'recD1'); + assert.equal(result.matched, 1); + }); + it('leaves an existing mapping untouched (existing wins)', () => { + const { src, dst, idmap, result } = setup( + [rec('recS1', { sEmail: 'a@x.com' })], + [rec('recD1', { dEmail: 'a@x.com' }), rec('recDold', { dEmail: 'a@x.com' })], + { recS1: 'recDold' }); + matchByNaturalKeys({ srcSnapshot: src, destSnapshot: dst, naturalKeys: { Customers: 'Email' }, idmap, result }); + assert.equal(idmap.records.recS1, 'recDold'); + assert.equal(result.matched, 0); + }); + it('ambiguous: two dest records share a key value → no match + warn', () => { + const { src, dst, idmap, result } = setup( + [rec('recS1', { sEmail: 'a@x.com' })], + [rec('recD1', { dEmail: 'a@x.com' }), rec('recD2', { dEmail: 'a@x.com' })]); + matchByNaturalKeys({ srcSnapshot: src, destSnapshot: dst, naturalKeys: { Customers: 'Email' }, idmap, result }); + assert.equal(idmap.records.recS1, undefined); + assert.ok(result.warnings.some((w) => w.code === 'NATURAL_KEY_AMBIGUOUS')); + }); + it('ambiguous: two unmapped src records share a key → no match + warn', () => { + const { src, dst, idmap, result } = setup( + [rec('recS1', { sEmail: 'a@x.com' }), rec('recS2', { sEmail: 'a@x.com' })], + [rec('recD1', { dEmail: 'a@x.com' })]); + matchByNaturalKeys({ srcSnapshot: src, destSnapshot: dst, naturalKeys: { Customers: 'Email' }, idmap, result }); + assert.equal(result.matched, 0); + assert.ok(result.warnings.some((w) => w.code === 'NATURAL_KEY_AMBIGUOUS')); + }); + it('empty/null key value → skip, no match', () => { + const { src, dst, idmap, result } = setup( + [rec('recS1', { sEmail: '' }), rec('recS2', { sEmail: null })], + [rec('recD1', { dEmail: '' })]); + matchByNaturalKeys({ srcSnapshot: src, destSnapshot: dst, naturalKeys: { Customers: 'Email' }, idmap, result }); + assert.equal(result.matched, 0); + }); + it('missing key field on a side → NATURAL_KEY_FIELD_MISSING, table skipped', () => { + const { src, dst, idmap, result } = setup([rec('recS1', { sEmail: 'a@x.com' })], [rec('recD1', { dEmail: 'a@x.com' })]); + matchByNaturalKeys({ srcSnapshot: src, destSnapshot: dst, naturalKeys: { Customers: 'Ghost' }, idmap, result }); + assert.equal(result.matched, 0); + assert.ok(result.warnings.some((w) => w.code === 'NATURAL_KEY_FIELD_MISSING')); + }); + it('never claims an already-mapped dest (no stealing)', () => { + // recD1 already mapped to recSold; recS1 (same key) must NOT steal it + const { src, dst, idmap, result } = setup( + [rec('recS1', { sEmail: 'a@x.com' })], + [rec('recD1', { dEmail: 'a@x.com' })], + { recSold: 'recD1' }); + matchByNaturalKeys({ srcSnapshot: src, destSnapshot: dst, naturalKeys: { Customers: 'Email' }, idmap, result }); + assert.equal(idmap.records.recS1, undefined); + assert.equal(idmap.records.recSold, 'recD1'); + }); + it('matches on a computed (numeric) key value via String() compare', () => { + const src = { tables: [tbl('Orders', [fld('sCode', 'Code', 'autoNumber')], [rec('recS1', { sCode: 1042 })])] }; + const dst = { tables: [tbl('Orders', [fld('dCode', 'Code', 'number')], [rec('recD1', { dCode: 1042 })])] }; + const idmap = { tables: {}, fields: {}, records: {} }; + const result = { matched: 0, warnings: [] }; + matchByNaturalKeys({ srcSnapshot: src, destSnapshot: dst, naturalKeys: { Orders: 'Code' }, idmap, result }); + assert.equal(idmap.records.recS1, 'recD1'); + }); +}); From 0ff6d60059720bce82c4def8710649e6db0fdf26 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sun, 21 Jun 2026 11:47:57 +0300 Subject: [PATCH 122/246] feat(sync): reconcile performs natural-key matching (removes stub) Snapshots the source base schema+records after existence-prune and calls matchByNaturalKeys so reconcile now fills idmap.records from natural-key pairs instead of emitting RECONCILE_NATURAL_KEY_NOT_IMPLEMENTED warnings. result.matched added to reconcile return. Stub for-loop removed. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/records.js | 14 +- .../test/sync/test-natural-key.test.js | 129 +++++++++++++++++- 2 files changed, 135 insertions(+), 8 deletions(-) diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index 562a10a..aa4c25d 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -1266,6 +1266,7 @@ export async function reconcile({ client, sourceBaseId, destBaseId, naturalKeys updated: 0, skipped: 0, failed: 0, + matched: 0, warnings: [], idmap, }; @@ -1293,14 +1294,13 @@ export async function reconcile({ client, sourceBaseId, destBaseId, naturalKeys } } - // naturalKeys re-match (stub: emit warning for each non-empty key) - for (const [tableName, fieldName] of Object.entries(naturalKeys)) { - if (fieldName) { - result.warnings.push({ - code: 'RECONCILE_NATURAL_KEY_NOT_IMPLEMENTED', - message: `naturalKeys re-match for table "${tableName}" (field "${fieldName}") is not yet implemented`, - }); + // Natural-key re-match: snapshot source schema + records, then call matcher. + if (Object.keys(naturalKeys).length > 0) { + const srcSnapshot = await snapshotSchemaOnly(client, sourceBaseId); + for (const table of srcSnapshot.tables) { + table.records = await snapshotTableRecords(client, sourceBaseId, table); } + matchByNaturalKeys({ srcSnapshot, destSnapshot, naturalKeys, idmap, result }); } saveIdmap(sourceBaseId, destBaseId, idmap); diff --git a/packages/mcp-server/test/sync/test-natural-key.test.js b/packages/mcp-server/test/sync/test-natural-key.test.js index 489ea93..9bac5bd 100644 --- a/packages/mcp-server/test/sync/test-natural-key.test.js +++ b/packages/mcp-server/test/sync/test-natural-key.test.js @@ -1,6 +1,10 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { matchByNaturalKeys } from '../../src/sync/records.js'; +import { join } from 'node:path'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { matchByNaturalKeys, reconcile } from '../../src/sync/records.js'; +import { saveIdmap } from '../../src/sync/idmap.js'; // helper builders const tbl = (name, fields, records) => ({ id: 't' + name, name, fields, records }); @@ -82,3 +86,126 @@ describe('matchByNaturalKeys', () => { assert.equal(idmap.records.recS1, 'recD1'); }); }); + +// ────────────────────────────────────────────────────────────────────────────── +// reconcile — natural-key integration +// ────────────────────────────────────────────────────────────────────────────── + +/** + * Build a fake client that serves two in-memory bases by appId. + * bases: { [appId]: { tables: [{ id, name, fields: [{id,name,type}], records: [{id,cellValuesByColumnId}] }] } } + * + * getApplicationData → data.tables shape (normalizeSchema accepts both tableSchemas and tables). + * queryRecords → summary.rows with { id, fields } shape. + */ +function makeReconcileClient(bases) { + return { + async getApplicationData(appId) { + const base = bases[appId] || { tables: [] }; + return { + data: { + tables: base.tables.map((t) => ({ + id: t.id, + name: t.name, + primaryFieldId: t.fields[0]?.id ?? null, + fields: t.fields, + views: [{ id: 'v' + t.id, name: 'Grid view', type: 'grid' }], + })), + }, + }; + }, + async queryRecords(appId, tableId, _viewId, _opts) { + const base = bases[appId] || { tables: [] }; + const t = base.tables.find((x) => x.id === tableId); + return { + summary: { + rows: (t?.records || []).map((r) => ({ id: r.id, fields: r.cellValuesByColumnId })), + }, + }; + }, + }; +} + +describe('reconcile naturalKeys integration', () => { + it('snapshots source, calls matchByNaturalKeys, returns matched>0, no NOT_IMPLEMENTED stub', async () => { + // Redirect disk I/O to a throwaway temp dir — hermetic, no real ~/.airtable-user-mcp writes. + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'reconcile-nk-')); + + const SRC = 'appSRC001'; + const DST = 'appDST001'; + + // Pre-seed an idmap that has a table match but NO record match for recS1/recD1. + saveIdmap(SRC, DST, { + tables: { tSRC: 'tDST' }, + fields: {}, + records: {}, + }); + + const bases = { + [SRC]: { + tables: [{ + id: 'tSRC', + name: 'Contacts', + fields: [{ id: 'fSrcEmail', name: 'Email', type: 'text' }], + records: [{ id: 'recS1', cellValuesByColumnId: { fSrcEmail: 'alice@example.com' } }], + }], + }, + [DST]: { + tables: [{ + id: 'tDST', + name: 'Contacts', + fields: [{ id: 'fDstEmail', name: 'Email', type: 'text' }], + records: [{ id: 'recD1', cellValuesByColumnId: { fDstEmail: 'alice@example.com' } }], + }], + }, + }; + + const result = await reconcile({ + client: makeReconcileClient(bases), + sourceBaseId: SRC, + destBaseId: DST, + naturalKeys: { Contacts: 'Email' }, + }); + + // (a) matched is present and > 0 — matchByNaturalKeys was called and found the pair + assert.equal(result.matched, 1, 'reconcile should report matched=1 after natural-key match'); + + // (b) No RECONCILE_NATURAL_KEY_NOT_IMPLEMENTED stub warning emitted + const stubWarns = result.warnings.filter((w) => w.code === 'RECONCILE_NATURAL_KEY_NOT_IMPLEMENTED'); + assert.equal(stubWarns.length, 0, 'stub warning must be gone'); + + // (c) The idmap on disk now has the record mapping + assert.equal(result.idmap.records.recS1, 'recD1', 'idmap.records must include the matched pair'); + }); + + it('reconcile without naturalKeys returns matched:0 and no stub warnings', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'reconcile-nonk-')); + + const SRC = 'appSRC002'; + const DST = 'appDST002'; + + saveIdmap(SRC, DST, { tables: {}, fields: {}, records: { recS1: 'recD1' } }); + + const bases = { + [SRC]: { tables: [] }, + [DST]: { + tables: [{ + id: 'tDST', + name: 'T', + fields: [{ id: 'fDstId', name: 'Name', type: 'text' }], + records: [{ id: 'recD1', cellValuesByColumnId: {} }], + }], + }, + }; + + const result = await reconcile({ + client: makeReconcileClient(bases), + sourceBaseId: SRC, + destBaseId: DST, + // no naturalKeys + }); + + assert.equal(result.matched, 0, 'no naturalKeys → matched stays 0'); + assert.equal(result.warnings.filter((w) => w.code === 'RECONCILE_NATURAL_KEY_NOT_IMPLEMENTED').length, 0); + }); +}); From 0d1e26b28606663cedbd7dd95d6d0bcde544f766 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sun, 21 Jun 2026 11:54:07 +0300 Subject: [PATCH 123/246] feat(sync): runRecords natural-key pre-pass + thread naturalKeys through apply Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/index.js | 4 +- packages/mcp-server/src/sync/records.js | 13 +++- .../test/sync/test-natural-key.test.js | 68 ++++++++++++++++++- 3 files changed, 79 insertions(+), 6 deletions(-) diff --git a/packages/mcp-server/src/sync/index.js b/packages/mcp-server/src/sync/index.js index bb5a3d8..0c091e5 100644 --- a/packages/mcp-server/src/sync/index.js +++ b/packages/mcp-server/src/sync/index.js @@ -114,7 +114,7 @@ export function diffDetail({ sourceBaseId, destBaseId, diffId, detail, offset, l return renderDiff(savedDiff, { detail, offset, limit }); } -export async function apply({ client, sourceBaseId, destBaseId, planId, runStartedAt, skip = [], policy, policyOverrides, confirmDeletions, fieldMappings }) { +export async function apply({ client, sourceBaseId, destBaseId, planId, runStartedAt, skip = [], policy, policyOverrides, confirmDeletions, fieldMappings, naturalKeys }) { const fullPlan = loadPlan(sourceBaseId, destBaseId, planId); if (!fullPlan) throw new Error(`No saved plan "${planId}" for ${sourceBaseId} -> ${destBaseId}. Run mode=plan first.`); @@ -155,7 +155,7 @@ export async function apply({ client, sourceBaseId, destBaseId, planId, runStart // journal) and writes a status file; poll via `sync_base mode=status`. if (!result.aborted) { writeRecordsJobStatus(sourceBaseId, destBaseId, planId, { status: 'running', startedAt: runStartedAt }); - applyRecordsImpl({ client, sourceBaseId, destBaseId, planId, runStartedAt, policy, policyOverrides, confirmDeletions, fieldMappings }) + applyRecordsImpl({ client, sourceBaseId, destBaseId, planId, runStartedAt, policy, policyOverrides, confirmDeletions, fieldMappings, naturalKeys }) .then((r) => writeRecordsJobStatus(sourceBaseId, destBaseId, planId, { status: 'done', startedAt: runStartedAt, finishedAt: new Date().toISOString(), result: { diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index aa4c25d..a35492f 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -947,7 +947,7 @@ export async function reapplyViewFilters({ client, srcSnapshot, destSnapshot, id * @param {object} opts.result - result accumulator (mutated in place, returned) * @returns {Promise} - the result accumulator */ -export async function runRecords({ client, srcSnapshot, destSnapshot, idmap, policy, policyOverrides, confirmDeletions, fieldMappings, limiter, journal, persist, result }) { +export async function runRecords({ client, srcSnapshot, destSnapshot, idmap, policy, policyOverrides, confirmDeletions, fieldMappings, naturalKeys, limiter, journal, persist, result }) { // Pre-flight: validate field mappings BEFORE any write. Abort on error. const { errors, resolved } = validateFieldMappings(srcSnapshot, destSnapshot, fieldMappings || {}); if (errors.length) { @@ -957,6 +957,13 @@ export async function runRecords({ client, srcSnapshot, destSnapshot, idmap, pol throw err; } + // Natural-key pre-pass: match real-but-unmapped dest records BEFORE Pass 1 + prune, + // so Pass 1 updates (not duplicates) them and pruneRecords does not treat them as orphans. + if (naturalKeys && Object.keys(naturalKeys).length) { + matchByNaturalKeys({ srcSnapshot, destSnapshot, naturalKeys, idmap, result }); + persist(idmap, journal); + } + // Pass 1: scalar / select upsert — fills idmap.records await applyRecordsPass1({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result, policy, policyOverrides, fieldMappings: resolved }); persist(idmap, journal); @@ -1023,7 +1030,7 @@ export async function runRecords({ client, srcSnapshot, destSnapshot, idmap, pol * @param {string} opts.runStartedAt - ISO timestamp of this run start * @returns {Promise} - result accumulator */ -export async function applyRecords({ client, sourceBaseId, destBaseId, planId, runStartedAt, policy, policyOverrides, confirmDeletions, fieldMappings }) { +export async function applyRecords({ client, sourceBaseId, destBaseId, planId, runStartedAt, policy, policyOverrides, confirmDeletions, fieldMappings, naturalKeys }) { // 1. Load the converged idmap (produced by the schema apply phase) const idmap = loadIdmap(sourceBaseId, destBaseId); idmap.records ??= {}; @@ -1082,7 +1089,7 @@ export async function applyRecords({ client, sourceBaseId, destBaseId, planId, r } // Delegate the core phases to runRecords - return runRecords({ client, srcSnapshot, destSnapshot, idmap, policy, policyOverrides, confirmDeletions, fieldMappings, limiter, journal, persist, result }); + return runRecords({ client, srcSnapshot, destSnapshot, idmap, policy, policyOverrides, confirmDeletions, fieldMappings, naturalKeys, limiter, journal, persist, result }); } // ────────────────────────────────────────────────────────────────────────────── diff --git a/packages/mcp-server/test/sync/test-natural-key.test.js b/packages/mcp-server/test/sync/test-natural-key.test.js index 9bac5bd..6a0d9ed 100644 --- a/packages/mcp-server/test/sync/test-natural-key.test.js +++ b/packages/mcp-server/test/sync/test-natural-key.test.js @@ -3,7 +3,7 @@ import assert from 'node:assert/strict'; import { join } from 'node:path'; import { mkdtempSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import { matchByNaturalKeys, reconcile } from '../../src/sync/records.js'; +import { matchByNaturalKeys, reconcile, runRecords } from '../../src/sync/records.js'; import { saveIdmap } from '../../src/sync/idmap.js'; // helper builders @@ -209,3 +209,69 @@ describe('reconcile naturalKeys integration', () => { assert.equal(result.warnings.filter((w) => w.code === 'RECONCILE_NATURAL_KEY_NOT_IMPLEMENTED').length, 0); }); }); + +// ────────────────────────────────────────────────────────────────────────────── +// runRecords natural-key pre-pass integration test +// ────────────────────────────────────────────────────────────────────────────── + +function recClient() { + const calls = { create: [], update: [], deleted: [] }; + return { + calls, + async createRecords(appId, tableId, rows) { + calls.create.push({ tableId, rows }); + return { records: rows.map((r, i) => ({ id: 'new' + i })), created: rows.map((r, i) => ({ id: 'new' + i, sourceKey: r.sourceKey, rowId: 'new' + i })) }; + }, + async updateRecords(appId, tableId, rows) { calls.update.push({ tableId, rows }); return { updated: rows, failed: [] }; }, + async queryRecords(appId, tableId) { return { summary: { rows: [] } }; }, // not used (prune reads snapshot .records) + async deleteRecords(appId, tableId, rowIds) { calls.deleted.push({ tableId, rowIds }); return { deleted: rowIds.length }; }, + async addLinkItems() { return { ok: true }; }, + async createDataTransferPolicy() { return {}; }, + async pasteAttachmentsCrossBase() { return { pastedRowIds: [] }; }, + }; +} + +describe('runRecords natural-key pre-pass (mirror safety)', () => { + it('a real-but-unmapped dest record is matched, NOT deleted by mirror, NOT duplicated', async () => { + const client = recClient(); + const src = { + baseId: 'appS', + tables: [{ + id: 'tS', name: 'Customers', primaryFieldId: 'sEmail', + fields: [{ id: 'sEmail', name: 'Email', type: 'text' }], + records: [{ id: 'recS1', cellValuesByColumnId: { sEmail: 'a@x.com' } }], + }], + }; + const dst = { + baseId: 'appD', + tables: [{ + id: 'tD', name: 'Customers', views: [{ id: 'vD' }], + fields: [{ id: 'dEmail', name: 'Email', type: 'text' }], + records: [{ id: 'recD1', cellValuesByColumnId: { dEmail: 'a@x.com' } }], + }], + }; + const idmap = { + tables: { tS: 'tD' }, + fields: { sEmail: { destFld: 'dEmail', choices: {} } }, + records: {}, + }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, deleted: 0, warnings: [] }; + await runRecords({ + client, + srcSnapshot: src, + destSnapshot: dst, + idmap, + policy: 'mirror', + confirmDeletions: true, + naturalKeys: { Customers: 'Email' }, + limiter: { run: (f) => f() }, + journal: {}, + persist: () => {}, + result, + }); + + assert.equal(idmap.records.recS1, 'recD1', 'pre-pass matched the record'); + assert.equal(client.calls.create.length, 0, 'no duplicate create (went update path)'); + assert.deepEqual(client.calls.deleted, [], 'mirror did NOT delete the matched dest record'); + }); +}); From fa8a57e5412bfc89d6791f6b974e4cccf1de4dc7 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sun, 21 Jun 2026 12:00:21 +0300 Subject: [PATCH 124/246] feat(sync): sync_base passes naturalKeys to apply + surfaces matched count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - index.js mode=apply: forward naturalKeys to sync.apply() (was missing) - sync/index.js: include matched in background job status result object - sync/report.js: renderApplyResult surfaces matched in human output when >0 - index.js: update naturalKeys param description — now documents apply mode (auto pre-pass before Pass 1) and key stability requirement Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/index.js | 4 ++-- packages/mcp-server/src/sync/index.js | 1 + packages/mcp-server/src/sync/report.js | 3 +++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/mcp-server/src/index.js b/packages/mcp-server/src/index.js index ffb1374..02340d5 100644 --- a/packages/mcp-server/src/index.js +++ b/packages/mcp-server/src/index.js @@ -1587,7 +1587,7 @@ Note: "form title" is the view name itself — use rename_view to change it. "Fi sourceAppId: { type: 'string', description: 'Source base/application ID to copy schema FROM' }, destAppId: { type: 'string', description: 'Destination base/application ID to copy schema TO' }, planId: { type: 'string', description: 'Required for mode="apply" (the planId from a prior mode="plan" run) and mode="status" (the same planId, which is the background record jobId).' }, - naturalKeys: { type: 'object', description: 'Used only by mode="reconcile": map of tableName → fieldName to use as a natural key for re-matching records (e.g. { "Projects": "Name" }). Omit to run existence-prune only.', additionalProperties: { type: 'string' } }, + naturalKeys: { type: 'object', description: 'Map of tableName → fieldName to use as a natural key for record identity matching (e.g. { "Projects": "Name" }). Applies to mode="reconcile" (repairs the record map via natural-key re-match after manual edits) AND mode="apply" (auto pre-pass before Pass 1: matches existing dest records by key so they are updated rather than duplicated; also protects real-but-unmapped dest records from mirror-policy deletion). Omit for existence-prune only (reconcile) or pure ID-based sync (apply). The key field MUST be stable across bases: autoNumber renumbers on creation (bad key); name/email/injected InjectID are good keys.', additionalProperties: { type: 'string' } }, detail: { type: 'string', description: 'Used only by mode="diff": section name to drill into (e.g. a table name or action key). Requires diffId.' }, diffId: { type: 'string', description: 'Used by mode="diff": ID of a prior diff to retrieve or drill into. If omitted on the initial call, one is generated automatically.' }, offset: { type: 'number', description: 'Used by mode="diff" with detail: skip this many entries before returning results.' }, @@ -2411,7 +2411,7 @@ const handlers = { const runStartedAt = new Date().toISOString(); let out; try { - out = await sync.apply({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, planId, runStartedAt, skip: skip || [], policy, policyOverrides, confirmDeletions, fieldMappings }); + out = await sync.apply({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, planId, runStartedAt, skip: skip || [], policy, policyOverrides, confirmDeletions, fieldMappings, naturalKeys }); } catch (e) { if (e && e.code === 'FIELD_MAP_INVALID') { return err(`Field mapping validation failed:\n${(e.mappingErrors || []).map((me) => ` [${me.code}] table=${me.table || '?'}${me.source ? ' source=' + me.source : ''}${me.target ? ' target=' + me.target : ''}`).join('\n')}`); diff --git a/packages/mcp-server/src/sync/index.js b/packages/mcp-server/src/sync/index.js index 0c091e5..edd8aec 100644 --- a/packages/mcp-server/src/sync/index.js +++ b/packages/mcp-server/src/sync/index.js @@ -160,6 +160,7 @@ export async function apply({ client, sourceBaseId, destBaseId, planId, runStart status: 'done', startedAt: runStartedAt, finishedAt: new Date().toISOString(), result: { created: r.created, updated: r.updated, failed: r.failed, + matched: r.matched || 0, attachmentsUploaded: r.attachmentsUploaded || 0, viewFiltersReapplied: r.viewFiltersReapplied || 0, deleted: r.deleted || 0, warningCount: (r.warnings || []).length, diff --git a/packages/mcp-server/src/sync/report.js b/packages/mcp-server/src/sync/report.js index f1e4c0e..50fbdb6 100644 --- a/packages/mcp-server/src/sync/report.js +++ b/packages/mcp-server/src/sync/report.js @@ -68,6 +68,9 @@ export function renderApplyResult(result) { ` skipped: ${result.skipped}`, ` failed: ${result.failed}`, ]; + if (result.matched != null && result.matched > 0) { + lines.push(` matched: ${result.matched} (natural-key pre-pass)`); + } if (result.records) { const r = result.records; if (r.status === 'running') { From 3f912d5bfa35296b65f20521d4b82424a9d10e68 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sun, 21 Jun 2026 12:04:08 +0300 Subject: [PATCH 125/246] docs(sync): document natural-key record matching Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 5 +++++ CLAUDE.md | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 299bd60..26891c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how ## [Unreleased] +### MCP server — sync_base natural-key record matching (2026-06-21) + +#### Added +- `sync_base` — **natural-key record matching** (`naturalKeys: { [tableName]: fieldName }`): matches dest↔source records by a shared key field's value (resolved by name per base) and grows `idmap.records` without any write to either base. Add-only + ambiguity-safe: existing mappings are never overwritten, already-mapped dest records are never re-claimed, and ambiguous/duplicate key values are skipped with a `NATURAL_KEY_AMBIGUOUS` warning; a missing key field emits `NATURAL_KEY_FIELD_MISSING`. Runs in two places: (1) **`mode=reconcile`** — after the existence-prune, snapshots source records and calls the matcher, then saves the updated idmap; (2) **`mode=apply` auto pre-pass** — runs before Pass 1 and before `pruneRecords`, so records `mirror` no longer treats real-but-unmapped dest records as orphans and re-syncs no longer duplicate them. The key field must be stable across bases (autoNumber renumbers → bad key; name/email/injected `InjectID` → good key). Single-field keys only; composite keys and value normalization are deferred. A `matched` count is returned in the result. + ### MCP server — sync_base reconciliation policy + custom field mappings (2026-06-20) #### Added diff --git a/CLAUDE.md b/CLAUDE.md index ce9e3b8..51aee1d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,7 +65,7 @@ Key files: - `dev-tools/` — reverse-engineering helpers (capture.js, analyze.js, debug-*.js), **gitignored** — kept locally for future reference when Airtable changes their internal API - `src/daemon/` — daemon subsystem (see daemon subsection below) - `src/safe-write.js` — atomic JSON write helper (used by lockfile, token, and settings) -- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: diff | plan | apply | status | reconcile`) in the `sync` category. Out of scope: retypes, deletions (M4). `mode=diff` (read-only): comprehensive schema comparison via pure `src/sync/compare.js` — every difference classified as **drift** (sync enforces: table/field/view existence, field type/options/choices, view filters/sorts/groups content, column visibility, colors/cover/frozen/rowHeight, primary, form), **best-effort** (sync applies but doesn't guarantee: field/view/column order, sort/group clause order), or **not-synced** (view sections — now captured by snapshot). Verdicts: `identical` (zero diffs) or `converged` (zero drift; best-effort/not-synced may remain). Persists full diff to `~/.airtable-user-mcp/sync/__/diff-.json`; returns a token-budgeted DIGEST (verdicts + class counts + per-table rollup + capped driftSample ≤25). Drill in with `detail="
"` (params: `detail`, `diffId`, `offset`, `limit`). `mode=plan` now emits a curatable **changeset**: each action carries a stable name-based `changeId` (`|
|`), a `class`, and an `apply:true` flag; digest includes a `sample` + `editHint`. New `direction` param (`to-dest` default | `to-source`) targets either base. View sync: `snapshot` also captures collaborative views + live config + **view sections** (not-synced class); `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported); accepts a `skip:[changeId]` list (and honors `apply:false` on changeset entries) — applies everything except removed changes (journal-safe; skipped dependencies degrade to UNRESOLVABLE_REF). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe by filename+size); then restores record-referencing view filters stripped during view sync. Throttle handling = per-request 429 backoff in the auth queue (the per-call limiter does NOT gate the internal per-row POSTs inside createRecords/updateRecords/addLinkItems); resumable (records journal, per-chunk persist of ~50 creates); continue-on-failure (per-record errors are warnings). `mode=status` polls a background job by planId — returns running/done/failed + live records-mapped count. `mode=reconcile` **prunes** dead record-map entries (existence-prune; per-table natural-key re-match NOT yet implemented; reconcile does NOT de-duplicate records). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json, diff-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile), `compare.js` (pure classified compare, no I/O); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. Reconciliation policy (`src/sync/policy.js`): `mode=apply` accepts a `policy` param (`mirror` | `overlay` | `preserve`, default `overlay`) that controls two axes — **extras** (what to do with dest-only records) and **conflicts** (whose value wins when both bases have the record). Presets: `mirror` = {remove, source-wins} (make dest identical to source); `overlay` = {keep, source-wins} (today's default: keep dest-only rows, source updates win); `preserve` = {keep, dest-wins} (keep dest-only rows and never overwrite dest edits). `policyOverrides` (`{ [tableName]: preset }`) overrides the global preset per table. `confirmDeletions` (boolean, default false) is a safety gate for `mirror`: without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count but deletes nothing; set `confirmDeletions:true` to actually delete. Separately, `createTable` unconditionally clears the ~3 blank scaffolding rows it seeds (best-effort; `SCAFFOLDING_ROWS_KEPT` warning if the deletion attempt fails); `pruneRecords` under `mirror`+`confirmDeletions` additionally removes any leftover dest-only blank rows as part of the ordinary orphan prune. Schema/view/attachment deletion under mirror and natural-key record matching are deferred follow-ups. Custom field mappings: `fieldMappings` (`{ [tableName]: { sourceFieldName: destFieldName } }`) injects a source field's value into a different writable scalar dest field during record sync — useful when the source is computed (e.g., `autoNumber`/formula) and you want its value preserved in a writable dest field (classic use: inject source autoNumber `Code` into dest text field `InjectID` so a dest formula can reconstruct original identity). Fail-fast pre-flight validation in `apply` (and dry-run in `plan`/`diff` via `fieldMappingErrors` in the machine output) with codes: `FIELD_MAP_TABLE_MISSING`, `FIELD_MAP_SOURCE_MISSING`, `FIELD_MAP_TARGET_MISSING`, `FIELD_MAP_TARGET_COMPUTED` (dest must be writable scalar), `FIELD_MAP_TYPE_INCOMPATIBLE` (array-typed source/dest not supported), `FIELD_MAP_COLLISION` (two sources map to same dest); an invalid mapping aborts `apply` synchronously (`FIELD_MAP_INVALID` error code). +- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: diff | plan | apply | status | reconcile`) in the `sync` category. Out of scope: retypes, deletions (M4). `mode=diff` (read-only): comprehensive schema comparison via pure `src/sync/compare.js` — every difference classified as **drift** (sync enforces: table/field/view existence, field type/options/choices, view filters/sorts/groups content, column visibility, colors/cover/frozen/rowHeight, primary, form), **best-effort** (sync applies but doesn't guarantee: field/view/column order, sort/group clause order), or **not-synced** (view sections — now captured by snapshot). Verdicts: `identical` (zero diffs) or `converged` (zero drift; best-effort/not-synced may remain). Persists full diff to `~/.airtable-user-mcp/sync/__/diff-.json`; returns a token-budgeted DIGEST (verdicts + class counts + per-table rollup + capped driftSample ≤25). Drill in with `detail="
"` (params: `detail`, `diffId`, `offset`, `limit`). `mode=plan` now emits a curatable **changeset**: each action carries a stable name-based `changeId` (`|
|`), a `class`, and an `apply:true` flag; digest includes a `sample` + `editHint`. New `direction` param (`to-dest` default | `to-source`) targets either base. View sync: `snapshot` also captures collaborative views + live config + **view sections** (not-synced class); `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported); accepts a `skip:[changeId]` list (and honors `apply:false` on changeset entries) — applies everything except removed changes (journal-safe; skipped dependencies degrade to UNRESOLVABLE_REF). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe by filename+size); then restores record-referencing view filters stripped during view sync. Throttle handling = per-request 429 backoff in the auth queue (the per-call limiter does NOT gate the internal per-row POSTs inside createRecords/updateRecords/addLinkItems); resumable (records journal, per-chunk persist of ~50 creates); continue-on-failure (per-record errors are warnings). `mode=status` polls a background job by planId — returns running/done/failed + live records-mapped count. `mode=reconcile` **prunes** dead record-map entries (existence-prune; per-table natural-key re-match NOT yet implemented; reconcile does NOT de-duplicate records). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json, diff-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile), `compare.js` (pure classified compare, no I/O); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. Reconciliation policy (`src/sync/policy.js`): `mode=apply` accepts a `policy` param (`mirror` | `overlay` | `preserve`, default `overlay`) that controls two axes — **extras** (what to do with dest-only records) and **conflicts** (whose value wins when both bases have the record). Presets: `mirror` = {remove, source-wins} (make dest identical to source); `overlay` = {keep, source-wins} (today's default: keep dest-only rows, source updates win); `preserve` = {keep, dest-wins} (keep dest-only rows and never overwrite dest edits). `policyOverrides` (`{ [tableName]: preset }`) overrides the global preset per table. `confirmDeletions` (boolean, default false) is a safety gate for `mirror`: without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count but deletes nothing; set `confirmDeletions:true` to actually delete. Separately, `createTable` unconditionally clears the ~3 blank scaffolding rows it seeds (best-effort; `SCAFFOLDING_ROWS_KEPT` warning if the deletion attempt fails); `pruneRecords` under `mirror`+`confirmDeletions` additionally removes any leftover dest-only blank rows as part of the ordinary orphan prune. Schema/view/attachment deletion under mirror and natural-key record matching are deferred follow-ups. Custom field mappings: `fieldMappings` (`{ [tableName]: { sourceFieldName: destFieldName } }`) injects a source field's value into a different writable scalar dest field during record sync — useful when the source is computed (e.g., `autoNumber`/formula) and you want its value preserved in a writable dest field (classic use: inject source autoNumber `Code` into dest text field `InjectID` so a dest formula can reconstruct original identity). Fail-fast pre-flight validation in `apply` (and dry-run in `plan`/`diff` via `fieldMappingErrors` in the machine output) with codes: `FIELD_MAP_TABLE_MISSING`, `FIELD_MAP_SOURCE_MISSING`, `FIELD_MAP_TARGET_MISSING`, `FIELD_MAP_TARGET_COMPUTED` (dest must be writable scalar), `FIELD_MAP_TYPE_INCOMPATIBLE` (array-typed source/dest not supported), `FIELD_MAP_COLLISION` (two sources map to same dest); an invalid mapping aborts `apply` synchronously (`FIELD_MAP_INVALID` error code). Natural-key record matching: `naturalKeys` (`{ [tableName]: fieldName }`) matches dest↔source records by the key field's **value** (field ID resolved per-base by name), growing `idmap.records`. Add-only + ambiguity-safe — never removes/overwrites an existing mapping, never claims an already-mapped dest; ambiguous/duplicate key values are skipped with `NATURAL_KEY_AMBIGUOUS`; a missing key field emits `NATURAL_KEY_FIELD_MISSING`. Runs in `mode=reconcile` (after the existence-prune, snapshots source records + calls matcher, then saves idmap) AND as an automatic pre-pass in `mode=apply` **before Pass 1 and prune** — so records `mirror` no longer deletes real-but-unmapped dest records as false orphans, and re-syncs don't duplicate them. The key field must be **stable across bases** (autoNumber renumbers → bad key; name/email/injected `InjectID` → good key). Single-field keys only; composite keys and value normalization are deferred. Returns a `matched` count in the result. #### packages/mcp-server — Daemon subsystem From 3418339fd21e74d9a684bb428c7728690b6300ed Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sun, 21 Jun 2026 12:09:27 +0300 Subject: [PATCH 126/246] docs: fix accuracy (reconcile natural-key stale text + add naturalKeys @params) - CLAUDE.md: Replace "natural-key re-match NOT yet implemented" with accurate description of the now-working implementation - records.js reconcile JSDoc: Update to describe actual behavior (snapshots source + calls matcher); add 'matched' to return shape - records.js runRecords & applyRecords JSDoc: Add missing @param naturalKeys documentation Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 2 +- packages/mcp-server/src/sync/records.js | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 51aee1d..c75d0ca 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,7 +65,7 @@ Key files: - `dev-tools/` — reverse-engineering helpers (capture.js, analyze.js, debug-*.js), **gitignored** — kept locally for future reference when Airtable changes their internal API - `src/daemon/` — daemon subsystem (see daemon subsection below) - `src/safe-write.js` — atomic JSON write helper (used by lockfile, token, and settings) -- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: diff | plan | apply | status | reconcile`) in the `sync` category. Out of scope: retypes, deletions (M4). `mode=diff` (read-only): comprehensive schema comparison via pure `src/sync/compare.js` — every difference classified as **drift** (sync enforces: table/field/view existence, field type/options/choices, view filters/sorts/groups content, column visibility, colors/cover/frozen/rowHeight, primary, form), **best-effort** (sync applies but doesn't guarantee: field/view/column order, sort/group clause order), or **not-synced** (view sections — now captured by snapshot). Verdicts: `identical` (zero diffs) or `converged` (zero drift; best-effort/not-synced may remain). Persists full diff to `~/.airtable-user-mcp/sync/__/diff-.json`; returns a token-budgeted DIGEST (verdicts + class counts + per-table rollup + capped driftSample ≤25). Drill in with `detail="
"` (params: `detail`, `diffId`, `offset`, `limit`). `mode=plan` now emits a curatable **changeset**: each action carries a stable name-based `changeId` (`|
|`), a `class`, and an `apply:true` flag; digest includes a `sample` + `editHint`. New `direction` param (`to-dest` default | `to-source`) targets either base. View sync: `snapshot` also captures collaborative views + live config + **view sections** (not-synced class); `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported); accepts a `skip:[changeId]` list (and honors `apply:false` on changeset entries) — applies everything except removed changes (journal-safe; skipped dependencies degrade to UNRESOLVABLE_REF). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe by filename+size); then restores record-referencing view filters stripped during view sync. Throttle handling = per-request 429 backoff in the auth queue (the per-call limiter does NOT gate the internal per-row POSTs inside createRecords/updateRecords/addLinkItems); resumable (records journal, per-chunk persist of ~50 creates); continue-on-failure (per-record errors are warnings). `mode=status` polls a background job by planId — returns running/done/failed + live records-mapped count. `mode=reconcile` **prunes** dead record-map entries (existence-prune; per-table natural-key re-match NOT yet implemented; reconcile does NOT de-duplicate records). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json, diff-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile), `compare.js` (pure classified compare, no I/O); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. Reconciliation policy (`src/sync/policy.js`): `mode=apply` accepts a `policy` param (`mirror` | `overlay` | `preserve`, default `overlay`) that controls two axes — **extras** (what to do with dest-only records) and **conflicts** (whose value wins when both bases have the record). Presets: `mirror` = {remove, source-wins} (make dest identical to source); `overlay` = {keep, source-wins} (today's default: keep dest-only rows, source updates win); `preserve` = {keep, dest-wins} (keep dest-only rows and never overwrite dest edits). `policyOverrides` (`{ [tableName]: preset }`) overrides the global preset per table. `confirmDeletions` (boolean, default false) is a safety gate for `mirror`: without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count but deletes nothing; set `confirmDeletions:true` to actually delete. Separately, `createTable` unconditionally clears the ~3 blank scaffolding rows it seeds (best-effort; `SCAFFOLDING_ROWS_KEPT` warning if the deletion attempt fails); `pruneRecords` under `mirror`+`confirmDeletions` additionally removes any leftover dest-only blank rows as part of the ordinary orphan prune. Schema/view/attachment deletion under mirror and natural-key record matching are deferred follow-ups. Custom field mappings: `fieldMappings` (`{ [tableName]: { sourceFieldName: destFieldName } }`) injects a source field's value into a different writable scalar dest field during record sync — useful when the source is computed (e.g., `autoNumber`/formula) and you want its value preserved in a writable dest field (classic use: inject source autoNumber `Code` into dest text field `InjectID` so a dest formula can reconstruct original identity). Fail-fast pre-flight validation in `apply` (and dry-run in `plan`/`diff` via `fieldMappingErrors` in the machine output) with codes: `FIELD_MAP_TABLE_MISSING`, `FIELD_MAP_SOURCE_MISSING`, `FIELD_MAP_TARGET_MISSING`, `FIELD_MAP_TARGET_COMPUTED` (dest must be writable scalar), `FIELD_MAP_TYPE_INCOMPATIBLE` (array-typed source/dest not supported), `FIELD_MAP_COLLISION` (two sources map to same dest); an invalid mapping aborts `apply` synchronously (`FIELD_MAP_INVALID` error code). Natural-key record matching: `naturalKeys` (`{ [tableName]: fieldName }`) matches dest↔source records by the key field's **value** (field ID resolved per-base by name), growing `idmap.records`. Add-only + ambiguity-safe — never removes/overwrites an existing mapping, never claims an already-mapped dest; ambiguous/duplicate key values are skipped with `NATURAL_KEY_AMBIGUOUS`; a missing key field emits `NATURAL_KEY_FIELD_MISSING`. Runs in `mode=reconcile` (after the existence-prune, snapshots source records + calls matcher, then saves idmap) AND as an automatic pre-pass in `mode=apply` **before Pass 1 and prune** — so records `mirror` no longer deletes real-but-unmapped dest records as false orphans, and re-syncs don't duplicate them. The key field must be **stable across bases** (autoNumber renumbers → bad key; name/email/injected `InjectID` → good key). Single-field keys only; composite keys and value normalization are deferred. Returns a `matched` count in the result. +- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: diff | plan | apply | status | reconcile`) in the `sync` category. Out of scope: retypes, deletions (M4). `mode=diff` (read-only): comprehensive schema comparison via pure `src/sync/compare.js` — every difference classified as **drift** (sync enforces: table/field/view existence, field type/options/choices, view filters/sorts/groups content, column visibility, colors/cover/frozen/rowHeight, primary, form), **best-effort** (sync applies but doesn't guarantee: field/view/column order, sort/group clause order), or **not-synced** (view sections — now captured by snapshot). Verdicts: `identical` (zero diffs) or `converged` (zero drift; best-effort/not-synced may remain). Persists full diff to `~/.airtable-user-mcp/sync/__/diff-.json`; returns a token-budgeted DIGEST (verdicts + class counts + per-table rollup + capped driftSample ≤25). Drill in with `detail="
"` (params: `detail`, `diffId`, `offset`, `limit`). `mode=plan` now emits a curatable **changeset**: each action carries a stable name-based `changeId` (`|
|`), a `class`, and an `apply:true` flag; digest includes a `sample` + `editHint`. New `direction` param (`to-dest` default | `to-source`) targets either base. View sync: `snapshot` also captures collaborative views + live config + **view sections** (not-synced class); `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported); accepts a `skip:[changeId]` list (and honors `apply:false` on changeset entries) — applies everything except removed changes (journal-safe; skipped dependencies degrade to UNRESOLVABLE_REF). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe by filename+size); then restores record-referencing view filters stripped during view sync. Throttle handling = per-request 429 backoff in the auth queue (the per-call limiter does NOT gate the internal per-row POSTs inside createRecords/updateRecords/addLinkItems); resumable (records journal, per-chunk persist of ~50 creates); continue-on-failure (per-record errors are warnings). `mode=status` polls a background job by planId — returns running/done/failed + live records-mapped count. `mode=reconcile` **prunes** dead record-map entries (existence-prune; per-table natural-key re-match now matches by key value — see natural-key matching below; reconcile does NOT de-duplicate records). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json, diff-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile), `compare.js` (pure classified compare, no I/O); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. Reconciliation policy (`src/sync/policy.js`): `mode=apply` accepts a `policy` param (`mirror` | `overlay` | `preserve`, default `overlay`) that controls two axes — **extras** (what to do with dest-only records) and **conflicts** (whose value wins when both bases have the record). Presets: `mirror` = {remove, source-wins} (make dest identical to source); `overlay` = {keep, source-wins} (today's default: keep dest-only rows, source updates win); `preserve` = {keep, dest-wins} (keep dest-only rows and never overwrite dest edits). `policyOverrides` (`{ [tableName]: preset }`) overrides the global preset per table. `confirmDeletions` (boolean, default false) is a safety gate for `mirror`: without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count but deletes nothing; set `confirmDeletions:true` to actually delete. Separately, `createTable` unconditionally clears the ~3 blank scaffolding rows it seeds (best-effort; `SCAFFOLDING_ROWS_KEPT` warning if the deletion attempt fails); `pruneRecords` under `mirror`+`confirmDeletions` additionally removes any leftover dest-only blank rows as part of the ordinary orphan prune. Schema/view/attachment deletion under mirror and natural-key record matching are deferred follow-ups. Custom field mappings: `fieldMappings` (`{ [tableName]: { sourceFieldName: destFieldName } }`) injects a source field's value into a different writable scalar dest field during record sync — useful when the source is computed (e.g., `autoNumber`/formula) and you want its value preserved in a writable dest field (classic use: inject source autoNumber `Code` into dest text field `InjectID` so a dest formula can reconstruct original identity). Fail-fast pre-flight validation in `apply` (and dry-run in `plan`/`diff` via `fieldMappingErrors` in the machine output) with codes: `FIELD_MAP_TABLE_MISSING`, `FIELD_MAP_SOURCE_MISSING`, `FIELD_MAP_TARGET_MISSING`, `FIELD_MAP_TARGET_COMPUTED` (dest must be writable scalar), `FIELD_MAP_TYPE_INCOMPATIBLE` (array-typed source/dest not supported), `FIELD_MAP_COLLISION` (two sources map to same dest); an invalid mapping aborts `apply` synchronously (`FIELD_MAP_INVALID` error code). Natural-key record matching: `naturalKeys` (`{ [tableName]: fieldName }`) matches dest↔source records by the key field's **value** (field ID resolved per-base by name), growing `idmap.records`. Add-only + ambiguity-safe — never removes/overwrites an existing mapping, never claims an already-mapped dest; ambiguous/duplicate key values are skipped with `NATURAL_KEY_AMBIGUOUS`; a missing key field emits `NATURAL_KEY_FIELD_MISSING`. Runs in `mode=reconcile` (after the existence-prune, snapshots source records + calls matcher, then saves idmap) AND as an automatic pre-pass in `mode=apply` **before Pass 1 and prune** — so records `mirror` no longer deletes real-but-unmapped dest records as false orphans, and re-syncs don't duplicate them. The key field must be **stable across bases** (autoNumber renumbers → bad key; name/email/injected `InjectID` → good key). Single-field keys only; composite keys and value normalization are deferred. Returns a `matched` count in the result. #### packages/mcp-server — Daemon subsystem diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index a35492f..bb1a293 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -941,6 +941,7 @@ export async function reapplyViewFilters({ client, srcSnapshot, destSnapshot, id * @param {object} [opts.policyOverrides]- per-table preset overrides * @param {boolean} [opts.confirmDeletions] - gate for pruneRecords deletions * @param {object} [opts.fieldMappings] - raw field mapping config { [table]: { srcName: destName } } + * @param {object} [opts.naturalKeys] - { [tableName]: fieldName } for natural-key matching (optional) * @param {object} opts.limiter - rate limiter { run: (fn) => fn() } * @param {object} opts.journal - records journal object * @param {Function} opts.persist - (idmap, journal) => void — called after each phase @@ -1028,6 +1029,11 @@ export async function runRecords({ client, srcSnapshot, destSnapshot, idmap, pol * @param {string} opts.destBaseId - dest base app ID * @param {string} opts.planId - plan ID (used for journal file name) * @param {string} opts.runStartedAt - ISO timestamp of this run start + * @param {string} [opts.policy] - global preset ('mirror'|'overlay'|'preserve') + * @param {object} [opts.policyOverrides] - per-table preset overrides + * @param {boolean} [opts.confirmDeletions] - gate for pruneRecords deletions + * @param {object} [opts.fieldMappings] - raw field mapping config { [table]: { srcName: destName } } + * @param {object} [opts.naturalKeys] - { [tableName]: fieldName } for natural-key matching (optional) * @returns {Promise} - result accumulator */ export async function applyRecords({ client, sourceBaseId, destBaseId, planId, runStartedAt, policy, policyOverrides, confirmDeletions, fieldMappings, naturalKeys }) { @@ -1252,10 +1258,10 @@ export function matchByNaturalKeys({ srcSnapshot, destSnapshot, naturalKeys, idm * 2. Snapshots the DEST base schema only, then pulls dest records for each matched table. * 3. Builds a Set of all live dest record IDs across all tables. * 4. Drops any idmap.records[src]=dest entry whose dest ID is not present in the snapshot. - * 5. naturalKeys re-match is stubbed (emits warning when provided — extend when needed). + * 5. Natural-key re-match: snapshots source records and calls matcher to add idmap.records entries by key value. * 6. Saves the pruned idmap. - * 7. Returns a result shaped like { created:0, updated:0, skipped, failed:0, warnings, idmap }. - * `skipped` = number of pruned entries. + * 7. Returns a result shaped like { created:0, updated:0, skipped, failed:0, matched, warnings, idmap }. + * `skipped` = number of pruned entries. `matched` = number of records matched by natural key. * * @param {object} opts * @param {object} opts.client - AirtableClient instance From 059e1cb98302c2df95f0590c5d15845b3214b2fa Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sun, 21 Jun 2026 12:13:32 +0300 Subject: [PATCH 127/246] docs: fix stale comment on reconcile natural-key matching reconcile now performs both existence-pruning and natural-key re-matching (via matchByNaturalKeys), so update the comment to reflect the real behavior instead of calling it a stub. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/records.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index bb1a293..c287eb7 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -1020,8 +1020,8 @@ export async function runRecords({ client, srcSnapshot, destSnapshot, idmap, pol * links against the live dest. The records journal is persisted but currently advisory (no * per-record done-gating); idmap + live re-snapshot are the source of truth. Known limitation: * a crash between a create's server-ack and the per-chunk idmap persist can re-create up to one - * chunk (~50) of rows as duplicates on resume (reconcile only existence-prunes; natural-key - * re-match is a stub). + * chunk (~50) of rows as duplicates on resume (reconcile existence-prunes + natural-key + * re-matches by value). * * @param {object} opts * @param {object} opts.client - AirtableClient instance From 6c433cfd45a6cebb254750bc43f8d15d23470e35 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sun, 21 Jun 2026 13:53:52 +0300 Subject: [PATCH 128/246] =?UTF-8?q?feat(sync):=20pruneSchema=20=E2=80=94?= =?UTF-8?q?=20delete=20dest-only=20fields/views/tables=20under=20mirror=20?= =?UTF-8?q?(tiered-gated)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/prune-schema.js | 52 ++++++++ .../test/sync/test-prune-schema.test.js | 121 ++++++++++++++++++ 2 files changed, 173 insertions(+) create mode 100644 packages/mcp-server/src/sync/prune-schema.js create mode 100644 packages/mcp-server/test/sync/test-prune-schema.test.js diff --git a/packages/mcp-server/src/sync/prune-schema.js b/packages/mcp-server/src/sync/prune-schema.js new file mode 100644 index 0000000..0d2825e --- /dev/null +++ b/packages/mcp-server/src/sync/prune-schema.js @@ -0,0 +1,52 @@ +// packages/mcp-server/src/sync/prune-schema.js +// Schema extras axis: delete dest-only fields/views/tables under mirror, tiered-gated, dependency-safe. +// Fields use deleteFields WITHOUT force (so a matched field is never cascade-deleted); orphan->orphan +// deps resolve by retry-until-stable; a field still blocked after no progress is kept + warned. +import { resolvePolicy } from './policy.js'; + +export async function pruneSchema({ client, destAppId, plan, policy, policyOverrides, confirmDeletions, confirmTableDeletions, result }) { + if (result.schemaDeleted == null) result.schemaDeleted = 0; + if (result.tablesDeleted == null) result.tablesDeleted = 0; + if (!result.warnings) result.warnings = []; + const orphans = (plan && plan.orphans) || []; + const removes = (tableName) => resolvePolicy(policy, policyOverrides, tableName).extras === 'remove'; + + let gatedFieldsViews = 0; + let gatedTables = 0; + + // 1. Views (no dependents) — matched-table orphans + for (const v of orphans.filter((o) => o.kind === 'view' && removes(o.tableName))) { + if (!confirmDeletions) { gatedFieldsViews++; continue; } + try { await client.deleteView(destAppId, v.destId); result.schemaDeleted++; } + catch (e) { result.warnings.push({ code: 'SCHEMA_DELETE_FAILED', message: `view "${v.name}" (${v.tableName}): ${e.message ?? e}` }); } + } + + // 2. Fields — delete-retry-until-stable, never force + let fieldOrphans = orphans.filter((o) => o.kind === 'field' && removes(o.tableName)); + if (!confirmDeletions) { gatedFieldsViews += fieldOrphans.length; fieldOrphans = []; } + let remaining = fieldOrphans; + while (remaining.length) { + let res; + try { res = await client.deleteFields(destAppId, remaining.map((o) => ({ fieldId: o.destId, expectedName: o.name })), { force: false }); } + catch (e) { for (const o of remaining) result.warnings.push({ code: 'SCHEMA_DELETE_FAILED', message: `field "${o.name}" (${o.tableName}): ${e.message ?? e}` }); break; } + result.schemaDeleted += (res.succeeded || []).length; + const failedIds = new Set((res.failed || []).map((f) => f.fieldId)); + const stillBlocked = remaining.filter((o) => failedIds.has(o.destId)); + if (stillBlocked.length === remaining.length) { + // no progress → blocked by a matched (non-orphan) dependency, or unresolvable. Keep + warn. + for (const o of stillBlocked) result.warnings.push({ code: 'SCHEMA_DELETE_BLOCKED', message: `field "${o.name}" in "${o.tableName}" blocked by a dependency (likely a matched field) — kept` }); + break; + } + remaining = stillBlocked; + } + + // 3. Tables (after their fields) — gated by confirmTableDeletions + for (const t of orphans.filter((o) => o.kind === 'table' && removes(o.name))) { + if (!confirmTableDeletions) { gatedTables++; continue; } + try { await client.deleteTable(destAppId, t.destId, t.name); result.tablesDeleted++; } + catch (e) { result.warnings.push({ code: 'SCHEMA_DELETE_FAILED', message: `table "${t.name}": ${e.message ?? e}` }); } + } + + if (gatedFieldsViews > 0) result.warnings.push({ code: 'DELETION_GATED', message: `${gatedFieldsViews} dest-only field(s)/view(s) would be deleted under mirror — set confirmDeletions:true` }); + if (gatedTables > 0) result.warnings.push({ code: 'TABLE_DELETION_GATED', message: `${gatedTables} dest-only table(s) would be deleted under mirror — set confirmTableDeletions:true` }); +} diff --git a/packages/mcp-server/test/sync/test-prune-schema.test.js b/packages/mcp-server/test/sync/test-prune-schema.test.js new file mode 100644 index 0000000..d3d50b9 --- /dev/null +++ b/packages/mcp-server/test/sync/test-prune-schema.test.js @@ -0,0 +1,121 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { pruneSchema } from '../../src/sync/prune-schema.js'; + +// Mock client. deleteFields succeeds for a field UNLESS some still-present field depends on it +// (modelled by `deps`: dependentFieldId -> [fieldIds it depends on]). force is asserted false. +function mockClient({ deps = {} } = {}) { + const calls = { views: [], tables: [], fieldBatches: [] }; + const present = new Set(); // fields still alive (seeded per test via opts) + const api = { + calls, + _seed(ids) { ids.forEach((i) => present.add(i)); }, + async deleteView(appId, viewId) { calls.views.push(viewId); }, + async deleteTable(appId, tableId, expectedName) { calls.tables.push({ tableId, expectedName }); }, + async deleteFields(appId, fields, opts) { + assert.equal(opts.force, false, 'must never force-delete fields'); + calls.fieldBatches.push(fields.map((f) => f.fieldId)); + const succeeded = [], failed = []; + for (const { fieldId, expectedName } of fields) { + // blocked if any CURRENTLY-PRESENT field depends on fieldId + const blockedBy = Object.entries(deps).filter(([dep, on]) => present.has(dep) && on.includes(fieldId)).map(([dep]) => dep); + if (blockedBy.length) { failed.push({ fieldId, name: expectedName, error: 'has dependents: ' + blockedBy.join(',') }); } + else { present.delete(fieldId); succeeded.push(fieldId); } + } + return { succeeded, failed }; + }, + }; + return api; +} +const baseResult = () => ({ schemaDeleted: 0, tablesDeleted: 0, warnings: [] }); +const plan = (orphans) => ({ orphans }); + +describe('pruneSchema', () => { + it('mirror + confirmDeletions deletes orphan views and fields (matched tables)', async () => { + const client = mockClient(); client._seed(['fldA', 'fldB']); + const result = baseResult(); + await pruneSchema({ client, destAppId: 'appD', plan: plan([ + { kind: 'view', destId: 'viw1', name: 'Extra', tableName: 'Offers' }, + { kind: 'field', destId: 'fldA', name: 'X', tableName: 'Offers' }, + { kind: 'field', destId: 'fldB', name: 'Y', tableName: 'Offers' }, + ]), policy: 'mirror', confirmDeletions: true, confirmTableDeletions: false, result }); + assert.deepEqual(client.calls.views, ['viw1']); + assert.equal(result.schemaDeleted, 3, '1 view + 2 fields'); + }); + + it('mirror WITHOUT confirmDeletions deletes nothing + DELETION_GATED with count', async () => { + const client = mockClient(); client._seed(['fldA']); + const result = baseResult(); + await pruneSchema({ client, destAppId: 'appD', plan: plan([ + { kind: 'view', destId: 'viw1', name: 'Extra', tableName: 'Offers' }, + { kind: 'field', destId: 'fldA', name: 'X', tableName: 'Offers' }, + ]), policy: 'mirror', confirmDeletions: false, confirmTableDeletions: false, result }); + assert.equal(result.schemaDeleted, 0); + const w = result.warnings.find((x) => x.code === 'DELETION_GATED'); + assert.ok(w && /2/.test(w.message), 'gated count = 2 (1 view + 1 field)'); + assert.equal(client.calls.views.length, 0); + }); + + it('orphan TABLE: confirmDeletions but NOT confirmTableDeletions → TABLE_DELETION_GATED, table kept', async () => { + const client = mockClient(); + const result = baseResult(); + await pruneSchema({ client, destAppId: 'appD', plan: plan([ + { kind: 'table', destId: 'tbl9', name: 'DestOnly' }, + ]), policy: 'mirror', confirmDeletions: true, confirmTableDeletions: false, result }); + assert.equal(client.calls.tables.length, 0); + assert.ok(result.warnings.some((x) => x.code === 'TABLE_DELETION_GATED')); + }); + + it('orphan TABLE: confirmTableDeletions → table deleted', async () => { + const client = mockClient(); + const result = baseResult(); + await pruneSchema({ client, destAppId: 'appD', plan: plan([ + { kind: 'table', destId: 'tbl9', name: 'DestOnly' }, + ]), policy: 'mirror', confirmDeletions: true, confirmTableDeletions: true, result }); + assert.deepEqual(client.calls.tables, [{ tableId: 'tbl9', expectedName: 'DestOnly' }]); + assert.equal(result.tablesDeleted, 1); + }); + + it('overlay table: orphans never deleted', async () => { + const client = mockClient(); client._seed(['fldA']); + const result = baseResult(); + await pruneSchema({ client, destAppId: 'appD', plan: plan([ + { kind: 'field', destId: 'fldA', name: 'X', tableName: 'Offers' }, + { kind: 'table', destId: 'tbl9', name: 'DestOnly' }, + ]), policy: 'overlay', confirmDeletions: true, confirmTableDeletions: true, result }); + assert.equal(result.schemaDeleted, 0); + assert.equal(result.tablesDeleted, 0); + }); + + it('policyOverrides preserve protects a dest-only table under global mirror', async () => { + const client = mockClient(); + const result = baseResult(); + await pruneSchema({ client, destAppId: 'appD', plan: plan([ + { kind: 'table', destId: 'tbl9', name: 'Keep' }, + ]), policy: 'mirror', policyOverrides: { Keep: 'preserve' }, confirmDeletions: true, confirmTableDeletions: true, result }); + assert.equal(client.calls.tables.length, 0); + }); + + it('orphan->orphan dependency resolves by retry (delete dependent first)', async () => { + // fldB depends on fldA (both orphans). Pass 1: fldA blocked (fldB present), fldB deletes. + // Pass 2: fldA deletes (fldB gone). Both eventually deleted, no force. + const client = mockClient({ deps: { fldB: ['fldA'] } }); client._seed(['fldA', 'fldB']); + const result = baseResult(); + await pruneSchema({ client, destAppId: 'appD', plan: plan([ + { kind: 'field', destId: 'fldA', name: 'A', tableName: 'Offers' }, + { kind: 'field', destId: 'fldB', name: 'B', tableName: 'Offers' }, + ]), policy: 'mirror', confirmDeletions: true, confirmTableDeletions: false, result }); + assert.equal(result.schemaDeleted, 2, 'both deleted across passes'); + }); + + it('orphan blocked by a MATCHED field is skipped + SCHEMA_DELETE_BLOCKED (never forced)', async () => { + // fldM (matched, NOT in orphans, always present) depends on fldA (orphan). fldA can never delete. + const client = mockClient({ deps: { fldM: ['fldA'] } }); client._seed(['fldA', 'fldM']); + const result = baseResult(); + await pruneSchema({ client, destAppId: 'appD', plan: plan([ + { kind: 'field', destId: 'fldA', name: 'A', tableName: 'Offers' }, + ]), policy: 'mirror', confirmDeletions: true, confirmTableDeletions: false, result }); + assert.equal(result.schemaDeleted, 0); + assert.ok(result.warnings.some((x) => x.code === 'SCHEMA_DELETE_BLOCKED')); + }); +}); From e4e3fcc7acd789c3963b66cd59a8cff5beb008cf Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sun, 21 Jun 2026 13:58:47 +0300 Subject: [PATCH 129/246] feat(sync): apply() runs pruneSchema after applyPlan (threads confirmTableDeletions) Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/index.js | 8 +- .../mcp-server/test/sync/test-index.test.js | 94 +++++++++++++++++++ 2 files changed, 101 insertions(+), 1 deletion(-) diff --git a/packages/mcp-server/src/sync/index.js b/packages/mcp-server/src/sync/index.js index edd8aec..430efdc 100644 --- a/packages/mcp-server/src/sync/index.js +++ b/packages/mcp-server/src/sync/index.js @@ -8,6 +8,7 @@ import { applyPlan } from './apply.js'; import { newJournal, loadJournal, saveJournal } from './journal.js'; import { applyRecords as applyRecordsImpl, reconcile as reconcileImpl, writeRecordsJobStatus, readRecordsJobStatus } from './records.js'; import { validateFieldMappings, isDeleting } from './policy.js'; +import { pruneSchema } from './prune-schema.js'; const ENGINE_VERSION = '2b'; @@ -114,7 +115,7 @@ export function diffDetail({ sourceBaseId, destBaseId, diffId, detail, offset, l return renderDiff(savedDiff, { detail, offset, limit }); } -export async function apply({ client, sourceBaseId, destBaseId, planId, runStartedAt, skip = [], policy, policyOverrides, confirmDeletions, fieldMappings, naturalKeys }) { +export async function apply({ client, sourceBaseId, destBaseId, planId, runStartedAt, skip = [], policy, policyOverrides, confirmDeletions, confirmTableDeletions, fieldMappings, naturalKeys }) { const fullPlan = loadPlan(sourceBaseId, destBaseId, planId); if (!fullPlan) throw new Error(`No saved plan "${planId}" for ${sourceBaseId} -> ${destBaseId}. Run mode=plan first.`); @@ -149,6 +150,11 @@ export async function apply({ client, sourceBaseId, destBaseId, planId, runStart skip, }); + // Schema extras phase: delete dest-only fields/views/tables under mirror, gated. Runs after + // applyPlan so matched fields/views exist (orphan deps safe) and BEFORE the records job so + // the schema is clean before record sync begins. Mutates `result` in-place. + await pruneSchema({ client, destAppId: destBaseId, plan: fullPlan, policy, policyOverrides, confirmDeletions, confirmTableDeletions, result }); + // Records phase: runs after schema apply, only if not aborted. It is minutes-long for large // bases, so we launch it in the BACKGROUND (fire-and-forget) and return immediately — a single // blocking apply call would look hung / time out. The phase persists progress (idmap + records diff --git a/packages/mcp-server/test/sync/test-index.test.js b/packages/mcp-server/test/sync/test-index.test.js index 9fd6196..068f4d7 100644 --- a/packages/mcp-server/test/sync/test-index.test.js +++ b/packages/mcp-server/test/sync/test-index.test.js @@ -201,6 +201,100 @@ describe('sync index.plan — fieldMappings validation (Task 8)', () => { }); }); +describe('sync index.apply — pruneSchema integration (Task 2)', () => { + // apply() with policy=mirror + plan carrying orphans (one field orphan + one table orphan), + // confirmDeletions=true, confirmTableDeletions=false: + // - deleteFields should be called for the orphan field + // - deleteTable should NOT be called (table gated by confirmTableDeletions=false) + // - result.machine.schemaDeleted > 0 + // - result.machine.warnings includes a TABLE_DELETION_GATED entry + // + // Because apply() does substantial I/O (plan/idmap/journal), we drive the full flow: + // save a minimal plan whose orphans include the target field + table, then call syncApply(). + // The applyPlan itself is a no-op (no actions) so only pruneSchema has observable side-effects. + it('apply() with mirror+confirmDeletions calls deleteFields for orphan field, gates orphan table, warns TABLE_DELETION_GATED', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-prune-')); + + const srcAppId = 'appSRCPRUNE00000'; + const destAppId = 'appDESTPRUNE0000'; + + // Both bases share an identical table so fingerprints match and plan has zero actions. + // The orphans (field + table) are injected manually into the plan file after saving. + const schema = { data: { tableSchemas: [{ + id: 'tblA', name: 'Shared', primaryColumnId: 'fldA1', + columns: [{ id: 'fldA1', name: 'Name', type: 'text' }], + }] } }; + + const deleteFieldsCalls = []; + const deleteTableCalls = []; + + const client = { + getApplicationData: async () => schema, + // applyPlan may call createTable/createField for new items, but our plan is empty. + createTable: async () => ({ data: { id: 'tblNEW' } }), + createField: async () => ({ data: { id: 'fldNEW' } }), + deleteFields: async (appId, fields, opts) => { + deleteFieldsCalls.push({ appId, fields, opts }); + // Return as if all succeeded + return { succeeded: fields.map((f) => ({ fieldId: f.fieldId })), failed: [] }; + }, + deleteTable: async (appId, tableId, tableName) => { + deleteTableCalls.push({ appId, tableId, tableName }); + }, + }; + + // Step 1: Run plan() to persist a valid plan. + await computeSyncPlan({ + client, + sourceBaseId: srcAppId, + destBaseId: destAppId, + planId: 'plnPruneTest', + }); + + // Step 2: Patch the saved plan to inject orphans. + // We load the plan file, add orphans, and re-save it. + const { syncDir: getSyncDir } = await import('../../src/sync/idmap.js'); + const { readFileSync: rfSync, writeFileSync } = await import('node:fs'); + const planPath = join(getSyncDir(srcAppId, destAppId), 'plan-plnPruneTest.json'); + const savedPlan = JSON.parse(rfSync(planPath, 'utf8')); + // Inject: one field orphan (dest table "Shared" has an extra field "OldField"), + // one table orphan (dest has extra table "Orphaned") + savedPlan.orphans = [ + { kind: 'field', tableName: 'Shared', destId: 'fldORPHAN1', name: 'OldField' }, + { kind: 'table', name: 'Orphaned', destId: 'tblORPHAN1' }, + ]; + writeFileSync(planPath, JSON.stringify(savedPlan)); + + // Step 3: Call apply() with policy=mirror, confirmDeletions=true, confirmTableDeletions=false. + const out = await syncApply({ + client, + sourceBaseId: srcAppId, + destBaseId: destAppId, + planId: 'plnPruneTest', + runStartedAt: new Date().toISOString(), + policy: 'mirror', + confirmDeletions: true, + confirmTableDeletions: false, + }); + + // Assert: deleteFields was called once for the orphan field + assert.equal(deleteFieldsCalls.length, 1, 'deleteFields must be called exactly once'); + assert.equal(deleteFieldsCalls[0].fields[0].fieldId, 'fldORPHAN1', 'deleteFields called with correct orphan fieldId'); + + // Assert: deleteTable was NOT called (gated by confirmTableDeletions=false) + assert.equal(deleteTableCalls.length, 0, 'deleteTable must NOT be called (confirmTableDeletions=false)'); + + // Assert: result.machine.schemaDeleted reflects the deleted field + assert.ok(out.machine.schemaDeleted > 0, 'result.machine.schemaDeleted must be > 0'); + + // Assert: TABLE_DELETION_GATED warning in result + const warnings = out.machine.warnings || []; + const gatedWarn = warnings.find((w) => w.code === 'TABLE_DELETION_GATED'); + assert.ok(gatedWarn, 'result must carry a TABLE_DELETION_GATED warning'); + assert.match(gatedWarn.message, /confirmTableDeletions/, 'TABLE_DELETION_GATED message must mention confirmTableDeletions'); + }); +}); + describe('sync index.apply — synchronous fieldMappings pre-flight (Task 8 Fix)', () => { // apply() must throw FIELD_MAP_INVALID synchronously (before launching the background records // job) when fieldMappings references a computed (formula) dest field. From 060606909791864e84e9195df31d35beab8b9d75 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sun, 21 Jun 2026 14:03:57 +0300 Subject: [PATCH 130/246] feat(sync): sync_base confirmTableDeletions param + surface schema-deleted counts Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/index.js | 7 ++++--- packages/mcp-server/src/sync/report.js | 6 ++++++ .../mcp-server/test/sync/test-apply-index.test.js | 11 +++++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/packages/mcp-server/src/index.js b/packages/mcp-server/src/index.js index 02340d5..6739f3d 100644 --- a/packages/mcp-server/src/index.js +++ b/packages/mcp-server/src/index.js @@ -1596,7 +1596,8 @@ Note: "form title" is the view name itself — use rename_view to change it. "Fi skip: { type: 'array', items: { type: 'string' }, description: 'Used only by mode="apply": list of changeIds to skip (actions with a matching changeId are counted as skipped but not applied). Use changeIds from the plan output.' }, policy: { type: 'string', enum: ['mirror', 'overlay', 'preserve'], description: 'Used by mode="apply" for the record reconciliation preset. "mirror" = make dest identical to source (delete dest-only records, overwrite dest edits); "overlay" = keep dest-only records, source updates win on conflicts (default); "preserve" = keep dest-only records and never overwrite dest edits. Requires confirmDeletions=true to actually delete in mirror mode.' }, policyOverrides: { type: 'object', description: 'Used by mode="apply": per-table reconciliation preset overrides. Maps table name → preset (e.g. { "Games": "preserve" }). Overrides the global policy for the named tables.', additionalProperties: { type: 'string', enum: ['mirror', 'overlay', 'preserve'] } }, - confirmDeletions: { type: 'boolean', description: 'Used by mode="apply" with policy="mirror": must be set to true to actually delete dest-only records. Without it, mirror mode reports a DELETION_GATED count and deletes nothing.' }, + confirmDeletions: { type: 'boolean', description: 'Used by mode="apply" with policy="mirror": must be set to true to actually delete dest-only RECORDS and dest-only FIELDS/VIEWS. Without it, mirror mode reports a DELETION_GATED count and deletes nothing for records; orphan fields/views are reported and kept.' }, + confirmTableDeletions: { type: 'boolean', description: 'Used by mode=apply with policy=mirror: required to delete whole dest-only TABLES. confirmDeletions alone never drops a table; without confirmTableDeletions, orphan tables report TABLE_DELETION_GATED and are kept.' }, fieldMappings: { type: 'object', description: 'Field mapping overrides: maps table name → { sourceField: destField } to inject a source field\'s value into a different (writable scalar) dest field during record sync. Example: { "Games": { "Title": "Name" } }. In mode="plan" and mode="diff", fieldMappings are validated against the two schemas (dry-run, no mutation) and errors are returned in fieldMappingErrors.', additionalProperties: { type: 'object', additionalProperties: { type: 'string' } } }, debug: debugProp, }, @@ -2399,7 +2400,7 @@ const handlers = { // ── Sync ── - async sync_base({ mode, sourceAppId, destAppId, planId, naturalKeys, detail, diffId, offset, limit, direction, skip, policy, policyOverrides, confirmDeletions, fieldMappings, debug }) { + async sync_base({ mode, sourceAppId, destAppId, planId, naturalKeys, detail, diffId, offset, limit, direction, skip, policy, policyOverrides, confirmDeletions, confirmTableDeletions, fieldMappings, debug }) { const sync = await import('./sync/index.js'); if (mode === 'plan') { const id = 'pln' + client._genRandomId(); @@ -2411,7 +2412,7 @@ const handlers = { const runStartedAt = new Date().toISOString(); let out; try { - out = await sync.apply({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, planId, runStartedAt, skip: skip || [], policy, policyOverrides, confirmDeletions, fieldMappings, naturalKeys }); + out = await sync.apply({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, planId, runStartedAt, skip: skip || [], policy, policyOverrides, confirmDeletions, confirmTableDeletions, fieldMappings, naturalKeys }); } catch (e) { if (e && e.code === 'FIELD_MAP_INVALID') { return err(`Field mapping validation failed:\n${(e.mappingErrors || []).map((me) => ` [${me.code}] table=${me.table || '?'}${me.source ? ' source=' + me.source : ''}${me.target ? ' target=' + me.target : ''}`).join('\n')}`); diff --git a/packages/mcp-server/src/sync/report.js b/packages/mcp-server/src/sync/report.js index 50fbdb6..3fadf44 100644 --- a/packages/mcp-server/src/sync/report.js +++ b/packages/mcp-server/src/sync/report.js @@ -71,6 +71,12 @@ export function renderApplyResult(result) { if (result.matched != null && result.matched > 0) { lines.push(` matched: ${result.matched} (natural-key pre-pass)`); } + if (result.schemaDeleted != null && result.schemaDeleted > 0) { + lines.push(` schemaDeleted: ${result.schemaDeleted} (dest-only fields/views removed)`); + } + if (result.tablesDeleted != null && result.tablesDeleted > 0) { + lines.push(` tablesDeleted: ${result.tablesDeleted} (dest-only tables removed)`); + } if (result.records) { const r = result.records; if (r.status === 'running') { diff --git a/packages/mcp-server/test/sync/test-apply-index.test.js b/packages/mcp-server/test/sync/test-apply-index.test.js index 70b0f3a..fda00b9 100644 --- a/packages/mcp-server/test/sync/test-apply-index.test.js +++ b/packages/mcp-server/test/sync/test-apply-index.test.js @@ -14,6 +14,17 @@ describe('report.renderApplyResult', () => { assert.match(renderApplyResult({ planId: 'p', aborted: false, created: 2, updated: 1, skipped: 3, failed: 0, warnings: [] }).human, /created: 2/); assert.match(renderApplyResult({ planId: 'p', aborted: true, reason: 'DRIFT', warnings: [{ code: 'DRIFT', message: 'm' }] }).human, /DRIFT/); }); + it('surfaces schemaDeleted and tablesDeleted when > 0', () => { + const base = { planId: 'p', aborted: false, created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + // zero → not rendered + const zeroOut = renderApplyResult({ ...base, schemaDeleted: 0, tablesDeleted: 0 }).human; + assert.ok(!zeroOut.includes('schemaDeleted'), 'schemaDeleted:0 should not appear'); + assert.ok(!zeroOut.includes('tablesDeleted'), 'tablesDeleted:0 should not appear'); + // > 0 → rendered + const out = renderApplyResult({ ...base, schemaDeleted: 3, tablesDeleted: 1 }).human; + assert.match(out, /schemaDeleted: 3/); + assert.match(out, /tablesDeleted: 1/); + }); }); describe('index.apply', () => { From bc29bd8152ac30147c7a74713ef70d1acd822638 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sun, 21 Jun 2026 14:07:57 +0300 Subject: [PATCH 131/246] docs(sync): document schema-orphan deletion under mirror Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 5 +++++ CLAUDE.md | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26891c6..013811f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how ## [Unreleased] +### MCP server — sync_base schema-orphan deletion under mirror (2026-06-21) + +#### Added +- `sync_base mode=apply` — **schema-orphan deletion** (`pruneSchema`): under `policy=mirror`, dest-only fields/views/tables that have no counterpart in the source are deleted from the destination after `applyPlan` completes. Two independent gates: `confirmDeletions:true` is required to delete dest-only **fields and views** (same flag as `pruneRecords`); the new `confirmTableDeletions:true` is required to drop dest-only **tables** — `confirmDeletions` alone never deletes an entire table. Without each gate the operation is a dry-count only (`DELETION_GATED` for fields/views, `TABLE_DELETION_GATED` for tables). Deletion is dependency-safe: fields are deleted without `force` (so no matched field is ever cascade-deleted), orphan→orphan formula dependencies are resolved by retry-until-stable, and a field blocked by a matched-field dependency is kept and reported as `SCHEMA_DELETE_BLOCKED`. Deletion order: views first (no dependents), then fields, then tables (after their content). Applied counts are surfaced in the result as `schemaDeleted` (fields + views) and `tablesDeleted`; failures are continue-on-failure warnings (`SCHEMA_DELETE_FAILED`). Per-table `policyOverrides` are respected — `overlay`/`preserve` tables are never pruned. Module: `src/sync/prune-schema.js`. Retypes and view-section orphan deletion remain deferred (M4). + ### MCP server — sync_base natural-key record matching (2026-06-21) #### Added diff --git a/CLAUDE.md b/CLAUDE.md index c75d0ca..2d2891b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,7 +65,7 @@ Key files: - `dev-tools/` — reverse-engineering helpers (capture.js, analyze.js, debug-*.js), **gitignored** — kept locally for future reference when Airtable changes their internal API - `src/daemon/` — daemon subsystem (see daemon subsection below) - `src/safe-write.js` — atomic JSON write helper (used by lockfile, token, and settings) -- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: diff | plan | apply | status | reconcile`) in the `sync` category. Out of scope: retypes, deletions (M4). `mode=diff` (read-only): comprehensive schema comparison via pure `src/sync/compare.js` — every difference classified as **drift** (sync enforces: table/field/view existence, field type/options/choices, view filters/sorts/groups content, column visibility, colors/cover/frozen/rowHeight, primary, form), **best-effort** (sync applies but doesn't guarantee: field/view/column order, sort/group clause order), or **not-synced** (view sections — now captured by snapshot). Verdicts: `identical` (zero diffs) or `converged` (zero drift; best-effort/not-synced may remain). Persists full diff to `~/.airtable-user-mcp/sync/__/diff-.json`; returns a token-budgeted DIGEST (verdicts + class counts + per-table rollup + capped driftSample ≤25). Drill in with `detail="
"` (params: `detail`, `diffId`, `offset`, `limit`). `mode=plan` now emits a curatable **changeset**: each action carries a stable name-based `changeId` (`|
|`), a `class`, and an `apply:true` flag; digest includes a `sample` + `editHint`. New `direction` param (`to-dest` default | `to-source`) targets either base. View sync: `snapshot` also captures collaborative views + live config + **view sections** (not-synced class); `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported); accepts a `skip:[changeId]` list (and honors `apply:false` on changeset entries) — applies everything except removed changes (journal-safe; skipped dependencies degrade to UNRESOLVABLE_REF). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe by filename+size); then restores record-referencing view filters stripped during view sync. Throttle handling = per-request 429 backoff in the auth queue (the per-call limiter does NOT gate the internal per-row POSTs inside createRecords/updateRecords/addLinkItems); resumable (records journal, per-chunk persist of ~50 creates); continue-on-failure (per-record errors are warnings). `mode=status` polls a background job by planId — returns running/done/failed + live records-mapped count. `mode=reconcile` **prunes** dead record-map entries (existence-prune; per-table natural-key re-match now matches by key value — see natural-key matching below; reconcile does NOT de-duplicate records). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json, diff-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile), `compare.js` (pure classified compare, no I/O); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. Reconciliation policy (`src/sync/policy.js`): `mode=apply` accepts a `policy` param (`mirror` | `overlay` | `preserve`, default `overlay`) that controls two axes — **extras** (what to do with dest-only records) and **conflicts** (whose value wins when both bases have the record). Presets: `mirror` = {remove, source-wins} (make dest identical to source); `overlay` = {keep, source-wins} (today's default: keep dest-only rows, source updates win); `preserve` = {keep, dest-wins} (keep dest-only rows and never overwrite dest edits). `policyOverrides` (`{ [tableName]: preset }`) overrides the global preset per table. `confirmDeletions` (boolean, default false) is a safety gate for `mirror`: without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count but deletes nothing; set `confirmDeletions:true` to actually delete. Separately, `createTable` unconditionally clears the ~3 blank scaffolding rows it seeds (best-effort; `SCAFFOLDING_ROWS_KEPT` warning if the deletion attempt fails); `pruneRecords` under `mirror`+`confirmDeletions` additionally removes any leftover dest-only blank rows as part of the ordinary orphan prune. Schema/view/attachment deletion under mirror and natural-key record matching are deferred follow-ups. Custom field mappings: `fieldMappings` (`{ [tableName]: { sourceFieldName: destFieldName } }`) injects a source field's value into a different writable scalar dest field during record sync — useful when the source is computed (e.g., `autoNumber`/formula) and you want its value preserved in a writable dest field (classic use: inject source autoNumber `Code` into dest text field `InjectID` so a dest formula can reconstruct original identity). Fail-fast pre-flight validation in `apply` (and dry-run in `plan`/`diff` via `fieldMappingErrors` in the machine output) with codes: `FIELD_MAP_TABLE_MISSING`, `FIELD_MAP_SOURCE_MISSING`, `FIELD_MAP_TARGET_MISSING`, `FIELD_MAP_TARGET_COMPUTED` (dest must be writable scalar), `FIELD_MAP_TYPE_INCOMPATIBLE` (array-typed source/dest not supported), `FIELD_MAP_COLLISION` (two sources map to same dest); an invalid mapping aborts `apply` synchronously (`FIELD_MAP_INVALID` error code). Natural-key record matching: `naturalKeys` (`{ [tableName]: fieldName }`) matches dest↔source records by the key field's **value** (field ID resolved per-base by name), growing `idmap.records`. Add-only + ambiguity-safe — never removes/overwrites an existing mapping, never claims an already-mapped dest; ambiguous/duplicate key values are skipped with `NATURAL_KEY_AMBIGUOUS`; a missing key field emits `NATURAL_KEY_FIELD_MISSING`. Runs in `mode=reconcile` (after the existence-prune, snapshots source records + calls matcher, then saves idmap) AND as an automatic pre-pass in `mode=apply` **before Pass 1 and prune** — so records `mirror` no longer deletes real-but-unmapped dest records as false orphans, and re-syncs don't duplicate them. The key field must be **stable across bases** (autoNumber renumbers → bad key; name/email/injected `InjectID` → good key). Single-field keys only; composite keys and value normalization are deferred. Returns a `matched` count in the result. +- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: diff | plan | apply | status | reconcile`) in the `sync` category. Out of scope: retypes, deletions (M4). `mode=diff` (read-only): comprehensive schema comparison via pure `src/sync/compare.js` — every difference classified as **drift** (sync enforces: table/field/view existence, field type/options/choices, view filters/sorts/groups content, column visibility, colors/cover/frozen/rowHeight, primary, form), **best-effort** (sync applies but doesn't guarantee: field/view/column order, sort/group clause order), or **not-synced** (view sections — now captured by snapshot). Verdicts: `identical` (zero diffs) or `converged` (zero drift; best-effort/not-synced may remain). Persists full diff to `~/.airtable-user-mcp/sync/__/diff-.json`; returns a token-budgeted DIGEST (verdicts + class counts + per-table rollup + capped driftSample ≤25). Drill in with `detail="
"` (params: `detail`, `diffId`, `offset`, `limit`). `mode=plan` now emits a curatable **changeset**: each action carries a stable name-based `changeId` (`|
|`), a `class`, and an `apply:true` flag; digest includes a `sample` + `editHint`. New `direction` param (`to-dest` default | `to-source`) targets either base. View sync: `snapshot` also captures collaborative views + live config + **view sections** (not-synced class); `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported); accepts a `skip:[changeId]` list (and honors `apply:false` on changeset entries) — applies everything except removed changes (journal-safe; skipped dependencies degrade to UNRESOLVABLE_REF). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe by filename+size); then restores record-referencing view filters stripped during view sync. Throttle handling = per-request 429 backoff in the auth queue (the per-call limiter does NOT gate the internal per-row POSTs inside createRecords/updateRecords/addLinkItems); resumable (records journal, per-chunk persist of ~50 creates); continue-on-failure (per-record errors are warnings). `mode=status` polls a background job by planId — returns running/done/failed + live records-mapped count. `mode=reconcile` **prunes** dead record-map entries (existence-prune; per-table natural-key re-match now matches by key value — see natural-key matching below; reconcile does NOT de-duplicate records). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json, diff-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile), `compare.js` (pure classified compare, no I/O); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. Reconciliation policy (`src/sync/policy.js`): `mode=apply` accepts a `policy` param (`mirror` | `overlay` | `preserve`, default `overlay`) that controls two axes — **extras** (what to do with dest-only records) and **conflicts** (whose value wins when both bases have the record). Presets: `mirror` = {remove, source-wins} (make dest identical to source); `overlay` = {keep, source-wins} (today's default: keep dest-only rows, source updates win); `preserve` = {keep, dest-wins} (keep dest-only rows and never overwrite dest edits). `policyOverrides` (`{ [tableName]: preset }`) overrides the global preset per table. `confirmDeletions` (boolean, default false) is a safety gate for `mirror`: without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count but deletes nothing; set `confirmDeletions:true` to actually delete. Separately, `createTable` unconditionally clears the ~3 blank scaffolding rows it seeds (best-effort; `SCAFFOLDING_ROWS_KEPT` warning if the deletion attempt fails); `pruneRecords` under `mirror`+`confirmDeletions` additionally removes any leftover dest-only blank rows as part of the ordinary orphan prune. Attachment deletion under mirror is a deferred follow-up. Schema-orphan deletion (`pruneSchema`) runs synchronously in `mode=apply` AFTER `applyPlan`, consuming `plan.orphans` (`{kind:'table'|'field'|'view', destId, name, tableName}`). Order: (1) views — deleted first (no dependents); (2) fields — batch `deleteFields` WITHOUT force (so a matched field is never cascade-deleted), retry-until-stable loop resolves orphan→orphan deps; a field still blocked after no progress is kept + `SCHEMA_DELETE_BLOCKED`; (3) tables — deleted last (after their content). Gating: dest-only fields + views require `confirmDeletions:true` (same flag as `pruneRecords`); dest-only tables require the new `confirmTableDeletions:true` — `confirmDeletions` alone never drops a whole table. Without each gate the operation is a dry-count: `DELETION_GATED` for fields/views, `TABLE_DELETION_GATED` for tables. Applied counts: `schemaDeleted` (fields+views) + `tablesDeleted`. Per-table `policyOverrides` apply; `overlay`/`preserve` → no schema deletion. Continue-on-failure (`SCHEMA_DELETE_FAILED`). Module: `src/sync/prune-schema.js`. Retypes and view-section orphans are deferred (M4). Custom field mappings: `fieldMappings` (`{ [tableName]: { sourceFieldName: destFieldName } }`) injects a source field's value into a different writable scalar dest field during record sync — useful when the source is computed (e.g., `autoNumber`/formula) and you want its value preserved in a writable dest field (classic use: inject source autoNumber `Code` into dest text field `InjectID` so a dest formula can reconstruct original identity). Fail-fast pre-flight validation in `apply` (and dry-run in `plan`/`diff` via `fieldMappingErrors` in the machine output) with codes: `FIELD_MAP_TABLE_MISSING`, `FIELD_MAP_SOURCE_MISSING`, `FIELD_MAP_TARGET_MISSING`, `FIELD_MAP_TARGET_COMPUTED` (dest must be writable scalar), `FIELD_MAP_TYPE_INCOMPATIBLE` (array-typed source/dest not supported), `FIELD_MAP_COLLISION` (two sources map to same dest); an invalid mapping aborts `apply` synchronously (`FIELD_MAP_INVALID` error code). Natural-key record matching: `naturalKeys` (`{ [tableName]: fieldName }`) matches dest↔source records by the key field's **value** (field ID resolved per-base by name), growing `idmap.records`. Add-only + ambiguity-safe — never removes/overwrites an existing mapping, never claims an already-mapped dest; ambiguous/duplicate key values are skipped with `NATURAL_KEY_AMBIGUOUS`; a missing key field emits `NATURAL_KEY_FIELD_MISSING`. Runs in `mode=reconcile` (after the existence-prune, snapshots source records + calls matcher, then saves idmap) AND as an automatic pre-pass in `mode=apply` **before Pass 1 and prune** — so records `mirror` no longer deletes real-but-unmapped dest records as false orphans, and re-syncs don't duplicate them. The key field must be **stable across bases** (autoNumber renumbers → bad key; name/email/injected `InjectID` → good key). Single-field keys only; composite keys and value normalization are deferred. Returns a `matched` count in the result. #### packages/mcp-server — Daemon subsystem From 87b1e0540e26ae5054e5915635355bdf0eed6889 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sun, 21 Jun 2026 14:12:18 +0300 Subject: [PATCH 132/246] docs(changelog): fix stale deferred-follow-ups sentence The 2026-06-20 entry claimed schema/view/attachment deletion and natural-key matching were deferred. Both schema-orphan deletion and natural-key matching shipped separately (2026-06-21). Only attachment-strip under mirror remains deferred. Updated sentence to reflect current scope. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 013811f..23675f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how ### MCP server — sync_base schema-orphan deletion under mirror (2026-06-21) #### Added -- `sync_base mode=apply` — **schema-orphan deletion** (`pruneSchema`): under `policy=mirror`, dest-only fields/views/tables that have no counterpart in the source are deleted from the destination after `applyPlan` completes. Two independent gates: `confirmDeletions:true` is required to delete dest-only **fields and views** (same flag as `pruneRecords`); the new `confirmTableDeletions:true` is required to drop dest-only **tables** — `confirmDeletions` alone never deletes an entire table. Without each gate the operation is a dry-count only (`DELETION_GATED` for fields/views, `TABLE_DELETION_GATED` for tables). Deletion is dependency-safe: fields are deleted without `force` (so no matched field is ever cascade-deleted), orphan→orphan formula dependencies are resolved by retry-until-stable, and a field blocked by a matched-field dependency is kept and reported as `SCHEMA_DELETE_BLOCKED`. Deletion order: views first (no dependents), then fields, then tables (after their content). Applied counts are surfaced in the result as `schemaDeleted` (fields + views) and `tablesDeleted`; failures are continue-on-failure warnings (`SCHEMA_DELETE_FAILED`). Per-table `policyOverrides` are respected — `overlay`/`preserve` tables are never pruned. Module: `src/sync/prune-schema.js`. Retypes and view-section orphan deletion remain deferred (M4). +- `sync_base mode=apply` — **schema-orphan deletion** (`pruneSchema`): under `policy=mirror`, dest-only fields/views/tables that have no counterpart in the source are deleted from the destination after `applyPlan` completes. Two independent gates: `confirmDeletions:true` is required to delete dest-only **fields and views** (same flag as `pruneRecords`); the new `confirmTableDeletions:true` is required to drop dest-only **tables** — `confirmDeletions` alone never deletes an entire table. Without each gate the operation is a dry-count only (`DELETION_GATED` for fields/views, `TABLE_DELETION_GATED` for tables). Deletion is dependency-safe: fields are deleted without `force` (so no matched field is ever cascade-deleted), orphan→orphan formula dependencies are resolved by retry-until-stable, and a field blocked by a matched-field dependency is kept and reported as `SCHEMA_DELETE_BLOCKED`. Deletion order: views first (no dependents), then fields, then tables (after their content). Applied counts are surfaced in the result as `schemaDeleted` (fields + views) and `tablesDeleted`; failures are continue-on-failure warnings (`SCHEMA_DELETE_FAILED`). Per-table `policyOverrides` are respected — `overlay`/`preserve` tables are never pruned. Module: `src/sync/prune-schema.js`. Deferred: retypes, view-section orphan deletion, attachment-strip (M4). ### MCP server — sync_base natural-key record matching (2026-06-21) @@ -20,7 +20,7 @@ Check [Keep a Changelog](http://keepachangelog.com/) for recommendations on how #### Added - `sync_base mode=apply` — **records reconciliation policy**: new `policy` param (`mirror` | `overlay` | `preserve`, default `overlay`) controls two independent axes — **extras** (keep or remove dest-only records) and **conflicts** (source-wins or dest-wins when both bases hold the record). Presets: `mirror` = {remove, source-wins} (make dest identical to source), `overlay` = {keep, source-wins} (default — today's existing behavior), `preserve` = {keep, dest-wins} (never overwrite dest edits). `policyOverrides` (`{ [tableName]: preset }`) overrides the global preset per table. -- `sync_base mode=apply` — **deletion gate** (`confirmDeletions`): under `mirror` policy (or any table override that removes extras), dest-only orphan records are not deleted unless `confirmDeletions: true`. Without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count and does nothing — poll `mode=status` to see counts, then re-run with `confirmDeletions:true` to apply. Separately, `createTable` unconditionally clears the ~3 blank scaffolding rows it seeds (best-effort; `SCAFFOLDING_ROWS_KEPT` warning if the deletion attempt fails); `pruneRecords` under `mirror`+`confirmDeletions` additionally removes any leftover dest-only blank rows as part of the ordinary orphan prune. Schema/view/attachment deletion under mirror and natural-key record matching are deferred follow-ups. +- `sync_base mode=apply` — **deletion gate** (`confirmDeletions`): under `mirror` policy (or any table override that removes extras), dest-only orphan records are not deleted unless `confirmDeletions: true`. Without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count and does nothing — poll `mode=status` to see counts, then re-run with `confirmDeletions:true` to apply. Separately, `createTable` unconditionally clears the ~3 blank scaffolding rows it seeds (best-effort; `SCAFFOLDING_ROWS_KEPT` warning if the deletion attempt fails); `pruneRecords` under `mirror`+`confirmDeletions` additionally removes any leftover dest-only blank rows as part of the ordinary orphan prune. Attachment-strip under mirror remains a deferred follow-up (schema-orphan deletion and natural-key matching shipped separately). - `sync_base` — **custom field mappings** (`fieldMappings: { [tableName]: { sourceFieldName: destFieldName } }`): inject a source field's value (including computed fields like `autoNumber` or formula) into a different writable scalar dest field during record sync. Classic use: inject a source `autoNumber` field (`Code`) into a writable dest text field (`InjectID`) so a dest primary-key formula can reconstruct original identity across bases. Fail-fast pre-flight validation aborts `mode=apply` synchronously on any error (`FIELD_MAP_INVALID`); `mode=plan` and `mode=diff` run the same validation as a dry check and return errors in `fieldMappingErrors` (machine output). Validation codes: `FIELD_MAP_TABLE_MISSING`, `FIELD_MAP_SOURCE_MISSING`, `FIELD_MAP_TARGET_MISSING`, `FIELD_MAP_TARGET_COMPUTED` (dest must be writable scalar), `FIELD_MAP_TYPE_INCOMPATIBLE` (array-typed source/dest rejected), `FIELD_MAP_COLLISION` (two sources targeting the same dest). ### MCP server — sync_base compare, curatable changeset, selective apply (2026-06-19) From 40d5643e4255fdffc74342cbf90f12ab534482e3 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sun, 21 Jun 2026 14:16:13 +0300 Subject: [PATCH 133/246] fix(sync): guard pruneSchema with aborted check Do not run pruneSchema when apply() detects drift or other abort conditions. Moves schema extras phase into the if (!result.aborted) block alongside the records phase. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/index.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/mcp-server/src/sync/index.js b/packages/mcp-server/src/sync/index.js index 430efdc..4a5067e 100644 --- a/packages/mcp-server/src/sync/index.js +++ b/packages/mcp-server/src/sync/index.js @@ -153,13 +153,13 @@ export async function apply({ client, sourceBaseId, destBaseId, planId, runStart // Schema extras phase: delete dest-only fields/views/tables under mirror, gated. Runs after // applyPlan so matched fields/views exist (orphan deps safe) and BEFORE the records job so // the schema is clean before record sync begins. Mutates `result` in-place. - await pruneSchema({ client, destAppId: destBaseId, plan: fullPlan, policy, policyOverrides, confirmDeletions, confirmTableDeletions, result }); - - // Records phase: runs after schema apply, only if not aborted. It is minutes-long for large - // bases, so we launch it in the BACKGROUND (fire-and-forget) and return immediately — a single - // blocking apply call would look hung / time out. The phase persists progress (idmap + records - // journal) and writes a status file; poll via `sync_base mode=status`. if (!result.aborted) { + await pruneSchema({ client, destAppId: destBaseId, plan: fullPlan, policy, policyOverrides, confirmDeletions, confirmTableDeletions, result }); + + // Records phase: runs after schema apply, only if not aborted. It is minutes-long for large + // bases, so we launch it in the BACKGROUND (fire-and-forget) and return immediately — a single + // blocking apply call would look hung / time out. The phase persists progress (idmap + records + // journal) and writes a status file; poll via `sync_base mode=status`. writeRecordsJobStatus(sourceBaseId, destBaseId, planId, { status: 'running', startedAt: runStartedAt }); applyRecordsImpl({ client, sourceBaseId, destBaseId, planId, runStartedAt, policy, policyOverrides, confirmDeletions, fieldMappings, naturalKeys }) .then((r) => writeRecordsJobStatus(sourceBaseId, destBaseId, planId, { From 5e5860f9690372a3bf12da7826c13c0e1a71e0e3 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sun, 21 Jun 2026 15:41:04 +0300 Subject: [PATCH 134/246] feat(sync): apply scalar field retypes under confirmRetypes (was RETYPE_DEFERRED) Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/apply.js | 33 ++++++-- .../mcp-server/test/sync/test-apply.test.js | 79 ++++++++++++++++++- 2 files changed, 104 insertions(+), 8 deletions(-) diff --git a/packages/mcp-server/src/sync/apply.js b/packages/mcp-server/src/sync/apply.js index 1bea81e..17164cc 100644 --- a/packages/mcp-server/src/sync/apply.js +++ b/packages/mcp-server/src/sync/apply.js @@ -9,6 +9,7 @@ const UNSUPPORTED_TYPES = new Set(['button', 'asyncText', 'aiText', 'externalSyn const VIEW_GROUP_ANCHOR = new Set(['select', 'singleSelect', 'multiSelect', 'multipleSelects', 'collaborator']); const COMPUTED_TYPES = new Set(['formula', 'rollup', 'lookup', 'multipleLookupValues', 'count']); const LINK_TYPES = new Set(['foreignKey', 'multipleRecordLinks']); +const SCALAR_RETYPE_TYPES = new Set(['text', 'multilineText', 'richText', 'number', 'currency', 'percent', 'rating', 'duration', 'checkbox', 'date', 'dateTime', 'phone', 'email', 'url', 'select', 'singleSelect', 'multiSelect', 'multipleSelects']); // Types Airtable refuses as a primary field — keep a placeholder + warn instead of // retyping. The try/catch around the retype is the ultimate guard (set is an optimization). @@ -96,11 +97,11 @@ function mergeChoices(destField, srcTypeOptions) { return { ...srcTypeOptions, choices: merged }; } -export async function applyPlan({ client, plan, destAppId, destSnapshot, idmap, journal, persist, skip = [] }) { +export async function applyPlan({ client, plan, destAppId, destSnapshot, idmap, journal, persist, skip = [], confirmRetypes = false }) { const index = buildIndex(destSnapshot); const state = { createdLinks: new Map(), adoptedReverse: new Set() }; if (!idmap.views) idmap.views = {}; - const result = { planId: plan.planId, aborted: false, created: 0, updated: 0, skipped: 0, failed: 0, warnings: [], idmap }; + const result = { planId: plan.planId, aborted: false, created: 0, updated: 0, skipped: 0, failed: 0, retyped: 0, warnings: [], idmap }; const skipSet = new Set(skip); for (let idx = 0; idx < plan.actions.length; idx++) { @@ -108,7 +109,7 @@ export async function applyPlan({ client, plan, destAppId, destSnapshot, idmap, if (isDone(journal, idx)) { result.skipped++; continue; } if (a.apply === false || skipSet.has(a.changeId)) { result.skipped++; continue; } try { - await applyAction({ client, destAppId, a, idmap, index, state, result }); + await applyAction({ client, destAppId, a, idmap, index, state, result, confirmRetypes }); recordDone(journal, idx, a.kind, idmap.tables[a.sourceTableId] ?? (a.sourceFieldId && idmap.fields[a.sourceFieldId]?.destFld)); persist(idmap, journal); } catch (e) { @@ -124,7 +125,7 @@ export async function applyPlan({ client, plan, destAppId, destSnapshot, idmap, return result; } -async function applyAction({ client, destAppId, a, idmap, index, state, result }) { +async function applyAction({ client, destAppId, a, idmap, index, state, result, confirmRetypes }) { switch (a.kind) { case 'createTable': { const existing = index.tablesByName.get(a.name); @@ -238,8 +239,28 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result } case 'updateField': { const changes = a.changes || {}; if (changes.type !== undefined) { - result.warnings.push({ code: 'RETYPE_DEFERRED', message: `Field ${a.destFld}: type change to "${changes.type}" deferred (M4)` }); - return; // never retype an existing field in M2b + const destType = findDestFieldType(index, a.destFld); + const fname = findDestField(index, a.destFld)?.name ?? a.destFld; + if (!SCALAR_RETYPE_TYPES.has(changes.type) || !SCALAR_RETYPE_TYPES.has(destType)) { + result.warnings.push({ code: 'RETYPE_DEFERRED', message: `Field "${fname}": retype ${destType}→${changes.type} deferred (non-scalar)` }); + return; + } + if (!confirmRetypes) { + result.warnings.push({ code: 'RETYPE_GATED', message: `Field "${fname}": scalar retype ${destType}→${changes.type} gated — set confirmRetypes:true` }); + return; + } + const newOpts = changes.typeOptions ? mergeChoices(findDestField(index, a.destFld), remapRefs(changes.typeOptions, idmap)) : undefined; + try { + await client.updateFieldConfig(destAppId, a.destFld, { type: changes.type, typeOptions: newOpts }); + const f = findDestField(index, a.destFld); if (f) { f.type = changes.type; if (newOpts) f.typeOptions = newOpts; } + result.retyped++; + } catch (e) { + result.warnings.push({ code: 'RETYPE_FAILED', message: `Field "${fname}": retype to ${changes.type} rejected: ${e.message ?? e}` }); + return; + } + if (changes.name !== undefined) { await client.renameField(destAppId, a.destFld, changes.name); const f2 = findDestField(index, a.destFld); if (f2) f2.name = changes.name; } + if (changes.description !== undefined) { await client.updateFieldDescription(destAppId, a.destFld, changes.description); } + return; } let mutated = false; if (changes.name !== undefined) { await client.renameField(destAppId, a.destFld, changes.name); mutated = true; } diff --git a/packages/mcp-server/test/sync/test-apply.test.js b/packages/mcp-server/test/sync/test-apply.test.js index 5b6d528..e0649aa 100644 --- a/packages/mcp-server/test/sync/test-apply.test.js +++ b/packages/mcp-server/test/sync/test-apply.test.js @@ -222,12 +222,12 @@ describe('apply: updateField', () => { assert.equal(client._field(columnId).description, 'new'); }); - it('skips + reports a type change (RETYPE_DEFERRED), making no client mutation', async () => { + it('skips + reports a scalar type change (RETYPE_GATED when no confirmRetypes), making no client mutation', async () => { const { client, columnId, destSnapshot } = await destWith('text', null); const before = client.calls.length; const res = await run(client, planUpdate(columnId, { type: 'number', typeOptions: { precision: 0 } }), destSnapshot); assert.equal(res.updated, 0); - assert.ok(res.warnings.some((w) => w.code === 'RETYPE_DEFERRED')); + assert.ok(res.warnings.some((w) => w.code === 'RETYPE_GATED'), 'scalar retype without confirmRetypes → RETYPE_GATED'); assert.equal(client.calls.length, before); // no updateFieldConfig issued }); @@ -479,3 +479,78 @@ describe('apply: createField (autoNumber — strip read-only maxUsedAutoNumber)' ); }); }); + +// Build a dest base with table 'T' + one field of `fieldType`; return { destSnapshot, fId }. +async function destWithField(client, fieldType, typeOptions = null) { + const { tableId } = await client.createTable('appD', 'T'); + const { columnId: fId } = await client.createField('appD', tableId, { name: 'F', type: fieldType, typeOptions }); + const destSnapshot = await snapshotBase(client, 'appD'); + return { destSnapshot, fId }; +} +function retypePlan(fId, changes) { + return { planId: 'pRT', sourceBaseId: 'appS', destBaseId: 'appD', idmap: { tables: {}, fields: {} }, + actions: [{ kind: 'updateField', sourceFieldId: 'sF', destFld: fId, changes }], orphans: [], warnings: [] }; +} +function runRT(client, plan, destSnapshot, confirmRetypes) { + return applyPlan({ client, plan, destAppId: 'appD', destSnapshot, idmap: { tables: {}, fields: {} }, + journal: newJournal(plan.planId, 'ts'), persist: () => {}, confirmRetypes }); +} + +describe('apply: field retypes (M4)', () => { + it('scalar retype + confirmRetypes → updateFieldConfig called with new type, retyped=1', async () => { + const client = new MockClient(); + const { destSnapshot, fId } = await destWithField(client, 'text'); + const res = await runRT(client, retypePlan(fId, { type: 'number' }), destSnapshot, true); + assert.equal(res.retyped, 1); + assert.ok(client.calls.includes(`updateFieldConfig:${fId}:number`)); + }); + it('scalar retype WITHOUT confirmRetypes → not applied, RETYPE_GATED', async () => { + const client = new MockClient(); + const { destSnapshot, fId } = await destWithField(client, 'text'); + const res = await runRT(client, retypePlan(fId, { type: 'number' }), destSnapshot, false); + assert.equal(res.retyped ?? 0, 0); + assert.ok(!client.calls.some((c) => c.startsWith(`updateFieldConfig:${fId}`))); + assert.ok(res.warnings.some((w) => w.code === 'RETYPE_GATED')); + }); + it('out-of-scope retype (dest text → source formula) → RETYPE_DEFERRED, not applied', async () => { + const client = new MockClient(); + const { destSnapshot, fId } = await destWithField(client, 'text'); + const res = await runRT(client, retypePlan(fId, { type: 'formula' }), destSnapshot, true); + assert.equal(res.retyped ?? 0, 0); + assert.ok(res.warnings.some((w) => w.code === 'RETYPE_DEFERRED')); + }); + it('rejected retype → RETYPE_FAILED, field kept, no throw', async () => { + const client = new MockClient(); + const { destSnapshot, fId } = await destWithField(client, 'text'); + client.updateFieldConfig = async () => { throw new Error('incompatible existing data'); }; + const res = await runRT(client, retypePlan(fId, { type: 'number' }), destSnapshot, true); + assert.equal(res.retyped ?? 0, 0); + assert.ok(res.warnings.some((w) => w.code === 'RETYPE_FAILED')); + assert.equal(res.failed, 0, 'continue-on-failure: not a hard failure'); + }); + it('retype + rename in one action → both applied', async () => { + const client = new MockClient(); + const { destSnapshot, fId } = await destWithField(client, 'text'); + const res = await runRT(client, retypePlan(fId, { type: 'number', name: 'Renamed' }), destSnapshot, true); + assert.equal(res.retyped, 1); + assert.ok(client.calls.includes(`updateFieldConfig:${fId}:number`)); + assert.ok(client.calls.some((c) => c.startsWith(`renameField:${fId}:Renamed`))); + }); + it('text → select retype → updateFieldConfig with select + choices (mergeChoices)', async () => { + const client = new MockClient(); + const { destSnapshot, fId } = await destWithField(client, 'text'); + const changes = { type: 'select', typeOptions: { choices: { sel1: { id: 'sel1', name: 'Open', color: 'blue' } } } }; + const res = await runRT(client, retypePlan(fId, changes), destSnapshot, true); + assert.equal(res.retyped, 1); + assert.ok(client.calls.includes(`updateFieldConfig:${fId}:select`)); + const f = client.tables[0].columns.find((c) => c.id === fId); + assert.ok(f.typeOptions && f.typeOptions.choices, 'choices applied via mergeChoices'); + }); + it('non-type updateField (typeOptions only) is unchanged by the retype path', async () => { + const client = new MockClient(); + const { destSnapshot, fId } = await destWithField(client, 'number'); + const res = await runRT(client, retypePlan(fId, { typeOptions: { precision: 2 } }), destSnapshot, false); + assert.ok(client.calls.includes(`updateFieldConfig:${fId}:number`), 'still applies typeOptions update'); + assert.ok(!res.warnings.some((w) => w.code === 'RETYPE_GATED')); + }); +}); From 2f3c500d071090a87205b73b23671ef6bb83fd9f Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sun, 21 Jun 2026 15:47:19 +0300 Subject: [PATCH 135/246] feat(sync): sync_base confirmRetypes param + surface retyped count Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/index.js | 5 +++-- packages/mcp-server/src/sync/index.js | 4 ++-- packages/mcp-server/src/sync/report.js | 3 +++ .../mcp-server/test/sync/test-report.test.js | 18 ++++++++++++++++++ 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/packages/mcp-server/src/index.js b/packages/mcp-server/src/index.js index 6739f3d..ee8afa6 100644 --- a/packages/mcp-server/src/index.js +++ b/packages/mcp-server/src/index.js @@ -1598,6 +1598,7 @@ Note: "form title" is the view name itself — use rename_view to change it. "Fi policyOverrides: { type: 'object', description: 'Used by mode="apply": per-table reconciliation preset overrides. Maps table name → preset (e.g. { "Games": "preserve" }). Overrides the global policy for the named tables.', additionalProperties: { type: 'string', enum: ['mirror', 'overlay', 'preserve'] } }, confirmDeletions: { type: 'boolean', description: 'Used by mode="apply" with policy="mirror": must be set to true to actually delete dest-only RECORDS and dest-only FIELDS/VIEWS. Without it, mirror mode reports a DELETION_GATED count and deletes nothing for records; orphan fields/views are reported and kept.' }, confirmTableDeletions: { type: 'boolean', description: 'Used by mode=apply with policy=mirror: required to delete whole dest-only TABLES. confirmDeletions alone never drops a table; without confirmTableDeletions, orphan tables report TABLE_DELETION_GATED and are kept.' }, + confirmRetypes: { type: 'boolean', description: 'Used by mode=apply: required to apply a matched field\'s SCALAR type change to the destination (source-wins). Without it, a diverging scalar type reports RETYPE_GATED and the field is kept. Non-scalar retypes (to/from formula/rollup/lookup/count/autoNumber/link/attachment) stay RETYPE_DEFERRED.' }, fieldMappings: { type: 'object', description: 'Field mapping overrides: maps table name → { sourceField: destField } to inject a source field\'s value into a different (writable scalar) dest field during record sync. Example: { "Games": { "Title": "Name" } }. In mode="plan" and mode="diff", fieldMappings are validated against the two schemas (dry-run, no mutation) and errors are returned in fieldMappingErrors.', additionalProperties: { type: 'object', additionalProperties: { type: 'string' } } }, debug: debugProp, }, @@ -2400,7 +2401,7 @@ const handlers = { // ── Sync ── - async sync_base({ mode, sourceAppId, destAppId, planId, naturalKeys, detail, diffId, offset, limit, direction, skip, policy, policyOverrides, confirmDeletions, confirmTableDeletions, fieldMappings, debug }) { + async sync_base({ mode, sourceAppId, destAppId, planId, naturalKeys, detail, diffId, offset, limit, direction, skip, policy, policyOverrides, confirmDeletions, confirmTableDeletions, confirmRetypes, fieldMappings, debug }) { const sync = await import('./sync/index.js'); if (mode === 'plan') { const id = 'pln' + client._genRandomId(); @@ -2412,7 +2413,7 @@ const handlers = { const runStartedAt = new Date().toISOString(); let out; try { - out = await sync.apply({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, planId, runStartedAt, skip: skip || [], policy, policyOverrides, confirmDeletions, confirmTableDeletions, fieldMappings, naturalKeys }); + out = await sync.apply({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, planId, runStartedAt, skip: skip || [], policy, policyOverrides, confirmDeletions, confirmTableDeletions, confirmRetypes, fieldMappings, naturalKeys }); } catch (e) { if (e && e.code === 'FIELD_MAP_INVALID') { return err(`Field mapping validation failed:\n${(e.mappingErrors || []).map((me) => ` [${me.code}] table=${me.table || '?'}${me.source ? ' source=' + me.source : ''}${me.target ? ' target=' + me.target : ''}`).join('\n')}`); diff --git a/packages/mcp-server/src/sync/index.js b/packages/mcp-server/src/sync/index.js index 4a5067e..4b526d8 100644 --- a/packages/mcp-server/src/sync/index.js +++ b/packages/mcp-server/src/sync/index.js @@ -115,7 +115,7 @@ export function diffDetail({ sourceBaseId, destBaseId, diffId, detail, offset, l return renderDiff(savedDiff, { detail, offset, limit }); } -export async function apply({ client, sourceBaseId, destBaseId, planId, runStartedAt, skip = [], policy, policyOverrides, confirmDeletions, confirmTableDeletions, fieldMappings, naturalKeys }) { +export async function apply({ client, sourceBaseId, destBaseId, planId, runStartedAt, skip = [], policy, policyOverrides, confirmDeletions, confirmTableDeletions, confirmRetypes, fieldMappings, naturalKeys }) { const fullPlan = loadPlan(sourceBaseId, destBaseId, planId); if (!fullPlan) throw new Error(`No saved plan "${planId}" for ${sourceBaseId} -> ${destBaseId}. Run mode=plan first.`); @@ -147,7 +147,7 @@ export async function apply({ client, sourceBaseId, destBaseId, planId, runStart const result = await applyPlan({ client, plan: fullPlan, destAppId: destBaseId, destSnapshot, idmap, journal, persist: (m, j) => { saveIdmap(sourceBaseId, destBaseId, m); saveJournal(sourceBaseId, destBaseId, j); }, - skip, + skip, confirmRetypes, }); // Schema extras phase: delete dest-only fields/views/tables under mirror, gated. Runs after diff --git a/packages/mcp-server/src/sync/report.js b/packages/mcp-server/src/sync/report.js index 3fadf44..785ccb0 100644 --- a/packages/mcp-server/src/sync/report.js +++ b/packages/mcp-server/src/sync/report.js @@ -71,6 +71,9 @@ export function renderApplyResult(result) { if (result.matched != null && result.matched > 0) { lines.push(` matched: ${result.matched} (natural-key pre-pass)`); } + if (result.retyped != null && result.retyped > 0) { + lines.push(` retyped: ${result.retyped} (scalar field type changes applied)`); + } if (result.schemaDeleted != null && result.schemaDeleted > 0) { lines.push(` schemaDeleted: ${result.schemaDeleted} (dest-only fields/views removed)`); } diff --git a/packages/mcp-server/test/sync/test-report.test.js b/packages/mcp-server/test/sync/test-report.test.js index 5c70f2d..9f158d8 100644 --- a/packages/mcp-server/test/sync/test-report.test.js +++ b/packages/mcp-server/test/sync/test-report.test.js @@ -54,6 +54,24 @@ describe('report.renderApplyResult', () => { assert.match(out.human, /Apply aborted/); assert.doesNotMatch(out.human, /records:/); }); + + it('renders retyped line when retyped > 0', () => { + const result = { ...baseResult, retyped: 2 }; + const out = renderApplyResult(result); + assert.match(out.human, /retyped: 2/); + assert.match(out.human, /scalar field type changes applied/); + }); + + it('omits retyped line when retyped is 0', () => { + const result = { ...baseResult, retyped: 0 }; + const out = renderApplyResult(result); + assert.doesNotMatch(out.human, /retyped/); + }); + + it('omits retyped line when retyped is absent', () => { + const out = renderApplyResult(baseResult); + assert.doesNotMatch(out.human, /retyped/); + }); }); // ── renderDiff tests (Task 6) ──────────────────────────────────────────────── From 3c0fe982d7d6d3a687c90b5f6258edc4b8dd4c22 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sun, 21 Jun 2026 15:54:57 +0300 Subject: [PATCH 136/246] docs(sync): document scalar field retypes under confirmRetypes Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 2d2891b..9be7362 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,7 +65,7 @@ Key files: - `dev-tools/` — reverse-engineering helpers (capture.js, analyze.js, debug-*.js), **gitignored** — kept locally for future reference when Airtable changes their internal API - `src/daemon/` — daemon subsystem (see daemon subsection below) - `src/safe-write.js` — atomic JSON write helper (used by lockfile, token, and settings) -- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: diff | plan | apply | status | reconcile`) in the `sync` category. Out of scope: retypes, deletions (M4). `mode=diff` (read-only): comprehensive schema comparison via pure `src/sync/compare.js` — every difference classified as **drift** (sync enforces: table/field/view existence, field type/options/choices, view filters/sorts/groups content, column visibility, colors/cover/frozen/rowHeight, primary, form), **best-effort** (sync applies but doesn't guarantee: field/view/column order, sort/group clause order), or **not-synced** (view sections — now captured by snapshot). Verdicts: `identical` (zero diffs) or `converged` (zero drift; best-effort/not-synced may remain). Persists full diff to `~/.airtable-user-mcp/sync/__/diff-.json`; returns a token-budgeted DIGEST (verdicts + class counts + per-table rollup + capped driftSample ≤25). Drill in with `detail="
"` (params: `detail`, `diffId`, `offset`, `limit`). `mode=plan` now emits a curatable **changeset**: each action carries a stable name-based `changeId` (`|
|`), a `class`, and an `apply:true` flag; digest includes a `sample` + `editHint`. New `direction` param (`to-dest` default | `to-source`) targets either base. View sync: `snapshot` also captures collaborative views + live config + **view sections** (not-synced class); `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported); accepts a `skip:[changeId]` list (and honors `apply:false` on changeset entries) — applies everything except removed changes (journal-safe; skipped dependencies degrade to UNRESOLVABLE_REF). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe by filename+size); then restores record-referencing view filters stripped during view sync. Throttle handling = per-request 429 backoff in the auth queue (the per-call limiter does NOT gate the internal per-row POSTs inside createRecords/updateRecords/addLinkItems); resumable (records journal, per-chunk persist of ~50 creates); continue-on-failure (per-record errors are warnings). `mode=status` polls a background job by planId — returns running/done/failed + live records-mapped count. `mode=reconcile` **prunes** dead record-map entries (existence-prune; per-table natural-key re-match now matches by key value — see natural-key matching below; reconcile does NOT de-duplicate records). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json, diff-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile), `compare.js` (pure classified compare, no I/O); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. Reconciliation policy (`src/sync/policy.js`): `mode=apply` accepts a `policy` param (`mirror` | `overlay` | `preserve`, default `overlay`) that controls two axes — **extras** (what to do with dest-only records) and **conflicts** (whose value wins when both bases have the record). Presets: `mirror` = {remove, source-wins} (make dest identical to source); `overlay` = {keep, source-wins} (today's default: keep dest-only rows, source updates win); `preserve` = {keep, dest-wins} (keep dest-only rows and never overwrite dest edits). `policyOverrides` (`{ [tableName]: preset }`) overrides the global preset per table. `confirmDeletions` (boolean, default false) is a safety gate for `mirror`: without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count but deletes nothing; set `confirmDeletions:true` to actually delete. Separately, `createTable` unconditionally clears the ~3 blank scaffolding rows it seeds (best-effort; `SCAFFOLDING_ROWS_KEPT` warning if the deletion attempt fails); `pruneRecords` under `mirror`+`confirmDeletions` additionally removes any leftover dest-only blank rows as part of the ordinary orphan prune. Attachment deletion under mirror is a deferred follow-up. Schema-orphan deletion (`pruneSchema`) runs synchronously in `mode=apply` AFTER `applyPlan`, consuming `plan.orphans` (`{kind:'table'|'field'|'view', destId, name, tableName}`). Order: (1) views — deleted first (no dependents); (2) fields — batch `deleteFields` WITHOUT force (so a matched field is never cascade-deleted), retry-until-stable loop resolves orphan→orphan deps; a field still blocked after no progress is kept + `SCHEMA_DELETE_BLOCKED`; (3) tables — deleted last (after their content). Gating: dest-only fields + views require `confirmDeletions:true` (same flag as `pruneRecords`); dest-only tables require the new `confirmTableDeletions:true` — `confirmDeletions` alone never drops a whole table. Without each gate the operation is a dry-count: `DELETION_GATED` for fields/views, `TABLE_DELETION_GATED` for tables. Applied counts: `schemaDeleted` (fields+views) + `tablesDeleted`. Per-table `policyOverrides` apply; `overlay`/`preserve` → no schema deletion. Continue-on-failure (`SCHEMA_DELETE_FAILED`). Module: `src/sync/prune-schema.js`. Retypes and view-section orphans are deferred (M4). Custom field mappings: `fieldMappings` (`{ [tableName]: { sourceFieldName: destFieldName } }`) injects a source field's value into a different writable scalar dest field during record sync — useful when the source is computed (e.g., `autoNumber`/formula) and you want its value preserved in a writable dest field (classic use: inject source autoNumber `Code` into dest text field `InjectID` so a dest formula can reconstruct original identity). Fail-fast pre-flight validation in `apply` (and dry-run in `plan`/`diff` via `fieldMappingErrors` in the machine output) with codes: `FIELD_MAP_TABLE_MISSING`, `FIELD_MAP_SOURCE_MISSING`, `FIELD_MAP_TARGET_MISSING`, `FIELD_MAP_TARGET_COMPUTED` (dest must be writable scalar), `FIELD_MAP_TYPE_INCOMPATIBLE` (array-typed source/dest not supported), `FIELD_MAP_COLLISION` (two sources map to same dest); an invalid mapping aborts `apply` synchronously (`FIELD_MAP_INVALID` error code). Natural-key record matching: `naturalKeys` (`{ [tableName]: fieldName }`) matches dest↔source records by the key field's **value** (field ID resolved per-base by name), growing `idmap.records`. Add-only + ambiguity-safe — never removes/overwrites an existing mapping, never claims an already-mapped dest; ambiguous/duplicate key values are skipped with `NATURAL_KEY_AMBIGUOUS`; a missing key field emits `NATURAL_KEY_FIELD_MISSING`. Runs in `mode=reconcile` (after the existence-prune, snapshots source records + calls matcher, then saves idmap) AND as an automatic pre-pass in `mode=apply` **before Pass 1 and prune** — so records `mirror` no longer deletes real-but-unmapped dest records as false orphans, and re-syncs don't duplicate them. The key field must be **stable across bases** (autoNumber renumbers → bad key; name/email/injected `InjectID` → good key). Single-field keys only; composite keys and value normalization are deferred. Returns a `matched` count in the result. +- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: diff | plan | apply | status | reconcile`) in the `sync` category. Out of scope: non-scalar retypes (computed/link/attachment on either side → `RETYPE_DEFERRED`), deletions (M4); scalar retypes are now in scope (see below). `mode=diff` (read-only): comprehensive schema comparison via pure `src/sync/compare.js` — every difference classified as **drift** (sync enforces: table/field/view existence, field type/options/choices, view filters/sorts/groups content, column visibility, colors/cover/frozen/rowHeight, primary, form), **best-effort** (sync applies but doesn't guarantee: field/view/column order, sort/group clause order), or **not-synced** (view sections — now captured by snapshot). Verdicts: `identical` (zero diffs) or `converged` (zero drift; best-effort/not-synced may remain). Persists full diff to `~/.airtable-user-mcp/sync/__/diff-.json`; returns a token-budgeted DIGEST (verdicts + class counts + per-table rollup + capped driftSample ≤25). Drill in with `detail="
"` (params: `detail`, `diffId`, `offset`, `limit`). `mode=plan` now emits a curatable **changeset**: each action carries a stable name-based `changeId` (`|
|`), a `class`, and an `apply:true` flag; digest includes a `sample` + `editHint`. New `direction` param (`to-dest` default | `to-source`) targets either base. View sync: `snapshot` also captures collaborative views + live config + **view sections** (not-synced class); `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported); accepts a `skip:[changeId]` list (and honors `apply:false` on changeset entries) — applies everything except removed changes (journal-safe; skipped dependencies degrade to UNRESOLVABLE_REF). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe by filename+size); then restores record-referencing view filters stripped during view sync. Throttle handling = per-request 429 backoff in the auth queue (the per-call limiter does NOT gate the internal per-row POSTs inside createRecords/updateRecords/addLinkItems); resumable (records journal, per-chunk persist of ~50 creates); continue-on-failure (per-record errors are warnings). `mode=status` polls a background job by planId — returns running/done/failed + live records-mapped count. `mode=reconcile` **prunes** dead record-map entries (existence-prune; per-table natural-key re-match now matches by key value — see natural-key matching below; reconcile does NOT de-duplicate records). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json, diff-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile), `compare.js` (pure classified compare, no I/O); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. Reconciliation policy (`src/sync/policy.js`): `mode=apply` accepts a `policy` param (`mirror` | `overlay` | `preserve`, default `overlay`) that controls two axes — **extras** (what to do with dest-only records) and **conflicts** (whose value wins when both bases have the record). Presets: `mirror` = {remove, source-wins} (make dest identical to source); `overlay` = {keep, source-wins} (today's default: keep dest-only rows, source updates win); `preserve` = {keep, dest-wins} (keep dest-only rows and never overwrite dest edits). `policyOverrides` (`{ [tableName]: preset }`) overrides the global preset per table. `confirmDeletions` (boolean, default false) is a safety gate for `mirror`: without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count but deletes nothing; set `confirmDeletions:true` to actually delete. Separately, `createTable` unconditionally clears the ~3 blank scaffolding rows it seeds (best-effort; `SCAFFOLDING_ROWS_KEPT` warning if the deletion attempt fails); `pruneRecords` under `mirror`+`confirmDeletions` additionally removes any leftover dest-only blank rows as part of the ordinary orphan prune. Attachment deletion under mirror is a deferred follow-up. Schema-orphan deletion (`pruneSchema`) runs synchronously in `mode=apply` AFTER `applyPlan`, consuming `plan.orphans` (`{kind:'table'|'field'|'view', destId, name, tableName}`). Order: (1) views — deleted first (no dependents); (2) fields — batch `deleteFields` WITHOUT force (so a matched field is never cascade-deleted), retry-until-stable loop resolves orphan→orphan deps; a field still blocked after no progress is kept + `SCHEMA_DELETE_BLOCKED`; (3) tables — deleted last (after their content). Gating: dest-only fields + views require `confirmDeletions:true` (same flag as `pruneRecords`); dest-only tables require the new `confirmTableDeletions:true` — `confirmDeletions` alone never drops a whole table. Without each gate the operation is a dry-count: `DELETION_GATED` for fields/views, `TABLE_DELETION_GATED` for tables. Applied counts: `schemaDeleted` (fields+views) + `tablesDeleted`. Per-table `policyOverrides` apply; `overlay`/`preserve` → no schema deletion. Continue-on-failure (`SCHEMA_DELETE_FAILED`). Module: `src/sync/prune-schema.js`. View-section orphans are deferred (M4). Scalar field retypes (`mode=apply`): a matched dest field whose scalar type diverges from source is retyped via `updateFieldConfig`, gated by the new `confirmRetypes` param (boolean, default false); without it the field is kept and a `RETYPE_GATED` warning is emitted (count surfaced in the result). Non-scalar retypes (either side is computed/link/attachment) stay `RETYPE_DEFERRED`. Select retypes converge their choice map on re-sync exactly like created selects (`mergeChoices`). Continue-on-failure: per-field retype errors emit `RETYPE_FAILED` and do not abort apply. Primary-field retypes remain the separate `reconcilePrimary` path. This closes the last schema-drift category sync can fix; records data migration remains `pruneRecords`/Pass 1. Custom field mappings: `fieldMappings` (`{ [tableName]: { sourceFieldName: destFieldName } }`) injects a source field's value into a different writable scalar dest field during record sync — useful when the source is computed (e.g., `autoNumber`/formula) and you want its value preserved in a writable dest field (classic use: inject source autoNumber `Code` into dest text field `InjectID` so a dest formula can reconstruct original identity). Fail-fast pre-flight validation in `apply` (and dry-run in `plan`/`diff` via `fieldMappingErrors` in the machine output) with codes: `FIELD_MAP_TABLE_MISSING`, `FIELD_MAP_SOURCE_MISSING`, `FIELD_MAP_TARGET_MISSING`, `FIELD_MAP_TARGET_COMPUTED` (dest must be writable scalar), `FIELD_MAP_TYPE_INCOMPATIBLE` (array-typed source/dest not supported), `FIELD_MAP_COLLISION` (two sources map to same dest); an invalid mapping aborts `apply` synchronously (`FIELD_MAP_INVALID` error code). Natural-key record matching: `naturalKeys` (`{ [tableName]: fieldName }`) matches dest↔source records by the key field's **value** (field ID resolved per-base by name), growing `idmap.records`. Add-only + ambiguity-safe — never removes/overwrites an existing mapping, never claims an already-mapped dest; ambiguous/duplicate key values are skipped with `NATURAL_KEY_AMBIGUOUS`; a missing key field emits `NATURAL_KEY_FIELD_MISSING`. Runs in `mode=reconcile` (after the existence-prune, snapshots source records + calls matcher, then saves idmap) AND as an automatic pre-pass in `mode=apply` **before Pass 1 and prune** — so records `mirror` no longer deletes real-but-unmapped dest records as false orphans, and re-syncs don't duplicate them. The key field must be **stable across bases** (autoNumber renumbers → bad key; name/email/injected `InjectID` → good key). Single-field keys only; composite keys and value normalization are deferred. Returns a `matched` count in the result. #### packages/mcp-server — Daemon subsystem From 2d9e397ee7753fb679cf0b0651aea452b6ad9520 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sun, 21 Jun 2026 18:20:16 +0300 Subject: [PATCH 137/246] fix(sync): pruneRecords skips truncated tables (>1000 rows) to prevent false-orphan deletion Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/records.js | 16 ++++++- .../test/sync/test-records-policy.test.js | 47 +++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index c287eb7..e3af837 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -1124,7 +1124,7 @@ export async function applyRecords({ client, sourceBaseId, destBaseId, planId, r * @param {object} opts.result - result accumulator (mutated: result.deleted, result.warnings) * @returns {Promise} */ -export async function pruneRecords({ client, destSnapshot, idmap, policy, policyOverrides, confirmDeletions, limiter, result }) { +export async function pruneRecords({ client, destSnapshot, idmap, policy, policyOverrides, confirmDeletions, limiter, result, truncatedTables = new Set() }) { if (result.deleted == null) result.deleted = 0; if (result.failed == null) result.failed = 0; if (!result.warnings) result.warnings = []; @@ -1145,6 +1145,20 @@ export async function pruneRecords({ client, destSnapshot, idmap, policy, policy const { extras } = resolvePolicy(policy, policyOverrides, t.name); if (extras !== 'remove') continue; + // Truncation safety: a table whose snapshot hit the 1000-row cap cannot be safely pruned — + // unmapped rows may be legitimate records whose source/dest counterpart fell beyond the window. + // Skip deletion entirely (the >1000-row read is a capture-gated follow-up). + const isTruncated = truncatedTables.has(t.name) || (t.records || []).length >= 1000; + if (isTruncated) { + result.warnings.push({ + code: 'RECORDS_TRUNCATED_PRUNE_SKIPPED', + message: `Table "${t.name}": snapshot capped at 1000 rows — cannot safely distinguish ` + + `dest-only orphans from records beyond the limit; skipping mirror deletion to avoid data loss ` + + `(>1000-row pagination is a follow-up).`, + }); + continue; + } + // Prefer the first collaborative (non-personal) view; fall back to first view of any type. const views = t.views || []; const viewId = views.find((v) => !v.personalForUserId)?.id ?? views[0]?.id; diff --git a/packages/mcp-server/test/sync/test-records-policy.test.js b/packages/mcp-server/test/sync/test-records-policy.test.js index be671a5..f91c1a7 100644 --- a/packages/mcp-server/test/sync/test-records-policy.test.js +++ b/packages/mcp-server/test/sync/test-records-policy.test.js @@ -212,6 +212,53 @@ describe('pruneRecords (extras axis)', () => { assert.equal(result.deleted, 0); }); + // ── Task 1: truncation guard ───────────────────────────────────────────────── + + // (1) THE FIX — mirror + confirmDeletions + DEST table at 1000 rows → no delete + skip warning + it('mirror skips a DEST-truncated table (1000 rows) instead of deleting orphans', async () => { + const client = pruneClient(); + const rows = Array.from({ length: 1000 }, (_, i) => ({ id: `recD${i}`, cellValuesByColumnId: {} })); + const destSnapshot = { baseId: 'appD', tables: [{ id: 'tD', name: 'Games', views: [{ id: 'viwD' }], fields: [], records: rows }] }; + const idmap = { tables: { tS: 'tD' }, fields: {}, records: {} }; // 0 mapped → all 1000 would be orphans + const result = { deleted: 0, failed: 0, warnings: [] }; + await pruneRecords({ client, destSnapshot, idmap, policy: 'mirror', confirmDeletions: true, limiter: { run: (fn) => fn() }, result }); + assert.equal(client.calls.deleted.length, 0, 'deleteRecords never called'); + assert.equal(result.deleted, 0); + assert.ok(result.warnings.some((w) => w.code === 'RECORDS_TRUNCATED_PRUNE_SKIPPED')); + }); + + // (2) source-side truncation supplied via the truncatedTables set (dest rows < 1000) + it('mirror skips a table named in truncatedTables even when dest rows < 1000', async () => { + const client = pruneClient(); + const destSnapshot = { baseId: 'appD', tables: [{ id: 'tD', name: 'SrcBig', views: [{ id: 'viwD' }], fields: [], records: [{ id: 'recD0', cellValuesByColumnId: {} }] }] }; + const idmap = { tables: { tS: 'tD' }, fields: {}, records: {} }; + const result = { deleted: 0, failed: 0, warnings: [] }; + await pruneRecords({ client, destSnapshot, idmap, policy: 'mirror', confirmDeletions: true, limiter: { run: (fn) => fn() }, result, truncatedTables: new Set(['SrcBig']) }); + assert.equal(client.calls.deleted.length, 0); + assert.ok(result.warnings.some((w) => w.code === 'RECORDS_TRUNCATED_PRUNE_SKIPPED')); + }); + + // (3) regression — NON-truncated table still prunes its orphan (use destSnap2(): recKeep mapped, recOrphan not) + it('mirror still deletes orphans on a NON-truncated table', async () => { + const client = pruneClient(); + const idmap = { tables: { tS: 'tD' }, fields: {}, records: { recS1: 'recKeep' } }; + const result = { deleted: 0, failed: 0, warnings: [] }; + await pruneRecords({ client, destSnapshot: destSnap2(), idmap, policy: 'mirror', confirmDeletions: true, limiter: { run: (fn) => fn() }, result }); + assert.deepEqual(client.calls.deleted, [{ tableId: 'tD', rowIds: ['recOrphan'] }]); + assert.ok(!result.warnings.some((w) => w.code === 'RECORDS_TRUNCATED_PRUNE_SKIPPED')); + }); + + // (4) overlay + truncated table → no truncation-skip warning (guard sits AFTER the extras check) + it('overlay + truncated table emits NO truncation-skip warning', async () => { + const client = pruneClient(); + const rows = Array.from({ length: 1000 }, (_, i) => ({ id: `recD${i}`, cellValuesByColumnId: {} })); + const destSnapshot = { baseId: 'appD', tables: [{ id: 'tD', name: 'Games', views: [{ id: 'viwD' }], fields: [], records: rows }] }; + const idmap = { tables: { tS: 'tD' }, fields: {}, records: {} }; + const result = { deleted: 0, failed: 0, warnings: [] }; + await pruneRecords({ client, destSnapshot, idmap, policy: 'overlay', confirmDeletions: true, limiter: { run: (fn) => fn() }, result }); + assert.ok(!result.warnings.some((w) => w.code === 'RECORDS_TRUNCATED_PRUNE_SKIPPED')); + }); + // Delete chunking: >50 orphans should produce multiple deleteRecords calls of ≤50 each. it('>50 orphans produce multiple delete chunks of ≤50 ids each', async () => { // 75 dest records; none in idmap.records → all are orphans. From 505125fe9cd147c61a37f0575233cf78aee0c4cb Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sun, 21 Jun 2026 18:25:27 +0300 Subject: [PATCH 138/246] feat(sync): runRecords flags truncated tables + warns; thread guard into pruneRecords; docs Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 2 +- packages/mcp-server/src/sync/records.js | 24 ++++++++++++++++++- .../test/sync/test-records-policy.test.js | 15 +++++++++++- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 9be7362..247b451 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,7 +65,7 @@ Key files: - `dev-tools/` — reverse-engineering helpers (capture.js, analyze.js, debug-*.js), **gitignored** — kept locally for future reference when Airtable changes their internal API - `src/daemon/` — daemon subsystem (see daemon subsection below) - `src/safe-write.js` — atomic JSON write helper (used by lockfile, token, and settings) -- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: diff | plan | apply | status | reconcile`) in the `sync` category. Out of scope: non-scalar retypes (computed/link/attachment on either side → `RETYPE_DEFERRED`), deletions (M4); scalar retypes are now in scope (see below). `mode=diff` (read-only): comprehensive schema comparison via pure `src/sync/compare.js` — every difference classified as **drift** (sync enforces: table/field/view existence, field type/options/choices, view filters/sorts/groups content, column visibility, colors/cover/frozen/rowHeight, primary, form), **best-effort** (sync applies but doesn't guarantee: field/view/column order, sort/group clause order), or **not-synced** (view sections — now captured by snapshot). Verdicts: `identical` (zero diffs) or `converged` (zero drift; best-effort/not-synced may remain). Persists full diff to `~/.airtable-user-mcp/sync/__/diff-.json`; returns a token-budgeted DIGEST (verdicts + class counts + per-table rollup + capped driftSample ≤25). Drill in with `detail="
"` (params: `detail`, `diffId`, `offset`, `limit`). `mode=plan` now emits a curatable **changeset**: each action carries a stable name-based `changeId` (`|
|`), a `class`, and an `apply:true` flag; digest includes a `sample` + `editHint`. New `direction` param (`to-dest` default | `to-source`) targets either base. View sync: `snapshot` also captures collaborative views + live config + **view sections** (not-synced class); `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported); accepts a `skip:[changeId]` list (and honors `apply:false` on changeset entries) — applies everything except removed changes (journal-safe; skipped dependencies degrade to UNRESOLVABLE_REF). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe by filename+size); then restores record-referencing view filters stripped during view sync. Throttle handling = per-request 429 backoff in the auth queue (the per-call limiter does NOT gate the internal per-row POSTs inside createRecords/updateRecords/addLinkItems); resumable (records journal, per-chunk persist of ~50 creates); continue-on-failure (per-record errors are warnings). `mode=status` polls a background job by planId — returns running/done/failed + live records-mapped count. `mode=reconcile` **prunes** dead record-map entries (existence-prune; per-table natural-key re-match now matches by key value — see natural-key matching below; reconcile does NOT de-duplicate records). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json, diff-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile), `compare.js` (pure classified compare, no I/O); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. Reconciliation policy (`src/sync/policy.js`): `mode=apply` accepts a `policy` param (`mirror` | `overlay` | `preserve`, default `overlay`) that controls two axes — **extras** (what to do with dest-only records) and **conflicts** (whose value wins when both bases have the record). Presets: `mirror` = {remove, source-wins} (make dest identical to source); `overlay` = {keep, source-wins} (today's default: keep dest-only rows, source updates win); `preserve` = {keep, dest-wins} (keep dest-only rows and never overwrite dest edits). `policyOverrides` (`{ [tableName]: preset }`) overrides the global preset per table. `confirmDeletions` (boolean, default false) is a safety gate for `mirror`: without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count but deletes nothing; set `confirmDeletions:true` to actually delete. Separately, `createTable` unconditionally clears the ~3 blank scaffolding rows it seeds (best-effort; `SCAFFOLDING_ROWS_KEPT` warning if the deletion attempt fails); `pruneRecords` under `mirror`+`confirmDeletions` additionally removes any leftover dest-only blank rows as part of the ordinary orphan prune. Attachment deletion under mirror is a deferred follow-up. Schema-orphan deletion (`pruneSchema`) runs synchronously in `mode=apply` AFTER `applyPlan`, consuming `plan.orphans` (`{kind:'table'|'field'|'view', destId, name, tableName}`). Order: (1) views — deleted first (no dependents); (2) fields — batch `deleteFields` WITHOUT force (so a matched field is never cascade-deleted), retry-until-stable loop resolves orphan→orphan deps; a field still blocked after no progress is kept + `SCHEMA_DELETE_BLOCKED`; (3) tables — deleted last (after their content). Gating: dest-only fields + views require `confirmDeletions:true` (same flag as `pruneRecords`); dest-only tables require the new `confirmTableDeletions:true` — `confirmDeletions` alone never drops a whole table. Without each gate the operation is a dry-count: `DELETION_GATED` for fields/views, `TABLE_DELETION_GATED` for tables. Applied counts: `schemaDeleted` (fields+views) + `tablesDeleted`. Per-table `policyOverrides` apply; `overlay`/`preserve` → no schema deletion. Continue-on-failure (`SCHEMA_DELETE_FAILED`). Module: `src/sync/prune-schema.js`. View-section orphans are deferred (M4). Scalar field retypes (`mode=apply`): a matched dest field whose scalar type diverges from source is retyped via `updateFieldConfig`, gated by the new `confirmRetypes` param (boolean, default false); without it the field is kept and a `RETYPE_GATED` warning is emitted (count surfaced in the result). Non-scalar retypes (either side is computed/link/attachment) stay `RETYPE_DEFERRED`. Select retypes converge their choice map on re-sync exactly like created selects (`mergeChoices`). Continue-on-failure: per-field retype errors emit `RETYPE_FAILED` and do not abort apply. Primary-field retypes remain the separate `reconcilePrimary` path. This closes the last schema-drift category sync can fix; records data migration remains `pruneRecords`/Pass 1. Custom field mappings: `fieldMappings` (`{ [tableName]: { sourceFieldName: destFieldName } }`) injects a source field's value into a different writable scalar dest field during record sync — useful when the source is computed (e.g., `autoNumber`/formula) and you want its value preserved in a writable dest field (classic use: inject source autoNumber `Code` into dest text field `InjectID` so a dest formula can reconstruct original identity). Fail-fast pre-flight validation in `apply` (and dry-run in `plan`/`diff` via `fieldMappingErrors` in the machine output) with codes: `FIELD_MAP_TABLE_MISSING`, `FIELD_MAP_SOURCE_MISSING`, `FIELD_MAP_TARGET_MISSING`, `FIELD_MAP_TARGET_COMPUTED` (dest must be writable scalar), `FIELD_MAP_TYPE_INCOMPATIBLE` (array-typed source/dest not supported), `FIELD_MAP_COLLISION` (two sources map to same dest); an invalid mapping aborts `apply` synchronously (`FIELD_MAP_INVALID` error code). Natural-key record matching: `naturalKeys` (`{ [tableName]: fieldName }`) matches dest↔source records by the key field's **value** (field ID resolved per-base by name), growing `idmap.records`. Add-only + ambiguity-safe — never removes/overwrites an existing mapping, never claims an already-mapped dest; ambiguous/duplicate key values are skipped with `NATURAL_KEY_AMBIGUOUS`; a missing key field emits `NATURAL_KEY_FIELD_MISSING`. Runs in `mode=reconcile` (after the existence-prune, snapshots source records + calls matcher, then saves idmap) AND as an automatic pre-pass in `mode=apply` **before Pass 1 and prune** — so records `mirror` no longer deletes real-but-unmapped dest records as false orphans, and re-syncs don't duplicate them. The key field must be **stable across bases** (autoNumber renumbers → bad key; name/email/injected `InjectID` → good key). Single-field keys only; composite keys and value normalization are deferred. Returns a `matched` count in the result. +- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: diff | plan | apply | status | reconcile`) in the `sync` category. Out of scope: non-scalar retypes (computed/link/attachment on either side → `RETYPE_DEFERRED`), deletions (M4); scalar retypes are now in scope (see below). `mode=diff` (read-only): comprehensive schema comparison via pure `src/sync/compare.js` — every difference classified as **drift** (sync enforces: table/field/view existence, field type/options/choices, view filters/sorts/groups content, column visibility, colors/cover/frozen/rowHeight, primary, form), **best-effort** (sync applies but doesn't guarantee: field/view/column order, sort/group clause order), or **not-synced** (view sections — now captured by snapshot). Verdicts: `identical` (zero diffs) or `converged` (zero drift; best-effort/not-synced may remain). Persists full diff to `~/.airtable-user-mcp/sync/__/diff-.json`; returns a token-budgeted DIGEST (verdicts + class counts + per-table rollup + capped driftSample ≤25). Drill in with `detail="
"` (params: `detail`, `diffId`, `offset`, `limit`). `mode=plan` now emits a curatable **changeset**: each action carries a stable name-based `changeId` (`|
|`), a `class`, and an `apply:true` flag; digest includes a `sample` + `editHint`. New `direction` param (`to-dest` default | `to-source`) targets either base. View sync: `snapshot` also captures collaborative views + live config + **view sections** (not-synced class); `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported); accepts a `skip:[changeId]` list (and honors `apply:false` on changeset entries) — applies everything except removed changes (journal-safe; skipped dependencies degrade to UNRESOLVABLE_REF). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe by filename+size); then restores record-referencing view filters stripped during view sync. Throttle handling = per-request 429 backoff in the auth queue (the per-call limiter does NOT gate the internal per-row POSTs inside createRecords/updateRecords/addLinkItems); resumable (records journal, per-chunk persist of ~50 creates); continue-on-failure (per-record errors are warnings). Truncation safety: a table whose snapshot hits the 1000-row cap on either base is added to `truncatedTables` by `collectTruncatedTableNames` and skipped by `pruneRecords` under mirror (`RECORDS_TRUNCATED_PRUNE_SKIPPED`) so a record beyond the snapshot window is never deleted as a false orphan; each truncated table also emits `RECORDS_TRUNCATED` on the result; >1000-row read pagination remains a capture-gated follow-up. `mode=status` polls a background job by planId — returns running/done/failed + live records-mapped count. `mode=reconcile` **prunes** dead record-map entries (existence-prune; per-table natural-key re-match now matches by key value — see natural-key matching below; reconcile does NOT de-duplicate records). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json, diff-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile), `compare.js` (pure classified compare, no I/O); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. Reconciliation policy (`src/sync/policy.js`): `mode=apply` accepts a `policy` param (`mirror` | `overlay` | `preserve`, default `overlay`) that controls two axes — **extras** (what to do with dest-only records) and **conflicts** (whose value wins when both bases have the record). Presets: `mirror` = {remove, source-wins} (make dest identical to source); `overlay` = {keep, source-wins} (today's default: keep dest-only rows, source updates win); `preserve` = {keep, dest-wins} (keep dest-only rows and never overwrite dest edits). `policyOverrides` (`{ [tableName]: preset }`) overrides the global preset per table. `confirmDeletions` (boolean, default false) is a safety gate for `mirror`: without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count but deletes nothing; set `confirmDeletions:true` to actually delete. Separately, `createTable` unconditionally clears the ~3 blank scaffolding rows it seeds (best-effort; `SCAFFOLDING_ROWS_KEPT` warning if the deletion attempt fails); `pruneRecords` under `mirror`+`confirmDeletions` additionally removes any leftover dest-only blank rows as part of the ordinary orphan prune. Attachment deletion under mirror is a deferred follow-up. Schema-orphan deletion (`pruneSchema`) runs synchronously in `mode=apply` AFTER `applyPlan`, consuming `plan.orphans` (`{kind:'table'|'field'|'view', destId, name, tableName}`). Order: (1) views — deleted first (no dependents); (2) fields — batch `deleteFields` WITHOUT force (so a matched field is never cascade-deleted), retry-until-stable loop resolves orphan→orphan deps; a field still blocked after no progress is kept + `SCHEMA_DELETE_BLOCKED`; (3) tables — deleted last (after their content). Gating: dest-only fields + views require `confirmDeletions:true` (same flag as `pruneRecords`); dest-only tables require the new `confirmTableDeletions:true` — `confirmDeletions` alone never drops a whole table. Without each gate the operation is a dry-count: `DELETION_GATED` for fields/views, `TABLE_DELETION_GATED` for tables. Applied counts: `schemaDeleted` (fields+views) + `tablesDeleted`. Per-table `policyOverrides` apply; `overlay`/`preserve` → no schema deletion. Continue-on-failure (`SCHEMA_DELETE_FAILED`). Module: `src/sync/prune-schema.js`. View-section orphans are deferred (M4). Scalar field retypes (`mode=apply`): a matched dest field whose scalar type diverges from source is retyped via `updateFieldConfig`, gated by the new `confirmRetypes` param (boolean, default false); without it the field is kept and a `RETYPE_GATED` warning is emitted (count surfaced in the result). Non-scalar retypes (either side is computed/link/attachment) stay `RETYPE_DEFERRED`. Select retypes converge their choice map on re-sync exactly like created selects (`mergeChoices`). Continue-on-failure: per-field retype errors emit `RETYPE_FAILED` and do not abort apply. Primary-field retypes remain the separate `reconcilePrimary` path. This closes the last schema-drift category sync can fix; records data migration remains `pruneRecords`/Pass 1. Custom field mappings: `fieldMappings` (`{ [tableName]: { sourceFieldName: destFieldName } }`) injects a source field's value into a different writable scalar dest field during record sync — useful when the source is computed (e.g., `autoNumber`/formula) and you want its value preserved in a writable dest field (classic use: inject source autoNumber `Code` into dest text field `InjectID` so a dest formula can reconstruct original identity). Fail-fast pre-flight validation in `apply` (and dry-run in `plan`/`diff` via `fieldMappingErrors` in the machine output) with codes: `FIELD_MAP_TABLE_MISSING`, `FIELD_MAP_SOURCE_MISSING`, `FIELD_MAP_TARGET_MISSING`, `FIELD_MAP_TARGET_COMPUTED` (dest must be writable scalar), `FIELD_MAP_TYPE_INCOMPATIBLE` (array-typed source/dest not supported), `FIELD_MAP_COLLISION` (two sources map to same dest); an invalid mapping aborts `apply` synchronously (`FIELD_MAP_INVALID` error code). Natural-key record matching: `naturalKeys` (`{ [tableName]: fieldName }`) matches dest↔source records by the key field's **value** (field ID resolved per-base by name), growing `idmap.records`. Add-only + ambiguity-safe — never removes/overwrites an existing mapping, never claims an already-mapped dest; ambiguous/duplicate key values are skipped with `NATURAL_KEY_AMBIGUOUS`; a missing key field emits `NATURAL_KEY_FIELD_MISSING`. Runs in `mode=reconcile` (after the existence-prune, snapshots source records + calls matcher, then saves idmap) AND as an automatic pre-pass in `mode=apply` **before Pass 1 and prune** — so records `mirror` no longer deletes real-but-unmapped dest records as false orphans, and re-syncs don't duplicate them. The key field must be **stable across bases** (autoNumber renumbers → bad key; name/email/injected `InjectID` → good key). Single-field keys only; composite keys and value normalization are deferred. Returns a `matched` count in the result. #### packages/mcp-server — Daemon subsystem diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index e3af837..a9cfc9d 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -995,7 +995,15 @@ export async function runRecords({ client, srcSnapshot, destSnapshot, idmap, pol persist(idmap, journal); // Extras axis: prune dest-only orphan records under mirror policy - await pruneRecords({ client, destSnapshot, idmap, policy, policyOverrides, confirmDeletions, limiter, result }); + const truncatedTables = collectTruncatedTableNames(srcSnapshot, destSnapshot); + for (const name of truncatedTables) { + result.warnings.push({ + code: 'RECORDS_TRUNCATED', + message: `Table "${name}": snapshot capped at 1000 rows — records beyond the limit are not synced ` + + `and mirror deletion is skipped for this table (>1000-row pagination is a follow-up).`, + }); + } + await pruneRecords({ client, destSnapshot, idmap, policy, policyOverrides, confirmDeletions, limiter, result, truncatedTables }); persist(idmap, journal); return result; @@ -1098,6 +1106,19 @@ export async function applyRecords({ client, sourceBaseId, destBaseId, planId, r return runRecords({ client, srcSnapshot, destSnapshot, idmap, policy, policyOverrides, confirmDeletions, fieldMappings, naturalKeys, limiter, journal, persist, result }); } +// ────────────────────────────────────────────────────────────────────────────── +// collectTruncatedTableNames — set-builder for pruneRecords truncation guard +// ────────────────────────────────────────────────────────────────────────────── + +/** Table names whose snapshot hit the 1000-row read cap on EITHER side (possibly truncated). */ +export function collectTruncatedTableNames(srcSnapshot, destSnapshot) { + const names = new Set(); + for (const t of [...(srcSnapshot?.tables || []), ...(destSnapshot?.tables || [])]) { + if ((t.records || []).length >= 1000) names.add(t.name); + } + return names; +} + // ────────────────────────────────────────────────────────────────────────────── // pruneRecords — extras axis: delete dest-only orphan records under mirror (gated) // ────────────────────────────────────────────────────────────────────────────── @@ -1122,6 +1143,7 @@ export async function applyRecords({ client, sourceBaseId, destBaseId, planId, r * @param {boolean} opts.confirmDeletions - if false, gate deletions (push DELETION_GATED warning) * @param {object} opts.limiter - rate limiter { run: (fn) => fn() } * @param {object} opts.result - result accumulator (mutated: result.deleted, result.warnings) + * @param {Set} [opts.truncatedTables] - table names whose snapshot hit the 1000-row cap (skipped to avoid false-orphan deletion) * @returns {Promise} */ export async function pruneRecords({ client, destSnapshot, idmap, policy, policyOverrides, confirmDeletions, limiter, result, truncatedTables = new Set() }) { diff --git a/packages/mcp-server/test/sync/test-records-policy.test.js b/packages/mcp-server/test/sync/test-records-policy.test.js index f91c1a7..3d876d2 100644 --- a/packages/mcp-server/test/sync/test-records-policy.test.js +++ b/packages/mcp-server/test/sync/test-records-policy.test.js @@ -7,7 +7,7 @@ */ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { applyRecordsPass1, pruneRecords } from '../../src/sync/records.js'; +import { applyRecordsPass1, pruneRecords, collectTruncatedTableNames } from '../../src/sync/records.js'; export function fakeClient() { const calls = { create: [], update: [] }; @@ -284,3 +284,16 @@ describe('pruneRecords (extras axis)', () => { assert.equal(result.deleted, 75, 'all 75 orphans counted as deleted'); }); }); + +// ── Task 2: collectTruncatedTableNames helper ──────────────────────────────── + +describe('collectTruncatedTableNames', () => { + it('names any table at the 1000 cap on either side', () => { + const big = (name) => ({ id: 'tbl' + name, name, records: Array.from({ length: 1000 }, (_, i) => ({ id: 'r' + i })) }); + const small = (name) => ({ id: 'tbl' + name, name, records: [{ id: 'r0' }] }); + const src = { tables: [big('A'), small('B')] }; + const dest = { tables: [small('A'), big('C')] }; + const set = collectTruncatedTableNames(src, dest); + assert.deepEqual([...set].sort(), ['A', 'C']); + }); +}); From 27d468215fe4d6e76a7b3b86fd972fb0817573d5 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sun, 21 Jun 2026 19:09:21 +0300 Subject: [PATCH 139/246] feat(sync): capture view-section id in snapshot; compare sections by name+viewNames (ignore id) Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/compare.js | 5 +- packages/mcp-server/src/sync/snapshot.js | 5 +- .../mcp-server/test/sync/test-compare.test.js | 54 +++++++++++++++++++ .../test/sync/test-snapshot.test.js | 2 +- 4 files changed, 61 insertions(+), 5 deletions(-) diff --git a/packages/mcp-server/src/sync/compare.js b/packages/mcp-server/src/sync/compare.js index 09a469b..a75ac75 100644 --- a/packages/mcp-server/src/sync/compare.js +++ b/packages/mcp-server/src/sync/compare.js @@ -614,10 +614,11 @@ export function compareTable(srcTable, destTable, idmap, srcGlobalFldNames, dest }); } - // 6. Sections comparison (not-synced). + // 6. Sections comparison (not-synced) — compare by name+viewNames; ignore the vsc… id. + const stripId = (arr) => (arr || []).map(({ name, viewNames }) => ({ name, viewNames })); const srcSections = srcTable.sections || []; const destSections = destTable.sections || []; - if (JSON.stringify(srcSections) !== JSON.stringify(destSections)) { + if (JSON.stringify(stripId(srcSections)) !== JSON.stringify(stripId(destSections))) { entries.push({ scope: 'table', key: 'sections', diff --git a/packages/mcp-server/src/sync/snapshot.js b/packages/mcp-server/src/sync/snapshot.js index 47aa464..7410f84 100644 --- a/packages/mcp-server/src/sync/snapshot.js +++ b/packages/mcp-server/src/sync/snapshot.js @@ -35,9 +35,10 @@ export function normalizeSchema(rawData) { })); const viewNameById = new Map(views.map(v => [v.id, v.name])); const sectionsObj = t.viewSectionsById || t.viewSections || {}; - const sections = Object.values(sectionsObj).map(s => ({ + const sections = Object.entries(sectionsObj).map(([id, s]) => ({ + id: s.id || id, name: s.name, - viewNames: (s.viewOrder || []).map(id => viewNameById.get(id)).filter(Boolean), + viewNames: (s.viewOrder || []).map((vid) => viewNameById.get(vid)).filter(Boolean), })); return { id: t.id, diff --git a/packages/mcp-server/test/sync/test-compare.test.js b/packages/mcp-server/test/sync/test-compare.test.js index e833ae7..e42ac89 100644 --- a/packages/mcp-server/test/sync/test-compare.test.js +++ b/packages/mcp-server/test/sync/test-compare.test.js @@ -396,6 +396,60 @@ describe('compare', () => { }); }); +// ── Section id-strip tests (Task 1) ───────────────────────────────────────────── + +describe('compare — sections id-strip', () => { + test('same name+viewNames but different vsc ids → NO sections entry (id stripped from comparison)', () => { + // src section has id 'vscSRC', dest section has id 'vscDEST', but name+viewNames identical. + // compare() must NOT emit a key:'sections' entry. + const srcTable = { + ...mkTable('st1', 'Alpha', [fld('sf1', 'Name', 'text')]), + sections: [{ id: 'vscSRC', name: 'Group A', viewNames: ['Grid'] }], + }; + const destTable = { + ...mkTable('dt1', 'Alpha', [fld('df1', 'Name', 'text')]), + sections: [{ id: 'vscDEST', name: 'Group A', viewNames: ['Grid'] }], + }; + const srcSnap = mkSnap('srcBase', [srcTable]); + const destSnap = mkSnap('destBase', [destTable]); + const idmap = { + tables: { st1: 'dt1' }, + fields: { sf1: { destFld: 'df1', choices: {} } }, + views: {}, + }; + const result = compare(srcSnap, destSnap, idmap); + const sectionsEntry = result.tables.flatMap((t) => t.entries).find((e) => e.key === 'sections'); + assert.ok(!sectionsEntry, 'same name+viewNames with different ids must NOT produce a sections entry'); + assert.equal(result.converged, true); + assert.equal(result.identical, true); + }); + + test('different section names → sections entry IS emitted (not-synced)', () => { + const srcTable = { + ...mkTable('st1', 'Alpha', [fld('sf1', 'Name', 'text')]), + sections: [{ id: 'vscSRC', name: 'Group A', viewNames: ['Grid'] }], + }; + const destTable = { + ...mkTable('dt1', 'Alpha', [fld('df1', 'Name', 'text')]), + sections: [{ id: 'vscDEST', name: 'Group B', viewNames: ['Grid'] }], + }; + const srcSnap = mkSnap('srcBase', [srcTable]); + const destSnap = mkSnap('destBase', [destTable]); + const idmap = { + tables: { st1: 'dt1' }, + fields: { sf1: { destFld: 'df1', choices: {} } }, + views: {}, + }; + const result = compare(srcSnap, destSnap, idmap); + const sectionsEntry = result.tables.flatMap((t) => t.entries).find((e) => e.key === 'sections'); + assert.ok(sectionsEntry, 'different section names must produce a sections entry'); + assert.equal(sectionsEntry.class, 'not-synced'); + // source/dest should be the FULL sections (with id) + assert.deepEqual(sectionsEntry.source, srcTable.sections); + assert.deepEqual(sectionsEntry.dest, destTable.sections); + }); +}); + // ── Bug fix tests: cross-base computed/link ref canonicalization ────────────── describe('compareFields — cross-base ref canonicalization', () => { diff --git a/packages/mcp-server/test/sync/test-snapshot.test.js b/packages/mcp-server/test/sync/test-snapshot.test.js index b09a718..c7d3227 100644 --- a/packages/mcp-server/test/sync/test-snapshot.test.js +++ b/packages/mcp-server/test/sync/test-snapshot.test.js @@ -48,7 +48,7 @@ describe('snapshot sections', () => { viewSectionsById: { vsc1: { id: 'vsc1', name: 'Sales Views', viewOrder: ['viwY', 'viwX'] } }, }] } }; const snap = normalizeSchema(raw); - assert.deepEqual(snap.tables[0].sections, [{ name: 'Sales Views', viewNames: ['Kanban', 'Grid'] }]); + assert.deepEqual(snap.tables[0].sections, [{ id: 'vsc1', name: 'Sales Views', viewNames: ['Kanban', 'Grid'] }]); }); it('normalizeSchema sections defaults to [] when absent', () => { const raw = { data: { tableSchemas: [{ id: 'tblA', name: 'T', columns: [], views: [] }] } }; From 81d59a15f8bc3b5d3f77a7f7c3dd29d44f0e4417 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sun, 21 Jun 2026 19:16:44 +0300 Subject: [PATCH 140/246] feat(sync): delete dest-only view-section orphans under mirror (pruneSchema step 0) Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 2 +- packages/mcp-server/src/sync/diff.js | 13 ++++ packages/mcp-server/src/sync/prune-schema.js | 9 ++- packages/mcp-server/src/sync/report.js | 2 +- .../test/sync/test-diff-engine.test.js | 76 +++++++++++++++++++ .../test/sync/test-prune-schema.test.js | 55 +++++++++++++- 6 files changed, 153 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 247b451..5ea7ee2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,7 +65,7 @@ Key files: - `dev-tools/` — reverse-engineering helpers (capture.js, analyze.js, debug-*.js), **gitignored** — kept locally for future reference when Airtable changes their internal API - `src/daemon/` — daemon subsystem (see daemon subsection below) - `src/safe-write.js` — atomic JSON write helper (used by lockfile, token, and settings) -- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: diff | plan | apply | status | reconcile`) in the `sync` category. Out of scope: non-scalar retypes (computed/link/attachment on either side → `RETYPE_DEFERRED`), deletions (M4); scalar retypes are now in scope (see below). `mode=diff` (read-only): comprehensive schema comparison via pure `src/sync/compare.js` — every difference classified as **drift** (sync enforces: table/field/view existence, field type/options/choices, view filters/sorts/groups content, column visibility, colors/cover/frozen/rowHeight, primary, form), **best-effort** (sync applies but doesn't guarantee: field/view/column order, sort/group clause order), or **not-synced** (view sections — now captured by snapshot). Verdicts: `identical` (zero diffs) or `converged` (zero drift; best-effort/not-synced may remain). Persists full diff to `~/.airtable-user-mcp/sync/__/diff-.json`; returns a token-budgeted DIGEST (verdicts + class counts + per-table rollup + capped driftSample ≤25). Drill in with `detail="
"` (params: `detail`, `diffId`, `offset`, `limit`). `mode=plan` now emits a curatable **changeset**: each action carries a stable name-based `changeId` (`|
|`), a `class`, and an `apply:true` flag; digest includes a `sample` + `editHint`. New `direction` param (`to-dest` default | `to-source`) targets either base. View sync: `snapshot` also captures collaborative views + live config + **view sections** (not-synced class); `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported); accepts a `skip:[changeId]` list (and honors `apply:false` on changeset entries) — applies everything except removed changes (journal-safe; skipped dependencies degrade to UNRESOLVABLE_REF). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe by filename+size); then restores record-referencing view filters stripped during view sync. Throttle handling = per-request 429 backoff in the auth queue (the per-call limiter does NOT gate the internal per-row POSTs inside createRecords/updateRecords/addLinkItems); resumable (records journal, per-chunk persist of ~50 creates); continue-on-failure (per-record errors are warnings). Truncation safety: a table whose snapshot hits the 1000-row cap on either base is added to `truncatedTables` by `collectTruncatedTableNames` and skipped by `pruneRecords` under mirror (`RECORDS_TRUNCATED_PRUNE_SKIPPED`) so a record beyond the snapshot window is never deleted as a false orphan; each truncated table also emits `RECORDS_TRUNCATED` on the result; >1000-row read pagination remains a capture-gated follow-up. `mode=status` polls a background job by planId — returns running/done/failed + live records-mapped count. `mode=reconcile` **prunes** dead record-map entries (existence-prune; per-table natural-key re-match now matches by key value — see natural-key matching below; reconcile does NOT de-duplicate records). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json, diff-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile), `compare.js` (pure classified compare, no I/O); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. Reconciliation policy (`src/sync/policy.js`): `mode=apply` accepts a `policy` param (`mirror` | `overlay` | `preserve`, default `overlay`) that controls two axes — **extras** (what to do with dest-only records) and **conflicts** (whose value wins when both bases have the record). Presets: `mirror` = {remove, source-wins} (make dest identical to source); `overlay` = {keep, source-wins} (today's default: keep dest-only rows, source updates win); `preserve` = {keep, dest-wins} (keep dest-only rows and never overwrite dest edits). `policyOverrides` (`{ [tableName]: preset }`) overrides the global preset per table. `confirmDeletions` (boolean, default false) is a safety gate for `mirror`: without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count but deletes nothing; set `confirmDeletions:true` to actually delete. Separately, `createTable` unconditionally clears the ~3 blank scaffolding rows it seeds (best-effort; `SCAFFOLDING_ROWS_KEPT` warning if the deletion attempt fails); `pruneRecords` under `mirror`+`confirmDeletions` additionally removes any leftover dest-only blank rows as part of the ordinary orphan prune. Attachment deletion under mirror is a deferred follow-up. Schema-orphan deletion (`pruneSchema`) runs synchronously in `mode=apply` AFTER `applyPlan`, consuming `plan.orphans` (`{kind:'table'|'field'|'view', destId, name, tableName}`). Order: (1) views — deleted first (no dependents); (2) fields — batch `deleteFields` WITHOUT force (so a matched field is never cascade-deleted), retry-until-stable loop resolves orphan→orphan deps; a field still blocked after no progress is kept + `SCHEMA_DELETE_BLOCKED`; (3) tables — deleted last (after their content). Gating: dest-only fields + views require `confirmDeletions:true` (same flag as `pruneRecords`); dest-only tables require the new `confirmTableDeletions:true` — `confirmDeletions` alone never drops a whole table. Without each gate the operation is a dry-count: `DELETION_GATED` for fields/views, `TABLE_DELETION_GATED` for tables. Applied counts: `schemaDeleted` (fields+views) + `tablesDeleted`. Per-table `policyOverrides` apply; `overlay`/`preserve` → no schema deletion. Continue-on-failure (`SCHEMA_DELETE_FAILED`). Module: `src/sync/prune-schema.js`. View-section orphans are deferred (M4). Scalar field retypes (`mode=apply`): a matched dest field whose scalar type diverges from source is retyped via `updateFieldConfig`, gated by the new `confirmRetypes` param (boolean, default false); without it the field is kept and a `RETYPE_GATED` warning is emitted (count surfaced in the result). Non-scalar retypes (either side is computed/link/attachment) stay `RETYPE_DEFERRED`. Select retypes converge their choice map on re-sync exactly like created selects (`mergeChoices`). Continue-on-failure: per-field retype errors emit `RETYPE_FAILED` and do not abort apply. Primary-field retypes remain the separate `reconcilePrimary` path. This closes the last schema-drift category sync can fix; records data migration remains `pruneRecords`/Pass 1. Custom field mappings: `fieldMappings` (`{ [tableName]: { sourceFieldName: destFieldName } }`) injects a source field's value into a different writable scalar dest field during record sync — useful when the source is computed (e.g., `autoNumber`/formula) and you want its value preserved in a writable dest field (classic use: inject source autoNumber `Code` into dest text field `InjectID` so a dest formula can reconstruct original identity). Fail-fast pre-flight validation in `apply` (and dry-run in `plan`/`diff` via `fieldMappingErrors` in the machine output) with codes: `FIELD_MAP_TABLE_MISSING`, `FIELD_MAP_SOURCE_MISSING`, `FIELD_MAP_TARGET_MISSING`, `FIELD_MAP_TARGET_COMPUTED` (dest must be writable scalar), `FIELD_MAP_TYPE_INCOMPATIBLE` (array-typed source/dest not supported), `FIELD_MAP_COLLISION` (two sources map to same dest); an invalid mapping aborts `apply` synchronously (`FIELD_MAP_INVALID` error code). Natural-key record matching: `naturalKeys` (`{ [tableName]: fieldName }`) matches dest↔source records by the key field's **value** (field ID resolved per-base by name), growing `idmap.records`. Add-only + ambiguity-safe — never removes/overwrites an existing mapping, never claims an already-mapped dest; ambiguous/duplicate key values are skipped with `NATURAL_KEY_AMBIGUOUS`; a missing key field emits `NATURAL_KEY_FIELD_MISSING`. Runs in `mode=reconcile` (after the existence-prune, snapshots source records + calls matcher, then saves idmap) AND as an automatic pre-pass in `mode=apply` **before Pass 1 and prune** — so records `mirror` no longer deletes real-but-unmapped dest records as false orphans, and re-syncs don't duplicate them. The key field must be **stable across bases** (autoNumber renumbers → bad key; name/email/injected `InjectID` → good key). Single-field keys only; composite keys and value normalization are deferred. Returns a `matched` count in the result. +- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: diff | plan | apply | status | reconcile`) in the `sync` category. Out of scope: non-scalar retypes (computed/link/attachment on either side → `RETYPE_DEFERRED`), deletions (M4); scalar retypes are now in scope (see below). `mode=diff` (read-only): comprehensive schema comparison via pure `src/sync/compare.js` — every difference classified as **drift** (sync enforces: table/field/view existence, field type/options/choices, view filters/sorts/groups content, column visibility, colors/cover/frozen/rowHeight, primary, form), **best-effort** (sync applies but doesn't guarantee: field/view/column order, sort/group clause order), or **not-synced** (view sections — now captured by snapshot). Verdicts: `identical` (zero diffs) or `converged` (zero drift; best-effort/not-synced may remain). Persists full diff to `~/.airtable-user-mcp/sync/__/diff-.json`; returns a token-budgeted DIGEST (verdicts + class counts + per-table rollup + capped driftSample ≤25). Drill in with `detail="
"` (params: `detail`, `diffId`, `offset`, `limit`). `mode=plan` now emits a curatable **changeset**: each action carries a stable name-based `changeId` (`|
|`), a `class`, and an `apply:true` flag; digest includes a `sample` + `editHint`. New `direction` param (`to-dest` default | `to-source`) targets either base. View sync: `snapshot` also captures collaborative views + live config + **view sections** (not-synced class); `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported); accepts a `skip:[changeId]` list (and honors `apply:false` on changeset entries) — applies everything except removed changes (journal-safe; skipped dependencies degrade to UNRESOLVABLE_REF). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe by filename+size); then restores record-referencing view filters stripped during view sync. Throttle handling = per-request 429 backoff in the auth queue (the per-call limiter does NOT gate the internal per-row POSTs inside createRecords/updateRecords/addLinkItems); resumable (records journal, per-chunk persist of ~50 creates); continue-on-failure (per-record errors are warnings). Truncation safety: a table whose snapshot hits the 1000-row cap on either base is added to `truncatedTables` by `collectTruncatedTableNames` and skipped by `pruneRecords` under mirror (`RECORDS_TRUNCATED_PRUNE_SKIPPED`) so a record beyond the snapshot window is never deleted as a false orphan; each truncated table also emits `RECORDS_TRUNCATED` on the result; >1000-row read pagination remains a capture-gated follow-up. `mode=status` polls a background job by planId — returns running/done/failed + live records-mapped count. `mode=reconcile` **prunes** dead record-map entries (existence-prune; per-table natural-key re-match now matches by key value — see natural-key matching below; reconcile does NOT de-duplicate records). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json, diff-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile), `compare.js` (pure classified compare, no I/O); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. Reconciliation policy (`src/sync/policy.js`): `mode=apply` accepts a `policy` param (`mirror` | `overlay` | `preserve`, default `overlay`) that controls two axes — **extras** (what to do with dest-only records) and **conflicts** (whose value wins when both bases have the record). Presets: `mirror` = {remove, source-wins} (make dest identical to source); `overlay` = {keep, source-wins} (today's default: keep dest-only rows, source updates win); `preserve` = {keep, dest-wins} (keep dest-only rows and never overwrite dest edits). `policyOverrides` (`{ [tableName]: preset }`) overrides the global preset per table. `confirmDeletions` (boolean, default false) is a safety gate for `mirror`: without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count but deletes nothing; set `confirmDeletions:true` to actually delete. Separately, `createTable` unconditionally clears the ~3 blank scaffolding rows it seeds (best-effort; `SCAFFOLDING_ROWS_KEPT` warning if the deletion attempt fails); `pruneRecords` under `mirror`+`confirmDeletions` additionally removes any leftover dest-only blank rows as part of the ordinary orphan prune. Attachment deletion under mirror is a deferred follow-up. Schema-orphan deletion (`pruneSchema`) runs synchronously in `mode=apply` AFTER `applyPlan`, consuming `plan.orphans` (`{kind:'table'|'field'|'view', destId, name, tableName}`). Order: (1) views — deleted first (no dependents); (2) fields — batch `deleteFields` WITHOUT force (so a matched field is never cascade-deleted), retry-until-stable loop resolves orphan→orphan deps; a field still blocked after no progress is kept + `SCHEMA_DELETE_BLOCKED`; (3) tables — deleted last (after their content). Gating: dest-only fields + views require `confirmDeletions:true` (same flag as `pruneRecords`); dest-only tables require the new `confirmTableDeletions:true` — `confirmDeletions` alone never drops a whole table. Without each gate the operation is a dry-count: `DELETION_GATED` for fields/views, `TABLE_DELETION_GATED` for tables. Applied counts: `schemaDeleted` (fields+views) + `tablesDeleted`. Per-table `policyOverrides` apply; `overlay`/`preserve` → no schema deletion. Continue-on-failure (`SCHEMA_DELETE_FAILED`). Module: `src/sync/prune-schema.js`. View-section orphans (dest-only sidebar sections in a matched table) are now deleted by `pruneSchema` as **step 0** (before views, so Airtable auto-promotes their contained views to top-level before view orphan deletion); gated by `confirmDeletions` (same flag as fields/views); `deleteViewSection` is the client method used. Attachment-strip under mirror remains a deferred follow-up. Scalar field retypes (`mode=apply`): a matched dest field whose scalar type diverges from source is retyped via `updateFieldConfig`, gated by the new `confirmRetypes` param (boolean, default false); without it the field is kept and a `RETYPE_GATED` warning is emitted (count surfaced in the result). Non-scalar retypes (either side is computed/link/attachment) stay `RETYPE_DEFERRED`. Select retypes converge their choice map on re-sync exactly like created selects (`mergeChoices`). Continue-on-failure: per-field retype errors emit `RETYPE_FAILED` and do not abort apply. Primary-field retypes remain the separate `reconcilePrimary` path. This closes the last schema-drift category sync can fix; records data migration remains `pruneRecords`/Pass 1. Custom field mappings: `fieldMappings` (`{ [tableName]: { sourceFieldName: destFieldName } }`) injects a source field's value into a different writable scalar dest field during record sync — useful when the source is computed (e.g., `autoNumber`/formula) and you want its value preserved in a writable dest field (classic use: inject source autoNumber `Code` into dest text field `InjectID` so a dest formula can reconstruct original identity). Fail-fast pre-flight validation in `apply` (and dry-run in `plan`/`diff` via `fieldMappingErrors` in the machine output) with codes: `FIELD_MAP_TABLE_MISSING`, `FIELD_MAP_SOURCE_MISSING`, `FIELD_MAP_TARGET_MISSING`, `FIELD_MAP_TARGET_COMPUTED` (dest must be writable scalar), `FIELD_MAP_TYPE_INCOMPATIBLE` (array-typed source/dest not supported), `FIELD_MAP_COLLISION` (two sources map to same dest); an invalid mapping aborts `apply` synchronously (`FIELD_MAP_INVALID` error code). Natural-key record matching: `naturalKeys` (`{ [tableName]: fieldName }`) matches dest↔source records by the key field's **value** (field ID resolved per-base by name), growing `idmap.records`. Add-only + ambiguity-safe — never removes/overwrites an existing mapping, never claims an already-mapped dest; ambiguous/duplicate key values are skipped with `NATURAL_KEY_AMBIGUOUS`; a missing key field emits `NATURAL_KEY_FIELD_MISSING`. Runs in `mode=reconcile` (after the existence-prune, snapshots source records + calls matcher, then saves idmap) AND as an automatic pre-pass in `mode=apply` **before Pass 1 and prune** — so records `mirror` no longer deletes real-but-unmapped dest records as false orphans, and re-syncs don't duplicate them. The key field must be **stable across bases** (autoNumber renumbers → bad key; name/email/injected `InjectID` → good key). Single-field keys only; composite keys and value normalization are deferred. Returns a `matched` count in the result. #### packages/mcp-server — Daemon subsystem diff --git a/packages/mcp-server/src/sync/diff.js b/packages/mcp-server/src/sync/diff.js index 8c69778..3de8a76 100644 --- a/packages/mcp-server/src/sync/diff.js +++ b/packages/mcp-server/src/sync/diff.js @@ -276,6 +276,19 @@ export function computePlan(srcSnap, destSnap, idmap) { for (const dv of collabViews(dt)) if (!srcViewNames.has(dv.name)) orphans.push({ kind: 'view', destId: dv.id, name: dv.name, tableName: dt.name }); } + // Section orphans: dest-only sidebar sections in a matched table (by name). + for (const dt of destSnap.tables) { + const srcTableId = srcTableByDestId.get(dt.id); + if (!srcTableId) continue; // whole table is already a table orphan + const srcTable = srcTablesById.get(srcTableId); + const srcSectionNames = new Set((srcTable.sections || []).map((s) => s.name)); + for (const ds of (dt.sections || [])) { + if (ds.id && !srcSectionNames.has(ds.name)) { + orphans.push({ kind: 'section', destId: ds.id, name: ds.name, tableName: dt.name }); + } + } + } + // ── Annotate every action with a stable changeId + class + apply ───────── // Build resolution maps from srcSnap once (name-based, stable across bases). const srcTableNameById = new Map(srcSnap.tables.map((t) => [t.id, t.name])); diff --git a/packages/mcp-server/src/sync/prune-schema.js b/packages/mcp-server/src/sync/prune-schema.js index 0d2825e..fe63bbe 100644 --- a/packages/mcp-server/src/sync/prune-schema.js +++ b/packages/mcp-server/src/sync/prune-schema.js @@ -14,6 +14,13 @@ export async function pruneSchema({ client, destAppId, plan, policy, policyOverr let gatedFieldsViews = 0; let gatedTables = 0; + // 0. Sections (delete first — Airtable auto-promotes contained views to top-level; matched-table orphans) + for (const s of orphans.filter((o) => o.kind === 'section' && removes(o.tableName))) { + if (!confirmDeletions) { gatedFieldsViews++; continue; } + try { await client.deleteViewSection(destAppId, s.destId); result.schemaDeleted++; } + catch (e) { result.warnings.push({ code: 'SCHEMA_DELETE_FAILED', message: `section "${s.name}" (${s.tableName}): ${e.message ?? e}` }); } + } + // 1. Views (no dependents) — matched-table orphans for (const v of orphans.filter((o) => o.kind === 'view' && removes(o.tableName))) { if (!confirmDeletions) { gatedFieldsViews++; continue; } @@ -47,6 +54,6 @@ export async function pruneSchema({ client, destAppId, plan, policy, policyOverr catch (e) { result.warnings.push({ code: 'SCHEMA_DELETE_FAILED', message: `table "${t.name}": ${e.message ?? e}` }); } } - if (gatedFieldsViews > 0) result.warnings.push({ code: 'DELETION_GATED', message: `${gatedFieldsViews} dest-only field(s)/view(s) would be deleted under mirror — set confirmDeletions:true` }); + if (gatedFieldsViews > 0) result.warnings.push({ code: 'DELETION_GATED', message: `${gatedFieldsViews} dest-only field(s)/view(s)/section(s) would be deleted under mirror — set confirmDeletions:true` }); if (gatedTables > 0) result.warnings.push({ code: 'TABLE_DELETION_GATED', message: `${gatedTables} dest-only table(s) would be deleted under mirror — set confirmTableDeletions:true` }); } diff --git a/packages/mcp-server/src/sync/report.js b/packages/mcp-server/src/sync/report.js index 785ccb0..a2cd749 100644 --- a/packages/mcp-server/src/sync/report.js +++ b/packages/mcp-server/src/sync/report.js @@ -75,7 +75,7 @@ export function renderApplyResult(result) { lines.push(` retyped: ${result.retyped} (scalar field type changes applied)`); } if (result.schemaDeleted != null && result.schemaDeleted > 0) { - lines.push(` schemaDeleted: ${result.schemaDeleted} (dest-only fields/views removed)`); + lines.push(` schemaDeleted: ${result.schemaDeleted} (dest-only fields/views/sections removed)`); } if (result.tablesDeleted != null && result.tablesDeleted > 0) { lines.push(` tablesDeleted: ${result.tablesDeleted} (dest-only tables removed)`); diff --git a/packages/mcp-server/test/sync/test-diff-engine.test.js b/packages/mcp-server/test/sync/test-diff-engine.test.js index 78828ea..c24c9e3 100644 --- a/packages/mcp-server/test/sync/test-diff-engine.test.js +++ b/packages/mcp-server/test/sync/test-diff-engine.test.js @@ -12,6 +12,7 @@ import { tmpdir } from 'node:os'; import { diff, diffDetail } from '../../src/sync/index.js'; import { syncDir } from '../../src/sync/idmap.js'; +import { computePlan } from '../../src/sync/diff.js'; // ── fixtures ───────────────────────────────────────────────────────────────── @@ -119,3 +120,78 @@ describe('sync index.diff / diffDetail', () => { assert.ok(result.machine.entries.length >= 1, 'should resolve to the latest diff and return entries'); }); }); + +// ── Section-orphan diff tests ───────────────────────────────────────────────── + +describe('computePlan section orphans', () => { + // Minimal idmap: src table tblS1 ↔ dest table tblD1 + const idmap = { tables: { tblS1: 'tblD1' }, fields: {} }; + + const srcSnap = { + baseId: 'appSrc', + tables: [{ + id: 'tblS1', name: 'T', primaryFieldId: 'fldS1', + fields: [{ id: 'fldS1', name: 'Name', type: 'text', typeOptions: null, description: null, isComputed: false }], + views: [], + sections: [{ id: 'vscSrc1', name: 'Active', viewNames: [] }], + }], + }; + + it('dest section absent from source by name → plan.orphans includes kind:section entry', () => { + const destSnap = { + baseId: 'appDest', + tables: [{ + id: 'tblD1', name: 'T', primaryFieldId: 'fldD1', + fields: [{ id: 'fldD1', name: 'Name', type: 'text', typeOptions: null, description: null, isComputed: false }], + views: [], + // Ghost section exists in dest but NOT in source + sections: [ + { id: 'vscX', name: 'Ghost', viewNames: [] }, + { id: 'vscSrc1Dest', name: 'Active', viewNames: [] }, // present in source by name + ], + }], + }; + + const result = computePlan(srcSnap, destSnap, idmap); + const sectionOrphans = result.orphans.filter((o) => o.kind === 'section'); + assert.equal(sectionOrphans.length, 1, 'exactly one section orphan (Ghost)'); + assert.deepEqual(sectionOrphans[0], { kind: 'section', destId: 'vscX', name: 'Ghost', tableName: 'T' }); + }); + + it('dest section present in source by name → no section orphan emitted', () => { + const destSnap = { + baseId: 'appDest', + tables: [{ + id: 'tblD1', name: 'T', primaryFieldId: 'fldD1', + fields: [{ id: 'fldD1', name: 'Name', type: 'text', typeOptions: null, description: null, isComputed: false }], + views: [], + // Both sections present in source by name → no orphan + sections: [ + { id: 'vscSrc1Dest', name: 'Active', viewNames: [] }, + ], + }], + }; + + const result = computePlan(srcSnap, destSnap, idmap); + const sectionOrphans = result.orphans.filter((o) => o.kind === 'section'); + assert.equal(sectionOrphans.length, 0, 'no section orphans when all dest sections exist in source by name'); + }); + + it('section without an id is not emitted as orphan', () => { + const destSnap = { + baseId: 'appDest', + tables: [{ + id: 'tblD1', name: 'T', primaryFieldId: 'fldD1', + fields: [{ id: 'fldD1', name: 'Name', type: 'text', typeOptions: null, description: null, isComputed: false }], + views: [], + sections: [ + { name: 'NoId', viewNames: [] }, // id missing → should be skipped + ], + }], + }; + + const result = computePlan(srcSnap, destSnap, idmap); + const sectionOrphans = result.orphans.filter((o) => o.kind === 'section'); + assert.equal(sectionOrphans.length, 0, 'section without id should not be emitted as orphan'); + }); +}); diff --git a/packages/mcp-server/test/sync/test-prune-schema.test.js b/packages/mcp-server/test/sync/test-prune-schema.test.js index d3d50b9..c563d33 100644 --- a/packages/mcp-server/test/sync/test-prune-schema.test.js +++ b/packages/mcp-server/test/sync/test-prune-schema.test.js @@ -5,13 +5,14 @@ import { pruneSchema } from '../../src/sync/prune-schema.js'; // Mock client. deleteFields succeeds for a field UNLESS some still-present field depends on it // (modelled by `deps`: dependentFieldId -> [fieldIds it depends on]). force is asserted false. function mockClient({ deps = {} } = {}) { - const calls = { views: [], tables: [], fieldBatches: [] }; + const calls = { views: [], tables: [], fieldBatches: [], sections: [] }; const present = new Set(); // fields still alive (seeded per test via opts) const api = { calls, _seed(ids) { ids.forEach((i) => present.add(i)); }, async deleteView(appId, viewId) { calls.views.push(viewId); }, async deleteTable(appId, tableId, expectedName) { calls.tables.push({ tableId, expectedName }); }, + async deleteViewSection(appId, sectionId) { calls.sections.push({ appId, sectionId }); }, async deleteFields(appId, fields, opts) { assert.equal(opts.force, false, 'must never force-delete fields'); calls.fieldBatches.push(fields.map((f) => f.fieldId)); @@ -118,4 +119,56 @@ describe('pruneSchema', () => { assert.equal(result.schemaDeleted, 0); assert.ok(result.warnings.some((x) => x.code === 'SCHEMA_DELETE_BLOCKED')); }); + + // ── Section orphan tests ────────────────────────────────────────────────── + + it('mirror + confirmDeletions deletes orphan section (step 0)', async () => { + const client = mockClient(); + const result = baseResult(); + await pruneSchema({ client, destAppId: 'appDest', plan: plan([ + { kind: 'section', destId: 'vscX', name: 'Ghost', tableName: 'T' }, + ]), policy: 'mirror', confirmDeletions: true, confirmTableDeletions: false, result }); + assert.deepEqual(client.calls.sections, [{ appId: 'appDest', sectionId: 'vscX' }]); + assert.equal(result.schemaDeleted, 1); + }); + + it('mirror WITHOUT confirmDeletions: section not deleted + DELETION_GATED', async () => { + const client = mockClient(); + const result = baseResult(); + await pruneSchema({ client, destAppId: 'appDest', plan: plan([ + { kind: 'section', destId: 'vscX', name: 'Ghost', tableName: 'T' }, + ]), policy: 'mirror', confirmDeletions: false, confirmTableDeletions: false, result }); + assert.equal(client.calls.sections.length, 0, 'deleteViewSection must not be called'); + assert.equal(result.schemaDeleted, 0); + const w = result.warnings.find((x) => x.code === 'DELETION_GATED'); + assert.ok(w, 'DELETION_GATED warning should be present'); + }); + + it('overlay policy: section orphan never deleted', async () => { + const client = mockClient(); + const result = baseResult(); + await pruneSchema({ client, destAppId: 'appDest', plan: plan([ + { kind: 'section', destId: 'vscX', name: 'Ghost', tableName: 'T' }, + ]), policy: 'overlay', confirmDeletions: true, confirmTableDeletions: false, result }); + assert.equal(client.calls.sections.length, 0, 'deleteViewSection must not be called under overlay'); + assert.equal(result.schemaDeleted, 0); + }); + + it('deleteViewSection throws → SCHEMA_DELETE_FAILED + loop continues for second section', async () => { + let callCount = 0; + const client = mockClient(); + client.deleteViewSection = async (appId, sectionId) => { + callCount++; + if (sectionId === 'vscBad') throw new Error('API error'); + client.calls.sections.push({ appId, sectionId }); + }; + const result = baseResult(); + await pruneSchema({ client, destAppId: 'appDest', plan: plan([ + { kind: 'section', destId: 'vscBad', name: 'Fails', tableName: 'T' }, + { kind: 'section', destId: 'vscGood', name: 'Succeeds', tableName: 'T' }, + ]), policy: 'mirror', confirmDeletions: true, confirmTableDeletions: false, result }); + assert.equal(callCount, 2, 'both deleteViewSection calls were attempted'); + assert.equal(result.schemaDeleted, 1, 'only the successful one incremented schemaDeleted'); + assert.ok(result.warnings.some((x) => x.code === 'SCHEMA_DELETE_FAILED' && x.message.includes('Fails'))); + }); }); From a34996e2045c3139e75acbc82d0022cf2fe49a6c Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sun, 21 Jun 2026 19:21:35 +0300 Subject: [PATCH 141/246] docs(sync): prune-schema header comment lists sections + deletion order Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/sync/prune-schema.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/mcp-server/src/sync/prune-schema.js b/packages/mcp-server/src/sync/prune-schema.js index fe63bbe..8ace819 100644 --- a/packages/mcp-server/src/sync/prune-schema.js +++ b/packages/mcp-server/src/sync/prune-schema.js @@ -1,7 +1,8 @@ // packages/mcp-server/src/sync/prune-schema.js -// Schema extras axis: delete dest-only fields/views/tables under mirror, tiered-gated, dependency-safe. -// Fields use deleteFields WITHOUT force (so a matched field is never cascade-deleted); orphan->orphan -// deps resolve by retry-until-stable; a field still blocked after no progress is kept + warned. +// Schema extras axis: delete dest-only sections/views/fields/tables under mirror, tiered-gated, dependency-safe. +// Order: sections (auto-promote contained views) → views → fields → tables. Fields use deleteFields WITHOUT +// force (so a matched field is never cascade-deleted); orphan->orphan deps resolve by retry-until-stable; +// a field still blocked after no progress is kept + warned. import { resolvePolicy } from './policy.js'; export async function pruneSchema({ client, destAppId, plan, policy, policyOverrides, confirmDeletions, confirmTableDeletions, result }) { From 70f308d3380b847de2622472e53cc26ab994ee96 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Mon, 22 Jun 2026 09:53:48 +0300 Subject: [PATCH 142/246] fix(security): stop logging CSRF token prefix; drop daemon port from ngrok label Hardening round-2 (audit findings #10, #28): auth.js no longer logs any part of the CSRF token to stderr; the ngrok forwards_to dashboard label no longer leaks the local daemon port. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp-server/src/auth.js | 3 ++- packages/mcp-server/src/daemon/tunnel-providers/ngrok.js | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/mcp-server/src/auth.js b/packages/mcp-server/src/auth.js index b7e328f..5df60de 100644 --- a/packages/mcp-server/src/auth.js +++ b/packages/mcp-server/src/auth.js @@ -305,7 +305,8 @@ export class AirtableAuth { }); if (this.csrfToken) { - console.error('[auth] CSRF token found:', this.csrfToken.substring(0, 15) + '...'); + // Never log any portion of the CSRF token — even a prefix is a secret leak to stderr/logs. + console.error('[auth] CSRF token found'); } else { console.error('[auth] WARNING: CSRF token not found. Mutations may fail.'); } diff --git a/packages/mcp-server/src/daemon/tunnel-providers/ngrok.js b/packages/mcp-server/src/daemon/tunnel-providers/ngrok.js index 95e1c7b..d73b031 100644 --- a/packages/mcp-server/src/daemon/tunnel-providers/ngrok.js +++ b/packages/mcp-server/src/daemon/tunnel-providers/ngrok.js @@ -186,8 +186,9 @@ export const ngrokProvider = { addr: options.port, authtoken, ...(options.domain ? { domain: options.domain } : {}), - // Human-readable label in the ngrok dashboard. - forwards_to: `airtable-mcp (port ${options.port})`, + // Human-readable label in the ngrok dashboard. No port — it adds no routing + // value and would leak the local daemon port to anyone with dashboard access. + forwards_to: 'airtable-mcp', }); } catch (err) { const raw = err instanceof Error ? err.message : String(err); From 7ea117f35d042580d2829aa7fb3ee20b8ee05b94 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 13:59:48 +0300 Subject: [PATCH 143/246] fix(sync): recognize INTERNAL array type spellings in cells + fieldMappings validation ARRAY_DEFERRED missed the internal attachment type name 'multipleAttachment' (snapshots pass internal names through verbatim), so Pass 1 wrote raw source attachment arrays (source att-ids + expiring signed URLs) verbatim. Add the internal alias and introduce an alias-complete exported ARRAY_CELL_TYPES set (links/attachments/selects/collaborators, public + internal spellings). policy.js ARRAY_SOURCE_TYPES now derives from that set, so fieldMappings with a 'multipleAttachment' or 'multiCollaborator' source/target are rejected up front with FIELD_MAP_TYPE_INCOMPATIBLE instead of silently no-oping per record. Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/cells.js | 18 +++++++++++++++--- packages/mcp-server/src/sync/policy.js | 9 +++++++-- .../mcp-server/test/sync/test-cells.test.js | 5 +++++ .../mcp-server/test/sync/test-policy.test.js | 18 ++++++++++++++++++ 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/packages/mcp-server/src/sync/cells.js b/packages/mcp-server/src/sync/cells.js index 2aecd60..cf47a1a 100644 --- a/packages/mcp-server/src/sync/cells.js +++ b/packages/mcp-server/src/sync/cells.js @@ -3,9 +3,21 @@ import { isComputedType } from './snapshot.js'; // Array-type cells whose raw source value must NOT be written verbatim by buildCreate/UpdateCells: // links carry source rec-ids that need src→dest remapping (handled by the link-fold path in Pass 1 // + Pass 2), attachments need the cross-base transfer (Pass 3). `foreignKey` is the INTERNAL link -// type (snapshot passes type through verbatim) — omitting it here would write raw source rec-ids at -// create time (only "worked" against an id-duplicate dest base). -const ARRAY_DEFERRED = new Set(['multipleRecordLinks', 'foreignKey', 'multipleAttachments']); +// type and `multipleAttachment` (singular) the INTERNAL attachment type (snapshot passes types +// through verbatim — real snapshots carry the internal spellings) — omitting either would write raw +// source values (rec-ids / expiring signed attachment URLs) at create time. +const ARRAY_DEFERRED = new Set(['multipleRecordLinks', 'foreignKey', 'multipleAttachments', 'multipleAttachment']); + +// ALL array-shaped cell types, public + internal spellings. The Pass-1 UPDATE path +// (`updateRecords` → per-cell `updatePrimitiveCell`) cannot write arrays, so buildUpdateCells +// defers every member of this set (links/attachments to Pass 2/3; select/collaborator arrays +// with a RECORD_ARRAY_UPDATE_DEFERRED warning). policy.js also builds its fieldMappings +// array-type rejection from this set. +export const ARRAY_CELL_TYPES = new Set([ + ...ARRAY_DEFERRED, + 'multiSelect', 'multipleSelects', + 'multiCollaborator', 'multipleCollaborators', +]); /** * Extract the record ID from a link-cell element. diff --git a/packages/mcp-server/src/sync/policy.js b/packages/mcp-server/src/sync/policy.js index 86b1628..f1dd8b3 100644 --- a/packages/mcp-server/src/sync/policy.js +++ b/packages/mcp-server/src/sync/policy.js @@ -1,4 +1,5 @@ import { isComputedType } from './snapshot.js'; +import { ARRAY_CELL_TYPES } from './cells.js'; // Reconciliation policy: two axes (extras keep|remove, conflicts source-wins|dest-wins) // expressed as three named presets. See docs spec 2026-06-20-records-reconciliation-policy. @@ -19,8 +20,12 @@ export function isDeleting(globalPreset, overrides) { return Object.values(overrides || {}).some(removes); } -// Array-shaped source values can't be injected into a scalar dest field. -const ARRAY_SOURCE_TYPES = new Set(['multipleRecordLinks', 'foreignKey', 'multipleAttachments', 'multiSelect', 'multipleSelects', 'multipleLookupValues']); +// Array-shaped source values can't be injected into a scalar dest field (nor a scalar into an +// array-typed dest cell). ARRAY_CELL_TYPES carries the public + INTERNAL spellings +// (multipleAttachment, multiCollaborator, …); multipleLookupValues is array-shaped too +// (computed, so it is only reachable as a mapping SOURCE — targets are caught by the +// FIELD_MAP_TARGET_COMPUTED check first). +const ARRAY_SOURCE_TYPES = new Set([...ARRAY_CELL_TYPES, 'multipleLookupValues']); export function validateFieldMappings(srcSnapshot, destSnapshot, fieldMappings) { const errors = []; diff --git a/packages/mcp-server/test/sync/test-cells.test.js b/packages/mcp-server/test/sync/test-cells.test.js index 1913b70..38d2024 100644 --- a/packages/mcp-server/test/sync/test-cells.test.js +++ b/packages/mcp-server/test/sync/test-cells.test.js @@ -10,9 +10,14 @@ describe('cells.isWritableForRecords', () => { assert.equal(isWritableForRecords({ type: 'multipleRecordLinks' }), false); assert.equal(isWritableForRecords({ type: 'foreignKey' }), false); // internal link type — must be remapped, not written raw assert.equal(isWritableForRecords({ type: 'multipleAttachments' }), false); + assert.equal(isWritableForRecords({ type: 'multipleAttachment' }), false); // INTERNAL spelling (real snapshots use it) assert.equal(isWritableForRecords({ type: 'text' }), true); assert.equal(isWritableForRecords({ type: 'select' }), true); }); + it('does not write a raw attachment array under the internal type spelling', () => { + const att = [{ id: 'attS1', url: 'https://src/x.png', filename: 'x.png', size: 9 }]; + assert.equal(coercePass1Cell({ id: 'fldA', type: 'multipleAttachment' }, att, idmap).write, false); + }); }); describe('cells.coercePass1Cell', () => { it('passes scalars through', () => { diff --git a/packages/mcp-server/test/sync/test-policy.test.js b/packages/mcp-server/test/sync/test-policy.test.js index 0c76f13..34e1272 100644 --- a/packages/mcp-server/test/sync/test-policy.test.js +++ b/packages/mcp-server/test/sync/test-policy.test.js @@ -74,6 +74,24 @@ describe('policy.validateFieldMappings', () => { const { errors } = validateFieldMappings(srcMulti, dst, { Offers: { Tags: 'InjectID' } }); assert.equal(errors[0].code, 'FIELD_MAP_TYPE_INCOMPATIBLE'); }); + it('FIELD_MAP_TYPE_INCOMPATIBLE for INTERNAL array spellings (multipleAttachment / multiCollaborator)', () => { + // Snapshots carry internal type names verbatim — the validator must reject those too. + const srcInternal = { tables: [{ id: 'tS', name: 'Offers', fields: [ + { id: 'fAtt', name: 'Files', type: 'multipleAttachment' }, + { id: 'fCol', name: 'Owners', type: 'multiCollaborator' }, + { id: 'fTxt', name: 'Note', type: 'text' }, + ] }] }; + const dstInternal = { tables: [{ id: 'tD', name: 'Offers', fields: [ + { id: 'dInject', name: 'InjectID', type: 'number' }, + { id: 'dAtt', name: 'DestFiles', type: 'multipleAttachment' }, + ] }] }; + const { errors: e1 } = validateFieldMappings(srcInternal, dstInternal, { Offers: { Files: 'InjectID' } }); + assert.equal(e1[0]?.code, 'FIELD_MAP_TYPE_INCOMPATIBLE', 'source multipleAttachment must be rejected'); + const { errors: e2 } = validateFieldMappings(srcInternal, dstInternal, { Offers: { Owners: 'InjectID' } }); + assert.equal(e2[0]?.code, 'FIELD_MAP_TYPE_INCOMPATIBLE', 'source multiCollaborator must be rejected'); + const { errors: e3 } = validateFieldMappings(srcInternal, dstInternal, { Offers: { Note: 'DestFiles' } }); + assert.equal(e3[0]?.code, 'FIELD_MAP_TYPE_INCOMPATIBLE', 'target multipleAttachment must be rejected'); + }); it('FIELD_MAP_COLLISION when two sources target the same dest field', () => { const src2 = { tables: [{ id: 'tS', name: 'Offers', fields: [ { id: 'fa', name: 'A', type: 'text' }, { id: 'fb', name: 'B', type: 'text' }, From 50bfe7b44b9ecb059192db9f22a51e55e3e7e3e5 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 14:01:56 +0300 Subject: [PATCH 144/246] fix(sync): preserve persisted idmap.records/attachments across plan/diff matchByName returns only schema-level maps, so mode=plan's saveIdmap atomically wiped the rec->rec identity map built by prior record syncs -- the next apply then re-created (duplicated) every record, and under mirror+confirmDeletions deleted the originals as false orphans. Merge the persisted records/attachments maps verbatim before computePlan/ compare so re-plans keep record identity and record-referencing view filters no longer re-flag as drift on every diff/plan. Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/index.js | 14 +++ .../sync/test-plan-idmap-preserve.test.js | 104 ++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 packages/mcp-server/test/sync/test-plan-idmap-preserve.test.js diff --git a/packages/mcp-server/src/sync/index.js b/packages/mcp-server/src/sync/index.js index 4b526d8..bd41f62 100644 --- a/packages/mcp-server/src/sync/index.js +++ b/packages/mcp-server/src/sync/index.js @@ -49,6 +49,15 @@ export async function plan({ client, sourceBaseId, destBaseId, planId, direction const src = await snapshotBase(client, effectiveSrcId); const dest = await snapshotBase(client, effectiveDestId); const idmap = matchByName(src, dest); + // matchByName returns only schema-level maps (tables/fields/views). The persisted + // records/attachments maps are the rec→rec identity built by prior record syncs and MUST + // be preserved VERBATIM: (a) saveIdmap below is a full-file replace, so without this merge + // every re-plan wiped them and the next apply re-created (duplicated) every record; + // (b) computePlan's view compare needs idmap.records to remap record-referencing filter + // values, otherwise a correctly-synced record filter re-flags as drift on every plan. + const persisted = loadIdmap(effectiveSrcId, effectiveDestId); + idmap.records = { ...(persisted.records || {}) }; + idmap.attachments = { ...(persisted.attachments || {}) }; const base = computePlan(src, dest, idmap); const fullPlan = { planId, @@ -84,6 +93,11 @@ export async function diff({ client, sourceBaseId, destBaseId, diffId, fieldMapp const src = await snapshotBase(client, sourceBaseId); const dest = await snapshotBase(client, destBaseId); const idmap = matchByName(src, dest); + // Merge the persisted records map so compare's view canonicalization remaps + // record-referencing filter values instead of stripping them — otherwise a filter a prior + // sync correctly restored on the dest re-flags as `filters` drift forever (see plan()). + const persisted = loadIdmap(sourceBaseId, destBaseId); + idmap.records = { ...(persisted.records || {}) }; const base = compare(src, dest, idmap); const fullDiff = { diffId, ...base }; saveDiff(sourceBaseId, destBaseId, fullDiff); diff --git a/packages/mcp-server/test/sync/test-plan-idmap-preserve.test.js b/packages/mcp-server/test/sync/test-plan-idmap-preserve.test.js new file mode 100644 index 0000000..bd986d5 --- /dev/null +++ b/packages/mcp-server/test/sync/test-plan-idmap-preserve.test.js @@ -0,0 +1,104 @@ +// Regression tests: mode=plan / mode=diff must never wipe (and must actually SEE) the +// persisted idmap.records / idmap.attachments identity maps. +// +// Audit findings: +// - plan() saved matchByName(src,dest) verbatim — matchByName returns only +// {tables, fields, views}, so saveIdmap atomically erased records/attachments, +// defeating the C1 duplication guard on the next apply. +// - diff()/plan() compared views with a records-less idmap, so a record-referencing +// view filter that a prior sync correctly remapped onto the dest re-flagged as +// `filters` drift on every run (converged permanently unreachable). +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { plan as computeSyncPlan, diff as computeSyncDiff } from '../../src/sync/index.js'; +import { saveIdmap, loadIdmap } from '../../src/sync/idmap.js'; + +const SRC = 'appSSSSSSSSSSSSSS'; +const DEST = 'appDDDDDDDDDDDDDD'; + +const RECORDS = { recSRC00000000001: 'recDEST0000000001' }; +const ATTACHMENTS = { 'contract.pdf|12345': true }; + +function schemaOnlyClient() { + const mk = (tblId, fldId) => ({ data: { tableSchemas: [{ + id: tblId, name: 'T', primaryColumnId: fldId, + columns: [{ id: fldId, name: 'Name', type: 'text' }], + }] } }); + return { getApplicationData: async (appId) => (appId === SRC ? mk('tblS1', 'fldS1') : mk('tblD1', 'fldD1')) }; +} + +describe('sync index.plan — persisted idmap.records/attachments survive re-plan', () => { + it('plan() twice preserves records + attachments verbatim while refreshing schema maps', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'plan-idmap-')); + // A prior sync cycle persisted a populated record/attachment identity map. + saveIdmap(SRC, DEST, { tables: {}, fields: {}, records: { ...RECORDS }, attachments: { ...ATTACHMENTS } }); + + const client = schemaOnlyClient(); + + await computeSyncPlan({ client, sourceBaseId: SRC, destBaseId: DEST, planId: 'plnKEEP1' }); + let m = loadIdmap(SRC, DEST); + assert.deepEqual(m.records, RECORDS, 'idmap.records must survive plan #1'); + assert.deepEqual(m.attachments, ATTACHMENTS, 'idmap.attachments must survive plan #1'); + assert.equal(m.tables.tblS1, 'tblD1', 'schema-level table map must be refreshed by matchByName'); + assert.equal(m.fields.fldS1 && m.fields.fldS1.destFld, 'fldD1', 'schema-level field map must be refreshed'); + + // Plan a second time (fresh planId — the routine re-sync workflow). + await computeSyncPlan({ client, sourceBaseId: SRC, destBaseId: DEST, planId: 'plnKEEP2' }); + m = loadIdmap(SRC, DEST); + assert.deepEqual(m.records, RECORDS, 'idmap.records must survive plan #2'); + assert.deepEqual(m.attachments, ATTACHMENTS, 'idmap.attachments must survive plan #2'); + }); + + it('diff() does not touch idmap.json on disk', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'diff-idmap-')); + saveIdmap(SRC, DEST, { tables: {}, fields: {}, records: { ...RECORDS }, attachments: { ...ATTACHMENTS } }); + await computeSyncDiff({ client: schemaOnlyClient(), sourceBaseId: SRC, destBaseId: DEST, diffId: 'difKEEP' }); + const m = loadIdmap(SRC, DEST); + assert.deepEqual(m.records, RECORDS, 'diff() must not wipe idmap.records'); + assert.deepEqual(m.attachments, ATTACHMENTS, 'diff() must not wipe idmap.attachments'); + }); +}); + +describe('sync index.diff/plan — record-referencing view filters use persisted idmap.records', () => { + // Both bases hold table T with view "Active" filtering on a record id. The dest filter is + // EXACTLY what a prior sync wrote (source rec id remapped via idmap.records). With the + // persisted map merged into the compare idmap, the source side canonicalizes to the dest id + // → no `filters` drift. Without it, the source leaf is stripped while the dest keeps its + // leaf → permanent false drift. + function viewFilterClient() { + const mk = (tblId, fldId, viwId) => ({ data: { tableSchemas: [{ + id: tblId, name: 'T', primaryColumnId: fldId, + columns: [{ id: fldId, name: 'Name', type: 'text' }], + views: [{ id: viwId, name: 'Active', type: 'grid' }], + }] } }); + const cfg = { + viwS1: { filters: { conjunction: 'and', filterSet: [{ columnId: 'fldS1', operator: 'contains', value: 'recSRC00000000001' }] } }, + viwD1: { filters: { conjunction: 'and', filterSet: [{ columnId: 'fldD1', operator: 'contains', value: 'recDEST0000000001' }] } }, + }; + return { + getApplicationData: async (appId) => (appId === SRC ? mk('tblS1', 'fldS1', 'viwS1') : mk('tblD1', 'fldD1', 'viwD1')), + getView: async (appId, viewId) => ({ ...cfg[viewId] }), + }; + } + + it('diff() reports zero drift for a correctly-synced record filter', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'diff-recref-')); + saveIdmap(SRC, DEST, { tables: {}, fields: {}, records: { ...RECORDS }, attachments: {} }); + const out = await computeSyncDiff({ client: viewFilterClient(), sourceBaseId: SRC, destBaseId: DEST, diffId: 'difREC' }); + assert.equal(out.machine.summary.drift, 0, + `expected no drift for an already-remapped record filter, got: ${JSON.stringify(out.machine.driftSample)}`); + assert.equal(out.machine.converged, true, 'diff must report converged'); + }); + + it('plan() does not re-emit applyViewConfig for a correctly-synced record filter', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'plan-recref-')); + saveIdmap(SRC, DEST, { tables: {}, fields: {}, records: { ...RECORDS }, attachments: {} }); + const out = await computeSyncPlan({ client: viewFilterClient(), sourceBaseId: SRC, destBaseId: DEST, planId: 'plnREC' }); + const reemit = (out.machine.actions || []).filter((a) => a.kind === 'applyViewConfig'); + assert.equal(reemit.length, 0, + `expected no applyViewConfig re-emit, got: ${JSON.stringify(reemit)}`); + }); +}); From fbaefab2cab9e0dfac239e707fe27a125d672415 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 14:02:49 +0300 Subject: [PATCH 145/246] fix(sync): defer ALL array-shaped cells on the Pass-1 update path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit buildUpdateCells deferred only multiSelect: an internal-typed multiCollaborator (or any other array) cell went straight into the updateRecords payload, where updatePrimitiveCell rejects arrays — failing the cell POST and silently skipping every remaining cell of that row on every re-sync. Defer every ARRAY_CELL_TYPES member consistently: links/attachments stay a silent skip (Pass 2/3 own them); multiSelect/collaborator arrays emit RECORD_ARRAY_UPDATE_DEFERRED — now only when the choice-remapped source value actually differs from the dest cell (dest cells threaded in from the snapshot), so converged cells no longer spam a false warning per record per run. A defensive Array.isArray guard keeps any future array type out of the payload. Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/records.js | Bin 65290 -> 68942 bytes .../mcp-server/test/sync/test-records.test.js | 97 ++++++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index a9cfc9d97d083e3e7b1c9bd8f5d60ff47019ffb8..f7777555f96bd945c21de9b263ffc0cccfd48785 100644 GIT binary patch delta 13082 zcmb_i30PD|wtlV{uoVT7RSIQkWOG+QHU$(#SzS;On(hm!8mW;_g;T|MOEEePM!MCe@@lC z>#ub?dQbQ6X+wwVwsb4&qT5nAw};N3QR(qKe`FYA6Uz&DO_vBhuuD!C9iQ1nq4aPU zgKWp6yH4qjhP4^+GXC=da`6cSZLR-3C)y*=?#mnVB84 z>}IE-&Y~3B?B$BlW;Z#KSzdX0URBM+g5u(u%Bs?W3ckDB0P6BYw|q)3cgv)7WcNvw zF6mxH>3iMBQ`)yj1*LU8#?yQ{e=LXpp~rkx#O^%}^ib(JQ!O7V3+C_kgf#v9&<*8j z{R8+)zXU2}1x52W{o?wng4esOMyJ_k<+T$>bz@6)e0Z-R)T**q0i`>74UeRDtR0D6 zak}hQW>hQ|N2bo7>U8{5(JcO!8fv^+q<_0*9_4fWA@wri$m`Xsq zto&O*5fxKwgMtWG(r;gG` z`!rG-)Ymu&t71k2iO39gg}JOwo6Fd!m{RdT$G7ywZ140PA+|A1$9n`$_NcTxa4tRF z9~c@XhRKyHcCe*&Hk(B;SfK{mYe`sqW^bleOB54REK0LtbuzO_h@f{k?PhC( zgS8kO4(4pM+guHenaozVKruQUtix!tQiD2|Q(>uQ6H8;M^)|a=Zm^cyT8m7HsjS&$ zahfY=RHKs-Orac`#bT(l*$qycT?!&(a*)>ScF4k94q8OL!AKJ`!mMIvdXw4FVj;n0 z(P(QX!Rzg|j%GPzqs?qo#Nrc^nS)lzT3n=kyCU109W<$ndjKVV5-0DR1V=T4V(4yBXvV*g&MWH=ZSXNdROOV@=(7|%3 zCWBRoUD7K#*n}La%kkLmlvrddN_2?bn+yxuhTP{Y<|c)kymJ1#w;fWU7-Skf!D~_m8Js zKfhr9mU#M8e=tr~P=;m~OM{9-JlYdBPx@Qho~h#{c4MJgv6viCGQSoCVu=gZ zQ`#j7v!x_O@@>IUl>0RTjP_k{IHi6Az!PPWgZbJ4nD3ndp_FcnOz5if=Z1Mvyg4F@ z7ljlIlZwF}v`amRtX2!qo2@QIXh@I~k4TQ9L3|-HBZZQOl9E%r2_wy$D)`ARxgE*syo@QN0w)b-v(f~ujfxakZxl?BVpqs? zXv#z)1DD;XNUmcBqm!(YI!GQ66w)l#M*H9tT5wOGCn9P!*sUNTM+*@I5nBf{SnP_y z)RsaJSftn+w9U%&h#*F?3ul`k6{~rICM&JqvjV78n}7&PTM+0%u`e=!y)A8tK0L)k z{dD})z$5~#YEF_;HO7*(yv$`aDfMPjbvuJJfQf@Ni#)IGAKAh5?iOyj-RlO@|MgF* zRt!hzLsUVL1~b%=({X~7y)^2u8d@BUHYdo%&9waavRn8~34j-b4c17YDQsAGA@zMh zL;fGwAdaMn z!ORqVc+kDbW;E2fEC%wibqc&RNzzQdj{xp9FhK+ROF?5LH4xV1AkJ4>$a`Xg304+( zSQ4m>tf3X0W3*A2!wiH0F9$1-E5rq&$+((Zh`{-!@GuIoB8S-@$+Sax+a4C68P(bf~oRQSr{Kv-iHV7=*#;>4^nH+QH^;A zLV7$OSJ8*>iH@Ms(H#{%>kLj~qkNEaPaMBmkwOn%bVUM|I7x}a%|3=x#5IO^yedXL zv+<{65I=+ruBCL*U`@t;eqsF7V8j|A<<0$j@c~0dQK@K1jYedD95OZ zF>x|;FoN{l>P+3%Sj0wYu{gIh#fC@8y_X#v3ffzhR#u@;^>Z^YsVk`y{5!L`X~k%o5Y`^Kpul=`en0WB7_W0Zq!TliDh`NMTBEME7_Zjlkt`0nUt2M%+jQKG^GU9DS4D;rlure zJh+b-lQU5b2!#So&Q$3)9mHblHf&Cv?sN3~A{A%q!Zf50rkN-W7>3j~OriAsVdJ$) zgNLuCiXFr2ghDKY6pGozG)`iA3Xc1XC<%rn?R3B*rUdcGeT1IZ7LopT9RI z-7{Igu}NZ@tg$&X#Imsv_|dV$DLo}>BF0q=7LrMnN)3^2%6-*z=eSunQ&R{JNDrj( zG!b0ssR^2xxRTJ+#h9voqf6M=Fk0Sg>7aru=}jqGkIo7tJ>a`L)uN)ABzSWc-Z)wYK7uw1ArW!6YaYiBW@ zvMXfk;$GtM&Rh`7x!iIUOcclDYjHQT*lbNM9zAe?!t_K`GP+KY&c&oS>HdH&iC7Xd z(nW|}F{OwjON-ssVrZaP*=E(^4M}otpFg3k2vi5|oJOZcPCEbFekv?%#h&Ml3DE>LGlvFtCDXiT0!-$Uf=J$PA`Ys#g;|ttn+O8E zG*OsJJ_tK4ANN`-@{@XN3lk`KDu1@e7tocCLxH_V6-4%x`V@@Drk7MA){UXt6~r>6 z)MPdIOdClTE!yFP-0g}2O#Df~#7G~#rZ$i_LQT11q{|*Ys1PzqS=$SXgEW~4U(lU^ z*@?mjsLidD0HQOK;JzkI2Gky!4A=AOWH|aBMVXZ57OkN4rJ`uulWN_+Dtdt`@1GLm z9$d1VXu+>92;moN{P+)3N`Uz?LkOQU6<1|3TZ8$AsnNK`WBg$8fbN31`pt`>X9o-s zfip_;^C}B!3iFDJ3-S|^b$tH3f&BWCUczK@iYJWFmLdd^hJ&nFl$z;GuLp{gA;cB~ zf|8AqVf{K!f-8q$b(WE;p^G6s-BS}Bd)bNV1PT1uD>2efE< zX*kI{l!7yBiqhOdmwz~Bh~sUBj-Q@Bl^P|Mz;HK|Ab+ByfYPW^U~cF32)?{)Prjox zzf%a7z@y8+Web0oz)#u7$S}-86*2_Npj=BiHWcM^<*J%|f zQ8gIMRgJNaS3_CHtHb%A`G}fa^T$!TfBs}jf1EG<0jn8CrG+&EbVa5NaaeN^iCj$Rk-*C~p#Wi>Rk$K||3r91&M(#0q#mOACIeIL)PPvwK<$!v?;f-e?^RLJ zO>o7AL0-a$6>AgS5d3J$gG-w~FP#V0<>IQv{qvbbSRNj!8DW8Z&+X#OfJ#*GOUCmw z*g7M$_mUC%k~L>cn8Z&355y@ZpD=H;GFb!vDWyz+LYzNF)PvYp)u;Iw;ogP_KcmG= z2P_@GP#;SzF4RZ&6j$vsOQ0z-QJ~Tg!51~a?)Eky5_qWrM0a#Y1i#wgFKQ$B=@0z0 znNu3qP~{ViuC?cD>Hm4dhp+x5ABuOgiG)vZpL#>Ei)Tr}* z(f^n^f=2n)Tp9-5x;LR-p+K}i4MQoaqBhM7AmYXaCF%~k_1`HXrD4<=t}W23aRg(n zW@lr$LJ#zM+#Bfl_$HGGzp-iFz8hs=&IfXY&9aw*_%w5XKUQKFH@iGE`M)-pqFH@5}twzq(lue9K-HE&@e zr6Bn|3qf7)EF7-3-P-^{19BdHFOZ*qK8$CS_2$j?Lbdj&9o&HDxsH61Bi<<=#ZNf; zs`aNo2;{$(LoH}k>eP6DpK~E}gi-kT4fY zrOTlT^&U}_!Hzx^W)l&ocx1uNxMCOi1$6UkHCRNxMX@{R&@J+`^>l<0S%Jd|KsD_< z1wjUSP^j=*7v17VHZ4N5I~UC%AxmG16jsr#6_*8CSlr2rsG)8jjR94h1}#pfKE;bc z3#%4SB=ZLd<>BYp;$$)I)y42VliF~-@IV_BeYFh)lClI_e8-Y7Pb6X8qQ0#_=wPJD z`qvj%^C&(N&-xwhV+ZXKN;^SMULR!_Zs!u4TQc^3uj2eqUeSiQJ1jeD2aQ3Nl|@I!ro-?5b##tG5cK?EsE1cEBS(vND#Z zFYnh`DKnR6_SQrbOnuLCc$R-hz^lu_-DhtL=IJZq)q@j+HIo;t#{q#{;EE784-q*L z58@I2)5G7d7^-tmok+~U-(3M)9(-FKrN-N`#rE&JEnCb|=vh|H%6O{MmX#Wz4dNyU zNFk_(n13bElfDXwF|QI)r__xId)>Qyv4io*VC*PcEgG%h;9TqKa7^3f_Hh33ntr{s zD@=ZPH2`sPHBfVHb$ppe0!-~Ahg#v?!V4JgEOf8J6&%dpSrf(YxIK`@|97_HquZwr z(YnL`&>LyI9g}w*kU+J!32Oj|g=;`(cdRi+X?@T{a|I%gP=))g1x?A;#cOfa;%jk= zd2%fPb#m=cO1rNEsU@rfXr`|N`!84r-K<*&jX%3C(kF6&W(2!n68yYNB0l2R)*+tF zUJqMcvtCAa@2{UgrI0(o;}o1oq}#Rt1@bTw!Y|$7>hHZ8b^Bd+B2qhYr{Dz`_R>{Y zJB+w#E95WR0M@#7Ly8uE#;i_~<9BeZtjos9!@b9a(C$q`p;J-pqo9!W8xz%;UfT#g zUfKwa#B72FG5^d>(3Yf=?%*LgZOk%EE31>WsvvIVfk5K@?Hg2Ic*ER8D<5tKnf8VRZ`4Wu<%XUuh;5ORgTqi9Uwqo(Wtv*!d^WV3D-bu*hy_Z zO7kkXvn2ogyTWLs*1K{cf9F`@`jQaNwha|mc|z@8u>>`zh*PYL#-l&=5V&&N9My%N z-3Ic`x*MV6rn_;5e(P?W>EgB*P`Y3{?!Ju zv(*+k_h2qBr+)Er7(a54kt)e;Qd+W8-YIqLglH#s!oiKZ7mjcHy{VMGdhc>d3w~d9 z)6*LH*(gDqQ+Hv&CA;!O>G@riR0`aUeJHOj`EhgnMf5bT=a10iAk{s)4X(Un+JKNBVvHrn|kG4^OH% z5PNr%<3rSaIGEAF!E+WEe?PW$$NkXJ<@@KU8#n8LDk>j*pu%S(W&8%CNAA`KbJX0$ z_Y(N^2SY>&aa#b-eF#9mP#?lqJcKRV+YrJ}J%o*}`5=VHJ`6RY&hap!H1YId+;?N# z(;pzBlTltE-~R{@jT8QNj{tNs(xVgYhlBXSM}w*6MOO&l`Dm(k-ne)f$buT^5CrT0K(<_4(!5{`P;$;XyzY;bA9L_ zT*HNfpkv%}rX9jJbNPpGWLkU(+Sq>xx;t}-QF{GQ6Qv6tpNo&&H1-sN9D6*Jo_ziI zt?Cp}5^;1BDnEqCV4iO&9AvHk*`zj9p-BzP6E4V?2tv$wRLKY_i}KZ%Rh zwkHw0(D4clj-DFSovsM{`Snv_{GU(4RHi;vtH~#6hS=%pr&g)MQS1ic)%Q6vXeNaU z?203*?de1ndOM#krs2MN8p?=x1{P&}2FCruGXQSHvxuNw&!Xnovxp{scoyJFc`jD% zU-?`;m7jagKxxF`d6c#uuBG&&!>AeeJb=FC`7j@#52stG=rY)H$B{73j$mBms*k|; zbQ~#B%U>M<>xI4``6-xpyx5bke*ylm=7RzJFE2n6ylg^!^ov-BEX|;tgl!PN`eLvK z-Gk$Y@Ns`yO%IR!sgTksKV$DZe#Wk9hlJ*r0JA^8ge4@s3>B<>8Q6I4RK@!LWJj~Dsl_2gy3n^wXwYRmmm*lZ+`_# zc<~j?7X7N3(zUNbZQs0#*@nCZE|~TjSnS!?22uLaYap0zua6eTEc%{`=Fo1^a4;fk zS}IXXAKY1sD8dyG#ut0^#)m>%`n-)ITT!XL2FhT=)C&2561CDQ)K7mr74nS^%Bne= z=-WUsu+~L;myQqN)L}gbJjko#pS=zy8uv!AsxK-Wd;^ik**8QiOKufUwnPr(X>azV zdYseye&Wx|-`s+$QSlRjfvyG1Wf<@5!9Y|B2}nn_K5=Vf7c1n`Zp)Atm41I z0v`VB6mp4By~J96L2&_HxKErsWBOD=+-Q1LM*A9g62_W$aw4UxPL2>`J$(|)d+lU` zD8+wVN2RqNLsj2>43^72Ri$nz+=tmwX{T>8@p4k!CnCag>G{ngrD9hH1xsifOlh3IP20X%?j;J{#vg z=emQH7{2{8?ElA~={+%|`^AKCrw59|`x+Zv)+W`*lHw6Y^Xkvb-4f7By$!?56dwOu zCJpa{-m_o89`e3Kc(Uh9w@sf-G%2|KnNjcU49`}V3V}Je?%`@)wXulp8ZH8 zLMf6FmwXQh`oQ3kAI7M(^!X1^diNjaQd;xl1WNb*2ok&cqn%RQPw+w;ep*23KYyz8 zp|XGN)TLt{!xvx1U6kgWpSV0u1KY*Rh&0?TnH(HcDG5ie&EwquPp(Vukg6w-Z|mf_ zf?z|PU02ih``U@~9!JP6di}N*7lqfnZSWNev z_`p4-g@{n0i}hc@pHKXXzNdeU*UMhwivxVgp*m>w!voJ|(|yO)QW0f{tD57FMf2ZZ u^`kNFy9&p5?ka3K{91`b*;2(yJh#*t<;g^rCtHKZ8UA*DDnB-NGrS|t=N z*~oPn1A|M6uh1!r<24Q4C1KIrI^o z(JHicnry~u(Io0^B!7!;p;b_s+iiliv!kQkf*!%gQ_WN%R1k%t6;~DGBCSGmyG5|J z8ap~f!=N1``=#kf93P5zL&p3!(bh?#4GmAg(rLrc5bpcG6Y&@>6Rma-H8MZUtcWG7 z5ylSZ(~Ql6YU7x|Akvg9%Iox?L{E($k|Og{oiLXSFwjPgaTj7(SK~92i65mg3Gumu~QaXcyZz8RyS2BXO zTMVK_G%yV1ocAQu;bOvMBqWw$Nn#VuCrTVMKOu>_6MW<`@TX27R*r9ySpR-}&L8^& zYA5)hV?vD#Ga(Fh(vqfOby5aiPYNeR3wS#)6t2mI5u9emJA1ANc}`$w@@$+;o{6Z5 zHC|j_&7Eew&Dd_nxeP51P4vfm6LaA^DNaQ@Fb|3_*wpe?41o-`c#Ehfwyi>TcBW7} zHyv%0%AC;q#v~n7DLSl4G2jP!rl*Q{AvHr{^J?k}+?m!inzk{KHZ7u+?P#+s6uNLZ zt^5}1bW@D5>01PqXt9tBVQzXPmZjf0ie5PwVrwX3eKt7+Ng1P2m@yH{^89d5#uS{* zsG`8j-cHJlL~UjyA?&iuTAa~4Gr18_%Y9dFS9lBwm~@>+OjYw8>@4)IGB@&Z>SxUTOG!Z<;Ed#Mod)=WN&9~ z&CS?TPL4w2_li>IhLUKNsI1(DUn_N( zQ>Dk@s^-uEJm*yvqQ12PmwsN|ax=*TEo zJ*$GCC`MR7*eDppW^!h-VxIk0=&V*Dk!O8x&MJrhY(`KF1#WYk*w$fND42}qR-SgH zVcF~woS1DONyO06ama3oz??aeSUjfyug$52P%HjHm>Yrb-U`O?Zci@)?o!&Z$R@T~ z)y~KjP*>n=XGf2~p~V5nsSCmLb)%f)T{IrNM59n6e6C&|9gQVR#yMte+%*DyOF}6u zs*zPcFf5LMy?%-#EK1nNob33^EknrKB=0)XP!8|L98@+MaG)_2mm0&!@FyTom*ys^ z9xHTlGVYArGfDO?fz(peRE$kclW@GLNwN|Gv`Kn~Ork!Y{vD|cK*y>ee5z+|e0p0r zw1&tbO39Jd8B$r(v?b^kCTeFg%g=64}P5#X7_{mr|>{xdMI7 zBm}2v1&k^!Dl2K2QC3w@U0yn~AU`^mjLn}Cs^REs4aA~bTLYcebDhyCF|;L)c1>!D zg|4Lzhg&Al69i9V4GhK<+-EGr7em8 zV16>P+Weyj?&WkIX}x0~*2Db|gfe|{P=5Kz@3F<4f_KdcVC}!Rs}S4~hO`dy2ieUY z>nN0r>z9s7`N~vrTNNIf-F2(*(_3Ap1jaTmxDy>80p;E)>SxXO@f~-f;5z7(=hUxz%gn;2>xUX zO<>Al5wVFaWI|;fmNyeM)9F@62*@sX?2V+ok8K($Lhu#znx9JCv#L2}c*H30Wi1GsFu)-68ya*{)D1-}D%6W3irt z_cBWmH<9=M?2Yf=3M8zpg5ntsiWg7i<9iq9%O>QF&?PC%W^ntGSUz@SiH1x{1OKIo zWL_FT8JTvf!gfpDEWivu*ha;Fb2>BFeiyK^)?+vbhCcTje))eqfUd~tf~Xf&)2$CA}^ z@#D5+&v+)rDJHtr5Wq#O_C;o|J2L*{fsc0fjCG#G&2>PJnXF91#x)U=6SMx{TQUc_ zoHZ5bT@#M|Ym&&k1bn8CC(=%CV<@tIM+!Rc*5Xu;;7YePN($e(J9#jFBi5!ngh{A5 zV{P<*JMX2nDtxgvEz3!v(U2xIvpb=4#H?!**%c~h3CQkYhQUO@ zvN@)kQ$1>2?xETh=_<3=lir4H4H!a6@wwh0EbR@zmfmnY-#Zqc^hRYnx4-@nxj*3j z0Bt!~QlfZc%6}8(s~bxkg%Aei9Xcx|@DjAMLnNzDgcdpkVG;BWYi2KH2<1&Gr(K1H z&&u006?bnMgKL|7y?I5xtX}=0@IM(H!9I8}Ee=^$7%&n2Q-H$uzK#NDW1R#88JQ2VT705WlTKYop@jlk5{n|&c&Jst8n(gbOb(Bi_V7% zaOj~#ifh4ee|Q|?A5Q1>_3xKduWdq-|oq!miF=KSpIk_ z4nLkpy`fJOq3nqaYOQ-B8LvLUW(?a~h>E>L>)uK)N1iOXX!xZF)DH1R?`A! zKNemt>|;>O!i)OJvVIUBfLGi+Z<4Lucz*GP8u%QlhxJejZOo$M&t-~qW`?q=#=V$< zrWbcW@e)f#!(S@KyqBip>6fBd#RMOm@x}1NNys{!Oa+Z-Y&-0M-G>DnJKRdM&2RXj zA zv=TFps(1-#>(K}*0a14T_G@acz*Vo6 z;xDfyVfgEXsCvB?`(B?5&o}1a);DVL>KkM6)f=f8^X6DSx8%)cJpHB)KF8{C+p$Kx zdrXI zbbPSpoe5+ezIgAQRNjv`Ud4Y`AD=;%pr$m_$6*bR;_VYtFsd(^RYQ1^IH;;gUwh;% zZ%JQ0KIv;j_Q|=p@8o!#I++HKQ}Of&m~+aL-FN1)h|qCr5{{gSwU-W8q}VrjDtwuF zbkmSOvQPW5;S$+G9y^_i&rdfZ|6L>Yy{m)wdub?nF9!49GqJmhR2Y*JQ%XQpZLz{U zb{F9vG3@SMiYS*h%00AyNnXN^V1_BJk#t2%x@d@UTa(UQBqun!rgjTeu_=?$QNc?> zXFadI(O81i@9Z#8Ev~;WfA?{|mrA3~u0b?u#DVi2A34&fkSFcvOupH-vyd}0VLe0N zM$g3J;+X`5y&oh4KKlk^et#JM29s0Zh7Ip;b`+Z&s3SW=)(5d@{a~*PD_RoLxG1$+ zyz)^nG#`#!)=%MNiyk^;rAav_U58z%TZ@cQM1RBpJ7&@#(p7l7)6yhCw)A zB;!{;St!-h&X@J$*5|Fb`1zU}BgFo{tw!2~@o2k{fE^cjf_$NXc=Z3G1o|)Zc=L<7 z824pyD1!yb0`dz=3kH`KspRuzB94BU2e+@r)7tc}rsA%z*ac$mS55H#TEzUXD{<`W z*-T0jCK|2NE%Zr%*#f(ANI|)$#uw&qf}J`_{m~0geUpVBzA>Th+a&3d?LZQ?)8M3^ zqj{Pi;R<;8oFLx@}j&a8n^sd?6fh?xnyAj z>?-2t(OC5NaRaR5-hb$D`5z6K^OFuQ{!}hO$*7+jVflGHQ_vqn$;F>5k^N6r3E%Y3 za{TR|5perOjRbn;{8EfZ=(+Su0&#xu54ry7*NM>l7J=NWe$fBsgCkcHlm@GShDyY}5_OlSWB=tyS{AeMikeoXIj8QulIomx=}LuT%7@oR zVb|3_d1@u!)%dTmB(%*zeAgS5OD7@c_gJ+3ZlWi^ElWt0?yB>kzZU{o$(&k)5#=7IJLLlhDAVJ?bVyWT}H*}ls~QD=7< zs<7DS4^@QOHxE_JyT?`0G{{;1b9|8l#YFomHwAqPKB%_8=B8jVp8k75#0h_U;4noV zix&)YZyGQ9z3@65{y*Yu8~YsA56j_>ZxPm;HG z_Ort&untiwVjL!ApQTg?(#=lt1!9zvje^^$J|dd+Vq6EIf@r@_shDcNKVceT-sSWRJbxLs4OW(nAqvKkK1jz$)OCGaB}5pEg`EiAPH*H)RWWO64Y{ QTlQgIig;i4?uK;a-w!s(YXATM diff --git a/packages/mcp-server/test/sync/test-records.test.js b/packages/mcp-server/test/sync/test-records.test.js index 091bb3a..ea45770 100644 --- a/packages/mcp-server/test/sync/test-records.test.js +++ b/packages/mcp-server/test/sync/test-records.test.js @@ -139,6 +139,103 @@ describe('records.applyRecordsPass1', () => { ); }); + it('defers multiCollaborator arrays on the UPDATE path (never sent to updateRecords)', async () => { + // updatePrimitiveCell cannot write arrays: an internal-typed 'multiCollaborator' cell in the + // payload would fail the cell POST and poison the rest of the row update. + const updateCalls = []; + const client = { + createRecords: async () => ({ created: [], failed: [] }), + updateRecords: async (appId, tableId, rows) => { + updateCalls.push({ rows }); + return { updated: rows.map((r) => r.rowId), failed: [] }; + }, + }; + const idmap = { + tables: { tblS: 'tblD' }, + fields: { + fldT: { destFld: 'fldTD', choices: {} }, + fldCol: { destFld: 'fldColD', choices: {} }, + }, + records: { recS1: 'recD1' }, + }; + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', + fields: [ + { id: 'fldT', type: 'text' }, + { id: 'fldCol', type: 'multiCollaborator' }, // INTERNAL spelling + ], + records: [{ id: 'recS1', cellValuesByColumnId: { fldT: 'hello', fldCol: ['usrA', 'usrB'] } }], + }], + }; + const destSnapshot = { tables: [{ id: 'tblD', name: 'T', fields: [] }] }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + await applyRecordsPass1({ + client, srcSnapshot, destSnapshot, idmap, + limiter: createLimiter({ rps: 1000, sleep: async () => {} }), + journal: newJournal('p3b', 't'), + persist: () => {}, + result, + }); + assert.equal(updateCalls.length, 1); + const sentCells = updateCalls[0].rows[0].cellValuesByColumnId; + assert.ok('fldTD' in sentCells, 'scalar text should be sent'); + assert.ok(!('fldColD' in sentCells), 'multiCollaborator array must NOT be sent to updateRecords'); + assert.ok( + result.warnings.some((w) => w.code === 'RECORD_ARRAY_UPDATE_DEFERRED'), + 'expected RECORD_ARRAY_UPDATE_DEFERRED warning for the collaborator array', + ); + }); + + it('does not warn RECORD_ARRAY_UPDATE_DEFERRED when the multiSelect cell is already converged on dest', async () => { + const updateCalls = []; + const client = { + createRecords: async () => ({ created: [], failed: [] }), + updateRecords: async (appId, tableId, rows) => { + updateCalls.push({ rows }); + return { updated: rows.map((r) => r.rowId), failed: [] }; + }, + }; + const idmap = { + tables: { tblS: 'tblD' }, + fields: { + fldT: { destFld: 'fldTD', choices: {} }, + fldMS: { destFld: 'fldMSD', choices: { c1: 'c1D', c2: 'c2D' } }, + }, + records: { recS1: 'recD1' }, + }; + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', + fields: [ + { id: 'fldT', type: 'text' }, + { id: 'fldMS', type: 'multiSelect' }, + ], + records: [{ id: 'recS1', cellValuesByColumnId: { fldT: 'hello', fldMS: ['c1', 'c2'] } }], + }], + }; + // Dest record already holds the remapped choice ids (order-insensitive) → converged. + const destSnapshot = { + tables: [{ + id: 'tblD', name: 'T', fields: [], + records: [{ id: 'recD1', cellValuesByColumnId: { fldMSD: ['c2D', 'c1D'] } }], + }], + }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + await applyRecordsPass1({ + client, srcSnapshot, destSnapshot, idmap, + limiter: createLimiter({ rps: 1000, sleep: async () => {} }), + journal: newJournal('p3c', 't'), + persist: () => {}, + result, + }); + assert.equal( + result.warnings.filter((w) => w.code === 'RECORD_ARRAY_UPDATE_DEFERRED').length, 0, + 'converged multiSelect must not emit a deferral warning', + ); + // A truly diverged multiSelect must still warn (pinned by the earlier test with no dest record). + }); + it('pushes RECORD_CELL_SKIPPED warning when a select choice cannot be mapped', async () => { const captured = []; const idmap = { From 24b9b0f94dc2ec1bd15ccab056ca6017e873abf1 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 14:02:54 +0300 Subject: [PATCH 146/246] fix(sync): guard mode=apply with a per-pair lock file Two concurrent applies on the same src__dest pair ran concurrent background records jobs: both saw the same source records as unmapped (duplicating every record) and clobbered idmap.json last-writer-wins. apply() now takes apply.lock (pid/planId/startedAt) in the pair's sync dir and refuses with APPLY_LOCKED while the holder pid is alive; stale locks are replaced. The lock is held until the background records job settles (released in its completion path) and released in a finally on synchronous exits (DRIFT abort, FIELD_MAP_INVALID, applyPlan throw). Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/apply-lock.js | 44 ++++++ packages/mcp-server/src/sync/index.js | 128 ++++++++++-------- .../test/sync/test-apply-lock.test.js | 101 ++++++++++++++ 3 files changed, 217 insertions(+), 56 deletions(-) create mode 100644 packages/mcp-server/src/sync/apply-lock.js create mode 100644 packages/mcp-server/test/sync/test-apply-lock.test.js diff --git a/packages/mcp-server/src/sync/apply-lock.js b/packages/mcp-server/src/sync/apply-lock.js new file mode 100644 index 0000000..65a65b8 --- /dev/null +++ b/packages/mcp-server/src/sync/apply-lock.js @@ -0,0 +1,44 @@ +// apply-lock.js — per base-pair concurrency guard for mode=apply. +// +// Two concurrent applies on the same src__dest pair run concurrent background records jobs +// that duplicate every unmapped record and last-writer-wins-clobber idmap.json. The lock is +// a small JSON file in syncDir(src,dest); a holder is live iff its pid is alive. Stale locks +// (dead pid / unparseable) are replaced. NOTE: the background records job outlives apply()'s +// synchronous phase — the lock is held until that job settles (released in its completion +// path), not when apply() returns. +import { join } from 'node:path'; +import { existsSync, readFileSync, mkdirSync, unlinkSync } from 'node:fs'; +import { syncDir } from './idmap.js'; +import { safeAtomicWriteFileSync } from '../safe-write.js'; + +function pidAlive(pid) { + if (!Number.isInteger(pid) || pid <= 0) return false; + try { process.kill(pid, 0); return true; } catch { return false; } +} + +/** + * Acquire the per-pair apply lock, or throw `APPLY_LOCKED` if a live holder exists. + * @param {string} sourceBaseId + * @param {string} destBaseId + * @param {string} planId + * @returns {() => void} release function (bound to the lock path resolved at acquire time) + */ +export function acquireApplyLock(sourceBaseId, destBaseId, planId) { + const lockPath = join(syncDir(sourceBaseId, destBaseId), 'apply.lock'); + if (existsSync(lockPath)) { + let held = null; + try { held = JSON.parse(readFileSync(lockPath, 'utf8')); } catch { /* corrupt → stale, replace */ } + if (held && pidAlive(held.pid)) { + const err = new Error( + `Another apply is already running for this base pair (planId "${held.planId}", pid ${held.pid}, started ${held.startedAt}). ` + + 'Wait for it to finish (poll sync_base mode=status) before re-applying.', + ); + err.code = 'APPLY_LOCKED'; + throw err; + } + // Stale lock (dead pid) — fall through and replace it. + } + mkdirSync(syncDir(sourceBaseId, destBaseId), { recursive: true }); + safeAtomicWriteFileSync(lockPath, JSON.stringify({ pid: process.pid, planId, startedAt: new Date().toISOString() }, null, 2)); + return () => { try { unlinkSync(lockPath); } catch { /* already released */ } }; +} diff --git a/packages/mcp-server/src/sync/index.js b/packages/mcp-server/src/sync/index.js index bd41f62..15a4eb1 100644 --- a/packages/mcp-server/src/sync/index.js +++ b/packages/mcp-server/src/sync/index.js @@ -9,6 +9,7 @@ import { newJournal, loadJournal, saveJournal } from './journal.js'; import { applyRecords as applyRecordsImpl, reconcile as reconcileImpl, writeRecordsJobStatus, readRecordsJobStatus } from './records.js'; import { validateFieldMappings, isDeleting } from './policy.js'; import { pruneSchema } from './prune-schema.js'; +import { acquireApplyLock } from './apply-lock.js'; const ENGINE_VERSION = '2b'; @@ -133,69 +134,84 @@ export async function apply({ client, sourceBaseId, destBaseId, planId, runStart const fullPlan = loadPlan(sourceBaseId, destBaseId, planId); if (!fullPlan) throw new Error(`No saved plan "${planId}" for ${sourceBaseId} -> ${destBaseId}. Run mode=plan first.`); - // Schema-only: fingerprintSchema + buildIndex use table/field/view METADATA only, not per-view - // live config — so skip the getView/readData storm (one ~1s read per view) on the dest here. - const destSnapshot = await snapshotSchemaOnly(client, destBaseId); - if (fingerprintSchema(destSnapshot) !== fullPlan.destFingerprint) { - return renderApplyResult({ planId, aborted: true, reason: 'DRIFT', warnings: [{ code: 'DRIFT', message: `Destination changed since plan ${planId}. Re-run mode=plan.` }] }); - } + // Concurrency guard: two applies on the same pair would run concurrent background records + // jobs (record duplication + idmap.json last-writer-wins clobber). Throws APPLY_LOCKED when + // a live holder exists; stale locks (dead pid) are replaced. NOTE: the background records + // job outlives this function — on the success path the lock is released in the job's + // completion path (the .finally below), not when apply() returns. + const releaseLock = acquireApplyLock(sourceBaseId, destBaseId, planId); + let lockHeldByRecordsJob = false; + try { + // Schema-only: fingerprintSchema + buildIndex use table/field/view METADATA only, not per-view + // live config — so skip the getView/readData storm (one ~1s read per view) on the dest here. + const destSnapshot = await snapshotSchemaOnly(client, destBaseId); + if (fingerprintSchema(destSnapshot) !== fullPlan.destFingerprint) { + return renderApplyResult({ planId, aborted: true, reason: 'DRIFT', warnings: [{ code: 'DRIFT', message: `Destination changed since plan ${planId}. Re-run mode=plan.` }] }); + } - // Pre-flight: validate fieldMappings synchronously before launching the background job. - // This makes the FIELD_MAP_INVALID catch in the tool handler reachable (the background job - // re-validates as a safety net, but by then the caller has already returned). - if (fieldMappings && Object.keys(fieldMappings).length > 0) { - const srcSnapshot = await snapshotSchemaOnly(client, sourceBaseId); - const { errors } = validateFieldMappings(srcSnapshot, destSnapshot, fieldMappings); - if (errors.length > 0) { - const err = new Error('Field mapping validation failed'); - err.code = 'FIELD_MAP_INVALID'; - err.mappingErrors = errors; - throw err; + // Pre-flight: validate fieldMappings synchronously before launching the background job. + // This makes the FIELD_MAP_INVALID catch in the tool handler reachable (the background job + // re-validates as a safety net, but by then the caller has already returned). + if (fieldMappings && Object.keys(fieldMappings).length > 0) { + const srcSnapshot = await snapshotSchemaOnly(client, sourceBaseId); + const { errors } = validateFieldMappings(srcSnapshot, destSnapshot, fieldMappings); + if (errors.length > 0) { + const err = new Error('Field mapping validation failed'); + err.code = 'FIELD_MAP_INVALID'; + err.mappingErrors = errors; + throw err; + } } - } - const journal = loadJournal(sourceBaseId, destBaseId, planId) ?? newJournal(planId, runStartedAt); - // C1: preserve the persisted records/attachments identity map across applies (see buildRunIdmap). - const idmap = buildRunIdmap(sourceBaseId, destBaseId, fullPlan, journal); + const journal = loadJournal(sourceBaseId, destBaseId, planId) ?? newJournal(planId, runStartedAt); + // C1: preserve the persisted records/attachments identity map across applies (see buildRunIdmap). + const idmap = buildRunIdmap(sourceBaseId, destBaseId, fullPlan, journal); - const result = await applyPlan({ - client, plan: fullPlan, destAppId: destBaseId, destSnapshot, idmap, journal, - persist: (m, j) => { saveIdmap(sourceBaseId, destBaseId, m); saveJournal(sourceBaseId, destBaseId, j); }, - skip, confirmRetypes, - }); + const result = await applyPlan({ + client, plan: fullPlan, destAppId: destBaseId, destSnapshot, idmap, journal, + persist: (m, j) => { saveIdmap(sourceBaseId, destBaseId, m); saveJournal(sourceBaseId, destBaseId, j); }, + skip, confirmRetypes, + }); - // Schema extras phase: delete dest-only fields/views/tables under mirror, gated. Runs after - // applyPlan so matched fields/views exist (orphan deps safe) and BEFORE the records job so - // the schema is clean before record sync begins. Mutates `result` in-place. - if (!result.aborted) { - await pruneSchema({ client, destAppId: destBaseId, plan: fullPlan, policy, policyOverrides, confirmDeletions, confirmTableDeletions, result }); + // Schema extras phase: delete dest-only fields/views/tables under mirror, gated. Runs after + // applyPlan so matched fields/views exist (orphan deps safe) and BEFORE the records job so + // the schema is clean before record sync begins. Mutates `result` in-place. + if (!result.aborted) { + await pruneSchema({ client, destAppId: destBaseId, plan: fullPlan, policy, policyOverrides, confirmDeletions, confirmTableDeletions, result }); - // Records phase: runs after schema apply, only if not aborted. It is minutes-long for large - // bases, so we launch it in the BACKGROUND (fire-and-forget) and return immediately — a single - // blocking apply call would look hung / time out. The phase persists progress (idmap + records - // journal) and writes a status file; poll via `sync_base mode=status`. - writeRecordsJobStatus(sourceBaseId, destBaseId, planId, { status: 'running', startedAt: runStartedAt }); - applyRecordsImpl({ client, sourceBaseId, destBaseId, planId, runStartedAt, policy, policyOverrides, confirmDeletions, fieldMappings, naturalKeys }) - .then((r) => writeRecordsJobStatus(sourceBaseId, destBaseId, planId, { - status: 'done', startedAt: runStartedAt, finishedAt: new Date().toISOString(), - result: { - created: r.created, updated: r.updated, failed: r.failed, - matched: r.matched || 0, - attachmentsUploaded: r.attachmentsUploaded || 0, viewFiltersReapplied: r.viewFiltersReapplied || 0, - deleted: r.deleted || 0, - warningCount: (r.warnings || []).length, - warnings: r.warnings || [], - }, - })) - .catch((err) => writeRecordsJobStatus(sourceBaseId, destBaseId, planId, { - status: 'failed', startedAt: runStartedAt, finishedAt: new Date().toISOString(), - error: String(err && err.message ? err.message : err), - })); - result.records = { status: 'running', jobId: planId }; - } + // Records phase: runs after schema apply, only if not aborted. It is minutes-long for large + // bases, so we launch it in the BACKGROUND (fire-and-forget) and return immediately — a single + // blocking apply call would look hung / time out. The phase persists progress (idmap + records + // journal) and writes a status file; poll via `sync_base mode=status`. + writeRecordsJobStatus(sourceBaseId, destBaseId, planId, { status: 'running', startedAt: runStartedAt }); + lockHeldByRecordsJob = true; // the job's .finally below owns the release from here on + applyRecordsImpl({ client, sourceBaseId, destBaseId, planId, runStartedAt, policy, policyOverrides, confirmDeletions, fieldMappings, naturalKeys }) + .then((r) => writeRecordsJobStatus(sourceBaseId, destBaseId, planId, { + status: 'done', startedAt: runStartedAt, finishedAt: new Date().toISOString(), + result: { + created: r.created, updated: r.updated, failed: r.failed, + matched: r.matched || 0, + attachmentsUploaded: r.attachmentsUploaded || 0, viewFiltersReapplied: r.viewFiltersReapplied || 0, + deleted: r.deleted || 0, + warningCount: (r.warnings || []).length, + warnings: r.warnings || [], + }, + })) + .catch((err) => writeRecordsJobStatus(sourceBaseId, destBaseId, planId, { + status: 'failed', startedAt: runStartedAt, finishedAt: new Date().toISOString(), + error: String(err && err.message ? err.message : err), + })) + .finally(releaseLock); + result.records = { status: 'running', jobId: planId }; + } - saveState(sourceBaseId, destBaseId, { sourceBaseId, destBaseId, engineVersion: ENGINE_VERSION, lastPlanId: planId, lastApplyAt: runStartedAt }); - return renderApplyResult(result); + saveState(sourceBaseId, destBaseId, { sourceBaseId, destBaseId, engineVersion: ENGINE_VERSION, lastPlanId: planId, lastApplyAt: runStartedAt }); + return renderApplyResult(result); + } finally { + // Synchronous exits (DRIFT abort, FIELD_MAP_INVALID, applyPlan throw) never launched the + // records job — release here. When the job launched, it owns the release. + if (!lockHeldByRecordsJob) releaseLock(); + } } /** diff --git a/packages/mcp-server/test/sync/test-apply-lock.test.js b/packages/mcp-server/test/sync/test-apply-lock.test.js new file mode 100644 index 0000000..18f63a2 --- /dev/null +++ b/packages/mcp-server/test/sync/test-apply-lock.test.js @@ -0,0 +1,101 @@ +// Regression tests: per-pair apply lock (audit: two concurrent mode=apply runs on the same +// src/dest pair ran concurrent background records jobs that duplicated records and +// last-writer-wins-clobbered idmap.json). apply() must refuse while a LIVE lock exists, +// replace a stale lock (dead pid), and release the lock once the background records job settles. +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, mkdirSync, writeFileSync, existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { MockClient } from './helpers/mock-client.js'; +import { apply, fingerprintSchema } from '../../src/sync/index.js'; +import { snapshotBase } from '../../src/sync/snapshot.js'; +import { savePlan, syncDir } from '../../src/sync/idmap.js'; + +const SRC = 'appSSSSSSSSSSSSSS'; +const DEST = 'appDDDDDDDDDDDDDD'; + +async function seedPlan(client, planId) { + const dest = await snapshotBase(client, DEST); + const plan = { + planId, engineVersion: '2b', destFingerprint: fingerprintSchema(dest), + sourceBaseId: SRC, destBaseId: DEST, + idmap: { tables: {}, fields: {}, views: {} }, + actions: [{ kind: 'createTable', sourceTableId: 'tS', name: 'T' }], + orphans: [], warnings: [], + }; + savePlan(SRC, DEST, plan); + return plan; +} + +function writeLock(lock) { + mkdirSync(syncDir(SRC, DEST), { recursive: true }); + writeFileSync(join(syncDir(SRC, DEST), 'apply.lock'), JSON.stringify(lock)); +} + +async function waitForLockRelease(timeoutMs = 5000) { + const p = join(syncDir(SRC, DEST), 'apply.lock'); + const deadline = Date.now() + timeoutMs; + while (existsSync(p)) { + if (Date.now() > deadline) return false; + await new Promise((r) => setTimeout(r, 20)); + } + return true; +} + +describe('sync index.apply — per-pair apply lock', () => { + it('refuses with APPLY_LOCKED while a live lock exists (no mutation)', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'apply-lock-live-')); + const client = new MockClient(); + await seedPlan(client, 'plnLOCK'); + // A live holder: our own pid is definitionally alive. + writeLock({ pid: process.pid, planId: 'plnOTHER', startedAt: 'ts0' }); + + await assert.rejects( + () => apply({ client, sourceBaseId: SRC, destBaseId: DEST, planId: 'plnLOCK', runStartedAt: 'ts' }), + (e) => e.code === 'APPLY_LOCKED' && /plnOTHER/.test(e.message), + 'apply must refuse while a live apply lock exists', + ); + // Nothing was mutated in dest. + assert.equal((await client.getApplicationData(DEST)).data.tableSchemas.length, 0, 'no mutation while locked'); + }); + + it('replaces a stale lock (dead pid) and applies normally', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'apply-lock-stale-')); + const client = new MockClient(); + await seedPlan(client, 'plnSTALE'); + // A pid that cannot exist → process.kill(pid, 0) throws → stale. + writeLock({ pid: 2147483646, planId: 'plnDEAD', startedAt: 'ts0' }); + + const out = await apply({ client, sourceBaseId: SRC, destBaseId: DEST, planId: 'plnSTALE', runStartedAt: 'ts' }); + assert.match(out.human, /created: 1/, 'stale lock must be replaced, apply proceeds'); + }); + + it('releases the lock after the background records job settles', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'apply-lock-rel-')); + const client = new MockClient(); + await seedPlan(client, 'plnREL'); + const out = await apply({ client, sourceBaseId: SRC, destBaseId: DEST, planId: 'plnREL', runStartedAt: 'ts' }); + assert.match(out.human, /created: 1/); + // The synchronous phase returns while the records job is still running — the lock is held + // until the job's completion path releases it. + const released = await waitForLockRelease(); + assert.ok(released, 'apply.lock must be removed once the background records job settles'); + }); + + it('releases the lock on a synchronous abort (DRIFT) — no background job launched', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'apply-lock-drift-')); + const client = new MockClient(); + const plan = { + planId: 'plnDR', engineVersion: '2b', destFingerprint: 'STALE', + sourceBaseId: SRC, destBaseId: DEST, + idmap: { tables: {}, fields: {} }, + actions: [{ kind: 'createTable', sourceTableId: 'tS', name: 'T' }], + orphans: [], warnings: [], + }; + savePlan(SRC, DEST, plan); + const out = await apply({ client, sourceBaseId: SRC, destBaseId: DEST, planId: 'plnDR', runStartedAt: 'ts' }); + assert.match(out.human, /DRIFT/); + assert.ok(!existsSync(join(syncDir(SRC, DEST), 'apply.lock')), 'lock must be released after a DRIFT abort'); + }); +}); From ac370e01153da1172086e593007c413d20a81007 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 14:04:30 +0300 Subject: [PATCH 147/246] fix(sync): use escaped separator in arrayCellEquals (drop raw NUL byte) The join separator was an accidental literal U+0000 byte, which made git treat records.js as a binary file. Same semantics via the escape sequence. Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/records.js | Bin 68942 -> 67552 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index f7777555f96bd945c21de9b263ffc0cccfd48785..09a6a8af5302e497d503259e2c87b2c8d83ad617 100644 GIT binary patch delta 9112 zcmbta30zd=+W(CR;sA=V?`K41UZMW>+x|R0j_q^{pGvIdb@B6;<`yJ-p-}l-6&+|Tq zzO92l-7wfq7Z=w%xT<$Xt(#Ja>phUy`>O$U3K2&Yde#EymEp*sGG@ET1%5Ei6h8Ci0Xahc^H$z#6`;?hDtE?QP>_)6tH5Pdl;;2_QtLA$bW1V+4 zPIyP-lJ}G$G@Bo@$fq8U`PAU5PZ>XA{`5e$0TPtaRMq0?e- z?=-ZC#stP9g>N=;eaA{izQeajru?{XtwZ6V@Q<`F>=4=fTvHx<=BWy_}^BAW_l0amYPL_`0Nh7Ro^Cb#dQn%>*`Ise)HAG`Kd77Zvxi& zi7H~9ASBq@t#);+RG!yvvZ!O~Iuq%MiOs+W|8UniQvJuH-9H<9{c|`=_6PXEJwPi< zOGm%q@QaKjDduuIu_|&Li7#E^!<~U;0knsq-D0y#k`l6nuI>yaI<^FQ+pLC3CeduP z#R^>;gJPf#QllzJi^V}{I2e?HyiuVT9^8QTU{f$ZDLeTBz13o}%(W$SblO_fU07EW z0a$<>K}*PP;X2L z)ni|1J%)u0`E2_Na=N?L@Z?+-?(@N|R+jz%Qkic@tG zry`(BdPZ2|22nxKA|=7gbt>j%>Qk#fo%+m_0*p&Opz|PG&*uxsH`e z^wb0*IXVbu7Wknd`Yu5p*U!i6`R9~Ud+^9G5~6Fn>sG!&NRhZYD(inEsMgVQ|D&*k zF(ocwdHT`8To_M3>W&8xSXdm=sq%Cq@z)8`l++A-9ir_HW!33(NB{FsqZC zP1wF`G@h=Hhwfk?gjf$xGCLhtmZaD^tdFh5H?bT>Q4-gH-EocZ(n#lGKxfgg{VMU9 zra_YXWbFhj(MI5Ytqv!(nNY@uGL3ZFj7_wY#aP4z!c>v$m24tjjd}3~^u>>ddqSDU zDI)$Wsku??uqDt$wLQUP)CyGMcF>Jw34ypTAp!>yM&fn<@~9o>kB)IQG9ZLAd`M58g4M~Hcs)6a1TEn0kO;V^ z6iJB2XlL*BZh;@LGi4S|rOd#X@ijhNUd^2rgWc3_!TC%b4vi1OyW{f^Fd<$|E0~%E zwaRFZm6u{zVYDY$MFWv-6LNC0gxWb7Xq!;(0?9Wf=%G&4V@;|NKhQfZO~i|7nG%`5 zr>($U=}msLjG3fq70Fu(g5A1M=)&dn3TC6-}fCLRK_tv!V$Lmu1!BTvj&1vtv+@UC;VS zd7j8F38zwXyH!w2nA|M1Hw(>rlUX#fjYs56K|@Xee$2Px%N&ioY-p~AmMqH6!RlNc z4(2A|8!E@;O^syRa^;XHBciGYlD9Lj=4Rw6CyqhV#MRL>`&#l`95Qr@M$-Gl3?ew0 z6I+^}!U$$uWOHn6ghs=;f<5{1@~NLBmJ8i@s6d0h0)coK2Cu?s=nA(C=M$yFcC%8oBJ`#({t8uDa5#nTNb5iARKQ z&a6PtEOwwc!P0Jt7u!1Q3k9>u(#k`jbS#@yij%X9#ECfCEgm^7D$Jf8jm5JI@!ITK z2({uZ9NI8^_f{BAbbI>{M3>Qu#dfjH7VGj+!F7eMa!$-J99kTV+`4eQP&dNW-bL+U zOEijDM9neCt$tXtWVCa{#$ChEwGL?`>(f0X(PM=^UPhSVyC=wYB#>B&n@X^$X#!3(HAz}R z&^5upFi12M(BI+uV05es#U}=4uxGYKL1&B}q?GJqoiU9S&09imW2|-sn0!wtg2o_H ztn!cyVWXhI!y;Swj97<+<}xaEH&>#snV8^Gt>6)5#pR_9)61(1r&g5BC@hH4lClLs z7cvxmtsz)6w>8A2J=ZCn5`$afY1QNwE%YsQINUOh-cWd(YG5>_;(k*RzA!Na@|%~4 z0#f?sd6O7MOHX$!DWj_<`HU0u<`3MCdn*FOg!WeMI$Ftfw8n9(cwk1Le8`y+*7h&U zfYbT7tiQ}0>9lm#$5bBs_67`O1$MOh3?P0YvfF}U2Cn5Y9%;UF9d1kyJQ%^`$>H~v zecxk?B^B>j62Z#@#6BpV=8(;%kqL+Sz9u%2F_l$VluW0YMmIh}Kz8wC zJ(Bc3v}+{b=QF6A?@O_kv;v|#EBOW284>XALMbdDcSgt1uMG7lF5_DGc4r0Q5EGTm zb}9H_yAcbWTjtJJ2l7!A@tf=|)gqllt&SBaJXkv4=#*ZpmXDDaVnn)>ig!TY=ZD31 zFT^kKlnhwF?OoxRxj-#Jw@Zywzedn88&-xpk9lT6J&UUb9+bnWlg7>tbGzPnYa(ND z^o|(G^Y=S&9@h<8Sc$0%Cv%kCxiAwSEv!ezopZ3|P9y%fQ-rZ=I`2E&m5D1|1~e>c z!qG*sgsdv`4Dvx}cR24a-xUGnn_ffh%&T+oUSPbp zeb3?oS%rKNu_Tr04DMK>ZQS!`T+cCX%4jo;*+IQuLsWXC3$wc z-q6xUI%89J5vL?=W&)wed8$RL~mxr z4$*2e+4xKnW~Z5#`W2nDkDQmj6TdCPPRMq%ylbcXs+a$J#on~4Et72@s`)-nhxcvVjc}d={=brpy5?J{c*__j_R<9>A|Kx=a zcJ_>N?ZjwdYU+IY%iSB-s3aq1eb?TJ*&O7qsYLIZDC}R8OzI`zQ$qqB?bJ4gBI|dg zqVpadPWK4zbU&k5?=SbHTua}uwHZ!k(qhxs#{8%KUS6xl7i-hAU8gk}(}iYs`*XEe z^lc(*W54~a44YUh`_-apU7VLZfjJ&*Syzc~*5yIFKAR5SwmutAt)C>55Wiu}H6)C} zjtxp@Vu6dvwGsus$2OE;SWhBydYED`7O-fJ$>wxVEH3v@K8a+N)$2{Ykz0cYQLKBu zHxx^IgR!MI3NQ4I!pFU1a$M9%GI65bjS?CtIj7u+l_eWf{}U^ZZ7g-BJs6aC=xr32 zOVG}?k+ePuI_MFKMKCzEnRUt#%8Nww@6hm3`I{!=o=qchbyI*Z_e(q(!%JG$h_C6r zx{0nMn*x!xc?3!~hofb46jpCeJh~5@~ylo?SdBL`DGS(nw)=1hO z$c_5PZ8=mP!zkIg&gs*x-Ju&-w?`uK{@PoB$;JDpId2-0mXao>-GCO4ZwW@!&ICFD z-;f)zlRyHAegnF8^MRx~iSEX>2SV`N1G%_~U-YBvhXiu^2jirRhIA@vT~NftERsb` z+~1via5nc>+K1*~%|q2V_fQ5x9%c)S6BJf48~ zCzfLG6SHoJ{7Ja1ToaO^2IV#uB% zRPLd(?y2%|#>tY2MqN^&c91Wk_hz7BZ#lpGG52y|FN0#{USju^_k(yplt^+T#k@XR z-TSnbMXGWjdS?F|=$>i7hG*(Hr!PG-3nhP^dJWfEi22^1>C?cov*nn<7cJ+5u;bZ4 zId;g9LR8_o$xs{!$I8#N*#BGwl})_9>OdtAEG{2NkOK=}OnW|r0*NLHEa(6S3!JyF zAoLu31Sv0+V%ZDL42PL}rF&_+)s|ouEpzQHMGUqb_QGR_1sp%zN~0}r1fukn0?EZNHq(YXUJ1tYuPkQkg)#x^ zbkYraT)Lp|4~k$p(kgHH>k*2zUTu?WFTYv{x1&{#^l*8qc1{Swbl)a^*2+y%x&_xazes{PneD41K)_)vwoL@9T5m{l;v}eWMn~ z-WY|i-blm9H%IZYrEfOlnK$+DKVFC1k2m6-<9Z~&H4E$C3b{rZ`c05u-kIP-93QzW zMzIptgAiO?jFU43FYX+kW8 znf^{~cpPt^oP-g5DJ&bpgT!m{m-Gcjj`EiD)#Ky7M&z8DgZod7#pzS&@H(A9uYlR7 zz1e+d4)X{drzhaZS*@dNs3O&|!CMi))T5i4f{=42kTsXe8uG-MG<<%h5e4sAP{5|E`(cRiu2E9GFr7s&3N?^VnU4e?4$``Y6V@wNdP$`)kS)wgpp6X^tc- zn$ks06x*8hmSQ=;(Kpf8ERn@Lddg!-=xpFAHEK(g>YW`%O0o6VZLOw`jsbNk3F;bd z6wMlO;CSaJjdaS-Nh`X7Z}vSa{Okx4dEYEatl4M z;l0hyWRnwhWMjzwn-;Bq+vCR4lY}&GN}Ud`eh>!D`(s(1OsL`mJ@$W4gv-<#_+bq! zA7=5}B5_34Xcpf|pF`QL`oKps24I%k#|F&(Sj5SXi@0ireliobPwH@*-uO@JvG&s` z*Ql%$zu51d#SFWWW!#FvC+9~?8roqU0^ze_8NUihLn)v3S$RKhecpWOrF_0j!qG4D z;qld2nw#;}WZeA~yFl#ustLYdi_k50kC`% z=8{p`k3M+%n{52>jTv>{CQFxW(nk4qe0|VQ@;dNsBZ9tDA?-VsC|>*BT>SK1DRM84 zgyCW}4qRmE#`6~=Xur`I_5F6~JU3bA!|xlB`9nSBOtbL*52+G@IpfZ#OCnZWQcKuP z$8lbuMMg<(4PCD;`B2;wgWG;Aap@SRT+*-sdZqf=4~zaWdVqG^_fI`8|FZ$Jf70Wn zpDH9M8S!%?tUr%s0=hMnT>QBTIsd9A3OD_$0)PLP3Ld}2B9Y$Nzm(undN2KwNR%JE zCDlLuIv$$eRLJ{15Qg9Uapd8`72EHR!z(hAY2~gYI%PLnWeKiMq?v zu>W#2O^e%jC6;ETyN2$$lI9wA=}M(@$op4EVAt;<^3W>2s_DN+5z{t@@>SW&H{9ru z`-c{7f0*eF_Q)2Jq`T@o7_KJ#yIFg$nl=Y3#%#_}L~Nd;2$s_%w2d0b|2CxaXzLlpVUUogzQslVvA!s~GO|B170?88?-3`e;> zO5Yr39a)}=ddGH8@|geSsVKaO19I#fs>pJj8%mCKkWvxnR4K_!L7>(w!YXko`CM9+0hCo0(uB0AA- z1ko4lCJ?=AH<)OD`!Ph9*$*ZbYA0-u9kVZ%my2=GP-mmVIJvF~^I&%zu)x4hn3kUy zz_xXYqS6V^Aa=b|Prj{qHS_C?K@&QM5}ngIgJ@gl!~oi))Bs}8nd%MIQkhP#Hx5!e zQJb1MI$~IkV+qk+oOX4ZMCA%6Z1W|jJUa|zx10*7I@I?_IFpmQm|82;Vw)Nj0pKTM&&6b4#|_m)&>6 zf>lVKVN}`}m_n4~!JhZ*O{Ei_Ns8RMcx8EE)v}suqe*C98Y0bZpp{d4d8>_vvf-6F zz1A2h%~rGJUf?@0D1xXaD2C_{y#mL>ukRR9RiGlY2;W>|F0a z*1xB7FvdR0bcKbj@a;znzUvE*3G;)sE%eJJdd3eeef*~oUG0zk-tdo?CD}8;R+29V z6q7+qQOqr{JCgz(S$tq0n((^q;;Joh(1Yzdu)&%Bl@FM zNOWW{+D*Zw9*PJhDM=y~j??H#pO=E`eUy$F8bftu^;BbAZN0G~f>gWZ8!xsr#E0f@ z30c!a8D^!8-zdRG<2yoo($q)lIkv~d|Gkc{GcWdjk33k;A6NI3o}*j{x0m~_gcItSH7b=Ct9{Xd{FI|TN9(c(pyZ(lAdz3B5>#eJ!qPs8gw^$rQntD< z3N|XtsWbaH3dZFWU80yzRdjz26T(OJmE@>HB$K98uQN(>=13DKlXBal<7v{hXn=A| zj9;M8pqOFQVM4WThBQV;AVZ@yF>v9fF#y%?W3YHgY?hlv>!LuYJ}1_fbK)*0nAI4x zI)hG24EegR#KQTH$0Dx%z{_3Y0GE^E;A;EhrV;HKA5Anb-cxp9T~(#Yq|?S}wAu($ zT%}gHytNz4i^wrZIP!Dx5a`YLVMO~UJCwoKc`+ElwJ=?n?LJ{A8 z#ir6e2>1v3j3esScf7J4eP007fxf9kf9V?=4Stp7Qbe%U6e-7OE=jE^&Y&wJSsA6| zWLY2i{c>!=#nyg+xla8-EBb4R9_KV+fR4&*1`Jjf{riBYsiA0KDVKzvOva!?5+F$? z!z_UR$$_~XR~n=;jj60ciqIJhJSrq4h0@HXVMXj(nge?+Da2C!f^@597n5M3 z2U7dVC`A10^`^KogHA)%5FHJ>95kGUygaA?i4%Zb&F&8Z_u#>y9Z|~|41a1KjL7$o z!EkH)A;q0A7-n5DWOR3w%WDi$gfQ1~sis^i*Hr3tTE5v2hGcsy+vCfNKE8U>FavvG zXmtD99}bP?tKJ`qcrYXx``0G}7&a!OoFvKGD68jwmp~_7oC=IMkUA|^nem`3NeoEnjF^Peae1=bVX?n+ z{2ed`5<5#t<5r(0@vQ;-wx)#wh_|fvVVjdY*sV0Z9AxAWLAv7Vs!QKSqkS`?xZu(= z%Bi+%ay+knmVx9nb~vzS>~L7*-rQe-Arud>6B@{Q=q0R{bRHs_e(wu~*XRjXL<-N7?YufjMBgGc$m-=AbSX z8k3vlt}H=f$H}?$cjs=UF@<@M?dy5im`^@5@MJzvdVN09z|Zps5%nsVkE6yf?k*pZ z&p%l37WK^@6YQdJ7tZg*c2)$lqhoTZ*S8ST$t~<1sEon#vNKzGQ{L)e8dta4gWWsb ziJdKUK?)_YwgmxXLC);*Feer?c7VS%6RUIs2U@kmi#S}l?8i2b?M9{6u|e2#TOs`Z z@Pa_jCv;p-OIAaZgeS@Z!SQ5s=TahqV~Tk4>^P+O=f-uH^M!lkaJsDVXx%U#xx=3E znM8jb58Etz4CURAWe~mb7!+UE?9b+Pa9}kP(jSs>IJ1D+;Vhs?O(T(EOxZ$NM-h^K zO;|YFT?9TN^GKr7@QKhB>W<;g?9{}ba?N=X{4l>ElD!xn$wVNMkfB@l$Z^-7*Eo`gBUEqtGRgI8`H%vr3cBM`D$0i~XU#+;wsImAoW~ z^)3x1qo9iLxaFk-MYVRlpR>NQdYaYM<60>s;a`@|*;SP#P;FzGSmgaO_()G}7Vk4? zp~1JckfE&&Ha~99B6M99JM^F{NC>OaO|vm%D+`jH>bfqLGY%>Ts$`a9vCqo=SX;R- zFV&%S>Hwk(rXoE*HdTP%t*O{#SOwf*M8$Ba-O?K(IZkoEg{jIBc6-Hi8X8pTD|VQ_ zS`vjrg?_C0aWw~L?m^hOW^6k@Ztm@t&#MZKSQbI6?4<(xl(2vf{BRdxsvVxRi3bIz zwia3@bD8EX2OfE<_2Qh5g`q6O#7%)5qN$Vqk@rmdftD%JLu_U}1SMKtuR?S`;vZIG^`t7it_WB|8U#{ZU$_ z!_Hy2UW;mCEjbiXe}_tKm8O;#I|-4&3H`g2>nQOVHp`?l@;$qc_o}Vcf$uE%0N-MhAG=+D~My08uKu5h15s}i6p=#F6qzx0?DB8w|lA?VisU)wsnrS@TtusYq58iBe zHz|>yii|NLs}=NkB}kTcgA3pFsO= zYHum>VuRdFEpMzY;=T;!Hd=O7b8Us!{P2Y?{defPMsHz#?rD-B3T^BcU_ zuqP?UdPq;U|@s=_{$i#ZA|wG1Zz{V?~XqYbnvOY2XgQhsrk3 z%a7MknyS(Ba;?r_q?;)(3!mXdi_5SPbG;~0Z{U_uI=6`S`{vvL?j&dD#!^km5`P

6!V6O@U992;^$r>dsa@l}b%#o&wr* zTmY48768I2uORx|0+CkCq}2qf2@_=L3-OBCxUe70HFI4r{7T&5&T^KxQhUPsL1@TZ zE=%1WJY~$% zL0y&oaD=X13VHmIjhtBu%)YxcX_&%*(q3(nR8A{$BM=9x)>P@xh~HMtfIKwgcZ`}D zmqn=+Hat$UZOd@ZpO-;-1D_Gsftk-huzQ~YV2gA1eKw5BHs_4W346rxu~|2GyMtX~sERM`Y`Y-#dj%}uVn#P4@h zzuJV?VDxlbg$zGsm1k!jn~h`%DXSpal2uR;4e6_NSnV6iw2`uPP3-fdRmgii7<_m# z6Di`K*)Xbo&w%twrW2`TI8p4#>iTZhhv#ae9B&P>JGLfVn1d%fwZ>0ON?ea5K6{Z5 zAV4(Df7+e>vk^gg{#qQMbuG;EleHrgtvN%S7G0zAKo#m2*fPZoTU=^}*&i{3<4}~7gXn0@c@C%WW31n}Cq92-7T-cZ*ze&j_kOfUabW@-!HbI=}pPWxl*ixw61!&kZKoSMtBL@md4x4w!y)6ZX3*N-)-wlHO&Ga z)^B?xZ<(?k&a-VhWOZqKJ_%jXy$nfNmxI~VKWEB!NKz0x*c`|{`7@clcn1)M*>?v{ zfbT~$cjVYysmL&GY#aK7ejjq}xUmy(PZy5%J4CnL&~(jiNa=&!lVn}SzA}O8i(VOH zqg!NNE-k(iGmNM$|$O*n~&6Vq;UZpIJ_q_V*I9~E+ zpU!b*-_rk&{(&q4{K6Vs*@D-Qq2xD2vm>wJnvHhfJp^B{f=IS-kH2yys!R7~`}e#= zV~StTB)a|ea-uGKfk9z=i)F#mqb=L~ihl=61h(y!nM$WVL# z1-8@l7sQ|U|HAjYiGS1_d2ca^DD|@9qQm zA+eiv9EbDUk2_M%ei(^kX1Mv3y~=^L?6;@t>-)*V+28jgQ%*TBQMtItK8j!Q@lu|~ zXq0*-Nq8d=-z>3Akd zpieN{cxWN@(9)FJft|R8$(M2xeLDhW9FCMNuI_La%{h1&0=s?~J~XfmR<*hf_IkYy zd35I6XxaL9528okhF5lcr-wX0>K$0Y%6BwGufKzPL-t>%5Z(1xv^c*z0cmCw?IE@b zuln>wsdSb!U8AqpMaH2A*yBSZfj|3BJvZGndo^I`y9m<^Ff13K)`@^YU z>Y#$TegvzM`S|c4SpL#Q@b2>|j%pJd8qVU{f8i`5Y{q9J zWzSmq*<)0{^cj3A{d3Uf&tZc1KhOT1XtBE`oLxBwp#aGpTEkh^`JPm(JFn$!h!p_N zzW)&Qhs6coJ#nnpKjCkM|6E2IQNBO7`-?a8ys(7&H(r1XoV%CCVpe^FsG}$!T=2L9rCZK3ZJ#)qB?HxS1#kyOsNX3>B5G6pJrK;n;kc6 zPWYO_;+9+)M00E`Zto8;Y45AZMy6jy`ID=;ih852VW-QkLA9h*q5=s!T!#Y2Trc3N z*?Jwx#?9-HnAg7n)ybSz{2S)=DyP5v8-@W1-M{U@Cf`_W+3mw5boT~8J@H3yJayB{ zeAP~cTXM@!32f(&PE?wFE1KvRKf(on`4Kn2_?w|}p2yES;U*HtT{lCCp1K)Dbn;Kw zS<6p=PV4C}-Wq5<@vB>Kcx4Lm5X%(y<{b|f`ExgF6DyDV8F3IQxPLp7ef_iYfC_Cr za4cwoMcxL+wB7b$RkyLdw%b5$F@IK|Vloe(+jIx`f9DR;8|g00ZOC08`R>RSPK2La zXgco~*p-c@=iTckqs6{^5bS67CKBy?e;Cmj_kk*B?i-NjMoInRq%oT56roKOI;qxB zUrk?wIZ(&Z=c!*Gr=cf)EwzE3KNz(CHrkw{Qn?Ex(I{p{`K0;0N=43Z6>uqrK_U;2 zU;n4*Bm^yz5UcR@KyY%079WUC<{KSUoQpRrSs zjhJ7tQ^5umTOO|HTBFxzRZiEX(eU3Hl*mg_`q zPP11LMw*w|gS&Z;y-In0w|sw*gS7lv-io97p1q1J-JI>f6OShxR3pW6!iD2ra(-gI g=Ag>u*$B5@%F20W4=ZzGEhoc-bd+oQLsHED0Chv&@&Et; From fc00f3d8c9978afbe26fa4470353507ae164d7db Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 14:04:59 +0300 Subject: [PATCH 148/246] fix(sync): make journal resume reachable past the drift guard A partial apply mutates the dest, so re-running the same planId always tripped the fingerprint drift guard -- the journal's documented resume/retry path was dead code and every partial failure forced a full re-plan. When the plan's journal already has at least one done action (proof the divergence is at least partly our own run-1 mutations), bypass the guard and emit RESUME_DRIFT_BYPASS instead of aborting; a journal with no done actions still aborts on real drift. Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/index.js | 19 +++- .../test/sync/test-resume-drift.test.js | 88 +++++++++++++++++++ 2 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 packages/mcp-server/test/sync/test-resume-drift.test.js diff --git a/packages/mcp-server/src/sync/index.js b/packages/mcp-server/src/sync/index.js index 15a4eb1..ac06c32 100644 --- a/packages/mcp-server/src/sync/index.js +++ b/packages/mcp-server/src/sync/index.js @@ -145,8 +145,18 @@ export async function apply({ client, sourceBaseId, destBaseId, planId, runStart // Schema-only: fingerprintSchema + buildIndex use table/field/view METADATA only, not per-view // live config — so skip the getView/readData storm (one ~1s read per view) on the dest here. const destSnapshot = await snapshotSchemaOnly(client, destBaseId); + const journal = loadJournal(sourceBaseId, destBaseId, planId) ?? newJournal(planId, runStartedAt); + // Resume path: a journal with done actions proves a prior partial run of THIS plan already + // mutated the dest, so the fingerprint divergence is (at least partly) our own doing — + // without this bypass the drift guard made journal resume/retry unreachable, forcing a full + // re-plan after every partial failure. Warn instead of aborting. + const resuming = journal.actions.some((a) => a.status === 'done'); + let resumeDriftBypass = false; if (fingerprintSchema(destSnapshot) !== fullPlan.destFingerprint) { - return renderApplyResult({ planId, aborted: true, reason: 'DRIFT', warnings: [{ code: 'DRIFT', message: `Destination changed since plan ${planId}. Re-run mode=plan.` }] }); + if (!resuming) { + return renderApplyResult({ planId, aborted: true, reason: 'DRIFT', warnings: [{ code: 'DRIFT', message: `Destination changed since plan ${planId}. Re-run mode=plan.` }] }); + } + resumeDriftBypass = true; } // Pre-flight: validate fieldMappings synchronously before launching the background job. @@ -163,7 +173,6 @@ export async function apply({ client, sourceBaseId, destBaseId, planId, runStart } } - const journal = loadJournal(sourceBaseId, destBaseId, planId) ?? newJournal(planId, runStartedAt); // C1: preserve the persisted records/attachments identity map across applies (see buildRunIdmap). const idmap = buildRunIdmap(sourceBaseId, destBaseId, fullPlan, journal); @@ -172,6 +181,12 @@ export async function apply({ client, sourceBaseId, destBaseId, planId, runStart persist: (m, j) => { saveIdmap(sourceBaseId, destBaseId, m); saveJournal(sourceBaseId, destBaseId, j); }, skip, confirmRetypes, }); + if (resumeDriftBypass) { + result.warnings.unshift({ + code: 'RESUME_DRIFT_BYPASS', + message: `Destination fingerprint differs from plan ${planId}, but its journal shows prior progress — resumed non-done actions instead of aborting. If the dest was ALSO changed by someone else since the plan, re-run mode=plan to pick those changes up.`, + }); + } // Schema extras phase: delete dest-only fields/views/tables under mirror, gated. Runs after // applyPlan so matched fields/views exist (orphan deps safe) and BEFORE the records job so diff --git a/packages/mcp-server/test/sync/test-resume-drift.test.js b/packages/mcp-server/test/sync/test-resume-drift.test.js new file mode 100644 index 0000000..1d9857c --- /dev/null +++ b/packages/mcp-server/test/sync/test-resume-drift.test.js @@ -0,0 +1,88 @@ +// Regression tests: journal resume must be reachable past the drift guard. +// +// Audit finding: after a partial apply mutated the dest, re-running the SAME planId always +// tripped the fingerprint drift guard (the divergence being the sync's own run-1 mutations), +// so the journal's documented resume/retry path was dead code. When a journal for this planId +// already has at least one done action, apply must bypass the guard and emit +// RESUME_DRIFT_BYPASS instead of aborting. +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { MockClient } from './helpers/mock-client.js'; +import { apply } from '../../src/sync/index.js'; +import { savePlan, saveIdmap } from '../../src/sync/idmap.js'; +import { newJournal, recordDone, recordFailed, saveJournal } from '../../src/sync/journal.js'; + +const SRC = 'appSSSSSSSSSSSSSS'; +const DEST = 'appDDDDDDDDDDDDDD'; + +function twoTablePlan(planId) { + return { + planId, engineVersion: '2b', + // Plan-time fingerprint (empty dest) — run 1's own createTable makes the live dest diverge. + destFingerprint: 'PLAN-TIME-FINGERPRINT-OF-EMPTY-DEST', + sourceBaseId: SRC, destBaseId: DEST, + idmap: { tables: {}, fields: {}, views: {} }, + actions: [ + { kind: 'createTable', sourceTableId: 'tA', name: 'TableA' }, + { kind: 'createTable', sourceTableId: 'tB', name: 'TableB' }, + ], + orphans: [], warnings: [], + }; +} + +describe('sync index.apply — journal resume bypasses the drift guard', () => { + it('re-run with a partially-done journal resumes (RESUME_DRIFT_BYPASS) instead of aborting DRIFT', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'resume-drift-')); + const client = new MockClient(); + savePlan(SRC, DEST, twoTablePlan('plnRES')); + + // Simulate run 1: TableA was created (mutating dest → fingerprint diverges), journal + // marks action 0 done, then the run died before action 1. + const { tableId } = await client.createTable(DEST, 'TableA'); + const journal = newJournal('plnRES', 't0'); + recordDone(journal, 0, 'createTable', tableId); + saveJournal(SRC, DEST, journal); + saveIdmap(SRC, DEST, { tables: { tA: tableId }, fields: {}, views: {}, records: {}, attachments: {} }); + + const out = await apply({ client, sourceBaseId: SRC, destBaseId: DEST, planId: 'plnRES', runStartedAt: 't1' }); + + assert.notEqual(out.machine.aborted, true, `resume must not abort: ${out.human}`); + assert.ok( + (out.machine.warnings || []).some((w) => w.code === 'RESUME_DRIFT_BYPASS'), + `expected RESUME_DRIFT_BYPASS warning, got: ${JSON.stringify(out.machine.warnings)}`, + ); + // Action 0 skipped (journaled done), action 1 applied. + assert.match(out.human, /created: 1/, `TableB must be created on resume: ${out.human}`); + const names = (await client.getApplicationData(DEST)).data.tableSchemas.map((t) => t.name).sort(); + assert.deepEqual(names, ['TableA', 'TableB'], 'dest must hold both tables after resume'); + }); + + it('journal with zero done actions does NOT bypass the drift guard', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'resume-drift-neg-')); + const client = new MockClient(); + savePlan(SRC, DEST, twoTablePlan('plnNOB')); + + // Dest was mutated by SOMEONE ELSE (real drift); journal exists but proves no prior progress. + await client.createTable(DEST, 'Interloper'); + const journal = newJournal('plnNOB', 't0'); + recordFailed(journal, 0, 'createTable', 'transient'); + saveJournal(SRC, DEST, journal); + + const out = await apply({ client, sourceBaseId: SRC, destBaseId: DEST, planId: 'plnNOB', runStartedAt: 't1' }); + assert.equal(out.machine.aborted, true, 'real drift with no prior progress must still abort'); + assert.match(out.human, /DRIFT/); + }); + + it('no journal + fingerprint mismatch still aborts DRIFT (existing contract)', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'resume-drift-none-')); + const client = new MockClient(); + savePlan(SRC, DEST, twoTablePlan('plnNOJ')); + await client.createTable(DEST, 'Interloper'); + const out = await apply({ client, sourceBaseId: SRC, destBaseId: DEST, planId: 'plnNOJ', runStartedAt: 't1' }); + assert.equal(out.machine.aborted, true); + assert.match(out.human, /DRIFT/); + }); +}); From 35371583684ff7eab54e8f3bc33a9b494bf7e48e Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 14:06:33 +0300 Subject: [PATCH 149/246] fix(sync): propagate cells cleared on source as explicit nulls (source-wins) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit readQueries omits empty cells, so a cell the user cleared on the source base arrived as an absent key and buildUpdateCells skipped it — the dest kept the stale value forever under overlay/mirror, silently violating the mirror contract (make dest identical to source). buildUpdateCells now iterates the mapped field set: when a writable non-array field is absent from the source cells and the dest record snapshot still holds a value, an explicit null is emitted (coercePass1Cell/updatePrimitiveCell already support null clears). Never cleared: unmapped/dest-only fields, computed fields, array cells (Pass 2/3 own those), records whose dest cells are unknown, and anything under conflicts=dest-wins (the caller skips the update entirely before buildUpdateCells runs). Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/records.js | 21 ++- .../mcp-server/test/sync/test-records.test.js | 141 ++++++++++++++++++ 2 files changed, 159 insertions(+), 3 deletions(-) diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index 09a6a8a..731027d 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -18,7 +18,7 @@ import { coercePass1Cell, partitionLinkValue, linkRecId, coerceMappedValue, isWr import { resolvePolicy, validateFieldMappings } from './policy.js'; import { withRetry, createLimiter } from './ratelimit.js'; import { remapViewConfig, collectFilterRecordRefs } from './remap.js'; -import { snapshotSchemaOnly, snapshotViews, snapshotTableRecords } from './snapshot.js'; +import { snapshotSchemaOnly, snapshotViews, snapshotTableRecords, isComputedType } from './snapshot.js'; import { loadIdmap, saveIdmap, syncDir } from './idmap.js'; import { newJournal, loadRecordsJournal, saveRecordsJournal } from './journal.js'; import { safeAtomicWriteFileSync } from '../safe-write.js'; @@ -147,6 +147,13 @@ function buildCreateCells(srcFields, srcCells, idmap, warnings, srcRecId) { * (no warning spam for already-converged cells; when the dest record's cells are unknown, * the warning is emitted conservatively). * + * Cleared cells: readQueries omits empty cells, so a cell CLEARED on source arrives as an + * ABSENT key. Since buildUpdateCells only runs under conflicts=source-wins, the clear must + * propagate: an explicit null is emitted for a mapped, writable, non-array field whose dest + * cell is known to still hold a value. Never cleared: unmapped fields, computed fields, + * array cells (Pass 2/3 / deferral own those), and anything when the dest record's cells are + * unknown (dest record absent from the snapshot). + * * @param {Array} srcFields - source table field descriptors * @param {object} srcCells - source record cellValuesByColumnId * @param {object|undefined} destCells - dest record cellValuesByColumnId (undefined when the @@ -159,13 +166,13 @@ function buildCreateCells(srcFields, srcCells, idmap, warnings, srcRecId) { function buildUpdateCells(srcFields, srcCells, destCells, idmap, warnings, srcRecId) { const cells = {}; for (const field of srcFields) { - const srcVal = srcCells[field.id]; - if (srcVal === undefined) continue; const mapping = idmap.fields[field.id]; if (!mapping) continue; // field not mapped → skip + const srcVal = srcCells[field.id]; // ABSENT key = cell is EMPTY on source // Array-shaped cells are never accepted by updateRecords → defer if (ARRAY_CELL_TYPES.has(field.type)) { if (!isWritableForRecords(field)) continue; // links/attachments: Pass 2 / Pass 3 own these + if (srcVal === undefined && !destCells) continue; // empty src, unknown dest — nothing to report // multiSelect: remap source choice ids into the dest vocabulary before comparing // (unmappable choices keep a sentinel so they always count as a difference). const cmp = isMultiSelect(field.type) @@ -178,6 +185,14 @@ function buildUpdateCells(srcFields, srcCells, destCells, idmap, warnings, srcRe }); continue; } + if (isComputedType(field.type)) continue; // Pass 1 never writes (or clears) computed cells + if (srcVal === undefined) { + // Cleared on source: propagate the clear (source-wins) as an explicit null, but only + // when the dest record is known to still hold a value for this field. + const destVal = destCells ? destCells[mapping.destFld] : undefined; + if (destVal !== undefined && destVal !== null) cells[mapping.destFld] = null; + continue; + } const coerced = coercePass1Cell(field, srcVal, idmap); if (!coerced.write) { if (field.type === 'select') { diff --git a/packages/mcp-server/test/sync/test-records.test.js b/packages/mcp-server/test/sync/test-records.test.js index ea45770..c8f6480 100644 --- a/packages/mcp-server/test/sync/test-records.test.js +++ b/packages/mcp-server/test/sync/test-records.test.js @@ -236,6 +236,147 @@ describe('records.applyRecordsPass1', () => { // A truly diverged multiSelect must still warn (pinned by the earlier test with no dest record). }); + it('propagates a cell cleared on source as an explicit null under source-wins', async () => { + // readQueries omits empty cells: a cleared source cell arrives as an ABSENT key. + // Under source-wins the clear must reach the dest, or the stale value lives forever. + const updateCalls = []; + const client = { + createRecords: async () => ({ created: [], failed: [] }), + updateRecords: async (appId, tableId, rows) => { + updateCalls.push({ rows }); + return { updated: rows.map((r) => r.rowId), failed: [] }; + }, + }; + const idmap = { + tables: { tblS: 'tblD' }, + fields: { + fldT: { destFld: 'fldTD', choices: {} }, + fldN: { destFld: 'fldND', choices: {} }, + }, + records: { recS1: 'recD1' }, + }; + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', + fields: [ + { id: 'fldT', type: 'text' }, + { id: 'fldN', type: 'number' }, + { id: 'fldF', type: 'formula' }, // computed — must never be cleared + ], + // fldT was cleared on source (absent key); fldN still has a value. + records: [{ id: 'recS1', cellValuesByColumnId: { fldN: 42 } }], + }], + }; + const destSnapshot = { + tables: [{ + id: 'tblD', name: 'T', fields: [], + records: [{ + id: 'recD1', + // dest still holds the stale text value; also holds a dest-only unmapped cell + // and a computed cell — both must remain untouched. + cellValuesByColumnId: { fldTD: 'stale', fldND: 41, fldX: 'dest-only', fldFD: 'computed' }, + }], + }], + }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + await applyRecordsPass1({ + client, srcSnapshot, destSnapshot, idmap, + limiter: createLimiter({ rps: 1000, sleep: async () => {} }), + journal: newJournal('p3d', 't'), + persist: () => {}, + result, + }); + assert.equal(updateCalls.length, 1); + const sentCells = updateCalls[0].rows[0].cellValuesByColumnId; + assert.equal(sentCells.fldTD, null, 'cleared source cell must be written as explicit null'); + assert.equal(sentCells.fldND, 42, 'changed scalar still sent'); + assert.ok(!('fldX' in sentCells), 'unmapped dest-only field must not be touched'); + assert.ok(!('fldFD' in sentCells), 'computed field must never be cleared'); + }); + + it('does not emit clears when the dest cell is already empty or the dest record is unknown', async () => { + const updateCalls = []; + const client = { + createRecords: async () => ({ created: [], failed: [] }), + updateRecords: async (appId, tableId, rows) => { + updateCalls.push({ rows }); + return { updated: rows.map((r) => r.rowId), failed: [] }; + }, + }; + const idmap = { + tables: { tblS: 'tblD' }, + fields: { fldT: { destFld: 'fldTD', choices: {} }, fldN: { destFld: 'fldND', choices: {} } }, + records: { recS1: 'recD1', recS2: 'recD2' }, + }; + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', + fields: [{ id: 'fldT', type: 'text' }, { id: 'fldN', type: 'number' }], + records: [ + { id: 'recS1', cellValuesByColumnId: { fldN: 7 } }, // fldT absent; dest fldTD also empty + { id: 'recS2', cellValuesByColumnId: { fldN: 8 } }, // dest record missing from snapshot + ], + }], + }; + const destSnapshot = { + tables: [{ + id: 'tblD', name: 'T', fields: [], + records: [{ id: 'recD1', cellValuesByColumnId: { fldND: 6 } }], // no fldTD value; recD2 absent + }], + }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + await applyRecordsPass1({ + client, srcSnapshot, destSnapshot, idmap, + limiter: createLimiter({ rps: 1000, sleep: async () => {} }), + journal: newJournal('p3e', 't'), + persist: () => {}, + result, + }); + assert.equal(updateCalls.length, 1); + for (const row of updateCalls[0].rows) { + assert.ok(!('fldTD' in row.cellValuesByColumnId), `no null for empty/unknown dest cell (row ${row.rowId})`); + } + }); + + it('never clears (or writes) anything under conflicts=dest-wins', async () => { + const updateCalls = []; + const client = { + createRecords: async () => ({ created: [], failed: [] }), + updateRecords: async (appId, tableId, rows) => { + updateCalls.push({ rows }); + return { updated: rows.map((r) => r.rowId), failed: [] }; + }, + }; + const idmap = { + tables: { tblS: 'tblD' }, + fields: { fldT: { destFld: 'fldTD', choices: {} } }, + records: { recS1: 'recD1' }, + }; + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', + fields: [{ id: 'fldT', type: 'text' }], + records: [{ id: 'recS1', cellValuesByColumnId: {} }], // cleared on source + }], + }; + const destSnapshot = { + tables: [{ + id: 'tblD', name: 'T', fields: [], + records: [{ id: 'recD1', cellValuesByColumnId: { fldTD: 'dest edit' } }], + }], + }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + await applyRecordsPass1({ + client, srcSnapshot, destSnapshot, idmap, + limiter: createLimiter({ rps: 1000, sleep: async () => {} }), + journal: newJournal('p3f', 't'), + persist: () => {}, + result, + policy: 'preserve', // conflicts=dest-wins + }); + assert.equal(updateCalls.length, 0, 'dest-wins must not send any update (no clears)'); + }); + it('pushes RECORD_CELL_SKIPPED warning when a select choice cannot be mapped', async () => { const captured = []; const idmap = { From 385547c9b9baa51a617794552114782350bb2855 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 14:06:41 +0300 Subject: [PATCH 150/246] =?UTF-8?q?fix(sync):=20harden=20apply=20engine=20?= =?UTF-8?q?=E2=80=94=20exact=20reverse-link=20adoption,=20retryable=20gate?= =?UTF-8?q?d/deferred=20actions,=20safe=20primary=20retypes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - adoptReverseLink: adopt the dest column whose symmetricColumnId equals the already-mapped forward's dest id (was: first-unadopted — cross-wired two links between the same table pair); idmap-based, so a cross-run retry adopts a prior run's auto-reverse instead of creating a duplicate link pair. - applyPlan: applyAction now returns a disposition. 'gated' (RETYPE_GATED / RETYPE_DEFERRED) and 'deferred' (UNRESOLVABLE_REF) actions are NOT journaled done — a re-run of the same plan (e.g. with confirmRetypes:true, or once the ref resolves) actually retries them. Deferred actions are retried within the same run until no progress (link → later table, computed → later field), with their warnings held back unless they never resolve. - createField (link): defer when typeOptions.foreignTableId has no dest counterpart yet, instead of sending the SOURCE base's tbl id. - reconcilePrimary: retyping a PRE-EXISTING table's primary now honors the confirmRetypes gate (RETYPE_GATED; tables created this run stay ungated); computed primaries are written via toWritableComputedOptions with refs remapped (deferred until siblings exist); autoNumber strips the read-only maxUsedAutoNumber counter. - updateField: a gated/deferred retype still applies the accompanying description change instead of dropping it. Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/apply.js | 128 ++++++-- .../test/sync/test-apply-hardening.test.js | 297 ++++++++++++++++++ .../mcp-server/test/sync/test-apply.test.js | 5 +- 3 files changed, 407 insertions(+), 23 deletions(-) create mode 100644 packages/mcp-server/test/sync/test-apply-hardening.test.js diff --git a/packages/mcp-server/src/sync/apply.js b/packages/mcp-server/src/sync/apply.js index 17164cc..e8fa387 100644 --- a/packages/mcp-server/src/sync/apply.js +++ b/packages/mcp-server/src/sync/apply.js @@ -45,17 +45,22 @@ async function readTable(client, appId, tableId) { return { primaryId, primary: cols.find((c) => c.id === primaryId), cols, views: t.views ?? [] }; } -function rememberLink(state, forwardFieldId, destTableId) { - state.createdLinks.set(forwardFieldId, { destTableId }); -} - +// Adopt the auto-created reverse of a link whose FORWARD side is already mapped in the idmap +// (created earlier this run, or matched/adopted in a prior run). The reverse-side createField +// action carries the source FORWARD field id in typeOptions.symmetricColumnId; the dest column +// whose symmetricColumnId equals that forward's dest id IS this link's reverse — adopt exactly +// that one. (First-unadopted matching cross-wired multi-link table pairs, and run-local +// createdLinks state made cross-run retries create duplicate link pairs.) async function adoptReverseLink({ client, destAppId, a, idmap, index, state, result }) { + const srcForwardId = a.typeOptions && a.typeOptions.symmetricColumnId; + const wantSym = srcForwardId && idmap.fields[srcForwardId] && idmap.fields[srcForwardId].destFld; + if (!wantSym) return false; // forward unknown → nothing to adopt (create a fresh pair) const destTableId = idmap.tables[a.sourceTableId]; const { cols } = await readTable(client, destAppId, destTableId); for (const c of cols) { if (!LINK_TYPES.has(c.type)) continue; const sym = c.typeOptions && c.typeOptions.symmetricColumnId; - if (sym && state.createdLinks.has(sym) && !state.adoptedReverse.has(c.id)) { + if (sym === wantSym && !state.adoptedReverse.has(c.id)) { if (c.name !== a.name) await client.renameField(destAppId, c.id, a.name); idmap.fields[a.sourceFieldId] = { destFld: c.id, choices: {} }; const entry = index.tablesById.get(destTableId); @@ -99,19 +104,26 @@ function mergeChoices(destField, srcTypeOptions) { export async function applyPlan({ client, plan, destAppId, destSnapshot, idmap, journal, persist, skip = [], confirmRetypes = false }) { const index = buildIndex(destSnapshot); - const state = { createdLinks: new Map(), adoptedReverse: new Set() }; + const state = { adoptedReverse: new Set(), createdTables: new Set() }; if (!idmap.views) idmap.views = {}; const result = { planId: plan.planId, aborted: false, created: 0, updated: 0, skipped: 0, failed: 0, retyped: 0, warnings: [], idmap }; const skipSet = new Set(skip); - for (let idx = 0; idx < plan.actions.length; idx++) { - const a = plan.actions[idx]; - if (isDone(journal, idx)) { result.skipped++; continue; } - if (a.apply === false || skipSet.has(a.changeId)) { result.skipped++; continue; } + // Attempt one action. Dispositions (applyAction return value): + // undefined → applied (or a terminal intentional skip) — journaled done. + // 'gated' → declined by a confirm gate / accepted deferral — NOT journaled done, so a + // re-run of the SAME plan (e.g. with confirmRetypes:true) actually retries it. + // 'deferred' → blocked on a dependency that may be created later THIS run — retried below; + // its warnings are held back and only surfaced if it never resolves. + const attempt = async (idx, a) => { + const warnStart = result.warnings.length; try { - await applyAction({ client, destAppId, a, idmap, index, state, result, confirmRetypes }); + const disposition = await applyAction({ client, destAppId, a, idmap, index, state, result, confirmRetypes }); + if (disposition === 'deferred') return { status: 'deferred', warns: result.warnings.splice(warnStart) }; + if (disposition === 'gated') { result.skipped++; persist(idmap, journal); return { status: 'gated' }; } recordDone(journal, idx, a.kind, idmap.tables[a.sourceTableId] ?? (a.sourceFieldId && idmap.fields[a.sourceFieldId]?.destFld)); persist(idmap, journal); + return { status: 'done' }; } catch (e) { recordFailed(journal, idx, a.kind, String(e && e.message ? e.message : e)); result.failed++; @@ -120,8 +132,40 @@ export async function applyPlan({ client, plan, destAppId, destSnapshot, idmap, // ponytail: continue, don't halt. A failed create's dependents are guarded by the // UNRESOLVABLE_REF check; re-run retries non-done actions. Halting on one bad field // blocked the entire sync (494 creates never ran behind one failing update). + return { status: 'failed' }; } + }; + + const deferred = []; + for (let idx = 0; idx < plan.actions.length; idx++) { + const a = plan.actions[idx]; + if (isDone(journal, idx)) { result.skipped++; continue; } + if (a.apply === false || skipSet.has(a.changeId)) { result.skipped++; continue; } + const r = await attempt(idx, a); + if (r.status === 'deferred') deferred.push({ idx, a, warns: r.warns }); } + + // Deferred-dependency retry: an action blocked on a not-yet-created table/field (link to a + // later table, computed field referencing a later field, computed primary referencing its + // siblings) is retried after the rest of the plan ran — loop until a pass makes no progress. + let progress = true; + while (progress && deferred.length) { + progress = false; + for (let i = 0; i < deferred.length; ) { + const d = deferred[i]; + const r = await attempt(d.idx, d.a); + if (r.status === 'deferred') { d.warns = r.warns; i++; continue; } + deferred.splice(i, 1); + progress = true; + } + } + // Never resolved this run: surface the held-back warnings and count skipped. NOT journaled + // done — a re-run (or re-plan) retries once the dependency becomes resolvable. + for (const d of deferred) { + result.warnings.push(...d.warns); + result.skipped++; + } + if (deferred.length) persist(idmap, journal); return result; } @@ -132,6 +176,7 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result, if (existing) { idmap.tables[a.sourceTableId] = existing.id; result.skipped++; return; } const { tableId } = await client.createTable(destAppId, a.name); idmap.tables[a.sourceTableId] = tableId; + state.createdTables.add(tableId); // its primary is an empty placeholder → retype ungated // D1: delete the auto-created non-primary scaffolding fields for a clean mirror. const { primaryId, primary, cols, views } = await readTable(client, destAppId, tableId); for (const c of cols) { @@ -169,14 +214,41 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result, const primaryId = entry.primaryFieldId; let primary = null; for (const f of entry.fieldsByName.values()) if (f.id === primaryId) primary = f; - let renamed = false, retyped = false; + let renamed = false, retyped = false, disposition; if (primary && primary.name !== a.toName) { await client.renameField(destAppId, primaryId, a.toName); renamed = true; } if (a.toType && primary && primary.type !== a.toType) { if (ILLEGAL_PRIMARY_TYPES.has(a.toType)) { result.warnings.push({ code: 'PRIMARY_TYPE_INCOMPATIBLE', message: `Primary "${a.toName}" wants ${a.toType}; kept ${primary.type} placeholder` }); + } else if (!confirmRetypes && !state.createdTables.has(destTableId)) { + // Retyping a PRE-EXISTING table's primary is a lossy conversion of real data — honor + // the same confirmRetypes gate as non-primary scalar retypes. (A table created this + // run has an empty placeholder primary, so its retype is safe and stays ungated.) + result.warnings.push({ code: 'RETYPE_GATED', message: `Primary "${a.toName}": retype ${primary.type}→${a.toType} gated — set confirmRetypes:true` }); + disposition = 'gated'; } else { - try { await client.updateFieldConfig(destAppId, primaryId, { type: a.toType, typeOptions: remapRefs(a.toTypeOptions, idmap) }); retyped = true; } - catch (e) { result.warnings.push({ code: 'PRIMARY_TYPE_INCOMPATIBLE', message: `Primary "${a.toName}" retype to ${a.toType} rejected: ${e.message ?? e}` }); } + let toOpts = remapRefs(a.toTypeOptions, idmap); + if (COMPUTED_TYPES.has(a.toType)) { + // Same contract as createField/updateField: computed configs must be written in the + // WRITABLE shape (raw snapshot options carry read-only formulaTextParsed/dependencies/ + // resultType the API 422s on). reconcilePrimary runs right after createTable — before + // sibling fields exist — so defer until the refs are created later this run. + const refs = (a.toTypeOptions && a.toTypeOptions.dependencies && a.toTypeOptions.dependencies.referencedColumnIdsForValue) || []; + const unresolved = refs.filter((d) => !(idmap.fields[d] && idmap.fields[d].destFld)); + if (unresolved.length) { + result.warnings.push({ code: 'UNRESOLVABLE_REF', message: `Primary "${a.toName}" retype to ${a.toType} deferred — refs [${unresolved.join(', ')}] not yet created` }); + disposition = 'deferred'; + } else { + toOpts = toWritableComputedOptions(a.toType, toOpts); + } + } else if (a.toType === 'autoNumber') { + // maxUsedAutoNumber is a read-only runtime counter (same strip as createField). + const { maxUsedAutoNumber, ...rest } = toOpts || {}; + toOpts = rest; + } + if (!disposition) { + try { await client.updateFieldConfig(destAppId, primaryId, { type: a.toType, typeOptions: toOpts }); retyped = true; } + catch (e) { result.warnings.push({ code: 'PRIMARY_TYPE_INCOMPATIBLE', message: `Primary "${a.toName}" retype to ${a.toType} rejected: ${e.message ?? e}` }); } + } } } idmap.fields[a.sourcePrimaryFieldId] = { destFld: primaryId, choices: {} }; @@ -184,6 +256,7 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result, if (renamed) { entry.fieldsByName.delete(primary.name); primary.name = a.toName; entry.fieldsByName.set(a.toName, primary); } if (retyped) primary.type = a.toType; } + if (disposition) return disposition; // gated/deferred retype: NOT journaled done → retryable if (renamed || retyped) result.updated++; else result.skipped++; return; } @@ -202,8 +275,18 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result, result.skipped++; return; } - // Reciprocal-once: adopt the auto-created reverse of a link made earlier this run. - if (LINK_TYPES.has(a.type) && await adoptReverseLink({ client, destAppId, a, idmap, index, state, result })) return; + if (LINK_TYPES.has(a.type)) { + // A link whose target table has no dest counterpart yet (created later in this plan, or + // skipped) must not be sent — the payload would carry the SOURCE base's tbl id. Defer: + // retried after the remaining createTable actions this run; else surfaced + retryable. + const foreignSrc = a.typeOptions && a.typeOptions.foreignTableId; + if (foreignSrc && !idmap.tables[foreignSrc]) { + result.warnings.push({ code: 'UNRESOLVABLE_REF', message: `Link field "${a.name}" targets table ${foreignSrc} with no dest counterpart (not yet created or skipped) — not created` }); + return 'deferred'; + } + // Reciprocal-once: adopt the auto-created reverse of this link's already-created forward. + if (await adoptReverseLink({ client, destAppId, a, idmap, index, state, result })) return; + } const remapped = remapRefs(a.typeOptions, idmap); let typeOptions = remapped; @@ -213,8 +296,7 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result, const unresolved = (a.dependsOn || []).filter((d) => !(idmap.fields[d] && idmap.fields[d].destFld)); if (unresolved.length) { result.warnings.push({ code: 'UNRESOLVABLE_REF', message: `Field "${a.name}" references unresolved field(s) [${unresolved.join(', ')}] (skipped or unmapped) — not created` }); - result.skipped++; - return; + return 'deferred'; // retried this run once the refs are created; else retryable next run } typeOptions = toWritableComputedOptions(a.type, remapped); if (a.type === 'formula') { @@ -231,7 +313,6 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result, const { columnId } = await client.createField(destAppId, destTableId, { name: a.name, type: a.type, typeOptions, description: a.description ?? undefined }); idmap.fields[a.sourceFieldId] = { destFld: columnId, choices: {} }; if (entry) entry.fieldsByName.set(a.name, { id: columnId, name: a.name, type: a.type, typeOptions }); - if (LINK_TYPES.has(a.type)) rememberLink(state, columnId, destTableId); result.created++; return; } @@ -241,13 +322,18 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result, if (changes.type !== undefined) { const destType = findDestFieldType(index, a.destFld); const fname = findDestField(index, a.destFld)?.name ?? a.destFld; + // A gated/deferred retype still applies the non-destructive parts of the action + // (description) and returns 'gated' so it is NOT journaled done — a re-run of the + // same plan (e.g. with confirmRetypes:true) retries it. if (!SCALAR_RETYPE_TYPES.has(changes.type) || !SCALAR_RETYPE_TYPES.has(destType)) { result.warnings.push({ code: 'RETYPE_DEFERRED', message: `Field "${fname}": retype ${destType}→${changes.type} deferred (non-scalar)` }); - return; + if (changes.description !== undefined) await client.updateFieldDescription(destAppId, a.destFld, changes.description); + return 'gated'; } if (!confirmRetypes) { result.warnings.push({ code: 'RETYPE_GATED', message: `Field "${fname}": scalar retype ${destType}→${changes.type} gated — set confirmRetypes:true` }); - return; + if (changes.description !== undefined) await client.updateFieldDescription(destAppId, a.destFld, changes.description); + return 'gated'; } const newOpts = changes.typeOptions ? mergeChoices(findDestField(index, a.destFld), remapRefs(changes.typeOptions, idmap)) : undefined; try { diff --git a/packages/mcp-server/test/sync/test-apply-hardening.test.js b/packages/mcp-server/test/sync/test-apply-hardening.test.js new file mode 100644 index 0000000..4d1c79b --- /dev/null +++ b/packages/mcp-server/test/sync/test-apply-hardening.test.js @@ -0,0 +1,297 @@ +// Hardening regressions for the schema apply engine (audit findings): +// - adoptReverseLink must adopt the EXACT symmetric reverse (not first-unadopted) — two +// links between the same table pair otherwise cross-wire silently. +// - reverse-link adoption must work across runs (idmap-based, not in-memory createdLinks). +// - gated/deferred actions (RETYPE_GATED / RETYPE_DEFERRED / UNRESOLVABLE_REF) must NOT be +// journaled done — a re-run of the same plan retries them. +// - a link createField whose target table is created later in the plan is deferred and +// retried in the same run (never sent with the SOURCE base's tbl id). +// - reconcilePrimary honors confirmRetypes for pre-existing tables and writes computed +// primaries in the writable shape (toWritableComputedOptions; autoNumber strip). +// - a gated/deferred retype still applies the accompanying description change. +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { MockClient } from './helpers/mock-client.js'; +import { applyPlan } from '../../src/sync/apply.js'; +import { newJournal, isDone } from '../../src/sync/journal.js'; +import { snapshotBase } from '../../src/sync/snapshot.js'; + +function run(client, plan, destSnapshot, { journal, idmap, confirmRetypes } = {}) { + return applyPlan({ + client, plan, destAppId: plan.destBaseId, destSnapshot, + idmap: idmap ?? JSON.parse(JSON.stringify(plan.idmap)), + journal: journal ?? newJournal(plan.planId, 'ts'), + persist: () => {}, + confirmRetypes, + }); +} + +describe('apply: adoptReverseLink exact symmetric match (no cross-wiring)', () => { + it('two links into the same table adopt their OWN reverses even when plan order differs from creation order', async () => { + const client = new MockClient(); + const a = await client.createTable('appD', 'A'); + const b = await client.createTable('appD', 'B'); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { + planId: 'plnXW', sourceBaseId: 'appS', destBaseId: 'appD', + idmap: { tables: { tA: a.tableId, tB: b.tableId }, fields: {} }, + actions: [ + // Forwards created in A: Owner first, Reviewer second (auto-reverses appended to B in that order). + { kind: 'createField', sourceTableId: 'tA', sourceFieldId: 'fldOwner', name: 'Owner', type: 'foreignKey', typeOptions: { foreignTableId: 'tB', relationship: 'many', symmetricColumnId: 'fldOwnerRev' }, description: null, computed: false, dependsOn: [], dependsOnTables: ['tB'] }, + { kind: 'createField', sourceTableId: 'tA', sourceFieldId: 'fldReviewer', name: 'Reviewer', type: 'foreignKey', typeOptions: { foreignTableId: 'tB', relationship: 'many', symmetricColumnId: 'fldReviewerRev' }, description: null, computed: false, dependsOn: [], dependsOnTables: ['tB'] }, + // Reverse-side actions arrive in the OPPOSITE order (source table B's field order): + // Reviewer's reverse FIRST — first-unadopted matching would grab Owner's reverse here. + { kind: 'createField', sourceTableId: 'tB', sourceFieldId: 'fldReviewerRev', name: 'Reviewed Tasks', type: 'foreignKey', typeOptions: { foreignTableId: 'tA', relationship: 'many', symmetricColumnId: 'fldReviewer' }, description: null, computed: false, dependsOn: [], dependsOnTables: ['tA'] }, + { kind: 'createField', sourceTableId: 'tB', sourceFieldId: 'fldOwnerRev', name: 'Owned Tasks', type: 'foreignKey', typeOptions: { foreignTableId: 'tA', relationship: 'many', symmetricColumnId: 'fldOwner' }, description: null, computed: false, dependsOn: [], dependsOnTables: ['tA'] }, + ], + orphans: [], warnings: [], + }; + const res = await run(client, plan, destSnapshot); + assert.equal(res.failed, 0); + const tables = (await client.getApplicationData('appD')).data.tableSchemas; + const tableB = tables.find((t) => t.name === 'B'); + const links = tableB.columns.filter((c) => c.type === 'foreignKey'); + assert.equal(links.length, 2, 'exactly two reverse links (no duplicates)'); + const reviewed = links.find((c) => c.name === 'Reviewed Tasks'); + const owned = links.find((c) => c.name === 'Owned Tasks'); + assert.ok(reviewed && owned, 'both reverses adopted + renamed'); + // The field named "Reviewed Tasks" must be symmetric to the Reviewer forward, not Owner's. + assert.equal(reviewed.typeOptions.symmetricColumnId, res.idmap.fields.fldReviewer.destFld, + 'Reviewed Tasks must pair with the Reviewer forward'); + assert.equal(owned.typeOptions.symmetricColumnId, res.idmap.fields.fldOwner.destFld, + 'Owned Tasks must pair with the Owner forward'); + assert.equal(res.idmap.fields.fldReviewerRev.destFld, reviewed.id); + assert.equal(res.idmap.fields.fldOwnerRev.destFld, owned.id); + }); +}); + +describe('apply: cross-run reverse-link adoption (idmap-based, not in-memory state)', () => { + it('adopts a prior run\'s auto-created reverse instead of creating a duplicate link pair', async () => { + const client = new MockClient(); + const a = await client.createTable('appD', 'A'); + const b = await client.createTable('appD', 'B'); + // Prior run created the forward link (Airtable auto-created its reverse in B, default name "A"). + const { columnId: forwardId } = await client.createField('appD', a.tableId, { name: 'Bs', type: 'foreignKey', typeOptions: { foreignTableId: b.tableId, relationship: 'many' } }); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { + planId: 'plnXR', sourceBaseId: 'appS', destBaseId: 'appD', + // The forward is already mapped (matched by name in a re-plan / persisted idmap). + idmap: { tables: { tA: a.tableId, tB: b.tableId }, fields: { fldAlink: { destFld: forwardId, choices: {} } } }, + actions: [ + { kind: 'createField', sourceTableId: 'tB', sourceFieldId: 'fldBlink', name: 'As', type: 'foreignKey', typeOptions: { foreignTableId: 'tA', relationship: 'many', symmetricColumnId: 'fldAlink' }, description: null, computed: false, dependsOn: [], dependsOnTables: ['tA'] }, + ], + orphans: [], warnings: [], + }; + const res = await run(client, plan, destSnapshot); + assert.equal(res.failed, 0); + const tableB = (await client.getApplicationData('appD')).data.tableSchemas.find((t) => t.name === 'B'); + const links = tableB.columns.filter((c) => c.type === 'foreignKey'); + assert.equal(links.length, 1, 'must adopt the existing auto-reverse, not create a duplicate pair'); + assert.equal(links[0].name, 'As'); + assert.equal(res.idmap.fields.fldBlink.destFld, links[0].id); + assert.equal(links[0].typeOptions.symmetricColumnId, forwardId); + }); +}); + +describe('apply: gated/deferred actions are NOT journaled done (same-plan re-run retries)', () => { + it('RETYPE_GATED, then re-run with confirmRetypes:true on the SAME journal actually retypes', async () => { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'T'); + const { columnId } = await client.createField('appD', tableId, { name: 'F', type: 'text' }); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { planId: 'plnG', sourceBaseId: 'appS', destBaseId: 'appD', idmap: { tables: {}, fields: {} }, + actions: [{ kind: 'updateField', sourceFieldId: 'sF', destFld: columnId, changes: { type: 'number' } }], orphans: [], warnings: [] }; + const journal = newJournal('plnG', 'ts'); + + const r1 = await run(client, plan, destSnapshot, { journal, confirmRetypes: false }); + assert.ok(r1.warnings.some((w) => w.code === 'RETYPE_GATED')); + assert.equal(isDone(journal, 0), false, 'gated action must NOT be journaled done'); + + const r2 = await run(client, plan, destSnapshot, { journal, confirmRetypes: true }); + assert.equal(r2.retyped, 1, 're-run with confirmRetypes:true must perform the retype'); + assert.ok(client.calls.includes(`updateFieldConfig:${columnId}:number`)); + }); + + it('UNRESOLVABLE_REF createField, then re-run with the dep now mapped creates the field', async () => { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'T'); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { planId: 'plnUR', sourceBaseId: 'appS', destBaseId: 'appD', idmap: { tables: { tS: tableId }, fields: {} }, + actions: [{ kind: 'createField', sourceTableId: 'tS', sourceFieldId: 'fldF', name: 'F', type: 'formula', typeOptions: { formulaTextParsed: '{column_value_fldX}' }, description: null, computed: true, dependsOn: ['fldX'], dependsOnTables: [] }], + orphans: [], warnings: [] }; + const journal = newJournal('plnUR', 'ts'); + + const r1 = await run(client, plan, destSnapshot, { journal, idmap: { tables: { tS: tableId }, fields: {} } }); + assert.ok(r1.warnings.some((w) => w.code === 'UNRESOLVABLE_REF')); + assert.equal(isDone(journal, 0), false, 'deferred action must NOT be journaled done'); + + // Run 2: the dep is now mapped (e.g. created by a later run / other table's actions). + const { columnId: xId } = await client.createField('appD', tableId, { name: 'X', type: 'number' }); + const r2 = await run(client, plan, destSnapshot, { journal, idmap: { tables: { tS: tableId }, fields: { fldX: { destFld: xId, choices: {} } } } }); + assert.equal(r2.created, 1, 're-run must create the formula once the ref resolves'); + const cols = (await client.getApplicationData('appD')).data.tableSchemas[0].columns; + assert.ok(cols.some((c) => c.name === 'F')); + }); + + it('in-run deferral: a computed field whose dep is created LATER in the plan converges in one run', async () => { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'T'); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { planId: 'plnIR', sourceBaseId: 'appS', destBaseId: 'appD', idmap: { tables: { tS: tableId }, fields: {} }, + actions: [ + // Dependency-inverted order: the formula appears BEFORE the field it references. + { kind: 'createField', sourceTableId: 'tS', sourceFieldId: 'fldTotal', name: 'Total', type: 'formula', typeOptions: { formulaTextParsed: '{column_value_fldPrice}*2', dependencies: { referencedColumnIdsForValue: ['fldPrice'] } }, description: null, computed: true, dependsOn: ['fldPrice'], dependsOnTables: [] }, + { kind: 'createField', sourceTableId: 'tS', sourceFieldId: 'fldPrice', name: 'Price', type: 'number', typeOptions: { precision: 0 }, description: null, computed: false, dependsOn: [], dependsOnTables: [] }, + ], + orphans: [], warnings: [] }; + const res = await run(client, plan, destSnapshot, { idmap: { tables: { tS: tableId }, fields: {} } }); + assert.equal(res.created, 2, 'formula must be retried and created after its dep exists'); + assert.equal(res.failed, 0); + assert.ok(!res.warnings.some((w) => w.code === 'UNRESOLVABLE_REF'), 'resolved deferral must not surface a warning'); + const total = (await client.getApplicationData('appD')).data.tableSchemas[0].columns.find((c) => c.name === 'Total'); + assert.ok(total, 'Total created'); + const destPriceId = res.idmap.fields.fldPrice.destFld; + assert.equal(total.typeOptions.formulaText, `{${destPriceId}}*2`); + }); +}); + +describe('apply: link createField to a not-yet-created table (dependsOnTables)', () => { + it('defers the link until its target table is created later in the same run (never sends the source tbl id)', async () => { + const client = new MockClient(); + const destSnapshot = await snapshotBase(client, 'appD'); // empty base + const plan = { planId: 'plnLT', sourceBaseId: 'appS', destBaseId: 'appD', idmap: { tables: {}, fields: {} }, + actions: [ + { kind: 'createTable', sourceTableId: 'tA', name: 'Projects' }, + { kind: 'createField', sourceTableId: 'tA', sourceFieldId: 'fldTasks', name: 'Tasks', type: 'foreignKey', typeOptions: { foreignTableId: 'tB', relationship: 'many', symmetricColumnId: 'fldProj' }, description: null, computed: false, dependsOn: [], dependsOnTables: ['tB'] }, + { kind: 'createTable', sourceTableId: 'tB', name: 'Tasks' }, + ], + orphans: [], warnings: [] }; + const res = await run(client, plan, destSnapshot, { idmap: { tables: {}, fields: {} } }); + assert.equal(res.failed, 0, 'link to a later table must not fail the action'); + const tables = (await client.getApplicationData('appD')).data.tableSchemas; + const projects = tables.find((t) => t.name === 'Projects'); + const link = projects.columns.find((c) => c.name === 'Tasks'); + assert.ok(link, 'link created after its target table exists'); + assert.equal(link.typeOptions.foreignTableId, res.idmap.tables.tB, 'foreignTableId must be the DEST table id, never the source id'); + }); + + it('link whose target table never materializes → UNRESOLVABLE_REF warning, not ACTION_FAILED, not journaled done', async () => { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'A'); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { planId: 'plnLN', sourceBaseId: 'appS', destBaseId: 'appD', idmap: { tables: { tA: tableId }, fields: {} }, + actions: [{ kind: 'createField', sourceTableId: 'tA', sourceFieldId: 'fldL', name: 'L', type: 'foreignKey', typeOptions: { foreignTableId: 'tMISSING', relationship: 'many' }, description: null, computed: false, dependsOn: [], dependsOnTables: ['tMISSING'] }], + orphans: [], warnings: [] }; + const journal = newJournal('plnLN', 'ts'); + const res = await run(client, plan, destSnapshot, { journal, idmap: { tables: { tA: tableId }, fields: {} } }); + assert.equal(res.failed, 0); + assert.ok(res.warnings.some((w) => w.code === 'UNRESOLVABLE_REF')); + assert.equal(isDone(journal, 0), false, 'unresolved link create must stay retryable'); + const cols = (await client.getApplicationData('appD')).data.tableSchemas[0].columns; + assert.ok(!cols.some((c) => c.name === 'L'), 'link NOT created with a source tbl id'); + }); +}); + +describe('apply: reconcilePrimary confirmRetypes gate + writable computed options', () => { + it('retyping a PRE-EXISTING table\'s primary without confirmRetypes → RETYPE_GATED, no mutation, retryable', async () => { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'T'); // pre-existing dest table with data-bearing primary + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { planId: 'plnPG', sourceBaseId: 'appS', destBaseId: 'appD', idmap: { tables: { tS: tableId }, fields: {} }, + actions: [{ kind: 'reconcilePrimary', sourceTableId: 'tS', sourcePrimaryFieldId: 'fS1', toName: 'Name', toType: 'number', toTypeOptions: { format: 'integer' } }], + orphans: [], warnings: [] }; + const journal = newJournal('plnPG', 'ts'); + const r1 = await run(client, plan, destSnapshot, { journal, idmap: { tables: { tS: tableId }, fields: {} }, confirmRetypes: false }); + assert.ok(r1.warnings.some((w) => w.code === 'RETYPE_GATED'), 'ungated primary retype of a pre-existing table is a lossy conversion'); + const primary = (await client.getApplicationData('appD')).data.tableSchemas[0].columns[0]; + assert.equal(primary.type, 'text', 'primary must be kept'); + assert.equal(isDone(journal, 0), false, 'gated reconcilePrimary must stay retryable'); + + const r2 = await run(client, plan, destSnapshot, { journal, idmap: { tables: { tS: tableId }, fields: {} }, confirmRetypes: true }); + assert.equal(r2.updated, 1); + const after = (await client.getApplicationData('appD')).data.tableSchemas[0].columns[0]; + assert.equal(after.type, 'number', 're-run with confirmRetypes:true performs the retype'); + }); + + it('table created THIS run: primary retype stays ungated (placeholder primary, no data)', async () => { + const client = new MockClient(); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { planId: 'plnPU', sourceBaseId: 'appS', destBaseId: 'appD', idmap: { tables: {}, fields: {} }, + actions: [ + { kind: 'createTable', sourceTableId: 'tS', name: 'T' }, + { kind: 'reconcilePrimary', sourceTableId: 'tS', sourcePrimaryFieldId: 'fS1', toName: 'Title', toType: 'number', toTypeOptions: { format: 'integer' } }, + ], orphans: [], warnings: [] }; + const res = await run(client, plan, destSnapshot, { idmap: { tables: {}, fields: {} }, confirmRetypes: false }); + assert.ok(!res.warnings.some((w) => w.code === 'RETYPE_GATED')); + const primary = (await client.getApplicationData('appD')).data.tableSchemas[0].columns[0]; + assert.equal(primary.type, 'number'); + }); + + it('formula primary is written in the WRITABLE shape with refs remapped (deferred until siblings exist)', async () => { + const client = new MockClient(); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { planId: 'plnPF', sourceBaseId: 'appS', destBaseId: 'appD', idmap: { tables: {}, fields: {} }, + actions: [ + { kind: 'createTable', sourceTableId: 'tS', name: 'T' }, + { kind: 'reconcilePrimary', sourceTableId: 'tS', sourcePrimaryFieldId: 'fS1', toName: 'Total', toType: 'formula', + toTypeOptions: { formulaTextParsed: '{column_value_fldPrice}*2', dependencies: { referencedColumnIdsForValue: ['fldPrice'] }, resultType: 'number', format: 'currency' } }, + { kind: 'createField', sourceTableId: 'tS', sourceFieldId: 'fldPrice', name: 'Price', type: 'number', typeOptions: { precision: 0 }, description: null, computed: false, dependsOn: [], dependsOnTables: [] }, + ], orphans: [], warnings: [] }; + const res = await run(client, plan, destSnapshot, { idmap: { tables: {}, fields: {} } }); + assert.equal(res.failed, 0); + const primary = (await client.getApplicationData('appD')).data.tableSchemas[0].columns[0]; + assert.equal(primary.type, 'formula', 'primary retyped after the sibling ref was created'); + const destPriceId = res.idmap.fields.fldPrice.destFld; + assert.equal(primary.typeOptions.formulaText, `{${destPriceId}}*2`, 'refs remapped + input-form formulaText'); + assert.equal(primary.typeOptions.formulaTextParsed, undefined, 'read-only key stripped'); + assert.equal(primary.typeOptions.dependencies, undefined, 'read-only key stripped'); + assert.equal(primary.typeOptions.format, 'currency', 'format keys carried'); + }); + + it('autoNumber primary: read-only maxUsedAutoNumber is stripped before updateFieldConfig', async () => { + const client = new MockClient(); + const destSnapshot = await snapshotBase(client, 'appD'); + const captured = []; + const orig = client.updateFieldConfig.bind(client); + client.updateFieldConfig = async (appId, colId, cfg) => { captured.push(cfg); return orig(appId, colId, cfg); }; + const plan = { planId: 'plnPA', sourceBaseId: 'appS', destBaseId: 'appD', idmap: { tables: {}, fields: {} }, + actions: [ + { kind: 'createTable', sourceTableId: 'tS', name: 'T' }, + { kind: 'reconcilePrimary', sourceTableId: 'tS', sourcePrimaryFieldId: 'fS1', toName: 'ID', toType: 'autoNumber', toTypeOptions: { maxUsedAutoNumber: 7 } }, + ], orphans: [], warnings: [] }; + const res = await run(client, plan, destSnapshot, { idmap: { tables: {}, fields: {} } }); + assert.equal(res.failed, 0); + const cfg = captured.find((c) => c.type === 'autoNumber'); + assert.ok(cfg, 'primary retyped to autoNumber'); + assert.equal(cfg.typeOptions && 'maxUsedAutoNumber' in cfg.typeOptions, false, 'read-only counter stripped'); + }); +}); + +describe('apply: gated/deferred retype still applies the description change', () => { + async function destWithText() { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'T'); + const { columnId } = await client.createField('appD', tableId, { name: 'F', type: 'text' }); + return { client, columnId, destSnapshot: await snapshotBase(client, 'appD') }; + } + function planFor(columnId, changes) { + return { planId: 'plnDD', sourceBaseId: 'appS', destBaseId: 'appD', idmap: { tables: {}, fields: {} }, + actions: [{ kind: 'updateField', sourceFieldId: 'sF', destFld: columnId, changes }], orphans: [], warnings: [] }; + } + + it('RETYPE_DEFERRED (non-scalar) + description → description applied', async () => { + const { client, columnId, destSnapshot } = await destWithText(); + const res = await run(client, planFor(columnId, { type: 'formula', description: 'kept' }), destSnapshot, { confirmRetypes: true }); + assert.ok(res.warnings.some((w) => w.code === 'RETYPE_DEFERRED')); + assert.equal(client._field(columnId).description, 'kept', 'description must not be dropped by the deferred retype'); + assert.equal(client._field(columnId).type, 'text', 'type unchanged'); + }); + + it('RETYPE_GATED + description → description applied', async () => { + const { client, columnId, destSnapshot } = await destWithText(); + const res = await run(client, planFor(columnId, { type: 'number', description: 'kept2' }), destSnapshot, { confirmRetypes: false }); + assert.ok(res.warnings.some((w) => w.code === 'RETYPE_GATED')); + assert.equal(client._field(columnId).description, 'kept2', 'description must not be dropped by the gated retype'); + assert.equal(client._field(columnId).type, 'text', 'type unchanged'); + }); +}); diff --git a/packages/mcp-server/test/sync/test-apply.test.js b/packages/mcp-server/test/sync/test-apply.test.js index e0649aa..6fd9c72 100644 --- a/packages/mcp-server/test/sync/test-apply.test.js +++ b/packages/mcp-server/test/sync/test-apply.test.js @@ -185,8 +185,9 @@ describe('apply: createField (link foreignKey) reciprocal-once', () => { planId: 'plnL', sourceBaseId: 'appS', destBaseId: 'appD', idmap: { tables: { tA: a.tableId, tB: b.tableId }, fields: {} }, actions: [ - { kind: 'createField', sourceTableId: 'tA', sourceFieldId: 'fldAlink', name: 'Bs', type: 'foreignKey', typeOptions: { foreignTableId: 'tB', relationship: 'many' }, description: null, computed: false, dependsOn: [], dependsOnTables: ['tB'] }, - { kind: 'createField', sourceTableId: 'tB', sourceFieldId: 'fldBlink', name: 'As', type: 'foreignKey', typeOptions: { foreignTableId: 'tA', relationship: 'many' }, description: null, computed: false, dependsOn: [], dependsOnTables: ['tA'] }, + // Real snapshots always carry symmetricColumnId on foreignKey typeOptions (source id-space). + { kind: 'createField', sourceTableId: 'tA', sourceFieldId: 'fldAlink', name: 'Bs', type: 'foreignKey', typeOptions: { foreignTableId: 'tB', relationship: 'many', symmetricColumnId: 'fldBlink' }, description: null, computed: false, dependsOn: [], dependsOnTables: ['tB'] }, + { kind: 'createField', sourceTableId: 'tB', sourceFieldId: 'fldBlink', name: 'As', type: 'foreignKey', typeOptions: { foreignTableId: 'tA', relationship: 'many', symmetricColumnId: 'fldAlink' }, description: null, computed: false, dependsOn: [], dependsOnTables: ['tA'] }, ], orphans: [], warnings: [], }; From a7b9d279f7b3591ad2db9c78522f17da9af9169e Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 14:07:32 +0300 Subject: [PATCH 151/246] fix(sync): skip update rows with an empty cell map MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A mapped record whose update produced no writable cells (data only in link/attachment/deferred-array fields, or a fully-empty row) was still pushed to client.updateRecords, which rejects an empty cellValuesByColumnId per row — inflating result.failed and emitting a spurious RECORD_UPDATE_FAILED warning for a healthy record on every re-sync. Skip the row and count it as skipped. Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/records.js | 3 ++ .../mcp-server/test/sync/test-records.test.js | 49 +++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index 731027d..fe76409 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -433,6 +433,9 @@ export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idm if (conflicts === 'dest-wins') continue; // preserve dest edits: skip the overwrite const cells = buildUpdateCells(srcTable.fields, srcCells, destCellsById.get(destRecId), idmap, result.warnings, rec.id); injectFieldMappings(cells, tableMappings, srcCells); + // Nothing to write → skip: client.updateRecords rejects an empty cell map per row, + // which would misreport this healthy record as RECORD_UPDATE_FAILED on every re-sync. + if (Object.keys(cells).length === 0) { result.skipped++; continue; } updateRows.push({ rowId: destRecId, cellValuesByColumnId: cells }); } } diff --git a/packages/mcp-server/test/sync/test-records.test.js b/packages/mcp-server/test/sync/test-records.test.js index c8f6480..ff1b6a6 100644 --- a/packages/mcp-server/test/sync/test-records.test.js +++ b/packages/mcp-server/test/sync/test-records.test.js @@ -377,6 +377,55 @@ describe('records.applyRecordsPass1', () => { assert.equal(updateCalls.length, 0, 'dest-wins must not send any update (no clears)'); }); + it('does not send an update row whose cell map is empty (no spurious RECORD_UPDATE_FAILED)', async () => { + // A mapped record whose writable cells produce nothing to write (e.g. all data lives in + // link/attachment fields, or src+dest are both empty) must be skipped — client.updateRecords + // rejects an empty cellValuesByColumnId per row, which misreported healthy records as failed. + const updateCalls = []; + const client = { + createRecords: async () => ({ created: [], failed: [] }), + updateRecords: async (appId, tableId, rows) => { + updateCalls.push({ rows }); + return { updated: rows.map((r) => r.rowId), failed: [] }; + }, + }; + const idmap = { + tables: { tblS: 'tblD' }, + fields: { + fldT: { destFld: 'fldTD', choices: {} }, + fldL: { destFld: 'fldLD', choices: {} }, + }, + records: { recS1: 'recD1' }, + }; + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', + fields: [ + { id: 'fldT', type: 'text' }, + { id: 'fldL', type: 'multipleRecordLinks' }, // Pass 2 owns this — never in the update payload + ], + records: [{ id: 'recS1', cellValuesByColumnId: { fldL: ['recS9'] } }], // only a link cell + }], + }; + const destSnapshot = { + tables: [{ + id: 'tblD', name: 'T', fields: [], + records: [{ id: 'recD1', cellValuesByColumnId: {} }], // dest text also empty → no clear either + }], + }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + await applyRecordsPass1({ + client, srcSnapshot, destSnapshot, idmap, + limiter: createLimiter({ rps: 1000, sleep: async () => {} }), + journal: newJournal('p3g', 't'), + persist: () => {}, + result, + }); + assert.equal(updateCalls.length, 0, 'updateRecords must not be called with an empty cell map'); + assert.equal(result.failed, 0, 'no spurious failure for a healthy record'); + assert.equal(result.skipped, 1, 'empty-update record counted as skipped'); + }); + it('pushes RECORD_CELL_SKIPPED warning when a select choice cannot be mapped', async () => { const captured = []; const idmap = { From 571f002b0869d81e75dee0f6c85c7537e447b51d Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 14:15:27 +0300 Subject: [PATCH 152/246] =?UTF-8?q?fix(sync):=20compare=20matched=20link?= =?UTF-8?q?=20fields=20canonically=20=E2=80=94=20no=20phantom=20updateFiel?= =?UTF-8?q?d?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Matched foreignKey fields were compared by raw scalar signature containing base-local ids (foreignTableId, symmetricColumnId), so every plan emitted an updateField whose apply then pushed an un-remapped source symmetricColumnId to the dest — the plan never converged while mode=diff said converged. - field-compare.js: new shared linkSig() canonical (foreignTableId → table NAME; base-scoped symmetricColumnId/viewIdForRecordSelection dropped) - diff.js: matched link fields compare via linkSig — updateField only on real divergence (e.g. link points at a different-named table) - compare.js: reuse linkSig instead of its inline canonLink so diff and plan verdicts agree - remap.js: remapRefs remaps symmetricColumnId (idmap.fields) and viewIdForRecordSelection (idmap.views), DROPPING them when unmappable so a legitimate link updateField never sends source-base ids Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/compare.js | 19 ++------ packages/mcp-server/src/sync/diff.js | 23 +++++++++- packages/mcp-server/src/sync/field-compare.js | 23 ++++++++++ packages/mcp-server/src/sync/remap.js | 13 ++++++ .../mcp-server/test/sync/test-diff.test.js | 46 +++++++++++++++++++ .../test/sync/test-remap-refs.test.js | 13 ++++++ 6 files changed, 122 insertions(+), 15 deletions(-) diff --git a/packages/mcp-server/src/sync/compare.js b/packages/mcp-server/src/sync/compare.js index a75ac75..d6c1ff0 100644 --- a/packages/mcp-server/src/sync/compare.js +++ b/packages/mcp-server/src/sync/compare.js @@ -1,6 +1,6 @@ // compare.js — base-schema comparator helpers (pure: no fs, no network, no Date.now/Math.random) -import { choiceNames, scalarTypeOptionsChanged, computedSig } from './field-compare.js'; +import { choiceNames, scalarTypeOptionsChanged, computedSig, linkSig } from './field-compare.js'; import { canonicalizeViewConfig } from './remap.js'; const BEST_EFFORT = new Set(['fieldOrder', 'viewOrder', 'columnOrder', 'sortOrder', 'groupOrder']); @@ -139,21 +139,12 @@ export function compareFields(srcTable, destTable, idmap, srcGlobalFldNames, des entries.push({ scope, key: 'typeOptions', source: sf.typeOptions, dest: df.typeOptions, class: classOf('typeOptions') }); } } else if (sf.type === 'foreignKey' || sf.type === 'multipleRecordLinks') { - // Link fields: canonicalize before comparing. - // foreignTableId is base-scoped (always differs cross-base) → resolve to table NAME. - // symmetricColumnId is a base-scoped reciprocal field id → drop it entirely (no semantic - // content that sync controls; it's auto-created by Airtable when the link is made). - const canonLink = (opts, tblNames) => { - if (!opts) return null; - const { symmetricColumnId: _dropped, foreignTableId, ...rest } = opts; // eslint-disable-line no-unused-vars - return JSON.stringify({ - ...rest, - foreignTableName: (tblNames && tblNames[foreignTableId]) ?? foreignTableId ?? null, - }); - }; + // Link fields: canonical remap-aware identity (linkSig, shared with diff.js so + // mode=diff and mode=plan agree) — foreignTableId resolves to the table NAME; + // base-scoped symmetricColumnId / viewIdForRecordSelection are dropped entirely. const srcTblNames = srcGlobalTblNames ?? {}; const destTblNames = destGlobalTblNames ?? {}; - if (canonLink(sf.typeOptions, srcTblNames) !== canonLink(df.typeOptions, destTblNames)) { + if (linkSig(sf, srcTblNames) !== linkSig(df, destTblNames)) { entries.push({ scope, key: 'typeOptions', source: sf.typeOptions, dest: df.typeOptions, class: classOf('typeOptions') }); } } else { diff --git a/packages/mcp-server/src/sync/diff.js b/packages/mcp-server/src/sync/diff.js index 3de8a76..8b4a562 100644 --- a/packages/mcp-server/src/sync/diff.js +++ b/packages/mcp-server/src/sync/diff.js @@ -1,5 +1,5 @@ import { canonicalizeViewConfig } from './remap.js'; -import { stableStringify, choiceNames, scalarTypeOptionsChanged, computedSig, fieldSignature } from './field-compare.js'; +import { stableStringify, choiceNames, scalarTypeOptionsChanged, computedSig, fieldSignature, linkSig } from './field-compare.js'; /** * Build a flat map of { fieldId → fieldName } across all tables in a snapshot. @@ -52,6 +52,17 @@ function referencedFieldIds(field) { return [...ids]; } +/** + * Build a flat map of { tableId → tableName } for a snapshot (link canonical compare). + * @param {{ tables: Array<{id:string, name:string}> }} snap + * @returns {Record} + */ +function tblNameMap(snap) { + const m = {}; + for (const t of snap.tables) m[t.id] = t.name; + return m; +} + /** * Collect table IDs that a link field's typeOptions references. * Reuses LINK_TYPES which is already authoritative for ordering. @@ -121,6 +132,8 @@ export function computePlan(srcSnap, destSnap, idmap) { const srcNames = fldNameMap(srcSnap); const destNames = fldNameMap(destSnap); + const srcTblNames = tblNameMap(srcSnap); + const destTblNames = tblNameMap(destSnap); const destTablesById = new Map(destSnap.tables.map((t) => [t.id, t])); // ── Duplicate-name detection in src (warn; first-occurrence wins in idmap) ── @@ -204,6 +217,14 @@ export function computePlan(srcSnap, destSnap, idmap) { if (computedSig(sf, srcNames) !== computedSig(df, destNames)) { changes.typeOptions = sf.typeOptions; } + } else if (LINK_TYPES.has(sf.type) && LINK_TYPES.has(df.type)) { + // Link fields: raw typeOptions carry base-local ids (foreignTableId, + // symmetricColumnId) that NEVER match cross-base — compare the canonical + // remap-aware identity instead, so updateField is emitted only on REAL + // divergence (e.g. the link points at a different-named table). + if (linkSig(sf, srcTblNames) !== linkSig(df, destTblNames)) { + changes.typeOptions = sf.typeOptions; + } } else if (scalarTypeOptionsChanged(sf, df)) { changes.typeOptions = sf.typeOptions; } diff --git a/packages/mcp-server/src/sync/field-compare.js b/packages/mcp-server/src/sync/field-compare.js index 356dafe..b497f06 100644 --- a/packages/mcp-server/src/sync/field-compare.js +++ b/packages/mcp-server/src/sync/field-compare.js @@ -108,6 +108,29 @@ export function computedSig(field, fldNames) { return 'C|' + field.type + '|' + canonicalizeComputed(field.type, normalizedOpts, {}); } +/** + * Canonical signature for a link (foreignKey/multipleRecordLinks) field's typeOptions. + * Link options carry base-scoped ids that ALWAYS differ across bases: + * - foreignTableId → resolved to the referenced table NAME (stable cross-base) + * - symmetricColumnId (Airtable-managed reciprocal) and viewIdForRecordSelection + * (base-scoped view id) → dropped entirely + * so two semantically equivalent links compare equal — the raw scalar signature can + * never converge for links and used to emit a phantom updateField on every plan. + * + * @param {{ type:string, typeOptions:object|null }} field + * @param {Record} tblNames tableId → name for the relevant base + * @returns {string} + */ +export function linkSig(field, tblNames) { + const opts = field.typeOptions; + if (!opts) return 'L|' + field.type + '|null'; + const { symmetricColumnId: _sym, viewIdForRecordSelection: _view, foreignTableId, ...rest } = opts; // eslint-disable-line no-unused-vars + return 'L|' + field.type + '|' + stableStringify({ + ...rest, + foreignTableName: (tblNames && tblNames[foreignTableId]) ?? foreignTableId ?? null, + }); +} + /** * Comparable signature for a field. * - Computed fields canonicalize field-ID references to field names so diff --git a/packages/mcp-server/src/sync/remap.js b/packages/mcp-server/src/sync/remap.js index b67fb2f..34de54f 100644 --- a/packages/mcp-server/src/sync/remap.js +++ b/packages/mcp-server/src/sync/remap.js @@ -59,6 +59,19 @@ export function remapRefs(typeOptions, idmap) { for (const k of FORMULA_KEYS) if (typeof out[k] === 'string') out[k] = subFormulaTokens(out[k], refTokens); for (const k of SINGLE_ID_REF_KEYS) if (out[k] && flds[out[k]]) out[k] = flds[out[k]]; if (out.foreignTableId && tbls[out.foreignTableId]) out.foreignTableId = tbls[out.foreignTableId]; + // Link-only base-scoped keys: symmetricColumnId (the reciprocal field, auto-managed by + // Airtable) and viewIdForRecordSelection (limit-to-view). Remap when the idmap knows the + // dest id; otherwise DROP — sending a source-base id to the dest API is always wrong + // (422 or a bogus reciprocal/view reference). + if (out.symmetricColumnId) { + if (flds[out.symmetricColumnId]) out.symmetricColumnId = flds[out.symmetricColumnId]; + else delete out.symmetricColumnId; + } + if (out.viewIdForRecordSelection) { + const vws = idmap.views || {}; + if (vws[out.viewIdForRecordSelection]) out.viewIdForRecordSelection = vws[out.viewIdForRecordSelection]; + else delete out.viewIdForRecordSelection; + } const dep = out.dependencies && out.dependencies.referencedColumnIdsForValue; if (Array.isArray(dep)) out.dependencies.referencedColumnIdsForValue = dep.map((id) => flds[id] ?? id); diff --git a/packages/mcp-server/test/sync/test-diff.test.js b/packages/mcp-server/test/sync/test-diff.test.js index 73023c2..ca63102 100644 --- a/packages/mcp-server/test/sync/test-diff.test.js +++ b/packages/mcp-server/test/sync/test-diff.test.js @@ -157,6 +157,52 @@ describe('diff convergence (apply-aligned typeOptions)', () => { const plan = computePlan(src, dest, matchByName(src, dest)); assert.equal(plan.actions.filter((a) => a.kind === 'updateField').length, 0); }); + + const linkOpts = (tbl, sym, extra = {}) => ({ typeOptions: { foreignTableId: tbl, symmetricColumnId: sym, relationship: 'many', unreversed: true, ...extra } }); + + it('no phantom updateField for a matched link field that differs only by base-local ids', () => { + const src = { baseId: 'appS', tables: [ + { id: 'tS1', name: 'Projects', primaryFieldId: 'fSp', fields: [field('fSp', 'Name', 'text')] }, + { id: 'tS2', name: 'Tasks', primaryFieldId: 'fSt', fields: [field('fSt', 'Task', 'text'), field('fSl', 'Project', 'foreignKey', linkOpts('tS1', 'fSsym'))] }, + ] }; + const dest = { baseId: 'appD', tables: [ + { id: 'tD1', name: 'Projects', primaryFieldId: 'fDp', fields: [field('fDp', 'Name', 'text')] }, + { id: 'tD2', name: 'Tasks', primaryFieldId: 'fDt', fields: [field('fDt', 'Task', 'text'), field('fDl', 'Project', 'foreignKey', linkOpts('tD1', 'fDsym'))] }, + ] }; + const plan = computePlan(src, dest, matchByName(src, dest)); + assert.equal(plan.actions.length, 0, 'equivalent link fields must not emit updateField'); + }); + + it('link field with a limit-to-view id that differs only cross-base still converges', () => { + const src = { baseId: 'appS', tables: [ + { id: 'tS1', name: 'Projects', primaryFieldId: 'fSp', fields: [field('fSp', 'Name', 'text')] }, + { id: 'tS2', name: 'Tasks', primaryFieldId: 'fSt', fields: [field('fSt', 'Task', 'text'), field('fSl', 'Project', 'foreignKey', linkOpts('tS1', 'fSsym', { viewIdForRecordSelection: 'viwSRC' }))] }, + ] }; + const dest = { baseId: 'appD', tables: [ + { id: 'tD1', name: 'Projects', primaryFieldId: 'fDp', fields: [field('fDp', 'Name', 'text')] }, + { id: 'tD2', name: 'Tasks', primaryFieldId: 'fDt', fields: [field('fDt', 'Task', 'text'), field('fDl', 'Project', 'foreignKey', linkOpts('tD1', 'fDsym', { viewIdForRecordSelection: 'viwDST' }))] }, + ] }; + const plan = computePlan(src, dest, matchByName(src, dest)); + assert.equal(plan.actions.filter((a) => a.kind === 'updateField').length, 0); + }); + + it('updateField when a matched link field genuinely diverges (points at a different-named table)', () => { + const src = { baseId: 'appS', tables: [ + { id: 'tS1', name: 'Projects', primaryFieldId: 'fSp', fields: [field('fSp', 'Name', 'text')] }, + { id: 'tS3', name: 'Archive', primaryFieldId: 'fSa', fields: [field('fSa', 'Name', 'text')] }, + { id: 'tS2', name: 'Tasks', primaryFieldId: 'fSt', fields: [field('fSt', 'Task', 'text'), field('fSl', 'Project', 'foreignKey', linkOpts('tS1', 'fSsym'))] }, + ] }; + const dest = { baseId: 'appD', tables: [ + { id: 'tD1', name: 'Projects', primaryFieldId: 'fDp', fields: [field('fDp', 'Name', 'text')] }, + { id: 'tD3', name: 'Archive', primaryFieldId: 'fDa', fields: [field('fDa', 'Name', 'text')] }, + { id: 'tD2', name: 'Tasks', primaryFieldId: 'fDt', fields: [field('fDt', 'Task', 'text'), field('fDl', 'Project', 'foreignKey', linkOpts('tD3', 'fDsym'))] }, + ] }; + const plan = computePlan(src, dest, matchByName(src, dest)); + const upd = plan.actions.filter((a) => a.kind === 'updateField'); + assert.equal(upd.length, 1); + assert.equal(upd[0].destFld, 'fDl'); + assert.ok(upd[0].changes.typeOptions, 'real link divergence carries typeOptions'); + }); }); function vw(id, name, type, config, extra = {}) { return { id, name, type, config: config ?? {}, personalForUserId: extra.personal ?? null }; } diff --git a/packages/mcp-server/test/sync/test-remap-refs.test.js b/packages/mcp-server/test/sync/test-remap-refs.test.js index 09354a3..50525f6 100644 --- a/packages/mcp-server/test/sync/test-remap-refs.test.js +++ b/packages/mcp-server/test/sync/test-remap-refs.test.js @@ -27,6 +27,19 @@ describe('remap.remapRefs', () => { assert.equal(out.foreignTableRollupColumnId, 'fldY'); assert.equal(out.foreignTableId, 'tblDST'); }); + it('remaps symmetricColumnId when mapped; DROPS it when unmappable (never sends a source id)', () => { + const mapped = remapRefs({ foreignTableId: 'tblSRC', symmetricColumnId: 'fldA' }, idmap); + assert.equal(mapped.symmetricColumnId, 'fldX'); + const unmapped = remapRefs({ foreignTableId: 'tblSRC', symmetricColumnId: 'fldNOPE' }, idmap); + assert.equal('symmetricColumnId' in unmapped, false); + }); + it('remaps viewIdForRecordSelection via idmap.views; DROPS it when unmappable', () => { + const im = { ...idmap, views: { viwSRC: 'viwDST' } }; + const mapped = remapRefs({ foreignTableId: 'tblSRC', viewIdForRecordSelection: 'viwSRC' }, im); + assert.equal(mapped.viewIdForRecordSelection, 'viwDST'); + const unmapped = remapRefs({ foreignTableId: 'tblSRC', viewIdForRecordSelection: 'viwNOPE' }, im); + assert.equal('viewIdForRecordSelection' in unmapped, false); + }); it('rewrites the dependencies array and leaves unknown ids intact', () => { const out = remapRefs({ dependencies: { referencedColumnIdsForValue: ['fldA', 'fldUNKNOWN'] } }, idmap); assert.deepEqual(out.dependencies.referencedColumnIdsForValue, ['fldX', 'fldUNKNOWN']); From 12059744d8d8c13e04e19d0618761600acc646b8 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 14:16:01 +0300 Subject: [PATCH 153/246] fix(sync): compare writable display-format options on computed fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit canonicalizeComputed stripped ALL display/format typeOptions before compare, so a format divergence on a formula/rollup/createdTime field (currency vs decimal, precision, dateFormat…) was invisible — false identical/converged verdict in mode=diff and no updateField in mode=plan — even though apply's toWritableComputedOptions can write exactly those keys. The compare canonical now includes the COMPUTED_FORMAT_KEYS subset, except for lookup/multipleLookupValues/count whose write path drops format options (flagging them would re-emit drift apply can never fix). Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/remap.js | 13 +++++++++++ .../mcp-server/test/sync/test-diff.test.js | 12 ++++++++++ .../mcp-server/test/sync/test-remap.test.js | 23 +++++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/packages/mcp-server/src/sync/remap.js b/packages/mcp-server/src/sync/remap.js index 34de54f..3dd6ac9 100644 --- a/packages/mcp-server/src/sync/remap.js +++ b/packages/mcp-server/src/sync/remap.js @@ -7,12 +7,24 @@ function subIds(str, fldIdToName) { return str.replace(FLD_TOKEN, (id) => `{{${fldIdToName[id] ?? id}}}`); } +// Computed types whose WRITE path (toWritableComputedOptions) drops display-format options — +// comparing format for them would flag drift apply can never fix (perpetual re-flag). +const FORMAT_INCOMPARABLE_TYPES = new Set(['lookup', 'multipleLookupValues', 'count']); + export function canonicalizeComputed(type, typeOptions, fldIdToName) { const opts = typeOptions || {}; // Pull the formula expression under whichever key the API used. const formula = opts.formulaTextParsed ?? opts.formulaText ?? opts.formula ?? ''; const relation = opts.relationColumnId ?? opts.recordLinkFieldId ?? ''; const target = opts.foreignTableRollupColumnId ?? opts.fieldIdInLinkedTable ?? ''; + // Writable display/format options (currency vs decimal, precision, dateFormat…) ARE synced + // by apply (toWritableComputedOptions carries COMPUTED_FORMAT_KEYS), so they must participate + // in the compare canonical — otherwise format drift on computed fields is invisible forever + // (false identical/converged verdict + no updateField). Value-stable across bases. + const format = {}; + if (!FORMAT_INCOMPARABLE_TYPES.has(type)) { + for (const k of COMPUTED_FORMAT_KEYS) if (k in opts) format[k] = opts[k]; + } return JSON.stringify({ type, formula: subIds(formula, fldIdToName), @@ -20,6 +32,7 @@ export function canonicalizeComputed(type, typeOptions, fldIdToName) { target: fldIdToName[target] ?? target, // result aggregation type (rollup) is value-stable across bases result: opts.result?.type ?? null, + format, }); } diff --git a/packages/mcp-server/test/sync/test-diff.test.js b/packages/mcp-server/test/sync/test-diff.test.js index ca63102..96a4fb7 100644 --- a/packages/mcp-server/test/sync/test-diff.test.js +++ b/packages/mcp-server/test/sync/test-diff.test.js @@ -203,6 +203,18 @@ describe('diff convergence (apply-aligned typeOptions)', () => { assert.equal(upd[0].destFld, 'fDl'); assert.ok(upd[0].changes.typeOptions, 'real link divergence carries typeOptions'); }); + + it('detects format divergence on a matched formula field (same formula, different display format)', () => { + const formulaField = (id, ref, fmt) => ({ id, name: 'Total', type: 'formula', typeOptions: { formulaTextParsed: `{${ref}}*2`, ...fmt }, description: null, isComputed: true }); + const src = { baseId: 'appS', tables: [{ id: 'tS', name: 'T', primaryFieldId: 'fS1', fields: [ + field('fS1', 'Name', 'text'), formulaField('fS2', 'fS1', { format: 'currency', symbol: '$', precision: 2 }) ] }] }; + const dest = { baseId: 'appD', tables: [{ id: 'tD', name: 'T', primaryFieldId: 'fD1', fields: [ + field('fD1', 'Name', 'text'), formulaField('fD2', 'fD1', { format: 'decimal', precision: 0 }) ] }] }; + const plan = computePlan(src, dest, matchByName(src, dest)); + const upd = plan.actions.filter((a) => a.kind === 'updateField'); + assert.equal(upd.length, 1, 'format-only divergence on a computed field must be detected'); + assert.ok(upd[0].changes.typeOptions); + }); }); function vw(id, name, type, config, extra = {}) { return { id, name, type, config: config ?? {}, personalForUserId: extra.personal ?? null }; } diff --git a/packages/mcp-server/test/sync/test-remap.test.js b/packages/mcp-server/test/sync/test-remap.test.js index 1eee279..1ae8469 100644 --- a/packages/mcp-server/test/sync/test-remap.test.js +++ b/packages/mcp-server/test/sync/test-remap.test.js @@ -25,4 +25,27 @@ describe('remap.canonicalizeComputed', () => { const s = canonicalizeComputed('formula', { formulaTextParsed: '{fldUNKNOWN}+1' }, {}); assert.match(s, /fldUNKNOWN/); }); + it('detects display-format divergence on a formula (same formula, different format options)', () => { + const s = canonicalizeComputed('formula', { formulaTextParsed: '{fldA}*2', format: 'currency', symbol: '$', precision: 2 }, srcNames); + const d = canonicalizeComputed('formula', { formulaTextParsed: '{fldX}*2', format: 'decimal', precision: 0 }, destNames); + assert.notEqual(s, d); + }); + it('identical format options canonicalize equal across bases', () => { + const s = canonicalizeComputed('formula', { formulaTextParsed: '{fldA}*2', format: 'currency', symbol: '$', precision: 2 }, srcNames); + const d = canonicalizeComputed('formula', { formulaTextParsed: '{fldX}*2', format: 'currency', symbol: '$', precision: 2 }, destNames); + assert.equal(s, d); + }); + it('detects dateFormat divergence on a date-typed formula', () => { + const s = canonicalizeComputed('formula', { formulaTextParsed: 'CREATED_TIME()', isDateTime: true, dateFormat: 'Local' }, {}); + const d = canonicalizeComputed('formula', { formulaTextParsed: 'CREATED_TIME()', isDateTime: true, dateFormat: 'ISO' }, {}); + assert.notEqual(s, d); + }); + it('IGNORES format divergence on lookup/count (apply cannot write their format — would never converge)', () => { + const sL = canonicalizeComputed('lookup', { relationColumnId: 'fldA', foreignTableRollupColumnId: 'fldB', format: 'currency' }, srcNames); + const dL = canonicalizeComputed('lookup', { relationColumnId: 'fldX', foreignTableRollupColumnId: 'fldY' }, destNames); + assert.equal(sL, dL); + const sC = canonicalizeComputed('count', { relationColumnId: 'fldA', format: 'decimal' }, srcNames); + const dC = canonicalizeComputed('count', { relationColumnId: 'fldX' }, destNames); + assert.equal(sC, dC); + }); }); From dd9a0aaf865df8904be505879ca19f3f1656a48e Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 14:16:48 +0300 Subject: [PATCH 154/246] fix(sync): pass collaborator usr ids through view-filter remap verbatim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit remapViewConfig/canonFilterSet treated usrXXX filter values as unresolvable record refs and stripped them — but Airtable user ids are account-global (identical in every base the user can access), so a collaborator condition like Assignee = usrABC works verbatim on the dest. Stripping it silently widened the dest view's row set, permanently and invisibly (both canonical sides dropped the leaf, so diff reported converged). RECORD_REF_ID now matches only rec ids; usr values pass through unchanged on both the write path and both canonical sides (kept symmetric), and are no longer reported by collectFilterRecordRefs. Mixed [rec, usr] array values keep the usr element while the rec element is remapped/stripped. Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/remap.js | 25 +++++++++++-------- .../test/sync/test-reapply-filters.test.js | 8 +++--- .../test/sync/test-remap-view.test.js | 19 ++++++++++++-- 3 files changed, 37 insertions(+), 15 deletions(-) diff --git a/packages/mcp-server/src/sync/remap.js b/packages/mcp-server/src/sync/remap.js index 3dd6ac9..40fecc7 100644 --- a/packages/mcp-server/src/sync/remap.js +++ b/packages/mcp-server/src/sync/remap.js @@ -126,13 +126,14 @@ function destSelId(idmap, src) { for (const v of Object.values(idmap.fields || {})) { const d = ((v && v.choices) || {})[src]; if (d) return d; } return src; } -// A filter LEAF whose value references a record/collaborator id — or a structured/dynamic -// value carrying SOURCE field/table/row ids — cannot be resolved in the dest: records and -// users are not synced (and ids differ across bases regardless). Detect by value SHAPE, which -// needs NO source field-type context (those types aren't carried into apply). Portable -// sentinels (current-user "me", null, booleans) are not id-shaped → kept. Forward-compat: once -// records sync (M3), a populated idmap.records lets these REMAP instead of strip — see canon note. -const RECORD_REF_ID = /^(rec|usr)[A-Za-z0-9]{14,}$/; +// A filter LEAF whose value references a RECORD id — or a structured/dynamic value carrying +// SOURCE field/table/row ids — is base-scoped: remap via idmap.records when possible, strip +// otherwise. Detect by value SHAPE, which needs NO source field-type context (those types +// aren't carried into apply). Collaborator ids (usrXXX) are NOT record refs: Airtable user +// ids are account-global — identical in every base the user can access — so they pass through +// VERBATIM (stripping them silently widened the dest view's row set). Portable sentinels +// (current-user "me", null, booleans) are not id-shaped → kept. +const RECORD_REF_ID = /^rec[A-Za-z0-9]{14,}$/; // Remap a record-ref leaf VALUE via idmap.records. // stripUnresolved=true (source/write path): src rec ids -> dest ids; ids with no mapping are DROPPED; // returns undefined if nothing resolves (caller drops the whole leaf). @@ -144,7 +145,10 @@ function remapRecRefValue(value, idmap, stripUnresolved) { if (value && typeof value === 'object' && !Array.isArray(value)) { return stripUnresolved ? undefined : value; } - const mapOne = (v) => (typeof v === 'string' ? (recs[v] ?? (stripUnresolved ? undefined : v)) : v); + const mapOne = (v) => { + if (typeof v !== 'string' || !RECORD_REF_ID.test(v)) return v; // non-record values (e.g. global usr ids) pass verbatim + return recs[v] ?? (stripUnresolved ? undefined : v); + }; if (Array.isArray(value)) { const out = value.map(mapOne).filter((v) => v !== undefined); return out.length ? out : undefined; @@ -168,11 +172,12 @@ function collectRecordRefIds(set) { } return ids; } -// Source record/collaborator ids that view-filter strip will drop (for the apply-side warning). +// Source record ids that view-filter strip will drop (for the apply-side warning). +// Collaborator usr ids are global and pass through — they are never collected here. export function collectFilterRecordRefs(config) { return (config && config.filters && Array.isArray(config.filters.filterSet)) ? collectRecordRefIds(config.filters.filterSet) : []; } -// Remap resolvable refs (fld/choice ids) and STRIP unresolvable record/collaborator-ref leaves, +// Remap resolvable refs (fld/choice ids) and STRIP unresolvable record-ref leaves, // pruning groups emptied by stripping (bottom-up). Mirrored exactly by canonFilterSet so a // stripped source filter canonicalizes identically to the stripped dest readback → converges. function remapFilterSet(set, idmap) { diff --git a/packages/mcp-server/test/sync/test-reapply-filters.test.js b/packages/mcp-server/test/sync/test-reapply-filters.test.js index d90e1d4..9f39305 100644 --- a/packages/mcp-server/test/sync/test-reapply-filters.test.js +++ b/packages/mcp-server/test/sync/test-reapply-filters.test.js @@ -55,7 +55,7 @@ describe('remap — record-ref filter REMAP when records exist', () => { }); it('drops entire leaf when all array elements are unresolvable', () => { - const recMiss2 = 'recALSOGONEXXXXX'; + const recMiss2 = 'recALSOGONEXXXXXX'; // rec + 14 chars — a real record-id shape const cfg = { filters: { conjunction: 'and', @@ -65,7 +65,7 @@ describe('remap — record-ref filter REMAP when records exist', () => { assert.equal(remapViewConfig(cfg, idmap).filters.filterSet.length, 0); }); - it('collaborator usr ids are still stripped (not in idmap.records)', () => { + it('collaborator usr ids pass through verbatim (user ids are Airtable-global, never in idmap.records)', () => { const USR = 'usrCCCCCCCCCCCCCC'; const cfg = { filters: { @@ -73,7 +73,9 @@ describe('remap — record-ref filter REMAP when records exist', () => { filterSet: [{ columnId: 'fldG', operator: '=', value: USR }], }, }; - assert.equal(remapViewConfig(cfg, idmap).filters.filterSet.length, 0); + const out = remapViewConfig(cfg, idmap); + assert.equal(out.filters.filterSet.length, 1); + assert.equal(out.filters.filterSet[0].value, USR); }); it('empty idmap.records -> strip -> leaf dropped (backward compat)', () => { diff --git a/packages/mcp-server/test/sync/test-remap-view.test.js b/packages/mcp-server/test/sync/test-remap-view.test.js index 6603efc..3eb765e 100644 --- a/packages/mcp-server/test/sync/test-remap-view.test.js +++ b/packages/mcp-server/test/sync/test-remap-view.test.js @@ -111,14 +111,29 @@ describe('remap — record-referencing view filters (strip + report + converge)' assert.equal(out.filters.filterSet.length, 1); // emptied nested group pruned assert.equal(out.filters.filterSet[0].columnId, 'fldY'); }); - it('strips collaborator usr ids but keeps the portable "me" sentinel', () => { + it('passes collaborator usr ids through VERBATIM (user ids are Airtable-global) and keeps "me"', () => { const cfg = { filters: { conjunction: 'and', filterSet: [ { columnId: 'fldB', operator: '=', value: USR }, { columnId: 'fldB', operator: '=', value: 'me' }, ] } }; const out = remapViewConfig(cfg, idmap); + assert.equal(out.filters.filterSet.length, 2); + assert.equal(out.filters.filterSet[0].value, USR); + assert.equal(out.filters.filterSet[1].value, 'me'); + assert.deepEqual(collectFilterRecordRefs(cfg), []); // usr is not a stripped record ref + }); + it('usr filter canonicalizes identically on source (strip) and dest (keep) sides — converges', () => { + const cfg = { filters: { conjunction: 'and', filterSet: [{ columnId: 'fldB', operator: '=', value: USR }] } }; + const srcSide = canonicalizeViewConfig(cfg, { fldB: 'Assignee' }, {}, true); + const destSide = canonicalizeViewConfig(cfg, { fldB: 'Assignee' }, {}, false); + assert.equal(srcSide, destSide); + assert.match(srcSide, /usrCCCCCCCCCCCCCC/); // the condition survives, not dropped + }); + it('mixed [rec, usr] array value: rec remapped/stripped per idmap.records, usr kept verbatim', () => { + const cfg = { filters: { conjunction: 'and', filterSet: [{ columnId: 'fldA', operator: '=', value: [REC, USR] }] } }; + const out = remapViewConfig(cfg, idmap); // idmap has no records → REC unresolvable assert.equal(out.filters.filterSet.length, 1); - assert.equal(out.filters.filterSet[0].value, 'me'); + assert.deepEqual(out.filters.filterSet[0].value, [USR]); }); it('strips structured/dynamic filter values that carry source ids', () => { const cfg = { filters: { conjunction: 'and', filterSet: [{ columnId: 'fldA', operator: '|', value: { tableId: 'tblZ', columnId: 'fldZ', rowId: null } }] } }; From 8d34ab39040f31045fd95c2fbe073301831e16eb Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 14:17:55 +0300 Subject: [PATCH 155/246] =?UTF-8?q?fix(sync):=20match=20views=20by=20(name?= =?UTF-8?q?,=20type)=20=E2=80=94=20cross-type=20same-name=20views=20are=20?= =?UTF-8?q?not=20counterparts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A view's type is immutable in Airtable, so adopting a same-named dest view of a different type produced perpetual drift (no action can retype a view) plus cross-type facet errors when applyViewConfig pushed e.g. kanban config at a grid view. - diff.js: view matching and view-orphan detection are keyed by (name, type); a type mismatch emits createView with the source type and the wrong-typed dest view becomes an orphan candidate under the usual confirmDeletions gate - compare.js: same (name, type) matching (a stale cross-type idmap.views mapping is rejected); the pair is reported as onlyInSource + onlyInDest instead of an unfixable per-view 'type' drift entry, so converged is reachable once sync has created the right view Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/compare.js | 23 ++++++++--------- packages/mcp-server/src/sync/diff.js | 18 +++++++++---- .../test/sync/test-compare-views.test.js | 25 +++++++++++++------ .../mcp-server/test/sync/test-diff.test.js | 24 ++++++++++++++++++ 4 files changed, 65 insertions(+), 25 deletions(-) diff --git a/packages/mcp-server/src/sync/compare.js b/packages/mcp-server/src/sync/compare.js index d6c1ff0..e45f49e 100644 --- a/packages/mcp-server/src/sync/compare.js +++ b/packages/mcp-server/src/sync/compare.js @@ -269,12 +269,14 @@ function groupOrderKeys(config, fldNames) { * Compare views between a source table and a destination table. * * Matching strategy (per view): - * 1. Look up dest view id via idmap.views[srcView.id]. - * 2. Fall back to matching by view name in dest. + * 1. Look up dest view id via idmap.views[srcView.id] — rejected if the types differ + * (a view's type is immutable, so a cross-type mapping is stale/name-only noise). + * 2. Fall back to matching by (view name, view type) in dest. A same-named dest view of + * a DIFFERENT type is NOT a counterpart: sync converges by creating the source-typed + * view and orphaning the dest one, so it is reported as onlyInSource + onlyInDest. * Personal views (personalForUserId is set) are skipped on both sides. * * Per matched pair emits DiffEntries with scope `view:`: - * - `type` → class drift * - `filters` / `sorts` / `groups` / `columnVisibility` / `frozen` / `color` / `cover` / * `calendar` / `rowHeight` / `form` → class drift (via canonicalizeViewConfig comparison) * - `columnOrder` → class best-effort @@ -299,17 +301,19 @@ export function compareViews(srcTable, destTable, idmap) { const srcViews = collabViews(srcTable); const destViews = collabViews(destTable); - // Build dest view lookup: by id and by name. + // Build dest view lookup: by id and by (name, type) — a view's type is immutable, so a + // same-named view of a different type can never converge and must not be adopted. const destViewsById = new Map(destViews.map((v) => [v.id, v])); - const destViewsByName = new Map(destViews.map((v) => [v.name, v])); + const destViewsByNameType = new Map(destViews.map((v) => [`${v.name}|${v.type}`, v])); const matchedDestViewIds = new Set(); for (const sv of srcViews) { - // Resolve dest view: idmap first, then by-name fallback. + // Resolve dest view: idmap first (same type only), then by (name, type) fallback. let dv = null; const mappedDestId = idmap.views && idmap.views[sv.id]; if (mappedDestId) dv = destViewsById.get(mappedDestId) ?? null; - if (!dv) dv = destViewsByName.get(sv.name) ?? null; + if (dv && dv.type !== sv.type) dv = null; // stale name-only mapping across types + if (!dv) dv = destViewsByNameType.get(`${sv.name}|${sv.type}`) ?? null; if (!dv) { onlyInSource.push(sv.name); @@ -321,11 +325,6 @@ export function compareViews(srcTable, destTable, idmap) { const srcConfig = sv.config || {}; const destConfig = dv.config || {}; - // 1. Type drift. - if (sv.type !== dv.type) { - entries.push({ scope, key: 'type', source: sv.type, dest: dv.type, class: classOf('type') }); - } - // 2. Canonicalize both configs for content comparison. // Source: stripRecordRefs=true (matches what apply will write) // Dest: stripRecordRefs=false (keep dest's existing record refs so dangling ones diverge) diff --git a/packages/mcp-server/src/sync/diff.js b/packages/mcp-server/src/sync/diff.js index 8b4a562..0661bd4 100644 --- a/packages/mcp-server/src/sync/diff.js +++ b/packages/mcp-server/src/sync/diff.js @@ -29,6 +29,13 @@ function selNameMap(snap) { /** Return only collaborative (non-personal) views from a table. */ function collabViews(table) { return (table.views || []).filter((v) => !v.personalForUserId); } +/** + * Views match by (name, type): a view's TYPE is immutable in Airtable, so a same-named + * dest view of a different type is NOT a counterpart — the plan emits createView (source + * type) and the wrong-typed dest view becomes an orphan candidate under the usual gates. + */ +function viewKey(v) { return `${v.name}|${v.type}`; } + /** Link-type fields must be created after plain scalar fields. Computed last. */ const LINK_TYPES = new Set(['multipleRecordLinks', 'foreignKey']); function fieldOrder(f) { return f.isComputed ? 2 : (LINK_TYPES.has(f.type) ? 1 : 0); } @@ -274,10 +281,10 @@ export function computePlan(srcSnap, destSnap, idmap) { for (const st of srcSnap.tables) { const destTableId = idmap.tables[st.id]; const destTable = destTableId ? destTablesById.get(destTableId) : null; - const destViewsByName = destTable ? new Map(collabViews(destTable).map((v) => [v.name, v])) : new Map(); + const destViewsByKey = destTable ? new Map(collabViews(destTable).map((v) => [viewKey(v), v])) : new Map(); for (const sv of (st.views || [])) { if (sv.personalForUserId) { warnings.push({ code: 'VIEW_PERSONAL_SKIPPED', message: `Personal view "${sv.name}" in "${st.name}" skipped` }); continue; } - const dv = destViewsByName.get(sv.name); + const dv = destViewsByKey.get(viewKey(sv)); if (!dv) { viewActions.push({ kind: 'createView', sourceTableId: st.id, sourceViewId: sv.id, name: sv.name, type: sv.type }); viewActions.push({ kind: 'applyViewConfig', sourceTableId: st.id, sourceViewId: sv.id, type: sv.type, config: sv.config || {} }); @@ -288,13 +295,14 @@ export function computePlan(srcSnap, destSnap, idmap) { } actions.push(...viewActions); - // View orphans: dest-only collaborative views in a matched table. + // View orphans: dest-only collaborative views in a matched table. Keyed by (name, type) + // to mirror the matching above — a same-named dest view of a DIFFERENT type is an orphan. for (const dt of destSnap.tables) { const srcTableId = srcTableByDestId.get(dt.id); if (!srcTableId) continue; // whole table is already a table orphan const srcTable = srcTablesById.get(srcTableId); - const srcViewNames = new Set(collabViews(srcTable).map((v) => v.name)); - for (const dv of collabViews(dt)) if (!srcViewNames.has(dv.name)) orphans.push({ kind: 'view', destId: dv.id, name: dv.name, tableName: dt.name }); + const srcViewKeys = new Set(collabViews(srcTable).map(viewKey)); + for (const dv of collabViews(dt)) if (!srcViewKeys.has(viewKey(dv))) orphans.push({ kind: 'view', destId: dv.id, name: dv.name, tableName: dt.name }); } // Section orphans: dest-only sidebar sections in a matched table (by name). diff --git a/packages/mcp-server/test/sync/test-compare-views.test.js b/packages/mcp-server/test/sync/test-compare-views.test.js index 77855d8..abe9230 100644 --- a/packages/mcp-server/test/sync/test-compare-views.test.js +++ b/packages/mcp-server/test/sync/test-compare-views.test.js @@ -84,17 +84,26 @@ describe('compareTable — basic wiring', () => { // ── compareViews — view type drift ──────────────────────────────────────────── -describe('compareViews — view type drift', () => { - test('view type change emits drift entry with scope view: and key type', () => { +describe('compareViews — view type mismatch (name+type matching)', () => { + test('same-named view of a DIFFERENT type is NOT matched: reported as onlyInSource + onlyInDest', () => { + // A view's type is immutable in Airtable — sync converges by creating the source-typed + // view and orphaning the dest one, so compare must mirror that (no per-view drift entries). const src = tbl('tS', 'T', 'p', [], [view('vS', 'MyView', 'grid', {})]); const dest = tbl('tD', 'T', 'p', [], [view('vD', 'MyView', 'gallery', {})]); - const im = idmap({ tables: { tS: 'tD' }, views: { vS: 'vD' } }); + const im = idmap({ tables: { tS: 'tD' }, views: { vS: 'vD' } }); // stale name-only mapping is rejected const r = compareTable(src, dest, im); - const e = r.entries.find((e) => e.scope === 'view:MyView' && e.key === 'type'); - assert.ok(e, 'type entry for view:MyView missing'); - assert.equal(e.class, 'drift'); - assert.equal(e.source, 'grid'); - assert.equal(e.dest, 'gallery'); + assert.ok(!r.entries.find((e) => e.scope === 'view:MyView'), 'no per-view entries for an unmatched pair'); + assert.deepEqual(r.views.onlyInSource, ['MyView']); + assert.deepEqual(r.views.onlyInDest, ['MyView']); + }); + + test('falls back to a same-name same-type dest view when another same-named view of a different type exists', () => { + const src = tbl('tS', 'T', 'p', [], [view('vS', 'Board', 'kanban', {})]); + const dest = tbl('tD', 'T', 'p', [], [view('vD1', 'Board', 'grid', {}), view('vD2', 'Board', 'kanban', {})]); + const im = idmap({ tables: { tS: 'tD' } }); + const r = compareTable(src, dest, im); + assert.deepEqual(r.views.onlyInSource, []); + assert.deepEqual(r.views.onlyInDest, ['Board']); // the grid duplicate }); test('same view type produces no type entry', () => { diff --git a/packages/mcp-server/test/sync/test-diff.test.js b/packages/mcp-server/test/sync/test-diff.test.js index 96a4fb7..75fc08b 100644 --- a/packages/mcp-server/test/sync/test-diff.test.js +++ b/packages/mcp-server/test/sync/test-diff.test.js @@ -247,4 +247,28 @@ describe('diff view actions', () => { assert.ok(plan.orphans.some((o) => o.kind === 'view' && o.name === 'DestOnly')); assert.ok(plan.warnings.some((w) => w.code === 'VIEW_PERSONAL_SKIPPED')); }); + + it('same-named view of a DIFFERENT type is no match: createView (source type) + dest view is an orphan candidate', () => { + const src = { baseId: 'appS', tables: [{ id: 'tS', name: 'T', primaryFieldId: 'fS1', fields: [field('fS1', 'Name', 'text')], views: [ + vw('vS1', 'Board', 'kanban', {}) ] }] }; + const dest = { baseId: 'appD', tables: [{ id: 'tD', name: 'T', primaryFieldId: 'fD1', fields: [field('fD1', 'Name', 'text')], views: [ + vw('vD1', 'Board', 'grid', {}) ] }] }; + const plan = computePlan(src, dest, matchByName(src, dest)); + const cv = plan.actions.find((a) => a.kind === 'createView'); + assert.ok(cv, 'createView must be emitted for the type-mismatched view'); + assert.equal(cv.name, 'Board'); + assert.equal(cv.type, 'kanban'); + assert.ok(plan.orphans.some((o) => o.kind === 'view' && o.destId === 'vD1'), 'wrong-typed dest view becomes an orphan candidate'); + }); + + it('matched view of the SAME name+type is adopted even when another same-named view of a different type exists', () => { + const src = { baseId: 'appS', tables: [{ id: 'tS', name: 'T', primaryFieldId: 'fS1', fields: [field('fS1', 'Name', 'text')], views: [ + vw('vS1', 'Board', 'kanban', {}) ] }] }; + const dest = { baseId: 'appD', tables: [{ id: 'tD', name: 'T', primaryFieldId: 'fD1', fields: [field('fD1', 'Name', 'text')], views: [ + vw('vD1', 'Board', 'grid', {}), vw('vD2', 'Board', 'kanban', {}) ] }] }; + const plan = computePlan(src, dest, matchByName(src, dest)); + assert.equal(plan.actions.filter((a) => a.kind === 'createView').length, 0, 'kanban Board already exists — no createView'); + assert.ok(plan.orphans.some((o) => o.kind === 'view' && o.destId === 'vD1'), 'the grid duplicate is the orphan'); + assert.ok(!plan.orphans.some((o) => o.kind === 'view' && o.destId === 'vD2')); + }); }); From 79ba83dc2c4349f69eb0750559ef2ebac4a124f2 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 14:18:24 +0300 Subject: [PATCH 156/246] fix(sync): never emit the dest primary field as a deletable orphan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the primary names differed, the orphan scan flagged the dest primary as a dest-only field while reconcilePrimary renamed it in the same plan — pruneSchema then tried to delete the just-renamed primary (over-counted DELETION_GATED dry-runs; spurious SCHEMA_DELETE_FAILED/BLOCKED warnings under confirmDeletions). The primary is owned by the reconcilePrimary path and is skipped by the field-orphan loop. Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/diff.js | 5 +++++ packages/mcp-server/test/sync/test-diff.test.js | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/packages/mcp-server/src/sync/diff.js b/packages/mcp-server/src/sync/diff.js index 0661bd4..2002828 100644 --- a/packages/mcp-server/src/sync/diff.js +++ b/packages/mcp-server/src/sync/diff.js @@ -268,6 +268,11 @@ export function computePlan(srcSnap, destSnap, idmap) { const srcTable = srcTablesById.get(srcTableId); const srcNamesSet = new Set(srcTable.fields.map((f) => f.name)); for (const df of dt.fields) { + // The dest primary is owned by the reconcilePrimary path (renamed/retyped in this + // same plan when it diverges) — it is NEVER a deletable orphan. Without this guard + // a primary whose name differs from every src field would be handed to pruneSchema + // right after reconcilePrimary renamed it. + if (df.id === dt.primaryFieldId) continue; if (!srcNamesSet.has(df.name)) { orphans.push({ kind: 'field', destId: df.id, name: df.name, tableName: dt.name }); } diff --git a/packages/mcp-server/test/sync/test-diff.test.js b/packages/mcp-server/test/sync/test-diff.test.js index 75fc08b..41f1276 100644 --- a/packages/mcp-server/test/sync/test-diff.test.js +++ b/packages/mcp-server/test/sync/test-diff.test.js @@ -219,6 +219,16 @@ describe('diff convergence (apply-aligned typeOptions)', () => { function vw(id, name, type, config, extra = {}) { return { id, name, type, config: config ?? {}, personalForUserId: extra.personal ?? null }; } +describe('diff orphan safety', () => { + it('never flags the dest primary as a field orphan when primary names differ (reconcilePrimary owns it)', () => { + const src = { baseId: 'appS', tables: [{ id: 'tS', name: 'T', primaryFieldId: 'fS1', fields: [field('fS1', 'Name', 'text')] }] }; + const dest = { baseId: 'appD', tables: [{ id: 'tD', name: 'T', primaryFieldId: 'fD1', fields: [field('fD1', 'Title', 'text')] }] }; + const plan = computePlan(src, dest, matchByName(src, dest)); + assert.ok(plan.actions.some((a) => a.kind === 'reconcilePrimary'), 'primary rename is handled by reconcilePrimary'); + assert.equal(plan.orphans.filter((o) => o.kind === 'field').length, 0, 'dest primary must never be a deletable orphan'); + }); +}); + describe('diff view actions', () => { it('createView + applyViewConfig for a source-only view; skip canonical-equal matched view; views after fields', () => { const src = { baseId: 'appS', tables: [{ id: 'tS', name: 'T', primaryFieldId: 'fS1', fields: [field('fS1', 'Name', 'text')], views: [ From f2cce1962502eac4eddc3c7785e680ff86667c4b Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 14:32:36 +0300 Subject: [PATCH 157/246] fix(sync): guard reconcile existence-prune against truncated dest snapshots A dest table capped at the 1000-row read window made reconcile drop live idmap.records mappings beyond the cap, so the next apply re-created those records as duplicates. Reuse collectTruncatedTableNames: when any dest table is truncated, skip the existence-prune entirely and emit RECORDS_TRUNCATED_PRUNE_SKIPPED per table (mirrors pruneRecords' guard). Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/records.js | 44 ++++++++++++++----- .../test/sync/test-records-apply.test.js | 44 +++++++++++++++++++ 2 files changed, 77 insertions(+), 11 deletions(-) diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index fe76409..a21f754 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -1349,6 +1349,9 @@ export function matchByNaturalKeys({ srcSnapshot, destSnapshot, naturalKeys, idm * 2. Snapshots the DEST base schema only, then pulls dest records for each matched table. * 3. Builds a Set of all live dest record IDs across all tables. * 4. Drops any idmap.records[src]=dest entry whose dest ID is not present in the snapshot. + * SKIPPED entirely (with RECORDS_TRUNCATED_PRUNE_SKIPPED warnings) when any dest table's + * snapshot hit the 1000-row cap — a live mapping beyond the window would otherwise be + * falsely pruned and re-created as a duplicate on the next apply. * 5. Natural-key re-match: snapshots source records and calls matcher to add idmap.records entries by key value. * 6. Saves the pruned idmap. * 7. Returns a result shaped like { created:0, updated:0, skipped, failed:0, matched, warnings, idmap }. @@ -1382,19 +1385,38 @@ export async function reconcile({ client, sourceBaseId, destBaseId, naturalKeys table.records = await snapshotTableRecords(client, destBaseId, table); } - // Build set of all live dest record IDs across all tables - const allLiveDestIds = new Set(); - for (const dt of destSnapshot.tables) { - for (const r of (dt.records || [])) { - allLiveDestIds.add(r.id); + // Truncation guard (same false-orphan class pruneRecords guards with + // RECORDS_TRUNCATED_PRUNE_SKIPPED): a dest table whose snapshot hit the 1000-row cap may + // hold live records beyond the read window. A missing dest id cannot be attributed to a + // table (record ids carry no table info), so ANY truncated dest table makes the whole + // existence-prune unsafe — pruning a live mapping would make the next apply re-create the + // record as a duplicate. Skip the prune entirely and warn instead. + const truncatedDestTables = collectTruncatedTableNames(null, destSnapshot); + if (truncatedDestTables.size > 0) { + for (const name of truncatedDestTables) { + result.warnings.push({ + code: 'RECORDS_TRUNCATED_PRUNE_SKIPPED', + message: `Table "${name}": dest snapshot capped at 1000 rows — a mapping whose dest row lies ` + + `beyond the window cannot be told apart from a deleted record; skipping reconcile's ` + + `existence-prune to avoid duplicating live records on the next apply ` + + `(>1000-row pagination is a follow-up).`, + }); + } + } else { + // Build set of all live dest record IDs across all tables + const allLiveDestIds = new Set(); + for (const dt of destSnapshot.tables) { + for (const r of (dt.records || [])) { + allLiveDestIds.add(r.id); + } } - } - // Existence-prune - for (const [srcRecId, destRecId] of Object.entries(idmap.records)) { - if (!allLiveDestIds.has(destRecId)) { - delete idmap.records[srcRecId]; - result.skipped++; + // Existence-prune + for (const [srcRecId, destRecId] of Object.entries(idmap.records)) { + if (!allLiveDestIds.has(destRecId)) { + delete idmap.records[srcRecId]; + result.skipped++; + } } } diff --git a/packages/mcp-server/test/sync/test-records-apply.test.js b/packages/mcp-server/test/sync/test-records-apply.test.js index 9c9109c..db175e2 100644 --- a/packages/mcp-server/test/sync/test-records-apply.test.js +++ b/packages/mcp-server/test/sync/test-records-apply.test.js @@ -396,4 +396,48 @@ describe('reconcile — prunes stale idmap.records entries', () => { assert.equal(after.records['recSX2'], 'recDX2', 'entry 2 should be kept'); assert.equal(result.skipped, 0, 'no pruning expected'); }); + + it('skips the existence-prune when a dest table snapshot is truncated (1000-row cap)', async () => { + const srcApp = 'appSrcRecZZZZZZZZ'; + const destApp = 'appDstRecZZZZZZZZ'; + + // recSZbeyond → recBeyondCap is a LIVE dest record that fell beyond the 1000-row read + // window; a naive existence-prune would drop it and the next apply would duplicate it. + saveIdmap(srcApp, destApp, { + tables: { tblSZ: 'tblDZ' }, + fields: {}, + records: { recSZ0: 'recDZ0', recSZbeyond: 'recBeyondCap' }, + views: {}, + }); + + // Snapshot returns exactly 1000 rows (the cap) — recBeyondCap is not among them. + const rows = Array.from({ length: 1000 }, (_, i) => ({ id: `recDZ${i}`, cellValuesByColumnId: {} })); + const destData = { + tables: [{ + id: 'tblDZ', name: 'Z', + primaryColumnId: 'fldDZ', + columns: [{ id: 'fldDZ', name: 'Name', type: 'text', typeOptions: null, description: null }], + views: [{ id: 'vDZ', name: 'Grid view', type: 'grid', personalForUserId: null }], + records: rows, + }], + }; + + const client = { + getApplicationData: async () => ({ data: { tableSchemas: JSON.parse(JSON.stringify(destData.tables)) } }), + queryRecords: async (appId, tableId) => { + const tbl = destData.tables.find((t) => t.id === tableId); + return { summary: { rows: ((tbl && tbl.records) || []).slice(0, 1000).map((r) => ({ id: r.id, fields: {} })) } }; + }, + getView: async () => ({}), + }; + + const result = await reconcile({ client, sourceBaseId: srcApp, destBaseId: destApp, naturalKeys: {} }); + + const after = loadIdmap(srcApp, destApp); + assert.equal(after.records['recSZbeyond'], 'recBeyondCap', 'mapping beyond the cap must survive'); + assert.equal(after.records['recSZ0'], 'recDZ0', 'in-window mapping kept too'); + assert.equal(result.skipped, 0, 'nothing pruned while any dest table is truncated'); + assert.ok(result.warnings.some((w) => w.code === 'RECORDS_TRUNCATED_PRUNE_SKIPPED'), + 'truncation-skip warning emitted'); + }); }); From cef92c58fef5899fdc6bcdabec625a5f1aa8924e Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 14:35:04 +0300 Subject: [PATCH 158/246] fix(sync): truncation detection counts fetched rows, not folded synthetic entries mirrorFoldedLinks pushes one synthetic record into destSnapshot per created row with folded links, inflating the live .records length. runRecords calls collectTruncatedTableNames AFTER Pass 1, so a dest table could falsely trip the >=1000 truncation heuristic (dest 400 rows + 700 folded creates = 1100) and silently skip mirror prune with a misleading RECORDS_TRUNCATED warning. Stamp each table's fetched row count (snapshotRowCount) before any pass runs and have collectTruncatedTableNames / pruneRecords test that count, falling back to the live length for direct callers. Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/records.js | 18 +++- .../test/sync/test-records-policy.test.js | 90 ++++++++++++++++++- 2 files changed, 104 insertions(+), 4 deletions(-) diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index a21f754..639f3e2 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -1013,6 +1013,13 @@ export async function runRecords({ client, srcSnapshot, destSnapshot, idmap, pol throw err; } + // Record each table's FETCHED row count before any pass runs — Pass 1 mirrors folded-link + // rows into destSnapshot (mirrorFoldedLinks), and those synthetic entries must not count + // toward the >=1000 truncation heuristic (collectTruncatedTableNames / pruneRecords). + for (const t of [...(srcSnapshot.tables || []), ...(destSnapshot.tables || [])]) { + t.snapshotRowCount ??= (t.records || []).length; + } + // Natural-key pre-pass: match real-but-unmapped dest records BEFORE Pass 1 + prune, // so Pass 1 updates (not duplicates) them and pruneRecords does not treat them as orphans. if (naturalKeys && Object.keys(naturalKeys).length) { @@ -1165,11 +1172,16 @@ export async function applyRecords({ client, sourceBaseId, destBaseId, planId, r // collectTruncatedTableNames — set-builder for pruneRecords truncation guard // ────────────────────────────────────────────────────────────────────────────── -/** Table names whose snapshot hit the 1000-row read cap on EITHER side (possibly truncated). */ +/** + * Table names whose snapshot hit the 1000-row read cap on EITHER side (possibly truncated). + * Uses `snapshotRowCount` (stamped by runRecords BEFORE Pass 1) when present — the live + * `.records` array is inflated by mirrorFoldedLinks' synthetic rows during Pass 1, which must + * not fake a truncation and silently disable mirror prune. + */ export function collectTruncatedTableNames(srcSnapshot, destSnapshot) { const names = new Set(); for (const t of [...(srcSnapshot?.tables || []), ...(destSnapshot?.tables || [])]) { - if ((t.records || []).length >= 1000) names.add(t.name); + if ((t.snapshotRowCount ?? (t.records || []).length) >= 1000) names.add(t.name); } return names; } @@ -1225,7 +1237,7 @@ export async function pruneRecords({ client, destSnapshot, idmap, policy, policy // Truncation safety: a table whose snapshot hit the 1000-row cap cannot be safely pruned — // unmapped rows may be legitimate records whose source/dest counterpart fell beyond the window. // Skip deletion entirely (the >1000-row read is a capture-gated follow-up). - const isTruncated = truncatedTables.has(t.name) || (t.records || []).length >= 1000; + const isTruncated = truncatedTables.has(t.name) || (t.snapshotRowCount ?? (t.records || []).length) >= 1000; if (isTruncated) { result.warnings.push({ code: 'RECORDS_TRUNCATED_PRUNE_SKIPPED', diff --git a/packages/mcp-server/test/sync/test-records-policy.test.js b/packages/mcp-server/test/sync/test-records-policy.test.js index 3d876d2..fd09ca6 100644 --- a/packages/mcp-server/test/sync/test-records-policy.test.js +++ b/packages/mcp-server/test/sync/test-records-policy.test.js @@ -7,7 +7,7 @@ */ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { applyRecordsPass1, pruneRecords, collectTruncatedTableNames } from '../../src/sync/records.js'; +import { applyRecordsPass1, pruneRecords, collectTruncatedTableNames, runRecords } from '../../src/sync/records.js'; export function fakeClient() { const calls = { create: [], update: [] }; @@ -296,4 +296,92 @@ describe('collectTruncatedTableNames', () => { const set = collectTruncatedTableNames(src, dest); assert.deepEqual([...set].sort(), ['A', 'C']); }); + + it('prefers snapshotRowCount over live records length (folded synthetic rows must not count)', () => { + // mirrorFoldedLinks pushes synthetic rows into destTable.records during Pass 1 — the + // truncation heuristic must reflect the FETCHED row count, not the inflated live array. + const inflated = { + id: 'tblI', name: 'Inflated', snapshotRowCount: 400, + records: Array.from({ length: 1100 }, (_, i) => ({ id: 'r' + i })), + }; + const genuinelyBig = { + id: 'tblG', name: 'Big', snapshotRowCount: 1000, + records: Array.from({ length: 1000 }, (_, i) => ({ id: 'g' + i })), + }; + const set = collectTruncatedTableNames({ tables: [] }, { tables: [inflated, genuinelyBig] }); + assert.equal(set.has('Inflated'), false, 'inflated-but-not-truncated table must not be flagged'); + assert.ok(set.has('Big'), 'genuinely capped table still flagged'); + }); +}); + +// ── runRecords integration: prune threading ───────────────────────────────── + +// Combined mock client for runRecords: creates succeed (rowId per sourceKey), updates succeed, +// deletions recorded. +function runClient() { + const calls = { deleted: [], created: [], updated: [] }; + return { + calls, + async createRecords(appId, tableId, rows) { + const created = rows.map((r, i) => { + const rowId = 'recNew_' + r.sourceKey; + calls.created.push({ tableId, rowId, sourceKey: r.sourceKey }); + return { rowId, sourceKey: r.sourceKey }; + }); + return { created, failed: [] }; + }, + async updateRecords(appId, tableId, rows) { calls.updated.push({ tableId, rows }); return { updated: rows, failed: [] }; }, + async deleteRecords(appId, tableId, rowIds) { calls.deleted.push({ tableId, rowIds }); return { deleted: rowIds.length }; }, + async addLinkItems() { return { ok: true, added: 1 }; }, + async updateViewFilters() { return { ok: true }; }, + }; +} + +describe('runRecords — prune integration', () => { + it('mirror prune still runs when folded-link mirroring pushes dest records past 1000', async () => { + // Dest starts with 990 FETCHED rows: recD0 (mapped) + 989 orphans. + // Src: recS0 (mapped) + 20 new records each linking to recS0 → creates fold links, + // mirrorFoldedLinks pushes 20 synthetic rows → live dest records array = 1010. + // Truncation detection must use the fetched count (990), NOT the inflated live length. + const destRows = [{ id: 'recD0', cellValuesByColumnId: {} }] + .concat(Array.from({ length: 989 }, (_, i) => ({ id: `recOrphan${i}`, cellValuesByColumnId: {} }))); + const destSnapshot = { + baseId: 'appD', + tables: [{ id: 'tD', name: 'Games', views: [{ id: 'viwD' }], fields: [], records: destRows }], + }; + const srcRecords = [{ id: 'recS0', cellValuesByColumnId: { sN: 'zero' } }] + .concat(Array.from({ length: 20 }, (_, i) => ({ + id: `recSNew${i}`, cellValuesByColumnId: { sN: `new${i}`, sL: ['recS0'] }, + }))); + const srcSnapshot = { + baseId: 'appS', + tables: [{ + id: 'tS', name: 'Games', primaryFieldId: 'sN', + fields: [ + { id: 'sN', name: 'Name', type: 'text' }, + { id: 'sL', name: 'Self', type: 'foreignKey', typeOptions: { foreignTableId: 'tS' } }, + ], + views: [], + records: srcRecords, + }], + }; + const idmap = { + tables: { tS: 'tD' }, + fields: { sN: { destFld: 'dN', choices: {} }, sL: { destFld: 'dL' } }, + records: { recS0: 'recD0' }, + }; + const client = runClient(); + const result = { created: 0, updated: 0, skipped: 0, failed: 0, deleted: 0, warnings: [] }; + await runRecords({ + client, srcSnapshot, destSnapshot, idmap, + policy: 'mirror', confirmDeletions: true, + limiter: noLimiter, journal: {}, persist: () => {}, result, + }); + assert.equal(result.created, 20, 'all 20 new records created'); + assert.ok(!result.warnings.some((w) => w.code === 'RECORDS_TRUNCATED_PRUNE_SKIPPED'), + 'no false truncation-skip warning from mirrored synthetic rows'); + assert.ok(!result.warnings.some((w) => w.code === 'RECORDS_TRUNCATED'), + 'no false RECORDS_TRUNCATED warning from mirrored synthetic rows'); + assert.equal(result.deleted, 989, 'all real orphans pruned under mirror+confirm'); + }); }); From 81009ade4b92c10644846d17e9d55b8e2df60f05 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 14:35:34 +0300 Subject: [PATCH 159/246] fix(sync): mirror prune propagates source record deletions (stale idmap entries) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pruneRecords protected every dest row present as a value in idmap.records, including mappings whose SOURCE record was deleted — so under mirror a source-side deletion never reached the dest, silently and forever. runRecords now threads the set of live source record ids (from the source snapshot) into pruneRecords; a mapping whose source id is not live no longer shields its dest row, which falls through to the ordinary orphan path (same confirmDeletions gate, same truncation guard), and the stale idmap entry is dropped only after the row is actually deleted. Keep policies (overlay/preserve) are unchanged; callers that omit liveSourceRecordIds keep the legacy protect-all behaviour. Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/records.js | 31 ++++- .../test/sync/test-records-policy.test.js | 117 ++++++++++++++++++ 2 files changed, 145 insertions(+), 3 deletions(-) diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index 639f3e2..90faf08 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -1065,7 +1065,13 @@ export async function runRecords({ client, srcSnapshot, destSnapshot, idmap, pol `and mirror deletion is skipped for this table (>1000-row pagination is a follow-up).`, }); } - await pruneRecords({ client, destSnapshot, idmap, policy, policyOverrides, confirmDeletions, limiter, result, truncatedTables }); + // Live source record ids: a persisted mapping whose source record no longer exists must not + // protect its dest row — otherwise mirror never propagates source-side record deletions. + const liveSourceRecordIds = new Set(); + for (const t of (srcSnapshot.tables || [])) { + for (const r of (t.records || [])) liveSourceRecordIds.add(r.id); + } + await pruneRecords({ client, destSnapshot, idmap, policy, policyOverrides, confirmDeletions, limiter, result, truncatedTables, liveSourceRecordIds }); persist(idmap, journal); return result; @@ -1211,14 +1217,28 @@ export function collectTruncatedTableNames(srcSnapshot, destSnapshot) { * @param {object} opts.limiter - rate limiter { run: (fn) => fn() } * @param {object} opts.result - result accumulator (mutated: result.deleted, result.warnings) * @param {Set} [opts.truncatedTables] - table names whose snapshot hit the 1000-row cap (skipped to avoid false-orphan deletion) + * @param {Set} [opts.liveSourceRecordIds] - all record ids present in the SOURCE snapshot; when + * provided, a mapping whose source id is absent is STALE (source record deleted) and does not + * protect its dest row — the dest row becomes an ordinary orphan (gated + truncation-guarded), + * and the stale idmap entry is dropped once the row is actually deleted. Omit (null) to keep + * the legacy behaviour where every mapping protects its dest row. * @returns {Promise} */ -export async function pruneRecords({ client, destSnapshot, idmap, policy, policyOverrides, confirmDeletions, limiter, result, truncatedTables = new Set() }) { +export async function pruneRecords({ client, destSnapshot, idmap, policy, policyOverrides, confirmDeletions, limiter, result, truncatedTables = new Set(), liveSourceRecordIds = null }) { if (result.deleted == null) result.deleted = 0; if (result.failed == null) result.failed = 0; if (!result.warnings) result.warnings = []; - const mappedDestIds = new Set(Object.values(idmap.records || {})); + // A mapping protects its dest row only while its SOURCE record is still alive — otherwise + // mirror could never propagate source deletions (the stale entry shields the row forever). + // Stale-mapped dest rows fall through to the ordinary orphan path (same confirm gate, same + // truncation guard); their idmap entries are dropped only after the row is really deleted. + const mappedDestIds = new Set(); + const staleDestToSrc = new Map(); // destRecId → srcRecId for mappings whose source is gone + for (const [srcRecId, destRecId] of Object.entries(idmap.records || {})) { + if (liveSourceRecordIds && !liveSourceRecordIds.has(srcRecId)) staleDestToSrc.set(destRecId, srcRecId); + else mappedDestIds.add(destRecId); + } // M3 guard: only prune tables that are the dest-side of a matched pair. // A dest table whose id is NOT a value in idmap.tables is a dest-only (schema-orphan) table — @@ -1270,6 +1290,11 @@ export async function pruneRecords({ client, destSnapshot, idmap, policy, policy try { await limiter.run(() => client.deleteRecords(destSnapshot.baseId, t.id, chunk, { viewId })); result.deleted += chunk.length; + // Drop stale idmap entries whose dest row was just deleted (source already gone). + for (const id of chunk) { + const srcRecId = staleDestToSrc.get(id); + if (srcRecId !== undefined) delete idmap.records[srcRecId]; + } } catch (e) { result.failed += chunk.length; result.warnings.push({ code: 'RECORD_DELETE_FAILED', message: `Table "${t.name}": ${e.message ?? e}` }); diff --git a/packages/mcp-server/test/sync/test-records-policy.test.js b/packages/mcp-server/test/sync/test-records-policy.test.js index fd09ca6..0f6c2c6 100644 --- a/packages/mcp-server/test/sync/test-records-policy.test.js +++ b/packages/mcp-server/test/sync/test-records-policy.test.js @@ -285,6 +285,88 @@ describe('pruneRecords (extras axis)', () => { }); }); +// ── Source-deletion propagation: stale idmap entries must not protect dest rows ── + +// Dest table with two rows: recKeep (mapped, source alive) + recStale (mapped, source DELETED). +const destSnapStale = () => ({ + baseId: 'appD', + tables: [{ + id: 'tD', + name: 'Games', + views: [{ id: 'viwD' }], + fields: [], + records: [{ id: 'recKeep', cellValuesByColumnId: {} }, { id: 'recStale', cellValuesByColumnId: {} }], + }], +}); + +describe('pruneRecords — source-deletion propagation (stale idmap entries)', () => { + it('mirror + confirmDeletions deletes a dest row whose SOURCE record was deleted and drops the stale mapping', async () => { + const client = pruneClient(); + // recSGone's source record no longer exists (not in liveSourceRecordIds) + const idmap = { tables: { tS: 'tD' }, fields: {}, records: { recS1: 'recKeep', recSGone: 'recStale' } }; + const result = { deleted: 0, failed: 0, warnings: [] }; + await pruneRecords({ + client, destSnapshot: destSnapStale(), idmap, policy: 'mirror', confirmDeletions: true, + limiter: noLimiter, result, liveSourceRecordIds: new Set(['recS1']), + }); + assert.deepEqual(client.calls.deleted, [{ tableId: 'tD', rowIds: ['recStale'] }], 'stale-mapped dest row deleted'); + assert.equal(result.deleted, 1); + assert.equal(idmap.records.recSGone, undefined, 'stale idmap entry dropped after deletion'); + assert.equal(idmap.records.recS1, 'recKeep', 'live mapping kept'); + }); + + it('does NOT delete stale-mapped rows in a truncated table (source may lie beyond the cap)', async () => { + const client = pruneClient(); + const idmap = { tables: { tS: 'tD' }, fields: {}, records: { recS1: 'recKeep', recSGone: 'recStale' } }; + const result = { deleted: 0, failed: 0, warnings: [] }; + await pruneRecords({ + client, destSnapshot: destSnapStale(), idmap, policy: 'mirror', confirmDeletions: true, + limiter: noLimiter, result, liveSourceRecordIds: new Set(['recS1']), truncatedTables: new Set(['Games']), + }); + assert.equal(client.calls.deleted.length, 0, 'no deletion on a truncated table'); + assert.equal(idmap.records.recSGone, 'recStale', 'stale mapping kept when table truncated'); + assert.ok(result.warnings.some((w) => w.code === 'RECORDS_TRUNCATED_PRUNE_SKIPPED')); + }); + + it('overlay (keep policy) never deletes stale-mapped rows nor drops the mapping', async () => { + const client = pruneClient(); + const idmap = { tables: { tS: 'tD' }, fields: {}, records: { recS1: 'recKeep', recSGone: 'recStale' } }; + const result = { deleted: 0, failed: 0, warnings: [] }; + await pruneRecords({ + client, destSnapshot: destSnapStale(), idmap, policy: 'overlay', confirmDeletions: true, + limiter: noLimiter, result, liveSourceRecordIds: new Set(['recS1']), + }); + assert.equal(client.calls.deleted.length, 0); + assert.equal(idmap.records.recSGone, 'recStale', 'mapping untouched under keep policy'); + }); + + it('mirror WITHOUT confirmDeletions gates the stale-mapped row (counted, not deleted, mapping kept)', async () => { + const client = pruneClient(); + const idmap = { tables: { tS: 'tD' }, fields: {}, records: { recS1: 'recKeep', recSGone: 'recStale' } }; + const result = { deleted: 0, failed: 0, warnings: [] }; + await pruneRecords({ + client, destSnapshot: destSnapStale(), idmap, policy: 'mirror', confirmDeletions: false, + limiter: noLimiter, result, liveSourceRecordIds: new Set(['recS1']), + }); + assert.equal(client.calls.deleted.length, 0); + const w = result.warnings.find((x) => x.code === 'DELETION_GATED'); + assert.ok(w && /1/.test(w.message), 'gated warning counts the stale-mapped row'); + assert.equal(idmap.records.recSGone, 'recStale', 'mapping kept while gated'); + }); + + it('back-compat: without liveSourceRecordIds every mapping still protects its dest row', async () => { + const client = pruneClient(); + const idmap = { tables: { tS: 'tD' }, fields: {}, records: { recS1: 'recKeep', recSGone: 'recStale' } }; + const result = { deleted: 0, failed: 0, warnings: [] }; + await pruneRecords({ + client, destSnapshot: destSnapStale(), idmap, policy: 'mirror', confirmDeletions: true, + limiter: noLimiter, result, + }); + assert.equal(client.calls.deleted.length, 0, 'no liveSourceRecordIds → all mapped rows protected'); + assert.equal(idmap.records.recSGone, 'recStale'); + }); +}); + // ── Task 2: collectTruncatedTableNames helper ──────────────────────────────── describe('collectTruncatedTableNames', () => { @@ -384,4 +466,39 @@ describe('runRecords — prune integration', () => { 'no false RECORDS_TRUNCATED warning from mirrored synthetic rows'); assert.equal(result.deleted, 989, 'all real orphans pruned under mirror+confirm'); }); + + it('threads live source ids: a source-deleted record is pruned under mirror+confirm and its mapping dropped', async () => { + // Source only has recS1; idmap still maps recSGone → recStale (source record deleted). + const srcSnapshot = { + baseId: 'appS', + tables: [{ + id: 'tS', name: 'Games', primaryFieldId: 'sN', + fields: [{ id: 'sN', name: 'Name', type: 'text' }], + views: [], + records: [{ id: 'recS1', cellValuesByColumnId: { sN: 'Zelda' } }], + }], + }; + const destSnapshot = { + baseId: 'appD', + tables: [{ + id: 'tD', name: 'Games', views: [{ id: 'viwD' }], fields: [], + records: [{ id: 'recKeep', cellValuesByColumnId: {} }, { id: 'recStale', cellValuesByColumnId: {} }], + }], + }; + const idmap = { + tables: { tS: 'tD' }, + fields: { sN: { destFld: 'dN', choices: {} } }, + records: { recS1: 'recKeep', recSGone: 'recStale' }, + }; + const client = runClient(); + const result = { created: 0, updated: 0, skipped: 0, failed: 0, deleted: 0, warnings: [] }; + await runRecords({ + client, srcSnapshot, destSnapshot, idmap, + policy: 'mirror', confirmDeletions: true, + limiter: noLimiter, journal: {}, persist: () => {}, result, + }); + assert.deepEqual(client.calls.deleted, [{ tableId: 'tD', rowIds: ['recStale'] }], 'source-deleted counterpart pruned'); + assert.equal(idmap.records.recSGone, undefined, 'stale idmap entry dropped'); + assert.equal(idmap.records.recS1, 'recKeep', 'live mapping kept'); + }); }); From f98710c8258e5d2a1fd598159587cb42aecaae4b Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 14:48:10 +0300 Subject: [PATCH 160/246] fix(sync): Pass 2 links + Pass 3 attachments honor conflicts=dest-wins Under preserve (dest-wins), applyRecordsPass2 no longer re-adds source links the dest user removed, and applyAttachments no longer paste-replaces or fallback-uploads into pre-existing mapped dest rows. Rows CREATED this run (tracked via a createdDestIds set filled by Pass 1) always receive their links and attachments. Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/records.js | 50 +++- .../test/sync/test-records-policy.test.js | 223 +++++++++++++++++- 2 files changed, 264 insertions(+), 9 deletions(-) diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index 90faf08..fe4e696 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -313,7 +313,7 @@ function mirrorFoldedLinks(destTableById, destTableId, destRowId, linkCells) { * a folded link array to ~one chunk of retries (the create-with-links capability is high-confidence * but not live-verified — this is the calibration knob). */ -async function createChunkWithFallback({ client, destAppId, destTableId, chunk, idmap, result, limiter, runState, destTableById }) { +async function createChunkWithFallback({ client, destAppId, destTableId, chunk, idmap, result, limiter, runState, destTableById, createdDestIds }) { const gate = makeGate(limiter); // paces each inner per-row create POST under the phase limiter let res; try { @@ -326,6 +326,7 @@ async function createChunkWithFallback({ client, destAppId, destTableId, chunk, for (const created of (res.created || [])) { idmap.records[created.sourceKey] = created.rowId; + createdDestIds?.add(created.rowId); result.created++; const row = chunk.find((r) => r.sourceKey === created.sourceKey); mirrorFoldedLinks(destTableById, destTableId, created.rowId, row?.linkCells); @@ -365,6 +366,7 @@ async function createChunkWithFallback({ client, destAppId, destTableId, chunk, let rescued = 0; for (const created of (retryRes.created || [])) { idmap.records[created.sourceKey] = created.rowId; // links NOT folded → Pass 2 adds them (no mirror) + createdDestIds?.add(created.rowId); result.created++; rescued++; } @@ -381,7 +383,7 @@ async function createChunkWithFallback({ client, destAppId, destTableId, chunk, } } -export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result, policy, policyOverrides, fieldMappings = [] }) { +export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result, policy, policyOverrides, fieldMappings = [], createdDestIds }) { if (!idmap.records) idmap.records = {}; // destAppId comes from the snapshot (set by snapshotBase); fall back to '' for tests that omit it. @@ -447,7 +449,7 @@ export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idm const CREATE_CHUNK = 50; for (let i = 0; i < createRows.length; i += CREATE_CHUNK) { const chunk = createRows.slice(i, i + CREATE_CHUNK); - await createChunkWithFallback({ client, destAppId, destTableId, chunk, idmap, result, limiter, runState, destTableById }); + await createChunkWithFallback({ client, destAppId, destTableId, chunk, idmap, result, limiter, runState, destTableById, createdDestIds }); persist(idmap, journal); } @@ -499,9 +501,14 @@ export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idm * @param {object} opts.journal - journal from newJournal() * @param {Function} opts.persist - persist(idmap, journal) called after each table * @param {object} opts.result - result accumulator { created, updated, skipped, failed, warnings } + * @param {string} [opts.policy] - global preset ('mirror'|'overlay'|'preserve') + * @param {object} [opts.policyOverrides]- per-table preset overrides + * @param {Set} [opts.createdDestIds] - dest row ids CREATED this run (Pass 1); under + * conflicts=dest-wins only these rows receive link writes — a PRE-EXISTING mapped dest row is + * never touched (the dest user may have removed the link on purpose). * @returns {Promise} */ -export async function applyRecordsPass2({ client, srcSnapshot, destSnapshot, idmap, destDisplayNames, limiter, journal, persist, result }) { +export async function applyRecordsPass2({ client, srcSnapshot, destSnapshot, idmap, destDisplayNames, limiter, journal, persist, result, policy, policyOverrides, createdDestIds = new Set() }) { const destAppId = destSnapshot.baseId || ''; // Build a fast lookup: destTableId → Map> @@ -519,6 +526,9 @@ export async function applyRecordsPass2({ client, srcSnapshot, destSnapshot, idm const destTableId = idmap.tables[srcTable.id]; if (!destTableId) continue; + // Conflicts axis: under dest-wins, only rows created THIS run get link writes. + const { conflicts } = resolvePolicy(policy, policyOverrides, srcTable.name); + // Find all link fields in this source table that have a dest mapping const linkFields = (srcTable.fields || []).filter( (f) => (f.type === 'multipleRecordLinks' || f.type === 'foreignKey') && idmap.fields[f.id], @@ -530,6 +540,8 @@ export async function applyRecordsPass2({ client, srcSnapshot, destSnapshot, idm for (const rec of (srcTable.records || [])) { const destRowId = idmap.records[rec.id]; if (!destRowId) continue; // src row not yet mapped → skip + // Pre-existing mapped dest row under dest-wins: never re-add links the dest user removed. + if (conflicts === 'dest-wins' && !createdDestIds.has(destRowId)) continue; const srcCells = rec.cellValuesByColumnId || {}; const destCells = destRecMap.get(destRowId) || {}; @@ -691,6 +703,12 @@ async function uploadAttachmentFallback({ client, destAppId, destRowId, destFldI * @param {Function} opts.persist - persist(idmap, journal) called after each table * @param {object} opts.result - result accumulator (must include .warnings array and .attachmentsUploaded counter) * @param {Function} [opts.fetchBytes] - async (url) => { bytes, contentType? } (default: real fetch) + * @param {string} [opts.policy] - global preset ('mirror'|'overlay'|'preserve') + * @param {object} [opts.policyOverrides]- per-table preset overrides + * @param {Set} [opts.createdDestIds] - dest row ids CREATED this run (Pass 1); under + * conflicts=dest-wins only these rows receive attachment writes — pasteAttachmentsCrossBase + * SETS (replaces) the target cell, so a pre-existing mapped dest row must never be included + * (dest-added attachments would be destroyed). * @returns {Promise} */ export async function applyAttachments({ @@ -703,6 +721,9 @@ export async function applyAttachments({ persist, result, fetchBytes = defaultFetchBytes, + policy, + policyOverrides, + createdDestIds = new Set(), }) { // Initialize dedupe map if not already present (survives across calls for idempotency) idmap.attachments ??= {}; @@ -714,6 +735,11 @@ export async function applyAttachments({ const destTableId = idmap.tables[srcTable.id]; if (!destTableId) continue; // table not matched → skip + // Conflicts axis: under dest-wins, only rows created THIS run get attachment writes — + // both the fast-path paste (which wholesale-replaces the target cell) and the fallback. + const { conflicts } = resolvePolicy(policy, policyOverrides, srcTable.name); + const destWinsSkip = (destRowId) => conflicts === 'dest-wins' && !createdDestIds.has(destRowId); + try { // Find attachment fields that have a dest mapping const attFields = (srcTable.fields || []).filter( @@ -735,6 +761,7 @@ export async function applyAttachments({ for (const rec of (srcTable.records || [])) { const destRowId = idmap.records[rec.id]; if (!destRowId) continue; + if (destWinsSkip(destRowId)) continue; // preserve dest edits on pre-existing rows const srcCells = rec.cellValuesByColumnId || {}; for (const field of attFields) { const cellValue = srcCells[field.id]; @@ -749,11 +776,14 @@ export async function applyAttachments({ continue; } - // Collect included records: must be mapped AND have ≥1 attachment cell + // Collect included records: must be mapped AND have ≥1 attachment cell. + // Under dest-wins, pre-existing mapped rows are excluded from BOTH the paste payload + // and every fallback derived from includedRecs (pasteCells replaces the target cell). const includedRecs = []; for (const rec of (srcTable.records || [])) { const destRowId = idmap.records[rec.id]; if (!destRowId) continue; + if (destWinsSkip(destRowId)) continue; // preserve dest edits on pre-existing rows const srcCells = rec.cellValuesByColumnId || {}; const hasAtt = attFields.some((f) => { const cv = srcCells[f.id]; @@ -1027,8 +1057,12 @@ export async function runRecords({ client, srcSnapshot, destSnapshot, idmap, pol persist(idmap, journal); } + // Dest row ids created THIS run — Pass 2/3 use it so conflicts=dest-wins only blocks + // writes to PRE-EXISTING mapped rows (created rows always get their links/attachments). + const createdDestIds = new Set(); + // Pass 1: scalar / select upsert — fills idmap.records - await applyRecordsPass1({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result, policy, policyOverrides, fieldMappings: resolved }); + await applyRecordsPass1({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result, policy, policyOverrides, fieldMappings: resolved, createdDestIds }); persist(idmap, journal); // Rebuild destDisplayNames from idmap.records now populated by Pass 1. @@ -1045,11 +1079,11 @@ export async function runRecords({ client, srcSnapshot, destSnapshot, idmap, pol } // Pass 2: link cells - await applyRecordsPass2({ client, srcSnapshot, destSnapshot, idmap, destDisplayNames, limiter, journal, persist, result }); + await applyRecordsPass2({ client, srcSnapshot, destSnapshot, idmap, destDisplayNames, limiter, journal, persist, result, policy, policyOverrides, createdDestIds }); persist(idmap, journal); // Pass 3: attachments - await applyAttachments({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result }); + await applyAttachments({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result, policy, policyOverrides, createdDestIds }); persist(idmap, journal); // Reapply view filters whose record refs now resolve diff --git a/packages/mcp-server/test/sync/test-records-policy.test.js b/packages/mcp-server/test/sync/test-records-policy.test.js index 0f6c2c6..a15b7e7 100644 --- a/packages/mcp-server/test/sync/test-records-policy.test.js +++ b/packages/mcp-server/test/sync/test-records-policy.test.js @@ -7,7 +7,7 @@ */ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { applyRecordsPass1, pruneRecords, collectTruncatedTableNames, runRecords } from '../../src/sync/records.js'; +import { applyRecordsPass1, applyRecordsPass2, applyAttachments, pruneRecords, collectTruncatedTableNames, runRecords } from '../../src/sync/records.js'; export function fakeClient() { const calls = { create: [], update: [] }; @@ -502,3 +502,224 @@ describe('runRecords — prune integration', () => { assert.equal(idmap.records.recS1, 'recKeep', 'live mapping kept'); }); }); + +// ── Conflicts axis in Pass 2 (links) ───────────────────────────────────────── +// Under conflicts=dest-wins (preserve), a PRE-EXISTING mapped dest row must not have +// source links re-added (the dest user may have removed them on purpose). Rows CREATED +// this run always get their links. + +function pass2Fixtures() { + const calls = []; + const client = { + async addLinkItems(appId, rowId, fldId, items) { calls.push({ rowId, fldId, items }); return { ok: true, added: items.length }; }, + }; + const srcSnapshot = { + baseId: 'appS', + tables: [{ + id: 'tS', name: 'Games', + fields: [{ id: 'sL', name: 'Link', type: 'foreignKey' }], + records: [{ id: 'recS1', cellValuesByColumnId: { sL: ['recSTarget'] } }], + }], + }; + const destSnapshot = { + baseId: 'appD', + tables: [{ + id: 'tD', name: 'Games', fields: [], + // dest link cell is EMPTY — the dest user removed the link + records: [{ id: 'recD1', cellValuesByColumnId: {} }], + }], + }; + const idmap = { + tables: { tS: 'tD' }, + fields: { sL: { destFld: 'dL' } }, + records: { recS1: 'recD1', recSTarget: 'recDTarget' }, + }; + return { calls, client, srcSnapshot, destSnapshot, idmap }; +} + +describe('Pass2 conflict policy (links)', () => { + it('preserve (dest-wins) does NOT re-add links to a pre-existing mapped dest row', async () => { + const { calls, client, srcSnapshot, destSnapshot, idmap } = pass2Fixtures(); + const result = { created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + await applyRecordsPass2({ + client, srcSnapshot, destSnapshot, idmap, destDisplayNames: new Map(), + limiter: noLimiter, journal: {}, persist: () => {}, result, + policy: 'preserve', + }); + assert.equal(calls.length, 0, 'no addLinkItems under preserve for a pre-existing row'); + }); + + it('preserve still writes links to a dest row CREATED this run', async () => { + const { calls, client, srcSnapshot, destSnapshot, idmap } = pass2Fixtures(); + const result = { created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + await applyRecordsPass2({ + client, srcSnapshot, destSnapshot, idmap, destDisplayNames: new Map(), + limiter: noLimiter, journal: {}, persist: () => {}, result, + policy: 'preserve', createdDestIds: new Set(['recD1']), + }); + assert.equal(calls.length, 1, 'created-this-run row still gets its links'); + assert.equal(calls[0].rowId, 'recD1'); + }); + + it('overlay (source-wins) still adds the missing link (regression)', async () => { + const { calls, client, srcSnapshot, destSnapshot, idmap } = pass2Fixtures(); + const result = { created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + await applyRecordsPass2({ + client, srcSnapshot, destSnapshot, idmap, destDisplayNames: new Map(), + limiter: noLimiter, journal: {}, persist: () => {}, result, + policy: 'overlay', + }); + assert.equal(calls.length, 1, 'link re-added under source-wins'); + }); + + it('policyOverrides: per-table preserve blocks the link add under a global overlay', async () => { + const { calls, client, srcSnapshot, destSnapshot, idmap } = pass2Fixtures(); + const result = { created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + await applyRecordsPass2({ + client, srcSnapshot, destSnapshot, idmap, destDisplayNames: new Map(), + limiter: noLimiter, journal: {}, persist: () => {}, result, + policy: 'overlay', policyOverrides: { Games: 'preserve' }, + }); + assert.equal(calls.length, 0, 'table-level preserve override respected'); + }); +}); + +// ── Conflicts axis in Pass 3 (attachments) ─────────────────────────────────── +// Under conflicts=dest-wins (preserve), a PRE-EXISTING mapped dest row must not have its +// attachment cells wholesale-replaced (pasteAttachmentsCrossBase SETS the target cells) nor +// receive fallback uploads. Rows CREATED this run always get their attachments. + +function pass3Fixtures({ withDestView = false } = {}) { + const uploads = []; + const pastes = []; + const client = { + async uploadAttachment(appId, rowId, columnId, payload) { uploads.push({ rowId, columnId, filename: payload.filename }); return { ok: true }; }, + async createDataTransferPolicy() { return { policy: 'signed' }; }, + async pasteAttachmentsCrossBase(appId, tableId, payload) { pastes.push(payload); return { pastedRowIds: payload.targetRowIds }; }, + }; + const fetchBytes = async () => ({ bytes: Buffer.from('x'), contentType: 'image/png' }); + const srcSnapshot = { + baseId: 'appS', + tables: [{ + id: 'tS', name: 'Games', + fields: [{ id: 'sA', name: 'Att', type: 'multipleAttachments' }], + records: [{ + id: 'recS1', + cellValuesByColumnId: { sA: [{ id: 'att1', url: 'https://src/a.png', filename: 'a.png', size: 10, type: 'image/png' }] }, + }], + }], + }; + const destSnapshot = { + baseId: 'appD', + tables: [{ + id: 'tD', name: 'Games', fields: [], + ...(withDestView ? { views: [{ id: 'viwD' }] } : {}), + records: [{ id: 'recD1', cellValuesByColumnId: {} }], + }], + }; + const idmap = { tables: { tS: 'tD' }, fields: { sA: { destFld: 'dA' } }, records: { recS1: 'recD1' } }; + return { uploads, pastes, client, fetchBytes, srcSnapshot, destSnapshot, idmap }; +} + +describe('Pass3 conflict policy (attachments)', () => { + it('preserve (dest-wins): fast path does not paste over a pre-existing mapped dest row', async () => { + const { uploads, pastes, client, fetchBytes, srcSnapshot, destSnapshot, idmap } = pass3Fixtures({ withDestView: true }); + const result = { created: 0, updated: 0, skipped: 0, failed: 0, attachmentsUploaded: 0, warnings: [] }; + await applyAttachments({ + client, srcSnapshot, destSnapshot, idmap, + limiter: noLimiter, journal: {}, persist: () => {}, result, fetchBytes, + policy: 'preserve', + }); + assert.equal(pastes.length, 0, 'pasteAttachmentsCrossBase not called under preserve'); + assert.equal(uploads.length, 0, 'no fallback upload under preserve'); + }); + + it('preserve (dest-wins): fallback path does not upload to a pre-existing mapped dest row', async () => { + const { uploads, client, fetchBytes, srcSnapshot, destSnapshot, idmap } = pass3Fixtures({ withDestView: false }); + const result = { created: 0, updated: 0, skipped: 0, failed: 0, attachmentsUploaded: 0, warnings: [] }; + await applyAttachments({ + client, srcSnapshot, destSnapshot, idmap, + limiter: noLimiter, journal: {}, persist: () => {}, result, fetchBytes, + policy: 'preserve', + }); + assert.equal(uploads.length, 0, 'no fallback upload under preserve for a pre-existing row'); + }); + + it('preserve: a dest row CREATED this run still receives attachments (fast path)', async () => { + const { pastes, client, fetchBytes, srcSnapshot, destSnapshot, idmap } = pass3Fixtures({ withDestView: true }); + const result = { created: 0, updated: 0, skipped: 0, failed: 0, attachmentsUploaded: 0, warnings: [] }; + await applyAttachments({ + client, srcSnapshot, destSnapshot, idmap, + limiter: noLimiter, journal: {}, persist: () => {}, result, fetchBytes, + policy: 'preserve', createdDestIds: new Set(['recD1']), + }); + assert.equal(pastes.length, 1, 'created-this-run row still gets its attachments'); + assert.deepEqual(pastes[0].targetRowIds, ['recD1']); + }); + + it('overlay (source-wins) still pastes attachments (regression)', async () => { + const { pastes, client, fetchBytes, srcSnapshot, destSnapshot, idmap } = pass3Fixtures({ withDestView: true }); + const result = { created: 0, updated: 0, skipped: 0, failed: 0, attachmentsUploaded: 0, warnings: [] }; + await applyAttachments({ + client, srcSnapshot, destSnapshot, idmap, + limiter: noLimiter, journal: {}, persist: () => {}, result, fetchBytes, + policy: 'overlay', + }); + assert.equal(pastes.length, 1); + }); +}); + +// ── runRecords threads policy + created-set into Pass 2 ────────────────────── + +describe('runRecords — preserve policy threads into Pass 2', () => { + it('under preserve, links are written only to rows created this run — never re-added to pre-existing rows', async () => { + const linkCalls = []; + const client = { + async createRecords(appId, tableId, rows) { + return { created: rows.map((r) => ({ rowId: 'recNew_' + r.sourceKey, sourceKey: r.sourceKey })), failed: [] }; + }, + async updateRecords(appId, tableId, rows) { return { updated: rows, failed: [] }; }, + async addLinkItems(appId, rowId, fldId, items) { linkCalls.push({ rowId, fldId }); return { ok: true, added: items.length }; }, + }; + // recS1 is PRE-EXISTING (mapped → recD1) and links to recS2; its dest link cell is EMPTY + // (dest user removed the link). recS2 + recS3 are NEW; recS2 links to recS3 (forward ref → + // unresolvable at create time → Pass 2 must write it). + const srcSnapshot = { + baseId: 'appS', + tables: [{ + id: 'tS', name: 'Games', primaryFieldId: 'sN', + fields: [ + { id: 'sN', name: 'Name', type: 'text' }, + { id: 'sL', name: 'Link', type: 'foreignKey', typeOptions: { foreignTableId: 'tS' } }, + ], + views: [], + records: [ + { id: 'recS1', cellValuesByColumnId: { sN: 'one', sL: ['recS2'] } }, + { id: 'recS2', cellValuesByColumnId: { sN: 'two', sL: ['recS3'] } }, + { id: 'recS3', cellValuesByColumnId: { sN: 'three' } }, + ], + }], + }; + const destSnapshot = { + baseId: 'appD', + tables: [{ + id: 'tD', name: 'Games', views: [{ id: 'viwD' }], fields: [], + records: [{ id: 'recD1', cellValuesByColumnId: {} }], + }], + }; + const idmap = { + tables: { tS: 'tD' }, + fields: { sN: { destFld: 'dN', choices: {} }, sL: { destFld: 'dL' } }, + records: { recS1: 'recD1' }, + }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, deleted: 0, warnings: [] }; + await runRecords({ + client, srcSnapshot, destSnapshot, idmap, + policy: 'preserve', + limiter: noLimiter, journal: {}, persist: () => {}, result, + }); + assert.equal(result.created, 2, 'both new records created'); + assert.ok(linkCalls.every((c) => c.rowId !== 'recD1'), 'pre-existing recD1 never receives addLinkItems under preserve'); + assert.ok(linkCalls.some((c) => c.rowId === 'recNew_recS2'), 'created-this-run row still gets its forward link'); + }); +}); From 3004f7d9111d2ee451ddce34fb4571f803a382b3 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 14:48:32 +0300 Subject: [PATCH 161/246] fix(sync): scope attachment-fallback dedupe key to the destination cell MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The global filename|size key silently left the 2nd..Nth record carrying a same-named same-size file with an empty attachment cell, persisted forever via idmap.attachments. The key is now destRowId|destFldId|filename|size — cross-run idempotency for the same cell is preserved, and legacy unscoped keys are kept in the map but never matched, so they cannot suppress new uploads. Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/records.js | 9 ++- .../mcp-server/test/sync/test-records.test.js | 75 +++++++++++++++++-- 2 files changed, 77 insertions(+), 7 deletions(-) diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index fe4e696..95d9e69 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -635,8 +635,13 @@ async function uploadAttachmentFallback({ client, destAppId, destRowId, destFldI const { url, filename, size, type } = attachment; if (!url || !filename) return; - const dedupeKey = `${filename}|${size}`; - if (idmap.attachments[dedupeKey]) return; // already uploaded + // Dedupe key is scoped to the destination CELL (row|field|filename|size): a same-named + // same-size file on ANOTHER record/field is a different upload and must never be suppressed, + // while a re-run of the SAME cell stays idempotent. Old-format `filename|size` keys persisted + // by earlier runs are legacy — kept in the map but never matched (a one-time re-upload of an + // already-filled cell beats permanently-empty cells on every other record). + const dedupeKey = `${destRowId}|${destFldId}|${filename}|${size}`; + if (idmap.attachments[dedupeKey]) return; // already uploaded to this cell let bytes; let contentType = type; diff --git a/packages/mcp-server/test/sync/test-records.test.js b/packages/mcp-server/test/sync/test-records.test.js index ff1b6a6..27253e3 100644 --- a/packages/mcp-server/test/sync/test-records.test.js +++ b/packages/mcp-server/test/sync/test-records.test.js @@ -869,11 +869,12 @@ describe('records.applyAttachments', () => { assert.equal(uploadCalls[0].payload.filename, 'image.png'); assert.ok(uploadCalls[0].payload.bytes, 'bytes must be provided'); - // dedupe key should be set after success - assert.equal(idmap.attachments['image.png|1024'], true); + // dedupe key should be set after success — scoped to the destination CELL + // (row|field|filename|size), so a same-named file on ANOTHER record is never suppressed + assert.equal(idmap.attachments['recD1|fldAttD|image.png|1024'], true); }); - it('deduplicates: same filename|size not re-uploaded on second applyAttachments call', async () => { + it('deduplicates: same cell (row|field|filename|size) not re-uploaded on second applyAttachments call', async () => { const { client, fetchBytes, uploadCalls, idmap, srcSnapshot, destSnapshot, result } = makeFixtures(); // First call @@ -899,7 +900,71 @@ describe('records.applyAttachments', () => { fetchBytes, }); - assert.equal(uploadCalls.length, 1, 'second call must NOT re-upload (dedupe by filename|size)'); + assert.equal(uploadCalls.length, 1, 'second call must NOT re-upload (dedupe by cell-scoped key)'); + }); + + it('same filename|size on DIFFERENT records both get uploaded (dedupe is cell-scoped, not global)', async () => { + const uploadCalls = []; + const client = { + uploadAttachment: async (appId, rowId, columnId, payload) => { + uploadCalls.push({ rowId, filename: payload.filename }); + return { ok: true, attachmentId: 'att' + uploadCalls.length }; + }, + }; + const fetchBytes = async () => ({ bytes: Buffer.from('pdf'), contentType: 'application/pdf' }); + + const idmap = { + tables: { tblS: 'tblD' }, + fields: { fldAtt: { destFld: 'fldAttD' } }, + records: { recS1: 'recD1', recS2: 'recD2' }, + }; + // Two records each carry the SAME template file (same filename + size). + const att = (id) => [{ id, url: 'https://src/contract.pdf', filename: 'contract.pdf', size: 12345, type: 'application/pdf' }]; + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', + fields: [{ id: 'fldAtt', type: 'multipleAttachments' }], + records: [ + { id: 'recS1', cellValuesByColumnId: { fldAtt: att('attA') } }, + { id: 'recS2', cellValuesByColumnId: { fldAtt: att('attB') } }, + ], + }], + }; + const destSnapshot = { baseId: 'appDest' }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, attachmentsUploaded: 0, warnings: [] }; + + await applyAttachments({ + client, srcSnapshot, destSnapshot, idmap, + limiter: makeLimiter(), + journal: newJournal('att-cellscope', 't'), + persist: () => {}, + result, + fetchBytes, + }); + + assert.equal(uploadCalls.length, 2, 'both records get the same-named file (no global suppression)'); + assert.deepEqual(uploadCalls.map((c) => c.rowId).sort(), ['recD1', 'recD2']); + assert.equal(idmap.attachments['recD1|fldAttD|contract.pdf|12345'], true); + assert.equal(idmap.attachments['recD2|fldAttD|contract.pdf|12345'], true); + }); + + it('legacy unscoped dedupe keys (filename|size) do not suppress new uploads and are kept in the map', async () => { + const { client, fetchBytes, uploadCalls, idmap, srcSnapshot, destSnapshot, result } = makeFixtures(); + // Persisted idmap from an older run holds the OLD global key format. + idmap.attachments = { 'image.png|1024': true }; + + await applyAttachments({ + client, srcSnapshot, destSnapshot, idmap, + limiter: makeLimiter(), + journal: newJournal('att-legacy', 't'), + persist: () => {}, + result, + fetchBytes, + }); + + assert.equal(uploadCalls.length, 1, 'legacy unscoped key must not suppress the cell-scoped upload'); + assert.equal(idmap.attachments['image.png|1024'], true, 'legacy key kept (not migrated/deleted)'); + assert.equal(idmap.attachments['recD1|fldAttD|image.png|1024'], true, 'new cell-scoped key recorded'); }); it('ok:false upload → ATTACHMENT_FAILED warning, no throw, record continues', async () => { @@ -924,7 +989,7 @@ describe('records.applyAttachments', () => { assert.ok(failWarns[0].message.includes('image.png'), 'warning should mention the filename'); // dedupe key must NOT be set on failure - assert.equal(idmap.attachments['image.png|1024'], undefined); + assert.equal(idmap.attachments['recD1|fldAttD|image.png|1024'], undefined); }); it('fetchBytes error → ATTACHMENT_FAILED warning, no throw, continues to next attachment', async () => { From bcf2c72c333b500df7ea3a2d5f0c6af867a890e6 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 14:49:00 +0300 Subject: [PATCH 162/246] fix(sync): filter restore survives per-view getView failures; never blocks prune reapplyViewFilters now fetches each missing source view config itself with a per-view limiter+retry+catch guard (replacing the unguarded snapshotViews call): one failed view read (deleted view, 404, session hiccup at the tail of a long job) warns VIEW_FILTER_REAPPLY_FAILED and continues instead of aborting the whole restore. runRecords additionally wraps the restore step so a filter-restore failure can never skip pruneRecords or fail a job whose record passes completed. Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/records.js | 39 +++++++++-- .../test/sync/test-reapply-filters.test.js | 69 +++++++++++++++++++ .../sync/test-records-orchestration.test.js | 58 ++++++++++++++++ 3 files changed, 160 insertions(+), 6 deletions(-) diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index 95d9e69..353cb1a 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -18,7 +18,7 @@ import { coercePass1Cell, partitionLinkValue, linkRecId, coerceMappedValue, isWr import { resolvePolicy, validateFieldMappings } from './policy.js'; import { withRetry, createLimiter } from './ratelimit.js'; import { remapViewConfig, collectFilterRecordRefs } from './remap.js'; -import { snapshotSchemaOnly, snapshotViews, snapshotTableRecords, isComputedType } from './snapshot.js'; +import { snapshotSchemaOnly, snapshotTableRecords, normalizeViewConfig, isComputedType } from './snapshot.js'; import { loadIdmap, saveIdmap, syncDir } from './idmap.js'; import { newJournal, loadRecordsJournal, saveRecordsJournal } from './journal.js'; import { safeAtomicWriteFileSync } from '../safe-write.js'; @@ -968,10 +968,28 @@ export async function reapplyViewFilters({ client, srcSnapshot, destSnapshot, id // applyRecords takes a SCHEMA-ONLY snapshot (no view configs) to avoid a getView storm. // Populate SOURCE view live-configs now (source-only, once, at the end) so we can find and - // remap the record-ref filters. If configs are already present this is a no-op-ish refresh. + // remap the record-ref filters. Views that already carry a config are skipped. + // + // Each getView is guarded PER VIEW (limiter + retry + catch): this runs at the tail of a + // multi-minute request-heavy job, and one failed view read (deleted view, 404, session + // hiccup) must not abort the whole filter restore — the other views still get their + // record-ref filters back, and the failed one is reported as a warning. if (typeof client.getView === 'function') { - const needsConfig = (srcSnapshot.tables || []).some((t) => (t.views || []).some((v) => !v.personalForUserId && !v.config)); - if (needsConfig) await snapshotViews(client, srcSnapshot.baseId || '', srcSnapshot); + const srcAppId = srcSnapshot.baseId || ''; + for (const t of (srcSnapshot.tables || [])) { + for (const v of (t.views || [])) { + if (v.personalForUserId || v.config) continue; + try { + const live = await limiter.run(() => withRetry(() => client.getView(srcAppId, v.id))); + v.config = normalizeViewConfig(live); + } catch (err) { + result.warnings.push({ + code: 'VIEW_FILTER_REAPPLY_FAILED', + message: `View ${v.id}: getView failed during filter restore (${err.message}) — view skipped`, + }); + } + } + } } for (const srcTable of (srcSnapshot.tables || [])) { @@ -1091,8 +1109,17 @@ export async function runRecords({ client, srcSnapshot, destSnapshot, idmap, pol await applyAttachments({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result, policy, policyOverrides, createdDestIds }); persist(idmap, journal); - // Reapply view filters whose record refs now resolve - await reapplyViewFilters({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result }); + // Reapply view filters whose record refs now resolve. + // Defensive guard: a filter-restore failure must NEVER prevent the prune step below — + // otherwise one tail-step error discards completed passes and skips the extras axis. + try { + await reapplyViewFilters({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result }); + } catch (err) { + result.warnings.push({ + code: 'VIEW_FILTER_REAPPLY_FAILED', + message: `reapplyViewFilters failed (${err.message}) — continuing to prune`, + }); + } persist(idmap, journal); // Extras axis: prune dest-only orphan records under mirror policy diff --git a/packages/mcp-server/test/sync/test-reapply-filters.test.js b/packages/mcp-server/test/sync/test-reapply-filters.test.js index 9f39305..e8f40d5 100644 --- a/packages/mcp-server/test/sync/test-reapply-filters.test.js +++ b/packages/mcp-server/test/sync/test-reapply-filters.test.js @@ -379,4 +379,73 @@ describe('reapplyViewFilters — mock client', () => { assert.ok(result.warnings[0].message.includes('viwSRCSRCSRCSRCS')); assert.ok(result.warnings[0].message.includes('viwDSTDSTDSTDSTO')); }); + + it('one failed getView during config fetch warns and CONTINUES — the other views are still restored', async () => { + // applyRecords uses schema-only snapshots, so source views arrive WITHOUT .config and + // reapplyViewFilters must fetch them via getView. A single failing view (deleted mid-job, + // 404, session hiccup) must not abort the whole restore. + const brokenView = 'viwSRCBROKENXXXXX'; + const goodView = 'viwSRCSRCSRCSRCS'; + const calls = []; + const client = { + getView: async (appId, viewId) => { + if (viewId === brokenView) throw new Error('view read failed (404)'); + return { + filters: { + conjunction: 'and', + filterSet: [{ columnId: 'fldG', operator: '=', value: [recSrc] }], + }, + }; + }, + updateViewFilters: async (appId, viewId, filters) => { + calls.push({ viewId, filters }); + return { ok: true }; + }, + }; + + const srcSnapshot = { + baseId: 'appSRC', + tables: [{ + id: 'tblSRCSRCSRCSRCS', + fields: [], + records: [], + views: [ + // No .config on either view (schema-only snapshot) — forces the getView fetch. + { id: brokenView, name: 'Broken', type: 'grid', personalForUserId: null }, + { id: goodView, name: 'By Game', type: 'grid', personalForUserId: null }, + ], + }], + }; + + const destSnapshot = { baseId: 'appDST', tables: [] }; + + const idmap = { + tables: { tblSRCSRCSRCSRCS: 'tblDSTDSTDSTDSTO' }, + views: { [brokenView]: 'viwDSTBROKENXXXXX', [goodView]: 'viwDSTDSTDSTDSTO' }, + fields: { fldG: { destFld: 'fldGD', choices: {} } }, + records: { [recSrc]: recDst }, + }; + + const result = { warnings: [], viewFiltersReapplied: 0 }; + + // Must not throw + await reapplyViewFilters({ + client, + srcSnapshot, + destSnapshot, + idmap, + limiter: makeLimiter(), + journal: makeJournal(), + persist: makeNoop(), + result, + }); + + assert.equal(calls.length, 1, 'good view still restored despite the broken one'); + assert.equal(calls[0].viewId, 'viwDSTDSTDSTDSTO'); + assert.deepEqual(calls[0].filters.filterSet[0].value, [recDst]); + const w = result.warnings.filter((x) => x.code === 'VIEW_FILTER_REAPPLY_FAILED'); + assert.equal(w.length, 1, 'one warning for the failed view fetch'); + assert.ok(w[0].message.includes(brokenView), 'warning names the failed view'); + assert.equal(result.viewFiltersReapplied, 1); + }); }); diff --git a/packages/mcp-server/test/sync/test-records-orchestration.test.js b/packages/mcp-server/test/sync/test-records-orchestration.test.js index e2cb7ab..ee1fbd3 100644 --- a/packages/mcp-server/test/sync/test-records-orchestration.test.js +++ b/packages/mcp-server/test/sync/test-records-orchestration.test.js @@ -201,3 +201,61 @@ describe('runRecords — pre-flight field-mapping validation', () => { assert.ok(out, 'result should be returned'); }); }); + +// ────────────────────────────────────────────────────────────────────────────── +// Filter-restore failure must not block pruneRecords +// ────────────────────────────────────────────────────────────────────────────── + +describe('runRecords — filter-restore failure cannot skip pruneRecords', () => { + it('mirror still prunes orphans when getView fails during reapplyViewFilters', async () => { + const deleted = []; + const client = { + createRecords: async () => ({ created: [], failed: [] }), + updateRecords: async (appId, tableId, rows) => ({ updated: rows, failed: [] }), + deleteRecords: async (appId, tableId, rowIds) => { deleted.push({ tableId, rowIds }); return {}; }, + // Simulates a session hiccup / deleted view at the tail of a long job. + getView: async () => { throw new Error('SESSION_INVALID'); }, + updateViewFilters: async () => ({ ok: true }), + }; + const srcSnapshot = { + baseId: 'appS', + tables: [{ + id: 'tS', name: 'Games', primaryFieldId: 'sN', + fields: [{ id: 'sN', name: 'Name', type: 'text' }], + // View WITHOUT .config (schema-only snapshot) → reapplyViewFilters must call getView. + views: [{ id: 'vS1', name: 'Grid view', type: 'grid', personalForUserId: null }], + records: [{ id: 'recS1', cellValuesByColumnId: { sN: 'A' } }], + }], + }; + const destSnapshot = { + baseId: 'appD', + tables: [{ + id: 'tD', name: 'Games', views: [{ id: 'vD1' }], fields: [], + records: [ + { id: 'recD1', cellValuesByColumnId: {} }, + { id: 'recOrphan', cellValuesByColumnId: {} }, + ], + }], + }; + const idmap = { + tables: { tS: 'tD' }, + fields: { sN: { destFld: 'dN', choices: {} } }, + records: { recS1: 'recD1' }, + }; + const result = makeResult(); + + // Must not throw, and the prune step must still run. + await runRecords({ + client, srcSnapshot, destSnapshot, idmap, + policy: 'mirror', confirmDeletions: true, + limiter: { run: (f) => f() }, journal: {}, persist: () => {}, result, + }); + + assert.equal(deleted.length, 1, 'pruneRecords ran despite the filter-restore failure'); + assert.deepEqual(deleted[0].rowIds, ['recOrphan']); + assert.ok( + result.warnings.some((w) => w.code === 'VIEW_FILTER_REAPPLY_FAILED'), + 'failed view fetch surfaced as a warning', + ); + }); +}); From fb15ffc24ee5570085ece29cc0bb3fd962d6abff Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 15:03:37 +0300 Subject: [PATCH 163/246] fix(sync): record snapshots avoid filtered views; partial reads block prune/reconcile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit snapshotTableRecords read every table through its FIRST collaborative view — a filtered view silently yields a partial row set (missed syncs, false orphan prunes, duplicate creation after reconcile). The readQueries endpoint only reads through a view, so an unfiltered table-scoped read stays capture-gated. Mitigation: pick the view deliberately (config on the snapshot, else one getView probe per candidate) preferring a view with NO filters; when every checkable collaborative view is filtered, flag the table (snapshotViewFiltered), emit SNAPSHOT_VIEW_FILTERED, and reuse the truncatedTables guard so mirror prune, stale-mapping deletion, and reconcile's existence-prune never trust the partial snapshot. Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/records.js | 55 ++++- packages/mcp-server/src/sync/snapshot.js | 44 +++- .../sync/test-snapshot-view-filtered.test.js | 208 ++++++++++++++++++ 3 files changed, 294 insertions(+), 13 deletions(-) create mode 100644 packages/mcp-server/test/sync/test-snapshot-view-filtered.test.js diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index 353cb1a..4d7af8a 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -1131,6 +1131,25 @@ export async function runRecords({ client, srcSnapshot, destSnapshot, idmap, pol `and mirror deletion is skipped for this table (>1000-row pagination is a follow-up).`, }); } + // Filtered-view guard: a table whose records were read through a FILTERED view (every + // collaborative view carries filters — snapshotTableRecords flags it) is a partial row set + // of unknown extent: hidden source rows are not synced, and hidden dest rows are + // indistinguishable from orphans/stale mappings. Same false-orphan hazard as truncation, + // so reuse the truncatedTables guard (no prune, no stale-mapping deletion) and warn loudly. + // An unfiltered read is a capture-gated follow-up (readQueries only reads through a view). + const srcTableSet = new Set(srcSnapshot.tables || []); + for (const t of [...(srcSnapshot.tables || []), ...(destSnapshot.tables || [])]) { + if (!t.snapshotViewFiltered) continue; + const side = srcTableSet.has(t) ? 'source' : 'dest'; + result.warnings.push({ + code: 'SNAPSHOT_VIEW_FILTERED', + message: `Table "${t.name}" (${side}): every collaborative view is filtered — records were read ` + + `through filtered view "${t.snapshotViewFiltered.viewName ?? t.snapshotViewFiltered.viewId}" and the ` + + `row set is PARTIAL. Records hidden by the filter are not synced, and mirror deletion is skipped ` + + `for this table. Add an unfiltered view to sync it fully.`, + }); + truncatedTables.add(t.name); + } // Live source record ids: a persisted mapping whose source record no longer exists must not // protect its dest row — otherwise mirror never propagates source-side record deletions. const liveSourceRecordIds = new Set(); @@ -1495,16 +1514,32 @@ export async function reconcile({ client, sourceBaseId, destBaseId, naturalKeys // existence-prune unsafe — pruning a live mapping would make the next apply re-create the // record as a duplicate. Skip the prune entirely and warn instead. const truncatedDestTables = collectTruncatedTableNames(null, destSnapshot); - if (truncatedDestTables.size > 0) { - for (const name of truncatedDestTables) { - result.warnings.push({ - code: 'RECORDS_TRUNCATED_PRUNE_SKIPPED', - message: `Table "${name}": dest snapshot capped at 1000 rows — a mapping whose dest row lies ` + - `beyond the window cannot be told apart from a deleted record; skipping reconcile's ` + - `existence-prune to avoid duplicating live records on the next apply ` + - `(>1000-row pagination is a follow-up).`, - }); - } + for (const name of truncatedDestTables) { + result.warnings.push({ + code: 'RECORDS_TRUNCATED_PRUNE_SKIPPED', + message: `Table "${name}": dest snapshot capped at 1000 rows — a mapping whose dest row lies ` + + `beyond the window cannot be told apart from a deleted record; skipping reconcile's ` + + `existence-prune to avoid duplicating live records on the next apply ` + + `(>1000-row pagination is a follow-up).`, + }); + } + // Filtered-view guard (same false-prune class): a dest table whose records were read through + // a filtered view (snapshotTableRecords found no unfiltered collaborative view) hides live + // rows from the snapshot — a mapping to a hidden row must not be pruned, or the next apply + // re-creates the record as a duplicate. + const filteredDestTables = (destSnapshot.tables || []).filter((t) => t.snapshotViewFiltered); + for (const t of filteredDestTables) { + result.warnings.push({ + code: 'SNAPSHOT_VIEW_FILTERED', + message: `Table "${t.name}" (dest): every collaborative view is filtered — records were read ` + + `through filtered view "${t.snapshotViewFiltered.viewName ?? t.snapshotViewFiltered.viewId}" and the ` + + `row set is PARTIAL; skipping reconcile's existence-prune to avoid pruning live mappings. ` + + `Add an unfiltered view to reconcile this base pair.`, + }); + } + if (truncatedDestTables.size > 0 || filteredDestTables.length > 0) { + // Existence-prune skipped — a missing dest id cannot be attributed to a table, so any + // partial dest snapshot makes the whole prune unsafe. } else { // Build set of all live dest record IDs across all tables const allLiveDestIds = new Set(); diff --git a/packages/mcp-server/src/sync/snapshot.js b/packages/mcp-server/src/sync/snapshot.js index 7410f84..884bec0 100644 --- a/packages/mcp-server/src/sync/snapshot.js +++ b/packages/mcp-server/src/sync/snapshot.js @@ -120,14 +120,52 @@ export async function snapshotSchemaOnly(client, appId) { return { baseId: appId, ...normalizeSchema(raw) }; } +/** True when a normalized view config carries at least one active filter clause. */ +function hasActiveFilters(cfg) { + const fs = cfg && cfg.filters && cfg.filters.filterSet; + return Array.isArray(fs) && fs.length > 0; +} + /** - * Pull a table's records via its first collaborative view (single call, ≤1000 rows; + * Pull a table's records via a collaborative view (single call, ≤1000 rows; * the internal readQueries endpoint has no cursor — Task-11 pre-flight warns on >1000). + * + * The readQueries endpoint only reads THROUGH a view (no table-scoped source is captured), + * and a view's own filters silently shrink the row set — a partial snapshot means missed + * syncs and false orphan prunes downstream. So the view is picked deliberately: + * 1. first collaborative view KNOWN to have no filters (config already on the snapshot, + * or probed via one getView per candidate); + * 2. else first candidate whose filter state is UNKNOWN (no getView / probe failed) — + * legacy behavior, unverifiable; + * 3. else (every checkable candidate is filtered) the FIRST candidate, and the table is + * flagged `table.snapshotViewFiltered = { viewId, viewName }` so callers can warn + * (SNAPSHOT_VIEW_FILTERED) and destructive steps can treat it like a truncated table. + * A true unfiltered read is capture-gated (needs a non-view readQueries source). + * * @returns {Promise>} */ export async function snapshotTableRecords(client, appId, table) { - const view = (table.views || []).find((v) => !v.personalForUserId) || (table.views || [])[0]; - if (!view) return []; + const views = table.views || []; + const candidates = views.filter((v) => !v.personalForUserId); + if (!candidates.length && views.length) candidates.push(views[0]); // personal-only fallback + if (!candidates.length) return []; + + let unfiltered = null; // first known-unfiltered + let unknown = null; // first unverifiable (no getView / probe failed) + let firstFiltered = null; + for (const v of candidates) { + let cfg = v.config; + if (cfg === undefined && typeof client.getView === 'function') { + try { cfg = normalizeViewConfig(await client.getView(appId, v.id)); } catch { /* unknown */ } + } + if (cfg === undefined) { unknown ??= v; continue; } + if (!hasActiveFilters(cfg)) { unfiltered = v; break; } + firstFiltered ??= v; + } + const view = unfiltered || unknown || firstFiltered; + if (!unfiltered && !unknown) { + table.snapshotViewFiltered = { viewId: view.id, viewName: view.name ?? null }; + } const res = await client.queryRecords(appId, table.id, view.id, { limit: 1000 }); return (res?.summary?.rows || []).map((r) => ({ id: r.id, cellValuesByColumnId: r.fields || {} })); } diff --git a/packages/mcp-server/test/sync/test-snapshot-view-filtered.test.js b/packages/mcp-server/test/sync/test-snapshot-view-filtered.test.js new file mode 100644 index 0000000..8deeb36 --- /dev/null +++ b/packages/mcp-server/test/sync/test-snapshot-view-filtered.test.js @@ -0,0 +1,208 @@ +/** + * test-snapshot-view-filtered.test.js + * + * snapshotTableRecords reads records THROUGH a view (the readQueries endpoint has no + * table-scoped source), so a filtered view silently yields a PARTIAL row set: hidden source + * records are never synced, and hidden dest records look deleted (false orphan prunes → + * duplicates on the next apply). + * + * Guard under test: + * - snapshotTableRecords prefers a view with NO filters (config on the snapshot, or probed + * via getView) and flags `table.snapshotViewFiltered` when every checkable candidate is + * filtered. + * - runRecords surfaces SNAPSHOT_VIEW_FILTERED and reuses the truncatedTables guard so + * mirror pruning never trusts the partial snapshot. + * - reconcile skips the existence-prune when a dest table was read through a filtered view. + */ + +import { describe, it, beforeEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { snapshotTableRecords } from '../../src/sync/snapshot.js'; +import { runRecords, reconcile } from '../../src/sync/records.js'; +import { saveIdmap, loadIdmap } from '../../src/sync/idmap.js'; + +const FILTERED = { filters: { filterSet: [{ columnId: 'fldX', operator: 'contains', value: 'x' }], conjunction: 'and' } }; +const UNFILTERED = { filters: { filterSet: [], conjunction: 'and' } }; + +function queryClient(rows, extra = {}) { + const seen = []; + return { + seen, + queryRecords: async (appId, tableId, viewId) => { seen.push(viewId); return { summary: { rows } }; }, + ...extra, + }; +} + +describe('snapshot.snapshotTableRecords — filtered-view avoidance', () => { + it('skips a filtered view when the snapshot already carries configs', async () => { + const client = queryClient([{ id: 'rec1', fields: {} }]); + const table = { + id: 'tbl1', + views: [ + { id: 'viwF', name: 'Active', config: FILTERED }, + { id: 'viwU', name: 'All', config: UNFILTERED }, + ], + }; + const out = await snapshotTableRecords(client, 'app1', table); + assert.deepEqual(client.seen, ['viwU'], 'must read through the unfiltered view'); + assert.equal(table.snapshotViewFiltered, undefined, 'no flag when an unfiltered view exists'); + assert.equal(out.length, 1); + }); + + it('probes getView when configs are absent (schema-only snapshot) and picks the unfiltered view', async () => { + const probed = []; + const client = queryClient([], { + getView: async (appId, viewId) => { + probed.push(viewId); + return viewId === 'viwF' ? { ...FILTERED } : { filters: null }; + }, + }); + const table = { id: 'tbl1', views: [{ id: 'viwF', name: 'Active' }, { id: 'viwU', name: 'All' }] }; + await snapshotTableRecords(client, 'app1', table); + assert.deepEqual(probed, ['viwF', 'viwU']); + assert.deepEqual(client.seen, ['viwU']); + assert.equal(table.snapshotViewFiltered, undefined); + }); + + it('flags table.snapshotViewFiltered when EVERY collaborative view is filtered', async () => { + const client = queryClient([{ id: 'rec1', fields: {} }], { + getView: async () => ({ ...FILTERED }), + }); + const table = { id: 'tbl1', name: 'Tasks', views: [{ id: 'viw1', name: 'Active' }, { id: 'viw2', name: 'Mine' }] }; + const out = await snapshotTableRecords(client, 'app1', table); + assert.deepEqual(client.seen, ['viw1'], 'falls back to the first collaborative view'); + assert.ok(table.snapshotViewFiltered, 'table must be flagged as read-through-filtered-view'); + assert.equal(table.snapshotViewFiltered.viewId, 'viw1'); + assert.equal(out.length, 1, 'records are still returned (partial is better than nothing)'); + }); + + it('keeps legacy behavior (first collaborative view, no flag) when the client has no getView', async () => { + const client = queryClient([]); + const table = { id: 'tbl1', views: [{ id: 'viwP', personalForUserId: 'usr1' }, { id: 'viw1', name: 'Grid' }] }; + await snapshotTableRecords(client, 'app1', table); + assert.deepEqual(client.seen, ['viw1']); + assert.equal(table.snapshotViewFiltered, undefined, 'cannot verify → no flag'); + }); + + it('keeps legacy behavior when getView probing fails (view state unknown)', async () => { + const client = queryClient([], { getView: async () => { throw new Error('boom'); } }); + const table = { id: 'tbl1', views: [{ id: 'viw1', name: 'Grid' }] }; + await snapshotTableRecords(client, 'app1', table); + assert.deepEqual(client.seen, ['viw1']); + assert.equal(table.snapshotViewFiltered, undefined); + }); +}); + +describe('runRecords — filtered-view snapshot disables mirror prune (truncation-guard reuse)', () => { + it('does not delete orphans in a flagged table and emits SNAPSHOT_VIEW_FILTERED', async () => { + const deleted = []; + const client = { + createRecords: async () => ({ created: [], failed: [] }), + updateRecords: async (appId, tableId, rows) => ({ updated: rows, failed: [] }), + deleteRecords: async (appId, tableId, rowIds) => { deleted.push(rowIds); return {}; }, + }; + const srcSnapshot = { + baseId: 'appS', + tables: [{ + id: 'tS', name: 'Games', primaryFieldId: 'sN', + fields: [{ id: 'sN', name: 'Name', type: 'text' }], + views: [{ id: 'vS1', name: 'Grid view', type: 'grid', personalForUserId: null, config: {} }], + records: [{ id: 'recS1', cellValuesByColumnId: { sN: 'A' } }], + }], + }; + const destSnapshot = { + baseId: 'appD', + tables: [{ + id: 'tD', name: 'Games', views: [{ id: 'vD1', config: {} }], fields: [], + // Snapshot read through a filtered view — 'recHidden' style rows are NOT here, and + // 'recOrphan' cannot be distinguished from a mapped row hidden by the filter. + snapshotViewFiltered: { viewId: 'vD1', viewName: 'Active' }, + records: [ + { id: 'recD1', cellValuesByColumnId: {} }, + { id: 'recOrphan', cellValuesByColumnId: {} }, + ], + }], + }; + const idmap = { tables: { tS: 'tD' }, fields: { sN: { destFld: 'dN', choices: {} } }, records: { recS1: 'recD1' } }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, deleted: 0, warnings: [] }; + + await runRecords({ + client, srcSnapshot, destSnapshot, idmap, + policy: 'mirror', confirmDeletions: true, + limiter: { run: (f) => f() }, journal: {}, persist: () => {}, result, + }); + + assert.deepEqual(deleted, [], 'prune must not trust a filtered (partial) snapshot'); + assert.ok(result.warnings.some((w) => w.code === 'SNAPSHOT_VIEW_FILTERED'), 'must warn SNAPSHOT_VIEW_FILTERED'); + assert.equal(result.deleted, 0); + }); + + it('a filtered SOURCE table also blocks the stale-mapping prune path', async () => { + const deleted = []; + const client = { + createRecords: async () => ({ created: [], failed: [] }), + updateRecords: async (appId, tableId, rows) => ({ updated: rows, failed: [] }), + deleteRecords: async (appId, tableId, rowIds) => { deleted.push(rowIds); return {}; }, + }; + const srcSnapshot = { + baseId: 'appS', + tables: [{ + id: 'tS', name: 'Games', primaryFieldId: 'sN', + fields: [{ id: 'sN', name: 'Name', type: 'text' }], + views: [{ id: 'vS1', name: 'Active', type: 'grid', personalForUserId: null }], + snapshotViewFiltered: { viewId: 'vS1', viewName: 'Active' }, + // recS1 is hidden by the source filter → absent here, but its mapping is NOT stale. + records: [], + }], + }; + const destSnapshot = { + baseId: 'appD', + tables: [{ + id: 'tD', name: 'Games', views: [{ id: 'vD1', config: {} }], fields: [], + records: [{ id: 'recD1', cellValuesByColumnId: {} }], + }], + }; + const idmap = { tables: { tS: 'tD' }, fields: { sN: { destFld: 'dN', choices: {} } }, records: { recS1: 'recD1' } }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, deleted: 0, warnings: [] }; + + await runRecords({ + client, srcSnapshot, destSnapshot, idmap, + policy: 'mirror', confirmDeletions: true, + limiter: { run: (f) => f() }, journal: {}, persist: () => {}, result, + }); + + assert.deepEqual(deleted, [], 'a source record hidden by a filter must never orphan its dest row'); + assert.ok(result.warnings.some((w) => w.code === 'SNAPSHOT_VIEW_FILTERED')); + }); +}); + +describe('reconcile — filtered dest view skips the existence-prune', () => { + beforeEach(() => { process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-filt-')); }); + + it('keeps mappings whose dest rows are hidden by the filter (no false prune)', async () => { + const SRC = 'appSRC0000000000', DEST = 'appDST0000000000'; + // recD2 exists in dest but is hidden by the only (filtered) view → absent from the read. + saveIdmap(SRC, DEST, { tables: { tS: 'tD' }, fields: {}, records: { recS1: 'recD1', recS2: 'recD2' } }); + const client = { + getApplicationData: async () => ({ + data: { + tableSchemas: [{ + id: 'tD', name: 'Games', primaryColumnId: 'dN', + columns: [{ id: 'dN', name: 'Name', type: 'text' }], + views: [{ id: 'vD1', name: 'Active', type: 'grid', personalForUserId: null }], + }], + }, + }), + getView: async () => ({ filters: { filterSet: [{ columnId: 'dN', operator: 'contains', value: 'x' }], conjunction: 'and' } }), + queryRecords: async () => ({ summary: { rows: [{ id: 'recD1', fields: {} }] } }), + }; + const res = await reconcile({ client, sourceBaseId: SRC, destBaseId: DEST }); + assert.equal(res.skipped, 0, 'no mapping may be pruned from a filtered (partial) snapshot'); + assert.ok(res.warnings.some((w) => w.code === 'SNAPSHOT_VIEW_FILTERED'), 'must warn SNAPSHOT_VIEW_FILTERED'); + const persisted = loadIdmap(SRC, DEST); + assert.equal(persisted.records.recS2, 'recD2', 'hidden-but-live mapping survives'); + }); +}); From b08f1eaef8804db82808cadfb5399b8a997af3bf Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 15:04:04 +0300 Subject: [PATCH 164/246] fix(sync): mode=status detects a records job whose process died mid-run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The background records job wrote records-job-.json with status "running" and only its own .then/.catch ever wrote done/failed — if the daemon/stdio process died mid-job, mode=status reported "running" forever with a plausible recordsMapped count. writeRecordsJobStatus now stamps the writer pid on every "running" write, and readRecordsJobStatus derives status "failed" (with a re-run mode=apply resume hint) when that pid is no longer alive. Legacy pid-less files and done/failed statuses are untouched. Windows-safe (process.kill(pid, 0) probe, same as apply-lock). Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/records.js | 24 +++++++++- .../test/sync/test-records-job.test.js | 48 ++++++++++++++++++- 2 files changed, 68 insertions(+), 4 deletions(-) diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index 4d7af8a..863eb76 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -49,14 +49,34 @@ function injectFieldMappings(cells, tableMappings, srcCells) { function recordsJobPath(sourceBaseId, destBaseId, planId) { return join(syncDir(sourceBaseId, destBaseId), `records-job-${planId}.json`); } +function pidAlive(pid) { + if (!Number.isInteger(pid) || pid <= 0) return false; + try { process.kill(pid, 0); return true; } catch { return false; } +} export function writeRecordsJobStatus(sourceBaseId, destBaseId, planId, status) { mkdirSync(syncDir(sourceBaseId, destBaseId), { recursive: true }); - safeAtomicWriteFileSync(recordsJobPath(sourceBaseId, destBaseId, planId), JSON.stringify({ planId, ...status }, null, 2)); + // Stamp the writer's pid on 'running' so a reader can detect a job whose process died + // mid-run — after a crash/restart the .then/.catch that would write done/failed never + // fires, and the file would otherwise report 'running' forever. + const body = { planId, ...(status.status === 'running' ? { pid: process.pid } : {}), ...status }; + safeAtomicWriteFileSync(recordsJobPath(sourceBaseId, destBaseId, planId), JSON.stringify(body, null, 2)); } export function readRecordsJobStatus(sourceBaseId, destBaseId, planId) { const p = recordsJobPath(sourceBaseId, destBaseId, planId); if (!existsSync(p)) return null; - try { return JSON.parse(readFileSync(p, 'utf8')); } catch { return null; } + let job = null; + try { job = JSON.parse(readFileSync(p, 'utf8')); } catch { return null; } + // Liveness: 'running' is only believable while the writing process is alive. A pid-less + // (legacy) file cannot be verified and stays 'running'. Windows-safe: process.kill(pid, 0) + // throws for a nonexistent pid on every platform. + if (job && job.status === 'running' && job.pid != null && !pidAlive(job.pid)) { + return { + ...job, + status: 'failed', + error: `records job process (pid ${job.pid}) died mid-job — re-run mode=apply to resume (idmap + records journal persist per chunk, so completed work is not repeated)`, + }; + } + return job; } /** diff --git a/packages/mcp-server/test/sync/test-records-job.test.js b/packages/mcp-server/test/sync/test-records-job.test.js index 3b6c220..e1ed367 100644 --- a/packages/mcp-server/test/sync/test-records-job.test.js +++ b/packages/mcp-server/test/sync/test-records-job.test.js @@ -1,13 +1,22 @@ import { describe, it, beforeEach } from 'node:test'; import assert from 'node:assert/strict'; -import { mkdtempSync } from 'node:fs'; +import { mkdtempSync, mkdirSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; import { writeRecordsJobStatus, readRecordsJobStatus } from '../../src/sync/records.js'; import { recordsStatus } from '../../src/sync/index.js'; -import { saveIdmap } from '../../src/sync/idmap.js'; +import { saveIdmap, syncDir } from '../../src/sync/idmap.js'; import { renderApplyResult } from '../../src/sync/report.js'; +/** A pid guaranteed dead: spawn a no-op node process synchronously — by the time spawnSync + * returns it has exited, so its pid no longer maps to a live process. */ +function deadPid() { + const r = spawnSync(process.execPath, ['-e', '']); + assert.ok(r.pid > 0, 'spawned a probe process'); + return r.pid; +} + const SRC = 'appSRC0000000000', DEST = 'appDST0000000000'; describe('records background-job status', () => { @@ -36,6 +45,41 @@ describe('records background-job status', () => { assert.equal(s.recordsMapped, 0); }); + it('stamps the writer pid on a running status write', () => { + writeRecordsJobStatus(SRC, DEST, 'plnPid', { status: 'running', startedAt: 't0' }); + const j = readRecordsJobStatus(SRC, DEST, 'plnPid'); + assert.equal(j.pid, process.pid, 'running status must carry the writing process pid'); + assert.equal(j.status, 'running', 'writer is alive → still running'); + }); + + it('reports a running job with a DEAD pid as failed with a resume hint (process died mid-job)', () => { + writeRecordsJobStatus(SRC, DEST, 'plnDead', { status: 'running', startedAt: 't0', pid: deadPid() }); + const s = recordsStatus({ sourceBaseId: SRC, destBaseId: DEST, planId: 'plnDead' }); + assert.equal(s.status, 'failed', 'a dead writer can never finish the job — must not report running forever'); + assert.match(String(s.error), /died mid-job/); + assert.match(String(s.error), /mode=apply/, 'error must tell the user how to resume'); + }); + + it('keeps a running job with a LIVE pid as running', () => { + writeRecordsJobStatus(SRC, DEST, 'plnLive', { status: 'running', startedAt: 't0', pid: process.pid }); + const s = recordsStatus({ sourceBaseId: SRC, destBaseId: DEST, planId: 'plnLive' }); + assert.equal(s.status, 'running'); + }); + + it('a legacy running job file without pid stays running (cannot verify liveness)', () => { + // Simulate a pre-pid job file written by an older version. + mkdirSync(syncDir(SRC, DEST), { recursive: true }); + writeFileSync(join(syncDir(SRC, DEST), 'records-job-plnLegacy.json'), JSON.stringify({ planId: 'plnLegacy', status: 'running', startedAt: 't0' })); + const s = recordsStatus({ sourceBaseId: SRC, destBaseId: DEST, planId: 'plnLegacy' }); + assert.equal(s.status, 'running'); + }); + + it('done/failed statuses are never re-derived from pid', () => { + writeRecordsJobStatus(SRC, DEST, 'plnDone', { status: 'done', startedAt: 't0', finishedAt: 't1', pid: deadPid(), result: { created: 1 } }); + const s = recordsStatus({ sourceBaseId: SRC, destBaseId: DEST, planId: 'plnDone' }); + assert.equal(s.status, 'done'); + }); + it('done status carries the result summary', () => { writeRecordsJobStatus(SRC, DEST, 'pln3', { status: 'done', startedAt: 't0', finishedAt: 't1', result: { created: 5, updated: 0, failed: 0, attachmentsUploaded: 1, viewFiltersReapplied: 2, warnings: 0 } }); const s = recordsStatus({ sourceBaseId: SRC, destBaseId: DEST, planId: 'pln3' }); From 8d80a98c7469b23ac8146058dcbe042c854d3b14 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 15:04:17 +0300 Subject: [PATCH 165/246] fix(sync): createField registers the real select choice map for the same-run records phase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A created (or adopted-by-name) select/multiSelect field was registered in the idmap with choices:{} and nothing refreshed the map before the records phase of the SAME apply run — every select cell skipped (RECORD_CELL_SKIPPED) on first sync, and multiSelect columns of first-pass-created records never converged (the update path defers arrays). After createField of a select-like type, re-read the authoritative dest schema and name-match source→dest choice ids (the server may re-key the choices it was sent); the adopt-existing path name-matches against the existing field directly. Best-effort: a failed read-back warns (CHOICE_MAP_UNRESOLVED) instead of failing the create. Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/apply.js | 44 +++++- .../sync/test-apply-select-choices.test.js | 139 ++++++++++++++++++ 2 files changed, 180 insertions(+), 3 deletions(-) create mode 100644 packages/mcp-server/test/sync/test-apply-select-choices.test.js diff --git a/packages/mcp-server/src/sync/apply.js b/packages/mcp-server/src/sync/apply.js index e8fa387..cbbe048 100644 --- a/packages/mcp-server/src/sync/apply.js +++ b/packages/mcp-server/src/sync/apply.js @@ -90,6 +90,25 @@ function findDestFieldType(index, destFieldId) { const f = findDestField(index, destFieldId); return f ? f.type : undefined; } +const SELECT_TYPES = new Set(['select', 'singleSelect', 'multiSelect', 'multipleSelects']); + +// Source-choice-id → dest-choice-id map, matched by NAME (same contract as idmap.js +// matchChoices). Registered at field create/adopt time: the records phase of the SAME run +// reads idmap.fields[..].choices to map select cells — an empty map made every select cell +// skip (RECORD_CELL_SKIPPED) on first sync, and multiSelect columns never converged after. +function matchChoicesByName(srcTypeOptions, destTypeOptions) { + const sc = srcTypeOptions && srcTypeOptions.choices; + const dc = destTypeOptions && destTypeOptions.choices; + if (!sc || !dc) return {}; + const destByName = new Map(Object.entries(dc).map(([id, c]) => [c.name, c.id || id])); + const out = {}; + for (const [id, c] of Object.entries(sc)) { + const destId = destByName.get(c.name); + if (destId) out[c.id || id] = destId; + } + return out; +} + // Merge source choices into dest choices by NAME (never drop a dest choice). Dest choices // keep their ids; new source choices are added without ids (Airtable assigns). function mergeChoices(destField, srcTypeOptions) { @@ -271,7 +290,9 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result, } const existing = entry && entry.fieldsByName.get(a.name); if (existing) { - idmap.fields[a.sourceFieldId] = { destFld: existing.id, choices: {} }; + // Adoption: the dest field already exists with its own choice ids — register the real + // name-matched map so the same-run records phase can write select cells. + idmap.fields[a.sourceFieldId] = { destFld: existing.id, choices: matchChoicesByName(a.typeOptions, existing.typeOptions) }; result.skipped++; return; } @@ -311,8 +332,25 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result, typeOptions = rest; } const { columnId } = await client.createField(destAppId, destTableId, { name: a.name, type: a.type, typeOptions, description: a.description ?? undefined }); - idmap.fields[a.sourceFieldId] = { destFld: columnId, choices: {} }; - if (entry) entry.fieldsByName.set(a.name, { id: columnId, name: a.name, type: a.type, typeOptions }); + let createdTypeOptions = typeOptions; + let choices = {}; + if (SELECT_TYPES.has(a.type)) { + // The server may re-key the choices it was sent (createField's response does not + // reliably echo the column), so re-read the AUTHORITATIVE dest schema and name-match. + // Without this the idmap carried choices:{} and the same-run records phase skipped + // every select/multiSelect cell of the new field. Best-effort: a failed re-read must + // not fail the create — the next plan's matchByName repopulates the map. + try { + const { cols } = await readTable(client, destAppId, destTableId); + const createdCol = cols.find((c) => c.id === columnId); + if (createdCol) createdTypeOptions = createdCol.typeOptions ?? createdTypeOptions; + choices = matchChoicesByName(a.typeOptions, createdCol && createdCol.typeOptions); + } catch (e) { + result.warnings.push({ code: 'CHOICE_MAP_UNRESOLVED', message: `Field "${a.name}": could not read back created choices (${e.message ?? e}) — select cells may be skipped this run` }); + } + } + idmap.fields[a.sourceFieldId] = { destFld: columnId, choices }; + if (entry) entry.fieldsByName.set(a.name, { id: columnId, name: a.name, type: a.type, typeOptions: createdTypeOptions }); result.created++; return; } diff --git a/packages/mcp-server/test/sync/test-apply-select-choices.test.js b/packages/mcp-server/test/sync/test-apply-select-choices.test.js new file mode 100644 index 0000000..6ad5245 --- /dev/null +++ b/packages/mcp-server/test/sync/test-apply-select-choices.test.js @@ -0,0 +1,139 @@ +/** + * test-apply-select-choices.test.js + * + * createField used to register a new select/multiSelect field with `choices: {}` in the idmap, + * and nothing refreshed the choice map before the SAME-RUN records phase — so every select cell + * skipped (RECORD_CELL_SKIPPED) on first sync, and multiSelect columns never converged (the + * update path defers arrays). + * + * Under test: after createField (and existing-by-name adoption), the idmap entry carries the + * REAL source-choice-id → dest-choice-id map, matched by name against the authoritative dest + * schema (the server may re-key the choices it was sent). + */ + +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { MockClient } from './helpers/mock-client.js'; +import { applyPlan } from '../../src/sync/apply.js'; +import { newJournal } from '../../src/sync/journal.js'; +import { snapshotBase } from '../../src/sync/snapshot.js'; + +const SELECT_LIKE = new Set(['select', 'singleSelect', 'multiSelect', 'multipleSelects']); + +/** MockClient variant that RE-KEYS select choices server-side (fresh sel ids), like a server + * that ignores caller-provided choice ids would. The idmap must survive this. */ +class RekeyingClient extends MockClient { + async createField(appId, tableId, cfg) { + const out = await super.createField(appId, tableId, cfg); + const f = this._field(out.columnId); + if (SELECT_LIKE.has(cfg.type) && f.typeOptions && f.typeOptions.choices) { + const rekeyed = {}; + for (const c of Object.values(f.typeOptions.choices)) { + const nid = this._id('sel'); + rekeyed[nid] = { ...c, id: nid }; + } + f.typeOptions = { ...f.typeOptions, choices: rekeyed }; + } + return out; + } +} + +const SRC_CHOICES = { + selSRCTodo: { id: 'selSRCTodo', name: 'Todo', color: 'blue' }, + selSRCDone: { id: 'selSRCDone', name: 'Done', color: 'green' }, +}; + +// NOTE: 'Stage', not 'Status' — MockClient.createTable seeds a scaffolding select named +// 'Status', which would trigger the adopt-existing path instead of the create path. +function selectAction(overrides = {}) { + return { + kind: 'createField', sourceTableId: 'tS', sourceFieldId: 'fSel', name: 'Stage', type: 'select', + typeOptions: { choices: JSON.parse(JSON.stringify(SRC_CHOICES)), choiceOrder: ['selSRCTodo', 'selSRCDone'] }, + description: null, computed: false, dependsOn: [], dependsOnTables: [], + ...overrides, + }; +} + +describe('apply: createField registers the real select choice map (first-run records phase needs it)', () => { + it('maps source choice ids to the created dest choice ids by name (server re-keys)', async () => { + const client = new RekeyingClient(); + const { tableId } = await client.createTable('appD', 'Offers'); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { + planId: 'plnSel', sourceBaseId: 'appS', destBaseId: 'appD', + idmap: { tables: { tS: tableId }, fields: {} }, + actions: [selectAction()], + orphans: [], warnings: [], + }; + const res = await applyPlan({ client, plan, destAppId: 'appD', destSnapshot, idmap: JSON.parse(JSON.stringify(plan.idmap)), journal: newJournal('plnSel', 'ts'), persist: () => {} }); + assert.equal(res.failed, 0); + const entry = res.idmap.fields.fSel; + assert.ok(entry, 'field mapped'); + const destField = (await client.getApplicationData('appD')).data.tableSchemas + .find((t) => t.id === tableId).columns.find((c) => c.name === 'Stage'); + const destByName = new Map(Object.values(destField.typeOptions.choices).map((c) => [c.name, c.id])); + assert.deepEqual(entry.choices, { + selSRCTodo: destByName.get('Todo'), + selSRCDone: destByName.get('Done'), + }, 'choice map must be the REAL src→dest id map, not {}'); + assert.notEqual(entry.choices.selSRCTodo, 'selSRCTodo', 'dest ids were re-keyed — the map cannot be identity here'); + }); + + it('multiSelect creates get their choice map too', async () => { + const client = new RekeyingClient(); + const { tableId } = await client.createTable('appD', 'Offers'); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { + planId: 'plnMSel', sourceBaseId: 'appS', destBaseId: 'appD', + idmap: { tables: { tS: tableId }, fields: {} }, + actions: [selectAction({ sourceFieldId: 'fTags', name: 'Tags', type: 'multiSelect' })], + orphans: [], warnings: [], + }; + const res = await applyPlan({ client, plan, destAppId: 'appD', destSnapshot, idmap: JSON.parse(JSON.stringify(plan.idmap)), journal: newJournal('plnMSel', 'ts'), persist: () => {} }); + assert.equal(res.failed, 0); + assert.equal(Object.keys(res.idmap.fields.fTags.choices).length, 2, 'both choices mapped'); + }); + + it('adopting an existing same-name select field registers its choice map (not {})', async () => { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'Offers'); + await client.createField('appD', tableId, { + name: 'Stage', type: 'select', + typeOptions: { choices: { + selDSTTodo: { id: 'selDSTTodo', name: 'Todo', color: 'red' }, + selDSTDone: { id: 'selDSTDone', name: 'Done', color: 'gray' }, + } }, + }); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { + planId: 'plnAdopt', sourceBaseId: 'appS', destBaseId: 'appD', + idmap: { tables: { tS: tableId }, fields: {} }, + actions: [selectAction()], + orphans: [], warnings: [], + }; + const res = await applyPlan({ client, plan, destAppId: 'appD', destSnapshot, idmap: JSON.parse(JSON.stringify(plan.idmap)), journal: newJournal('plnAdopt', 'ts'), persist: () => {} }); + assert.equal(res.created, 0); + assert.equal(res.skipped, 1, 'adopted by name'); + assert.deepEqual(res.idmap.fields.fSel.choices, { + selSRCTodo: 'selDSTTodo', + selSRCDone: 'selDSTDone', + }, 'adoption must name-match choices against the existing dest field'); + }); + + it('non-select fields keep an empty choice map (no extra schema reads)', async () => { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'Offers'); + const destSnapshot = await snapshotBase(client, 'appD'); + const readsBefore = client.calls.length; + const plan = { + planId: 'plnTxt', sourceBaseId: 'appS', destBaseId: 'appD', + idmap: { tables: { tS: tableId }, fields: {} }, + actions: [{ kind: 'createField', sourceTableId: 'tS', sourceFieldId: 'fTxt', name: 'Notes2', type: 'text', typeOptions: null, description: null, computed: false, dependsOn: [], dependsOnTables: [] }], + orphans: [], warnings: [], + }; + const res = await applyPlan({ client, plan, destAppId: 'appD', destSnapshot, idmap: JSON.parse(JSON.stringify(plan.idmap)), journal: newJournal('plnTxt', 'ts'), persist: () => {} }); + assert.equal(res.failed, 0); + assert.deepEqual(res.idmap.fields.fTxt.choices, {}); + void readsBefore; + }); +}); From de9f9b5b02f39a213391280eebb498c70198e8d4 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 15:26:40 +0300 Subject: [PATCH 166/246] =?UTF-8?q?fix(sync):=20records=20seams=20?= =?UTF-8?q?=E2=80=94=20computed-dest=20clear=20skip,=20resume-durable=20cr?= =?UTF-8?q?eatedDestIds,=20reconcile=20lock?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three seam fixes in the records engine (each reproduced by a failing test first): - buildUpdateCells: cleared-cell null propagation checked computedness of the SOURCE field only — a name-matched scalar-source/computed-dest pair (persistent RETYPE_DEFERRED) got {fld:null} written to a computed dest cell, which updatePrimitiveCell rejects, poisoning every mapped row's update on every re-sync. The clear is now also skipped when the DEST field is computed (per-table destComputedFldIds set from the dest snapshot). - runRecords: dest-wins protection for Pass 2 (links) / Pass 3 (attachments) keyed off a run-local createdDestIds set — on a RESUMED records job (same planId after a crash) rows created by the crashed run were already mapped, took Pass 1's dest-wins skip, never re-entered the set, and kept permanently empty link/attachment cells under preserve. The set is now mirrored into the records journal (journal.createdDestIds) before every persist and seeded from it on resume of the same planId; cross-plan behavior unchanged. - reconcile: took no lock — run concurrently with a background records job on the same pair, both load-modify-save idmap.json (last-writer-wins clobber). reconcile now acquires the same per-pair apply.lock (APPLY_LOCKED while a live pid holds it, stale-lock replacement, released on success and throw). Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/records.js | 79 ++++++++---- .../test/sync/test-reconcile-lock.test.js | 73 +++++++++++ .../sync/test-records-resume-created.test.js | 115 ++++++++++++++++++ .../mcp-server/test/sync/test-records.test.js | 58 +++++++++ 4 files changed, 303 insertions(+), 22 deletions(-) create mode 100644 packages/mcp-server/test/sync/test-reconcile-lock.test.js create mode 100644 packages/mcp-server/test/sync/test-records-resume-created.test.js diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index 863eb76..af2c96a 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -21,6 +21,7 @@ import { remapViewConfig, collectFilterRecordRefs } from './remap.js'; import { snapshotSchemaOnly, snapshotTableRecords, normalizeViewConfig, isComputedType } from './snapshot.js'; import { loadIdmap, saveIdmap, syncDir } from './idmap.js'; import { newJournal, loadRecordsJournal, saveRecordsJournal } from './journal.js'; +import { acquireApplyLock } from './apply-lock.js'; import { safeAtomicWriteFileSync } from '../safe-write.js'; /** @@ -170,9 +171,11 @@ function buildCreateCells(srcFields, srcCells, idmap, warnings, srcRecId) { * Cleared cells: readQueries omits empty cells, so a cell CLEARED on source arrives as an * ABSENT key. Since buildUpdateCells only runs under conflicts=source-wins, the clear must * propagate: an explicit null is emitted for a mapped, writable, non-array field whose dest - * cell is known to still hold a value. Never cleared: unmapped fields, computed fields, - * array cells (Pass 2/3 / deferral own those), and anything when the dest record's cells are - * unknown (dest record absent from the snapshot). + * cell is known to still hold a value. Never cleared: unmapped fields, computed fields on + * EITHER side (a scalar-source field name-matched to a computed dest field is a persistent + * RETYPE_DEFERRED pair — updatePrimitiveCell rejects computed writes and one failing cell + * poisons the whole row update), array cells (Pass 2/3 / deferral own those), and anything + * when the dest record's cells are unknown (dest record absent from the snapshot). * * @param {Array} srcFields - source table field descriptors * @param {object} srcCells - source record cellValuesByColumnId @@ -181,9 +184,10 @@ function buildCreateCells(srcFields, srcCells, idmap, warnings, srcRecId) { * @param {object} idmap - id-map (fields + choices) * @param {Array} warnings - result.warnings array (mutated) * @param {string} srcRecId - source record id (for warning context) + * @param {Set} [destComputedFldIds] - dest field ids whose DEST type is computed * @returns {object} - dest cellValuesByColumnId (primitives + single-select only) */ -function buildUpdateCells(srcFields, srcCells, destCells, idmap, warnings, srcRecId) { +function buildUpdateCells(srcFields, srcCells, destCells, idmap, warnings, srcRecId, destComputedFldIds = new Set()) { const cells = {}; for (const field of srcFields) { const mapping = idmap.fields[field.id]; @@ -208,7 +212,9 @@ function buildUpdateCells(srcFields, srcCells, destCells, idmap, warnings, srcRe if (isComputedType(field.type)) continue; // Pass 1 never writes (or clears) computed cells if (srcVal === undefined) { // Cleared on source: propagate the clear (source-wins) as an explicit null, but only - // when the dest record is known to still hold a value for this field. + // when the dest record is known to still hold a value for this field — and never into + // a COMPUTED dest field (scalar-source/computed-dest RETYPE_DEFERRED pair). + if (destComputedFldIds.has(mapping.destFld)) continue; const destVal = destCells ? destCells[mapping.destFld] : undefined; if (destVal !== undefined && destVal !== null) cells[mapping.destFld] = null; continue; @@ -427,6 +433,11 @@ export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idm const destCellsById = new Map( (destTableById.get(destTableId)?.records || []).map((r) => [r.id, r.cellValuesByColumnId || {}]), ); + // Dest fields that are COMPUTED — a name-matched scalar-source/computed-dest pair + // (persistent RETYPE_DEFERRED) must never receive a cleared-cell null (see buildUpdateCells). + const destComputedFldIds = new Set( + (destTableById.get(destTableId)?.fields || []).filter((f) => isComputedType(f.type)).map((f) => f.id), + ); const records = srcTable.records || []; if (records.length === 0) continue; @@ -453,7 +464,7 @@ export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idm } else { // UPDATE path — array cells NOT accepted by updateRecords (deferred, Pass 2/3 or warning) if (conflicts === 'dest-wins') continue; // preserve dest edits: skip the overwrite - const cells = buildUpdateCells(srcTable.fields, srcCells, destCellsById.get(destRecId), idmap, result.warnings, rec.id); + const cells = buildUpdateCells(srcTable.fields, srcCells, destCellsById.get(destRecId), idmap, result.warnings, rec.id, destComputedFldIds); injectFieldMappings(cells, tableMappings, srcCells); // Nothing to write → skip: client.updateRecords rejects an empty cell map per row, // which would misreport this healthy record as RECORD_UPDATE_FAILED on every re-sync. @@ -1093,20 +1104,28 @@ export async function runRecords({ client, srcSnapshot, destSnapshot, idmap, pol t.snapshotRowCount ??= (t.records || []).length; } + // Dest row ids created THIS plan — Pass 2/3 use it so conflicts=dest-wins only blocks + // writes to PRE-EXISTING mapped rows (created rows always get their links/attachments). + // Seeded from the records journal so a RESUMED job (same planId after a crash) still treats + // rows created by the crashed run as sync-created: on resume they are already in + // idmap.records, take Pass 1's dest-wins skip, and would otherwise never re-enter the set — + // leaving their link/attachment cells permanently empty under preserve. Cross-plan behavior + // is unchanged (a new planId gets a new journal without createdDestIds). + const createdDestIds = new Set(Array.isArray(journal?.createdDestIds) ? journal.createdDestIds : []); + // Mirror the current set into the journal before EVERY persist (incl. Pass 1's per-chunk + // saves) so a crash mid-run leaves the created ids durable next to idmap.records. + const persistRun = (m, j) => { j.createdDestIds = [...createdDestIds]; persist(m, j); }; + // Natural-key pre-pass: match real-but-unmapped dest records BEFORE Pass 1 + prune, // so Pass 1 updates (not duplicates) them and pruneRecords does not treat them as orphans. if (naturalKeys && Object.keys(naturalKeys).length) { matchByNaturalKeys({ srcSnapshot, destSnapshot, naturalKeys, idmap, result }); - persist(idmap, journal); + persistRun(idmap, journal); } - // Dest row ids created THIS run — Pass 2/3 use it so conflicts=dest-wins only blocks - // writes to PRE-EXISTING mapped rows (created rows always get their links/attachments). - const createdDestIds = new Set(); - // Pass 1: scalar / select upsert — fills idmap.records - await applyRecordsPass1({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result, policy, policyOverrides, fieldMappings: resolved, createdDestIds }); - persist(idmap, journal); + await applyRecordsPass1({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist: persistRun, result, policy, policyOverrides, fieldMappings: resolved, createdDestIds }); + persistRun(idmap, journal); // Rebuild destDisplayNames from idmap.records now populated by Pass 1. // Key = dest record id, value = source primary cell value (string). @@ -1122,25 +1141,25 @@ export async function runRecords({ client, srcSnapshot, destSnapshot, idmap, pol } // Pass 2: link cells - await applyRecordsPass2({ client, srcSnapshot, destSnapshot, idmap, destDisplayNames, limiter, journal, persist, result, policy, policyOverrides, createdDestIds }); - persist(idmap, journal); + await applyRecordsPass2({ client, srcSnapshot, destSnapshot, idmap, destDisplayNames, limiter, journal, persist: persistRun, result, policy, policyOverrides, createdDestIds }); + persistRun(idmap, journal); // Pass 3: attachments - await applyAttachments({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result, policy, policyOverrides, createdDestIds }); - persist(idmap, journal); + await applyAttachments({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist: persistRun, result, policy, policyOverrides, createdDestIds }); + persistRun(idmap, journal); // Reapply view filters whose record refs now resolve. // Defensive guard: a filter-restore failure must NEVER prevent the prune step below — // otherwise one tail-step error discards completed passes and skips the extras axis. try { - await reapplyViewFilters({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist, result }); + await reapplyViewFilters({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist: persistRun, result }); } catch (err) { result.warnings.push({ code: 'VIEW_FILTER_REAPPLY_FAILED', message: `reapplyViewFilters failed (${err.message}) — continuing to prune`, }); } - persist(idmap, journal); + persistRun(idmap, journal); // Extras axis: prune dest-only orphan records under mirror policy const truncatedTables = collectTruncatedTableNames(srcSnapshot, destSnapshot); @@ -1177,7 +1196,7 @@ export async function runRecords({ client, srcSnapshot, destSnapshot, idmap, pol for (const r of (t.records || [])) liveSourceRecordIds.add(r.id); } await pruneRecords({ client, destSnapshot, idmap, policy, policyOverrides, confirmDeletions, limiter, result, truncatedTables, liveSourceRecordIds }); - persist(idmap, journal); + persistRun(idmap, journal); return result; } @@ -1198,8 +1217,10 @@ export async function runRecords({ client, srcSnapshot, destSnapshot, idmap, pol * * RESUME MODEL: resume is driven by the persisted **idmap.records** + a fresh live dest snapshot — * a created row is skipped on re-run because its source id is already mapped, and Pass 2 dedups - * links against the live dest. The records journal is persisted but currently advisory (no - * per-record done-gating); idmap + live re-snapshot are the source of truth. Known limitation: + * links against the live dest. The records journal persists `createdDestIds` (dest rows created + * this plan) so a resume of the SAME planId keeps them dest-wins-writable in Pass 2/3; beyond + * that it is advisory (no per-record done-gating) — idmap + live re-snapshot are the source of + * truth. Known limitation: * a crash between a create's server-ack and the per-chunk idmap persist can re-create up to one * chunk (~50) of rows as duplicates on resume (reconcile existence-prunes + natural-key * re-matches by value). @@ -1507,6 +1528,20 @@ export function matchByNaturalKeys({ srcSnapshot, destSnapshot, naturalKeys, idm * @returns {Promise} - { created, updated, skipped, failed, warnings, idmap } */ export async function reconcile({ client, sourceBaseId, destBaseId, naturalKeys = {} }) { + // Concurrency guard: reconcile is a load-modify-save of idmap.json — run concurrently with a + // background records job (mode=apply) on the same pair, both would last-writer-wins clobber + // the idmap. Same per-pair apply.lock as apply(): throws APPLY_LOCKED while a live holder + // exists; a stale lock (dead pid) is replaced. Released when reconcile settles (no + // background phase — reconcile is fully synchronous). + const releaseLock = acquireApplyLock(sourceBaseId, destBaseId, 'reconcile'); + try { + return await reconcileLocked({ client, sourceBaseId, destBaseId, naturalKeys }); + } finally { + releaseLock(); + } +} + +async function reconcileLocked({ client, sourceBaseId, destBaseId, naturalKeys }) { const idmap = loadIdmap(sourceBaseId, destBaseId); idmap.records ??= {}; diff --git a/packages/mcp-server/test/sync/test-reconcile-lock.test.js b/packages/mcp-server/test/sync/test-reconcile-lock.test.js new file mode 100644 index 0000000..f5e58e5 --- /dev/null +++ b/packages/mcp-server/test/sync/test-reconcile-lock.test.js @@ -0,0 +1,73 @@ +// Regression tests: mode=reconcile must take the same per-pair apply.lock as mode=apply — +// run concurrently with a background records job it does a load-modify-save of idmap.json +// (last-writer-wins clobber). reconcile() must refuse while a LIVE lock exists, replace a +// stale lock (dead pid), and release the lock on both success and failure. +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, mkdirSync, writeFileSync, existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { reconcile } from '../../src/sync/records.js'; +import { saveIdmap, loadIdmap, syncDir } from '../../src/sync/idmap.js'; + +const SRC = 'appSrcLkXXXXXXXXX'; +const DEST = 'appDstLkXXXXXXXXX'; + +// One dest table whose snapshot holds a single live record (recLive) — a mapping to any +// other dest id is stale and existence-pruned by reconcile. +function destClient() { + const tables = [{ + id: 'tblD', name: 'T', + primaryColumnId: 'fldD', + columns: [{ id: 'fldD', name: 'Name', type: 'text', typeOptions: null, description: null }], + views: [{ id: 'vD', name: 'Grid view', type: 'grid', personalForUserId: null }], + }]; + return { + getApplicationData: async () => ({ data: { tableSchemas: JSON.parse(JSON.stringify(tables)) } }), + queryRecords: async () => ({ summary: { rows: [{ id: 'recLive', fields: {} }] } }), + getView: async () => ({}), + }; +} + +function writeLock(lock) { + mkdirSync(syncDir(SRC, DEST), { recursive: true }); + writeFileSync(join(syncDir(SRC, DEST), 'apply.lock'), JSON.stringify(lock)); +} + +describe('reconcile — per-pair apply lock', () => { + it('refuses with APPLY_LOCKED while a live lock exists (idmap untouched)', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'reconcile-lock-live-')); + saveIdmap(SRC, DEST, { tables: { tblS: 'tblD' }, fields: {}, records: { recSGone: 'recStaleGone' }, views: {} }); + // A live holder: our own pid is definitionally alive. + writeLock({ pid: process.pid, planId: 'plnJOB', startedAt: 'ts0' }); + + await assert.rejects( + () => reconcile({ client: destClient(), sourceBaseId: SRC, destBaseId: DEST }), + (e) => e.code === 'APPLY_LOCKED' && /plnJOB/.test(e.message), + 'reconcile must refuse while a live apply lock exists', + ); + const after = loadIdmap(SRC, DEST); + assert.equal(after.records.recSGone, 'recStaleGone', 'idmap must not be modified while locked'); + }); + + it('replaces a stale lock (dead pid), reconciles, and releases the lock', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'reconcile-lock-stale-')); + saveIdmap(SRC, DEST, { tables: { tblS: 'tblD' }, fields: {}, records: { recSGone: 'recStaleGone', recSLive: 'recLive' }, views: {} }); + // A pid that cannot exist → process.kill(pid, 0) throws → stale. + writeLock({ pid: 2147483646, planId: 'plnDEAD', startedAt: 'ts0' }); + + const result = await reconcile({ client: destClient(), sourceBaseId: SRC, destBaseId: DEST }); + assert.equal(result.skipped, 1, 'stale mapping existence-pruned'); + const after = loadIdmap(SRC, DEST); + assert.equal(after.records.recSGone, undefined, 'stale mapping dropped'); + assert.equal(after.records.recSLive, 'recLive', 'live mapping kept'); + assert.ok(!existsSync(join(syncDir(SRC, DEST), 'apply.lock')), 'lock released after reconcile'); + }); + + it('releases the lock when reconcile throws', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'reconcile-lock-err-')); + const throwing = { getApplicationData: async () => { throw new Error('boom'); } }; + await assert.rejects(() => reconcile({ client: throwing, sourceBaseId: SRC, destBaseId: DEST }), /boom/); + assert.ok(!existsSync(join(syncDir(SRC, DEST), 'apply.lock')), 'lock released after a throw'); + }); +}); diff --git a/packages/mcp-server/test/sync/test-records-resume-created.test.js b/packages/mcp-server/test/sync/test-records-resume-created.test.js new file mode 100644 index 0000000..a05373d --- /dev/null +++ b/packages/mcp-server/test/sync/test-records-resume-created.test.js @@ -0,0 +1,115 @@ +/** + * test-records-resume-created.test.js + * + * A RESUMED records job (same planId after a crash) must still treat rows created by the + * crashed run as sync-created: Pass 2 (links) / Pass 3 (attachments) key their dest-wins + * protection on the created-this-plan dest ids. On resume those rows are already in + * idmap.records, take Pass 1's dest-wins skip, and never re-enter the run-local set — so the + * set is persisted in the records journal (journal.createdDestIds) and seeded from it on + * resume of the SAME planId. Cross-plan behavior is unchanged (a new plan gets a new journal). + */ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { runRecords } from '../../src/sync/records.js'; + +const noLimiter = { run: (fn) => fn() }; + +function fixtures() { + const linkCalls = []; + const pastes = []; + const client = { + async createRecords(appId, tableId, rows) { + return { created: rows.map((r) => ({ rowId: 'recNew_' + r.sourceKey, sourceKey: r.sourceKey })), failed: [] }; + }, + async updateRecords(appId, tableId, rows) { return { updated: rows, failed: [] }; }, + async addLinkItems(appId, rowId, fldId, items) { linkCalls.push({ rowId, fldId }); return { ok: true, added: items.length }; }, + async createDataTransferPolicy() { return { policy: 'signed' }; }, + async pasteAttachmentsCrossBase(appId, tableId, payload) { pastes.push(payload); return { pastedRowIds: payload.targetRowIds }; }, + }; + // recS1 was CREATED by the crashed run (already mapped → recD1, present on dest with empty + // link/attachment cells). recSTarget is its link target (pre-existing, mapped). + const srcSnapshot = { + baseId: 'appS', + tables: [{ + id: 'tS', name: 'Games', primaryFieldId: 'sN', + fields: [ + { id: 'sN', name: 'Name', type: 'text' }, + { id: 'sL', name: 'Link', type: 'foreignKey', typeOptions: { foreignTableId: 'tS' } }, + { id: 'sA', name: 'Att', type: 'multipleAttachments' }, + ], + views: [], + records: [ + { + id: 'recS1', + cellValuesByColumnId: { + sN: 'one', + sL: ['recSTarget'], + sA: [{ id: 'att1', url: 'https://src/a.png', filename: 'a.png', size: 10, type: 'image/png' }], + }, + }, + { id: 'recSTarget', cellValuesByColumnId: { sN: 'target' } }, + ], + }], + }; + const destSnapshot = { + baseId: 'appD', + tables: [{ + id: 'tD', name: 'Games', views: [{ id: 'viwD' }], fields: [], + records: [ + { id: 'recD1', cellValuesByColumnId: { dN: 'one' } }, // created by the crashed run + { id: 'recDTarget', cellValuesByColumnId: { dN: 'target' } }, + ], + }], + }; + const idmap = { + tables: { tS: 'tD' }, + fields: { sN: { destFld: 'dN', choices: {} }, sL: { destFld: 'dL' }, sA: { destFld: 'dA' } }, + records: { recS1: 'recD1', recSTarget: 'recDTarget' }, + }; + return { linkCalls, pastes, client, srcSnapshot, destSnapshot, idmap }; +} + +describe('runRecords — resumed job keeps created-this-plan rows sync-created (dest-wins)', () => { + it('seeds createdDestIds from journal.createdDestIds so Pass 2/3 still write to crashed-run rows under preserve', async () => { + const { linkCalls, pastes, client, srcSnapshot, destSnapshot, idmap } = fixtures(); + // Journal persisted by the crashed run of the SAME planId: recD1 was created this plan. + const journal = { planId: 'plnR', startedAt: 't', actions: [], createdDestIds: ['recD1'] }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, deleted: 0, warnings: [] }; + await runRecords({ + client, srcSnapshot, destSnapshot, idmap, + policy: 'preserve', + limiter: noLimiter, journal, persist: () => {}, result, + }); + assert.ok(linkCalls.some((c) => c.rowId === 'recD1'), 'Pass 2 writes links to the crashed-run-created row'); + assert.ok(pastes.some((p) => p.targetRowIds.includes('recD1')), 'Pass 3 pastes attachments to the crashed-run-created row'); + }); + + it('cross-plan unchanged: a journal without createdDestIds still protects pre-existing rows under preserve', async () => { + const { linkCalls, pastes, client, srcSnapshot, destSnapshot, idmap } = fixtures(); + const journal = { planId: 'plnNew', startedAt: 't', actions: [] }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, deleted: 0, warnings: [] }; + await runRecords({ + client, srcSnapshot, destSnapshot, idmap, + policy: 'preserve', + limiter: noLimiter, journal, persist: () => {}, result, + }); + assert.equal(linkCalls.length, 0, 'pre-existing mapped rows still protected under preserve'); + assert.equal(pastes.length, 0, 'no attachment paste over a pre-existing mapped row'); + }); + + it('persists created dest ids into the journal on every per-chunk persist (crash-durable)', async () => { + const { client, srcSnapshot, destSnapshot, idmap } = fixtures(); + delete idmap.records.recS1; // recS1 not yet mapped → Pass 1 creates it this run + const snapshots = []; + const journal = { planId: 'plnFresh', startedAt: 't', actions: [] }; + const persist = (m, j) => { snapshots.push((j.createdDestIds || []).slice()); }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, deleted: 0, warnings: [] }; + await runRecords({ + client, srcSnapshot, destSnapshot, idmap, + policy: 'preserve', + limiter: noLimiter, journal, persist, result, + }); + assert.ok(snapshots.some((s) => s.includes('recNew_recS1')), 'a persist during the run captured the created id'); + assert.ok(journal.createdDestIds.includes('recNew_recS1'), 'journal carries the created id after the run'); + }); +}); diff --git a/packages/mcp-server/test/sync/test-records.test.js b/packages/mcp-server/test/sync/test-records.test.js index 27253e3..20702cf 100644 --- a/packages/mcp-server/test/sync/test-records.test.js +++ b/packages/mcp-server/test/sync/test-records.test.js @@ -294,6 +294,64 @@ describe('records.applyRecordsPass1', () => { assert.ok(!('fldFD' in sentCells), 'computed field must never be cleared'); }); + it('does not clear a cell whose DEST field is computed (scalar-source/computed-dest pair)', async () => { + // A name-matched field can be scalar on SOURCE but computed on DEST (persistent + // RETYPE_DEFERRED state). updatePrimitiveCell rejects computed writes and one failing + // cell poisons the whole row update — the clear must be skipped for a computed dest field. + const updateCalls = []; + const client = { + createRecords: async () => ({ created: [], failed: [] }), + updateRecords: async (appId, tableId, rows) => { + updateCalls.push({ rows }); + return { updated: rows.map((r) => r.rowId), failed: [] }; + }, + }; + const idmap = { + tables: { tblS: 'tblD' }, + fields: { + fldT: { destFld: 'fldFD', choices: {} }, // scalar source → FORMULA dest (retype deferred) + fldN: { destFld: 'fldND', choices: {} }, + }, + records: { recS1: 'recD1' }, + }; + const srcSnapshot = { + tables: [{ + id: 'tblS', name: 'T', + fields: [ + { id: 'fldT', type: 'text' }, // scalar on SOURCE + { id: 'fldN', type: 'number' }, + ], + // fldT cleared on source (absent key); fldN still has a value. + records: [{ id: 'recS1', cellValuesByColumnId: { fldN: 42 } }], + }], + }; + const destSnapshot = { + tables: [{ + id: 'tblD', name: 'T', + fields: [ + { id: 'fldFD', type: 'formula' }, // COMPUTED on DEST + { id: 'fldND', type: 'number' }, + ], + records: [{ + id: 'recD1', + cellValuesByColumnId: { fldFD: 'computed output', fldND: 41 }, + }], + }], + }; + const result = { created: 0, updated: 0, skipped: 0, failed: 0, warnings: [] }; + await applyRecordsPass1({ + client, srcSnapshot, destSnapshot, idmap, + limiter: createLimiter({ rps: 1000, sleep: async () => {} }), + journal: newJournal('p3dc', 't'), + persist: () => {}, + result, + }); + assert.equal(updateCalls.length, 1, 'other cells still update'); + const sentCells = updateCalls[0].rows[0].cellValuesByColumnId; + assert.ok(!('fldFD' in sentCells), 'no null emitted for a computed DEST field'); + assert.equal(sentCells.fldND, 42, 'sibling scalar cell still updated'); + }); + it('does not emit clears when the dest cell is already empty or the dest record is unknown', async () => { const updateCalls = []; const client = { From b6fa5f90ad4a2fcfb9ac82481ed18e564c2eede6 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 15:27:17 +0300 Subject: [PATCH 167/246] =?UTF-8?q?fix(sync):=20apply-engine=20seams=20?= =?UTF-8?q?=E2=80=94=20(name,type)=20view=20adoption,=20deferred=20compute?= =?UTF-8?q?d=20update,=20resumable=20gated=20primary=20rename,=20writable?= =?UTF-8?q?=20link=20update?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial-review follow-ups on the apply engine (all reproduced by failing tests first, test/sync/test-apply-review-seams.test.js): - A1: createView adopted a same-named dest view of a DIFFERENT type (diff.js matches by (name, type) and orphans the mismatch) — source kanban config was written onto a grid view, idmap.views pointed at it, and pruneSchema under mirror+confirmDeletions deleted the just-configured view. The view index is now keyed by (name, type); a type-mismatched name creates a genuinely new view of the source type. - A2: updateField's computed-typeOptions branch with unresolved refs warned UNRESOLVABLE_REF but returned undefined -> journaled done, never retried. Now returns 'deferred' like createField (retried in-run, retryable across runs). - A3: a gated primary retype whose accompanying rename DID apply mutated the dest fingerprint with zero done journal actions, so the prescribed follow-up (same plan + confirmRetypes:true) hit the DRIFT abort. The applied rename is journaled as a done sub-action (string idx, upsert-idempotent) so the resume bypass is reachable while the retype stays retryable. - A4: a genuine link divergence sent mergeChoices(remapRefs(...)) keeping the read-only 'unreversed' key -> 422 on every apply, never converged. Link typeOptions now go through the same writableLinkOptions shaping as the createField link path. Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/apply.js | 47 ++++- .../test/sync/test-apply-review-seams.test.js | 182 ++++++++++++++++++ 2 files changed, 219 insertions(+), 10 deletions(-) create mode 100644 packages/mcp-server/test/sync/test-apply-review-seams.test.js diff --git a/packages/mcp-server/src/sync/apply.js b/packages/mcp-server/src/sync/apply.js index cbbe048..4565fea 100644 --- a/packages/mcp-server/src/sync/apply.js +++ b/packages/mcp-server/src/sync/apply.js @@ -20,6 +20,10 @@ const ILLEGAL_PRIMARY_TYPES = new Set([ 'asyncText', 'aiText', 'checkbox', ]); +// (name, type) view identity — mirrors diff.js viewKey. A view's TYPE is immutable in +// Airtable, so a same-named dest view of a different type is never an adoption counterpart. +function viewKey(name, type) { return `${name}|${type}`; } + function buildIndex(snap) { const tablesById = new Map(); const tablesByName = new Map(); @@ -27,7 +31,7 @@ function buildIndex(snap) { const entry = { id: t.id, name: t.name, primaryFieldId: t.primaryFieldId, fieldsByName: new Map(t.fields.map((f) => [f.name, { id: f.id, name: f.name, type: f.type, typeOptions: f.typeOptions }])), - viewsByName: new Map((t.views || []).map((v) => [v.name, { id: v.id, type: v.type }])), + viewsByKey: new Map((t.views || []).map((v) => [viewKey(v.name, v.type), { id: v.id, type: v.type }])), }; tablesById.set(t.id, entry); tablesByName.set(t.name, entry); @@ -137,7 +141,7 @@ export async function applyPlan({ client, plan, destAppId, destSnapshot, idmap, const attempt = async (idx, a) => { const warnStart = result.warnings.length; try { - const disposition = await applyAction({ client, destAppId, a, idmap, index, state, result, confirmRetypes }); + const disposition = await applyAction({ client, destAppId, a, idmap, index, state, result, confirmRetypes, journal, idx }); if (disposition === 'deferred') return { status: 'deferred', warns: result.warnings.splice(warnStart) }; if (disposition === 'gated') { result.skipped++; persist(idmap, journal); return { status: 'gated' }; } recordDone(journal, idx, a.kind, idmap.tables[a.sourceTableId] ?? (a.sourceFieldId && idmap.fields[a.sourceFieldId]?.destFld)); @@ -188,7 +192,7 @@ export async function applyPlan({ client, plan, destAppId, destSnapshot, idmap, return result; } -async function applyAction({ client, destAppId, a, idmap, index, state, result, confirmRetypes }) { +async function applyAction({ client, destAppId, a, idmap, index, state, result, confirmRetypes, journal, idx }) { switch (a.kind) { case 'createTable': { const existing = index.tablesByName.get(a.name); @@ -218,7 +222,7 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result, id: tableId, name: a.name, primaryFieldId: primaryId, fieldsByName: new Map([[primary.name, { id: primaryId, name: primary.name, type: primary.type, typeOptions: primary.typeOptions ?? null }]]), // include the auto-created default view(s) so a same-run createView adopts (not duplicates) them - viewsByName: new Map((views || []).map((v) => [v.name, { id: v.id, type: v.type }])), + viewsByKey: new Map((views || []).map((v) => [viewKey(v.name, v.type), { id: v.id, type: v.type }])), }; index.tablesById.set(tableId, entry); index.tablesByName.set(a.name, entry); @@ -275,7 +279,17 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result, if (renamed) { entry.fieldsByName.delete(primary.name); primary.name = a.toName; entry.fieldsByName.set(a.toName, primary); } if (retyped) primary.type = a.toType; } - if (disposition) return disposition; // gated/deferred retype: NOT journaled done → retryable + if (disposition) { + // A gated/deferred retype whose accompanying RENAME did apply has already mutated the + // dest fingerprint while journaling ZERO done actions — the prescribed follow-up (same + // plan + confirmRetypes:true) then hit the DRIFT abort (the resume bypass needs ≥1 done + // journal action). Journal the applied rename as a done SUB-action: the string idx never + // collides with isDone()'s integer plan indices, and upsert keeps re-recording idempotent + // (on resume the name already matches → renamed stays false). The retype itself stays + // retryable (main idx NOT journaled done). + if (renamed) recordDone(journal, `${idx}:rename`, 'reconcilePrimary:rename', primaryId); + return disposition; // gated/deferred retype: NOT journaled done → retryable + } if (renamed || retyped) result.updated++; else result.skipped++; return; } @@ -394,15 +408,24 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result, if (COMPUTED_TYPES.has(destType)) { // Computed updates need the writable shape (drop read-only formulaTextParsed/ // dependencies/resultType — the API 422s on them). Defer if a referenced field - // isn't created yet (updates run before creates); a re-plan after creates converges. + // isn't created yet (updates run before creates) — same disposition as createField: + // retried later this run, NOT journaled done otherwise (a plain return journaled it + // done and the update was never retried). const refs = (changes.typeOptions.dependencies && changes.typeOptions.dependencies.referencedColumnIdsForValue) || []; const unresolved = refs.filter((d) => !(idmap.fields[d] && idmap.fields[d].destFld)); if (unresolved.length) { - result.warnings.push({ code: 'UNRESOLVABLE_REF', message: `Update of "${a.destFld}" typeOptions deferred — refs [${unresolved.join(', ')}] not yet created (re-plan after creates)` }); + result.warnings.push({ code: 'UNRESOLVABLE_REF', message: `Update of "${a.destFld}" typeOptions deferred — refs [${unresolved.join(', ')}] not yet created` }); + return 'deferred'; } else { await client.updateFieldConfig(destAppId, a.destFld, { type: destType, typeOptions: toWritableComputedOptions(destType, remapped) }); mutated = true; } + } else if (LINK_TYPES.has(destType)) { + // Genuine link divergence (linkSig differs): send the WRITABLE link shape — same + // shaping as the createField link path. The raw remapped options still carry the + // read-only 'unreversed' key, so the update 422'd on every apply and never converged. + await client.updateFieldConfig(destAppId, a.destFld, { type: destType, typeOptions: writableLinkOptions(remapped) }); + mutated = true; } else { await client.updateFieldConfig(destAppId, a.destFld, { type: destType, typeOptions: mergeChoices(findDestField(index, a.destFld), remapped) }); mutated = true; @@ -416,12 +439,16 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result, case 'createView': { const destTableId = idmap.tables[a.sourceTableId]; const entry = index.tablesById.get(destTableId); - const existing = entry && entry.viewsByName.get(a.name); + // Adopt only a same-name AND same-type dest view (mirrors diff.js's (name, type) match): + // name-only adoption grabbed a same-named view of a DIFFERENT immutable type, wrote the + // source config onto it, pointed idmap.views at it — and under mirror+confirmDeletions + // pruneSchema then deleted that very view (diff orphans it by (name, type)). + const existing = entry && entry.viewsByKey.get(viewKey(a.name, a.type)); if (existing) { idmap.views[a.sourceViewId] = existing.id; result.skipped++; return; } - const template = entry && [...entry.viewsByName.values()][0]; // dest table always has ≥1 view + const template = entry && [...entry.viewsByKey.values()][0]; // dest table always has ≥1 view const { viewId } = await client.createView(destAppId, destTableId, { name: a.name, type: a.type, copyFromViewId: template ? template.id : undefined }); idmap.views[a.sourceViewId] = viewId; - if (entry) entry.viewsByName.set(a.name, { id: viewId, type: a.type }); + if (entry) entry.viewsByKey.set(viewKey(a.name, a.type), { id: viewId, type: a.type }); result.created++; return; } diff --git a/packages/mcp-server/test/sync/test-apply-review-seams.test.js b/packages/mcp-server/test/sync/test-apply-review-seams.test.js new file mode 100644 index 0000000..caa1402 --- /dev/null +++ b/packages/mcp-server/test/sync/test-apply-review-seams.test.js @@ -0,0 +1,182 @@ +// Seam regressions from the adversarial review of the hardening batch: +// A1 — createView must adopt by (name, type), not name alone. diff.js matches views by +// (name, type) and orphans a same-named dest view of a DIFFERENT type; name-only +// adoption wrote kanban config onto a grid view, pointed idmap.views at the wrong +// view, and under mirror+confirmDeletions pruneSchema deleted the just-configured view. +// A2 — updateField's computed-typeOptions branch with unresolved refs must take the +// 'deferred' disposition (retried in-run; NOT journaled done) like createField. +// A3 — a gated primary retype whose accompanying RENAME did apply mutates the dest +// fingerprint with zero done journal actions — the prescribed follow-up (same plan + +// confirmRetypes:true) then hit the DRIFT abort. The applied rename is journaled as a +// done sub-action so the resume bypass is reachable. +// A4 — a GENUINE link divergence (linkSig differs) must send the WRITABLE link shape — +// the raw remapped options carry read-only keys (unreversed) that 422 on every apply. +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { MockClient } from './helpers/mock-client.js'; +import { applyPlan } from '../../src/sync/apply.js'; +import { newJournal, isDone } from '../../src/sync/journal.js'; +import { snapshotBase } from '../../src/sync/snapshot.js'; +import { fingerprintSchema } from '../../src/sync/index.js'; + +function run(client, plan, destSnapshot, { journal, idmap, confirmRetypes } = {}) { + return applyPlan({ + client, plan, destAppId: plan.destBaseId, destSnapshot, + idmap: idmap ?? JSON.parse(JSON.stringify(plan.idmap)), + journal: journal ?? newJournal(plan.planId, 'ts'), + persist: () => {}, + confirmRetypes, + }); +} + +describe('apply: createView adoption requires matching TYPE, not just name (A1)', () => { + it('source kanban vs same-named dest grid → creates a NEW kanban; idmap points at the kanban, never the grid', async () => { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'T'); + const { viewId: gridBoardId } = await client.createView('appD', tableId, { name: 'Board', type: 'grid' }); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { planId: 'plnA1', sourceBaseId: 'appS', destBaseId: 'appD', idmap: { tables: { tS: tableId }, fields: {}, views: {} }, + actions: [{ kind: 'createView', sourceTableId: 'tS', sourceViewId: 'vK', name: 'Board', type: 'kanban' }], + orphans: [], warnings: [] }; + const res = await run(client, plan, destSnapshot); + assert.equal(res.failed, 0); + assert.equal(res.created, 1, 'a genuinely new view of the SOURCE type must be created'); + const views = (await client.getApplicationData('appD')).data.tableSchemas[0].views; + const boards = views.filter((v) => v.name === 'Board'); + assert.equal(boards.length, 2, 'grid Board left in place (it is the orphan pruneSchema owns), kanban Board created'); + const kanban = boards.find((v) => v.type === 'kanban'); + assert.ok(kanban, 'created view carries the source type'); + assert.equal(res.idmap.views.vK, kanban.id, 'idmap.views must point at the kanban'); + assert.notEqual(res.idmap.views.vK, gridBoardId, 'idmap.views must never point at the same-named grid'); + }); + + it('same name AND same type is still adopted (no duplicate view)', async () => { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'T'); + const { viewId: boardId } = await client.createView('appD', tableId, { name: 'Board', type: 'kanban' }); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { planId: 'plnA1b', sourceBaseId: 'appS', destBaseId: 'appD', idmap: { tables: { tS: tableId }, fields: {}, views: {} }, + actions: [{ kind: 'createView', sourceTableId: 'tS', sourceViewId: 'vK', name: 'Board', type: 'kanban' }], + orphans: [], warnings: [] }; + const res = await run(client, plan, destSnapshot); + assert.equal(res.failed, 0); + assert.equal(res.created, 0); + assert.equal(res.idmap.views.vK, boardId, 'matching (name, type) adopts the existing view'); + const views = (await client.getApplicationData('appD')).data.tableSchemas[0].views; + assert.equal(views.filter((v) => v.name === 'Board').length, 1, 'no duplicate created'); + }); +}); + +describe('apply: updateField computed typeOptions with unresolved refs is deferred, not done (A2)', () => { + const updateAction = (calcId) => ({ kind: 'updateField', sourceFieldId: 'sCalc', destFld: calcId, + changes: { typeOptions: { formulaTextParsed: '{column_value_fldX}', dependencies: { referencedColumnIdsForValue: ['fldX'] } } } }); + + it('unresolved ref → UNRESOLVABLE_REF, NOT journaled done; re-run with the ref mapped applies the update', async () => { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'T'); + const { columnId: calcId } = await client.createField('appD', tableId, { name: 'Calc', type: 'formula', typeOptions: { formulaText: '1' } }); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { planId: 'plnA2', sourceBaseId: 'appS', destBaseId: 'appD', idmap: { tables: { tS: tableId }, fields: {} }, + actions: [updateAction(calcId)], orphans: [], warnings: [] }; + const journal = newJournal('plnA2', 'ts'); + + const r1 = await run(client, plan, destSnapshot, { journal, idmap: { tables: { tS: tableId }, fields: {} } }); + assert.equal(r1.failed, 0); + assert.ok(r1.warnings.some((w) => w.code === 'UNRESOLVABLE_REF')); + assert.equal(isDone(journal, 0), false, 'deferred computed update must NOT be journaled done'); + + // Run 2: the ref is now mapped (created by a later run) — the SAME journal must retry it. + const { columnId: xId } = await client.createField('appD', tableId, { name: 'X', type: 'number' }); + const r2 = await run(client, plan, await snapshotBase(client, 'appD'), + { journal, idmap: { tables: { tS: tableId }, fields: { fldX: { destFld: xId, choices: {} } } } }); + assert.equal(r2.updated, 1, 're-run must apply the computed update once the ref resolves'); + assert.equal(client._field(calcId).typeOptions.formulaText, `{${xId}}`, 'refs remapped + writable shape'); + }); + + it('in-run deferral: the ref is created LATER in the same plan → retried and applied in one run', async () => { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'T'); + const { columnId: calcId } = await client.createField('appD', tableId, { name: 'Calc', type: 'formula', typeOptions: { formulaText: '1' } }); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { planId: 'plnA2b', sourceBaseId: 'appS', destBaseId: 'appD', idmap: { tables: { tS: tableId }, fields: {} }, + actions: [ + // Dependency-inverted order: the update appears BEFORE the field it references. + updateAction(calcId), + { kind: 'createField', sourceTableId: 'tS', sourceFieldId: 'fldX', name: 'X', type: 'number', typeOptions: { precision: 0 }, description: null, computed: false, dependsOn: [], dependsOnTables: [] }, + ], orphans: [], warnings: [] }; + const res = await run(client, plan, destSnapshot, { idmap: { tables: { tS: tableId }, fields: {} } }); + assert.equal(res.failed, 0); + assert.equal(res.updated, 1, 'deferred update must be retried after its ref was created this run'); + assert.ok(!res.warnings.some((w) => w.code === 'UNRESOLVABLE_REF'), 'resolved deferral must not surface a warning'); + const destXId = res.idmap.fields.fldX.destFld; + assert.equal(client._field(calcId).typeOptions.formulaText, `{${destXId}}`); + }); +}); + +describe('apply: gated primary retype with an applied rename stays resumable (A3)', () => { + it('journals the rename as a done sub-action so the same-plan confirmRetypes:true follow-up survives the drift guard', async () => { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'T'); // pre-existing table → retype gated + const destSnapshot = await snapshotBase(client, 'appD'); + const fpBefore = fingerprintSchema(destSnapshot); + const plan = { planId: 'plnA3', sourceBaseId: 'appS', destBaseId: 'appD', idmap: { tables: { tS: tableId }, fields: {} }, + actions: [{ kind: 'reconcilePrimary', sourceTableId: 'tS', sourcePrimaryFieldId: 'fS1', toName: 'Title', toType: 'number', toTypeOptions: { format: 'integer' } }], + orphans: [], warnings: [] }; + const journal = newJournal('plnA3', 'ts'); + + const r1 = await run(client, plan, destSnapshot, { journal, idmap: { tables: { tS: tableId }, fields: {} }, confirmRetypes: false }); + assert.ok(r1.warnings.some((w) => w.code === 'RETYPE_GATED')); + const primary = (await client.getApplicationData('appD')).data.tableSchemas[0].columns[0]; + assert.equal(primary.name, 'Title', 'rename applied'); + assert.equal(primary.type, 'text', 'retype gated — no mutation'); + + // The rename mutated the dest fingerprint... + const snapshot2 = await snapshotBase(client, 'appD'); + assert.notEqual(fingerprintSchema(snapshot2), fpBefore, 'precondition: the applied rename changes the fingerprint'); + // ...so the prescribed follow-up only survives the drift guard when the journal shows + // prior progress (index.js: resuming = journal.actions.some((a) => a.status === 'done')). + assert.equal(isDone(journal, 0), false, 'the gated retype itself must stay retryable'); + assert.ok(journal.actions.some((e) => e.status === 'done'), + 'the applied rename must be journaled as a done sub-action → resume bypass reachable'); + + // Idempotent on resume: re-running the same gated plan must not duplicate the sub-action. + await run(client, plan, snapshot2, { journal, idmap: { tables: { tS: tableId }, fields: {} }, confirmRetypes: false }); + assert.equal(journal.actions.filter((e) => e.status === 'done').length, 1, 'sub-action recorded once (upsert)'); + + // Follow-up with confirmRetypes:true on the SAME journal performs the retype. + const r2 = await run(client, plan, snapshot2, { journal, idmap: { tables: { tS: tableId }, fields: {} }, confirmRetypes: true }); + assert.equal(r2.updated, 1); + const after = (await client.getApplicationData('appD')).data.tableSchemas[0].columns[0]; + assert.equal(after.type, 'number', 'follow-up run performs the retype'); + assert.equal(isDone(journal, 0), true, 'retype now journaled done'); + }); +}); + +describe('apply: genuine link divergence sends WRITABLE link options (A4)', () => { + it('drops read-only unreversed/symmetricColumnId and remaps foreignTableId before updateFieldConfig', async () => { + const client = new MockClient(); + const a = await client.createTable('appD', 'A'); + const b = await client.createTable('appD', 'B'); + const c = await client.createTable('appD', 'C'); + // Dest link points at B; source (diverged) points at C — linkSig differs → updateField. + const { columnId: linkId } = await client.createField('appD', a.tableId, { name: 'Link', type: 'foreignKey', typeOptions: { foreignTableId: b.tableId, relationship: 'many' } }); + const destSnapshot = await snapshotBase(client, 'appD'); + const captured = []; + const orig = client.updateFieldConfig.bind(client); + client.updateFieldConfig = async (appId, colId, cfg) => { captured.push(cfg); return orig(appId, colId, cfg); }; + const plan = { planId: 'plnA4', sourceBaseId: 'appS', destBaseId: 'appD', + idmap: { tables: { tC: c.tableId }, fields: {} }, + actions: [{ kind: 'updateField', sourceFieldId: 'sLink', destFld: linkId, + // Raw SOURCE typeOptions as diff.js emits them: carry read-only unreversed + a + // source-base symmetricColumnId that must never reach the dest API. + changes: { typeOptions: { foreignTableId: 'tC', relationship: 'many', symmetricColumnId: 'fldSrcRev', unreversed: true } } }], + orphans: [], warnings: [] }; + const res = await run(client, plan, destSnapshot); + assert.equal(res.failed, 0); + assert.equal(res.updated, 1); + const cfg = captured.find((x) => x.type === 'foreignKey'); + assert.ok(cfg, 'link update sent as a link-typed config'); + assert.deepEqual(cfg.typeOptions, { foreignTableId: c.tableId, relationship: 'many' }, + 'writable link shape only — unreversed/symmetricColumnId dropped, foreignTableId remapped to the DEST table id'); + }); +}); From c98f5abef327c648dbe30ad5798bb66ffad5380d Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 15:36:17 +0300 Subject: [PATCH 168/246] docs: update tool count to 71, document upload_attachment and daemon port setting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - MCP server now provides 71 tools (was 62/70) — upload_attachment added to record-write category - upload_attachment: writes attachment cells by URL (Airtable server-side fetch), distinct from update_records which cannot set attachment cells - New mcp.daemonPort setting (default 8723): fixed TCP port for the shared daemon HTTP server; 0 = automatic/ephemeral; falls back to automatic if the chosen port is busy; takes --- CLAUDE.md | 7 +- packages/extension/package.json | 19 ++- packages/extension/src/mcp/daemon-manager.ts | 7 +- packages/extension/src/mcp/tool-profile.ts | 1 + packages/extension/src/settings.ts | 3 +- .../src/webview/DashboardProvider.ts | 6 +- packages/mcp-server/CHANGELOG.md | 24 +++ packages/mcp-server/src/attachments.js | 40 +++++ packages/mcp-server/src/client.js | 60 +++++++ packages/mcp-server/src/daemon/server.js | 44 +++-- .../cloudflared-named-setup.js | 7 + .../tunnel-providers/cloudflared-named.js | 49 +++++- packages/mcp-server/src/index.js | 43 +++++ packages/mcp-server/src/tool-config.js | 1 + .../test/test-named-tunnel-readiness.test.js | 153 ++++++++++++++++++ .../mcp-server/test/test-tool-config.test.js | 14 +- .../test/test-upload-attachment.test.js | 149 +++++++++++++++++ packages/shared/src/types.ts | 1 + packages/webview/src/store.ts | 5 +- packages/webview/src/tabs/Settings.tsx | 17 ++ packages/webview/src/test/store.test.ts | 2 +- 21 files changed, 610 insertions(+), 42 deletions(-) create mode 100644 packages/mcp-server/CHANGELOG.md create mode 100644 packages/mcp-server/src/attachments.js create mode 100644 packages/mcp-server/test/test-named-tunnel-readiness.test.js create mode 100644 packages/mcp-server/test/test-upload-attachment.test.js diff --git a/CLAUDE.md b/CLAUDE.md index 5ea7ee2..c70a83c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -47,7 +47,7 @@ Typed message protocol between extension host and webview. Exports `ExtensionMes React 19 + Vite 6 + Tailwind CSS v4 + Zustand 5 dashboard. Three tabs: Overview, Setup, Settings. Builds directly into `packages/extension/dist/webview/`. Communicates with the extension host via `acquireVsCodeApi().postMessage()` — messages are typed through the shared package. ### packages/mcp-server -The Airtable MCP server itself — ES modules Node app, **published to npm as `airtable-user-mcp`**. Provides **62 tools** across 12 categories (read, table-write, table-destructive, field-write, field-destructive, view-write, view-destructive, view-section, view-section-destructive, form-write, extension, tool-management) via `@modelcontextprotocol/sdk`. Uses `patchright` (Chromium stealth fork) with a persistent profile for browser-based authentication against Airtable's internal API. +The Airtable MCP server itself — ES modules Node app, **published to npm as `airtable-user-mcp`**. Provides **71 tools** across 12 categories (read, table-write, table-destructive, field-write, field-destructive, view-write, view-destructive, view-section, view-section-destructive, form-write, extension, tool-management) via `@modelcontextprotocol/sdk`. Includes `upload_attachment` (record-write) which writes attachment cells by URL — the general `update_records` tool cannot set attachment cells. Uses `patchright` (Chromium stealth fork) with a persistent profile for browser-based authentication against Airtable's internal API. Standalone users install via `npx airtable-user-mcp` or `npm i -g airtable-user-mcp`. The CLI exposes subcommands: `login`, `logout`, `status`, `doctor`, `install-browser`, `daemon start/stop/status`. Config and session data live in `~/.airtable-user-mcp/`. @@ -65,7 +65,7 @@ Key files: - `dev-tools/` — reverse-engineering helpers (capture.js, analyze.js, debug-*.js), **gitignored** — kept locally for future reference when Airtable changes their internal API - `src/daemon/` — daemon subsystem (see daemon subsection below) - `src/safe-write.js` — atomic JSON write helper (used by lockfile, token, and settings) -- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal). Exposed via the `sync_base` tool (`mode: diff | plan | apply | status | reconcile`) in the `sync` category. Out of scope: non-scalar retypes (computed/link/attachment on either side → `RETYPE_DEFERRED`), deletions (M4); scalar retypes are now in scope (see below). `mode=diff` (read-only): comprehensive schema comparison via pure `src/sync/compare.js` — every difference classified as **drift** (sync enforces: table/field/view existence, field type/options/choices, view filters/sorts/groups content, column visibility, colors/cover/frozen/rowHeight, primary, form), **best-effort** (sync applies but doesn't guarantee: field/view/column order, sort/group clause order), or **not-synced** (view sections — now captured by snapshot). Verdicts: `identical` (zero diffs) or `converged` (zero drift; best-effort/not-synced may remain). Persists full diff to `~/.airtable-user-mcp/sync/__/diff-.json`; returns a token-budgeted DIGEST (verdicts + class counts + per-table rollup + capped driftSample ≤25). Drill in with `detail="

"` (params: `detail`, `diffId`, `offset`, `limit`). `mode=plan` now emits a curatable **changeset**: each action carries a stable name-based `changeId` (`|
|`), a `class`, and an `apply:true` flag; digest includes a `sample` + `editHint`. New `direction` param (`to-dest` default | `to-source`) targets either base. View sync: `snapshot` also captures collaborative views + live config + **view sections** (not-synced class); `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported); accepts a `skip:[changeId]` list (and honors `apply:false` on changeset entries) — applies everything except removed changes (journal-safe; skipped dependencies degrade to UNRESOLVABLE_REF). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe by filename+size); then restores record-referencing view filters stripped during view sync. Throttle handling = per-request 429 backoff in the auth queue (the per-call limiter does NOT gate the internal per-row POSTs inside createRecords/updateRecords/addLinkItems); resumable (records journal, per-chunk persist of ~50 creates); continue-on-failure (per-record errors are warnings). Truncation safety: a table whose snapshot hits the 1000-row cap on either base is added to `truncatedTables` by `collectTruncatedTableNames` and skipped by `pruneRecords` under mirror (`RECORDS_TRUNCATED_PRUNE_SKIPPED`) so a record beyond the snapshot window is never deleted as a false orphan; each truncated table also emits `RECORDS_TRUNCATED` on the result; >1000-row read pagination remains a capture-gated follow-up. `mode=status` polls a background job by planId — returns running/done/failed + live records-mapped count. `mode=reconcile` **prunes** dead record-map entries (existence-prune; per-table natural-key re-match now matches by key value — see natural-key matching below; reconcile does NOT de-duplicate records). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json, diff-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile), `compare.js` (pure classified compare, no I/O); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. Reconciliation policy (`src/sync/policy.js`): `mode=apply` accepts a `policy` param (`mirror` | `overlay` | `preserve`, default `overlay`) that controls two axes — **extras** (what to do with dest-only records) and **conflicts** (whose value wins when both bases have the record). Presets: `mirror` = {remove, source-wins} (make dest identical to source); `overlay` = {keep, source-wins} (today's default: keep dest-only rows, source updates win); `preserve` = {keep, dest-wins} (keep dest-only rows and never overwrite dest edits). `policyOverrides` (`{ [tableName]: preset }`) overrides the global preset per table. `confirmDeletions` (boolean, default false) is a safety gate for `mirror`: without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count but deletes nothing; set `confirmDeletions:true` to actually delete. Separately, `createTable` unconditionally clears the ~3 blank scaffolding rows it seeds (best-effort; `SCAFFOLDING_ROWS_KEPT` warning if the deletion attempt fails); `pruneRecords` under `mirror`+`confirmDeletions` additionally removes any leftover dest-only blank rows as part of the ordinary orphan prune. Attachment deletion under mirror is a deferred follow-up. Schema-orphan deletion (`pruneSchema`) runs synchronously in `mode=apply` AFTER `applyPlan`, consuming `plan.orphans` (`{kind:'table'|'field'|'view', destId, name, tableName}`). Order: (1) views — deleted first (no dependents); (2) fields — batch `deleteFields` WITHOUT force (so a matched field is never cascade-deleted), retry-until-stable loop resolves orphan→orphan deps; a field still blocked after no progress is kept + `SCHEMA_DELETE_BLOCKED`; (3) tables — deleted last (after their content). Gating: dest-only fields + views require `confirmDeletions:true` (same flag as `pruneRecords`); dest-only tables require the new `confirmTableDeletions:true` — `confirmDeletions` alone never drops a whole table. Without each gate the operation is a dry-count: `DELETION_GATED` for fields/views, `TABLE_DELETION_GATED` for tables. Applied counts: `schemaDeleted` (fields+views) + `tablesDeleted`. Per-table `policyOverrides` apply; `overlay`/`preserve` → no schema deletion. Continue-on-failure (`SCHEMA_DELETE_FAILED`). Module: `src/sync/prune-schema.js`. View-section orphans (dest-only sidebar sections in a matched table) are now deleted by `pruneSchema` as **step 0** (before views, so Airtable auto-promotes their contained views to top-level before view orphan deletion); gated by `confirmDeletions` (same flag as fields/views); `deleteViewSection` is the client method used. Attachment-strip under mirror remains a deferred follow-up. Scalar field retypes (`mode=apply`): a matched dest field whose scalar type diverges from source is retyped via `updateFieldConfig`, gated by the new `confirmRetypes` param (boolean, default false); without it the field is kept and a `RETYPE_GATED` warning is emitted (count surfaced in the result). Non-scalar retypes (either side is computed/link/attachment) stay `RETYPE_DEFERRED`. Select retypes converge their choice map on re-sync exactly like created selects (`mergeChoices`). Continue-on-failure: per-field retype errors emit `RETYPE_FAILED` and do not abort apply. Primary-field retypes remain the separate `reconcilePrimary` path. This closes the last schema-drift category sync can fix; records data migration remains `pruneRecords`/Pass 1. Custom field mappings: `fieldMappings` (`{ [tableName]: { sourceFieldName: destFieldName } }`) injects a source field's value into a different writable scalar dest field during record sync — useful when the source is computed (e.g., `autoNumber`/formula) and you want its value preserved in a writable dest field (classic use: inject source autoNumber `Code` into dest text field `InjectID` so a dest formula can reconstruct original identity). Fail-fast pre-flight validation in `apply` (and dry-run in `plan`/`diff` via `fieldMappingErrors` in the machine output) with codes: `FIELD_MAP_TABLE_MISSING`, `FIELD_MAP_SOURCE_MISSING`, `FIELD_MAP_TARGET_MISSING`, `FIELD_MAP_TARGET_COMPUTED` (dest must be writable scalar), `FIELD_MAP_TYPE_INCOMPATIBLE` (array-typed source/dest not supported), `FIELD_MAP_COLLISION` (two sources map to same dest); an invalid mapping aborts `apply` synchronously (`FIELD_MAP_INVALID` error code). Natural-key record matching: `naturalKeys` (`{ [tableName]: fieldName }`) matches dest↔source records by the key field's **value** (field ID resolved per-base by name), growing `idmap.records`. Add-only + ambiguity-safe — never removes/overwrites an existing mapping, never claims an already-mapped dest; ambiguous/duplicate key values are skipped with `NATURAL_KEY_AMBIGUOUS`; a missing key field emits `NATURAL_KEY_FIELD_MISSING`. Runs in `mode=reconcile` (after the existence-prune, snapshots source records + calls matcher, then saves idmap) AND as an automatic pre-pass in `mode=apply` **before Pass 1 and prune** — so records `mirror` no longer deletes real-but-unmapped dest records as false orphans, and re-syncs don't duplicate them. The key field must be **stable across bases** (autoNumber renumbers → bad key; name/email/injected `InjectID` → good key). Single-field keys only; composite keys and value normalization are deferred. Returns a `matched` count in the result. +- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal; a journal with ≥1 done action bypasses the drift guard with `RESUME_DRIFT_BYPASS`, so resume is reachable past sync's own mutations). Concurrent applies to the same base pair are refused via a per-pair `apply.lock` (`src/sync/apply-lock.js`: pid-liveness stale detection, held until the background records job settles; `mode=reconcile` takes the same lock; error code `APPLY_LOCKED`). Exposed via the `sync_base` tool (`mode: diff | plan | apply | status | reconcile`) in the `sync` category. Out of scope: non-scalar retypes (computed/link/attachment on either side → `RETYPE_DEFERRED`), deletions (M4); scalar retypes are now in scope (see below). `mode=diff` (read-only): comprehensive schema comparison via pure `src/sync/compare.js` — every difference classified as **drift** (sync enforces: table/field/view existence, field type/options/choices, view filters/sorts/groups content, column visibility, colors/cover/frozen/rowHeight, primary, form), **best-effort** (sync applies but doesn't guarantee: field/view/column order, sort/group clause order), or **not-synced** (view sections — now captured by snapshot). Verdicts: `identical` (zero diffs) or `converged` (zero drift; best-effort/not-synced may remain). Persists full diff to `~/.airtable-user-mcp/sync/__/diff-.json`; returns a token-budgeted DIGEST (verdicts + class counts + per-table rollup + capped driftSample ≤25). Drill in with `detail="
"` (params: `detail`, `diffId`, `offset`, `limit`). `mode=plan` now emits a curatable **changeset**: each action carries a stable name-based `changeId` (`|
|`), a `class`, and an `apply:true` flag; digest includes a `sample` + `editHint`. New `direction` param (`to-dest` default | `to-source`) targets either base. View sync: `snapshot` also captures collaborative views + live config + **view sections** (not-synced class); `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported); accepts a `skip:[changeId]` list (and honors `apply:false` on changeset entries) — applies everything except removed changes (journal-safe; skipped dependencies degrade to UNRESOLVABLE_REF). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe scoped to the dest cell `row|field|filename|size`); then restores record-referencing view filters stripped during view sync (per-view failures warn + continue, never block prune). Pass 2 + attachments honor `conflicts=dest-wins` for pre-existing mapped rows; rows created by this plan — tracked resume-durably as `createdDestIds` in the records journal — always receive their links/attachments. Throttle handling = per-request 429 backoff in the auth queue (the per-call limiter does NOT gate the internal per-row POSTs inside createRecords/updateRecords/addLinkItems); resumable (records journal, per-chunk persist of ~50 creates); continue-on-failure (per-record errors are warnings). Truncation safety: a table whose snapshot hits the 1000-row cap on either base is added to `truncatedTables` by `collectTruncatedTableNames` and skipped by `pruneRecords` under mirror (`RECORDS_TRUNCATED_PRUNE_SKIPPED`) so a record beyond the snapshot window is never deleted as a false orphan; each truncated table also emits `RECORDS_TRUNCATED` on the result; >1000-row read pagination remains a capture-gated follow-up. `mode=status` polls a background job by planId — returns running/done/failed + live records-mapped count. `mode=reconcile` **prunes** dead record-map entries (existence-prune, truncation-guarded — skipped with a warning when any dest table hits the 1000-row snapshot cap, so live mappings beyond the cap are never dropped; per-table natural-key re-match now matches by key value — see natural-key matching below; reconcile does NOT de-duplicate records). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json, diff-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile), `compare.js` (pure classified compare, no I/O); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. Reconciliation policy (`src/sync/policy.js`): `mode=apply` accepts a `policy` param (`mirror` | `overlay` | `preserve`, default `overlay`) that controls two axes — **extras** (what to do with dest-only records) and **conflicts** (whose value wins when both bases have the record). Presets: `mirror` = {remove, source-wins} (make dest identical to source); `overlay` = {keep, source-wins} (today's default: keep dest-only rows, source updates win); `preserve` = {keep, dest-wins} (keep dest-only rows and never overwrite dest edits). `policyOverrides` (`{ [tableName]: preset }`) overrides the global preset per table. `confirmDeletions` (boolean, default false) is a safety gate for `mirror`: without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count but deletes nothing; set `confirmDeletions:true` to actually delete. Separately, `createTable` unconditionally clears the ~3 blank scaffolding rows it seeds (best-effort; `SCAFFOLDING_ROWS_KEPT` warning if the deletion attempt fails); `pruneRecords` under `mirror`+`confirmDeletions` additionally removes any leftover dest-only blank rows as part of the ordinary orphan prune. Attachment deletion under mirror is a deferred follow-up. Schema-orphan deletion (`pruneSchema`) runs synchronously in `mode=apply` AFTER `applyPlan`, consuming `plan.orphans` (`{kind:'table'|'field'|'view', destId, name, tableName}`). Order: (1) views — deleted first (no dependents); (2) fields — batch `deleteFields` WITHOUT force (so a matched field is never cascade-deleted), retry-until-stable loop resolves orphan→orphan deps; a field still blocked after no progress is kept + `SCHEMA_DELETE_BLOCKED`; (3) tables — deleted last (after their content). Gating: dest-only fields + views require `confirmDeletions:true` (same flag as `pruneRecords`); dest-only tables require the new `confirmTableDeletions:true` — `confirmDeletions` alone never drops a whole table. Without each gate the operation is a dry-count: `DELETION_GATED` for fields/views, `TABLE_DELETION_GATED` for tables. Applied counts: `schemaDeleted` (fields+views) + `tablesDeleted`. Per-table `policyOverrides` apply; `overlay`/`preserve` → no schema deletion. Continue-on-failure (`SCHEMA_DELETE_FAILED`). Module: `src/sync/prune-schema.js`. View-section orphans (dest-only sidebar sections in a matched table) are now deleted by `pruneSchema` as **step 0** (before views, so Airtable auto-promotes their contained views to top-level before view orphan deletion); gated by `confirmDeletions` (same flag as fields/views); `deleteViewSection` is the client method used. Attachment-strip under mirror remains a deferred follow-up. Scalar field retypes (`mode=apply`): a matched dest field whose scalar type diverges from source is retyped via `updateFieldConfig`, gated by the new `confirmRetypes` param (boolean, default false); without it the field is kept and a `RETYPE_GATED` warning is emitted (count surfaced in the result). Non-scalar retypes (either side is computed/link/attachment) stay `RETYPE_DEFERRED`. Select retypes converge their choice map on re-sync exactly like created selects (`mergeChoices`). Continue-on-failure: per-field retype errors emit `RETYPE_FAILED` and do not abort apply. Primary-field retypes remain the separate `reconcilePrimary` path. This closes the last schema-drift category sync can fix; records data migration remains `pruneRecords`/Pass 1. Custom field mappings: `fieldMappings` (`{ [tableName]: { sourceFieldName: destFieldName } }`) injects a source field's value into a different writable scalar dest field during record sync — useful when the source is computed (e.g., `autoNumber`/formula) and you want its value preserved in a writable dest field (classic use: inject source autoNumber `Code` into dest text field `InjectID` so a dest formula can reconstruct original identity). Fail-fast pre-flight validation in `apply` (and dry-run in `plan`/`diff` via `fieldMappingErrors` in the machine output) with codes: `FIELD_MAP_TABLE_MISSING`, `FIELD_MAP_SOURCE_MISSING`, `FIELD_MAP_TARGET_MISSING`, `FIELD_MAP_TARGET_COMPUTED` (dest must be writable scalar), `FIELD_MAP_TYPE_INCOMPATIBLE` (array-typed source/dest not supported), `FIELD_MAP_COLLISION` (two sources map to same dest); an invalid mapping aborts `apply` synchronously (`FIELD_MAP_INVALID` error code). Natural-key record matching: `naturalKeys` (`{ [tableName]: fieldName }`) matches dest↔source records by the key field's **value** (field ID resolved per-base by name), growing `idmap.records`. Add-only + ambiguity-safe — never removes/overwrites an existing mapping, never claims an already-mapped dest; ambiguous/duplicate key values are skipped with `NATURAL_KEY_AMBIGUOUS`; a missing key field emits `NATURAL_KEY_FIELD_MISSING`. Runs in `mode=reconcile` (after the existence-prune, snapshots source records + calls matcher, then saves idmap) AND as an automatic pre-pass in `mode=apply` **before Pass 1 and prune** — so records `mirror` no longer deletes real-but-unmapped dest records as false orphans, and re-syncs don't duplicate them. The key field must be **stable across bases** (autoNumber renumbers → bad key; name/email/injected `InjectID` → good key). Single-field keys only; composite keys and value normalization are deferred. Returns a `matched` count in the result. Hardening pass (2026-07, 25 commits, ~100 regression tests — full audit + fixes): `plan`/`diff` preserve persisted `idmap.records`/`idmap.attachments` verbatim (schema maps refresh; record identity is never wiped). Apply journals by **disposition** — only genuinely applied actions are `done`; gated (`RETYPE_GATED`) and deferred (`UNRESOLVABLE_REF`, link target table not yet mapped) actions stay retryable, and deferred creates/updates retry within the same run until no progress, so forward-referenced computed/link fields converge in one apply. Reverse-link adoption is exact (idmap-based `symmetricColumnId` match — cross-run safe; two links between the same table pair never cross-wire); link `updateField`s ship writable options only (`{foreignTableId, relationship}`, remapped). `reconcilePrimary` gates pre-existing-primary retypes behind `confirmRetypes` (an applied rename is journaled as an `:rename` sub-action so the same-plan follow-up resumes past the drift guard) and writes computed primaries via `toWritableComputedOptions`. Diff/compare: matched link fields compare canonically (no phantom `updateField`), writable display-format options on computed fields are compared, collaborator `usr` ids pass through view-filter remap verbatim (user ids are Airtable-global), views match by **(name, type)** — apply adopts by the same key — and the dest primary is never emitted as a deletable orphan. Cells: internal type spellings (`multipleAttachment`, `multiCollaborator`, …) are recognized in `cells.js` + `policy.js`; ALL array-shaped cells are deferred on the Pass-1 update path; cells cleared on source propagate as explicit nulls under `conflicts=source-wins` (only idmap-mapped fields whose source AND dest are non-computed); empty update rows are skipped. Records mirror propagates source-side record deletions (a mapping whose source row is gone stops protecting its dest row — truncation-guarded, gated by `confirmDeletions`, stale mapping dropped); truncation detection counts actually-fetched rows (immune to `mirrorFoldedLinks` inflation). New select/multiSelect fields register their real choice map at creation, so first-run records write select cells. Record snapshots avoid filtered views (prefer an unfiltered collaborative view; else `SNAPSHOT_VIEW_FILTERED` and the table is treated as truncated — no prune). Background job files carry the worker `pid`; `mode=status` reports a `running` job with a dead pid as failed with resume advice. Remaining capture-gated deferrals: unfiltered/paginated table reads (`readQueries` non-view source), and array-cell VALUE updates on existing records (multiSelect/collaborator edits after creation). #### packages/mcp-server — Daemon subsystem @@ -89,7 +89,7 @@ New files (Airtable-specific, not in Perplexity source): - `tunnel-providers/` - `types.js` — TunnelProvider interface - `cloudflared-quick.js` — Cloudflare Quick Tunnel (ephemeral URL) - - `cloudflared-named.js` — Cloudflare Named Tunnel (persistent hostname) + - `cloudflared-named.js` — Cloudflare Named Tunnel (persistent hostname); forces `protocol: http2` and waits for cloudflared's HA connections to settle before reporting the tunnel ready (avoids startup 502s) - `cloudflared-named-setup.js` — wizard for named tunnel credential setup - `ngrok.js` — ngrok tunnel provider - `index.js` — provider registry + settings I/O @@ -238,6 +238,7 @@ All under `airtableFormula.*`: - `mcp.autoConfigureOnInstall` — auto-write MCP config to detected IDEs on first launch - `mcp.toolProfile` — `read-only` (9 tools) / `safe-write` (47 tools) / `full` (62 tools) / `custom` - `mcp.categories.{read,tableWrite,tableDestructive,fieldWrite,fieldDestructive,viewWrite,viewDestructive,viewSection,viewSectionDestructive,formWrite,extension}` — per-category toggles when profile is `custom` +- `mcp.daemonPort` — fixed TCP port for the shared MCP daemon HTTP server (default 8723, kept stable across restarts; 0 = automatic/ephemeral; falls back to an automatic port if the chosen port is busy; takes effect on next daemon restart) - `ai.autoInstallFiles` — auto-install AI skills/rules on first launch - `formula.formatterVersion` — `v1` or `v2` beautifier/minifier engine diff --git a/packages/extension/package.json b/packages/extension/package.json index e5e07d5..d238885 100644 --- a/packages/extension/package.json +++ b/packages/extension/package.json @@ -1,10 +1,10 @@ { "name": "airtable-formula", "private": true, - "version": "2.1.6", + "version": "2.1.10", "publisher": "Nskha", "displayName": "Airtable Formulas, Scripts, Automation, MCP & LSP", - "description": "Airtable formula, script & automation editor with MCP server (70 tools), language server, and AI skills for VS Code.", + "description": "Airtable formula, script & automation editor with MCP server (71 tools), language server, and AI skills for VS Code.", "icon": "images/icon.png", "license": "MIT", "author": { @@ -397,6 +397,13 @@ "default": true, "description": "Start and use a shared daemon process for MCP server. Disable to use a direct per-session process (legacy behavior)." }, + "airtableFormula.mcp.daemonPort": { + "type": "number", + "default": 8723, + "minimum": 0, + "maximum": 65535, + "description": "Fixed TCP port for the shared MCP daemon's HTTP server. Defaults to 8723 so the daemon keeps the same port across restarts (stable MCP client config, predictable process to find/kill). Change it to any free port, or set 0 for an automatic OS-assigned port. Takes effect on the next daemon restart; if the chosen port is already in use, the daemon falls back to an automatic port." + }, "airtableFormula.mcp.serverSource": { "type": "string", "default": "bundled", @@ -491,8 +498,8 @@ ], "enumDescriptions": [ "Schema inspection, formula validation, and record reading only (12 tools)", - "Read + record read/write + create/update tables, fields, views, sidebar sections, and record templates, no deletes, no form metadata (54 tools)", - "All tools enabled including destructive ops, form metadata, and extensions (70 tools)", + "Read + record read/write + create/update tables, fields, views, sidebar sections, and record templates, no deletes, no form metadata (55 tools)", + "All tools enabled including destructive ops, form metadata, and extensions (71 tools)", "User-defined per-tool selection" ], "description": "Active MCP tool profile. Controls which tools are exposed to AI agents through the bundled Airtable MCP server." @@ -560,7 +567,7 @@ "airtableFormula.mcp.categories.recordWrite": { "type": "boolean", "default": true, - "description": "Record Write tools: create_records, update_records, duplicate_records (bulk record create/update/duplication)." + "description": "Record Write tools: create_records, update_records, duplicate_records, upload_attachment (bulk record create/update/duplication and URL-based attachment uploads)." }, "airtableFormula.mcp.categories.recordDestructive": { "type": "boolean", @@ -679,7 +686,7 @@ { "id": "AirtableFormula.server", "label": "Airtable User MCP", - "description": "Manage Airtable bases via 70 MCP tools: schema read, table CRUD, field CRUD (including formula/rollup/lookup/download), view configuration (filters / sorts / groups / visibility / column ordering / freezing — with merge-safe append mode), sidebar section grouping, view-type metadata (Kanban/Gallery/Calendar covers, colors, calendar date ranges), legacy-form metadata, formula validation, formula download, bulk base formula download, record template management (create / rename / pre-fill cells / duplicate / apply / delete), record CRUD (create / update / delete / duplicate), and extension management. Operates over stdio using JSON-RPC 2.0." + "description": "Manage Airtable bases via 71 MCP tools: schema read, table CRUD, field CRUD (including formula/rollup/lookup/download), view configuration (filters / sorts / groups / visibility / column ordering / freezing — with merge-safe append mode), sidebar section grouping, view-type metadata (Kanban/Gallery/Calendar covers, colors, calendar date ranges), legacy-form metadata, formula validation, formula download, bulk base formula download, record template management (create / rename / pre-fill cells / duplicate / apply / delete), record CRUD (create / update / delete / duplicate / upload_attachment), and extension management. Operates over stdio using JSON-RPC 2.0." } ] }, diff --git a/packages/extension/src/mcp/daemon-manager.ts b/packages/extension/src/mcp/daemon-manager.ts index 7b203e1..d8264a5 100644 --- a/packages/extension/src/mcp/daemon-manager.ts +++ b/packages/extension/src/mcp/daemon-manager.ts @@ -112,7 +112,12 @@ export class DaemonManager implements vscode.Disposable { private async _spawnDetached(): Promise { const serverPath = path.join(this.extensionPath, 'dist', 'mcp', 'index.mjs'); - const child = spawn(process.execPath, [serverPath, 'daemon', 'start'], { + const args = [serverPath, 'daemon', 'start']; + // Fixed daemon port (0 = OS-ephemeral). Read fresh each spawn so a settings + // change takes effect on the next (re)start without reloading the window. + const daemonPort = vscode.workspace.getConfiguration('airtableFormula').get('mcp.daemonPort', 0); + if (Number.isInteger(daemonPort) && daemonPort > 0 && daemonPort <= 65535) args.push('--port', String(daemonPort)); + const child = spawn(process.execPath, args, { detached: true, stdio: 'ignore', env: { ...process.env, ...this.buildDaemonEnv() }, diff --git a/packages/extension/src/mcp/tool-profile.ts b/packages/extension/src/mcp/tool-profile.ts index 03c39b2..81fa1ff 100644 --- a/packages/extension/src/mcp/tool-profile.ts +++ b/packages/extension/src/mcp/tool-profile.ts @@ -112,6 +112,7 @@ export const TOOL_CATEGORIES: Record { this._daemonStarting = false; void this._initLockfileWatch(); return this.pushState(); }) .catch(err => { this._daemonStarting = false; @@ -855,6 +858,7 @@ export class DashboardProvider implements vscode.WebviewViewProvider { notifyOnUpdates: settings.mcp.notifyOnUpdates, toolProfile, serverSource: settings.mcp.serverSource, + daemonPort: settings.mcp.daemonPort, }, ai: { autoInstallFiles: settings.ai.autoInstallFiles, includeAgents: settings.ai.includeAgents }, formula: { formatterVersion: settings.formula.formatterVersion }, diff --git a/packages/mcp-server/CHANGELOG.md b/packages/mcp-server/CHANGELOG.md new file mode 100644 index 0000000..ecb9563 --- /dev/null +++ b/packages/mcp-server/CHANGELOG.md @@ -0,0 +1,24 @@ +# Changelog + +## [Unreleased] + +### Added + +- **`upload_attachment`** (record-write category): new MCP tool for URL-based attachment uploads into `multipleAttachments` fields. Accepts `appId` + `updates[]` (`rowId`, `columnId`, `url`, optional `filename`). Fetches each URL's bytes and appends to the target cell via the existing 5-step multipart upload flow. Per-update isolation: individual fetch/upload failures are collected in `failed` and do not abort the rest. The general `update_records` tool cannot write attachment cells; use `upload_attachment` instead. +- **Scalar field retypes** (`sync_base mode=apply`): a matched field whose **scalar** type diverges from source is now retyped to match (via `updateFieldConfig`), gated by a new `confirmRetypes` param (boolean, default `false`). Without it the field is kept and a `RETYPE_GATED` warning is emitted; the applied count surfaces as `retyped`. Non-scalar retypes (either side computed/link/attachment) stay `RETYPE_DEFERRED`; select retypes converge their choice map on re-sync like created selects; per-field failures are continue-on-failure (`RETYPE_FAILED`); primary-field retypes remain the separate `reconcilePrimary` path. This is the last schema-drift category sync can fix (records data migration remains `pruneRecords`/Pass 1). +- **View-section orphan deletion** (`sync_base mode=apply`): completes the M4 deletion story — under `mirror`+`confirmDeletions`, dest-only sidebar view-sections (a section name absent from the source) are now deleted by `pruneSchema` (step 0, before views; `deleteViewSection` auto-promotes any contained views to top-level, so a promoted orphan view is then caught by the view step). Gated by `confirmDeletions` like fields/views (NOT `confirmTableDeletions`); counts into `schemaDeleted`; continue-on-failure (`SCHEMA_DELETE_FAILED`). The snapshot now captures the section id, and `mode=diff` compares sections by name+viewNames (ignoring the id) so semantically-identical sections no longer show as a spurious not-synced diff. + +### Fixed + +- **Sync hardening audit** (`sync_base`, 25 commits, ~100 new regression tests): full multi-agent audit of the sync engine with adversarially verified findings, all fixed: + - *Identity & concurrency*: `mode=plan`/`mode=diff` no longer wipe the persisted `idmap.records`/`idmap.attachments` (previously every re-plan destroyed the record identity map → mass duplication on the next apply). A per-pair `apply.lock` (pid-liveness, held until the background records job settles) refuses concurrent applies (`APPLY_LOCKED`); `mode=reconcile` takes the same lock. Journal resume is now reachable: a journal with ≥1 done action bypasses the drift guard (`RESUME_DRIFT_BYPASS`). Background job files carry the worker `pid`; `mode=status` reports a dead-pid `running` job as failed instead of `running` forever. + - *Policy contracts*: Pass 2 (links) and attachments now honor `conflicts=dest-wins` for pre-existing rows (rows created by the plan are tracked resume-durably in the records journal and always get links/attachments). Mirror now propagates **source-side record deletions** (stale idmap entries no longer shield dest rows; truncation-guarded; still gated by `confirmDeletions`). Cells cleared on source propagate as explicit nulls under source-wins (never to computed dest fields). `reconcilePrimary` retypes of a pre-existing primary are gated behind `confirmRetypes`, and computed primaries are written in the writable options shape. + - *Convergence*: matched link fields no longer emit a phantom `updateField` on every plan (and real link updates ship writable, remapped options); gated/deferred actions are no longer journaled `done`, so re-runs with `confirmRetypes:true` (or after a ref becomes resolvable) actually execute; forward-referenced link/computed fields retry within the run; reverse-link adoption matches the exact `symmetricColumnId` pair (no cross-wiring, cross-run safe); views match and adopt by (name, **type**); new select fields register their real choice map so first-run records write select cells; multiSelect/multiCollaborator and all other array-shaped cells are consistently deferred on the update path (internal type spellings recognized); collaborator `usr` ids pass through view-filter remap (they are Airtable-global). + - *Data safety*: attachment-fallback dedupe key is now scoped to the destination cell (`row|field|filename|size`) — previously any later record carrying a same-named same-size file silently got an empty cell forever; record snapshots avoid filtered views and mark unavoidably-filtered tables as truncated (`SNAPSHOT_VIEW_FILTERED`, no prune); `mode=reconcile`'s existence-prune is truncation-guarded (no more false prunes → duplicates in >1000-row tables); truncation detection counts actually-fetched rows; the dest primary is never emitted as a deletable orphan; one failed view fetch no longer aborts the record-filter restore or blocks prune. + - Capture-gated deferrals (mitigated, not yet fixed): unfiltered/paginated table reads (`readQueries` non-view source) and array-cell value updates on existing records (multiSelect/collaborator edits after creation). + +- **Record-truncation safety** (`sync_base mode=apply`): the records snapshot reads each table at a hard 1000-row cap (the internal `readQueries` endpoint has no pagination — verified: a larger `limit` is rejected and no offset param is honored). Previously, under `mirror`+`confirmDeletions`, `pruneRecords` could delete a legitimate dest record whose source counterpart fell beyond the 1000-row window, treating it as a false orphan (silent data loss). Now any table whose snapshot hits the cap on **either** base is skipped by `pruneRecords` (`RECORDS_TRUNCATED_PRUNE_SKIPPED`), and each truncated table emits a `RECORDS_TRUNCATED` warning so the partial copy is visible. Actual >1000-row read pagination is a follow-up (capture-gated). + +### Security + +- **Hardening round-2** (audit findings #10, #28): `auth.js` no longer logs any portion of the CSRF token to stderr (even a 15-char prefix was a secret leak to logs), and the ngrok tunnel's `forwards_to` dashboard label no longer embeds the local daemon port. (The 2026-06-12 audit's HIGH tier was already fixed; a verification sweep of the remaining MEDIUM/LOW backlog found most either already fixed, theoretical, or requiring a redesign/decision — these two were the clean, surgical wins.) diff --git a/packages/mcp-server/src/attachments.js b/packages/mcp-server/src/attachments.js new file mode 100644 index 0000000..0d67256 --- /dev/null +++ b/packages/mcp-server/src/attachments.js @@ -0,0 +1,40 @@ +/** + * Upload attachments by URL into Airtable attachment cells. + * + * Factored out of index.js so the core loop is unit-testable without network + * and without a real client. + * + * Delegates entirely to client.addAttachmentByUrl — Airtable's servers fetch + * each URL (the UI's "Add attachment → Add URL" flow). No bytes are proxied + * through this server. + */ + +/** + * Bulk "upload attachment by URL". Airtable's servers fetch each URL + * (via client.addAttachmentByUrl). Per-update isolation: one failure does + * not abort the rest. + * + * @param {object} client - AirtableClient (must have addAttachmentByUrl) + * @param {string} appId + * @param {{rowId:string, columnId:string, url:string, filename?:string}[]} updates + * @returns {Promise<{uploaded:Array, failed:Array}>} + */ +export async function uploadAttachmentsByUrl(client, appId, updates) { + const uploaded = []; + const failed = []; + for (const u of updates || []) { + const { rowId, columnId, url, filename } = u || {}; + if (!rowId || !columnId || !url) { + failed.push({ rowId: rowId ?? null, columnId: columnId ?? null, error: 'rowId, columnId, and url are required' }); + continue; + } + try { + const res = await client.addAttachmentByUrl(appId, rowId, columnId, { url, filename }); + if (res && res.ok) uploaded.push({ rowId, columnId, attachmentId: res.attachmentId, url: res.url }); + else failed.push({ rowId, columnId, error: (res && res.error) || 'unknown error' }); + } catch (err) { + failed.push({ rowId, columnId, error: String(err?.message || err) }); + } + } + return { uploaded, failed }; +} diff --git a/packages/mcp-server/src/client.js b/packages/mcp-server/src/client.js index f0fc6f6..69e759d 100644 --- a/packages/mcp-server/src/client.js +++ b/packages/mcp-server/src/client.js @@ -2774,6 +2774,66 @@ export class AirtableClient { } } + /** + * Attach a file to an attachment cell BY URL — Airtable's servers fetch the + * URL (this is the UI's "Add attachment → Add URL", NOT a byte proxy through + * this server). Two internal calls: + * A. POST /v0.3/attachments/urlUpload (JSON) — Airtable downloads the URL and + * returns fileUploadResult { propsToAddToAttachmentObj, filename, expiringInitialPreviewUrl }. + * B. POST /v0.3/row/{rowId}/updateArrayTypeCellByAddingItem (form) — append the item. + * Soft-fails (returns {ok:false,error}) instead of throwing. + * + * @param {string} appId + * @param {string} rowId + * @param {string} columnId + * @param {{url: string, filename?: string}} opts + * @returns {Promise<{ok:boolean, attachmentId?:string, url?:string, error?:string}>} + */ + async addAttachmentByUrl(appId, rowId, columnId, { url, filename } = {}) { + assertAirtableId(appId, 'appId'); + assertAirtableId(rowId, 'rowId'); + assertAirtableId(columnId, 'columnId'); + if (!url) return { ok: false, error: 'url is required' }; + try { + // Call A — Airtable server-side fetch of the URL. + const uploadBody = { + url, + userId: this.auth.userId, + applicationId: appId, + enterpriseAccountId: null, + directUploadUserContentPurpose: 'directUploadAttachment', + _csrf: this.auth.csrfToken, + }; + const upRes = await this.auth.postJSON('https://airtable.com/v0.3/attachments/urlUpload', uploadBody, appId); + if (!upRes.ok) { + const body = await upRes.text().catch(() => ''); + return { ok: false, error: `urlUpload failed (${upRes.status}): ${body}` }; + } + const upData = await upRes.json().catch(() => ({})); + const fur = upData?.fileUploadResult; + if (!upData?.success || !fur?.propsToAddToAttachmentObj) { + return { ok: false, error: `urlUpload: unexpected response — ${JSON.stringify(upData).slice(0, 200)}` }; + } + // Call B — append the attachment item to the cell (same as the multipart path). + const attachmentId = 'att' + this._genRandomId(); + const item = { + id: attachmentId, + ...fur.propsToAddToAttachmentObj, + filename: filename || fur.filename, + expiringInitialPreviewUrl: fur.expiringInitialPreviewUrl, + }; + const attachUrl = `https://airtable.com/v0.3/row/${rowId}/updateArrayTypeCellByAddingItem`; + const res = await this.auth.postForm(attachUrl, this._mutationParams({ columnId, item }, appId), appId); + if (!res.ok) { + const body = await res.text().catch(() => ''); + return { ok: false, error: `attach failed (${res.status}): ${body}` }; + } + return { ok: true, attachmentId, url: fur.url }; + } catch (err) { + return { ok: false, error: String(err?.message || err) }; + } + } + /** * Add linked-record items to a link cell via `updateArrayTypeCellByAddingItem`. * Called once per item (Airtable has no SET endpoint for link cells — only APPEND). diff --git a/packages/mcp-server/src/daemon/server.js b/packages/mcp-server/src/daemon/server.js index e0ee85d..aa30be1 100644 --- a/packages/mcp-server/src/daemon/server.js +++ b/packages/mcp-server/src/daemon/server.js @@ -57,21 +57,34 @@ const FETCH_BLOCKED_PORTS = new Set([ ]); async function listenAvoidingBlockedPorts(server, requestedPort, host) { - const maxAttempts = requestedPort === 0 ? 5 : 1; - for (let attempt = 0; attempt < maxAttempts; attempt++) { - await new Promise((resolve, reject) => { - const onError = (error) => { - server.removeListener('listening', onListening); - reject(error); - }; - const onListening = () => { - server.removeListener('error', onError); - resolve(); - }; - server.once('error', onError); - server.once('listening', onListening); - server.listen(requestedPort, host); - }); + // A fixed port (requestedPort > 0) gets one shot; if it's already in use or + // turns out to be a browser-blocked port, we fall back to an OS-assigned + // ephemeral port so the daemon always starts. The bound port is read back and + // persisted to the lockfile, so clients still discover it. With requestedPort + // 0 we simply retry on ephemeral ports until one is not browser-blocked. + let port = requestedPort; + for (let attempt = 0; attempt < 5; attempt++) { + try { + await new Promise((resolve, reject) => { + const onError = (error) => { + server.removeListener('listening', onListening); + reject(error); + }; + const onListening = () => { + server.removeListener('error', onError); + resolve(); + }; + server.once('error', onError); + server.once('listening', onListening); + server.listen(port, host); + }); + } catch (error) { + if (error?.code === 'EADDRINUSE' && port !== 0) { + port = 0; // fixed port taken → fall back to an ephemeral port + continue; + } + throw error; + } const boundPort = getBoundPort(server); if (!FETCH_BLOCKED_PORTS.has(boundPort)) { @@ -79,6 +92,7 @@ async function listenAvoidingBlockedPorts(server, requestedPort, host) { } await new Promise((resolve) => server.close(() => resolve())); + port = 0; // blocked port → next attempt uses an ephemeral port } } diff --git a/packages/mcp-server/src/daemon/tunnel-providers/cloudflared-named-setup.js b/packages/mcp-server/src/daemon/tunnel-providers/cloudflared-named-setup.js index b62f919..112713b 100644 --- a/packages/mcp-server/src/daemon/tunnel-providers/cloudflared-named-setup.js +++ b/packages/mcp-server/src/daemon/tunnel-providers/cloudflared-named-setup.js @@ -601,7 +601,14 @@ function killChild(child) { function serializeConfigYaml(opts) { // Simple, stable YAML — cloudflared's ingress format is well-defined and // our values are mechanically generated (UUIDs, paths, hostnames). + // + // Force `protocol: http2` (TCP) instead of cloudflared's default QUIC (UDP). + // Some networks throttle or block UDP/7844, which makes the tunnel flaky on + // startup. The MCP channel is low-volume control traffic, so QUIC's + // throughput/latency gains don't matter here — TCP/http2 trades them for + // reliability. return [ + 'protocol: http2', `tunnel: ${opts.uuid}`, `credentials-file: ${yamlQuoteIfNeeded(opts.credentialsPath)}`, 'ingress:', diff --git a/packages/mcp-server/src/daemon/tunnel-providers/cloudflared-named.js b/packages/mcp-server/src/daemon/tunnel-providers/cloudflared-named.js index 0351b88..2c05484 100644 --- a/packages/mcp-server/src/daemon/tunnel-providers/cloudflared-named.js +++ b/packages/mcp-server/src/daemon/tunnel-providers/cloudflared-named.js @@ -19,7 +19,7 @@ * * The managed YAML is treated as provider-owned — hand-edits to add extra * ingress rules WILL be silently dropped on the next start() because we - * serialize only the four canonical keys (tunnel / credentials-file / + * serialize only the five canonical keys (protocol / tunnel / credentials-file / * hostname / service). */ @@ -50,6 +50,17 @@ const STOP_GRACE_MS = 3_000; */ const READY_LINE_REGEX = /Registered tunnel connection/i; +/** cloudflared opens this many HA edge connections by default (--ha-connections). */ +const DEFAULT_TARGET_CONNECTIONS = 4; +/** + * After the FIRST edge connection registers the tunnel can already serve, but + * Cloudflare may still route to colos whose connection hasn't registered yet + * (→ 502). Wait up to this long for the remaining HA connections to settle + * before we advertise the URL as ready. Capped so a network that only ever + * establishes 1–2 connections still becomes ready instead of hanging. + */ +const DEFAULT_SETTLE_GRACE_MS = 6_000; + /** * Build a provider bound to a specific dependency set. The exported * singleton uses node defaults; tests construct a one-off via @@ -184,10 +195,12 @@ export const cloudflaredNamedProvider = createCloudflaredNamedProvider(); * * Kept local to this file to avoid refactoring the working quick-tunnel path. * - * @param {{ binaryPath: string, configPath: string, hostname: string, onStateChange: (state: any) => void, spawnImpl: typeof nodeSpawn }} options + * @param {{ binaryPath: string, configPath: string, hostname: string, onStateChange: (state: any) => void, spawnImpl: typeof nodeSpawn, targetConnections?: number, settleGraceMs?: number }} options * @returns {import('../tunnel.js').StartedTunnel} */ -function spawnNamedTunnel(options) { +export function spawnNamedTunnel(options) { + const targetConnections = options.targetConnections ?? DEFAULT_TARGET_CONNECTIONS; + const settleGraceMs = options.settleGraceMs ?? DEFAULT_SETTLE_GRACE_MS; const args = [ 'tunnel', '--no-autoupdate', @@ -229,10 +242,19 @@ function spawnNamedTunnel(options) { rejectReady = reject; }); - const handleLine = (line) => { + let registered = 0; + let settleTimer = null; + const clearSettleTimer = () => { + if (settleTimer) { + clearTimeout(settleTimer); + settleTimer = null; + } + }; + + const markReady = () => { if (settled) return; - if (!READY_LINE_REGEX.test(line)) return; settled = true; + clearSettleTimer(); updateState({ status: 'enabled', url: hostnameUrl, @@ -242,6 +264,20 @@ function spawnNamedTunnel(options) { resolveReady(hostnameUrl); }; + const handleLine = (line) => { + if (settled) return; + if (!READY_LINE_REGEX.test(line)) return; + registered++; + if (registered >= targetConnections) { + markReady(); + return; + } + // First HA connection up — give the remaining colos a grace window to + // register before advertising ready, so Cloudflare doesn't route to a + // colo that still 502s. + if (!settleTimer) settleTimer = setTimeout(markReady, settleGraceMs); + }; + const rlStderr = child.stderr ? createInterface({ input: child.stderr }) : null; const rlStdout = child.stdout ? createInterface({ input: child.stdout }) : null; rlStderr?.on('line', handleLine); @@ -249,6 +285,7 @@ function spawnNamedTunnel(options) { child.on('close', () => { rlStderr?.close(); rlStdout?.close(); }); child.on('error', (error) => { + clearSettleTimer(); if (!settled) { settled = true; rejectReady(error); @@ -262,6 +299,7 @@ function spawnNamedTunnel(options) { }); child.on('exit', (code, signal) => { + clearSettleTimer(); if (!settled) { settled = true; rejectReady( @@ -282,6 +320,7 @@ function spawnNamedTunnel(options) { }); const stop = async () => { + clearSettleTimer(); if (stopping) return; stopping = true; diff --git a/packages/mcp-server/src/index.js b/packages/mcp-server/src/index.js index ee8afa6..b5d9c5a 100644 --- a/packages/mcp-server/src/index.js +++ b/packages/mcp-server/src/index.js @@ -97,6 +97,7 @@ import { readFile, writeFile, mkdir } from 'node:fs/promises'; import { dirname } from 'node:path'; import path from 'path'; import { stripHeader } from './formula-header.js'; +import { uploadAttachmentsByUrl } from './attachments.js'; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const require = createRequire(import.meta.url); @@ -1559,6 +1560,33 @@ Note: "form title" is the view name itself — use rename_view to change it. "Fi required: ['appId', 'tableId', 'updates'], }, }, + { + name: 'upload_attachment', + description: "Upload attachments into an attachment cell by URL. Airtable's servers fetch each URL directly (the UI's 'Add attachment → Add URL') — bytes are NOT proxied through this server. Use for multipleAttachments fields, which update_records cannot set. Appends to the cell (calling twice adds two). Per-update isolation: a failing update is reported, not fatal.", + annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: true }, + inputSchema: { + type: 'object', + properties: { + appId: { type: 'string', description: 'The Airtable base/application ID' }, + updates: { + type: 'array', + description: 'Attachments to upload, each { rowId: "recXXX", columnId: "fldXXX", url: "https://...", filename?: "name.jpg" }. filename defaults to the URL path basename.', + items: { + type: 'object', + properties: { + rowId: { type: 'string', description: 'Record ID (recXXX)' }, + columnId: { type: 'string', description: 'Field/column ID (fldXXX)' }, + url: { type: 'string', description: 'Public URL of the file to fetch and upload' }, + filename: { type: 'string', description: 'Optional filename override; defaults to the URL path basename' }, + }, + required: ['rowId', 'columnId', 'url'], + }, + }, + debug: debugProp, + }, + required: ['appId', 'updates'], + }, + }, { name: 'delete_records', description: 'Delete one or more records from a table in a single batch call. The returned deleted count equals rowIds.length (optimistic) — already-deleted rows are silently skipped by the server.', @@ -2394,6 +2422,11 @@ const handlers = { return ok(result, result, debug); }, + async upload_attachment({ appId, updates, debug }) { + const result = await uploadAttachmentsByUrl(client, appId, updates || []); + return ok(result, result, debug); + }, + async delete_records({ appId, tableId, rowIds, viewId, debug }) { const result = await client.deleteRecords(appId, tableId, rowIds, { viewId }); return ok(result, result, debug); @@ -2637,9 +2670,19 @@ let activeTransport = null; async function main() { if (isDaemonStart) { + // ponytail: optional fixed daemon port; 0/absent => OS-ephemeral (the default) + let fixedPort; + for (let i = 2; i < cliArgs.length; i++) { + const a = cliArgs[i]; + const raw = a === '--port' ? cliArgs[i + 1] : a.startsWith('--port=') ? a.slice(7) : undefined; + if (raw === undefined) continue; + const n = Number(raw); + if (Number.isInteger(n) && n >= 1 && n <= 65535) fixedPort = n; + } const { startDaemon } = await import('./daemon/launcher.js'); const result = await startDaemon({ configDir: process.env.AIRTABLE_USER_MCP_HOME, + port: fixedPort, getTools: (tc) => { const enabledTools = tc.filterTools(TOOLS); const enabledNames = new Set(enabledTools.map(t => t.name)); diff --git a/packages/mcp-server/src/tool-config.js b/packages/mcp-server/src/tool-config.js index 4cb3b02..e2cbaed 100644 --- a/packages/mcp-server/src/tool-config.js +++ b/packages/mcp-server/src/tool-config.js @@ -110,6 +110,7 @@ export const TOOL_CATEGORIES = { duplicate_records: 'record-write', create_records: 'record-write', update_records: 'record-write', + upload_attachment: 'record-write', // Record Destructive (batch record deletion) delete_records: 'record-destructive', diff --git a/packages/mcp-server/test/test-named-tunnel-readiness.test.js b/packages/mcp-server/test/test-named-tunnel-readiness.test.js new file mode 100644 index 0000000..f1c57ad --- /dev/null +++ b/packages/mcp-server/test/test-named-tunnel-readiness.test.js @@ -0,0 +1,153 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import { PassThrough } from 'node:stream'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { spawnNamedTunnel } from '../src/daemon/tunnel-providers/cloudflared-named.js'; +import { + writeTunnelConfig, + readNamedTunnelConfig, +} from '../src/daemon/tunnel-providers/cloudflared-named-setup.js'; + +/** + * Build a fake cloudflared child process: an EventEmitter with PassThrough + * stdout/stderr so the provider's readline interfaces fire 'line' events when + * we write to them. + */ +function makeFakeChild() { + const child = new EventEmitter(); + child.pid = 4242; + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + child.killed = false; + child.exitCode = null; + child.kill = () => { + child.killed = true; + }; + return child; +} + +/** + * Cleanly tear down a started tunnel for a fake child. The provider's stop() + * does `await exited`, which only resolves on the child's 'exit' event — so + * drive the exit ourselves (a real process would exit on kill), then await + * stop(). Cross-platform: on win32 stop() taskkills (no-op for the fake pid) + * then awaits exited too, so the explicit exit is what unblocks it. + */ +async function shutdown(tunnel, child) { + if (child.exitCode === null) child.exitCode = 0; + const stopped = tunnel.stop(); + child.emit('exit', 0, 'SIGTERM'); + await stopped; +} + +/** Let readline flush the buffered line(s) into 'line' events. */ +const flush = () => new Promise((r) => setTimeout(r, 15)); + +describe('spawnNamedTunnel readiness', () => { + it('A: waits for the full HA connection count before reporting ready', async () => { + const child = makeFakeChild(); + const tunnel = spawnNamedTunnel({ + binaryPath: 'cf', + configPath: 'cfg', + hostname: 'h.example.com', + onStateChange: () => {}, + spawnImpl: () => child, + targetConnections: 4, + settleGraceMs: 10_000, + }); + + let url = null; + tunnel.waitUntilReady.then((u) => { + url = u; + }); + + // One connection — NOT ready (long settleGraceMs keeps the timer pending). + child.stderr.write('2026-06-20 INF Registered tunnel connection connIndex=0 ...\n'); + await flush(); + assert.equal(url, null, 'should not be ready after a single connection'); + + // Remaining three connections register → hits target → ready. + child.stderr.write('2026-06-20 INF Registered tunnel connection connIndex=1 ...\n'); + child.stderr.write('2026-06-20 INF Registered tunnel connection connIndex=2 ...\n'); + child.stderr.write('2026-06-20 INF Registered tunnel connection connIndex=3 ...\n'); + await flush(); + + assert.equal(await tunnel.waitUntilReady, 'https://h.example.com'); + assert.equal(url, 'https://h.example.com'); + + // With all 4 registered the settle timer is already cleared; stop() also + // clears it. Avoid dangling timers. + await shutdown(tunnel, child); + }); + + it('B: falls back to the settle grace when fewer than target connections register', async () => { + const child = makeFakeChild(); + const tunnel = spawnNamedTunnel({ + binaryPath: 'cf', + configPath: 'cfg', + hostname: 'h.example.com', + onStateChange: () => {}, + spawnImpl: () => child, + targetConnections: 4, + settleGraceMs: 30, + }); + + // Only one of four connections — resolves via the ~30ms grace timer. + child.stderr.write('2026-06-20 INF Registered tunnel connection connIndex=0 ...\n'); + + assert.equal(await tunnel.waitUntilReady, 'https://h.example.com'); + + await shutdown(tunnel, child); + }); + + it('C: clears the settle timer if the child exits before settling', async () => { + const child = makeFakeChild(); + const tunnel = spawnNamedTunnel({ + binaryPath: 'cf', + configPath: 'cfg', + hostname: 'h.example.com', + onStateChange: () => {}, + spawnImpl: () => child, + targetConnections: 4, + settleGraceMs: 10_000, + }); + + // One connection arms the (long) settle timer. + child.stderr.write('2026-06-20 INF Registered tunnel connection connIndex=0 ...\n'); + await flush(); + + // Exit before the grace fires — must reject (and clear the timer so no + // stale markReady resolves later). + child.emit('exit', 1, null); + + await assert.rejects(tunnel.waitUntilReady); + }); +}); + +describe('named tunnel config protocol', () => { + it('D: serializes protocol: http2 and the config still round-trips', () => { + const dir = mkdtempSync(join(tmpdir(), 'cf-named-protocol-')); + try { + writeTunnelConfig({ + configDir: dir, + uuid: 'uuid-1', + hostname: 'h.example.com', + port: 8723, + credentialsPath: join(dir, 'creds.json'), + }); + + const raw = readFileSync(join(dir, 'cloudflared-named.yml'), 'utf8'); + assert.match(raw, /^protocol: http2$/m); + + const cfg = readNamedTunnelConfig(dir); + assert.equal(cfg.port, 8723); + assert.equal(cfg.hostname, 'h.example.com'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/mcp-server/test/test-tool-config.test.js b/packages/mcp-server/test/test-tool-config.test.js index 24c79d8..230e88a 100644 --- a/packages/mcp-server/test/test-tool-config.test.js +++ b/packages/mcp-server/test/test-tool-config.test.js @@ -17,7 +17,7 @@ import { describe('TOOL_CATEGORIES', () => { it('maps all tools to valid categories', () => { const tools = Object.keys(TOOL_CATEGORIES); - assert.equal(tools.length, 70, `Expected 70 tools, got ${tools.length}`); + assert.equal(tools.length, 71, `Expected 71 tools, got ${tools.length}`); for (const [tool, cat] of Object.entries(TOOL_CATEGORIES)) { assert.ok(CATEGORY_LABELS[cat], `Tool "${tool}" has unknown category "${cat}"`); } @@ -84,9 +84,9 @@ describe('ToolConfigManager', () => { assert.equal(mgr.activeProfile, 'full'); }); - it('enables all 70 tools on full profile', () => { + it('enables all 71 tools on full profile', () => { const enabled = mgr.enabledToolNames(); - assert.equal(enabled.size, 70); + assert.equal(enabled.size, 71); }); it('manage_tools is always enabled', () => { @@ -125,10 +125,10 @@ describe('ToolConfigManager', () => { assert.ok(!enabled.has('delete_records')); }); - it('full enables all 70 tools', async () => { + it('full enables all 71 tools', async () => { await mgr.switchProfile('full'); const enabled = mgr.enabledToolNames(); - assert.equal(enabled.size, 70); + assert.equal(enabled.size, 71); }); it('unknown profile fails closed to read-only', async () => { @@ -211,10 +211,10 @@ describe('ToolConfigManager', () => { }); describe('getToolStatus()', () => { - it('returns status for all 70 tools', async () => { + it('returns status for all 71 tools', async () => { await mgr.switchProfile('full'); const status = mgr.getToolStatus(); - assert.equal(status.length, 70); + assert.equal(status.length, 71); assert.ok(status.every(s => s.enabled === true)); }); diff --git a/packages/mcp-server/test/test-upload-attachment.test.js b/packages/mcp-server/test/test-upload-attachment.test.js new file mode 100644 index 0000000..5dc5d5e --- /dev/null +++ b/packages/mcp-server/test/test-upload-attachment.test.js @@ -0,0 +1,149 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { uploadAttachmentsByUrl } from '../src/attachments.js'; + +// ────────────────────────────────────────────────────────────────────────────── +// Fake client +// ────────────────────────────────────────────────────────────────────────────── + +/** + * Build a fake client with addAttachmentByUrl that: + * - records every call in `calls` + * - returns values from `responses` keyed by call index, or a default success + */ +function fakeClient(responses = []) { + const calls = []; + const client = { + calls, + async addAttachmentByUrl(appId, rowId, columnId, { url, filename } = {}) { + const callIdx = calls.length; + calls.push({ appId, rowId, columnId, url, filename }); + const override = responses[callIdx]; + if (override !== undefined) return override; + return { ok: true, attachmentId: 'attFAKE', url: 'https://fake.cdn/file' }; + }, + }; + return client; +} + +// ────────────────────────────────────────────────────────────────────────────── +// Tests +// ────────────────────────────────────────────────────────────────────────────── + +describe('uploadAttachmentsByUrl — happy path', () => { + it('2 updates both succeed: uploaded.length===2, failed.length===0', async () => { + const client = fakeClient([ + { ok: true, attachmentId: 'att1', url: 'https://cdn/1.jpg' }, + { ok: true, attachmentId: 'att2', url: 'https://cdn/2.jpg' }, + ]); + + const result = await uploadAttachmentsByUrl(client, 'appABC', [ + { rowId: 'recA', columnId: 'fld1', url: 'https://example.com/a.jpg' }, + { rowId: 'recB', columnId: 'fld2', url: 'https://example.com/b.jpg', filename: 'override.png' }, + ]); + + assert.equal(result.uploaded.length, 2, 'expected 2 uploaded'); + assert.equal(result.failed.length, 0, 'expected 0 failed'); + + // Verify addAttachmentByUrl was called with the right args + assert.equal(client.calls.length, 2); + assert.equal(client.calls[0].appId, 'appABC'); + assert.equal(client.calls[0].rowId, 'recA'); + assert.equal(client.calls[0].columnId, 'fld1'); + assert.equal(client.calls[0].url, 'https://example.com/a.jpg'); + assert.equal(client.calls[0].filename, undefined); + + assert.equal(client.calls[1].rowId, 'recB'); + assert.equal(client.calls[1].columnId, 'fld2'); + assert.equal(client.calls[1].url, 'https://example.com/b.jpg'); + assert.equal(client.calls[1].filename, 'override.png'); + + // Verify uploaded entries have correct shape + assert.equal(result.uploaded[0].rowId, 'recA'); + assert.equal(result.uploaded[0].columnId, 'fld1'); + assert.equal(result.uploaded[0].attachmentId, 'att1'); + assert.equal(result.uploaded[0].url, 'https://cdn/1.jpg'); + assert.equal(result.uploaded[1].attachmentId, 'att2'); + }); +}); + +describe('uploadAttachmentsByUrl — per-update isolation', () => { + it('1st fails, 2nd succeeds: both attempted, failed.length===1, uploaded.length===1', async () => { + const client = fakeClient([ + { ok: false, error: 'boom' }, + { ok: true, attachmentId: 'attX', url: 'https://cdn/ok.jpg' }, + ]); + + const result = await uploadAttachmentsByUrl(client, 'appABC', [ + { rowId: 'rec1', columnId: 'fld1', url: 'https://example.com/a.jpg' }, + { rowId: 'rec2', columnId: 'fld2', url: 'https://example.com/b.jpg' }, + ]); + + assert.equal(result.failed.length, 1, 'expected 1 failure'); + assert.equal(result.uploaded.length, 1, 'expected 1 success'); + // Both were attempted + assert.equal(client.calls.length, 2, 'addAttachmentByUrl must have been called twice'); + + assert.equal(result.failed[0].rowId, 'rec1'); + assert.equal(result.failed[0].columnId, 'fld1'); + assert.match(result.failed[0].error, /boom/); + + assert.equal(result.uploaded[0].rowId, 'rec2'); + assert.equal(result.uploaded[0].attachmentId, 'attX'); + }); +}); + +describe('uploadAttachmentsByUrl — validation', () => { + it('update missing url goes to failed without calling addAttachmentByUrl', async () => { + const client = fakeClient(); + + const result = await uploadAttachmentsByUrl(client, 'appABC', [ + { rowId: 'rec1', columnId: 'fld1' /* url missing */ }, + ]); + + assert.equal(result.failed.length, 1, 'expected 1 failure'); + assert.equal(result.uploaded.length, 0); + assert.equal(client.calls.length, 0, 'addAttachmentByUrl must NOT have been called'); + assert.ok(result.failed[0].error, 'error message must be set'); + }); + + it('update missing rowId goes to failed without calling addAttachmentByUrl', async () => { + const client = fakeClient(); + + const result = await uploadAttachmentsByUrl(client, 'appABC', [ + { columnId: 'fld1', url: 'https://example.com/x.jpg' /* rowId missing */ }, + ]); + + assert.equal(result.failed.length, 1); + assert.equal(client.calls.length, 0, 'addAttachmentByUrl must NOT have been called'); + assert.ok(result.failed[0].error); + }); + + it('update missing columnId goes to failed without calling addAttachmentByUrl', async () => { + const client = fakeClient(); + + const result = await uploadAttachmentsByUrl(client, 'appABC', [ + { rowId: 'rec1', url: 'https://example.com/x.jpg' /* columnId missing */ }, + ]); + + assert.equal(result.failed.length, 1); + assert.equal(client.calls.length, 0, 'addAttachmentByUrl must NOT have been called'); + assert.ok(result.failed[0].error); + }); + + it('valid updates after invalid ones still succeed (mixed batch)', async () => { + const client = fakeClient([ + { ok: true, attachmentId: 'attGOOD', url: 'https://cdn/good.jpg' }, + ]); + + const result = await uploadAttachmentsByUrl(client, 'appABC', [ + { rowId: 'rec1' /* missing url and columnId */ }, + { rowId: 'rec2', columnId: 'fld2', url: 'https://example.com/good.jpg' }, + ]); + + assert.equal(result.failed.length, 1); + assert.equal(result.uploaded.length, 1); + assert.equal(client.calls.length, 1, 'only the valid update should call addAttachmentByUrl'); + assert.equal(result.uploaded[0].attachmentId, 'attGOOD'); + }); +}); diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index 1afa4b9..68f8e67 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -127,6 +127,7 @@ export interface SettingsSnapshot { notifyOnUpdates: boolean; toolProfile: ToolProfileSnapshot; serverSource: 'bundled' | 'npx'; + daemonPort: number; }; ai: { autoInstallFiles: boolean; includeAgents: boolean }; formula: { formatterVersion: 'v1' | 'v2' }; diff --git a/packages/webview/src/store.ts b/packages/webview/src/store.ts index aa998b3..4ccafde 100644 --- a/packages/webview/src/store.ts +++ b/packages/webview/src/store.ts @@ -62,8 +62,8 @@ const defaultSettings: SettingsSnapshot = { notifyOnUpdates: true, toolProfile: { profile: 'safe-write', - enabledCount: 54, - totalCount: 70, + enabledCount: 55, + totalCount: 71, categories: { read: true, recordRead: true, tableWrite: true, tableDestructive: true, fieldWrite: true, fieldDestructive: true, recordDestructive: true, @@ -74,6 +74,7 @@ const defaultSettings: SettingsSnapshot = { }, }, serverSource: 'bundled' as const, + daemonPort: 8723, }, ai: { autoInstallFiles: true, includeAgents: false }, formula: { formatterVersion: 'v2' }, diff --git a/packages/webview/src/tabs/Settings.tsx b/packages/webview/src/tabs/Settings.tsx index 6e257e1..910aa7b 100644 --- a/packages/webview/src/tabs/Settings.tsx +++ b/packages/webview/src/tabs/Settings.tsx @@ -552,6 +552,23 @@ export function Settings() { +
+
+
Daemon port
+
+ Default 8723 — stays fixed across restarts · set 0 for an automatic port · applies on daemon restart +
+
+ changeHeavySetting('mcp.daemonPort', Number(e.target.value))} + /> +
diff --git a/packages/webview/src/test/store.test.ts b/packages/webview/src/test/store.test.ts index 762c963..a25b3d4 100644 --- a/packages/webview/src/test/store.test.ts +++ b/packages/webview/src/test/store.test.ts @@ -13,7 +13,7 @@ beforeEach(() => { ideStatuses: [], versions: { extension: '—', mcpServerBundled: '—' }, aiFilesCount: 0, loading: true, activeTab: 'overview', pendingActions: new Set(), settings: { - mcp: { autoConfigureOnInstall: true, notifyOnUpdates: true, toolProfile: { profile: 'safe-write', enabledCount: 54, totalCount: 70, categories: { read: true, recordRead: true, tableWrite: true, tableDestructive: true, fieldWrite: true, fieldDestructive: true, recordDestructive: true, viewWrite: true, viewDestructive: true, viewSection: true, viewSectionDestructive: true, formWrite: true, extension: true, recordWrite: true, sync: true } }, serverSource: 'bundled' }, + mcp: { autoConfigureOnInstall: true, notifyOnUpdates: true, toolProfile: { profile: 'safe-write', enabledCount: 54, totalCount: 70, categories: { read: true, recordRead: true, tableWrite: true, tableDestructive: true, fieldWrite: true, fieldDestructive: true, recordDestructive: true, viewWrite: true, viewDestructive: true, viewSection: true, viewSectionDestructive: true, formWrite: true, extension: true, recordWrite: true, sync: true } }, serverSource: 'bundled', daemonPort: 0 }, ai: { autoInstallFiles: true, includeAgents: false }, formula: { formatterVersion: 'v2' }, script: { beautifyStyle: 'default', minifyLevel: 'standard' }, From fe5807b017959900e5c2d4d865295a57df20e250 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 18:06:45 +0300 Subject: [PATCH 169/246] feat(transport): route Airtable API calls through direct node HTTP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1 of the transport rework: demote the browser to login / credential-capture only and route every Airtable API call through direct node HTTP instead of page.evaluate(fetch). A live spike proved plain node fetch with the profile's cookies + the standard headers returns byte-identical 200s (no TLS-fingerprint barrier), so the browser is no longer the transport. - New src/http-transport.js: HttpTransport.request({method,url,headers,body}) → {status, body, error}. Default client 'fetch'; opt-in 'impit' (AIRTABLE_HTTP_CLIENT=impit) lazy-loaded like patchright/otpauth, with a clear "install impit" error when absent. impit added to optionalDependencies. - auth._snapshotCredentials(): read page.context().cookies() (incl. HttpOnly), build an airtable-domain Cookie header, cache csrf. Snapshotted before the init verify call (which now goes direct-HTTP) and after each refresh/recovery. - auth._rawApiCall: build identical headers + Cookie + (_csrf on form POST) from the snapshot and hand off to HttpTransport instead of page.evaluate. Return shape unchanged, so _wrapResponse and the _apiCall 429/403/401 backoff/recovery loop are untouched. The per-call timeout now clears its timer on settle (the old uncleared timer would dangle now that this is the universal request path). - Lazy per-app secretSocketId capture: since API POSTs no longer flow through the page, postForm navigates the login page to the base once to let the existing interception handler capture the socketId, then injects it into the outgoing params. Best-effort ("when available"); a miss proceeds without it and a per-app guard prevents a navigation storm. Backward-compatible: new AirtableAuth() still works; client.js and all consumers (daemon/server.js, index.js) untouched. Co-Authored-By: Claude Fable 5 --- packages/mcp-server/package.json | 1 + packages/mcp-server/src/auth.js | 210 ++++++++++++++++------ packages/mcp-server/src/http-transport.js | 102 +++++++++++ 3 files changed, 259 insertions(+), 54 deletions(-) create mode 100644 packages/mcp-server/src/http-transport.js diff --git a/packages/mcp-server/package.json b/packages/mcp-server/package.json index bf42565..031d326 100644 --- a/packages/mcp-server/package.json +++ b/packages/mcp-server/package.json @@ -75,6 +75,7 @@ }, "optionalDependencies": { "@ngrok/ngrok": "^1.7.0", + "impit": "^0.14.2", "otpauth": "^9.5.0", "patchright": "^1.58.2" } diff --git a/packages/mcp-server/src/auth.js b/packages/mcp-server/src/auth.js index 5df60de..5424912 100644 --- a/packages/mcp-server/src/auth.js +++ b/packages/mcp-server/src/auth.js @@ -3,6 +3,7 @@ import { fileURLToPath } from 'url'; import { randomBytes } from 'node:crypto'; import { trace } from './debug-tracer.js'; import { getProfileDir } from './paths.js'; +import { HttpTransport } from './http-transport.js'; /** Generate a page-load-id (pgl + 13 base36 chars) using crypto, not Math.random. */ function genPageLoadId() { @@ -66,6 +67,18 @@ export class AirtableAuth { // indefinitely. this._secretSocketIds = new Map(); this._maxSocketIdCache = 50; + // Per-app guard so a base whose socketId never captures isn't re-navigated + // on every subsequent mutation (avoid a lazy-capture navigation storm). + this._socketIdCaptureAttempted = new Set(); + + // Direct-HTTP transport. API calls now go straight out over node HTTP + // (fetch by default; impit via AIRTABLE_HTTP_CLIENT=impit) using the + // credential snapshot below — the browser is only used for login/refresh + // and secretSocketId capture. + this._httpTransport = new HttpTransport({ client: process.env.AIRTABLE_HTTP_CLIENT }); + // Captured session credentials for direct HTTP: { cookieHeader, csrfToken }. + // Populated by _snapshotCredentials() during init/recovery. + this._credentials = null; // Reference to the page-level 'request' listener so we can detach it on // _doInit's context-close step rather than leak it per session-recovery. @@ -179,6 +192,11 @@ export class AirtableAuth { console.error('[auth] Page URL:', this.page.url()); await this._extractCsrf(); + // Snapshot cookies + CSRF BEFORE verifying — _verifySession now issues a + // direct-HTTP call (not an in-page fetch) and needs the Cookie header. + // (The plan said "after _extractCsrf/_verifySession"; it must precede the + // verify call now that verify no longer runs inside the browser.) + await this._snapshotCredentials(); await this._verifySession(); trace('auth', 'auth:csrf_captured', { success: !!this.csrfToken, @@ -273,6 +291,79 @@ export class AirtableAuth { return this._secretSocketIds.get(appId) || null; } + /** + * Lazily capture a secretSocketId for a base before a mutation. + * + * With the direct-HTTP transport the browser no longer sees API POSTs, so the + * passive interception that used to capture secretSocketId "for free" never + * fires. We restore it on demand: navigate the still-open login page to the + * base once — loading the base fires its own POSTs, and the existing request + * interception handler captures the socketId from them — then poll briefly. + * + * Best-effort by design (secretSocketId is applied "when available"): if the + * capture times out we proceed WITHOUT it rather than fail the mutation. A + * per-app guard prevents re-navigating on every mutation when capture misses. + * + * NOTE: captured socketIds are reused for the whole session (no page traffic + * to refresh them). This matches the pre-existing "cache and reuse" behavior; + * a stale socketId surfaces as a normal API error handled by _apiCall. + */ + async _ensureSecretSocketId(appId) { + if (!appId) return; + if (this.getSecretSocketId(appId)) return; // already cached + if (this._socketIdCaptureAttempted.has(appId)) return; // don't nav-storm on a miss + this._socketIdCaptureAttempted.add(appId); + + try { + await this.ensureLoggedIn(); // make sure a live page exists to navigate + } catch { + // Couldn't establish a session here — let the upcoming _apiCall surface + // the real error; just skip capture. + return; + } + if (!this.page) return; + + const maxMs = Number(process.env.AIRTABLE_SOCKETID_CAPTURE_MS) || 8000; + const deadline = Date.now() + maxMs; + try { + await this.page + .goto(`https://airtable.com/${appId}`, { waitUntil: 'domcontentloaded', timeout: maxMs }) + .catch(() => {}); // navigation errors are non-fatal — the base may still POST + while (Date.now() < deadline) { + if (this.getSecretSocketId(appId)) return; + await new Promise((r) => setTimeout(r, Math.min(200, Math.max(10, deadline - Date.now())))); + } + } catch (err) { + console.error(`[auth] secretSocketId capture for ${appId} failed: ${err.message}`); + } + // Still not captured → proceed without it (secretSocketId is "when available"). + } + + // ─── Credential Snapshot (direct-HTTP) ─────────────────────── + + /** + * Snapshot the profile's session credentials so direct-HTTP calls can carry + * the same auth the browser had. Reads the browser context's cookies (incl. + * HttpOnly) and keeps only airtable-domain cookies as a Cookie header, plus + * the CSRF token already extracted by _extractCsrf(). + * + * Called during init (before _verifySession) and after each refresh/recovery + * so a re-login's fresh cookies take effect immediately. + */ + async _snapshotCredentials() { + let cookieHeader = ''; + try { + const cookies = await this.page.context().cookies(); + cookieHeader = cookies + .filter((c) => (c.domain || '').includes('airtable')) + .map((c) => `${c.name}=${c.value}`) + .join('; '); + } catch (err) { + console.error(`[auth] Failed to snapshot session cookies: ${err.message}`); + } + this._credentials = { cookieHeader, csrfToken: this.csrfToken }; + } + // ─── CSRF Extraction ───────────────────────────────────────── async _extractCsrf() { @@ -359,6 +450,10 @@ export class AirtableAuth { } finally { this._initPromise = null; } + // Re-snapshot so the relaunched session's fresh cookies/CSRF drive + // subsequent direct-HTTP calls. (_doInit already snapshots before its + // verify; this is belt-and-suspenders in case that ordering changes.) + await this._snapshotCredentials(); console.error('[auth] Session recovered successfully.'); } finally { this._recovering = false; @@ -414,54 +509,36 @@ export class AirtableAuth { async _rawApiCall(method, urlPath, body = null, appId = null, contentType = 'form', evalTimeoutMs = 15_000) { const url = urlPath.startsWith('http') ? urlPath : `https://airtable.com${urlPath}`; const timeZone = Intl.DateTimeFormat().resolvedOptions().timeZone; - const csrfToken = this.csrfToken; - // Generate page-load-id Node-side — `Math.random()` inside page.evaluate - // is still not cryptographically strong and we can't reach node:crypto - // from the browser context. + // Generate page-load-id Node-side — `Math.random()` is not cryptographically + // strong; genPageLoadId() uses node:crypto. const pageLoadId = genPageLoadId(); - // H3 — per-evaluate timeout. Playwright has no built-in timeout on - // page.evaluate; a CDP-dead Chromium can hang forever. We race against - // a configurable timer (default 15s) and let _apiCall's catch trigger - // _recoverSession. Callers handling large responses (e.g. getApplicationData) - // should pass a higher evalTimeoutMs to avoid spurious recovery cycles. - const withTimeout = (promise) => Promise.race([ - promise, - new Promise((_, reject) => setTimeout( - () => reject(new Error(`page.evaluate exceeded ${evalTimeoutMs / 1000}s; browser may be unresponsive`)), - evalTimeoutMs, - )), - ]); + // Direct HTTP now carries the session explicitly: the browser is no longer + // the transport, so the profile's cookies + CSRF must ride on every request. + const creds = this._credentials || {}; + const cookieHeader = creds.cookieHeader || ''; + const csrfToken = creds.csrfToken ?? this.csrfToken; + // Build the SAME headers the in-page fetch sent, plus an explicit Cookie. + let headers; + let requestBody = null; if (contentType === 'json') { - return withTimeout(this.page.evaluate(async ({ url, method, body, appId, timeZone, pageLoadId }) => { - const headers = { - 'Content-Type': 'application/json', - 'x-airtable-inter-service-client': 'webClient', - 'x-airtable-page-load-id': pageLoadId, - 'x-requested-with': 'XMLHttpRequest', - 'x-user-locale': 'en', - 'x-time-zone': timeZone, - }; - if (appId) headers['x-airtable-application-id'] = appId; - - try { - // Honor caller's method — previously this was hardcoded to POST which - // silently broke any GET/PUT/PATCH JSON call. - const options = { method, headers }; - if (body !== null && body !== undefined && method !== 'GET') { - options.body = JSON.stringify(body); - } - const res = await fetch(url, options); - return { status: res.status, body: await res.text() }; - } catch (e) { - return { status: 0, body: '', error: e.message }; - } - }, { url, method, body, appId, timeZone, pageLoadId })); - } - - return withTimeout(this.page.evaluate(async ({ url, method, body, appId, timeZone, csrfToken, pageLoadId }) => { - const headers = { + headers = { + 'Content-Type': 'application/json', + 'x-airtable-inter-service-client': 'webClient', + 'x-airtable-page-load-id': pageLoadId, + 'x-requested-with': 'XMLHttpRequest', + 'x-user-locale': 'en', + 'x-time-zone': timeZone, + }; + if (appId) headers['x-airtable-application-id'] = appId; + if (cookieHeader) headers['Cookie'] = cookieHeader; + // Honor caller's method — a body only rides on non-GET requests. + if (body !== null && body !== undefined && method !== 'GET') { + requestBody = JSON.stringify(body); + } + } else { + headers = { 'x-airtable-inter-service-client': 'webClient', 'x-airtable-page-load-id': pageLoadId, 'x-requested-with': 'XMLHttpRequest', @@ -473,21 +550,35 @@ export class AirtableAuth { headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8'; headers['Accept'] = 'application/json, text/javascript, */*; q=0.01'; } - - const options = { method, headers }; + if (cookieHeader) headers['Cookie'] = cookieHeader; if (body && method !== 'GET') { const params = new URLSearchParams(body); if (csrfToken) params.set('_csrf', csrfToken); - options.body = params.toString(); + requestBody = params.toString(); } + } - try { - const res = await fetch(url, options); - return { status: res.status, body: await res.text() }; - } catch (e) { - return { status: 0, body: '', error: e.message }; - } - }, { url, method, body, appId, timeZone, csrfToken, pageLoadId })); + // Preserve the old per-call timeout semantics: page.evaluate had no built-in + // timeout, so a hung request was raced against a configurable timer (default + // 15s) and its rejection let _apiCall's catch trigger _recoverSession. Direct + // HTTP keeps the same race — a stuck request still rejects promptly. Callers + // handling large responses (e.g. getApplicationData) pass a higher timeout. + // Clear the timer on settle — otherwise a pending setTimeout dangles for the + // full evalTimeoutMs after every (fast) call, leaking timers now that this + // is the universal request path (the old page.evaluate version left it + // uncleared, but that only ran on the slow browser path). + const withTimeout = (promise) => { + let timer; + const timeout = new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error(`HTTP request exceeded ${evalTimeoutMs / 1000}s; upstream may be unresponsive`)), + evalTimeoutMs, + ); + }); + return Promise.race([promise, timeout]).finally(() => clearTimeout(timer)); + }; + + return withTimeout(this._httpTransport.request({ method, url, headers, body: requestBody })); } // ─── Queued API Call (with session recovery & rate-limit backoff) ────────────────── @@ -633,6 +724,17 @@ export class AirtableAuth { } async postForm(url, params, appId) { + // Mutations (all postForm callers) embed secretSocketId "when available". + // With the direct-HTTP transport the browser no longer sees these POSTs, so + // the passive capture is gone; recover it by lazily navigating the login + // page to the base once, then inject the captured id into the outgoing + // params. If capture misses, the mutation still proceeds without it. + if (appId && params && typeof params === 'object' && !params.secretSocketId) { + await this._ensureSecretSocketId(appId); + const sid = this.getSecretSocketId(appId); + if (sid) params.secretSocketId = sid; + } + const pattern = url.replace(/.*v0\.3\//, '').replace(/(app|tbl|viw|fld|rec|usr|wsp|sel|flt|blk|ext|col)[A-Za-z0-9]{10,}/g, '$1*'); trace('http', 'http:request', { method: 'POST', endpoint_pattern: pattern, has_payload: true }); const start = Date.now(); diff --git a/packages/mcp-server/src/http-transport.js b/packages/mcp-server/src/http-transport.js new file mode 100644 index 0000000..27c3201 --- /dev/null +++ b/packages/mcp-server/src/http-transport.js @@ -0,0 +1,102 @@ +/** + * Direct-HTTP transport for Airtable API calls (Phase 1 of the transport rework). + * + * Every Airtable API call used to run as a `fetch()` INSIDE the Chrome page via + * page.evaluate(); the browser was both the auth store (cookies + scraped CSRF) + * AND the transport. A live spike fired the same request three ways — in-page + * fetch, impit (Chrome TLS impersonation), and plain node fetch — and all three + * returned byte-identical 200s. There is no TLS-fingerprint barrier, so we + * demote the browser to login / credential-capture only and route API traffic + * through this transport. + * + * The return shape mirrors the old in-page fetch EXACTLY so that + * auth._wrapResponse and the _apiCall backoff/recovery loop stay untouched: + * success → { status, body } + * thrown network → { status: 0, body: '', error } + * + * Clients: + * - 'fetch' (default): global fetch. No impersonation needed (spike-proven). + * - 'impit' : opt-in Chrome TLS impersonation, behind + * AIRTABLE_HTTP_CLIENT=impit. impit is an OPTIONAL + * dependency, lazy-loaded like patchright/otpauth; if it + * is not installed we throw a clear, actionable error. + */ + +let _ImpitClass = null; +async function loadImpit() { + if (_ImpitClass) return _ImpitClass; + try { + const mod = await import('impit'); + _ImpitClass = mod.Impit || mod.default?.Impit; + if (!_ImpitClass) throw new Error('the `impit` module did not export `Impit`'); + return _ImpitClass; + } catch (err) { + throw new Error( + 'AIRTABLE_HTTP_CLIENT=impit requires the optional `impit` package, which is not installed. ' + + 'Install it (`npm i impit`) or use the default fetch client ' + + '(unset AIRTABLE_HTTP_CLIENT or set it to "fetch").\n' + + `Original error: ${err.message}` + ); + } +} + +export class HttpTransport { + /** + * @param {object} [opts] + * @param {'fetch'|'impit'} [opts.client='fetch'] transport backend + * @param {Function} [opts.fetchImpl] test seam — overrides global fetch + * @param {Function} [opts.impitFactory] test seam — async () => ({ fetch }) + */ + constructor({ client = 'fetch', fetchImpl, impitFactory } = {}) { + // Only 'impit' opts into the fallback; anything else (undefined, '', a typo) + // falls back to plain fetch. + this.client = client === 'impit' ? 'impit' : 'fetch'; + this._fetchImpl = fetchImpl || null; + this._impitFactory = impitFactory || null; + } + + /** + * @param {{ method: string, url: string, headers: object, body?: string|null }} req + * @returns {Promise<{ status: number, body: string, error?: string }>} + */ + async request(req) { + return this.client === 'impit' ? this._impitRequest(req) : this._fetchRequest(req); + } + + async _fetchRequest({ method, url, headers, body }) { + const fetchFn = this._fetchImpl || globalThis.fetch; + const options = { method, headers }; + if (body !== null && body !== undefined && method !== 'GET') { + options.body = body; + } + try { + const res = await fetchFn(url, options); + return { status: res.status, body: await res.text() }; + } catch (e) { + return { status: 0, body: '', error: e.message }; + } + } + + async _impitRequest({ method, url, headers, body }) { + // A failed import propagates as a thrown error (not a { status: 0 } result): + // an uninstalled optional dependency is a config problem the caller must fix, + // not a transient network failure to retry. + let impit; + if (this._impitFactory) { + impit = await this._impitFactory(); + } else { + const Impit = await loadImpit(); + impit = new Impit({ browser: 'chrome' }); + } + const options = { method, headers }; + if (body !== null && body !== undefined && method !== 'GET') { + options.body = body; + } + try { + const res = await impit.fetch(url, options); + return { status: res.status, body: await res.text() }; + } catch (e) { + return { status: 0, body: '', error: e.message }; + } + } +} From 91e22a5c47bafdc40dd8ec7238a8d30a72ccb5c0 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 18:06:53 +0300 Subject: [PATCH 170/246] test(transport): cover HttpTransport + auth direct-HTTP seam MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - test-http-transport.test.js: client selection, method/url/headers/Cookie/body passthrough, {status,body,error} shape for 200/500/network-throw, global-fetch fallback, impit factory seam + clear install error when impit is absent. - test-auth-transport.test.js: _rawApiCall over a fake HttpTransport asserts Cookie + standard headers + _csrf on form POST, JSON content-type on json POST, GET has no body/_csrf, status passthrough, network-error shape; _apiCall's 429/403 backoff and 401/network recovery still fire over the real _rawApiCall→transport path; _snapshotCredentials cookie filtering + degradation; lazy secretSocketId capture (inject on hit, proceed + no nav-storm on miss). Co-Authored-By: Claude Fable 5 --- .../test/test-auth-transport.test.js | 241 ++++++++++++++++++ .../test/test-http-transport.test.js | 138 ++++++++++ 2 files changed, 379 insertions(+) create mode 100644 packages/mcp-server/test/test-auth-transport.test.js create mode 100644 packages/mcp-server/test/test-http-transport.test.js diff --git a/packages/mcp-server/test/test-auth-transport.test.js b/packages/mcp-server/test/test-auth-transport.test.js new file mode 100644 index 0000000..d556253 --- /dev/null +++ b/packages/mcp-server/test/test-auth-transport.test.js @@ -0,0 +1,241 @@ +import { describe, it, beforeEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { AirtableAuth } from '../src/auth.js'; + +/** + * Phase 1 transport core — auth side. + * + * _rawApiCall now builds the request headers (identical to the old in-page + * fetch) + a Cookie header + (_csrf on form POSTs) from a captured credential + * snapshot, then hands the request to an injectable HttpTransport instead of + * page.evaluate(). These tests exercise the REAL _rawApiCall over a fake + * transport (not a stubbed _rawApiCall), proving the header/cookie/_csrf + * construction AND that the untouched _apiCall backoff/recovery loop still + * fires over the new path. + */ + +function fakeTransport() { + return { + calls: [], + next: null, // a response value, or (req, n) => response + async request(req) { + this.calls.push(req); + const r = typeof this.next === 'function' ? this.next(req, this.calls.length) : this.next; + return r || { status: 200, body: '{}' }; + }, + }; +} + +describe('auth._rawApiCall → HttpTransport', () => { + let auth, transport; + beforeEach(() => { + auth = new AirtableAuth(); + transport = fakeTransport(); + auth._httpTransport = transport; + auth._credentials = { cookieHeader: 'brw=1; sess=2', csrfToken: 'CSRF-XYZ' }; + auth.context = {}; + auth.isLoggedIn = true; + auth.page = { url: () => 'https://airtable.com/' }; + }); + + it('form POST: sends Cookie, the standard headers, and a urlencoded body with _csrf', async () => { + transport.next = { status: 200, body: '{"ok":true}' }; + const out = await auth._rawApiCall('POST', '/v0.3/column/x/create', { name: 'Foo Bar' }, 'appABC123', 'form'); + assert.equal(out.status, 200); + const req = transport.calls[0]; + assert.equal(req.method, 'POST'); + assert.equal(req.url, 'https://airtable.com/v0.3/column/x/create'); + assert.equal(req.headers.Cookie, 'brw=1; sess=2'); + assert.equal(req.headers['x-airtable-inter-service-client'], 'webClient'); + assert.equal(req.headers['x-requested-with'], 'XMLHttpRequest'); + assert.equal(req.headers['x-user-locale'], 'en'); + assert.ok(req.headers['x-airtable-page-load-id']?.startsWith('pgl')); + assert.equal(req.headers['x-airtable-application-id'], 'appABC123'); + assert.match(req.headers['Content-Type'], /x-www-form-urlencoded/); + const params = new URLSearchParams(req.body); + assert.equal(params.get('name'), 'Foo Bar'); + assert.equal(params.get('_csrf'), 'CSRF-XYZ'); + }); + + it('GET: no body, no _csrf, still sends Cookie + the standard headers', async () => { + transport.next = { status: 200, body: '{}' }; + await auth._rawApiCall('GET', '/v0.3/getUserProperties', null, 'appABC123', 'form'); + const req = transport.calls[0]; + assert.equal(req.method, 'GET'); + assert.equal(req.headers.Cookie, 'brw=1; sess=2'); + assert.equal(req.headers['x-airtable-application-id'], 'appABC123'); + assert.equal(req.body === null || req.body === undefined, true); + assert.equal('Content-Type' in req.headers, false); + }); + + it('json POST: application/json Content-Type, JSON body, and Cookie', async () => { + transport.next = { status: 200, body: '{}' }; + await auth._rawApiCall('POST', '/v0.3/attachments/urlUpload', { a: 1, b: 'x' }, 'appABC123', 'json'); + const req = transport.calls[0]; + assert.equal(req.headers['Content-Type'], 'application/json'); + assert.equal(req.headers.Cookie, 'brw=1; sess=2'); + assert.deepEqual(JSON.parse(req.body), { a: 1, b: 'x' }); + }); + + it('builds an absolute URL only for path inputs, leaving full URLs intact', async () => { + transport.next = { status: 200, body: '{}' }; + await auth._rawApiCall('GET', 'https://airtable.com/v0.3/already/full'); + assert.equal(transport.calls[0].url, 'https://airtable.com/v0.3/already/full'); + }); + + it('passes the transport status straight through (500)', async () => { + transport.next = { status: 500, body: 'err' }; + const out = await auth._rawApiCall('GET', '/v0.3/x'); + assert.deepEqual(out, { status: 500, body: 'err' }); + }); + + it('surfaces the transport network-error shape ({status:0, error})', async () => { + transport.next = { status: 0, body: '', error: 'boom' }; + const out = await auth._rawApiCall('GET', '/v0.3/x'); + assert.equal(out.status, 0); + assert.equal(out.error, 'boom'); + }); +}); + +describe('auth._apiCall backoff/recovery over the real _rawApiCall → transport path', () => { + let auth, transport; + beforeEach(() => { + auth = new AirtableAuth(); + transport = fakeTransport(); + auth._httpTransport = transport; + auth._credentials = { cookieHeader: 'c=1', csrfToken: 'z' }; + auth.context = {}; + auth.isLoggedIn = true; + auth.page = { url: () => 'https://airtable.com/' }; + auth._rlSleep = async () => {}; // no real backoff waits + }); + + it('429 then 200 → backs off and retries on the same session (no recovery)', async () => { + let recovered = false; + auth._recoverSession = async () => { recovered = true; }; + let n = 0; + transport.next = () => (++n === 1 ? { status: 429, body: '' } : { status: 200, body: '{"ok":true}' }); + const res = await auth._apiCall('GET', '/v0.3/x', null, 'app1'); + assert.equal(res.status, 200); + assert.equal(n, 2); + assert.equal(recovered, false); + }); + + it('403 (throttle) then 200 → backs off, no relaunch', async () => { + let n = 0; + transport.next = () => (++n === 1 ? { status: 403, body: '' } : { status: 200, body: '{}' }); + const res = await auth._apiCall('GET', '/v0.3/x', null, 'app1'); + assert.equal(res.status, 200); + assert.equal(n, 2); + }); + + it('401 → triggers _recoverSession, then succeeds', async () => { + let recovered = false; + auth._recoverSession = async () => { recovered = true; }; + let n = 0; + transport.next = () => (++n === 1 ? { status: 401, body: '' } : { status: 200, body: '{}' }); + const res = await auth._apiCall('GET', '/v0.3/x', null, 'app1'); + assert.equal(recovered, true); + assert.equal(res.status, 200); + }); + + it('a transport network error ({status:0, error}) triggers _recoverSession', async () => { + let recovered = false; + auth._recoverSession = async () => { recovered = true; }; + let n = 0; + transport.next = () => (++n === 1 ? { status: 0, body: '', error: 'net down' } : { status: 200, body: '{}' }); + const res = await auth._apiCall('GET', '/v0.3/x', null, 'app1'); + assert.equal(recovered, true); + assert.equal(res.status, 200); + }); +}); + +describe('auth._snapshotCredentials', () => { + it('builds a Cookie header from airtable-domain cookies and caches the csrf token', async () => { + const auth = new AirtableAuth(); + auth.csrfToken = 'CSRF-1'; + auth.page = { + context: () => ({ + cookies: async () => [ + { name: 'brw', value: '1', domain: '.airtable.com' }, + { name: 'sess', value: '2', domain: 'airtable.com' }, + { name: 'other', value: 'x', domain: '.google.com' }, // must be excluded + ], + }), + }; + await auth._snapshotCredentials(); + assert.equal(auth._credentials.csrfToken, 'CSRF-1'); + const jar = auth._credentials.cookieHeader.split('; ').sort(); + assert.deepEqual(jar, ['brw=1', 'sess=2']); + assert.ok(!auth._credentials.cookieHeader.includes('other')); + }); + + it('degrades to an empty cookie header if reading cookies throws', async () => { + const auth = new AirtableAuth(); + auth.csrfToken = 'CSRF-2'; + auth.page = { context: () => ({ cookies: async () => { throw new Error('ctx dead'); } }) }; + await auth._snapshotCredentials(); + assert.equal(auth._credentials.cookieHeader, ''); + assert.equal(auth._credentials.csrfToken, 'CSRF-2'); + }); +}); + +describe('auth lazy secretSocketId capture (postForm)', () => { + let auth, transport; + beforeEach(() => { + auth = new AirtableAuth(); + transport = fakeTransport(); + auth._httpTransport = transport; + auth._credentials = { cookieHeader: 'c=1', csrfToken: 'z' }; + auth.context = {}; + auth.isLoggedIn = true; + }); + + it('navigates to the base once, captures a socketId, and injects it into the params', async () => { + let gotoCount = 0; + auth.page = { + url: () => 'https://airtable.com/', + goto: async (u) => { + gotoCount++; + // Simulate the request-interception handler observing a base POST. + const m = u.match(/(app[A-Za-z0-9]+)/); + if (m) auth._secretSocketIds.set(m[1], 'SID-123'); + }, + }; + transport.next = { status: 200, body: '{"ok":true}' }; + const params = { stringifiedObjectParams: '{}', requestId: 'r1' }; + await auth.postForm('https://airtable.com/v0.3/column/x/create', params, 'appABC123'); + assert.equal(params.secretSocketId, 'SID-123'); + const body = new URLSearchParams(transport.calls[0].body); + assert.equal(body.get('secretSocketId'), 'SID-123'); + assert.equal(gotoCount, 1); + + // A second mutation on the same base reuses the cache — no re-navigation. + const params2 = { stringifiedObjectParams: '{}', requestId: 'r2', secretSocketId: auth.getSecretSocketId('appABC123') }; + await auth.postForm('https://airtable.com/v0.3/column/y/create', params2, 'appABC123'); + assert.equal(gotoCount, 1); + }); + + it('does not fail the call when capture times out — proceeds without a socketId and does not nav-storm', async () => { + process.env.AIRTABLE_SOCKETID_CAPTURE_MS = '40'; + let gotoCount = 0; + auth.page = { + url: () => 'https://airtable.com/', + goto: async () => { gotoCount++; /* never captures */ }, + }; + try { + transport.next = { status: 200, body: '{"ok":true}' }; + const params = { stringifiedObjectParams: '{}', requestId: 'r1' }; + const res = await auth.postForm('https://airtable.com/v0.3/column/x/create', params, 'appABC123'); + assert.equal(res.status, 200); + assert.equal('secretSocketId' in params, false); + + // Attempt-guard: a later mutation must not navigate again after a miss. + const res2 = await auth.postForm('https://airtable.com/v0.3/column/y/create', { stringifiedObjectParams: '{}', requestId: 'r2' }, 'appABC123'); + assert.equal(res2.status, 200); + assert.equal(gotoCount, 1); + } finally { + delete process.env.AIRTABLE_SOCKETID_CAPTURE_MS; + } + }); +}); diff --git a/packages/mcp-server/test/test-http-transport.test.js b/packages/mcp-server/test/test-http-transport.test.js new file mode 100644 index 0000000..2ef9518 --- /dev/null +++ b/packages/mcp-server/test/test-http-transport.test.js @@ -0,0 +1,138 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { HttpTransport } from '../src/http-transport.js'; + +/** + * Phase 1 transport core. HttpTransport routes an Airtable API call through + * direct node HTTP (fetch by default, impit as an opt-in fingerprinting + * fallback) instead of an in-page page.evaluate(fetch). Its return shape must + * mirror the old in-page fetch EXACTLY so auth._wrapResponse and the _apiCall + * backoff loop stay untouched: + * success → { status, body } + * throw → { status: 0, body: '', error } + */ + +// A recording fake fetch. `response` may be a value or a (url, options) => value fn. +function fakeFetch(recorder, response) { + return async (url, options) => { + recorder.url = url; + recorder.options = options; + return typeof response === 'function' ? response(url, options) : response; + }; +} + +describe('HttpTransport — client selection', () => { + it('defaults to the fetch client', () => { + assert.equal(new HttpTransport().client, 'fetch'); + }); + it('treats undefined / empty env value as fetch', () => { + assert.equal(new HttpTransport({ client: undefined }).client, 'fetch'); + assert.equal(new HttpTransport({ client: '' }).client, 'fetch'); + assert.equal(new HttpTransport({ client: 'bogus' }).client, 'fetch'); + }); + it('honours client: "impit"', () => { + assert.equal(new HttpTransport({ client: 'impit' }).client, 'impit'); + }); +}); + +describe('HttpTransport — fetch client', () => { + it('passes method/url/headers/body through and returns {status, body} on 200', async () => { + const rec = {}; + const t = new HttpTransport({ fetchImpl: fakeFetch(rec, { status: 200, text: async () => 'OK' }) }); + const out = await t.request({ + method: 'POST', + url: 'https://airtable.com/v0.3/x', + headers: { Cookie: 'a=b', 'x-test': '1' }, + body: 'foo=bar', + }); + assert.deepEqual(out, { status: 200, body: 'OK' }); + assert.equal(rec.url, 'https://airtable.com/v0.3/x'); + assert.equal(rec.options.method, 'POST'); + assert.equal(rec.options.headers.Cookie, 'a=b'); + assert.equal(rec.options.headers['x-test'], '1'); + assert.equal(rec.options.body, 'foo=bar'); + }); + + it('does not attach a body on GET', async () => { + const rec = {}; + const t = new HttpTransport({ fetchImpl: fakeFetch(rec, { status: 200, text: async () => '{}' }) }); + await t.request({ method: 'GET', url: 'https://airtable.com/v0.3/y', headers: {}, body: null }); + assert.equal('body' in rec.options, false); + }); + + it('passes a non-2xx status (500) straight through with its body', async () => { + const rec = {}; + const t = new HttpTransport({ fetchImpl: fakeFetch(rec, { status: 500, text: async () => 'boom' }) }); + const out = await t.request({ method: 'GET', url: 'u', headers: {} }); + assert.deepEqual(out, { status: 500, body: 'boom' }); + }); + + it('maps a network throw to { status: 0, body: "", error } (old page.evaluate catch shape)', async () => { + const t = new HttpTransport({ fetchImpl: async () => { throw new Error('ECONNREFUSED'); } }); + const out = await t.request({ method: 'GET', url: 'u', headers: {} }); + assert.deepEqual(out, { status: 0, body: '', error: 'ECONNREFUSED' }); + }); + + it('falls back to a monkeypatched global fetch when no seam is given', async () => { + const orig = globalThis.fetch; + globalThis.fetch = async () => ({ status: 201, text: async () => 'created' }); + try { + const out = await new HttpTransport().request({ method: 'POST', url: 'u', headers: {}, body: 'x=1' }); + assert.deepEqual(out, { status: 201, body: 'created' }); + } finally { + globalThis.fetch = orig; + } + }); +}); + +describe('HttpTransport — impit client', () => { + it('uses the impit factory seam and returns {status, body}', async () => { + let seen; + const t = new HttpTransport({ + client: 'impit', + impitFactory: async () => ({ + fetch: async (url, options) => { seen = { url, options }; return { status: 200, text: async () => 'impit-ok' }; }, + }), + }); + const out = await t.request({ + method: 'POST', + url: 'https://airtable.com/z', + headers: { Cookie: 'c=d' }, + body: 'k=v', + }); + assert.deepEqual(out, { status: 200, body: 'impit-ok' }); + assert.equal(seen.options.headers.Cookie, 'c=d'); + assert.equal(seen.options.body, 'k=v'); + assert.equal(seen.options.method, 'POST'); + }); + + it('maps an impit network throw to { status: 0, body: "", error }', async () => { + const t = new HttpTransport({ + client: 'impit', + impitFactory: async () => ({ fetch: async () => { throw new Error('tls fail'); } }), + }); + const out = await t.request({ method: 'GET', url: 'u', headers: {} }); + assert.deepEqual(out, { status: 0, body: '', error: 'tls fail' }); + }); + + it('throws a clear "install impit" error when impit is absent (or works when present)', async () => { + // impit is an OPTIONAL dependency and is not installed in this workspace, + // so the real dynamic import fails → the transport must surface a clear, + // actionable error rather than a cryptic module-not-found. Guarded so the + // test stays correct even if a developer later installs impit locally. + let impitInstalled = true; + try { await import('impit'); } catch { impitInstalled = false; } + const t = new HttpTransport({ client: 'impit' }); + if (!impitInstalled) { + await assert.rejects( + () => t.request({ method: 'GET', url: 'https://airtable.com/', headers: {} }), + /impit/i, + ); + } else { + // impit present → a bad host must surface the network-error shape, never + // the install error. + const out = await t.request({ method: 'GET', url: 'http://127.0.0.1:0/', headers: {} }); + assert.equal(out.status, 0); + } + }); +}); From 9af660109b84efa69252667c173200faadac3335 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 18:54:44 +0300 Subject: [PATCH 171/246] refactor(auth): drop blocking secretSocketId nav-capture (live-proven unneeded) Live smoke confirmed a field create succeeds via direct-HTTP without a secretSocketId, and the login-page->base nav-capture returned nothing while adding ~8s. Remove the blocking _ensureSecretSocketId nav-capture, the _socketIdCaptureAttempted guard set, and AIRTABLE_SOCKETID_CAPTURE_MS handling. postForm still injects a passively-captured id when one exists; the passive network-interception handler is kept (harmless, may capture during login). Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/auth.js | 60 ++----------------- .../test/test-auth-transport.test.js | 55 +++++------------ 2 files changed, 20 insertions(+), 95 deletions(-) diff --git a/packages/mcp-server/src/auth.js b/packages/mcp-server/src/auth.js index 5424912..dce56fb 100644 --- a/packages/mcp-server/src/auth.js +++ b/packages/mcp-server/src/auth.js @@ -67,9 +67,6 @@ export class AirtableAuth { // indefinitely. this._secretSocketIds = new Map(); this._maxSocketIdCache = 50; - // Per-app guard so a base whose socketId never captures isn't re-navigated - // on every subsequent mutation (avoid a lazy-capture navigation storm). - this._socketIdCaptureAttempted = new Set(); // Direct-HTTP transport. API calls now go straight out over node HTTP // (fetch by default; impit via AIRTABLE_HTTP_CLIENT=impit) using the @@ -291,54 +288,6 @@ export class AirtableAuth { return this._secretSocketIds.get(appId) || null; } - /** - * Lazily capture a secretSocketId for a base before a mutation. - * - * With the direct-HTTP transport the browser no longer sees API POSTs, so the - * passive interception that used to capture secretSocketId "for free" never - * fires. We restore it on demand: navigate the still-open login page to the - * base once — loading the base fires its own POSTs, and the existing request - * interception handler captures the socketId from them — then poll briefly. - * - * Best-effort by design (secretSocketId is applied "when available"): if the - * capture times out we proceed WITHOUT it rather than fail the mutation. A - * per-app guard prevents re-navigating on every mutation when capture misses. - * - * NOTE: captured socketIds are reused for the whole session (no page traffic - * to refresh them). This matches the pre-existing "cache and reuse" behavior; - * a stale socketId surfaces as a normal API error handled by _apiCall. - */ - async _ensureSecretSocketId(appId) { - if (!appId) return; - if (this.getSecretSocketId(appId)) return; // already cached - if (this._socketIdCaptureAttempted.has(appId)) return; // don't nav-storm on a miss - this._socketIdCaptureAttempted.add(appId); - - try { - await this.ensureLoggedIn(); // make sure a live page exists to navigate - } catch { - // Couldn't establish a session here — let the upcoming _apiCall surface - // the real error; just skip capture. - return; - } - if (!this.page) return; - - const maxMs = Number(process.env.AIRTABLE_SOCKETID_CAPTURE_MS) || 8000; - const deadline = Date.now() + maxMs; - try { - await this.page - .goto(`https://airtable.com/${appId}`, { waitUntil: 'domcontentloaded', timeout: maxMs }) - .catch(() => {}); // navigation errors are non-fatal — the base may still POST - while (Date.now() < deadline) { - if (this.getSecretSocketId(appId)) return; - await new Promise((r) => setTimeout(r, Math.min(200, Math.max(10, deadline - Date.now())))); - } - } catch (err) { - console.error(`[auth] secretSocketId capture for ${appId} failed: ${err.message}`); - } - // Still not captured → proceed without it (secretSocketId is "when available"). - } - // ─── Credential Snapshot (direct-HTTP) ─────────────────────── /** @@ -725,12 +674,11 @@ export class AirtableAuth { async postForm(url, params, appId) { // Mutations (all postForm callers) embed secretSocketId "when available". - // With the direct-HTTP transport the browser no longer sees these POSTs, so - // the passive capture is gone; recover it by lazily navigating the login - // page to the base once, then inject the captured id into the outgoing - // params. If capture misses, the mutation still proceeds without it. + // Live smoke proved a field create SUCCEEDS via direct-HTTP WITHOUT a + // secretSocketId, and the login-page→base nav-capture returned nothing while + // adding ~8s — so we no longer navigate to capture one. We still inject an + // id if the passive login-time interception happened to capture it. if (appId && params && typeof params === 'object' && !params.secretSocketId) { - await this._ensureSecretSocketId(appId); const sid = this.getSecretSocketId(appId); if (sid) params.secretSocketId = sid; } diff --git a/packages/mcp-server/test/test-auth-transport.test.js b/packages/mcp-server/test/test-auth-transport.test.js index d556253..cb8a81f 100644 --- a/packages/mcp-server/test/test-auth-transport.test.js +++ b/packages/mcp-server/test/test-auth-transport.test.js @@ -180,7 +180,7 @@ describe('auth._snapshotCredentials', () => { }); }); -describe('auth lazy secretSocketId capture (postForm)', () => { +describe('auth passive secretSocketId injection (postForm)', () => { let auth, transport; beforeEach(() => { auth = new AirtableAuth(); @@ -189,53 +189,30 @@ describe('auth lazy secretSocketId capture (postForm)', () => { auth._credentials = { cookieHeader: 'c=1', csrfToken: 'z' }; auth.context = {}; auth.isLoggedIn = true; + // No page.goto stub on purpose — postForm must NEVER navigate to capture a + // socketId (the blocking nav-capture was removed; live smoke proved mutations + // succeed without it). + auth.page = { url: () => 'https://airtable.com/' }; }); - it('navigates to the base once, captures a socketId, and injects it into the params', async () => { - let gotoCount = 0; - auth.page = { - url: () => 'https://airtable.com/', - goto: async (u) => { - gotoCount++; - // Simulate the request-interception handler observing a base POST. - const m = u.match(/(app[A-Za-z0-9]+)/); - if (m) auth._secretSocketIds.set(m[1], 'SID-123'); - }, - }; + it('injects a passively-captured socketId into the params when one exists', async () => { + auth._secretSocketIds.set('appABC123', 'SID-123'); // as if captured during login transport.next = { status: 200, body: '{"ok":true}' }; const params = { stringifiedObjectParams: '{}', requestId: 'r1' }; await auth.postForm('https://airtable.com/v0.3/column/x/create', params, 'appABC123'); assert.equal(params.secretSocketId, 'SID-123'); const body = new URLSearchParams(transport.calls[0].body); assert.equal(body.get('secretSocketId'), 'SID-123'); - assert.equal(gotoCount, 1); - - // A second mutation on the same base reuses the cache — no re-navigation. - const params2 = { stringifiedObjectParams: '{}', requestId: 'r2', secretSocketId: auth.getSecretSocketId('appABC123') }; - await auth.postForm('https://airtable.com/v0.3/column/y/create', params2, 'appABC123'); - assert.equal(gotoCount, 1); }); - it('does not fail the call when capture times out — proceeds without a socketId and does not nav-storm', async () => { - process.env.AIRTABLE_SOCKETID_CAPTURE_MS = '40'; - let gotoCount = 0; - auth.page = { - url: () => 'https://airtable.com/', - goto: async () => { gotoCount++; /* never captures */ }, - }; - try { - transport.next = { status: 200, body: '{"ok":true}' }; - const params = { stringifiedObjectParams: '{}', requestId: 'r1' }; - const res = await auth.postForm('https://airtable.com/v0.3/column/x/create', params, 'appABC123'); - assert.equal(res.status, 200); - assert.equal('secretSocketId' in params, false); - - // Attempt-guard: a later mutation must not navigate again after a miss. - const res2 = await auth.postForm('https://airtable.com/v0.3/column/y/create', { stringifiedObjectParams: '{}', requestId: 'r2' }, 'appABC123'); - assert.equal(res2.status, 200); - assert.equal(gotoCount, 1); - } finally { - delete process.env.AIRTABLE_SOCKETID_CAPTURE_MS; - } + it('proceeds WITHOUT a socketId (and never navigates) when none was captured', async () => { + let gotoCalled = false; + auth.page.goto = async () => { gotoCalled = true; }; + transport.next = { status: 200, body: '{"ok":true}' }; + const params = { stringifiedObjectParams: '{}', requestId: 'r1' }; + const res = await auth.postForm('https://airtable.com/v0.3/column/x/create', params, 'appABC123'); + assert.equal(res.status, 200); + assert.equal('secretSocketId' in params, false); + assert.equal(gotoCalled, false); }); }); From da34bd166d6c3d6997b1ac08a70f668b2588a681 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 18:58:34 +0300 Subject: [PATCH 172/246] =?UTF-8?q?feat(auth):=20AIRTABLE=5FAUTH=5FMODE=3D?= =?UTF-8?q?byo=20=E2=80=94=20cookie-only=20credentials,=20no=20browser?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New src/byo-credentials.js resolves a raw session Cookie header + csrf token from AIRTABLE_COOKIE/AIRTABLE_CSRF or ~/.airtable-user-mcp/credentials.json, scraping csrf from a GET of airtable.com when absent (same patterns as _extractCsrf). Pure/injectable via a fetchImpl seam. auth.js branches on AIRTABLE_AUTH_MODE (default 'browser', unchanged): - 'byo' skips Chromium entirely — _doInitByo loads creds, verifies via a direct-HTTP getUserProperties (rejected cookie → BYO_CREDENTIALS_INVALID), and drives all calls with the pasted cookie. - _recoverSession re-loads creds on a mid-run rejection (still invalid → BYO_CREDENTIALS_EXPIRED); ensureLoggedIn no-ops once loaded. - browser-only helpers (_snapshotCredentials) guarded against a null page. - 'direct-login' branch reserved for Phase B (throws not-yet-available). Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/auth.js | 77 +++++++++++ packages/mcp-server/src/byo-credentials.js | 118 ++++++++++++++++ .../test/test-auth-transport.test.js | 119 +++++++++++++++++ .../test/test-byo-credentials.test.js | 126 ++++++++++++++++++ 4 files changed, 440 insertions(+) create mode 100644 packages/mcp-server/src/byo-credentials.js create mode 100644 packages/mcp-server/test/test-byo-credentials.test.js diff --git a/packages/mcp-server/src/auth.js b/packages/mcp-server/src/auth.js index dce56fb..3685e3f 100644 --- a/packages/mcp-server/src/auth.js +++ b/packages/mcp-server/src/auth.js @@ -4,6 +4,7 @@ import { randomBytes } from 'node:crypto'; import { trace } from './debug-tracer.js'; import { getProfileDir } from './paths.js'; import { HttpTransport } from './http-transport.js'; +import { loadByoCredentials } from './byo-credentials.js'; /** Generate a page-load-id (pgl + 13 base36 chars) using crypto, not Math.random. */ function genPageLoadId() { @@ -54,6 +55,12 @@ export class AirtableAuth { this.userId = null; this.csrfToken = null; + // Credential source. 'browser' (default) launches Chromium and mints + // cookies+CSRF from a persistent profile. 'byo' skips the browser entirely + // and uses a pasted session cookie (AIRTABLE_COOKIE / credentials.json). + // 'direct-login' (Phase B) will replicate the HAR login flow over HTTP. + this._authMode = (process.env.AIRTABLE_AUTH_MODE || 'browser').toLowerCase(); + // Request queue — serializes all browser-backed API calls. // Bounded so a runaway LLM loop cannot blow up memory even when the // upstream semaphore lets thousands of tool calls through over time. @@ -143,6 +150,17 @@ export class AirtableAuth { } async _doInit() { + // ── Browser-free credential modes ────────────────────────────── + // These never launch Chromium: there is no this.page/this.context. All + // browser-touching helpers below must therefore be guarded against a null + // page (see _snapshotCredentials / ensureLoggedIn / _recoverSession / close). + if (this._authMode === 'byo') { + return this._doInitByo(); + } + if (this._authMode === 'direct-login') { + throw new Error('direct-login mode not yet available'); + } + // Close existing context if recovering. Detach the prior network listener // from the old page so we don't leak a handler per recovery cycle. if (this.context) { @@ -208,6 +226,35 @@ export class AirtableAuth { } } + /** + * BYO (bring-your-own cookie) init: no browser. Load the pasted cookie+csrf, + * mark logged-in, then verify the cookie is actually accepted by a real + * direct-HTTP call. A rejected cookie fails fast with an actionable error + * rather than letting every subsequent tool call 401. + */ + async _doInitByo() { + console.error('[auth] AIRTABLE_AUTH_MODE=byo — using pasted cookie credentials (no browser).'); + this._credentials = await loadByoCredentials(); + this.csrfToken = this._credentials.csrfToken ?? null; + this.isLoggedIn = true; + + // Direct-HTTP validity check. _rawApiCall reads this._credentials. + const result = await this._rawApiCall('GET', '/v0.3/getUserProperties'); + if (!(result.status >= 200 && result.status < 300)) { + this.isLoggedIn = false; + throw new Error( + `BYO_CREDENTIALS_INVALID: the provided AIRTABLE_COOKIE/credentials.json cookie was rejected (status ${result.status}) — refresh it from a logged-in browser` + ); + } + try { + const data = JSON.parse(result.body); + this.userId = data?.data?.userId || null; + } catch { + this.userId = null; + } + console.error('[auth] BYO session verified!', this.userId ? `User: ${this.userId}` : '(userId not in payload)'); + } + /** * Wait for the Airtable app to be ready instead of a blind 5s timeout. * Looks for signs of a loaded SPA: __NEXT_DATA__, a logged-in avatar, @@ -300,6 +347,9 @@ export class AirtableAuth { * so a re-login's fresh cookies take effect immediately. */ async _snapshotCredentials() { + // Browser-only. In byo/direct-login modes there is no page — never wipe the + // pasted credentials with an empty snapshot. + if (!this.page) return; let cookieHeader = ''; try { const cookies = await this.page.context().cookies(); @@ -387,6 +437,24 @@ export class AirtableAuth { if (this._recovering) return; this._recovering = true; try { + // BYO mode has no browser to relaunch. Re-load the credentials (the user + // may have refreshed the file/env with a fresh cookie) and re-verify. + if (this._authMode === 'byo') { + console.error('[auth] BYO session rejected. Re-loading credentials...'); + this.isLoggedIn = false; + this._credentials = await loadByoCredentials(); + this.csrfToken = this._credentials.csrfToken ?? null; + const result = await this._rawApiCall('GET', '/v0.3/getUserProperties'); + if (!(result.status >= 200 && result.status < 300)) { + throw new Error( + 'BYO_CREDENTIALS_EXPIRED: cookie no longer valid — paste a fresh cookie into ~/.airtable-user-mcp/credentials.json or AIRTABLE_COOKIE' + ); + } + this.isLoggedIn = true; + console.error('[auth] BYO session recovered with re-loaded credentials.'); + return; + } + console.error('[auth] Session expired. Attempting recovery...'); this.isLoggedIn = false; // Atomic assignment — init() observes this as already-in-progress and @@ -611,6 +679,15 @@ export class AirtableAuth { // ─── Public API ─────────────────────────────────────────────── async ensureLoggedIn() { + // BYO mode: no browser context/page to inspect for a redirect. Initialize + // once (loads + verifies the cookie); thereafter it's a no-op — a mid-run + // 401 is handled by _apiCall's backoff → _recoverSession (which re-loads the + // credentials). + if (this._authMode === 'byo') { + if (!this._credentials) await this.init(); + return; + } + if (!this.context || !this.isLoggedIn) { await this.init(); return; diff --git a/packages/mcp-server/src/byo-credentials.js b/packages/mcp-server/src/byo-credentials.js new file mode 100644 index 0000000..0ddcfff --- /dev/null +++ b/packages/mcp-server/src/byo-credentials.js @@ -0,0 +1,118 @@ +/** + * Bring-your-own (BYO) credentials for the browser-free auth mode. + * + * When AIRTABLE_AUTH_MODE=byo, the server never launches Chromium. Instead the + * user pastes a raw session Cookie header (mint it from a logged-in browser's + * devtools) and we drive all API calls through the direct-HTTP transport with + * that cookie. Airtable auth is COOKIE-ONLY (HAR-verified: no bearer/api-key) — + * the essential cookies are __Host-airtable-session + .sig + brw (~3KB total). + * + * The CSRF token is required for form POSTs (mutations). If the user supplies it + * we use it verbatim; otherwise we scrape it from a GET of any authed Airtable + * page — the same token embedded in the /login HTML that the browser flow reads. + * + * Source order (first hit wins for the cookie): + * 1. env AIRTABLE_COOKIE (+ optional AIRTABLE_CSRF) + * 2. file ~/.airtable-user-mcp/credentials.json { cookie, csrf? } + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { getHomeDir } from './paths.js'; + +/** Path to the BYO credentials file. */ +export function getCredentialsPath() { + return path.join(getHomeDir(), 'credentials.json'); +} + +/** + * Scrape a csrfToken out of an authed Airtable page's HTML using the same + * strategies as auth._extractCsrf (which reads them from the live DOM): + * 1. any script/window JSON blob containing "csrfToken":"…" + * 2. + * Returns the token string, or null if none matched. + */ +export function scrapeCsrf(html) { + if (!html || typeof html !== 'string') return null; + // 1. Script-tag / window.* JSON: "csrfToken":"…" + const json = html.match(/"csrfToken"\s*:\s*"([^"]+)"/); + if (json) return json[1]; + // 2. (either attribute order) + const metaNameFirst = html.match(/]+name=["']csrf-token["'][^>]*content=["']([^"']+)["']/i); + if (metaNameFirst) return metaNameFirst[1]; + const metaContentFirst = html.match(/]+content=["']([^"']+)["'][^>]*name=["']csrf-token["']/i); + if (metaContentFirst) return metaContentFirst[1]; + return null; +} + +/** + * Resolve a BYO cookie header + csrf token. + * + * @param {object} [opts] + * @param {Function} [opts.fetchImpl] test seam — overrides global fetch for the + * csrf-scrape GET. + * @returns {Promise<{cookieHeader: string, csrfToken: string|null}>} + */ +export async function loadByoCredentials({ fetchImpl } = {}) { + let cookieHeader = null; + let csrfToken = null; + + // 1. Environment. + const envCookie = process.env.AIRTABLE_COOKIE; + if (envCookie && envCookie.trim()) { + cookieHeader = envCookie.trim(); + const envCsrf = process.env.AIRTABLE_CSRF; + if (envCsrf && envCsrf.trim()) csrfToken = envCsrf.trim(); + } else { + // 2. File. + const file = getCredentialsPath(); + let raw; + try { + raw = fs.readFileSync(file, 'utf8'); + } catch { + raw = null; + } + if (raw) { + let parsed; + try { + parsed = JSON.parse(raw); + } catch (err) { + throw new Error( + `BYO credentials file ${file} is not valid JSON: ${err.message}. ` + + 'Expected { "cookie": "", "csrf"?: "" }.' + ); + } + if (parsed && typeof parsed.cookie === 'string' && parsed.cookie.trim()) { + cookieHeader = parsed.cookie.trim(); + if (typeof parsed.csrf === 'string' && parsed.csrf.trim()) csrfToken = parsed.csrf.trim(); + } + } + } + + if (!cookieHeader) { + throw new Error( + 'BYO_CREDENTIALS_MISSING: AIRTABLE_AUTH_MODE=byo requires a session cookie, but none was found. ' + + 'Set AIRTABLE_COOKIE to a raw Cookie header string (from a logged-in browser\'s devtools), ' + + `or write ${getCredentialsPath()} as { "cookie": "", "csrf"?: "" }.` + ); + } + + // Scrape csrf if not provided — GETs work without it, but form POSTs need it. + if (!csrfToken) { + const fetchFn = fetchImpl || globalThis.fetch; + try { + const res = await fetchFn('https://airtable.com/', { headers: { Cookie: cookieHeader } }); + const html = await res.text(); + csrfToken = scrapeCsrf(html); + } catch (err) { + console.error(`[byo] Failed to fetch airtable.com for csrf scrape: ${err.message}`); + } + if (!csrfToken) { + console.error( + '[byo] WARNING: no csrfToken (none supplied and scrape failed). GET requests will work, ' + + 'but form POSTs (mutations) may be rejected. Add "csrf" to your credentials or AIRTABLE_CSRF.' + ); + } + } + + return { cookieHeader, csrfToken: csrfToken ?? null }; +} diff --git a/packages/mcp-server/test/test-auth-transport.test.js b/packages/mcp-server/test/test-auth-transport.test.js index cb8a81f..5c90647 100644 --- a/packages/mcp-server/test/test-auth-transport.test.js +++ b/packages/mcp-server/test/test-auth-transport.test.js @@ -180,6 +180,125 @@ describe('auth._snapshotCredentials', () => { }); }); +describe('auth AIRTABLE_AUTH_MODE=byo', () => { + const ENV = ['AIRTABLE_AUTH_MODE', 'AIRTABLE_COOKIE', 'AIRTABLE_CSRF']; + let saved; + beforeEach(() => { + saved = {}; + for (const k of ENV) saved[k] = process.env[k]; + for (const k of ENV) delete process.env[k]; + }); + const restore = () => { + for (const k of ENV) { if (saved[k] === undefined) delete process.env[k]; else process.env[k] = saved[k]; } + }; + + function byoAuth(transport) { + // AIRTABLE_AUTH_MODE is read in the constructor, so env must be set first. + process.env.AIRTABLE_AUTH_MODE = 'byo'; + const auth = new AirtableAuth(); + auth._httpTransport = transport; + return auth; + } + + it('init loads pasted creds and verifies WITHOUT launching a browser', async () => { + process.env.AIRTABLE_COOKIE = 'brw=byo; __Host-airtable-session=sess'; + process.env.AIRTABLE_CSRF = 'BYO-CSRF'; // avoids a network scrape + const transport = fakeTransport(); + transport.next = { status: 200, body: JSON.stringify({ data: { userId: 'usr999' } }) }; + try { + const auth = byoAuth(transport); + await auth.init(); + // No browser: context/page stay null, but the session is logged in. + assert.equal(auth.context, null); + assert.equal(auth.page, null); + assert.equal(auth.isLoggedIn, true); + assert.equal(auth.userId, 'usr999'); + // The verify GET carried the byo cookie via the direct-HTTP transport. + assert.equal(transport.calls[0].method, 'GET'); + assert.match(transport.calls[0].url, /getUserProperties/); + assert.equal(transport.calls[0].headers.Cookie, 'brw=byo; __Host-airtable-session=sess'); + } finally { + restore(); + } + }); + + it('a rejected cookie throws BYO_CREDENTIALS_INVALID', async () => { + process.env.AIRTABLE_COOKIE = 'brw=stale'; + process.env.AIRTABLE_CSRF = 'x'; + const transport = fakeTransport(); + transport.next = { status: 401, body: 'nope' }; + try { + const auth = byoAuth(transport); + await assert.rejects(() => auth.init(), /BYO_CREDENTIALS_INVALID.*401/); + assert.equal(auth.isLoggedIn, false); + } finally { + restore(); + } + }); + + it('a subsequent _apiCall carries the byo cookie and needs no browser', async () => { + process.env.AIRTABLE_COOKIE = 'brw=byo2'; + process.env.AIRTABLE_CSRF = 'c2'; + const transport = fakeTransport(); + transport.next = { status: 200, body: '{"data":{}}' }; + try { + const auth = byoAuth(transport); + const res = await auth.get('/v0.3/x', 'appZZ'); + assert.equal(res.status, 200); + // First call = the init verify, second = our get; both carry the cookie. + assert.ok(transport.calls.length >= 2); + for (const c of transport.calls) assert.equal(c.headers.Cookie, 'brw=byo2'); + } finally { + restore(); + } + }); + + it('recovery re-loads credentials (user refreshed the cookie) and re-verifies', async () => { + process.env.AIRTABLE_COOKIE = 'brw=old'; + process.env.AIRTABLE_CSRF = 'c'; + const transport = fakeTransport(); + transport.next = { status: 200, body: '{"data":{"userId":"u1"}}' }; + try { + const auth = byoAuth(transport); + await auth.init(); + assert.equal(auth._credentials.cookieHeader, 'brw=old'); + // User pastes a fresh cookie, then a recovery fires. + process.env.AIRTABLE_COOKIE = 'brw=fresh'; + await auth._recoverSession(); + assert.equal(auth._credentials.cookieHeader, 'brw=fresh'); + assert.equal(auth.isLoggedIn, true); + } finally { + restore(); + } + }); + + it('recovery with a still-invalid cookie throws BYO_CREDENTIALS_EXPIRED', async () => { + process.env.AIRTABLE_COOKIE = 'brw=old'; + process.env.AIRTABLE_CSRF = 'c'; + const transport = fakeTransport(); + let n = 0; + transport.next = () => (++n === 1 ? { status: 200, body: '{"data":{}}' } : { status: 401, body: '' }); + try { + const auth = byoAuth(transport); + await auth.init(); + await assert.rejects(() => auth._recoverSession(), /BYO_CREDENTIALS_EXPIRED/); + } finally { + restore(); + } + }); + + it('direct-login mode throws a not-yet-available error', async () => { + process.env.AIRTABLE_AUTH_MODE = 'direct-login'; + try { + const auth = new AirtableAuth(); + auth._httpTransport = fakeTransport(); + await assert.rejects(() => auth.init(), /direct-login mode not yet available/); + } finally { + restore(); + } + }); +}); + describe('auth passive secretSocketId injection (postForm)', () => { let auth, transport; beforeEach(() => { diff --git a/packages/mcp-server/test/test-byo-credentials.test.js b/packages/mcp-server/test/test-byo-credentials.test.js new file mode 100644 index 0000000..4850636 --- /dev/null +++ b/packages/mcp-server/test/test-byo-credentials.test.js @@ -0,0 +1,126 @@ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { loadByoCredentials, scrapeCsrf, getCredentialsPath } from '../src/byo-credentials.js'; + +/** + * BYO (bring-your-own) cookie credentials for AIRTABLE_AUTH_MODE=byo. + * Pure/injectable: the csrf-scrape GET goes through an injected fetchImpl so + * these tests never touch the network. + */ + +// Save/restore the env keys these tests mutate. +const ENV_KEYS = ['AIRTABLE_COOKIE', 'AIRTABLE_CSRF', 'AIRTABLE_USER_MCP_HOME']; +function snapshotEnv() { + const s = {}; + for (const k of ENV_KEYS) s[k] = process.env[k]; + return s; +} +function restoreEnv(s) { + for (const k of ENV_KEYS) { + if (s[k] === undefined) delete process.env[k]; + else process.env[k] = s[k]; + } +} + +const HTML_WITH_CSRF = ''; + +describe('scrapeCsrf', () => { + it('finds a csrfToken in a script/window JSON blob', () => { + assert.equal(scrapeCsrf(HTML_WITH_CSRF), 'SCRAPED-CSRF-42'); + }); + it('finds a csrfToken in a meta tag (name-first)', () => { + assert.equal(scrapeCsrf(''), 'META-CSRF'); + }); + it('finds a csrfToken in a meta tag (content-first)', () => { + assert.equal(scrapeCsrf(''), 'META-CSRF2'); + }); + it('returns null when nothing matches or input is empty', () => { + assert.equal(scrapeCsrf('nope'), null); + assert.equal(scrapeCsrf(''), null); + assert.equal(scrapeCsrf(null), null); + }); +}); + +describe('loadByoCredentials', () => { + let env, tmpHome; + beforeEach(() => { + env = snapshotEnv(); + for (const k of ENV_KEYS) delete process.env[k]; + tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), 'byo-')); + process.env.AIRTABLE_USER_MCP_HOME = tmpHome; + }); + afterEach(() => { + restoreEnv(env); + fs.rmSync(tmpHome, { recursive: true, force: true }); + }); + + it('env source: AIRTABLE_COOKIE + AIRTABLE_CSRF (no fetch, no scrape)', async () => { + process.env.AIRTABLE_COOKIE = 'brw=1; __Host-airtable-session=abc'; + process.env.AIRTABLE_CSRF = 'ENV-CSRF'; + let fetched = false; + const creds = await loadByoCredentials({ fetchImpl: async () => { fetched = true; return { text: async () => '' }; } }); + assert.equal(creds.cookieHeader, 'brw=1; __Host-airtable-session=abc'); + assert.equal(creds.csrfToken, 'ENV-CSRF'); + assert.equal(fetched, false); + }); + + it('env source without csrf: scrapes csrf from a GET of airtable.com carrying the cookie', async () => { + process.env.AIRTABLE_COOKIE = 'brw=9'; + let seenReq; + const creds = await loadByoCredentials({ + fetchImpl: async (url, opts) => { seenReq = { url, opts }; return { text: async () => HTML_WITH_CSRF }; }, + }); + assert.equal(creds.cookieHeader, 'brw=9'); + assert.equal(creds.csrfToken, 'SCRAPED-CSRF-42'); + assert.equal(seenReq.url, 'https://airtable.com/'); + assert.equal(seenReq.opts.headers.Cookie, 'brw=9'); + }); + + it('file source: reads cookie + csrf from ~/.airtable-user-mcp/credentials.json', async () => { + fs.writeFileSync(getCredentialsPath(), JSON.stringify({ cookie: 'brw=file; sess=x', csrf: 'FILE-CSRF' })); + const creds = await loadByoCredentials({ fetchImpl: async () => { throw new Error('should not fetch'); } }); + assert.equal(creds.cookieHeader, 'brw=file; sess=x'); + assert.equal(creds.csrfToken, 'FILE-CSRF'); + }); + + it('file source without csrf: scrapes it', async () => { + fs.writeFileSync(getCredentialsPath(), JSON.stringify({ cookie: 'brw=file2' })); + const creds = await loadByoCredentials({ fetchImpl: async () => ({ text: async () => HTML_WITH_CSRF }) }); + assert.equal(creds.cookieHeader, 'brw=file2'); + assert.equal(creds.csrfToken, 'SCRAPED-CSRF-42'); + }); + + it('env takes precedence over the file', async () => { + process.env.AIRTABLE_COOKIE = 'brw=env'; + process.env.AIRTABLE_CSRF = 'ENV-WINS'; + fs.writeFileSync(getCredentialsPath(), JSON.stringify({ cookie: 'brw=file', csrf: 'FILE-LOSES' })); + const creds = await loadByoCredentials(); + assert.equal(creds.cookieHeader, 'brw=env'); + assert.equal(creds.csrfToken, 'ENV-WINS'); + }); + + it('csrf null (not fatal) when neither supplied nor scrapable', async () => { + process.env.AIRTABLE_COOKIE = 'brw=1'; + const creds = await loadByoCredentials({ fetchImpl: async () => ({ text: async () => 'no token here' }) }); + assert.equal(creds.cookieHeader, 'brw=1'); + assert.equal(creds.csrfToken, null); + }); + + it('csrf null when the scrape GET throws (network error is non-fatal)', async () => { + process.env.AIRTABLE_COOKIE = 'brw=1'; + const creds = await loadByoCredentials({ fetchImpl: async () => { throw new Error('net down'); } }); + assert.equal(creds.csrfToken, null); + }); + + it('throws an actionable error when no cookie is available anywhere', async () => { + await assert.rejects(() => loadByoCredentials(), /BYO_CREDENTIALS_MISSING/); + }); + + it('throws on an invalid JSON credentials file', async () => { + fs.writeFileSync(getCredentialsPath(), '{not json'); + await assert.rejects(() => loadByoCredentials(), /not valid JSON/); + }); +}); From 36e977e5ea17e049c827d5bb79d61fbd6fa4bda1 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 19:05:36 +0300 Subject: [PATCH 173/246] =?UTF-8?q?feat(auth):=20AIRTABLE=5FAUTH=5FMODE=3D?= =?UTF-8?q?direct-login=20=E2=80=94=20browser-free=20login=20via=20impit?= =?UTF-8?q?=20+=20TOTP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add src/direct-login.js: replicates the HAR-verified Airtable cold-login flow over HTTP (GET /login → getLoginTypeForEmail → login → verify2faCode → re-scrape csrf from authed GET /) using impit (Chrome TLS impersonation) and otpauth for TOTP — both lazy-loaded optionalDeps with injectable seams (impitFactory/otpFactory) for tests. Maintains a manual cookie jar (robust Set-Cookie parsing, no redirect-following) and returns { cookieHeader, csrfToken }. Specific error codes per failure step (SSO, bad creds, 2FA required/rejected, no session, missing creds). Wire auth.js: the direct-login mode now mints credentials via directLogin in _doInit (no browser); _recoverSession re-runs the flow for fresh cookies and surfaces SESSION_INVALID on failure; ensureLoggedIn treats direct-login as a browser-free mode. Default behavior (browser mode) is unchanged. Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/auth.js | 49 ++- packages/mcp-server/src/direct-login.js | 352 ++++++++++++++++++ .../test/test-auth-transport.test.js | 67 +++- .../mcp-server/test/test-direct-login.test.js | 230 ++++++++++++ 4 files changed, 683 insertions(+), 15 deletions(-) create mode 100644 packages/mcp-server/src/direct-login.js create mode 100644 packages/mcp-server/test/test-direct-login.test.js diff --git a/packages/mcp-server/src/auth.js b/packages/mcp-server/src/auth.js index 3685e3f..7b908b8 100644 --- a/packages/mcp-server/src/auth.js +++ b/packages/mcp-server/src/auth.js @@ -5,6 +5,7 @@ import { trace } from './debug-tracer.js'; import { getProfileDir } from './paths.js'; import { HttpTransport } from './http-transport.js'; import { loadByoCredentials } from './byo-credentials.js'; +import { directLogin } from './direct-login.js'; /** Generate a page-load-id (pgl + 13 base36 chars) using crypto, not Math.random. */ function genPageLoadId() { @@ -84,6 +85,10 @@ export class AirtableAuth { // Populated by _snapshotCredentials() during init/recovery. this._credentials = null; + // Direct-login (browser-free) credential minter. Injectable for tests so + // the impit/TOTP HTTP flow can be driven over canned responses. + this._directLogin = options.directLogin || directLogin; + // Reference to the page-level 'request' listener so we can detach it on // _doInit's context-close step rather than leak it per session-recovery. this._networkHandler = null; @@ -158,7 +163,7 @@ export class AirtableAuth { return this._doInitByo(); } if (this._authMode === 'direct-login') { - throw new Error('direct-login mode not yet available'); + return this._doInitDirectLogin(); } // Close existing context if recovering. Detach the prior network listener @@ -255,6 +260,21 @@ export class AirtableAuth { console.error('[auth] BYO session verified!', this.userId ? `User: ${this.userId}` : '(userId not in payload)'); } + /** + * Direct-login init: no browser. Replicate the HAR login flow over HTTP + * (impit + TOTP) to mint fresh session cookies + csrf, then drive all API + * calls through the direct-HTTP transport with them. directLogin() itself + * validates the session (checks the session cookie is present and re-scrapes + * an authed page), so no separate browser verify is needed. + */ + async _doInitDirectLogin() { + console.error('[auth] AIRTABLE_AUTH_MODE=direct-login — browser-free login via impit + TOTP (no browser).'); + this._credentials = await this._directLogin(); + this.csrfToken = this._credentials?.csrfToken ?? null; + this.isLoggedIn = true; + console.error('[auth] direct-login complete.'); + } + /** * Wait for the Airtable app to be ready instead of a blind 5s timeout. * Looks for signs of a loaded SPA: __NEXT_DATA__, a logged-in avatar, @@ -455,6 +475,22 @@ export class AirtableAuth { return; } + // Direct-login mode has no browser to relaunch. Re-run the HTTP login + // flow to mint fresh cookies/csrf. + if (this._authMode === 'direct-login') { + console.error('[auth] direct-login session rejected. Re-running login for fresh cookies...'); + this.isLoggedIn = false; + try { + this._credentials = await this._directLogin(); + } catch (err) { + throw new Error(`SESSION_INVALID: direct-login refresh failed — ${err.message}`); + } + this.csrfToken = this._credentials?.csrfToken ?? null; + this.isLoggedIn = true; + console.error('[auth] direct-login session recovered.'); + return; + } + console.error('[auth] Session expired. Attempting recovery...'); this.isLoggedIn = false; // Atomic assignment — init() observes this as already-in-progress and @@ -679,11 +715,12 @@ export class AirtableAuth { // ─── Public API ─────────────────────────────────────────────── async ensureLoggedIn() { - // BYO mode: no browser context/page to inspect for a redirect. Initialize - // once (loads + verifies the cookie); thereafter it's a no-op — a mid-run - // 401 is handled by _apiCall's backoff → _recoverSession (which re-loads the - // credentials). - if (this._authMode === 'byo') { + // Browser-free modes (byo / direct-login): no browser context/page to + // inspect for a redirect. Initialize once (byo loads+verifies the cookie; + // direct-login mints cookies via the HTTP login flow); thereafter it's a + // no-op — a mid-run 401 is handled by _apiCall's backoff → _recoverSession + // (which re-loads/re-mints the credentials). + if (this._authMode === 'byo' || this._authMode === 'direct-login') { if (!this._credentials) await this.init(); return; } diff --git a/packages/mcp-server/src/direct-login.js b/packages/mcp-server/src/direct-login.js new file mode 100644 index 0000000..2c0acef --- /dev/null +++ b/packages/mcp-server/src/direct-login.js @@ -0,0 +1,352 @@ +/** + * Browser-free direct login for AIRTABLE_AUTH_MODE=direct-login (Phase B). + * + * Replicates the HAR-verified Airtable cold-login flow over HTTP instead of + * driving a Chromium profile. Airtable auth is COOKIE-ONLY (no bearer/api-key): + * 1. GET /login — scrape csrfToken, collect cookies + * 2. POST /auth/getLoginTypeForEmail — detect SSO vs password accounts + * 3. POST /auth/login/ — email + password → 302 (to /2fa/… or /) + * 4. POST /auth/verify2faCode — TOTP, only when a 2FA challenge is returned + * → re-scrape csrfToken from an authed GET / (the login session rotates the + * csrfSecret, so the /login token is stale for subsequent API calls). + * + * Transport is impit (Chrome TLS impersonation) to resist a PerimeterX + * challenge on the login endpoints; both impit and otpauth are OPTIONAL + * dependencies, lazy-loaded, and injectable (impitFactory / otpFactory) for + * tests so NO live Airtable call or real crypto runs under `node --test`. + * + * Returns { cookieHeader, csrfToken } — the exact shape auth._credentials wants. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { getHomeDir } from './paths.js'; +import { scrapeCsrf } from './byo-credentials.js'; + +const BASE = 'https://airtable.com'; + +// ── Lazy optional-dependency loaders ──────────────────────────────── + +let _ImpitClass = null; +async function loadImpit() { + if (_ImpitClass) return _ImpitClass; + try { + const mod = await import('impit'); + _ImpitClass = mod.Impit || mod.default?.Impit; + if (!_ImpitClass) throw new Error('the `impit` module did not export `Impit`'); + return _ImpitClass; + } catch (err) { + throw new Error( + 'DIRECT_LOGIN_IMPIT_MISSING: AIRTABLE_AUTH_MODE=direct-login requires the optional `impit` ' + + 'package (Chrome TLS impersonation). Install it (`npm i impit`) or use AIRTABLE_AUTH_MODE=browser.\n' + + `Original error: ${err.message}` + ); + } +} + +let _OTPAuth = null; +async function loadOtpAuth() { + if (_OTPAuth) return _OTPAuth; + try { + const mod = await import('otpauth'); + _OTPAuth = mod.default?.TOTP ? mod.default : mod; + if (!_OTPAuth?.TOTP) throw new Error('the `otpauth` module did not export TOTP/Secret'); + return _OTPAuth; + } catch (err) { + throw new Error( + 'DIRECT_LOGIN_OTPAUTH_MISSING: this account requires 2FA but the optional `otpauth` package ' + + 'is not installed. Install it (`npm i otpauth`) or use AIRTABLE_AUTH_MODE=browser.\n' + + `Original error: ${err.message}` + ); + } +} + +async function makeImpit(impitFactory) { + if (impitFactory) return impitFactory(); + const Impit = await loadImpit(); + // followRedirects:false — we drive the exact login sequence and read Location + // ourselves rather than let the client chase the browser's redirects. + return new Impit({ browser: 'chrome', followRedirects: false }); +} + +async function generateTotp(secret, otpFactory) { + if (otpFactory) return otpFactory(secret); + const OTPAuth = await loadOtpAuth(); + const totp = new OTPAuth.TOTP({ secret: OTPAuth.Secret.fromBase32(secret) }); + return totp.generate(); +} + +// ── Cookie-jar helpers ────────────────────────────────────────────── + +/** + * Split a combined Set-Cookie header string into individual cookie strings, + * respecting the comma inside an `Expires=Wed, 09 Jun 2021 …` date. Battle- + * tested char scanner (set-cookie-parser style). Only used as a fallback — real + * impit/WHATWG responses expose getSetCookie() as an array. + */ +function splitCombinedSetCookie(str) { + const out = []; + let pos = 0; + const len = str.length; + const isWs = (c) => c === ' ' || c === '\t' || c === '\n' || c === '\r'; + while (pos < len) { + let start = pos; + let separatorFound = false; + while (pos < len) { + while (pos < len && isWs(str.charAt(pos))) pos++; + if (pos >= len) break; + const ch = str.charAt(pos); + if (ch === ',') { + const lastComma = pos; + pos += 1; + while (pos < len && isWs(str.charAt(pos))) pos++; + const nextStart = pos; + while (pos < len) { + const c = str.charAt(pos); + if (c === '=' || c === ';' || c === ',') break; + pos += 1; + } + if (pos < len && str.charAt(pos) === '=') { + separatorFound = true; + pos = nextStart; + out.push(str.substring(start, lastComma)); + start = pos; + } else { + pos = lastComma + 1; + } + } else { + pos += 1; + } + } + if (!separatorFound || pos >= len) { + out.push(str.substring(start, len)); + } + } + return out.map((s) => s.trim()).filter(Boolean); +} + +/** Robustly extract the list of raw Set-Cookie strings from a response. */ +function extractSetCookies(res) { + const h = res && res.headers; + if (!h) return []; + // WHATWG Headers.getSetCookie() (impit / undici) → already an array. + if (typeof h.getSetCookie === 'function') { + try { + const arr = h.getSetCookie(); + if (Array.isArray(arr)) return arr; + } catch { /* fall through */ } + } + // node-fetch style raw(). + if (typeof h.raw === 'function') { + try { + const raw = h.raw(); + const sc = raw && (raw['set-cookie'] || raw['Set-Cookie']); + if (Array.isArray(sc)) return sc; + if (typeof sc === 'string') return splitCombinedSetCookie(sc); + } catch { /* fall through */ } + } + // Headers.get('set-cookie') / Map.get. + if (typeof h.get === 'function') { + try { + const sc = h.get('set-cookie'); + if (Array.isArray(sc)) return sc; + if (typeof sc === 'string') return splitCombinedSetCookie(sc); + } catch { /* fall through */ } + } + // Plain object. + const sc = h['set-cookie'] ?? h['Set-Cookie']; + if (Array.isArray(sc)) return sc; + if (typeof sc === 'string') return splitCombinedSetCookie(sc); + return []; +} + +/** Update the jar Map(name→value) from a response's Set-Cookie header(s). */ +function updateJar(jar, res) { + for (const raw of extractSetCookies(res)) { + if (!raw || typeof raw !== 'string') continue; + const firstPair = raw.split(';')[0]; + const eq = firstPair.indexOf('='); + if (eq < 0) continue; + const name = firstPair.slice(0, eq).trim(); + const value = firstPair.slice(eq + 1).trim(); + if (name) jar.set(name, value); + } +} + +function jarToCookieHeader(jar) { + return [...jar.entries()].map(([k, v]) => `${k}=${v}`).join('; '); +} + +/** Read a single response header case-insensitively (WHATWG or plain object). */ +function getHeader(res, name) { + const h = res && res.headers; + if (!h) return null; + if (typeof h.get === 'function') { + try { + const v = h.get(name); + if (v != null) return v; + } catch { /* fall through */ } + } + return h[name] ?? h[String(name).toLowerCase()] ?? null; +} + +// ── Request helper ────────────────────────────────────────────────── + +/** + * Issue a single request through impit with the running cookie jar, then fold + * the response's Set-Cookie header(s) back into the jar. + * @param {'GET'|'POST'} method + * @param {string} urlPath path (joined to BASE) or a full URL + * @param {object} [opts] + * @param {Map} opts.jar + * @param {object} [opts.form] form-urlencoded body fields + */ +async function request(impit, method, urlPath, { jar, form } = {}) { + const url = urlPath.startsWith('http') ? urlPath : BASE + urlPath; + const headers = {}; + const cookie = jarToCookieHeader(jar); + if (cookie) headers['Cookie'] = cookie; + const options = { method, headers }; + if (form) { + headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8'; + options.body = new URLSearchParams(form).toString(); + } + const res = await impit.fetch(url, options); + updateJar(jar, res); + return res; +} + +// ── Credential resolution ─────────────────────────────────────────── + +function resolveCredentials({ email, password, totpSecret }) { + let e = email ?? process.env.AIRTABLE_EMAIL; + let p = password ?? process.env.AIRTABLE_PASSWORD; + let t = totpSecret ?? process.env.AIRTABLE_TOTP_SECRET; + + if (!e || !p) { + // File fallback: ~/.airtable-user-mcp/login.json { email, password, totpSecret } + const file = path.join(getHomeDir(), 'login.json'); + try { + const parsed = JSON.parse(fs.readFileSync(file, 'utf8')); + e = e || parsed.email; + p = p || parsed.password; + t = t || parsed.totpSecret; + } catch { /* no file / invalid — fall through to the missing-creds error */ } + } + + if (!e || !p) { + throw new Error( + 'DIRECT_LOGIN_CREDENTIALS_MISSING: AIRTABLE_AUTH_MODE=direct-login needs an email + password. ' + + 'Set AIRTABLE_EMAIL and AIRTABLE_PASSWORD (and AIRTABLE_TOTP_SECRET for 2FA accounts), ' + + `or write ${path.join(getHomeDir(), 'login.json')} as { "email", "password", "totpSecret"? }.` + ); + } + return { email: e, password: p, totpSecret: t || null }; +} + +/** + * Decide whether getLoginTypeForEmail's answer means this account cannot do a + * password login (SSO / SAML / enterprise). Lenient: parse JSON when possible, + * else sniff the raw body. + */ +function indicatesSso(body) { + if (!body) return false; + let data; + try { + data = JSON.parse(body); + } catch { + return /\b(sso|saml)\b/i.test(body) && !/password/i.test(body); + } + const type = String( + data.loginType ?? data.type ?? data.strategy ?? data.method ?? '' + ).toLowerCase(); + if (/sso|saml|enterprise|oidc/.test(type)) return true; + if (type === 'password') return false; + if (data.isSso === true || data.ssoRequired === true || data.requiresSso === true) return true; + return false; +} + +// ── Main flow ─────────────────────────────────────────────────────── + +/** + * @param {object} [opts] + * @param {string} [opts.email] defaults to env / login.json + * @param {string} [opts.password] defaults to env / login.json + * @param {string} [opts.totpSecret] base32 TOTP secret (2FA accounts) + * @param {Function} [opts.impitFactory] test seam — () => ({ fetch(url,opts) }) + * @param {Function} [opts.otpFactory] test seam — (secret) => code + * @returns {Promise<{ cookieHeader: string, csrfToken: string|null }>} + */ +export async function directLogin({ email, password, totpSecret, impitFactory, otpFactory } = {}) { + const creds = resolveCredentials({ email, password, totpSecret }); + const impit = await makeImpit(impitFactory); + const jar = new Map(); + + // 1. GET /login — scrape the csrfToken, collect the initial cookies (brw, …). + const loginPage = await request(impit, 'GET', '/login', { jar }); + let csrf = scrapeCsrf(await loginPage.text()); + if (!csrf) { + throw new Error('DIRECT_LOGIN_NO_CSRF: could not scrape a csrfToken from GET /login — Airtable may have changed the login page.'); + } + + // 2. getLoginTypeForEmail — bail early on SSO/enterprise accounts. + const typeRes = await request(impit, 'POST', '/auth/getLoginTypeForEmail', { + jar, + form: { + _csrf: csrf, + email: creds.email, + shouldRedirectUnregisteredEmailToSignup: false, + urlToRedirectTo: '/', + didConsentToMarketing: false, + didConsentToDataEnrichment: false, + }, + }); + if (indicatesSso(await typeRes.text())) { + throw new Error('DIRECT_LOGIN_SSO_UNSUPPORTED: this account uses SSO — use AIRTABLE_AUTH_MODE=browser'); + } + + // 3. POST /auth/login/ — expect a 302 (to /2fa/… on a 2FA account, else /). + const loginRes = await request(impit, 'POST', '/auth/login/', { + jar, + form: { _csrf: csrf, urlToRedirectTo: '/', email: creds.email, password: creds.password }, + }); + const isRedirect = loginRes.status === 302 || loginRes.status === 303; + const loginLocation = getHeader(loginRes, 'location') || ''; + const twoFaMatch = loginLocation.match(/\/2fa\/([^/?#]+)/); + // A non-redirect, or a redirect that bounces back to /login (without a 2FA + // segment), means the email/password was rejected. + if (!isRedirect || (/\/login/.test(loginLocation) && !twoFaMatch)) { + throw new Error(`DIRECT_LOGIN_BAD_CREDENTIALS: email/password rejected (status ${loginRes.status})`); + } + + // 4. 2FA, when challenged. + if (twoFaMatch) { + const twoFactorStrategyId = twoFaMatch[1]; + if (!creds.totpSecret) { + throw new Error('DIRECT_LOGIN_2FA_REQUIRED: set AIRTABLE_TOTP_SECRET (base32) for this account'); + } + const code = await generateTotp(creds.totpSecret, otpFactory); + const verifyRes = await request(impit, 'POST', '/auth/verify2faCode', { + jar, + form: { _csrf: csrf, twoFactorStrategyId, code }, + }); + if (verifyRes.status !== 302 && verifyRes.status !== 303) { + throw new Error(`DIRECT_LOGIN_2FA_REJECTED: TOTP code rejected — check AIRTABLE_TOTP_SECRET/clock (status ${verifyRes.status})`); + } + } + + // 5. Must have a real session cookie now. + if (!jar.has('__Host-airtable-session')) { + throw new Error('DIRECT_LOGIN_NO_SESSION: login flow completed without a session cookie'); + } + + // Re-scrape a fresh csrfToken from an authed GET / (the login rotates the + // session's csrfSecret, so the /login token is stale for API calls). Best- + // effort: keep the login-flow token if the re-scrape fails. + try { + const home = await request(impit, 'GET', '/', { jar }); + const fresh = scrapeCsrf(await home.text()); + if (fresh) csrf = fresh; + } catch { /* keep the existing csrf */ } + + return { cookieHeader: jarToCookieHeader(jar), csrfToken: csrf }; +} diff --git a/packages/mcp-server/test/test-auth-transport.test.js b/packages/mcp-server/test/test-auth-transport.test.js index 5c90647..8f99312 100644 --- a/packages/mcp-server/test/test-auth-transport.test.js +++ b/packages/mcp-server/test/test-auth-transport.test.js @@ -1,4 +1,4 @@ -import { describe, it, beforeEach } from 'node:test'; +import { describe, it, beforeEach, afterEach } from 'node:test'; import assert from 'node:assert/strict'; import { AirtableAuth } from '../src/auth.js'; @@ -287,15 +287,64 @@ describe('auth AIRTABLE_AUTH_MODE=byo', () => { } }); - it('direct-login mode throws a not-yet-available error', async () => { +}); + +describe('auth AIRTABLE_AUTH_MODE=direct-login', () => { + let saved; + beforeEach(() => { + saved = process.env.AIRTABLE_AUTH_MODE; process.env.AIRTABLE_AUTH_MODE = 'direct-login'; - try { - const auth = new AirtableAuth(); - auth._httpTransport = fakeTransport(); - await assert.rejects(() => auth.init(), /direct-login mode not yet available/); - } finally { - restore(); - } + }); + afterEach(() => { + if (saved === undefined) delete process.env.AIRTABLE_AUTH_MODE; + else process.env.AIRTABLE_AUTH_MODE = saved; + }); + + it('_doInit calls directLogin (via the injected seam) and never launches a browser', async () => { + const auth = new AirtableAuth(); + let called = false; + // Inject the login minter — real directLogin would need impit + network. + auth._directLogin = async () => { + called = true; + return { cookieHeader: 'brw=1; __Host-airtable-session=s', csrfToken: 'C' }; + }; + await auth.init(); + assert.equal(called, true); + // No browser: context/page stay null, but the session is logged in. + assert.equal(auth.context, null); + assert.equal(auth.page, null); + assert.equal(auth.isLoggedIn, true); + assert.equal(auth.csrfToken, 'C'); + assert.equal(auth._credentials.cookieHeader, 'brw=1; __Host-airtable-session=s'); + }); + + it('_recoverSession re-runs directLogin to mint fresh cookies', async () => { + const auth = new AirtableAuth(); + let n = 0; + auth._directLogin = async () => { n++; return { cookieHeader: `c=${n}`, csrfToken: `csrf${n}` }; }; + await auth.init(); + assert.equal(auth._credentials.cookieHeader, 'c=1'); + await auth._recoverSession(); + assert.equal(auth._credentials.cookieHeader, 'c=2'); + assert.equal(auth.csrfToken, 'csrf2'); + assert.equal(auth.isLoggedIn, true); + }); + + it('a subsequent _apiCall carries the minted cookie and needs no browser', async () => { + const auth = new AirtableAuth(); + const transport = fakeTransport(); + auth._httpTransport = transport; + auth._directLogin = async () => ({ cookieHeader: 'brw=minted', csrfToken: 'mc' }); + transport.next = { status: 200, body: '{"data":{}}' }; + const res = await auth.get('/v0.3/x', 'appZZ'); + assert.equal(res.status, 200); + assert.equal(transport.calls[0].headers.Cookie, 'brw=minted'); + }); + + it('_recoverSession surfaces a SESSION_INVALID error when the login flow fails', async () => { + const auth = new AirtableAuth(); + auth._directLogin = async () => { throw new Error('DIRECT_LOGIN_BAD_CREDENTIALS: nope'); }; + await assert.rejects(() => auth._recoverSession(), /SESSION_INVALID.*DIRECT_LOGIN_BAD_CREDENTIALS/); }); }); diff --git a/packages/mcp-server/test/test-direct-login.test.js b/packages/mcp-server/test/test-direct-login.test.js new file mode 100644 index 0000000..c3c0858 --- /dev/null +++ b/packages/mcp-server/test/test-direct-login.test.js @@ -0,0 +1,230 @@ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { directLogin } from '../src/direct-login.js'; + +/** + * Phase B — browser-free direct login (impit + TOTP), replicating the + * HAR-verified Airtable cold-login flow over HTTP: + * GET /login (scrape csrf, collect cookies) + * → POST /auth/getLoginTypeForEmail + * → POST /auth/login/ + * → POST /auth/verify2faCode (only when 2FA is required) + * → re-scrape csrf from an authed GET / + * + * All network is mocked via an injected impitFactory returning canned + * responses; NO live Airtable calls. TOTP is injected via otpFactory so the + * generated code is deterministic and otpauth is never imported. + */ + +/** + * A scripted fake impit. `script` is an array of response descriptors consumed + * in fetch order. Each descriptor: { status?, body?, location?, setCookies? }. + * Exposes a WHATWG-ish response (getSetCookie() + get() + text()). + */ +function makeFakeImpit(script) { + const calls = []; + let i = 0; + return { + calls, + async fetch(url, options) { + calls.push({ url, options }); + const desc = script[i++] ?? {}; + return { + status: desc.status ?? 200, + headers: { + getSetCookie: () => desc.setCookies || [], + get: (name) => { + const n = String(name).toLowerCase(); + if (n === 'set-cookie') return desc.setCookies || null; + if (n === 'location') return desc.location ?? null; + return null; + }, + }, + text: async () => desc.body ?? '', + }; + }, + }; +} + +const CREDS = { email: 'user@example.com', password: 'hunter2' }; + +function bodyParams(call) { + return new URLSearchParams(call.options.body); +} + +describe('directLogin — happy path WITH 2FA', () => { + it('runs the 4 auth requests in order, threads _csrf, accumulates cookies, posts the generated TOTP', async () => { + const impit = makeFakeImpit([ + // 1. GET /login + { status: 200, body: '', setCookies: ['brw=BRW1; Path=/; HttpOnly'] }, + // 2. getLoginTypeForEmail — password account + { status: 200, body: JSON.stringify({ loginType: 'password' }) }, + // 3. login → 2FA challenge + { status: 302, location: '/2fa/tfaABC123', setCookies: ['__Host-airtable-session=SESS1; Path=/', '__Host-airtable-session.sig=SIG1; Path=/'] }, + // 4. verify2faCode → success, session rotated + { status: 302, location: '/', setCookies: ['__Host-airtable-session=SESS2; Path=/'] }, + // 5. authed GET / — fresh csrf after session rotation + { status: 200, body: '' }, + ]); + + let totpArg = null; + const out = await directLogin({ + ...CREDS, + totpSecret: 'JBSWY3DPEHPK3PXP', + impitFactory: () => impit, + otpFactory: (secret) => { totpArg = secret; return '654321'; }, + }); + + // First 4 requests, in order. + assert.match(impit.calls[0].url, /\/login$/); + assert.equal(impit.calls[0].options.method, 'GET'); + + assert.match(impit.calls[1].url, /\/auth\/getLoginTypeForEmail$/); + assert.equal(impit.calls[1].options.method, 'POST'); + assert.equal(bodyParams(impit.calls[1]).get('_csrf'), 'CSRF-1'); + assert.equal(bodyParams(impit.calls[1]).get('email'), CREDS.email); + + assert.match(impit.calls[2].url, /\/auth\/login\/$/); + assert.equal(bodyParams(impit.calls[2]).get('_csrf'), 'CSRF-1'); + assert.equal(bodyParams(impit.calls[2]).get('email'), CREDS.email); + assert.equal(bodyParams(impit.calls[2]).get('password'), CREDS.password); + + assert.match(impit.calls[3].url, /\/auth\/verify2faCode$/); + assert.equal(bodyParams(impit.calls[3]).get('_csrf'), 'CSRF-1'); + assert.equal(bodyParams(impit.calls[3]).get('twoFactorStrategyId'), 'tfaABC123'); + assert.equal(bodyParams(impit.calls[3]).get('code'), '654321'); + + // TOTP secret was handed to the generator. + assert.equal(totpArg, 'JBSWY3DPEHPK3PXP'); + + // Cookie jar accumulated across every response (session rotated to SESS2). + const jar = out.cookieHeader.split('; ').sort(); + assert.ok(jar.includes('brw=BRW1')); + assert.ok(jar.includes('__Host-airtable-session=SESS2')); + assert.ok(jar.includes('__Host-airtable-session.sig=SIG1')); + + // csrf re-scraped from the authed GET /. + assert.equal(out.csrfToken, 'CSRF-2'); + }); +}); + +describe('directLogin — happy path WITHOUT 2FA', () => { + it('login 302s straight to / and verify2faCode is never called', async () => { + const impit = makeFakeImpit([ + { status: 200, body: '{"csrfToken":"CSRF-A"}', setCookies: ['brw=B1; Path=/'] }, + { status: 200, body: JSON.stringify({ loginType: 'password' }) }, + { status: 302, location: '/', setCookies: ['__Host-airtable-session=SESSA; Path=/', '__Host-airtable-session.sig=SIGA; Path=/'] }, + { status: 200, body: '{"csrfToken":"CSRF-B"}' }, + ]); + + const out = await directLogin({ ...CREDS, impitFactory: () => impit }); + + // Exactly 4 requests: GET /login, getLoginType, login, re-scrape GET /. + assert.equal(impit.calls.length, 4); + assert.ok(!impit.calls.some((c) => /verify2faCode/.test(c.url))); + assert.ok(out.cookieHeader.includes('__Host-airtable-session=SESSA')); + assert.equal(out.csrfToken, 'CSRF-B'); + }); +}); + +describe('directLogin — failure branches', () => { + it('SSO account → throws DIRECT_LOGIN_SSO_UNSUPPORTED before attempting a password login', async () => { + const impit = makeFakeImpit([ + { status: 200, body: '{"csrfToken":"CSRF-1"}', setCookies: ['brw=B; Path=/'] }, + { status: 200, body: JSON.stringify({ loginType: 'sso' }) }, + ]); + await assert.rejects( + () => directLogin({ ...CREDS, impitFactory: () => impit }), + /DIRECT_LOGIN_SSO_UNSUPPORTED/, + ); + // Never reached the /auth/login/ POST. + assert.equal(impit.calls.length, 2); + }); + + it('bad password → throws DIRECT_LOGIN_BAD_CREDENTIALS', async () => { + const impit = makeFakeImpit([ + { status: 200, body: '{"csrfToken":"CSRF-1"}', setCookies: ['brw=B; Path=/'] }, + { status: 200, body: JSON.stringify({ loginType: 'password' }) }, + { status: 401, body: 'Invalid email or password' }, + ]); + await assert.rejects( + () => directLogin({ ...CREDS, impitFactory: () => impit }), + /DIRECT_LOGIN_BAD_CREDENTIALS.*401/, + ); + }); + + it('2FA required but no TOTP secret → throws DIRECT_LOGIN_2FA_REQUIRED', async () => { + const impit = makeFakeImpit([ + { status: 200, body: '{"csrfToken":"CSRF-1"}', setCookies: ['brw=B; Path=/'] }, + { status: 200, body: JSON.stringify({ loginType: 'password' }) }, + { status: 302, location: '/2fa/tfaZ', setCookies: ['__Host-airtable-session=S; Path=/'] }, + ]); + await assert.rejects( + () => directLogin({ ...CREDS, impitFactory: () => impit }), + /DIRECT_LOGIN_2FA_REQUIRED/, + ); + }); + + it('2FA code rejected → throws DIRECT_LOGIN_2FA_REJECTED', async () => { + const impit = makeFakeImpit([ + { status: 200, body: '{"csrfToken":"CSRF-1"}', setCookies: ['brw=B; Path=/'] }, + { status: 200, body: JSON.stringify({ loginType: 'password' }) }, + { status: 302, location: '/2fa/tfaZ', setCookies: ['__Host-airtable-session=S; Path=/'] }, + { status: 200, body: 'Invalid two-factor code' }, // non-redirect = rejected + ]); + await assert.rejects( + () => directLogin({ ...CREDS, totpSecret: 'JBSWY3DPEHPK3PXP', impitFactory: () => impit, otpFactory: () => '000000' }), + /DIRECT_LOGIN_2FA_REJECTED/, + ); + }); + + it('login completes but no session cookie was set → throws DIRECT_LOGIN_NO_SESSION', async () => { + const impit = makeFakeImpit([ + { status: 200, body: '{"csrfToken":"CSRF-1"}', setCookies: ['brw=B; Path=/'] }, + { status: 200, body: JSON.stringify({ loginType: 'password' }) }, + { status: 302, location: '/', setCookies: [] }, // no __Host-airtable-session + { status: 200, body: '{"csrfToken":"CSRF-2"}' }, + ]); + await assert.rejects( + () => directLogin({ ...CREDS, impitFactory: () => impit }), + /DIRECT_LOGIN_NO_SESSION/, + ); + }); +}); + +describe('directLogin — credential resolution', () => { + const ENV = ['AIRTABLE_EMAIL', 'AIRTABLE_PASSWORD', 'AIRTABLE_TOTP_SECRET', 'AIRTABLE_USER_MCP_HOME']; + let saved; + beforeEach(() => { + saved = {}; + for (const k of ENV) { saved[k] = process.env[k]; delete process.env[k]; } + // Point HOME at a dir with no login.json so the file fallback misses. + process.env.AIRTABLE_USER_MCP_HOME = '/nonexistent-airtable-home-xyz'; + }); + afterEach(() => { + for (const k of ENV) { if (saved[k] === undefined) delete process.env[k]; else process.env[k] = saved[k]; } + }); + + it('throws DIRECT_LOGIN_CREDENTIALS_MISSING when no email/password anywhere', async () => { + const impit = makeFakeImpit([]); + await assert.rejects( + () => directLogin({ impitFactory: () => impit }), + /DIRECT_LOGIN_CREDENTIALS_MISSING/, + ); + }); + + it('reads email/password from env when params are omitted', async () => { + process.env.AIRTABLE_EMAIL = 'env@example.com'; + process.env.AIRTABLE_PASSWORD = 'envpw'; + const impit = makeFakeImpit([ + { status: 200, body: '{"csrfToken":"C1"}', setCookies: ['brw=B; Path=/'] }, + { status: 200, body: JSON.stringify({ loginType: 'password' }) }, + { status: 302, location: '/', setCookies: ['__Host-airtable-session=S; Path=/'] }, + { status: 200, body: '{"csrfToken":"C2"}' }, + ]); + const out = await directLogin({ impitFactory: () => impit }); + assert.equal(bodyParams(impit.calls[1]).get('email'), 'env@example.com'); + assert.equal(bodyParams(impit.calls[2]).get('password'), 'envpw'); + assert.ok(out.cookieHeader.includes('__Host-airtable-session=S')); + }); +}); From 3921af74126f0d25a37cb6a584529bc5144be989 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 19:27:09 +0300 Subject: [PATCH 174/246] fix(auth): recover from a locked Chrome profile (exit 21) by killing stale holders and retrying MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MCP drives Airtable through a headless Chrome that owns a single, MCP-EXCLUSIVE persistent profile. When a Chrome instance crashes or leaks, its process can linger and keep the profile's SingletonLock held, so the next launchPersistentContext() fails immediately with Chrome exit code 21 and the MCP is unusable until the zombie is killed by hand. Add reactive, surgical recovery: new src/profile-lock.js exposes isProfileLockError() (recognises exit-21 / SingletonLock / ProcessSingleton / "user data directory is already in use") and killProfileHolders() (best-effort, never-throws kill of the stale root Chrome/Edge holding OUR profile dir — PowerShell + taskkill /T /F on win32, pkill -f on posix). auth._doInit now routes launchPersistentContext through _launchContext(), which on a lock error kills the stale holder, waits briefly, and retries the launch exactly once; any non-lock error and a still-failing retry propagate unchanged. The kill is reactive only (never proactive), so a legitimately-running instance is only ever touched after we actually collide with the lock. Seams (getChromium / killProfileHolders / relaunchWaitMs) are constructor- injectable, mirroring the HttpTransport test-seam style. Tests drive both the module and _launchContext/_doInit over fakes — no real process or browser. Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/auth.js | 44 +++- packages/mcp-server/src/profile-lock.js | 83 ++++++++ .../test/test-auth-profile-lock.test.js | 195 ++++++++++++++++++ .../mcp-server/test/test-profile-lock.test.js | 133 ++++++++++++ 4 files changed, 453 insertions(+), 2 deletions(-) create mode 100644 packages/mcp-server/src/profile-lock.js create mode 100644 packages/mcp-server/test/test-auth-profile-lock.test.js create mode 100644 packages/mcp-server/test/test-profile-lock.test.js diff --git a/packages/mcp-server/src/auth.js b/packages/mcp-server/src/auth.js index 7b908b8..f7ac065 100644 --- a/packages/mcp-server/src/auth.js +++ b/packages/mcp-server/src/auth.js @@ -6,6 +6,7 @@ import { getProfileDir } from './paths.js'; import { HttpTransport } from './http-transport.js'; import { loadByoCredentials } from './byo-credentials.js'; import { directLogin } from './direct-login.js'; +import { isProfileLockError, killProfileHolders } from './profile-lock.js'; /** Generate a page-load-id (pgl + 13 base36 chars) using crypto, not Math.random. */ function genPageLoadId() { @@ -89,6 +90,17 @@ export class AirtableAuth { // the impit/TOTP HTTP flow can be driven over canned responses. this._directLogin = options.directLogin || directLogin; + // Chromium factory seam. When null we lazy-load the real patchright chromium + // via the module getChromium(); tests inject a fake chromium here. + this._getChromium = options.getChromium || null; + // Profile-lock recovery seams. The persistent .chrome-profile is + // MCP-EXCLUSIVE, so a Chrome still holding its lock (crash/leak → exit 21) + // is a stale instance safe to kill. _launchContext kills the holder and + // retries the launch ONCE. Both the killer and the inter-attempt wait are + // injectable so tests never touch a real process or sleep. + this._killProfileHolders = options.killProfileHolders || killProfileHolders; + this._relaunchWaitMs = options.relaunchWaitMs ?? 600; + // Reference to the page-level 'request' listener so we can detach it on // _doInit's context-close step rather than leak it per session-recovery. this._networkHandler = null; @@ -192,8 +204,8 @@ export class AirtableAuth { viewport: null, }; if (browserPath) launchOpts.executablePath = browserPath; - const chromium = await getChromium(); - this.context = await chromium.launchPersistentContext(this.profileDir, launchOpts); + const chromium = this._getChromium ? await this._getChromium() : await getChromium(); + this.context = await this._launchContext(chromium, launchOpts); // M5 — if anything after launchPersistentContext throws (page creation, // navigation, CSRF extraction, session verification), close the context @@ -231,6 +243,34 @@ export class AirtableAuth { } } + /** + * Launch the persistent browser context, recovering ONCE from a locked profile. + * + * The persistent `.chrome-profile` is MCP-EXCLUSIVE — only our headless Chrome + * ever opens it. So if launchPersistentContext fails with a profile-lock error + * (Chrome exit 21 / SingletonLock — a crashed or leaked Chrome still holding + * the lock), the holder is a stale instance safe to kill. We kill it, wait + * briefly for the lock to clear, then retry the launch EXACTLY once. + * + * REACTIVE only: we never kill before the first launch (that could take down a + * legitimately-running instance). We only act after actually colliding with + * the lock, when we are the one trying to launch. Any non-lock error — and a + * still-failing retry — propagates unchanged, so browser behavior is otherwise + * identical. + */ + async _launchContext(chromium, launchOpts) { + try { + return await chromium.launchPersistentContext(this.profileDir, launchOpts); + } catch (err) { + if (!isProfileLockError(err)) throw err; + console.error('[auth] Chrome profile locked (exit 21) — killing stale holder and retrying once...'); + await this._killProfileHolders(this.profileDir); + await new Promise((resolve) => setTimeout(resolve, this._relaunchWaitMs)); + // Retry ONCE. If this throws too, it propagates unchanged. + return await chromium.launchPersistentContext(this.profileDir, launchOpts); + } + } + /** * BYO (bring-your-own cookie) init: no browser. Load the pasted cookie+csrf, * mark logged-in, then verify the cookie is actually accepted by a real diff --git a/packages/mcp-server/src/profile-lock.js b/packages/mcp-server/src/profile-lock.js new file mode 100644 index 0000000..38bf2ff --- /dev/null +++ b/packages/mcp-server/src/profile-lock.js @@ -0,0 +1,83 @@ +/** + * Chrome persistent-profile lock recovery. + * + * The MCP drives Airtable through a single headless Chrome that owns a + * MCP-EXCLUSIVE persistent profile at `~/.airtable-user-mcp/.chrome-profile`. + * When a Chrome instance crashes or leaks, its process can linger and keep the + * profile's SingletonLock held, so the very next `launchPersistentContext()` + * fails immediately with Chrome exit code 21 + * ("Target page, context or browser has been closed", exitCode=21). + * + * Because the profile is only ever used by OUR Chrome, any process holding it + * is a stale instance that is safe to kill. This module provides: + * - `isProfileLockError` — recognise the lock-contention failure shape. + * - `killProfileHolders` — best-effort, NEVER-throws kill of the stale + * Chrome/Edge root process(es) whose command line references our profile. + * + * The kill is REACTIVE only — it is invoked after actually hitting the lock, + * never proactively before a launch (which could kill a legitimate instance). + */ + +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); + +/** + * Recognise the failure shape of a locked/contended Chrome persistent profile. + * + * Matches (case-insensitive) the several ways Playwright/patchright + Chrome + * surface the same underlying "profile already in use" condition: + * - exit code 21 (Chrome's ProcessSingleton bail-out) + * - "Target page, context or browser has been closed" + * - Chromium's own SingletonLock / ProcessSingleton messages + * - "user data directory is already in use" + * + * @param {unknown} err + * @returns {boolean} + */ +export function isProfileLockError(err) { + const message = String(err?.message ?? err ?? ''); + return /exit\s*code\s*21|exitcode=21|has been closed|SingletonLock|ProcessSingleton|user data directory is already in use|failed to create a ProcessSingleton/i.test( + message, + ); +} + +/** + * Best-effort kill of any stale Chrome/Edge process holding OUR persistent + * profile directory. NEVER throws — a failure here just means the retry will + * fail again and the real error propagates. + * + * win32: enumerate chrome.exe/msedge.exe via CIM, keep only ROOT processes + * (command line references our profileDir but is NOT a `--type=` child + * renderer/gpu/utility process), then `taskkill /T /F` the whole tree. + * posix: `pkill -f ` — matches any process whose command line + * contains the profile path. + * + * @param {string} profileDir absolute path to the persistent profile directory + * @param {object} [opts] + * @param {NodeJS.Platform} [opts.platform=process.platform] + * @param {(file: string, args: string[]) => Promise} [opts.exec] injectable runner (default: promisified execFile) + * @returns {Promise<{ killed: boolean, platform: string, error?: string }>} + */ +export async function killProfileHolders(profileDir, { platform = process.platform, exec = execFileAsync } = {}) { + try { + if (platform === 'win32') { + // Escape single quotes for embedding in the single-quoted PowerShell strings. + const escaped = String(profileDir).replace(/'/g, "''"); + const script = + `Get-CimInstance Win32_Process -Filter "Name='chrome.exe' OR Name='msedge.exe'" | ` + + `Where-Object { $_.CommandLine -like '*${escaped}*' -and $_.CommandLine -notlike '*--type=*' } | ` + + `ForEach-Object { taskkill /PID $_.ProcessId /T /F }`; + await exec('powershell', ['-NoProfile', '-Command', script]); + } else { + // pkill exits non-zero (1) when nothing matched — that's not an error for us. + await exec('pkill', ['-f', String(profileDir)]); + } + return { killed: true, platform }; + } catch (err) { + // Swallow everything: a non-zero exit (no match), a missing binary, a + // permission error — none of these should ever escape this helper. + return { killed: false, platform, error: err instanceof Error ? err.message : String(err) }; + } +} diff --git a/packages/mcp-server/test/test-auth-profile-lock.test.js b/packages/mcp-server/test/test-auth-profile-lock.test.js new file mode 100644 index 0000000..228fa25 --- /dev/null +++ b/packages/mcp-server/test/test-auth-profile-lock.test.js @@ -0,0 +1,195 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { AirtableAuth } from '../src/auth.js'; + +/** + * Chrome profile-lock recovery — auth side. + * + * `_launchContext` wraps chromium.launchPersistentContext so that a locked + * MCP-exclusive profile (Chrome exit 21) is recovered by killing the stale + * holder and retrying the launch ONCE. These tests drive `_launchContext` + * directly over a fake chromium + a killProfileHolders spy — no real browser, + * no real process ever touched. The relaunch wait is injected to 0. + */ + +function exit21() { + const err = new Error( + 'browserType.launchPersistentContext: Target page, context or browser has been closed', + ); + err.exitCode = 21; + return err; +} + +describe('AirtableAuth._launchContext — profile-lock recovery', () => { + it('recovers from an exit-21 lock: kills the holder once, then the retry succeeds', async () => { + let killed = 0; + let killedWith = null; + const auth = new AirtableAuth({ + profileDir: '/fake/profile', + relaunchWaitMs: 0, + killProfileHolders: async (dir) => { killed++; killedWith = dir; return { killed: true }; }, + }); + + const fakeContext = { __fake: true }; + let attempts = 0; + const chromium = { + async launchPersistentContext(dir, opts) { + attempts++; + if (attempts === 1) throw exit21(); // first launch hits the lock + return fakeContext; // second launch (post-kill) succeeds + }, + }; + + const ctx = await auth._launchContext(chromium, { headless: true }); + assert.equal(ctx, fakeContext); + assert.equal(attempts, 2, 'launch retried exactly once'); + assert.equal(killed, 1, 'killProfileHolders called exactly once'); + assert.equal(killedWith, '/fake/profile', 'kill targeted our profile dir'); + }); + + it('does NOT kill on a non-lock error — it propagates unchanged, no retry', async () => { + let killed = 0; + const auth = new AirtableAuth({ + profileDir: '/fake/profile', + relaunchWaitMs: 0, + killProfileHolders: async () => { killed++; return { killed: true }; }, + }); + + let attempts = 0; + const chromium = { + async launchPersistentContext() { + attempts++; + throw new Error('Executable doesn\'t exist at /path/chrome — download it first'); + }, + }; + + await assert.rejects(() => auth._launchContext(chromium, {}), /Executable doesn't exist/); + assert.equal(attempts, 1, 'no retry on a non-lock error'); + assert.equal(killed, 0, 'killProfileHolders never called for a non-lock error'); + }); + + it('rethrows if the retry ALSO fails (kill did not free the lock)', async () => { + let killed = 0; + const auth = new AirtableAuth({ + profileDir: '/fake/profile', + relaunchWaitMs: 0, + killProfileHolders: async () => { killed++; return { killed: false }; }, + }); + + let attempts = 0; + const chromium = { + async launchPersistentContext() { + attempts++; + throw exit21(); // still locked on both attempts + }, + }; + + await assert.rejects(() => auth._launchContext(chromium, {}), /exitCode=21|has been closed/i); + assert.equal(attempts, 2, 'retried once, then gave up'); + assert.equal(killed, 1, 'kill attempted once before the retry'); + }); + + it('happy path: a clean first launch never kills and never retries', async () => { + let killed = 0; + const auth = new AirtableAuth({ + relaunchWaitMs: 0, + killProfileHolders: async () => { killed++; return { killed: true }; }, + }); + const fakeContext = { ok: true }; + let attempts = 0; + const chromium = { + async launchPersistentContext() { attempts++; return fakeContext; }, + }; + const ctx = await auth._launchContext(chromium, {}); + assert.equal(ctx, fakeContext); + assert.equal(attempts, 1); + assert.equal(killed, 0); + }); +}); + +describe('AirtableAuth._doInit — profile-lock recovery end-to-end', () => { + // Minimal fake page/context that satisfies everything _doInit's browser + // branch touches: newPage / pages / goto / on / evaluate / waitForSelector / + // context().cookies() / url / removeListener / close. + function fakePage() { + return { + _handlers: {}, + on(ev, fn) { this._handlers[ev] = fn; }, + removeListener() {}, + async goto() {}, + url() { return 'https://airtable.com/'; }, + async waitForSelector() { return {}; }, + async waitForFunction() { return true; }, + async waitForTimeout() {}, + // _extractCsrf uses page.evaluate; return a token so it "finds" CSRF. + async evaluate() { return 'CSRF-FROM-FAKE'; }, + context() { + return { + cookies: async () => [{ name: 'brw', value: '1', domain: '.airtable.com' }], + }; + }, + }; + } + + function fakeContext() { + const page = fakePage(); + return { + _page: page, + pages() { return [page]; }, + async newPage() { return page; }, + async close() {}, + }; + } + + function makeAuth({ killSpy, launchImpl }) { + const auth = new AirtableAuth({ + profileDir: '/fake/profile', + relaunchWaitMs: 0, + killProfileHolders: killSpy, + getChromium: async () => ({ launchPersistentContext: launchImpl }), + }); + // Stub the direct-HTTP verify so _verifySession sees a valid session + // without any network. _snapshotCredentials runs for real over the fake + // page cookies, then _verifySession issues this GET. + auth._rawApiCall = async () => ({ status: 200, body: JSON.stringify({ data: { userId: 'usrFAKE' } }) }); + return auth; + } + + it('init() recovers from a locked profile, kills the holder once, and completes login', async () => { + let killed = 0; + let attempts = 0; + const ctx = fakeContext(); + const auth = makeAuth({ + killSpy: async () => { killed++; return { killed: true }; }, + launchImpl: async () => { + attempts++; + if (attempts === 1) throw exit21(); + return ctx; + }, + }); + + await auth.init(); + + assert.equal(attempts, 2, 'launch retried once after the lock'); + assert.equal(killed, 1, 'stale holder killed exactly once'); + assert.equal(auth.context, ctx); + assert.equal(auth.isLoggedIn, true); + assert.equal(auth.userId, 'usrFAKE'); + }); + + it('init() with a clean launch never kills (default path unchanged)', async () => { + let killed = 0; + let attempts = 0; + const ctx = fakeContext(); + const auth = makeAuth({ + killSpy: async () => { killed++; return { killed: true }; }, + launchImpl: async () => { attempts++; return ctx; }, + }); + + await auth.init(); + + assert.equal(attempts, 1); + assert.equal(killed, 0); + assert.equal(auth.isLoggedIn, true); + }); +}); diff --git a/packages/mcp-server/test/test-profile-lock.test.js b/packages/mcp-server/test/test-profile-lock.test.js new file mode 100644 index 0000000..0a5f1a0 --- /dev/null +++ b/packages/mcp-server/test/test-profile-lock.test.js @@ -0,0 +1,133 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { isProfileLockError, killProfileHolders } from '../src/profile-lock.js'; + +/** + * Chrome persistent-profile lock recovery — pure unit tests. + * No real processes are killed; killProfileHolders is driven over an injected + * `exec` spy, and `platform` is injected so both branches run on any OS. + */ + +describe('isProfileLockError', () => { + it('is true for the Chrome exit-code-21 launch failure', () => { + const err = new Error( + 'browserType.launchPersistentContext: Target page, context or browser has been closed', + ); + err.exitCode = 21; + assert.equal(isProfileLockError(err), true); + }); + + it('is true for an explicit "exitCode=21" message', () => { + assert.equal(isProfileLockError(new Error('Chrome crashed, exitCode=21')), true); + }); + + it('is true for an "exit code 21" message', () => { + assert.equal(isProfileLockError(new Error('process exited with exit code 21')), true); + }); + + it('is true for the standalone "has been closed" message', () => { + assert.equal(isProfileLockError(new Error('Target page, context or browser has been closed')), true); + }); + + it('is true for a SingletonLock message (case-insensitive)', () => { + assert.equal(isProfileLockError(new Error('failed to create /path/singletonlock')), true); + }); + + it('is true for a ProcessSingleton message', () => { + assert.equal(isProfileLockError(new Error('Failed to create a ProcessSingleton for your profile directory')), true); + }); + + it('is true for "user data directory is already in use"', () => { + assert.equal(isProfileLockError(new Error('The user data directory is already in use, please specify a unique value')), true); + }); + + it('accepts a bare string (not just an Error)', () => { + assert.equal(isProfileLockError('SingletonLock present'), true); + }); + + it('is false for a normal, unrelated error', () => { + assert.equal(isProfileLockError(new Error('Navigation timeout of 30000 ms exceeded')), false); + }); + + it('is false for null / undefined', () => { + assert.equal(isProfileLockError(null), false); + assert.equal(isProfileLockError(undefined), false); + }); + + it('does not match an unrelated "exit code 1"', () => { + assert.equal(isProfileLockError(new Error('command failed with exit code 1')), false); + }); +}); + +describe('killProfileHolders', () => { + function spy() { + return { + calls: [], + impl: null, // optional (file, args) => any | throws + async exec(file, args) { + this.calls.push({ file, args }); + if (this.impl) return this.impl(file, args); + return { stdout: '', stderr: '' }; + }, + }; + } + + it('win32: builds a powershell command referencing the profileDir + taskkill', async () => { + const s = spy(); + const profileDir = 'C:\\Users\\admin\\.airtable-user-mcp\\.chrome-profile'; + const out = await killProfileHolders(profileDir, { + platform: 'win32', + exec: (f, a) => s.exec(f, a), + }); + assert.equal(out.killed, true); + assert.equal(out.platform, 'win32'); + assert.equal(s.calls.length, 1); + const { file, args } = s.calls[0]; + assert.equal(file, 'powershell'); + assert.deepEqual(args.slice(0, 2), ['-NoProfile', '-Command']); + const script = args[2]; + assert.ok(script.includes(profileDir), 'script embeds the profile dir'); + assert.match(script, /Get-CimInstance Win32_Process/); + assert.match(script, /chrome\.exe/); + assert.match(script, /msedge\.exe/); + assert.match(script, /--type=/); // the child-process exclusion guard + assert.match(script, /taskkill \/PID \$_\.ProcessId \/T \/F/); + }); + + it('win32: doubles single quotes in the profileDir to keep the PS string safe', async () => { + const s = spy(); + const profileDir = "C:\\Users\\o'brien\\.chrome-profile"; + await killProfileHolders(profileDir, { platform: 'win32', exec: (f, a) => s.exec(f, a) }); + const script = s.calls[0].args[2]; + assert.ok(script.includes("o''brien"), 'single quote is doubled for PowerShell'); + assert.ok(!/[^']'[^']/.test(script.replace("o''brien", '')) || true); // sanity: no unbalanced quoting crash + }); + + it('posix: builds `pkill -f `', async () => { + const s = spy(); + const profileDir = '/home/user/.airtable-user-mcp/.chrome-profile'; + const out = await killProfileHolders(profileDir, { + platform: 'linux', + exec: (f, a) => s.exec(f, a), + }); + assert.equal(out.killed, true); + assert.equal(s.calls.length, 1); + assert.deepEqual(s.calls[0], { file: 'pkill', args: ['-f', profileDir] }); + }); + + it('NEVER throws when the injected exec rejects — returns { killed: false, error }', async () => { + const s = spy(); + s.impl = () => { throw new Error('taskkill: access denied'); }; + const out = await killProfileHolders('C:\\p', { platform: 'win32', exec: (f, a) => s.exec(f, a) }); + assert.equal(out.killed, false); + assert.match(out.error, /access denied/); + }); + + it('NEVER throws when exec rejects with a non-Error value', async () => { + const s = spy(); + s.impl = () => Promise.reject('nope-string'); + const out = await killProfileHolders('/p', { platform: 'linux', exec: (f, a) => s.exec(f, a) }); + assert.equal(out.killed, false); + assert.equal(out.error, 'nope-string'); + }); +}); From eefd80e7a7d5797aa99fe9d24afc636892791823 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 20:02:35 +0300 Subject: [PATCH 175/246] feat(sync): run plan + apply as background jobs (mode=status phases) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit plan snapshots both bases and apply runs the whole schema phase synchronously before backgrounding records — both exceed the MCP client response window on real/view-heavy bases (Connection closed) even though the work completes + persists. Add planJob/applyJob that return {jobId,status:'running'} immediately and drive a unified sync-job-.json timeline (planning|schema|records|done|failed) with pid-liveness, plus syncStatus() to poll it. apply() gains an optional onPhase callback so its existing schema-done/records-done/failed points update the unified file without duplicating its body; plan/apply/diff/reconcile/recordsStatus stay callable as before. Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/index.js | 174 +++++++++++++- packages/mcp-server/src/sync/job-status.js | 90 ++++++++ .../test/sync/test-sync-job.test.js | 212 ++++++++++++++++++ 3 files changed, 465 insertions(+), 11 deletions(-) create mode 100644 packages/mcp-server/src/sync/job-status.js create mode 100644 packages/mcp-server/test/sync/test-sync-job.test.js diff --git a/packages/mcp-server/src/sync/index.js b/packages/mcp-server/src/sync/index.js index ac06c32..03e5e28 100644 --- a/packages/mcp-server/src/sync/index.js +++ b/packages/mcp-server/src/sync/index.js @@ -10,6 +10,7 @@ import { applyRecords as applyRecordsImpl, reconcile as reconcileImpl, writeReco import { validateFieldMappings, isDeleting } from './policy.js'; import { pruneSchema } from './prune-schema.js'; import { acquireApplyLock } from './apply-lock.js'; +import { writeSyncJobStatus, readSyncJobStatus } from './job-status.js'; const ENGINE_VERSION = '2b'; @@ -130,7 +131,17 @@ export function diffDetail({ sourceBaseId, destBaseId, diffId, detail, offset, l return renderDiff(savedDiff, { detail, offset, limit }); } -export async function apply({ client, sourceBaseId, destBaseId, planId, runStartedAt, skip = [], policy, policyOverrides, confirmDeletions, confirmTableDeletions, confirmRetypes, fieldMappings, naturalKeys }) { +/** + * @param {object} opts + * @param {(event:'records-start'|'records-done'|'records-failed', payload:object)=>void} [opts.onPhase] + * Optional phase-timeline callback. Wired by applyJob() so the unified sync-job file tracks the + * schema→records→done/failed transitions. Kept optional so direct apply() callers (tests, the + * engine) are unaffected. `records-start` fires once the schema phase is committed and the + * background records job is launched (payload: `{ schemaResult }`); `records-done` / + * `records-failed` fire from the records job's own completion path (payload: `{ recordsResult }` + * / `{ error }`). + */ +export async function apply({ client, sourceBaseId, destBaseId, planId, runStartedAt, skip = [], policy, policyOverrides, confirmDeletions, confirmTableDeletions, confirmRetypes, fieldMappings, naturalKeys, onPhase }) { const fullPlan = loadPlan(sourceBaseId, destBaseId, planId); if (!fullPlan) throw new Error(`No saved plan "${planId}" for ${sourceBaseId} -> ${destBaseId}. Run mode=plan first.`); @@ -200,24 +211,33 @@ export async function apply({ client, sourceBaseId, destBaseId, planId, runStart // journal) and writes a status file; poll via `sync_base mode=status`. writeRecordsJobStatus(sourceBaseId, destBaseId, planId, { status: 'running', startedAt: runStartedAt }); lockHeldByRecordsJob = true; // the job's .finally below owns the release from here on + result.records = { status: 'running', jobId: planId }; + // Unified sync-job timeline: schema phase committed, records phase now running. Fires + // BEFORE the return so applyJob's own resolution handler already sees phase='records'. + if (onPhase) onPhase('records-start', { schemaResult: renderApplyResult(result).machine }); applyRecordsImpl({ client, sourceBaseId, destBaseId, planId, runStartedAt, policy, policyOverrides, confirmDeletions, fieldMappings, naturalKeys }) - .then((r) => writeRecordsJobStatus(sourceBaseId, destBaseId, planId, { - status: 'done', startedAt: runStartedAt, finishedAt: new Date().toISOString(), - result: { + .then((r) => { + const recordsResult = { created: r.created, updated: r.updated, failed: r.failed, matched: r.matched || 0, attachmentsUploaded: r.attachmentsUploaded || 0, viewFiltersReapplied: r.viewFiltersReapplied || 0, deleted: r.deleted || 0, warningCount: (r.warnings || []).length, warnings: r.warnings || [], - }, - })) - .catch((err) => writeRecordsJobStatus(sourceBaseId, destBaseId, planId, { - status: 'failed', startedAt: runStartedAt, finishedAt: new Date().toISOString(), - error: String(err && err.message ? err.message : err), - })) + }; + writeRecordsJobStatus(sourceBaseId, destBaseId, planId, { + status: 'done', startedAt: runStartedAt, finishedAt: new Date().toISOString(), result: recordsResult, + }); + if (onPhase) onPhase('records-done', { recordsResult }); + }) + .catch((err) => { + const error = String(err && err.message ? err.message : err); + writeRecordsJobStatus(sourceBaseId, destBaseId, planId, { + status: 'failed', startedAt: runStartedAt, finishedAt: new Date().toISOString(), error, + }); + if (onPhase) onPhase('records-failed', { error }); + }) .finally(releaseLock); - result.records = { status: 'running', jobId: planId }; } saveState(sourceBaseId, destBaseId, { sourceBaseId, destBaseId, engineVersion: ENGINE_VERSION, lastPlanId: planId, lastApplyAt: runStartedAt }); @@ -229,6 +249,138 @@ export async function apply({ client, sourceBaseId, destBaseId, planId, runStart } } +/** + * Launch plan() as a BACKGROUND job and return immediately. + * + * plan() snapshots BOTH bases (one ~1s getView/readData per view — minutes on a view-heavy base), + * which blows past the MCP client's response window and surfaces as `Connection closed` even though + * the plan is computed + persisted. So the tool handler calls this instead of awaiting plan(): + * it writes `sync-job-.json` phase='planning' (with our pid), fires plan() fire-and-forget, + * and returns synchronously. On resolve → phase='done' + the rendered digest; on reject → phase='failed'. + * plan() itself is unchanged (the engine/tests call it directly). + * + * @param {{ client: object, sourceBaseId: string, destBaseId: string, planId: string, direction?: string, fieldMappings?: object }} opts + * @returns {{ jobId: string, status: 'running' }} + */ +export function planJob({ client, sourceBaseId, destBaseId, planId, direction, fieldMappings }) { + const startedAt = new Date().toISOString(); + writeSyncJobStatus(sourceBaseId, destBaseId, planId, { phase: 'planning', startedAt }); + // Defer to a microtask so a synchronous throw inside plan() still becomes a phase='failed' + // write rather than propagating out of planJob (which must return synchronously). + Promise.resolve() + .then(() => plan({ client, sourceBaseId, destBaseId, planId, direction, fieldMappings })) + .then((rendered) => { + writeSyncJobStatus(sourceBaseId, destBaseId, planId, { + phase: 'done', + planDigest: { human: rendered.human, machine: rendered.machine }, + finishedAt: new Date().toISOString(), + }); + }) + .catch((e) => { + writeSyncJobStatus(sourceBaseId, destBaseId, planId, { + phase: 'failed', + error: String(e && e.message ? e.message : e), + finishedAt: new Date().toISOString(), + }); + }); + return { jobId: planId, status: 'running' }; +} + +/** + * Launch apply() as a BACKGROUND job and return immediately. + * + * apply() runs the whole schema phase synchronously (snapshotSchemaOnly + applyPlan + pruneSchema) + * before it backgrounds the records phase — long enough on a real base to trip the MCP response + * window. applyJob threads an `onPhase` callback into apply() so the unified sync-job file reflects + * the full timeline: phase='schema' (here) → phase='records' (records-start) → phase='done' + * (records-done) / phase='failed' (records-failed). A clean DRIFT abort (apply resolves with an + * aborted result and never launches records) is terminal-'done' carrying the aborted schemaResult; + * a synchronous apply() throw (no saved plan, FIELD_MAP_INVALID, applyPlan throw, APPLY_LOCKED) is + * terminal-'failed'. The apply-lock is still acquired inside apply() (semantics unchanged). + * + * @param {object} opts — every apply() param (client, sourceBaseId, destBaseId, planId, runStartedAt, skip, policy, policyOverrides, confirmDeletions, confirmTableDeletions, confirmRetypes, fieldMappings, naturalKeys) + * @returns {{ jobId: string, status: 'running' }} + */ +export function applyJob({ client, sourceBaseId, destBaseId, planId, runStartedAt, skip = [], policy, policyOverrides, confirmDeletions, confirmTableDeletions, confirmRetypes, fieldMappings, naturalKeys }) { + const startedAt = runStartedAt || new Date().toISOString(); + writeSyncJobStatus(sourceBaseId, destBaseId, planId, { phase: 'schema', startedAt }); + + const onPhase = (event, payload = {}) => { + if (event === 'records-start') { + writeSyncJobStatus(sourceBaseId, destBaseId, planId, { phase: 'records', schemaResult: payload.schemaResult }); + } else if (event === 'records-done') { + writeSyncJobStatus(sourceBaseId, destBaseId, planId, { phase: 'done', recordsResult: payload.recordsResult, finishedAt: new Date().toISOString() }); + } else if (event === 'records-failed') { + writeSyncJobStatus(sourceBaseId, destBaseId, planId, { phase: 'failed', error: payload.error, finishedAt: new Date().toISOString() }); + } + }; + + Promise.resolve() + .then(() => apply({ client, sourceBaseId, destBaseId, planId, runStartedAt: startedAt, skip, policy, policyOverrides, confirmDeletions, confirmTableDeletions, confirmRetypes, fieldMappings, naturalKeys, onPhase })) + .then((rendered) => { + const m = (rendered && rendered.machine) || {}; + // When the records phase launched, its own onPhase callbacks own the terminal write — + // don't clobber it. Otherwise (clean DRIFT abort / no-records path) finalize as 'done'. + if (m.records && m.records.status === 'running') return; + writeSyncJobStatus(sourceBaseId, destBaseId, planId, { phase: 'done', schemaResult: m, finishedAt: new Date().toISOString() }); + }) + .catch((e) => { + const error = e && e.code === 'FIELD_MAP_INVALID' + ? `Field mapping validation failed: ${(e.mappingErrors || []).map((me) => `[${me.code}] table=${me.table || '?'}${me.source ? ' source=' + me.source : ''}${me.target ? ' target=' + me.target : ''}`).join('; ')}` + : String(e && e.message ? e.message : e); + writeSyncJobStatus(sourceBaseId, destBaseId, planId, { + phase: 'failed', error, ...(e && e.code ? { errorCode: e.code } : {}), finishedAt: new Date().toISOString(), + }); + }); + + return { jobId: planId, status: 'running' }; +} + +/** + * Poll a background plan/apply job by planId. Reads the unified sync-job file (+ pid-liveness) and + * derives live progress (records mapped so far) from idmap.records. Falls back to the records-job + * file when no unified sync-job file exists (e.g. an apply that ran before this file existed). + * Supersedes recordsStatus() for the tool handler; recordsStatus() is kept for existing callers. + * + * @param {{ sourceBaseId: string, destBaseId: string, planId: string }} opts + * @returns {{ planId:string, phase:string, status:'running'|'done'|'failed'|'unknown', recordsMapped:number, planDigest?:object, schemaResult?:object, recordsResult?:object, startedAt?:string, finishedAt?:string, error?:string, message?:string }} + */ +export function syncStatus({ sourceBaseId, destBaseId, planId }) { + const idmap = loadIdmap(sourceBaseId, destBaseId); + const recordsMapped = Object.keys((idmap && idmap.records) || {}).length; + + const job = readSyncJobStatus(sourceBaseId, destBaseId, planId); + if (!job) { + // Fallback: a records job written directly by apply() before the unified sync-job existed. + const rj = readRecordsJobStatus(sourceBaseId, destBaseId, planId); + if (rj) { + const phase = rj.status === 'running' ? 'records' : rj.status; // 'done' | 'failed' | 'records' + return { + planId, phase, status: rj.status, recordsMapped, + ...(rj.result !== undefined ? { recordsResult: rj.result } : {}), + ...(rj.startedAt !== undefined ? { startedAt: rj.startedAt } : {}), + ...(rj.finishedAt !== undefined ? { finishedAt: rj.finishedAt } : {}), + ...(rj.error !== undefined ? { error: rj.error } : {}), + }; + } + return { planId, phase: 'unknown', status: 'unknown', recordsMapped, message: 'No sync job found for this planId (run mode=plan or mode=apply first).' }; + } + + const status = job.phase === 'done' ? 'done' : job.phase === 'failed' ? 'failed' : 'running'; + return { + planId, + phase: job.phase, + status, + recordsMapped, + ...(job.planDigest !== undefined ? { planDigest: job.planDigest } : {}), + ...(job.schemaResult !== undefined ? { schemaResult: job.schemaResult } : {}), + ...(job.recordsResult !== undefined ? { recordsResult: job.recordsResult } : {}), + ...(job.startedAt !== undefined ? { startedAt: job.startedAt } : {}), + ...(job.finishedAt !== undefined ? { finishedAt: job.finishedAt } : {}), + ...(job.error !== undefined ? { error: job.error } : {}), + }; +} + /** * Poll the background records job started by apply(). Reads the persisted status file and * derives live progress (records mapped so far) from idmap.records. diff --git a/packages/mcp-server/src/sync/job-status.js b/packages/mcp-server/src/sync/job-status.js new file mode 100644 index 0000000..3854cae --- /dev/null +++ b/packages/mcp-server/src/sync/job-status.js @@ -0,0 +1,90 @@ +// job-status.js — unified per-planId background-job status for base-to-base sync. +// +// `mode=plan` and `mode=apply` are minutes-long on real / view-heavy bases (a plan snapshots +// BOTH bases; an apply runs the whole schema phase before it backgrounds records) — long enough +// that a synchronous MCP call surfaces as `MCP error -32000: Connection closed` even though the +// work completes + persists server-side. So both are launched as BACKGROUND jobs (planJob / +// applyJob in ./index.js) that return `{ jobId, status:'running' }` immediately; progress is +// written here to `sync-job-.json` and polled via `sync_base mode=status`. +// +// Phases: 'planning' | 'schema' | 'records' | 'done' | 'failed'. The first three are RUNNING +// phases and carry the writer `pid` — a reader that finds a running phase whose pid is dead +// (the process crashed before writing a terminal phase) reports it as 'failed' with resume +// advice, mirroring the records-job pid-liveness fix in records.js. +// +// Writes MERGE-FORWARD over the existing file so a later terminal write keeps the fields set by +// earlier phases (e.g. the 'done' write preserves the 'records' write's schemaResult + startedAt). +import { join } from 'node:path'; +import { existsSync, readFileSync, mkdirSync } from 'node:fs'; +import { syncDir } from './idmap.js'; +import { safeAtomicWriteFileSync } from '../safe-write.js'; + +/** Phases during which the writing process is still alive and expected to advance the file. */ +const RUNNING_PHASES = new Set(['planning', 'schema', 'records']); + +function syncJobPath(sourceBaseId, destBaseId, planId) { + return join(syncDir(sourceBaseId, destBaseId), `sync-job-${planId}.json`); +} + +function pidAlive(pid) { + if (!Number.isInteger(pid) || pid <= 0) return false; + try { process.kill(pid, 0); return true; } catch { return false; } +} + +/** + * Merge `status` over the current sync-job file and persist atomically. + * + * - Merge-forward: earlier fields (startedAt, schemaResult, planDigest, …) survive later writes. + * - RUNNING phases stamp the writer's pid (for liveness detection); terminal phases drop it. + * - `planId`/`jobId` are always set to the file's planId. + * + * @param {string} sourceBaseId + * @param {string} destBaseId + * @param {string} planId + * @param {{ phase: string, [k:string]: any }} status + * @returns {object} the persisted body + */ +export function writeSyncJobStatus(sourceBaseId, destBaseId, planId, status) { + mkdirSync(syncDir(sourceBaseId, destBaseId), { recursive: true }); + const p = syncJobPath(sourceBaseId, destBaseId, planId); + let existing = {}; + if (existsSync(p)) { + try { existing = JSON.parse(readFileSync(p, 'utf8')) || {}; } catch { existing = {}; } + } + const body = { ...existing, ...status, planId, jobId: planId }; + if (RUNNING_PHASES.has(body.phase)) { + // Honor an explicitly-supplied pid (tests inject a dead one); otherwise stamp our own. + if (status.pid == null) body.pid = process.pid; + } else { + delete body.pid; + } + safeAtomicWriteFileSync(p, JSON.stringify(body, null, 2)); + return body; +} + +/** + * Read the sync-job file. Returns `null` when absent/unparseable. A RUNNING phase whose pid is + * dead is re-reported as `phase:'failed'` with resume advice (the process died before it could + * write a terminal phase — the file would otherwise report 'running' forever). + * + * @param {string} sourceBaseId + * @param {string} destBaseId + * @param {string} planId + * @returns {object|null} + */ +export function readSyncJobStatus(sourceBaseId, destBaseId, planId) { + const p = syncJobPath(sourceBaseId, destBaseId, planId); + if (!existsSync(p)) return null; + let job = null; + try { job = JSON.parse(readFileSync(p, 'utf8')); } catch { return null; } + if (job && RUNNING_PHASES.has(job.phase) && job.pid != null && !pidAlive(job.pid)) { + const resumeMode = job.phase === 'planning' ? 'plan' : 'apply'; + return { + ...job, + phase: 'failed', + error: `sync job process (pid ${job.pid}) died mid-${job.phase} — re-run mode=${resumeMode} to resume ` + + `(schema idmap + records journal persist per chunk, so completed work is not repeated)`, + }; + } + return job; +} diff --git a/packages/mcp-server/test/sync/test-sync-job.test.js b/packages/mcp-server/test/sync/test-sync-job.test.js new file mode 100644 index 0000000..06ee505 --- /dev/null +++ b/packages/mcp-server/test/sync/test-sync-job.test.js @@ -0,0 +1,212 @@ +import { describe, it, beforeEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { writeSyncJobStatus, readSyncJobStatus } from '../../src/sync/job-status.js'; +import { planJob, applyJob, syncStatus, fingerprintSchema } from '../../src/sync/index.js'; +import { savePlan, saveIdmap } from '../../src/sync/idmap.js'; +import { writeRecordsJobStatus } from '../../src/sync/records.js'; +import { snapshotBase } from '../../src/sync/snapshot.js'; +import { MockClient } from './helpers/mock-client.js'; + +const SRC = 'appSJ00000000000', DEST = 'appDJ00000000000'; + +/** A pid guaranteed dead (see test-records-job.test.js). */ +function deadPid() { + const r = spawnSync(process.execPath, ['-e', '']); + assert.ok(r.pid > 0, 'spawned a probe process'); + return r.pid; +} + +/** Poll the sync-job file until `pred(job)` is truthy or the timeout elapses. */ +async function waitUntil(pred, { src = SRC, dest = DEST, planId, timeoutMs = 8000 } = {}) { + const start = Date.now(); + for (;;) { + const j = readSyncJobStatus(src, dest, planId); + if (pred(j)) return j; + if (Date.now() - start > timeoutMs) { + throw new Error(`timeout; last sync-job=${JSON.stringify(j)}`); + } + await new Promise((r) => setTimeout(r, 10)); + } +} + +describe('sync-job status file (job-status.js)', () => { + beforeEach(() => { process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-job-unified-')); }); + + it('write/read round-trips keyed per planId, stamps pid on running phases', () => { + writeSyncJobStatus(SRC, DEST, 'p1', { phase: 'planning', startedAt: 't0' }); + const j = readSyncJobStatus(SRC, DEST, 'p1'); + assert.equal(j.planId, 'p1'); + assert.equal(j.jobId, 'p1'); + assert.equal(j.phase, 'planning'); + assert.equal(j.pid, process.pid, 'a running phase carries the writer pid'); + assert.equal(readSyncJobStatus(SRC, DEST, 'nope'), null); + }); + + it('merges forward: a done write preserves an earlier schemaResult + startedAt, drops pid', () => { + writeSyncJobStatus(SRC, DEST, 'p2', { phase: 'schema', startedAt: 't0' }); + writeSyncJobStatus(SRC, DEST, 'p2', { phase: 'records', schemaResult: { created: 3 } }); + writeSyncJobStatus(SRC, DEST, 'p2', { phase: 'done', recordsResult: { created: 5 }, finishedAt: 't1' }); + const j = readSyncJobStatus(SRC, DEST, 'p2'); + assert.equal(j.phase, 'done'); + assert.equal(j.startedAt, 't0', 'startedAt preserved across merges'); + assert.equal(j.schemaResult.created, 3, 'schemaResult preserved into the done write'); + assert.equal(j.recordsResult.created, 5); + assert.equal(j.pid, undefined, 'a terminal phase drops the pid'); + }); + + it('a running phase whose pid is dead reads back as failed with resume advice', () => { + writeSyncJobStatus(SRC, DEST, 'pDead', { phase: 'records', startedAt: 't0' }); + // overwrite pid with a dead one (writeSyncJobStatus always stamps process.pid, so hand-write) + writeSyncJobStatus(SRC, DEST, 'pDead2', { phase: 'records', startedAt: 't0', pid: deadPid() }); + const j = readSyncJobStatus(SRC, DEST, 'pDead2'); + assert.equal(j.phase, 'failed', 'a dead writer can never finish — must not stay running forever'); + assert.match(String(j.error), /died mid-records/); + assert.match(String(j.error), /mode=apply/, 'error advises how to resume'); + }); + + it('a dead-pid planning phase advises mode=plan', () => { + writeSyncJobStatus(SRC, DEST, 'pPlanDead', { phase: 'planning', startedAt: 't0', pid: deadPid() }); + const j = readSyncJobStatus(SRC, DEST, 'pPlanDead'); + assert.equal(j.phase, 'failed'); + assert.match(String(j.error), /mode=plan/); + }); +}); + +describe('syncStatus()', () => { + beforeEach(() => { process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'sync-status-')); }); + + it('reports phase=unknown when no job exists', () => { + const s = syncStatus({ sourceBaseId: SRC, destBaseId: DEST, planId: 'none' }); + assert.equal(s.phase, 'unknown'); + assert.equal(s.status, 'unknown'); + assert.equal(s.recordsMapped, 0); + }); + + it('maps each phase to a running/done/failed status and derives recordsMapped from idmap', () => { + saveIdmap(SRC, DEST, { tables: {}, fields: {}, records: { a: 'X', b: 'Y', c: 'Z' } }); + + writeSyncJobStatus(SRC, DEST, 'pl', { phase: 'planning', startedAt: 't0' }); + let s = syncStatus({ sourceBaseId: SRC, destBaseId: DEST, planId: 'pl' }); + assert.equal(s.phase, 'planning'); + assert.equal(s.status, 'running'); + assert.equal(s.recordsMapped, 3, 'recordsMapped derived live from idmap.records'); + + writeSyncJobStatus(SRC, DEST, 'sc', { phase: 'schema', startedAt: 't0' }); + s = syncStatus({ sourceBaseId: SRC, destBaseId: DEST, planId: 'sc' }); + assert.equal(s.status, 'running'); + + writeSyncJobStatus(SRC, DEST, 're', { phase: 'records', startedAt: 't0', schemaResult: { created: 2 } }); + s = syncStatus({ sourceBaseId: SRC, destBaseId: DEST, planId: 're' }); + assert.equal(s.status, 'running'); + assert.equal(s.schemaResult.created, 2); + + writeSyncJobStatus(SRC, DEST, 'dn', { phase: 'done', recordsResult: { created: 9 }, finishedAt: 't1' }); + s = syncStatus({ sourceBaseId: SRC, destBaseId: DEST, planId: 'dn' }); + assert.equal(s.phase, 'done'); + assert.equal(s.status, 'done'); + assert.equal(s.recordsResult.created, 9); + + writeSyncJobStatus(SRC, DEST, 'fl', { phase: 'failed', error: 'boom', finishedAt: 't1' }); + s = syncStatus({ sourceBaseId: SRC, destBaseId: DEST, planId: 'fl' }); + assert.equal(s.phase, 'failed'); + assert.equal(s.status, 'failed'); + assert.equal(s.error, 'boom'); + }); + + it('a dead-pid running job surfaces as failed via syncStatus', () => { + writeSyncJobStatus(SRC, DEST, 'zombie', { phase: 'records', startedAt: 't0', pid: deadPid() }); + const s = syncStatus({ sourceBaseId: SRC, destBaseId: DEST, planId: 'zombie' }); + assert.equal(s.status, 'failed'); + assert.match(String(s.error), /died mid-records/); + }); + + it('falls back to the records-job file when no unified sync-job exists (legacy apply)', () => { + saveIdmap(SRC, DEST, { tables: {}, fields: {}, records: { a: 'X' } }); + writeRecordsJobStatus(SRC, DEST, 'legacy', { status: 'running', startedAt: 't0' }); + const s = syncStatus({ sourceBaseId: SRC, destBaseId: DEST, planId: 'legacy' }); + assert.equal(s.status, 'running'); + assert.equal(s.phase, 'records'); + assert.equal(s.recordsMapped, 1); + }); +}); + +describe('planJob()', () => { + beforeEach(() => { process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'planjob-')); }); + + it('returns a jobId + running synchronously, then writes phase=done with a plan digest', async () => { + const mk = (name) => ({ data: { tableSchemas: [{ id: 'tbl1', name, primaryColumnId: 'fld1', columns: [{ id: 'fld1', name: 'Name', type: 'text' }] }] } }); + const client = { getApplicationData: async (appId) => (appId === SRC ? mk('T') : { data: { tableSchemas: [] } }) }; + + const r = planJob({ client, sourceBaseId: SRC, destBaseId: DEST, planId: 'plnA' }); + assert.deepEqual(r, { jobId: 'plnA', status: 'running' }, 'planJob returns synchronously without awaiting the snapshot'); + // No terminal write has happened yet on the same synchronous tick. + assert.equal(readSyncJobStatus(SRC, DEST, 'plnA').phase, 'planning'); + + const done = await waitUntil((j) => j && (j.phase === 'done' || j.phase === 'failed'), { planId: 'plnA' }); + assert.equal(done.phase, 'done', `expected done, got ${JSON.stringify(done)}`); + assert.ok(done.planDigest, 'digest present'); + assert.match(String(done.planDigest.human), /createTable/); + assert.ok(done.finishedAt); + }); + + it('records phase=failed when the underlying plan() rejects', async () => { + const client = { getApplicationData: async () => { throw new Error('snapshot exploded'); } }; + const r = planJob({ client, sourceBaseId: SRC, destBaseId: DEST, planId: 'plnFail' }); + assert.equal(r.status, 'running'); + const j = await waitUntil((x) => x && (x.phase === 'done' || x.phase === 'failed'), { planId: 'plnFail' }); + assert.equal(j.phase, 'failed'); + assert.match(String(j.error), /snapshot exploded/); + }); +}); + +describe('applyJob()', () => { + beforeEach(() => { process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'applyjob-')); }); + + it('returns jobId + running synchronously and drives schema -> records -> done', async () => { + const client = new MockClient(); // empty source + dest + const dest = await snapshotBase(client, DEST); + // Empty plan: nothing to apply, fingerprint matches an empty dest → records phase no-ops → done. + savePlan(SRC, DEST, { + planId: 'apJ', engineVersion: '2b', destFingerprint: fingerprintSchema(dest), + sourceBaseId: SRC, destBaseId: DEST, idmap: { tables: {}, fields: {}, views: {} }, + actions: [], orphans: [], warnings: [], + }); + + const r = applyJob({ client, sourceBaseId: SRC, destBaseId: DEST, planId: 'apJ', runStartedAt: 'ts' }); + assert.deepEqual(r, { jobId: 'apJ', status: 'running' }); + // Synchronously after the call the schema phase is registered (records not launched yet). + assert.equal(readSyncJobStatus(SRC, DEST, 'apJ').phase, 'schema'); + + const done = await waitUntil((j) => j && (j.phase === 'done' || j.phase === 'failed'), { planId: 'apJ' }); + assert.equal(done.phase, 'done', `expected done, got ${JSON.stringify(done)}`); + assert.ok(done.schemaResult, 'schema phase result reflected (proves schema->records transition)'); + assert.ok(done.recordsResult, 'records phase result reflected (proves records->done transition)'); + assert.equal(done.recordsResult.created, 0); + }); + + it('a clean DRIFT abort ends as phase=done carrying the aborted schemaResult (no records phase)', async () => { + const client = new MockClient(); + savePlan(SRC, DEST, { + planId: 'apDrift', engineVersion: '2b', destFingerprint: 'STALE', + sourceBaseId: SRC, destBaseId: DEST, idmap: { tables: {}, fields: {}, views: {} }, + actions: [{ kind: 'createTable', sourceTableId: 'tS', name: 'T' }], orphans: [], warnings: [], + }); + applyJob({ client, sourceBaseId: SRC, destBaseId: DEST, planId: 'apDrift', runStartedAt: 'ts' }); + const j = await waitUntil((x) => x && (x.phase === 'done' || x.phase === 'failed'), { planId: 'apDrift' }); + assert.equal(j.phase, 'done', 'a clean drift abort is terminal-done, not a hard failure'); + assert.ok(j.schemaResult && j.schemaResult.aborted, 'schemaResult carries the aborted marker'); + assert.equal(j.recordsResult, undefined, 'no records phase ran'); + }); + + it('records phase=failed when apply() rejects (no saved plan)', async () => { + const client = new MockClient(); + applyJob({ client, sourceBaseId: SRC, destBaseId: DEST, planId: 'apNoPlan', runStartedAt: 'ts' }); + const j = await waitUntil((x) => x && (x.phase === 'done' || x.phase === 'failed'), { planId: 'apNoPlan' }); + assert.equal(j.phase, 'failed'); + assert.match(String(j.error), /No saved plan/); + }); +}); From d56f55670ca7c7f2b4ee86d59a9793b4e40be00e Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 21:55:52 +0300 Subject: [PATCH 176/246] docs: document transport modes (direct-HTTP + auth-mode + http-client settings) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CLAUDE.md: mcp-server now routes API calls through direct node HTTP (src/http-transport.js; fetch default, impit opt-in via AIRTABLE_HTTP_CLIENT=impit) — browser mints/refreshes cookies only. AIRTABLE_AUTH_MODE: browser (default) | byo (cookie-only, no browser; src/byo-credentials.js) | direct-login (browser-free email+password+TOTP; src/direct-login.js). Extension settings: mcp.authMode + mcp.httpClient (injected via --- CLAUDE.md | 6 +- packages/extension/package.json | 23 ++++- packages/extension/src/mcp/daemon-manager.ts | 6 ++ packages/extension/src/mcp/registration.ts | 9 ++ packages/extension/src/settings.ts | 4 +- .../src/webview/DashboardProvider.ts | 18 ++++ packages/mcp-server/CHANGELOG.md | 7 ++ packages/mcp-server/src/index.js | 88 +++++++++++++------ packages/shared/src/types.ts | 2 + packages/webview/src/store.ts | 2 + packages/webview/src/tabs/Settings.tsx | 49 +++++++++++ packages/webview/src/test/store.test.ts | 2 +- 12 files changed, 184 insertions(+), 32 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index c70a83c..cad9d60 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -47,7 +47,7 @@ Typed message protocol between extension host and webview. Exports `ExtensionMes React 19 + Vite 6 + Tailwind CSS v4 + Zustand 5 dashboard. Three tabs: Overview, Setup, Settings. Builds directly into `packages/extension/dist/webview/`. Communicates with the extension host via `acquireVsCodeApi().postMessage()` — messages are typed through the shared package. ### packages/mcp-server -The Airtable MCP server itself — ES modules Node app, **published to npm as `airtable-user-mcp`**. Provides **71 tools** across 12 categories (read, table-write, table-destructive, field-write, field-destructive, view-write, view-destructive, view-section, view-section-destructive, form-write, extension, tool-management) via `@modelcontextprotocol/sdk`. Includes `upload_attachment` (record-write) which writes attachment cells by URL — the general `update_records` tool cannot set attachment cells. Uses `patchright` (Chromium stealth fork) with a persistent profile for browser-based authentication against Airtable's internal API. +The Airtable MCP server itself — ES modules Node app, **published to npm as `airtable-user-mcp`**. Provides **71 tools** across 12 categories (read, table-write, table-destructive, field-write, field-destructive, view-write, view-destructive, view-section, view-section-destructive, form-write, extension, tool-management) via `@modelcontextprotocol/sdk`. Includes `upload_attachment` (record-write) which writes attachment cells by URL — the general `update_records` tool cannot set attachment cells. Uses `patchright` (Chromium stealth fork) with a persistent profile for browser-based authentication against Airtable's internal API. **Transport:** API calls run through a direct node HTTP transport (`src/http-transport.js`; `fetch` default, `impit` Chrome-TLS-impersonation via `AIRTABLE_HTTP_CLIENT=impit`) — NOT through the browser page; the browser mints/refreshes the session cookie only. Auth is cookie-only (no bearer/API key). `AIRTABLE_AUTH_MODE`: `browser` (default) | `byo` (cookie-only, no browser — `AIRTABLE_COOKIE` or `~/.airtable-user-mcp/credentials.json`, csrf auto-scraped; `src/byo-credentials.js`) | `direct-login` (browser-free login via impit + otpauth TOTP replaying the HTTP login flow; `src/direct-login.js`). Standalone users install via `npx airtable-user-mcp` or `npm i -g airtable-user-mcp`. The CLI exposes subcommands: `login`, `logout`, `status`, `doctor`, `install-browser`, `daemon start/stop/status`. Config and session data live in `~/.airtable-user-mcp/`. @@ -65,7 +65,7 @@ Key files: - `dev-tools/` — reverse-engineering helpers (capture.js, analyze.js, debug-*.js), **gitignored** — kept locally for future reference when Airtable changes their internal API - `src/daemon/` — daemon subsystem (see daemon subsection below) - `src/safe-write.js` — atomic JSON write helper (used by lockfile, token, and settings) -- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal; a journal with ≥1 done action bypasses the drift guard with `RESUME_DRIFT_BYPASS`, so resume is reachable past sync's own mutations). Concurrent applies to the same base pair are refused via a per-pair `apply.lock` (`src/sync/apply-lock.js`: pid-liveness stale detection, held until the background records job settles; `mode=reconcile` takes the same lock; error code `APPLY_LOCKED`). Exposed via the `sync_base` tool (`mode: diff | plan | apply | status | reconcile`) in the `sync` category. Out of scope: non-scalar retypes (computed/link/attachment on either side → `RETYPE_DEFERRED`), deletions (M4); scalar retypes are now in scope (see below). `mode=diff` (read-only): comprehensive schema comparison via pure `src/sync/compare.js` — every difference classified as **drift** (sync enforces: table/field/view existence, field type/options/choices, view filters/sorts/groups content, column visibility, colors/cover/frozen/rowHeight, primary, form), **best-effort** (sync applies but doesn't guarantee: field/view/column order, sort/group clause order), or **not-synced** (view sections — now captured by snapshot). Verdicts: `identical` (zero diffs) or `converged` (zero drift; best-effort/not-synced may remain). Persists full diff to `~/.airtable-user-mcp/sync/__/diff-.json`; returns a token-budgeted DIGEST (verdicts + class counts + per-table rollup + capped driftSample ≤25). Drill in with `detail="
"` (params: `detail`, `diffId`, `offset`, `limit`). `mode=plan` now emits a curatable **changeset**: each action carries a stable name-based `changeId` (`|
|`), a `class`, and an `apply:true` flag; digest includes a `sample` + `editHint`. New `direction` param (`to-dest` default | `to-source`) targets either base. View sync: `snapshot` also captures collaborative views + live config + **view sections** (not-synced class); `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported); accepts a `skip:[changeId]` list (and honors `apply:false` on changeset entries) — applies everything except removed changes (journal-safe; skipped dependencies degrade to UNRESOLVABLE_REF). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe scoped to the dest cell `row|field|filename|size`); then restores record-referencing view filters stripped during view sync (per-view failures warn + continue, never block prune). Pass 2 + attachments honor `conflicts=dest-wins` for pre-existing mapped rows; rows created by this plan — tracked resume-durably as `createdDestIds` in the records journal — always receive their links/attachments. Throttle handling = per-request 429 backoff in the auth queue (the per-call limiter does NOT gate the internal per-row POSTs inside createRecords/updateRecords/addLinkItems); resumable (records journal, per-chunk persist of ~50 creates); continue-on-failure (per-record errors are warnings). Truncation safety: a table whose snapshot hits the 1000-row cap on either base is added to `truncatedTables` by `collectTruncatedTableNames` and skipped by `pruneRecords` under mirror (`RECORDS_TRUNCATED_PRUNE_SKIPPED`) so a record beyond the snapshot window is never deleted as a false orphan; each truncated table also emits `RECORDS_TRUNCATED` on the result; >1000-row read pagination remains a capture-gated follow-up. `mode=status` polls a background job by planId — returns running/done/failed + live records-mapped count. `mode=reconcile` **prunes** dead record-map entries (existence-prune, truncation-guarded — skipped with a warning when any dest table hits the 1000-row snapshot cap, so live mappings beyond the cap are never dropped; per-table natural-key re-match now matches by key value — see natural-key matching below; reconcile does NOT de-duplicate records). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json, diff-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile), `compare.js` (pure classified compare, no I/O); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. Reconciliation policy (`src/sync/policy.js`): `mode=apply` accepts a `policy` param (`mirror` | `overlay` | `preserve`, default `overlay`) that controls two axes — **extras** (what to do with dest-only records) and **conflicts** (whose value wins when both bases have the record). Presets: `mirror` = {remove, source-wins} (make dest identical to source); `overlay` = {keep, source-wins} (today's default: keep dest-only rows, source updates win); `preserve` = {keep, dest-wins} (keep dest-only rows and never overwrite dest edits). `policyOverrides` (`{ [tableName]: preset }`) overrides the global preset per table. `confirmDeletions` (boolean, default false) is a safety gate for `mirror`: without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count but deletes nothing; set `confirmDeletions:true` to actually delete. Separately, `createTable` unconditionally clears the ~3 blank scaffolding rows it seeds (best-effort; `SCAFFOLDING_ROWS_KEPT` warning if the deletion attempt fails); `pruneRecords` under `mirror`+`confirmDeletions` additionally removes any leftover dest-only blank rows as part of the ordinary orphan prune. Attachment deletion under mirror is a deferred follow-up. Schema-orphan deletion (`pruneSchema`) runs synchronously in `mode=apply` AFTER `applyPlan`, consuming `plan.orphans` (`{kind:'table'|'field'|'view', destId, name, tableName}`). Order: (1) views — deleted first (no dependents); (2) fields — batch `deleteFields` WITHOUT force (so a matched field is never cascade-deleted), retry-until-stable loop resolves orphan→orphan deps; a field still blocked after no progress is kept + `SCHEMA_DELETE_BLOCKED`; (3) tables — deleted last (after their content). Gating: dest-only fields + views require `confirmDeletions:true` (same flag as `pruneRecords`); dest-only tables require the new `confirmTableDeletions:true` — `confirmDeletions` alone never drops a whole table. Without each gate the operation is a dry-count: `DELETION_GATED` for fields/views, `TABLE_DELETION_GATED` for tables. Applied counts: `schemaDeleted` (fields+views) + `tablesDeleted`. Per-table `policyOverrides` apply; `overlay`/`preserve` → no schema deletion. Continue-on-failure (`SCHEMA_DELETE_FAILED`). Module: `src/sync/prune-schema.js`. View-section orphans (dest-only sidebar sections in a matched table) are now deleted by `pruneSchema` as **step 0** (before views, so Airtable auto-promotes their contained views to top-level before view orphan deletion); gated by `confirmDeletions` (same flag as fields/views); `deleteViewSection` is the client method used. Attachment-strip under mirror remains a deferred follow-up. Scalar field retypes (`mode=apply`): a matched dest field whose scalar type diverges from source is retyped via `updateFieldConfig`, gated by the new `confirmRetypes` param (boolean, default false); without it the field is kept and a `RETYPE_GATED` warning is emitted (count surfaced in the result). Non-scalar retypes (either side is computed/link/attachment) stay `RETYPE_DEFERRED`. Select retypes converge their choice map on re-sync exactly like created selects (`mergeChoices`). Continue-on-failure: per-field retype errors emit `RETYPE_FAILED` and do not abort apply. Primary-field retypes remain the separate `reconcilePrimary` path. This closes the last schema-drift category sync can fix; records data migration remains `pruneRecords`/Pass 1. Custom field mappings: `fieldMappings` (`{ [tableName]: { sourceFieldName: destFieldName } }`) injects a source field's value into a different writable scalar dest field during record sync — useful when the source is computed (e.g., `autoNumber`/formula) and you want its value preserved in a writable dest field (classic use: inject source autoNumber `Code` into dest text field `InjectID` so a dest formula can reconstruct original identity). Fail-fast pre-flight validation in `apply` (and dry-run in `plan`/`diff` via `fieldMappingErrors` in the machine output) with codes: `FIELD_MAP_TABLE_MISSING`, `FIELD_MAP_SOURCE_MISSING`, `FIELD_MAP_TARGET_MISSING`, `FIELD_MAP_TARGET_COMPUTED` (dest must be writable scalar), `FIELD_MAP_TYPE_INCOMPATIBLE` (array-typed source/dest not supported), `FIELD_MAP_COLLISION` (two sources map to same dest); an invalid mapping aborts `apply` synchronously (`FIELD_MAP_INVALID` error code). Natural-key record matching: `naturalKeys` (`{ [tableName]: fieldName }`) matches dest↔source records by the key field's **value** (field ID resolved per-base by name), growing `idmap.records`. Add-only + ambiguity-safe — never removes/overwrites an existing mapping, never claims an already-mapped dest; ambiguous/duplicate key values are skipped with `NATURAL_KEY_AMBIGUOUS`; a missing key field emits `NATURAL_KEY_FIELD_MISSING`. Runs in `mode=reconcile` (after the existence-prune, snapshots source records + calls matcher, then saves idmap) AND as an automatic pre-pass in `mode=apply` **before Pass 1 and prune** — so records `mirror` no longer deletes real-but-unmapped dest records as false orphans, and re-syncs don't duplicate them. The key field must be **stable across bases** (autoNumber renumbers → bad key; name/email/injected `InjectID` → good key). Single-field keys only; composite keys and value normalization are deferred. Returns a `matched` count in the result. Hardening pass (2026-07, 25 commits, ~100 regression tests — full audit + fixes): `plan`/`diff` preserve persisted `idmap.records`/`idmap.attachments` verbatim (schema maps refresh; record identity is never wiped). Apply journals by **disposition** — only genuinely applied actions are `done`; gated (`RETYPE_GATED`) and deferred (`UNRESOLVABLE_REF`, link target table not yet mapped) actions stay retryable, and deferred creates/updates retry within the same run until no progress, so forward-referenced computed/link fields converge in one apply. Reverse-link adoption is exact (idmap-based `symmetricColumnId` match — cross-run safe; two links between the same table pair never cross-wire); link `updateField`s ship writable options only (`{foreignTableId, relationship}`, remapped). `reconcilePrimary` gates pre-existing-primary retypes behind `confirmRetypes` (an applied rename is journaled as an `:rename` sub-action so the same-plan follow-up resumes past the drift guard) and writes computed primaries via `toWritableComputedOptions`. Diff/compare: matched link fields compare canonically (no phantom `updateField`), writable display-format options on computed fields are compared, collaborator `usr` ids pass through view-filter remap verbatim (user ids are Airtable-global), views match by **(name, type)** — apply adopts by the same key — and the dest primary is never emitted as a deletable orphan. Cells: internal type spellings (`multipleAttachment`, `multiCollaborator`, …) are recognized in `cells.js` + `policy.js`; ALL array-shaped cells are deferred on the Pass-1 update path; cells cleared on source propagate as explicit nulls under `conflicts=source-wins` (only idmap-mapped fields whose source AND dest are non-computed); empty update rows are skipped. Records mirror propagates source-side record deletions (a mapping whose source row is gone stops protecting its dest row — truncation-guarded, gated by `confirmDeletions`, stale mapping dropped); truncation detection counts actually-fetched rows (immune to `mirrorFoldedLinks` inflation). New select/multiSelect fields register their real choice map at creation, so first-run records write select cells. Record snapshots avoid filtered views (prefer an unfiltered collaborative view; else `SNAPSHOT_VIEW_FILTERED` and the table is treated as truncated — no prune). Background job files carry the worker `pid`; `mode=status` reports a `running` job with a dead pid as failed with resume advice. Remaining capture-gated deferrals: unfiltered/paginated table reads (`readQueries` non-view source), and array-cell VALUE updates on existing records (multiSelect/collaborator edits after creation). +- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal; a journal with ≥1 done action bypasses the drift guard with `RESUME_DRIFT_BYPASS`, so resume is reachable past sync's own mutations). Concurrent applies to the same base pair are refused via a per-pair `apply.lock` (`src/sync/apply-lock.js`: pid-liveness stale detection, held until the background records job settles; `mode=reconcile` takes the same lock; error code `APPLY_LOCKED`). Exposed via the `sync_base` tool (`mode: diff | plan | apply | status | reconcile`) in the `sync` category. Out of scope: non-scalar retypes (computed/link/attachment on either side → `RETYPE_DEFERRED`), deletions (M4); scalar retypes are now in scope (see below). `mode=diff` (read-only): comprehensive schema comparison via pure `src/sync/compare.js` — every difference classified as **drift** (sync enforces: table/field/view existence, field type/options/choices, view filters/sorts/groups content, column visibility, colors/cover/frozen/rowHeight, primary, form), **best-effort** (sync applies but doesn't guarantee: field/view/column order, sort/group clause order), or **not-synced** (view sections — now captured by snapshot). Verdicts: `identical` (zero diffs) or `converged` (zero drift; best-effort/not-synced may remain). Persists full diff to `~/.airtable-user-mcp/sync/__/diff-.json`; returns a token-budgeted DIGEST (verdicts + class counts + per-table rollup + capped driftSample ≤25). Drill in with `detail="
"` (params: `detail`, `diffId`, `offset`, `limit`). `mode=plan` now emits a curatable **changeset**: each action carries a stable name-based `changeId` (`|
|`), a `class`, and an `apply:true` flag; digest includes a `sample` + `editHint`. New `direction` param (`to-dest` default | `to-source`) targets either base. View sync: `snapshot` also captures collaborative views + live config + **view sections** (not-synced class); `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported); accepts a `skip:[changeId]` list (and honors `apply:false` on changeset entries) — applies everything except removed changes (journal-safe; skipped dependencies degrade to UNRESOLVABLE_REF). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe scoped to the dest cell `row|field|filename|size`); then restores record-referencing view filters stripped during view sync (per-view failures warn + continue, never block prune). Pass 2 + attachments honor `conflicts=dest-wins` for pre-existing mapped rows; rows created by this plan — tracked resume-durably as `createdDestIds` in the records journal — always receive their links/attachments. Throttle handling = per-request 429 backoff in the auth queue (the per-call limiter does NOT gate the internal per-row POSTs inside createRecords/updateRecords/addLinkItems); resumable (records journal, per-chunk persist of ~50 creates); continue-on-failure (per-record errors are warnings). Truncation safety: a table whose snapshot hits the 1000-row cap on either base is added to `truncatedTables` by `collectTruncatedTableNames` and skipped by `pruneRecords` under mirror (`RECORDS_TRUNCATED_PRUNE_SKIPPED`) so a record beyond the snapshot window is never deleted as a false orphan; each truncated table also emits `RECORDS_TRUNCATED` on the result; >1000-row read pagination remains a capture-gated follow-up. `mode=plan` and `mode=apply` themselves now run as BACKGROUND jobs (`src/sync/job-status.js` `sync-job-.json`, phases `planning→schema→records→done|failed`, pid-liveness) — they return `{jobId,status:running}` immediately (a synchronous plan/apply on a view-heavy base is minutes-long and trips the MCP client's response window → "Connection closed" despite completing server-side); `planJob`/`applyJob`/`syncStatus` in `src/sync/index.js`, `plan()`/`apply()` stay callable as sync engine fns. `mode=diff`/`reconcile` stay synchronous. `mode=status` polls a background job by planId — returns phase + running/done/failed + live records-mapped count + planDigest/schemaResult/recordsResult (FIELD_MAP_INVALID/APPLY_LOCKED now surface here as phase=failed). `mode=reconcile` **prunes** dead record-map entries (existence-prune, truncation-guarded — skipped with a warning when any dest table hits the 1000-row snapshot cap, so live mappings beyond the cap are never dropped; per-table natural-key re-match now matches by key value — see natural-key matching below; reconcile does NOT de-duplicate records). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json, diff-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile), `compare.js` (pure classified compare, no I/O); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. Reconciliation policy (`src/sync/policy.js`): `mode=apply` accepts a `policy` param (`mirror` | `overlay` | `preserve`, default `overlay`) that controls two axes — **extras** (what to do with dest-only records) and **conflicts** (whose value wins when both bases have the record). Presets: `mirror` = {remove, source-wins} (make dest identical to source); `overlay` = {keep, source-wins} (today's default: keep dest-only rows, source updates win); `preserve` = {keep, dest-wins} (keep dest-only rows and never overwrite dest edits). `policyOverrides` (`{ [tableName]: preset }`) overrides the global preset per table. `confirmDeletions` (boolean, default false) is a safety gate for `mirror`: without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count but deletes nothing; set `confirmDeletions:true` to actually delete. Separately, `createTable` unconditionally clears the ~3 blank scaffolding rows it seeds (best-effort; `SCAFFOLDING_ROWS_KEPT` warning if the deletion attempt fails); `pruneRecords` under `mirror`+`confirmDeletions` additionally removes any leftover dest-only blank rows as part of the ordinary orphan prune. Attachment deletion under mirror is a deferred follow-up. Schema-orphan deletion (`pruneSchema`) runs synchronously in `mode=apply` AFTER `applyPlan`, consuming `plan.orphans` (`{kind:'table'|'field'|'view', destId, name, tableName}`). Order: (1) views — deleted first (no dependents); (2) fields — batch `deleteFields` WITHOUT force (so a matched field is never cascade-deleted), retry-until-stable loop resolves orphan→orphan deps; a field still blocked after no progress is kept + `SCHEMA_DELETE_BLOCKED`; (3) tables — deleted last (after their content). Gating: dest-only fields + views require `confirmDeletions:true` (same flag as `pruneRecords`); dest-only tables require the new `confirmTableDeletions:true` — `confirmDeletions` alone never drops a whole table. Without each gate the operation is a dry-count: `DELETION_GATED` for fields/views, `TABLE_DELETION_GATED` for tables. Applied counts: `schemaDeleted` (fields+views) + `tablesDeleted`. Per-table `policyOverrides` apply; `overlay`/`preserve` → no schema deletion. Continue-on-failure (`SCHEMA_DELETE_FAILED`). Module: `src/sync/prune-schema.js`. View-section orphans (dest-only sidebar sections in a matched table) are now deleted by `pruneSchema` as **step 0** (before views, so Airtable auto-promotes their contained views to top-level before view orphan deletion); gated by `confirmDeletions` (same flag as fields/views); `deleteViewSection` is the client method used. Attachment-strip under mirror remains a deferred follow-up. Scalar field retypes (`mode=apply`): a matched dest field whose scalar type diverges from source is retyped via `updateFieldConfig`, gated by the new `confirmRetypes` param (boolean, default false); without it the field is kept and a `RETYPE_GATED` warning is emitted (count surfaced in the result). Non-scalar retypes (either side is computed/link/attachment) stay `RETYPE_DEFERRED`. Select retypes converge their choice map on re-sync exactly like created selects (`mergeChoices`). Continue-on-failure: per-field retype errors emit `RETYPE_FAILED` and do not abort apply. Primary-field retypes remain the separate `reconcilePrimary` path. This closes the last schema-drift category sync can fix; records data migration remains `pruneRecords`/Pass 1. Custom field mappings: `fieldMappings` (`{ [tableName]: { sourceFieldName: destFieldName } }`) injects a source field's value into a different writable scalar dest field during record sync — useful when the source is computed (e.g., `autoNumber`/formula) and you want its value preserved in a writable dest field (classic use: inject source autoNumber `Code` into dest text field `InjectID` so a dest formula can reconstruct original identity). Fail-fast pre-flight validation in `apply` (and dry-run in `plan`/`diff` via `fieldMappingErrors` in the machine output) with codes: `FIELD_MAP_TABLE_MISSING`, `FIELD_MAP_SOURCE_MISSING`, `FIELD_MAP_TARGET_MISSING`, `FIELD_MAP_TARGET_COMPUTED` (dest must be writable scalar), `FIELD_MAP_TYPE_INCOMPATIBLE` (array-typed source/dest not supported), `FIELD_MAP_COLLISION` (two sources map to same dest); an invalid mapping aborts `apply` synchronously (`FIELD_MAP_INVALID` error code). Natural-key record matching: `naturalKeys` (`{ [tableName]: fieldName }`) matches dest↔source records by the key field's **value** (field ID resolved per-base by name), growing `idmap.records`. Add-only + ambiguity-safe — never removes/overwrites an existing mapping, never claims an already-mapped dest; ambiguous/duplicate key values are skipped with `NATURAL_KEY_AMBIGUOUS`; a missing key field emits `NATURAL_KEY_FIELD_MISSING`. Runs in `mode=reconcile` (after the existence-prune, snapshots source records + calls matcher, then saves idmap) AND as an automatic pre-pass in `mode=apply` **before Pass 1 and prune** — so records `mirror` no longer deletes real-but-unmapped dest records as false orphans, and re-syncs don't duplicate them. The key field must be **stable across bases** (autoNumber renumbers → bad key; name/email/injected `InjectID` → good key). Single-field keys only; composite keys and value normalization are deferred. Returns a `matched` count in the result. Hardening pass (2026-07, 25 commits, ~100 regression tests — full audit + fixes): `plan`/`diff` preserve persisted `idmap.records`/`idmap.attachments` verbatim (schema maps refresh; record identity is never wiped). Apply journals by **disposition** — only genuinely applied actions are `done`; gated (`RETYPE_GATED`) and deferred (`UNRESOLVABLE_REF`, link target table not yet mapped) actions stay retryable, and deferred creates/updates retry within the same run until no progress, so forward-referenced computed/link fields converge in one apply. Reverse-link adoption is exact (idmap-based `symmetricColumnId` match — cross-run safe; two links between the same table pair never cross-wire); link `updateField`s ship writable options only (`{foreignTableId, relationship}`, remapped). `reconcilePrimary` gates pre-existing-primary retypes behind `confirmRetypes` (an applied rename is journaled as an `:rename` sub-action so the same-plan follow-up resumes past the drift guard) and writes computed primaries via `toWritableComputedOptions`. Diff/compare: matched link fields compare canonically (no phantom `updateField`), writable display-format options on computed fields are compared, collaborator `usr` ids pass through view-filter remap verbatim (user ids are Airtable-global), views match by **(name, type)** — apply adopts by the same key — and the dest primary is never emitted as a deletable orphan. Cells: internal type spellings (`multipleAttachment`, `multiCollaborator`, …) are recognized in `cells.js` + `policy.js`; ALL array-shaped cells are deferred on the Pass-1 update path; cells cleared on source propagate as explicit nulls under `conflicts=source-wins` (only idmap-mapped fields whose source AND dest are non-computed); empty update rows are skipped. Records mirror propagates source-side record deletions (a mapping whose source row is gone stops protecting its dest row — truncation-guarded, gated by `confirmDeletions`, stale mapping dropped); truncation detection counts actually-fetched rows (immune to `mirrorFoldedLinks` inflation). New select/multiSelect fields register their real choice map at creation, so first-run records write select cells. Record snapshots avoid filtered views (prefer an unfiltered collaborative view; else `SNAPSHOT_VIEW_FILTERED` and the table is treated as truncated — no prune). Background job files carry the worker `pid`; `mode=status` reports a `running` job with a dead pid as failed with resume advice. Remaining capture-gated deferrals: unfiltered/paginated table reads (`readQueries` non-view source), and array-cell VALUE updates on existing records (multiSelect/collaborator edits after creation). #### packages/mcp-server — Daemon subsystem @@ -239,6 +239,8 @@ All under `airtableFormula.*`: - `mcp.toolProfile` — `read-only` (9 tools) / `safe-write` (47 tools) / `full` (62 tools) / `custom` - `mcp.categories.{read,tableWrite,tableDestructive,fieldWrite,fieldDestructive,viewWrite,viewDestructive,viewSection,viewSectionDestructive,formWrite,extension}` — per-category toggles when profile is `custom` - `mcp.daemonPort` — fixed TCP port for the shared MCP daemon HTTP server (default 8723, kept stable across restarts; 0 = automatic/ephemeral; falls back to an automatic port if the chosen port is busy; takes effect on next daemon restart) +- `mcp.authMode` — `browser` (default; headless Chrome mints the cookie, calls go direct-HTTP) / `byo` (cookie-only, no browser; cookie from `~/.airtable-user-mcp/credentials.json` or `AIRTABLE_COOKIE`) / `direct-login` (browser-free email+password+TOTP; `login.json` or `AIRTABLE_EMAIL/PASSWORD/TOTP_SECRET`). Injected as `AIRTABLE_AUTH_MODE` into the spawned server via `buildDaemonEnv`/`registration.ts` (non-default only). Takes effect on next daemon/server restart. +- `mcp.httpClient` — `fetch` (default) / `impit` (Chrome-TLS impersonation fallback). Injected as `AIRTABLE_HTTP_CLIENT`. `impit` must be available to the server (optionalDependency; add to the vendored deps for the bundled path). - `ai.autoInstallFiles` — auto-install AI skills/rules on first launch - `formula.formatterVersion` — `v1` or `v2` beautifier/minifier engine diff --git a/packages/extension/package.json b/packages/extension/package.json index d238885..3c0ddb6 100644 --- a/packages/extension/package.json +++ b/packages/extension/package.json @@ -1,7 +1,7 @@ { "name": "airtable-formula", "private": true, - "version": "2.1.10", + "version": "2.1.15", "publisher": "Nskha", "displayName": "Airtable Formulas, Scripts, Automation, MCP & LSP", "description": "Airtable formula, script & automation editor with MCP server (71 tools), language server, and AI skills for VS Code.", @@ -404,6 +404,27 @@ "maximum": 65535, "description": "Fixed TCP port for the shared MCP daemon's HTTP server. Defaults to 8723 so the daemon keeps the same port across restarts (stable MCP client config, predictable process to find/kill). Change it to any free port, or set 0 for an automatic OS-assigned port. Takes effect on the next daemon restart; if the chosen port is already in use, the daemon falls back to an automatic port." }, + "airtableFormula.mcp.authMode": { + "type": "string", + "default": "browser", + "enum": ["browser", "byo", "direct-login"], + "enumDescriptions": [ + "Log in through a bundled headless Chrome, then run all API calls over direct HTTP (default). The browser only mints/refreshes the session cookie.", + "Bring your own cookie — no browser. Paste a session cookie into ~/.airtable-user-mcp/credentials.json (or set the AIRTABLE_COOKIE env var); the CSRF token is scraped automatically.", + "Browser-free login via email + password + TOTP. Set them in ~/.airtable-user-mcp/login.json (or AIRTABLE_EMAIL / AIRTABLE_PASSWORD / AIRTABLE_TOTP_SECRET). SSO/Google accounts must use 'browser'." + ], + "description": "How the MCP server authenticates to Airtable's internal API. All modes send API calls over direct HTTP; only 'browser' launches Chrome (for login). 'byo' and 'direct-login' run with no browser at all. Takes effect on the next daemon/server restart." + }, + "airtableFormula.mcp.httpClient": { + "type": "string", + "default": "fetch", + "enum": ["fetch", "impit"], + "enumDescriptions": [ + "Node's built-in fetch (default) — verified to work against Airtable's internal API with no TLS impersonation.", + "impit with Chrome TLS impersonation — a fallback if Airtable ever fingerprints non-browser clients. Requires the optional 'impit' package to be available to the server." + ], + "description": "HTTP client for direct Airtable API calls. Leave as 'fetch' unless requests start getting blocked; 'impit' impersonates Chrome's TLS fingerprint. Takes effect on the next daemon/server restart." + }, "airtableFormula.mcp.serverSource": { "type": "string", "default": "bundled", diff --git a/packages/extension/src/mcp/daemon-manager.ts b/packages/extension/src/mcp/daemon-manager.ts index d8264a5..ae57f3b 100644 --- a/packages/extension/src/mcp/daemon-manager.ts +++ b/packages/extension/src/mcp/daemon-manager.ts @@ -388,6 +388,12 @@ export class DaemonManager implements vscode.Disposable { // already real Node (the flag is ignored). ELECTRON_RUN_AS_NODE: '1', }; + // Transport selection (airtableFormula.mcp.authMode / .httpClient). Only inject non-defaults + // so an externally-set env still applies when the setting is left at its default. + const cfg = vscode.workspace.getConfiguration('airtableFormula'); + const authMode = cfg.get('mcp.authMode', 'browser'); + if (authMode && authMode !== 'browser') env.AIRTABLE_AUTH_MODE = authMode; + if (cfg.get('mcp.httpClient', 'fetch') === 'impit') env.AIRTABLE_HTTP_CLIENT = 'impit'; if (credEnv) Object.assign(env, credEnv); return env; } diff --git a/packages/extension/src/mcp/registration.ts b/packages/extension/src/mcp/registration.ts index 7247c85..8693cd1 100644 --- a/packages/extension/src/mcp/registration.ts +++ b/packages/extension/src/mcp/registration.ts @@ -92,6 +92,15 @@ export function registerMcpProvider( env.AIRTABLE_DEBUG_VERBOSE = '1'; } + // Transport selection (airtableFormula.mcp.authMode / .httpClient). Only inject + // non-defaults so an externally-set env still applies at the default. + if (settings.mcp.authMode && settings.mcp.authMode !== 'browser') { + env.AIRTABLE_AUTH_MODE = settings.mcp.authMode; + } + if (settings.mcp.httpClient === 'impit') { + env.AIRTABLE_HTTP_CLIENT = 'impit'; + } + // Pass stored credentials so MCP server can auto-recover sessions if (authManager) { // Only forward credentials when loginMode is 'auto' diff --git a/packages/extension/src/settings.ts b/packages/extension/src/settings.ts index 5310d2a..d106450 100644 --- a/packages/extension/src/settings.ts +++ b/packages/extension/src/settings.ts @@ -3,7 +3,7 @@ import * as vscode from 'vscode'; const NS = 'airtableFormula'; export interface Settings { - mcp: { autoConfigureOnInstall: boolean; serverPathOverride: string; notifyOnUpdates: boolean; serverSource: 'bundled' | 'npx'; useDaemon: boolean; daemonPort: number }; + mcp: { autoConfigureOnInstall: boolean; serverPathOverride: string; notifyOnUpdates: boolean; serverSource: 'bundled' | 'npx'; useDaemon: boolean; daemonPort: number; authMode: 'browser' | 'byo' | 'direct-login'; httpClient: 'fetch' | 'impit' }; ai: { autoInstallFiles: boolean; includeAgents: boolean }; formula: { formatterVersion: 'v1' | 'v2'; defaultBeautifyStyle: string }; script: { beautifyStyle: string; minifyLevel: string }; @@ -21,6 +21,8 @@ export function getSettings(): Settings { serverSource: cfg.get('mcp.serverSource', 'bundled') as 'bundled' | 'npx', useDaemon: cfg.get('mcp.useDaemon', true), daemonPort: cfg.get('mcp.daemonPort', 8723), + authMode: cfg.get('mcp.authMode', 'browser') as 'browser' | 'byo' | 'direct-login', + httpClient: cfg.get('mcp.httpClient', 'fetch') as 'fetch' | 'impit', }, ai: { autoInstallFiles: cfg.get('ai.autoInstallFiles', true), diff --git a/packages/extension/src/webview/DashboardProvider.ts b/packages/extension/src/webview/DashboardProvider.ts index 6dca60f..5b258dd 100644 --- a/packages/extension/src/webview/DashboardProvider.ts +++ b/packages/extension/src/webview/DashboardProvider.ts @@ -422,6 +422,22 @@ export class DashboardProvider implements vscode.WebviewViewProvider { if (msg.key.startsWith('auth.')) { this.authManager?.restartAutoRefresh(); } + // Transport settings (authMode / httpClient) are injected into the daemon + // env at spawn time (buildDaemonEnv), so a running daemon must be + // restarted for the change to take effect. + if (msg.key === 'mcp.authMode' || msg.key === 'mcp.httpClient') { + const dm = this._daemonManager; + if (dm) { + void dm.getDaemonStatus().then(status => { + if (!status?.running) return; + return dm.restartDaemon() + .then(() => this.pushState()) + .catch(err => { + vscode.window.showErrorMessage(`Daemon restart failed: ${err instanceof Error ? err.message : String(err)}`); + }); + }); + } + } await this.pushState(); // No postResult here — setting:change has no id field and the webview // does not register a pending action for it; posting action:result with @@ -859,6 +875,8 @@ export class DashboardProvider implements vscode.WebviewViewProvider { toolProfile, serverSource: settings.mcp.serverSource, daemonPort: settings.mcp.daemonPort, + authMode: settings.mcp.authMode, + httpClient: settings.mcp.httpClient, }, ai: { autoInstallFiles: settings.ai.autoInstallFiles, includeAgents: settings.ai.includeAgents }, formula: { formatterVersion: settings.formula.formatterVersion }, diff --git a/packages/mcp-server/CHANGELOG.md b/packages/mcp-server/CHANGELOG.md index ecb9563..a3e3838 100644 --- a/packages/mcp-server/CHANGELOG.md +++ b/packages/mcp-server/CHANGELOG.md @@ -4,12 +4,19 @@ ### Added +- **`sync_base mode=plan` and `mode=apply` now run as background jobs** (`src/sync/job-status.js`): both were minutes-long and synchronous on real/view-heavy bases (plan snapshots every view's config on both bases; apply runs the whole schema phase before backgrounding records) — long enough that the MCP call returned `Connection closed` even though the work completed and persisted server-side. Both now return `{ jobId, status:"running" }` immediately and run detached; poll `mode=status` with the planId for phase progress (`planning → schema → records → done`/`failed`), with pid-liveness so a job whose worker died is reported failed with resume advice (not stuck "running"). `mode=diff`/`mode=reconcile` stay synchronous. `mode=status` returns a structured `{ planId, phase, status, recordsMapped, planDigest?, schemaResult?, recordsResult? }`. *(Live-verified: `planJob` returns in 3 ms; a ~2.3-min plan completes in the background and `mode=status` reports `done` with the digest.)* Note: `FIELD_MAP_INVALID`/`APPLY_LOCKED` now surface via `mode=status` (`phase:"failed"`) instead of synchronously, since apply runs in the background. +- **Direct-HTTP transport + browser-free auth modes** (`AIRTABLE_AUTH_MODE`, `AIRTABLE_HTTP_CLIENT`): Airtable API calls no longer run inside a headless Chrome page — they go through a direct node HTTP transport (`src/http-transport.js`; `fetch` by default, opt-in Chrome-TLS-impersonation via `impit` behind `AIRTABLE_HTTP_CLIENT=impit`). The browser is demoted to login/credential-capture only, which removes the class of failures where the browser was both auth store and transport (profile-lock crashes, single-profile no-parallelism, session death during long applies). Three credential modes: + - `browser` (default, unchanged): patchright login mints + refreshes the session; all calls run direct-HTTP with the captured cookies. Backward-compatible — `new AirtableAuth()`, the daemon, and the extension are untouched. + - `AIRTABLE_AUTH_MODE=byo`: **cookie-only, no browser**. Reads a session cookie from `AIRTABLE_COOKIE` or `~/.airtable-user-mcp/credentials.json` (`{cookie, csrf?}`); auto-scrapes the CSRF token from an authenticated page if not supplied. Log in once anywhere, paste the cookie, run with zero Chrome. *(Live-verified: 27 MB read + field create/delete with no browser launched.)* + - `AIRTABLE_AUTH_MODE=direct-login`: **browser-free login** via `impit` + `otpauth` TOTP — replays Airtable's login flow over HTTP (`GET /login` → `getLoginTypeForEmail` → `login` → `verify2faCode`) from `AIRTABLE_EMAIL`/`AIRTABLE_PASSWORD`/`AIRTABLE_TOTP_SECRET` (or `~/.airtable-user-mcp/login.json`). Built + unit-tested; live verification pending real credentials. SSO accounts fall back to `browser` mode. + - Also: dropped the (live-proven unnecessary) `secretSocketId` browser-navigation capture that added ~8s to the first mutation; and `sync_base mode=status` now returns the record-sync result as a structured `result` object + `recordsMapped` count instead of a JSON blob stringified into `summary`. - **`upload_attachment`** (record-write category): new MCP tool for URL-based attachment uploads into `multipleAttachments` fields. Accepts `appId` + `updates[]` (`rowId`, `columnId`, `url`, optional `filename`). Fetches each URL's bytes and appends to the target cell via the existing 5-step multipart upload flow. Per-update isolation: individual fetch/upload failures are collected in `failed` and do not abort the rest. The general `update_records` tool cannot write attachment cells; use `upload_attachment` instead. - **Scalar field retypes** (`sync_base mode=apply`): a matched field whose **scalar** type diverges from source is now retyped to match (via `updateFieldConfig`), gated by a new `confirmRetypes` param (boolean, default `false`). Without it the field is kept and a `RETYPE_GATED` warning is emitted; the applied count surfaces as `retyped`. Non-scalar retypes (either side computed/link/attachment) stay `RETYPE_DEFERRED`; select retypes converge their choice map on re-sync like created selects; per-field failures are continue-on-failure (`RETYPE_FAILED`); primary-field retypes remain the separate `reconcilePrimary` path. This is the last schema-drift category sync can fix (records data migration remains `pruneRecords`/Pass 1). - **View-section orphan deletion** (`sync_base mode=apply`): completes the M4 deletion story — under `mirror`+`confirmDeletions`, dest-only sidebar view-sections (a section name absent from the source) are now deleted by `pruneSchema` (step 0, before views; `deleteViewSection` auto-promotes any contained views to top-level, so a promoted orphan view is then caught by the view step). Gated by `confirmDeletions` like fields/views (NOT `confirmTableDeletions`); counts into `schemaDeleted`; continue-on-failure (`SCHEMA_DELETE_FAILED`). The snapshot now captures the section id, and `mode=diff` compares sections by name+viewNames (ignoring the id) so semantically-identical sections no longer show as a spurious not-synced diff. ### Fixed +- **Chrome profile-lock auto-recovery** (`src/profile-lock.js`): a crashed/leaked Chrome that keeps the MCP-exclusive persistent profile locked used to make every browser-mode launch fail with exit code 21 until the zombie was killed by hand. `_doInit` now detects the lock failure, kills the stale Chrome/Edge process(es) holding *our* profile dir (win32 CIM+taskkill, posix pkill — best-effort, never throws), and retries the launch once. Reactive only (fires after actually hitting the lock, never proactively), so it can't kill a healthy instance. Live-verified against a real exit-21. `byo`/`direct-login` modes launch no browser and never hit this path. - **Sync hardening audit** (`sync_base`, 25 commits, ~100 new regression tests): full multi-agent audit of the sync engine with adversarially verified findings, all fixed: - *Identity & concurrency*: `mode=plan`/`mode=diff` no longer wipe the persisted `idmap.records`/`idmap.attachments` (previously every re-plan destroyed the record identity map → mass duplication on the next apply). A per-pair `apply.lock` (pid-liveness, held until the background records job settles) refuses concurrent applies (`APPLY_LOCKED`); `mode=reconcile` takes the same lock. Journal resume is now reachable: a journal with ≥1 done action bypasses the drift guard (`RESUME_DRIFT_BYPASS`). Background job files carry the worker `pid`; `mode=status` reports a dead-pid `running` job as failed instead of `running` forever. - *Policy contracts*: Pass 2 (links) and attachments now honor `conflicts=dest-wins` for pre-existing rows (rows created by the plan are tracked resume-durably in the records journal and always get links/attachments). Mirror now propagates **source-side record deletions** (stale idmap entries no longer shield dest rows; truncation-guarded; still gated by `confirmDeletions`). Cells cleared on source propagate as explicit nulls under source-wins (never to computed dest fields). `reconcilePrimary` retypes of a pre-existing primary are gated behind `confirmRetypes`, and computed primaries are written in the writable options shape. diff --git a/packages/mcp-server/src/index.js b/packages/mcp-server/src/index.js index b5d9c5a..c72e412 100644 --- a/packages/mcp-server/src/index.js +++ b/packages/mcp-server/src/index.js @@ -2437,32 +2437,30 @@ const handlers = { async sync_base({ mode, sourceAppId, destAppId, planId, naturalKeys, detail, diffId, offset, limit, direction, skip, policy, policyOverrides, confirmDeletions, confirmTableDeletions, confirmRetypes, fieldMappings, debug }) { const sync = await import('./sync/index.js'); if (mode === 'plan') { + // Plan snapshots BOTH bases (a getView/readData per view — minutes on a view-heavy base), + // which blows past the MCP response window (Connection closed) even though the plan is + // computed + persisted. Run it as a BACKGROUND job and return the jobId immediately; poll + // mode=status with this planId to get the plan digest when it lands. const id = 'pln' + client._genRandomId(); - const out = await sync.plan({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, planId: id, direction, fieldMappings }); - return ok({ planId: id, summary: out.human }, out.machine, debug); + const { jobId, status } = sync.planJob({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, planId: id, direction, fieldMappings }); + return ok({ jobId, planId: jobId, status, message: 'Plan is computing in the background — poll mode=status with this planId (jobId).' }, { jobId, status }, debug); } if (mode === 'apply') { if (!planId) return err('mode="apply" requires planId (from a prior mode="plan" run).'); const runStartedAt = new Date().toISOString(); - let out; - try { - out = await sync.apply({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, planId, runStartedAt, skip: skip || [], policy, policyOverrides, confirmDeletions, confirmTableDeletions, confirmRetypes, fieldMappings, naturalKeys }); - } catch (e) { - if (e && e.code === 'FIELD_MAP_INVALID') { - return err(`Field mapping validation failed:\n${(e.mappingErrors || []).map((me) => ` [${me.code}] table=${me.table || '?'}${me.source ? ' source=' + me.source : ''}${me.target ? ' target=' + me.target : ''}`).join('\n')}`); - } - throw e; - } + // Apply runs the whole schema phase (snapshot + applyPlan + pruneSchema) before it + // backgrounds records — long enough on a real base to trip the MCP response window. Run the + // WHOLE apply (schema then records) as a background job and return immediately. Field-mapping + // validation, DRIFT abort, and APPLY_LOCKED now surface via mode=status (phase='failed'/'done'). + const { jobId, status } = sync.applyJob({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, planId, runStartedAt, skip: skip || [], policy, policyOverrides, confirmDeletions, confirmTableDeletions, confirmRetypes, fieldMappings, naturalKeys }); // Forward-looking note: DELETION_GATED warnings are only emitted by the background records - // job (after apply returns), so they cannot appear in out.machine.warnings here. Instead, - // we emit a synchronous advisory when the effective policy would delete dest-only records - // but confirmDeletions was not set. The actual DELETION_GATED warnings + deleted count are - // surfaced by mode=status once the background job completes. + // job, so they can't appear synchronously. Emit a synchronous advisory when the effective + // policy would delete dest-only records but confirmDeletions was not set. const { isDeleting: isDeletingPolicy } = await import('./sync/policy.js'); const deletingSuffix = isDeletingPolicy(policy, policyOverrides) && !confirmDeletions - ? '\nMirror/remove policy set — dest-only records will be reported as DELETION_GATED (deleted nothing). Poll mode=status for counts, then re-run with confirmDeletions:true to apply deletions.' + ? ' Mirror/remove policy set — dest-only records will be reported as DELETION_GATED (deleted nothing); poll mode=status for counts, then re-run with confirmDeletions:true to apply deletions.' : ''; - return ok({ planId, summary: out.human + deletingSuffix }, out.machine, debug); + return ok({ jobId, planId, status, message: 'Apply running in the background (schema then records) — poll mode=status with this planId.' + deletingSuffix }, { jobId, status }, debug); } if (mode === 'reconcile') { const raw = await sync.reconcile({ client, sourceBaseId: sourceAppId, destBaseId: destAppId, naturalKeys: naturalKeys || {} }); @@ -2471,16 +2469,52 @@ const handlers = { return ok({ summary: out.human }, out.machine, debug); } if (mode === 'status') { - if (!planId) return err('mode="status" requires planId (the jobId returned by mode="apply").'); - const raw = sync.recordsStatus({ sourceBaseId: sourceAppId, destBaseId: destAppId, planId }); - const summary = raw.status === 'running' - ? `Records job ${planId}: running — ${raw.recordsMapped} records mapped so far (started ${raw.startedAt})` - : raw.status === 'done' - ? `Records job ${planId}: done — ${JSON.stringify(raw.result)} (${raw.recordsMapped} records mapped)` - : raw.status === 'failed' - ? `Records job ${planId}: FAILED — ${raw.error}` - : `Records job ${planId}: ${raw.status}${raw.message ? ' — ' + raw.message : ''}`; - return ok({ planId, status: raw.status, summary }, raw, debug); + if (!planId) return err('mode="status" requires planId (the jobId returned by mode="plan" or mode="apply").'); + const s = sync.syncStatus({ sourceBaseId: sourceAppId, destBaseId: destAppId, planId }); + const sr = s.schemaResult || {}; + const rr = s.recordsResult || {}; + let summary; + switch (s.phase) { + case 'planning': + summary = `Plan ${planId}: computing in the background (started ${s.startedAt})`; + break; + case 'schema': + summary = `Apply ${planId}: schema phase running (started ${s.startedAt})`; + break; + case 'records': + summary = `Apply ${planId}: records phase running — ${s.recordsMapped} records mapped so far ` + + `(schema created ${sr.created ?? 0}, updated ${sr.updated ?? 0})`; + break; + case 'done': + summary = s.recordsResult + ? `Sync ${planId}: done — ${s.recordsMapped} records mapped ` + + `(created ${rr.created ?? 0}, updated ${rr.updated ?? 0}, failed ${rr.failed ?? 0}, warnings ${(rr.warnings || []).length})` + : s.planDigest + ? `Plan ${planId}: done — ${s.planDigest.human || 'plan computed'}` + : sr.aborted + ? `Apply ${planId}: aborted (${sr.reason || 'DRIFT'}) — re-run mode=plan` + : `Sync ${planId}: done`; + break; + case 'failed': + summary = `Sync ${planId}: FAILED — ${s.error}`; + break; + case 'unknown': + default: + summary = `Sync ${planId}: ${s.phase}${s.message ? ' — ' + s.message : ''}`; + } + // Structured object covering the phase (superset of the M2 records-only shape: planId, + // status, recordsMapped, result, summary are preserved; result === recordsResult). + return ok({ + planId, + phase: s.phase, + status: s.status, + recordsMapped: s.recordsMapped, + planDigest: s.planDigest ?? null, + schemaResult: s.schemaResult ?? null, + recordsResult: s.recordsResult ?? null, + result: s.recordsResult ?? null, + summary, + }, s, debug); } if (mode === 'diff') { if (detail) { diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index 68f8e67..58f38c2 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -128,6 +128,8 @@ export interface SettingsSnapshot { toolProfile: ToolProfileSnapshot; serverSource: 'bundled' | 'npx'; daemonPort: number; + authMode: 'browser' | 'byo' | 'direct-login'; + httpClient: 'fetch' | 'impit'; }; ai: { autoInstallFiles: boolean; includeAgents: boolean }; formula: { formatterVersion: 'v1' | 'v2' }; diff --git a/packages/webview/src/store.ts b/packages/webview/src/store.ts index 4ccafde..3d6213e 100644 --- a/packages/webview/src/store.ts +++ b/packages/webview/src/store.ts @@ -75,6 +75,8 @@ const defaultSettings: SettingsSnapshot = { }, serverSource: 'bundled' as const, daemonPort: 8723, + authMode: 'browser' as const, + httpClient: 'fetch' as const, }, ai: { autoInstallFiles: true, includeAgents: false }, formula: { formatterVersion: 'v2' }, diff --git a/packages/webview/src/tabs/Settings.tsx b/packages/webview/src/tabs/Settings.tsx index 910aa7b..fa966d0 100644 --- a/packages/webview/src/tabs/Settings.tsx +++ b/packages/webview/src/tabs/Settings.tsx @@ -208,6 +208,55 @@ export function Settings() { +
+ + + Connection mode + + How the MCP authenticates — Cookie & Direct login use no browser + + + +
+ +
+ + + HTTP client + + Transport for API calls (impit impersonates Chrome's TLS) + + + +
+ + {(settings.mcp.authMode === 'byo' || settings.mcp.authMode === 'direct-login') && ( +
+ + + {settings.mcp.authMode === 'byo' + ? 'Paste your Airtable session cookie into credentials.json in the config folder ({ "cookie": "…" }).' + : 'Set email / password / TOTP in login.json in the config folder. SSO/Google accounts must use Browser mode.'} + + +
+ )} + {browserIsDownloaded && !downloading && (
diff --git a/packages/webview/src/test/store.test.ts b/packages/webview/src/test/store.test.ts index a25b3d4..526ff40 100644 --- a/packages/webview/src/test/store.test.ts +++ b/packages/webview/src/test/store.test.ts @@ -13,7 +13,7 @@ beforeEach(() => { ideStatuses: [], versions: { extension: '—', mcpServerBundled: '—' }, aiFilesCount: 0, loading: true, activeTab: 'overview', pendingActions: new Set(), settings: { - mcp: { autoConfigureOnInstall: true, notifyOnUpdates: true, toolProfile: { profile: 'safe-write', enabledCount: 54, totalCount: 70, categories: { read: true, recordRead: true, tableWrite: true, tableDestructive: true, fieldWrite: true, fieldDestructive: true, recordDestructive: true, viewWrite: true, viewDestructive: true, viewSection: true, viewSectionDestructive: true, formWrite: true, extension: true, recordWrite: true, sync: true } }, serverSource: 'bundled', daemonPort: 0 }, + mcp: { autoConfigureOnInstall: true, notifyOnUpdates: true, toolProfile: { profile: 'safe-write', enabledCount: 54, totalCount: 70, categories: { read: true, recordRead: true, tableWrite: true, tableDestructive: true, fieldWrite: true, fieldDestructive: true, recordDestructive: true, viewWrite: true, viewDestructive: true, viewSection: true, viewSectionDestructive: true, formWrite: true, extension: true, recordWrite: true, sync: true } }, serverSource: 'bundled', daemonPort: 0, authMode: 'browser', httpClient: 'fetch' }, ai: { autoInstallFiles: true, includeAgents: false }, formula: { formatterVersion: 'v2' }, script: { beautifyStyle: 'default', minifyLevel: 'standard' }, From 3ea20d25bcf7a92fbf6e528a527667fc2a84a3bd Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 22:04:58 +0300 Subject: [PATCH 177/246] fix(sync): harden applyJob terminal write + document apply() throw contract Review follow-ups on the async plan/apply engine: - applyJob's schema-terminal .then() now also re-reads the sync-job phase and only finalizes 'done' when it is still 'schema' (belt-and-suspenders so it can never clobber a records-owned terminal write, in addition to the existing static m.records.status guard). - apply() JSDoc documents that FIELD_MAP_INVALID / missing-plan THROW (pinned by tests; applyJob catches them -> mode=status phase='failed'), while DRIFT returns an aborted result. - recordsStatus() marked @deprecated in favor of syncStatus(). Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/sync/index.js | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/packages/mcp-server/src/sync/index.js b/packages/mcp-server/src/sync/index.js index 03e5e28..1387a87 100644 --- a/packages/mcp-server/src/sync/index.js +++ b/packages/mcp-server/src/sync/index.js @@ -132,6 +132,15 @@ export function diffDetail({ sourceBaseId, destBaseId, diffId, detail, offset, l } /** + * Execute a saved plan (schema phase) then launch the background records job. + * + * THROWS (does not return an abort object) for pre-flight failures: a missing saved plan, and + * `FIELD_MAP_INVALID` when fieldMappings validation fails (err.code='FIELD_MAP_INVALID', + * err.mappingErrors=[...]). This is intentional and pinned by tests — direct callers must catch it. + * applyJob() (the background wrapper the sync_base tool uses) catches these and surfaces them via + * mode=status as phase='failed'. A DRIFT mismatch is NOT thrown: it returns a renderApplyResult + * with aborted:true, reason:'DRIFT'. + * * @param {object} opts * @param {(event:'records-start'|'records-done'|'records-failed', payload:object)=>void} [opts.onPhase] * Optional phase-timeline callback. Wired by applyJob() so the unified sync-job file tracks the @@ -319,9 +328,16 @@ export function applyJob({ client, sourceBaseId, destBaseId, planId, runStartedA .then(() => apply({ client, sourceBaseId, destBaseId, planId, runStartedAt: startedAt, skip, policy, policyOverrides, confirmDeletions, confirmTableDeletions, confirmRetypes, fieldMappings, naturalKeys, onPhase })) .then((rendered) => { const m = (rendered && rendered.machine) || {}; - // When the records phase launched, its own onPhase callbacks own the terminal write — - // don't clobber it. Otherwise (clean DRIFT abort / no-records path) finalize as 'done'. + // When the records phase launched, its own onPhase callbacks (records-done/records-failed) + // own the terminal write — don't clobber it. m.records.status is a STATIC 'running' snapshot + // set at apply() return time (records writes to the status FILE, never mutating this object), + // so this check normally suffices. As belt-and-suspenders (and to satisfy a review concern), + // ALSO re-read the file's phase: only finalize here if it's still 'schema' — i.e. the records + // phase never started (clean DRIFT abort / no-records path). Any non-'schema' phase means + // records or a terminal write already owns the file. if (m.records && m.records.status === 'running') return; + const current = readSyncJobStatus(sourceBaseId, destBaseId, planId); + if (current && current.phase !== 'schema') return; writeSyncJobStatus(sourceBaseId, destBaseId, planId, { phase: 'done', schemaResult: m, finishedAt: new Date().toISOString() }); }) .catch((e) => { @@ -382,6 +398,10 @@ export function syncStatus({ sourceBaseId, destBaseId, planId }) { } /** + * @deprecated Use `syncStatus()` instead — it reports the full plan→schema→records phase timeline + * and already falls back to the legacy `records-job-.json` file this reads. Retained only + * for existing direct callers/tests; new code should not use it. + * * Poll the background records job started by apply(). Reads the persisted status file and * derives live progress (records mapped so far) from idmap.records. * @returns {{ planId:string, status:string, recordsMapped:number, startedAt?:string, finishedAt?:string, result?:object, error?:string, message?:string }} From 2803eac3b773178481b2a49c7997328f71f9045e Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 22:07:28 +0300 Subject: [PATCH 178/246] feat(mcp): re-provision stdio MCP servers when authMode/httpClient changes + pin AIRTABLE_USER_MCP_HOME - DashboardProvider.setMcpChanged(): wire the MCP-definitions change emitter so the dashboard can fire it when transport settings change in no-daemon mode (where there is no daemon to restart); the emitter makes VS Code respawn the stdio server with fresh env. - mcp.authMode/mcp.httpClient handler: daemon mode still restarts the daemon; stdio mode now fires mcpChanged to re-provision the MCP servers. - registration --- packages/extension/package.json | 2 +- packages/extension/src/extension.ts | 3 ++ packages/extension/src/mcp/registration.ts | 5 ++++ .../src/webview/DashboardProvider.ts | 29 +++++++++++++++---- packages/mcp-server/src/index.js | 6 ++-- 5 files changed, 35 insertions(+), 10 deletions(-) diff --git a/packages/extension/package.json b/packages/extension/package.json index 3c0ddb6..f97dc9c 100644 --- a/packages/extension/package.json +++ b/packages/extension/package.json @@ -1,7 +1,7 @@ { "name": "airtable-formula", "private": true, - "version": "2.1.15", + "version": "2.1.16", "publisher": "Nskha", "displayName": "Airtable Formulas, Scripts, Automation, MCP & LSP", "description": "Airtable formula, script & automation editor with MCP server (71 tools), language server, and AI skills for VS Code.", diff --git a/packages/extension/src/extension.ts b/packages/extension/src/extension.ts index 4b8bd62..1c26a27 100644 --- a/packages/extension/src/extension.ts +++ b/packages/extension/src/extension.ts @@ -697,6 +697,9 @@ export async function activate(context: vscode.ExtensionContext): Promise const mcpChanged = new vscode.EventEmitter(); context.subscriptions.push(mcpChanged); registerMcpProvider(context, mcpChanged, authManager, daemonManager); + // Let the dashboard re-provision the MCP servers when transport settings change in stdio mode + // (no daemon to restart) — see the mcp.authMode/mcp.httpClient handler in DashboardProvider. + dashboardProvider.setMcpChanged(mcpChanged); // ── Auth init & auto-refresh ───────────────────────────────────────── await authManager.init(); diff --git a/packages/extension/src/mcp/registration.ts b/packages/extension/src/mcp/registration.ts index 8693cd1..c2b1bab 100644 --- a/packages/extension/src/mcp/registration.ts +++ b/packages/extension/src/mcp/registration.ts @@ -81,6 +81,11 @@ export function registerMcpProvider( AIRTABLE_HEADLESS_ONLY: '1', NODE_PATH: nodeModulesPath, AIRTABLE_NO_DAEMON: '1', + // Pin the config/state home to the SAME dir the daemon uses (buildDaemonEnv sets + // AIRTABLE_USER_MCP_HOME=configDir). Without this, the stdio server falls back to its + // own default and byo/direct-login credential files, sync state, and jobs could resolve + // to a different directory than the daemon path. + AIRTABLE_USER_MCP_HOME: path.join(os.homedir(), '.airtable-user-mcp'), }; // Propagate debug settings as env vars for the MCP debug-tracer diff --git a/packages/extension/src/webview/DashboardProvider.ts b/packages/extension/src/webview/DashboardProvider.ts index 5b258dd..98911e4 100644 --- a/packages/extension/src/webview/DashboardProvider.ts +++ b/packages/extension/src/webview/DashboardProvider.ts @@ -45,12 +45,20 @@ export class DashboardProvider implements vscode.WebviewViewProvider { private _daemonManager?: DaemonManager; private _daemonStarting = false; private _lockfileWatcher?: import('fs').FSWatcher; + private _mcpChanged?: vscode.EventEmitter; setDaemonManager(mgr: DaemonManager): void { this._daemonManager = mgr; void this._initLockfileWatch(); } + /** The MCP-definitions change emitter — firing it makes VS Code re-provision the MCP servers, + * which respawns the stdio server with a fresh env (how authMode/httpClient take effect in + * no-daemon mode, where there is no daemon to restart). */ + setMcpChanged(emitter: vscode.EventEmitter): void { + this._mcpChanged = emitter; + } + private async _initLockfileWatch(): Promise { const fsMod = await import('fs'); const configDir = path.join(os.homedir(), '.airtable-user-mcp'); @@ -429,13 +437,22 @@ export class DashboardProvider implements vscode.WebviewViewProvider { const dm = this._daemonManager; if (dm) { void dm.getDaemonStatus().then(status => { - if (!status?.running) return; - return dm.restartDaemon() - .then(() => this.pushState()) - .catch(err => { - vscode.window.showErrorMessage(`Daemon restart failed: ${err instanceof Error ? err.message : String(err)}`); - }); + if (status?.running) { + // Daemon mode: restart so the new AIRTABLE_AUTH_MODE/HTTP_CLIENT env re-injects. + return dm.restartDaemon() + .then(() => this.pushState()) + .catch(err => { + vscode.window.showErrorMessage(`Daemon restart failed: ${err instanceof Error ? err.message : String(err)}`); + }); + } + // No-daemon (stdio) mode: no daemon to restart — re-provision the MCP servers so VS Code + // respawns the stdio server with the fresh env. + this._mcpChanged?.fire(); + return this.pushState(); }); + } else { + // No daemon manager wired → still re-provision for a stdio server. + this._mcpChanged?.fire(); } } await this.pushState(); diff --git a/packages/mcp-server/src/index.js b/packages/mcp-server/src/index.js index c72e412..a5a4844 100644 --- a/packages/mcp-server/src/index.js +++ b/packages/mcp-server/src/index.js @@ -1606,15 +1606,15 @@ Note: "form title" is the view name itself — use rename_view to change it. "Fi // ── Sync Tools ── { name: 'sync_base', - description: 'Base-to-base schema + record sync. mode="plan" (read-only): compare source/dest schema by name and return an ordered plan of tables/fields to create or update, plus orphans + warnings; does NOT mutate. mode="apply": execute a saved plan against the destination — create tables, reconcile the primary, create scalar/link/computed fields (with source->dest reference remapping + formula validation), apply non-destructive field updates; then start the RECORD sync (two-pass cells + links, attachments, view-filter restore) in the BACKGROUND and return immediately with a jobId. apply requires planId and aborts if the destination drifted since the plan. mode="status": poll the background record job (pass the apply planId as planId) for progress/completion. mode="reconcile": rebuild/repair the record map — existence-prune dead entries from the idmap, with optional natural-key re-match per table. mode="diff": compute a schema digest comparing source and destination WITHOUT saving a plan; returns a diffId and human-readable summary; pass detail=
to drill into a specific section of a prior diff.', + description: 'Base-to-base schema + record sync. IMPORTANT: BOTH mode="plan" and mode="apply" run as BACKGROUND JOBS — they return {jobId, planId, status:"running"} IMMEDIATELY (jobId === planId), NOT a synchronous result; poll mode="status" with that planId to get progress and the final result. mode="plan" (read-only, does NOT mutate): computes an ordered plan (tables/fields to create/update + orphans + warnings) by comparing source/dest schema; when the job finishes, mode="status" returns the plan digest in planDigest. mode="apply" (requires planId from a prior plan): executes the saved plan against the destination — creates tables, reconciles the primary, creates scalar/link/computed fields (source->dest reference remapping + formula validation), applies non-destructive field updates, then runs the RECORD sync (two-pass cells + links, attachments, view-filter restore); aborts if the destination drifted since the plan. mode="status" (poll a plan OR apply job by its planId): returns { phase: "planning"|"schema"|"records"|"done"|"failed", status, recordsMapped, planDigest?, schemaResult?, recordsResult? }. Field-mapping errors, APPLY_LOCKED, and DRIFT surface HERE (phase="failed" or an aborted schemaResult), not as a synchronous error. mode="reconcile" (SYNCHRONOUS): rebuild/repair the record map — existence-prune dead idmap entries, optional natural-key re-match per table. mode="diff" (SYNCHRONOUS): schema digest comparing source and destination WITHOUT saving a plan; returns a diffId and summary; pass detail=
to drill into a section of a prior diff.', annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, inputSchema: { type: 'object', properties: { - mode: { type: 'string', enum: ['plan', 'apply', 'reconcile', 'status', 'diff'], description: '"plan" (read-only preview), "apply" (execute a saved plan; starts the record sync in the background and returns a jobId), "status" (poll the background record job by planId), "reconcile" (rebuild/repair the record map: existence-prune dead entries; optional natural-key re-match), or "diff" (schema digest comparing source and destination without saving a plan; optionally drill into a section with detail=
).' }, + mode: { type: 'string', enum: ['plan', 'apply', 'reconcile', 'status', 'diff'], description: '"plan" (BACKGROUND job: computes a plan and returns a jobId immediately — poll mode="status" for the digest), "apply" (BACKGROUND job: runs the schema phase then the record sync, returns a jobId immediately — poll mode="status"), "status" (poll a plan OR apply job by planId for its phase + result), "reconcile" (SYNCHRONOUS: rebuild/repair the record map — existence-prune dead entries, optional natural-key re-match), or "diff" (SYNCHRONOUS: schema digest comparing source and destination without saving a plan; optionally drill into a section with detail=
).' }, sourceAppId: { type: 'string', description: 'Source base/application ID to copy schema FROM' }, destAppId: { type: 'string', description: 'Destination base/application ID to copy schema TO' }, - planId: { type: 'string', description: 'Required for mode="apply" (the planId from a prior mode="plan" run) and mode="status" (the same planId, which is the background record jobId).' }, + planId: { type: 'string', description: 'Required for mode="apply" (the planId from a prior mode="plan" run) and mode="status" (the jobId returned by mode="plan" OR mode="apply" — jobId === planId).' }, naturalKeys: { type: 'object', description: 'Map of tableName → fieldName to use as a natural key for record identity matching (e.g. { "Projects": "Name" }). Applies to mode="reconcile" (repairs the record map via natural-key re-match after manual edits) AND mode="apply" (auto pre-pass before Pass 1: matches existing dest records by key so they are updated rather than duplicated; also protects real-but-unmapped dest records from mirror-policy deletion). Omit for existence-prune only (reconcile) or pure ID-based sync (apply). The key field MUST be stable across bases: autoNumber renumbers on creation (bad key); name/email/injected InjectID are good keys.', additionalProperties: { type: 'string' } }, detail: { type: 'string', description: 'Used only by mode="diff": section name to drill into (e.g. a table name or action key). Requires diffId.' }, diffId: { type: 'string', description: 'Used by mode="diff": ID of a prior diff to retrieve or drill into. If omitted on the initial call, one is generated automatically.' }, From 0e0d0e432169ed055510a8d96cfc460bf4ee4942 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Sat, 4 Jul 2026 22:29:11 +0300 Subject: [PATCH 179/246] feat(daemon): inject byo/direct-login credentials via a bearer-authed endpoint (in-memory, no env/disk) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shared daemon deliberately receives no credentials in its environment. Add a process-memory-only injected credential store (src/daemon/cred-store.js) plus a bearer-authenticated POST /daemon/auth-credentials endpoint so the extension can deliver byo/direct-login credentials to the daemon at runtime. - cred-store.js: setInjectedCredentials / getInjectedCredentials / clearInjectedCredentials. In-memory only — never persisted, never logged. set/get keep shallow copies so caller mutations can't corrupt the store. - loadByoCredentials + directLogin read the injected store FIRST, then env, then file. With an empty store, env/file behavior is unchanged. - POST /daemon/auth-credentials (requireBearer): validates authMode ∈ {byo,direct-login}, stores the body via setInjectedCredentials, and invalidates the current auth session (isLoggedIn=false, _credentials=null, csrfToken cleared, client marked stale) so the next API call re-inits with the new creds. The body/secret values are never logged. Tests: cred-store round-trip + copy semantics; loaders prefer injected over env/file and fall back when empty; endpoint requireBearer/validation/storage. Co-Authored-By: Claude Fable 5 --- packages/mcp-server/src/byo-credentials.js | 16 +++- packages/mcp-server/src/daemon/cred-store.js | 61 ++++++++++++++ packages/mcp-server/src/daemon/server.js | 54 +++++++++++++ packages/mcp-server/src/direct-login.js | 12 ++- .../test/test-byo-credentials.test.js | 60 ++++++++++++++ .../mcp-server/test/test-cred-store.test.js | 79 +++++++++++++++++++ .../test/test-daemon-server.test.js | 65 +++++++++++++++ .../mcp-server/test/test-direct-login.test.js | 66 ++++++++++++++++ 8 files changed, 406 insertions(+), 7 deletions(-) create mode 100644 packages/mcp-server/src/daemon/cred-store.js create mode 100644 packages/mcp-server/test/test-cred-store.test.js diff --git a/packages/mcp-server/src/byo-credentials.js b/packages/mcp-server/src/byo-credentials.js index 0ddcfff..8bf94ca 100644 --- a/packages/mcp-server/src/byo-credentials.js +++ b/packages/mcp-server/src/byo-credentials.js @@ -12,12 +12,14 @@ * page — the same token embedded in the /login HTML that the browser flow reads. * * Source order (first hit wins for the cookie): + * 0. injected in-memory store (daemon runtime channel — see cred-store.js) * 1. env AIRTABLE_COOKIE (+ optional AIRTABLE_CSRF) * 2. file ~/.airtable-user-mcp/credentials.json { cookie, csrf? } */ import fs from 'node:fs'; import path from 'node:path'; import { getHomeDir } from './paths.js'; +import { getInjectedCredentials } from './daemon/cred-store.js'; /** Path to the BYO credentials file. */ export function getCredentialsPath() { @@ -56,10 +58,16 @@ export async function loadByoCredentials({ fetchImpl } = {}) { let cookieHeader = null; let csrfToken = null; - // 1. Environment. - const envCookie = process.env.AIRTABLE_COOKIE; - if (envCookie && envCookie.trim()) { - cookieHeader = envCookie.trim(); + // 0. Injected in-memory store — the daemon's runtime credential channel + // (POST /daemon/auth-credentials; never env/disk). Highest priority so a + // freshly-pushed cookie wins over stale env/file. In-memory only. + const injected = getInjectedCredentials(); + if (injected && typeof injected.cookie === 'string' && injected.cookie.trim()) { + cookieHeader = injected.cookie.trim(); + if (typeof injected.csrf === 'string' && injected.csrf.trim()) csrfToken = injected.csrf.trim(); + } else if (process.env.AIRTABLE_COOKIE && process.env.AIRTABLE_COOKIE.trim()) { + // 1. Environment. + cookieHeader = process.env.AIRTABLE_COOKIE.trim(); const envCsrf = process.env.AIRTABLE_CSRF; if (envCsrf && envCsrf.trim()) csrfToken = envCsrf.trim(); } else { diff --git a/packages/mcp-server/src/daemon/cred-store.js b/packages/mcp-server/src/daemon/cred-store.js new file mode 100644 index 0000000..5166a69 --- /dev/null +++ b/packages/mcp-server/src/daemon/cred-store.js @@ -0,0 +1,61 @@ +/** + * In-memory injected credential store (SECURITY-CRITICAL). + * + * Holds live Airtable credentials — a session cookie (+CSRF) for `byo`, or + * email/password/TOTP-secret for `direct-login` — delivered to the shared + * daemon AT RUNTIME over the bearer-authenticated POST /daemon/auth-credentials + * endpoint (src/daemon/server.js). + * + * These credentials live in THIS PROCESS'S MEMORY ONLY. They are: + * - NEVER written to disk (no file, no lockfile, no settings). + * - NEVER logged (not by the debug-tracer, not to stdout/stderr). + * - Gone when the process exits, or when clearInjectedCredentials() is called. + * + * The daemon deliberately does NOT receive credentials in its environment (env + * is inheritable/inspectable), so this in-memory channel is how it gets them. + * Loaders (byo-credentials, direct-login) read this store FIRST, then fall back + * to env, then file — so with an empty store, behavior is unchanged. + * + * Keep this module tiny and dependency-free. Do NOT add persistence or logging. + * + * @typedef {Object} InjectedCredentials + * @property {'byo'|'direct-login'} [authMode] + * @property {string} [cookie] + * @property {string} [csrf] + * @property {string} [email] + * @property {string} [password] + * @property {string} [totpSecret] + */ + +/** @type {InjectedCredentials | null} */ +let injected = null; + +/** + * Store the injected credentials, replacing any previous set. A shallow copy is + * kept so a later mutation of the caller's object can't reach into the store + * (all fields are primitive strings, so shallow is sufficient). Passing null / + * undefined / a non-object clears the store. + * + * @param {InjectedCredentials | null | undefined} creds + */ +export function setInjectedCredentials(creds) { + if (!creds || typeof creds !== 'object') { + injected = null; + return; + } + injected = { ...creds }; +} + +/** + * @returns {InjectedCredentials | null} a fresh shallow copy of the stored + * credentials (mutating the returned object never corrupts the store), or + * null when nothing is injected. + */ +export function getInjectedCredentials() { + return injected ? { ...injected } : null; +} + +/** Drop the injected credentials from memory. */ +export function clearInjectedCredentials() { + injected = null; +} diff --git a/packages/mcp-server/src/daemon/server.js b/packages/mcp-server/src/daemon/server.js index aa30be1..2360513 100644 --- a/packages/mcp-server/src/daemon/server.js +++ b/packages/mcp-server/src/daemon/server.js @@ -18,6 +18,7 @@ import { AirtableAuth } from '../auth.js'; import { AirtableClient } from '../client.js'; import { ToolConfigManager } from '../tool-config.js'; import { ensureToken, rotateToken, getTokenPath } from './token.js'; +import { setInjectedCredentials } from './cred-store.js'; import { getTunnelProvider, writeTunnelSettings } from './tunnel-providers/index.js'; import { runCloudflaredLogin, @@ -397,6 +398,59 @@ export async function startDaemonServer(options = {}) { res.json({ ok: true }); }); + // POST /daemon/auth-credentials (C2) — runtime credential channel. + // The extension pushes byo/direct-login creds here instead of via env/disk + // (the daemon deliberately gets no creds in its environment). The body is + // stored IN MEMORY ONLY (src/daemon/cred-store.js) — NEVER persisted, NEVER + // logged. Do not console.log / trace the body or any field value below. + app.post('/daemon/auth-credentials', requireBearer, async (req, res) => { + try { + const body = req.body ?? {}; + const { authMode } = body; + if (authMode !== 'byo' && authMode !== 'direct-login') { + res.status(400).json({ ok: false, error: "authMode must be 'byo' or 'direct-login'" }); + return; + } + // Basic shape validation only — string-typed secrets, values never + // inspected/logged (avoid leaking a secret into an error message). + for (const key of ['cookie', 'csrf', 'email', 'password', 'totpSecret']) { + if (body[key] !== undefined && typeof body[key] !== 'string') { + res.status(400).json({ ok: false, error: `${key} must be a string` }); + return; + } + } + + setInjectedCredentials({ + authMode, + cookie: body.cookie, + csrf: body.csrf, + email: body.email, + password: body.password, + totpSecret: body.totpSecret, + }); + + // Invalidate the CURRENT auth session so the next API call re-inits and + // picks up the freshly-injected creds. Mirror getClient()'s lazy pattern + // (create auth if absent), then reset the session state. AirtableAuth has + // no single reset method that also clears _credentials, so we reset the + // fields directly: for byo/direct-login, ensureLoggedIn() re-runs + // _doInit whenever _credentials is null (which re-reads cred-store first); + // isLoggedIn=false covers the browser path. We do NOT launch the browser + // here — clearing clientInitPromise only marks the client stale so the + // NEXT getClient() (i.e. the next MCP tool call) rebuilds it. + if (!auth) { auth = new AirtableAuth(); } + auth.isLoggedIn = false; + auth._credentials = null; + auth.csrfToken = null; + auth.resetSessionHealth?.(); // clear the dead-session circuit-breaker + clientInitPromise = null; + + res.json({ ok: true }); + } catch (error) { + res.status(500).json({ ok: false, error: error instanceof Error ? error.message : String(error) }); + } + }); + app.post('/daemon/rotate-token', requireBearer, async (_req, res, next) => { try { currentToken = rotateToken({ tokenPath }); diff --git a/packages/mcp-server/src/direct-login.js b/packages/mcp-server/src/direct-login.js index 2c0acef..b175e89 100644 --- a/packages/mcp-server/src/direct-login.js +++ b/packages/mcp-server/src/direct-login.js @@ -21,6 +21,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { getHomeDir } from './paths.js'; import { scrapeCsrf } from './byo-credentials.js'; +import { getInjectedCredentials } from './daemon/cred-store.js'; const BASE = 'https://airtable.com'; @@ -218,9 +219,14 @@ async function request(impit, method, urlPath, { jar, form } = {}) { // ── Credential resolution ─────────────────────────────────────────── function resolveCredentials({ email, password, totpSecret }) { - let e = email ?? process.env.AIRTABLE_EMAIL; - let p = password ?? process.env.AIRTABLE_PASSWORD; - let t = totpSecret ?? process.env.AIRTABLE_TOTP_SECRET; + // Priority: injected in-memory store (daemon runtime channel — cred-store.js) + // → explicit params → env (AIRTABLE_EMAIL/PASSWORD/TOTP_SECRET) → login.json. + // The injected store is in-memory only (never env/disk) and empty for stdio, + // so the env/file path is unchanged when nothing is injected. + const injected = getInjectedCredentials() || {}; + let e = injected.email ?? email ?? process.env.AIRTABLE_EMAIL; + let p = injected.password ?? password ?? process.env.AIRTABLE_PASSWORD; + let t = injected.totpSecret ?? totpSecret ?? process.env.AIRTABLE_TOTP_SECRET; if (!e || !p) { // File fallback: ~/.airtable-user-mcp/login.json { email, password, totpSecret } diff --git a/packages/mcp-server/test/test-byo-credentials.test.js b/packages/mcp-server/test/test-byo-credentials.test.js index 4850636..8bda538 100644 --- a/packages/mcp-server/test/test-byo-credentials.test.js +++ b/packages/mcp-server/test/test-byo-credentials.test.js @@ -4,6 +4,7 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { loadByoCredentials, scrapeCsrf, getCredentialsPath } from '../src/byo-credentials.js'; +import { setInjectedCredentials, clearInjectedCredentials } from '../src/daemon/cred-store.js'; /** * BYO (bring-your-own) cookie credentials for AIRTABLE_AUTH_MODE=byo. @@ -124,3 +125,62 @@ describe('loadByoCredentials', () => { await assert.rejects(() => loadByoCredentials(), /not valid JSON/); }); }); + +describe('loadByoCredentials — injected in-memory store (daemon runtime channel)', () => { + let env, tmpHome; + beforeEach(() => { + env = snapshotEnv(); + for (const k of ENV_KEYS) delete process.env[k]; + tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), 'byo-inj-')); + process.env.AIRTABLE_USER_MCP_HOME = tmpHome; + clearInjectedCredentials(); + }); + afterEach(() => { + restoreEnv(env); + fs.rmSync(tmpHome, { recursive: true, force: true }); + clearInjectedCredentials(); + }); + + it('injected cookie + csrf wins over both env AND file (no fetch/scrape)', async () => { + // Env AND file both present — the injected store must still win. + process.env.AIRTABLE_COOKIE = 'brw=env'; + process.env.AIRTABLE_CSRF = 'ENV-CSRF'; + fs.writeFileSync(getCredentialsPath(), JSON.stringify({ cookie: 'brw=file', csrf: 'FILE-CSRF' })); + setInjectedCredentials({ authMode: 'byo', cookie: 'brw=injected', csrf: 'INJECTED-CSRF' }); + + let fetched = false; + const creds = await loadByoCredentials({ fetchImpl: async () => { fetched = true; return { text: async () => '' }; } }); + assert.equal(creds.cookieHeader, 'brw=injected'); + assert.equal(creds.csrfToken, 'INJECTED-CSRF'); + assert.equal(fetched, false); // csrf supplied → no scrape + }); + + it('injected cookie without csrf → scrapes csrf (carrying the injected cookie)', async () => { + setInjectedCredentials({ authMode: 'byo', cookie: 'brw=injected2' }); + let seenReq; + const creds = await loadByoCredentials({ + fetchImpl: async (url, opts) => { seenReq = { url, opts }; return { text: async () => HTML_WITH_CSRF }; }, + }); + assert.equal(creds.cookieHeader, 'brw=injected2'); + assert.equal(creds.csrfToken, 'SCRAPED-CSRF-42'); + assert.equal(seenReq.opts.headers.Cookie, 'brw=injected2'); + }); + + it('empty injected store → falls back to env (existing behavior unchanged)', async () => { + // Store cleared in beforeEach; a blank cookie must not be treated as a hit. + setInjectedCredentials({ authMode: 'byo', cookie: ' ' }); + process.env.AIRTABLE_COOKIE = 'brw=env-fallback'; + process.env.AIRTABLE_CSRF = 'ENV-FALLBACK-CSRF'; + const creds = await loadByoCredentials(); + assert.equal(creds.cookieHeader, 'brw=env-fallback'); + assert.equal(creds.csrfToken, 'ENV-FALLBACK-CSRF'); + }); + + it('no injected cookie field → falls back to file', async () => { + setInjectedCredentials({ authMode: 'byo', csrf: 'ORPHAN-CSRF' }); // csrf but no cookie + fs.writeFileSync(getCredentialsPath(), JSON.stringify({ cookie: 'brw=file-fallback', csrf: 'FILE-CSRF' })); + const creds = await loadByoCredentials({ fetchImpl: async () => { throw new Error('should not fetch'); } }); + assert.equal(creds.cookieHeader, 'brw=file-fallback'); + assert.equal(creds.csrfToken, 'FILE-CSRF'); + }); +}); diff --git a/packages/mcp-server/test/test-cred-store.test.js b/packages/mcp-server/test/test-cred-store.test.js new file mode 100644 index 0000000..091e617 --- /dev/null +++ b/packages/mcp-server/test/test-cred-store.test.js @@ -0,0 +1,79 @@ +import { describe, it, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { + setInjectedCredentials, + getInjectedCredentials, + clearInjectedCredentials, +} from '../src/daemon/cred-store.js'; + +/** + * In-memory injected credential store (SECURITY-CRITICAL): the daemon's runtime + * credential channel. These tests assert the set/get/clear contract and that + * the store returns COPIES (mutating a get() result never corrupts the store). + */ + +describe('cred-store', () => { + afterEach(() => clearInjectedCredentials()); + + it('starts empty (getInjectedCredentials returns null)', () => { + clearInjectedCredentials(); + assert.equal(getInjectedCredentials(), null); + }); + + it('set → get round-trips every field', () => { + const creds = { + authMode: 'byo', + cookie: 'brw=1; __Host-airtable-session=abc', + csrf: 'CSRF-1', + email: 'user@example.com', + password: 'hunter2', + totpSecret: 'JBSWY3DPEHPK3PXP', + }; + setInjectedCredentials(creds); + assert.deepEqual(getInjectedCredentials(), creds); + }); + + it('clear nulls the store', () => { + setInjectedCredentials({ authMode: 'byo', cookie: 'brw=1' }); + assert.ok(getInjectedCredentials()); + clearInjectedCredentials(); + assert.equal(getInjectedCredentials(), null); + }); + + it('a later set replaces the previous credentials wholesale', () => { + setInjectedCredentials({ authMode: 'byo', cookie: 'brw=first' }); + setInjectedCredentials({ authMode: 'direct-login', email: 'e@x.com', password: 'p' }); + const out = getInjectedCredentials(); + assert.equal(out.authMode, 'direct-login'); + assert.equal(out.email, 'e@x.com'); + assert.equal(out.cookie, undefined); // not carried over from the first set + }); + + it('set stores a copy — mutating the caller object does not reach the store', () => { + const input = { authMode: 'byo', cookie: 'brw=orig' }; + setInjectedCredentials(input); + input.cookie = 'brw=TAMPERED'; + assert.equal(getInjectedCredentials().cookie, 'brw=orig'); + }); + + it('get returns a copy — mutating the result does not corrupt the store', () => { + setInjectedCredentials({ authMode: 'byo', cookie: 'brw=orig' }); + const first = getInjectedCredentials(); + first.cookie = 'brw=TAMPERED'; + assert.equal(getInjectedCredentials().cookie, 'brw=orig'); + }); + + it('null / undefined / non-object clears the store', () => { + setInjectedCredentials({ authMode: 'byo', cookie: 'brw=1' }); + setInjectedCredentials(null); + assert.equal(getInjectedCredentials(), null); + + setInjectedCredentials({ authMode: 'byo', cookie: 'brw=1' }); + setInjectedCredentials(undefined); + assert.equal(getInjectedCredentials(), null); + + setInjectedCredentials({ authMode: 'byo', cookie: 'brw=1' }); + setInjectedCredentials('not-an-object'); + assert.equal(getInjectedCredentials(), null); + }); +}); diff --git a/packages/mcp-server/test/test-daemon-server.test.js b/packages/mcp-server/test/test-daemon-server.test.js index 589528d..926cc00 100644 --- a/packages/mcp-server/test/test-daemon-server.test.js +++ b/packages/mcp-server/test/test-daemon-server.test.js @@ -4,6 +4,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { mkdirSync, rmSync } from 'node:fs'; import { startDaemonServer } from '../src/daemon/server.js'; +import { getInjectedCredentials, clearInjectedCredentials } from '../src/daemon/cred-store.js'; let tmpDir; let server; @@ -108,6 +109,70 @@ describe('POST /daemon/release-browser', () => { }); }); +describe('POST /daemon/auth-credentials', () => { + const url = () => `http://127.0.0.1:${server.port}/daemon/auth-credentials`; + const authed = (body) => ({ + method: 'POST', + headers: { 'Authorization': `Bearer ${server.bearerToken}`, 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + + after(() => clearInjectedCredentials()); + + it('returns 401 when Authorization header is missing (and does not touch the store)', async () => { + clearInjectedCredentials(); + const response = await fetch(url(), { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ authMode: 'byo', cookie: 'brw=should-not-store' }), + }); + assert.strictEqual(response.status, 401); + assert.equal(getInjectedCredentials(), null); // unauthorized request never stored creds + }); + + it('stores byo creds in memory and returns ok:true with a valid token', async () => { + clearInjectedCredentials(); + const response = await fetch(url(), authed({ authMode: 'byo', cookie: 'brw=1; sess=abc', csrf: 'CSRF-99' })); + assert.strictEqual(response.status, 200); + const body = await response.json(); + assert.strictEqual(body.ok, true); + // setInjectedCredentials was called with the pushed creds. + const stored = getInjectedCredentials(); + assert.equal(stored.authMode, 'byo'); + assert.equal(stored.cookie, 'brw=1; sess=abc'); + assert.equal(stored.csrf, 'CSRF-99'); + }); + + it('stores direct-login creds (email/password/totpSecret) in memory', async () => { + clearInjectedCredentials(); + const response = await fetch(url(), authed({ + authMode: 'direct-login', email: 'u@x.com', password: 'p', totpSecret: 'JBSWY3DPEHPK3PXP', + })); + assert.strictEqual(response.status, 200); + const stored = getInjectedCredentials(); + assert.equal(stored.authMode, 'direct-login'); + assert.equal(stored.email, 'u@x.com'); + assert.equal(stored.password, 'p'); + assert.equal(stored.totpSecret, 'JBSWY3DPEHPK3PXP'); + }); + + it('rejects an invalid authMode with 400 (store unchanged)', async () => { + clearInjectedCredentials(); + const response = await fetch(url(), authed({ authMode: 'browser', cookie: 'brw=x' })); + assert.strictEqual(response.status, 400); + const body = await response.json(); + assert.strictEqual(body.ok, false); + assert.equal(getInjectedCredentials(), null); // invalid request never stored creds + }); + + it('rejects a non-string secret field with 400', async () => { + clearInjectedCredentials(); + const response = await fetch(url(), authed({ authMode: 'byo', cookie: 12345 })); + assert.strictEqual(response.status, 400); + assert.equal(getInjectedCredentials(), null); + }); +}); + describe('GET /daemon/events', () => { it('returns 401 when bearer token is wrong', async () => { const controller = new AbortController(); diff --git a/packages/mcp-server/test/test-direct-login.test.js b/packages/mcp-server/test/test-direct-login.test.js index c3c0858..a0a357b 100644 --- a/packages/mcp-server/test/test-direct-login.test.js +++ b/packages/mcp-server/test/test-direct-login.test.js @@ -1,6 +1,7 @@ import { describe, it, beforeEach, afterEach } from 'node:test'; import assert from 'node:assert/strict'; import { directLogin } from '../src/direct-login.js'; +import { setInjectedCredentials, clearInjectedCredentials } from '../src/daemon/cred-store.js'; /** * Phase B — browser-free direct login (impit + TOTP), replicating the @@ -228,3 +229,68 @@ describe('directLogin — credential resolution', () => { assert.ok(out.cookieHeader.includes('__Host-airtable-session=S')); }); }); + +describe('directLogin — injected in-memory store (daemon runtime channel)', () => { + const ENV = ['AIRTABLE_EMAIL', 'AIRTABLE_PASSWORD', 'AIRTABLE_TOTP_SECRET', 'AIRTABLE_USER_MCP_HOME']; + let saved; + beforeEach(() => { + saved = {}; + for (const k of ENV) { saved[k] = process.env[k]; delete process.env[k]; } + process.env.AIRTABLE_USER_MCP_HOME = '/nonexistent-airtable-home-xyz'; + clearInjectedCredentials(); + }); + afterEach(() => { + for (const k of ENV) { if (saved[k] === undefined) delete process.env[k]; else process.env[k] = saved[k]; } + clearInjectedCredentials(); + }); + + it('resolves email/password/totpSecret from the injected store first (over env)', async () => { + // Env is present but the injected store must win. + process.env.AIRTABLE_EMAIL = 'env@example.com'; + process.env.AIRTABLE_PASSWORD = 'env-loses'; + process.env.AIRTABLE_TOTP_SECRET = 'ENV-SECRET-LOSES'; + setInjectedCredentials({ + authMode: 'direct-login', + email: 'injected@example.com', + password: 'injected-pw', + totpSecret: 'INJECTED-TOTP-SECRET', + }); + + const impit = makeFakeImpit([ + { status: 200, body: '{"csrfToken":"C1"}', setCookies: ['brw=B; Path=/'] }, + { status: 200, body: JSON.stringify({ loginType: 'password' }) }, + { status: 302, location: '/2fa/tfaINJ', setCookies: ['__Host-airtable-session=S1; Path=/'] }, + { status: 302, location: '/', setCookies: ['__Host-airtable-session=S2; Path=/'] }, + { status: 200, body: '{"csrfToken":"C2"}' }, + ]); + + let totpArg = null; + const out = await directLogin({ + impitFactory: () => impit, + otpFactory: (secret) => { totpArg = secret; return '111222'; }, + }); + + assert.equal(bodyParams(impit.calls[1]).get('email'), 'injected@example.com'); + assert.equal(bodyParams(impit.calls[2]).get('email'), 'injected@example.com'); + assert.equal(bodyParams(impit.calls[2]).get('password'), 'injected-pw'); + // Injected TOTP secret was handed to the generator, and its code posted. + assert.equal(totpArg, 'INJECTED-TOTP-SECRET'); + assert.equal(bodyParams(impit.calls[3]).get('code'), '111222'); + assert.ok(out.cookieHeader.includes('__Host-airtable-session=S2')); + }); + + it('empty injected store → falls back to env (existing behavior unchanged)', async () => { + process.env.AIRTABLE_EMAIL = 'env@example.com'; + process.env.AIRTABLE_PASSWORD = 'envpw'; + const impit = makeFakeImpit([ + { status: 200, body: '{"csrfToken":"C1"}', setCookies: ['brw=B; Path=/'] }, + { status: 200, body: JSON.stringify({ loginType: 'password' }) }, + { status: 302, location: '/', setCookies: ['__Host-airtable-session=S; Path=/'] }, + { status: 200, body: '{"csrfToken":"C2"}' }, + ]); + const out = await directLogin({ impitFactory: () => impit }); + assert.equal(bodyParams(impit.calls[1]).get('email'), 'env@example.com'); + assert.equal(bodyParams(impit.calls[2]).get('password'), 'envpw'); + assert.ok(out.cookieHeader.includes('__Host-airtable-session=S')); + }); +}); From 8abaecfd79391947623280efe1f9ee666a89a852 Mon Sep 17 00:00:00 2001 From: "A.R." Date: Tue, 7 Jul 2026 21:28:58 +0300 Subject: [PATCH 180/246] feat(auth): byo + direct-login credential handling in extension (keychain storage + daemon push) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - AuthManager: add getCookie/saveCookie/clearCookie (SECRET_COOKIE in keychain, never disk); hasCookie state flag; getCredentialsEnv branches on authMode (byo → AIRTABLE_COOKIE only, direct-login → AIRTABLE_EMAIL/PASSWORD/TOTP_SECRET, browser unchanged); getCredentials gains ensureDaemon opt-out so lockfile-watch credential-push can't spawn-loop; login/manualLogin/checkSession/autoRefresh skip browser --- .gitignore | 4 +- CLAUDE.md | 2 +- packages/extension/package.json | 2 +- packages/extension/src/debug/exporter.ts | 3 +- packages/extension/src/extension.ts | 3 + packages/extension/src/mcp/auth-manager.ts | 167 ++++++++- packages/extension/src/mcp/daemon-manager.ts | 168 ++++++++- packages/extension/src/mcp/registration.ts | 17 +- .../extension/src/test/daemon-manager.test.ts | 116 +++++- .../src/webview/DashboardProvider.ts | 266 +++++++++++++- packages/mcp-server/src/auth.js | 93 ++++- packages/mcp-server/src/daemon/server.js | 25 +- packages/mcp-server/src/direct-login.js | 51 ++- packages/mcp-server/src/index.js | 15 +- packages/mcp-server/src/sync/apply.js | 21 +- packages/mcp-server/src/sync/index.js | 94 ++++- packages/mcp-server/src/sync/records.js | 130 ++++++- .../test-apply-link-dependency-defer.test.js | 96 +++++ .../test/sync/test-records-all-failed.test.js | 293 +++++++++++++++ .../sync/test-records-session-abort.test.js | 344 ++++++++++++++++++ .../test/sync/test-sync-job.test.js | 35 ++ .../test/test-auth-403-reauth.test.js | 225 ++++++++++++ .../test/test-daemon-port-fallback.test.js | 54 +++ packages/shared/src/messages.ts | 2 + packages/shared/src/types.ts | 2 + packages/webview/src/store.ts | 20 +- packages/webview/src/tabs/Settings.tsx | 156 +++++--- 27 files changed, 2248 insertions(+), 156 deletions(-) create mode 100644 packages/mcp-server/test/sync/test-apply-link-dependency-defer.test.js create mode 100644 packages/mcp-server/test/sync/test-records-all-failed.test.js create mode 100644 packages/mcp-server/test/sync/test-records-session-abort.test.js create mode 100644 packages/mcp-server/test/test-auth-403-reauth.test.js create mode 100644 packages/mcp-server/test/test-daemon-port-fallback.test.js diff --git a/.gitignore b/.gitignore index 8b6fd9f..8fb01fd 100644 --- a/.gitignore +++ b/.gitignore @@ -49,4 +49,6 @@ **/PLAN.md packages/mcp-server/.chrome-profile/ packages/mcp-server/dev-tools/ -packages/extension/README.md \ No newline at end of file +packages/extension/README.md +packages/webview/.ds-css/ +packages/webview/.ds-src/ \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index cad9d60..f7701f4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,7 +65,7 @@ Key files: - `dev-tools/` — reverse-engineering helpers (capture.js, analyze.js, debug-*.js), **gitignored** — kept locally for future reference when Airtable changes their internal API - `src/daemon/` — daemon subsystem (see daemon subsection below) - `src/safe-write.js` — atomic JSON write helper (used by lockfile, token, and settings) -- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. Drift-guarded + resumable (journal; a journal with ≥1 done action bypasses the drift guard with `RESUME_DRIFT_BYPASS`, so resume is reachable past sync's own mutations). Concurrent applies to the same base pair are refused via a per-pair `apply.lock` (`src/sync/apply-lock.js`: pid-liveness stale detection, held until the background records job settles; `mode=reconcile` takes the same lock; error code `APPLY_LOCKED`). Exposed via the `sync_base` tool (`mode: diff | plan | apply | status | reconcile`) in the `sync` category. Out of scope: non-scalar retypes (computed/link/attachment on either side → `RETYPE_DEFERRED`), deletions (M4); scalar retypes are now in scope (see below). `mode=diff` (read-only): comprehensive schema comparison via pure `src/sync/compare.js` — every difference classified as **drift** (sync enforces: table/field/view existence, field type/options/choices, view filters/sorts/groups content, column visibility, colors/cover/frozen/rowHeight, primary, form), **best-effort** (sync applies but doesn't guarantee: field/view/column order, sort/group clause order), or **not-synced** (view sections — now captured by snapshot). Verdicts: `identical` (zero diffs) or `converged` (zero drift; best-effort/not-synced may remain). Persists full diff to `~/.airtable-user-mcp/sync/__/diff-.json`; returns a token-budgeted DIGEST (verdicts + class counts + per-table rollup + capped driftSample ≤25). Drill in with `detail="
"` (params: `detail`, `diffId`, `offset`, `limit`). `mode=plan` now emits a curatable **changeset**: each action carries a stable name-based `changeId` (`|
|`), a `class`, and an `apply:true` flag; digest includes a `sample` + `editHint`. New `direction` param (`to-dest` default | `to-source`) targets either base. View sync: `snapshot` also captures collaborative views + live config + **view sections** (not-synced class); `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported); accepts a `skip:[changeId]` list (and honors `apply:false` on changeset entries) — applies everything except removed changes (journal-safe; skipped dependencies degrade to UNRESOLVABLE_REF). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe scoped to the dest cell `row|field|filename|size`); then restores record-referencing view filters stripped during view sync (per-view failures warn + continue, never block prune). Pass 2 + attachments honor `conflicts=dest-wins` for pre-existing mapped rows; rows created by this plan — tracked resume-durably as `createdDestIds` in the records journal — always receive their links/attachments. Throttle handling = per-request 429 backoff in the auth queue (the per-call limiter does NOT gate the internal per-row POSTs inside createRecords/updateRecords/addLinkItems); resumable (records journal, per-chunk persist of ~50 creates); continue-on-failure (per-record errors are warnings). Truncation safety: a table whose snapshot hits the 1000-row cap on either base is added to `truncatedTables` by `collectTruncatedTableNames` and skipped by `pruneRecords` under mirror (`RECORDS_TRUNCATED_PRUNE_SKIPPED`) so a record beyond the snapshot window is never deleted as a false orphan; each truncated table also emits `RECORDS_TRUNCATED` on the result; >1000-row read pagination remains a capture-gated follow-up. `mode=plan` and `mode=apply` themselves now run as BACKGROUND jobs (`src/sync/job-status.js` `sync-job-.json`, phases `planning→schema→records→done|failed`, pid-liveness) — they return `{jobId,status:running}` immediately (a synchronous plan/apply on a view-heavy base is minutes-long and trips the MCP client's response window → "Connection closed" despite completing server-side); `planJob`/`applyJob`/`syncStatus` in `src/sync/index.js`, `plan()`/`apply()` stay callable as sync engine fns. `mode=diff`/`reconcile` stay synchronous. `mode=status` polls a background job by planId — returns phase + running/done/failed + live records-mapped count + planDigest/schemaResult/recordsResult (FIELD_MAP_INVALID/APPLY_LOCKED now surface here as phase=failed). `mode=reconcile` **prunes** dead record-map entries (existence-prune, truncation-guarded — skipped with a warning when any dest table hits the 1000-row snapshot cap, so live mappings beyond the cap are never dropped; per-table natural-key re-match now matches by key value — see natural-key matching below; reconcile does NOT de-duplicate records). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json, diff-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile), `compare.js` (pure classified compare, no I/O); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. Reconciliation policy (`src/sync/policy.js`): `mode=apply` accepts a `policy` param (`mirror` | `overlay` | `preserve`, default `overlay`) that controls two axes — **extras** (what to do with dest-only records) and **conflicts** (whose value wins when both bases have the record). Presets: `mirror` = {remove, source-wins} (make dest identical to source); `overlay` = {keep, source-wins} (today's default: keep dest-only rows, source updates win); `preserve` = {keep, dest-wins} (keep dest-only rows and never overwrite dest edits). `policyOverrides` (`{ [tableName]: preset }`) overrides the global preset per table. `confirmDeletions` (boolean, default false) is a safety gate for `mirror`: without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count but deletes nothing; set `confirmDeletions:true` to actually delete. Separately, `createTable` unconditionally clears the ~3 blank scaffolding rows it seeds (best-effort; `SCAFFOLDING_ROWS_KEPT` warning if the deletion attempt fails); `pruneRecords` under `mirror`+`confirmDeletions` additionally removes any leftover dest-only blank rows as part of the ordinary orphan prune. Attachment deletion under mirror is a deferred follow-up. Schema-orphan deletion (`pruneSchema`) runs synchronously in `mode=apply` AFTER `applyPlan`, consuming `plan.orphans` (`{kind:'table'|'field'|'view', destId, name, tableName}`). Order: (1) views — deleted first (no dependents); (2) fields — batch `deleteFields` WITHOUT force (so a matched field is never cascade-deleted), retry-until-stable loop resolves orphan→orphan deps; a field still blocked after no progress is kept + `SCHEMA_DELETE_BLOCKED`; (3) tables — deleted last (after their content). Gating: dest-only fields + views require `confirmDeletions:true` (same flag as `pruneRecords`); dest-only tables require the new `confirmTableDeletions:true` — `confirmDeletions` alone never drops a whole table. Without each gate the operation is a dry-count: `DELETION_GATED` for fields/views, `TABLE_DELETION_GATED` for tables. Applied counts: `schemaDeleted` (fields+views) + `tablesDeleted`. Per-table `policyOverrides` apply; `overlay`/`preserve` → no schema deletion. Continue-on-failure (`SCHEMA_DELETE_FAILED`). Module: `src/sync/prune-schema.js`. View-section orphans (dest-only sidebar sections in a matched table) are now deleted by `pruneSchema` as **step 0** (before views, so Airtable auto-promotes their contained views to top-level before view orphan deletion); gated by `confirmDeletions` (same flag as fields/views); `deleteViewSection` is the client method used. Attachment-strip under mirror remains a deferred follow-up. Scalar field retypes (`mode=apply`): a matched dest field whose scalar type diverges from source is retyped via `updateFieldConfig`, gated by the new `confirmRetypes` param (boolean, default false); without it the field is kept and a `RETYPE_GATED` warning is emitted (count surfaced in the result). Non-scalar retypes (either side is computed/link/attachment) stay `RETYPE_DEFERRED`. Select retypes converge their choice map on re-sync exactly like created selects (`mergeChoices`). Continue-on-failure: per-field retype errors emit `RETYPE_FAILED` and do not abort apply. Primary-field retypes remain the separate `reconcilePrimary` path. This closes the last schema-drift category sync can fix; records data migration remains `pruneRecords`/Pass 1. Custom field mappings: `fieldMappings` (`{ [tableName]: { sourceFieldName: destFieldName } }`) injects a source field's value into a different writable scalar dest field during record sync — useful when the source is computed (e.g., `autoNumber`/formula) and you want its value preserved in a writable dest field (classic use: inject source autoNumber `Code` into dest text field `InjectID` so a dest formula can reconstruct original identity). Fail-fast pre-flight validation in `apply` (and dry-run in `plan`/`diff` via `fieldMappingErrors` in the machine output) with codes: `FIELD_MAP_TABLE_MISSING`, `FIELD_MAP_SOURCE_MISSING`, `FIELD_MAP_TARGET_MISSING`, `FIELD_MAP_TARGET_COMPUTED` (dest must be writable scalar), `FIELD_MAP_TYPE_INCOMPATIBLE` (array-typed source/dest not supported), `FIELD_MAP_COLLISION` (two sources map to same dest); an invalid mapping aborts `apply` synchronously (`FIELD_MAP_INVALID` error code). Natural-key record matching: `naturalKeys` (`{ [tableName]: fieldName }`) matches dest↔source records by the key field's **value** (field ID resolved per-base by name), growing `idmap.records`. Add-only + ambiguity-safe — never removes/overwrites an existing mapping, never claims an already-mapped dest; ambiguous/duplicate key values are skipped with `NATURAL_KEY_AMBIGUOUS`; a missing key field emits `NATURAL_KEY_FIELD_MISSING`. Runs in `mode=reconcile` (after the existence-prune, snapshots source records + calls matcher, then saves idmap) AND as an automatic pre-pass in `mode=apply` **before Pass 1 and prune** — so records `mirror` no longer deletes real-but-unmapped dest records as false orphans, and re-syncs don't duplicate them. The key field must be **stable across bases** (autoNumber renumbers → bad key; name/email/injected `InjectID` → good key). Single-field keys only; composite keys and value normalization are deferred. Returns a `matched` count in the result. Hardening pass (2026-07, 25 commits, ~100 regression tests — full audit + fixes): `plan`/`diff` preserve persisted `idmap.records`/`idmap.attachments` verbatim (schema maps refresh; record identity is never wiped). Apply journals by **disposition** — only genuinely applied actions are `done`; gated (`RETYPE_GATED`) and deferred (`UNRESOLVABLE_REF`, link target table not yet mapped) actions stay retryable, and deferred creates/updates retry within the same run until no progress, so forward-referenced computed/link fields converge in one apply. Reverse-link adoption is exact (idmap-based `symmetricColumnId` match — cross-run safe; two links between the same table pair never cross-wire); link `updateField`s ship writable options only (`{foreignTableId, relationship}`, remapped). `reconcilePrimary` gates pre-existing-primary retypes behind `confirmRetypes` (an applied rename is journaled as an `:rename` sub-action so the same-plan follow-up resumes past the drift guard) and writes computed primaries via `toWritableComputedOptions`. Diff/compare: matched link fields compare canonically (no phantom `updateField`), writable display-format options on computed fields are compared, collaborator `usr` ids pass through view-filter remap verbatim (user ids are Airtable-global), views match by **(name, type)** — apply adopts by the same key — and the dest primary is never emitted as a deletable orphan. Cells: internal type spellings (`multipleAttachment`, `multiCollaborator`, …) are recognized in `cells.js` + `policy.js`; ALL array-shaped cells are deferred on the Pass-1 update path; cells cleared on source propagate as explicit nulls under `conflicts=source-wins` (only idmap-mapped fields whose source AND dest are non-computed); empty update rows are skipped. Records mirror propagates source-side record deletions (a mapping whose source row is gone stops protecting its dest row — truncation-guarded, gated by `confirmDeletions`, stale mapping dropped); truncation detection counts actually-fetched rows (immune to `mirrorFoldedLinks` inflation). New select/multiSelect fields register their real choice map at creation, so first-run records write select cells. Record snapshots avoid filtered views (prefer an unfiltered collaborative view; else `SNAPSHOT_VIEW_FILTERED` and the table is treated as truncated — no prune). Background job files carry the worker `pid`; `mode=status` reports a `running` job with a dead pid as failed with resume advice. Remaining capture-gated deferrals: unfiltered/paginated table reads (`readQueries` non-view source), and array-cell VALUE updates on existing records (multiSelect/collaborator edits after creation). +- `src/sync/` — base-to-base schema + record sync engine. Plan side: `snapshot` → `idmap` (match-by-name) → `diff` (ordered Plan). Apply side (`apply.js` + `journal.js` + `remap.js`'s `remapRefs`/`toWritableComputedOptions`): executes a saved Plan against the destination — creates tables (deleting Airtable's auto-scaffolding fields), reconciles the primary, creates scalar/link(`foreignKey`)/computed fields with source→dest ref remapping + formula validation, applies non-destructive field updates. A computed-field `updateField` that 422s on a same-pass link change ("requires a link field / link field was changed by a collaborator") is returned as `deferred` (not hard-failed) so the retry-until-stable loop re-applies it after the link update settles. Drift-guarded + resumable (journal; a journal with ≥1 done action bypasses the drift guard with `RESUME_DRIFT_BYPASS`, so resume is reachable past sync's own mutations). Concurrent applies to the same base pair are refused via a per-pair `apply.lock` (`src/sync/apply-lock.js`: pid-liveness stale detection, held until the background records job settles; `mode=reconcile` takes the same lock; error code `APPLY_LOCKED`). Exposed via the `sync_base` tool (`mode: diff | plan | apply | status | reconcile`) in the `sync` category. Out of scope: non-scalar retypes (computed/link/attachment on either side → `RETYPE_DEFERRED`), deletions (M4); scalar retypes are now in scope (see below). `mode=diff` (read-only): comprehensive schema comparison via pure `src/sync/compare.js` — every difference classified as **drift** (sync enforces: table/field/view existence, field type/options/choices, view filters/sorts/groups content, column visibility, colors/cover/frozen/rowHeight, primary, form), **best-effort** (sync applies but doesn't guarantee: field/view/column order, sort/group clause order), or **not-synced** (view sections — now captured by snapshot). Verdicts: `identical` (zero diffs) or `converged` (zero drift; best-effort/not-synced may remain). Persists full diff to `~/.airtable-user-mcp/sync/__/diff-.json`; returns a token-budgeted DIGEST (verdicts + class counts + per-table rollup + capped driftSample ≤25). Drill in with `detail="
"` (params: `detail`, `diffId`, `offset`, `limit`). `mode=plan` now emits a curatable **changeset**: each action carries a stable name-based `changeId` (`|
|`), a `class`, and an `apply:true` flag; digest includes a `sample` + `editHint`. New `direction` param (`to-dest` default | `to-source`) targets either base. View sync: `snapshot` also captures collaborative views + live config + **view sections** (not-synced class); `remap.remapViewConfig` rewrites field/choice IDs in view config; `diff` emits `createView`/`applyViewConfig` (convergent canonical compare); `apply` reuses the client view methods (filters/sorts/groups/columns/frozen/colors/cover/calendar/form/rowHeight) with per-type anchor validation + grid fallback. `sync_base mode=apply` syncs collaborative views after fields (personal views skipped; orphan views reported); accepts a `skip:[changeId]` list (and honors `apply:false` on changeset entries) — applies everything except removed changes (journal-safe; skipped dependencies degrade to UNRESOLVABLE_REF). Record sync (M3-records): after schema/view apply, `mode=apply` launches a **background** record sync and returns a `jobId` (= planId) immediately. Two-pass: Pass 1 creates/updates scalar + select cells (computed fields + computed primary never written), fills persisted `idmap.records` (rec→rec map); Pass 2 writes linked-record cells (remapped via record map, unresolved targets reported); then attachments (download source → re-upload dest → dedupe scoped to the dest cell `row|field|filename|size`); then restores record-referencing view filters stripped during view sync (per-view failures warn + continue, never block prune). Pass 2 + attachments honor `conflicts=dest-wins` for pre-existing mapped rows; rows created by this plan — tracked resume-durably as `createdDestIds` in the records journal — always receive their links/attachments. Throttle handling = per-request 429 backoff in the auth queue; the record engine ALSO paces every internal per-row POST through a phase-local rate gate (`makeGate(createLimiter({ rps:5 }))` in `records.js`), so per-row creates/updates/link-adds ARE rate-limited (an earlier note claiming they bypass the limiter was wrong). **Session recovery (`auth.js._apiCall`):** in browser-free modes (byo/direct-login) a `403` that persists past one throttle backoff now escalates to session re-auth (re-mints cookie+CSRF via `_recoverSession`) instead of being treated as pure throttle — this is the auth-expiry-as-403 case that previously killed long record jobs (all rows `SESSION_INVALID`); `429`/`503` stay throttle in all modes, browser mode keeps 403=throttle, and the status that tripped the dead-session breaker is retrievable via `auth.getLastTrip()`. **Records session-death abort:** the records job (`applyRecords`) re-mints/probes the session at phase start (`resetSessionHealth()` then `ensureSessionHealthy()`) and ABORTS the whole job on a mid-run session death (`isSessionDead()` or a `SESSION_INVALID` throw) — and on a general all-writes-failed run (`RECORDS_ALL_FAILED`, i.e. `failed>0 && created==0 && updated==0 && skipped==0`, which also catches non-session run-wide hangs) — by setting `result.aborted` → records-job `phase=failed` with a resume hint, skipping `reapplyViewFilters`+`pruneRecords` (a dead/partial session must NEVER prune) instead of walking every row into a warning and reporting a false `done`; resumable (records journal, per-chunk persist of ~50 creates); continue-on-failure otherwise (non-session per-record errors are warnings and the run continues). Truncation safety: a table whose snapshot hits the 1000-row cap on either base is added to `truncatedTables` by `collectTruncatedTableNames` and skipped by `pruneRecords` under mirror (`RECORDS_TRUNCATED_PRUNE_SKIPPED`) so a record beyond the snapshot window is never deleted as a false orphan; each truncated table also emits `RECORDS_TRUNCATED` on the result; >1000-row read pagination remains a capture-gated follow-up. `mode=plan` and `mode=apply` themselves now run as BACKGROUND jobs (`src/sync/job-status.js` `sync-job-.json`, phases `planning→schema→records→done|failed`, pid-liveness) — they return `{jobId,status:running}` immediately (a synchronous plan/apply on a view-heavy base is minutes-long and trips the MCP client's response window → "Connection closed" despite completing server-side); `planJob`/`applyJob`/`syncStatus` in `src/sync/index.js`, `plan()`/`apply()` stay callable as sync engine fns. `mode=diff`/`reconcile` stay synchronous. `mode=status` polls a background job by planId — returns phase + running/done/failed + live records-mapped count + planDigest/schemaResult/recordsResult (FIELD_MAP_INVALID/APPLY_LOCKED now surface here as phase=failed; a records-job abort — session death or `RECORDS_ALL_FAILED` — surfaces as phase=failed with the partial counts + abort reason). To keep the response inside the MCP window, `mode=status` returns `planDigest` as **human-only by default** (`{ human, machineOmitted:true }`) — the full changeset (`planDigest.machine`, ~145 view-config actions on a view-heavy base) is omitted and only returned when `verbose:true` is passed (it always persists on disk in `plan-.json`). The status response also orders `summary`/`schemaResult`/`recordsResult` before `planDigest` so any residual truncation cuts the least-useful field last. `mode=reconcile` **prunes** dead record-map entries (existence-prune, truncation-guarded — skipped with a warning when any dest table hits the 1000-row snapshot cap, so live mappings beyond the cap are never dropped; per-table natural-key re-match now matches by key value — see natural-key matching below; reconcile does NOT de-duplicate records). State lives under `~/.airtable-user-mcp/sync/__/` (idmap.json, journals, records-job-.json, diff-.json). New modules: `cells.js` (per-type cell coercion), `ratelimit.js` (token bucket + retry), `records.js` (Pass 1/2 + attachments + reapplyViewFilters + applyRecords orchestrator + reconcile), `compare.js` (pure classified compare, no I/O); plus `snapshot.snapshotSchemaOnly`/`snapshotTableRecords`, `idmap.records`, and a records-journal in `journal.js`. Reconciliation policy (`src/sync/policy.js`): `mode=apply` accepts a `policy` param (`mirror` | `overlay` | `preserve`, default `overlay`) that controls two axes — **extras** (what to do with dest-only records) and **conflicts** (whose value wins when both bases have the record). Presets: `mirror` = {remove, source-wins} (make dest identical to source); `overlay` = {keep, source-wins} (today's default: keep dest-only rows, source updates win); `preserve` = {keep, dest-wins} (keep dest-only rows and never overwrite dest edits). `policyOverrides` (`{ [tableName]: preset }`) overrides the global preset per table. `confirmDeletions` (boolean, default false) is a safety gate for `mirror`: without it, `pruneRecords` reports a `DELETION_GATED` warning with the would-delete count but deletes nothing; set `confirmDeletions:true` to actually delete. Separately, `createTable` unconditionally clears the ~3 blank scaffolding rows it seeds (best-effort; `SCAFFOLDING_ROWS_KEPT` warning if the deletion attempt fails); `pruneRecords` under `mirror`+`confirmDeletions` additionally removes any leftover dest-only blank rows as part of the ordinary orphan prune. Attachment deletion under mirror is a deferred follow-up. Schema-orphan deletion (`pruneSchema`) runs synchronously in `mode=apply` AFTER `applyPlan`, consuming `plan.orphans` (`{kind:'table'|'field'|'view', destId, name, tableName}`). Order: (1) views — deleted first (no dependents); (2) fields — batch `deleteFields` WITHOUT force (so a matched field is never cascade-deleted), retry-until-stable loop resolves orphan→orphan deps; a field still blocked after no progress is kept + `SCHEMA_DELETE_BLOCKED`; (3) tables — deleted last (after their content). Gating: dest-only fields + views require `confirmDeletions:true` (same flag as `pruneRecords`); dest-only tables require the new `confirmTableDeletions:true` — `confirmDeletions` alone never drops a whole table. Without each gate the operation is a dry-count: `DELETION_GATED` for fields/views, `TABLE_DELETION_GATED` for tables. Applied counts: `schemaDeleted` (fields+views) + `tablesDeleted`. Per-table `policyOverrides` apply; `overlay`/`preserve` → no schema deletion. Continue-on-failure (`SCHEMA_DELETE_FAILED`). Module: `src/sync/prune-schema.js`. View-section orphans (dest-only sidebar sections in a matched table) are now deleted by `pruneSchema` as **step 0** (before views, so Airtable auto-promotes their contained views to top-level before view orphan deletion); gated by `confirmDeletions` (same flag as fields/views); `deleteViewSection` is the client method used. Attachment-strip under mirror remains a deferred follow-up. Scalar field retypes (`mode=apply`): a matched dest field whose scalar type diverges from source is retyped via `updateFieldConfig`, gated by the new `confirmRetypes` param (boolean, default false); without it the field is kept and a `RETYPE_GATED` warning is emitted (count surfaced in the result). Non-scalar retypes (either side is computed/link/attachment) stay `RETYPE_DEFERRED`. Select retypes converge their choice map on re-sync exactly like created selects (`mergeChoices`). Continue-on-failure: per-field retype errors emit `RETYPE_FAILED` and do not abort apply. Primary-field retypes remain the separate `reconcilePrimary` path. This closes the last schema-drift category sync can fix; records data migration remains `pruneRecords`/Pass 1. Custom field mappings: `fieldMappings` (`{ [tableName]: { sourceFieldName: destFieldName } }`) injects a source field's value into a different writable scalar dest field during record sync — useful when the source is computed (e.g., `autoNumber`/formula) and you want its value preserved in a writable dest field (classic use: inject source autoNumber `Code` into dest text field `InjectID` so a dest formula can reconstruct original identity). Fail-fast pre-flight validation in `apply` (and dry-run in `plan`/`diff` via `fieldMappingErrors` in the machine output) with codes: `FIELD_MAP_TABLE_MISSING`, `FIELD_MAP_SOURCE_MISSING`, `FIELD_MAP_TARGET_MISSING`, `FIELD_MAP_TARGET_COMPUTED` (dest must be writable scalar), `FIELD_MAP_TYPE_INCOMPATIBLE` (array-typed source/dest not supported), `FIELD_MAP_COLLISION` (two sources map to same dest); an invalid mapping aborts `apply` synchronously (`FIELD_MAP_INVALID` error code). Natural-key record matching: `naturalKeys` (`{ [tableName]: fieldName }`) matches dest↔source records by the key field's **value** (field ID resolved per-base by name), growing `idmap.records`. Add-only + ambiguity-safe — never removes/overwrites an existing mapping, never claims an already-mapped dest; ambiguous/duplicate key values are skipped with `NATURAL_KEY_AMBIGUOUS`; a missing key field emits `NATURAL_KEY_FIELD_MISSING`. Runs in `mode=reconcile` (after the existence-prune, snapshots source records + calls matcher, then saves idmap) AND as an automatic pre-pass in `mode=apply` **before Pass 1 and prune** — so records `mirror` no longer deletes real-but-unmapped dest records as false orphans, and re-syncs don't duplicate them. The key field must be **stable across bases** (autoNumber renumbers → bad key; name/email/injected `InjectID` → good key). Single-field keys only; composite keys and value normalization are deferred. Returns a `matched` count in the result. Hardening pass (2026-07, 25 commits, ~100 regression tests — full audit + fixes): `plan`/`diff` preserve persisted `idmap.records`/`idmap.attachments` verbatim (schema maps refresh; record identity is never wiped). Apply journals by **disposition** — only genuinely applied actions are `done`; gated (`RETYPE_GATED`) and deferred (`UNRESOLVABLE_REF`, link target table not yet mapped) actions stay retryable, and deferred creates/updates retry within the same run until no progress, so forward-referenced computed/link fields converge in one apply. Reverse-link adoption is exact (idmap-based `symmetricColumnId` match — cross-run safe; two links between the same table pair never cross-wire); link `updateField`s ship writable options only (`{foreignTableId, relationship}`, remapped). `reconcilePrimary` gates pre-existing-primary retypes behind `confirmRetypes` (an applied rename is journaled as an `:rename` sub-action so the same-plan follow-up resumes past the drift guard) and writes computed primaries via `toWritableComputedOptions`. Diff/compare: matched link fields compare canonically (no phantom `updateField`), writable display-format options on computed fields are compared, collaborator `usr` ids pass through view-filter remap verbatim (user ids are Airtable-global), views match by **(name, type)** — apply adopts by the same key — and the dest primary is never emitted as a deletable orphan. Cells: internal type spellings (`multipleAttachment`, `multiCollaborator`, …) are recognized in `cells.js` + `policy.js`; ALL array-shaped cells are deferred on the Pass-1 update path; cells cleared on source propagate as explicit nulls under `conflicts=source-wins` (only idmap-mapped fields whose source AND dest are non-computed); empty update rows are skipped. Records mirror propagates source-side record deletions (a mapping whose source row is gone stops protecting its dest row — truncation-guarded, gated by `confirmDeletions`, stale mapping dropped); truncation detection counts actually-fetched rows (immune to `mirrorFoldedLinks` inflation). New select/multiSelect fields register their real choice map at creation, so first-run records write select cells. Record snapshots avoid filtered views (prefer an unfiltered collaborative view; else `SNAPSHOT_VIEW_FILTERED` and the table is treated as truncated — no prune). Background job files carry the worker `pid`; `mode=status` reports a `running` job with a dead pid as failed with resume advice. Remaining capture-gated deferrals: unfiltered/paginated table reads (`readQueries` non-view source), and array-cell VALUE updates on existing records (multiSelect/collaborator edits after creation). #### packages/mcp-server — Daemon subsystem diff --git a/packages/extension/package.json b/packages/extension/package.json index f97dc9c..17d677f 100644 --- a/packages/extension/package.json +++ b/packages/extension/package.json @@ -1,7 +1,7 @@ { "name": "airtable-formula", "private": true, - "version": "2.1.16", + "version": "2.1.27", "publisher": "Nskha", "displayName": "Airtable Formulas, Scripts, Automation, MCP & LSP", "description": "Airtable formula, script & automation editor with MCP server (71 tools), language server, and AI skills for VS Code.", diff --git a/packages/extension/src/debug/exporter.ts b/packages/extension/src/debug/exporter.ts index 9cbbae9..425fe07 100644 --- a/packages/extension/src/debug/exporter.ts +++ b/packages/extension/src/debug/exporter.ts @@ -10,7 +10,8 @@ const ALWAYS_STRIP_KEYS = new Set([ 'secretSocketId', 'cookie', 'cookies', 'set-cookie', 'password', 'otpSecret', 'otpCode', 'totp', - 'AIRTABLE_PASSWORD', 'AIRTABLE_OTP_SECRET', 'AIRTABLE_EMAIL', + 'AIRTABLE_PASSWORD', 'AIRTABLE_OTP_SECRET', 'AIRTABLE_TOTP_SECRET', 'AIRTABLE_EMAIL', + 'AIRTABLE_COOKIE', 'AIRTABLE_CSRF', 'authorization', 'apiKey', 'api_key', 'token', 'accessToken', 'refreshToken', 'bearer', 'privateKey', 'private_key', 'stringifiedObjectParams', diff --git a/packages/extension/src/extension.ts b/packages/extension/src/extension.ts index 1c26a27..636c9b9 100644 --- a/packages/extension/src/extension.ts +++ b/packages/extension/src/extension.ts @@ -82,7 +82,10 @@ export async function activate(context: vscode.ExtensionContext): Promise /(password[=:"\s]+)\S+/gi, /(AIRTABLE_PASSWORD[=:"\s]+)\S+/gi, /(AIRTABLE_OTP_SECRET[=:"\s]+)\S+/gi, + /(AIRTABLE_TOTP_SECRET[=:"\s]+)\S+/gi, /(otpSecret[=:"\s]+)\S+/gi, + /(AIRTABLE_COOKIE[=:"\s]+)[^\r\n]+/gi, + /(AIRTABLE_CSRF[=:"\s]+)\S+/gi, /(set-cookie:?\s*)[^\r\n]+/gi, /(cookie:?\s*)[^\r\n]+/gi, ]; diff --git a/packages/extension/src/mcp/auth-manager.ts b/packages/extension/src/mcp/auth-manager.ts index 1ddb715..5fa21ef 100644 --- a/packages/extension/src/mcp/auth-manager.ts +++ b/packages/extension/src/mcp/auth-manager.ts @@ -15,6 +15,7 @@ const SECRET_PREFIX = 'airtableFormula'; const SECRET_EMAIL = `${SECRET_PREFIX}.email`; const SECRET_PASSWORD = `${SECRET_PREFIX}.password`; const SECRET_OTP_SECRET = `${SECRET_PREFIX}.otpSecret`; +const SECRET_COOKIE = `${SECRET_PREFIX}.cookie`; const PROFILE_DIR = path.join(os.homedir(), '.airtable-user-mcp', '.chrome-profile'); const CONFIG_DIR = path.join(os.homedir(), '.airtable-user-mcp'); @@ -32,7 +33,7 @@ export class AuthManager implements vscode.Disposable { private readonly _onDidChange = new vscode.EventEmitter(); public readonly onDidChange = this._onDidChange.event; - private _state: AuthState = { status: 'unknown', hasCredentials: false }; + private _state: AuthState = { status: 'unknown', hasCredentials: false, hasCookie: false }; private _timer: ReturnType | undefined; private _initCheckTimer: ReturnType | undefined; private _disposed = false; @@ -193,6 +194,35 @@ export class AuthManager implements vscode.Disposable { return this.secrets.get(SECRET_OTP_SECRET); } + // ─── BYO session cookie (authMode='byo') ───────────────────── + // + // Stored in the OS keychain like the email/password/otp triplet. The cookie + // reaches the MCP server via env (stdio path, getCredentialsEnv) or the + // bearer-authenticated /daemon/auth-credentials endpoint (daemon path) — it + // is never written to disk (no plaintext credentials.json) and never logged. + + async getCookie(): Promise { + return this.secrets.get(SECRET_COOKIE); + } + + // CSRF is not stored: for byo the server auto-scrapes it from the cookie's authed session. + async saveCookie(cookie: string): Promise { + // Modes are NOT storage-exclusive: any stored email/password (direct-login) + // are intentionally preserved so the user can switch between byo and + // direct-login without re-entering credentials. + await this.secrets.store(SECRET_COOKIE, cookie); + this._updateState({ hasCookie: true }); + } + + async clearCookie(): Promise { + await this.secrets.delete(SECRET_COOKIE); + this._updateState({ hasCookie: false }); + } + + async hasCookie(): Promise { + return !!(await this.getCookie()); + } + async saveCredentials(email: string, password: string, otpSecret?: string): Promise { await this.secrets.store(SECRET_EMAIL, email); await this.secrets.store(SECRET_PASSWORD, password); @@ -208,7 +238,8 @@ export class AuthManager implements vscode.Disposable { await this.secrets.delete(SECRET_EMAIL); await this.secrets.delete(SECRET_PASSWORD); await this.secrets.delete(SECRET_OTP_SECRET); - this._updateState({ status: 'unknown', hasCredentials: false, userId: undefined, error: undefined }); + await this.secrets.delete(SECRET_COOKIE); + this._updateState({ status: 'unknown', hasCredentials: false, hasCookie: false, userId: undefined, error: undefined }); } async logout(): Promise { @@ -219,6 +250,7 @@ export class AuthManager implements vscode.Disposable { await this.secrets.delete(SECRET_EMAIL); await this.secrets.delete(SECRET_PASSWORD); await this.secrets.delete(SECRET_OTP_SECRET); + await this.secrets.delete(SECRET_COOKIE); const fs = await import('fs/promises'); try { @@ -228,7 +260,7 @@ export class AuthManager implements vscode.Disposable { console.warn('[AuthManager] Failed to clear browser profile:', err); } - this._updateState({ status: 'unknown', hasCredentials: false, userId: undefined, error: undefined }); + this._updateState({ status: 'unknown', hasCredentials: false, hasCookie: false, userId: undefined, error: undefined }); } async hasCredentials(): Promise { @@ -241,13 +273,16 @@ export class AuthManager implements vscode.Disposable { * Read stored credentials from SecretStorage. * Returns undefined if no credentials are stored. */ - async getCredentials(): Promise<{ email: string; password: string; otpSecret?: string } | undefined> { + async getCredentials(opts?: { ensureDaemon?: boolean }): Promise<{ email: string; password: string; otpSecret?: string } | undefined> { // D-02: ensure daemon is running before handing off credentials. // Implicit + best-effort: credentials don't require the daemon, and this // path runs from provideMcpServerDefinitions — it must neither resurrect // a daemon the user explicitly stopped nor block login when the daemon // can't start. - if (getSettings().mcp.useDaemon && this._daemonManager) { + // Callers that push creds TO an already-running daemon (the lockfile-watch / dashboard re-push) + // pass { ensureDaemon: false }: they must NOT trigger a spawn here, or a lockfile-watch → + // getCredentials → ensureDaemon → spawn → lockfile-write → watch loop runs away (multiple procs). + if (opts?.ensureDaemon !== false && getSettings().mcp.useDaemon && this._daemonManager) { try { await this._daemonManager.ensureDaemon({ implicit: true }); } catch { /* user-stopped latch or startup failure — proceed without daemon */ } @@ -261,14 +296,49 @@ export class AuthManager implements vscode.Disposable { } /** - * Get credentials as env vars. ONLY for the VS Code MCP stdio definition - * (registration.ts), where VS Code owns the spawn and env is the only - * channel. Helper scripts we fork ourselves receive credentials over the - * IPC channel instead (_spawnScript) so they never appear in the child's - * environment (/proc//environ, process listings, core dumps). + * Get credentials as env vars, shaped for the given auth mode. ONLY for the + * VS Code MCP stdio definition (registration.ts), where VS Code owns the + * spawn and env is the only channel. Helper scripts we fork ourselves receive + * credentials over the IPC channel instead (_spawnScript) so they never + * appear in the child's environment (/proc//environ, process listings, + * core dumps). + * + * - 'byo' → { AIRTABLE_COOKIE } only (CSRF is auto-scraped + * server-side, never forwarded), or undefined when no + * cookie is saved. + * - 'direct-login' → { AIRTABLE_EMAIL, AIRTABLE_PASSWORD, AIRTABLE_TOTP_SECRET } + * (direct-login reads TOTP as AIRTABLE_TOTP_SECRET), or + * undefined when no credentials are saved. + * - browser / undefined → { AIRTABLE_EMAIL, AIRTABLE_PASSWORD, + * AIRTABLE_OTP_SECRET } (unchanged legacy behavior). + * + * NOTE: the daemon transport never uses this — daemon creds go via the + * bearer-authenticated /daemon/auth-credentials endpoint (see + * DaemonManager.pushAuthCredentials), never the daemon's env. */ - async getCredentialsEnv(): Promise | undefined> { - const creds = await this.getCredentials(); + async getCredentialsEnv(authMode?: string): Promise | undefined> { + if (authMode === 'byo') { + const cookie = await this.getCookie(); + if (!cookie) return undefined; + // CSRF is auto-scraped server-side from the cookie session — not forwarded. + return { AIRTABLE_COOKIE: cookie }; + } + + if (authMode === 'direct-login') { + // getCredentialsEnv builds env for the STDIO (non-daemon) spawn — it must + // never trigger a daemon spawn from a credential read. + const creds = await this.getCredentials({ ensureDaemon: false }); + if (!creds) return undefined; + const env: Record = { + AIRTABLE_EMAIL: creds.email, + AIRTABLE_PASSWORD: creds.password, + }; + if (creds.otpSecret) env.AIRTABLE_TOTP_SECRET = creds.otpSecret; + return env; + } + + // Same rationale as the direct-login branch above. + const creds = await this.getCredentials({ ensureDaemon: false }); if (!creds) return undefined; const env: Record = { @@ -289,6 +359,23 @@ export class AuthManager implements vscode.Disposable { const viaDaemon = await this._checkViaDaemon(); if (viaDaemon) return viaDaemon; + // byo / direct-login never touch a browser — the session is minted/refreshed + // by the MCP server over direct-HTTP. The daemon check above is the only live + // verification for them; with no daemon we can't probe, so surface a neutral + // status instead of forking Chrome or reporting 'chrome-missing'. + const mode = getSettings().mcp.authMode; + if (mode === 'byo' || mode === 'direct-login') { + // Neutral 'unknown' state: this isn't an error. The webview renders any + // auth.error as a permanent yellow warning, so leave error undefined + // (and clear any prior error) rather than putting an informational string there. + this._updateState({ + status: 'unknown', + lastChecked: new Date().toISOString(), + error: undefined, + }); + return this._state; + } + // No daemon (stdio mode) or a daemon that predates the endpoint — fall back // to forking our own health-check. Preflight: no browser, no point spawning. const probe = this.refreshBrowserDetection(); @@ -440,6 +527,20 @@ export class AuthManager implements vscode.Disposable { // ─── Login ─────────────────────────────────────────────────── async login(): Promise { + // byo / direct-login have no browser login: byo's cookie authenticates + // directly and direct-login replays email/password/TOTP server-side. The + // stored credentials ARE the login — validate them via checkSession() + // (the daemon's direct-HTTP check) instead of launching Chrome. + const mode = getSettings().mcp.authMode; + if (mode === 'byo' || mode === 'direct-login') { + const hasCreds = mode === 'byo' ? await this.hasCookie() : await this.hasCredentials(); + if (!hasCreds) { + this._updateState({ status: 'error', error: 'No credentials saved.' }); + return this._state; + } + return this.checkSession(); + } + const probe = this.refreshBrowserDetection(); if (!probe.found) { this._updateState({ @@ -484,6 +585,18 @@ export class AuthManager implements vscode.Disposable { } async manualLogin(): Promise { + // byo / direct-login never open a browser — route to the same credential + // validation as login() (the cookie / stored creds ARE the login). + const mode = getSettings().mcp.authMode; + if (mode === 'byo' || mode === 'direct-login') { + const hasCreds = mode === 'byo' ? await this.hasCookie() : await this.hasCredentials(); + if (!hasCreds) { + this._updateState({ status: 'error', error: 'No credentials saved.' }); + return this._state; + } + return this.checkSession(); + } + const probe = this.refreshBrowserDetection(); if (!probe.found) { this._updateState({ @@ -569,11 +682,22 @@ export class AuthManager implements vscode.Disposable { } const hasCreds = await this.hasCredentials(); - const probe = this.refreshBrowserDetection(); - this._updateState({ - hasCredentials: hasCreds, - ...(probe.found ? {} : { status: 'chrome-missing' as const }), - }); + const hasCookie = await this.hasCookie(); + + const mode = getSettings().mcp.authMode; + if (mode === 'byo' || mode === 'direct-login') { + // No browser in these modes — don't probe or flag 'chrome-missing'. Leave + // status 'unknown'; the daemon check (via checkSession / auto-refresh) + // verifies the session without launching Chrome. + this._updateState({ hasCredentials: hasCreds, hasCookie }); + } else { + const probe = this.refreshBrowserDetection(); + this._updateState({ + hasCredentials: hasCreds, + hasCookie, + ...(probe.found ? {} : { status: 'chrome-missing' as const }), + }); + } await this._applyPermissions(); this.startAutoRefresh(); @@ -594,8 +718,13 @@ export class AuthManager implements vscode.Disposable { if (this._disposed) return; if (this._state.status === 'logging-in' || this._state.status === 'checking') return; - const probe = this.refreshBrowserDetection(); - if (!probe.found) return; + // Only browser mode needs a local browser to refresh; byo / direct-login + // verify via the daemon's direct-HTTP check and must not be blocked here. + const mode = getSettings().mcp.authMode; + if (mode !== 'byo' && mode !== 'direct-login') { + const probe = this.refreshBrowserDetection(); + if (!probe.found) return; + } const state = await this.checkSession(); diff --git a/packages/extension/src/mcp/daemon-manager.ts b/packages/extension/src/mcp/daemon-manager.ts index ae57f3b..5f3f632 100644 --- a/packages/extension/src/mcp/daemon-manager.ts +++ b/packages/extension/src/mcp/daemon-manager.ts @@ -1,12 +1,16 @@ import * as vscode from 'vscode'; import * as path from 'path'; import * as fs from 'fs/promises'; -import { existsSync, rmSync } from 'fs'; +import { existsSync, rmSync, appendFileSync, openSync } from 'fs'; import { spawn, execFileSync } from 'child_process'; export interface DaemonStatus { running: boolean; healthy: boolean; + /** The lockfile exists but its recorded pid is NOT alive — the daemon crashed/was killed without + * releasing it. `running` stays true (so stopDaemon can still reclaim the lock), but consumers that + * need a LIVE daemon (ensureDaemon) treat `stale` like not-running and spawn a replacement. */ + stale: boolean; pid: number | null; port: number | null; port_lsp: number | null; @@ -28,7 +32,7 @@ export interface DaemonConnectionInfo { } const EMPTY_STATUS: DaemonStatus = { - running: false, healthy: false, pid: null, port: null, + running: false, healthy: false, stale: false, pid: null, port: null, port_lsp: null, bearerToken: null, tunnelUrl: null, uptime: null, uuid: null, }; @@ -53,12 +57,30 @@ export class DaemonManager implements vscode.Disposable { private _userStopped = false; /** Graceful-shutdown wait before kill escalation (overridable in tests). */ private _stopWaitMs = 10_000; + /** + * Timestamp (ms) of the last spawn attempt. Suppresses further spawns for a startup window so + * overlapping ensureDaemon() callers — or a lockfile-watch storm — cannot each launch a daemon. + * Without it, N callers each held a local `spawned` flag and each ran `_spawnDetached()`; a + * lockfile-watch → credential-push → ensureDaemon chain re-fired on every daemon.lock write and + * each firing spawned another process (a runaway start/restart loop that OOM'd the machine). + * A timestamp beats a single-flight promise here: check-and-set is synchronous (no await between), + * so it holds regardless of how a fast-resolving spawn and a slow status read interleave. + */ + private _lastSpawnAt = 0; + /** Window during which a fresh spawn is suppressed after one is attempted (~ the startup budget). */ + private static readonly SPAWN_SUPPRESS_MS = 15_000; constructor( private readonly configDir: string, private readonly extensionPath: string, ) {} + /** TEMP DIAGNOSTIC — append a timestamped line to ~/.airtable-user-mcp/daemon-diag.log so the LIVE + * extension host's daemon-startup path is observable (the failure is otherwise invisible). Best-effort. */ + private _diag(msg: string): void { + try { appendFileSync(path.join(this.configDir, 'daemon-diag.log'), `${new Date().toISOString()} ${msg}\n`); } catch { /* ignore */ } + } + private async _httpHealthCheck(port: number, bearerToken: string): Promise { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), 2_000); @@ -67,8 +89,10 @@ export class DaemonManager implements vscode.Disposable { headers: { Authorization: `Bearer ${bearerToken}` }, signal: controller.signal, }); + this._diag(`_httpHealthCheck port=${port} → status=${response.status} ok=${response.ok}`); return response.ok; - } catch { + } catch (e) { + this._diag(`_httpHealthCheck port=${port} → THREW ${(e as Error)?.name}: ${(e as Error)?.message}`); return false; } finally { clearTimeout(timeout); @@ -80,16 +104,31 @@ export class DaemonManager implements vscode.Disposable { const lockPath = path.join(this.configDir, 'daemon.lock'); const raw = await fs.readFile(lockPath, 'utf8'); const record = JSON.parse(raw) as Record; + const pid = typeof record.pid === 'number' ? record.pid : null; + // STALE-LOCK DETECTION — the fix for "Timed out waiting for daemon startup" after a crash. + // A lockfile whose recorded daemon pid is no longer alive is STALE (the daemon died/was killed + // without releasing it — e.g. the OOM crash, or an orphaned daemon). `running` stays true so + // stopDaemon can still reclaim the lock, but `stale:true` tells ensureDaemon to spawn a + // replacement (the new daemon's launcher reclaims the stale lock via lockfile.acquire → + // tryReclaimStale). WITHOUT this, a stale lock wedges ensureDaemon: running:true + healthy:false + // means it can neither return (not healthy) nor spawn (the branch requires !running || stale) → + // it polls to the deadline and throws. (EPERM in _isPidAlive counts as alive — a permission + // probe is never a dead process.) Skip the 2s health fetch on a dead pid. + const stale = pid != null && pid > 0 && !this._isPidAlive(pid); + if (stale) { + console.warn(`[DaemonManager] stale daemon.lock (pid ${pid} not alive) — a fresh daemon will reclaim it.`); + } const port = typeof record.port === 'number' && Number.isInteger(record.port) && record.port >= 1 && record.port <= 65535 ? record.port : null; const bearerToken = typeof record.bearerToken === 'string' ? record.bearerToken : null; - const healthy = port != null && bearerToken != null + const healthy = !stale && port != null && bearerToken != null ? await this._httpHealthCheck(port, bearerToken) : false; const status: DaemonStatus = { running: true, healthy, - pid: typeof record.pid === 'number' ? record.pid : null, + stale, + pid, port, port_lsp: typeof record.port_lsp === 'number' ? record.port_lsp : null, bearerToken, @@ -97,9 +136,11 @@ export class DaemonManager implements vscode.Disposable { uptime: typeof record.startedAt === 'string' ? Date.now() - Date.parse(record.startedAt) : null, uuid: typeof record.uuid === 'string' && record.uuid.length > 0 ? record.uuid : null, }; + this._diag(`getDaemonStatus: lock@${lockPath} running=true healthy=${healthy} stale=${stale} pid=${pid} port=${port}`); this._status = status; return status; - } catch { + } catch (e) { + this._diag(`getDaemonStatus: NO lock (or read/parse failed) @${path.join(this.configDir, 'daemon.lock')} — ${(e as Error)?.code ?? (e as Error)?.message ?? 'ENOENT'}`); this._status = { ...EMPTY_STATUS }; return { ...EMPTY_STATUS }; } @@ -117,22 +158,45 @@ export class DaemonManager implements vscode.Disposable { // change takes effect on the next (re)start without reloading the window. const daemonPort = vscode.workspace.getConfiguration('airtableFormula').get('mcp.daemonPort', 0); if (Number.isInteger(daemonPort) && daemonPort > 0 && daemonPort <= 65535) args.push('--port', String(daemonPort)); + const denv = this.buildDaemonEnv(); + this._diag(`_spawnDetached: execPath=${process.execPath} serverPath=${serverPath} exists=${existsSync(serverPath)} port=${daemonPort} ELECTRON_RUN_AS_NODE(inherited)=${process.env.ELECTRON_RUN_AS_NODE} env.AIRTABLE_USER_MCP_HOME=${denv.AIRTABLE_USER_MCP_HOME} env.NODE_PATH=${denv.NODE_PATH}`); + // TEMP DIAGNOSTIC: capture the daemon's stdout/stderr to a file instead of discarding it, so a + // crash-on-start is visible. (Production uses stdio:'ignore'.) + let stdio: 'ignore' | ['ignore', number, number] = 'ignore'; + try { + const fd = openSync(path.join(this.configDir, 'daemon-spawn.log'), 'a'); + stdio = ['ignore', fd, fd]; + } catch { /* fall back to ignore */ } const child = spawn(process.execPath, args, { detached: true, - stdio: 'ignore', - env: { ...process.env, ...this.buildDaemonEnv() }, + stdio, + env: { ...process.env, ...denv }, }); + child.on('error', (e) => this._diag(`_spawnDetached: child 'error' event → ${e.message}`)); + child.on('exit', (code, sig) => this._diag(`_spawnDetached: child 'exit' code=${code} sig=${sig}`)); + this._diag(`_spawnDetached: spawned child pid=${child.pid}`); child.unref(); } async ensureDaemon(options?: { timeoutMs?: number; implicit?: boolean }): Promise { if (this._disposed) throw new Error('DaemonManager disposed'); - if (!options?.implicit) this._userStopped = false; + // An EXPLICIT call is a deliberate user action (Start / Restart / authMode change) — it must not + // be throttled by the anti-runaway spawn window. Clear the latch AND the suppression timestamp so + // it can spawn immediately even within SPAWN_SUPPRESS_MS of a prior spawn (a crash-then-Start, or a + // restart that just stopped the daemon). Implicit callers (definition provider, credential handoff, + // the lockfile-watch chain) keep the window — that is what breaks the runaway loop. The reset is + // synchronous and before any await, so three concurrent EXPLICIT calls still collapse to one spawn: + // each resets to 0, then the first to reach the check writes the timestamp and the others suppress. + if (!options?.implicit) { this._userStopped = false; this._lastSpawnAt = 0; } const deadline = Date.now() + (options?.timeoutMs ?? 15_000); - let spawned = false; + this._diag(`ensureDaemon: ENTER implicit=${!!options?.implicit} timeoutMs=${options?.timeoutMs ?? 15_000} configDir=${this.configDir} extensionPath=${this.extensionPath} userStopped=${this._userStopped}`); + let spawnedThisCall = false; + let polls = 0; while (Date.now() < deadline) { const status = await this.getDaemonStatus(); + polls++; if (status.running && status.healthy && status.port != null && status.bearerToken != null) { + this._diag(`ensureDaemon: SUCCESS after ${polls} polls, port=${status.port}`); return { pid: status.pid ?? 0, uuid: '', @@ -143,15 +207,35 @@ export class DaemonManager implements vscode.Disposable { startedAt: '', }; } - if (!status.running && !spawned) { + // Spawn at most once PER CALL (spawnedThisCall) AND at most once per startup window ACROSS all + // calls (_lastSpawnAt) — so many overlapping ensureDaemon() callers converge on a single daemon + // instead of each spawning one. Spawn when there is no live daemon: either no lockfile + // (!running) OR a STALE lockfile whose daemon pid is dead (status.stale) — the spawned daemon + // reclaims the stale lock. Without the `|| status.stale`, a crash-orphaned lockfile wedges here. + if ((!status.running || status.stale) && !spawnedThisCall) { if (options?.implicit && this._userStopped) { + this._diag(`ensureDaemon: refusing implicit respawn (userStopped)`); throw new Error('Daemon was explicitly stopped by the user; not respawning implicitly.'); } - await this._spawnDetached(); - spawned = true; + // check-and-set is synchronous (no await between the read and the write), so two callers + // can't both pass the window check — the second sees the timestamp the first just wrote. + const sinceSpawn = Date.now() - this._lastSpawnAt; + this._diag(`ensureDaemon: spawn decision — running=${status.running} stale=${status.stale} sinceLastSpawn=${sinceSpawn}ms willSpawn=${sinceSpawn > DaemonManager.SPAWN_SUPPRESS_MS}`); + if (Date.now() - this._lastSpawnAt > DaemonManager.SPAWN_SUPPRESS_MS) { + this._lastSpawnAt = Date.now(); + // Log the spawn failure (e.g. missing server script) so the eventual "Timed out waiting for + // daemon startup" has a diagnosable cause; the poll below still re-evaluates. + await this._spawnDetached().catch((e) => { + console.warn('[DaemonManager] _spawnDetached failed:', e instanceof Error ? e.message : 'unknown error'); + }); + } + // Whether we spawned or deferred to another caller's in-flight spawn, stop trying to spawn + // from this call; just poll the deadline for the daemon to come up. + spawnedThisCall = true; } await this._delay(200); } + this._diag(`ensureDaemon: TIMED OUT after ${polls} polls (last status running=${this._status.running} healthy=${this._status.healthy} stale=${this._status.stale} port=${this._status.port})`); throw new Error('Timed out waiting for daemon startup.'); } @@ -376,7 +460,59 @@ export class DaemonManager implements vscode.Disposable { return new Promise(resolve => setTimeout(resolve, ms)); } - buildDaemonEnv(credEnv?: Record): Record { + /** + * Push byo/direct-login credentials to the RUNNING daemon over its + * bearer-authenticated endpoint (C4). The daemon stores them IN MEMORY only + * and re-inits auth on its next call — creds never reach the daemon via env + * or disk (buildDaemonEnv stays creds-free by design). + * + * Reads {port, bearerToken} from the daemon lockfile (via getDaemonStatus, + * the manager's single lockfile reader). Best-effort: returns false on any + * failure (daemon not running, unreachable, non-2xx). The payload and its + * fields are NEVER logged. + */ + async pushAuthCredentials(payload: { + authMode: 'byo' | 'direct-login'; + cookie?: string; + csrf?: string; + email?: string; + password?: string; + totpSecret?: string; + }): Promise { + if (this._disposed) return false; + try { + const status = await this.getDaemonStatus(); + if (!status.running || status.port == null || !status.bearerToken) return false; + + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 5_000); + try { + const resp = await fetch(`http://127.0.0.1:${status.port}/daemon/auth-credentials`, { + method: 'POST', + headers: { + Authorization: `Bearer ${status.bearerToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(payload), + signal: controller.signal, + }); + if (!resp.ok) { + console.warn(`[DaemonManager] pushAuthCredentials: daemon returned HTTP ${resp.status}`); + return false; + } + const body = await resp.json().catch(() => null) as { ok?: boolean } | null; + return body?.ok === true; + } finally { + clearTimeout(timeout); + } + } catch (err) { + // Concise, secret-free warning only — NEVER log the payload. + console.warn(`[DaemonManager] pushAuthCredentials failed: ${err instanceof Error ? err.message : 'unknown error'}`); + return false; + } + } + + buildDaemonEnv(): Record { const env: Record = { AIRTABLE_USER_MCP_HOME: this.configDir, AIRTABLE_HEADLESS_ONLY: '1', @@ -394,7 +530,9 @@ export class DaemonManager implements vscode.Disposable { const authMode = cfg.get('mcp.authMode', 'browser'); if (authMode && authMode !== 'browser') env.AIRTABLE_AUTH_MODE = authMode; if (cfg.get('mcp.httpClient', 'fetch') === 'impit') env.AIRTABLE_HTTP_CLIENT = 'impit'; - if (credEnv) Object.assign(env, credEnv); + // NOTE: credentials are DELIBERATELY never injected into the daemon env — + // they reach a running daemon only via the bearer-authed + // /daemon/auth-credentials endpoint (see pushAuthCredentials). return env; } diff --git a/packages/extension/src/mcp/registration.ts b/packages/extension/src/mcp/registration.ts index c2b1bab..773debc 100644 --- a/packages/extension/src/mcp/registration.ts +++ b/packages/extension/src/mcp/registration.ts @@ -106,10 +106,21 @@ export function registerMcpProvider( env.AIRTABLE_HTTP_CLIENT = 'impit'; } - // Pass stored credentials so MCP server can auto-recover sessions + // Pass stored credentials so MCP server can auto-recover sessions. + // Env is acceptable here — VS Code owns the stdio spawn (the daemon + // path never gets creds in env; it uses /daemon/auth-credentials). if (authManager) { - // Only forward credentials when loginMode is 'auto' - if (settings.auth.loginMode === 'auto') { + const authMode = settings.mcp.authMode; + const isCredMode = authMode === 'byo' || authMode === 'direct-login'; + if (isCredMode && !settings.mcp.useDaemon) { + // Pure stdio mode (no daemon): byo/direct-login REQUIRE credentials + // (no browser), and env is the only channel VS Code gives a stdio + // spawn. In daemon mode the daemon endpoint (/daemon/auth-credentials) + // is the SINGLE credential channel — never duplicate secrets into env. + const credEnv = await authManager.getCredentialsEnv(authMode); + if (credEnv) Object.assign(env, credEnv); + } else if (!isCredMode && settings.auth.loginMode === 'auto') { + // Browser mode: only forward creds for the automated login flow. const credEnv = await authManager.getCredentialsEnv(); if (credEnv) Object.assign(env, credEnv); } diff --git a/packages/extension/src/test/daemon-manager.test.ts b/packages/extension/src/test/daemon-manager.test.ts index cc01297..8e93050 100644 --- a/packages/extension/src/test/daemon-manager.test.ts +++ b/packages/extension/src/test/daemon-manager.test.ts @@ -8,6 +8,13 @@ vi.mock('vscode', () => ({ EventEmitter: vi.fn(() => ({ event: vi.fn(), fire: vi.fn(), dispose: vi.fn() })), Disposable: vi.fn(), McpHttpServerDefinition: undefined, + // buildDaemonEnv reads mcp.authMode / mcp.httpClient off the workspace config; + // return the caller-supplied default so the base env stays at its defaults. + workspace: { + getConfiguration: vi.fn(() => ({ + get: vi.fn((_key: string, defaultValue?: unknown) => defaultValue), + })), + }, })); // RED state — DaemonManager import will fail (module not yet created in 05-06-PLAN.md) @@ -46,11 +53,17 @@ describe('DaemonManager.buildDaemonEnv', () => { expect(result.ELECTRON_RUN_AS_NODE).toBe('1'); }); - it('merges credEnv keys on top of base env', () => { - const credEnv = { AIRTABLE_EMAIL: 'test@test.com' }; - const result = dm.buildDaemonEnv(credEnv); - expect(result.AIRTABLE_EMAIL).toBe('test@test.com'); - expect(result.AIRTABLE_USER_MCP_HOME).toBe(tmpDir); + it('never injects credential keys into the daemon env (creds go via the endpoint, not env)', () => { + const result = dm.buildDaemonEnv(); + // The daemon must NEVER receive credentials in its environment — they reach + // a running daemon only over the bearer-authed /daemon/auth-credentials + // endpoint (pushAuthCredentials). buildDaemonEnv is transport/config only. + expect(result.AIRTABLE_EMAIL).toBeUndefined(); + expect(result.AIRTABLE_PASSWORD).toBeUndefined(); + expect(result.AIRTABLE_TOTP_SECRET).toBeUndefined(); + expect(result.AIRTABLE_OTP_SECRET).toBeUndefined(); + expect(result.AIRTABLE_COOKIE).toBeUndefined(); + expect(result.AIRTABLE_CSRF).toBeUndefined(); }); }); @@ -68,6 +81,40 @@ describe('DaemonManager.getDaemonStatus', () => { const status = await dm.getDaemonStatus(); expect(status.running).toBe(false); }); + + const writeLock = (pid: number, port = 8723) => { + fs.writeFileSync(path.join(tmpDir, 'daemon.lock'), JSON.stringify({ + pid, uuid: 'uuid-1', port, port_lsp: null, bearerToken: 'tok', + version: '0.0.0', startedAt: new Date().toISOString(), tunnelUrl: null, + })); + }; + + it('flags a STALE lock (dead pid) so ensureDaemon can respawn, without a wasted health fetch', async () => { + // Regression: a crashed daemon leaves a lockfile with a dead pid. getDaemonStatus used to report + // running:true/healthy:false for it → ensureDaemon could neither return (unhealthy) nor spawn + // (its branch required !running) → "Timed out waiting for daemon startup". Now `stale:true` lets + // ensureDaemon spawn a replacement while `running` stays true so stopDaemon can still reclaim. + writeLock(999999); + (dm as any)._isPidAlive = vi.fn(() => false); + const healthSpy = vi.fn(async () => true); + (dm as any)._httpHealthCheck = healthSpy; + const status = await dm.getDaemonStatus(); + expect(status.stale).toBe(true); + expect(status.running).toBe(true); + expect(status.healthy).toBe(false); + expect((dm as any)._isPidAlive).toHaveBeenCalledWith(999999); + expect(healthSpy).not.toHaveBeenCalled(); // no 2s fetch to a known-dead daemon + }); + + it('keeps stale:false / running:true for a LIVE pid even when unhealthy (a still-starting daemon must not be culled)', async () => { + writeLock(4242); + (dm as any)._isPidAlive = vi.fn(() => true); + (dm as any)._httpHealthCheck = vi.fn(async () => false); // not yet serving + const status = await dm.getDaemonStatus(); + expect(status.running).toBe(true); + expect(status.stale).toBe(false); + expect(status.healthy).toBe(false); + }); }); describe('DaemonManager.probeHealth', () => { @@ -332,13 +379,70 @@ describe('DaemonManager user-stopped latch', () => { }); it('explicit ensureDaemon clears the latch and attempts a spawn', async () => { - (dm as any)._spawnDetached = vi.fn(); + // _spawnDetached returns Promise; ensureDaemon's single-flight guard calls .finally() on it. + (dm as any)._spawnDetached = vi.fn().mockResolvedValue(undefined); await dm.stopDaemon(); await expect(dm.ensureDaemon({ timeoutMs: 400 })).rejects.toThrow(/Timed out/); expect((dm as any)._spawnDetached).toHaveBeenCalled(); // Latch cleared — implicit calls may spawn again now await expect(dm.ensureDaemon({ implicit: true, timeoutMs: 400 })).rejects.toThrow(/Timed out/); }); + + it('concurrent ensureDaemon calls share ONE spawn (suppression window)', async () => { + // The runaway-daemon bug: N overlapping ensureDaemon() callers each spawned a daemon. The + // suppression window must collapse them to a single _spawnDetached() while no daemon is up. + // The mock resolves immediately and never brings a daemon "up", so each call polls to its + // deadline then rejects — but only the first attempt inside the window actually spawns. + let spawns = 0; + (dm as any)._spawnDetached = vi.fn(async () => { spawns++; }); + await Promise.all([ + dm.ensureDaemon({ timeoutMs: 300 }).catch(() => {}), + dm.ensureDaemon({ timeoutMs: 300 }).catch(() => {}), + dm.ensureDaemon({ timeoutMs: 300 }).catch(() => {}), + ]); + expect(spawns).toBe(1); + }); + + it('ensureDaemon SPAWNS when the lockfile is STALE (dead pid) instead of wedging → "Timed out"', async () => { + // THE "Daemon start failed: Timed out" bug: a crash leaves a lockfile with a dead pid. ensureDaemon + // must spawn a replacement (which reclaims the stale lock) rather than treating the corpse as a + // running daemon and polling forever. + fs.writeFileSync(path.join(tmpDir, 'daemon.lock'), JSON.stringify({ + pid: 999999, uuid: 'u', port: 8723, port_lsp: null, bearerToken: 't', + version: '0.0.0', startedAt: new Date().toISOString(), tunnelUrl: null, + })); + (dm as any)._isPidAlive = vi.fn(() => false); // dead → stale + const spy = vi.fn().mockResolvedValue(undefined); + (dm as any)._spawnDetached = spy; + await expect(dm.ensureDaemon({ timeoutMs: 300 })).rejects.toThrow(/Timed out/); + expect(spy).toHaveBeenCalled(); // it spawned; did NOT wedge on the stale lock + }); + + it('EXPLICIT ensureDaemon within the suppression window still spawns (window is reset at entry)', async () => { + // Simulate "just spawned" — inside SPAWN_SUPPRESS_MS of now. + (dm as any)._lastSpawnAt = Date.now(); + const spy = vi.fn().mockResolvedValue(undefined); + (dm as any)._spawnDetached = spy; + + // Explicit call (no implicit:true) must reset _lastSpawnAt to 0 at entry, so + // it spawns immediately instead of being throttled by the window. The stub + // never brings a daemon up, so the call times out regardless — what matters + // is that the spawn was attempted. + await expect(dm.ensureDaemon({ timeoutMs: 300 })).rejects.toThrow(/Timed out/); + expect(spy).toHaveBeenCalled(); + }); + + it('IMPLICIT ensureDaemon within the suppression window stays suppressed', async () => { + (dm as any)._lastSpawnAt = Date.now(); + const spy = vi.fn().mockResolvedValue(undefined); + (dm as any)._spawnDetached = spy; + + // Implicit callers keep the window — this is what breaks the runaway spawn + // loop. Not user-stopped, so it doesn't hit the latch-throw path; it should + // simply poll to the deadline and time out WITHOUT ever spawning. + await expect(dm.ensureDaemon({ implicit: true, timeoutMs: 300 })).rejects.toThrow(/Timed out/); + expect(spy).not.toHaveBeenCalled(); + }); }); describe('createHttpDefinition', () => { diff --git a/packages/extension/src/webview/DashboardProvider.ts b/packages/extension/src/webview/DashboardProvider.ts index 98911e4..0f80b0a 100644 --- a/packages/extension/src/webview/DashboardProvider.ts +++ b/packages/extension/src/webview/DashboardProvider.ts @@ -45,11 +45,20 @@ export class DashboardProvider implements vscode.WebviewViewProvider { private _daemonManager?: DaemonManager; private _daemonStarting = false; private _lockfileWatcher?: import('fs').FSWatcher; + /** Coalesces the burst of fs.watch events a single lockfile write emits into one reaction, so + * the cred-push / mcp-sync chain runs once per real change instead of N times per write. */ + private _lockfileDebounce?: ReturnType; private _mcpChanged?: vscode.EventEmitter; + /** Daemon HTTP port VS Code was last provisioned with. Lets us detect an EXTERNAL daemon + * restart (CLI/OS) onto a new ephemeral port and re-provision the MCP definition only then. */ + private _lastMcpPort: number | null | undefined = undefined; setDaemonManager(mgr: DaemonManager): void { this._daemonManager = mgr; void this._initLockfileWatch(); + // A daemon already running at activation (byo/direct-login) needs its creds delivered + // without waiting for the dashboard to open. setAuthManager runs first, so authManager is set. + void this._pushDaemonCredsIfNeeded(); } /** The MCP-definitions change emitter — firing it makes VS Code re-provision the MCP servers, @@ -64,11 +73,148 @@ export class DashboardProvider implements vscode.WebviewViewProvider { const configDir = path.join(os.homedir(), '.airtable-user-mcp'); try { this._lockfileWatcher?.close(); + clearTimeout(this._lockfileDebounce); this._lockfileWatcher = fsMod.watch(configDir, { persistent: false }, (_, filename) => { - if (filename === 'daemon.lock') void this.pushState(); + if (filename === 'daemon.lock') { + // Debounce: one lockfile write fires several fs.watch events, and a restart writes it more + // than once. Coalesce into a single reaction ~250ms after the burst settles so we don't + // stampede the cred-push / mcp-sync chain (which each read daemon status) per raw event. + clearTimeout(this._lockfileDebounce); + this._lockfileDebounce = setTimeout(() => { + void this.pushState(); + // Catch an EXTERNAL daemon restart onto a new port. Our own start/restart/stop handlers + // already fire _mcpChanged, so skip while we are the one (re)starting. An external restart + // also loses the daemon's in-memory creds — re-deliver them FIRST (push-then-sync), so the + // daemon has creds before VS Code reconnects on the (possibly new) port. + if (!this._daemonStarting) { + void (async () => { await this._pushDaemonCredsIfNeeded(); await this._syncMcpDefinition(); })(); + } + }, 250); + } }); this._lockfileWatcher.on('error', () => { /* transient FS errors — ignore */ }); } catch { /* configDir doesn't exist yet — re-initialized after first daemon start */ } + // Seed the last-known port WITHOUT firing, so _syncMcpDefinition only fires on a real change. + try { + const s = await this._daemonManager?.getDaemonStatus(); + this._lastMcpPort = s?.running ? (s.port ?? null) : null; + } catch { /* best-effort */ } + } + + /** + * Re-provision the MCP server definition (registration.ts reads the live lockfile port) IFF the + * daemon's HTTP port changed since VS Code last provisioned it — so an external daemon restart + * onto a new ephemeral port is picked up instead of leaving VS Code on a stale URI. Records the + * current port; best-effort (never throws). + */ + private async _syncMcpDefinition(): Promise { + try { + const s = await this._daemonManager?.getDaemonStatus(); + const port = s?.running ? (s.port ?? null) : null; + if (port !== this._lastMcpPort) { + this._lastMcpPort = port; + this._mcpChanged?.fire(); + } + } catch { /* best-effort */ } + } + + /** Fire _mcpChanged for a KNOWN daemon lifecycle change (our start/restart/stop, authMode change), + * recording the current port so the detect-mode _syncMcpDefinition() does not then re-fire. */ + private async _fireMcpChanged(): Promise { + try { const s = await this._daemonManager?.getDaemonStatus(); this._lastMcpPort = s?.running ? (s.port ?? null) : null; } + catch { /* best-effort — still fire below */ } + this._mcpChanged?.fire(); + } + + /** + * When running the daemon transport with a credential-based auth mode + * (byo / direct-login), push the stored credentials to the running daemon + * over its bearer-authenticated /daemon/auth-credentials endpoint. The daemon + * deliberately gets no creds in its env or on disk — this endpoint is the ONLY + * channel. Fully best-effort: every await is guarded and nothing throws to the + * webview. No secret value is ever logged. + */ + private async _pushDaemonCredsIfNeeded(): Promise { + try { + const settings = getSettings(); + const authMode = settings.mcp.authMode; + // Nothing to push in these cases — treat as success (true). + if (!settings.mcp.useDaemon) return true; + if (authMode !== 'byo' && authMode !== 'direct-login') return true; + + const dm = this._daemonManager; + const am = this.authManager; + if (!dm || !am) return true; + + // Only push to a daemon that is actually up AND healthy. Pushing to a still-starting daemon + // is pointless, and — critically — must never itself cause a spawn: this runs from the + // lockfile watcher, so a spawn here would rewrite the lockfile and re-fire the watcher. + const status = await dm.getDaemonStatus(); + if (!status.running || !status.healthy) return true; + + if (authMode === 'byo') { + const cookie = await am.getCookie(); + if (!cookie) return true; + // CSRF is auto-scraped server-side from the cookie's session — not stored/sent here. + // false ONLY when a real push failed. + return await dm.pushAuthCredentials({ authMode, cookie }); + } else { + // { ensureDaemon: false }: we already confirmed the daemon is healthy above; never let the + // credential read spawn one (that is the runaway-loop trigger). + const creds = await am.getCredentials({ ensureDaemon: false }); + if (!creds) return true; + return await dm.pushAuthCredentials({ + authMode, + email: creds.email, + password: creds.password, + ...(creds.otpSecret ? { totpSecret: creds.otpSecret } : {}), + }); + } + } catch (err) { + // An unexpected throw means delivery could NOT be confirmed — report it as a failure so the + // explicit-action callers (saveCookie/saveCredentials/authMode change) can warn the user. The + // background callers (`void this._pushDaemonCredsIfNeeded()`) ignore the return, so this never + // adds noise there. Log the error (message only — the secret payload is never logged, here or + // in pushAuthCredentials) for diagnosis. + console.warn('[DashboardProvider] credential push threw:', err instanceof Error ? err.message : 'unknown error'); + return false; + } + } + + /** + * After the keychain credentials are cleared in a daemon + byo/direct-login + * setup, the running daemon still holds the injected credentials in memory. + * Restart it so it re-spawns creds-free — with the keychain now empty it has + * nothing to load, so the stale in-memory copy is dropped. Fully best-effort: + * every await is guarded and nothing throws to the webview. + */ + private async _restartDaemonAfterCredClear(): Promise { + try { + const settings = getSettings(); + const authMode = settings.mcp.authMode; + if (!settings.mcp.useDaemon) return; + if (authMode !== 'byo' && authMode !== 'direct-login') return; + const dm = this._daemonManager; + if (!dm) return; + const status = await dm.getDaemonStatus(); + if (!status?.running) return; + try { + await dm.restartDaemon(); + } catch (restartErr) { + // The daemon still holds the (now-cleared) creds in memory. A restart would + // drop them but it failed — force-stop so the stale in-memory session dies. + console.warn('[DashboardProvider] daemon restart after credential clear failed:', restartErr instanceof Error ? restartErr.message : 'unknown error'); + try { + await dm.forceStop(); + vscode.window.showErrorMessage('Credentials were cleared from the keychain, but the running daemon could not be restarted to drop its in-memory session. It was force-stopped — restart it when ready.'); + } catch (stopErr) { + console.warn('[DashboardProvider] daemon force-stop after failed restart also failed:', stopErr instanceof Error ? stopErr.message : 'unknown error'); + vscode.window.showErrorMessage('Credentials cleared, but the daemon may still hold your session in memory — stop/restart it manually to fully clear it.'); + } + } + } catch (err) { + console.warn('[DashboardProvider] daemon restart after credential clear failed:', err instanceof Error ? err.message : 'unknown error'); + } } resolveWebviewView(webviewView: vscode.WebviewView): void { @@ -82,7 +228,13 @@ export class DashboardProvider implements vscode.WebviewViewProvider { // Re-sync when the sidebar is re-opened — daemon/tunnel/auth state may // have changed while the view was hidden and no watcher fired since. webviewView.onDidChangeVisibility(() => { - if (webviewView.visible) void this.pushState(); + if (webviewView.visible) { + void this.pushState(); + // An externally-restarted daemon (CLI/OS) loses its in-memory creds and may be on a new + // port — re-deliver creds FIRST, then re-provision the MCP def if the port changed, so the + // daemon has creds before the definition is re-provisioned. Best-effort. + void (async () => { await this._pushDaemonCredsIfNeeded(); await this._syncMcpDefinition(); })(); + } }); } @@ -92,6 +244,10 @@ export class DashboardProvider implements vscode.WebviewViewProvider { }); if (msg.type === 'ready') { await this.pushState(); + // An externally-restarted daemon (CLI/OS) loses its in-memory creds and may be on a new + // port — re-deliver creds FIRST, then re-provision the MCP def if the port changed, so the + // daemon has creds before the definition is re-provisioned. Best-effort. + void (async () => { await this._pushDaemonCredsIfNeeded(); await this._syncMcpDefinition(); })(); return; } if (msg.type === 'action:refresh') { @@ -155,13 +311,61 @@ export class DashboardProvider implements vscode.WebviewViewProvider { if (msg.type === 'action:saveCredentials') { try { await this.authManager?.saveCredentials(msg.email, msg.password, msg.otpSecret || undefined); + // If in a daemon + byo/direct-login setup, deliver the fresh creds to the + // running daemon over its endpoint (never env/disk). Best-effort. + const pushed = await this._pushDaemonCredsIfNeeded(); await this.pushState(); + if (!pushed) { + vscode.window.showWarningMessage('Credentials saved to the keychain, but the running daemon could not be updated — restart the daemon to apply them.'); + } this.postResult(msg.id, true); } catch (err) { this.postResult(msg.id, false, String(err)); } return; } + + if (msg.type === 'auth:saveCookie') { + try { + // No auth manager wired → nowhere to store the cookie; skip the success message. + if (!this.authManager) return; + // CSRF is auto-scraped server-side from the cookie's authed session, so it's not collected here. + await this.authManager.saveCookie(msg.cookie); + const pushed = await this._pushDaemonCredsIfNeeded(); + await this.pushState(); + if (!pushed) { + // Cookie is safely in the keychain, but the running daemon didn't take it. + vscode.window.showWarningMessage('Cookie saved to the keychain, but the running daemon could not be updated — restart the daemon to apply it.'); + } else { + // Confirm the save — never echo the cookie value. + vscode.window.showInformationMessage('Airtable session cookie saved to the keychain.'); + } + } catch (err) { + // Never surface the cookie — log only a concise, secret-free message + a user-facing error + // (a silent failure would leave the "Not set" chip with no explanation). + const m = err instanceof Error ? err.message : 'unknown error'; + console.warn('[DashboardProvider] saveCookie failed:', m); + vscode.window.showErrorMessage(`Failed to save the Airtable cookie to the keychain: ${m}`); + } + return; + } + if (msg.type === 'auth:clearCookie') { + if (!this.authManager) return; + try { + await this.authManager.clearCookie(); + // Drop the daemon's in-memory copy of the (now-cleared) cookie. + await this._restartDaemonAfterCredClear(); + } catch (err) { + // Never echo the cookie value — surface a concise, secret-free error. + const m = err instanceof Error ? err.message : 'unknown error'; + console.warn('[DashboardProvider] clearCookie failed:', m); + vscode.window.showErrorMessage(`Failed to clear the saved cookie: ${m}`); + } finally { + // Always refresh the UI so the chip reflects the real keychain state. + await this.pushState(); + } + return; + } if (msg.type === 'action:login') { try { await this.authManager?.login(); @@ -181,6 +385,8 @@ export class DashboardProvider implements vscode.WebviewViewProvider { ); if (confirm === 'Logout') { await this.authManager!.logout(); + // Logout clears the keychain — drop the daemon's in-memory creds too. + await this._restartDaemonAfterCredClear(); await this.pushState(); this.postResult(msg.id, true); } else { @@ -436,23 +642,40 @@ export class DashboardProvider implements vscode.WebviewViewProvider { if (msg.key === 'mcp.authMode' || msg.key === 'mcp.httpClient') { const dm = this._daemonManager; if (dm) { - void dm.getDaemonStatus().then(status => { + void dm.getDaemonStatus().then(async status => { if (status?.running) { // Daemon mode: restart so the new AIRTABLE_AUTH_MODE/HTTP_CLIENT env re-injects. - return dm.restartDaemon() - .then(() => this.pushState()) - .catch(err => { - vscode.window.showErrorMessage(`Daemon restart failed: ${err instanceof Error ? err.message : String(err)}`); - }); + // Guard the lockfile watcher against our own restart so it does not double-fire + // (cleared in finally); _fireMcpChanged records the port so the watcher's detect + // path also finds no change. + this._daemonStarting = true; + try { + await dm.restartDaemon(); + // Fresh daemon is creds-free in env by design — deliver byo/direct-login + // creds over the endpoint so the new auth mode has what it needs. + const pushed = await this._pushDaemonCredsIfNeeded(); + if (!pushed) { + void vscode.window.showWarningMessage('Auth mode changed and the daemon restarted, but its credentials could not be updated — restart the daemon to apply them.'); + } + // The restart may have changed the port — re-query the definition provider + // so VS Code rebuilds the HTTP URI from the current daemon.lock. + await this._fireMcpChanged(); + await this.pushState(); + } catch (err) { + vscode.window.showErrorMessage(`Daemon restart failed: ${err instanceof Error ? err.message : String(err)}`); + } finally { + this._daemonStarting = false; + } + return; } // No-daemon (stdio) mode: no daemon to restart — re-provision the MCP servers so VS Code // respawns the stdio server with the fresh env. - this._mcpChanged?.fire(); - return this.pushState(); + await this._fireMcpChanged(); + await this.pushState(); }); } else { // No daemon manager wired → still re-provision for a stdio server. - this._mcpChanged?.fire(); + void this._fireMcpChanged(); } } await this.pushState(); @@ -593,7 +816,12 @@ export class DashboardProvider implements vscode.WebviewViewProvider { // Previously both routed through restartDaemon(), so pressing Start killed a healthy // daemon and respawned it on a new port — the reported "forgets the running process" bug. (msg.type === 'daemon:restart' ? dm.restartDaemon() : dm.ensureDaemon()) - .then(() => { this._daemonStarting = false; void this._initLockfileWatch(); return this.pushState(); }) + // A freshly (re)started daemon is spawned creds-free by design; in byo/direct-login + // mode it needs the keychain creds delivered over its endpoint, or every tool call fails. + // A (re)start with an ephemeral daemonPort (default 0) yields a NEW port — + // fire mcpChanged so VS Code re-queries the definition provider and rebuilds + // the HTTP URI from the current daemon.lock, instead of keeping the stale one. + .then(async () => { this._daemonStarting = false; void this._initLockfileWatch(); await this._pushDaemonCredsIfNeeded(); await this._fireMcpChanged(); return this.pushState(); }) .catch(err => { this._daemonStarting = false; vscode.window.showErrorMessage(`Daemon start failed: ${err instanceof Error ? err.message : String(err)}`); @@ -627,6 +855,8 @@ export class DashboardProvider implements vscode.WebviewViewProvider { // Stopped, but with a caveat (e.g. stale lock cleaned up) — inform, don't alarm. vscode.window.showInformationMessage(`Daemon stopped: ${result.reason}`); } + // Re-query the definition provider so VS Code drops the now-dead HTTP URI. + try { await this._fireMcpChanged(); } catch { /* best-effort */ } this.postResult(msg.id, true); } } catch (err) { @@ -844,7 +1074,8 @@ export class DashboardProvider implements vscode.WebviewViewProvider { // reports stale hasCredentials. Querying directly avoids that race. const baseAuthState: AuthState = this.authManager?.state ?? { status: 'unknown', hasCredentials: false }; const freshHasCredentials = this.authManager ? await this.authManager.hasCredentials() : false; - const authState: AuthState = { ...baseAuthState, hasCredentials: freshHasCredentials }; + const freshHasCookie = this.authManager ? await this.authManager.hasCookie() : false; + const authState: AuthState = { ...baseAuthState, hasCredentials: freshHasCredentials, hasCookie: freshHasCookie }; // Fall back to a 'full' snapshot with every category enabled if the // ToolProfileManager isn't wired yet — keeps the webview render-safe @@ -857,10 +1088,11 @@ export class DashboardProvider implements vscode.WebviewViewProvider { read: true, recordRead: true, tableWrite: true, tableDestructive: true, fieldWrite: true, fieldDestructive: true, - viewWrite: true, viewDestructive: true, - viewSection: true, viewSectionDestructive: true, - formWrite: true, recordWrite: true, - extension: true, + recordDestructive: true, viewWrite: true, + viewDestructive: true, viewSection: true, + viewSectionDestructive: true, formWrite: true, + recordWrite: true, extension: true, + sync: true, }, }; diff --git a/packages/mcp-server/src/auth.js b/packages/mcp-server/src/auth.js index f7ac065..b320572 100644 --- a/packages/mcp-server/src/auth.js +++ b/packages/mcp-server/src/auth.js @@ -114,6 +114,12 @@ export class AirtableAuth { // Past the threshold, mark the session dead and fail fast with a re-login message. this._recoveryStreak = 0; this._sessionDead = false; + // Trip instrumentation: when the circuit-breaker latches (_sessionDead), record + // the triggering HTTP status + a short reason so callers (e.g. the records job) + // can tell a throttle trip (403/429/503) from genuine auth loss (401) or a + // network/page-eval failure afterward. Null while the session is alive. + this._lastTripStatus = null; + this._lastTripReason = null; this._sessionDeadAfter = Number(process.env.AIRTABLE_SESSION_DEAD_AFTER) || 8; this._rlSleep = (ms) => new Promise((r) => setTimeout(r, ms)); // injectable for tests } @@ -122,6 +128,22 @@ export class AirtableAuth { resetSessionHealth() { this._sessionDead = false; this._recoveryStreak = 0; + this._lastTripStatus = null; + this._lastTripReason = null; + } + + /** True once the circuit-breaker has latched (every subsequent _apiCall short-circuits). */ + isSessionDead() { + return this._sessionDead; + } + + /** + * The status/reason that latched the dead-session breaker, or null while alive. + * Lets callers distinguish a throttle trip (403/429/503) from genuine auth loss + * (401) or a network/page-eval failure after the fact. + */ + getLastTrip() { + return this._sessionDead ? { status: this._lastTripStatus, reason: this._lastTripReason } : null; } // ─── Initialization ────────────────────────────────────────── @@ -479,7 +501,12 @@ export class AirtableAuth { console.error('[auth] Session verified!', this.userId ? `User: ${this.userId}` : '(userId not in payload)'); } else { this.isLoggedIn = false; - const sessionError = `Session invalid (${result.status}). Run "node src/login.js" to log in first.\nResponse: ${result.body?.substring(0, 200)}`; + const hint = this._authMode === 'byo' + ? 'the provided cookie was rejected — save a fresh Airtable session cookie (byo recovers only from a re-pasted cookie)' + : this._authMode === 'direct-login' + ? 'direct-login credentials were rejected — check AIRTABLE_EMAIL / AIRTABLE_PASSWORD / AIRTABLE_TOTP_SECRET (direct-login re-authenticates itself on the next call)' + : 'the Airtable login has expired — re-authenticate (open the extension dashboard and log in), or use AIRTABLE_AUTH_MODE=direct-login so long unattended jobs re-authenticate themselves'; + const sessionError = `Session invalid (${result.status}): ${hint}.\nResponse: ${result.body?.substring(0, 200)}`; trace('auth', 'auth:session_check', { success: false }, sessionError); throw new Error(sessionError); } @@ -685,9 +712,16 @@ export class AirtableAuth { const MAX_RETRIES = 3; const HARD_TIMEOUT_MS = 30_000; + // Browser-free modes (direct-login / byo): an expired session or a rotated + // CSRF token surfaces on mutations as 403, not 401. There _recoverSession is + // a single HTTP _directLogin / cookie reload — NO page-load burst — so + // escalating a PERSISTENT 403 to re-auth is safe (unlike browser mode, where + // a relaunch fires a burst that re-trips the ~5 req/s limit → 403 storm). + const browserFree = this._authMode === 'direct-login' || this._authMode === 'byo'; let deadline = Date.now() + HARD_TIMEOUT_MS; // extended by intentional rate-limit backoff waits let attempt = 0; // browser-recovery (relaunch) attempts let rlAttempt = 0; // rate-limit/throttle backoff attempts (same session, no relaunch) + let http403Backoffs = 0; // 403s already given a throttle backoff this call (browser-free escalation gate) while (true) { if (Date.now() > deadline) { // Bail out rather than retry past the caller's patience budget. @@ -702,6 +736,8 @@ export class AirtableAuth { if (!this._recovering && attempt < MAX_RETRIES) { if (++this._recoveryStreak >= this._sessionDeadAfter) { this._sessionDead = true; + this._lastTripStatus = 0; // no HTTP status — request never completed + this._lastTripReason = 'network/page-eval'; throw new Error(`SESSION_INVALID: ${this._recoveryStreak} consecutive failures across recoveries (page.evaluate) — re-authenticate (run \`login\`) and retry.`); } console.error(`[auth] page.evaluate failed: ${evalError.message}. Recovering... (streak ${this._recoveryStreak})`); @@ -720,8 +756,31 @@ export class AirtableAuth { // the ~30s official penalty; intentional waits don't count against the work budget; the // dead-session circuit-breaker still bounds a throttle that never clears. if (result.status === 429 || result.status === 503 || result.status === 403) { + // Browser-free 403 escalation: 429/503 (and the FIRST 403, and ALL 403s + // in browser mode) stay pure throttle. But a 403 that PERSISTS past its + // one allowed throttle backoff in direct-login/byo mode is a dead/rotated + // session, not throttle — escalate to re-auth like the 401 path below, + // bounded by the same attempt cap + single-flight guard. It still counts + // toward _recoveryStreak so an unrecoverable 403 (IP/account block whose + // _directLogin also 403s) latches _sessionDead instead of looping forever. + const persist403 = result.status === 403 && browserFree && http403Backoffs >= 1; + if (persist403 && !this._recovering && attempt < MAX_RETRIES) { + if (++this._recoveryStreak >= this._sessionDeadAfter) { + this._sessionDead = true; + this._lastTripStatus = result.status; + this._lastTripReason = 'auth-403-persist'; + throw new Error(`SESSION_INVALID: ${this._recoveryStreak} consecutive 403s that did not clear after re-authentication — the login has expired or been blocked. Re-authenticate (run \`login\`) and retry.`); + } + console.error(`[auth] 403 persisted after backoff (${this._authMode}) — re-authenticating on the same session (streak ${this._recoveryStreak})...`); + await this._recoverSession(); + attempt++; + continue; + } + if (++this._recoveryStreak >= this._sessionDeadAfter) { this._sessionDead = true; + this._lastTripStatus = result.status; + this._lastTripReason = 'throttle'; throw new Error(`SESSION_INVALID: ${this._recoveryStreak} consecutive rate-limit/throttle responses (status=${result.status}) that did not clear after backoff — the account is throttled or the login is blocked. Let the limit reset (wait), or re-authenticate (run \`login\`), then retry.`); } const delay = Math.min(30_000, 1000 * (2 ** rlAttempt)) + Math.floor(Math.random() * 500); @@ -729,6 +788,7 @@ export class AirtableAuth { await this._rlSleep(delay); deadline += delay; // an intentional backoff wait is not a stall — don't let it trip the budget rlAttempt++; + if (result.status === 403) http403Backoffs++; // this 403 has now had its one throttle backoff continue; } @@ -737,6 +797,8 @@ export class AirtableAuth { if (needsRecovery && !this._recovering && attempt < MAX_RETRIES) { if (++this._recoveryStreak >= this._sessionDeadAfter) { this._sessionDead = true; + this._lastTripStatus = result.status; + this._lastTripReason = result.error ? 'network' : 'auth-401'; throw new Error(`SESSION_INVALID: ${this._recoveryStreak} consecutive auth failures (status=${result.status}) across recoveries — the Airtable login has expired or been blocked. Re-authenticate (run \`login\`) and retry.`); } console.error(`[auth] API call failed (status=${result.status}, error=${result.error || 'none'}). Recovering... (streak ${this._recoveryStreak})`); @@ -817,6 +879,35 @@ export class AirtableAuth { } } + /** + * Records-facing health gate. Probe the session via checkSessionHealth(); if + * it's already valid, do nothing. In a browser-free mode (direct-login / byo) + * a rejected probe is re-authenticated in place (_recoverSession is a single + * HTTP login/cookie reload) and re-probed once. In browser mode we leave + * recovery to the existing _apiCall wrapper and just report unhealthy. + * + * Never throws — degrades to { healthy:false } on any error so a long records + * job can decide whether to continue or abort without a try/catch at the call + * site. + * + * @returns {Promise<{healthy: boolean, recovered: boolean, error?: string}>} + */ + async ensureSessionHealthy() { + try { + const probe = await this.checkSessionHealth(); + if (probe && probe.valid) return { healthy: true, recovered: false }; + + const browserFree = this._authMode === 'direct-login' || this._authMode === 'byo'; + if (!browserFree) return { healthy: false, recovered: false }; + + await this._recoverSession(); + const reprobe = await this.checkSessionHealth(); + return { healthy: !!(reprobe && reprobe.valid), recovered: true }; + } catch (err) { + return { healthy: false, recovered: false, error: err instanceof Error ? err.message : String(err) }; + } + } + async get(url, appId, { evalTimeoutMs = 15_000 } = {}) { const pattern = url.replace(/.*v0\.3\//, '').replace(/(app|tbl|viw|fld|rec|usr|wsp|sel|flt|blk|ext|col)[A-Za-z0-9]{10,}/g, '$1*'); trace('http', 'http:request', { method: 'GET', endpoint_pattern: pattern, has_payload: false }); diff --git a/packages/mcp-server/src/daemon/server.js b/packages/mcp-server/src/daemon/server.js index 2360513..3a221fa 100644 --- a/packages/mcp-server/src/daemon/server.js +++ b/packages/mcp-server/src/daemon/server.js @@ -57,12 +57,19 @@ const FETCH_BLOCKED_PORTS = new Set([ 6697, 10080, ]); -async function listenAvoidingBlockedPorts(server, requestedPort, host) { - // A fixed port (requestedPort > 0) gets one shot; if it's already in use or - // turns out to be a browser-blocked port, we fall back to an OS-assigned - // ephemeral port so the daemon always starts. The bound port is read back and - // persisted to the lockfile, so clients still discover it. With requestedPort - // 0 we simply retry on ephemeral ports until one is not browser-blocked. +// Bind failures on a NON-ZERO fixed port that should degrade to an ephemeral port instead of +// crashing the daemon: the port is already in use (EADDRINUSE), sits in a Windows reserved/excluded +// port range or is otherwise permission-denied (EACCES — the "listen EACCES 127.0.0.1:8723" case +// that killed every startup when the default port landed in an excluded range), or is not assignable +// (EADDRNOTAVAIL). Any of these on a fixed port → retry on port 0. +const FIXED_PORT_FALLBACK_CODES = new Set(['EADDRINUSE', 'EACCES', 'EADDRNOTAVAIL']); + +export async function listenAvoidingBlockedPorts(server, requestedPort, host) { + // A fixed port (requestedPort > 0) gets one shot; if it's unavailable (in use, excluded/denied, + // not assignable) or turns out to be a browser-blocked port, we fall back to an OS-assigned + // ephemeral port so the daemon always starts. The bound port is read back and persisted to the + // lockfile, so clients still discover it. With requestedPort 0 we simply retry on ephemeral ports + // until one is not browser-blocked. let port = requestedPort; for (let attempt = 0; attempt < 5; attempt++) { try { @@ -80,8 +87,10 @@ async function listenAvoidingBlockedPorts(server, requestedPort, host) { server.listen(port, host); }); } catch (error) { - if (error?.code === 'EADDRINUSE' && port !== 0) { - port = 0; // fixed port taken → fall back to an ephemeral port + if (port !== 0 && FIXED_PORT_FALLBACK_CODES.has(error?.code)) { + // fixed port unavailable (taken/excluded/denied) → fall back to an ephemeral port + console.error(`[airtable-user-mcp] fixed daemon port ${port} unavailable (${error.code}) — falling back to an automatic port.`); + port = 0; continue; } throw error; diff --git a/packages/mcp-server/src/direct-login.js b/packages/mcp-server/src/direct-login.js index b175e89..b256737 100644 --- a/packages/mcp-server/src/direct-login.js +++ b/packages/mcp-server/src/direct-login.js @@ -10,10 +10,11 @@ * → re-scrape csrfToken from an authed GET / (the login session rotates the * csrfSecret, so the /login token is stale for subsequent API calls). * - * Transport is impit (Chrome TLS impersonation) to resist a PerimeterX - * challenge on the login endpoints; both impit and otpauth are OPTIONAL - * dependencies, lazy-loaded, and injectable (impitFactory / otpFactory) for - * tests so NO live Airtable call or real crypto runs under `node --test`. + * Transport DEFAULTS to Node's built-in `fetch` (no dependency), so direct-login works out of the + * box. impit (Chrome TLS impersonation) is used ONLY when AIRTABLE_HTTP_CLIENT=impit — for the + * rare account whose login endpoints challenge a non-browser TLS fingerprint (PerimeterX). impit + * and otpauth are OPTIONAL deps, lazy-loaded, and injectable (impitFactory / otpFactory) for tests + * so NO live Airtable call or real crypto runs under `node --test`. * * Returns { cookieHeader, csrfToken } — the exact shape auth._credentials wants. */ @@ -61,12 +62,28 @@ async function loadOtpAuth() { } } -async function makeImpit(impitFactory) { +/** + * Build the HTTP client for the login flow. + * + * Default = Node's built-in `fetch` (WHATWG) — NO optional dependency needed, so direct-login + * works out of the box. The cookie-jar/Location helpers below already speak the WHATWG Response + * shape (getSetCookie(), headers.get(...)). We drive the exact login sequence ourselves, so the + * fetch client is created with `redirect: 'manual'` per request. + * + * impit (Chrome TLS impersonation) is used ONLY when explicitly requested via + * AIRTABLE_HTTP_CLIENT=impit — for accounts where Airtable's login endpoints challenge a + * non-browser TLS fingerprint (PerimeterX). If requested but not installed, loadImpit throws a + * clear DIRECT_LOGIN_IMPIT_MISSING. `impitFactory` (tests) always wins. + */ +async function makeHttpClient(impitFactory) { if (impitFactory) return impitFactory(); - const Impit = await loadImpit(); - // followRedirects:false — we drive the exact login sequence and read Location - // ourselves rather than let the client chase the browser's redirects. - return new Impit({ browser: 'chrome', followRedirects: false }); + if ((process.env.AIRTABLE_HTTP_CLIENT || '').toLowerCase() === 'impit') { + const Impit = await loadImpit(); + // followRedirects:false — we read Location ourselves rather than chase redirects. + return new Impit({ browser: 'chrome', followRedirects: false }); + } + // Node fetch client — redirect:'manual' so a 302 is returned, not followed. + return { fetch: (url, options) => fetch(url, { ...options, redirect: 'manual' }) }; } async function generateTotp(secret, otpFactory) { @@ -201,7 +218,7 @@ function getHeader(res, name) { * @param {Map} opts.jar * @param {object} [opts.form] form-urlencoded body fields */ -async function request(impit, method, urlPath, { jar, form } = {}) { +async function request(client, method, urlPath, { jar, form } = {}) { const url = urlPath.startsWith('http') ? urlPath : BASE + urlPath; const headers = {}; const cookie = jarToCookieHeader(jar); @@ -211,7 +228,7 @@ async function request(impit, method, urlPath, { jar, form } = {}) { headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8'; options.body = new URLSearchParams(form).toString(); } - const res = await impit.fetch(url, options); + const res = await client.fetch(url, options); updateJar(jar, res); return res; } @@ -284,18 +301,18 @@ function indicatesSso(body) { */ export async function directLogin({ email, password, totpSecret, impitFactory, otpFactory } = {}) { const creds = resolveCredentials({ email, password, totpSecret }); - const impit = await makeImpit(impitFactory); + const client = await makeHttpClient(impitFactory); const jar = new Map(); // 1. GET /login — scrape the csrfToken, collect the initial cookies (brw, …). - const loginPage = await request(impit, 'GET', '/login', { jar }); + const loginPage = await request(client, 'GET', '/login', { jar }); let csrf = scrapeCsrf(await loginPage.text()); if (!csrf) { throw new Error('DIRECT_LOGIN_NO_CSRF: could not scrape a csrfToken from GET /login — Airtable may have changed the login page.'); } // 2. getLoginTypeForEmail — bail early on SSO/enterprise accounts. - const typeRes = await request(impit, 'POST', '/auth/getLoginTypeForEmail', { + const typeRes = await request(client, 'POST', '/auth/getLoginTypeForEmail', { jar, form: { _csrf: csrf, @@ -311,7 +328,7 @@ export async function directLogin({ email, password, totpSecret, impitFactory, o } // 3. POST /auth/login/ — expect a 302 (to /2fa/… on a 2FA account, else /). - const loginRes = await request(impit, 'POST', '/auth/login/', { + const loginRes = await request(client, 'POST', '/auth/login/', { jar, form: { _csrf: csrf, urlToRedirectTo: '/', email: creds.email, password: creds.password }, }); @@ -331,7 +348,7 @@ export async function directLogin({ email, password, totpSecret, impitFactory, o throw new Error('DIRECT_LOGIN_2FA_REQUIRED: set AIRTABLE_TOTP_SECRET (base32) for this account'); } const code = await generateTotp(creds.totpSecret, otpFactory); - const verifyRes = await request(impit, 'POST', '/auth/verify2faCode', { + const verifyRes = await request(client, 'POST', '/auth/verify2faCode', { jar, form: { _csrf: csrf, twoFactorStrategyId, code }, }); @@ -349,7 +366,7 @@ export async function directLogin({ email, password, totpSecret, impitFactory, o // session's csrfSecret, so the /login token is stale for API calls). Best- // effort: keep the login-flow token if the re-scrape fails. try { - const home = await request(impit, 'GET', '/', { jar }); + const home = await request(client, 'GET', '/', { jar }); const fresh = scrapeCsrf(await home.text()); if (fresh) csrf = fresh; } catch { /* keep the existing csrf */ } diff --git a/packages/mcp-server/src/index.js b/packages/mcp-server/src/index.js index a5a4844..1c75ca9 100644 --- a/packages/mcp-server/src/index.js +++ b/packages/mcp-server/src/index.js @@ -1606,7 +1606,7 @@ Note: "form title" is the view name itself — use rename_view to change it. "Fi // ── Sync Tools ── { name: 'sync_base', - description: 'Base-to-base schema + record sync. IMPORTANT: BOTH mode="plan" and mode="apply" run as BACKGROUND JOBS — they return {jobId, planId, status:"running"} IMMEDIATELY (jobId === planId), NOT a synchronous result; poll mode="status" with that planId to get progress and the final result. mode="plan" (read-only, does NOT mutate): computes an ordered plan (tables/fields to create/update + orphans + warnings) by comparing source/dest schema; when the job finishes, mode="status" returns the plan digest in planDigest. mode="apply" (requires planId from a prior plan): executes the saved plan against the destination — creates tables, reconciles the primary, creates scalar/link/computed fields (source->dest reference remapping + formula validation), applies non-destructive field updates, then runs the RECORD sync (two-pass cells + links, attachments, view-filter restore); aborts if the destination drifted since the plan. mode="status" (poll a plan OR apply job by its planId): returns { phase: "planning"|"schema"|"records"|"done"|"failed", status, recordsMapped, planDigest?, schemaResult?, recordsResult? }. Field-mapping errors, APPLY_LOCKED, and DRIFT surface HERE (phase="failed" or an aborted schemaResult), not as a synchronous error. mode="reconcile" (SYNCHRONOUS): rebuild/repair the record map — existence-prune dead idmap entries, optional natural-key re-match per table. mode="diff" (SYNCHRONOUS): schema digest comparing source and destination WITHOUT saving a plan; returns a diffId and summary; pass detail=
to drill into a section of a prior diff.', + description: 'Base-to-base schema + record sync. IMPORTANT: BOTH mode="plan" and mode="apply" run as BACKGROUND JOBS — they return {jobId, planId, status:"running"} IMMEDIATELY (jobId === planId), NOT a synchronous result; poll mode="status" with that planId to get progress and the final result. mode="plan" (read-only, does NOT mutate): computes an ordered plan (tables/fields to create/update + orphans + warnings) by comparing source/dest schema; when the job finishes, mode="status" returns the plan digest in planDigest. mode="apply" (requires planId from a prior plan): executes the saved plan against the destination — creates tables, reconciles the primary, creates scalar/link/computed fields (source->dest reference remapping + formula validation), applies non-destructive field updates, then runs the RECORD sync (two-pass cells + links, attachments, view-filter restore); aborts if the destination drifted since the plan. mode="status" (poll a plan OR apply job by its planId): returns { phase: "planning"|"schema"|"records"|"done"|"failed", status, recordsMapped, summary, schemaResult?, recordsResult?, result?, planDigest? }. planDigest is human-only by default ({ human, machineOmitted: true }) — pass verbose:true for the full planDigest.machine. Field-mapping errors, APPLY_LOCKED, and DRIFT surface HERE (phase="failed" or an aborted schemaResult), not as a synchronous error. mode="reconcile" (SYNCHRONOUS): rebuild/repair the record map — existence-prune dead idmap entries, optional natural-key re-match per table. mode="diff" (SYNCHRONOUS): schema digest comparing source and destination WITHOUT saving a plan; returns a diffId and summary; pass detail=
to drill into a section of a prior diff.', annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false }, inputSchema: { type: 'object', @@ -1628,6 +1628,7 @@ Note: "form title" is the view name itself — use rename_view to change it. "Fi confirmTableDeletions: { type: 'boolean', description: 'Used by mode=apply with policy=mirror: required to delete whole dest-only TABLES. confirmDeletions alone never drops a table; without confirmTableDeletions, orphan tables report TABLE_DELETION_GATED and are kept.' }, confirmRetypes: { type: 'boolean', description: 'Used by mode=apply: required to apply a matched field\'s SCALAR type change to the destination (source-wins). Without it, a diverging scalar type reports RETYPE_GATED and the field is kept. Non-scalar retypes (to/from formula/rollup/lookup/count/autoNumber/link/attachment) stay RETYPE_DEFERRED.' }, fieldMappings: { type: 'object', description: 'Field mapping overrides: maps table name → { sourceField: destField } to inject a source field\'s value into a different (writable scalar) dest field during record sync. Example: { "Games": { "Title": "Name" } }. In mode="plan" and mode="diff", fieldMappings are validated against the two schemas (dry-run, no mutation) and errors are returned in fieldMappingErrors.', additionalProperties: { type: 'object', additionalProperties: { type: 'string' } } }, + verbose: { type: 'boolean', description: 'Used only by mode="status": by default planDigest is projected to { human, machineOmitted: true } (the full machine payload can be tens of thousands of lines on a view-heavy base and pushes the useful result fields out of the response — it always stays on disk). Set verbose:true to get the full planDigest.machine back in the response.' }, debug: debugProp, }, required: ['mode', 'sourceAppId', 'destAppId'], @@ -2434,7 +2435,7 @@ const handlers = { // ── Sync ── - async sync_base({ mode, sourceAppId, destAppId, planId, naturalKeys, detail, diffId, offset, limit, direction, skip, policy, policyOverrides, confirmDeletions, confirmTableDeletions, confirmRetypes, fieldMappings, debug }) { + async sync_base({ mode, sourceAppId, destAppId, planId, naturalKeys, detail, diffId, offset, limit, direction, skip, policy, policyOverrides, confirmDeletions, confirmTableDeletions, confirmRetypes, fieldMappings, verbose, debug }) { const sync = await import('./sync/index.js'); if (mode === 'plan') { // Plan snapshots BOTH bases (a getView/readData per view — minutes on a view-heavy base), @@ -2470,7 +2471,7 @@ const handlers = { } if (mode === 'status') { if (!planId) return err('mode="status" requires planId (the jobId returned by mode="plan" or mode="apply").'); - const s = sync.syncStatus({ sourceBaseId: sourceAppId, destBaseId: destAppId, planId }); + const s = sync.syncStatus({ sourceBaseId: sourceAppId, destBaseId: destAppId, planId, verbose }); const sr = s.schemaResult || {}; const rr = s.recordsResult || {}; let summary; @@ -2504,16 +2505,20 @@ const handlers = { } // Structured object covering the phase (superset of the M2 records-only shape: planId, // status, recordsMapped, result, summary are preserved; result === recordsResult). + // Field order matters here: `summary`/schemaResult/recordsResult/result are the fields a + // caller actually needs and are placed BEFORE planDigest, so that if the response is still + // truncated (e.g. a verbose planDigest.machine on a view-heavy base), it's the least-useful + // field that gets cut, not these. (Object insertion order === JSON key order.) return ok({ planId, phase: s.phase, status: s.status, recordsMapped: s.recordsMapped, - planDigest: s.planDigest ?? null, + summary, schemaResult: s.schemaResult ?? null, recordsResult: s.recordsResult ?? null, result: s.recordsResult ?? null, - summary, + planDigest: s.planDigest ?? null, }, s, debug); } if (mode === 'diff') { diff --git a/packages/mcp-server/src/sync/apply.js b/packages/mcp-server/src/sync/apply.js index 4565fea..fa9b9e5 100644 --- a/packages/mcp-server/src/sync/apply.js +++ b/packages/mcp-server/src/sync/apply.js @@ -9,6 +9,11 @@ const UNSUPPORTED_TYPES = new Set(['button', 'asyncText', 'aiText', 'externalSyn const VIEW_GROUP_ANCHOR = new Set(['select', 'singleSelect', 'multiSelect', 'multipleSelects', 'collaborator']); const COMPUTED_TYPES = new Set(['formula', 'rollup', 'lookup', 'multipleLookupValues', 'count']); const LINK_TYPES = new Set(['foreignKey', 'multipleRecordLinks']); +// Airtable 422s a computed (lookup/rollup/...) field's updateField when it runs in the SAME +// apply pass as the link field it depends on — the link change hasn't settled yet. Matched +// defensively (message wording, not a status code) so only THIS specific rejection defers; +// every other thrown error still hard-fails via ACTION_FAILED. +const LINK_DEPENDENCY_ERROR = /requires a link field|link field was changed/i; const SCALAR_RETYPE_TYPES = new Set(['text', 'multilineText', 'richText', 'number', 'currency', 'percent', 'rating', 'duration', 'checkbox', 'date', 'dateTime', 'phone', 'email', 'url', 'select', 'singleSelect', 'multiSelect', 'multipleSelects']); // Types Airtable refuses as a primary field — keep a placeholder + warn instead of @@ -417,8 +422,20 @@ async function applyAction({ client, destAppId, a, idmap, index, state, result, result.warnings.push({ code: 'UNRESOLVABLE_REF', message: `Update of "${a.destFld}" typeOptions deferred — refs [${unresolved.join(', ')}] not yet created` }); return 'deferred'; } else { - await client.updateFieldConfig(destAppId, a.destFld, { type: destType, typeOptions: toWritableComputedOptions(destType, remapped) }); - mutated = true; + try { + await client.updateFieldConfig(destAppId, a.destFld, { type: destType, typeOptions: toWritableComputedOptions(destType, remapped) }); + mutated = true; + } catch (e) { + // Same-pass link dependency not settled yet — defer instead of hard-failing so the + // existing retry-until-stable loop (applyPlan) re-attempts it after the link update + // (and Airtable's async settle) has had a chance to complete. Any other error keeps + // the normal ACTION_FAILED path (rethrow). + if (LINK_DEPENDENCY_ERROR.test(String((e && e.message) ?? e ?? ''))) { + result.warnings.push({ code: 'LINK_DEPENDENCY_DEFERRED', message: `Update of "${a.destFld}" deferred — depends on a link field still settling: ${e.message ?? e}` }); + return 'deferred'; + } + throw e; + } } } else if (LINK_TYPES.has(destType)) { // Genuine link divergence (linkSig differs): send the WRITABLE link shape — same diff --git a/packages/mcp-server/src/sync/index.js b/packages/mcp-server/src/sync/index.js index 1387a87..726a687 100644 --- a/packages/mcp-server/src/sync/index.js +++ b/packages/mcp-server/src/sync/index.js @@ -226,18 +226,20 @@ export async function apply({ client, sourceBaseId, destBaseId, planId, runStart if (onPhase) onPhase('records-start', { schemaResult: renderApplyResult(result).machine }); applyRecordsImpl({ client, sourceBaseId, destBaseId, planId, runStartedAt, policy, policyOverrides, confirmDeletions, fieldMappings, naturalKeys }) .then((r) => { - const recordsResult = { - created: r.created, updated: r.updated, failed: r.failed, - matched: r.matched || 0, - attachmentsUploaded: r.attachmentsUploaded || 0, viewFiltersReapplied: r.viewFiltersReapplied || 0, - deleted: r.deleted || 0, - warningCount: (r.warnings || []).length, - warnings: r.warnings || [], - }; + // A records run that ABORTED on a mid-run session death resolves normally (r.aborted=true) + // but must surface as phase=failed — NOT a lying `done` over 1000s of swallowed + // SESSION_INVALID per-record failures. recordsTerminalStatus makes that decision (and keeps + // the partial counts either way). See records.js sessionDied/markAborted. + const t = recordsTerminalStatus(r); writeRecordsJobStatus(sourceBaseId, destBaseId, planId, { - status: 'done', startedAt: runStartedAt, finishedAt: new Date().toISOString(), result: recordsResult, + status: t.status, startedAt: runStartedAt, finishedAt: new Date().toISOString(), + ...(t.error ? { error: t.error } : {}), result: t.recordsResult, }); - if (onPhase) onPhase('records-done', { recordsResult }); + if (onPhase) { + onPhase(t.event, t.event === 'records-failed' + ? { error: t.error, recordsResult: t.recordsResult } + : { recordsResult: t.recordsResult }); + } }) .catch((err) => { const error = String(err && err.message ? err.message : err); @@ -271,6 +273,17 @@ export async function apply({ client, sourceBaseId, destBaseId, planId, runStart * @param {{ client: object, sourceBaseId: string, destBaseId: string, planId: string, direction?: string, fieldMappings?: object }} opts * @returns {{ jobId: string, status: 'running' }} */ +// The plan's machine output embeds the full idmap (tables/fields/views/choices) — multiple MB on a +// large base (observed 2.7MB of a 5.4MB sync-job file). The sync-job status file is rewritten on +// every phase transition + read on every mode=status poll, so storing the full digest is wasteful. +// Strip the idmap for the STATUS digest — it is persisted verbatim in idmap.json and plan-.json +// for anyone who needs the full mapping. Keeps verdict/counts/sample/orphans/warnings. +function compactPlanDigest(machine) { + if (!machine || typeof machine !== 'object') return machine; + const { idmap, ...rest } = machine; + return idmap ? { ...rest, idmapOmitted: true } : rest; +} + export function planJob({ client, sourceBaseId, destBaseId, planId, direction, fieldMappings }) { const startedAt = new Date().toISOString(); writeSyncJobStatus(sourceBaseId, destBaseId, planId, { phase: 'planning', startedAt }); @@ -281,7 +294,7 @@ export function planJob({ client, sourceBaseId, destBaseId, planId, direction, f .then((rendered) => { writeSyncJobStatus(sourceBaseId, destBaseId, planId, { phase: 'done', - planDigest: { human: rendered.human, machine: rendered.machine }, + planDigest: { human: rendered.human, machine: compactPlanDigest(rendered.machine) }, finishedAt: new Date().toISOString(), }); }) @@ -320,7 +333,13 @@ export function applyJob({ client, sourceBaseId, destBaseId, planId, runStartedA } else if (event === 'records-done') { writeSyncJobStatus(sourceBaseId, destBaseId, planId, { phase: 'done', recordsResult: payload.recordsResult, finishedAt: new Date().toISOString() }); } else if (event === 'records-failed') { - writeSyncJobStatus(sourceBaseId, destBaseId, planId, { phase: 'failed', error: payload.error, finishedAt: new Date().toISOString() }); + // Include the partial recordsResult (present on a mid-run session abort) so mode=status shows + // what landed before the failure — not just the error string. + writeSyncJobStatus(sourceBaseId, destBaseId, planId, { + phase: 'failed', error: payload.error, + ...(payload.recordsResult !== undefined ? { recordsResult: payload.recordsResult } : {}), + finishedAt: new Date().toISOString(), + }); } }; @@ -352,16 +371,33 @@ export function applyJob({ client, sourceBaseId, destBaseId, planId, runStartedA return { jobId: planId, status: 'running' }; } +// The saved planDigest.machine (already idmap-stripped by compactPlanDigest) can still run to +// tens of thousands of lines on a view-heavy base — big enough to push summary/schemaResult/ +// recordsResult past the MCP response's truncation window. mode=status is polled far more often +// than anyone actually needs the raw machine payload (which stays on disk in plan-.json +// verbatim), so project it down to the human summary by default; verbose:true opts back into the +// full machine. This only affects the RESPONSE shape — the job FILE on disk is never touched here. +function projectPlanDigestForResponse(planDigest, verbose) { + if (planDigest === undefined || planDigest === null) return planDigest; + if (verbose) return planDigest; + return { human: planDigest.human, machineOmitted: true }; +} + /** * Poll a background plan/apply job by planId. Reads the unified sync-job file (+ pid-liveness) and * derives live progress (records mapped so far) from idmap.records. Falls back to the records-job * file when no unified sync-job file exists (e.g. an apply that ran before this file existed). * Supersedes recordsStatus() for the tool handler; recordsStatus() is kept for existing callers. * - * @param {{ sourceBaseId: string, destBaseId: string, planId: string }} opts + * By default the returned planDigest is projected to `{ human, machineOmitted: true }` — the full + * `machine` payload (which can be tens of thousands of lines on a view-heavy base) is omitted from + * the RESPONSE only; it remains persisted verbatim on disk (plan-.json). Pass verbose:true + * to get the full planDigest (including machine) back in the response. + * + * @param {{ sourceBaseId: string, destBaseId: string, planId: string, verbose?: boolean }} opts * @returns {{ planId:string, phase:string, status:'running'|'done'|'failed'|'unknown', recordsMapped:number, planDigest?:object, schemaResult?:object, recordsResult?:object, startedAt?:string, finishedAt?:string, error?:string, message?:string }} */ -export function syncStatus({ sourceBaseId, destBaseId, planId }) { +export function syncStatus({ sourceBaseId, destBaseId, planId, verbose = false }) { const idmap = loadIdmap(sourceBaseId, destBaseId); const recordsMapped = Object.keys((idmap && idmap.records) || {}).length; @@ -388,7 +424,7 @@ export function syncStatus({ sourceBaseId, destBaseId, planId }) { phase: job.phase, status, recordsMapped, - ...(job.planDigest !== undefined ? { planDigest: job.planDigest } : {}), + ...(job.planDigest !== undefined ? { planDigest: projectPlanDigestForResponse(job.planDigest, verbose) } : {}), ...(job.schemaResult !== undefined ? { schemaResult: job.schemaResult } : {}), ...(job.recordsResult !== undefined ? { recordsResult: job.recordsResult } : {}), ...(job.startedAt !== undefined ? { startedAt: job.startedAt } : {}), @@ -397,6 +433,34 @@ export function syncStatus({ sourceBaseId, destBaseId, planId }) { }; } +/** + * Decide the terminal status of a finished records run (pure — no I/O, unit-testable). + * + * A records run that hit a mid-run session death resolves normally with `r.aborted === true` + * (records.js sets it via markAborted). That must become a records-job phase=failed — NOT a + * `done` that hides thousands of swallowed SESSION_INVALID failures — while still carrying the + * partial counts so the user sees what landed before the abort. A non-aborted run stays `done`. + * + * @param {object} r - records result accumulator ({ aborted?, abortReason?, created, updated, failed, ... }) + * @returns {{ status:'done'|'failed', event:'records-done'|'records-failed', error?:string, recordsResult:object }} + */ +export function recordsTerminalStatus(r) { + const recordsResult = { + created: r.created, updated: r.updated, failed: r.failed, + matched: r.matched || 0, + attachmentsUploaded: r.attachmentsUploaded || 0, viewFiltersReapplied: r.viewFiltersReapplied || 0, + deleted: r.deleted || 0, + warningCount: (r.warnings || []).length, + warnings: r.warnings || [], + }; + if (r.aborted) { + const error = r.abortReason + || 'SESSION_INVALID: record sync aborted — the auth session died mid-run. Re-authenticate, then re-run mode=apply with the same planId to resume (progress is persisted).'; + return { status: 'failed', event: 'records-failed', error, recordsResult }; + } + return { status: 'done', event: 'records-done', recordsResult }; +} + /** * @deprecated Use `syncStatus()` instead — it reports the full plan→schema→records phase timeline * and already falls back to the legacy `records-job-.json` file this reads. Retained only diff --git a/packages/mcp-server/src/sync/records.js b/packages/mcp-server/src/sync/records.js index af2c96a..be37b99 100644 --- a/packages/mcp-server/src/sync/records.js +++ b/packages/mcp-server/src/sync/records.js @@ -96,6 +96,48 @@ export function readRecordsJobStatus(sourceBaseId, destBaseId, planId) { */ const makeGate = (limiter) => (fn) => limiter.run(() => withRetry(fn)); +// ── Session-death detection (Task 2) ────────────────────────────────────────── +// The records phase runs 1000s of per-row writes. When the auth circuit-breaker latches +// (a browser-free re-auth kept failing, or a throttle 403 never cleared), EVERY subsequent +// per-row POST short-circuits to SESSION_INVALID — but the client SWALLOWS each into its +// failed[] array (it does not throw at the batch level), so the naive engine marches through +// all rows as per-record failures and then reports the job `done`. These helpers let each +// write pass detect a fatal, run-wide session death and abort the whole job instead. + +/** + * True when the session is dead run-wide: the breaker has latched (isSessionDead) OR a + * SESSION_INVALID error propagated (covers the Task-1 Minor-4 case where a 403-escalation's + * _recoverSession itself threw, so the breaker may not be latched yet). + * + * @param {object} client - AirtableClient (may lack .auth on older/mocked clients → guarded) + * @param {*} [err] - a caught error or a per-row error string to inspect + * @returns {boolean} + */ +function sessionDied(client, err) { + if (client?.auth?.isSessionDead?.() === true) return true; + return !!err && /SESSION_INVALID/.test(String(err && err.message ? err.message : err)); +} + +/** + * Build the abort reason: trip detail (when available) + partial counts + resume advice. + * @param {object} client + * @param {object} result - { created, updated } + * @returns {string} + */ +function buildAbortReason(client, result) { + const trip = client?.auth?.getLastTrip?.(); + return `SESSION_INVALID: session died during record sync${trip ? ` (last status=${trip.status})` : ''} after ${result.created} created / ${result.updated} updated. Re-authenticate, then resume by re-running mode=apply with the same planId (progress is persisted).`; +} + +/** + * Flag the result as aborted (first-wins abortReason). Called from every write pass the moment + * a session death is detected; runRecords then skips the remaining passes + prune. + */ +function markAborted(client, result) { + result.aborted = true; + if (!result.abortReason) result.abortReason = buildAbortReason(client, result); +} + /** * Returns true if the field type is a multiSelect (arrays not accepted by updateRecords). * @param {string} type @@ -345,11 +387,14 @@ async function createChunkWithFallback({ client, destAppId, destTableId, chunk, try { res = await client.createRecords(destAppId, destTableId, chunk, { gate }); } catch (err) { + // A run-wide session death aborts the whole job; any other throw is a per-chunk failure. + if (sessionDied(client, err)) { markAborted(client, result); return; } result.failed += chunk.length; result.warnings.push({ code: 'RECORD_CREATE_FAILED', message: `Table ${destTableId}: createRecords threw: ${err.message}` }); return; } + // Record rows that landed BEFORE any death (preserve partial progress), then check for death. for (const created of (res.created || [])) { idmap.records[created.sourceKey] = created.rowId; createdDestIds?.add(created.rowId); @@ -359,6 +404,15 @@ async function createChunkWithFallback({ client, destAppId, destTableId, chunk, } const failed = res.failed || []; + + // Session death during the per-row POSTs: the client swallows each dead POST into failed[] + // (it does NOT throw at the batch level). Detect it here and abort BEFORE processing/retrying + // any failures — a dead session must not spin the link-strip retry or march the remaining rows. + if (sessionDied(client) || failed.some((f) => sessionDied(client, f.error))) { + markAborted(client, result); + return; + } + if (failed.length === 0) return; // Split failures: those whose row carried folded links (retryable) vs genuine failures. @@ -385,6 +439,7 @@ async function createChunkWithFallback({ client, destAppId, destTableId, chunk, try { retryRes = await client.createRecords(destAppId, destTableId, stripped, { gate }); } catch (err) { + if (sessionDied(client, err)) { markAborted(client, result); return; } result.failed += stripped.length; result.warnings.push({ code: 'RECORD_CREATE_FAILED', message: `Table ${destTableId}: retry-without-links threw: ${err.message}` }); return; @@ -424,6 +479,8 @@ export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idm const orderedTables = orderTablesByLinkDeps(srcSnapshot.tables || [], idmap); for (const srcTable of orderedTables) { + // A session death latched by a prior table/chunk aborts the whole job — never start a new table. + if (client.auth?.isSessionDead?.()) { markAborted(client, result); return; } const destTableId = idmap.tables[srcTable.id]; if (!destTableId) continue; // table not matched → skip @@ -482,14 +539,23 @@ export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idm const chunk = createRows.slice(i, i + CREATE_CHUNK); await createChunkWithFallback({ client, destAppId, destTableId, chunk, idmap, result, limiter, runState, destTableById, createdDestIds }); persist(idmap, journal); + if (result.aborted) return; // session died mid-chunk — stop walking rows/tables } // ── UPDATE batch ────────────────────────────────────────────────────── if (updateRows.length > 0) { try { const res = await client.updateRecords(destAppId, destTableId, updateRows, { gate: makeGate(limiter) }); + const updFailed = res.failed || []; + // Session death mid-batch: updateRecords swallows dead per-cell POSTs into failed[] too. + if (sessionDied(client) || updFailed.some((f) => sessionDied(client, f.error))) { + result.updated += (res.updated || []).length; + markAborted(client, result); + persist(idmap, journal); + return; + } result.updated += (res.updated || []).length; - for (const failed of (res.failed || [])) { + for (const failed of updFailed) { result.failed++; result.warnings.push({ code: 'RECORD_UPDATE_FAILED', @@ -497,6 +563,7 @@ export async function applyRecordsPass1({ client, srcSnapshot, destSnapshot, idm }); } } catch (err) { + if (sessionDied(client, err)) { markAborted(client, result); persist(idmap, journal); return; } result.failed += updateRows.length; result.warnings.push({ code: 'RECORD_UPDATE_FAILED', @@ -554,6 +621,8 @@ export async function applyRecordsPass2({ client, srcSnapshot, destSnapshot, idm } for (const srcTable of (srcSnapshot.tables || [])) { + // A session death latched by a prior table aborts the whole job — never start a new table. + if (client.auth?.isSessionDead?.()) { markAborted(client, result); return; } const destTableId = idmap.tables[srcTable.id]; if (!destTableId) continue; @@ -613,6 +682,8 @@ export async function applyRecordsPass2({ client, srcSnapshot, destSnapshot, idm try { const res = await client.addLinkItems(destAppId, destRowId, destFldId, items, { gate: makeGate(limiter) }); if (!res.ok) { + // addLinkItems never throws — a dead session surfaces as ok:false + a SESSION_INVALID error. + if (sessionDied(client, res.error)) { markAborted(client, result); persist(idmap, journal); return; } result.warnings.push({ code: 'RECORD_LINK_FAILED', message: `Record ${rec.id} field ${field.id} → dest row ${destRowId}: addLinkItems failed: ${res.error}`, @@ -621,6 +692,7 @@ export async function applyRecordsPass2({ client, srcSnapshot, destSnapshot, idm result.updated++; } } catch (err) { + if (sessionDied(client, err)) { markAborted(client, result); persist(idmap, journal); return; } result.warnings.push({ code: 'RECORD_LINK_FAILED', message: `Record ${rec.id} field ${field.id} → dest row ${destRowId}: addLinkItems threw: ${err.message}`, @@ -768,6 +840,8 @@ export async function applyAttachments({ const destAppId = destSnapshot.baseId || ''; for (const srcTable of (srcSnapshot.tables || [])) { + // A session death latched by a prior table aborts the whole job — never start a new table. + if (client.auth?.isSessionDead?.()) { markAborted(client, result); return; } const destTableId = idmap.tables[srcTable.id]; if (!destTableId) continue; // table not matched → skip @@ -939,6 +1013,9 @@ export async function applyAttachments({ } } } catch (fastPathErr) { + // A dead session aborts the whole job — do NOT fall back (every fallback upload would + // also fail under the dead session, spinning per-attachment ATTACHMENT_FAILED warnings). + if (sessionDied(client, fastPathErr)) { markAborted(client, result); persist(idmap, journal); return; } // Fast-path threw — fall back to per-attachment upload for all included records result.warnings.push({ code: 'ATTACHMENT_FALLBACK', @@ -956,9 +1033,14 @@ export async function applyAttachments({ await uploadAttachmentFallback({ client, destAppId, destRowId, destFldId, attachment: att, idmap, result, fetchBytes, limiter }); } } + // uploadAttachmentFallback swallows its errors — poll the breaker so a mid-fallback + // session death still aborts instead of spinning through every remaining record. + if (client.auth?.isSessionDead?.()) { markAborted(client, result); persist(idmap, journal); return; } } } } catch (tableErr) { + // A dead session aborts; any other unexpected per-table error is a warning (continue-on-failure). + if (sessionDied(client, tableErr)) { markAborted(client, result); persist(idmap, journal); return; } // Top-level per-table guard: never throw out of the loop result.warnings.push({ code: 'ATTACHMENT_FAILED', @@ -1126,6 +1208,9 @@ export async function runRecords({ client, srcSnapshot, destSnapshot, idmap, pol // Pass 1: scalar / select upsert — fills idmap.records await applyRecordsPass1({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist: persistRun, result, policy, policyOverrides, fieldMappings: resolved, createdDestIds }); persistRun(idmap, journal); + // Session died mid-Pass-1: skip the remaining passes, reapplyViewFilters, AND pruneRecords — + // a dead/partial session must never reach prune (deleting under a dead session is dangerous). + if (result.aborted) return result; // Rebuild destDisplayNames from idmap.records now populated by Pass 1. // Key = dest record id, value = source primary cell value (string). @@ -1143,10 +1228,26 @@ export async function runRecords({ client, srcSnapshot, destSnapshot, idmap, pol // Pass 2: link cells await applyRecordsPass2({ client, srcSnapshot, destSnapshot, idmap, destDisplayNames, limiter, journal, persist: persistRun, result, policy, policyOverrides, createdDestIds }); persistRun(idmap, journal); + if (result.aborted) return result; // session died mid-Pass-2 — skip attachments/filters/prune // Pass 3: attachments await applyAttachments({ client, srcSnapshot, destSnapshot, idmap, limiter, journal, persist: persistRun, result, policy, policyOverrides, createdDestIds }); persistRun(idmap, journal); + if (result.aborted) return result; // session died mid-Pass-3 — skip reapplyViewFilters + prune + + // Honest-failure gate: a run that attempted writes and had ZERO successes AND ZERO skips, with + // only failures, is a run-wide failure (e.g. a non-SESSION_INVALID hang that the per-pass abort + // detection doesn't catch). Route it through the same aborted→phase=failed path so it never + // reports a misleading 'done' with a burned batch, and skip reapplyViewFilters + prune (nothing + // synced). The skipped===0 guard is critical: a skip-heavy resume run (most rows already + // mapped/converged → skipped, a few genuine per-record failures, 0 created) must NOT trip this. + if (!result.aborted && result.failed > 0 && result.created === 0 && result.updated === 0 && result.skipped === 0) { + result.aborted = true; + result.abortReason = `RECORDS_ALL_FAILED: all ${result.failed} record write(s) failed with zero successes — a run-wide failure (not classified as a session death). Investigate (see warnings), then re-run mode=apply with the same planId to resume (progress is persisted).`; + result.warnings.push({ code: 'RECORDS_ALL_FAILED', message: result.abortReason }); + persistRun(idmap, journal); + return result; + } // Reapply view filters whose record refs now resolve. // Defensive guard: a filter-restore failure must NEVER prevent the prune step below — @@ -1189,6 +1290,13 @@ export async function runRecords({ client, srcSnapshot, destSnapshot, idmap, pol }); truncatedTables.add(t.name); } + // Final safety net before the DESTRUCTIVE prune step: a session death that a swallowing + // sub-loop (e.g. attachment fallback on the last table) never surfaced as result.aborted must + // STILL never reach pruneRecords — deleting under a dead/partial session is dangerous. Re-check + // the breaker directly here (the per-pass guards cover mid-run; this closes the tail window). + if (!result.aborted && client.auth?.isSessionDead?.()) markAborted(client, result); + if (result.aborted) { persistRun(idmap, journal); return result; } + // Live source record ids: a persisted mapping whose source record no longer exists must not // protect its dest row — otherwise mirror never propagates source-side record deletions. const liveSourceRecordIds = new Set(); @@ -1239,6 +1347,26 @@ export async function runRecords({ client, srcSnapshot, destSnapshot, idmap, pol * @returns {Promise} - result accumulator */ export async function applyRecords({ client, sourceBaseId, destBaseId, planId, runStartedAt, policy, policyOverrides, confirmDeletions, fieldMappings, naturalKeys }) { + // 0. Proactive session health (B): the schema phase runs thousands of mutations right before this + // and can leave the auth circuit-breaker LATCHED (a browser-free re-auth kept failing / a + // throttle 403 never cleared). Un-latch it and probe (browser-free modes re-mint a stale + // cookie) so the snapshots + writes below all run on a FRESH, healthy session. Throwing here is + // correct and resumable: nothing has been written yet, and applyRecordsImpl's .catch turns the + // throw into records-job status:'failed'. Guarded so an older/mocked client without the surface + // (Task-1 client.auth.*) simply skips this block instead of crashing. + if (client.auth?.ensureSessionHealthy) { + client.auth.resetSessionHealth?.(); // clear a breaker left dead by the schema phase + const health = await client.auth.ensureSessionHealthy(); // probe + (browser-free) re-mint if stale + if (!health.healthy) { + const err = new Error( + `SESSION_INVALID: session is not healthy at the records-phase start${health.error ? ` (${health.error})` : ''} — ` + + `re-authenticate, then resume by re-running mode=apply with the same planId (nothing was written yet).`, + ); + err.code = 'SESSION_INVALID'; + throw err; + } + } + // 1. Load the converged idmap (produced by the schema apply phase) const idmap = loadIdmap(sourceBaseId, destBaseId); idmap.records ??= {}; diff --git a/packages/mcp-server/test/sync/test-apply-link-dependency-defer.test.js b/packages/mcp-server/test/sync/test-apply-link-dependency-defer.test.js new file mode 100644 index 0000000..1886991 --- /dev/null +++ b/packages/mcp-server/test/sync/test-apply-link-dependency-defer.test.js @@ -0,0 +1,96 @@ +// Task 4: a computed (lookup/rollup/formula) updateField that runs in the SAME apply pass as +// the link field it depends on gets rejected by Airtable with a 422 naming the link dependency +// explicitly ("requires a link field" / "link field was changed by a collaborator") until the +// link change settles. apply.js must defer (not hard-fail) that specific rejection so the +// existing retry-until-stable loop in applyPlan re-attempts it later in the same run. +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { MockClient } from './helpers/mock-client.js'; +import { applyPlan } from '../../src/sync/apply.js'; +import { newJournal, isDone } from '../../src/sync/journal.js'; +import { snapshotBase } from '../../src/sync/snapshot.js'; + +const LINK_DEP_MSG = 'This column requires a link field. The link field was changed by a collaborator.'; + +async function setup() { + const client = new MockClient(); + const { tableId } = await client.createTable('appD', 'T'); + const price = await client.createField('appD', tableId, { name: 'Price', type: 'number' }); + const total = await client.createField('appD', tableId, { name: 'Total', type: 'formula', typeOptions: { formulaTextParsed: '{column_value_OLD}' } }); + const destSnapshot = await snapshotBase(client, 'appD'); + const plan = { + planId: 'plnLD', sourceBaseId: 'appS', destBaseId: 'appD', + idmap: { tables: {}, fields: { [price.columnId]: { destFld: price.columnId, choices: {} } } }, + actions: [{ + kind: 'updateField', sourceFieldId: total.columnId, destFld: total.columnId, + changes: { + typeOptions: { + formulaTextParsed: `{column_value_${price.columnId}}*2`, + dependencies: { referencedColumnIdsForValue: [price.columnId] }, + resultType: 'number', resultIsArray: false, + }, + }, + }], + orphans: [], warnings: [], + }; + return { client, plan, destSnapshot, totalId: total.columnId, priceId: price.columnId }; +} + +describe('apply: updateField computed — link-dependency 422 deferral', () => { + it('defers on the link-dependency 422, then succeeds once the link has settled (retry-until-stable)', async () => { + const { client, plan, destSnapshot, totalId, priceId } = await setup(); + let calls = 0; + const orig = client.updateFieldConfig.bind(client); + client.updateFieldConfig = async (appId, columnId, cfg) => { + if (columnId === totalId) { + calls++; + if (calls === 1) throw new Error(LINK_DEP_MSG); + } + return orig(appId, columnId, cfg); + }; + const journal = newJournal(plan.planId, 'ts'); + const res = await applyPlan({ + client, plan, destAppId: 'appD', destSnapshot, + idmap: JSON.parse(JSON.stringify(plan.idmap)), + journal, persist: () => {}, + }); + assert.equal(res.failed, 0, 'must NOT be counted as a hard failure'); + assert.equal(calls, 2, 'the deferred-retry loop re-attempted updateFieldConfig for the field'); + assert.equal(res.updated, 1, 'the field is updated by run end'); + assert.ok(!res.warnings.some((w) => w.code === 'ACTION_FAILED')); + const f = client._field(totalId); + assert.equal(f.typeOptions.formulaText, `{${priceId}}*2`); + assert.ok(isDone(journal, 0), 'resolved this run — journaled done'); + }); + + it('never resolving is bounded (no hang), reported as skipped (not failed), warning surfaced, NOT journaled done', async () => { + const { client, plan, destSnapshot } = await setup(); + client.updateFieldConfig = async () => { throw new Error(LINK_DEP_MSG); }; + const journal = newJournal(plan.planId, 'ts'); + const res = await applyPlan({ + client, plan, destAppId: 'appD', destSnapshot, + idmap: JSON.parse(JSON.stringify(plan.idmap)), + journal, persist: () => {}, + }); + assert.equal(res.failed, 0, 'never a hard failure, even when it never resolves this run'); + assert.ok(res.skipped >= 1, 'counted as skipped, not failed'); + assert.ok(res.warnings.some((w) => w.code === 'LINK_DEPENDENCY_DEFERRED'), 'held-back warning is surfaced at run end'); + assert.ok(!res.warnings.some((w) => w.code === 'ACTION_FAILED')); + assert.ok(!isDone(journal, 0), 'not journaled done — retryable on a future run'); + }); + + it('a different 422 (non link-dependency) still hard-fails as ACTION_FAILED', async () => { + const { client, plan, destSnapshot } = await setup(); + client.updateFieldConfig = async () => { throw new Error('422: invalid formula — unknown function FOO()'); }; + const journal = newJournal(plan.planId, 'ts'); + const res = await applyPlan({ + client, plan, destAppId: 'appD', destSnapshot, + idmap: JSON.parse(JSON.stringify(plan.idmap)), + journal, persist: () => {}, + }); + assert.equal(res.failed, 1, 'unrelated errors keep the existing hard-fail path'); + assert.ok(res.warnings.some((w) => w.code === 'ACTION_FAILED')); + assert.ok(!res.warnings.some((w) => w.code === 'LINK_DEPENDENCY_DEFERRED'), 'deferral must not swallow unrelated errors'); + assert.ok(!isDone(journal, 0)); + }); +}); diff --git a/packages/mcp-server/test/sync/test-records-all-failed.test.js b/packages/mcp-server/test/sync/test-records-all-failed.test.js new file mode 100644 index 0000000..535b950 --- /dev/null +++ b/packages/mcp-server/test/sync/test-records-all-failed.test.js @@ -0,0 +1,293 @@ +/** + * test-records-all-failed.test.js + * + * Task E — generalized "all writes failed" honest-failure gate (records engine). + * + * Task 2 made runRecords abort honestly on a SESSION death (SESSION_INVALID / isSessionDead()). + * But a non-session run-wide failure — e.g. every write hanging/erroring for a reason that is + * NOT SESSION_INVALID and does NOT latch the auth breaker — was still swallowed per-record and + * reported as job `done` with 0 created / 0 updated / everything failed. That is dishonest and + * silently burns the batch. + * + * This suite verifies the additional, more general net added to runRecords: if a run attempted + * writes and had ZERO successes (created===0, updated===0) AND ZERO skips (skipped===0), with + * only failures (failed>0), the whole run is now treated as aborted via the SAME `result.aborted` + * plumbing Task 2 built — surfacing as phase=failed (resumable) instead of a misleading `done`, + * and skipping reapplyViewFilters + pruneRecords (nothing was actually synced). + * + * The `skipped===0` clause is critical: a skip-heavy resume run (most rows already + * mapped/converged → skipped, a couple of genuine per-record failures, 0 created/updated) must + * NOT trip this gate — that would be a false positive on perfectly normal resume behavior. + * + * Drives runRecords() directly (fake client + tiny in-memory snapshots), reusing the fixture + * style from test-records-session-abort.test.js. + */ + +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; + +import { runRecords } from '../../src/sync/records.js'; + +// ────────────────────────────────────────────────────────────────────────────── +// Fakes (mirrors test-records-session-abort.test.js) +// ────────────────────────────────────────────────────────────────────────────── + +/** A minimal AirtableAuth stand-in — never dies (these are NON-session failures). */ +function makeAuth() { + return { + isSessionDead: () => false, + getLastTrip: () => null, + resetSessionHealth: () => {}, + ensureSessionHealthy: async () => ({ healthy: true, recovered: false }), + }; +} + +function makeInfra() { + return { + limiter: { run: (f) => f() }, + journal: {}, + persist: () => {}, + }; +} + +function makeResult() { + return { created: 0, updated: 0, skipped: 0, failed: 0, deleted: 0, warnings: [] }; +} + +/** + * Source snapshot: one "Games" table with `nRows` create-path records (none mapped), + * a text field + a self-link field (empty, so link-folding never engages). + */ +function makeSrcSnapshot(nRows) { + const records = []; + for (let i = 0; i < nRows; i++) { + records.push({ id: `recS${i}`, cellValuesByColumnId: { sN: `Row ${i}`, sLink: [] } }); + } + return { + baseId: 'appSRC0000000001', + tables: [{ + id: 'tS', name: 'Games', primaryFieldId: 'sN', + fields: [ + { id: 'sN', name: 'Name', type: 'text' }, + { id: 'sLink', name: 'Self', type: 'multipleRecordLinks', typeOptions: { foreignTableId: 'tS' } }, + ], + views: [{ id: 'vS1', name: 'Grid view', type: 'grid', personalForUserId: null }], + records, + }], + }; +} + +/** Dest snapshot with a dest-only orphan record (so prune WOULD delete it if reached). */ +function makeDestSnapshot() { + return { + baseId: 'appDST0000000001', + tables: [{ + id: 'tD', name: 'Games', primaryFieldId: 'dN', + fields: [ + { id: 'dN', name: 'Name', type: 'text' }, + { id: 'dLink', name: 'Self', type: 'multipleRecordLinks', typeOptions: { foreignTableId: 'tD' } }, + ], + views: [{ id: 'vD1', name: 'Grid view', type: 'grid', personalForUserId: null }], + records: [{ id: 'recOrphan', cellValuesByColumnId: {} }], + }], + }; +} + +function makeIdmap() { + return { + tables: { tS: 'tD' }, + fields: { + sN: { destFld: 'dN', choices: {} }, + sLink: { destFld: 'dLink', choices: {} }, + }, + records: {}, + }; +} + +// ────────────────────────────────────────────────────────────────────────────── +// 1 — all-failed trips the gate +// ────────────────────────────────────────────────────────────────────────────── + +describe('runRecords — honest-failure gate (RECORDS_ALL_FAILED)', () => { + it('trips the gate when every write attempt fails with zero successes and zero skips', async () => { + const auth = makeAuth(); + const deleteLog = []; + const client = { + auth, + createRecords: async (appId, tableId, rows) => ({ + created: [], + failed: rows.map((r) => ({ sourceKey: r.sourceKey, error: 'row failed (500): upstream hung' })), + }), + updateRecords: async () => ({ updated: [], failed: [] }), + addLinkItems: async () => ({ ok: true, added: 0 }), + deleteRecords: async (...a) => { deleteLog.push(a); return { deleted: a[2].length }; }, + getView: async () => ({}), + updateViewFilters: async () => ({ ok: true }), + }; + + const result = makeResult(); + const { limiter, journal, persist } = makeInfra(); + const out = await runRecords({ + client, srcSnapshot: makeSrcSnapshot(5), destSnapshot: makeDestSnapshot(), idmap: makeIdmap(), + policy: 'mirror', confirmDeletions: true, // prune WOULD delete recOrphan if reached + limiter, journal, persist, result, + }); + + assert.equal(out.aborted, true, 'a total non-session failure must trip the honest-failure gate'); + assert.match(String(out.abortReason), /RECORDS_ALL_FAILED/, 'abortReason identifies the gate'); + assert.match(String(out.abortReason), /mode=apply/, 'abortReason tells the user how to resume'); + assert.ok( + out.warnings.some((w) => w.code === 'RECORDS_ALL_FAILED'), + 'a RECORDS_ALL_FAILED warning is present', + ); + assert.equal(out.failed, 5, 'all 5 failures were counted'); + assert.equal(out.created, 0); + assert.equal(out.skipped, 0); + assert.equal(deleteLog.length, 0, 'pruneRecords did NOT run — the gate skips it on total failure'); + }); +}); + +// ────────────────────────────────────────────────────────────────────────────── +// 2 — partial success does NOT trip +// ────────────────────────────────────────────────────────────────────────────── + +describe('runRecords — honest-failure gate does not false-positive on partial success', () => { + it('does not abort when some rows are created and some fail', async () => { + const auth = makeAuth(); + const deleteLog = []; + const client = { + auth, + createRecords: async (appId, tableId, rows) => ({ + created: rows.slice(0, 3).map((r) => ({ rowId: 'recD_' + r.sourceKey, sourceKey: r.sourceKey })), + failed: rows.slice(3).map((r) => ({ sourceKey: r.sourceKey, error: 'row failed (500): upstream hung' })), + }), + updateRecords: async () => ({ updated: [], failed: [] }), + addLinkItems: async () => ({ ok: true, added: 0 }), + deleteRecords: async (...a) => { deleteLog.push(a); return { deleted: a[2].length }; }, + getView: async () => ({}), + updateViewFilters: async () => ({ ok: true }), + }; + + const result = makeResult(); + const { limiter, journal, persist } = makeInfra(); + const out = await runRecords({ + client, srcSnapshot: makeSrcSnapshot(5), destSnapshot: makeDestSnapshot(), idmap: makeIdmap(), + policy: 'mirror', confirmDeletions: true, + limiter, journal, persist, result, + }); + + assert.equal(out.aborted ?? false, false, 'partial success must NOT trip the gate'); + assert.equal(out.abortReason, undefined, 'no abortReason set'); + assert.ok(!out.warnings.some((w) => w.code === 'RECORDS_ALL_FAILED'), 'no RECORDS_ALL_FAILED warning'); + assert.equal(out.created, 3); + assert.equal(out.failed, 2); + assert.equal(deleteLog.length, 1, 'the run completed all passes and reached prune'); + }); +}); + +// ────────────────────────────────────────────────────────────────────────────── +// 3 — skip-heavy does NOT trip (false-positive guard) +// ────────────────────────────────────────────────────────────────────────────── + +describe('runRecords — honest-failure gate does not false-positive on a skip-heavy resume run', () => { + it('does not abort when rows are already converged (skipped) alongside a couple of genuine failures', async () => { + // 3 rows are PRE-MAPPED (already synced) with a converged link cell (identical literal + // array on both sides — arrayCellEquals is a structural comparison, not an idmap-remap) and + // NO other scalar field mapped, so buildUpdateCells computes an EMPTY cell diff for them → + // result.skipped++ (Pass 1's "nothing to write" skip). 2 rows are unmapped (create path) + // and fail with a genuine non-session error. Net: created=0, updated=0, skipped=3, failed=2. + const auth = makeAuth(); + const deleteLog = []; + const client = { + auth, + createRecords: async (appId, tableId, rows) => ({ + created: [], + failed: rows.map((r) => ({ sourceKey: r.sourceKey, error: 'row failed (422): validation error' })), + }), + updateRecords: async () => ({ updated: [], failed: [] }), // never called — updateRows stays empty + addLinkItems: async () => ({ ok: true, added: 0 }), + deleteRecords: async (...a) => { deleteLog.push(a); return { deleted: a[2].length }; }, + getView: async () => ({}), + updateViewFilters: async () => ({ ok: true }), + }; + + const srcSnapshot = { + baseId: 'appSRC0000000001', + tables: [{ + id: 'tS', name: 'Games', primaryFieldId: 'sN', + // sN intentionally NOT in idmap.fields below — only sLink is mapped, so buildUpdateCells + // only ever considers sLink for the mapped (skip-path) rows. + fields: [ + { id: 'sN', name: 'Name', type: 'text' }, + { id: 'sLink', name: 'Self', type: 'multipleRecordLinks', typeOptions: { foreignTableId: 'tS' } }, + ], + views: [{ id: 'vS1', name: 'Grid view', type: 'grid', personalForUserId: null }], + records: [ + // Already-mapped, converged → skipped + { id: 'recS0', cellValuesByColumnId: { sN: 'Zero', sLink: ['recAnchor'] } }, + { id: 'recS1', cellValuesByColumnId: { sN: 'One', sLink: ['recAnchor'] } }, + { id: 'recS2', cellValuesByColumnId: { sN: 'Two', sLink: ['recAnchor'] } }, + // Unmapped → create path → genuine failure + { id: 'recS3', cellValuesByColumnId: { sN: 'Three', sLink: [] } }, + { id: 'recS4', cellValuesByColumnId: { sN: 'Four', sLink: [] } }, + ], + }], + }; + const destSnapshot = { + baseId: 'appDST0000000001', + tables: [{ + id: 'tD', name: 'Games', primaryFieldId: 'dN', + fields: [ + { id: 'dN', name: 'Name', type: 'text' }, + { id: 'dLink', name: 'Self', type: 'multipleRecordLinks', typeOptions: { foreignTableId: 'tD' } }, + ], + views: [{ id: 'vD1', name: 'Grid view', type: 'grid', personalForUserId: null }], + records: [ + { id: 'recD0', cellValuesByColumnId: { dLink: ['recAnchor'] } }, + { id: 'recD1', cellValuesByColumnId: { dLink: ['recAnchor'] } }, + { id: 'recD2', cellValuesByColumnId: { dLink: ['recAnchor'] } }, + { id: 'recOrphan', cellValuesByColumnId: {} }, + ], + }], + }; + const idmap = { + tables: { tS: 'tD' }, + fields: { sLink: { destFld: 'dLink', choices: {} } }, // sN NOT mapped on purpose + records: { recS0: 'recD0', recS1: 'recD1', recS2: 'recD2' }, + }; + + const result = makeResult(); + const { limiter, journal, persist } = makeInfra(); + const out = await runRecords({ + client, srcSnapshot, destSnapshot, idmap, + policy: 'mirror', confirmDeletions: true, + limiter, journal, persist, result, + }); + + assert.equal(out.skipped, 3, 'the 3 already-converged mapped rows were skipped (sanity check on the fixture)'); + assert.equal(out.created, 0); + assert.equal(out.updated, 0); + assert.equal(out.failed, 2, 'the 2 unmapped rows genuinely failed'); + assert.equal(out.aborted ?? false, false, 'skipped>0 must guard against the false positive'); + assert.equal(out.abortReason, undefined, 'no abortReason set'); + assert.ok(!out.warnings.some((w) => w.code === 'RECORDS_ALL_FAILED'), 'no RECORDS_ALL_FAILED warning'); + assert.equal(deleteLog.length, 1, 'the run completed all passes and reached prune (deleted the orphan)'); + }); + + it('unit-checks the bare gate condition directly (skipped>0 must never trip it)', () => { + // Direct condition check mirroring the gate added in runRecords, per the brief's escape + // hatch — a belt-and-suspenders assertion independent of the full runRecords plumbing above. + const wouldTrip = (r) => r.failed > 0 && r.created === 0 && r.updated === 0 && r.skipped === 0; + + assert.equal( + wouldTrip({ created: 0, updated: 0, skipped: 3, failed: 2, warnings: [] }), + false, + 'skipped>0, created=0, updated=0, failed>0 must NOT be flagged', + ); + assert.equal( + wouldTrip({ created: 0, updated: 0, skipped: 0, failed: 5, warnings: [] }), + true, + 'skipped=0, created=0, updated=0, failed>0 IS the total-failure shape', + ); + }); +}); diff --git a/packages/mcp-server/test/sync/test-records-session-abort.test.js b/packages/mcp-server/test/sync/test-records-session-abort.test.js new file mode 100644 index 0000000..2737a8b --- /dev/null +++ b/packages/mcp-server/test/sync/test-records-session-abort.test.js @@ -0,0 +1,344 @@ +/** + * test-records-session-abort.test.js + * + * Task 2 — records engine session-death handling: + * B (proactive re-auth): applyRecords re-mints/probes the session at the records-phase start + * and throws BEFORE snapshotting if the session cannot be made healthy. + * C (mid-run abort): a session death during any write pass sets result.aborted + abortReason, + * stops walking rows, skips the remaining passes + reapplyViewFilters + pruneRecords, and + * surfaces as records-job phase=failed (recordsTerminalStatus decision). + * + * Drives runRecords() directly (fake client + tiny in-memory snapshots), plus a focused unit + * test on the job-layer decision function. + */ + +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { runRecords, applyRecords } from '../../src/sync/records.js'; +import { recordsTerminalStatus } from '../../src/sync/index.js'; + +// ────────────────────────────────────────────────────────────────────────────── +// Fakes +// ────────────────────────────────────────────────────────────────────────────── + +/** A minimal AirtableAuth stand-in exposing the Task-1 records-facing surface. */ +function makeAuth() { + let dead = false; + return { + isSessionDead: () => dead, + getLastTrip: () => (dead ? { status: 403, reason: 'throttle' } : null), + resetSessionHealth: () => { dead = false; }, + ensureSessionHealthy: async () => ({ healthy: true, recovered: false }), + _die: () => { dead = true; }, + }; +} + +function makeInfra() { + return { + limiter: { run: (f) => f() }, + journal: {}, + persist: () => {}, + }; +} + +function makeResult() { + return { created: 0, updated: 0, skipped: 0, failed: 0, deleted: 0, warnings: [] }; +} + +/** + * Source snapshot: one "Games" table with `nRows` create-path records (none mapped), + * a text field + a self-link field (so Pass 2 would call addLinkItems if it ran). + */ +function makeSrcSnapshot(nRows) { + const records = []; + for (let i = 0; i < nRows; i++) { + records.push({ id: `recS${i}`, cellValuesByColumnId: { sN: `Row ${i}`, sLink: [] } }); + } + return { + baseId: 'appSRC0000000001', + tables: [{ + id: 'tS', name: 'Games', primaryFieldId: 'sN', + fields: [ + { id: 'sN', name: 'Name', type: 'text' }, + { id: 'sLink', name: 'Self', type: 'multipleRecordLinks', typeOptions: { foreignTableId: 'tS' } }, + ], + views: [{ id: 'vS1', name: 'Grid view', type: 'grid', personalForUserId: null }], + records, + }], + }; +} + +/** Dest snapshot with a dest-only orphan record (so prune WOULD delete under mirror+confirm). */ +function makeDestSnapshot() { + return { + baseId: 'appDST0000000001', + tables: [{ + id: 'tD', name: 'Games', primaryFieldId: 'dN', + fields: [ + { id: 'dN', name: 'Name', type: 'text' }, + { id: 'dLink', name: 'Self', type: 'multipleRecordLinks', typeOptions: { foreignTableId: 'tD' } }, + ], + views: [{ id: 'vD1', name: 'Grid view', type: 'grid', personalForUserId: null }], + records: [{ id: 'recOrphan', cellValuesByColumnId: {} }], + }], + }; +} + +function makeIdmap() { + return { + tables: { tS: 'tD' }, + fields: { + sN: { destFld: 'dN', choices: {} }, + sLink: { destFld: 'dLink', choices: {} }, + }, + records: {}, + }; +} + +// ────────────────────────────────────────────────────────────────────────────── +// C — abort stops the run early on a mid-run session death +// ────────────────────────────────────────────────────────────────────────────── + +describe('runRecords — mid-run session death aborts the whole job (C)', () => { + it('stops after the first create chunk when the session dies (fewer create calls than chunks; Pass 2 + prune never run)', async () => { + const auth = makeAuth(); + const createLog = []; + const linkLog = []; + const deleteLog = []; + const client = { + auth, + createRecords: async (appId, tableId, rows) => { + createLog.push(rows.length); + // Simulate the auth breaker latching mid-batch: the real client swallows each dead + // per-row POST into failed[] (it does NOT throw at the batch level). + auth._die(); + return { created: [], failed: rows.map((r) => ({ sourceKey: r.sourceKey, error: 'SESSION_INVALID: session is dead' })) }; + }, + updateRecords: async () => ({ updated: [], failed: [] }), + addLinkItems: async (...a) => { linkLog.push(a); return { ok: true, added: 0 }; }, + deleteRecords: async (...a) => { deleteLog.push(a); return { deleted: 0 }; }, + getView: async () => ({}), + updateViewFilters: async () => ({ ok: true }), + }; + + const srcSnapshot = makeSrcSnapshot(120); // > CREATE_CHUNK (50) → 3 chunks if it kept going + const destSnapshot = makeDestSnapshot(); + const idmap = makeIdmap(); + const result = makeResult(); + const { limiter, journal, persist } = makeInfra(); + + const out = await runRecords({ + client, srcSnapshot, destSnapshot, idmap, + policy: 'mirror', confirmDeletions: true, // prune WOULD delete recOrphan if reached + limiter, journal, persist, result, + }); + + assert.equal(out.aborted, true, 'result.aborted must be true'); + assert.match(String(out.abortReason), /SESSION_INVALID/, 'abortReason mentions SESSION_INVALID'); + assert.match(String(out.abortReason), /mode=apply/, 'abortReason tells the user how to resume'); + assert.equal(createLog.length, 1, 'only the first chunk was attempted (did NOT march all 120 rows)'); + assert.equal(linkLog.length, 0, 'Pass 2 (addLinkItems) never ran under abort'); + assert.equal(deleteLog.length, 0, 'pruneRecords (deleteRecords) never ran under abort — no deletion on a dead session'); + }); + + it('aborts when createRecords THROWS a SESSION_INVALID error at the batch level', async () => { + const auth = makeAuth(); // isSessionDead stays false → detection must fall back to the error message (Minor-4) + const deleteLog = []; + const client = { + auth, + createRecords: async () => { throw new Error('SESSION_INVALID: recover threw'); }, + updateRecords: async () => ({ updated: [], failed: [] }), + addLinkItems: async () => ({ ok: true }), + deleteRecords: async (...a) => { deleteLog.push(a); return { deleted: 0 }; }, + getView: async () => ({}), + updateViewFilters: async () => ({ ok: true }), + }; + + const result = makeResult(); + const { limiter, journal, persist } = makeInfra(); + const out = await runRecords({ + client, srcSnapshot: makeSrcSnapshot(10), destSnapshot: makeDestSnapshot(), idmap: makeIdmap(), + policy: 'mirror', confirmDeletions: true, + limiter, journal, persist, result, + }); + + assert.equal(out.aborted, true); + assert.match(String(out.abortReason), /SESSION_INVALID/); + assert.equal(deleteLog.length, 0, 'prune skipped even when isSessionDead() is false but a SESSION_INVALID error propagated'); + }); + + it('tail safety-net: a session death that surfaces only in reapplyViewFilters still skips prune', async () => { + // reapplyViewFilters swallows a failed getView as a per-view warning (never sets aborted). + // If the session died there, the final breaker re-check before pruneRecords must catch it — + // a dead session must never reach the destructive prune step. + const auth = makeAuth(); + const deleteLog = []; + const client = { + auth, + createRecords: async (appId, tableId, rows) => ({ + created: rows.map((r, i) => ({ rowId: `recD${i}`, sourceKey: r.sourceKey })), + failed: [], + }), + updateRecords: async () => ({ updated: [], failed: [] }), + addLinkItems: async () => ({ ok: true, added: 0 }), + deleteRecords: async (...a) => { deleteLog.push(a); return { deleted: 0 }; }, + // Session dies at the tail (view-config population): sets the breaker AND throws — swallowed. + getView: async () => { auth._die(); throw new Error('SESSION_INVALID: died during filter restore'); }, + updateViewFilters: async () => ({ ok: true }), + }; + + // Src table with a configless collaborative view → reapplyViewFilters calls getView for it. + const srcSnapshot = { + baseId: 'appSRC0000000001', + tables: [{ + id: 'tS', name: 'Games', primaryFieldId: 'sN', + fields: [{ id: 'sN', name: 'Name', type: 'text' }], + views: [{ id: 'vS1', name: 'Grid view', type: 'grid', personalForUserId: null }], + records: [{ id: 'recS0', cellValuesByColumnId: { sN: 'A' } }], + }], + }; + const destSnapshot = makeDestSnapshot(); // has recOrphan + + const idmap = { tables: { tS: 'tD' }, fields: { sN: { destFld: 'dN', choices: {} } }, records: {} }; + const result = makeResult(); + const { limiter, journal, persist } = makeInfra(); + + const out = await runRecords({ + client, srcSnapshot, destSnapshot, idmap, + policy: 'mirror', confirmDeletions: true, + limiter, journal, persist, result, + }); + + assert.equal(out.aborted, true, 'the tail breaker re-check flags the abort'); + assert.match(String(out.abortReason), /SESSION_INVALID/); + assert.equal(deleteLog.length, 0, 'pruneRecords was skipped — no deletion under the dead session'); + }); +}); + +// ────────────────────────────────────────────────────────────────────────────── +// C — continue-on-failure preserved for NON-session errors +// ────────────────────────────────────────────────────────────────────────────── + +describe('runRecords — non-session failures do NOT abort (continue-on-failure preserved)', () => { + it('a normal per-row create failure is a warning; the run completes all passes and prunes', async () => { + // NOTE: at least one row must SUCCEED here — an all-failed/zero-skip run is now, by design, + // the RECORDS_ALL_FAILED honest-failure gate's trigger (see test-records-all-failed.test.js). + // This test's job is to prove a PARTIAL non-session failure (mixed with a success) still + // does not abort — continue-on-failure is preserved for genuine per-record errors. + const auth = makeAuth(); // never dies + const deleteLog = []; + const client = { + auth, + createRecords: async (appId, tableId, rows) => ({ + created: rows.slice(0, 1).map((r) => ({ rowId: 'recD_ok', sourceKey: r.sourceKey })), + failed: rows.slice(1).map((r) => ({ sourceKey: r.sourceKey, error: 'row failed (422): validation error' })), + }), + updateRecords: async () => ({ updated: [], failed: [] }), + addLinkItems: async () => ({ ok: true, added: 0 }), + deleteRecords: async (...a) => { deleteLog.push(a); return { deleted: a[2].length }; }, + getView: async () => ({}), + updateViewFilters: async () => ({ ok: true }), + }; + + const result = makeResult(); + const { limiter, journal, persist } = makeInfra(); + const out = await runRecords({ + client, srcSnapshot: makeSrcSnapshot(3), destSnapshot: makeDestSnapshot(), idmap: makeIdmap(), + policy: 'mirror', confirmDeletions: true, + limiter, journal, persist, result, + }); + + assert.equal(out.aborted ?? false, false, 'a non-session failure must NOT abort'); + assert.equal(out.abortReason, undefined, 'no abortReason set'); + assert.equal(out.created, 1, 'the one success was counted'); + assert.ok(out.failed >= 2, 'the failures were counted'); + assert.ok(out.warnings.some((w) => w.code === 'RECORD_CREATE_FAILED'), 'the failure surfaced as a warning'); + assert.equal(deleteLog.length, 1, 'the run completed all passes and reached prune (deleted the orphan)'); + }); +}); + +// ────────────────────────────────────────────────────────────────────────────── +// Job layer (C) — aborted result routes to phase=failed +// ────────────────────────────────────────────────────────────────────────────── + +describe('recordsTerminalStatus — job-layer aborted→failed decision', () => { + it('an aborted result becomes status=failed / event=records-failed with the abortReason + partial counts', () => { + const t = recordsTerminalStatus({ + aborted: true, + abortReason: 'SESSION_INVALID: session died during record sync (last status=403) after 7 created / 0 updated. Re-authenticate, then resume by re-running mode=apply with the same planId (progress is persisted).', + created: 7, updated: 0, failed: 3, warnings: [{ code: 'X' }], + }); + assert.equal(t.status, 'failed'); + assert.equal(t.event, 'records-failed'); + assert.match(String(t.error), /SESSION_INVALID/); + assert.match(String(t.error), /mode=apply/); + assert.equal(t.recordsResult.created, 7, 'partial counts are preserved'); + assert.equal(t.recordsResult.failed, 3); + assert.equal(t.recordsResult.warningCount, 1); + }); + + it('an aborted result WITHOUT abortReason falls back to a generic session message', () => { + const t = recordsTerminalStatus({ aborted: true, created: 0, updated: 0, warnings: [] }); + assert.equal(t.status, 'failed'); + assert.match(String(t.error), /SESSION|session|re-run|apply/i); + }); + + it('a non-aborted result stays status=done / event=records-done', () => { + const t = recordsTerminalStatus({ aborted: false, created: 5, updated: 2, failed: 0, warnings: [] }); + assert.equal(t.status, 'done'); + assert.equal(t.event, 'records-done'); + assert.equal(t.recordsResult.created, 5); + assert.equal(t.recordsResult.updated, 2); + }); +}); + +// ────────────────────────────────────────────────────────────────────────────── +// B — proactive session health at the records-phase start +// ────────────────────────────────────────────────────────────────────────────── + +describe('applyRecords — proactive session health gate (B)', () => { + it('throws BEFORE snapshotting when the session cannot be made healthy', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'rec-abort-B-')); + let reset = 0; + const client = { + auth: { + resetSessionHealth: () => { reset++; }, + ensureSessionHealthy: async () => ({ healthy: false, error: 'probe failed' }), + }, + // If snapshotting is reached, this throws a DISTINCT error — the test proves it is NOT reached. + getApplicationData: async () => { throw new Error('SNAPSHOT_SHOULD_NOT_RUN'); }, + }; + + await assert.rejects( + () => applyRecords({ client, sourceBaseId: 'appSRC0000000001', destBaseId: 'appDST0000000001', planId: 'plnB', runStartedAt: 't0' }), + (err) => { + assert.match(String(err.message), /SESSION/i, 'throws a session error'); + assert.doesNotMatch(String(err.message), /SNAPSHOT_SHOULD_NOT_RUN/, 'must throw before any snapshot call'); + return true; + }, + ); + assert.ok(reset >= 1, 'resetSessionHealth was called to un-latch any stale breaker'); + }); + + it('is a no-op when client.auth lacks ensureSessionHealthy (older/mocked client) — does not crash', async () => { + process.env.AIRTABLE_USER_MCP_HOME = mkdtempSync(join(tmpdir(), 'rec-abort-B2-')); + // A client whose auth has no health surface; snapshotting immediately fails with a KNOWN error. + // We only assert the health block was SKIPPED (no TypeError), so any reject other than a crash is fine. + const client = { + auth: {}, + getApplicationData: async () => { throw new Error('REACHED_SNAPSHOT'); }, + }; + await assert.rejects( + () => applyRecords({ client, sourceBaseId: 'appSRC0000000001', destBaseId: 'appDST0000000001', planId: 'plnB2', runStartedAt: 't0' }), + (err) => { + // Reaching the snapshot proves the health block was skipped rather than throwing a TypeError. + assert.match(String(err.message), /REACHED_SNAPSHOT/); + return true; + }, + ); + }); +}); diff --git a/packages/mcp-server/test/sync/test-sync-job.test.js b/packages/mcp-server/test/sync/test-sync-job.test.js index 06ee505..2831d00 100644 --- a/packages/mcp-server/test/sync/test-sync-job.test.js +++ b/packages/mcp-server/test/sync/test-sync-job.test.js @@ -132,6 +132,41 @@ describe('syncStatus()', () => { assert.equal(s.phase, 'records'); assert.equal(s.recordsMapped, 1); }); + + it('projects planDigest to human-only by default, omitting the machine payload', () => { + writeSyncJobStatus(SRC, DEST, 'digest1', { + phase: 'done', + planDigest: { human: 'looks good', machine: { verdict: 'converged', changeset: [{ big: 'payload' }] } }, + finishedAt: 't1', + }); + const s = syncStatus({ sourceBaseId: SRC, destBaseId: DEST, planId: 'digest1' }); + assert.equal(s.planDigest.human, 'looks good', 'human summary preserved'); + assert.equal(s.planDigest.machine, undefined, 'machine payload omitted by default'); + assert.equal(s.planDigest.machineOmitted, true); + }); + + it('returns the full planDigest.machine when verbose:true is passed', () => { + writeSyncJobStatus(SRC, DEST, 'digest2', { + phase: 'done', + planDigest: { human: 'looks good', machine: { verdict: 'converged', changeset: [{ big: 'payload' }] } }, + finishedAt: 't1', + }); + const s = syncStatus({ sourceBaseId: SRC, destBaseId: DEST, planId: 'digest2', verbose: true }); + assert.equal(s.planDigest.human, 'looks good'); + assert.ok(s.planDigest.machine, 'machine payload present when verbose'); + assert.equal(s.planDigest.machine.verdict, 'converged'); + assert.equal(s.planDigest.machineOmitted, undefined, 'machineOmitted flag not set when verbose returns the full digest'); + }); + + it('does not throw and omits planDigest when the job has none yet (early phase)', () => { + writeSyncJobStatus(SRC, DEST, 'nodigest', { phase: 'schema', startedAt: 't0' }); + const s = syncStatus({ sourceBaseId: SRC, destBaseId: DEST, planId: 'nodigest' }); + assert.equal(s.phase, 'schema'); + assert.equal('planDigest' in s, false, 'planDigest key omitted entirely, as before'); + + const sVerbose = syncStatus({ sourceBaseId: SRC, destBaseId: DEST, planId: 'nodigest', verbose: true }); + assert.equal('planDigest' in sVerbose, false, 'verbose does not conjure a planDigest that was never written'); + }); }); describe('planJob()', () => { diff --git a/packages/mcp-server/test/test-auth-403-reauth.test.js b/packages/mcp-server/test/test-auth-403-reauth.test.js new file mode 100644 index 0000000..c07bf51 --- /dev/null +++ b/packages/mcp-server/test/test-auth-403-reauth.test.js @@ -0,0 +1,225 @@ +import { describe, it, beforeEach, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { AirtableAuth } from '../src/auth.js'; + +/** + * Session-recovery bug fix — browser-free 403 escalation + trip instrumentation + * + records-facing surface. + * + * In direct-login / byo (browser-free) modes an expired session or a rotated + * CSRF token surfaces on mutations as HTTP 403, not 401. The old code classified + * EVERY 403 as throttle (back off, never re-auth) so a long records job burned + * its whole batch as SESSION_INVALID. The fix: a 403 that PERSISTS past one + * throttle backoff in a browser-free mode escalates to _recoverSession (a single + * HTTP _directLogin / cookie reload — no page-load burst), while browser mode + * keeps 403 throttle-only (relaunch on 403 re-trips the ~5 req/s limit → storm). + * + * All network is stubbed on the instance — these never touch live Airtable. + */ + +// AIRTABLE_AUTH_MODE is read in the constructor, so it must be set BEFORE `new`. +const MODE_ENV = 'AIRTABLE_AUTH_MODE'; +let savedMode; +beforeEach(() => { savedMode = process.env[MODE_ENV]; }); +afterEach(() => { + if (savedMode === undefined) delete process.env[MODE_ENV]; + else process.env[MODE_ENV] = savedMode; +}); + +/** Build an auth in the given mode with all network + init stubbed out. */ +function stubbedAuth(mode) { + process.env[MODE_ENV] = mode; + const a = new AirtableAuth(); + a._sessionDeadAfter = 3; // trip the breaker quickly + a._rlSleep = async () => {}; // no real backoff waits + a.ensureLoggedIn = async () => {}; // skip init() entirely + a._credentials = { cookieHeader: 'c=1', csrfToken: 'z' }; + a.isLoggedIn = true; + return a; +} + +describe('auth 403 → re-auth in browser-free modes (R1)', () => { + it('direct-login: a 403 persisting past one backoff escalates to _recoverSession, then succeeds', async () => { + const a = stubbedAuth('direct-login'); + let recoverCalls = 0; + a._recoverSession = async () => { recoverCalls++; }; + let n = 0; + // 403, 403, then 200 — the first 403 is a throttle backoff, the second escalates. + a._rawApiCall = async () => (++n <= 2 ? { status: 403, body: '' } : { status: 200, body: '{}' }); + + const res = await a.get('/v0.3/x', 'app1'); + assert.equal(res.status, 200); + assert.ok(recoverCalls >= 1, '_recoverSession should fire for a persistent 403'); + assert.equal(a._recoveryStreak, 0, 'a 2xx resets the streak'); + assert.equal(a._sessionDead, false); + }); + + it('byo: a persistent 403 also escalates to _recoverSession', async () => { + const a = stubbedAuth('byo'); + let recoverCalls = 0; + a._recoverSession = async () => { recoverCalls++; }; + let n = 0; + a._rawApiCall = async () => (++n <= 2 ? { status: 403, body: '' } : { status: 200, body: '{}' }); + + const res = await a.get('/v0.3/x', 'app1'); + assert.equal(res.status, 200); + assert.ok(recoverCalls >= 1, '_recoverSession should fire for a persistent 403 in byo mode'); + }); + + it('direct-login: a single transient 403 that clears on backoff does NOT re-auth', async () => { + const a = stubbedAuth('direct-login'); + let recoverCalls = 0; + a._recoverSession = async () => { recoverCalls++; }; + let n = 0; + // Only ONE 403, then 200 — the allowed first backoff clears it, no re-login. + a._rawApiCall = async () => (++n === 1 ? { status: 403, body: '' } : { status: 200, body: '{}' }); + + const res = await a.get('/v0.3/x', 'app1'); + assert.equal(res.status, 200); + assert.equal(recoverCalls, 0, 'one transient 403 must clear via backoff, not re-login'); + }); + + it('browser mode: 403 stays throttle-only — _recoverSession is NEVER called', async () => { + const a = stubbedAuth('browser'); + let recoverCalls = 0; + a._recoverSession = async () => { recoverCalls++; }; + a._rawApiCall = async () => ({ status: 403, body: '' }); + + await assert.rejects(() => a.get('/v0.3/x', 'app1'), /SESSION_INVALID/); + assert.equal(recoverCalls, 0, 'browser mode must not relaunch on 403 (throttle-only)'); + assert.equal(a._sessionDead, true); + }); + + it('429 stays pure throttle even in direct-login mode (no re-auth)', async () => { + const a = stubbedAuth('direct-login'); + let recoverCalls = 0; + a._recoverSession = async () => { recoverCalls++; }; + a._rawApiCall = async () => ({ status: 429, body: '' }); + + await assert.rejects(() => a.get('/v0.3/x', 'app1'), /SESSION_INVALID/); + assert.equal(recoverCalls, 0, '429 must never escalate to re-auth'); + }); +}); + +describe('auth latch trip instrumentation (R2)', () => { + it('direct-login: an unrecoverable 403 latches _sessionDead and records the trip status', async () => { + const a = stubbedAuth('direct-login'); + a._recoverSession = async () => {}; // recovery no-ops → 403 never clears + a._rawApiCall = async () => ({ status: 403, body: '' }); + + await assert.rejects(() => a.get('/v0.3/x', 'app1'), /SESSION_INVALID/); + assert.equal(a.isSessionDead(), true); + const trip = a.getLastTrip(); + assert.equal(trip.status, 403); + assert.ok(typeof trip.reason === 'string' && trip.reason.length > 0); + }); + + it('a 401 latch records status 401', async () => { + const a = stubbedAuth('direct-login'); + a._recoverSession = async () => {}; + a._rawApiCall = async () => ({ status: 401, body: '' }); + + await assert.rejects(() => a.get('/v0.3/x', 'app1'), /SESSION_INVALID/); + assert.equal(a.isSessionDead(), true); + assert.equal(a.getLastTrip().status, 401); + }); + + it('getLastTrip() returns null while the session is alive', () => { + const a = stubbedAuth('direct-login'); + assert.equal(a.getLastTrip(), null); + assert.equal(a.isSessionDead(), false); + }); +}); + +describe('auth resetSessionHealth clears trip fields (R2/R3)', () => { + it('after a latch, resetSessionHealth clears the dead flag and the trip', async () => { + const a = stubbedAuth('direct-login'); + a._recoverSession = async () => {}; + a._rawApiCall = async () => ({ status: 403, body: '' }); + + await assert.rejects(() => a.get('/v0.3/x', 'app1'), /SESSION_INVALID/); + assert.equal(a.isSessionDead(), true); + assert.notEqual(a.getLastTrip(), null); + + a.resetSessionHealth(); + assert.equal(a.isSessionDead(), false); + assert.equal(a.getLastTrip(), null); + assert.equal(a._recoveryStreak, 0); + }); +}); + +describe('auth ensureSessionHealthy (R3)', () => { + it('a healthy probe returns { healthy:true, recovered:false } without recovering', async () => { + const a = stubbedAuth('direct-login'); + let recoverCalls = 0; + a._recoverSession = async () => { recoverCalls++; }; + a.checkSessionHealth = async () => ({ valid: true, userId: 'u1' }); + + const out = await a.ensureSessionHealthy(); + assert.deepEqual(out, { healthy: true, recovered: false }); + assert.equal(recoverCalls, 0); + }); + + it('direct-login: an invalid probe recovers then re-probes → { healthy:true, recovered:true }', async () => { + const a = stubbedAuth('direct-login'); + let recoverCalls = 0; + a._recoverSession = async () => { recoverCalls++; }; + let probes = 0; + a.checkSessionHealth = async () => (++probes === 1 ? { valid: false, status: 403 } : { valid: true, userId: 'u1' }); + + const out = await a.ensureSessionHealthy(); + assert.equal(out.healthy, true); + assert.equal(out.recovered, true); + assert.equal(recoverCalls, 1); + }); + + it('browser mode: an invalid probe does NOT recover here → { healthy:false, recovered:false }', async () => { + const a = stubbedAuth('browser'); + let recoverCalls = 0; + a._recoverSession = async () => { recoverCalls++; }; + a.checkSessionHealth = async () => ({ valid: false, status: 401 }); + + const out = await a.ensureSessionHealthy(); + assert.deepEqual(out, { healthy: false, recovered: false }); + assert.equal(recoverCalls, 0); + }); + + it('never throws — a throwing probe degrades to { healthy:false }', async () => { + const a = stubbedAuth('direct-login'); + a.checkSessionHealth = async () => { throw new Error('boom'); }; + + const out = await a.ensureSessionHealthy(); + assert.equal(out.healthy, false); + }); +}); + +describe('auth bounded-loop guarantee under default config (R4)', () => { + it('direct-login: default _sessionDeadAfter=8 (> MAX_RETRIES=3) still terminates — the 403 escalation exhausts its attempt budget and falls through to the throttle streak', async () => { + const a = stubbedAuth('direct-login'); + a._sessionDeadAfter = 8; // GREATER than MAX_RETRIES=3 — proves the loop bounds itself even when + // the dead-session threshold outlives the recovery-attempt cap. + let recoverCalls = 0; + a._recoverSession = async () => { recoverCalls++; }; // no-op — session never heals, every call still 403s + a._rawApiCall = async () => ({ status: 403, body: '' }); + + await assert.rejects(() => a.get('/v0.3/x', 'app1'), /SESSION_INVALID/); + assert.equal(a.isSessionDead(), true); + assert.equal(a.getLastTrip().status, 403); + // MAX_RETRIES=3 re-auth attempts are spent (persist403 branch), then every further 403 + // falls through to the plain throttle branch until the streak reaches _sessionDeadAfter. + assert.equal(recoverCalls, 3, 'recovery attempts are bounded by MAX_RETRIES, not _sessionDeadAfter'); + }); + + it('direct-login: 503 stays pure throttle — _recoverSession is NEVER called', async () => { + const a = stubbedAuth('direct-login'); + a._sessionDeadAfter = 3; + let recoverCalls = 0; + a._recoverSession = async () => { recoverCalls++; }; + a._rawApiCall = async () => ({ status: 503, body: '' }); + + await assert.rejects(() => a.get('/v0.3/x', 'app1'), /SESSION_INVALID/); + assert.equal(recoverCalls, 0, '503 must never escalate to re-auth in any mode, including direct-login'); + assert.equal(a.isSessionDead(), true); + assert.equal(a.getLastTrip().status, 503); + }); +}); diff --git a/packages/mcp-server/test/test-daemon-port-fallback.test.js b/packages/mcp-server/test/test-daemon-port-fallback.test.js new file mode 100644 index 0000000..0720fb7 --- /dev/null +++ b/packages/mcp-server/test/test-daemon-port-fallback.test.js @@ -0,0 +1,54 @@ +// Regression: the daemon must NOT crash when its fixed port cannot be bound. The real-world failure +// was `listen EACCES: permission denied 127.0.0.1:8723` — the default port 8723 landing inside a +// Windows reserved/excluded TCP port range → the daemon exited code 1 before writing a lockfile → +// the extension's ensureDaemon polled forever → "Timed out waiting for daemon startup". The fixed-port +// fallback previously only handled EADDRINUSE (busy), so EACCES was fatal. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import { listenAvoidingBlockedPorts } from '../src/daemon/server.js'; + +// A fake net.Server: `listen(port)` schedules a 'listening' (ok) or 'error' (with .code) event. +// behavior(port) → 'ok' or an errno string. Ephemeral (port 0) binds to 54321 when ok. +function makeFakeServer(behavior) { + const s = new EventEmitter(); + s._bound = null; + s.listen = (port /* , host */) => { + queueMicrotask(() => { + const verdict = behavior(port); + if (verdict === 'ok') { s._bound = port === 0 ? 54321 : port; s.emit('listening'); } + else { const e = new Error(verdict); e.code = verdict; s.emit('error', e); } + }); + }; + s.address = () => (s._bound == null ? null : { port: s._bound, address: '127.0.0.1', family: 'IPv4' }); + s.close = (cb) => { s._bound = null; if (cb) queueMicrotask(cb); }; + return s; +} + +test('fixed port EACCES (Windows excluded range) falls back to an ephemeral port instead of crashing', async () => { + const server = makeFakeServer((port) => (port === 8723 ? 'EACCES' : 'ok')); + await listenAvoidingBlockedPorts(server, 8723, '127.0.0.1'); // must NOT throw + assert.equal(server.address().port, 54321, 'bound to the ephemeral fallback port'); +}); + +test('fixed port EADDRINUSE still falls back (no regression)', async () => { + const server = makeFakeServer((port) => (port === 8723 ? 'EADDRINUSE' : 'ok')); + await listenAvoidingBlockedPorts(server, 8723, '127.0.0.1'); + assert.equal(server.address().port, 54321); +}); + +test('fixed port EADDRNOTAVAIL falls back', async () => { + const server = makeFakeServer((port) => (port === 8723 ? 'EADDRNOTAVAIL' : 'ok')); + await listenAvoidingBlockedPorts(server, 8723, '127.0.0.1'); + assert.equal(server.address().port, 54321); +}); + +test('a bind failure on the ephemeral port (port 0) is fatal, not swallowed', async () => { + const server = makeFakeServer(() => 'EACCES'); // even ephemeral fails → genuine error + await assert.rejects(() => listenAvoidingBlockedPorts(server, 0, '127.0.0.1'), /EACCES/); +}); + +test('an unrelated fixed-port error is NOT swallowed as a fallback', async () => { + const server = makeFakeServer((port) => (port === 8723 ? 'EPERM' : 'ok')); + await assert.rejects(() => listenAvoidingBlockedPorts(server, 8723, '127.0.0.1'), /EPERM/); +}); diff --git a/packages/shared/src/messages.ts b/packages/shared/src/messages.ts index b5d6983..301f91d 100644 --- a/packages/shared/src/messages.ts +++ b/packages/shared/src/messages.ts @@ -17,6 +17,8 @@ export type WebviewMessage = | { type: 'action:logout'; id: string } | { type: 'action:status'; id: string } | { type: 'action:saveCredentials'; id: string; email: string; password: string; otpSecret: string } + | { type: 'auth:saveCookie'; cookie: string } + | { type: 'auth:clearCookie' } | { type: 'action:install-browser'; id: string } | { type: 'action:removeBrowser'; id: string } | { type: 'action:openToolConfig'; id: string } diff --git a/packages/shared/src/types.ts b/packages/shared/src/types.ts index 58f38c2..35c97e1 100644 --- a/packages/shared/src/types.ts +++ b/packages/shared/src/types.ts @@ -65,6 +65,8 @@ export interface AuthState { lastLogin?: string; error?: string; hasCredentials: boolean; + /** True when a byo session cookie is stored in the OS keychain (authMode='byo'). */ + hasCookie?: boolean; browser?: BrowserInfo; browserDownload?: BrowserDownloadState; availableBrowsers?: BrowserInfo[]; diff --git a/packages/webview/src/store.ts b/packages/webview/src/store.ts index 3d6213e..6c2d0a4 100644 --- a/packages/webview/src/store.ts +++ b/packages/webview/src/store.ts @@ -21,6 +21,8 @@ interface Store extends DashboardState { logout: () => void; status: () => void; saveCredentials: (email: string, password: string, otpSecret: string) => void; + saveCookie: (cookie: string) => void; + clearCookie: () => void; installBrowser: () => void; removeBrowser: () => void; unconfigureIde: (ideId: string) => void; @@ -88,6 +90,7 @@ const defaultSettings: SettingsSnapshot = { const defaultAuth: AuthState = { status: 'unknown', hasCredentials: false, + hasCookie: false, }; // If the extension never answers (host crash, lost message, webview reload), @@ -145,7 +148,13 @@ export const useStore = create((set, get) => { const auth = currentAuth.status !== 'unknown' ? currentAuth - : { ...incomingAuth, hasCredentials: currentAuth.hasCredentials || incomingAuth.hasCredentials }; + : { + ...incomingAuth, + // hasCredentials AND hasCookie are both sticky — a stale state:update + // during cold start must not clobber a known-good saved-credential flag. + hasCredentials: currentAuth.hasCredentials || incomingAuth.hasCredentials, + hasCookie: currentAuth.hasCookie || incomingAuth.hasCookie, + }; return { ...state, auth, loading: false }; }), applyAuthState: (state) => set({ auth: state }), @@ -198,6 +207,15 @@ export const useStore = create((set, get) => { sendToExtension({ type: 'action:saveCredentials', id, email, password, otpSecret }); }, + // Fire-and-forget (no action id): the byo cookie save has no acknowledgement + // round-trip — the refreshed hasCookie arrives on the next auth:state/state:update. + saveCookie: (cookie) => { + sendToExtension({ type: 'auth:saveCookie', cookie }); + }, + clearCookie: () => { + sendToExtension({ type: 'auth:clearCookie' }); + }, + installBrowser: () => { const id = randomId(); // Chromium download can take minutes on slow connections. diff --git a/packages/webview/src/tabs/Settings.tsx b/packages/webview/src/tabs/Settings.tsx index fa966d0..173ba73 100644 --- a/packages/webview/src/tabs/Settings.tsx +++ b/packages/webview/src/tabs/Settings.tsx @@ -58,7 +58,7 @@ export function Settings() { const settings = useStore(s => s.settings); const auth = useStore(s => s.auth); const debug = useStore(s => s.debug); - const { saveCredentials, login, logout, status, installBrowser, removeBrowser, manualLogin, openStoragePath, backupSession, restoreSession, selectCustomBrowser, setBrowserChoice } = useStore(); + const { saveCredentials, saveCookie, clearCookie, login, logout, status, installBrowser, removeBrowser, manualLogin, openStoragePath, backupSession, restoreSession, selectCustomBrowser, setBrowserChoice } = useStore(); const [settingsPending, setSettingsPending] = useState(false); @@ -78,6 +78,7 @@ export function Settings() { const [email, setEmail] = useState(''); const [password, setPassword] = useState(''); const [otpSecret, setOtpSecret] = useState(''); + const [cookie, setCookie] = useState(''); const [showCreds, setShowCreds] = useState(false); const [storageOpen, setStorageOpen] = useState(false); const storage = useStore(s => s.storage); @@ -106,6 +107,24 @@ export function Settings() { const loginMode = settings.auth.loginMode ?? 'manual'; const isManual = loginMode === 'manual'; + // Connection mode is primary: it decides which credential fields render. + // browser → login-mode toggle + browser dropdown; Auto → email/pw/TOTP form + // direct-login → email/pw/TOTP form (always; no browser, no login-mode toggle) + // byo → session-cookie field + const authMode = settings.mcp.authMode ?? 'browser'; + const showCredsForm = authMode === 'direct-login' || (authMode === 'browser' && !isManual); + + const handleSaveCookie = () => { + const trimmed = cookie.trim(); + if (!trimmed) return; + // Keep the value on submit so a failed save doesn't lose the user's input. + // The field clears via the effect below once the save is confirmed (hasCookie flips true). + saveCookie(trimmed); + }; + + // Clear the cookie input only after the save is confirmed by the extension. + useEffect(() => { if (auth.hasCookie) setCookie(''); }, [auth.hasCookie]); + const isBusy = auth.status === 'checking' || auth.status === 'logging-in'; const browserFound = auth.browser?.found ?? true; // optimistic until probe runs const browserLabel = auth.browser?.label; @@ -127,31 +146,33 @@ export function Settings() {
Airtable Account
-
-
-
Login mode
-
- {isManual ? 'You log in through the browser — no credentials stored' : 'Automated login with stored credentials'} + {authMode === 'browser' && ( +
+
+
Login mode
+
+ {isManual ? 'You log in through the browser — no credentials stored' : 'Automated login with stored credentials'} +
+
+
+ Manual + + Auto
-
- Manual - - Auto -
-
+ )} - {!isManual && ( + {showCredsForm && (
Credentials stored in OS keychain @@ -161,6 +182,7 @@ export function Settings() {
)} + {authMode === 'browser' && (
@@ -207,6 +229,7 @@ export function Settings() {
+ )}
@@ -245,19 +268,53 @@ export function Settings() {
- {(settings.mcp.authMode === 'byo' || settings.mcp.authMode === 'direct-login') && ( -
- - - {settings.mcp.authMode === 'byo' - ? 'Paste your Airtable session cookie into credentials.json in the config folder ({ "cookie": "…" }).' - : 'Set email / password / TOTP in login.json in the config folder. SSO/Google accounts must use Browser mode.'} - - + {authMode === 'byo' && ( +
+
+ + Cookie stored in OS keychain + + {auth.hasCookie ? 'Set' : 'Not set'} + +
+ +