From e5db119277c7b3908c1c97862f6482a2789547d0 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 21 May 2025 16:26:10 +0200 Subject: [PATCH 01/48] Removed zongji type mappings which are now provided by the Zongji package directly Added check for tablemap events --- modules/module-mysql/package.json | 2 +- .../src/replication/zongji/zongji-utils.ts | 5 + .../src/replication/zongji/zongji.d.ts | 129 ------------------ modules/module-mysql/test/tsconfig.json | 2 +- pnpm-lock.yaml | 10 +- 5 files changed, 12 insertions(+), 136 deletions(-) delete mode 100644 modules/module-mysql/src/replication/zongji/zongji.d.ts diff --git a/modules/module-mysql/package.json b/modules/module-mysql/package.json index 07fe335f..c87e698b 100644 --- a/modules/module-mysql/package.json +++ b/modules/module-mysql/package.json @@ -33,7 +33,7 @@ "@powersync/service-sync-rules": "workspace:*", "@powersync/service-types": "workspace:*", "@powersync/service-jsonbig": "workspace:*", - "@powersync/mysql-zongji": "^0.1.0", + "@powersync/mysql-zongji": "0.0.0-dev-20250521092520", "semver": "^7.5.4", "async": "^3.2.4", "mysql2": "^3.11.0", diff --git a/modules/module-mysql/src/replication/zongji/zongji-utils.ts b/modules/module-mysql/src/replication/zongji/zongji-utils.ts index 36122b63..24d01663 100644 --- a/modules/module-mysql/src/replication/zongji/zongji-utils.ts +++ b/modules/module-mysql/src/replication/zongji/zongji-utils.ts @@ -3,6 +3,7 @@ import { BinLogGTIDLogEvent, BinLogMutationEvent, BinLogRotationEvent, + BinLogTableMapEvent, BinLogUpdateEvent, BinLogXidEvent } from '@powersync/mysql-zongji'; @@ -11,6 +12,10 @@ export function eventIsGTIDLog(event: BinLogEvent): event is BinLogGTIDLogEvent return event.getEventName() == 'gtidlog'; } +export function eventIsTableMap(event: BinLogEvent): event is BinLogTableMapEvent { + return event.getEventName() == 'tablemap'; +} + export function eventIsXid(event: BinLogEvent): event is BinLogXidEvent { return event.getEventName() == 'xid'; } diff --git a/modules/module-mysql/src/replication/zongji/zongji.d.ts b/modules/module-mysql/src/replication/zongji/zongji.d.ts deleted file mode 100644 index f5640497..00000000 --- a/modules/module-mysql/src/replication/zongji/zongji.d.ts +++ /dev/null @@ -1,129 +0,0 @@ -declare module '@powersync/mysql-zongji' { - import { Socket } from 'net'; - - export type ZongjiOptions = { - host: string; - user: string; - password: string; - dateStrings?: boolean; - timeZone?: string; - }; - - interface DatabaseFilter { - [databaseName: string]: string[] | true; - } - - export type StartOptions = { - includeEvents?: string[]; - excludeEvents?: string[]; - /** - * Describe which databases and tables to include (Only for row events). Use database names as the key and pass an array of table names or true (for the entire database). - * Example: { 'my_database': ['allow_table', 'another_table'], 'another_db': true } - */ - includeSchema?: DatabaseFilter; - /** - * Object describing which databases and tables to exclude (Same format as includeSchema) - * Example: { 'other_db': ['disallowed_table'], 'ex_db': true } - */ - excludeSchema?: DatabaseFilter; - /** - * BinLog position filename to start reading events from - */ - filename?: string; - /** - * BinLog position offset to start reading events from in file specified - */ - position?: number; - - /** - * Unique server ID for this replication client. - */ - serverId?: number; - }; - - export type ColumnSchema = { - COLUMN_NAME: string; - COLLATION_NAME: string; - CHARACTER_SET_NAME: string; - COLUMN_COMMENT: string; - COLUMN_TYPE: string; - }; - - export type ColumnDefinition = { - name: string; - charset: string; - type: number; - metadata: Record; - }; - - export type TableMapEntry = { - columnSchemas: ColumnSchema[]; - parentSchema: string; - tableName: string; - columns: ColumnDefinition[]; - }; - - export type BaseBinLogEvent = { - timestamp: number; - getEventName(): string; - - /** - * Next position in BinLog file to read from after - * this event. - */ - nextPosition: number; - /** - * Size of this event - */ - size: number; - flags: number; - useChecksum: boolean; - }; - - export type BinLogRotationEvent = BaseBinLogEvent & { - binlogName: string; - position: number; - }; - - export type BinLogGTIDLogEvent = BaseBinLogEvent & { - serverId: Buffer; - transactionRange: number; - }; - - export type BinLogXidEvent = BaseBinLogEvent & { - xid: number; - }; - - export type BinLogMutationEvent = BaseBinLogEvent & { - tableId: number; - numberOfColumns: number; - tableMap: Record; - rows: Record[]; - }; - - export type BinLogUpdateEvent = Omit & { - rows: { - before: Record; - after: Record; - }[]; - }; - - export type BinLogEvent = BinLogRotationEvent | BinLogGTIDLogEvent | BinLogXidEvent | BinLogMutationEvent; - - // @vlasky/mysql Connection - export interface MySQLConnection { - _socket?: Socket; - /** There are other forms of this method as well - this is the most basic one. */ - query(sql: string, callback: (error: any, results: any, fields: any) => void): void; - } - - export default class ZongJi { - connection: MySQLConnection; - constructor(options: ZongjiOptions); - - start(options: StartOptions): void; - stop(): void; - - on(type: 'binlog' | string, callback: (event: BinLogEvent) => void); - } -} diff --git a/modules/module-mysql/test/tsconfig.json b/modules/module-mysql/test/tsconfig.json index 5257b273..18898c4e 100644 --- a/modules/module-mysql/test/tsconfig.json +++ b/modules/module-mysql/test/tsconfig.json @@ -13,7 +13,7 @@ "@core-tests/*": ["../../../packages/service-core/test/src/*"] } }, - "include": ["src", "../src/replication/zongji/zongji.d.ts"], + "include": ["src"], "references": [ { "path": "../" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6f0c20ca..2a61565d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -271,8 +271,8 @@ importers: specifier: workspace:* version: link:../../libs/lib-services '@powersync/mysql-zongji': - specifier: ^0.1.0 - version: 0.1.0 + specifier: 0.0.0-dev-20250521092520 + version: 0.0.0-dev-20250521092520 '@powersync/service-core': specifier: workspace:* version: link:../../packages/service-core @@ -1312,8 +1312,8 @@ packages: resolution: {integrity: sha512-UA91GwWPhFExt3IizW6bOeY/pQ0BkuNwKjk9iQW9KqxluGCrg4VenZ0/L+2Y0+ZOtme72EVvg6v0zo3AMQRCeA==} engines: {node: '>=12'} - '@powersync/mysql-zongji@0.1.0': - resolution: {integrity: sha512-2GjOxVws+wtbb+xFUJe4Ozzkp/f0Gsna0fje9art76bmz6yfLCW4K3Mf2/M310xMnAIp8eP9hsJ6DYwwZCo1RA==} + '@powersync/mysql-zongji@0.0.0-dev-20250521092520': + resolution: {integrity: sha512-AZ03eO5O/LQ8MFl/Z6OWyLJ4Mykd/gSbfIA8Iy0XImIKQt+XY8MqvtU/u3LLIZOJ+1ea43h0BfPvnFMsgwVxZg==} engines: {node: '>=20.0.0'} '@powersync/service-jsonbig@0.17.10': @@ -4778,7 +4778,7 @@ snapshots: '@pnpm/network.ca-file': 1.0.2 config-chain: 1.1.13 - '@powersync/mysql-zongji@0.1.0': + '@powersync/mysql-zongji@0.0.0-dev-20250521092520': dependencies: '@vlasky/mysql': 2.18.6 big-integer: 1.6.51 From d03360ea89e36f0322389b3afd600cbc62db4f8d Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 21 May 2025 16:29:53 +0200 Subject: [PATCH 02/48] Moved most of the binlog event handling logic to a separate BinlogListener class. Introduced a mechanism to limit the maximum size of the binlog processing queue, thus also limiting memory usage. This maximum processing queue size is configurable --- .../src/replication/MySQLConnectionManager.ts | 2 +- .../src/replication/zongji/BinlogListener.ts | 186 ++++++++++++++++++ modules/module-mysql/src/types/types.ts | 9 +- .../test/src/BinlogListener.test.ts | 136 +++++++++++++ .../test/src/mysql-to-sqlite.test.ts | 2 +- 5 files changed, 332 insertions(+), 3 deletions(-) create mode 100644 modules/module-mysql/src/replication/zongji/BinlogListener.ts create mode 100644 modules/module-mysql/test/src/BinlogListener.test.ts diff --git a/modules/module-mysql/src/replication/MySQLConnectionManager.ts b/modules/module-mysql/src/replication/MySQLConnectionManager.ts index 4548d838..d464a975 100644 --- a/modules/module-mysql/src/replication/MySQLConnectionManager.ts +++ b/modules/module-mysql/src/replication/MySQLConnectionManager.ts @@ -2,8 +2,8 @@ import { NormalizedMySQLConnectionConfig } from '../types/types.js'; import mysqlPromise from 'mysql2/promise'; import mysql, { FieldPacket, RowDataPacket } from 'mysql2'; import * as mysql_utils from '../utils/mysql-utils.js'; -import ZongJi from '@powersync/mysql-zongji'; import { logger } from '@powersync/lib-services-framework'; +import { ZongJi } from '@powersync/mysql-zongji'; export class MySQLConnectionManager { /** diff --git a/modules/module-mysql/src/replication/zongji/BinlogListener.ts b/modules/module-mysql/src/replication/zongji/BinlogListener.ts new file mode 100644 index 00000000..f67af740 --- /dev/null +++ b/modules/module-mysql/src/replication/zongji/BinlogListener.ts @@ -0,0 +1,186 @@ +import * as common from '../../common/common-index.js'; +import async from 'async'; +import { BinLogEvent, StartOptions, TableMapEntry, ZongJi } from '@powersync/mysql-zongji'; +import * as zongji_utils from './zongji-utils.js'; +import { logger } from '@powersync/lib-services-framework'; +import { MySQLConnectionManager } from '../MySQLConnectionManager.js'; + +export type Row = Record; + +export interface BinlogEventHandler { + onWrite: (rows: Row[], tableMap: TableMapEntry) => Promise; + onUpdate: (rowsAfter: Row[], rowsBefore: Row[], tableMap: TableMapEntry) => Promise; + onDelete: (rows: Row[], tableMap: TableMapEntry) => Promise; + onCommit: (lsn: string) => Promise; +} + +export interface BinlogListenerOptions { + connectionManager: MySQLConnectionManager; + eventHandler: BinlogEventHandler; + includedTables: string[]; + serverId: number; + startPosition: common.BinLogPosition; + abortSignal: AbortSignal; +} + +export class BinlogListener { + private connectionManager: MySQLConnectionManager; + private eventHandler: BinlogEventHandler; + private binLogPosition: common.BinLogPosition; + private currentGTID: common.ReplicatedGTID | null; + + zongji: ZongJi; + processingQueue: async.QueueObject; + + constructor(public options: BinlogListenerOptions) { + this.connectionManager = options.connectionManager; + this.eventHandler = options.eventHandler; + this.binLogPosition = options.startPosition; + this.currentGTID = null; + + this.processingQueue = async.queue(this.createQueueWorker(), 1); + this.zongji = this.createZongjiListener(); + } + + public async start(): Promise { + logger.info(`Starting replication. Created replica client with serverId:${this.options.serverId}`); + // Set a heartbeat interval for the Zongji replication connection + // Zongji does not explicitly handle the heartbeat events - they are categorized as event:unknown + // The heartbeat events are enough to keep the connection alive for setTimeout to work on the socket. + await new Promise((resolve, reject) => { + this.zongji.connection.query( + // In nanoseconds, 10^9 = 1s + 'set @master_heartbeat_period=28*1000000000', + function (error: any, results: any, fields: any) { + if (error) { + reject(error); + } else { + resolve(results); + } + } + ); + }); + logger.info('Successfully set up replication connection heartbeat...'); + + // The _socket member is only set after a query is run on the connection, so we set the timeout after setting the heartbeat. + // The timeout here must be greater than the master_heartbeat_period. + const socket = this.zongji.connection._socket!; + socket.setTimeout(60_000, () => { + socket.destroy(new Error('Replication connection timeout.')); + }); + + logger.info(`Reading binlog from: ${this.binLogPosition.filename}:${this.binLogPosition.offset}`); + this.zongji.start({ + // We ignore the unknown/heartbeat event since it currently serves no purpose other than to keep the connection alive + // tablemap events always need to be included for the other row events to work + includeEvents: ['tablemap', 'writerows', 'updaterows', 'deleterows', 'xid', 'rotate', 'gtidlog'], + includeSchema: { [this.connectionManager.databaseName]: this.options.includedTables }, + filename: this.binLogPosition.filename, + position: this.binLogPosition.offset, + serverId: this.options.serverId + } satisfies StartOptions); + + await new Promise((resolve, reject) => { + this.zongji.on('error', (error) => { + logger.error('Binlog listener error:', error); + this.zongji.stop(); + this.processingQueue.kill(); + reject(error); + }); + + this.processingQueue.error((error) => { + logger.error('BinlogEvent processing error:', error); + this.zongji.stop(); + this.processingQueue.kill(); + reject(error); + }); + + this.zongji.on('stopped', () => { + logger.info('Binlog listener stopped. Replication ended.'); + resolve(); + }); + + const stop = () => { + logger.info('Abort signal received, stopping replication...'); + this.zongji.stop(); + this.processingQueue.kill(); + resolve(); + }; + + this.options.abortSignal.addEventListener('abort', stop, { once: true }); + + if (this.options.abortSignal.aborted) { + // Generally this should have been picked up early, but we add this here as a failsafe. + stop(); + } + }); + } + + private createZongjiListener(): ZongJi { + const zongji = this.connectionManager.createBinlogListener(); + + zongji.on('binlog', async (evt) => { + logger.info(`Received Binlog event:${evt.getEventName()}`); + this.processingQueue.push(evt); + + // When the processing queue grows past the threshold, we pause the binlog listener + if (this.processingQueue.length() > this.connectionManager.options.max_binlog_queue_size) { + logger.info( + `Max Binlog processing queue length [${this.connectionManager.options.max_binlog_queue_size}] reached. Pausing Binlog listener.` + ); + zongji.pause(); + await this.processingQueue.empty(); + logger.info(`Binlog processing queue backlog cleared. Resuming Binlog listener.`); + zongji.resume(); + } + }); + + return zongji; + } + + private createQueueWorker() { + return async (evt: BinLogEvent) => { + switch (true) { + case zongji_utils.eventIsGTIDLog(evt): + this.currentGTID = common.ReplicatedGTID.fromBinLogEvent({ + raw_gtid: { + server_id: evt.serverId, + transaction_range: evt.transactionRange + }, + position: { + filename: this.binLogPosition.filename, + offset: evt.nextPosition + } + }); + break; + case zongji_utils.eventIsRotation(evt): + this.binLogPosition.filename = evt.binlogName; + this.binLogPosition.offset = evt.position; + break; + case zongji_utils.eventIsWriteMutation(evt): + await this.eventHandler.onWrite(evt.rows, evt.tableMap[evt.tableId]); + break; + case zongji_utils.eventIsUpdateMutation(evt): + await this.eventHandler.onUpdate( + evt.rows.map((row) => row.after), + evt.rows.map((row) => row.before), + evt.tableMap[evt.tableId] + ); + break; + case zongji_utils.eventIsDeleteMutation(evt): + await this.eventHandler.onDelete(evt.rows, evt.tableMap[evt.tableId]); + break; + case zongji_utils.eventIsXid(evt): + const LSN = new common.ReplicatedGTID({ + raw_gtid: this.currentGTID!.raw, + position: { + filename: this.binLogPosition.filename, + offset: evt.nextPosition + } + }).comparable; + await this.eventHandler.onCommit(LSN); + break; + } + }; + } +} diff --git a/modules/module-mysql/src/types/types.ts b/modules/module-mysql/src/types/types.ts index c0826187..d8079d1f 100644 --- a/modules/module-mysql/src/types/types.ts +++ b/modules/module-mysql/src/types/types.ts @@ -23,6 +23,8 @@ export interface NormalizedMySQLConnectionConfig { client_private_key?: string; lookup?: LookupFunction; + + max_binlog_queue_size: number; } export const MySQLConnectionConfig = service_types.configFile.DataSourceConfig.and( @@ -40,7 +42,9 @@ export const MySQLConnectionConfig = service_types.configFile.DataSourceConfig.a client_certificate: t.string.optional(), client_private_key: t.string.optional(), - reject_ip_ranges: t.array(t.string).optional() + reject_ip_ranges: t.array(t.string).optional(), + // The maximum number of binlog events that can be queued in memory before throttling is applied. + max_binlog_queue_size: t.number.optional() }) ); @@ -114,6 +118,9 @@ export function normalizeConnectionConfig(options: MySQLConnectionConfig): Norma server_id: options.server_id ?? 1, + // Based on profiling, a queue size of 1000 uses about 50MB of memory. + max_binlog_queue_size: options.max_binlog_queue_size ?? 1000, + lookup }; } diff --git a/modules/module-mysql/test/src/BinlogListener.test.ts b/modules/module-mysql/test/src/BinlogListener.test.ts new file mode 100644 index 00000000..278812cd --- /dev/null +++ b/modules/module-mysql/test/src/BinlogListener.test.ts @@ -0,0 +1,136 @@ +import { describe, test, beforeEach, vi, expect, afterEach } from 'vitest'; +import { BinlogEventHandler, BinlogListener, Row } from '@module/replication/zongji/BinlogListener.js'; +import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManager.js'; +import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js'; +import { v4 as uuid } from 'uuid'; +import * as common from '@module/common/common-index.js'; +import { createRandomServerId } from '@module/utils/mysql-utils.js'; +import { TableMapEntry } from '@powersync/mysql-zongji'; + +describe('BinlogListener tests', () => { + const MAX_QUEUE_SIZE = 10; + const BINLOG_LISTENER_CONNECTION_OPTIONS = { + ...TEST_CONNECTION_OPTIONS, + max_binlog_queue_size: MAX_QUEUE_SIZE + }; + + let connectionManager: MySQLConnectionManager; + let abortController: AbortController; + let eventHandler: TestBinlogEventHandler; + let binlogListener: BinlogListener; + + beforeEach(async () => { + connectionManager = new MySQLConnectionManager(BINLOG_LISTENER_CONNECTION_OPTIONS, {}); + const connection = await connectionManager.getConnection(); + await clearTestDb(connection); + await connection.query(`CREATE TABLE test_DATA (id CHAR(36) PRIMARY KEY, description text)`); + connection.release(); + const fromGTID = await getFromGTID(connectionManager); + + abortController = new AbortController(); + eventHandler = new TestBinlogEventHandler(); + binlogListener = new BinlogListener({ + connectionManager: connectionManager, + eventHandler: eventHandler, + startPosition: fromGTID.position, + includedTables: ['test_DATA'], + serverId: createRandomServerId(1), + abortSignal: abortController.signal + }); + }); + + afterEach(async () => { + await connectionManager.end(); + }); + + test('Binlog listener stops on abort signal', async () => { + const stopSpy = vi.spyOn(binlogListener.zongji, 'stop'); + + setTimeout(() => abortController.abort(), 10); + await expect(binlogListener.start()).resolves.toBeUndefined(); + expect(stopSpy).toHaveBeenCalled(); + }); + + test('Pause Zongji binlog listener when processing queue reaches max size', async () => { + const pauseSpy = vi.spyOn(binlogListener.zongji, 'pause'); + const resumeSpy = vi.spyOn(binlogListener.zongji, 'resume'); + const queueSpy = vi.spyOn(binlogListener.processingQueue, 'length'); + + const ROW_COUNT = 100; + await insertRows(connectionManager, ROW_COUNT); + + const startPromise = binlogListener.start(); + + await vi.waitFor(() => expect(eventHandler.rowsWritten).equals(ROW_COUNT), { timeout: 5000 }); + abortController.abort(); + await expect(startPromise).resolves.toBeUndefined(); + + // Count how many times the queue reached the max size. Consequently, we expect the listener to have paused and resumed that many times. + const overThresholdCount = queueSpy.mock.results.map((r) => r.value).filter((v) => v === MAX_QUEUE_SIZE).length; + expect(pauseSpy).toHaveBeenCalledTimes(overThresholdCount); + expect(resumeSpy).toHaveBeenCalledTimes(overThresholdCount); + }); + + test('Binlog events are correctly forwarded to provided binlog events handler', async () => { + const startPromise = binlogListener.start(); + + const ROW_COUNT = 10; + await insertRows(connectionManager, ROW_COUNT); + await vi.waitFor(() => expect(eventHandler.rowsWritten).equals(ROW_COUNT), { timeout: 5000 }); + expect(eventHandler.commitCount).equals(ROW_COUNT); + + await updateRows(connectionManager); + await vi.waitFor(() => expect(eventHandler.rowsUpdated).equals(ROW_COUNT), { timeout: 5000 }); + + await deleteRows(connectionManager); + await vi.waitFor(() => expect(eventHandler.rowsDeleted).equals(ROW_COUNT), { timeout: 5000 }); + + abortController.abort(); + await expect(startPromise).resolves.toBeUndefined(); + }); +}); + +async function getFromGTID(connectionManager: MySQLConnectionManager) { + const connection = await connectionManager.getConnection(); + const fromGTID = await common.readExecutedGtid(connection); + connection.release(); + + return fromGTID; +} + +async function insertRows(connectionManager: MySQLConnectionManager, count: number) { + for (let i = 0; i < count; i++) { + await connectionManager.query(`INSERT INTO test_DATA(id, description) VALUES('${uuid()}','test${i}')`); + } +} + +async function updateRows(connectionManager: MySQLConnectionManager) { + await connectionManager.query(`UPDATE test_DATA SET description='updated'`); +} + +async function deleteRows(connectionManager: MySQLConnectionManager) { + await connectionManager.query(`DELETE FROM test_DATA`); +} + +class TestBinlogEventHandler implements BinlogEventHandler { + rowsWritten = 0; + rowsUpdated = 0; + rowsDeleted = 0; + commitCount = 0; + + async onWrite(rows: Row[], tableMap: TableMapEntry) { + this.rowsWritten = this.rowsWritten + rows.length; + } + + async onUpdate(afterRows: Row[], beforeRows: Row[], tableMap: TableMapEntry) { + this.rowsUpdated = this.rowsUpdated + afterRows.length; + } + + async onDelete(rows: Row[], tableMap: TableMapEntry) { + this.rowsDeleted = this.rowsDeleted + rows.length; + } + + async onCommit(lsn: string) { + this.commitCount++; + } +} diff --git a/modules/module-mysql/test/src/mysql-to-sqlite.test.ts b/modules/module-mysql/test/src/mysql-to-sqlite.test.ts index 3b172955..97c5db93 100644 --- a/modules/module-mysql/test/src/mysql-to-sqlite.test.ts +++ b/modules/module-mysql/test/src/mysql-to-sqlite.test.ts @@ -3,7 +3,7 @@ import { afterAll, describe, expect, test } from 'vitest'; import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js'; import { eventIsWriteMutation, eventIsXid } from '@module/replication/zongji/zongji-utils.js'; import * as common from '@module/common/common-index.js'; -import ZongJi, { BinLogEvent } from '@powersync/mysql-zongji'; +import { BinLogEvent, ZongJi } from '@powersync/mysql-zongji'; import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManager.js'; import { toColumnDescriptors } from '@module/common/common-index.js'; From 924ecd87f81d979436d5b44e496a2c8017064ba5 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 21 May 2025 16:30:30 +0200 Subject: [PATCH 03/48] Updated the BinLogStream to use the new BinLogListener --- .../src/replication/BinLogStream.ts | 245 +++++------------- 1 file changed, 65 insertions(+), 180 deletions(-) diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index a1af98ac..be4fabfb 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -1,6 +1,5 @@ import { logger, ReplicationAbortedError, ReplicationAssertionError } from '@powersync/lib-services-framework'; import * as sync_rules from '@powersync/service-sync-rules'; -import async from 'async'; import { ColumnDescriptor, @@ -9,16 +8,15 @@ import { MetricsEngine, storage } from '@powersync/service-core'; -import mysql, { FieldPacket } from 'mysql2'; - -import { BinLogEvent, StartOptions, TableMapEntry } from '@powersync/mysql-zongji'; +import mysql from 'mysql2'; import mysqlPromise from 'mysql2/promise'; + +import { TableMapEntry } from '@powersync/mysql-zongji'; import * as common from '../common/common-index.js'; -import { isBinlogStillAvailable, ReplicatedGTID, toColumnDescriptors } from '../common/common-index.js'; import { createRandomServerId, escapeMysqlTableName } from '../utils/mysql-utils.js'; import { MySQLConnectionManager } from './MySQLConnectionManager.js'; -import * as zongji_utils from './zongji/zongji-utils.js'; import { ReplicationMetric } from '@powersync/service-types'; +import { BinlogEventHandler, BinlogListener, Row } from './zongji/BinlogListener.js'; export interface BinLogStreamOptions { connections: MySQLConnectionManager; @@ -34,16 +32,14 @@ interface MysqlRelId { interface WriteChangePayload { type: storage.SaveOperationTag; - data: Data; - previous_data?: Data; + row: Row; + previous_row?: Row; database: string; table: string; sourceTable: storage.SourceTable; columns: Map; } -export type Data = Record; - export class BinlogConfigurationError extends Error { constructor(message: string) { super(message); @@ -247,7 +243,7 @@ AND table_type = 'BASE TABLE';`, // Check if the binlog is still available. If it isn't we need to snapshot again. const connection = await this.connections.getConnection(); try { - const isAvailable = await isBinlogStillAvailable(connection, lastKnowGTID.position.filename); + const isAvailable = await common.isBinlogStillAvailable(connection, lastKnowGTID.position.filename); if (!isAvailable) { logger.info( `Binlog file ${lastKnowGTID.position.filename} is no longer available, starting initial replication again.` @@ -288,7 +284,7 @@ AND table_type = 'BASE TABLE';`, const sourceTables = this.syncRules.getSourceTables(); await this.storage.startBatch( - { zeroLSN: ReplicatedGTID.ZERO.comparable, defaultSchema: this.defaultSchema, storeCurrentData: true }, + { zeroLSN: common.ReplicatedGTID.ZERO.comparable, defaultSchema: this.defaultSchema, storeCurrentData: true }, async (batch) => { for (let tablePattern of sourceTables) { const tables = await this.getQualifiedTableNames(batch, tablePattern); @@ -324,9 +320,9 @@ AND table_type = 'BASE TABLE';`, const stream = query.stream(); let columns: Map | undefined = undefined; - stream.on('fields', (fields: FieldPacket[]) => { + stream.on('fields', (fields: mysql.FieldPacket[]) => { // Map the columns and their types - columns = toColumnDescriptors(fields); + columns = common.toColumnDescriptors(fields); }); for await (let row of stream) { @@ -383,7 +379,7 @@ AND table_type = 'BASE TABLE';`, // This is needed for includeSchema to work correctly. const sourceTables = this.syncRules.getSourceTables(); await this.storage.startBatch( - { zeroLSN: ReplicatedGTID.ZERO.comparable, defaultSchema: this.defaultSchema, storeCurrentData: true }, + { zeroLSN: common.ReplicatedGTID.ZERO.comparable, defaultSchema: this.defaultSchema, storeCurrentData: true }, async (batch) => { for (let tablePattern of sourceTables) { await this.getQualifiedTableNames(batch, tablePattern); @@ -407,14 +403,12 @@ AND table_type = 'BASE TABLE';`, // Auto-activate as soon as initial replication is done await this.storage.autoActivate(); const serverId = createRandomServerId(this.storage.group_id); - logger.info(`Starting replication. Created replica client with serverId:${serverId}`); const connection = await this.connections.getConnection(); const { checkpoint_lsn } = await this.storage.getStatus(); if (checkpoint_lsn) { logger.info(`Existing checkpoint found: ${checkpoint_lsn}`); } - const fromGTID = checkpoint_lsn ? common.ReplicatedGTID.fromSerialized(checkpoint_lsn) : await common.readExecutedGtid(connection); @@ -423,179 +417,70 @@ AND table_type = 'BASE TABLE';`, if (!this.stopped) { await this.storage.startBatch( - { zeroLSN: ReplicatedGTID.ZERO.comparable, defaultSchema: this.defaultSchema, storeCurrentData: true }, + { zeroLSN: common.ReplicatedGTID.ZERO.comparable, defaultSchema: this.defaultSchema, storeCurrentData: true }, async (batch) => { - const zongji = this.connections.createBinlogListener(); - - let currentGTID: common.ReplicatedGTID | null = null; - - const queue = async.queue(async (evt: BinLogEvent) => { - // State machine - switch (true) { - case zongji_utils.eventIsGTIDLog(evt): - currentGTID = common.ReplicatedGTID.fromBinLogEvent({ - raw_gtid: { - server_id: evt.serverId, - transaction_range: evt.transactionRange - }, - position: { - filename: binLogPositionState.filename, - offset: evt.nextPosition - } - }); - break; - case zongji_utils.eventIsRotation(evt): - // Update the position - binLogPositionState.filename = evt.binlogName; - binLogPositionState.offset = evt.position; - break; - case zongji_utils.eventIsWriteMutation(evt): - const writeTableInfo = evt.tableMap[evt.tableId]; - await this.writeChanges(batch, { - type: storage.SaveOperationTag.INSERT, - data: evt.rows, - tableEntry: writeTableInfo - }); - break; - case zongji_utils.eventIsUpdateMutation(evt): - const updateTableInfo = evt.tableMap[evt.tableId]; - await this.writeChanges(batch, { - type: storage.SaveOperationTag.UPDATE, - data: evt.rows.map((row) => row.after), - previous_data: evt.rows.map((row) => row.before), - tableEntry: updateTableInfo - }); - break; - case zongji_utils.eventIsDeleteMutation(evt): - const deleteTableInfo = evt.tableMap[evt.tableId]; - await this.writeChanges(batch, { - type: storage.SaveOperationTag.DELETE, - data: evt.rows, - tableEntry: deleteTableInfo - }); - break; - case zongji_utils.eventIsXid(evt): - this.metrics.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED).add(1); - // Need to commit with a replicated GTID with updated next position - await batch.commit( - new common.ReplicatedGTID({ - raw_gtid: currentGTID!.raw, - position: { - filename: binLogPositionState.filename, - offset: evt.nextPosition - } - }).comparable - ); - currentGTID = null; - // chunks_replicated_total.add(1); - break; - } - }, 1); - - zongji.on('binlog', (evt: BinLogEvent) => { - if (!this.stopped) { - logger.info(`Received Binlog event:${evt.getEventName()}`); - queue.push(evt); - } else { - logger.info(`Replication is busy stopping, ignoring event ${evt.getEventName()}`); - } - }); - - // Set a heartbeat interval for the Zongji replication connection - // Zongji does not explicitly handle the heartbeat events - they are categorized as event:unknown - // The heartbeat events are enough to keep the connection alive for setTimeout to work on the socket. - await new Promise((resolve, reject) => { - zongji.connection.query( - // In nanoseconds, 10^9 = 1s - 'set @master_heartbeat_period=28*1000000000', - function (error: any, results: any, fields: any) { - if (error) { - reject(error); - } else { - resolve(results); - } - } - ); - }); - logger.info('Successfully set up replication connection heartbeat...'); - - // The _socket member is only set after a query is run on the connection, so we set the timeout after setting the heartbeat. - // The timeout here must be greater than the master_heartbeat_period. - const socket = zongji.connection._socket!; - socket.setTimeout(60_000, () => { - socket.destroy(new Error('Replication connection timeout.')); - }); - - if (this.stopped) { - // Powersync is shutting down, don't start replicating - return; - } - - logger.info(`Reading binlog from: ${binLogPositionState.filename}:${binLogPositionState.offset}`); + const binlogEventHandler = this.createBinlogEventHandler(batch); // Only listen for changes to tables in the sync rules const includedTables = [...this.tableCache.values()].map((table) => table.table); - zongji.start({ - // We ignore the unknown/heartbeat event since it currently serves no purpose other than to keep the connection alive - includeEvents: ['tablemap', 'writerows', 'updaterows', 'deleterows', 'xid', 'rotate', 'gtidlog'], - excludeEvents: [], - includeSchema: { [this.defaultSchema]: includedTables }, - filename: binLogPositionState.filename, - position: binLogPositionState.offset, - serverId: serverId - } satisfies StartOptions); - - // Forever young - await new Promise((resolve, reject) => { - zongji.on('error', (error) => { - logger.error('Binlog listener error:', error); - zongji.stop(); - queue.kill(); - reject(error); - }); - - zongji.on('stopped', () => { - logger.info('Binlog listener stopped. Replication ended.'); - resolve(); - }); - - queue.error((error) => { - logger.error('Binlog listener queue error:', error); - zongji.stop(); - queue.kill(); - reject(error); - }); - - const stop = () => { - logger.info('Abort signal received, stopping replication...'); - zongji.stop(); - queue.kill(); - resolve(); - }; - - this.abortSignal.addEventListener('abort', stop, { once: true }); - - if (this.stopped) { - // Generally this should have been picked up early, but we add this here as a failsafe. - stop(); - } + const binlogListener = new BinlogListener({ + abortSignal: this.abortSignal, + includedTables: includedTables, + startPosition: binLogPositionState, + connectionManager: this.connections, + serverId: serverId, + eventHandler: binlogEventHandler }); + + await binlogListener.start(); } ); } } + private createBinlogEventHandler(batch: storage.BucketStorageBatch): BinlogEventHandler { + return { + onWrite: async (rows: Row[], tableMap: TableMapEntry) => { + await this.writeChanges(batch, { + type: storage.SaveOperationTag.INSERT, + rows: rows, + tableEntry: tableMap + }); + }, + + onUpdate: async (rowsAfter: Row[], rowsBefore: Row[], tableMap: TableMapEntry) => { + await this.writeChanges(batch, { + type: storage.SaveOperationTag.UPDATE, + rows: rowsAfter, + rows_before: rowsBefore, + tableEntry: tableMap + }); + }, + onDelete: async (rows: Row[], tableMap: TableMapEntry) => { + await this.writeChanges(batch, { + type: storage.SaveOperationTag.DELETE, + rows: rows, + tableEntry: tableMap + }); + }, + onCommit: async (lsn: string) => { + this.metrics.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED).add(1); + await batch.commit(lsn); + } + }; + } + private async writeChanges( batch: storage.BucketStorageBatch, msg: { type: storage.SaveOperationTag; - data: Data[]; - previous_data?: Data[]; + rows: Row[]; + rows_before?: Row[]; tableEntry: TableMapEntry; } ): Promise { - const columns = toColumnDescriptors(msg.tableEntry); + const columns = common.toColumnDescriptors(msg.tableEntry); - for (const [index, row] of msg.data.entries()) { + for (const [index, row] of msg.rows.entries()) { await this.writeChange(batch, { type: msg.type, database: msg.tableEntry.parentSchema, @@ -607,8 +492,8 @@ AND table_type = 'BASE TABLE';`, ), table: msg.tableEntry.tableName, columns: columns, - data: row, - previous_data: msg.previous_data?.[index] + row: row, + previous_row: msg.rows_before?.[index] }); } return null; @@ -621,7 +506,7 @@ AND table_type = 'BASE TABLE';`, switch (payload.type) { case storage.SaveOperationTag.INSERT: this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); - const record = common.toSQLiteRow(payload.data, payload.columns); + const record = common.toSQLiteRow(payload.row, payload.columns); return await batch.save({ tag: storage.SaveOperationTag.INSERT, sourceTable: payload.sourceTable, @@ -634,10 +519,10 @@ AND table_type = 'BASE TABLE';`, this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); // "before" may be null if the replica id columns are unchanged // It's fine to treat that the same as an insert. - const beforeUpdated = payload.previous_data - ? common.toSQLiteRow(payload.previous_data, payload.columns) + const beforeUpdated = payload.previous_row + ? common.toSQLiteRow(payload.previous_row, payload.columns) : undefined; - const after = common.toSQLiteRow(payload.data, payload.columns); + const after = common.toSQLiteRow(payload.row, payload.columns); return await batch.save({ tag: storage.SaveOperationTag.UPDATE, @@ -652,7 +537,7 @@ AND table_type = 'BASE TABLE';`, case storage.SaveOperationTag.DELETE: this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); - const beforeDeleted = common.toSQLiteRow(payload.data, payload.columns); + const beforeDeleted = common.toSQLiteRow(payload.row, payload.columns); return await batch.save({ tag: storage.SaveOperationTag.DELETE, From 404dcdeddf1a6812fed1a6f4c788375113f2e2d8 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 21 May 2025 16:35:02 +0200 Subject: [PATCH 04/48] Renamed BinlogListener to BinLogListener --- .../src/replication/BinLogStream.ts | 6 ++--- .../{BinlogListener.ts => BinLogListener.ts} | 12 ++++----- ...istener.test.ts => BinLogListener.test.ts} | 26 +++++++++---------- 3 files changed, 22 insertions(+), 22 deletions(-) rename modules/module-mysql/src/replication/zongji/{BinlogListener.ts => BinLogListener.ts} (96%) rename modules/module-mysql/test/src/{BinlogListener.test.ts => BinLogListener.test.ts} (85%) diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index be4fabfb..ff1120a5 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -16,7 +16,7 @@ import * as common from '../common/common-index.js'; import { createRandomServerId, escapeMysqlTableName } from '../utils/mysql-utils.js'; import { MySQLConnectionManager } from './MySQLConnectionManager.js'; import { ReplicationMetric } from '@powersync/service-types'; -import { BinlogEventHandler, BinlogListener, Row } from './zongji/BinlogListener.js'; +import { BinLogEventHandler, BinLogListener, Row } from './zongji/BinLogListener.js'; export interface BinLogStreamOptions { connections: MySQLConnectionManager; @@ -422,7 +422,7 @@ AND table_type = 'BASE TABLE';`, const binlogEventHandler = this.createBinlogEventHandler(batch); // Only listen for changes to tables in the sync rules const includedTables = [...this.tableCache.values()].map((table) => table.table); - const binlogListener = new BinlogListener({ + const binlogListener = new BinLogListener({ abortSignal: this.abortSignal, includedTables: includedTables, startPosition: binLogPositionState, @@ -437,7 +437,7 @@ AND table_type = 'BASE TABLE';`, } } - private createBinlogEventHandler(batch: storage.BucketStorageBatch): BinlogEventHandler { + private createBinlogEventHandler(batch: storage.BucketStorageBatch): BinLogEventHandler { return { onWrite: async (rows: Row[], tableMap: TableMapEntry) => { await this.writeChanges(batch, { diff --git a/modules/module-mysql/src/replication/zongji/BinlogListener.ts b/modules/module-mysql/src/replication/zongji/BinLogListener.ts similarity index 96% rename from modules/module-mysql/src/replication/zongji/BinlogListener.ts rename to modules/module-mysql/src/replication/zongji/BinLogListener.ts index f67af740..9ee8563b 100644 --- a/modules/module-mysql/src/replication/zongji/BinlogListener.ts +++ b/modules/module-mysql/src/replication/zongji/BinLogListener.ts @@ -7,32 +7,32 @@ import { MySQLConnectionManager } from '../MySQLConnectionManager.js'; export type Row = Record; -export interface BinlogEventHandler { +export interface BinLogEventHandler { onWrite: (rows: Row[], tableMap: TableMapEntry) => Promise; onUpdate: (rowsAfter: Row[], rowsBefore: Row[], tableMap: TableMapEntry) => Promise; onDelete: (rows: Row[], tableMap: TableMapEntry) => Promise; onCommit: (lsn: string) => Promise; } -export interface BinlogListenerOptions { +export interface BinLogListenerOptions { connectionManager: MySQLConnectionManager; - eventHandler: BinlogEventHandler; + eventHandler: BinLogEventHandler; includedTables: string[]; serverId: number; startPosition: common.BinLogPosition; abortSignal: AbortSignal; } -export class BinlogListener { +export class BinLogListener { private connectionManager: MySQLConnectionManager; - private eventHandler: BinlogEventHandler; + private eventHandler: BinLogEventHandler; private binLogPosition: common.BinLogPosition; private currentGTID: common.ReplicatedGTID | null; zongji: ZongJi; processingQueue: async.QueueObject; - constructor(public options: BinlogListenerOptions) { + constructor(public options: BinLogListenerOptions) { this.connectionManager = options.connectionManager; this.eventHandler = options.eventHandler; this.binLogPosition = options.startPosition; diff --git a/modules/module-mysql/test/src/BinlogListener.test.ts b/modules/module-mysql/test/src/BinLogListener.test.ts similarity index 85% rename from modules/module-mysql/test/src/BinlogListener.test.ts rename to modules/module-mysql/test/src/BinLogListener.test.ts index 278812cd..038621c6 100644 --- a/modules/module-mysql/test/src/BinlogListener.test.ts +++ b/modules/module-mysql/test/src/BinLogListener.test.ts @@ -1,5 +1,5 @@ import { describe, test, beforeEach, vi, expect, afterEach } from 'vitest'; -import { BinlogEventHandler, BinlogListener, Row } from '@module/replication/zongji/BinlogListener.js'; +import { BinLogEventHandler, BinLogListener, Row } from '@module/replication/zongji/BinLogListener.js'; import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManager.js'; import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js'; import { v4 as uuid } from 'uuid'; @@ -16,8 +16,8 @@ describe('BinlogListener tests', () => { let connectionManager: MySQLConnectionManager; let abortController: AbortController; - let eventHandler: TestBinlogEventHandler; - let binlogListener: BinlogListener; + let eventHandler: TestBinLogEventHandler; + let binLogListener: BinLogListener; beforeEach(async () => { connectionManager = new MySQLConnectionManager(BINLOG_LISTENER_CONNECTION_OPTIONS, {}); @@ -28,8 +28,8 @@ describe('BinlogListener tests', () => { const fromGTID = await getFromGTID(connectionManager); abortController = new AbortController(); - eventHandler = new TestBinlogEventHandler(); - binlogListener = new BinlogListener({ + eventHandler = new TestBinLogEventHandler(); + binLogListener = new BinLogListener({ connectionManager: connectionManager, eventHandler: eventHandler, startPosition: fromGTID.position, @@ -44,22 +44,22 @@ describe('BinlogListener tests', () => { }); test('Binlog listener stops on abort signal', async () => { - const stopSpy = vi.spyOn(binlogListener.zongji, 'stop'); + const stopSpy = vi.spyOn(binLogListener.zongji, 'stop'); setTimeout(() => abortController.abort(), 10); - await expect(binlogListener.start()).resolves.toBeUndefined(); + await expect(binLogListener.start()).resolves.toBeUndefined(); expect(stopSpy).toHaveBeenCalled(); }); test('Pause Zongji binlog listener when processing queue reaches max size', async () => { - const pauseSpy = vi.spyOn(binlogListener.zongji, 'pause'); - const resumeSpy = vi.spyOn(binlogListener.zongji, 'resume'); - const queueSpy = vi.spyOn(binlogListener.processingQueue, 'length'); + const pauseSpy = vi.spyOn(binLogListener.zongji, 'pause'); + const resumeSpy = vi.spyOn(binLogListener.zongji, 'resume'); + const queueSpy = vi.spyOn(binLogListener.processingQueue, 'length'); const ROW_COUNT = 100; await insertRows(connectionManager, ROW_COUNT); - const startPromise = binlogListener.start(); + const startPromise = binLogListener.start(); await vi.waitFor(() => expect(eventHandler.rowsWritten).equals(ROW_COUNT), { timeout: 5000 }); abortController.abort(); @@ -72,7 +72,7 @@ describe('BinlogListener tests', () => { }); test('Binlog events are correctly forwarded to provided binlog events handler', async () => { - const startPromise = binlogListener.start(); + const startPromise = binLogListener.start(); const ROW_COUNT = 10; await insertRows(connectionManager, ROW_COUNT); @@ -112,7 +112,7 @@ async function deleteRows(connectionManager: MySQLConnectionManager) { await connectionManager.query(`DELETE FROM test_DATA`); } -class TestBinlogEventHandler implements BinlogEventHandler { +class TestBinLogEventHandler implements BinLogEventHandler { rowsWritten = 0; rowsUpdated = 0; rowsDeleted = 0; From b1b8c30074dc0d33e7a4d1cf4288f784e35149a6 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 21 May 2025 16:42:21 +0200 Subject: [PATCH 05/48] Added changeset --- .changeset/honest-ties-crash.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/honest-ties-crash.md diff --git a/.changeset/honest-ties-crash.md b/.changeset/honest-ties-crash.md new file mode 100644 index 00000000..1644a321 --- /dev/null +++ b/.changeset/honest-ties-crash.md @@ -0,0 +1,5 @@ +--- +'@powersync/service-module-mysql': minor +--- + +Added a configurable limit for the MySQL binlog processing queue to limit memory usage. From 126f9b3104122bafe71068a7d0f66578a409b79e Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Thu, 22 May 2025 15:25:12 +0200 Subject: [PATCH 06/48] Simplified BinLogListener stopping mechanism Cleaned up BinLogStream logs a bit --- modules/module-mysql/package.json | 2 +- .../src/replication/BinLogStream.ts | 16 +++++++++--- .../src/replication/zongji/BinLogListener.ts | 25 ++++++++----------- .../src/replication/zongji/zongji-utils.ts | 10 ++++---- .../test/src/BinLogListener.test.ts | 19 +++++++------- pnpm-lock.yaml | 20 +++++++++------ 6 files changed, 51 insertions(+), 41 deletions(-) diff --git a/modules/module-mysql/package.json b/modules/module-mysql/package.json index e4bea339..2d19145a 100644 --- a/modules/module-mysql/package.json +++ b/modules/module-mysql/package.json @@ -33,7 +33,7 @@ "@powersync/service-sync-rules": "workspace:*", "@powersync/service-types": "workspace:*", "@powersync/service-jsonbig": "workspace:*", - "@powersync/mysql-zongji": "0.0.0-dev-20250521092520", + "@powersync/mysql-zongji": "0.0.0-dev-20250522110942", "async": "^3.2.4", "mysql2": "^3.11.0", "semver": "^7.5.4", diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index ff1120a5..f8be521b 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -246,7 +246,7 @@ AND table_type = 'BASE TABLE';`, const isAvailable = await common.isBinlogStillAvailable(connection, lastKnowGTID.position.filename); if (!isAvailable) { logger.info( - `Binlog file ${lastKnowGTID.position.filename} is no longer available, starting initial replication again.` + `BinLog file ${lastKnowGTID.position.filename} is no longer available, starting initial replication again.` ); } return isAvailable; @@ -355,7 +355,7 @@ AND table_type = 'BASE TABLE';`, // all connections automatically closed, including this one. await this.initReplication(); await this.streamChanges(); - logger.info('BinlogStream has been shut down'); + logger.info('BinLogStream has been shut down.'); } catch (e) { await this.storage.reportError(e); throw e; @@ -368,7 +368,7 @@ AND table_type = 'BASE TABLE';`, connection.release(); if (errors.length > 0) { - throw new BinlogConfigurationError(`Binlog Configuration Errors: ${errors.join(', ')}`); + throw new BinlogConfigurationError(`BinLog Configuration Errors: ${errors.join(', ')}`); } const initialReplicationCompleted = await this.checkInitialReplicated(); @@ -423,7 +423,6 @@ AND table_type = 'BASE TABLE';`, // Only listen for changes to tables in the sync rules const includedTables = [...this.tableCache.values()].map((table) => table.table); const binlogListener = new BinLogListener({ - abortSignal: this.abortSignal, includedTables: includedTables, startPosition: binLogPositionState, connectionManager: this.connections, @@ -431,6 +430,15 @@ AND table_type = 'BASE TABLE';`, eventHandler: binlogEventHandler }); + this.abortSignal.addEventListener( + 'abort', + () => { + logger.info('Abort signal received, stopping replication...'); + binlogListener.stop(); + }, + { once: true } + ); + await binlogListener.start(); } ); diff --git a/modules/module-mysql/src/replication/zongji/BinLogListener.ts b/modules/module-mysql/src/replication/zongji/BinLogListener.ts index 9ee8563b..d0dfef4a 100644 --- a/modules/module-mysql/src/replication/zongji/BinLogListener.ts +++ b/modules/module-mysql/src/replication/zongji/BinLogListener.ts @@ -20,9 +20,12 @@ export interface BinLogListenerOptions { includedTables: string[]; serverId: number; startPosition: common.BinLogPosition; - abortSignal: AbortSignal; } +/** + * Wrapper class for the Zongji BinLog listener. Internally handles the creation and management of the listener and posts + * events on the provided BinLogEventHandler. + */ export class BinLogListener { private connectionManager: MySQLConnectionManager; private eventHandler: BinLogEventHandler; @@ -96,24 +99,16 @@ export class BinLogListener { }); this.zongji.on('stopped', () => { - logger.info('Binlog listener stopped. Replication ended.'); - resolve(); - }); - - const stop = () => { - logger.info('Abort signal received, stopping replication...'); - this.zongji.stop(); this.processingQueue.kill(); resolve(); - }; + }); + }); - this.options.abortSignal.addEventListener('abort', stop, { once: true }); + logger.info('BinLog listener stopped. Replication ended.'); + } - if (this.options.abortSignal.aborted) { - // Generally this should have been picked up early, but we add this here as a failsafe. - stop(); - } - }); + public stop(): void { + this.zongji.stop(); } private createZongjiListener(): ZongJi { diff --git a/modules/module-mysql/src/replication/zongji/zongji-utils.ts b/modules/module-mysql/src/replication/zongji/zongji-utils.ts index 24d01663..ee9e4c53 100644 --- a/modules/module-mysql/src/replication/zongji/zongji-utils.ts +++ b/modules/module-mysql/src/replication/zongji/zongji-utils.ts @@ -1,10 +1,10 @@ import { BinLogEvent, BinLogGTIDLogEvent, - BinLogMutationEvent, + BinLogRowEvent, BinLogRotationEvent, BinLogTableMapEvent, - BinLogUpdateEvent, + BinLogRowUpdateEvent, BinLogXidEvent } from '@powersync/mysql-zongji'; @@ -24,14 +24,14 @@ export function eventIsRotation(event: BinLogEvent): event is BinLogRotationEven return event.getEventName() == 'rotate'; } -export function eventIsWriteMutation(event: BinLogEvent): event is BinLogMutationEvent { +export function eventIsWriteMutation(event: BinLogEvent): event is BinLogRowEvent { return event.getEventName() == 'writerows'; } -export function eventIsDeleteMutation(event: BinLogEvent): event is BinLogMutationEvent { +export function eventIsDeleteMutation(event: BinLogEvent): event is BinLogRowEvent { return event.getEventName() == 'deleterows'; } -export function eventIsUpdateMutation(event: BinLogEvent): event is BinLogUpdateEvent { +export function eventIsUpdateMutation(event: BinLogEvent): event is BinLogRowUpdateEvent { return event.getEventName() == 'updaterows'; } diff --git a/modules/module-mysql/test/src/BinLogListener.test.ts b/modules/module-mysql/test/src/BinLogListener.test.ts index 038621c6..cf385f52 100644 --- a/modules/module-mysql/test/src/BinLogListener.test.ts +++ b/modules/module-mysql/test/src/BinLogListener.test.ts @@ -15,7 +15,6 @@ describe('BinlogListener tests', () => { }; let connectionManager: MySQLConnectionManager; - let abortController: AbortController; let eventHandler: TestBinLogEventHandler; let binLogListener: BinLogListener; @@ -27,15 +26,13 @@ describe('BinlogListener tests', () => { connection.release(); const fromGTID = await getFromGTID(connectionManager); - abortController = new AbortController(); eventHandler = new TestBinLogEventHandler(); binLogListener = new BinLogListener({ connectionManager: connectionManager, eventHandler: eventHandler, startPosition: fromGTID.position, includedTables: ['test_DATA'], - serverId: createRandomServerId(1), - abortSignal: abortController.signal + serverId: createRandomServerId(1) }); }); @@ -43,12 +40,16 @@ describe('BinlogListener tests', () => { await connectionManager.end(); }); - test('Binlog listener stops on abort signal', async () => { + test('Stop binlog listener', async () => { const stopSpy = vi.spyOn(binLogListener.zongji, 'stop'); + const queueStopSpy = vi.spyOn(binLogListener.processingQueue, 'kill'); - setTimeout(() => abortController.abort(), 10); - await expect(binLogListener.start()).resolves.toBeUndefined(); + const startPromise = binLogListener.start(); + setTimeout(async () => binLogListener.stop(), 50); + + await expect(startPromise).resolves.toBeUndefined(); expect(stopSpy).toHaveBeenCalled(); + expect(queueStopSpy).toHaveBeenCalled(); }); test('Pause Zongji binlog listener when processing queue reaches max size', async () => { @@ -62,7 +63,7 @@ describe('BinlogListener tests', () => { const startPromise = binLogListener.start(); await vi.waitFor(() => expect(eventHandler.rowsWritten).equals(ROW_COUNT), { timeout: 5000 }); - abortController.abort(); + binLogListener.stop(); await expect(startPromise).resolves.toBeUndefined(); // Count how many times the queue reached the max size. Consequently, we expect the listener to have paused and resumed that many times. @@ -85,7 +86,7 @@ describe('BinlogListener tests', () => { await deleteRows(connectionManager); await vi.waitFor(() => expect(eventHandler.rowsDeleted).equals(ROW_COUNT), { timeout: 5000 }); - abortController.abort(); + binLogListener.stop(); await expect(startPromise).resolves.toBeUndefined(); }); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 60d6c7b2..33c54c7d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -258,8 +258,8 @@ importers: specifier: workspace:* version: link:../../libs/lib-services '@powersync/mysql-zongji': - specifier: 0.0.0-dev-20250521092520 - version: 0.0.0-dev-20250521092520 + specifier: 0.0.0-dev-20250522110942 + version: 0.0.0-dev-20250522110942 '@powersync/service-core': specifier: workspace:* version: link:../../packages/service-core @@ -1281,9 +1281,9 @@ packages: resolution: {integrity: sha512-UA91GwWPhFExt3IizW6bOeY/pQ0BkuNwKjk9iQW9KqxluGCrg4VenZ0/L+2Y0+ZOtme72EVvg6v0zo3AMQRCeA==} engines: {node: '>=12'} - '@powersync/mysql-zongji@0.0.0-dev-20250521092520': - resolution: {integrity: sha512-AZ03eO5O/LQ8MFl/Z6OWyLJ4Mykd/gSbfIA8Iy0XImIKQt+XY8MqvtU/u3LLIZOJ+1ea43h0BfPvnFMsgwVxZg==} - engines: {node: '>=20.0.0'} + '@powersync/mysql-zongji@0.0.0-dev-20250522110942': + resolution: {integrity: sha512-6Sx6FUQeWBdOxUp8NucRAAyMKc+jkWIKZoPW4V2Y8hiEJZsqEOK7+HEOvkVMTHKF/dK5MOGtNk5tcUqbd23d2g==} + engines: {node: '>=22.0.0'} '@powersync/service-jsonbig@0.17.10': resolution: {integrity: sha512-BgxgUewuw4HFCM9MzuzlIuRKHya6rimNPYqUItt7CO3ySUeUnX8Qn9eZpMxu9AT5Y8zqkSyxvduY36zZueNojg==} @@ -1753,6 +1753,10 @@ packages: resolution: {integrity: sha512-GPEid2Y9QU1Exl1rpO9B2IPJGHPSupF5GnVIP0blYvNOMer2bTvSWs1jGOUg04hTmu67nmLsQ9TBo1puaotBHg==} engines: {node: '>=0.6'} + big-integer@1.6.52: + resolution: {integrity: sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==} + engines: {node: '>=0.6'} + bignumber.js@9.1.1: resolution: {integrity: sha512-pHm4LsMJ6lzgNGVfZHjMoO8sdoRhOzOH4MLmY65Jg70bpxCKu5iOHNJyfF6OyvYw7t8Fpf35RuzUyqnQsj8Vig==} @@ -4744,10 +4748,10 @@ snapshots: '@pnpm/network.ca-file': 1.0.2 config-chain: 1.1.13 - '@powersync/mysql-zongji@0.0.0-dev-20250521092520': + '@powersync/mysql-zongji@0.0.0-dev-20250522110942': dependencies: '@vlasky/mysql': 2.18.6 - big-integer: 1.6.51 + big-integer: 1.6.52 iconv-lite: 0.6.3 '@powersync/service-jsonbig@0.17.10': @@ -5192,6 +5196,8 @@ snapshots: big-integer@1.6.51: {} + big-integer@1.6.52: {} + bignumber.js@9.1.1: {} binary-extensions@2.3.0: {} From a03260f8203db56f7c78190d5f0fea04626774b8 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Mon, 26 May 2025 14:22:13 +0200 Subject: [PATCH 07/48] Corrected BinLogListener name. Simplified BinLogListener stopping mechanism --- modules/module-mysql/package.json | 2 +- modules/module-mysql/src/replication/BinLogStream.ts | 4 +++- .../src/replication/zongji/BinLogListener.ts | 2 +- pnpm-lock.yaml | 10 +++++----- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/modules/module-mysql/package.json b/modules/module-mysql/package.json index 2d19145a..2db0f5e9 100644 --- a/modules/module-mysql/package.json +++ b/modules/module-mysql/package.json @@ -33,7 +33,7 @@ "@powersync/service-sync-rules": "workspace:*", "@powersync/service-types": "workspace:*", "@powersync/service-jsonbig": "workspace:*", - "@powersync/mysql-zongji": "0.0.0-dev-20250522110942", + "@powersync/mysql-zongji": "0.0.0-dev-20250526121208", "async": "^3.2.4", "mysql2": "^3.11.0", "semver": "^7.5.4", diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index f8be521b..5829fdaf 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -439,7 +439,9 @@ AND table_type = 'BASE TABLE';`, { once: true } ); - await binlogListener.start(); + if (!this.stopped) { + await binlogListener.start(); + } } ); } diff --git a/modules/module-mysql/src/replication/zongji/BinLogListener.ts b/modules/module-mysql/src/replication/zongji/BinLogListener.ts index d0dfef4a..7e2e08cf 100644 --- a/modules/module-mysql/src/replication/zongji/BinLogListener.ts +++ b/modules/module-mysql/src/replication/zongji/BinLogListener.ts @@ -73,7 +73,7 @@ export class BinLogListener { }); logger.info(`Reading binlog from: ${this.binLogPosition.filename}:${this.binLogPosition.offset}`); - this.zongji.start({ + await this.zongji.start({ // We ignore the unknown/heartbeat event since it currently serves no purpose other than to keep the connection alive // tablemap events always need to be included for the other row events to work includeEvents: ['tablemap', 'writerows', 'updaterows', 'deleterows', 'xid', 'rotate', 'gtidlog'], diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 33c54c7d..129b7ac3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -258,8 +258,8 @@ importers: specifier: workspace:* version: link:../../libs/lib-services '@powersync/mysql-zongji': - specifier: 0.0.0-dev-20250522110942 - version: 0.0.0-dev-20250522110942 + specifier: 0.0.0-dev-20250526121208 + version: 0.0.0-dev-20250526121208 '@powersync/service-core': specifier: workspace:* version: link:../../packages/service-core @@ -1281,8 +1281,8 @@ packages: resolution: {integrity: sha512-UA91GwWPhFExt3IizW6bOeY/pQ0BkuNwKjk9iQW9KqxluGCrg4VenZ0/L+2Y0+ZOtme72EVvg6v0zo3AMQRCeA==} engines: {node: '>=12'} - '@powersync/mysql-zongji@0.0.0-dev-20250522110942': - resolution: {integrity: sha512-6Sx6FUQeWBdOxUp8NucRAAyMKc+jkWIKZoPW4V2Y8hiEJZsqEOK7+HEOvkVMTHKF/dK5MOGtNk5tcUqbd23d2g==} + '@powersync/mysql-zongji@0.0.0-dev-20250526121208': + resolution: {integrity: sha512-YOMYUU7oTHDlMrgboy3Zj0StThlBc26yley4UCvtykwxcc75+OkZZ5f3flCkKuhUk8aghG5aNNpMtdAaR9uOWQ==} engines: {node: '>=22.0.0'} '@powersync/service-jsonbig@0.17.10': @@ -4748,7 +4748,7 @@ snapshots: '@pnpm/network.ca-file': 1.0.2 config-chain: 1.1.13 - '@powersync/mysql-zongji@0.0.0-dev-20250522110942': + '@powersync/mysql-zongji@0.0.0-dev-20250526121208': dependencies: '@vlasky/mysql': 2.18.6 big-integer: 1.6.52 From e147318136303eb5774abeaad0c446b35bac3f83 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Tue, 27 May 2025 09:42:13 +0200 Subject: [PATCH 08/48] Supply port for binlog listener connections. --- modules/module-mysql/src/replication/MySQLConnectionManager.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/module-mysql/src/replication/MySQLConnectionManager.ts b/modules/module-mysql/src/replication/MySQLConnectionManager.ts index d464a975..b648ab26 100644 --- a/modules/module-mysql/src/replication/MySQLConnectionManager.ts +++ b/modules/module-mysql/src/replication/MySQLConnectionManager.ts @@ -46,6 +46,7 @@ export class MySQLConnectionManager { createBinlogListener(): ZongJi { const listener = new ZongJi({ host: this.options.hostname, + port: this.options.port, user: this.options.username, password: this.options.password }); From 999a8dce6f2ba3efe38fcc6150e90f94e0665ac8 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Tue, 27 May 2025 11:57:33 +0200 Subject: [PATCH 09/48] Only set up binlog heartbeat once the listener is fully started up. Added a few more defensive stopped checks to the binlog listener --- modules/module-mysql/package.json | 2 +- .../src/replication/BinLogStream.ts | 5 +- .../src/replication/zongji/BinLogListener.ts | 103 +++++++++++------- pnpm-lock.yaml | 10 +- 4 files changed, 72 insertions(+), 48 deletions(-) diff --git a/modules/module-mysql/package.json b/modules/module-mysql/package.json index 2db0f5e9..eb0de824 100644 --- a/modules/module-mysql/package.json +++ b/modules/module-mysql/package.json @@ -33,7 +33,7 @@ "@powersync/service-sync-rules": "workspace:*", "@powersync/service-types": "workspace:*", "@powersync/service-jsonbig": "workspace:*", - "@powersync/mysql-zongji": "0.0.0-dev-20250526121208", + "@powersync/mysql-zongji": "0.0.0-dev-20250527085137", "async": "^3.2.4", "mysql2": "^3.11.0", "semver": "^7.5.4", diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index 5829fdaf..fe137ae0 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -439,9 +439,8 @@ AND table_type = 'BASE TABLE';`, { once: true } ); - if (!this.stopped) { - await binlogListener.start(); - } + // Only returns when the replication is stopped or interrupted by an error + await binlogListener.start(); } ); } diff --git a/modules/module-mysql/src/replication/zongji/BinLogListener.ts b/modules/module-mysql/src/replication/zongji/BinLogListener.ts index 7e2e08cf..21163f84 100644 --- a/modules/module-mysql/src/replication/zongji/BinLogListener.ts +++ b/modules/module-mysql/src/replication/zongji/BinLogListener.ts @@ -46,34 +46,12 @@ export class BinLogListener { } public async start(): Promise { + if (this.isStopped) { + return; + } logger.info(`Starting replication. Created replica client with serverId:${this.options.serverId}`); - // Set a heartbeat interval for the Zongji replication connection - // Zongji does not explicitly handle the heartbeat events - they are categorized as event:unknown - // The heartbeat events are enough to keep the connection alive for setTimeout to work on the socket. - await new Promise((resolve, reject) => { - this.zongji.connection.query( - // In nanoseconds, 10^9 = 1s - 'set @master_heartbeat_period=28*1000000000', - function (error: any, results: any, fields: any) { - if (error) { - reject(error); - } else { - resolve(results); - } - } - ); - }); - logger.info('Successfully set up replication connection heartbeat...'); - - // The _socket member is only set after a query is run on the connection, so we set the timeout after setting the heartbeat. - // The timeout here must be greater than the master_heartbeat_period. - const socket = this.zongji.connection._socket!; - socket.setTimeout(60_000, () => { - socket.destroy(new Error('Replication connection timeout.')); - }); - logger.info(`Reading binlog from: ${this.binLogPosition.filename}:${this.binLogPosition.offset}`); - await this.zongji.start({ + this.zongji.start({ // We ignore the unknown/heartbeat event since it currently serves no purpose other than to keep the connection alive // tablemap events always need to be included for the other row events to work includeEvents: ['tablemap', 'writerows', 'updaterows', 'deleterows', 'xid', 'rotate', 'gtidlog'], @@ -83,32 +61,49 @@ export class BinLogListener { serverId: this.options.serverId } satisfies StartOptions); - await new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { this.zongji.on('error', (error) => { - logger.error('Binlog listener error:', error); - this.zongji.stop(); - this.processingQueue.kill(); - reject(error); + if (!this.isStopped) { + logger.error('Binlog listener error:', error); + this.stop(); + reject(error); + } else { + logger.warn('Binlog listener error during shutdown:', error); + } }); this.processingQueue.error((error) => { - logger.error('BinlogEvent processing error:', error); - this.zongji.stop(); - this.processingQueue.kill(); - reject(error); + if (!this.isStopped) { + logger.error('BinlogEvent processing error:', error); + this.stop(); + reject(error); + } else { + logger.warn('BinlogEvent processing error during shutdown:', error); + } }); this.zongji.on('stopped', () => { - this.processingQueue.kill(); resolve(); + logger.info('BinLog listener stopped. Replication ended.'); }); - }); - logger.info('BinLog listener stopped. Replication ended.'); + // Handle the edge case where the listener has already been stopped before completing startup + if (this.isStopped) { + logger.info('BinLog listener was stopped before startup completed.'); + resolve(); + } + }); } public stop(): void { - this.zongji.stop(); + if (!this.isStopped) { + this.zongji.stop(); + this.processingQueue.kill(); + } + } + + private get isStopped(): boolean { + return this.zongji.stopped; } private createZongjiListener(): ZongJi { @@ -130,6 +125,36 @@ export class BinLogListener { } }); + zongji.on('ready', async () => { + // Set a heartbeat interval for the Zongji replication connection + // Zongji does not explicitly handle the heartbeat events - they are categorized as event:unknown + // The heartbeat events are enough to keep the connection alive for setTimeout to work on the socket. + await new Promise((resolve, reject) => { + this.zongji.connection.query( + // In nanoseconds, 10^9 = 1s + 'set @master_heartbeat_period=28*1000000000', + function (error: any, results: any, fields: any) { + if (error) { + reject(error); + } else { + logger.info('Successfully set up replication connection heartbeat...'); + resolve(results); + } + } + ); + }); + + // The _socket member is only set after a query is run on the connection, so we set the timeout after setting the heartbeat. + // The timeout here must be greater than the master_heartbeat_period. + const socket = this.zongji.connection._socket!; + socket.setTimeout(60_000, () => { + socket.destroy(new Error('Replication connection timeout.')); + }); + logger.info( + `BinLog listener setup complete. Reading binlog from: ${this.binLogPosition.filename}:${this.binLogPosition.offset}` + ); + }); + return zongji; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 129b7ac3..94b215cf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -258,8 +258,8 @@ importers: specifier: workspace:* version: link:../../libs/lib-services '@powersync/mysql-zongji': - specifier: 0.0.0-dev-20250526121208 - version: 0.0.0-dev-20250526121208 + specifier: 0.0.0-dev-20250527085137 + version: 0.0.0-dev-20250527085137 '@powersync/service-core': specifier: workspace:* version: link:../../packages/service-core @@ -1281,8 +1281,8 @@ packages: resolution: {integrity: sha512-UA91GwWPhFExt3IizW6bOeY/pQ0BkuNwKjk9iQW9KqxluGCrg4VenZ0/L+2Y0+ZOtme72EVvg6v0zo3AMQRCeA==} engines: {node: '>=12'} - '@powersync/mysql-zongji@0.0.0-dev-20250526121208': - resolution: {integrity: sha512-YOMYUU7oTHDlMrgboy3Zj0StThlBc26yley4UCvtykwxcc75+OkZZ5f3flCkKuhUk8aghG5aNNpMtdAaR9uOWQ==} + '@powersync/mysql-zongji@0.0.0-dev-20250527085137': + resolution: {integrity: sha512-3NPUfq1rcLpTCMkOwCGsJeS/zKoidDYsPqrJ/sy4Vcsgp1vXMkdnLlQyiAq7vYu5a15zUTK+mQoVruwUi82ANg==} engines: {node: '>=22.0.0'} '@powersync/service-jsonbig@0.17.10': @@ -4748,7 +4748,7 @@ snapshots: '@pnpm/network.ca-file': 1.0.2 config-chain: 1.1.13 - '@powersync/mysql-zongji@0.0.0-dev-20250526121208': + '@powersync/mysql-zongji@0.0.0-dev-20250527085137': dependencies: '@vlasky/mysql': 2.18.6 big-integer: 1.6.52 From 07201e8e321a9aa5a63a530ae5829aeddc70c057 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Tue, 27 May 2025 13:29:25 +0200 Subject: [PATCH 10/48] Updated changeset --- .changeset/honest-ties-crash.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.changeset/honest-ties-crash.md b/.changeset/honest-ties-crash.md index 1644a321..71533ce1 100644 --- a/.changeset/honest-ties-crash.md +++ b/.changeset/honest-ties-crash.md @@ -3,3 +3,5 @@ --- Added a configurable limit for the MySQL binlog processing queue to limit memory usage. +Removed MySQL Zongji type definitions, they are now instead imported from the `@powersync/mysql-zongji` package. +Now passing in port for the Zongji connection, so that it can be used with MySQL servers that are not running on the default port 3306. From 079a2f5f8d693735fc92871e31328324fb57746e Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Thu, 29 May 2025 13:02:44 +0200 Subject: [PATCH 11/48] Changed binlog backpressure mechanism to be based on processing queue memory usage rather than number of events --- modules/module-mysql/package.json | 2 +- .../src/replication/zongji/BinLogListener.ts | 36 +++++++++++---- modules/module-mysql/src/types/types.ts | 10 ++--- .../test/src/BinLogListener.test.ts | 45 ++++++++++++++----- pnpm-lock.yaml | 10 ++--- 5 files changed, 72 insertions(+), 31 deletions(-) diff --git a/modules/module-mysql/package.json b/modules/module-mysql/package.json index eb0de824..d8a4ec0d 100644 --- a/modules/module-mysql/package.json +++ b/modules/module-mysql/package.json @@ -33,7 +33,7 @@ "@powersync/service-sync-rules": "workspace:*", "@powersync/service-types": "workspace:*", "@powersync/service-jsonbig": "workspace:*", - "@powersync/mysql-zongji": "0.0.0-dev-20250527085137", + "@powersync/mysql-zongji": "0.0.0-dev-20250528105319", "async": "^3.2.4", "mysql2": "^3.11.0", "semver": "^7.5.4", diff --git a/modules/module-mysql/src/replication/zongji/BinLogListener.ts b/modules/module-mysql/src/replication/zongji/BinLogListener.ts index 21163f84..0a1adfce 100644 --- a/modules/module-mysql/src/replication/zongji/BinLogListener.ts +++ b/modules/module-mysql/src/replication/zongji/BinLogListener.ts @@ -34,6 +34,10 @@ export class BinLogListener { zongji: ZongJi; processingQueue: async.QueueObject; + /** + * The combined size in bytes of all the binlog events currently in the processing queue. + */ + queueMemoryUsage: number; constructor(public options: BinLogListenerOptions) { this.connectionManager = options.connectionManager; @@ -42,9 +46,18 @@ export class BinLogListener { this.currentGTID = null; this.processingQueue = async.queue(this.createQueueWorker(), 1); + this.queueMemoryUsage = 0; this.zongji = this.createZongjiListener(); } + /** + * The queue memory limit in bytes as defined in the connection options. + * @private + */ + private get queueMemoryLimit(): number { + return this.connectionManager.options.binlog_queue_memory_limit * 1024 * 1024; + } + public async start(): Promise { if (this.isStopped) { return; @@ -62,6 +75,12 @@ export class BinLogListener { } satisfies StartOptions); return new Promise((resolve, reject) => { + // Handle an edge case where the listener has already been stopped before completing startup + if (this.isStopped) { + logger.info('BinLog listener was stopped before startup completed.'); + resolve(); + } + this.zongji.on('error', (error) => { if (!this.isStopped) { logger.error('Binlog listener error:', error); @@ -86,12 +105,6 @@ export class BinLogListener { resolve(); logger.info('BinLog listener stopped. Replication ended.'); }); - - // Handle the edge case where the listener has already been stopped before completing startup - if (this.isStopped) { - logger.info('BinLog listener was stopped before startup completed.'); - resolve(); - } }); } @@ -112,11 +125,12 @@ export class BinLogListener { zongji.on('binlog', async (evt) => { logger.info(`Received Binlog event:${evt.getEventName()}`); this.processingQueue.push(evt); + this.queueMemoryUsage += evt.size; // When the processing queue grows past the threshold, we pause the binlog listener - if (this.processingQueue.length() > this.connectionManager.options.max_binlog_queue_size) { + if (this.isQueueOverCapacity()) { logger.info( - `Max Binlog processing queue length [${this.connectionManager.options.max_binlog_queue_size}] reached. Pausing Binlog listener.` + `Binlog processing queue has reached its memory limit of [${this.connectionManager.options.binlog_queue_memory_limit}MB]. Pausing Binlog listener.` ); zongji.pause(); await this.processingQueue.empty(); @@ -201,6 +215,12 @@ export class BinLogListener { await this.eventHandler.onCommit(LSN); break; } + + this.queueMemoryUsage -= evt.size; }; } + + isQueueOverCapacity(): boolean { + return this.queueMemoryUsage >= this.queueMemoryLimit; + } } diff --git a/modules/module-mysql/src/types/types.ts b/modules/module-mysql/src/types/types.ts index d8079d1f..aae72f5b 100644 --- a/modules/module-mysql/src/types/types.ts +++ b/modules/module-mysql/src/types/types.ts @@ -24,7 +24,7 @@ export interface NormalizedMySQLConnectionConfig { lookup?: LookupFunction; - max_binlog_queue_size: number; + binlog_queue_memory_limit: number; } export const MySQLConnectionConfig = service_types.configFile.DataSourceConfig.and( @@ -43,8 +43,8 @@ export const MySQLConnectionConfig = service_types.configFile.DataSourceConfig.a client_private_key: t.string.optional(), reject_ip_ranges: t.array(t.string).optional(), - // The maximum number of binlog events that can be queued in memory before throttling is applied. - max_binlog_queue_size: t.number.optional() + // The combined size of binlog events that can be queued in memory before throttling is applied. + binlog_queue_memory_limit: t.number.optional() }) ); @@ -118,8 +118,8 @@ export function normalizeConnectionConfig(options: MySQLConnectionConfig): Norma server_id: options.server_id ?? 1, - // Based on profiling, a queue size of 1000 uses about 50MB of memory. - max_binlog_queue_size: options.max_binlog_queue_size ?? 1000, + // Binlog processing queue memory limit before throttling is applied. + binlog_queue_memory_limit: options.binlog_queue_memory_limit ?? 50, lookup }; diff --git a/modules/module-mysql/test/src/BinLogListener.test.ts b/modules/module-mysql/test/src/BinLogListener.test.ts index cf385f52..263eff26 100644 --- a/modules/module-mysql/test/src/BinLogListener.test.ts +++ b/modules/module-mysql/test/src/BinLogListener.test.ts @@ -6,12 +6,13 @@ import { v4 as uuid } from 'uuid'; import * as common from '@module/common/common-index.js'; import { createRandomServerId } from '@module/utils/mysql-utils.js'; import { TableMapEntry } from '@powersync/mysql-zongji'; +import crypto from 'crypto'; describe('BinlogListener tests', () => { - const MAX_QUEUE_SIZE = 10; + const MAX_QUEUE_CAPACITY_MB = 1; const BINLOG_LISTENER_CONNECTION_OPTIONS = { ...TEST_CONNECTION_OPTIONS, - max_binlog_queue_size: MAX_QUEUE_SIZE + binlog_queue_memory_limit: MAX_QUEUE_CAPACITY_MB }; let connectionManager: MySQLConnectionManager; @@ -22,7 +23,7 @@ describe('BinlogListener tests', () => { connectionManager = new MySQLConnectionManager(BINLOG_LISTENER_CONNECTION_OPTIONS, {}); const connection = await connectionManager.getConnection(); await clearTestDb(connection); - await connection.query(`CREATE TABLE test_DATA (id CHAR(36) PRIMARY KEY, description text)`); + await connection.query(`CREATE TABLE test_DATA (id CHAR(36) PRIMARY KEY, description MEDIUMTEXT)`); connection.release(); const fromGTID = await getFromGTID(connectionManager); @@ -52,24 +53,30 @@ describe('BinlogListener tests', () => { expect(queueStopSpy).toHaveBeenCalled(); }); - test('Pause Zongji binlog listener when processing queue reaches max size', async () => { + test('Pause Zongji binlog listener when processing queue reaches maximum memory size', async () => { const pauseSpy = vi.spyOn(binLogListener.zongji, 'pause'); const resumeSpy = vi.spyOn(binLogListener.zongji, 'resume'); - const queueSpy = vi.spyOn(binLogListener.processingQueue, 'length'); - const ROW_COUNT = 100; + // Pause the event handler to force a backlog on the processing queue + eventHandler.pause(); + + const ROW_COUNT = 10; await insertRows(connectionManager, ROW_COUNT); const startPromise = binLogListener.start(); + // Wait for listener to pause due to queue reaching capacity + await vi.waitFor(() => expect(pauseSpy).toHaveBeenCalled(), { timeout: 5000 }); + + expect(binLogListener.isQueueOverCapacity()).toBeTruthy(); + // Resume event processing + eventHandler.unpause!(); + await vi.waitFor(() => expect(eventHandler.rowsWritten).equals(ROW_COUNT), { timeout: 5000 }); binLogListener.stop(); await expect(startPromise).resolves.toBeUndefined(); - - // Count how many times the queue reached the max size. Consequently, we expect the listener to have paused and resumed that many times. - const overThresholdCount = queueSpy.mock.results.map((r) => r.value).filter((v) => v === MAX_QUEUE_SIZE).length; - expect(pauseSpy).toHaveBeenCalledTimes(overThresholdCount); - expect(resumeSpy).toHaveBeenCalledTimes(overThresholdCount); + // Confirm resume was called after unpausing + expect(resumeSpy).toHaveBeenCalled(); }); test('Binlog events are correctly forwarded to provided binlog events handler', async () => { @@ -101,7 +108,9 @@ async function getFromGTID(connectionManager: MySQLConnectionManager) { async function insertRows(connectionManager: MySQLConnectionManager, count: number) { for (let i = 0; i < count; i++) { - await connectionManager.query(`INSERT INTO test_DATA(id, description) VALUES('${uuid()}','test${i}')`); + await connectionManager.query( + `INSERT INTO test_DATA(id, description) VALUES('${uuid()}','test${i} ${crypto.randomBytes(100_000).toString('hex')}')` + ); } } @@ -119,7 +128,19 @@ class TestBinLogEventHandler implements BinLogEventHandler { rowsDeleted = 0; commitCount = 0; + unpause: ((value: void | PromiseLike) => void) | undefined; + private pausedPromise: Promise | undefined; + + pause() { + this.pausedPromise = new Promise((resolve) => { + this.unpause = resolve; + }); + } + async onWrite(rows: Row[], tableMap: TableMapEntry) { + if (this.pausedPromise) { + await this.pausedPromise; + } this.rowsWritten = this.rowsWritten + rows.length; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 94b215cf..bba138f3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -258,8 +258,8 @@ importers: specifier: workspace:* version: link:../../libs/lib-services '@powersync/mysql-zongji': - specifier: 0.0.0-dev-20250527085137 - version: 0.0.0-dev-20250527085137 + specifier: 0.0.0-dev-20250528105319 + version: 0.0.0-dev-20250528105319 '@powersync/service-core': specifier: workspace:* version: link:../../packages/service-core @@ -1281,8 +1281,8 @@ packages: resolution: {integrity: sha512-UA91GwWPhFExt3IizW6bOeY/pQ0BkuNwKjk9iQW9KqxluGCrg4VenZ0/L+2Y0+ZOtme72EVvg6v0zo3AMQRCeA==} engines: {node: '>=12'} - '@powersync/mysql-zongji@0.0.0-dev-20250527085137': - resolution: {integrity: sha512-3NPUfq1rcLpTCMkOwCGsJeS/zKoidDYsPqrJ/sy4Vcsgp1vXMkdnLlQyiAq7vYu5a15zUTK+mQoVruwUi82ANg==} + '@powersync/mysql-zongji@0.0.0-dev-20250528105319': + resolution: {integrity: sha512-67MRLJi7hHb0371/6gffkZlAaDoAFy1pVBGKP17i3MltumAF+ZlukC8Q0nKsqOcEwVMvbhmaalMeVgtJeZ/VfA==} engines: {node: '>=22.0.0'} '@powersync/service-jsonbig@0.17.10': @@ -4748,7 +4748,7 @@ snapshots: '@pnpm/network.ca-file': 1.0.2 config-chain: 1.1.13 - '@powersync/mysql-zongji@0.0.0-dev-20250527085137': + '@powersync/mysql-zongji@0.0.0-dev-20250528105319': dependencies: '@vlasky/mysql': 2.18.6 big-integer: 1.6.52 From 286ba164ec23a7afb55f9bdc08a2a0b1bdfc70a0 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Thu, 29 May 2025 16:20:23 +0200 Subject: [PATCH 12/48] Changed binlog backpressure mechanism to be based on processing queue memory usage rather than number of events. Introduced a maximum timeout that the binlog processing queue can be paused before auto-resuming. This is to prevent the replication connection timing out. --- modules/module-mysql/package.json | 2 +- .../src/replication/zongji/BinLogListener.ts | 12 +++++++++++- pnpm-lock.yaml | 10 +++++----- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/modules/module-mysql/package.json b/modules/module-mysql/package.json index 384326eb..09016a42 100644 --- a/modules/module-mysql/package.json +++ b/modules/module-mysql/package.json @@ -33,7 +33,7 @@ "@powersync/service-sync-rules": "workspace:*", "@powersync/service-types": "workspace:*", "@powersync/service-jsonbig": "workspace:*", - "@powersync/mysql-zongji": "0.0.0-dev-20250528105319", + "@powersync/mysql-zongji": "0.2.0", "async": "^3.2.4", "mysql2": "^3.11.0", "semver": "^7.5.4", diff --git a/modules/module-mysql/src/replication/zongji/BinLogListener.ts b/modules/module-mysql/src/replication/zongji/BinLogListener.ts index 0a1adfce..e22ff8b2 100644 --- a/modules/module-mysql/src/replication/zongji/BinLogListener.ts +++ b/modules/module-mysql/src/replication/zongji/BinLogListener.ts @@ -5,6 +5,10 @@ import * as zongji_utils from './zongji-utils.js'; import { logger } from '@powersync/lib-services-framework'; import { MySQLConnectionManager } from '../MySQLConnectionManager.js'; +// Maximum time the processing queue can be paused before resuming automatically +// MySQL server will automatically terminate replication connections after 60 seconds of inactivity, so this guards against that. +const MAX_QUEUE_PAUSE_TIME_MS = 45_000; + export type Row = Record; export interface BinLogEventHandler { @@ -133,7 +137,12 @@ export class BinLogListener { `Binlog processing queue has reached its memory limit of [${this.connectionManager.options.binlog_queue_memory_limit}MB]. Pausing Binlog listener.` ); zongji.pause(); - await this.processingQueue.empty(); + const resumeTimeoutPromise = new Promise((resolve) => { + setTimeout(() => resolve('timeout'), MAX_QUEUE_PAUSE_TIME_MS); + }); + + await Promise.race([this.processingQueue.empty(), resumeTimeoutPromise]); + logger.info(`Binlog processing queue backlog cleared. Resuming Binlog listener.`); zongji.resume(); } @@ -162,6 +171,7 @@ export class BinLogListener { // The timeout here must be greater than the master_heartbeat_period. const socket = this.zongji.connection._socket!; socket.setTimeout(60_000, () => { + logger.info('Destroying socket due to replication connection timeout.'); socket.destroy(new Error('Replication connection timeout.')); }); logger.info( diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bba138f3..c7367649 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -258,8 +258,8 @@ importers: specifier: workspace:* version: link:../../libs/lib-services '@powersync/mysql-zongji': - specifier: 0.0.0-dev-20250528105319 - version: 0.0.0-dev-20250528105319 + specifier: 0.2.0 + version: 0.2.0 '@powersync/service-core': specifier: workspace:* version: link:../../packages/service-core @@ -1281,8 +1281,8 @@ packages: resolution: {integrity: sha512-UA91GwWPhFExt3IizW6bOeY/pQ0BkuNwKjk9iQW9KqxluGCrg4VenZ0/L+2Y0+ZOtme72EVvg6v0zo3AMQRCeA==} engines: {node: '>=12'} - '@powersync/mysql-zongji@0.0.0-dev-20250528105319': - resolution: {integrity: sha512-67MRLJi7hHb0371/6gffkZlAaDoAFy1pVBGKP17i3MltumAF+ZlukC8Q0nKsqOcEwVMvbhmaalMeVgtJeZ/VfA==} + '@powersync/mysql-zongji@0.2.0': + resolution: {integrity: sha512-ua/n7WFfoiXmqfgwLikcm/AaDE6+t5gFVTWHWsbiuRQMNtXE1F2gXpZJdwKhr8WsOCYkB/A1ZOgbJKi4tK342g==} engines: {node: '>=22.0.0'} '@powersync/service-jsonbig@0.17.10': @@ -4748,7 +4748,7 @@ snapshots: '@pnpm/network.ca-file': 1.0.2 config-chain: 1.1.13 - '@powersync/mysql-zongji@0.0.0-dev-20250528105319': + '@powersync/mysql-zongji@0.2.0': dependencies: '@vlasky/mysql': 2.18.6 big-integer: 1.6.52 From 9a00b8ba424eae86959f12cf9ed505f7209121b9 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 4 Jun 2025 13:24:00 +0200 Subject: [PATCH 13/48] Added optional columns field to SourceEntityDescriptor Made SourceTable implement SourceEntityDescriptor interface --- .../implementation/MongoBucketBatch.ts | 18 ++--- .../implementation/MongoSyncBucketStorage.ts | 47 ++++++------- .../src/api/MongoRouteAPIAdapter.ts | 36 +++++----- .../src/replication/ChangeStream.ts | 4 +- .../src/replication/MongoRelation.ts | 2 +- .../src/api/MySQLRouteAPIAdapter.ts | 12 +++- .../src/replication/BinLogStream.ts | 6 +- modules/module-mysql/src/utils/mysql-utils.ts | 2 +- .../src/storage/PostgresSyncRulesStorage.ts | 49 ++++++------- .../src/storage/batch/PostgresBucketBatch.ts | 18 ++--- .../src/replication/PgRelation.ts | 4 +- .../src/replication/WalStream.ts | 4 +- .../src/replication/replication-utils.ts | 12 +++- .../src/test-utils/general-utils.ts | 18 ++--- packages/service-core/src/api/diagnostics.ts | 2 +- .../service-core/src/storage/SourceEntity.ts | 14 ++-- .../service-core/src/storage/SourceTable.ts | 69 ++++++++++++------- packages/sync-rules/src/BaseSqlDataQuery.ts | 4 +- .../sync-rules/src/SourceTableInterface.ts | 2 +- packages/sync-rules/src/SqlDataQuery.ts | 2 +- packages/sync-rules/src/StaticSchema.ts | 4 +- packages/sync-rules/src/TablePattern.ts | 4 +- .../src/events/SqlEventDescriptor.ts | 2 +- packages/sync-rules/src/types.ts | 2 +- packages/sync-rules/test/src/util.ts | 2 +- 25 files changed, 190 insertions(+), 149 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index c1640c89..c55fd71d 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -876,15 +876,15 @@ export class MongoBucketBatch } }); return tables.map((table) => { - const copy = new storage.SourceTable( - table.id, - table.connectionTag, - table.objectId, - table.schema, - table.table, - table.replicaIdColumns, - table.snapshotComplete - ); + const copy = new storage.SourceTable({ + id: table.id, + connectionTag: table.connectionTag, + objectId: table.objectId, + schema: table.schema, + name: table.name, + replicaIdColumns: table.replicaIdColumns, + snapshotComplete: table.snapshotComplete + }); copy.syncData = table.syncData; copy.syncParameters = table.syncParameters; return copy; diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 354b4aab..9f1ad8b9 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -162,9 +162,9 @@ export class MongoSyncBucketStorage async resolveTable(options: storage.ResolveTableOptions): Promise { const { group_id, connection_id, connection_tag, entity_descriptor } = options; - const { schema, name: table, objectId, replicationColumns } = entity_descriptor; + const { schema, name, objectId, replicaIdColumns } = entity_descriptor; - const columns = replicationColumns.map((column) => ({ + const columns = replicaIdColumns.map((column) => ({ name: column.name, type: column.type, type_oid: column.typeId @@ -176,7 +176,7 @@ export class MongoSyncBucketStorage group_id: group_id, connection_id: connection_id, schema_name: schema, - table_name: table, + table_name: name, replica_id_columns2: columns }; if (objectId != null) { @@ -190,7 +190,7 @@ export class MongoSyncBucketStorage connection_id: connection_id, relation_id: objectId, schema_name: schema, - table_name: table, + table_name: name, replica_id_columns: null, replica_id_columns2: columns, snapshot_done: false @@ -198,22 +198,22 @@ export class MongoSyncBucketStorage await col.insertOne(doc, { session }); } - const sourceTable = new storage.SourceTable( - doc._id, - connection_tag, - objectId, - schema, - table, - replicationColumns, - doc.snapshot_done ?? true - ); + const sourceTable = new storage.SourceTable({ + id: doc._id, + connectionTag: connection_tag, + objectId: objectId, + schema: schema, + name: name, + replicaIdColumns: replicaIdColumns, + snapshotComplete: doc.snapshot_done ?? true + }); sourceTable.syncEvent = options.sync_rules.tableTriggersEvent(sourceTable); sourceTable.syncData = options.sync_rules.tableSyncsData(sourceTable); sourceTable.syncParameters = options.sync_rules.tableSyncsParameters(sourceTable); let dropTables: storage.SourceTable[] = []; // Detect tables that are either renamed, or have different replica_id_columns - let truncateFilter = [{ schema_name: schema, table_name: table }] as any[]; + let truncateFilter = [{ schema_name: schema, table_name: name }] as any[]; if (objectId != null) { // Only detect renames if the source uses relation ids. truncateFilter.push({ relation_id: objectId }); @@ -231,15 +231,16 @@ export class MongoSyncBucketStorage .toArray(); dropTables = truncate.map( (doc) => - new storage.SourceTable( - doc._id, - connection_tag, - doc.relation_id, - doc.schema_name, - doc.table_name, - doc.replica_id_columns2?.map((c) => ({ name: c.name, typeOid: c.type_oid, type: c.type })) ?? [], - doc.snapshot_done ?? true - ) + new storage.SourceTable({ + id: doc._id, + connectionTag: connection_tag, + objectId: doc.relation_id, + schema: doc.schema_name, + name: doc.table_name, + replicaIdColumns: + doc.replica_id_columns2?.map((c) => ({ name: c.name, typeOid: c.type_oid, type: c.type })) ?? [], + snapshotComplete: doc.snapshot_done ?? true + }) ); result = { diff --git a/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts index 958517b9..2a74bc98 100644 --- a/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts +++ b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts @@ -137,15 +137,15 @@ export class MongoRouteAPIAdapter implements api.RouteAPI { if (tablePattern.isWildcard) { patternResult.tables = []; for (let collection of collections) { - const sourceTable = new SourceTable( - 0, - this.connectionTag, - collection.name, - schema, - collection.name, - [], - true - ); + const sourceTable = new SourceTable({ + id: 0, + connectionTag: this.connectionTag, + objectId: collection.name, + schema: schema, + name: collection.name, + replicaIdColumns: [], + snapshotComplete: true + }); let errors: service_types.ReplicationError[] = []; if (collection.type == 'view') { errors.push({ level: 'warning', message: `Collection ${schema}.${tablePattern.name} is a view` }); @@ -164,15 +164,15 @@ export class MongoRouteAPIAdapter implements api.RouteAPI { }); } } else { - const sourceTable = new SourceTable( - 0, - this.connectionTag, - tablePattern.name, - schema, - tablePattern.name, - [], - true - ); + const sourceTable = new SourceTable({ + id: 0, + connectionTag: this.connectionTag, + objectId: tablePattern.name, + schema: schema, + name: tablePattern.name, + replicaIdColumns: [], + snapshotComplete: true + }); const syncData = sqlSyncRules.tableSyncsData(sourceTable); const syncParameters = sqlSyncRules.tableSyncsParameters(sourceTable); diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index 95887221..7454e145 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -179,7 +179,7 @@ export class ChangeStream { async estimatedCount(table: storage.SourceTable): Promise { const db = this.client.db(table.schema); - const count = await db.collection(table.table).estimatedDocumentCount(); + const count = await db.collection(table.name).estimatedDocumentCount(); return `~${count}`; } @@ -307,7 +307,7 @@ export class ChangeStream { const estimatedCount = await this.estimatedCount(table); let at = 0; const db = this.client.db(table.schema); - const collection = db.collection(table.table); + const collection = db.collection(table.name); const cursor = collection.find({}, { batchSize: 6_000, readConcern: 'majority' }); let lastBatch = performance.now(); diff --git a/modules/module-mongodb/src/replication/MongoRelation.ts b/modules/module-mongodb/src/replication/MongoRelation.ts index 736ff149..e651ebcb 100644 --- a/modules/module-mongodb/src/replication/MongoRelation.ts +++ b/modules/module-mongodb/src/replication/MongoRelation.ts @@ -13,7 +13,7 @@ export function getMongoRelation(source: mongo.ChangeStreamNameSpace): storage.S schema: source.db, // Not relevant for MongoDB - we use db + coll name as the identifier objectId: undefined, - replicationColumns: [{ name: '_id' }] + replicaIdColumns: [{ name: '_id' }] } satisfies storage.SourceEntityDescriptor; } diff --git a/modules/module-mysql/src/api/MySQLRouteAPIAdapter.ts b/modules/module-mysql/src/api/MySQLRouteAPIAdapter.ts index ab0a7481..fd055672 100644 --- a/modules/module-mysql/src/api/MySQLRouteAPIAdapter.ts +++ b/modules/module-mysql/src/api/MySQLRouteAPIAdapter.ts @@ -217,7 +217,15 @@ export class MySQLRouteAPIAdapter implements api.RouteAPI { } const idColumns = idColumnsResult?.columns ?? []; - const sourceTable = new storage.SourceTable(0, this.config.tag, tableName, schema, tableName, idColumns, true); + const sourceTable = new storage.SourceTable({ + id: 0, + connectionTag: this.config.tag, + objectId: tableName, + schema: schema, + name: tableName, + replicaIdColumns: idColumns, + snapshotComplete: true + }); const syncData = syncRules.tableSyncsData(sourceTable); const syncParameters = syncRules.tableSyncsParameters(sourceTable); @@ -232,7 +240,7 @@ export class MySQLRouteAPIAdapter implements api.RouteAPI { let selectError: service_types.ReplicationError | null = null; try { await this.retriedQuery({ - query: `SELECT * FROM ${sourceTable.table} LIMIT 1` + query: `SELECT * FROM ${sourceTable.name} LIMIT 1` }); } catch (e) { selectError = { level: 'fatal', message: e.message }; diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index fe137ae0..4d7bfec1 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -207,7 +207,7 @@ AND table_type = 'BASE TABLE';`, } const connection = await this.connections.getConnection(); - const replicationColumns = await common.getReplicationIdentityColumns({ + const replicaIdColumns = await common.getReplicationIdentityColumns({ connection: connection, schema: tablePattern.schema, table_name: tablePattern.name @@ -220,7 +220,7 @@ AND table_type = 'BASE TABLE';`, name, schema: tablePattern.schema, objectId: getMysqlRelId(tablePattern), - replicationColumns: replicationColumns.columns + replicaIdColumns: replicaIdColumns.columns }, false ); @@ -421,7 +421,7 @@ AND table_type = 'BASE TABLE';`, async (batch) => { const binlogEventHandler = this.createBinlogEventHandler(batch); // Only listen for changes to tables in the sync rules - const includedTables = [...this.tableCache.values()].map((table) => table.table); + const includedTables = [...this.tableCache.values()].map((table) => table.name); const binlogListener = new BinLogListener({ includedTables: includedTables, startPosition: binLogPositionState, diff --git a/modules/module-mysql/src/utils/mysql-utils.ts b/modules/module-mysql/src/utils/mysql-utils.ts index 61a2e5d3..958c770b 100644 --- a/modules/module-mysql/src/utils/mysql-utils.ts +++ b/modules/module-mysql/src/utils/mysql-utils.ts @@ -86,5 +86,5 @@ export function isVersionAtLeast(version: string, minimumVersion: string): boole } export function escapeMysqlTableName(table: SourceTable): string { - return `\`${table.schema.replaceAll('`', '``')}\`.\`${table.table.replaceAll('`', '``')}\``; + return `\`${table.schema.replaceAll('`', '``')}\`.\`${table.name.replaceAll('`', '``')}\``; } diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index e785549e..8fa226da 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -150,9 +150,9 @@ export class PostgresSyncRulesStorage async resolveTable(options: storage.ResolveTableOptions): Promise { const { group_id, connection_id, connection_tag, entity_descriptor } = options; - const { schema, name: table, objectId, replicationColumns } = entity_descriptor; + const { schema, name: table, objectId, replicaIdColumns } = entity_descriptor; - const columns = replicationColumns.map((column) => ({ + const columns = replicaIdColumns.map((column) => ({ name: column.name, type: column.type, // The PGWire returns this as a BigInt. We want to store this as JSONB @@ -224,15 +224,15 @@ export class PostgresSyncRulesStorage sourceTableRow = row; } - const sourceTable = new storage.SourceTable( - sourceTableRow!.id, - connection_tag, - objectId, - schema, - table, - replicationColumns, - sourceTableRow!.snapshot_done ?? true - ); + const sourceTable = new storage.SourceTable({ + id: sourceTableRow!.id, + connectionTag: connection_tag, + objectId: objectId, + schema: schema, + name: table, + replicaIdColumns: replicaIdColumns, + snapshotComplete: sourceTableRow!.snapshot_done ?? true + }); sourceTable.syncEvent = options.sync_rules.tableTriggersEvent(sourceTable); sourceTable.syncData = options.sync_rules.tableSyncsData(sourceTable); sourceTable.syncParameters = options.sync_rules.tableSyncsParameters(sourceTable); @@ -283,19 +283,20 @@ export class PostgresSyncRulesStorage table: sourceTable, dropTables: truncatedTables.map( (doc) => - new storage.SourceTable( - doc.id, - connection_tag, - doc.relation_id?.object_id ?? 0, - doc.schema_name, - doc.table_name, - doc.replica_id_columns?.map((c) => ({ - name: c.name, - typeOid: c.typeId, - type: c.type - })) ?? [], - doc.snapshot_done ?? true - ) + new storage.SourceTable({ + id: doc.id, + connectionTag: connection_tag, + objectId: doc.relation_id?.object_id ?? 0, + schema: doc.schema_name, + name: doc.table_name, + replicaIdColumns: + doc.replica_id_columns?.map((c) => ({ + name: c.name, + typeOid: c.typeId, + type: c.type + })) ?? [], + snapshotComplete: doc.snapshot_done ?? true + }) ) }; }); diff --git a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts index f11b730d..a400144b 100644 --- a/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts +++ b/modules/module-postgres-storage/src/storage/batch/PostgresBucketBatch.ts @@ -433,15 +433,15 @@ export class PostgresBucketBatch } }); return tables.map((table) => { - const copy = new storage.SourceTable( - table.id, - table.connectionTag, - table.objectId, - table.schema, - table.table, - table.replicaIdColumns, - table.snapshotComplete - ); + const copy = new storage.SourceTable({ + id: table.id, + connectionTag: table.connectionTag, + objectId: table.objectId, + schema: table.schema, + name: table.name, + replicaIdColumns: table.replicaIdColumns, + snapshotComplete: table.snapshotComplete + }); copy.syncData = table.syncData; copy.syncParameters = table.syncParameters; return copy; diff --git a/modules/module-postgres/src/replication/PgRelation.ts b/modules/module-postgres/src/replication/PgRelation.ts index 08cb87c7..cc3d9a84 100644 --- a/modules/module-postgres/src/replication/PgRelation.ts +++ b/modules/module-postgres/src/replication/PgRelation.ts @@ -1,4 +1,4 @@ -import { ReplicationAssertionError, ServiceError } from '@powersync/lib-services-framework'; +import { ReplicationAssertionError } from '@powersync/lib-services-framework'; import { storage } from '@powersync/service-core'; import { PgoutputRelation } from '@powersync/service-jpgwire'; @@ -27,6 +27,6 @@ export function getPgOutputRelation(source: PgoutputRelation): storage.SourceEnt name: source.name, schema: source.schema, objectId: getRelId(source), - replicationColumns: getReplicaIdColumns(source) + replicaIdColumns: getReplicaIdColumns(source) } satisfies storage.SourceEntityDescriptor; } diff --git a/modules/module-postgres/src/replication/WalStream.ts b/modules/module-postgres/src/replication/WalStream.ts index df589c63..1662461f 100644 --- a/modules/module-postgres/src/replication/WalStream.ts +++ b/modules/module-postgres/src/replication/WalStream.ts @@ -195,7 +195,7 @@ export class WalStream { name, schema, objectId: relid, - replicationColumns: cresult.replicationColumns + replicaIdColumns: cresult.replicationColumns } as SourceEntityDescriptor, false ); @@ -437,7 +437,7 @@ WHERE oid = $1::regclass`, const estimatedCount = await this.estimatedCount(db, table); let at = 0; let lastLogIndex = 0; - const cursor = db.stream({ statement: `SELECT * FROM ${table.escapedIdentifier}` }); + const cursor = db.stream({ statement: `SELECT * FROM ${table.qualifiedName}` }); let columns: { i: number; name: string }[] = []; // pgwire streams rows in chunks. // These chunks can be quite small (as little as 16KB), so we don't flush chunks automatically. diff --git a/modules/module-postgres/src/replication/replication-utils.ts b/modules/module-postgres/src/replication/replication-utils.ts index 26fb8e19..02ec535c 100644 --- a/modules/module-postgres/src/replication/replication-utils.ts +++ b/modules/module-postgres/src/replication/replication-utils.ts @@ -260,7 +260,15 @@ export async function getDebugTableInfo(options: GetDebugTableInfoOptions): Prom const id_columns = id_columns_result?.replicationColumns ?? []; - const sourceTable = new storage.SourceTable(0, connectionTag, relationId ?? 0, schema, name, id_columns, true); + const sourceTable = new storage.SourceTable({ + id: 0, + connectionTag: connectionTag, + objectId: relationId ?? 0, + schema: schema, + name: name, + replicaIdColumns: id_columns, + snapshotComplete: true + }); const syncData = syncRules.tableSyncsData(sourceTable); const syncParameters = syncRules.tableSyncsParameters(sourceTable); @@ -287,7 +295,7 @@ export async function getDebugTableInfo(options: GetDebugTableInfoOptions): Prom let selectError = null; try { - await lib_postgres.retriedQuery(db, `SELECT * FROM ${sourceTable.escapedIdentifier} LIMIT 1`); + await lib_postgres.retriedQuery(db, `SELECT * FROM ${sourceTable.qualifiedName} LIMIT 1`); } catch (e) { selectError = { level: 'fatal', message: e.message }; } diff --git a/packages/service-core-tests/src/test-utils/general-utils.ts b/packages/service-core-tests/src/test-utils/general-utils.ts index 03a81879..f34f7b44 100644 --- a/packages/service-core-tests/src/test-utils/general-utils.ts +++ b/packages/service-core-tests/src/test-utils/general-utils.ts @@ -35,15 +35,15 @@ export function testRules(content: string): storage.PersistedSyncRulesContent { export function makeTestTable(name: string, replicaIdColumns?: string[] | undefined) { const relId = utils.hashData('table', name, (replicaIdColumns ?? ['id']).join(',')); const id = new bson.ObjectId('6544e3899293153fa7b38331'); - return new storage.SourceTable( - id, - storage.SourceTable.DEFAULT_TAG, - relId, - 'public', - name, - (replicaIdColumns ?? ['id']).map((column) => ({ name: column, type: 'VARCHAR', typeId: 25 })), - true - ); + return new storage.SourceTable({ + id: id, + connectionTag: storage.SourceTable.DEFAULT_TAG, + objectId: relId, + schema: 'public', + name: name, + replicaIdColumns: (replicaIdColumns ?? ['id']).map((column) => ({ name: column, type: 'VARCHAR', typeId: 25 })), + snapshotComplete: true + }); } export function getBatchData( diff --git a/packages/service-core/src/api/diagnostics.ts b/packages/service-core/src/api/diagnostics.ts index 3d562e5f..e3f3e80f 100644 --- a/packages/service-core/src/api/diagnostics.ts +++ b/packages/service-core/src/api/diagnostics.ts @@ -105,7 +105,7 @@ export async function getSyncRulesStatus( const source: SourceTableInterface = { connectionTag: tag, schema: pattern.schema, - table: pattern.tablePattern + name: pattern.tablePattern }; const syncData = rules.tableSyncsData(source); const syncParameters = rules.tableSyncsParameters(source); diff --git a/packages/service-core/src/storage/SourceEntity.ts b/packages/service-core/src/storage/SourceEntity.ts index 1de25388..1845ff2d 100644 --- a/packages/service-core/src/storage/SourceEntity.ts +++ b/packages/service-core/src/storage/SourceEntity.ts @@ -10,17 +10,21 @@ export interface ColumnDescriptor { typeId?: number; } -// TODO: This needs to be consolidated with SourceTable into something new. export interface SourceEntityDescriptor { /** - * The internal id of the data source structure in the database. - * + * The internal id of the source entity structure in the database. * If undefined, the schema and name are used as the identifier. - * * If specified, this is specifically used to detect renames. */ objectId: number | string | undefined; schema: string; name: string; - replicationColumns: ColumnDescriptor[]; + /** + * The columns that are used to uniquely identify a record in the source entity. + */ + replicaIdColumns: ColumnDescriptor[]; + /** + * Description of the columns/fields of the source entity. + */ + columns?: ColumnDescriptor[]; } diff --git a/packages/service-core/src/storage/SourceTable.ts b/packages/service-core/src/storage/SourceTable.ts index ab415c91..d9b5d823 100644 --- a/packages/service-core/src/storage/SourceTable.ts +++ b/packages/service-core/src/storage/SourceTable.ts @@ -1,8 +1,19 @@ import { DEFAULT_TAG } from '@powersync/service-sync-rules'; import * as util from '../util/util-index.js'; -import { ColumnDescriptor } from './SourceEntity.js'; +import { ColumnDescriptor, SourceEntityDescriptor } from './SourceEntity.js'; -export class SourceTable { +export interface SourceTableOptions { + id: any; + connectionTag: string; + objectId: number | string | undefined; + schema: string; + name: string; + replicaIdColumns: ColumnDescriptor[]; + snapshotComplete: boolean; + columns?: ColumnDescriptor[]; +} + +export class SourceTable implements SourceEntityDescriptor { static readonly DEFAULT_TAG = DEFAULT_TAG; /** @@ -32,37 +43,45 @@ export class SourceTable { */ public syncEvent = true; - constructor( - public readonly id: any, - public readonly connectionTag: string, - public readonly objectId: number | string | undefined, - public readonly schema: string, - public readonly table: string, + constructor(public readonly options: SourceTableOptions) {} - public readonly replicaIdColumns: ColumnDescriptor[], - public readonly snapshotComplete: boolean - ) {} + get id() { + return this.options.id; + } - get hasReplicaIdentity() { - return this.replicaIdColumns.length > 0; + get connectionTag() { + return this.options.connectionTag; } - /** - * Use for postgres only. - * - * Usage: db.query({statement: `SELECT $1::regclass`, params: [{type: 'varchar', value: table.qualifiedName}]}) - */ - get qualifiedName() { - return this.escapedIdentifier; + get objectId() { + return this.options.objectId; + } + + get schema() { + return this.options.schema; + } + get name() { + return this.options.name; + } + + get replicaIdColumns() { + return this.options.replicaIdColumns; + } + + get snapshotComplete() { + return this.options.snapshotComplete; + } + + get columns() { + return this.options.columns; } /** - * Use for postgres and logs only. - * - * Usage: db.query(`SELECT * FROM ${table.escapedIdentifier}`) + * Sanitized name of the entity in the format of "{schema}.{entity name}" + * Suitable for safe use in queries. */ - get escapedIdentifier() { - return `${util.escapeIdentifier(this.schema)}.${util.escapeIdentifier(this.table)}`; + get qualifiedName() { + return `${util.escapeIdentifier(this.schema)}.${util.escapeIdentifier(this.name)}`; } get syncAny() { diff --git a/packages/sync-rules/src/BaseSqlDataQuery.ts b/packages/sync-rules/src/BaseSqlDataQuery.ts index a3bba1eb..75d6fcf2 100644 --- a/packages/sync-rules/src/BaseSqlDataQuery.ts +++ b/packages/sync-rules/src/BaseSqlDataQuery.ts @@ -93,7 +93,7 @@ export class BaseSqlDataQuery { if (this.sourceTable.isWildcard) { return { ...row, - _table_suffix: this.sourceTable.suffix(table.table) + _table_suffix: this.sourceTable.suffix(table.name) }; } else { return row; @@ -130,7 +130,7 @@ export class BaseSqlDataQuery { this.getColumnOutputsFor(schemaTable, output); result.push({ - name: this.getOutputName(schemaTable.table), + name: this.getOutputName(schemaTable.name), columns: Object.values(output) }); } diff --git a/packages/sync-rules/src/SourceTableInterface.ts b/packages/sync-rules/src/SourceTableInterface.ts index 47f0cfdc..09e5d12a 100644 --- a/packages/sync-rules/src/SourceTableInterface.ts +++ b/packages/sync-rules/src/SourceTableInterface.ts @@ -1,5 +1,5 @@ export interface SourceTableInterface { readonly connectionTag: string; readonly schema: string; - readonly table: string; + readonly name: string; } diff --git a/packages/sync-rules/src/SqlDataQuery.ts b/packages/sync-rules/src/SqlDataQuery.ts index dda09b74..9f6f75c1 100644 --- a/packages/sync-rules/src/SqlDataQuery.ts +++ b/packages/sync-rules/src/SqlDataQuery.ts @@ -204,7 +204,7 @@ export class SqlDataQuery extends BaseSqlDataQuery { // anything. id = castAsText(id) ?? ''; } - const outputTable = this.getOutputName(table.table); + const outputTable = this.getOutputName(table.name); return bucketIds.map((bucketId) => { return { diff --git a/packages/sync-rules/src/StaticSchema.ts b/packages/sync-rules/src/StaticSchema.ts index aa27114c..3ae1c8c3 100644 --- a/packages/sync-rules/src/StaticSchema.ts +++ b/packages/sync-rules/src/StaticSchema.ts @@ -46,13 +46,13 @@ export interface SourceConnectionDefinition { class SourceTableDetails implements SourceTableInterface, SourceSchemaTable { readonly connectionTag: string; readonly schema: string; - readonly table: string; + readonly name: string; private readonly columns: Record; constructor(connection: SourceConnectionDefinition, schema: SourceSchemaDefinition, table: SourceTableDefinition) { this.connectionTag = connection.tag; this.schema = schema.name; - this.table = table.name; + this.name = table.name; this.columns = Object.fromEntries( table.columns.map((column) => { return [column.name, mapColumn(column)]; diff --git a/packages/sync-rules/src/TablePattern.ts b/packages/sync-rules/src/TablePattern.ts index 55c90ec9..89b98109 100644 --- a/packages/sync-rules/src/TablePattern.ts +++ b/packages/sync-rules/src/TablePattern.ts @@ -49,9 +49,9 @@ export class TablePattern { return false; } if (this.isWildcard) { - return table.table.startsWith(this.tablePrefix); + return table.name.startsWith(this.tablePrefix); } else { - return this.tablePattern == table.table; + return this.tablePattern == table.name; } } diff --git a/packages/sync-rules/src/events/SqlEventDescriptor.ts b/packages/sync-rules/src/events/SqlEventDescriptor.ts index 0947fc2a..526e0f1f 100644 --- a/packages/sync-rules/src/events/SqlEventDescriptor.ts +++ b/packages/sync-rules/src/events/SqlEventDescriptor.ts @@ -43,7 +43,7 @@ export class SqlEventDescriptor { const matchingQuery = this.sourceQueries.find((q) => q.applies(options.sourceTable)); if (!matchingQuery) { return { - errors: [{ error: `No marching source query found for table ${options.sourceTable.table}` }] + errors: [{ error: `No marching source query found for table ${options.sourceTable.name}` }] }; } diff --git a/packages/sync-rules/src/types.ts b/packages/sync-rules/src/types.ts index dc699af8..23c7de03 100644 --- a/packages/sync-rules/src/types.ts +++ b/packages/sync-rules/src/types.ts @@ -341,7 +341,7 @@ export type CompiledClause = RowValueClause | ParameterMatchClause | ParameterVa export type TrueIfParametersMatch = FilterParameters[]; export interface SourceSchemaTable { - table: string; + name: string; getColumn(column: string): ColumnDefinition | undefined; getColumns(): ColumnDefinition[]; } diff --git a/packages/sync-rules/test/src/util.ts b/packages/sync-rules/test/src/util.ts index e1ed5b80..b780cd0c 100644 --- a/packages/sync-rules/test/src/util.ts +++ b/packages/sync-rules/test/src/util.ts @@ -10,7 +10,7 @@ export class TestSourceTable implements SourceTableInterface { readonly connectionTag = DEFAULT_TAG; readonly schema = 'test_schema'; - constructor(public readonly table: string) {} + constructor(public readonly name: string) {} } export const PARSE_OPTIONS = { From 3aebffd191c8906cd492a72fad40596e689722d2 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 4 Jun 2025 13:34:51 +0200 Subject: [PATCH 14/48] Cleanup unused imports --- modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts | 2 +- packages/sync-rules/src/events/SqlEventDescriptor.ts | 1 - packages/sync-rules/src/types.ts | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts index 2a74bc98..6ef377b0 100644 --- a/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts +++ b/modules/module-mongodb/src/api/MongoRouteAPIAdapter.ts @@ -5,7 +5,7 @@ import * as sync_rules from '@powersync/service-sync-rules'; import * as service_types from '@powersync/service-types'; import { MongoManager } from '../replication/MongoManager.js'; -import { constructAfterRecord, createCheckpoint, STANDALONE_CHECKPOINT_ID } from '../replication/MongoRelation.js'; +import { constructAfterRecord, STANDALONE_CHECKPOINT_ID } from '../replication/MongoRelation.js'; import { CHECKPOINTS_COLLECTION } from '../replication/replication-utils.js'; import * as types from '../types/types.js'; import { escapeRegExp } from '../utils.js'; diff --git a/packages/sync-rules/src/events/SqlEventDescriptor.ts b/packages/sync-rules/src/events/SqlEventDescriptor.ts index 526e0f1f..1d76407e 100644 --- a/packages/sync-rules/src/events/SqlEventDescriptor.ts +++ b/packages/sync-rules/src/events/SqlEventDescriptor.ts @@ -1,5 +1,4 @@ import { SqlRuleError } from '../errors.js'; -import { IdSequence } from '../IdSequence.js'; import { SourceTableInterface } from '../SourceTableInterface.js'; import { QueryParseResult } from '../SqlBucketDescriptor.js'; import { SyncRulesOptions } from '../SqlSyncRules.js'; diff --git a/packages/sync-rules/src/types.ts b/packages/sync-rules/src/types.ts index 23c7de03..ac6d9035 100644 --- a/packages/sync-rules/src/types.ts +++ b/packages/sync-rules/src/types.ts @@ -4,7 +4,7 @@ import { SourceTableInterface } from './SourceTableInterface.js'; import { SyncRulesOptions } from './SqlSyncRules.js'; import { TablePattern } from './TablePattern.js'; import { toSyncRulesParameters } from './utils.js'; -import { BucketDescription, BucketPriority } from './BucketDescription.js'; +import { BucketPriority } from './BucketDescription.js'; import { ParameterLookup } from './BucketParameterQuerier.js'; export interface SyncRules { From bf481c871ecf83d48e01972908c21028ef7858d1 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Thu, 5 Jun 2025 10:55:33 +0200 Subject: [PATCH 15/48] Ensure column values are preserved when available Report 0 storage metrics instead of ignoring them. SourceTable. Moved MySQL table detail retrieval logic to utility function. --- .../implementation/MongoSyncBucketStorage.ts | 11 +- .../src/api/MySQLRouteAPIAdapter.ts | 2 +- .../module-mysql/src/common/common-index.ts | 3 +- .../src/common/get-tables-from-pattern.ts | 44 ------ ...replication-columns.ts => schema-utils.ts} | 130 +++++++++++++----- .../src/replication/BinLogStream.ts | 59 ++------ .../src/storage/PostgresSyncRulesStorage.ts | 13 +- .../service-core/src/storage/SourceTable.ts | 2 +- 8 files changed, 125 insertions(+), 139 deletions(-) delete mode 100644 modules/module-mysql/src/common/get-tables-from-pattern.ts rename modules/module-mysql/src/common/{get-replication-columns.ts => schema-utils.ts} (57%) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 9f1ad8b9..4d676c6a 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -162,9 +162,9 @@ export class MongoSyncBucketStorage async resolveTable(options: storage.ResolveTableOptions): Promise { const { group_id, connection_id, connection_tag, entity_descriptor } = options; - const { schema, name, objectId, replicaIdColumns } = entity_descriptor; + const { schema, name, objectId, replicaIdColumns, columns } = entity_descriptor; - const columns = replicaIdColumns.map((column) => ({ + const normalizedReplicaIdColumns = replicaIdColumns.map((column) => ({ name: column.name, type: column.type, type_oid: column.typeId @@ -177,7 +177,7 @@ export class MongoSyncBucketStorage connection_id: connection_id, schema_name: schema, table_name: name, - replica_id_columns2: columns + replica_id_columns2: normalizedReplicaIdColumns }; if (objectId != null) { filter.relation_id = objectId; @@ -192,7 +192,7 @@ export class MongoSyncBucketStorage schema_name: schema, table_name: name, replica_id_columns: null, - replica_id_columns2: columns, + replica_id_columns2: normalizedReplicaIdColumns, snapshot_done: false }; @@ -205,7 +205,8 @@ export class MongoSyncBucketStorage schema: schema, name: name, replicaIdColumns: replicaIdColumns, - snapshotComplete: doc.snapshot_done ?? true + snapshotComplete: doc.snapshot_done ?? true, + columns: columns }); sourceTable.syncEvent = options.sync_rules.tableTriggersEvent(sourceTable); sourceTable.syncData = options.sync_rules.tableSyncsData(sourceTable); diff --git a/modules/module-mysql/src/api/MySQLRouteAPIAdapter.ts b/modules/module-mysql/src/api/MySQLRouteAPIAdapter.ts index fd055672..56272162 100644 --- a/modules/module-mysql/src/api/MySQLRouteAPIAdapter.ts +++ b/modules/module-mysql/src/api/MySQLRouteAPIAdapter.ts @@ -208,7 +208,7 @@ export class MySQLRouteAPIAdapter implements api.RouteAPI { idColumnsResult = await common.getReplicationIdentityColumns({ connection: connection, schema, - table_name: tableName + tableName: tableName }); } catch (ex) { idColumnsError = { level: 'fatal', message: ex.message }; diff --git a/modules/module-mysql/src/common/common-index.ts b/modules/module-mysql/src/common/common-index.ts index 6da00571..63112da4 100644 --- a/modules/module-mysql/src/common/common-index.ts +++ b/modules/module-mysql/src/common/common-index.ts @@ -1,6 +1,5 @@ export * from './check-source-configuration.js'; -export * from './get-replication-columns.js'; -export * from './get-tables-from-pattern.js'; +export * from './schema-utils.js'; export * from './mysql-to-sqlite.js'; export * from './read-executed-gtid.js'; export * from './ReplicatedGTID.js'; diff --git a/modules/module-mysql/src/common/get-tables-from-pattern.ts b/modules/module-mysql/src/common/get-tables-from-pattern.ts deleted file mode 100644 index 166bf93a..00000000 --- a/modules/module-mysql/src/common/get-tables-from-pattern.ts +++ /dev/null @@ -1,44 +0,0 @@ -import * as sync_rules from '@powersync/service-sync-rules'; -import mysql from 'mysql2/promise'; - -export type GetDebugTablesInfoOptions = { - connection: mysql.Connection; - tablePattern: sync_rules.TablePattern; -}; - -export async function getTablesFromPattern(options: GetDebugTablesInfoOptions): Promise> { - const { connection, tablePattern } = options; - const schema = tablePattern.schema; - - if (tablePattern.isWildcard) { - const [results] = await connection.query( - `SELECT - TABLE_NAME AS table_name - FROM - INFORMATION_SCHEMA.TABLES - WHERE - TABLE_SCHEMA = ? - AND TABLE_NAME LIKE ?`, - [schema, tablePattern.tablePattern] - ); - - return new Set( - results - .filter((result) => result.table_name.startsWith(tablePattern.tablePrefix)) - .map((result) => result.table_name) - ); - } else { - const [[match]] = await connection.query( - `SELECT - TABLE_NAME AS table_name - FROM - INFORMATION_SCHEMA.TABLES - WHERE - TABLE_SCHEMA = ? - AND TABLE_NAME = ?`, - [tablePattern.schema, tablePattern.tablePattern] - ); - // Only return the first result - return new Set([match.table_name]); - } -} diff --git a/modules/module-mysql/src/common/get-replication-columns.ts b/modules/module-mysql/src/common/schema-utils.ts similarity index 57% rename from modules/module-mysql/src/common/get-replication-columns.ts rename to modules/module-mysql/src/common/schema-utils.ts index fa0eb8fd..7d3c1736 100644 --- a/modules/module-mysql/src/common/get-replication-columns.ts +++ b/modules/module-mysql/src/common/schema-utils.ts @@ -1,23 +1,64 @@ -import { storage } from '@powersync/service-core'; import mysqlPromise from 'mysql2/promise'; import * as mysql_utils from '../utils/mysql-utils.js'; +import { ColumnDescriptor } from '@powersync/service-core'; +import { TablePattern } from '@powersync/service-sync-rules'; -export type GetReplicationColumnsOptions = { +export interface GetColumnsOptions { connection: mysqlPromise.Connection; schema: string; - table_name: string; -}; + tableName: string; +} + +export async function getColumns(options: GetColumnsOptions): Promise { + const { connection, schema, tableName } = options; + + const [allColumns] = await mysql_utils.retriedQuery({ + connection: connection, + query: ` + SELECT + s.COLUMN_NAME AS name, + c.DATA_TYPE as type + FROM + INFORMATION_SCHEMA.COLUMNS s + JOIN + INFORMATION_SCHEMA.COLUMNS c + ON + s.TABLE_SCHEMA = c.TABLE_SCHEMA + AND s.TABLE_NAME = c.TABLE_NAME + AND s.COLUMN_NAME = c.COLUMN_NAME + WHERE + s.TABLE_SCHEMA = ? + AND s.TABLE_NAME = ? + ORDER BY + s.ORDINAL_POSITION; + `, + params: [schema, tableName] + }); + + return allColumns.map((row) => { + return { + name: row.name, + type: row.type + }; + }); +} + +export interface GetReplicationIdentityColumnsOptions { + connection: mysqlPromise.Connection; + schema: string; + tableName: string; +} -export type ReplicationIdentityColumnsResult = { - columns: storage.ColumnDescriptor[]; +export interface ReplicationIdentityColumnsResult { + columns: ColumnDescriptor[]; // TODO maybe export an enum from the core package identity: string; -}; +} export async function getReplicationIdentityColumns( - options: GetReplicationColumnsOptions + options: GetReplicationIdentityColumnsOptions ): Promise { - const { connection, schema, table_name } = options; + const { connection, schema, tableName } = options; const [primaryKeyColumns] = await mysql_utils.retriedQuery({ connection: connection, query: ` @@ -39,7 +80,7 @@ export async function getReplicationIdentityColumns( ORDER BY s.SEQ_IN_INDEX; `, - params: [schema, table_name] + params: [schema, tableName] }); if (primaryKeyColumns.length) { @@ -78,7 +119,7 @@ export async function getReplicationIdentityColumns( AND s.NON_UNIQUE = 0 ORDER BY s.SEQ_IN_INDEX; `, - params: [schema, table_name] + params: [schema, tableName] }); if (uniqueKeyColumns.length > 0) { @@ -91,34 +132,53 @@ export async function getReplicationIdentityColumns( }; } - const [allColumns] = await mysql_utils.retriedQuery({ + const allColumns = await getColumns({ connection: connection, - query: ` - SELECT - s.COLUMN_NAME AS name, - c.DATA_TYPE as type - FROM - INFORMATION_SCHEMA.COLUMNS s - JOIN - INFORMATION_SCHEMA.COLUMNS c - ON - s.TABLE_SCHEMA = c.TABLE_SCHEMA - AND s.TABLE_NAME = c.TABLE_NAME - AND s.COLUMN_NAME = c.COLUMN_NAME - WHERE - s.TABLE_SCHEMA = ? - AND s.TABLE_NAME = ? - ORDER BY - s.ORDINAL_POSITION; - `, - params: [schema, table_name] + schema: schema, + tableName: tableName }); return { - columns: allColumns.map((row) => ({ - name: row.name, - type: row.type - })), + columns: allColumns, identity: 'full' }; } + +export async function getTablesFromPattern( + connection: mysqlPromise.Connection, + tablePattern: TablePattern +): Promise { + const schema = tablePattern.schema; + + if (tablePattern.isWildcard) { + const [results] = await mysql_utils.retriedQuery({ + connection: connection, + query: ` + SELECT TABLE_NAME + FROM information_schema.tables + WHERE TABLE_SCHEMA = ? + AND TABLE_NAME LIKE ? + AND table_type = 'BASE TABLE' + `, + params: [schema, tablePattern.tablePattern] + }); + + return results + .map((row) => row.TABLE_NAME) + .filter((tableName: string) => tableName.startsWith(tablePattern.tablePrefix)); + } else { + const [results] = await mysql_utils.retriedQuery({ + connection: connection, + query: ` + SELECT TABLE_NAME + FROM information_schema.tables + WHERE TABLE_SCHEMA = ? + AND TABLE_NAME = ? + AND table_type = 'BASE TABLE' + `, + params: [schema, tablePattern.tablePattern] + }); + + return results.map((row) => row.TABLE_NAME); + } +} diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index 4d7bfec1..c455df6d 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -165,68 +165,37 @@ export class BinLogStream { return []; } - let tableRows: any[]; - const prefix = tablePattern.isWildcard ? tablePattern.tablePrefix : undefined; - if (tablePattern.isWildcard) { - const result = await this.connections.query( - `SELECT TABLE_NAME -FROM information_schema.tables -WHERE TABLE_SCHEMA = ? AND TABLE_NAME LIKE ?; -`, - [tablePattern.schema, tablePattern.tablePattern] - ); - tableRows = result[0]; - } else { - const result = await this.connections.query( - `SELECT TABLE_NAME -FROM information_schema.tables -WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?; -`, - [tablePattern.schema, tablePattern.tablePattern] - ); - tableRows = result[0]; - } - let tables: storage.SourceTable[] = []; - - for (let row of tableRows) { - const name = row['TABLE_NAME'] as string; - if (prefix && !name.startsWith(prefix)) { - continue; - } - - const result = await this.connections.query( - `SELECT 1 -FROM information_schema.tables -WHERE table_schema = ? AND table_name = ? -AND table_type = 'BASE TABLE';`, - [tablePattern.schema, tablePattern.name] - ); - if (result[0].length == 0) { - logger.info(`Skipping ${tablePattern.schema}.${name} - no table exists/is not a base table`); - continue; - } + const connection = await this.connections.getConnection(); + const matchedTables: string[] = await common.getTablesFromPattern(connection, tablePattern); - const connection = await this.connections.getConnection(); + let tables: storage.SourceTable[] = []; + for (const matchedTable of matchedTables) { const replicaIdColumns = await common.getReplicationIdentityColumns({ connection: connection, schema: tablePattern.schema, - table_name: tablePattern.name + tableName: matchedTable + }); + const columns = await common.getColumns({ + connection: connection, + schema: tablePattern.schema, + tableName: matchedTable }); - connection.release(); const table = await this.handleRelation( batch, { - name, + name: matchedTable, schema: tablePattern.schema, objectId: getMysqlRelId(tablePattern), - replicaIdColumns: replicaIdColumns.columns + replicaIdColumns: replicaIdColumns.columns, + columns: columns }, false ); tables.push(table); } + connection.release(); return tables; } diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index 8fa226da..036a731c 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -150,9 +150,9 @@ export class PostgresSyncRulesStorage async resolveTable(options: storage.ResolveTableOptions): Promise { const { group_id, connection_id, connection_tag, entity_descriptor } = options; - const { schema, name: table, objectId, replicaIdColumns } = entity_descriptor; + const { schema, name: table, objectId, replicaIdColumns, columns } = entity_descriptor; - const columns = replicaIdColumns.map((column) => ({ + const normalizedReplicaIdColumns = replicaIdColumns.map((column) => ({ name: column.name, type: column.type, // The PGWire returns this as a BigInt. We want to store this as JSONB @@ -172,7 +172,7 @@ export class PostgresSyncRulesStorage AND relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }} AND schema_name = ${{ type: 'varchar', value: schema }} AND table_name = ${{ type: 'varchar', value: table }} - AND replica_id_columns = ${{ type: 'jsonb', value: columns }} + AND replica_id_columns = ${{ type: 'jsonb', value: normalizedReplicaIdColumns }} ` .decoded(models.SourceTable) .first(); @@ -187,7 +187,7 @@ export class PostgresSyncRulesStorage AND connection_id = ${{ type: 'int4', value: connection_id }} AND schema_name = ${{ type: 'varchar', value: schema }} AND table_name = ${{ type: 'varchar', value: table }} - AND replica_id_columns = ${{ type: 'jsonb', value: columns }} + AND replica_id_columns = ${{ type: 'jsonb', value: normalizedReplicaIdColumns }} ` .decoded(models.SourceTable) .first(); @@ -214,7 +214,7 @@ export class PostgresSyncRulesStorage ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}, ${{ type: 'varchar', value: schema }}, ${{ type: 'varchar', value: table }}, - ${{ type: 'jsonb', value: columns }} + ${{ type: 'jsonb', value: normalizedReplicaIdColumns }} ) RETURNING * @@ -231,7 +231,8 @@ export class PostgresSyncRulesStorage schema: schema, name: table, replicaIdColumns: replicaIdColumns, - snapshotComplete: sourceTableRow!.snapshot_done ?? true + snapshotComplete: sourceTableRow!.snapshot_done ?? true, + columns: columns }); sourceTable.syncEvent = options.sync_rules.tableTriggersEvent(sourceTable); sourceTable.syncData = options.sync_rules.tableSyncsData(sourceTable); diff --git a/packages/service-core/src/storage/SourceTable.ts b/packages/service-core/src/storage/SourceTable.ts index d9b5d823..2d215a8c 100644 --- a/packages/service-core/src/storage/SourceTable.ts +++ b/packages/service-core/src/storage/SourceTable.ts @@ -78,7 +78,7 @@ export class SourceTable implements SourceEntityDescriptor { /** * Sanitized name of the entity in the format of "{schema}.{entity name}" - * Suitable for safe use in queries. + * Suitable for safe use in Postgres queries. */ get qualifiedName() { return `${util.escapeIdentifier(this.schema)}.${util.escapeIdentifier(this.name)}`; From b673609203c928ec222928a9da94abac375ccd5c Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Thu, 5 Jun 2025 15:02:22 +0200 Subject: [PATCH 16/48] Added basic schema change handling for MySQL --- modules/module-mysql/package.json | 4 +- .../src/replication/BinLogStream.ts | 40 +++++++++++++++++++ .../src/replication/zongji/BinLogListener.ts | 11 +++++ .../test/src/BinLogListener.test.ts | 22 +++++++++- pnpm-lock.yaml | 16 +++++--- 5 files changed, 85 insertions(+), 8 deletions(-) diff --git a/modules/module-mysql/package.json b/modules/module-mysql/package.json index 09016a42..976df29b 100644 --- a/modules/module-mysql/package.json +++ b/modules/module-mysql/package.json @@ -33,12 +33,13 @@ "@powersync/service-sync-rules": "workspace:*", "@powersync/service-types": "workspace:*", "@powersync/service-jsonbig": "workspace:*", - "@powersync/mysql-zongji": "0.2.0", + "@powersync/mysql-zongji": "0.0.0-dev-20250605124659", "async": "^3.2.4", "mysql2": "^3.11.0", "semver": "^7.5.4", "ts-codec": "^1.3.0", "uri-js": "^4.4.1", + "lodash": "^4.17.21", "uuid": "^11.1.0" }, "devDependencies": { @@ -46,6 +47,7 @@ "@powersync/service-module-mongodb-storage": "workspace:*", "@powersync/service-module-postgres-storage": "workspace:*", "@types/async": "^3.2.24", + "@types/lodash": "^4.17.5", "@types/semver": "^7.5.4" } } diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index c455df6d..cf737c26 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -10,6 +10,7 @@ import { } from '@powersync/service-core'; import mysql from 'mysql2'; import mysqlPromise from 'mysql2/promise'; +import _ from 'lodash'; import { TableMapEntry } from '@powersync/mysql-zongji'; import * as common from '../common/common-index.js'; @@ -443,10 +444,49 @@ export class BinLogStream { onCommit: async (lsn: string) => { this.metrics.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED).add(1); await batch.commit(lsn); + }, + onSchemaChange: async (tableMap: TableMapEntry) => { + if (!this.hasSchemaChange(tableMap)) { + logger.info(`Skipping schema change for ${tableMap.parentSchema}.${tableMap.tableName} - no changes`); + return; + } + await this.handleRelation( + batch, + { + name: tableMap.tableName, + schema: tableMap.parentSchema, + objectId: getMysqlRelId({ + schema: tableMap.parentSchema, + name: tableMap.tableName + }), + replicaIdColumns: Array.from(common.toColumnDescriptors(tableMap).values()) + }, + true + ); } }; } + private hasSchemaChange(tableMap: TableMapEntry) { + // Check if the table is already in the cache + const cachedTable = this.tableCache.get( + getMysqlRelId({ + schema: tableMap.parentSchema, + name: tableMap.tableName + }) + ); + if (cachedTable) { + // The table already exists in the cache with the same name, check if the columns are the same + const existingColumns = cachedTable.columns!.sort((a, b) => a.name.localeCompare(b.name)); + const newColumns = Array.from(common.toColumnDescriptors(tableMap).values()).sort((a, b) => + a.name.localeCompare(b.name) + ); + return _.isEqual(existingColumns, newColumns); + } + // If the table is not in the cache, it is a new table, or has been renamed + return true; + } + private async writeChanges( batch: storage.BucketStorageBatch, msg: { diff --git a/modules/module-mysql/src/replication/zongji/BinLogListener.ts b/modules/module-mysql/src/replication/zongji/BinLogListener.ts index e22ff8b2..df3abf38 100644 --- a/modules/module-mysql/src/replication/zongji/BinLogListener.ts +++ b/modules/module-mysql/src/replication/zongji/BinLogListener.ts @@ -16,6 +16,7 @@ export interface BinLogEventHandler { onUpdate: (rowsAfter: Row[], rowsBefore: Row[], tableMap: TableMapEntry) => Promise; onDelete: (rows: Row[], tableMap: TableMapEntry) => Promise; onCommit: (lsn: string) => Promise; + onSchemaChange: (tableMap: TableMapEntry) => Promise; } export interface BinLogListenerOptions { @@ -224,6 +225,16 @@ export class BinLogListener { }).comparable; await this.eventHandler.onCommit(LSN); break; + case zongji_utils.eventIsTableMap(evt): + const tableMapEntry = evt.tableMap[evt.tableId]; + logger.info( + `Potential schema change detected for table: ${tableMapEntry.tableName}. Pausing Binlog listener.)` + ); + this.zongji.pause(); + await this.eventHandler.onSchemaChange(evt.tableMap[evt.tableId]); + logger.info(`Schema change for table ${tableMapEntry.tableName} handled. Resuming Binlog listener.`); + this.zongji.resume(); + break; } this.queueMemoryUsage -= evt.size; diff --git a/modules/module-mysql/test/src/BinLogListener.test.ts b/modules/module-mysql/test/src/BinLogListener.test.ts index 263eff26..8f37a8bf 100644 --- a/modules/module-mysql/test/src/BinLogListener.test.ts +++ b/modules/module-mysql/test/src/BinLogListener.test.ts @@ -32,7 +32,7 @@ describe('BinlogListener tests', () => { connectionManager: connectionManager, eventHandler: eventHandler, startPosition: fromGTID.position, - includedTables: ['test_DATA'], + includedTables: ['test_DATA', 'test_DATA_new'], serverId: createRandomServerId(1) }); }); @@ -79,7 +79,7 @@ describe('BinlogListener tests', () => { expect(resumeSpy).toHaveBeenCalled(); }); - test('Binlog events are correctly forwarded to provided binlog events handler', async () => { + test('Binlog row events are correctly forwarded to provided binlog events handler', async () => { const startPromise = binLogListener.start(); const ROW_COUNT = 10; @@ -96,6 +96,19 @@ describe('BinlogListener tests', () => { binLogListener.stop(); await expect(startPromise).resolves.toBeUndefined(); }); + + test('Binlog schema change events are correctly forwarded to provided binlog events handler', async () => { + const startPromise = binLogListener.start(); + await connectionManager.query(`RENAME TABLE test_DATA TO test_DATA_new`); + // Table map events are only emitted before row events + await connectionManager.query( + `INSERT INTO test_DATA_new(id, description) VALUES('${uuid()}','test ${crypto.randomBytes(100).toString('hex')}')` + ); + await vi.waitFor(() => expect(eventHandler.latestSchemaChange).toBeDefined(), { timeout: 5000 }); + binLogListener.stop(); + await expect(startPromise).resolves.toBeUndefined(); + expect(eventHandler.latestSchemaChange?.tableName).toEqual('test_DATA_new'); + }); }); async function getFromGTID(connectionManager: MySQLConnectionManager) { @@ -127,6 +140,7 @@ class TestBinLogEventHandler implements BinLogEventHandler { rowsUpdated = 0; rowsDeleted = 0; commitCount = 0; + latestSchemaChange: TableMapEntry | undefined; unpause: ((value: void | PromiseLike) => void) | undefined; private pausedPromise: Promise | undefined; @@ -155,4 +169,8 @@ class TestBinLogEventHandler implements BinLogEventHandler { async onCommit(lsn: string) { this.commitCount++; } + + async onSchemaChange(tableMap: TableMapEntry) { + this.latestSchemaChange = tableMap; + } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c7367649..504a4d13 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -258,8 +258,8 @@ importers: specifier: workspace:* version: link:../../libs/lib-services '@powersync/mysql-zongji': - specifier: 0.2.0 - version: 0.2.0 + specifier: 0.0.0-dev-20250605124659 + version: 0.0.0-dev-20250605124659 '@powersync/service-core': specifier: workspace:* version: link:../../packages/service-core @@ -275,6 +275,9 @@ importers: async: specifier: ^3.2.4 version: 3.2.5 + lodash: + specifier: ^4.17.21 + version: 4.17.21 mysql2: specifier: ^3.11.0 version: 3.11.3 @@ -303,6 +306,9 @@ importers: '@types/async': specifier: ^3.2.24 version: 3.2.24 + '@types/lodash': + specifier: ^4.17.5 + version: 4.17.6 '@types/semver': specifier: ^7.5.4 version: 7.5.8 @@ -1281,8 +1287,8 @@ packages: resolution: {integrity: sha512-UA91GwWPhFExt3IizW6bOeY/pQ0BkuNwKjk9iQW9KqxluGCrg4VenZ0/L+2Y0+ZOtme72EVvg6v0zo3AMQRCeA==} engines: {node: '>=12'} - '@powersync/mysql-zongji@0.2.0': - resolution: {integrity: sha512-ua/n7WFfoiXmqfgwLikcm/AaDE6+t5gFVTWHWsbiuRQMNtXE1F2gXpZJdwKhr8WsOCYkB/A1ZOgbJKi4tK342g==} + '@powersync/mysql-zongji@0.0.0-dev-20250605124659': + resolution: {integrity: sha512-HYPaejOMY8yok7g2DIBgdnm2QiFHb28QlsCorq+K5KSKeT8NLR0xpTPZ9JvvYhmE5ooFQVnO7ZZBEOFiQzEv6Q==} engines: {node: '>=22.0.0'} '@powersync/service-jsonbig@0.17.10': @@ -4748,7 +4754,7 @@ snapshots: '@pnpm/network.ca-file': 1.0.2 config-chain: 1.1.13 - '@powersync/mysql-zongji@0.2.0': + '@powersync/mysql-zongji@0.0.0-dev-20250605124659': dependencies: '@vlasky/mysql': 2.18.6 big-integer: 1.6.52 From f707e2bcf104c95d6a0686947c79f2f65aeb2c00 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 18 Jun 2025 09:26:25 +0200 Subject: [PATCH 17/48] Revert columns field addition to SourceEntityDescriptor --- .../src/storage/implementation/MongoSyncBucketStorage.ts | 5 ++--- .../src/storage/PostgresSyncRulesStorage.ts | 5 ++--- packages/service-core/src/storage/SourceEntity.ts | 4 ---- packages/service-core/src/storage/SourceTable.ts | 5 ----- 4 files changed, 4 insertions(+), 15 deletions(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 4d676c6a..4407ab69 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -162,7 +162,7 @@ export class MongoSyncBucketStorage async resolveTable(options: storage.ResolveTableOptions): Promise { const { group_id, connection_id, connection_tag, entity_descriptor } = options; - const { schema, name, objectId, replicaIdColumns, columns } = entity_descriptor; + const { schema, name, objectId, replicaIdColumns } = entity_descriptor; const normalizedReplicaIdColumns = replicaIdColumns.map((column) => ({ name: column.name, @@ -205,8 +205,7 @@ export class MongoSyncBucketStorage schema: schema, name: name, replicaIdColumns: replicaIdColumns, - snapshotComplete: doc.snapshot_done ?? true, - columns: columns + snapshotComplete: doc.snapshot_done ?? true }); sourceTable.syncEvent = options.sync_rules.tableTriggersEvent(sourceTable); sourceTable.syncData = options.sync_rules.tableSyncsData(sourceTable); diff --git a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts index 036a731c..f3371cc8 100644 --- a/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts +++ b/modules/module-postgres-storage/src/storage/PostgresSyncRulesStorage.ts @@ -150,7 +150,7 @@ export class PostgresSyncRulesStorage async resolveTable(options: storage.ResolveTableOptions): Promise { const { group_id, connection_id, connection_tag, entity_descriptor } = options; - const { schema, name: table, objectId, replicaIdColumns, columns } = entity_descriptor; + const { schema, name: table, objectId, replicaIdColumns } = entity_descriptor; const normalizedReplicaIdColumns = replicaIdColumns.map((column) => ({ name: column.name, @@ -231,8 +231,7 @@ export class PostgresSyncRulesStorage schema: schema, name: table, replicaIdColumns: replicaIdColumns, - snapshotComplete: sourceTableRow!.snapshot_done ?? true, - columns: columns + snapshotComplete: sourceTableRow!.snapshot_done ?? true }); sourceTable.syncEvent = options.sync_rules.tableTriggersEvent(sourceTable); sourceTable.syncData = options.sync_rules.tableSyncsData(sourceTable); diff --git a/packages/service-core/src/storage/SourceEntity.ts b/packages/service-core/src/storage/SourceEntity.ts index 1845ff2d..a139eb08 100644 --- a/packages/service-core/src/storage/SourceEntity.ts +++ b/packages/service-core/src/storage/SourceEntity.ts @@ -23,8 +23,4 @@ export interface SourceEntityDescriptor { * The columns that are used to uniquely identify a record in the source entity. */ replicaIdColumns: ColumnDescriptor[]; - /** - * Description of the columns/fields of the source entity. - */ - columns?: ColumnDescriptor[]; } diff --git a/packages/service-core/src/storage/SourceTable.ts b/packages/service-core/src/storage/SourceTable.ts index 2d215a8c..fdd31914 100644 --- a/packages/service-core/src/storage/SourceTable.ts +++ b/packages/service-core/src/storage/SourceTable.ts @@ -10,7 +10,6 @@ export interface SourceTableOptions { name: string; replicaIdColumns: ColumnDescriptor[]; snapshotComplete: boolean; - columns?: ColumnDescriptor[]; } export class SourceTable implements SourceEntityDescriptor { @@ -72,10 +71,6 @@ export class SourceTable implements SourceEntityDescriptor { return this.options.snapshotComplete; } - get columns() { - return this.options.columns; - } - /** * Sanitized name of the entity in the format of "{schema}.{entity name}" * Suitable for safe use in Postgres queries. From 74adb22d35388670bee69bb5de137dd14f555543 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 18 Jun 2025 09:51:17 +0200 Subject: [PATCH 18/48] Added schema change handling for the MySQL binlog replication. --- modules/module-mysql/package.json | 3 +- .../src/replication/BinLogStream.ts | 97 +++-- .../src/replication/zongji/BinLogListener.ts | 368 +++++++++++++----- .../src/replication/zongji/zongji-utils.ts | 7 +- .../types/node-sql-parser-extended-types.ts | 17 + .../test/src/BinLogListener.test.ts | 148 +++++-- pnpm-lock.yaml | 27 +- 7 files changed, 505 insertions(+), 162 deletions(-) create mode 100644 modules/module-mysql/src/types/node-sql-parser-extended-types.ts diff --git a/modules/module-mysql/package.json b/modules/module-mysql/package.json index 976df29b..edc0598f 100644 --- a/modules/module-mysql/package.json +++ b/modules/module-mysql/package.json @@ -33,9 +33,10 @@ "@powersync/service-sync-rules": "workspace:*", "@powersync/service-types": "workspace:*", "@powersync/service-jsonbig": "workspace:*", - "@powersync/mysql-zongji": "0.0.0-dev-20250605124659", + "@powersync/mysql-zongji": "0.0.0-dev-20250610140703", "async": "^3.2.4", "mysql2": "^3.11.0", + "node-sql-parser": "^5.3.9", "semver": "^7.5.4", "ts-codec": "^1.3.0", "uri-js": "^4.4.1", diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index cf737c26..d40696b3 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -10,14 +10,13 @@ import { } from '@powersync/service-core'; import mysql from 'mysql2'; import mysqlPromise from 'mysql2/promise'; -import _ from 'lodash'; import { TableMapEntry } from '@powersync/mysql-zongji'; import * as common from '../common/common-index.js'; import { createRandomServerId, escapeMysqlTableName } from '../utils/mysql-utils.js'; import { MySQLConnectionManager } from './MySQLConnectionManager.js'; import { ReplicationMetric } from '@powersync/service-types'; -import { BinLogEventHandler, BinLogListener, Row } from './zongji/BinLogListener.js'; +import { BinLogEventHandler, BinLogListener, Row, SchemaChange, SchemaChangeType } from './zongji/BinLogListener.js'; export interface BinLogStreamOptions { connections: MySQLConnectionManager; @@ -139,7 +138,7 @@ export class BinLogStream { const promiseConnection = (connection as mysql.Connection).promise(); try { await promiseConnection.query(`SET time_zone = '+00:00'`); - await promiseConnection.query('BEGIN'); + await promiseConnection.query('START TRANSACTION'); try { gtid = await common.readExecutedGtid(promiseConnection); await this.snapshotTable(connection.connection, batch, result.table); @@ -176,11 +175,6 @@ export class BinLogStream { schema: tablePattern.schema, tableName: matchedTable }); - const columns = await common.getColumns({ - connection: connection, - schema: tablePattern.schema, - tableName: matchedTable - }); const table = await this.handleRelation( batch, @@ -188,8 +182,7 @@ export class BinLogStream { name: matchedTable, schema: tablePattern.schema, objectId: getMysqlRelId(tablePattern), - replicaIdColumns: replicaIdColumns.columns, - columns: columns + replicaIdColumns: replicaIdColumns.columns }, false ); @@ -402,15 +395,15 @@ export class BinLogStream { this.abortSignal.addEventListener( 'abort', - () => { + async () => { logger.info('Abort signal received, stopping replication...'); - binlogListener.stop(); + await binlogListener.stop(); }, { once: true } ); - // Only returns when the replication is stopped or interrupted by an error await binlogListener.start(); + await binlogListener.replicateUntilStopped(); } ); } @@ -445,46 +438,66 @@ export class BinLogStream { this.metrics.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED).add(1); await batch.commit(lsn); }, - onSchemaChange: async (tableMap: TableMapEntry) => { - if (!this.hasSchemaChange(tableMap)) { - logger.info(`Skipping schema change for ${tableMap.parentSchema}.${tableMap.tableName} - no changes`); - return; - } + onSchemaChange: async (change: SchemaChange) => { + await this.handleSchemaChange(batch, change); + } + }; + } + + private async handleSchemaChange(batch: storage.BucketStorageBatch, change: SchemaChange): Promise { + const tableId = getMysqlRelId({ + schema: this.connections.databaseName, + name: change.table + }); + + const table = this.tableCache.get(tableId); + if (table) { + if (change.type === SchemaChangeType.TRUNCATE_TABLE) { + await batch.truncate([table]); + } else if (change.type === SchemaChangeType.DROP_TABLE) { + await batch.drop([table]); + this.tableCache.delete(tableId); + } else if (change.type === SchemaChangeType.RENAME_TABLE) { + await batch.drop([table]); + this.tableCache.delete(tableId); await this.handleRelation( batch, { - name: tableMap.tableName, - schema: tableMap.parentSchema, + name: change.newTable!, + schema: this.connections.databaseName, objectId: getMysqlRelId({ - schema: tableMap.parentSchema, - name: tableMap.tableName + schema: this.connections.databaseName, + name: change.newTable! }), - replicaIdColumns: Array.from(common.toColumnDescriptors(tableMap).values()) + replicaIdColumns: table.replicaIdColumns }, true ); + } else { + await this.handleRelation(batch, table, true); } - }; - } - - private hasSchemaChange(tableMap: TableMapEntry) { - // Check if the table is already in the cache - const cachedTable = this.tableCache.get( - getMysqlRelId({ - schema: tableMap.parentSchema, - name: tableMap.tableName - }) - ); - if (cachedTable) { - // The table already exists in the cache with the same name, check if the columns are the same - const existingColumns = cachedTable.columns!.sort((a, b) => a.name.localeCompare(b.name)); - const newColumns = Array.from(common.toColumnDescriptors(tableMap).values()).sort((a, b) => - a.name.localeCompare(b.name) + } else if (change.type === SchemaChangeType.CREATE_TABLE) { + const connection = await this.connections.getConnection(); + const replicaIdColumns = await common.getReplicationIdentityColumns({ + connection, + schema: this.connections.databaseName, + tableName: change.table + }); + connection.release(); + await this.handleRelation( + batch, + { + name: change.newTable!, + schema: this.connections.databaseName, + objectId: getMysqlRelId({ + schema: this.connections.databaseName, + name: change.newTable! + }), + replicaIdColumns: replicaIdColumns.columns + }, + true ); - return _.isEqual(existingColumns, newColumns); } - // If the table is not in the cache, it is a new table, or has been renamed - return true; } private async writeChanges( diff --git a/modules/module-mysql/src/replication/zongji/BinLogListener.ts b/modules/module-mysql/src/replication/zongji/BinLogListener.ts index df3abf38..33631c8a 100644 --- a/modules/module-mysql/src/replication/zongji/BinLogListener.ts +++ b/modules/module-mysql/src/replication/zongji/BinLogListener.ts @@ -4,6 +4,8 @@ import { BinLogEvent, StartOptions, TableMapEntry, ZongJi } from '@powersync/mys import * as zongji_utils from './zongji-utils.js'; import { logger } from '@powersync/lib-services-framework'; import { MySQLConnectionManager } from '../MySQLConnectionManager.js'; +import { AST, BaseFrom, Parser, RenameStatement, TruncateStatement } from 'node-sql-parser'; +import timers from 'timers/promises'; // Maximum time the processing queue can be paused before resuming automatically // MySQL server will automatically terminate replication connections after 60 seconds of inactivity, so this guards against that. @@ -11,12 +13,39 @@ const MAX_QUEUE_PAUSE_TIME_MS = 45_000; export type Row = Record; +export enum SchemaChangeType { + CREATE_TABLE = 'create_table', + RENAME_TABLE = 'rename_table', + DROP_TABLE = 'drop_table', + TRUNCATE_TABLE = 'truncate_table', + MODIFY_COLUMN = 'modify_column', + DROP_COLUMN = 'drop_column', + ADD_COLUMN = 'add_column', + RENAME_COLUMN = 'rename_column' +} + +export interface SchemaChange { + type: SchemaChangeType; + /** + * The table that the schema change applies to. + */ + table: string; + newTable?: string; // Only for table renames + column?: { + /** + * The column that the schema change applies to. + */ + column: string; + newColumn?: string; // Only for column renames + }; +} + export interface BinLogEventHandler { onWrite: (rows: Row[], tableMap: TableMapEntry) => Promise; onUpdate: (rowsAfter: Row[], rowsBefore: Row[], tableMap: TableMapEntry) => Promise; onDelete: (rows: Row[], tableMap: TableMapEntry) => Promise; onCommit: (lsn: string) => Promise; - onSchemaChange: (tableMap: TableMapEntry) => Promise; + onSchemaChange: (change: SchemaChange) => Promise; } export interface BinLogListenerOptions { @@ -32,11 +61,14 @@ export interface BinLogListenerOptions { * events on the provided BinLogEventHandler. */ export class BinLogListener { + private sqlParser: Parser; private connectionManager: MySQLConnectionManager; private eventHandler: BinLogEventHandler; private binLogPosition: common.BinLogPosition; private currentGTID: common.ReplicatedGTID | null; + isStopped: boolean = false; + isStopping: boolean = false; zongji: ZongJi; processingQueue: async.QueueObject; /** @@ -49,8 +81,8 @@ export class BinLogListener { this.eventHandler = options.eventHandler; this.binLogPosition = options.startPosition; this.currentGTID = null; - - this.processingQueue = async.queue(this.createQueueWorker(), 1); + this.sqlParser = new Parser(); + this.processingQueue = this.createProcessingQueue(); this.queueMemoryUsage = 0; this.zongji = this.createZongjiListener(); } @@ -63,121 +95,130 @@ export class BinLogListener { return this.connectionManager.options.binlog_queue_memory_limit * 1024 * 1024; } - public async start(): Promise { + public async start(isRestart: boolean = false): Promise { if (this.isStopped) { return; } - logger.info(`Starting replication. Created replica client with serverId:${this.options.serverId}`); + + logger.info( + `${isRestart ? 'Restarting' : 'Starting'} BinLog Listener with replica client id:${this.options.serverId}...` + ); + + // Set a heartbeat interval for the Zongji replication connection + // Zongji does not explicitly handle the heartbeat events - they are categorized as event:unknown + // The heartbeat events are enough to keep the connection alive for setTimeout to work on the socket. + // The heartbeat needs to be set before starting the listener, since the replication connection is locked once replicating + await new Promise((resolve, reject) => { + this.zongji.connection.query( + // In nanoseconds, 10^9 = 1s + 'set @master_heartbeat_period=28*1000000000', + (error: any, results: any, _fields: any) => { + if (error) { + reject(error); + } else { + logger.info('Successfully set up replication connection heartbeat...'); + resolve(results); + } + } + ); + }); + + // The _socket member is only set after a query is run on the connection, so we set the timeout after setting the heartbeat. + // The timeout here must be greater than the master_heartbeat_period. + const socket = this.zongji.connection._socket!; + socket.setTimeout(60_000, () => { + logger.info('Destroying socket due to replication connection timeout.'); + socket.destroy(new Error('Replication connection timeout.')); + }); this.zongji.start({ // We ignore the unknown/heartbeat event since it currently serves no purpose other than to keep the connection alive // tablemap events always need to be included for the other row events to work - includeEvents: ['tablemap', 'writerows', 'updaterows', 'deleterows', 'xid', 'rotate', 'gtidlog'], + includeEvents: ['tablemap', 'writerows', 'updaterows', 'deleterows', 'xid', 'rotate', 'gtidlog', 'query'], includeSchema: { [this.connectionManager.databaseName]: this.options.includedTables }, filename: this.binLogPosition.filename, position: this.binLogPosition.offset, serverId: this.options.serverId } satisfies StartOptions); - return new Promise((resolve, reject) => { - // Handle an edge case where the listener has already been stopped before completing startup - if (this.isStopped) { - logger.info('BinLog listener was stopped before startup completed.'); + return new Promise((resolve) => { + this.zongji.once('ready', () => { + logger.info( + `BinLog Listener ${isRestart ? 'restarted' : 'started'}. Listening for events from position: ${this.binLogPosition.filename}:${this.binLogPosition.offset}.` + ); resolve(); - } - - this.zongji.on('error', (error) => { - if (!this.isStopped) { - logger.error('Binlog listener error:', error); - this.stop(); - reject(error); - } else { - logger.warn('Binlog listener error during shutdown:', error); - } }); + }); + } - this.processingQueue.error((error) => { - if (!this.isStopped) { - logger.error('BinlogEvent processing error:', error); - this.stop(); - reject(error); - } else { - logger.warn('BinlogEvent processing error during shutdown:', error); + public async stop(): Promise { + if (!(this.isStopped || this.isStopping)) { + logger.info('Stopping BinLog Listener...'); + this.isStopping = true; + await new Promise((resolve) => { + if (this.isStopped) { + resolve(); } + this.zongji.once('stopped', () => { + this.isStopped = true; + logger.info('BinLog Listener stopped. Replication ended.'); + resolve(); + }); + this.zongji.stop(); + this.processingQueue.kill(); }); - - this.zongji.on('stopped', () => { - resolve(); - logger.info('BinLog listener stopped. Replication ended.'); - }); - }); + } } - public stop(): void { - if (!this.isStopped) { - this.zongji.stop(); - this.processingQueue.kill(); + public async replicateUntilStopped(): Promise { + while (!this.isStopped) { + await timers.setTimeout(1_000); } } - private get isStopped(): boolean { - return this.zongji.stopped; + private createProcessingQueue(): async.QueueObject { + const queue = async.queue(this.createQueueWorker(), 1); + + queue.error((error) => { + if (!(this.isStopped || this.isStopping)) { + logger.error('Error processing BinLog event:', error); + this.stop(); + } else { + logger.warn('Error processing BinLog event during shutdown:', error); + } + }); + + return queue; } private createZongjiListener(): ZongJi { const zongji = this.connectionManager.createBinlogListener(); zongji.on('binlog', async (evt) => { - logger.info(`Received Binlog event:${evt.getEventName()}`); + logger.info(`Received BinLog event:${evt.getEventName()}`); this.processingQueue.push(evt); this.queueMemoryUsage += evt.size; // When the processing queue grows past the threshold, we pause the binlog listener if (this.isQueueOverCapacity()) { logger.info( - `Binlog processing queue has reached its memory limit of [${this.connectionManager.options.binlog_queue_memory_limit}MB]. Pausing Binlog listener.` + `BinLog processing queue has reached its memory limit of [${this.connectionManager.options.binlog_queue_memory_limit}MB]. Pausing BinLog Listener.` ); zongji.pause(); - const resumeTimeoutPromise = new Promise((resolve) => { - setTimeout(() => resolve('timeout'), MAX_QUEUE_PAUSE_TIME_MS); - }); - + const resumeTimeoutPromise = timers.setTimeout(MAX_QUEUE_PAUSE_TIME_MS); await Promise.race([this.processingQueue.empty(), resumeTimeoutPromise]); - - logger.info(`Binlog processing queue backlog cleared. Resuming Binlog listener.`); + logger.info(`BinLog processing queue backlog cleared. Resuming BinLog Listener.`); zongji.resume(); } }); - zongji.on('ready', async () => { - // Set a heartbeat interval for the Zongji replication connection - // Zongji does not explicitly handle the heartbeat events - they are categorized as event:unknown - // The heartbeat events are enough to keep the connection alive for setTimeout to work on the socket. - await new Promise((resolve, reject) => { - this.zongji.connection.query( - // In nanoseconds, 10^9 = 1s - 'set @master_heartbeat_period=28*1000000000', - function (error: any, results: any, fields: any) { - if (error) { - reject(error); - } else { - logger.info('Successfully set up replication connection heartbeat...'); - resolve(results); - } - } - ); - }); - - // The _socket member is only set after a query is run on the connection, so we set the timeout after setting the heartbeat. - // The timeout here must be greater than the master_heartbeat_period. - const socket = this.zongji.connection._socket!; - socket.setTimeout(60_000, () => { - logger.info('Destroying socket due to replication connection timeout.'); - socket.destroy(new Error('Replication connection timeout.')); - }); - logger.info( - `BinLog listener setup complete. Reading binlog from: ${this.binLogPosition.filename}:${this.binLogPosition.offset}` - ); + zongji.on('error', (error) => { + if (!(this.isStopped || this.isStopping)) { + logger.error('BinLog Listener error:', error); + this.stop(); + } else { + logger.warn('Ignored BinLog Listener error during shutdown:', error); + } }); return zongji; @@ -197,13 +238,26 @@ export class BinLogListener { offset: evt.nextPosition } }); + this.binLogPosition.offset = evt.nextPosition; + logger.info( + `Processed GTID log event. Next position in BinLog: ${this.binLogPosition.filename}:${this.binLogPosition.offset}` + ); break; case zongji_utils.eventIsRotation(evt): + if (this.binLogPosition.filename !== evt.binlogName) { + logger.info( + `Processed Rotate log event. New BinLog file is: ${this.binLogPosition.filename}:${this.binLogPosition.offset}` + ); + } this.binLogPosition.filename = evt.binlogName; this.binLogPosition.offset = evt.position; + break; case zongji_utils.eventIsWriteMutation(evt): await this.eventHandler.onWrite(evt.rows, evt.tableMap[evt.tableId]); + logger.info( + `Processed Write row event of ${evt.rows.length} rows for table: ${evt.tableMap[evt.tableId].tableName}` + ); break; case zongji_utils.eventIsUpdateMutation(evt): await this.eventHandler.onUpdate( @@ -211,29 +265,40 @@ export class BinLogListener { evt.rows.map((row) => row.before), evt.tableMap[evt.tableId] ); + logger.info( + `Processed Update row event of ${evt.rows.length} rows for table: ${evt.tableMap[evt.tableId].tableName}` + ); break; case zongji_utils.eventIsDeleteMutation(evt): await this.eventHandler.onDelete(evt.rows, evt.tableMap[evt.tableId]); + logger.info( + `Processed Delete row event of ${evt.rows.length} rows for table: ${evt.tableMap[evt.tableId].tableName}` + ); break; case zongji_utils.eventIsXid(evt): + this.binLogPosition.offset = evt.nextPosition; const LSN = new common.ReplicatedGTID({ raw_gtid: this.currentGTID!.raw, - position: { - filename: this.binLogPosition.filename, - offset: evt.nextPosition - } + position: this.binLogPosition }).comparable; await this.eventHandler.onCommit(LSN); + logger.info(`Processed Xid log event. Transaction LSN: ${LSN}.`); break; - case zongji_utils.eventIsTableMap(evt): - const tableMapEntry = evt.tableMap[evt.tableId]; - logger.info( - `Potential schema change detected for table: ${tableMapEntry.tableName}. Pausing Binlog listener.)` - ); - this.zongji.pause(); - await this.eventHandler.onSchemaChange(evt.tableMap[evt.tableId]); - logger.info(`Schema change for table ${tableMapEntry.tableName} handled. Resuming Binlog listener.`); - this.zongji.resume(); + case zongji_utils.eventIsQuery(evt): + // Ignore BEGIN queries + if (evt.query === 'BEGIN') { + break; + } + const ast = this.sqlParser.astify(evt.query, { database: 'MySQL' }); + const statements = Array.isArray(ast) ? ast : [ast]; + const schemaChanges = this.toSchemaChanges(statements); + if (schemaChanges.length > 0) { + await this.handleSchemaChanges(schemaChanges, evt.nextPosition); + } else { + // Still have to update the binlog position, even if there were no effective schema changes + this.binLogPosition.offset = evt.nextPosition; + } + break; } @@ -241,6 +306,131 @@ export class BinLogListener { }; } + private async handleSchemaChanges(changes: SchemaChange[], nextPosition: number): Promise { + logger.info(`Stopping BinLog Listener to process ${changes.length} schema change events...`); + // Since handling the schema changes can take a long time, we need to stop the Zongji listener instead of pausing it. + await new Promise((resolve) => { + this.zongji.once('stopped', () => { + resolve(); + }); + this.zongji.stop(); + }); + for (const change of changes) { + logger.info(`Processing schema change: ${change.type} for table: ${change.table}`); + await this.eventHandler.onSchemaChange(change); + } + + this.binLogPosition.offset = nextPosition; + // Wait until all the current events in the processing queue are processed and the binlog position has been updated + // otherwise we risk processing the same events again. + if (this.processingQueue.length() > 0) { + await this.processingQueue.empty(); + } + this.zongji = this.createZongjiListener(); + logger.info(`Successfully processed schema changes.`); + // Restart the Zongji listener + await this.start(true); + } + + private toSchemaChanges(statements: AST[]): SchemaChange[] { + // TODO: We need to check if schema filtering is also required + const changes: SchemaChange[] = []; + for (const statement of statements) { + // @ts-ignore + if (statement.type === 'rename') { + const renameStatement = statement as RenameStatement; + for (const table of renameStatement.table) { + changes.push({ + type: SchemaChangeType.RENAME_TABLE, + table: table[0].table, + newTable: table[1].table + }); + } + } // @ts-ignore + else if (statement.type === 'truncate' && statement.keyword === 'table') { + const truncateStatement = statement as TruncateStatement; + // Truncate statements can apply to multiple tables + for (const entity of truncateStatement.name) { + changes.push({ type: SchemaChangeType.TRUNCATE_TABLE, table: entity.table }); + } + } else if (statement.type === 'create' && statement.keyword === 'table' && statement.temporary === null) { + changes.push({ + type: SchemaChangeType.CREATE_TABLE, + table: statement.table![0].table + }); + } else if (statement.type === 'drop' && statement.keyword === 'table') { + // Drop statements can apply to multiple tables + for (const entity of statement.name) { + changes.push({ type: SchemaChangeType.DROP_TABLE, table: entity.table }); + } + } else if (statement.type === 'alter') { + const expression = statement.expr[0]; + const fromTable = statement.table[0] as BaseFrom; + if (expression.resource === 'table') { + if (expression.action === 'rename') { + changes.push({ + type: SchemaChangeType.RENAME_TABLE, + table: fromTable.table, + newTable: expression.table + }); + } + } else if (expression.resource === 'column') { + const column = expression.column; + if (expression.action === 'drop') { + changes.push({ + type: SchemaChangeType.DROP_COLUMN, + table: fromTable.table, + column: { + column: column.column + } + }); + } else if (expression.action === 'add') { + changes.push({ + type: SchemaChangeType.ADD_COLUMN, + table: fromTable.table, + column: { + column: column.column + } + }); + } else if (expression.action === 'modify') { + changes.push({ + type: SchemaChangeType.MODIFY_COLUMN, + table: fromTable.table, + column: { + column: column.column + } + }); + } else if (expression.action === 'change') { + changes.push({ + type: SchemaChangeType.RENAME_COLUMN, + table: fromTable.table, + column: { + column: expression.old_column.column, + newColumn: column.column + } + }); + } else if (expression.action === 'rename') { + changes.push({ + type: SchemaChangeType.RENAME_COLUMN, + table: fromTable.table, + column: { + column: expression.old_column.column, + newColumn: column.column + } + }); + } + } + } + } + + // Filter out schema changes that are not relevant to the included tables + return changes.filter( + (change) => + this.options.includedTables.includes(change.table) || + (change.newTable && this.options.includedTables.includes(change.newTable)) + ); + } + isQueueOverCapacity(): boolean { return this.queueMemoryUsage >= this.queueMemoryLimit; } diff --git a/modules/module-mysql/src/replication/zongji/zongji-utils.ts b/modules/module-mysql/src/replication/zongji/zongji-utils.ts index ee9e4c53..b1d2c579 100644 --- a/modules/module-mysql/src/replication/zongji/zongji-utils.ts +++ b/modules/module-mysql/src/replication/zongji/zongji-utils.ts @@ -5,7 +5,8 @@ import { BinLogRotationEvent, BinLogTableMapEvent, BinLogRowUpdateEvent, - BinLogXidEvent + BinLogXidEvent, + BinLogQueryEvent } from '@powersync/mysql-zongji'; export function eventIsGTIDLog(event: BinLogEvent): event is BinLogGTIDLogEvent { @@ -35,3 +36,7 @@ export function eventIsDeleteMutation(event: BinLogEvent): event is BinLogRowEve export function eventIsUpdateMutation(event: BinLogEvent): event is BinLogRowUpdateEvent { return event.getEventName() == 'updaterows'; } + +export function eventIsQuery(event: BinLogEvent): event is BinLogQueryEvent { + return event.getEventName() == 'query'; +} diff --git a/modules/module-mysql/src/types/node-sql-parser-extended-types.ts b/modules/module-mysql/src/types/node-sql-parser-extended-types.ts new file mode 100644 index 00000000..de1e7687 --- /dev/null +++ b/modules/module-mysql/src/types/node-sql-parser-extended-types.ts @@ -0,0 +1,17 @@ +import 'node-sql-parser'; + +/** + * Missing Type definitions for the node-sql-parser + */ +declare module 'node-sql-parser' { + interface RenameStatement { + type: 'rename'; + table: { db: string | null; table: string }[][]; + } + + interface TruncateStatement { + type: 'truncate'; + keyword: 'table'; // There are more keywords possible, but we only care about 'table' for now + name: { db: string | null; table: string; as: string | null }[]; + } +} diff --git a/modules/module-mysql/test/src/BinLogListener.test.ts b/modules/module-mysql/test/src/BinLogListener.test.ts index 8f37a8bf..b0c4dac7 100644 --- a/modules/module-mysql/test/src/BinLogListener.test.ts +++ b/modules/module-mysql/test/src/BinLogListener.test.ts @@ -1,5 +1,11 @@ import { describe, test, beforeEach, vi, expect, afterEach } from 'vitest'; -import { BinLogEventHandler, BinLogListener, Row } from '@module/replication/zongji/BinLogListener.js'; +import { + BinLogEventHandler, + BinLogListener, + Row, + SchemaChange, + SchemaChangeType +} from '@module/replication/zongji/BinLogListener.js'; import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManager.js'; import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js'; import { v4 as uuid } from 'uuid'; @@ -32,7 +38,7 @@ describe('BinlogListener tests', () => { connectionManager: connectionManager, eventHandler: eventHandler, startPosition: fromGTID.position, - includedTables: ['test_DATA', 'test_DATA_new'], + includedTables: ['test_DATA'], serverId: createRandomServerId(1) }); }); @@ -45,10 +51,9 @@ describe('BinlogListener tests', () => { const stopSpy = vi.spyOn(binLogListener.zongji, 'stop'); const queueStopSpy = vi.spyOn(binLogListener.processingQueue, 'kill'); - const startPromise = binLogListener.start(); - setTimeout(async () => binLogListener.stop(), 50); + await binLogListener.start(); + await binLogListener.stop(); - await expect(startPromise).resolves.toBeUndefined(); expect(stopSpy).toHaveBeenCalled(); expect(queueStopSpy).toHaveBeenCalled(); }); @@ -63,7 +68,7 @@ describe('BinlogListener tests', () => { const ROW_COUNT = 10; await insertRows(connectionManager, ROW_COUNT); - const startPromise = binLogListener.start(); + await binLogListener.start(); // Wait for listener to pause due to queue reaching capacity await vi.waitFor(() => expect(pauseSpy).toHaveBeenCalled(), { timeout: 5000 }); @@ -73,14 +78,13 @@ describe('BinlogListener tests', () => { eventHandler.unpause!(); await vi.waitFor(() => expect(eventHandler.rowsWritten).equals(ROW_COUNT), { timeout: 5000 }); - binLogListener.stop(); - await expect(startPromise).resolves.toBeUndefined(); + await binLogListener.stop(); // Confirm resume was called after unpausing expect(resumeSpy).toHaveBeenCalled(); }); test('Binlog row events are correctly forwarded to provided binlog events handler', async () => { - const startPromise = binLogListener.start(); + await binLogListener.start(); const ROW_COUNT = 10; await insertRows(connectionManager, ROW_COUNT); @@ -93,21 +97,117 @@ describe('BinlogListener tests', () => { await deleteRows(connectionManager); await vi.waitFor(() => expect(eventHandler.rowsDeleted).equals(ROW_COUNT), { timeout: 5000 }); - binLogListener.stop(); - await expect(startPromise).resolves.toBeUndefined(); + await binLogListener.stop(); }); - test('Binlog schema change events are correctly forwarded to provided binlog events handler', async () => { - const startPromise = binLogListener.start(); + test('ALTER TABLE RENAME schema changes', async () => { + await binLogListener.start(); + await connectionManager.query(`ALTER TABLE test_DATA RENAME test_DATA_new`); + await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); + await binLogListener.stop(); + expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.RENAME_TABLE); + expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); + expect(eventHandler.schemaChanges[0].newTable).toEqual('test_DATA_new'); + }); + + test('RENAME TABLE schema changes', async () => { + await binLogListener.start(); await connectionManager.query(`RENAME TABLE test_DATA TO test_DATA_new`); - // Table map events are only emitted before row events - await connectionManager.query( - `INSERT INTO test_DATA_new(id, description) VALUES('${uuid()}','test ${crypto.randomBytes(100).toString('hex')}')` - ); - await vi.waitFor(() => expect(eventHandler.latestSchemaChange).toBeDefined(), { timeout: 5000 }); - binLogListener.stop(); - await expect(startPromise).resolves.toBeUndefined(); - expect(eventHandler.latestSchemaChange?.tableName).toEqual('test_DATA_new'); + await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); + await binLogListener.stop(); + expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.RENAME_TABLE); + expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); + expect(eventHandler.schemaChanges[0].newTable).toEqual('test_DATA_new'); + }); + + test('RENAME TABLE multipe table schema changes', async () => { + await binLogListener.start(); + await connectionManager.query(`RENAME TABLE + test_DATA TO test_DATA_new, + test_DATA_new TO test_DATA + `); + await vi.waitFor(() => expect(eventHandler.schemaChanges.length == 2).toBeTruthy(), { timeout: 5000 }); + await binLogListener.stop(); + expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.RENAME_TABLE); + expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); + expect(eventHandler.schemaChanges[0].newTable).toEqual('test_DATA_new'); + + expect(eventHandler.schemaChanges[1].type).toBe(SchemaChangeType.RENAME_TABLE); + expect(eventHandler.schemaChanges[1].table).toEqual('test_DATA_new'); + expect(eventHandler.schemaChanges[1].newTable).toEqual('test_DATA'); + }); + + test('TRUNCATE TABLE schema changes', async () => { + await binLogListener.start(); + await connectionManager.query(`TRUNCATE TABLE test_DATA`); + await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); + await binLogListener.stop(); + expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.TRUNCATE_TABLE); + expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); + }); + + test('DROP AND CREATE TABLE schema changes', async () => { + await binLogListener.start(); + await connectionManager.query(`DROP TABLE test_DATA`); + await connectionManager.query(`CREATE TABLE test_DATA (id CHAR(36) PRIMARY KEY, description MEDIUMTEXT)`); + await vi.waitFor(() => expect(eventHandler.schemaChanges.length === 2).toBeTruthy(), { timeout: 5000 }); + await binLogListener.stop(); + expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.DROP_TABLE); + expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); + expect(eventHandler.schemaChanges[1].type).toBe(SchemaChangeType.CREATE_TABLE); + expect(eventHandler.schemaChanges[1].table).toEqual('test_DATA'); + }); + + test('ALTER TABLE DROP COLUMN schema changes', async () => { + await binLogListener.start(); + await connectionManager.query(`ALTER TABLE test_DATA DROP COLUMN description`); + await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); + await binLogListener.stop(); + expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.DROP_COLUMN); + expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); + expect(eventHandler.schemaChanges[0].column?.column).toEqual('description'); + }); + + test('ALTER TABLE ADD COLUMN schema changes', async () => { + await binLogListener.start(); + await connectionManager.query(`ALTER TABLE test_DATA ADD COLUMN new_column VARCHAR(255)`); + await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); + await binLogListener.stop(); + expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.ADD_COLUMN); + expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); + expect(eventHandler.schemaChanges[0].column?.column).toEqual('new_column'); + }); + + test('ALTER TABLE MODIFY COLUMN type schema changes', async () => { + await binLogListener.start(); + await connectionManager.query(`ALTER TABLE test_DATA MODIFY COLUMN description TEXT`); + await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); + await binLogListener.stop(); + expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.MODIFY_COLUMN); + expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); + expect(eventHandler.schemaChanges[0].column?.column).toEqual('description'); + }); + + test('ALTER TABLE CHANGE COLUMN column rename schema changes', async () => { + await binLogListener.start(); + await connectionManager.query(`ALTER TABLE test_DATA CHANGE COLUMN description description_new MEDIUMTEXT`); + await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); + await binLogListener.stop(); + expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.RENAME_COLUMN); + expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); + expect(eventHandler.schemaChanges[0].column?.column).toEqual('description'); + expect(eventHandler.schemaChanges[0].column?.newColumn).toEqual('description_new'); + }); + + test('ALTER TABLE RENAME COLUMN column rename schema changes', async () => { + await binLogListener.start(); + await connectionManager.query(`ALTER TABLE test_DATA RENAME COLUMN description TO description_new`); + await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); + await binLogListener.stop(); + expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.RENAME_COLUMN); + expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); + expect(eventHandler.schemaChanges[0].column?.column).toEqual('description'); + expect(eventHandler.schemaChanges[0].column?.newColumn).toEqual('description_new'); }); }); @@ -140,7 +240,7 @@ class TestBinLogEventHandler implements BinLogEventHandler { rowsUpdated = 0; rowsDeleted = 0; commitCount = 0; - latestSchemaChange: TableMapEntry | undefined; + schemaChanges: SchemaChange[] = []; unpause: ((value: void | PromiseLike) => void) | undefined; private pausedPromise: Promise | undefined; @@ -170,7 +270,7 @@ class TestBinLogEventHandler implements BinLogEventHandler { this.commitCount++; } - async onSchemaChange(tableMap: TableMapEntry) { - this.latestSchemaChange = tableMap; + async onSchemaChange(change: SchemaChange) { + this.schemaChanges.push(change); } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 504a4d13..ace5dd83 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -258,8 +258,8 @@ importers: specifier: workspace:* version: link:../../libs/lib-services '@powersync/mysql-zongji': - specifier: 0.0.0-dev-20250605124659 - version: 0.0.0-dev-20250605124659 + specifier: 0.0.0-dev-20250610140703 + version: 0.0.0-dev-20250610140703 '@powersync/service-core': specifier: workspace:* version: link:../../packages/service-core @@ -281,6 +281,9 @@ importers: mysql2: specifier: ^3.11.0 version: 3.11.3 + node-sql-parser: + specifier: ^5.3.9 + version: 5.3.9 semver: specifier: ^7.5.4 version: 7.6.2 @@ -1287,8 +1290,8 @@ packages: resolution: {integrity: sha512-UA91GwWPhFExt3IizW6bOeY/pQ0BkuNwKjk9iQW9KqxluGCrg4VenZ0/L+2Y0+ZOtme72EVvg6v0zo3AMQRCeA==} engines: {node: '>=12'} - '@powersync/mysql-zongji@0.0.0-dev-20250605124659': - resolution: {integrity: sha512-HYPaejOMY8yok7g2DIBgdnm2QiFHb28QlsCorq+K5KSKeT8NLR0xpTPZ9JvvYhmE5ooFQVnO7ZZBEOFiQzEv6Q==} + '@powersync/mysql-zongji@0.0.0-dev-20250610140703': + resolution: {integrity: sha512-chcUCpawar5I89CkDTHmSwoAdto9DptAMGwznXS9KqpigFcfZz7jGbGpwUCxMM03VhSwICrND9XyxCQezOm9fg==} engines: {node: '>=22.0.0'} '@powersync/service-jsonbig@0.17.10': @@ -1535,6 +1538,9 @@ packages: '@types/node@22.5.5': resolution: {integrity: sha512-Xjs4y5UPO/CLdzpgR6GirZJx36yScjh73+2NlLlkFRSoQN8B0DpfXPdZGnvVmLRLOsqDpOfTNv7D9trgGhmOIA==} + '@types/pegjs@0.10.6': + resolution: {integrity: sha512-eLYXDbZWXh2uxf+w8sXS8d6KSoXTswfps6fvCUuVAGN8eRpfe7h9eSRydxiSJvo9Bf+GzifsDOr9TMQlmJdmkw==} + '@types/pg-pool@2.0.4': resolution: {integrity: sha512-qZAvkv1K3QbmHHFYSNRYPkRjOWRLBYrL4B9c+wG0GSVGBw0NtJwPcgx/DSddeDJvRGMHCEQ4VMEVfuJ/0gZ3XQ==} @@ -2900,6 +2906,10 @@ packages: resolution: {integrity: sha512-2YEOR5qlI1zUFbGMLKNfsrR5JUvFg9LxIRVE+xJe962pfVLH0rnItqLzv96XVs1Y1UIR8FxsXAuvX/lYAWZ2BQ==} engines: {node: '>=8'} + node-sql-parser@5.3.9: + resolution: {integrity: sha512-yJuCNCUWqS296krMKooLIJcZy+Q0OL6ZsNDxrwqj+HdGMHVT0ChPIFF1vqCexDe51VWKEwhU65OgvKyiyRcQLg==} + engines: {node: '>=8'} + nodemon@3.1.4: resolution: {integrity: sha512-wjPBbFhtpJwmIeY2yP7QF+UKzPfltVGtfce1g/bB15/8vCGZj8uxD62b/b9M9/WVgme0NZudpownKN+c0plXlQ==} engines: {node: '>=10'} @@ -4754,7 +4764,7 @@ snapshots: '@pnpm/network.ca-file': 1.0.2 config-chain: 1.1.13 - '@powersync/mysql-zongji@0.0.0-dev-20250605124659': + '@powersync/mysql-zongji@0.0.0-dev-20250610140703': dependencies: '@vlasky/mysql': 2.18.6 big-integer: 1.6.52 @@ -4988,6 +4998,8 @@ snapshots: dependencies: undici-types: 6.19.8 + '@types/pegjs@0.10.6': {} + '@types/pg-pool@2.0.4': dependencies: '@types/pg': 8.6.1 @@ -6390,6 +6402,11 @@ snapshots: dependencies: big-integer: 1.6.51 + node-sql-parser@5.3.9: + dependencies: + '@types/pegjs': 0.10.6 + big-integer: 1.6.52 + nodemon@3.1.4: dependencies: chokidar: 3.6.0 From 12709390ad258f3669310b7e61852ef6d314154b Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 18 Jun 2025 09:51:54 +0200 Subject: [PATCH 19/48] Include powersync core version in metrics metadata --- packages/service-core/src/metrics/open-telemetry/util.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/service-core/src/metrics/open-telemetry/util.ts b/packages/service-core/src/metrics/open-telemetry/util.ts index af4af406..a3b25659 100644 --- a/packages/service-core/src/metrics/open-telemetry/util.ts +++ b/packages/service-core/src/metrics/open-telemetry/util.ts @@ -7,6 +7,8 @@ import { OpenTelemetryMetricsFactory } from './OpenTelemetryMetricsFactory.js'; import { MetricsFactory } from '../metrics-interfaces.js'; import { logger } from '@powersync/lib-services-framework'; +import pkg from '../../../package.json' with { type: 'json' }; + export interface RuntimeMetadata { [key: string]: string | number | undefined; } @@ -61,7 +63,8 @@ export function createOpenTelemetryMetricsFactory(context: ServiceContext): Metr const meterProvider = new MeterProvider({ resource: new Resource( { - ['service']: 'PowerSync' + ['service']: 'PowerSync', + ['service.version']: pkg.version }, runtimeMetadata ), From 7500bed45367967a00e4c54cc72ec082e43422cb Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 18 Jun 2025 09:58:35 +0200 Subject: [PATCH 20/48] Code analysis cleanup --- .../src/storage/implementation/MongoSyncBucketStorage.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index 4407ab69..92eeacf0 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -550,7 +550,6 @@ export class MongoSyncBucketStorage `${this.slot_name} Cleared batch of data in ${lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, continuing...` ); await timers.setTimeout(lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS / 5); - continue; } else { throw e; } From 54e6a9da3eaddc072c38519e58afedf3e31ddc68 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 18 Jun 2025 12:15:55 +0200 Subject: [PATCH 21/48] Merge conflicts --- modules/module-mongodb/src/replication/ChangeStream.ts | 2 +- modules/module-mongodb/src/replication/MongoRelation.ts | 2 +- modules/module-mysql/dev/docker/mysql/docker-compose.yaml | 6 +++--- modules/module-mysql/src/replication/BinLogStream.ts | 2 +- modules/module-postgres/src/replication/SnapshotQuery.ts | 8 ++++---- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index 8b9d3732..c5f40fc3 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -215,7 +215,7 @@ export class ChangeStream { async estimatedCountNumber(table: storage.SourceTable): Promise { const db = this.client.db(table.schema); - return await db.collection(table.table).estimatedDocumentCount(); + return await db.collection(table.name).estimatedDocumentCount(); } private async getSnapshotLsn(): Promise { diff --git a/modules/module-mongodb/src/replication/MongoRelation.ts b/modules/module-mongodb/src/replication/MongoRelation.ts index cc471e6e..4ac11efb 100644 --- a/modules/module-mongodb/src/replication/MongoRelation.ts +++ b/modules/module-mongodb/src/replication/MongoRelation.ts @@ -22,7 +22,7 @@ export function getMongoRelation(source: mongo.ChangeStreamNameSpace): storage.S */ export function getCacheIdentifier(source: storage.SourceEntityDescriptor | storage.SourceTable): string { if (source instanceof storage.SourceTable) { - return `${source.schema}.${source.table}`; + return `${source.schema}.${source.name}`; } return `${source.schema}.${source.name}`; } diff --git a/modules/module-mysql/dev/docker/mysql/docker-compose.yaml b/modules/module-mysql/dev/docker/mysql/docker-compose.yaml index 50dfd2d2..274e53b6 100644 --- a/modules/module-mysql/dev/docker/mysql/docker-compose.yaml +++ b/modules/module-mysql/dev/docker/mysql/docker-compose.yaml @@ -1,5 +1,5 @@ services: - mysql: + mysql_test: image: mysql:8.0 environment: MYSQL_ROOT_PASSWORD: root_password @@ -11,7 +11,7 @@ services: volumes: - ./init-scripts/my.cnf:/etc/mysql/my.cnf - ./init-scripts/mysql.sql:/docker-entrypoint-initdb.d/init_user.sql - - mysql_data:/var/lib/mysql + - mysql_test_data:/var/lib/mysql volumes: - mysql_data: + mysql_test_data: diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index c1cd4c91..e00a8443 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -427,7 +427,7 @@ export class BinLogStream { this.abortSignal.addEventListener( 'abort', async () => { - logger.info('Abort signal received, stopping replication...'); + this.logger.info('Abort signal received, stopping replication...'); await binlogListener.stop(); }, { once: true } diff --git a/modules/module-postgres/src/replication/SnapshotQuery.ts b/modules/module-postgres/src/replication/SnapshotQuery.ts index 438637cb..b826c215 100644 --- a/modules/module-postgres/src/replication/SnapshotQuery.ts +++ b/modules/module-postgres/src/replication/SnapshotQuery.ts @@ -36,7 +36,7 @@ export class SimpleSnapshotQuery implements SnapshotQuery { ) {} public async initialize(): Promise { - await this.connection.query(`DECLARE snapshot_cursor CURSOR FOR SELECT * FROM ${this.table.escapedIdentifier}`); + await this.connection.query(`DECLARE snapshot_cursor CURSOR FOR SELECT * FROM ${this.table.qualifiedName}`); } public nextChunk(): AsyncIterableIterator { @@ -121,7 +121,7 @@ export class ChunkedSnapshotQuery implements SnapshotQuery { const escapedKeyName = escapeIdentifier(this.key.name); if (this.lastKey == null) { stream = this.connection.stream( - `SELECT * FROM ${this.table.escapedIdentifier} ORDER BY ${escapedKeyName} LIMIT ${this.chunkSize}` + `SELECT * FROM ${this.table.qualifiedName} ORDER BY ${escapedKeyName} LIMIT ${this.chunkSize}` ); } else { if (this.key.typeId == null) { @@ -129,7 +129,7 @@ export class ChunkedSnapshotQuery implements SnapshotQuery { } let type: StatementParam['type'] = Number(this.key.typeId); stream = this.connection.stream({ - statement: `SELECT * FROM ${this.table.escapedIdentifier} WHERE ${escapedKeyName} > $1 ORDER BY ${escapedKeyName} LIMIT ${this.chunkSize}`, + statement: `SELECT * FROM ${this.table.qualifiedName} WHERE ${escapedKeyName} > $1 ORDER BY ${escapedKeyName} LIMIT ${this.chunkSize}`, params: [{ value: this.lastKey, type }] }); } @@ -197,7 +197,7 @@ export class IdSnapshotQuery implements SnapshotQuery { throw new Error(`Cannot determine primary key array type for ${JSON.stringify(keyDefinition)}`); } yield* this.connection.stream({ - statement: `SELECT * FROM ${this.table.escapedIdentifier} WHERE ${escapeIdentifier(keyDefinition.name)} = ANY($1)`, + statement: `SELECT * FROM ${this.table.qualifiedName} WHERE ${escapeIdentifier(keyDefinition.name)} = ANY($1)`, params: [ { type: type, From a5582b11c3e7cc16607a554d01b58e9d5ea13f27 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 18 Jun 2025 12:17:50 +0200 Subject: [PATCH 22/48] Fixed parser import --- .../src/replication/zongji/BinLogListener.ts | 14 +++++++------- packages/service-core/src/storage/SourceTable.ts | 12 +++++------- pnpm-lock.yaml | 6 ------ 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/modules/module-mysql/src/replication/zongji/BinLogListener.ts b/modules/module-mysql/src/replication/zongji/BinLogListener.ts index 95e1ca15..fe2f5e22 100644 --- a/modules/module-mysql/src/replication/zongji/BinLogListener.ts +++ b/modules/module-mysql/src/replication/zongji/BinLogListener.ts @@ -4,7 +4,7 @@ import { BinLogEvent, StartOptions, TableMapEntry, ZongJi } from '@powersync/mys import * as zongji_utils from './zongji-utils.js'; import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework'; import { MySQLConnectionManager } from '../MySQLConnectionManager.js'; -import { AST, BaseFrom, Parser, RenameStatement, TruncateStatement } from 'node-sql-parser'; +import * as parser from 'node-sql-parser'; import timers from 'timers/promises'; // Maximum time the processing queue can be paused before resuming automatically @@ -64,7 +64,7 @@ export interface BinLogListenerOptions { * events on the provided BinLogEventHandler. */ export class BinLogListener { - private sqlParser: Parser; + private sqlParser: parser.Parser; private connectionManager: MySQLConnectionManager; private eventHandler: BinLogEventHandler; private binLogPosition: common.BinLogPosition; @@ -86,7 +86,7 @@ export class BinLogListener { this.eventHandler = options.eventHandler; this.binLogPosition = options.startPosition; this.currentGTID = null; - this.sqlParser = new Parser(); + this.sqlParser = new parser.Parser(); this.processingQueue = this.createProcessingQueue(); this.queueMemoryUsage = 0; this.zongji = this.createZongjiListener(); @@ -338,13 +338,13 @@ export class BinLogListener { await this.start(true); } - private toSchemaChanges(statements: AST[]): SchemaChange[] { + private toSchemaChanges(statements: parser.AST[]): SchemaChange[] { // TODO: We need to check if schema filtering is also required const changes: SchemaChange[] = []; for (const statement of statements) { // @ts-ignore if (statement.type === 'rename') { - const renameStatement = statement as RenameStatement; + const renameStatement = statement as parser.RenameStatement; for (const table of renameStatement.table) { changes.push({ type: SchemaChangeType.RENAME_TABLE, @@ -354,7 +354,7 @@ export class BinLogListener { } } // @ts-ignore else if (statement.type === 'truncate' && statement.keyword === 'table') { - const truncateStatement = statement as TruncateStatement; + const truncateStatement = statement as parser.TruncateStatement; // Truncate statements can apply to multiple tables for (const entity of truncateStatement.name) { changes.push({ type: SchemaChangeType.TRUNCATE_TABLE, table: entity.table }); @@ -371,7 +371,7 @@ export class BinLogListener { } } else if (statement.type === 'alter') { const expression = statement.expr[0]; - const fromTable = statement.table[0] as BaseFrom; + const fromTable = statement.table[0] as parser.BaseFrom; if (expression.resource === 'table') { if (expression.action === 'rename') { changes.push({ diff --git a/packages/service-core/src/storage/SourceTable.ts b/packages/service-core/src/storage/SourceTable.ts index 857c28d8..8e595154 100644 --- a/packages/service-core/src/storage/SourceTable.ts +++ b/packages/service-core/src/storage/SourceTable.ts @@ -55,8 +55,11 @@ export class SourceTable implements SourceEntityDescriptor { */ public snapshotStatus: TableSnapshotStatus | undefined = undefined; + public snapshotComplete: boolean; - constructor(public readonly options: SourceTableOptions) {} + constructor(public readonly options: SourceTableOptions) { + this.snapshotComplete = options.snapshotComplete; + } get id() { return this.options.id; @@ -81,10 +84,6 @@ export class SourceTable implements SourceEntityDescriptor { return this.options.replicaIdColumns; } - get snapshotComplete() { - return this.options.snapshotComplete; - } - /** * Sanitized name of the entity in the format of "{schema}.{entity name}" * Suitable for safe use in Postgres queries. @@ -109,8 +108,7 @@ export class SourceTable implements SourceEntityDescriptor { name: this.name, replicaIdColumns: this.replicaIdColumns, snapshotComplete: this.snapshotComplete - } - ); + }); copy.syncData = this.syncData; copy.syncParameters = this.syncParameters; copy.snapshotStatus = this.snapshotStatus; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ace5dd83..c9292285 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -275,9 +275,6 @@ importers: async: specifier: ^3.2.4 version: 3.2.5 - lodash: - specifier: ^4.17.21 - version: 4.17.21 mysql2: specifier: ^3.11.0 version: 3.11.3 @@ -309,9 +306,6 @@ importers: '@types/async': specifier: ^3.2.24 version: 3.2.24 - '@types/lodash': - specifier: ^4.17.5 - version: 4.17.6 '@types/semver': specifier: ^7.5.4 version: 7.5.8 From d34f8faa1dc37673672a8de5ea9e338b32f64b52 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 25 Jun 2025 16:26:20 +0200 Subject: [PATCH 23/48] Fixed mysql->sqlite rows parsing that would filter out columns with null values --- .../src/common/mysql-to-sqlite.ts | 136 +++++++++--------- 1 file changed, 67 insertions(+), 69 deletions(-) diff --git a/modules/module-mysql/src/common/mysql-to-sqlite.ts b/modules/module-mysql/src/common/mysql-to-sqlite.ts index aa5aa949..e9c38571 100644 --- a/modules/module-mysql/src/common/mysql-to-sqlite.ts +++ b/modules/module-mysql/src/common/mysql-to-sqlite.ts @@ -109,80 +109,78 @@ export function toSQLiteRow(row: Record, columns: Map Date: Wed, 25 Jun 2025 16:29:05 +0200 Subject: [PATCH 24/48] Cleaned up SchemaChange handling in BinLogListener Improved binlog table filtering Added extended type definitions for node-sql-parser package --- modules/module-mysql/package.json | 2 +- .../src/replication/BinLogStream.ts | 147 ++++++--- .../src/replication/zongji/BinLogListener.ts | 279 ++++++++++-------- .../types/node-sql-parser-extended-types.ts | 2 +- pnpm-lock.yaml | 10 +- 5 files changed, 261 insertions(+), 179 deletions(-) diff --git a/modules/module-mysql/package.json b/modules/module-mysql/package.json index a1495a6b..299c8838 100644 --- a/modules/module-mysql/package.json +++ b/modules/module-mysql/package.json @@ -33,7 +33,7 @@ "@powersync/service-sync-rules": "workspace:*", "@powersync/service-types": "workspace:*", "@powersync/service-jsonbig": "workspace:*", - "@powersync/mysql-zongji": "0.0.0-dev-20250610140703", + "@powersync/mysql-zongji": "0.0.0-dev-20250619101606", "async": "^3.2.4", "mysql2": "^3.11.0", "node-sql-parser": "^5.3.9", diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index e00a8443..594064ea 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -83,7 +83,7 @@ export class BinLogStream { * Keep track of whether we have done a commit or keepalive yet. * We can only compute replication lag if isStartingReplication == false, or oldestUncommittedChange is present. */ - private isStartingReplication = true; + isStartingReplication = true; constructor(private options: BinLogStreamOptions) { this.logger = options.logger ?? defaultLogger; @@ -161,7 +161,7 @@ export class BinLogStream { await promiseConnection.query('START TRANSACTION'); try { gtid = await common.readExecutedGtid(promiseConnection); - await this.snapshotTable(connection.connection, batch, result.table); + await this.snapshotTable(connection as mysql.Connection, batch, result.table); await promiseConnection.query('COMMIT'); } catch (e) { await this.tryRollback(promiseConnection); @@ -187,29 +187,28 @@ export class BinLogStream { const connection = await this.connections.getConnection(); const matchedTables: string[] = await common.getTablesFromPattern(connection, tablePattern); + connection.release(); let tables: storage.SourceTable[] = []; for (const matchedTable of matchedTables) { - const replicaIdColumns = await common.getReplicationIdentityColumns({ - connection: connection, - schema: tablePattern.schema, - tableName: matchedTable - }); + const replicaIdColumns = await this.getReplicaIdColumns(matchedTable); const table = await this.handleRelation( batch, { name: matchedTable, schema: tablePattern.schema, - objectId: getMysqlRelId(tablePattern), - replicaIdColumns: replicaIdColumns.columns + objectId: getMysqlRelId({ + schema: tablePattern.schema, + name: matchedTable + }), + replicaIdColumns: replicaIdColumns }, false ); tables.push(table); } - connection.release(); return tables; } @@ -415,9 +414,10 @@ export class BinLogStream { const binlogEventHandler = this.createBinlogEventHandler(batch); // Only listen for changes to tables in the sync rules const includedTables = [...this.tableCache.values()].map((table) => table.name); + const binlogListener = new BinLogListener({ logger: this.logger, - includedTables: includedTables, + tableFilter: (table: string) => this.matchesTable(table), startPosition: binLogPositionState, connectionManager: this.connections, serverId: serverId, @@ -440,6 +440,18 @@ export class BinLogStream { } } + private matchesTable(tableName: string): boolean { + const sourceTables = this.syncRules.getSourceTables(); + + return ( + sourceTables.findIndex((table) => + table.isWildcard + ? tableName.startsWith(table.tablePattern.substring(0, table.tablePattern.length - 1)) + : tableName === table.name + ) !== -1 + ); + } + private createBinlogEventHandler(batch: storage.BucketStorageBatch): BinLogEventHandler { return { onWrite: async (rows: Row[], tableMap: TableMapEntry) => { @@ -488,21 +500,37 @@ export class BinLogStream { } private async handleSchemaChange(batch: storage.BucketStorageBatch, change: SchemaChange): Promise { - const tableId = getMysqlRelId({ - schema: this.connections.databaseName, - name: change.table - }); + if (change.type === SchemaChangeType.CREATE_TABLE) { + const replicaIdColumns = await this.getReplicaIdColumns(change.table); + await this.handleRelation( + batch, + { + name: change.table, + schema: this.connections.databaseName, + objectId: getMysqlRelId({ + schema: this.connections.databaseName, + name: change.table + }), + replicaIdColumns: replicaIdColumns + }, + true + ); + return; + } else if (change.type === SchemaChangeType.RENAME_TABLE) { + const oldTableId = getMysqlRelId({ + schema: this.connections.databaseName, + name: change.table + }); - const table = this.tableCache.get(tableId); - if (table) { - if (change.type === SchemaChangeType.TRUNCATE_TABLE) { - await batch.truncate([table]); - } else if (change.type === SchemaChangeType.DROP_TABLE) { - await batch.drop([table]); - this.tableCache.delete(tableId); - } else if (change.type === SchemaChangeType.RENAME_TABLE) { + const table = this.tableCache.get(oldTableId); + // Old table needs to be cleaned up + if (table) { await batch.drop([table]); - this.tableCache.delete(tableId); + this.tableCache.delete(oldTableId); + } + // If the new table matches the sync rules, we need to add it to the cache and snapshot it + if (this.matchesTable(change.newTable!)) { + const replicaIdColumns = await this.getReplicaIdColumns(change.newTable!); await this.handleRelation( batch, { @@ -512,37 +540,64 @@ export class BinLogStream { schema: this.connections.databaseName, name: change.newTable! }), - replicaIdColumns: table.replicaIdColumns + replicaIdColumns: replicaIdColumns }, true ); - } else { - await this.handleRelation(batch, table, true); } - } else if (change.type === SchemaChangeType.CREATE_TABLE) { - const connection = await this.connections.getConnection(); - const replicaIdColumns = await common.getReplicationIdentityColumns({ - connection, + } else { + const tableId = getMysqlRelId({ schema: this.connections.databaseName, - tableName: change.table + name: change.table }); - connection.release(); - await this.handleRelation( - batch, - { - name: change.newTable!, - schema: this.connections.databaseName, - objectId: getMysqlRelId({ - schema: this.connections.databaseName, - name: change.newTable! - }), - replicaIdColumns: replicaIdColumns.columns - }, - true - ); + + const table = this.getTable(tableId); + + switch (change.type) { + case SchemaChangeType.ADD_COLUMN: + case SchemaChangeType.DROP_COLUMN: + case SchemaChangeType.MODIFY_COLUMN: + case SchemaChangeType.RENAME_COLUMN: + case SchemaChangeType.REPLICATION_IDENTITY: + // For these changes, we need to update the table's replica id columns + const replicaIdColumns = await this.getReplicaIdColumns(change.table); + await this.handleRelation( + batch, + { + name: change.table, + schema: this.connections.databaseName, + objectId: tableId, + replicaIdColumns: replicaIdColumns + }, + true + ); + break; + case SchemaChangeType.TRUNCATE_TABLE: + await batch.truncate([table]); + break; + case SchemaChangeType.DROP_TABLE: + await batch.drop([table]); + this.tableCache.delete(tableId); + break; + default: + // No action needed for other schema changes + break; + } } } + private async getReplicaIdColumns(tableName: string) { + const connection = await this.connections.getConnection(); + const replicaIdColumns = await common.getReplicationIdentityColumns({ + connection, + schema: this.connections.databaseName, + tableName + }); + connection.release(); + + return replicaIdColumns.columns; + } + private async writeChanges( batch: storage.BucketStorageBatch, msg: { diff --git a/modules/module-mysql/src/replication/zongji/BinLogListener.ts b/modules/module-mysql/src/replication/zongji/BinLogListener.ts index fe2f5e22..95bbe711 100644 --- a/modules/module-mysql/src/replication/zongji/BinLogListener.ts +++ b/modules/module-mysql/src/replication/zongji/BinLogListener.ts @@ -1,15 +1,17 @@ import * as common from '../../common/common-index.js'; import async from 'async'; -import { BinLogEvent, StartOptions, TableMapEntry, ZongJi } from '@powersync/mysql-zongji'; +import { BinLogEvent, BinLogQueryEvent, StartOptions, TableMapEntry, ZongJi } from '@powersync/mysql-zongji'; import * as zongji_utils from './zongji-utils.js'; import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework'; import { MySQLConnectionManager } from '../MySQLConnectionManager.js'; -import * as parser from 'node-sql-parser'; import timers from 'timers/promises'; +import pkg, { BaseFrom, Parser as ParserType, RenameStatement, TruncateStatement } from 'node-sql-parser'; -// Maximum time the processing queue can be paused before resuming automatically -// MySQL server will automatically terminate replication connections after 60 seconds of inactivity, so this guards against that. -const MAX_QUEUE_PAUSE_TIME_MS = 45_000; +const { Parser } = pkg; + +// Maximum time the Zongji listener can be paused before resuming automatically +// MySQL server automatically terminates replication connections after 60 seconds of inactivity +const MAX_PAUSE_TIME_MS = 45_000; export type Row = Record; @@ -21,7 +23,8 @@ export enum SchemaChangeType { MODIFY_COLUMN = 'modify_column', DROP_COLUMN = 'drop_column', ADD_COLUMN = 'add_column', - RENAME_COLUMN = 'rename_column' + RENAME_COLUMN = 'rename_column', + REPLICATION_IDENTITY = 'replication_identity' } export interface SchemaChange { @@ -31,6 +34,9 @@ export interface SchemaChange { */ table: string; newTable?: string; // Only for table renames + /** + * ColumnDetails. Only applicable for column schema changes. + */ column?: { /** * The column that the schema change applies to. @@ -53,7 +59,8 @@ export interface BinLogEventHandler { export interface BinLogListenerOptions { connectionManager: MySQLConnectionManager; eventHandler: BinLogEventHandler; - includedTables: string[]; + // Filter for tables to include in the replication + tableFilter: (tableName: string) => boolean; serverId: number; startPosition: common.BinLogPosition; logger?: Logger; @@ -64,21 +71,22 @@ export interface BinLogListenerOptions { * events on the provided BinLogEventHandler. */ export class BinLogListener { - private sqlParser: parser.Parser; + private sqlParser: ParserType; private connectionManager: MySQLConnectionManager; private eventHandler: BinLogEventHandler; private binLogPosition: common.BinLogPosition; private currentGTID: common.ReplicatedGTID | null; private logger: Logger; - isStopped: boolean = false; - isStopping: boolean = false; zongji: ZongJi; processingQueue: async.QueueObject; + + isStopped: boolean = false; + isStopping: boolean = false; /** * The combined size in bytes of all the binlog events currently in the processing queue. */ - queueMemoryUsage: number; + queueMemoryUsage: number = 0; constructor(public options: BinLogListenerOptions) { this.logger = options.logger ?? defaultLogger; @@ -86,9 +94,8 @@ export class BinLogListener { this.eventHandler = options.eventHandler; this.binLogPosition = options.startPosition; this.currentGTID = null; - this.sqlParser = new parser.Parser(); + this.sqlParser = new Parser(); this.processingQueue = this.createProcessingQueue(); - this.queueMemoryUsage = 0; this.zongji = this.createZongjiListener(); } @@ -140,7 +147,7 @@ export class BinLogListener { // We ignore the unknown/heartbeat event since it currently serves no purpose other than to keep the connection alive // tablemap events always need to be included for the other row events to work includeEvents: ['tablemap', 'writerows', 'updaterows', 'deleterows', 'xid', 'rotate', 'gtidlog', 'query'], - includeSchema: { [this.connectionManager.databaseName]: this.options.includedTables }, + includeSchema: { [this.connectionManager.databaseName]: this.options.tableFilter }, filename: this.binLogPosition.filename, position: this.binLogPosition.offset, serverId: this.options.serverId @@ -156,6 +163,25 @@ export class BinLogListener { }); } + private async restartZongji(): Promise { + this.zongji = this.createZongjiListener(); + await this.start(true); + } + + private async stopZongji(): Promise { + await new Promise((resolve) => { + this.zongji.once('stopped', () => { + resolve(); + }); + this.zongji.stop(); + }); + + // Wait until all the current events in the processing queue are also processed + if (this.processingQueue.length() > 0) { + await this.processingQueue.empty(); + } + } + public async stop(): Promise { if (!(this.isStopped || this.isStopping)) { this.logger.info('Stopping BinLog Listener...'); @@ -201,19 +227,25 @@ export class BinLogListener { zongji.on('binlog', async (evt) => { this.logger.info(`Received BinLog event:${evt.getEventName()}`); - this.processingQueue.push(evt); - this.queueMemoryUsage += evt.size; + // We have to handle schema change events before handling more binlog events, + // This avoids a bunch of possible race conditions + if (zongji_utils.eventIsQuery(evt)) { + await this.processQueryEvent(evt); + } else { + this.processingQueue.push(evt); + this.queueMemoryUsage += evt.size; - // When the processing queue grows past the threshold, we pause the binlog listener - if (this.isQueueOverCapacity()) { - this.logger.info( - `BinLog processing queue has reached its memory limit of [${this.connectionManager.options.binlog_queue_memory_limit}MB]. Pausing BinLog Listener.` - ); - zongji.pause(); - const resumeTimeoutPromise = timers.setTimeout(MAX_QUEUE_PAUSE_TIME_MS); - await Promise.race([this.processingQueue.empty(), resumeTimeoutPromise]); - this.logger.info(`BinLog processing queue backlog cleared. Resuming BinLog Listener.`); - zongji.resume(); + // When the processing queue grows past the threshold, we pause the binlog listener + if (this.isQueueOverCapacity()) { + this.logger.info( + `BinLog processing queue has reached its memory limit of [${this.connectionManager.options.binlog_queue_memory_limit}MB]. Pausing BinLog Listener.` + ); + zongji.pause(); + const resumeTimeoutPromise = timers.setTimeout(MAX_PAUSE_TIME_MS); + await Promise.race([this.processingQueue.empty(), resumeTimeoutPromise]); + this.logger.info(`BinLog processing queue backlog cleared. Resuming BinLog Listener.`); + zongji.resume(); + } } }); @@ -229,6 +261,10 @@ export class BinLogListener { return zongji; } + isQueueOverCapacity(): boolean { + return this.queueMemoryUsage >= this.queueMemoryLimit; + } + private createQueueWorker() { return async (evt: BinLogEvent) => { switch (true) { @@ -250,14 +286,15 @@ export class BinLogListener { ); break; case zongji_utils.eventIsRotation(evt): - if (this.binLogPosition.filename !== evt.binlogName) { - this.logger.info( - `Processed Rotate log event. New BinLog file is: ${this.binLogPosition.filename}:${this.binLogPosition.offset}` - ); - } + const newFile = this.binLogPosition.filename !== evt.binlogName; this.binLogPosition.filename = evt.binlogName; this.binLogPosition.offset = evt.position; await this.eventHandler.onRotate(); + + this.logger.info( + `Processed Rotate log event. ${newFile ? `New BinLog file is: ${this.binLogPosition.filename}:${this.binLogPosition.offset}` : ''}` + ); + break; case zongji_utils.eventIsWriteMutation(evt): await this.eventHandler.onWrite(evt.rows, evt.tableMap[evt.tableId]); @@ -289,22 +326,6 @@ export class BinLogListener { }).comparable; await this.eventHandler.onCommit(LSN); this.logger.info(`Processed Xid log event. Transaction LSN: ${LSN}.`); - break; - case zongji_utils.eventIsQuery(evt): - // Ignore BEGIN queries - if (evt.query === 'BEGIN') { - break; - } - const ast = this.sqlParser.astify(evt.query, { database: 'MySQL' }); - const statements = Array.isArray(ast) ? ast : [ast]; - const schemaChanges = this.toSchemaChanges(statements); - if (schemaChanges.length > 0) { - await this.handleSchemaChanges(schemaChanges, evt.nextPosition); - } else { - // Still have to update the binlog position, even if there were no effective schema changes - this.binLogPosition.offset = evt.nextPosition; - } - break; } @@ -312,39 +333,56 @@ export class BinLogListener { }; } - private async handleSchemaChanges(changes: SchemaChange[], nextPosition: number): Promise { - this.logger.info(`Stopping BinLog Listener to process ${changes.length} schema change events...`); - // Since handling the schema changes can take a long time, we need to stop the Zongji listener instead of pausing it. - await new Promise((resolve) => { - this.zongji.once('stopped', () => { - resolve(); - }); - this.zongji.stop(); - }); - for (const change of changes) { - this.logger.info(`Processing schema change: ${change.type} for table: ${change.table}`); - await this.eventHandler.onSchemaChange(change); + private async processQueryEvent(event: BinLogQueryEvent): Promise { + const { query, nextPosition } = event; + + // Ignore BEGIN queries + if (query === 'BEGIN') { + return; } - this.binLogPosition.offset = nextPosition; - // Wait until all the current events in the processing queue are processed and the binlog position has been updated - // otherwise we risk processing the same events again. - if (this.processingQueue.length() > 0) { - await this.processingQueue.empty(); + const schemaChanges = this.toSchemaChanges(query); + if (schemaChanges.length > 0) { + this.logger.info(`Processing schema change query: ${query}`); + this.logger.info(`Stopping BinLog Listener to process ${schemaChanges.length} schema change events...`); + // Since handling the schema changes can take a long time, we need to stop the Zongji listener instead of pausing it. + await this.stopZongji(); + + for (const change of schemaChanges) { + await this.eventHandler.onSchemaChange(change); + } + + // DDL queries are auto commited, and so do not come with a corresponding Xid event. + // This is problematic for DDL queries which result in row events, so we manually commit here. + this.binLogPosition.offset = nextPosition; + const LSN = new common.ReplicatedGTID({ + raw_gtid: this.currentGTID!.raw, + position: this.binLogPosition + }).comparable; + await this.eventHandler.onCommit(LSN); + + this.logger.info(`Successfully processed schema changes.`); + // Restart the Zongji listener + await this.restartZongji(); } - this.zongji = this.createZongjiListener(); - this.logger.info(`Successfully processed schema changes.`); - // Restart the Zongji listener - await this.start(true); } - private toSchemaChanges(statements: parser.AST[]): SchemaChange[] { - // TODO: We need to check if schema filtering is also required + /** + * Function that interprets a DDL query for any applicable schema changes. + * If the query does not contain any relevant schema changes, an empty array is returned. + * + * @param query + */ + private toSchemaChanges(query: string): SchemaChange[] { + const ast = this.sqlParser.astify(query, { database: 'MySQL' }); + const statements = Array.isArray(ast) ? ast : [ast]; + const changes: SchemaChange[] = []; for (const statement of statements) { // @ts-ignore if (statement.type === 'rename') { - const renameStatement = statement as parser.RenameStatement; + const renameStatement = statement as RenameStatement; + // Rename statements can apply to multiple tables for (const table of renameStatement.table) { changes.push({ type: SchemaChangeType.RENAME_TABLE, @@ -352,18 +390,18 @@ export class BinLogListener { newTable: table[1].table }); } - } // @ts-ignore - else if (statement.type === 'truncate' && statement.keyword === 'table') { - const truncateStatement = statement as parser.TruncateStatement; - // Truncate statements can apply to multiple tables - for (const entity of truncateStatement.name) { - changes.push({ type: SchemaChangeType.TRUNCATE_TABLE, table: entity.table }); - } } else if (statement.type === 'create' && statement.keyword === 'table' && statement.temporary === null) { changes.push({ type: SchemaChangeType.CREATE_TABLE, table: statement.table![0].table }); + } // @ts-ignore + else if (statement.type === 'truncate') { + const truncateStatement = statement as TruncateStatement; + // Truncate statements can apply to multiple tables + for (const entity of truncateStatement.name) { + changes.push({ type: SchemaChangeType.TRUNCATE_TABLE, table: entity.table }); + } } else if (statement.type === 'drop' && statement.keyword === 'table') { // Drop statements can apply to multiple tables for (const entity of statement.name) { @@ -371,7 +409,7 @@ export class BinLogListener { } } else if (statement.type === 'alter') { const expression = statement.expr[0]; - const fromTable = statement.table[0] as parser.BaseFrom; + const fromTable = statement.table[0] as BaseFrom; if (expression.resource === 'table') { if (expression.action === 'rename') { changes.push({ @@ -381,50 +419,28 @@ export class BinLogListener { }); } } else if (expression.resource === 'column') { - const column = expression.column; - if (expression.action === 'drop') { - changes.push({ - type: SchemaChangeType.DROP_COLUMN, - table: fromTable.table, - column: { - column: column.column - } - }); - } else if (expression.action === 'add') { - changes.push({ - type: SchemaChangeType.ADD_COLUMN, - table: fromTable.table, - column: { - column: column.column - } - }); - } else if (expression.action === 'modify') { - changes.push({ - type: SchemaChangeType.MODIFY_COLUMN, - table: fromTable.table, - column: { - column: column.column - } - }); - } else if (expression.action === 'change') { - changes.push({ - type: SchemaChangeType.RENAME_COLUMN, - table: fromTable.table, - column: { - column: expression.old_column.column, - newColumn: column.column - } - }); - } else if (expression.action === 'rename') { - changes.push({ - type: SchemaChangeType.RENAME_COLUMN, - table: fromTable.table, - column: { - column: expression.old_column.column, - newColumn: column.column - } - }); + const columnChange: SchemaChange = { + type: this.toColumnSchemaChangeType(expression.action), + table: fromTable.table, + column: { + column: expression.column.column + } + }; + + if (expression.action === 'change' || expression.action === 'rename') { + columnChange.column = { + column: expression.old_column.column, + newColumn: expression.column.column + }; } + changes.push(columnChange); + } else if (expression.resource === 'key' && expression.keyword === 'primary key') { + // This is a special case for MySQL, where the primary key is being set or changed + // We treat this as a replication identity change + changes.push({ + type: SchemaChangeType.REPLICATION_IDENTITY, + table: fromTable.table + }); } } } @@ -432,12 +448,23 @@ export class BinLogListener { // Filter out schema changes that are not relevant to the included tables return changes.filter( (change) => - this.options.includedTables.includes(change.table) || - (change.newTable && this.options.includedTables.includes(change.newTable)) + this.options.tableFilter(change.table) || (change.newTable && this.options.tableFilter(change.newTable)) ); } - isQueueOverCapacity(): boolean { - return this.queueMemoryUsage >= this.queueMemoryLimit; + private toColumnSchemaChangeType(action: string): SchemaChangeType { + switch (action) { + case 'drop': + return SchemaChangeType.DROP_COLUMN; + case 'add': + return SchemaChangeType.ADD_COLUMN; + case 'modify': + return SchemaChangeType.MODIFY_COLUMN; + case 'change': + case 'rename': + return SchemaChangeType.RENAME_COLUMN; + default: + throw new Error(`Unknown column schema change action: ${action}`); + } } } diff --git a/modules/module-mysql/src/types/node-sql-parser-extended-types.ts b/modules/module-mysql/src/types/node-sql-parser-extended-types.ts index de1e7687..834b2579 100644 --- a/modules/module-mysql/src/types/node-sql-parser-extended-types.ts +++ b/modules/module-mysql/src/types/node-sql-parser-extended-types.ts @@ -11,7 +11,7 @@ declare module 'node-sql-parser' { interface TruncateStatement { type: 'truncate'; - keyword: 'table'; // There are more keywords possible, but we only care about 'table' for now + keyword: 'table'; // There are more keywords possible, but we only care about 'table' name: { db: string | null; table: string; as: string | null }[]; } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c9292285..65446370 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -258,8 +258,8 @@ importers: specifier: workspace:* version: link:../../libs/lib-services '@powersync/mysql-zongji': - specifier: 0.0.0-dev-20250610140703 - version: 0.0.0-dev-20250610140703 + specifier: 0.0.0-dev-20250619101606 + version: 0.0.0-dev-20250619101606 '@powersync/service-core': specifier: workspace:* version: link:../../packages/service-core @@ -1284,8 +1284,8 @@ packages: resolution: {integrity: sha512-UA91GwWPhFExt3IizW6bOeY/pQ0BkuNwKjk9iQW9KqxluGCrg4VenZ0/L+2Y0+ZOtme72EVvg6v0zo3AMQRCeA==} engines: {node: '>=12'} - '@powersync/mysql-zongji@0.0.0-dev-20250610140703': - resolution: {integrity: sha512-chcUCpawar5I89CkDTHmSwoAdto9DptAMGwznXS9KqpigFcfZz7jGbGpwUCxMM03VhSwICrND9XyxCQezOm9fg==} + '@powersync/mysql-zongji@0.0.0-dev-20250619101606': + resolution: {integrity: sha512-llMHZyLv6+F7R74opP0ZiFp1zPVhsmtuQXCx9N2SCatY8Yl48LO3ICyntBy5wZ6mUv2SGynMNX+dxEgWJ+tbqw==} engines: {node: '>=22.0.0'} '@powersync/service-jsonbig@0.17.10': @@ -4758,7 +4758,7 @@ snapshots: '@pnpm/network.ca-file': 1.0.2 config-chain: 1.1.13 - '@powersync/mysql-zongji@0.0.0-dev-20250610140703': + '@powersync/mysql-zongji@0.0.0-dev-20250619101606': dependencies: '@vlasky/mysql': 2.18.6 big-integer: 1.6.52 From 1d1e9451d00075e70156256ab648252bb75e8cec Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 25 Jun 2025 16:34:35 +0200 Subject: [PATCH 25/48] Added schema change tests Cleaned up MySQL tests in general and added a few new test utils --- .../test/src/BinLogListener.test.ts | 43 ++- .../test/src/BinLogStream.test.ts | 75 ++-- .../test/src/BinlogStreamUtils.ts | 13 +- .../test/src/schema-changes.test.ts | 350 ++++++++++++++++++ 4 files changed, 423 insertions(+), 58 deletions(-) create mode 100644 modules/module-mysql/test/src/schema-changes.test.ts diff --git a/modules/module-mysql/test/src/BinLogListener.test.ts b/modules/module-mysql/test/src/BinLogListener.test.ts index c2614968..72dbb406 100644 --- a/modules/module-mysql/test/src/BinLogListener.test.ts +++ b/modules/module-mysql/test/src/BinLogListener.test.ts @@ -38,7 +38,7 @@ describe('BinlogListener tests', () => { connectionManager: connectionManager, eventHandler: eventHandler, startPosition: fromGTID.position, - includedTables: ['test_DATA'], + tableFilter: (table) => ['test_DATA'].includes(table), serverId: createRandomServerId(1) }); }); @@ -58,7 +58,7 @@ describe('BinlogListener tests', () => { expect(queueStopSpy).toHaveBeenCalled(); }); - test('Pause Zongji binlog listener when processing queue reaches maximum memory size', async () => { + test('Zongji listener is paused when processing queue reaches maximum memory size', async () => { const pauseSpy = vi.spyOn(binLogListener.zongji, 'pause'); const resumeSpy = vi.spyOn(binLogListener.zongji, 'resume'); @@ -83,7 +83,7 @@ describe('BinlogListener tests', () => { expect(resumeSpy).toHaveBeenCalled(); }); - test('Binlog row events are correctly forwarded to provided binlog events handler', async () => { + test('Row event handling', async () => { await binLogListener.start(); const ROW_COUNT = 10; @@ -100,7 +100,7 @@ describe('BinlogListener tests', () => { await binLogListener.stop(); }); - test('ALTER TABLE RENAME schema changes', async () => { + test('Schema change event handling - ALTER TABLE RENAME', async () => { await binLogListener.start(); await connectionManager.query(`ALTER TABLE test_DATA RENAME test_DATA_new`); await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); @@ -110,7 +110,7 @@ describe('BinlogListener tests', () => { expect(eventHandler.schemaChanges[0].newTable).toEqual('test_DATA_new'); }); - test('RENAME TABLE schema changes', async () => { + test('Schema change event handling - RENAME TABLE', async () => { await binLogListener.start(); await connectionManager.query(`RENAME TABLE test_DATA TO test_DATA_new`); await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); @@ -120,7 +120,7 @@ describe('BinlogListener tests', () => { expect(eventHandler.schemaChanges[0].newTable).toEqual('test_DATA_new'); }); - test('RENAME TABLE multipe table schema changes', async () => { + test('Schema change event handling - RENAME TABLE multiple', async () => { await binLogListener.start(); await connectionManager.query(`RENAME TABLE test_DATA TO test_DATA_new, @@ -137,7 +137,7 @@ describe('BinlogListener tests', () => { expect(eventHandler.schemaChanges[1].newTable).toEqual('test_DATA'); }); - test('TRUNCATE TABLE schema changes', async () => { + test('Schema change event handling - TRUNCATE TABLE', async () => { await binLogListener.start(); await connectionManager.query(`TRUNCATE TABLE test_DATA`); await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); @@ -146,7 +146,7 @@ describe('BinlogListener tests', () => { expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); }); - test('DROP AND CREATE TABLE schema changes', async () => { + test('Schema change event handling - DROP AND CREATE TABLE ', async () => { await binLogListener.start(); await connectionManager.query(`DROP TABLE test_DATA`); await connectionManager.query(`CREATE TABLE test_DATA (id CHAR(36) PRIMARY KEY, description MEDIUMTEXT)`); @@ -158,7 +158,7 @@ describe('BinlogListener tests', () => { expect(eventHandler.schemaChanges[1].table).toEqual('test_DATA'); }); - test('ALTER TABLE DROP COLUMN schema changes', async () => { + test('Schema change event handling - ALTER TABLE DROP COLUMN', async () => { await binLogListener.start(); await connectionManager.query(`ALTER TABLE test_DATA DROP COLUMN description`); await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); @@ -168,7 +168,7 @@ describe('BinlogListener tests', () => { expect(eventHandler.schemaChanges[0].column?.column).toEqual('description'); }); - test('ALTER TABLE ADD COLUMN schema changes', async () => { + test('Schema change event handling - ALTER TABLE ADD COLUMN', async () => { await binLogListener.start(); await connectionManager.query(`ALTER TABLE test_DATA ADD COLUMN new_column VARCHAR(255)`); await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); @@ -178,7 +178,7 @@ describe('BinlogListener tests', () => { expect(eventHandler.schemaChanges[0].column?.column).toEqual('new_column'); }); - test('ALTER TABLE MODIFY COLUMN type schema changes', async () => { + test('Schema change event handling - ALTER TABLE MODIFY COLUMN', async () => { await binLogListener.start(); await connectionManager.query(`ALTER TABLE test_DATA MODIFY COLUMN description TEXT`); await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); @@ -188,7 +188,7 @@ describe('BinlogListener tests', () => { expect(eventHandler.schemaChanges[0].column?.column).toEqual('description'); }); - test('ALTER TABLE CHANGE COLUMN column rename schema changes', async () => { + test('Schema change event handling - ALTER TABLE CHANGE COLUMN column rename', async () => { await binLogListener.start(); await connectionManager.query(`ALTER TABLE test_DATA CHANGE COLUMN description description_new MEDIUMTEXT`); await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); @@ -199,7 +199,7 @@ describe('BinlogListener tests', () => { expect(eventHandler.schemaChanges[0].column?.newColumn).toEqual('description_new'); }); - test('ALTER TABLE RENAME COLUMN column rename schema changes', async () => { + test('Schema change event handling - ALTER TABLE RENAME COLUMN column rename', async () => { await binLogListener.start(); await connectionManager.query(`ALTER TABLE test_DATA RENAME COLUMN description TO description_new`); await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); @@ -209,6 +209,23 @@ describe('BinlogListener tests', () => { expect(eventHandler.schemaChanges[0].column?.column).toEqual('description'); expect(eventHandler.schemaChanges[0].column?.newColumn).toEqual('description_new'); }); + + test('Schema changes for non-matching tables are ignored', async () => { + const stopSpy = vi.spyOn(binLogListener.zongji, 'stop'); + + // TableFilter = only match 'test_DATA' + await binLogListener.start(); + await connectionManager.query(`CREATE TABLE test_ignored (id CHAR(36) PRIMARY KEY, description TEXT)`); + await connectionManager.query(`ALTER TABLE test_ignored ADD COLUMN new_column VARCHAR(10)`); + await connectionManager.query(`DROP TABLE test_ignored`); + + // "Anchor" event to latch onto, ensuring that the schema change events have finished + await insertRows(connectionManager, 1); + await vi.waitFor(() => expect(eventHandler.rowsWritten).equals(1), { timeout: 5000 }); + await binLogListener.stop(); + + expect(eventHandler.schemaChanges.length).toBe(0); + }); }); async function getFromGTID(connectionManager: MySQLConnectionManager) { diff --git a/modules/module-mysql/test/src/BinLogStream.test.ts b/modules/module-mysql/test/src/BinLogStream.test.ts index 1e7708eb..1a6cb420 100644 --- a/modules/module-mysql/test/src/BinLogStream.test.ts +++ b/modules/module-mysql/test/src/BinLogStream.test.ts @@ -34,7 +34,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; - context.startStreaming(); + await context.startStreaming(); const testId = uuid(); await connectionManager.query( `INSERT INTO test_data(id, description, num) VALUES('${testId}', 'test1', 1152921504606846976)` @@ -65,7 +65,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; - context.startStreaming(); + await context.startStreaming(); const testId = uuid(); await connectionManager.query(`INSERT INTO test_DATA(id, description) VALUES('${testId}','test1')`); @@ -79,50 +79,37 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { expect(endTxCount - startTxCount).toEqual(1); }); - // TODO: Not supported yet - // test('replicating TRUNCATE', async () => { - // await using context = await BinlogStreamTestContext.create(factory); - // const { connectionManager } = context; - // const syncRuleContent = ` - // bucket_definitions: - // global: - // data: - // - SELECT id, description FROM "test_data" - // by_test_data: - // parameters: SELECT id FROM test_data WHERE id = token_parameters.user_id - // data: [] - // `; - // await context.updateSyncRules(syncRuleContent); - // await connectionManager.query(`DROP TABLE IF EXISTS test_data`); - // await connectionManager.query( - // `CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)` - // ); - - // await context.replicateSnapshot(); - // context.startStreaming(); - - // const [{ test_id }] = pgwireRows( - // await connectionManager.query(`INSERT INTO test_data(description) VALUES('test1') returning id as test_id`) - // ); - // await connectionManager.query(`TRUNCATE test_data`); - - // const data = await context.getBucketData('global[]'); - - // expect(data).toMatchObject([ - // putOp('test_data', { id: test_id, description: 'test1' }), - // removeOp('test_data', test_id) - // ]); - // }); + test('replicating TRUNCATE', async () => { + await using context = await BinlogStreamTestContext.open(factory); + await context.updateSyncRules(BASIC_SYNC_RULES); + + const { connectionManager } = context; + await connectionManager.query(`CREATE TABLE test_data (id CHAR(36) PRIMARY KEY, description text)`); + + await context.replicateSnapshot(); + await context.startStreaming(); + + const testId = uuid(); + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('${testId}','test1')`); + await connectionManager.query(`TRUNCATE TABLE test_data`); + + const data = await context.getBucketData('global[]'); + + expect(data).toMatchObject([ + putOp('test_data', { id: testId, description: 'test1' }), + removeOp('test_data', testId) + ]); + }); test('replicating changing primary key', async () => { await using context = await BinlogStreamTestContext.open(factory); - const { connectionManager } = context; await context.updateSyncRules(BASIC_SYNC_RULES); + const { connectionManager } = context; await connectionManager.query(`CREATE TABLE test_data (id CHAR(36) PRIMARY KEY, description text)`); await context.replicateSnapshot(); - context.startStreaming(); + await context.startStreaming(); const testId1 = uuid(); await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('${testId1}','test1')`); @@ -167,7 +154,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; await context.replicateSnapshot(); - context.startStreaming(); + await context.startStreaming(); const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; const data = await context.getBucketData('global[]'); @@ -195,7 +182,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { `); await context.replicateSnapshot(); - context.startStreaming(); + await context.startStreaming(); const data = await context.getBucketData('global[]'); expect(data).toMatchObject([ @@ -228,7 +215,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; - context.startStreaming(); + await context.startStreaming(); const testId = uuid(); await connectionManager.query(` @@ -271,7 +258,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; - context.startStreaming(); + await context.startStreaming(); await connectionManager.query(`INSERT INTO test_donotsync(id, description) VALUES('${uuid()}','test1')`); const data = await context.getBucketData('global[]'); @@ -300,7 +287,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { await connectionManager.query(`CREATE TABLE test_data (id CHAR(36) PRIMARY KEY, description TEXT, num BIGINT)`); await context.replicateSnapshot(); - context.startStreaming(); + await context.startStreaming(); await connectionManager.query( `INSERT INTO test_data(id, description, num) VALUES('${testId1}', 'test1', 1152921504606846976)` ); @@ -315,7 +302,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { await context.loadActiveSyncRules(); // Does not actually do a snapshot again - just does the required intialization. await context.replicateSnapshot(); - context.startStreaming(); + await context.startStreaming(); await connectionManager.query(`INSERT INTO test_data(id, description, num) VALUES('${testId2}', 'test2', 0)`); const data = await context.getBucketData('global[]'); diff --git a/modules/module-mysql/test/src/BinlogStreamUtils.ts b/modules/module-mysql/test/src/BinlogStreamUtils.ts index 575f50ae..228b15e6 100644 --- a/modules/module-mysql/test/src/BinlogStreamUtils.ts +++ b/modules/module-mysql/test/src/BinlogStreamUtils.ts @@ -16,6 +16,7 @@ import { import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; import mysqlPromise from 'mysql2/promise'; import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js'; +import timers from 'timers/promises'; /** * Tests operating on the binlog stream need to configure the stream and manage asynchronous @@ -116,11 +117,21 @@ export class BinlogStreamTestContext { this.replicationDone = true; } - startStreaming() { + async startStreaming() { if (!this.replicationDone) { throw new Error('Call replicateSnapshot() before startStreaming()'); } this.streamPromise = this.binlogStream.streamChanges(); + + // Wait for the replication to start before returning. + // This avoids a bunch of unpredictable race conditions that appear in testing + return new Promise(async (resolve) => { + while (this.binlogStream.isStartingReplication) { + await timers.setTimeout(50); + } + + resolve(); + }); } async getCheckpoint(options?: { timeout?: number }): Promise { diff --git a/modules/module-mysql/test/src/schema-changes.test.ts b/modules/module-mysql/test/src/schema-changes.test.ts new file mode 100644 index 00000000..4c226fbe --- /dev/null +++ b/modules/module-mysql/test/src/schema-changes.test.ts @@ -0,0 +1,350 @@ +import { compareIds, putOp, removeOp, test_utils } from '@powersync/service-core-tests'; +import { describe, expect, test } from 'vitest'; + +import { storage } from '@powersync/service-core'; +import { describeWithStorage } from './util.js'; +import { BinlogStreamTestContext } from './BinlogStreamUtils.js'; + +describe('MySQL Schema Changes', () => { + describeWithStorage({ timeout: 20_000 }, defineTests); +}); + +const BASIC_SYNC_RULES = ` +bucket_definitions: + global: + data: + - SELECT id, * FROM "test_data" +`; + +const PUT_T1 = test_utils.putOp('test_data', { id: 't1', description: 'test1' }); +const PUT_T2 = test_utils.putOp('test_data', { id: 't2', description: 'test2' }); +const PUT_T3 = test_utils.putOp('test_data', { id: 't3', description: 'test3' }); + +const REMOVE_T1 = test_utils.removeOp('test_data', 't1'); +const REMOVE_T2 = test_utils.removeOp('test_data', 't2'); + +function defineTests(factory: storage.TestStorageFactory) { + test('Re-create table', async () => { + await using context = await BinlogStreamTestContext.open(factory); + // Drop a table and re-create it. + await context.updateSyncRules(BASIC_SYNC_RULES); + + const { connectionManager } = context; + await connectionManager.query(`CREATE TABLE test_data (id CHAR(36) PRIMARY KEY, description TEXT)`); + + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t1','test1')`); + + await context.replicateSnapshot(); + await context.startStreaming(); + + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t2','test2')`); + + await connectionManager.query(`DROP TABLE test_data`); + await connectionManager.query(`CREATE TABLE test_data (id CHAR(36) PRIMARY KEY, description TEXT)`); + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t3','test3')`); + + const data = await context.getBucketData('global[]'); + + // Initial inserts + expect(data.slice(0, 2)).toMatchObject([PUT_T1, PUT_T2]); + + // Truncate - order doesn't matter + expect(data.slice(2, 4).sort(compareIds)).toMatchObject([REMOVE_T1, REMOVE_T2]); + + // Due to the async nature of this replication test, + // the insert for t3 is picked up both in the snapshot and in the replication stream. + expect(data.slice(4)).toMatchObject([ + PUT_T3, // Snapshot insert + PUT_T3 // Insert from binlog replication stream + ]); + }); + + test('Add a table that is in the sync rules', async () => { + await using context = await BinlogStreamTestContext.open(factory); + const { connectionManager } = context; + await context.updateSyncRules(BASIC_SYNC_RULES); + + await context.replicateSnapshot(); + await context.startStreaming(); + + // Add table after initial replication + await connectionManager.query(`CREATE TABLE test_data (id CHAR(36) PRIMARY KEY, description TEXT)`); + + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t1','test1')`); + + const data = await context.getBucketData('global[]'); + + expect(data).toMatchObject([PUT_T1, PUT_T1]); + }); + + test('(1) Rename a table not in the sync rules to one in the sync rules', async () => { + await using context = await BinlogStreamTestContext.open(factory); + const { connectionManager } = context; + await context.updateSyncRules(BASIC_SYNC_RULES); + + // Rename table not that is not in sync rules -> in sync rules + await connectionManager.query(`CREATE TABLE test_data_old (id CHAR(36) PRIMARY KEY, description TEXT)`); + await connectionManager.query(`INSERT INTO test_data_old(id, description) VALUES('t1','test1')`); + + await context.replicateSnapshot(); + await context.startStreaming(); + + await connectionManager.query(`RENAME TABLE test_data_old TO test_data`); + + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t2','test2')`); + + const data = await context.getBucketData('global[]'); + + expect(data).toMatchObject([ + // Snapshot insert + PUT_T1, + PUT_T2, + // Replicated insert + PUT_T2 + ]); + }); + + test('(2) Rename a table in the sync rules to another table also in the sync rules', async () => { + await using context = await BinlogStreamTestContext.open(factory); + + await context.updateSyncRules(` + bucket_definitions: + global: + data: + - SELECT id, * FROM "test_data%" + `); + + const { connectionManager } = context; + await connectionManager.query(`CREATE TABLE test_data1 (id CHAR(36) PRIMARY KEY, description TEXT)`); + await connectionManager.query(`INSERT INTO test_data1(id, description) VALUES('t1','test1')`); + + await context.replicateSnapshot(); + await context.startStreaming(); + + await connectionManager.query(`RENAME TABLE test_data1 TO test_data2`); + await connectionManager.query(`INSERT INTO test_data2(id, description) VALUES('t2','test2')`); + + const data = await context.getBucketData('global[]'); + + expect(data.slice(0, 2)).toMatchObject([ + // Initial replication + putOp('test_data1', { id: 't1', description: 'test1' }), + // Initial truncate + removeOp('test_data1', 't1') + ]); + + expect(data.slice(2, 4).sort(compareIds)).toMatchObject([ + // Snapshot insert + putOp('test_data2', { id: 't1', description: 'test1' }), + putOp('test_data2', { id: 't2', description: 'test2' }) + ]); + expect(data.slice(4)).toMatchObject([ + // Replicated insert + // We may eventually be able to de-duplicate this + putOp('test_data2', { id: 't2', description: 'test2' }) + ]); + }); + + test('(3) Rename table in the sync rules to one not in the sync rules', async () => { + await using context = await BinlogStreamTestContext.open(factory); + await context.updateSyncRules(BASIC_SYNC_RULES); + + const { connectionManager } = context; + await connectionManager.query(`CREATE TABLE test_data (id CHAR(36) PRIMARY KEY, description TEXT)`); + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t1','test1')`); + + await context.replicateSnapshot(); + await context.startStreaming(); + + await connectionManager.query(`RENAME TABLE test_data TO test_data_not_in_sync_rules`); + await connectionManager.query(`INSERT INTO test_data_not_in_sync_rules(id, description) VALUES('t2','test2')`); + + const data = await context.getBucketData('global[]'); + + expect(data).toMatchObject([ + // Initial replication + PUT_T1, + // Truncate + REMOVE_T1 + ]); + }); + + test('(1) Change Replication Identity default by dropping the primary key', async () => { + await using context = await BinlogStreamTestContext.open(factory); + // Change replica id from default (PK) to full + // Requires re-snapshotting the table. + + await context.updateSyncRules(BASIC_SYNC_RULES); + const { connectionManager } = context; + await connectionManager.query(`CREATE TABLE test_data (id CHAR(36) PRIMARY KEY, description TEXT)`); + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t1','test1')`); + + await context.replicateSnapshot(); + await context.startStreaming(); + + await connectionManager.query(`ALTER TABLE test_data DROP PRIMARY KEY;`); + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t2','test2')`); + + const data = await context.getBucketData('global[]'); + + expect(data.slice(0, 2)).toMatchObject([ + // Initial inserts + PUT_T1, + // Truncate + REMOVE_T1 + ]); + + expect(data.slice(2)).toMatchObject([ + // Snapshot inserts + PUT_T1, + PUT_T2, + // Replicated insert + PUT_T2 + ]); + }); + + test('(2) Change Replication Identity full by adding a column', async () => { + await using context = await BinlogStreamTestContext.open(factory); + // Change replica id from full by adding column + // Causes a re-import of the table. + // Other changes such as renaming column would have the same effect + + await context.updateSyncRules(BASIC_SYNC_RULES); + + const { connectionManager } = context; + // No primary key, no unique column, so full replication identity will be used + await connectionManager.query(`CREATE TABLE test_data (id CHAR(36), description TEXT)`); + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t1','test1')`); + + await context.replicateSnapshot(); + await context.startStreaming(); + + await connectionManager.query(`ALTER TABLE test_data ADD COLUMN new_column TEXT`); + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t2','test2')`); + + const data = await context.getBucketData('global[]'); + + expect(data.slice(0, 2)).toMatchObject([ + // Initial inserts + PUT_T1, + // Truncate + REMOVE_T1 + ]); + + // Snapshot - order doesn't matter + expect(data.slice(2)).toMatchObject([ + // Snapshot inserts + putOp('test_data', { id: 't1', description: 'test1', new_column: null }), + putOp('test_data', { id: 't2', description: 'test2', new_column: null }), + // Replicated insert + putOp('test_data', { id: 't2', description: 'test2', new_column: null }) + ]); + }); + + test('(3) Change Replication Identity default by modifying primary key column type', async () => { + await using context = await BinlogStreamTestContext.open(factory); + await context.updateSyncRules(BASIC_SYNC_RULES); + + const { connectionManager } = context; + await connectionManager.query(`CREATE TABLE test_data (id CHAR(36) PRIMARY KEY, description TEXT)`); + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t1','test1')`); + + await context.replicateSnapshot(); + await context.startStreaming(); + + await connectionManager.query(`ALTER TABLE test_data MODIFY COLUMN id VARCHAR(36)`); + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t2','test2')`); + + const data = await context.getBucketData('global[]'); + + expect(data.slice(0, 2)).toMatchObject([ + // Initial inserts + PUT_T1, + // Truncate + REMOVE_T1 + ]); + + expect(data.slice(2)).toMatchObject([ + // Snapshot inserts + PUT_T1, + PUT_T2, + // Replicated insert + PUT_T2 + ]); + }); + + test('(4) Change Replication Identity by changing the type of a column in a compound unique index', async () => { + await using context = await BinlogStreamTestContext.open(factory); + // Change index replica id by changing column type + // Causes a re-import of the table. + // Secondary functionality tested here is that replica id column order stays + // the same between initial and incremental replication. + await context.updateSyncRules(BASIC_SYNC_RULES); + + const { connectionManager } = context; + await connectionManager.query(`CREATE TABLE test_data (id CHAR(36), description CHAR(100))`); + await connectionManager.query(`ALTER TABLE test_data ADD INDEX (id, description)`); + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t1','test1')`); + + await context.replicateSnapshot(); + await context.startStreaming(); + + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t2','test2')`); + + await connectionManager.query(`ALTER TABLE test_data MODIFY COLUMN id VARCHAR(36)`); + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t3','test3')`); + + const data = await context.getBucketData('global[]'); + + expect(data.slice(0, 2)).toMatchObject([ + // Initial snapshot + PUT_T1, + // Streamed + PUT_T2 + ]); + + expect(data.slice(2, 4).sort(compareIds)).toMatchObject([ + // Truncate - any order + REMOVE_T1, + REMOVE_T2 + ]); + + // Snapshot - order doesn't matter + expect(data.slice(4, 7).sort(compareIds)).toMatchObject([PUT_T1, PUT_T2, PUT_T3]); + + expect(data.slice(7).sort(compareIds)).toMatchObject([ + // Replicated insert + PUT_T3 + ]); + }); + + test('Drop a table in the sync rules', async () => { + await using context = await BinlogStreamTestContext.open(factory); + // Technically not a schema change, but fits here. + await context.updateSyncRules(BASIC_SYNC_RULES); + + const { connectionManager } = context; + await connectionManager.query(`CREATE TABLE test_data (id CHAR(36), description CHAR(100))`); + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t1','test1')`); + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t2','test2')`); + + await context.replicateSnapshot(); + await context.startStreaming(); + + await connectionManager.query(`DROP TABLE test_data`); + + const data = await context.getBucketData('global[]'); + + expect(data.slice(0, 2)).toMatchObject([ + // Initial inserts + PUT_T1, + PUT_T2 + ]); + + expect(data.slice(2).sort(compareIds)).toMatchObject([ + // Drop + REMOVE_T1, + REMOVE_T2 + ]); + }); +} From ce8cb9cca644c02958458e1ecc690a0aed8aecd2 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 25 Jun 2025 17:13:09 +0200 Subject: [PATCH 26/48] Change binlog event receive log message to debug --- modules/module-mysql/src/replication/zongji/BinLogListener.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/module-mysql/src/replication/zongji/BinLogListener.ts b/modules/module-mysql/src/replication/zongji/BinLogListener.ts index 95bbe711..ed8ee826 100644 --- a/modules/module-mysql/src/replication/zongji/BinLogListener.ts +++ b/modules/module-mysql/src/replication/zongji/BinLogListener.ts @@ -226,7 +226,7 @@ export class BinLogListener { const zongji = this.connectionManager.createBinlogListener(); zongji.on('binlog', async (evt) => { - this.logger.info(`Received BinLog event:${evt.getEventName()}`); + this.logger.debug(`Received BinLog event:${evt.getEventName()}`); // We have to handle schema change events before handling more binlog events, // This avoids a bunch of possible race conditions if (zongji_utils.eventIsQuery(evt)) { From 2411f216ed3597cb74ec1ec87a37aaf1745b80ea Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 25 Jun 2025 21:41:52 +0200 Subject: [PATCH 27/48] Revert and fix mysql->sqlite row conversion for null value columns --- .../src/common/mysql-to-sqlite.ts | 139 +++++++++--------- 1 file changed, 72 insertions(+), 67 deletions(-) diff --git a/modules/module-mysql/src/common/mysql-to-sqlite.ts b/modules/module-mysql/src/common/mysql-to-sqlite.ts index e9c38571..46a4b058 100644 --- a/modules/module-mysql/src/common/mysql-to-sqlite.ts +++ b/modules/module-mysql/src/common/mysql-to-sqlite.ts @@ -109,78 +109,83 @@ export function toSQLiteRow(row: Record, columns: Map Date: Wed, 25 Jun 2025 22:40:43 +0200 Subject: [PATCH 28/48] Added conditional skip of mysql schema test for syntax that does not exist in version 5.7 --- modules/module-mysql/src/utils/mysql-utils.ts | 9 ++++- .../test/src/BinLogListener.test.ts | 39 ++++++++++++------- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/modules/module-mysql/src/utils/mysql-utils.ts b/modules/module-mysql/src/utils/mysql-utils.ts index 7439c30d..e998cbd7 100644 --- a/modules/module-mysql/src/utils/mysql-utils.ts +++ b/modules/module-mysql/src/utils/mysql-utils.ts @@ -2,7 +2,7 @@ import { logger } from '@powersync/lib-services-framework'; import mysql from 'mysql2'; import mysqlPromise from 'mysql2/promise'; import * as types from '../types/types.js'; -import { coerce, gte } from 'semver'; +import { coerce, eq, gte } from 'semver'; import { SourceTable } from '@powersync/service-core'; export type RetriedQueryOptions = { @@ -86,6 +86,13 @@ export function isVersionAtLeast(version: string, minimumVersion: string): boole return gte(coercedVersion!, coercedMinimumVersion!, { loose: true }); } +export function isVersion(version: string, targetVersion: string): boolean { + const coercedVersion = coerce(version); + const coercedTargetVersion = coerce(targetVersion); + + return eq(coercedVersion!, coercedTargetVersion!, { loose: true }); +} + export function escapeMysqlTableName(table: SourceTable): string { return `\`${table.schema.replaceAll('`', '``')}\`.\`${table.name.replaceAll('`', '``')}\``; } diff --git a/modules/module-mysql/test/src/BinLogListener.test.ts b/modules/module-mysql/test/src/BinLogListener.test.ts index 72dbb406..9ba91624 100644 --- a/modules/module-mysql/test/src/BinLogListener.test.ts +++ b/modules/module-mysql/test/src/BinLogListener.test.ts @@ -1,4 +1,4 @@ -import { describe, test, beforeEach, vi, expect, afterEach } from 'vitest'; +import { describe, test, beforeEach, vi, expect, beforeAll, afterAll } from 'vitest'; import { BinLogEventHandler, BinLogListener, @@ -10,7 +10,7 @@ import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManag import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js'; import { v4 as uuid } from 'uuid'; import * as common from '@module/common/common-index.js'; -import { createRandomServerId } from '@module/utils/mysql-utils.js'; +import { createRandomServerId, getMySQLVersion, isVersion, isVersionAtLeast } from '@module/utils/mysql-utils.js'; import { TableMapEntry } from '@powersync/mysql-zongji'; import crypto from 'crypto'; @@ -24,9 +24,17 @@ describe('BinlogListener tests', () => { let connectionManager: MySQLConnectionManager; let eventHandler: TestBinLogEventHandler; let binLogListener: BinLogListener; + let isMySQL57: boolean = false; - beforeEach(async () => { + beforeAll(async () => { connectionManager = new MySQLConnectionManager(BINLOG_LISTENER_CONNECTION_OPTIONS, {}); + const connection = await connectionManager.getConnection(); + const version = await getMySQLVersion(connection); + isMySQL57 = isVersion(version, '5.7.0'); + connection.release(); + }); + + beforeEach(async () => { const connection = await connectionManager.getConnection(); await clearTestDb(connection); await connection.query(`CREATE TABLE test_DATA (id CHAR(36) PRIMARY KEY, description MEDIUMTEXT)`); @@ -43,7 +51,7 @@ describe('BinlogListener tests', () => { }); }); - afterEach(async () => { + afterAll(async () => { await connectionManager.end(); }); @@ -199,17 +207,6 @@ describe('BinlogListener tests', () => { expect(eventHandler.schemaChanges[0].column?.newColumn).toEqual('description_new'); }); - test('Schema change event handling - ALTER TABLE RENAME COLUMN column rename', async () => { - await binLogListener.start(); - await connectionManager.query(`ALTER TABLE test_DATA RENAME COLUMN description TO description_new`); - await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); - await binLogListener.stop(); - expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.RENAME_COLUMN); - expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); - expect(eventHandler.schemaChanges[0].column?.column).toEqual('description'); - expect(eventHandler.schemaChanges[0].column?.newColumn).toEqual('description_new'); - }); - test('Schema changes for non-matching tables are ignored', async () => { const stopSpy = vi.spyOn(binLogListener.zongji, 'stop'); @@ -226,6 +223,18 @@ describe('BinlogListener tests', () => { expect(eventHandler.schemaChanges.length).toBe(0); }); + + // Syntax ALTER TABLE RENAME COLUMN was only introduced in MySQL 8.0.0 + test.skipIf(isMySQL57)('Schema change event handling - ALTER TABLE RENAME COLUMN column rename', async () => { + await binLogListener.start(); + await connectionManager.query(`ALTER TABLE test_DATA RENAME COLUMN description TO description_new`); + await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); + await binLogListener.stop(); + expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.RENAME_COLUMN); + expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); + expect(eventHandler.schemaChanges[0].column?.column).toEqual('description'); + expect(eventHandler.schemaChanges[0].column?.newColumn).toEqual('description_new'); + }); }); async function getFromGTID(connectionManager: MySQLConnectionManager) { From 3adea0417fee3643edd0deb2e02a5fb72b0a8f9d Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Thu, 26 Jun 2025 11:54:41 +0200 Subject: [PATCH 29/48] Fixed version checking for mysql 5.7 incompatible test --- modules/module-mysql/src/utils/mysql-utils.ts | 5 ++--- modules/module-mysql/test/src/BinLogListener.test.ts | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/modules/module-mysql/src/utils/mysql-utils.ts b/modules/module-mysql/src/utils/mysql-utils.ts index e998cbd7..81601770 100644 --- a/modules/module-mysql/src/utils/mysql-utils.ts +++ b/modules/module-mysql/src/utils/mysql-utils.ts @@ -2,7 +2,7 @@ import { logger } from '@powersync/lib-services-framework'; import mysql from 'mysql2'; import mysqlPromise from 'mysql2/promise'; import * as types from '../types/types.js'; -import { coerce, eq, gte } from 'semver'; +import { coerce, eq, gte, satisfies } from 'semver'; import { SourceTable } from '@powersync/service-core'; export type RetriedQueryOptions = { @@ -88,9 +88,8 @@ export function isVersionAtLeast(version: string, minimumVersion: string): boole export function isVersion(version: string, targetVersion: string): boolean { const coercedVersion = coerce(version); - const coercedTargetVersion = coerce(targetVersion); - return eq(coercedVersion!, coercedTargetVersion!, { loose: true }); + return satisfies(coercedVersion!, targetVersion!, { loose: true }); } export function escapeMysqlTableName(table: SourceTable): string { diff --git a/modules/module-mysql/test/src/BinLogListener.test.ts b/modules/module-mysql/test/src/BinLogListener.test.ts index 9ba91624..46357ac6 100644 --- a/modules/module-mysql/test/src/BinLogListener.test.ts +++ b/modules/module-mysql/test/src/BinLogListener.test.ts @@ -1,4 +1,4 @@ -import { describe, test, beforeEach, vi, expect, beforeAll, afterAll } from 'vitest'; +import { afterAll, beforeAll, beforeEach, describe, expect, test, vi } from 'vitest'; import { BinLogEventHandler, BinLogListener, @@ -10,7 +10,7 @@ import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManag import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js'; import { v4 as uuid } from 'uuid'; import * as common from '@module/common/common-index.js'; -import { createRandomServerId, getMySQLVersion, isVersion, isVersionAtLeast } from '@module/utils/mysql-utils.js'; +import { createRandomServerId, getMySQLVersion, isVersion } from '@module/utils/mysql-utils.js'; import { TableMapEntry } from '@powersync/mysql-zongji'; import crypto from 'crypto'; @@ -30,7 +30,7 @@ describe('BinlogListener tests', () => { connectionManager = new MySQLConnectionManager(BINLOG_LISTENER_CONNECTION_OPTIONS, {}); const connection = await connectionManager.getConnection(); const version = await getMySQLVersion(connection); - isMySQL57 = isVersion(version, '5.7.0'); + isMySQL57 = isVersion(version, '5.7'); connection.release(); }); From 79bd14ee1eae4cab6cbf8bf9ab09acb4545917fd Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Thu, 26 Jun 2025 12:21:56 +0200 Subject: [PATCH 30/48] Fix skip test on mysql 5.7 schema change --- modules/module-mysql/src/utils/mysql-utils.ts | 2 +- .../test/src/BinLogListener.test.ts | 26 ++++++++++--------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/modules/module-mysql/src/utils/mysql-utils.ts b/modules/module-mysql/src/utils/mysql-utils.ts index 81601770..5d6b85fb 100644 --- a/modules/module-mysql/src/utils/mysql-utils.ts +++ b/modules/module-mysql/src/utils/mysql-utils.ts @@ -86,7 +86,7 @@ export function isVersionAtLeast(version: string, minimumVersion: string): boole return gte(coercedVersion!, coercedMinimumVersion!, { loose: true }); } -export function isVersion(version: string, targetVersion: string): boolean { +export function satisfiesVersion(version: string, targetVersion: string): boolean { const coercedVersion = coerce(version); return satisfies(coercedVersion!, targetVersion!, { loose: true }); diff --git a/modules/module-mysql/test/src/BinLogListener.test.ts b/modules/module-mysql/test/src/BinLogListener.test.ts index 46357ac6..b51e97a8 100644 --- a/modules/module-mysql/test/src/BinLogListener.test.ts +++ b/modules/module-mysql/test/src/BinLogListener.test.ts @@ -10,7 +10,7 @@ import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManag import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js'; import { v4 as uuid } from 'uuid'; import * as common from '@module/common/common-index.js'; -import { createRandomServerId, getMySQLVersion, isVersion } from '@module/utils/mysql-utils.js'; +import { createRandomServerId, getMySQLVersion, satisfiesVersion } from '@module/utils/mysql-utils.js'; import { TableMapEntry } from '@powersync/mysql-zongji'; import crypto from 'crypto'; @@ -30,7 +30,7 @@ describe('BinlogListener tests', () => { connectionManager = new MySQLConnectionManager(BINLOG_LISTENER_CONNECTION_OPTIONS, {}); const connection = await connectionManager.getConnection(); const version = await getMySQLVersion(connection); - isMySQL57 = isVersion(version, '5.7'); + isMySQL57 = satisfiesVersion(version, '5.7.x'); connection.release(); }); @@ -224,16 +224,18 @@ describe('BinlogListener tests', () => { expect(eventHandler.schemaChanges.length).toBe(0); }); - // Syntax ALTER TABLE RENAME COLUMN was only introduced in MySQL 8.0.0 - test.skipIf(isMySQL57)('Schema change event handling - ALTER TABLE RENAME COLUMN column rename', async () => { - await binLogListener.start(); - await connectionManager.query(`ALTER TABLE test_DATA RENAME COLUMN description TO description_new`); - await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); - await binLogListener.stop(); - expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.RENAME_COLUMN); - expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); - expect(eventHandler.schemaChanges[0].column?.column).toEqual('description'); - expect(eventHandler.schemaChanges[0].column?.newColumn).toEqual('description_new'); + test('Schema change event handling - ALTER TABLE RENAME COLUMN column rename', async () => { + // Syntax ALTER TABLE RENAME COLUMN was only introduced in MySQL 8.0.0 + if (!isMySQL57) { + await binLogListener.start(); + await connectionManager.query(`ALTER TABLE test_DATA RENAME COLUMN description TO description_new`); + await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); + await binLogListener.stop(); + expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.RENAME_COLUMN); + expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); + expect(eventHandler.schemaChanges[0].column?.column).toEqual('description'); + expect(eventHandler.schemaChanges[0].column?.newColumn).toEqual('description_new'); + } }); }); From 81b437f9eebce72e6427cc2d5633c32915155de5 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Thu, 26 Jun 2025 15:05:04 +0200 Subject: [PATCH 31/48] Reverted mysql dev docker compose Updated to released zongji listener version --- .../module-mysql/dev/docker/mysql/docker-compose.yaml | 6 +++--- modules/module-mysql/package.json | 2 +- pnpm-lock.yaml | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/module-mysql/dev/docker/mysql/docker-compose.yaml b/modules/module-mysql/dev/docker/mysql/docker-compose.yaml index 274e53b6..50dfd2d2 100644 --- a/modules/module-mysql/dev/docker/mysql/docker-compose.yaml +++ b/modules/module-mysql/dev/docker/mysql/docker-compose.yaml @@ -1,5 +1,5 @@ services: - mysql_test: + mysql: image: mysql:8.0 environment: MYSQL_ROOT_PASSWORD: root_password @@ -11,7 +11,7 @@ services: volumes: - ./init-scripts/my.cnf:/etc/mysql/my.cnf - ./init-scripts/mysql.sql:/docker-entrypoint-initdb.d/init_user.sql - - mysql_test_data:/var/lib/mysql + - mysql_data:/var/lib/mysql volumes: - mysql_test_data: + mysql_data: diff --git a/modules/module-mysql/package.json b/modules/module-mysql/package.json index cd4fe0fc..a022f80e 100644 --- a/modules/module-mysql/package.json +++ b/modules/module-mysql/package.json @@ -33,7 +33,7 @@ "@powersync/service-sync-rules": "workspace:*", "@powersync/service-types": "workspace:*", "@powersync/service-jsonbig": "workspace:*", - "@powersync/mysql-zongji": "0.0.0-dev-20250619101606", + "@powersync/mysql-zongji": "0.3.0", "async": "^3.2.4", "mysql2": "^3.11.0", "node-sql-parser": "^5.3.9", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 65446370..cb942bde 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -258,8 +258,8 @@ importers: specifier: workspace:* version: link:../../libs/lib-services '@powersync/mysql-zongji': - specifier: 0.0.0-dev-20250619101606 - version: 0.0.0-dev-20250619101606 + specifier: 0.3.0 + version: 0.3.0 '@powersync/service-core': specifier: workspace:* version: link:../../packages/service-core @@ -1284,8 +1284,8 @@ packages: resolution: {integrity: sha512-UA91GwWPhFExt3IizW6bOeY/pQ0BkuNwKjk9iQW9KqxluGCrg4VenZ0/L+2Y0+ZOtme72EVvg6v0zo3AMQRCeA==} engines: {node: '>=12'} - '@powersync/mysql-zongji@0.0.0-dev-20250619101606': - resolution: {integrity: sha512-llMHZyLv6+F7R74opP0ZiFp1zPVhsmtuQXCx9N2SCatY8Yl48LO3ICyntBy5wZ6mUv2SGynMNX+dxEgWJ+tbqw==} + '@powersync/mysql-zongji@0.3.0': + resolution: {integrity: sha512-Y113htGr0vSz3+IIWnooE4H/qXcps4Qes3e9HJECwQq02i5Eh3MHlhUxJoODBTv9LWlmsBXNeD4AuM0P0Ijtuw==} engines: {node: '>=22.0.0'} '@powersync/service-jsonbig@0.17.10': @@ -4758,7 +4758,7 @@ snapshots: '@pnpm/network.ca-file': 1.0.2 config-chain: 1.1.13 - '@powersync/mysql-zongji@0.0.0-dev-20250619101606': + '@powersync/mysql-zongji@0.3.0': dependencies: '@vlasky/mysql': 2.18.6 big-integer: 1.6.52 From b8e631bd348ef8a4a5ac686e543dc19f62c5ddf0 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Mon, 30 Jun 2025 17:17:40 +0200 Subject: [PATCH 32/48] Moved schema change handling to processing queue Catch parsing errors, and log an error if the DDL query might apply to one of the tables in the sync rules. --- .../src/replication/zongji/BinLogListener.ts | 113 +++++++++--------- modules/module-mysql/src/utils/mysql-utils.ts | 28 ++++- .../test/src/BinLogListener.test.ts | 17 ++- .../test/src/BinLogStream.test.ts | 19 +-- .../module-mysql/test/src/mysql-utils.test.ts | 21 +++- 5 files changed, 132 insertions(+), 66 deletions(-) diff --git a/modules/module-mysql/src/replication/zongji/BinLogListener.ts b/modules/module-mysql/src/replication/zongji/BinLogListener.ts index ed8ee826..26284425 100644 --- a/modules/module-mysql/src/replication/zongji/BinLogListener.ts +++ b/modules/module-mysql/src/replication/zongji/BinLogListener.ts @@ -6,6 +6,7 @@ import { Logger, logger as defaultLogger } from '@powersync/lib-services-framewo import { MySQLConnectionManager } from '../MySQLConnectionManager.js'; import timers from 'timers/promises'; import pkg, { BaseFrom, Parser as ParserType, RenameStatement, TruncateStatement } from 'node-sql-parser'; +import { matchedSchemaChangeQuery } from '../../utils/mysql-utils.js'; const { Parser } = pkg; @@ -156,7 +157,7 @@ export class BinLogListener { return new Promise((resolve) => { this.zongji.once('ready', () => { this.logger.info( - `BinLog Listener ${isRestart ? 'restarted' : 'started'}. Listening for events from position: ${this.binLogPosition.filename}:${this.binLogPosition.offset}.` + `BinLog Listener ${isRestart ? 'restarted' : 'started'}. Listening for events from position: ${this.binLogPosition.filename}:${this.binLogPosition.offset}` ); resolve(); }); @@ -169,35 +170,23 @@ export class BinLogListener { } private async stopZongji(): Promise { + this.logger.info('Stopping BinLog Listener...'); await new Promise((resolve) => { this.zongji.once('stopped', () => { resolve(); }); this.zongji.stop(); }); - - // Wait until all the current events in the processing queue are also processed - if (this.processingQueue.length() > 0) { - await this.processingQueue.empty(); - } + this.logger.info('BinLog Listener stopped.'); } public async stop(): Promise { if (!(this.isStopped || this.isStopping)) { - this.logger.info('Stopping BinLog Listener...'); this.isStopping = true; - await new Promise((resolve) => { - if (this.isStopped) { - resolve(); - } - this.zongji.once('stopped', () => { - this.isStopped = true; - this.logger.info('BinLog Listener stopped. Replication ended.'); - resolve(); - }); - this.zongji.stop(); - this.processingQueue.kill(); - }); + await this.stopZongji(); + this.processingQueue.kill(); + + this.isStopped = true; } } @@ -227,25 +216,20 @@ export class BinLogListener { zongji.on('binlog', async (evt) => { this.logger.debug(`Received BinLog event:${evt.getEventName()}`); - // We have to handle schema change events before handling more binlog events, - // This avoids a bunch of possible race conditions - if (zongji_utils.eventIsQuery(evt)) { - await this.processQueryEvent(evt); - } else { - this.processingQueue.push(evt); - this.queueMemoryUsage += evt.size; - // When the processing queue grows past the threshold, we pause the binlog listener - if (this.isQueueOverCapacity()) { - this.logger.info( - `BinLog processing queue has reached its memory limit of [${this.connectionManager.options.binlog_queue_memory_limit}MB]. Pausing BinLog Listener.` - ); - zongji.pause(); - const resumeTimeoutPromise = timers.setTimeout(MAX_PAUSE_TIME_MS); - await Promise.race([this.processingQueue.empty(), resumeTimeoutPromise]); - this.logger.info(`BinLog processing queue backlog cleared. Resuming BinLog Listener.`); - zongji.resume(); - } + this.processingQueue.push(evt); + this.queueMemoryUsage += evt.size; + + // When the processing queue grows past the threshold, we pause the binlog listener + if (this.isQueueOverCapacity()) { + this.logger.info( + `BinLog processing queue has reached its memory limit of [${this.connectionManager.options.binlog_queue_memory_limit}MB]. Pausing BinLog Listener.` + ); + zongji.pause(); + const resumeTimeoutPromise = timers.setTimeout(MAX_PAUSE_TIME_MS); + await Promise.race([this.processingQueue.empty(), resumeTimeoutPromise]); + this.logger.info(`BinLog processing queue backlog cleared. Resuming BinLog Listener.`); + zongji.resume(); } }); @@ -281,8 +265,8 @@ export class BinLogListener { }); this.binLogPosition.offset = evt.nextPosition; await this.eventHandler.onTransactionStart({ timestamp: new Date(evt.timestamp) }); - this.logger.info( - `Processed GTID log event. Next position in BinLog: ${this.binLogPosition.filename}:${this.binLogPosition.offset}` + this.logger.debug( + `Processed GTID event. Next position in BinLog: ${this.binLogPosition.filename}:${this.binLogPosition.offset}` ); break; case zongji_utils.eventIsRotation(evt): @@ -291,15 +275,16 @@ export class BinLogListener { this.binLogPosition.offset = evt.position; await this.eventHandler.onRotate(); - this.logger.info( - `Processed Rotate log event. ${newFile ? `New BinLog file is: ${this.binLogPosition.filename}:${this.binLogPosition.offset}` : ''}` - ); - + if (newFile) { + this.logger.info( + `Processed Rotate event. New BinLog file is: ${this.binLogPosition.filename}:${this.binLogPosition.offset}` + ); + } break; case zongji_utils.eventIsWriteMutation(evt): await this.eventHandler.onWrite(evt.rows, evt.tableMap[evt.tableId]); this.logger.info( - `Processed Write row event of ${evt.rows.length} rows for table: ${evt.tableMap[evt.tableId].tableName}` + `Processed Write event for table:${evt.tableMap[evt.tableId].tableName}. ${evt.rows.length} row(s) inserted.` ); break; case zongji_utils.eventIsUpdateMutation(evt): @@ -309,13 +294,13 @@ export class BinLogListener { evt.tableMap[evt.tableId] ); this.logger.info( - `Processed Update row event of ${evt.rows.length} rows for table: ${evt.tableMap[evt.tableId].tableName}` + `Processed Update event for table:${evt.tableMap[evt.tableId].tableName}. ${evt.rows.length} row(s) updated.` ); break; case zongji_utils.eventIsDeleteMutation(evt): await this.eventHandler.onDelete(evt.rows, evt.tableMap[evt.tableId]); this.logger.info( - `Processed Delete row event of ${evt.rows.length} rows for table: ${evt.tableMap[evt.tableId].tableName}` + `Processed Delete event for table:${evt.tableMap[evt.tableId].tableName}. ${evt.rows.length} row(s) deleted.` ); break; case zongji_utils.eventIsXid(evt): @@ -325,7 +310,10 @@ export class BinLogListener { position: this.binLogPosition }).comparable; await this.eventHandler.onCommit(LSN); - this.logger.info(`Processed Xid log event. Transaction LSN: ${LSN}.`); + this.logger.debug(`Processed Xid event - transaction complete. LSN: ${LSN}.`); + break; + case zongji_utils.eventIsQuery(evt): + await this.processQueryEvent(evt); break; } @@ -336,15 +324,25 @@ export class BinLogListener { private async processQueryEvent(event: BinLogQueryEvent): Promise { const { query, nextPosition } = event; - // Ignore BEGIN queries + // BEGIN query events mark the start of a transaction before any row events. They are not relevant for schema changes if (query === 'BEGIN') { return; } - const schemaChanges = this.toSchemaChanges(query); + let schemaChanges: SchemaChange[] = []; + try { + schemaChanges = this.toSchemaChanges(query); + } catch (error) { + if (matchedSchemaChangeQuery(query, this.options.tableFilter)) { + this.logger.warn( + `Failed to parse query: [${query}]. + Please review for the schema changes and manually redeploy the sync rules if required.` + ); + } + return; + } if (schemaChanges.length > 0) { - this.logger.info(`Processing schema change query: ${query}`); - this.logger.info(`Stopping BinLog Listener to process ${schemaChanges.length} schema change events...`); + this.logger.info(`Processing ${schemaChanges.length} schema change(s)...`); // Since handling the schema changes can take a long time, we need to stop the Zongji listener instead of pausing it. await this.stopZongji(); @@ -361,9 +359,16 @@ export class BinLogListener { }).comparable; await this.eventHandler.onCommit(LSN); - this.logger.info(`Successfully processed schema changes.`); - // Restart the Zongji listener - await this.restartZongji(); + this.logger.info(`Successfully processed ${schemaChanges.length} schema change(s).`); + + // If there are still events in the processing queue, we need to process those before restarting Zongji + if (this.processingQueue.length() > 0) { + this.processingQueue.empty(async () => { + await this.restartZongji(); + }); + } else { + await this.restartZongji(); + } } } diff --git a/modules/module-mysql/src/utils/mysql-utils.ts b/modules/module-mysql/src/utils/mysql-utils.ts index 5d6b85fb..97a0c565 100644 --- a/modules/module-mysql/src/utils/mysql-utils.ts +++ b/modules/module-mysql/src/utils/mysql-utils.ts @@ -2,7 +2,7 @@ import { logger } from '@powersync/lib-services-framework'; import mysql from 'mysql2'; import mysqlPromise from 'mysql2/promise'; import * as types from '../types/types.js'; -import { coerce, eq, gte, satisfies } from 'semver'; +import { coerce, gte, satisfies } from 'semver'; import { SourceTable } from '@powersync/service-core'; export type RetriedQueryOptions = { @@ -95,3 +95,29 @@ export function satisfiesVersion(version: string, targetVersion: string): boolea export function escapeMysqlTableName(table: SourceTable): string { return `\`${table.schema.replaceAll('`', '``')}\`.\`${table.name.replaceAll('`', '``')}\``; } + +const DDL_KEYWORDS = ['create table', 'alter table', 'drop table', 'truncate table', 'rename table']; + +/** + * Check if a query is a DDL statement that applies to tables matching the provided matcher function. + * @param query + * @param matcher + */ +export function matchedSchemaChangeQuery(query: string, matcher: (tableName: string) => boolean): boolean { + // Normalize case and remove backticks for matching + const normalizedQuery = query.toLowerCase().replace(/`/g, ''); + + const isDDLQuery = DDL_KEYWORDS.some((keyword) => normalizedQuery.includes(keyword)); + if (isDDLQuery) { + const tokens = normalizedQuery.split(/[^a-zA-Z0-9_`]+/); + // Check if any matched table names appear in the query + for (const token of tokens!) { + const matchFound = matcher(token); + if (matchFound) { + return true; + } + } + } + + return false; +} diff --git a/modules/module-mysql/test/src/BinLogListener.test.ts b/modules/module-mysql/test/src/BinLogListener.test.ts index b51e97a8..829db5cc 100644 --- a/modules/module-mysql/test/src/BinLogListener.test.ts +++ b/modules/module-mysql/test/src/BinLogListener.test.ts @@ -208,8 +208,6 @@ describe('BinlogListener tests', () => { }); test('Schema changes for non-matching tables are ignored', async () => { - const stopSpy = vi.spyOn(binLogListener.zongji, 'stop'); - // TableFilter = only match 'test_DATA' await binLogListener.start(); await connectionManager.query(`CREATE TABLE test_ignored (id CHAR(36) PRIMARY KEY, description TEXT)`); @@ -237,6 +235,21 @@ describe('BinlogListener tests', () => { expect(eventHandler.schemaChanges[0].column?.newColumn).toEqual('description_new'); } }); + + test('Unparseable query events that dont match tables in the sync rules are ignored', async () => { + binLogListener.options.tableFilter = (table) => ['test_DATA', 'test_unparseable'].includes(table); + await binLogListener.start(); + await connectionManager.query( + `CREATE TABLE test_unparseable (sale_date DATE) PARTITION BY RANGE (YEAR(sale_date)) + (PARTITION p2023 VALUES LESS THAN (2024))` + ); + + // "Anchor" event to latch onto, ensuring that the schema change events have finished + await insertRows(connectionManager, 1); + await vi.waitFor(() => expect(eventHandler.rowsWritten).equals(1), { timeout: 5000 }); + await binLogListener.stop(); + expect(eventHandler.schemaChanges.length).toBe(0); + }); }); async function getFromGTID(connectionManager: MySQLConnectionManager) { diff --git a/modules/module-mysql/test/src/BinLogStream.test.ts b/modules/module-mysql/test/src/BinLogStream.test.ts index 1a6cb420..3f78b16d 100644 --- a/modules/module-mysql/test/src/BinLogStream.test.ts +++ b/modules/module-mysql/test/src/BinLogStream.test.ts @@ -13,7 +13,7 @@ bucket_definitions: - SELECT id, description FROM "test_data" `; -describe('BigLog stream', () => { +describe('BigLogStream tests', () => { describeWithStorage({ timeout: 20_000 }, defineBinlogStreamTests); }); @@ -48,7 +48,10 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { expect(endTxCount - startTxCount).toEqual(1); }); - test('replicating case sensitive table', async () => { + test('Replicate case sensitive table', async () => { + // MySQL inherits the case sensitivity of the underlying OS filesystem. + // So Unix-based systems will have case-sensitive tables, but Windows won't. + // https://dev.mysql.com/doc/refman/8.4/en/identifier-case-sensitivity.html await using context = await BinlogStreamTestContext.open(factory); const { connectionManager } = context; await context.updateSyncRules(` @@ -79,7 +82,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { expect(endTxCount - startTxCount).toEqual(1); }); - test('replicating TRUNCATE', async () => { + test('Handle table TRUNCATE events', async () => { await using context = await BinlogStreamTestContext.open(factory); await context.updateSyncRules(BASIC_SYNC_RULES); @@ -101,7 +104,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { ]); }); - test('replicating changing primary key', async () => { + test('Handle changes in a replicated table primary key', async () => { await using context = await BinlogStreamTestContext.open(factory); await context.updateSyncRules(BASIC_SYNC_RULES); @@ -141,7 +144,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { ]); }); - test('initial sync', async () => { + test('Initial snapshot sync', async () => { await using context = await BinlogStreamTestContext.open(factory); const { connectionManager } = context; await context.updateSyncRules(BASIC_SYNC_RULES); @@ -162,7 +165,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { expect(endRowCount - startRowCount).toEqual(1); }); - test('snapshot with date values', async () => { + test('Snapshot with date values', async () => { await using context = await BinlogStreamTestContext.open(factory); const { connectionManager } = context; await context.updateSyncRules(` @@ -196,7 +199,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { ]); }); - test('replication with date values', async () => { + test('Replication with date values', async () => { await using context = await BinlogStreamTestContext.open(factory); const { connectionManager } = context; await context.updateSyncRules(` @@ -246,7 +249,7 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { expect(endTxCount - startTxCount).toEqual(2); }); - test('table not in sync rules', async () => { + test('Replication for tables not in the sync rules are ignored', async () => { await using context = await BinlogStreamTestContext.open(factory); const { connectionManager } = context; await context.updateSyncRules(BASIC_SYNC_RULES); diff --git a/modules/module-mysql/test/src/mysql-utils.test.ts b/modules/module-mysql/test/src/mysql-utils.test.ts index 03975626..533ca290 100644 --- a/modules/module-mysql/test/src/mysql-utils.test.ts +++ b/modules/module-mysql/test/src/mysql-utils.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { isVersionAtLeast } from '@module/utils/mysql-utils.js'; +import { isVersionAtLeast, matchedSchemaChangeQuery } from '@module/utils/mysql-utils.js'; describe('MySQL Utility Tests', () => { test('Minimum version checking ', () => { @@ -14,4 +14,23 @@ describe('MySQL Utility Tests', () => { expect(isVersionAtLeast(olderVersion, '8.0')).toBeFalsy(); expect(isVersionAtLeast(improperSemver, '5.7')).toBeTruthy(); }); + + test('matchedSchemaChangeQuery function', () => { + const matcher = (tableName: string) => tableName === 'users'; + + // DDL matches and table name matches + expect(matchedSchemaChangeQuery('CREATE TABLE clientSchema.users (id INT)', matcher)).toBeTruthy(); + expect(matchedSchemaChangeQuery('ALTER TABLE users ADD COLUMN name VARCHAR(255)', matcher)).toBeTruthy(); + expect(matchedSchemaChangeQuery('DROP TABLE users', matcher)).toBeTruthy(); + expect(matchedSchemaChangeQuery('TRUNCATE TABLE users', matcher)).toBeTruthy(); + expect(matchedSchemaChangeQuery('RENAME TABLE new_users TO users', matcher)).toBeTruthy(); + + // Can handle backticks in table names + expect(matchedSchemaChangeQuery('CREATE TABLE `clientSchema`.`users` (id INT)', matcher)).toBeTruthy(); + + // DDL matches, but table name does not match + expect(matchedSchemaChangeQuery('CREATE TABLE clientSchema.clients (id INT)', matcher)).toBeFalsy(); + // No DDL match + expect(matchedSchemaChangeQuery('SELECT * FROM users', matcher)).toBeFalsy(); + }); }); From ac968013add55ca01c32bb8c2dcf68df3d73077c Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Tue, 1 Jul 2025 14:15:48 +0200 Subject: [PATCH 33/48] Fixed bug where multiple zongji listeners could be started if multiple schema change events were in the processing queue Added small timeout to test to prevent rare race condition --- .../src/replication/zongji/BinLogListener.ts | 25 ++++++++++++------- .../test/src/schema-changes.test.ts | 5 ++++ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/modules/module-mysql/src/replication/zongji/BinLogListener.ts b/modules/module-mysql/src/replication/zongji/BinLogListener.ts index 26284425..c9472244 100644 --- a/modules/module-mysql/src/replication/zongji/BinLogListener.ts +++ b/modules/module-mysql/src/replication/zongji/BinLogListener.ts @@ -165,19 +165,23 @@ export class BinLogListener { } private async restartZongji(): Promise { - this.zongji = this.createZongjiListener(); - await this.start(true); + if (this.zongji.stopped) { + this.zongji = this.createZongjiListener(); + await this.start(true); + } } private async stopZongji(): Promise { this.logger.info('Stopping BinLog Listener...'); - await new Promise((resolve) => { - this.zongji.once('stopped', () => { - resolve(); + if (!this.zongji.stopped) { + await new Promise((resolve) => { + this.zongji.once('stopped', () => { + resolve(); + }); + this.zongji.stop(); }); - this.zongji.stop(); - }); - this.logger.info('BinLog Listener stopped.'); + this.logger.info('BinLog Listener stopped.'); + } } public async stop(): Promise { @@ -342,11 +346,11 @@ export class BinLogListener { return; } if (schemaChanges.length > 0) { - this.logger.info(`Processing ${schemaChanges.length} schema change(s)...`); // Since handling the schema changes can take a long time, we need to stop the Zongji listener instead of pausing it. await this.stopZongji(); for (const change of schemaChanges) { + this.logger.info(`Processing ${change.type} for table:${change.table}...`); await this.eventHandler.onSchemaChange(change); } @@ -363,6 +367,9 @@ export class BinLogListener { // If there are still events in the processing queue, we need to process those before restarting Zongji if (this.processingQueue.length() > 0) { + this.logger.info( + `Finish processing [${this.processingQueue.length()}] events(s) before resuming...` + ); this.processingQueue.empty(async () => { await this.restartZongji(); }); diff --git a/modules/module-mysql/test/src/schema-changes.test.ts b/modules/module-mysql/test/src/schema-changes.test.ts index 4c226fbe..fecb6959 100644 --- a/modules/module-mysql/test/src/schema-changes.test.ts +++ b/modules/module-mysql/test/src/schema-changes.test.ts @@ -4,6 +4,7 @@ import { describe, expect, test } from 'vitest'; import { storage } from '@powersync/service-core'; import { describeWithStorage } from './util.js'; import { BinlogStreamTestContext } from './BinlogStreamUtils.js'; +import timers from 'timers/promises'; describe('MySQL Schema Changes', () => { describeWithStorage({ timeout: 20_000 }, defineTests); @@ -39,6 +40,10 @@ function defineTests(factory: storage.TestStorageFactory) { await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t2','test2')`); + // Dropping the table immediately leads to a rare race condition where Zongji tries to get the table information + // for the previous write event, but the table is already gone. Without the table info the tablemap event can't be correctly + // populated and replication will fail. + await timers.setTimeout(50); await connectionManager.query(`DROP TABLE test_data`); await connectionManager.query(`CREATE TABLE test_data (id CHAR(36) PRIMARY KEY, description TEXT)`); await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t3','test3')`); From 301345c7fe26e267718992ebf72f91d3568407ff Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 9 Jul 2025 12:20:35 +0200 Subject: [PATCH 34/48] Extended node-sql-parser type definitions Added util functions to identify the different types of DDL statements --- .../module-mysql/src/common/schema-utils.ts | 3 +- .../types/node-sql-parser-extended-types.ts | 8 ++ modules/module-mysql/src/utils/mysql-utils.ts | 26 ------- .../module-mysql/src/utils/parser-utils.ts | 73 +++++++++++++++++++ .../module-mysql/test/src/mysql-utils.test.ts | 21 +----- .../test/src/parser-utils.test.ts | 24 ++++++ 6 files changed, 107 insertions(+), 48 deletions(-) create mode 100644 modules/module-mysql/src/utils/parser-utils.ts create mode 100644 modules/module-mysql/test/src/parser-utils.test.ts diff --git a/modules/module-mysql/src/common/schema-utils.ts b/modules/module-mysql/src/common/schema-utils.ts index 7d3c1736..4339920c 100644 --- a/modules/module-mysql/src/common/schema-utils.ts +++ b/modules/module-mysql/src/common/schema-utils.ts @@ -93,8 +93,7 @@ export async function getReplicationIdentityColumns( }; } - // TODO: test code with tables with unique keys, compound key etc. - // No primary key, find the first valid unique key + // No primary key, check if any of the columns have a unique constraint we can use const [uniqueKeyColumns] = await mysql_utils.retriedQuery({ connection: connection, query: ` diff --git a/modules/module-mysql/src/types/node-sql-parser-extended-types.ts b/modules/module-mysql/src/types/node-sql-parser-extended-types.ts index 834b2579..dd6af40e 100644 --- a/modules/module-mysql/src/types/node-sql-parser-extended-types.ts +++ b/modules/module-mysql/src/types/node-sql-parser-extended-types.ts @@ -14,4 +14,12 @@ declare module 'node-sql-parser' { keyword: 'table'; // There are more keywords possible, but we only care about 'table' name: { db: string | null; table: string; as: string | null }[]; } + + // This custom type more accurately describes what the structure of a Drop statement looks like for indexes. + interface DropIndexStatement { + type: 'drop'; + keyword: 'index'; + table: { db: string | null; table: string }; + name: any[]; + } } diff --git a/modules/module-mysql/src/utils/mysql-utils.ts b/modules/module-mysql/src/utils/mysql-utils.ts index 97a0c565..1ccf8e22 100644 --- a/modules/module-mysql/src/utils/mysql-utils.ts +++ b/modules/module-mysql/src/utils/mysql-utils.ts @@ -95,29 +95,3 @@ export function satisfiesVersion(version: string, targetVersion: string): boolea export function escapeMysqlTableName(table: SourceTable): string { return `\`${table.schema.replaceAll('`', '``')}\`.\`${table.name.replaceAll('`', '``')}\``; } - -const DDL_KEYWORDS = ['create table', 'alter table', 'drop table', 'truncate table', 'rename table']; - -/** - * Check if a query is a DDL statement that applies to tables matching the provided matcher function. - * @param query - * @param matcher - */ -export function matchedSchemaChangeQuery(query: string, matcher: (tableName: string) => boolean): boolean { - // Normalize case and remove backticks for matching - const normalizedQuery = query.toLowerCase().replace(/`/g, ''); - - const isDDLQuery = DDL_KEYWORDS.some((keyword) => normalizedQuery.includes(keyword)); - if (isDDLQuery) { - const tokens = normalizedQuery.split(/[^a-zA-Z0-9_`]+/); - // Check if any matched table names appear in the query - for (const token of tokens!) { - const matchFound = matcher(token); - if (matchFound) { - return true; - } - } - } - - return false; -} diff --git a/modules/module-mysql/src/utils/parser-utils.ts b/modules/module-mysql/src/utils/parser-utils.ts new file mode 100644 index 00000000..aa84be09 --- /dev/null +++ b/modules/module-mysql/src/utils/parser-utils.ts @@ -0,0 +1,73 @@ +import { Alter, AST, Create, Drop, TruncateStatement, RenameStatement, DropIndexStatement } from 'node-sql-parser'; + +// We ignore create table statements, since even in the worst case we will pick up the changes when row events for that +// table are received. +const DDL_KEYWORDS = ['alter table', 'drop table', 'truncate table', 'rename table']; + +/** + * Check if a query is a DDL statement that applies to tables matching the provided matcher function. + * @param query + * @param matcher + */ +export function matchedSchemaChangeQuery(query: string, matcher: (tableName: string) => boolean): boolean { + // Normalize case and remove backticks for matching + const normalizedQuery = query.toLowerCase().replace(/`/g, ''); + + const isDDLQuery = DDL_KEYWORDS.some((keyword) => normalizedQuery.includes(keyword)); + if (isDDLQuery) { + const tokens = normalizedQuery.split(/[^a-zA-Z0-9_`]+/); + // Check if any matched table names appear in the query + for (const token of tokens!) { + const matchFound = matcher(token); + if (matchFound) { + return true; + } + } + } + + return false; +} + +// @ts-ignore +export function isTruncate(statement: AST): statement is TruncateStatement { + // @ts-ignore + return statement.type === 'truncate'; +} + +// @ts-ignore +export function isRenameTable(statement: AST): statement is RenameStatement { + // @ts-ignore + return statement.type === 'rename'; +} + +export function isAlterTable(statement: AST): statement is Alter { + return statement.type === 'alter'; +} + +export function isRenameExpression(expression: any): boolean { + return expression.resource === 'table' && expression.action === 'rename'; +} + +export function isColumnExpression(expression: any): boolean { + return expression.resource === 'column'; +} + +export function isConstraintExpression(expression: any): boolean { + return ( + (expression.resource === 'key' && expression.keyword === 'primary key') || + expression.resource === 'constraint' || + (expression.resource === 'index' && expression.action === 'drop') + ); +} + +export function isDropTable(statement: AST): statement is Drop { + return statement.type === 'drop' && statement.keyword === 'table'; +} + +export function isDropIndex(statement: AST): statement is DropIndexStatement { + return statement.type === 'drop' && statement.keyword === 'index'; +} + +export function isCreateUniqueIndex(statement: AST): statement is Create { + return statement.type === 'create' && statement.keyword === 'index' && statement.index_type === 'unique'; +} diff --git a/modules/module-mysql/test/src/mysql-utils.test.ts b/modules/module-mysql/test/src/mysql-utils.test.ts index 533ca290..03975626 100644 --- a/modules/module-mysql/test/src/mysql-utils.test.ts +++ b/modules/module-mysql/test/src/mysql-utils.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { isVersionAtLeast, matchedSchemaChangeQuery } from '@module/utils/mysql-utils.js'; +import { isVersionAtLeast } from '@module/utils/mysql-utils.js'; describe('MySQL Utility Tests', () => { test('Minimum version checking ', () => { @@ -14,23 +14,4 @@ describe('MySQL Utility Tests', () => { expect(isVersionAtLeast(olderVersion, '8.0')).toBeFalsy(); expect(isVersionAtLeast(improperSemver, '5.7')).toBeTruthy(); }); - - test('matchedSchemaChangeQuery function', () => { - const matcher = (tableName: string) => tableName === 'users'; - - // DDL matches and table name matches - expect(matchedSchemaChangeQuery('CREATE TABLE clientSchema.users (id INT)', matcher)).toBeTruthy(); - expect(matchedSchemaChangeQuery('ALTER TABLE users ADD COLUMN name VARCHAR(255)', matcher)).toBeTruthy(); - expect(matchedSchemaChangeQuery('DROP TABLE users', matcher)).toBeTruthy(); - expect(matchedSchemaChangeQuery('TRUNCATE TABLE users', matcher)).toBeTruthy(); - expect(matchedSchemaChangeQuery('RENAME TABLE new_users TO users', matcher)).toBeTruthy(); - - // Can handle backticks in table names - expect(matchedSchemaChangeQuery('CREATE TABLE `clientSchema`.`users` (id INT)', matcher)).toBeTruthy(); - - // DDL matches, but table name does not match - expect(matchedSchemaChangeQuery('CREATE TABLE clientSchema.clients (id INT)', matcher)).toBeFalsy(); - // No DDL match - expect(matchedSchemaChangeQuery('SELECT * FROM users', matcher)).toBeFalsy(); - }); }); diff --git a/modules/module-mysql/test/src/parser-utils.test.ts b/modules/module-mysql/test/src/parser-utils.test.ts new file mode 100644 index 00000000..5d903ee5 --- /dev/null +++ b/modules/module-mysql/test/src/parser-utils.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, test } from 'vitest'; +import { matchedSchemaChangeQuery } from '@module/utils/parser-utils.js'; + +describe('MySQL Parser Util Tests', () => { + test('matchedSchemaChangeQuery function', () => { + const matcher = (tableName: string) => tableName === 'users'; + + // DDL matches and table name matches + expect(matchedSchemaChangeQuery('ALTER TABLE users ADD COLUMN name VARCHAR(255)', matcher)).toBeTruthy(); + expect(matchedSchemaChangeQuery('DROP TABLE users', matcher)).toBeTruthy(); + expect(matchedSchemaChangeQuery('TRUNCATE TABLE users', matcher)).toBeTruthy(); + expect(matchedSchemaChangeQuery('RENAME TABLE new_users TO users', matcher)).toBeTruthy(); + + // Can handle backticks in table names + expect( + matchedSchemaChangeQuery('ALTER TABLE `clientSchema`.`users` ADD COLUMN name VARCHAR(255)', matcher) + ).toBeTruthy(); + + // DDL matches, but table name does not match + expect(matchedSchemaChangeQuery('DROP TABLE clientSchema.clients', matcher)).toBeFalsy(); + // No DDL match + expect(matchedSchemaChangeQuery('SELECT * FROM users', matcher)).toBeFalsy(); + }); +}); From 3898db70168c83781c94e45860115d162da69233 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 9 Jul 2025 12:39:20 +0200 Subject: [PATCH 35/48] - Simplified schema change types - Added more detections of constraint changes - Removed detection of create table statements since they can be detected and reacted to when row events are received for new tables - Added multiple extra test cases --- .../src/replication/BinLogStream.ts | 92 +++---- .../src/replication/zongji/BinLogListener.ts | 156 ++++++------ .../test/src/BinLogListener.test.ts | 158 +++++++----- .../test/src/BinLogStream.test.ts | 36 +++ .../test/src/schema-changes.test.ts | 231 +++++++++++++++++- 5 files changed, 463 insertions(+), 210 deletions(-) diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index 1343d990..acfa75b2 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -11,6 +11,7 @@ import { framework, getUuidReplicaIdentityBson, MetricsEngine, + SourceTable, storage } from '@powersync/service-core'; import mysql from 'mysql2'; @@ -500,23 +501,7 @@ export class BinLogStream { } private async handleSchemaChange(batch: storage.BucketStorageBatch, change: SchemaChange): Promise { - if (change.type === SchemaChangeType.CREATE_TABLE) { - const replicaIdColumns = await this.getReplicaIdColumns(change.table); - await this.handleRelation( - batch, - { - name: change.table, - schema: this.connections.databaseName, - objectId: getMysqlRelId({ - schema: this.connections.databaseName, - name: change.table - }), - replicaIdColumns: replicaIdColumns - }, - true - ); - return; - } else if (change.type === SchemaChangeType.RENAME_TABLE) { + if (change.type === SchemaChangeType.RENAME_TABLE) { const oldTableId = getMysqlRelId({ schema: this.connections.databaseName, name: change.table @@ -530,20 +515,7 @@ export class BinLogStream { } // If the new table matches the sync rules, we need to add it to the cache and snapshot it if (this.matchesTable(change.newTable!)) { - const replicaIdColumns = await this.getReplicaIdColumns(change.newTable!); - await this.handleRelation( - batch, - { - name: change.newTable!, - schema: this.connections.databaseName, - objectId: getMysqlRelId({ - schema: this.connections.databaseName, - name: change.newTable! - }), - replicaIdColumns: replicaIdColumns - }, - true - ); + await this.handleCreateOrUpdateTable(batch, change.newTable!, this.connections.databaseName); } } else { const tableId = getMysqlRelId({ @@ -554,23 +526,10 @@ export class BinLogStream { const table = this.getTable(tableId); switch (change.type) { - case SchemaChangeType.ADD_COLUMN: - case SchemaChangeType.DROP_COLUMN: - case SchemaChangeType.MODIFY_COLUMN: - case SchemaChangeType.RENAME_COLUMN: + case SchemaChangeType.ALTER_TABLE_COLUMN: case SchemaChangeType.REPLICATION_IDENTITY: - // For these changes, we need to update the table's replica id columns - const replicaIdColumns = await this.getReplicaIdColumns(change.table); - await this.handleRelation( - batch, - { - name: change.table, - schema: this.connections.databaseName, - objectId: tableId, - replicaIdColumns: replicaIdColumns - }, - true - ); + // For these changes, we need to update the table if the replication identity columns have changed. + await this.handleCreateOrUpdateTable(batch, change.table, this.connections.databaseName); break; case SchemaChangeType.TRUNCATE_TABLE: await batch.truncate([table]); @@ -598,6 +557,27 @@ export class BinLogStream { return replicaIdColumns.columns; } + private async handleCreateOrUpdateTable( + batch: storage.BucketStorageBatch, + tableName: string, + schema: string + ): Promise { + const replicaIdColumns = await this.getReplicaIdColumns(tableName); + return await this.handleRelation( + batch, + { + name: tableName, + schema: schema, + objectId: getMysqlRelId({ + schema: schema, + name: tableName + }), + replicaIdColumns: replicaIdColumns + }, + true + ); + } + private async writeChanges( batch: storage.BucketStorageBatch, msg: { @@ -608,17 +588,23 @@ export class BinLogStream { } ): Promise { const columns = common.toColumnDescriptors(msg.tableEntry); + const tableId = getMysqlRelId({ + schema: msg.tableEntry.parentSchema, + name: msg.tableEntry.tableName + }); + + let table = this.tableCache.get(tableId); + if (table == null) { + // This write event is for a new table that matches a table in the sync rules + // We need to create the table in the storage and cache it. + table = await this.handleCreateOrUpdateTable(batch, msg.tableEntry.tableName, msg.tableEntry.parentSchema); + } for (const [index, row] of msg.rows.entries()) { await this.writeChange(batch, { type: msg.type, database: msg.tableEntry.parentSchema, - sourceTable: this.getTable( - getMysqlRelId({ - schema: msg.tableEntry.parentSchema, - name: msg.tableEntry.tableName - }) - ), + sourceTable: table!, table: msg.tableEntry.tableName, columns: columns, row: row, diff --git a/modules/module-mysql/src/replication/zongji/BinLogListener.ts b/modules/module-mysql/src/replication/zongji/BinLogListener.ts index c9472244..0e1fd094 100644 --- a/modules/module-mysql/src/replication/zongji/BinLogListener.ts +++ b/modules/module-mysql/src/replication/zongji/BinLogListener.ts @@ -5,8 +5,25 @@ import * as zongji_utils from './zongji-utils.js'; import { Logger, logger as defaultLogger } from '@powersync/lib-services-framework'; import { MySQLConnectionManager } from '../MySQLConnectionManager.js'; import timers from 'timers/promises'; -import pkg, { BaseFrom, Parser as ParserType, RenameStatement, TruncateStatement } from 'node-sql-parser'; -import { matchedSchemaChangeQuery } from '../../utils/mysql-utils.js'; +import pkg, { + BaseFrom, + DropIndexStatement, + Parser as ParserType, + RenameStatement, + TruncateStatement +} from 'node-sql-parser'; +import { + isAlterTable, + isColumnExpression, + isConstraintExpression, + isCreateUniqueIndex, + isDropIndex, + isDropTable, + isRenameExpression, + isRenameTable, + isTruncate, + matchedSchemaChangeQuery +} from '../../utils/parser-utils.js'; const { Parser } = pkg; @@ -16,15 +33,16 @@ const MAX_PAUSE_TIME_MS = 45_000; export type Row = Record; +/** + * Schema changes that can be detected by inspecting query events. + * Note that create table statements are not included here, since new tables are automatically detected when row events + * are received for them. + */ export enum SchemaChangeType { - CREATE_TABLE = 'create_table', RENAME_TABLE = 'rename_table', DROP_TABLE = 'drop_table', TRUNCATE_TABLE = 'truncate_table', - MODIFY_COLUMN = 'modify_column', - DROP_COLUMN = 'drop_column', - ADD_COLUMN = 'add_column', - RENAME_COLUMN = 'rename_column', + ALTER_TABLE_COLUMN = 'alter_table_column', REPLICATION_IDENTITY = 'replication_identity' } @@ -35,16 +53,6 @@ export interface SchemaChange { */ table: string; newTable?: string; // Only for table renames - /** - * ColumnDetails. Only applicable for column schema changes. - */ - column?: { - /** - * The column that the schema change applies to. - */ - column: string; - newColumn?: string; // Only for column renames - }; } export interface BinLogEventHandler { @@ -219,7 +227,7 @@ export class BinLogListener { const zongji = this.connectionManager.createBinlogListener(); zongji.on('binlog', async (evt) => { - this.logger.debug(`Received BinLog event:${evt.getEventName()}`); + this.logger.info(`Received BinLog event:${evt.getEventName()}`); this.processingQueue.push(evt); this.queueMemoryUsage += evt.size; @@ -231,7 +239,7 @@ export class BinLogListener { ); zongji.pause(); const resumeTimeoutPromise = timers.setTimeout(MAX_PAUSE_TIME_MS); - await Promise.race([this.processingQueue.empty(), resumeTimeoutPromise]); + await Promise.race([this.processingQueue.drain(), resumeTimeoutPromise]); this.logger.info(`BinLog processing queue backlog cleared. Resuming BinLog Listener.`); zongji.resume(); } @@ -366,11 +374,9 @@ export class BinLogListener { this.logger.info(`Successfully processed ${schemaChanges.length} schema change(s).`); // If there are still events in the processing queue, we need to process those before restarting Zongji - if (this.processingQueue.length() > 0) { - this.logger.info( - `Finish processing [${this.processingQueue.length()}] events(s) before resuming...` - ); - this.processingQueue.empty(async () => { + if (!this.processingQueue.idle()) { + this.logger.info(`Finish processing [${this.processingQueue.length()}] events(s) before resuming...`); + this.processingQueue.drain(async () => { await this.restartZongji(); }); } else { @@ -391,8 +397,30 @@ export class BinLogListener { const changes: SchemaChange[] = []; for (const statement of statements) { - // @ts-ignore - if (statement.type === 'rename') { + if (isTruncate(statement)) { + const truncateStatement = statement as TruncateStatement; + // Truncate statements can apply to multiple tables + for (const entity of truncateStatement.name) { + changes.push({ type: SchemaChangeType.TRUNCATE_TABLE, table: entity.table }); + } + } else if (isDropTable(statement)) { + for (const entity of statement.name) { + changes.push({ type: SchemaChangeType.DROP_TABLE, table: entity.table }); + } + } else if (isDropIndex(statement)) { + const dropStatement = statement as DropIndexStatement; + changes.push({ + type: SchemaChangeType.REPLICATION_IDENTITY, + table: dropStatement.table.table + }); + } else if (isCreateUniqueIndex(statement)) { + // Potential change to the replication identity if the table has no prior unique constraint + changes.push({ + type: SchemaChangeType.REPLICATION_IDENTITY, + // @ts-ignore - The type definitions for node-sql-parser do not reflect the correct structure here + table: statement.table!.table + }); + } else if (isRenameTable(statement)) { const renameStatement = statement as RenameStatement; // Rename statements can apply to multiple tables for (const table of renameStatement.table) { @@ -402,81 +430,35 @@ export class BinLogListener { newTable: table[1].table }); } - } else if (statement.type === 'create' && statement.keyword === 'table' && statement.temporary === null) { - changes.push({ - type: SchemaChangeType.CREATE_TABLE, - table: statement.table![0].table - }); - } // @ts-ignore - else if (statement.type === 'truncate') { - const truncateStatement = statement as TruncateStatement; - // Truncate statements can apply to multiple tables - for (const entity of truncateStatement.name) { - changes.push({ type: SchemaChangeType.TRUNCATE_TABLE, table: entity.table }); - } - } else if (statement.type === 'drop' && statement.keyword === 'table') { - // Drop statements can apply to multiple tables - for (const entity of statement.name) { - changes.push({ type: SchemaChangeType.DROP_TABLE, table: entity.table }); - } - } else if (statement.type === 'alter') { - const expression = statement.expr[0]; + } else if (isAlterTable(statement)) { const fromTable = statement.table[0] as BaseFrom; - if (expression.resource === 'table') { - if (expression.action === 'rename') { + for (const expression of statement.expr) { + if (isRenameExpression(expression)) { changes.push({ type: SchemaChangeType.RENAME_TABLE, table: fromTable.table, newTable: expression.table }); + } else if (isColumnExpression(expression)) { + changes.push({ + type: SchemaChangeType.ALTER_TABLE_COLUMN, + table: fromTable.table + }); + } else if (isConstraintExpression(expression)) { + // Potential changes to the replication identity + changes.push({ + type: SchemaChangeType.REPLICATION_IDENTITY, + table: fromTable.table + }); } - } else if (expression.resource === 'column') { - const columnChange: SchemaChange = { - type: this.toColumnSchemaChangeType(expression.action), - table: fromTable.table, - column: { - column: expression.column.column - } - }; - - if (expression.action === 'change' || expression.action === 'rename') { - columnChange.column = { - column: expression.old_column.column, - newColumn: expression.column.column - }; - } - changes.push(columnChange); - } else if (expression.resource === 'key' && expression.keyword === 'primary key') { - // This is a special case for MySQL, where the primary key is being set or changed - // We treat this as a replication identity change - changes.push({ - type: SchemaChangeType.REPLICATION_IDENTITY, - table: fromTable.table - }); } + break; } } - // Filter out schema changes that are not relevant to the included tables return changes.filter( (change) => this.options.tableFilter(change.table) || (change.newTable && this.options.tableFilter(change.newTable)) ); } - - private toColumnSchemaChangeType(action: string): SchemaChangeType { - switch (action) { - case 'drop': - return SchemaChangeType.DROP_COLUMN; - case 'add': - return SchemaChangeType.ADD_COLUMN; - case 'modify': - return SchemaChangeType.MODIFY_COLUMN; - case 'change': - case 'rename': - return SchemaChangeType.RENAME_COLUMN; - default: - throw new Error(`Unknown column schema change action: ${action}`); - } - } } diff --git a/modules/module-mysql/test/src/BinLogListener.test.ts b/modules/module-mysql/test/src/BinLogListener.test.ts index 829db5cc..8f4fb102 100644 --- a/modules/module-mysql/test/src/BinLogListener.test.ts +++ b/modules/module-mysql/test/src/BinLogListener.test.ts @@ -91,7 +91,7 @@ describe('BinlogListener tests', () => { expect(resumeSpy).toHaveBeenCalled(); }); - test('Row event handling', async () => { + test('Row events: Write, update, delete', async () => { await binLogListener.start(); const ROW_COUNT = 10; @@ -108,33 +108,25 @@ describe('BinlogListener tests', () => { await binLogListener.stop(); }); - test('Schema change event handling - ALTER TABLE RENAME', async () => { + test('Schema change event: Rename table', async () => { await binLogListener.start(); await connectionManager.query(`ALTER TABLE test_DATA RENAME test_DATA_new`); - await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); + await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 }); await binLogListener.stop(); expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.RENAME_TABLE); expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); expect(eventHandler.schemaChanges[0].newTable).toEqual('test_DATA_new'); }); - test('Schema change event handling - RENAME TABLE', async () => { - await binLogListener.start(); - await connectionManager.query(`RENAME TABLE test_DATA TO test_DATA_new`); - await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); - await binLogListener.stop(); - expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.RENAME_TABLE); - expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); - expect(eventHandler.schemaChanges[0].newTable).toEqual('test_DATA_new'); - }); - - test('Schema change event handling - RENAME TABLE multiple', async () => { + test('Schema change event: Rename multiple tables', async () => { + // RENAME TABLE supports renaming multiple tables in a single statement + // We generate a schema change event for each table renamed await binLogListener.start(); await connectionManager.query(`RENAME TABLE test_DATA TO test_DATA_new, test_DATA_new TO test_DATA `); - await vi.waitFor(() => expect(eventHandler.schemaChanges.length == 2).toBeTruthy(), { timeout: 5000 }); + await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(2), { timeout: 5000 }); await binLogListener.stop(); expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.RENAME_TABLE); expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); @@ -145,109 +137,151 @@ describe('BinlogListener tests', () => { expect(eventHandler.schemaChanges[1].newTable).toEqual('test_DATA'); }); - test('Schema change event handling - TRUNCATE TABLE', async () => { + test('Schema change event: Truncate table', async () => { await binLogListener.start(); await connectionManager.query(`TRUNCATE TABLE test_DATA`); - await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); + await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 }); await binLogListener.stop(); expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.TRUNCATE_TABLE); expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); }); - test('Schema change event handling - DROP AND CREATE TABLE ', async () => { + test('Schema change event: Drop table', async () => { await binLogListener.start(); await connectionManager.query(`DROP TABLE test_DATA`); await connectionManager.query(`CREATE TABLE test_DATA (id CHAR(36) PRIMARY KEY, description MEDIUMTEXT)`); - await vi.waitFor(() => expect(eventHandler.schemaChanges.length === 2).toBeTruthy(), { timeout: 5000 }); + await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 }); await binLogListener.stop(); expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.DROP_TABLE); expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); - expect(eventHandler.schemaChanges[1].type).toBe(SchemaChangeType.CREATE_TABLE); - expect(eventHandler.schemaChanges[1].table).toEqual('test_DATA'); }); - test('Schema change event handling - ALTER TABLE DROP COLUMN', async () => { + test('Schema change event: Drop column', async () => { await binLogListener.start(); await connectionManager.query(`ALTER TABLE test_DATA DROP COLUMN description`); - await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); + await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 }); await binLogListener.stop(); - expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.DROP_COLUMN); + expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.ALTER_TABLE_COLUMN); expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); - expect(eventHandler.schemaChanges[0].column?.column).toEqual('description'); }); - test('Schema change event handling - ALTER TABLE ADD COLUMN', async () => { + test('Schema change event: Add column', async () => { await binLogListener.start(); await connectionManager.query(`ALTER TABLE test_DATA ADD COLUMN new_column VARCHAR(255)`); - await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); + await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 }); await binLogListener.stop(); - expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.ADD_COLUMN); + expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.ALTER_TABLE_COLUMN); expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); - expect(eventHandler.schemaChanges[0].column?.column).toEqual('new_column'); }); - test('Schema change event handling - ALTER TABLE MODIFY COLUMN', async () => { + test('Schema change event: Modify column', async () => { await binLogListener.start(); await connectionManager.query(`ALTER TABLE test_DATA MODIFY COLUMN description TEXT`); - await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); + await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 }); await binLogListener.stop(); - expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.MODIFY_COLUMN); + expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.ALTER_TABLE_COLUMN); expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); - expect(eventHandler.schemaChanges[0].column?.column).toEqual('description'); }); - test('Schema change event handling - ALTER TABLE CHANGE COLUMN column rename', async () => { + test('Schema change event: Rename column via change statement', async () => { await binLogListener.start(); await connectionManager.query(`ALTER TABLE test_DATA CHANGE COLUMN description description_new MEDIUMTEXT`); - await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); + await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 }); await binLogListener.stop(); - expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.RENAME_COLUMN); + expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.ALTER_TABLE_COLUMN); expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); - expect(eventHandler.schemaChanges[0].column?.column).toEqual('description'); - expect(eventHandler.schemaChanges[0].column?.newColumn).toEqual('description_new'); }); - test('Schema changes for non-matching tables are ignored', async () => { - // TableFilter = only match 'test_DATA' - await binLogListener.start(); - await connectionManager.query(`CREATE TABLE test_ignored (id CHAR(36) PRIMARY KEY, description TEXT)`); - await connectionManager.query(`ALTER TABLE test_ignored ADD COLUMN new_column VARCHAR(10)`); - await connectionManager.query(`DROP TABLE test_ignored`); - - // "Anchor" event to latch onto, ensuring that the schema change events have finished - await insertRows(connectionManager, 1); - await vi.waitFor(() => expect(eventHandler.rowsWritten).equals(1), { timeout: 5000 }); - await binLogListener.stop(); - - expect(eventHandler.schemaChanges.length).toBe(0); - }); - - test('Schema change event handling - ALTER TABLE RENAME COLUMN column rename', async () => { + test('Schema change event: Rename column via rename statement', async () => { // Syntax ALTER TABLE RENAME COLUMN was only introduced in MySQL 8.0.0 if (!isMySQL57) { await binLogListener.start(); await connectionManager.query(`ALTER TABLE test_DATA RENAME COLUMN description TO description_new`); - await vi.waitFor(() => expect(eventHandler.schemaChanges.length > 0).toBeTruthy(), { timeout: 5000 }); + await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 }); await binLogListener.stop(); - expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.RENAME_COLUMN); + expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.ALTER_TABLE_COLUMN); expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); - expect(eventHandler.schemaChanges[0].column?.column).toEqual('description'); - expect(eventHandler.schemaChanges[0].column?.newColumn).toEqual('description_new'); } }); - test('Unparseable query events that dont match tables in the sync rules are ignored', async () => { - binLogListener.options.tableFilter = (table) => ['test_DATA', 'test_unparseable'].includes(table); + test('Schema change event: Multiple column changes', async () => { + // ALTER TABLE can have multiple column changes in a single statement await binLogListener.start(); await connectionManager.query( - `CREATE TABLE test_unparseable (sale_date DATE) PARTITION BY RANGE (YEAR(sale_date)) - (PARTITION p2023 VALUES LESS THAN (2024))` + `ALTER TABLE test_DATA DROP COLUMN description, ADD COLUMN new_description TEXT, MODIFY COLUMN id VARCHAR(50)` ); + await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(3), { timeout: 5000 }); + await binLogListener.stop(); + expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.ALTER_TABLE_COLUMN); + expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); + + expect(eventHandler.schemaChanges[1].type).toBe(SchemaChangeType.ALTER_TABLE_COLUMN); + expect(eventHandler.schemaChanges[1].table).toEqual('test_DATA'); + + expect(eventHandler.schemaChanges[2].type).toBe(SchemaChangeType.ALTER_TABLE_COLUMN); + expect(eventHandler.schemaChanges[2].table).toEqual('test_DATA'); + }); + + test('Schema change event: Drop and Add primary key', async () => { + await connectionManager.query(`CREATE TABLE test_constraints (id CHAR(36), description VARCHAR(100))`); + binLogListener.options.tableFilter = (table) => table === 'test_constraints'; + await binLogListener.start(); + await connectionManager.query(`ALTER TABLE test_constraints ADD PRIMARY KEY (id)`); + await connectionManager.query(`ALTER TABLE test_constraints DROP PRIMARY KEY`); + await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(2), { timeout: 5000 }); + await binLogListener.stop(); + // Event for the add + expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.REPLICATION_IDENTITY); + expect(eventHandler.schemaChanges[0].table).toEqual('test_constraints'); + // Event for the drop + expect(eventHandler.schemaChanges[1].type).toBe(SchemaChangeType.REPLICATION_IDENTITY); + expect(eventHandler.schemaChanges[1].table).toEqual('test_constraints'); + }); + + test('Schema change event: Add and drop unique constraint', async () => { + await connectionManager.query(`CREATE TABLE test_constraints (id CHAR(36), description VARCHAR(100))`); + binLogListener.options.tableFilter = (table) => table === 'test_constraints'; + await binLogListener.start(); + await connectionManager.query(`ALTER TABLE test_constraints ADD UNIQUE (description)`); + await connectionManager.query(`ALTER TABLE test_constraints DROP INDEX description`); + await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(2), { timeout: 5000 }); + await binLogListener.stop(); + // Event for the creation + expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.REPLICATION_IDENTITY); + expect(eventHandler.schemaChanges[0].table).toEqual('test_constraints'); + // Event for the drop + expect(eventHandler.schemaChanges[1].type).toBe(SchemaChangeType.REPLICATION_IDENTITY); + expect(eventHandler.schemaChanges[1].table).toEqual('test_constraints'); + }); + + test('Schema change event: Add and drop a unique index', async () => { + await connectionManager.query(`CREATE TABLE test_constraints (id CHAR(36), description VARCHAR(100))`); + binLogListener.options.tableFilter = (table) => table === 'test_constraints'; + await binLogListener.start(); + await connectionManager.query(`CREATE UNIQUE INDEX description_idx ON test_constraints (description)`); + await connectionManager.query(`DROP INDEX description_idx ON test_constraints`); + await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(2), { timeout: 5000 }); + await binLogListener.stop(); + // Event for the creation + expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.REPLICATION_IDENTITY); + expect(eventHandler.schemaChanges[0].table).toEqual('test_constraints'); + // Event for the drop + expect(eventHandler.schemaChanges[1].type).toBe(SchemaChangeType.REPLICATION_IDENTITY); + expect(eventHandler.schemaChanges[1].table).toEqual('test_constraints'); + }); + + test('Schema changes for non-matching tables are ignored', async () => { + // TableFilter = only match 'test_DATA' + await binLogListener.start(); + await connectionManager.query(`CREATE TABLE test_ignored (id CHAR(36) PRIMARY KEY, description TEXT)`); + await connectionManager.query(`ALTER TABLE test_ignored ADD COLUMN new_column VARCHAR(10)`); + await connectionManager.query(`DROP TABLE test_ignored`); // "Anchor" event to latch onto, ensuring that the schema change events have finished await insertRows(connectionManager, 1); await vi.waitFor(() => expect(eventHandler.rowsWritten).equals(1), { timeout: 5000 }); await binLogListener.stop(); + expect(eventHandler.schemaChanges.length).toBe(0); }); }); diff --git a/modules/module-mysql/test/src/BinLogStream.test.ts b/modules/module-mysql/test/src/BinLogStream.test.ts index 3f78b16d..89b542e9 100644 --- a/modules/module-mysql/test/src/BinLogStream.test.ts +++ b/modules/module-mysql/test/src/BinLogStream.test.ts @@ -82,6 +82,42 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { expect(endTxCount - startTxCount).toEqual(1); }); + test('Replicate matched wild card tables in sync rules', async () => { + await using context = await BinlogStreamTestContext.open(factory); + const { connectionManager } = context; + await context.updateSyncRules(` + bucket_definitions: + global: + data: + - SELECT id, description FROM "test_data_%"`); + + await connectionManager.query(`CREATE TABLE test_data_1 (id CHAR(36) PRIMARY KEY, description TEXT)`); + await connectionManager.query(`CREATE TABLE test_data_2 (id CHAR(36) PRIMARY KEY, description TEXT)`); + + const testId11 = uuid(); + await connectionManager.query(`INSERT INTO test_data_1(id, description) VALUES('${testId11}','test11')`); + + const testId21 = uuid(); + await connectionManager.query(`INSERT INTO test_data_2(id, description) VALUES('${testId21}','test21')`); + + await context.replicateSnapshot(); + await context.startStreaming(); + + const testId12 = uuid(); + await connectionManager.query(`INSERT INTO test_data_1(id, description) VALUES('${testId12}', 'test12')`); + + const testId22 = uuid(); + await connectionManager.query(`INSERT INTO test_data_2(id, description) VALUES('${testId22}', 'test22')`); + const data = await context.getBucketData('global[]'); + + expect(data).toMatchObject([ + putOp('test_data_1', { id: testId11, description: 'test11' }), + putOp('test_data_2', { id: testId21, description: 'test21' }), + putOp('test_data_1', { id: testId12, description: 'test12' }), + putOp('test_data_2', { id: testId22, description: 'test22' }) + ]); + }); + test('Handle table TRUNCATE events', async () => { await using context = await BinlogStreamTestContext.open(factory); await context.updateSyncRules(BASIC_SYNC_RULES); diff --git a/modules/module-mysql/test/src/schema-changes.test.ts b/modules/module-mysql/test/src/schema-changes.test.ts index fecb6959..c88a62c0 100644 --- a/modules/module-mysql/test/src/schema-changes.test.ts +++ b/modules/module-mysql/test/src/schema-changes.test.ts @@ -64,7 +64,7 @@ function defineTests(factory: storage.TestStorageFactory) { ]); }); - test('Add a table that is in the sync rules', async () => { + test('Create table: New table in is in the sync rules', async () => { await using context = await BinlogStreamTestContext.open(factory); const { connectionManager } = context; await context.updateSyncRules(BASIC_SYNC_RULES); @@ -82,7 +82,55 @@ function defineTests(factory: storage.TestStorageFactory) { expect(data).toMatchObject([PUT_T1, PUT_T1]); }); - test('(1) Rename a table not in the sync rules to one in the sync rules', async () => { + test('Create table: New table is created from existing data', async () => { + await using context = await BinlogStreamTestContext.open(factory); + const { connectionManager } = context; + await context.updateSyncRules(BASIC_SYNC_RULES); + + await connectionManager.query(`CREATE TABLE test_data_from (id CHAR(36) PRIMARY KEY, description TEXT)`); + await connectionManager.query(`INSERT INTO test_data_from(id, description) VALUES('t1','test1')`); + await connectionManager.query(`INSERT INTO test_data_from(id, description) VALUES('t2','test2')`); + await connectionManager.query(`INSERT INTO test_data_from(id, description) VALUES('t3','test3')`); + + await context.replicateSnapshot(); + await context.startStreaming(); + + // Add table after initial replication + await connectionManager.query(`CREATE TABLE test_data SELECT * FROM test_data_from`); + const data = await context.getBucketData('global[]'); + + // Interestingly, the create with select triggers binlog row write events + expect(data).toMatchObject([ + // From snapshot + PUT_T1, + PUT_T2, + PUT_T3, + // From replication stream + PUT_T1, + PUT_T2, + PUT_T3 + ]); + }); + + test('Create table: New table is not in the sync rules', async () => { + await using context = await BinlogStreamTestContext.open(factory); + const { connectionManager } = context; + await context.updateSyncRules(BASIC_SYNC_RULES); + + await context.replicateSnapshot(); + await context.startStreaming(); + + // Add table after initial replication + await connectionManager.query(`CREATE TABLE test_data_ignored (id CHAR(36) PRIMARY KEY, description TEXT)`); + + await connectionManager.query(`INSERT INTO test_data_ignored(id, description) VALUES('t1','test ignored')`); + + const data = await context.getBucketData('global[]'); + + expect(data).toMatchObject([]); + }); + + test('Rename table: Table not in the sync rules to one in the sync rules', async () => { await using context = await BinlogStreamTestContext.open(factory); const { connectionManager } = context; await context.updateSyncRules(BASIC_SYNC_RULES); @@ -109,7 +157,7 @@ function defineTests(factory: storage.TestStorageFactory) { ]); }); - test('(2) Rename a table in the sync rules to another table also in the sync rules', async () => { + test('Rename table: Table in the sync rules to another table in the sync rules', async () => { await using context = await BinlogStreamTestContext.open(factory); await context.updateSyncRules(` @@ -150,7 +198,7 @@ function defineTests(factory: storage.TestStorageFactory) { ]); }); - test('(3) Rename table in the sync rules to one not in the sync rules', async () => { + test('Rename table: Table in the sync rules to not in the sync rules', async () => { await using context = await BinlogStreamTestContext.open(factory); await context.updateSyncRules(BASIC_SYNC_RULES); @@ -174,7 +222,7 @@ function defineTests(factory: storage.TestStorageFactory) { ]); }); - test('(1) Change Replication Identity default by dropping the primary key', async () => { + test('Change Replication Identity default to full by dropping the primary key', async () => { await using context = await BinlogStreamTestContext.open(factory); // Change replica id from default (PK) to full // Requires re-snapshotting the table. @@ -208,7 +256,7 @@ function defineTests(factory: storage.TestStorageFactory) { ]); }); - test('(2) Change Replication Identity full by adding a column', async () => { + test('Change Replication Identity full by adding a column', async () => { await using context = await BinlogStreamTestContext.open(factory); // Change replica id from full by adding column // Causes a re-import of the table. @@ -246,7 +294,114 @@ function defineTests(factory: storage.TestStorageFactory) { ]); }); - test('(3) Change Replication Identity default by modifying primary key column type', async () => { + test('Change Replication Identity from full to index by adding a unique constraint', async () => { + await using context = await BinlogStreamTestContext.open(factory); + // Change replica id full by adding a unique index that can serve as the replication id + + await context.updateSyncRules(BASIC_SYNC_RULES); + + const { connectionManager } = context; + // No primary key, no unique column, so full replication identity will be used + await connectionManager.query(`CREATE TABLE test_data (id CHAR(36), description TEXT)`); + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t1','test1')`); + + await context.replicateSnapshot(); + await context.startStreaming(); + + await connectionManager.query(`ALTER TABLE test_data ADD UNIQUE (id)`); + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t2','test2')`); + + const data = await context.getBucketData('global[]'); + + expect(data.slice(0, 2)).toMatchObject([ + // Initial inserts + PUT_T1, + // Truncate + REMOVE_T1 + ]); + + // Snapshot - order doesn't matter + expect(data.slice(2)).toMatchObject([ + // Snapshot inserts + PUT_T1, + PUT_T2, + // Replicated insert + PUT_T2 + ]); + }); + + test('Change Replication Identity from full to index by adding a unique index', async () => { + await using context = await BinlogStreamTestContext.open(factory); + // Change replica id full by adding a unique index that can serve as the replication id + await context.updateSyncRules(BASIC_SYNC_RULES); + + const { connectionManager } = context; + // No primary key, no unique column, so full replication identity will be used + await connectionManager.query(`CREATE TABLE test_data (id CHAR(36), description TEXT)`); + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t1','test1')`); + + await context.replicateSnapshot(); + await context.startStreaming(); + + await connectionManager.query(`CREATE UNIQUE INDEX id_idx ON test_data (id)`); + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t2','test2')`); + + const data = await context.getBucketData('global[]'); + + expect(data.slice(0, 2)).toMatchObject([ + // Initial inserts + PUT_T1, + // Truncate + REMOVE_T1 + ]); + + // Snapshot - order doesn't matter + expect(data.slice(2)).toMatchObject([ + // Snapshot inserts + PUT_T1, + PUT_T2, + // Replicated insert + PUT_T2 + ]); + }); + + test('Change Replication Identity from index by dropping the unique constraint', async () => { + await using context = await BinlogStreamTestContext.open(factory); + // Change replica id full by adding a unique index that can serve as the replication id + + await context.updateSyncRules(BASIC_SYNC_RULES); + + const { connectionManager } = context; + // Unique constraint on id + await connectionManager.query(`CREATE TABLE test_data (id CHAR(36), description TEXT, UNIQUE (id))`); + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t1','test1')`); + + await context.replicateSnapshot(); + await context.startStreaming(); + + await connectionManager.query(`ALTER TABLE test_data DROP INDEX id`); + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t2','test2')`); + + const data = await context.getBucketData('global[]'); + + expect(data.slice(0, 2)).toMatchObject([ + // Initial inserts + PUT_T1, + // Truncate + REMOVE_T1 + ]); + + // Snapshot - order doesn't matter + expect(data.slice(2)).toMatchObject([ + // Snapshot inserts + PUT_T1, + PUT_T2, + // Replicated insert + PUT_T2 + ]); + }); + + test('Change Replication Identity default by modifying primary key column type', async () => { await using context = await BinlogStreamTestContext.open(factory); await context.updateSyncRules(BASIC_SYNC_RULES); @@ -278,7 +433,7 @@ function defineTests(factory: storage.TestStorageFactory) { ]); }); - test('(4) Change Replication Identity by changing the type of a column in a compound unique index', async () => { + test('Change Replication Identity by changing the type of a column in a compound unique index', async () => { await using context = await BinlogStreamTestContext.open(factory); // Change index replica id by changing column type // Causes a re-import of the table. @@ -323,6 +478,66 @@ function defineTests(factory: storage.TestStorageFactory) { ]); }); + test('Add column: New non replication identity column does not trigger re-sync', async () => { + await using context = await BinlogStreamTestContext.open(factory); + // Added column not in replication identity so it should not cause a re-import + + await context.updateSyncRules(BASIC_SYNC_RULES); + + const { connectionManager } = context; + await connectionManager.query(`CREATE TABLE test_data (id CHAR(36) PRIMARY KEY, description TEXT)`); + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t1','test1')`); + + await context.replicateSnapshot(); + await context.startStreaming(); + + await connectionManager.query(`ALTER TABLE test_data ADD COLUMN new_column TEXT`); + await connectionManager.query( + `INSERT INTO test_data(id, description, new_column) VALUES('t2','test2', 'new_data')` + ); + + const data = await context.getBucketData('global[]'); + + expect(data.slice(0, 1)).toMatchObject([PUT_T1]); + + expect(data.slice(1)).toMatchObject([ + // Snapshot inserts + putOp('test_data', { id: 't2', description: 'test2', new_column: 'new_data' }) + ]); + }); + + test('Modify non replication identity column', async () => { + await using context = await BinlogStreamTestContext.open(factory); + // Changing the type of a column that is not part of the replication identity does not cause a re-sync of the table. + await context.updateSyncRules(BASIC_SYNC_RULES); + + const { connectionManager } = context; + await connectionManager.query(`CREATE TABLE test_data (id CHAR(36) PRIMARY KEY, description TEXT)`); + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t1','test1')`); + + await context.replicateSnapshot(); + await context.startStreaming(); + + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t2','test2')`); + + await connectionManager.query(`ALTER TABLE test_data MODIFY COLUMN description VARCHAR(100)`); + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t3','test3')`); + + const data = await context.getBucketData('global[]'); + + expect(data.slice(0, 2)).toMatchObject([ + // Initial snapshot + PUT_T1, + // Streamed + PUT_T2 + ]); + + expect(data.slice(2)).toMatchObject([ + // Replicated insert + PUT_T3 + ]); + }); + test('Drop a table in the sync rules', async () => { await using context = await BinlogStreamTestContext.open(factory); // Technically not a schema change, but fits here. From a339ec83764d4e613813bc15e8d118b3360d9c23 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 9 Jul 2025 12:40:57 +0200 Subject: [PATCH 36/48] Removed unused constant --- modules/module-mysql/src/replication/BinLogStream.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index acfa75b2..c2296571 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -413,9 +413,6 @@ export class BinLogStream { { zeroLSN: common.ReplicatedGTID.ZERO.comparable, defaultSchema: this.defaultSchema, storeCurrentData: true }, async (batch) => { const binlogEventHandler = this.createBinlogEventHandler(batch); - // Only listen for changes to tables in the sync rules - const includedTables = [...this.tableCache.values()].map((table) => table.name); - const binlogListener = new BinLogListener({ logger: this.logger, tableFilter: (table: string) => this.matchesTable(table), From 5df001b15fc7ba55c8b18712f87882919a14f62d Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 9 Jul 2025 13:05:26 +0200 Subject: [PATCH 37/48] Skip unsupported schema test for MySQL 5.7 --- .../test/src/schema-changes.test.ts | 81 ++++++++++++------- 1 file changed, 52 insertions(+), 29 deletions(-) diff --git a/modules/module-mysql/test/src/schema-changes.test.ts b/modules/module-mysql/test/src/schema-changes.test.ts index c88a62c0..af06e9a2 100644 --- a/modules/module-mysql/test/src/schema-changes.test.ts +++ b/modules/module-mysql/test/src/schema-changes.test.ts @@ -1,10 +1,12 @@ import { compareIds, putOp, removeOp, test_utils } from '@powersync/service-core-tests'; -import { describe, expect, test } from 'vitest'; +import { beforeAll, describe, expect, test } from 'vitest'; import { storage } from '@powersync/service-core'; -import { describeWithStorage } from './util.js'; +import { describeWithStorage, TEST_CONNECTION_OPTIONS } from './util.js'; import { BinlogStreamTestContext } from './BinlogStreamUtils.js'; import timers from 'timers/promises'; +import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManager.js'; +import { getMySQLVersion, satisfiesVersion } from '@module/utils/mysql-utils.js'; describe('MySQL Schema Changes', () => { describeWithStorage({ timeout: 20_000 }, defineTests); @@ -25,6 +27,17 @@ const REMOVE_T1 = test_utils.removeOp('test_data', 't1'); const REMOVE_T2 = test_utils.removeOp('test_data', 't2'); function defineTests(factory: storage.TestStorageFactory) { + let isMySQL57: boolean = false; + + beforeAll(async () => { + const connectionManager = new MySQLConnectionManager(TEST_CONNECTION_OPTIONS, {}); + const connection = await connectionManager.getConnection(); + const version = await getMySQLVersion(connection); + isMySQL57 = satisfiesVersion(version, '5.7.x'); + connection.release(); + await connectionManager.end(); + }); + test('Re-create table', async () => { await using context = await BinlogStreamTestContext.open(factory); // Drop a table and re-create it. @@ -83,33 +96,43 @@ function defineTests(factory: storage.TestStorageFactory) { }); test('Create table: New table is created from existing data', async () => { - await using context = await BinlogStreamTestContext.open(factory); - const { connectionManager } = context; - await context.updateSyncRules(BASIC_SYNC_RULES); - - await connectionManager.query(`CREATE TABLE test_data_from (id CHAR(36) PRIMARY KEY, description TEXT)`); - await connectionManager.query(`INSERT INTO test_data_from(id, description) VALUES('t1','test1')`); - await connectionManager.query(`INSERT INTO test_data_from(id, description) VALUES('t2','test2')`); - await connectionManager.query(`INSERT INTO test_data_from(id, description) VALUES('t3','test3')`); - - await context.replicateSnapshot(); - await context.startStreaming(); - - // Add table after initial replication - await connectionManager.query(`CREATE TABLE test_data SELECT * FROM test_data_from`); - const data = await context.getBucketData('global[]'); - - // Interestingly, the create with select triggers binlog row write events - expect(data).toMatchObject([ - // From snapshot - PUT_T1, - PUT_T2, - PUT_T3, - // From replication stream - PUT_T1, - PUT_T2, - PUT_T3 - ]); + // Create table with select from is not allowed in MySQL 5.7 when enforce_gtid_consistency=ON + if (!isMySQL57) { + await using context = await BinlogStreamTestContext.open(factory); + const { connectionManager } = context; + await context.updateSyncRules(BASIC_SYNC_RULES); + + await connectionManager.query(`CREATE TABLE test_data_from + ( + id CHAR(36) PRIMARY KEY, + description TEXT + )`); + await connectionManager.query(`INSERT INTO test_data_from(id, description) + VALUES ('t1', 'test1')`); + await connectionManager.query(`INSERT INTO test_data_from(id, description) + VALUES ('t2', 'test2')`); + await connectionManager.query(`INSERT INTO test_data_from(id, description) + VALUES ('t3', 'test3')`); + + await context.replicateSnapshot(); + await context.startStreaming(); + + // Add table after initial replication + await connectionManager.query(`CREATE TABLE test_data SELECT * FROM test_data_from`); + const data = await context.getBucketData('global[]'); + + // Interestingly, the create with select triggers binlog row write events + expect(data).toMatchObject([ + // From snapshot + PUT_T1, + PUT_T2, + PUT_T3, + // From replication stream + PUT_T1, + PUT_T2, + PUT_T3 + ]); + } }); test('Create table: New table is not in the sync rules', async () => { From 57fcfec42433c5899f2b5f8d2bc00f096cbcbe59 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Thu, 10 Jul 2025 14:51:13 +0200 Subject: [PATCH 38/48] Added error handling for zongji emitted schema errors --- modules/module-mysql/package.json | 2 +- .../src/replication/zongji/BinLogListener.ts | 17 +++++-- .../test/src/BinLogListener.test.ts | 46 +++++++++++++++++++ pnpm-lock.yaml | 10 ++-- 4 files changed, 64 insertions(+), 11 deletions(-) diff --git a/modules/module-mysql/package.json b/modules/module-mysql/package.json index 78730b83..36f7de32 100644 --- a/modules/module-mysql/package.json +++ b/modules/module-mysql/package.json @@ -33,7 +33,7 @@ "@powersync/service-sync-rules": "workspace:*", "@powersync/service-types": "workspace:*", "@powersync/service-jsonbig": "workspace:*", - "@powersync/mysql-zongji": "0.3.0", + "@powersync/mysql-zongji": "^0.4.0", "async": "^3.2.4", "mysql2": "^3.11.0", "node-sql-parser": "^5.3.9", diff --git a/modules/module-mysql/src/replication/zongji/BinLogListener.ts b/modules/module-mysql/src/replication/zongji/BinLogListener.ts index 0e1fd094..9ff53a5b 100644 --- a/modules/module-mysql/src/replication/zongji/BinLogListener.ts +++ b/modules/module-mysql/src/replication/zongji/BinLogListener.ts @@ -86,6 +86,7 @@ export class BinLogListener { private binLogPosition: common.BinLogPosition; private currentGTID: common.ReplicatedGTID | null; private logger: Logger; + private listenerError: Error | null; zongji: ZongJi; processingQueue: async.QueueObject; @@ -106,6 +107,7 @@ export class BinLogListener { this.sqlParser = new Parser(); this.processingQueue = this.createProcessingQueue(); this.zongji = this.createZongjiListener(); + this.listenerError = null; } /** @@ -180,8 +182,8 @@ export class BinLogListener { } private async stopZongji(): Promise { - this.logger.info('Stopping BinLog Listener...'); if (!this.zongji.stopped) { + this.logger.info('Stopping BinLog Listener...'); await new Promise((resolve) => { this.zongji.once('stopped', () => { resolve(); @@ -206,6 +208,11 @@ export class BinLogListener { while (!this.isStopped) { await timers.setTimeout(1_000); } + + if (this.listenerError) { + this.logger.error('BinLog Listener stopped due to an error:', this.listenerError); + throw this.listenerError; + } } private createProcessingQueue(): async.QueueObject { @@ -213,7 +220,7 @@ export class BinLogListener { queue.error((error) => { if (!(this.isStopped || this.isStopping)) { - this.logger.error('Error processing BinLog event:', error); + this.listenerError = error; this.stop(); } else { this.logger.warn('Error processing BinLog event during shutdown:', error); @@ -227,7 +234,7 @@ export class BinLogListener { const zongji = this.connectionManager.createBinlogListener(); zongji.on('binlog', async (evt) => { - this.logger.info(`Received BinLog event:${evt.getEventName()}`); + this.logger.debug(`Received BinLog event:${evt.getEventName()}`); this.processingQueue.push(evt); this.queueMemoryUsage += evt.size; @@ -247,7 +254,7 @@ export class BinLogListener { zongji.on('error', (error) => { if (!(this.isStopped || this.isStopping)) { - this.logger.error('BinLog Listener error:', error); + this.listenerError = error; this.stop(); } else { this.logger.warn('Ignored BinLog Listener error during shutdown:', error); @@ -375,7 +382,7 @@ export class BinLogListener { // If there are still events in the processing queue, we need to process those before restarting Zongji if (!this.processingQueue.idle()) { - this.logger.info(`Finish processing [${this.processingQueue.length()}] events(s) before resuming...`); + this.logger.info(`Processing [${this.processingQueue.length()}] events(s) before resuming...`); this.processingQueue.drain(async () => { await this.restartZongji(); }); diff --git a/modules/module-mysql/test/src/BinLogListener.test.ts b/modules/module-mysql/test/src/BinLogListener.test.ts index 8f4fb102..63430f5f 100644 --- a/modules/module-mysql/test/src/BinLogListener.test.ts +++ b/modules/module-mysql/test/src/BinLogListener.test.ts @@ -284,6 +284,52 @@ describe('BinlogListener tests', () => { expect(eventHandler.schemaChanges.length).toBe(0); }); + + test('Sequential schema change handling', async () => { + // If there are multiple schema changes in the binlog processing queue, we only restart the binlog listener once + // all the schema changes have been processed + await connectionManager.query(`CREATE TABLE test_multiple (id CHAR(36), description VARCHAR(100))`); + await connectionManager.query(`ALTER TABLE test_multiple ADD COLUMN new_column VARCHAR(10)`); + await connectionManager.query(`ALTER TABLE test_multiple ADD PRIMARY KEY (id)`); + await connectionManager.query(`ALTER TABLE test_multiple MODIFY COLUMN new_column TEXT`); + await connectionManager.query(`DROP TABLE test_multiple`); + + binLogListener.options.tableFilter = (table) => table === 'test_multiple'; + await binLogListener.start(); + + await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(4), { timeout: 5000 }); + await binLogListener.stop(); + expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.ALTER_TABLE_COLUMN); + expect(eventHandler.schemaChanges[1].type).toBe(SchemaChangeType.REPLICATION_IDENTITY); + expect(eventHandler.schemaChanges[2].type).toBe(SchemaChangeType.ALTER_TABLE_COLUMN); + expect(eventHandler.schemaChanges[3].type).toBe(SchemaChangeType.DROP_TABLE); + }); + + test('Unprocessed binlog event received that does match the current table schema', async () => { + // If we process a binlog event for a table which has since had its schema changed, we expect the binlog listener to stop with an error + await connectionManager.query(`CREATE TABLE test_failure (id CHAR(36), description VARCHAR(100))`); + await connectionManager.query(`INSERT INTO test_failure(id, description) VALUES('${uuid()}','test_failure')`); + await connectionManager.query(`ALTER TABLE test_failure DROP COLUMN description`); + + binLogListener.options.tableFilter = (table) => table === 'test_failure'; + await binLogListener.start(); + + await expect(() => binLogListener.replicateUntilStopped()).rejects.toThrow( + /that does not match its current schema/ + ); + }); + + test('Unprocessed binlog event received for a dropped table', async () => { + // If we process a binlog event for a table which has since been dropped, we expect the binlog listener to stop with an error + await connectionManager.query(`CREATE TABLE test_failure (id CHAR(36), description VARCHAR(100))`); + await connectionManager.query(`INSERT INTO test_failure(id, description) VALUES('${uuid()}','test_failure')`); + await connectionManager.query(`DROP TABLE test_failure`); + + binLogListener.options.tableFilter = (table) => table === 'test_failure'; + await binLogListener.start(); + + await expect(() => binLogListener.replicateUntilStopped()).rejects.toThrow(/or the table has been dropped/); + }); }); async function getFromGTID(connectionManager: MySQLConnectionManager) { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cb942bde..975271a9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -258,8 +258,8 @@ importers: specifier: workspace:* version: link:../../libs/lib-services '@powersync/mysql-zongji': - specifier: 0.3.0 - version: 0.3.0 + specifier: ^0.4.0 + version: 0.4.0 '@powersync/service-core': specifier: workspace:* version: link:../../packages/service-core @@ -1284,8 +1284,8 @@ packages: resolution: {integrity: sha512-UA91GwWPhFExt3IizW6bOeY/pQ0BkuNwKjk9iQW9KqxluGCrg4VenZ0/L+2Y0+ZOtme72EVvg6v0zo3AMQRCeA==} engines: {node: '>=12'} - '@powersync/mysql-zongji@0.3.0': - resolution: {integrity: sha512-Y113htGr0vSz3+IIWnooE4H/qXcps4Qes3e9HJECwQq02i5Eh3MHlhUxJoODBTv9LWlmsBXNeD4AuM0P0Ijtuw==} + '@powersync/mysql-zongji@0.4.0': + resolution: {integrity: sha512-O5zGYF3mzHO50SOSj3/6EnXYebC2Lvu1BTashbbz6eLAwaR3TkxwMfPGqFEsuecIm22djSBlgXjKM2FzVVG/VQ==} engines: {node: '>=22.0.0'} '@powersync/service-jsonbig@0.17.10': @@ -4758,7 +4758,7 @@ snapshots: '@pnpm/network.ca-file': 1.0.2 config-chain: 1.1.13 - '@powersync/mysql-zongji@0.3.0': + '@powersync/mysql-zongji@0.4.0': dependencies: '@vlasky/mysql': 2.18.6 big-integer: 1.6.52 From e6240818f8f96b5a853d85de00d8bbdf7abc159f Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Fri, 11 Jul 2025 08:04:29 +0200 Subject: [PATCH 39/48] Added changeset --- .changeset/wet-berries-enjoy.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .changeset/wet-berries-enjoy.md diff --git a/.changeset/wet-berries-enjoy.md b/.changeset/wet-berries-enjoy.md new file mode 100644 index 00000000..5d07184b --- /dev/null +++ b/.changeset/wet-berries-enjoy.md @@ -0,0 +1,21 @@ +--- +'@powersync/service-module-postgres-storage': minor +'@powersync/service-module-mongodb-storage': minor +'@powersync/service-core-tests': minor +'@powersync/service-module-postgres': minor +'@powersync/service-module-mongodb': minor +'@powersync/service-core': minor +'@powersync/service-module-mysql': minor +'@powersync/service-sync-rules': minor +--- + +MySQL: +- Added schema change handling + - Except for some edge cases, the following schema changes are now handled automatically: + - Creation, renaming, dropping and truncation of tables. + - Creation and dropping of unique indexes and primary keys. + - Adding, modifying, dropping and renaming of table columns. + - If a schema change cannot handled automatically, a warning with details will be logged. + - Mismatches in table schema from the Zongji binlog listener are now handled more gracefully. +- Replication of wildcard tables is now supported. +- Improved logging for binlog event processing. From 462e08d8b393c4539a75eddcd314d393bd1ea628 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Fri, 11 Jul 2025 10:10:00 +0200 Subject: [PATCH 40/48] Typo fixes from pr feedback --- modules/module-mysql/src/replication/zongji/BinLogListener.ts | 1 - modules/module-mysql/test/src/BinLogStream.test.ts | 2 +- packages/sync-rules/src/events/SqlEventDescriptor.ts | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/module-mysql/src/replication/zongji/BinLogListener.ts b/modules/module-mysql/src/replication/zongji/BinLogListener.ts index 9ff53a5b..08be7193 100644 --- a/modules/module-mysql/src/replication/zongji/BinLogListener.ts +++ b/modules/module-mysql/src/replication/zongji/BinLogListener.ts @@ -459,7 +459,6 @@ export class BinLogListener { }); } } - break; } } // Filter out schema changes that are not relevant to the included tables diff --git a/modules/module-mysql/test/src/BinLogStream.test.ts b/modules/module-mysql/test/src/BinLogStream.test.ts index 89b542e9..3b871969 100644 --- a/modules/module-mysql/test/src/BinLogStream.test.ts +++ b/modules/module-mysql/test/src/BinLogStream.test.ts @@ -13,7 +13,7 @@ bucket_definitions: - SELECT id, description FROM "test_data" `; -describe('BigLogStream tests', () => { +describe('BinLogStream tests', () => { describeWithStorage({ timeout: 20_000 }, defineBinlogStreamTests); }); diff --git a/packages/sync-rules/src/events/SqlEventDescriptor.ts b/packages/sync-rules/src/events/SqlEventDescriptor.ts index 1d76407e..ff09ab25 100644 --- a/packages/sync-rules/src/events/SqlEventDescriptor.ts +++ b/packages/sync-rules/src/events/SqlEventDescriptor.ts @@ -42,7 +42,7 @@ export class SqlEventDescriptor { const matchingQuery = this.sourceQueries.find((q) => q.applies(options.sourceTable)); if (!matchingQuery) { return { - errors: [{ error: `No marching source query found for table ${options.sourceTable.name}` }] + errors: [{ error: `No matching source query found for table ${options.sourceTable.name}` }] }; } From 15183589cc9b409f56e01ba7626ccf80fbcbaa38 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 23 Jul 2025 13:49:29 +0200 Subject: [PATCH 41/48] Removed filters from mysql dev docker config --- modules/module-mysql/dev/docker/mysql/init-scripts/my.cnf | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/module-mysql/dev/docker/mysql/init-scripts/my.cnf b/modules/module-mysql/dev/docker/mysql/init-scripts/my.cnf index 99f01c70..ea21db79 100644 --- a/modules/module-mysql/dev/docker/mysql/init-scripts/my.cnf +++ b/modules/module-mysql/dev/docker/mysql/init-scripts/my.cnf @@ -4,6 +4,4 @@ enforce-gtid-consistency = ON # Row format required for ZongJi binlog_format = row log_bin=mysql-bin -server-id=1 -binlog-do-db=mydatabase -replicate-do-table=mydatabase.lists \ No newline at end of file +server-id=1 \ No newline at end of file From 22c42a24f9ffbc6b20aa18e6a3e3d186d5c47a21 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 23 Jul 2025 13:50:24 +0200 Subject: [PATCH 42/48] Added safeguard for gtid splitting when no transactions have been run on the mysql database yet. --- modules/module-mysql/src/common/ReplicatedGTID.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/module-mysql/src/common/ReplicatedGTID.ts b/modules/module-mysql/src/common/ReplicatedGTID.ts index d51d43a7..dc7713e9 100644 --- a/modules/module-mysql/src/common/ReplicatedGTID.ts +++ b/modules/module-mysql/src/common/ReplicatedGTID.ts @@ -92,10 +92,15 @@ export class ReplicatedGTID { * @returns A comparable string in the format * `padded_end_transaction|raw_gtid|binlog_filename|binlog_position` */ - get comparable() { + get comparable(): string { const { raw, position } = this; const [, transactionRanges] = this.raw.split(':'); + // This means no transactions have been executed on the database yet + if (!transactionRanges) { + return ReplicatedGTID.ZERO.comparable; + } + let maxTransactionId = 0; for (const range of transactionRanges.split(',')) { From 6565b978db03123bfd69fbb60df979b9d3d3c211 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 23 Jul 2025 13:56:50 +0200 Subject: [PATCH 43/48] BinLog listener now correctly takes schema into account for replication. TableFilter creation is now internally handled in the BinLog listener Pause/unpause binlog listening now uses the same stop start functionality used for schema change handling. --- .../src/replication/zongji/BinLogListener.ts | 158 ++++++++++++------ 1 file changed, 104 insertions(+), 54 deletions(-) diff --git a/modules/module-mysql/src/replication/zongji/BinLogListener.ts b/modules/module-mysql/src/replication/zongji/BinLogListener.ts index 08be7193..4de30f53 100644 --- a/modules/module-mysql/src/replication/zongji/BinLogListener.ts +++ b/modules/module-mysql/src/replication/zongji/BinLogListener.ts @@ -6,6 +6,7 @@ import { Logger, logger as defaultLogger } from '@powersync/lib-services-framewo import { MySQLConnectionManager } from '../MySQLConnectionManager.js'; import timers from 'timers/promises'; import pkg, { + AST, BaseFrom, DropIndexStatement, Parser as ParserType, @@ -24,13 +25,10 @@ import { isTruncate, matchedSchemaChangeQuery } from '../../utils/parser-utils.js'; +import { TablePattern } from '@powersync/service-sync-rules'; const { Parser } = pkg; -// Maximum time the Zongji listener can be paused before resuming automatically -// MySQL server automatically terminates replication connections after 60 seconds of inactivity -const MAX_PAUSE_TIME_MS = 45_000; - export type Row = Record; /** @@ -39,11 +37,11 @@ export type Row = Record; * are received for them. */ export enum SchemaChangeType { - RENAME_TABLE = 'rename_table', - DROP_TABLE = 'drop_table', - TRUNCATE_TABLE = 'truncate_table', - ALTER_TABLE_COLUMN = 'alter_table_column', - REPLICATION_IDENTITY = 'replication_identity' + RENAME_TABLE = 'Rename Table', + DROP_TABLE = 'Drop Table', + TRUNCATE_TABLE = 'Truncate Table', + ALTER_TABLE_COLUMN = 'Alter Table Column', + REPLICATION_IDENTITY = 'Alter Replication Identity' } export interface SchemaChange { @@ -52,7 +50,11 @@ export interface SchemaChange { * The table that the schema change applies to. */ table: string; - newTable?: string; // Only for table renames + schema: string; + /** + * Populated for table renames if the newTable was matched by the DatabaseFilter + */ + newTable?: string; } export interface BinLogEventHandler { @@ -68,8 +70,7 @@ export interface BinLogEventHandler { export interface BinLogListenerOptions { connectionManager: MySQLConnectionManager; eventHandler: BinLogEventHandler; - // Filter for tables to include in the replication - tableFilter: (tableName: string) => boolean; + sourceTables: TablePattern[]; serverId: number; startPosition: common.BinLogPosition; logger?: Logger; @@ -87,6 +88,7 @@ export class BinLogListener { private currentGTID: common.ReplicatedGTID | null; private logger: Logger; private listenerError: Error | null; + private databaseFilter: { [schema: string]: (table: string) => boolean }; zongji: ZongJi; processingQueue: async.QueueObject; @@ -108,6 +110,7 @@ export class BinLogListener { this.processingQueue = this.createProcessingQueue(); this.zongji = this.createZongjiListener(); this.listenerError = null; + this.databaseFilter = this.createDatabaseFilter(options.sourceTables); } /** @@ -139,7 +142,7 @@ export class BinLogListener { if (error) { reject(error); } else { - this.logger.info('Successfully set up replication connection heartbeat...'); + this.logger.info('Successfully set up replication connection heartbeat.'); resolve(results); } } @@ -158,7 +161,7 @@ export class BinLogListener { // We ignore the unknown/heartbeat event since it currently serves no purpose other than to keep the connection alive // tablemap events always need to be included for the other row events to work includeEvents: ['tablemap', 'writerows', 'updaterows', 'deleterows', 'xid', 'rotate', 'gtidlog', 'query'], - includeSchema: { [this.connectionManager.databaseName]: this.options.tableFilter }, + includeSchema: this.databaseFilter, filename: this.binLogPosition.filename, position: this.binLogPosition.offset, serverId: this.options.serverId @@ -244,11 +247,10 @@ export class BinLogListener { this.logger.info( `BinLog processing queue has reached its memory limit of [${this.connectionManager.options.binlog_queue_memory_limit}MB]. Pausing BinLog Listener.` ); - zongji.pause(); - const resumeTimeoutPromise = timers.setTimeout(MAX_PAUSE_TIME_MS); - await Promise.race([this.processingQueue.drain(), resumeTimeoutPromise]); + await this.stopZongji(); + await this.processingQueue.drain(); this.logger.info(`BinLog processing queue backlog cleared. Resuming BinLog Listener.`); - zongji.resume(); + await this.restartZongji(); } }); @@ -284,14 +286,11 @@ export class BinLogListener { }); this.binLogPosition.offset = evt.nextPosition; await this.eventHandler.onTransactionStart({ timestamp: new Date(evt.timestamp) }); - this.logger.debug( - `Processed GTID event. Next position in BinLog: ${this.binLogPosition.filename}:${this.binLogPosition.offset}` - ); + this.logger.info(`Processed GTID event: ${this.currentGTID.comparable}`); break; case zongji_utils.eventIsRotation(evt): const newFile = this.binLogPosition.filename !== evt.binlogName; this.binLogPosition.filename = evt.binlogName; - this.binLogPosition.offset = evt.position; await this.eventHandler.onRotate(); if (newFile) { @@ -301,9 +300,10 @@ export class BinLogListener { } break; case zongji_utils.eventIsWriteMutation(evt): - await this.eventHandler.onWrite(evt.rows, evt.tableMap[evt.tableId]); + const tableMap = evt.tableMap[evt.tableId]; + await this.eventHandler.onWrite(evt.rows, tableMap); this.logger.info( - `Processed Write event for table:${evt.tableMap[evt.tableId].tableName}. ${evt.rows.length} row(s) inserted.` + `Processed Write event for table [${tableMap.parentSchema}.${tableMap.tableName}]. ${evt.rows.length} row(s) inserted.` ); break; case zongji_utils.eventIsUpdateMutation(evt): @@ -313,13 +313,13 @@ export class BinLogListener { evt.tableMap[evt.tableId] ); this.logger.info( - `Processed Update event for table:${evt.tableMap[evt.tableId].tableName}. ${evt.rows.length} row(s) updated.` + `Processed Update event for table [${evt.tableMap[evt.tableId].tableName}]. ${evt.rows.length} row(s) updated.` ); break; case zongji_utils.eventIsDeleteMutation(evt): await this.eventHandler.onDelete(evt.rows, evt.tableMap[evt.tableId]); this.logger.info( - `Processed Delete event for table:${evt.tableMap[evt.tableId].tableName}. ${evt.rows.length} row(s) deleted.` + `Processed Delete event for table [${evt.tableMap[evt.tableId].tableName}]. ${evt.rows.length} row(s) deleted.` ); break; case zongji_utils.eventIsXid(evt): @@ -329,13 +329,15 @@ export class BinLogListener { position: this.binLogPosition }).comparable; await this.eventHandler.onCommit(LSN); - this.logger.debug(`Processed Xid event - transaction complete. LSN: ${LSN}.`); + this.logger.info(`Processed Xid event - transaction complete. LSN: ${LSN}.`); break; case zongji_utils.eventIsQuery(evt): await this.processQueryEvent(evt); break; } + // Update the binlog position after processing the event + this.binLogPosition.offset = evt.nextPosition; this.queueMemoryUsage -= evt.size; }; } @@ -348,29 +350,19 @@ export class BinLogListener { return; } - let schemaChanges: SchemaChange[] = []; - try { - schemaChanges = this.toSchemaChanges(query); - } catch (error) { - if (matchedSchemaChangeQuery(query, this.options.tableFilter)) { - this.logger.warn( - `Failed to parse query: [${query}]. - Please review for the schema changes and manually redeploy the sync rules if required.` - ); - } - return; - } + const schemaChanges = this.toSchemaChanges(query, event.schema); if (schemaChanges.length > 0) { // Since handling the schema changes can take a long time, we need to stop the Zongji listener instead of pausing it. await this.stopZongji(); for (const change of schemaChanges) { - this.logger.info(`Processing ${change.type} for table:${change.table}...`); + this.logger.info(`Processing schema change ${change.type} for table [${change.schema}.${change.table}]`); await this.eventHandler.onSchemaChange(change); } - // DDL queries are auto commited, and so do not come with a corresponding Xid event. - // This is problematic for DDL queries which result in row events, so we manually commit here. + // DDL queries are auto commited, but do not come with a corresponding Xid event. + // This is problematic for DDL queries which result in row events because the checkpoint is not moved on, + // so we manually commit here. this.binLogPosition.offset = nextPosition; const LSN = new common.ReplicatedGTID({ raw_gtid: this.currentGTID!.raw, @@ -395,12 +387,26 @@ export class BinLogListener { /** * Function that interprets a DDL query for any applicable schema changes. * If the query does not contain any relevant schema changes, an empty array is returned. + * The defaultSchema is derived from the database set on the MySQL Node.js connection client. + * It is used as a fallback when the schema/database cannot be determined from the query DDL. * * @param query + * @param defaultSchema */ - private toSchemaChanges(query: string): SchemaChange[] { - const ast = this.sqlParser.astify(query, { database: 'MySQL' }); - const statements = Array.isArray(ast) ? ast : [ast]; + private toSchemaChanges(query: string, defaultSchema: string): SchemaChange[] { + let statements: AST[] = []; + try { + const ast = this.sqlParser.astify(query, { database: 'MySQL' }); + statements = Array.isArray(ast) ? ast : [ast]; + } catch (error) { + if (matchedSchemaChangeQuery(query, Object.values(this.databaseFilter))) { + this.logger.warn( + `Failed to parse query: [${query}]. + Please review for the schema changes and manually redeploy the sync rules if required.` + ); + } + return []; + } const changes: SchemaChange[] = []; for (const statement of statements) { @@ -408,33 +414,43 @@ export class BinLogListener { const truncateStatement = statement as TruncateStatement; // Truncate statements can apply to multiple tables for (const entity of truncateStatement.name) { - changes.push({ type: SchemaChangeType.TRUNCATE_TABLE, table: entity.table }); + changes.push({ + type: SchemaChangeType.TRUNCATE_TABLE, + table: entity.table, + schema: entity.db ?? defaultSchema + }); } } else if (isDropTable(statement)) { for (const entity of statement.name) { - changes.push({ type: SchemaChangeType.DROP_TABLE, table: entity.table }); + changes.push({ type: SchemaChangeType.DROP_TABLE, table: entity.table, schema: entity.db ?? defaultSchema }); } } else if (isDropIndex(statement)) { const dropStatement = statement as DropIndexStatement; changes.push({ type: SchemaChangeType.REPLICATION_IDENTITY, - table: dropStatement.table.table + table: dropStatement.table.table, + schema: dropStatement.table.db ?? defaultSchema }); } else if (isCreateUniqueIndex(statement)) { // Potential change to the replication identity if the table has no prior unique constraint changes.push({ type: SchemaChangeType.REPLICATION_IDENTITY, // @ts-ignore - The type definitions for node-sql-parser do not reflect the correct structure here - table: statement.table!.table + table: statement.table!.table, + // @ts-ignore + schema: statement.table!.db ?? defaultSchema }); } else if (isRenameTable(statement)) { const renameStatement = statement as RenameStatement; // Rename statements can apply to multiple tables for (const table of renameStatement.table) { + const schema = table[0].db ?? defaultSchema; + const isNewTableIncluded = this.databaseFilter[schema](table[1].table); changes.push({ type: SchemaChangeType.RENAME_TABLE, table: table[0].table, - newTable: table[1].table + newTable: isNewTableIncluded ? table[1].table : undefined, + schema }); } } else if (isAlterTable(statement)) { @@ -444,18 +460,21 @@ export class BinLogListener { changes.push({ type: SchemaChangeType.RENAME_TABLE, table: fromTable.table, - newTable: expression.table + newTable: expression.table, + schema: fromTable.db ?? defaultSchema }); } else if (isColumnExpression(expression)) { changes.push({ type: SchemaChangeType.ALTER_TABLE_COLUMN, - table: fromTable.table + table: fromTable.table, + schema: fromTable.db ?? defaultSchema }); } else if (isConstraintExpression(expression)) { // Potential changes to the replication identity changes.push({ type: SchemaChangeType.REPLICATION_IDENTITY, - table: fromTable.table + table: fromTable.table, + schema: fromTable.db ?? defaultSchema }); } } @@ -464,7 +483,38 @@ export class BinLogListener { // Filter out schema changes that are not relevant to the included tables return changes.filter( (change) => - this.options.tableFilter(change.table) || (change.newTable && this.options.tableFilter(change.newTable)) + this.isTableIncluded(change.table, change.schema) || + (change.newTable && this.isTableIncluded(change.newTable, change.schema)) ); } + + private isTableIncluded(tableName: string, schema: string): boolean { + return this.databaseFilter[schema] && this.databaseFilter[schema](tableName); + } + + private createDatabaseFilter(sourceTables: TablePattern[]): { [schema: string]: (table: string) => boolean } { + // Group sync rule tables by schema + const schemaMap = new Map(); + for (const table of sourceTables) { + if (!schemaMap.has(table.schema)) { + const tables = [table]; + schemaMap.set(table.schema, tables); + } else { + schemaMap.get(table.schema)!.push(table); + } + } + + const databaseFilter: { [schema: string]: (table: string) => boolean } = {}; + for (const entry of schemaMap.entries()) { + const [schema, sourceTables] = entry; + databaseFilter[schema] = (table: string) => + sourceTables.findIndex((sourceTable) => + sourceTable.isWildcard + ? table.startsWith(sourceTable.tablePattern.substring(0, sourceTable.tablePattern.length - 1)) + : table === sourceTable.name + ) !== -1; + } + + return databaseFilter; + } } From 113ae83a88088afcef26cdf5809b9be786090901 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 23 Jul 2025 13:59:35 +0200 Subject: [PATCH 44/48] BinLog stream now correctly honors multiple schemas in the sync rules. --- .../src/replication/BinLogStream.ts | 57 +++++++++---------- 1 file changed, 26 insertions(+), 31 deletions(-) diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index c2296571..16bdad08 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -19,7 +19,7 @@ import mysqlPromise from 'mysql2/promise'; import { TableMapEntry } from '@powersync/mysql-zongji'; import * as common from '../common/common-index.js'; -import { createRandomServerId, escapeMysqlTableName } from '../utils/mysql-utils.js'; +import { createRandomServerId, qualifiedMySQLTable, retriedQuery } from '../utils/mysql-utils.js'; import { MySQLConnectionManager } from './MySQLConnectionManager.js'; import { ReplicationMetric } from '@powersync/service-types'; import { BinLogEventHandler, BinLogListener, Row, SchemaChange, SchemaChangeType } from './zongji/BinLogListener.js'; @@ -61,6 +61,13 @@ function getMysqlRelId(source: MysqlRelId): string { return `${source.schema}.${source.name}`; } +export async function sendKeepAlive(connection: mysqlPromise.Connection) { + await retriedQuery({ connection: connection, query: `XA START 'powersync_keepalive'` }); + await retriedQuery({ connection: connection, query: `XA END 'powersync_keepalive'` }); + await retriedQuery({ connection: connection, query: `XA PREPARE 'powersync_keepalive'` }); + await retriedQuery({ connection: connection, query: `XA COMMIT 'powersync_keepalive'` }); +} + export class BinLogStream { private readonly syncRules: sync_rules.SqlSyncRules; private readonly groupId: number; @@ -192,7 +199,7 @@ export class BinLogStream { let tables: storage.SourceTable[] = []; for (const matchedTable of matchedTables) { - const replicaIdColumns = await this.getReplicaIdColumns(matchedTable); + const replicaIdColumns = await this.getReplicaIdColumns(matchedTable, tablePattern.schema); const table = await this.handleRelation( batch, @@ -300,11 +307,11 @@ export class BinLogStream { batch: storage.BucketStorageBatch, table: storage.SourceTable ) { - this.logger.info(`Replicating ${table.qualifiedName}`); + this.logger.info(`Replicating ${qualifiedMySQLTable(table)}`); // TODO count rows and log progress at certain batch sizes // MAX_EXECUTION_TIME(0) hint disables execution timeout for this query - const query = connection.query(`SELECT /*+ MAX_EXECUTION_TIME(0) */ * FROM ${escapeMysqlTableName(table)}`); + const query = connection.query(`SELECT /*+ MAX_EXECUTION_TIME(0) */ * FROM ${qualifiedMySQLTable(table)}`); const stream = query.stream(); let columns: Map | undefined = undefined; @@ -415,7 +422,7 @@ export class BinLogStream { const binlogEventHandler = this.createBinlogEventHandler(batch); const binlogListener = new BinLogListener({ logger: this.logger, - tableFilter: (table: string) => this.matchesTable(table), + sourceTables: this.syncRules.getSourceTables(), startPosition: binLogPositionState, connectionManager: this.connections, serverId: serverId, @@ -438,18 +445,6 @@ export class BinLogStream { } } - private matchesTable(tableName: string): boolean { - const sourceTables = this.syncRules.getSourceTables(); - - return ( - sourceTables.findIndex((table) => - table.isWildcard - ? tableName.startsWith(table.tablePattern.substring(0, table.tablePattern.length - 1)) - : tableName === table.name - ) !== -1 - ); - } - private createBinlogEventHandler(batch: storage.BucketStorageBatch): BinLogEventHandler { return { onWrite: async (rows: Row[], tableMap: TableMapEntry) => { @@ -499,24 +494,24 @@ export class BinLogStream { private async handleSchemaChange(batch: storage.BucketStorageBatch, change: SchemaChange): Promise { if (change.type === SchemaChangeType.RENAME_TABLE) { - const oldTableId = getMysqlRelId({ - schema: this.connections.databaseName, + const fromTableId = getMysqlRelId({ + schema: change.schema, name: change.table }); - const table = this.tableCache.get(oldTableId); + const fromTable = this.tableCache.get(fromTableId); // Old table needs to be cleaned up - if (table) { - await batch.drop([table]); - this.tableCache.delete(oldTableId); + if (fromTable) { + await batch.drop([fromTable]); + this.tableCache.delete(fromTableId); } - // If the new table matches the sync rules, we need to add it to the cache and snapshot it - if (this.matchesTable(change.newTable!)) { - await this.handleCreateOrUpdateTable(batch, change.newTable!, this.connections.databaseName); + // The new table matched a table in the sync rules + if (change.newTable) { + await this.handleCreateOrUpdateTable(batch, change.newTable!, change.schema); } } else { const tableId = getMysqlRelId({ - schema: this.connections.databaseName, + schema: change.schema, name: change.table }); @@ -526,7 +521,7 @@ export class BinLogStream { case SchemaChangeType.ALTER_TABLE_COLUMN: case SchemaChangeType.REPLICATION_IDENTITY: // For these changes, we need to update the table if the replication identity columns have changed. - await this.handleCreateOrUpdateTable(batch, change.table, this.connections.databaseName); + await this.handleCreateOrUpdateTable(batch, change.table, change.schema); break; case SchemaChangeType.TRUNCATE_TABLE: await batch.truncate([table]); @@ -542,11 +537,11 @@ export class BinLogStream { } } - private async getReplicaIdColumns(tableName: string) { + private async getReplicaIdColumns(tableName: string, schema: string) { const connection = await this.connections.getConnection(); const replicaIdColumns = await common.getReplicationIdentityColumns({ connection, - schema: this.connections.databaseName, + schema, tableName }); connection.release(); @@ -559,7 +554,7 @@ export class BinLogStream { tableName: string, schema: string ): Promise { - const replicaIdColumns = await this.getReplicaIdColumns(tableName); + const replicaIdColumns = await this.getReplicaIdColumns(tableName, schema); return await this.handleRelation( batch, { From 805b609d79a5d11594414892592676eff86d14b7 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 23 Jul 2025 14:01:18 +0200 Subject: [PATCH 45/48] Added tests for multi schema support --- modules/module-mysql/src/utils/mysql-utils.ts | 13 +- .../module-mysql/src/utils/parser-utils.ts | 10 +- .../test/src/BinLogListener.test.ts | 277 +++++++++++++----- .../test/src/BinLogStream.test.ts | 52 +++- .../test/src/parser-utils.test.ts | 14 +- .../test/src/schema-changes.test.ts | 74 ++++- modules/module-mysql/test/src/util.ts | 6 + 7 files changed, 363 insertions(+), 83 deletions(-) diff --git a/modules/module-mysql/src/utils/mysql-utils.ts b/modules/module-mysql/src/utils/mysql-utils.ts index 1ccf8e22..5c6149a3 100644 --- a/modules/module-mysql/src/utils/mysql-utils.ts +++ b/modules/module-mysql/src/utils/mysql-utils.ts @@ -92,6 +92,15 @@ export function satisfiesVersion(version: string, targetVersion: string): boolea return satisfies(coercedVersion!, targetVersion!, { loose: true }); } -export function escapeMysqlTableName(table: SourceTable): string { - return `\`${table.schema.replaceAll('`', '``')}\`.\`${table.name.replaceAll('`', '``')}\``; +export function qualifiedMySQLTable(table: SourceEntityDescriptor): string; +export function qualifiedMySQLTable(table: string, schema: string): string; + +export function qualifiedMySQLTable(table: SourceEntityDescriptor | string, schema?: string): string { + if (typeof table === 'object') { + return `\`${table.schema.replaceAll('`', '``')}\`.\`${table.name.replaceAll('`', '``')}\``; + } else if (schema) { + return `\`${schema.replaceAll('`', '``')}\`.\`${table.replaceAll('`', '``')}\``; + } else { + return `\`${table.replaceAll('`', '``')}\``; + } } diff --git a/modules/module-mysql/src/utils/parser-utils.ts b/modules/module-mysql/src/utils/parser-utils.ts index aa84be09..d3647dcd 100644 --- a/modules/module-mysql/src/utils/parser-utils.ts +++ b/modules/module-mysql/src/utils/parser-utils.ts @@ -5,11 +5,11 @@ import { Alter, AST, Create, Drop, TruncateStatement, RenameStatement, DropIndex const DDL_KEYWORDS = ['alter table', 'drop table', 'truncate table', 'rename table']; /** - * Check if a query is a DDL statement that applies to tables matching the provided matcher function. + * Check if a query is a DDL statement that applies to tables matching any of the provided matcher functions. * @param query - * @param matcher + * @param matchers */ -export function matchedSchemaChangeQuery(query: string, matcher: (tableName: string) => boolean): boolean { +export function matchedSchemaChangeQuery(query: string, matchers: ((table: string) => boolean)[]) { // Normalize case and remove backticks for matching const normalizedQuery = query.toLowerCase().replace(/`/g, ''); @@ -17,8 +17,8 @@ export function matchedSchemaChangeQuery(query: string, matcher: (tableName: str if (isDDLQuery) { const tokens = normalizedQuery.split(/[^a-zA-Z0-9_`]+/); // Check if any matched table names appear in the query - for (const token of tokens!) { - const matchFound = matcher(token); + for (const token of tokens) { + const matchFound = matchers.some((matcher) => matcher(token)); if (matchFound) { return true; } diff --git a/modules/module-mysql/test/src/BinLogListener.test.ts b/modules/module-mysql/test/src/BinLogListener.test.ts index 63430f5f..c7447dd7 100644 --- a/modules/module-mysql/test/src/BinLogListener.test.ts +++ b/modules/module-mysql/test/src/BinLogListener.test.ts @@ -7,12 +7,18 @@ import { SchemaChangeType } from '@module/replication/zongji/BinLogListener.js'; import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManager.js'; -import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js'; +import { clearTestDb, createTestDb, TEST_CONNECTION_OPTIONS } from './util.js'; import { v4 as uuid } from 'uuid'; import * as common from '@module/common/common-index.js'; -import { createRandomServerId, getMySQLVersion, satisfiesVersion } from '@module/utils/mysql-utils.js'; +import { + createRandomServerId, + getMySQLVersion, + qualifiedMySQLTable, + satisfiesVersion +} from '@module/utils/mysql-utils.js'; import { TableMapEntry } from '@powersync/mysql-zongji'; import crypto from 'crypto'; +import { TablePattern } from '@powersync/service-sync-rules'; describe('BinlogListener tests', () => { const MAX_QUEUE_CAPACITY_MB = 1; @@ -37,18 +43,10 @@ describe('BinlogListener tests', () => { beforeEach(async () => { const connection = await connectionManager.getConnection(); await clearTestDb(connection); - await connection.query(`CREATE TABLE test_DATA (id CHAR(36) PRIMARY KEY, description MEDIUMTEXT)`); + await connectionManager.query(`CREATE TABLE test_DATA (id CHAR(36) PRIMARY KEY, description MEDIUMTEXT)`); connection.release(); - const fromGTID = await getFromGTID(connectionManager); - eventHandler = new TestBinLogEventHandler(); - binLogListener = new BinLogListener({ - connectionManager: connectionManager, - eventHandler: eventHandler, - startPosition: fromGTID.position, - tableFilter: (table) => ['test_DATA'].includes(table), - serverId: createRandomServerId(1) - }); + binLogListener = await createBinlogListener(); }); afterAll(async () => { @@ -66,9 +64,8 @@ describe('BinlogListener tests', () => { expect(queueStopSpy).toHaveBeenCalled(); }); - test('Zongji listener is paused when processing queue reaches maximum memory size', async () => { - const pauseSpy = vi.spyOn(binLogListener.zongji, 'pause'); - const resumeSpy = vi.spyOn(binLogListener.zongji, 'resume'); + test('Zongji listener is stopped when processing queue reaches maximum memory size', async () => { + const stopSpy = vi.spyOn(binLogListener.zongji, 'stop'); // Pause the event handler to force a backlog on the processing queue eventHandler.pause(); @@ -78,17 +75,18 @@ describe('BinlogListener tests', () => { await binLogListener.start(); - // Wait for listener to pause due to queue reaching capacity - await vi.waitFor(() => expect(pauseSpy).toHaveBeenCalled(), { timeout: 5000 }); + // Wait for listener to stop due to queue reaching capacity + await vi.waitFor(() => expect(stopSpy).toHaveBeenCalled(), { timeout: 5000 }); expect(binLogListener.isQueueOverCapacity()).toBeTruthy(); // Resume event processing eventHandler.unpause!(); + const restartSpy = vi.spyOn(binLogListener, 'start'); await vi.waitFor(() => expect(eventHandler.rowsWritten).equals(ROW_COUNT), { timeout: 5000 }); await binLogListener.stop(); // Confirm resume was called after unpausing - expect(resumeSpy).toHaveBeenCalled(); + expect(restartSpy).toHaveBeenCalled(); }); test('Row events: Write, update, delete', async () => { @@ -113,9 +111,13 @@ describe('BinlogListener tests', () => { await connectionManager.query(`ALTER TABLE test_DATA RENAME test_DATA_new`); await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 }); await binLogListener.stop(); - expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.RENAME_TABLE); - expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); - expect(eventHandler.schemaChanges[0].newTable).toEqual('test_DATA_new'); + assertSchemaChange( + eventHandler.schemaChanges[0], + SchemaChangeType.RENAME_TABLE, + connectionManager.databaseName, + 'test_DATA', + 'test_DATA_new' + ); }); test('Schema change event: Rename multiple tables', async () => { @@ -128,13 +130,22 @@ describe('BinlogListener tests', () => { `); await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(2), { timeout: 5000 }); await binLogListener.stop(); - expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.RENAME_TABLE); - expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); - expect(eventHandler.schemaChanges[0].newTable).toEqual('test_DATA_new'); - - expect(eventHandler.schemaChanges[1].type).toBe(SchemaChangeType.RENAME_TABLE); - expect(eventHandler.schemaChanges[1].table).toEqual('test_DATA_new'); - expect(eventHandler.schemaChanges[1].newTable).toEqual('test_DATA'); + assertSchemaChange( + eventHandler.schemaChanges[0], + SchemaChangeType.RENAME_TABLE, + connectionManager.databaseName, + 'test_DATA' + ); + // New table name is undefined since the renamed table is not included by the database filter + expect(eventHandler.schemaChanges[0].newTable).toBeUndefined(); + + assertSchemaChange( + eventHandler.schemaChanges[1], + SchemaChangeType.RENAME_TABLE, + connectionManager.databaseName, + 'test_DATA_new', + 'test_DATA' + ); }); test('Schema change event: Truncate table', async () => { @@ -142,8 +153,12 @@ describe('BinlogListener tests', () => { await connectionManager.query(`TRUNCATE TABLE test_DATA`); await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 }); await binLogListener.stop(); - expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.TRUNCATE_TABLE); - expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); + assertSchemaChange( + eventHandler.schemaChanges[0], + SchemaChangeType.TRUNCATE_TABLE, + connectionManager.databaseName, + 'test_DATA' + ); }); test('Schema change event: Drop table', async () => { @@ -152,8 +167,12 @@ describe('BinlogListener tests', () => { await connectionManager.query(`CREATE TABLE test_DATA (id CHAR(36) PRIMARY KEY, description MEDIUMTEXT)`); await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 }); await binLogListener.stop(); - expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.DROP_TABLE); - expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); + assertSchemaChange( + eventHandler.schemaChanges[0], + SchemaChangeType.DROP_TABLE, + connectionManager.databaseName, + 'test_DATA' + ); }); test('Schema change event: Drop column', async () => { @@ -161,8 +180,12 @@ describe('BinlogListener tests', () => { await connectionManager.query(`ALTER TABLE test_DATA DROP COLUMN description`); await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 }); await binLogListener.stop(); - expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.ALTER_TABLE_COLUMN); - expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); + assertSchemaChange( + eventHandler.schemaChanges[0], + SchemaChangeType.ALTER_TABLE_COLUMN, + connectionManager.databaseName, + 'test_DATA' + ); }); test('Schema change event: Add column', async () => { @@ -170,8 +193,12 @@ describe('BinlogListener tests', () => { await connectionManager.query(`ALTER TABLE test_DATA ADD COLUMN new_column VARCHAR(255)`); await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 }); await binLogListener.stop(); - expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.ALTER_TABLE_COLUMN); - expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); + assertSchemaChange( + eventHandler.schemaChanges[0], + SchemaChangeType.ALTER_TABLE_COLUMN, + connectionManager.databaseName, + 'test_DATA' + ); }); test('Schema change event: Modify column', async () => { @@ -179,8 +206,12 @@ describe('BinlogListener tests', () => { await connectionManager.query(`ALTER TABLE test_DATA MODIFY COLUMN description TEXT`); await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 }); await binLogListener.stop(); - expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.ALTER_TABLE_COLUMN); - expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); + assertSchemaChange( + eventHandler.schemaChanges[0], + SchemaChangeType.ALTER_TABLE_COLUMN, + connectionManager.databaseName, + 'test_DATA' + ); }); test('Schema change event: Rename column via change statement', async () => { @@ -188,8 +219,12 @@ describe('BinlogListener tests', () => { await connectionManager.query(`ALTER TABLE test_DATA CHANGE COLUMN description description_new MEDIUMTEXT`); await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 }); await binLogListener.stop(); - expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.ALTER_TABLE_COLUMN); - expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); + assertSchemaChange( + eventHandler.schemaChanges[0], + SchemaChangeType.ALTER_TABLE_COLUMN, + connectionManager.databaseName, + 'test_DATA' + ); }); test('Schema change event: Rename column via rename statement', async () => { @@ -199,8 +234,12 @@ describe('BinlogListener tests', () => { await connectionManager.query(`ALTER TABLE test_DATA RENAME COLUMN description TO description_new`); await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 }); await binLogListener.stop(); - expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.ALTER_TABLE_COLUMN); - expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); + assertSchemaChange( + eventHandler.schemaChanges[0], + SchemaChangeType.ALTER_TABLE_COLUMN, + connectionManager.databaseName, + 'test_DATA' + ); } }); @@ -212,62 +251,101 @@ describe('BinlogListener tests', () => { ); await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(3), { timeout: 5000 }); await binLogListener.stop(); - expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.ALTER_TABLE_COLUMN); - expect(eventHandler.schemaChanges[0].table).toEqual('test_DATA'); + assertSchemaChange( + eventHandler.schemaChanges[0], + SchemaChangeType.ALTER_TABLE_COLUMN, + connectionManager.databaseName, + 'test_DATA' + ); - expect(eventHandler.schemaChanges[1].type).toBe(SchemaChangeType.ALTER_TABLE_COLUMN); - expect(eventHandler.schemaChanges[1].table).toEqual('test_DATA'); + assertSchemaChange( + eventHandler.schemaChanges[1], + SchemaChangeType.ALTER_TABLE_COLUMN, + connectionManager.databaseName, + 'test_DATA' + ); - expect(eventHandler.schemaChanges[2].type).toBe(SchemaChangeType.ALTER_TABLE_COLUMN); - expect(eventHandler.schemaChanges[2].table).toEqual('test_DATA'); + assertSchemaChange( + eventHandler.schemaChanges[2], + SchemaChangeType.ALTER_TABLE_COLUMN, + connectionManager.databaseName, + 'test_DATA' + ); }); test('Schema change event: Drop and Add primary key', async () => { await connectionManager.query(`CREATE TABLE test_constraints (id CHAR(36), description VARCHAR(100))`); - binLogListener.options.tableFilter = (table) => table === 'test_constraints'; + const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_constraints')]; + binLogListener = await createBinlogListener(sourceTables); await binLogListener.start(); await connectionManager.query(`ALTER TABLE test_constraints ADD PRIMARY KEY (id)`); await connectionManager.query(`ALTER TABLE test_constraints DROP PRIMARY KEY`); await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(2), { timeout: 5000 }); await binLogListener.stop(); // Event for the add - expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.REPLICATION_IDENTITY); - expect(eventHandler.schemaChanges[0].table).toEqual('test_constraints'); + assertSchemaChange( + eventHandler.schemaChanges[0], + SchemaChangeType.REPLICATION_IDENTITY, + connectionManager.databaseName, + 'test_constraints' + ); // Event for the drop - expect(eventHandler.schemaChanges[1].type).toBe(SchemaChangeType.REPLICATION_IDENTITY); - expect(eventHandler.schemaChanges[1].table).toEqual('test_constraints'); + assertSchemaChange( + eventHandler.schemaChanges[1], + SchemaChangeType.REPLICATION_IDENTITY, + connectionManager.databaseName, + 'test_constraints' + ); }); test('Schema change event: Add and drop unique constraint', async () => { await connectionManager.query(`CREATE TABLE test_constraints (id CHAR(36), description VARCHAR(100))`); - binLogListener.options.tableFilter = (table) => table === 'test_constraints'; + const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_constraints')]; + binLogListener = await createBinlogListener(sourceTables); await binLogListener.start(); await connectionManager.query(`ALTER TABLE test_constraints ADD UNIQUE (description)`); await connectionManager.query(`ALTER TABLE test_constraints DROP INDEX description`); await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(2), { timeout: 5000 }); await binLogListener.stop(); // Event for the creation - expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.REPLICATION_IDENTITY); - expect(eventHandler.schemaChanges[0].table).toEqual('test_constraints'); + assertSchemaChange( + eventHandler.schemaChanges[0], + SchemaChangeType.REPLICATION_IDENTITY, + connectionManager.databaseName, + 'test_constraints' + ); // Event for the drop - expect(eventHandler.schemaChanges[1].type).toBe(SchemaChangeType.REPLICATION_IDENTITY); - expect(eventHandler.schemaChanges[1].table).toEqual('test_constraints'); + assertSchemaChange( + eventHandler.schemaChanges[1], + SchemaChangeType.REPLICATION_IDENTITY, + connectionManager.databaseName, + 'test_constraints' + ); }); test('Schema change event: Add and drop a unique index', async () => { await connectionManager.query(`CREATE TABLE test_constraints (id CHAR(36), description VARCHAR(100))`); - binLogListener.options.tableFilter = (table) => table === 'test_constraints'; + const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_constraints')]; + binLogListener = await createBinlogListener(sourceTables); await binLogListener.start(); await connectionManager.query(`CREATE UNIQUE INDEX description_idx ON test_constraints (description)`); await connectionManager.query(`DROP INDEX description_idx ON test_constraints`); await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(2), { timeout: 5000 }); await binLogListener.stop(); // Event for the creation - expect(eventHandler.schemaChanges[0].type).toBe(SchemaChangeType.REPLICATION_IDENTITY); - expect(eventHandler.schemaChanges[0].table).toEqual('test_constraints'); + assertSchemaChange( + eventHandler.schemaChanges[0], + SchemaChangeType.REPLICATION_IDENTITY, + connectionManager.databaseName, + 'test_constraints' + ); // Event for the drop - expect(eventHandler.schemaChanges[1].type).toBe(SchemaChangeType.REPLICATION_IDENTITY); - expect(eventHandler.schemaChanges[1].table).toEqual('test_constraints'); + assertSchemaChange( + eventHandler.schemaChanges[1], + SchemaChangeType.REPLICATION_IDENTITY, + connectionManager.databaseName, + 'test_constraints' + ); }); test('Schema changes for non-matching tables are ignored', async () => { @@ -288,13 +366,15 @@ describe('BinlogListener tests', () => { test('Sequential schema change handling', async () => { // If there are multiple schema changes in the binlog processing queue, we only restart the binlog listener once // all the schema changes have been processed + const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_multiple')]; + binLogListener = await createBinlogListener(sourceTables); + await connectionManager.query(`CREATE TABLE test_multiple (id CHAR(36), description VARCHAR(100))`); await connectionManager.query(`ALTER TABLE test_multiple ADD COLUMN new_column VARCHAR(10)`); await connectionManager.query(`ALTER TABLE test_multiple ADD PRIMARY KEY (id)`); await connectionManager.query(`ALTER TABLE test_multiple MODIFY COLUMN new_column TEXT`); await connectionManager.query(`DROP TABLE test_multiple`); - binLogListener.options.tableFilter = (table) => table === 'test_multiple'; await binLogListener.start(); await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(4), { timeout: 5000 }); @@ -307,11 +387,13 @@ describe('BinlogListener tests', () => { test('Unprocessed binlog event received that does match the current table schema', async () => { // If we process a binlog event for a table which has since had its schema changed, we expect the binlog listener to stop with an error + const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_failure')]; + binLogListener = await createBinlogListener(sourceTables); + await connectionManager.query(`CREATE TABLE test_failure (id CHAR(36), description VARCHAR(100))`); await connectionManager.query(`INSERT INTO test_failure(id, description) VALUES('${uuid()}','test_failure')`); await connectionManager.query(`ALTER TABLE test_failure DROP COLUMN description`); - binLogListener.options.tableFilter = (table) => table === 'test_failure'; await binLogListener.start(); await expect(() => binLogListener.replicateUntilStopped()).rejects.toThrow( @@ -320,16 +402,79 @@ describe('BinlogListener tests', () => { }); test('Unprocessed binlog event received for a dropped table', async () => { + const sourceTables = [new TablePattern(connectionManager.databaseName, 'test_failure')]; + binLogListener = await createBinlogListener(sourceTables); + // If we process a binlog event for a table which has since been dropped, we expect the binlog listener to stop with an error await connectionManager.query(`CREATE TABLE test_failure (id CHAR(36), description VARCHAR(100))`); await connectionManager.query(`INSERT INTO test_failure(id, description) VALUES('${uuid()}','test_failure')`); await connectionManager.query(`DROP TABLE test_failure`); - binLogListener.options.tableFilter = (table) => table === 'test_failure'; await binLogListener.start(); await expect(() => binLogListener.replicateUntilStopped()).rejects.toThrow(/or the table has been dropped/); }); + + test('Multi database events', async () => { + await createTestDb(connectionManager, 'multi_schema'); + const testTable = qualifiedMySQLTable('test_DATA_multi', 'multi_schema'); + await connectionManager.query(`CREATE TABLE ${testTable} (id CHAR(36) PRIMARY KEY,description TEXT);`); + + const sourceTables = [ + new TablePattern(connectionManager.databaseName, 'test_DATA'), + new TablePattern('multi_schema', 'test_DATA_multi') + ]; + binLogListener = await createBinlogListener(sourceTables); + await binLogListener.start(); + + // Default database insert into test_DATA + await insertRows(connectionManager, 1); + // multi_schema database insert into test_DATA_multi + await connectionManager.query(`INSERT INTO ${testTable}(id, description) VALUES('${uuid()}','test')`); + await connectionManager.query(`DROP TABLE ${testTable}`); + + await vi.waitFor(() => expect(eventHandler.schemaChanges.length).toBe(1), { timeout: 5000 }); + await binLogListener.stop(); + expect(eventHandler.rowsWritten).toBe(2); + assertSchemaChange(eventHandler.schemaChanges[0], SchemaChangeType.DROP_TABLE, 'multi_schema', 'test_DATA_multi'); + }); + + async function createBinlogListener( + sourceTables?: TablePattern[], + startPosition?: common.BinLogPosition + ): Promise { + if (!sourceTables) { + sourceTables = [new TablePattern(connectionManager.databaseName, 'test_DATA')]; + } + + if (!startPosition) { + const fromGTID = await getFromGTID(connectionManager); + startPosition = fromGTID.position; + } + + return new BinLogListener({ + connectionManager: connectionManager, + eventHandler: eventHandler, + startPosition: startPosition, + sourceTables: sourceTables, + serverId: createRandomServerId(1) + }); + } + + function assertSchemaChange( + change: SchemaChange, + type: SchemaChangeType, + schema: string, + table: string, + newTable?: string + ) { + expect(change.type).toBe(type); + expect(change.schema).toBe(schema); + expect(change.table).toEqual(table); + if (newTable) { + expect(change.newTable).toEqual(newTable); + } + } }); async function getFromGTID(connectionManager: MySQLConnectionManager) { diff --git a/modules/module-mysql/test/src/BinLogStream.test.ts b/modules/module-mysql/test/src/BinLogStream.test.ts index 3b871969..5d35428b 100644 --- a/modules/module-mysql/test/src/BinLogStream.test.ts +++ b/modules/module-mysql/test/src/BinLogStream.test.ts @@ -4,7 +4,8 @@ import { ReplicationMetric } from '@powersync/service-types'; import { v4 as uuid } from 'uuid'; import { describe, expect, test } from 'vitest'; import { BinlogStreamTestContext } from './BinlogStreamUtils.js'; -import { describeWithStorage } from './util.js'; +import { createTestDb, describeWithStorage } from './util.js'; +import { qualifiedMySQLTable } from '@module/utils/mysql-utils.js'; const BASIC_SYNC_RULES = ` bucket_definitions: @@ -48,6 +49,55 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { expect(endTxCount - startTxCount).toEqual(1); }); + test('Replicate multi schema sync rules', async () => { + await using context = await BinlogStreamTestContext.open(factory); + const { connectionManager } = context; + await context.updateSyncRules(` + bucket_definitions: + default_schema_test_data: + data: + - SELECT id, description, num FROM "${connectionManager.databaseName}"."test_data" + multi_schema_test_data: + data: + - SELECT id, description, num FROM "multi_schema"."test_data" + `); + + await createTestDb(connectionManager, 'multi_schema'); + + await connectionManager.query(`CREATE TABLE test_data (id CHAR(36) PRIMARY KEY, description TEXT, num BIGINT)`); + const testTable = qualifiedMySQLTable('test_data', 'multi_schema'); + await connectionManager.query( + `CREATE TABLE IF NOT EXISTS ${testTable} (id CHAR(36) PRIMARY KEY,description TEXT);` + ); + await context.replicateSnapshot(); + + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; + + await context.startStreaming(); + + const testId = uuid(); + await connectionManager.query( + `INSERT INTO test_data(id, description, num) VALUES('${testId}', 'test1', 1152921504606846976)` + ); + + const testId2 = uuid(); + await connectionManager.query(`INSERT INTO ${testTable}(id, description) VALUES('${testId2}', 'test2')`); + + const default_data = await context.getBucketData('default_schema_test_data[]'); + expect(default_data).toMatchObject([ + putOp('test_data', { id: testId, description: 'test1', num: 1152921504606846976n }) + ]); + + const multi_schema_data = await context.getBucketData('multi_schema_test_data[]'); + expect(multi_schema_data).toMatchObject([putOp('test_data', { id: testId2, description: 'test2' })]); + + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; + expect(endRowCount - startRowCount).toEqual(2); + expect(endTxCount - startTxCount).toEqual(2); + }); + test('Replicate case sensitive table', async () => { // MySQL inherits the case sensitivity of the underlying OS filesystem. // So Unix-based systems will have case-sensitive tables, but Windows won't. diff --git a/modules/module-mysql/test/src/parser-utils.test.ts b/modules/module-mysql/test/src/parser-utils.test.ts index 5d903ee5..802efc04 100644 --- a/modules/module-mysql/test/src/parser-utils.test.ts +++ b/modules/module-mysql/test/src/parser-utils.test.ts @@ -6,19 +6,19 @@ describe('MySQL Parser Util Tests', () => { const matcher = (tableName: string) => tableName === 'users'; // DDL matches and table name matches - expect(matchedSchemaChangeQuery('ALTER TABLE users ADD COLUMN name VARCHAR(255)', matcher)).toBeTruthy(); - expect(matchedSchemaChangeQuery('DROP TABLE users', matcher)).toBeTruthy(); - expect(matchedSchemaChangeQuery('TRUNCATE TABLE users', matcher)).toBeTruthy(); - expect(matchedSchemaChangeQuery('RENAME TABLE new_users TO users', matcher)).toBeTruthy(); + expect(matchedSchemaChangeQuery('ALTER TABLE users ADD COLUMN name VARCHAR(255)', [matcher])).toBeTruthy(); + expect(matchedSchemaChangeQuery('DROP TABLE users', [matcher])).toBeTruthy(); + expect(matchedSchemaChangeQuery('TRUNCATE TABLE users', [matcher])).toBeTruthy(); + expect(matchedSchemaChangeQuery('RENAME TABLE new_users TO users', [matcher])).toBeTruthy(); // Can handle backticks in table names expect( - matchedSchemaChangeQuery('ALTER TABLE `clientSchema`.`users` ADD COLUMN name VARCHAR(255)', matcher) + matchedSchemaChangeQuery('ALTER TABLE `clientSchema`.`users` ADD COLUMN name VARCHAR(255)', [matcher]) ).toBeTruthy(); // DDL matches, but table name does not match - expect(matchedSchemaChangeQuery('DROP TABLE clientSchema.clients', matcher)).toBeFalsy(); + expect(matchedSchemaChangeQuery('DROP TABLE clientSchema.clients', [matcher])).toBeFalsy(); // No DDL match - expect(matchedSchemaChangeQuery('SELECT * FROM users', matcher)).toBeFalsy(); + expect(matchedSchemaChangeQuery('SELECT * FROM users', [matcher])).toBeFalsy(); }); }); diff --git a/modules/module-mysql/test/src/schema-changes.test.ts b/modules/module-mysql/test/src/schema-changes.test.ts index af06e9a2..511ca902 100644 --- a/modules/module-mysql/test/src/schema-changes.test.ts +++ b/modules/module-mysql/test/src/schema-changes.test.ts @@ -2,11 +2,11 @@ import { compareIds, putOp, removeOp, test_utils } from '@powersync/service-core import { beforeAll, describe, expect, test } from 'vitest'; import { storage } from '@powersync/service-core'; -import { describeWithStorage, TEST_CONNECTION_OPTIONS } from './util.js'; +import { createTestDb, describeWithStorage, TEST_CONNECTION_OPTIONS } from './util.js'; import { BinlogStreamTestContext } from './BinlogStreamUtils.js'; import timers from 'timers/promises'; import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManager.js'; -import { getMySQLVersion, satisfiesVersion } from '@module/utils/mysql-utils.js'; +import { getMySQLVersion, qualifiedMySQLTable, satisfiesVersion } from '@module/utils/mysql-utils.js'; describe('MySQL Schema Changes', () => { describeWithStorage({ timeout: 20_000 }, defineTests); @@ -119,6 +119,7 @@ function defineTests(factory: storage.TestStorageFactory) { // Add table after initial replication await connectionManager.query(`CREATE TABLE test_data SELECT * FROM test_data_from`); + const data = await context.getBucketData('global[]'); // Interestingly, the create with select triggers binlog row write events @@ -590,4 +591,73 @@ function defineTests(factory: storage.TestStorageFactory) { REMOVE_T2 ]); }); + + test('Schema changes for tables in other schemas in the sync rules', async () => { + await using context = await BinlogStreamTestContext.open(factory); + // Technically not a schema change, but fits here. + await context.updateSyncRules(` + bucket_definitions: + multi_schema_test_data: + data: + - SELECT id, description, num FROM "multi_schema"."test_data" + `); + + const { connectionManager } = context; + await createTestDb(connectionManager, 'multi_schema'); + const testTable = qualifiedMySQLTable('test_data', 'multi_schema'); + await connectionManager.query( + `CREATE TABLE IF NOT EXISTS ${testTable} (id CHAR(36) PRIMARY KEY,description TEXT);` + ); + await connectionManager.query(`INSERT INTO ${testTable}(id, description) VALUES('t1','test1')`); + await connectionManager.query(`INSERT INTO ${testTable}(id, description) VALUES('t2','test2')`); + + await context.replicateSnapshot(); + await context.startStreaming(); + + await connectionManager.query(`DROP TABLE ${testTable}`); + + const data = await context.getBucketData('multi_schema_test_data[]'); + + expect(data.slice(0, 2)).toMatchObject([ + // Initial inserts + PUT_T1, + PUT_T2 + ]); + + expect(data.slice(2).sort(compareIds)).toMatchObject([ + // Drop + REMOVE_T1, + REMOVE_T2 + ]); + }); + + test('Changes for tables in schemas not in the sync rules are ignored', async () => { + await using context = await BinlogStreamTestContext.open(factory); + await context.updateSyncRules(BASIC_SYNC_RULES); + + const { connectionManager } = context; + await connectionManager.query(`CREATE TABLE test_data (id CHAR(36), description CHAR(100))`); + + await createTestDb(connectionManager, 'multi_schema'); + const testTable = qualifiedMySQLTable('test_data_ignored', 'multi_schema'); + await connectionManager.query( + `CREATE TABLE IF NOT EXISTS ${testTable} (id CHAR(36) PRIMARY KEY,description TEXT);` + ); + await connectionManager.query(`INSERT INTO ${testTable}(id, description) VALUES('t1','test1')`); + await connectionManager.query(`INSERT INTO ${testTable}(id, description) VALUES('t2','test2')`); + + await context.replicateSnapshot(); + await context.startStreaming(); + + await connectionManager.query(`INSERT INTO ${testTable}(id, description) VALUES('t3','test3')`); + await connectionManager.query(`DROP TABLE ${testTable}`); + + // Force a commit on the watched schema to advance the checkpoint + await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('t1','test1')`); + + const data = await context.getBucketData('global[]'); + + // Should only include the entry used to advance the checkpoint + expect(data).toMatchObject([PUT_T1]); + }); } diff --git a/modules/module-mysql/test/src/util.ts b/modules/module-mysql/test/src/util.ts index 9126f744..8f1bab67 100644 --- a/modules/module-mysql/test/src/util.ts +++ b/modules/module-mysql/test/src/util.ts @@ -6,6 +6,7 @@ import mysqlPromise from 'mysql2/promise'; import { env } from './env.js'; import { describe, TestOptions } from 'vitest'; import { TestStorageFactory } from '@powersync/service-core'; +import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManager.js'; export const TEST_URI = env.MYSQL_TEST_URI; @@ -52,3 +53,8 @@ export async function clearTestDb(connection: mysqlPromise.Connection) { } } } + +export async function createTestDb(connectionManager: MySQLConnectionManager, dbName: string) { + await connectionManager.query(`DROP DATABASE IF EXISTS ${dbName}`); + await connectionManager.query(`CREATE DATABASE IF NOT EXISTS ${dbName}`); +} From 6fb5be36e8207c56a17fc49b5b052985df83fd07 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 23 Jul 2025 15:02:20 +0200 Subject: [PATCH 46/48] MySQL util fix post merge --- modules/module-mysql/src/utils/mysql-utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/module-mysql/src/utils/mysql-utils.ts b/modules/module-mysql/src/utils/mysql-utils.ts index 5c6149a3..623e8973 100644 --- a/modules/module-mysql/src/utils/mysql-utils.ts +++ b/modules/module-mysql/src/utils/mysql-utils.ts @@ -3,7 +3,7 @@ import mysql from 'mysql2'; import mysqlPromise from 'mysql2/promise'; import * as types from '../types/types.js'; import { coerce, gte, satisfies } from 'semver'; -import { SourceTable } from '@powersync/service-core'; +import { SourceEntityDescriptor } from '@powersync/service-core'; export type RetriedQueryOptions = { connection: mysqlPromise.Connection; From 541235b7fb0606b6308e68c4b5be6bbda7319ccb Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 23 Jul 2025 16:32:14 +0200 Subject: [PATCH 47/48] Removed accidentally commited keepalive code in BinLogStream. --- modules/module-mysql/src/replication/BinLogStream.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index 1f20745b..e651143a 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -19,7 +19,7 @@ import mysqlPromise from 'mysql2/promise'; import { TableMapEntry } from '@powersync/mysql-zongji'; import * as common from '../common/common-index.js'; -import { createRandomServerId, qualifiedMySQLTable, retriedQuery } from '../utils/mysql-utils.js'; +import { createRandomServerId, qualifiedMySQLTable } from '../utils/mysql-utils.js'; import { MySQLConnectionManager } from './MySQLConnectionManager.js'; import { ReplicationMetric } from '@powersync/service-types'; import { BinLogEventHandler, BinLogListener, Row, SchemaChange, SchemaChangeType } from './zongji/BinLogListener.js'; @@ -61,13 +61,6 @@ function getMysqlRelId(source: MysqlRelId): string { return `${source.schema}.${source.name}`; } -export async function sendKeepAlive(connection: mysqlPromise.Connection) { - await retriedQuery({ connection: connection, query: `XA START 'powersync_keepalive'` }); - await retriedQuery({ connection: connection, query: `XA END 'powersync_keepalive'` }); - await retriedQuery({ connection: connection, query: `XA PREPARE 'powersync_keepalive'` }); - await retriedQuery({ connection: connection, query: `XA COMMIT 'powersync_keepalive'` }); -} - export class BinLogStream { private readonly syncRules: sync_rules.SqlSyncRules; private readonly groupId: number; From 9c1d34be7a1c472257b13f0379ab38b1630fcbbe Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Thu, 24 Jul 2025 12:26:24 +0200 Subject: [PATCH 48/48] Cleaned up Binlog docs and comments a bit --- .../src/replication/BinLogStream.ts | 63 +++++++------------ .../src/replication/zongji/BinLogListener.ts | 4 +- 2 files changed, 25 insertions(+), 42 deletions(-) diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index e651143a..d6bc18f1 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -32,11 +32,6 @@ export interface BinLogStreamOptions { logger?: Logger; } -interface MysqlRelId { - schema: string; - name: string; -} - interface WriteChangePayload { type: storage.SaveOperationTag; row: Row; @@ -54,11 +49,14 @@ export class BinlogConfigurationError extends Error { } /** - * MySQL does not have same relation structure. Just returning unique key as string. - * @param source + * Unlike Postgres' relation id, MySQL's tableId is only guaranteed to be unique and stay the same + * in the context of a single replication session. + * Instead, we create a unique key by combining the source schema and table name + * @param schema + * @param tableName */ -function getMysqlRelId(source: MysqlRelId): string { - return `${source.schema}.${source.name}`; +function createTableId(schema: string, tableName: string): string { + return `${schema}.${tableName}`; } export class BinLogStream { @@ -69,11 +67,11 @@ export class BinLogStream { private readonly connections: MySQLConnectionManager; - private abortSignal: AbortSignal; + private readonly abortSignal: AbortSignal; - private tableCache = new Map(); + private readonly logger: Logger; - private logger: Logger; + private tableCache = new Map(); /** * Time of the oldest uncommitted change, according to the source db. @@ -135,15 +133,15 @@ export class BinLogStream { entity_descriptor: entity, sync_rules: this.syncRules }); - // objectId is always defined for mysql + // Since we create the objectId ourselves, this is always defined this.tableCache.set(entity.objectId!, result.table); - // Drop conflicting tables. This includes for example renamed tables. + // Drop conflicting tables. In the MySQL case with ObjectIds created from the table name, renames cannot be detected by the storage. await batch.drop(result.dropTables); // Snapshot if: // 1. Snapshot is requested (false for initial snapshot, since that process handles it elsewhere) - // 2. Snapshot is not already done, AND: + // 2. Snapshot is not done yet, AND: // 3. The table is used in sync rules. const shouldSnapshot = snapshot && !result.table.snapshotComplete && result.table.syncAny; @@ -199,10 +197,7 @@ export class BinLogStream { { name: matchedTable, schema: tablePattern.schema, - objectId: getMysqlRelId({ - schema: tablePattern.schema, - name: matchedTable - }), + objectId: createTableId(tablePattern.schema, matchedTable), replicaIdColumns: replicaIdColumns }, false @@ -214,7 +209,7 @@ export class BinLogStream { } /** - * Checks if the initial sync has been completed yet. + * Checks if the initial sync has already been completed */ protected async checkInitialReplicated(): Promise { const status = await this.storage.getStatus(); @@ -223,7 +218,7 @@ export class BinLogStream { this.logger.info(`Initial replication already done.`); if (lastKnowGTID) { - // Check if the binlog is still available. If it isn't we need to snapshot again. + // Check if the specific binlog file is still available. If it isn't, we need to snapshot again. const connection = await this.connections.getConnection(); try { const isAvailable = await common.isBinlogStillAvailable(connection, lastKnowGTID.position.filename); @@ -485,10 +480,7 @@ export class BinLogStream { private async handleSchemaChange(batch: storage.BucketStorageBatch, change: SchemaChange): Promise { if (change.type === SchemaChangeType.RENAME_TABLE) { - const fromTableId = getMysqlRelId({ - schema: change.schema, - name: change.table - }); + const fromTableId = createTableId(change.schema, change.table); const fromTable = this.tableCache.get(fromTableId); // Old table needs to be cleaned up @@ -501,10 +493,7 @@ export class BinLogStream { await this.handleCreateOrUpdateTable(batch, change.newTable!, change.schema); } } else { - const tableId = getMysqlRelId({ - schema: change.schema, - name: change.table - }); + const tableId = createTableId(change.schema, change.table); const table = this.getTable(tableId); @@ -551,10 +540,7 @@ export class BinLogStream { { name: tableName, schema: schema, - objectId: getMysqlRelId({ - schema: schema, - name: tableName - }), + objectId: createTableId(schema, tableName), replicaIdColumns: replicaIdColumns }, true @@ -571,14 +557,11 @@ export class BinLogStream { } ): Promise { const columns = common.toColumnDescriptors(msg.tableEntry); - const tableId = getMysqlRelId({ - schema: msg.tableEntry.parentSchema, - name: msg.tableEntry.tableName - }); + const tableId = createTableId(msg.tableEntry.parentSchema, msg.tableEntry.tableName); let table = this.tableCache.get(tableId); if (table == null) { - // This write event is for a new table that matches a table in the sync rules + // This is an insert for a new table that matches a table in the sync rules // We need to create the table in the storage and cache it. table = await this.handleCreateOrUpdateTable(batch, msg.tableEntry.tableName, msg.tableEntry.parentSchema); } @@ -615,7 +598,7 @@ export class BinLogStream { }); case storage.SaveOperationTag.UPDATE: this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); - // "before" may be null if the replica id columns are unchanged + // The previous row may be null if the replica id columns are unchanged. // It's fine to treat that the same as an insert. const beforeUpdated = payload.previous_row ? common.toSQLiteRow(payload.previous_row, payload.columns) @@ -656,7 +639,7 @@ export class BinLogStream { // We don't have anything to compute replication lag with yet. return undefined; } else { - // We don't have any uncommitted changes, so replication is up-to-date. + // We don't have any uncommitted changes, so replication is up to date. return 0; } } diff --git a/modules/module-mysql/src/replication/zongji/BinLogListener.ts b/modules/module-mysql/src/replication/zongji/BinLogListener.ts index 4de30f53..f7b2ccd3 100644 --- a/modules/module-mysql/src/replication/zongji/BinLogListener.ts +++ b/modules/module-mysql/src/replication/zongji/BinLogListener.ts @@ -32,8 +32,8 @@ const { Parser } = pkg; export type Row = Record; /** - * Schema changes that can be detected by inspecting query events. - * Note that create table statements are not included here, since new tables are automatically detected when row events + * Schema changes that are detectable by inspecting query events. + * Create table statements are not included here, since new tables are automatically detected when row events * are received for them. */ export enum SchemaChangeType {