From 6a16c3dd61711e5f560b54734cb5e9694aa378fc Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Mon, 3 Mar 2025 18:11:54 +0200 Subject: [PATCH 01/18] Added basic metrics interfaces, a metrics factory and an implementation of those interfaces for Opentelemetry Added a MetricsEngine that handles creation and registration of metrics --- .../src/test-utils/metrics-utils.ts | 14 - packages/service-core/src/metrics/Metrics.ts | 255 ------------------ .../service-core/src/metrics/MetricsEngine.ts | 98 +++++++ .../service-core/src/metrics/metrics-index.ts | 4 + .../src/metrics/metrics-interfaces.ts | 41 +++ .../OpenTelemetryMetricsFactory.ts | 66 +++++ .../src/metrics/open-telemetry/util.ts | 74 +++++ 7 files changed, 283 insertions(+), 269 deletions(-) delete mode 100644 packages/service-core-tests/src/test-utils/metrics-utils.ts delete mode 100644 packages/service-core/src/metrics/Metrics.ts create mode 100644 packages/service-core/src/metrics/MetricsEngine.ts create mode 100644 packages/service-core/src/metrics/metrics-index.ts create mode 100644 packages/service-core/src/metrics/metrics-interfaces.ts create mode 100644 packages/service-core/src/metrics/open-telemetry/OpenTelemetryMetricsFactory.ts create mode 100644 packages/service-core/src/metrics/open-telemetry/util.ts diff --git a/packages/service-core-tests/src/test-utils/metrics-utils.ts b/packages/service-core-tests/src/test-utils/metrics-utils.ts deleted file mode 100644 index 8f61d250..00000000 --- a/packages/service-core-tests/src/test-utils/metrics-utils.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { Metrics } from '@powersync/service-core'; - -export const initMetrics = async () => { - await Metrics.initialise({ - disable_telemetry_sharing: true, - powersync_instance_id: 'test', - internal_metrics_endpoint: 'unused.for.tests.com' - }); - await resetMetrics(); -}; - -export const resetMetrics = async () => { - Metrics.getInstance().resetCounters(); -}; diff --git a/packages/service-core/src/metrics/Metrics.ts b/packages/service-core/src/metrics/Metrics.ts deleted file mode 100644 index 7a168216..00000000 --- a/packages/service-core/src/metrics/Metrics.ts +++ /dev/null @@ -1,255 +0,0 @@ -import { Attributes, Counter, ObservableGauge, UpDownCounter, ValueType } from '@opentelemetry/api'; -import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'; -import { PrometheusExporter } from '@opentelemetry/exporter-prometheus'; -import { Resource } from '@opentelemetry/resources'; -import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; -import { logger, ServiceAssertionError } from '@powersync/lib-services-framework'; -import * as storage from '../storage/storage-index.js'; -import * as util from '../util/util-index.js'; - -export interface MetricsOptions { - disable_telemetry_sharing: boolean; - powersync_instance_id: string; - internal_metrics_endpoint: string; -} - -export class Metrics { - private static instance: Metrics; - - private prometheusExporter: PrometheusExporter; - private meterProvider: MeterProvider; - - // Metrics - // 1. Data processing / month - - // 1a. Postgres -> PowerSync - // Record on replication pod - public data_replicated_bytes: Counter; - // 1b. PowerSync -> clients - // Record on API pod - public data_synced_bytes: Counter; - // Unused for pricing - // Record on replication pod - public rows_replicated_total: Counter; - // Unused for pricing - // Record on replication pod - public transactions_replicated_total: Counter; - // Unused for pricing - // Record on replication pod - public chunks_replicated_total: Counter; - - // 2. Sync operations / month - - // Record on API pod - public operations_synced_total: Counter; - - // 3. Data hosted on PowerSync sync service - - // Record on replication pod - // 3a. Replication storage -> raw data as received from Postgres. - public replication_storage_size_bytes: ObservableGauge; - // 3b. Operations storage -> transformed history, as will be synced to clients - public operation_storage_size_bytes: ObservableGauge; - // 3c. Parameter storage -> used for parameter queries - public parameter_storage_size_bytes: ObservableGauge; - - // 4. Peak concurrent connections - - // Record on API pod - public concurrent_connections: UpDownCounter; - - private constructor(meterProvider: MeterProvider, prometheusExporter: PrometheusExporter) { - this.meterProvider = meterProvider; - this.prometheusExporter = prometheusExporter; - const meter = meterProvider.getMeter('powersync'); - - this.data_replicated_bytes = meter.createCounter('powersync_data_replicated_bytes_total', { - description: 'Uncompressed size of replicated data', - unit: 'bytes', - valueType: ValueType.INT - }); - - this.data_synced_bytes = meter.createCounter('powersync_data_synced_bytes_total', { - description: 'Uncompressed size of synced data', - unit: 'bytes', - valueType: ValueType.INT - }); - - this.rows_replicated_total = meter.createCounter('powersync_rows_replicated_total', { - description: 'Total number of replicated rows', - valueType: ValueType.INT - }); - - this.transactions_replicated_total = meter.createCounter('powersync_transactions_replicated_total', { - description: 'Total number of replicated transactions', - valueType: ValueType.INT - }); - - this.chunks_replicated_total = meter.createCounter('powersync_chunks_replicated_total', { - description: 'Total number of replication chunks', - valueType: ValueType.INT - }); - - this.operations_synced_total = meter.createCounter('powersync_operations_synced_total', { - description: 'Number of operations synced', - valueType: ValueType.INT - }); - - this.replication_storage_size_bytes = meter.createObservableGauge('powersync_replication_storage_size_bytes', { - description: 'Size of current data stored in PowerSync', - unit: 'bytes', - valueType: ValueType.INT - }); - - this.operation_storage_size_bytes = meter.createObservableGauge('powersync_operation_storage_size_bytes', { - description: 'Size of operations stored in PowerSync', - unit: 'bytes', - valueType: ValueType.INT - }); - - this.parameter_storage_size_bytes = meter.createObservableGauge('powersync_parameter_storage_size_bytes', { - description: 'Size of parameter data stored in PowerSync', - unit: 'bytes', - valueType: ValueType.INT - }); - - this.concurrent_connections = meter.createUpDownCounter('powersync_concurrent_connections', { - description: 'Number of concurrent sync connections', - valueType: ValueType.INT - }); - } - - // Generally only useful for tests. Note: gauges are ignored here. - resetCounters() { - this.data_replicated_bytes.add(0); - this.data_synced_bytes.add(0); - this.rows_replicated_total.add(0); - this.transactions_replicated_total.add(0); - this.chunks_replicated_total.add(0); - this.operations_synced_total.add(0); - this.concurrent_connections.add(0); - } - - public static getInstance(): Metrics { - if (!Metrics.instance) { - throw new ServiceAssertionError('Metrics have not been initialized'); - } - - return Metrics.instance; - } - - public static async initialise(options: MetricsOptions): Promise { - if (Metrics.instance) { - return; - } - logger.info('Configuring telemetry.'); - - logger.info( - ` -Attention: -PowerSync collects completely anonymous telemetry regarding usage. -This information is used to shape our roadmap to better serve our customers. -You can learn more, including how to opt-out if you'd not like to participate in this anonymous program, by visiting the following URL: -https://docs.powersync.com/self-hosting/telemetry -Anonymous telemetry is currently: ${options.disable_telemetry_sharing ? 'disabled' : 'enabled'} - `.trim() - ); - - const configuredExporters: MetricReader[] = []; - - const port: number = util.env.METRICS_PORT ?? 0; - const prometheusExporter = new PrometheusExporter({ port: port, preventServerStart: true }); - configuredExporters.push(prometheusExporter); - - if (!options.disable_telemetry_sharing) { - logger.info('Sharing anonymous telemetry'); - const periodicExporter = new PeriodicExportingMetricReader({ - exporter: new OTLPMetricExporter({ - url: options.internal_metrics_endpoint - }), - exportIntervalMillis: 1000 * 60 * 5 // 5 minutes - }); - - configuredExporters.push(periodicExporter); - } - - const meterProvider = new MeterProvider({ - resource: new Resource({ - ['service']: 'PowerSync', - ['instance_id']: options.powersync_instance_id - }), - readers: configuredExporters - }); - - if (port > 0) { - await prometheusExporter.startServer(); - } - - Metrics.instance = new Metrics(meterProvider, prometheusExporter); - - logger.info('Telemetry configuration complete.'); - } - - public async shutdown(): Promise { - await this.meterProvider.shutdown(); - } - - public configureApiMetrics() { - // Initialize the metric, so that it reports a value before connections - // have been opened. - this.concurrent_connections.add(0); - } - - public configureReplicationMetrics(bucketStorage: storage.BucketStorageFactory) { - // Rate limit collection of these stats, since it may be an expensive query - const MINIMUM_INTERVAL = 60_000; - - let cachedRequest: Promise | undefined = undefined; - let cacheTimestamp = 0; - - function getMetrics() { - if (cachedRequest == null || Date.now() - cacheTimestamp > MINIMUM_INTERVAL) { - cachedRequest = bucketStorage.getStorageMetrics().catch((e) => { - logger.error(`Failed to get storage metrics`, e); - return null; - }); - cacheTimestamp = Date.now(); - } - return cachedRequest; - } - - this.operation_storage_size_bytes.addCallback(async (result) => { - const metrics = await getMetrics(); - if (metrics) { - result.observe(metrics.operations_size_bytes); - } - }); - - this.parameter_storage_size_bytes.addCallback(async (result) => { - const metrics = await getMetrics(); - if (metrics) { - result.observe(metrics.parameters_size_bytes); - } - }); - - this.replication_storage_size_bytes.addCallback(async (result) => { - const metrics = await getMetrics(); - if (metrics) { - result.observe(metrics.replication_size_bytes); - } - }); - } - - public async getMetricValueForTests(name: string): Promise { - const metrics = await this.prometheusExporter.collect(); - const scoped = metrics.resourceMetrics.scopeMetrics[0].metrics; - const metric = scoped.find((metric) => metric.descriptor.name == name); - if (metric == null) { - throw new Error( - `Cannot find metric ${name}. Options: ${scoped.map((metric) => metric.descriptor.name).join(',')}` - ); - } - const point = metric.dataPoints[metric.dataPoints.length - 1]; - return point?.value as number; - } -} diff --git a/packages/service-core/src/metrics/MetricsEngine.ts b/packages/service-core/src/metrics/MetricsEngine.ts new file mode 100644 index 00000000..f98cf587 --- /dev/null +++ b/packages/service-core/src/metrics/MetricsEngine.ts @@ -0,0 +1,98 @@ +import { logger, ServiceAssertionError } from '@powersync/lib-services-framework'; +import { Counter, UpDownCounter, ObservableGauge, MetricMetadata, MetricsFactory } from './metrics-interfaces.js'; + +export interface MetricsEngineOptions { + factory: MetricsFactory; + disable_telemetry_sharing: boolean; +} + +export class MetricsEngine { + private counters: Map; + private upDownCounters: Map; + private observableGauges: Map; + + constructor(private options: MetricsEngineOptions) { + this.counters = new Map(); + this.upDownCounters = new Map(); + this.observableGauges = new Map(); + } + + private get factory(): MetricsFactory { + return this.options.factory; + } + + createCounter(metadata: MetricMetadata): Counter { + if (this.counters.has(metadata.name)) { + logger.warn(`Counter with name ${metadata.name} already created and registered, skipping.`); + return this.counters.get(metadata.name)!; + } + + const counter = this.factory.createCounter(metadata); + this.counters.set(metadata.name, counter); + return counter; + } + createUpDownCounter(metadata: MetricMetadata): UpDownCounter { + if (this.upDownCounters.has(metadata.name)) { + logger.warn(`UpDownCounter with name ${metadata.name} already created and registered, skipping.`); + return this.upDownCounters.get(metadata.name)!; + } + + const upDownCounter = this.factory.createUpDownCounter(metadata); + this.upDownCounters.set(metadata.name, upDownCounter); + return upDownCounter; + } + + createObservableGauge(metadata: MetricMetadata): ObservableGauge { + if (this.observableGauges.has(metadata.name)) { + logger.warn(`ObservableGauge with name ${metadata.name} already created and registered, skipping.`); + return this.observableGauges.get(metadata.name)!; + } + + const observableGauge = this.factory.createObservableGauge(metadata); + this.observableGauges.set(metadata.name, observableGauge); + return observableGauge; + } + + getCounter(name: string): Counter { + const counter = this.counters.get(name); + if (!counter) { + throw new ServiceAssertionError(`Counter '${name}' has not been created and registered yet.`); + } + return counter; + } + + getUpDownCounter(name: string): UpDownCounter { + const upDownCounter = this.upDownCounters.get(name); + if (!upDownCounter) { + throw new ServiceAssertionError(`UpDownCounter '${name}' has not been created and registered yet.`); + } + return upDownCounter; + } + + getObservableGauge(name: string): ObservableGauge { + const observableGauge = this.observableGauges.get(name); + if (!observableGauge) { + throw new ServiceAssertionError(`ObservableGauge '${name}' has not been created and registered yet.`); + } + return observableGauge; + } + + public async start(): Promise { + logger.info( + ` +Attention: +PowerSync collects completely anonymous telemetry regarding usage. +This information is used to shape our roadmap to better serve our customers. +You can learn more, including how to opt-out if you'd not like to participate in this anonymous program, by visiting the following URL: +https://docs.powersync.com/self-hosting/lifecycle-maintenance/telemetry + `.trim() + ); + logger.info(`Anonymous telemetry is currently: ${this.options.disable_telemetry_sharing ? 'disabled' : 'enabled'}`); + + logger.info('Successfully started Metrics Engine.'); + } + + public async shutdown(): Promise { + logger.info('Successfully shut down Metrics Engine.'); + } +} diff --git a/packages/service-core/src/metrics/metrics-index.ts b/packages/service-core/src/metrics/metrics-index.ts new file mode 100644 index 00000000..9c52331c --- /dev/null +++ b/packages/service-core/src/metrics/metrics-index.ts @@ -0,0 +1,4 @@ +export * from './MetricsEngine.js'; +export * from './metrics-interfaces.js'; +export * from './open-telemetry/OpenTelemetryMetricsFactory.js'; +export * from './open-telemetry/util.js'; diff --git a/packages/service-core/src/metrics/metrics-interfaces.ts b/packages/service-core/src/metrics/metrics-interfaces.ts new file mode 100644 index 00000000..d2847595 --- /dev/null +++ b/packages/service-core/src/metrics/metrics-interfaces.ts @@ -0,0 +1,41 @@ +export interface Counter { + /** + * Increment the counter by the given value. Only positive numbers are valid. + * @param value + */ + add(value: number): void; +} + +export interface UpDownCounter { + /** + * Increment or decrement(if negative) the counter by the given value. + * @param value + */ + add(value: number): void; +} + +export interface ObservableGauge { + /** + * Set a value provider that provides the value for the gauge at the time of observation. + * @param valueProvider + */ + setValueProvider(valueProvider: () => Promise): void; +} + +export enum Precision { + INT = 'int', + DOUBLE = 'double' +} + +export interface MetricMetadata { + name: string; + description?: string; + unit?: string; + precision?: Precision; +} + +export interface MetricsFactory { + createCounter(metadata: MetricMetadata): Counter; + createUpDownCounter(metadata: MetricMetadata): UpDownCounter; + createObservableGauge(metadata: MetricMetadata): ObservableGauge; +} diff --git a/packages/service-core/src/metrics/open-telemetry/OpenTelemetryMetricsFactory.ts b/packages/service-core/src/metrics/open-telemetry/OpenTelemetryMetricsFactory.ts new file mode 100644 index 00000000..13013cb5 --- /dev/null +++ b/packages/service-core/src/metrics/open-telemetry/OpenTelemetryMetricsFactory.ts @@ -0,0 +1,66 @@ +import { Meter, ValueType } from '@opentelemetry/api'; +import { + Counter, + ObservableGauge, + UpDownCounter, + MetricMetadata, + MetricsFactory, + Precision +} from '../metrics-interfaces.js'; + +export class OpenTelemetryMetricsFactory implements MetricsFactory { + private meter: Meter; + + constructor(meter: Meter) { + this.meter = meter; + } + + createCounter(metadata: MetricMetadata): Counter { + return this.meter.createCounter(metadata.name, { + description: metadata.description, + unit: metadata.unit, + valueType: this.toValueType(metadata.precision) + }); + } + + createObservableGauge(metadata: MetricMetadata): ObservableGauge { + const gauge = this.meter.createObservableGauge(metadata.name, { + description: metadata.description, + unit: metadata.unit, + valueType: this.toValueType(metadata.precision) + }); + + return { + setValueProvider(valueProvider: () => Promise) { + gauge.addCallback(async (result) => { + const value = await valueProvider(); + + if (value) { + result.observe(value); + } + }); + } + }; + } + + createUpDownCounter(metadata: MetricMetadata): UpDownCounter { + return this.meter.createUpDownCounter(metadata.name, { + description: metadata.description, + unit: metadata.unit, + valueType: this.toValueType(metadata.precision) + }); + } + + private toValueType(precision?: Precision): ValueType { + if (!precision) { + return ValueType.INT; + } + + switch (precision) { + case Precision.INT: + return ValueType.INT; + case Precision.DOUBLE: + return ValueType.DOUBLE; + } + } +} diff --git a/packages/service-core/src/metrics/open-telemetry/util.ts b/packages/service-core/src/metrics/open-telemetry/util.ts new file mode 100644 index 00000000..5396d795 --- /dev/null +++ b/packages/service-core/src/metrics/open-telemetry/util.ts @@ -0,0 +1,74 @@ +import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; +import { env } from '../../util/env.js'; +import { PrometheusExporter } from '@opentelemetry/exporter-prometheus'; +import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'; +import { Resource } from '@opentelemetry/resources'; +import { ServiceContext } from '../../system/ServiceContext.js'; +import { OpenTelemetryMetricsFactory } from './OpenTelemetryMetricsFactory.js'; +import { MetricsFactory } from '../metrics-interfaces.js'; + +export interface RuntimeMetadata { + [key: string]: string | number | undefined; +} + +export function createOpenTelemetryMetricsFactory(context: ServiceContext): MetricsFactory { + const { configuration, lifeCycleEngine, storageEngine } = context; + const configuredExporters: MetricReader[] = []; + + if (env.METRICS_PORT) { + const prometheusExporter = new PrometheusExporter({ port: env.METRICS_PORT, preventServerStart: true }); + configuredExporters.push(prometheusExporter); + + lifeCycleEngine.withLifecycle(prometheusExporter, { + start: () => prometheusExporter.startServer() + }); + } + + if (!configuration.telemetry.disable_telemetry_sharing) { + const periodicExporter = new PeriodicExportingMetricReader({ + exporter: new OTLPMetricExporter({ + url: configuration.telemetry.internal_service_endpoint + }), + exportIntervalMillis: 1000 * 60 * 5 // 5 minutes + }); + + configuredExporters.push(periodicExporter); + } + + let resolvedMetadata: (metadata: RuntimeMetadata) => void; + const runtimeMetadata: Promise = new Promise((resolve) => { + resolvedMetadata = resolve; + }); + + lifeCycleEngine.withLifecycle(null, { + start: async () => { + const bucketStorage = storageEngine.activeBucketStorage; + try { + const instanceId = await bucketStorage.getPowerSyncInstanceId(); + resolvedMetadata({ ['instance_id']: instanceId }); + } catch (err) { + resolvedMetadata({ ['instance_id']: 'Unknown' }); + } + } + }); + + const meterProvider = new MeterProvider({ + resource: new Resource( + { + ['service']: 'PowerSync' + }, + runtimeMetadata + ), + readers: configuredExporters + }); + + lifeCycleEngine.withLifecycle(meterProvider, { + stop: async () => { + await meterProvider.shutdown(); + } + }); + + const meter = meterProvider.getMeter('powersync'); + + return new OpenTelemetryMetricsFactory(meter); +} From 82385bd1ecf9d2dbc5bfe958960aa74d810c446f Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Mon, 3 Mar 2025 18:13:55 +0200 Subject: [PATCH 02/18] Added creation and initialization helper methods for the existing replication, storage and API metrics Added the MetricsEngine to the service context --- packages/service-core/src/api/api-index.ts | 1 + packages/service-core/src/api/api-metrics.ts | 43 +++++++++++ packages/service-core/src/index.ts | 4 +- .../src/replication/replication-index.ts | 1 + .../src/replication/replication-metrics.ts | 55 ++++++++++++++ .../service-core/src/storage/storage-index.ts | 1 + .../src/storage/storage-metrics.ts | 75 +++++++++++++++++++ .../service-core/src/system/ServiceContext.ts | 13 +++- 8 files changed, 187 insertions(+), 6 deletions(-) create mode 100644 packages/service-core/src/api/api-metrics.ts create mode 100644 packages/service-core/src/replication/replication-metrics.ts create mode 100644 packages/service-core/src/storage/storage-metrics.ts diff --git a/packages/service-core/src/api/api-index.ts b/packages/service-core/src/api/api-index.ts index 0f90b173..2074af6a 100644 --- a/packages/service-core/src/api/api-index.ts +++ b/packages/service-core/src/api/api-index.ts @@ -1,3 +1,4 @@ export * from './diagnostics.js'; export * from './RouteAPI.js'; export * from './schema.js'; +export * from './api-metrics.js'; diff --git a/packages/service-core/src/api/api-metrics.ts b/packages/service-core/src/api/api-metrics.ts new file mode 100644 index 00000000..766aaf82 --- /dev/null +++ b/packages/service-core/src/api/api-metrics.ts @@ -0,0 +1,43 @@ +import { MetricsEngine } from '../metrics/MetricsEngine.js'; + +export enum APIMetricType { + // Uncompressed size of synced data from PowerSync to Clients + DATA_SYNCED_BYTES = 'powersync_data_synced_bytes_total', + // Number of operations synced + OPERATIONS_SYNCED_TOTAL = 'powersync_operations_synced_total', + // Number of concurrent sync connections + CONCURRENT_CONNECTIONS = 'powersync_concurrent_connections' +} + +/** + * Create and register the core API metrics. + * @param engine + */ +export function createCoreAPIMetrics(engine: MetricsEngine): void { + engine.createCounter({ + name: APIMetricType.DATA_SYNCED_BYTES, + description: 'Uncompressed size of synced data', + unit: 'bytes' + }); + + engine.createCounter({ + name: APIMetricType.OPERATIONS_SYNCED_TOTAL, + description: 'Number of operations synced' + }); + + engine.createUpDownCounter({ + name: APIMetricType.CONCURRENT_CONNECTIONS, + description: 'Number of concurrent sync connections' + }); +} + +/** + * Initialise the core API metrics. This should be called after the metrics have been created. + * @param engine + */ +export function initializeCoreAPIMetrics(engine: MetricsEngine): void { + const concurrent_connections = engine.getUpDownCounter(APIMetricType.CONCURRENT_CONNECTIONS); + + // Initialize the metric, so that it reports a value before connections have been opened. + concurrent_connections.add(0); +} diff --git a/packages/service-core/src/index.ts b/packages/service-core/src/index.ts index 9a90ceeb..73ce3d7c 100644 --- a/packages/service-core/src/index.ts +++ b/packages/service-core/src/index.ts @@ -12,8 +12,8 @@ export * as entry from './entry/entry-index.js'; // Re-export framework for easy use of Container API export * as framework from '@powersync/lib-services-framework'; -export * from './metrics/Metrics.js'; -export * as metrics from './metrics/Metrics.js'; +export * from './metrics/metrics-index.js'; +export * as metrics from './metrics/metrics-index.js'; export * from './migrations/migrations-index.js'; export * as migrations from './migrations/migrations-index.js'; diff --git a/packages/service-core/src/replication/replication-index.ts b/packages/service-core/src/replication/replication-index.ts index 0b37534c..eb1f4705 100644 --- a/packages/service-core/src/replication/replication-index.ts +++ b/packages/service-core/src/replication/replication-index.ts @@ -3,3 +3,4 @@ export * from './AbstractReplicator.js'; export * from './ErrorRateLimiter.js'; export * from './ReplicationEngine.js'; export * from './ReplicationModule.js'; +export * from './replication-metrics.js'; diff --git a/packages/service-core/src/replication/replication-metrics.ts b/packages/service-core/src/replication/replication-metrics.ts new file mode 100644 index 00000000..6c387fa3 --- /dev/null +++ b/packages/service-core/src/replication/replication-metrics.ts @@ -0,0 +1,55 @@ +import { MetricsEngine } from '../metrics/metrics-index.js'; + +export enum ReplicationMetricType { + // Uncompressed size of replicated data from data source to PowerSync + DATA_REPLICATED_BYTES = 'powersync_data_replicated_bytes_total', + // Total number of replicated rows. Not used for pricing. + ROWS_REPLICATED_TOTAL = 'powersync_rows_replicated_total', + // Total number of replicated transactions. Not used for pricing. + TRANSACTIONS_REPLICATED_TOTAL = 'powersync_transactions_replicated_total', + // Total number of replication chunks. Not used for pricing. + CHUNKS_REPLICATED_TOTAL = 'powersync_chunks_replicated_total' +} + +/** + * Create and register the core replication metrics. + * @param engine + */ +export function createCoreReplicationMetrics(engine: MetricsEngine): void { + engine.createCounter({ + name: ReplicationMetricType.DATA_REPLICATED_BYTES, + description: 'Uncompressed size of replicated data', + unit: 'bytes' + }); + + engine.createCounter({ + name: ReplicationMetricType.ROWS_REPLICATED_TOTAL, + description: 'Total number of replicated rows' + }); + + engine.createCounter({ + name: ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL, + description: 'Total number of replicated transactions' + }); + + engine.createCounter({ + name: ReplicationMetricType.CHUNKS_REPLICATED_TOTAL, + description: 'Total number of replication chunks' + }); +} + +/** + * Initialise the core replication metrics. This should be called after the metrics have been created. + * @param engine + */ +export function initializeCoreReplicationMetrics(engine: MetricsEngine): void { + const data_replicated_bytes = engine.getCounter(ReplicationMetricType.DATA_REPLICATED_BYTES); + const rows_replicated_total = engine.getCounter(ReplicationMetricType.ROWS_REPLICATED_TOTAL); + const transactions_replicated_total = engine.getCounter(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL); + const chunks_replicated_total = engine.getCounter(ReplicationMetricType.CHUNKS_REPLICATED_TOTAL); + + data_replicated_bytes.add(0); + rows_replicated_total.add(0); + transactions_replicated_total.add(0); + chunks_replicated_total.add(0); +} diff --git a/packages/service-core/src/storage/storage-index.ts b/packages/service-core/src/storage/storage-index.ts index b3e2c15b..9787afd3 100644 --- a/packages/service-core/src/storage/storage-index.ts +++ b/packages/service-core/src/storage/storage-index.ts @@ -6,4 +6,5 @@ export * from './SourceEntity.js'; export * from './SourceTable.js'; export * from './StorageEngine.js'; export * from './StorageProvider.js'; +export * from './storage-metrics.js'; export * from './WriteCheckpointAPI.js'; diff --git a/packages/service-core/src/storage/storage-metrics.ts b/packages/service-core/src/storage/storage-metrics.ts new file mode 100644 index 00000000..0f5af3bb --- /dev/null +++ b/packages/service-core/src/storage/storage-metrics.ts @@ -0,0 +1,75 @@ +import { MetricsEngine } from '../metrics/MetricsEngine.js'; +import { BucketStorageFactory, StorageMetrics } from './BucketStorage.js'; +import { logger } from '@powersync/lib-services-framework'; + +export enum StorageMetricType { + // Size of current replication data stored in PowerSync + REPLICATION_SIZE_BYTES = 'powersync_replication_storage_size_bytes', + // Size of operations data stored in PowerSync + OPERATION_SIZE_BYTES = 'powersync_operation_storage_size_bytes', + // Size of parameter data stored in PowerSync + PARAMETER_SIZE_BYTES = 'powersync_parameter_storage_size_bytes' +} + +export function createCoreStorageMetrics(engine: MetricsEngine): void { + engine.createObservableGauge({ + name: StorageMetricType.REPLICATION_SIZE_BYTES, + description: 'Size of current data stored in PowerSync', + unit: 'bytes' + }); + + engine.createObservableGauge({ + name: StorageMetricType.OPERATION_SIZE_BYTES, + description: 'Size of operations stored in PowerSync', + unit: 'bytes' + }); + + engine.createObservableGauge({ + name: StorageMetricType.PARAMETER_SIZE_BYTES, + description: 'Size of parameter data stored in PowerSync', + unit: 'bytes' + }); +} + +export function initializeCoreStorageMetrics(engine: MetricsEngine, storage: BucketStorageFactory): void { + const replication_storage_size_bytes = engine.getObservableGauge(StorageMetricType.REPLICATION_SIZE_BYTES); + const operation_storage_size_bytes = engine.getObservableGauge(StorageMetricType.OPERATION_SIZE_BYTES); + const parameter_storage_size_bytes = engine.getObservableGauge(StorageMetricType.PARAMETER_SIZE_BYTES); + + const MINIMUM_INTERVAL = 60_000; + + let cachedRequest: Promise | undefined = undefined; + let cacheTimestamp = 0; + + const getMetrics = () => { + if (cachedRequest == null || Date.now() - cacheTimestamp > MINIMUM_INTERVAL) { + cachedRequest = storage.getStorageMetrics().catch((e) => { + logger.error(`Failed to get storage metrics`, e); + return null; + }); + cacheTimestamp = Date.now(); + } + return cachedRequest; + }; + + replication_storage_size_bytes.setValueProvider(async () => { + const metrics = await getMetrics(); + if (metrics) { + return metrics.replication_size_bytes; + } + }); + + operation_storage_size_bytes.setValueProvider(async () => { + const metrics = await getMetrics(); + if (metrics) { + return metrics.operations_size_bytes; + } + }); + + parameter_storage_size_bytes.setValueProvider(async () => { + const metrics = await getMetrics(); + if (metrics) { + return metrics.parameters_size_bytes; + } + }); +} diff --git a/packages/service-core/src/system/ServiceContext.ts b/packages/service-core/src/system/ServiceContext.ts index aa322dcd..9788b220 100644 --- a/packages/service-core/src/system/ServiceContext.ts +++ b/packages/service-core/src/system/ServiceContext.ts @@ -1,7 +1,7 @@ import { LifeCycledSystem, MigrationManager, ServiceIdentifier, container } from '@powersync/lib-services-framework'; import { framework } from '../index.js'; -import * as metrics from '../metrics/Metrics.js'; +import * as metrics from '../metrics/MetricsEngine.js'; import { PowerSyncMigrationManager } from '../migrations/PowerSyncMigrationManager.js'; import * as replication from '../replication/replication-index.js'; import * as routes from '../routes/routes-index.js'; @@ -11,7 +11,7 @@ import * as utils from '../util/util-index.js'; export interface ServiceContext { configuration: utils.ResolvedPowerSyncConfig; lifeCycleEngine: LifeCycledSystem; - metrics: metrics.Metrics | null; + metricsEngine: metrics.MetricsEngine; replicationEngine: replication.ReplicationEngine | null; routerEngine: routes.RouterEngine | null; storageEngine: storage.StorageEngine; @@ -34,6 +34,11 @@ export class ServiceContextContainer implements ServiceContext { configuration }); + this.lifeCycleEngine.withLifecycle(this.storageEngine, { + start: (storageEngine) => storageEngine.start(), + stop: (storageEngine) => storageEngine.shutDown() + }); + const migrationManager = new MigrationManager(); container.register(framework.ContainerImplementation.MIGRATION_MANAGER, migrationManager); @@ -56,8 +61,8 @@ export class ServiceContextContainer implements ServiceContext { return container.getOptional(routes.RouterEngine); } - get metrics(): metrics.Metrics | null { - return container.getOptional(metrics.Metrics); + get metricsEngine(): metrics.MetricsEngine { + return container.getImplementation(metrics.MetricsEngine); } get migrations(): PowerSyncMigrationManager { From c655147931dfd3859b9d01468628e8c73bbf306b Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Mon, 3 Mar 2025 18:16:02 +0200 Subject: [PATCH 03/18] Updated service runners to initialize the MetricsEngine and the existing metrics --- service/src/metrics.ts | 61 ++++++++++++++++++--------- service/src/runners/stream-worker.ts | 4 +- service/src/runners/unified-runner.ts | 2 +- 3 files changed, 43 insertions(+), 24 deletions(-) diff --git a/service/src/metrics.ts b/service/src/metrics.ts index 57ef04db..6e0e9ea0 100644 --- a/service/src/metrics.ts +++ b/service/src/metrics.ts @@ -1,8 +1,17 @@ import * as core from '@powersync/service-core'; +import { + createCoreAPIMetrics, + createCoreReplicationMetrics, + createCoreStorageMetrics, + initializeCoreAPIMetrics, + initializeCoreReplicationMetrics, + initializeCoreStorageMetrics +} from '@powersync/service-core'; export enum MetricModes { API = 'api', - REPLICATION = 'replication' + REPLICATION = 'replication', + STORAGE = 'storage' } export type MetricsRegistrationOptions = { @@ -13,28 +22,38 @@ export type MetricsRegistrationOptions = { export const registerMetrics = async (options: MetricsRegistrationOptions) => { const { service_context, modes } = options; - // This requires an instantiated bucket storage, which is only created when the lifecycle starts - service_context.lifeCycleEngine.withLifecycle(null, { - start: async () => { - const instanceId = await service_context.storageEngine.activeBucketStorage.getPowerSyncInstanceId(); - await core.metrics.Metrics.initialise({ - powersync_instance_id: instanceId, - disable_telemetry_sharing: service_context.configuration.telemetry.disable_telemetry_sharing, - internal_metrics_endpoint: service_context.configuration.telemetry.internal_service_endpoint - }); - - // TODO remove singleton - const instance = core.Metrics.getInstance(); - service_context.register(core.metrics.Metrics, instance); - - if (modes.includes(MetricModes.API)) { - instance.configureApiMetrics(); - } + const metricsFactory = core.metrics.createOpenTelemetryMetricsFactory(service_context); + const metricsEngine = new core.metrics.MetricsEngine({ + factory: metricsFactory, + disable_telemetry_sharing: service_context.configuration.telemetry.disable_telemetry_sharing + }); + service_context.register(core.metrics.MetricsEngine, metricsEngine); + + if (modes.includes(MetricModes.API)) { + createCoreAPIMetrics(metricsEngine); + initializeCoreAPIMetrics(metricsEngine); + } - if (modes.includes(MetricModes.REPLICATION)) { - instance.configureReplicationMetrics(service_context.storageEngine.activeBucketStorage); + if (modes.includes(MetricModes.REPLICATION)) { + createCoreReplicationMetrics(metricsEngine); + initializeCoreReplicationMetrics(metricsEngine); + } + + if (modes.includes(MetricModes.STORAGE)) { + createCoreStorageMetrics(metricsEngine); + + // This requires an instantiated bucket storage, which is only created when the lifecycle starts + service_context.storageEngine.registerListener({ + storageActivated: (bucketStorage) => { + initializeCoreStorageMetrics(metricsEngine, bucketStorage); } + }); + } + + service_context.lifeCycleEngine.withLifecycle(metricsEngine, { + start: async () => { + await metricsEngine.start(); }, - stop: () => service_context.metrics!.shutdown() + stop: () => metricsEngine.shutdown() }); }; diff --git a/service/src/runners/stream-worker.ts b/service/src/runners/stream-worker.ts index 323d7117..42d5382a 100644 --- a/service/src/runners/stream-worker.ts +++ b/service/src/runners/stream-worker.ts @@ -21,14 +21,14 @@ export const startStreamRunner = async (runnerConfig: core.utils.RunnerConfig) = const config = await core.utils.loadConfig(runnerConfig); - // Self hosted version allows for automatic migrations + // Self-hosted version allows for automatic migrations const serviceContext = new core.system.ServiceContextContainer(config); registerReplicationServices(serviceContext); await registerMetrics({ service_context: serviceContext, - modes: [MetricModes.REPLICATION] + modes: [MetricModes.REPLICATION, MetricModes.STORAGE] }); const moduleManager = container.getImplementation(core.modules.ModuleManager); diff --git a/service/src/runners/unified-runner.ts b/service/src/runners/unified-runner.ts index 5e505e2f..b2fc684f 100644 --- a/service/src/runners/unified-runner.ts +++ b/service/src/runners/unified-runner.ts @@ -20,7 +20,7 @@ export const startUnifiedRunner = async (runnerConfig: core.utils.RunnerConfig) await registerMetrics({ service_context: serviceContext, - modes: [MetricModes.API, MetricModes.REPLICATION] + modes: [MetricModes.API, MetricModes.REPLICATION, MetricModes.STORAGE] }); const moduleManager = container.getImplementation(core.modules.ModuleManager); From 437bd7a77b65ce65c4901ee7bcd26d88bb871031 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Mon, 3 Mar 2025 18:18:48 +0200 Subject: [PATCH 04/18] Updated replication modules to use the MetricsEngine instead of the old singleton metrics instance --- .../module-mongodb/src/module/MongoModule.ts | 3 +++ .../src/replication/ChangeStream.ts | 16 +++++++++--- .../replication/ChangeStreamReplicationJob.ts | 1 + .../src/replication/ChangeStreamReplicator.ts | 1 + .../module-mysql/src/module/MySQLModule.ts | 5 ++-- .../src/replication/BinLogReplicationJob.ts | 1 + .../src/replication/BinLogReplicator.ts | 1 + .../src/replication/BinLogStream.ts | 26 ++++++++++++++----- .../src/module/PostgresModule.ts | 18 ++++++------- .../src/replication/WalStream.ts | 24 +++++++++++------ .../replication/WalStreamReplicationJob.ts | 1 + .../src/replication/WalStreamReplicator.ts | 1 + .../src/replication/AbstractReplicationJob.ts | 2 ++ .../src/replication/AbstractReplicator.ts | 7 +++++ .../src/replication/ReplicationModule.ts | 10 +++++++ .../src/routes/endpoints/socket-route.ts | 10 +++---- .../src/routes/endpoints/sync-stream.ts | 12 ++++----- .../service-core/src/sync/RequestTracker.ts | 11 +++++--- 18 files changed, 106 insertions(+), 44 deletions(-) diff --git a/modules/module-mongodb/src/module/MongoModule.ts b/modules/module-mongodb/src/module/MongoModule.ts index b38c0e03..3e7b754a 100644 --- a/modules/module-mongodb/src/module/MongoModule.ts +++ b/modules/module-mongodb/src/module/MongoModule.ts @@ -37,11 +37,14 @@ export class MongoModule extends replication.ReplicationModule {} + /** * Combines base config with normalized connection settings */ diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index c0e9e56e..46e939b4 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -8,7 +8,14 @@ import { ReplicationAssertionError, ServiceError } from '@powersync/lib-services-framework'; -import { Metrics, SaveOperationTag, SourceEntityDescriptor, SourceTable, storage } from '@powersync/service-core'; +import { + MetricsEngine, + ReplicationMetricType, + SaveOperationTag, + SourceEntityDescriptor, + SourceTable, + storage +} from '@powersync/service-core'; import { DatabaseInputRow, SqliteRow, SqlSyncRules, TablePattern } from '@powersync/service-sync-rules'; import { MongoLSN } from '../common/MongoLSN.js'; import { PostImagesOption } from '../types/types.js'; @@ -20,6 +27,7 @@ import { CHECKPOINTS_COLLECTION } from './replication-utils.js'; export interface ChangeStreamOptions { connections: MongoManager; storage: storage.SyncRulesBucketStorage; + metrics: MetricsEngine; abort_signal: AbortSignal; } @@ -52,6 +60,7 @@ export class ChangeStream { private connections: MongoManager; private readonly client: mongo.MongoClient; private readonly defaultDb: mongo.Db; + private readonly metrics: MetricsEngine; private abort_signal: AbortSignal; @@ -59,6 +68,7 @@ export class ChangeStream { constructor(options: ChangeStreamOptions) { this.storage = options.storage; + this.metrics = options.metrics; this.group_id = options.storage.group_id; this.connections = options.connections; this.client = this.connections.client; @@ -322,7 +332,7 @@ export class ChangeStream { logger.info(`[${this.group_id}] Replicating ${table.qualifiedName} ${at}/${estimatedCount}`); lastLogIndex = at; } - Metrics.getInstance().rows_replicated_total.add(1); + this.metrics.getCounter(ReplicationMetricType.ROWS_REPLICATED_TOTAL).add(1); await touch(); } @@ -438,7 +448,7 @@ export class ChangeStream { return null; } - Metrics.getInstance().rows_replicated_total.add(1); + this.metrics.getCounter(ReplicationMetricType.ROWS_REPLICATED_TOTAL).add(1); if (change.operationType == 'insert') { const baseRecord = constructAfterRecord(change.fullDocument); return await batch.save({ diff --git a/modules/module-mongodb/src/replication/ChangeStreamReplicationJob.ts b/modules/module-mongodb/src/replication/ChangeStreamReplicationJob.ts index e2ba6a24..a2f9cfc7 100644 --- a/modules/module-mongodb/src/replication/ChangeStreamReplicationJob.ts +++ b/modules/module-mongodb/src/replication/ChangeStreamReplicationJob.ts @@ -71,6 +71,7 @@ export class ChangeStreamReplicationJob extends replication.AbstractReplicationJ const stream = new ChangeStream({ abort_signal: this.abortController.signal, storage: this.options.storage, + metrics: this.options.metrics, connections: connectionManager }); await stream.replicate(); diff --git a/modules/module-mongodb/src/replication/ChangeStreamReplicator.ts b/modules/module-mongodb/src/replication/ChangeStreamReplicator.ts index 9f6aae16..99930e2e 100644 --- a/modules/module-mongodb/src/replication/ChangeStreamReplicator.ts +++ b/modules/module-mongodb/src/replication/ChangeStreamReplicator.ts @@ -20,6 +20,7 @@ export class ChangeStreamReplicator extends replication.AbstractReplicator { - await super.initialize(context); - } + async onInitialized(context: system.ServiceContextContainer): Promise {} protected createRouteAPIAdapter(): api.RouteAPI { return new MySQLRouteAPIAdapter(this.resolveConfig(this.decodedConfig!)); @@ -42,6 +40,7 @@ export class MySQLModule extends replication.ReplicationModule(); - constructor(protected options: BinLogStreamOptions) { + constructor(private options: BinLogStreamOptions) { this.storage = options.storage; this.connections = options.connections; this.syncRules = options.storage.getParsedSyncRules({ defaultSchema: this.defaultSchema }); @@ -74,6 +82,10 @@ export class BinLogStream { return this.connections.connectionTag; } + private get metrics() { + return this.options.metrics; + } + get connectionId() { const { connectionId } = this.connections; // Default to 1 if not set @@ -334,6 +346,8 @@ AND table_type = 'BASE TABLE';`, after: record, afterReplicaId: getUuidReplicaIdentityBson(record, table.replicaIdColumns) }); + + this.metrics.getCounter(ReplicationMetricType.ROWS_REPLICATED_TOTAL).add(1); } await batch.flush(); } @@ -460,7 +474,7 @@ AND table_type = 'BASE TABLE';`, }); break; case zongji_utils.eventIsXid(evt): - Metrics.getInstance().transactions_replicated_total.add(1); + this.metrics.getCounter(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL).add(1); // Need to commit with a replicated GTID with updated next position await batch.commit( new common.ReplicatedGTID({ @@ -602,7 +616,7 @@ AND table_type = 'BASE TABLE';`, ): Promise { switch (payload.type) { case storage.SaveOperationTag.INSERT: - Metrics.getInstance().rows_replicated_total.add(1); + this.metrics.getCounter(ReplicationMetricType.ROWS_REPLICATED_TOTAL).add(1); const record = common.toSQLiteRow(payload.data, payload.columns); return await batch.save({ tag: storage.SaveOperationTag.INSERT, @@ -613,7 +627,7 @@ AND table_type = 'BASE TABLE';`, afterReplicaId: getUuidReplicaIdentityBson(record, payload.sourceTable.replicaIdColumns) }); case storage.SaveOperationTag.UPDATE: - Metrics.getInstance().rows_replicated_total.add(1); + this.metrics.getCounter(ReplicationMetricType.ROWS_REPLICATED_TOTAL).add(1); // "before" may be null if the replica id columns are unchanged // It's fine to treat that the same as an insert. const beforeUpdated = payload.previous_data @@ -633,7 +647,7 @@ AND table_type = 'BASE TABLE';`, }); case storage.SaveOperationTag.DELETE: - Metrics.getInstance().rows_replicated_total.add(1); + this.metrics.getCounter(ReplicationMetricType.ROWS_REPLICATED_TOTAL).add(1); const beforeDeleted = common.toSQLiteRow(payload.data, payload.columns); return await batch.save({ diff --git a/modules/module-postgres/src/module/PostgresModule.ts b/modules/module-postgres/src/module/PostgresModule.ts index d9b7ff47..ee4f0e76 100644 --- a/modules/module-postgres/src/module/PostgresModule.ts +++ b/modules/module-postgres/src/module/PostgresModule.ts @@ -5,6 +5,7 @@ import { ConnectionTestResult, modules, replication, + ReplicationMetricType, system } from '@powersync/service-core'; import * as jpgwire from '@powersync/service-jpgwire'; @@ -29,9 +30,7 @@ export class PostgresModule extends replication.ReplicationModule { - await super.initialize(context); - + async onInitialized(context: system.ServiceContextContainer): Promise { const client_auth = context.configuration.base_config.client_auth; if (client_auth?.supabase && client_auth?.supabase_jwt_secret == null) { @@ -46,13 +45,11 @@ export class PostgresModule extends replication.ReplicationModule { protected logger: winston.Logger; + /** * Map of replication jobs by sync rule id. Usually there is only one running job, but there could be two when * transitioning to a new set of sync rules. @@ -72,6 +75,10 @@ export abstract class AbstractReplicator { this.runLoop().catch((e) => { this.logger.error('Data source fatal replication error', e); diff --git a/packages/service-core/src/replication/ReplicationModule.ts b/packages/service-core/src/replication/ReplicationModule.ts index 410c022a..5c69074b 100644 --- a/packages/service-core/src/replication/ReplicationModule.ts +++ b/packages/service-core/src/replication/ReplicationModule.ts @@ -64,6 +64,14 @@ export abstract class ReplicationModule */ protected abstract createReplicator(context: system.ServiceContext): AbstractReplicator; + /** + * Any additional initialization specific to the module should be added here. Will be called if necessary after the + * main initialization has been completed + * @param context + * @protected + */ + protected abstract onInitialized(context: system.ServiceContext): Promise; + public abstract testConnection(config: TConfig): Promise; /** @@ -93,6 +101,8 @@ export abstract class ReplicationModule context.replicationEngine?.register(this.createReplicator(context)); context.routerEngine?.registerAPI(this.createRouteAPIAdapter()); + + await this.onInitialized(context); } protected decodeConfig(config: TConfig): void { diff --git a/packages/service-core/src/routes/endpoints/socket-route.ts b/packages/service-core/src/routes/endpoints/socket-route.ts index eca06932..d21a7ccf 100644 --- a/packages/service-core/src/routes/endpoints/socket-route.ts +++ b/packages/service-core/src/routes/endpoints/socket-route.ts @@ -2,18 +2,18 @@ import { ErrorCode, errors, logger, schema } from '@powersync/lib-services-frame import { RequestParameters } from '@powersync/service-sync-rules'; import { serialize } from 'bson'; -import { Metrics } from '../../metrics/Metrics.js'; import * as sync from '../../sync/sync-index.js'; import * as util from '../../util/util-index.js'; import { SocketRouteGenerator } from '../router-socket.js'; import { SyncRoutes } from './sync-stream.js'; +import { APIMetricType } from '../../api/api-metrics.js'; export const syncStreamReactive: SocketRouteGenerator = (router) => router.reactiveStream(SyncRoutes.STREAM, { validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }), handler: async ({ context, params, responder, observer, initialN, signal: upstreamSignal }) => { const { service_context } = context; - const { routerEngine } = service_context; + const { routerEngine, metricsEngine } = service_context; // Create our own controller that we can abort directly const controller = new AbortController(); @@ -67,8 +67,8 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => controller.abort(); }); - Metrics.getInstance().concurrent_connections.add(1); - const tracker = new sync.RequestTracker(); + metricsEngine.getUpDownCounter(APIMetricType.CONCURRENT_CONNECTIONS).add(1); + const tracker = new sync.RequestTracker(metricsEngine); try { for await (const data of sync.streamResponse({ storage: activeBucketStorage, @@ -144,7 +144,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => operations_synced: tracker.operationsSynced, data_synced_bytes: tracker.dataSyncedBytes }); - Metrics.getInstance().concurrent_connections.add(-1); + metricsEngine.getUpDownCounter(APIMetricType.CONCURRENT_CONNECTIONS).add(-1); } } }); diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index aaf84ae0..1c1cfe7c 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -5,9 +5,9 @@ import { Readable } from 'stream'; import * as sync from '../../sync/sync-index.js'; import * as util from '../../util/util-index.js'; -import { Metrics } from '../../metrics/Metrics.js'; import { authUser } from '../auth.js'; import { routeDefinition } from '../router.js'; +import { APIMetricType } from '../../api/api-metrics.js'; export enum SyncRoutes { STREAM = '/sync/stream' @@ -20,7 +20,7 @@ export const syncStreamed = routeDefinition({ validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }), handler: async (payload) => { const { service_context } = payload.context; - const { routerEngine, storageEngine } = service_context; + const { routerEngine, storageEngine, metricsEngine } = service_context; const headers = payload.request.headers; const userAgent = headers['x-user-agent'] ?? headers['user-agent']; const clientId = payload.params.client_id; @@ -46,9 +46,9 @@ export const syncStreamed = routeDefinition({ }); } const controller = new AbortController(); - const tracker = new sync.RequestTracker(); + const tracker = new sync.RequestTracker(metricsEngine); try { - Metrics.getInstance().concurrent_connections.add(1); + metricsEngine.getUpDownCounter(APIMetricType.CONCURRENT_CONNECTIONS).add(1); const stream = Readable.from( sync.transformToBytesTracked( sync.ndjson( @@ -92,7 +92,7 @@ export const syncStreamed = routeDefinition({ data: stream, afterSend: async () => { controller.abort(); - Metrics.getInstance().concurrent_connections.add(-1); + metricsEngine.getUpDownCounter(APIMetricType.CONCURRENT_CONNECTIONS).add(-1); logger.info(`Sync stream complete`, { user_id: syncParams.user_id, client_id: clientId, @@ -104,7 +104,7 @@ export const syncStreamed = routeDefinition({ }); } catch (ex) { controller.abort(); - Metrics.getInstance().concurrent_connections.add(-1); + metricsEngine.getUpDownCounter(APIMetricType.CONCURRENT_CONNECTIONS).add(-1); } } }); diff --git a/packages/service-core/src/sync/RequestTracker.ts b/packages/service-core/src/sync/RequestTracker.ts index 81d717d2..a4684f5e 100644 --- a/packages/service-core/src/sync/RequestTracker.ts +++ b/packages/service-core/src/sync/RequestTracker.ts @@ -1,4 +1,5 @@ -import { Metrics } from '../metrics/Metrics.js'; +import { MetricsEngine } from '../metrics/MetricsEngine.js'; +import { APIMetricType } from '../api/api-metrics.js'; /** * Record sync stats per request stream. @@ -7,15 +8,19 @@ export class RequestTracker { operationsSynced = 0; dataSyncedBytes = 0; + constructor(private metrics: MetricsEngine) { + this.metrics = metrics; + } + addOperationsSynced(operations: number) { this.operationsSynced += operations; - Metrics.getInstance().operations_synced_total.add(operations); + this.metrics.getCounter(APIMetricType.OPERATIONS_SYNCED_TOTAL).add(operations); } addDataSynced(bytes: number) { this.dataSyncedBytes += bytes; - Metrics.getInstance().data_synced_bytes.add(bytes); + this.metrics.getCounter(APIMetricType.DATA_SYNCED_BYTES).add(bytes); } } From 4ab4112382413635b9cfe55be34e77252feac595 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Mon, 3 Mar 2025 18:19:49 +0200 Subject: [PATCH 05/18] Updated tests Added a MetricsHelper to aid with test setup for metrics --- .../module-mongodb-storage/test/src/setup.ts | 5 +- .../test/src/change_stream_utils.ts | 17 ++++-- modules/module-mongodb/test/src/env.ts | 2 +- modules/module-mongodb/test/src/setup.ts | 6 +-- .../test/src/BinLogStream.test.ts | 45 +++++++++------- .../test/src/BinlogStreamUtils.ts | 10 +++- modules/module-mysql/test/src/setup.ts | 5 +- .../test/src/large_batch.test.ts | 8 +-- modules/module-postgres/test/src/setup.ts | 5 +- .../test/src/slow_tests.test.ts | 17 ++++-- .../test/src/wal_stream.test.ts | 31 ++++++----- .../test/src/wal_stream_utils.ts | 17 ++++-- packages/service-core-tests/package.json | 3 +- .../src/test-utils/MetricsHelper.ts | 54 +++++++++++++++++++ .../src/test-utils/test-utils-index.ts | 2 +- .../src/tests/register-sync-tests.ts | 8 +-- 16 files changed, 168 insertions(+), 67 deletions(-) create mode 100644 packages/service-core-tests/src/test-utils/MetricsHelper.ts diff --git a/modules/module-mongodb-storage/test/src/setup.ts b/modules/module-mongodb-storage/test/src/setup.ts index 43946a1a..0cdd8be9 100644 --- a/modules/module-mongodb-storage/test/src/setup.ts +++ b/modules/module-mongodb-storage/test/src/setup.ts @@ -1,13 +1,12 @@ import { container } from '@powersync/lib-services-framework'; -import { test_utils } from '@powersync/service-core-tests'; import { beforeAll, beforeEach } from 'vitest'; +import { METRICS_HELPER } from '@powersync/service-core-tests'; beforeAll(async () => { // Executes for every test file container.registerDefaults(); - await test_utils.initMetrics(); }); beforeEach(async () => { - await test_utils.resetMetrics(); + METRICS_HELPER.resetMetrics(); }); diff --git a/modules/module-mongodb/test/src/change_stream_utils.ts b/modules/module-mongodb/test/src/change_stream_utils.ts index 90763fd0..0a4cfd49 100644 --- a/modules/module-mongodb/test/src/change_stream_utils.ts +++ b/modules/module-mongodb/test/src/change_stream_utils.ts @@ -1,6 +1,13 @@ import { mongo } from '@powersync/lib-service-mongodb'; -import { ActiveCheckpoint, BucketStorageFactory, OpId, SyncRulesBucketStorage } from '@powersync/service-core'; -import { test_utils } from '@powersync/service-core-tests'; +import { + ActiveCheckpoint, + BucketStorageFactory, + createCoreReplicationMetrics, + initializeCoreReplicationMetrics, + OpId, + SyncRulesBucketStorage +} from '@powersync/service-core'; +import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; import { ChangeStream, ChangeStreamOptions } from '@module/replication/ChangeStream.js'; import { MongoManager } from '@module/replication/MongoManager.js'; @@ -32,7 +39,10 @@ export class ChangeStreamTestContext { constructor( public factory: BucketStorageFactory, public connectionManager: MongoManager - ) {} + ) { + createCoreReplicationMetrics(METRICS_HELPER.metricsEngine); + initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine); + } async dispose() { this.abortController.abort(); @@ -72,6 +82,7 @@ export class ChangeStreamTestContext { } const options: ChangeStreamOptions = { storage: this.storage, + metrics: METRICS_HELPER.metricsEngine, connections: this.connectionManager, abort_signal: this.abortController.signal }; diff --git a/modules/module-mongodb/test/src/env.ts b/modules/module-mongodb/test/src/env.ts index ad0f171e..971bf241 100644 --- a/modules/module-mongodb/test/src/env.ts +++ b/modules/module-mongodb/test/src/env.ts @@ -7,5 +7,5 @@ export const env = utils.collectEnvironmentVariables({ CI: utils.type.boolean.default('false'), SLOW_TESTS: utils.type.boolean.default('false'), TEST_MONGO_STORAGE: utils.type.boolean.default('true'), - TEST_POSTGRES_STORAGE: utils.type.boolean.default('true') + TEST_POSTGRES_STORAGE: utils.type.boolean.default('false') }); diff --git a/modules/module-mongodb/test/src/setup.ts b/modules/module-mongodb/test/src/setup.ts index c89537d5..e83b0f22 100644 --- a/modules/module-mongodb/test/src/setup.ts +++ b/modules/module-mongodb/test/src/setup.ts @@ -1,15 +1,13 @@ import { container } from '@powersync/lib-services-framework'; -import { test_utils } from '@powersync/service-core-tests'; +import { METRICS_HELPER } from '@powersync/service-core-tests'; import { beforeEach } from 'node:test'; import { beforeAll } from 'vitest'; beforeAll(async () => { // Executes for every test file container.registerDefaults(); - - await test_utils.initMetrics(); }); beforeEach(async () => { - await test_utils.resetMetrics(); + METRICS_HELPER.resetMetrics(); }); diff --git a/modules/module-mysql/test/src/BinLogStream.test.ts b/modules/module-mysql/test/src/BinLogStream.test.ts index 2ed56098..4ab5e759 100644 --- a/modules/module-mysql/test/src/BinLogStream.test.ts +++ b/modules/module-mysql/test/src/BinLogStream.test.ts @@ -1,5 +1,5 @@ -import { Metrics, storage } from '@powersync/service-core'; -import { putOp, removeOp } from '@powersync/service-core-tests'; +import { ReplicationMetricType, storage } from '@powersync/service-core'; +import { METRICS_HELPER, putOp, removeOp } from '@powersync/service-core-tests'; import { v4 as uuid } from 'uuid'; import { describe, expect, test } from 'vitest'; import { BinlogStreamTestContext } from './BinlogStreamUtils.js'; @@ -35,9 +35,10 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { await context.replicateSnapshot(); - const startRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; + const startRowCount = + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; const startTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; context.startStreaming(); const testId = uuid(); @@ -47,9 +48,9 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { const data = await context.getBucketData('global[]'); expect(data).toMatchObject([putOp('test_data', { id: testId, description: 'test1', num: 1152921504606846976n })]); - const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; const endTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; expect(endRowCount - startRowCount).toEqual(1); expect(endTxCount - startTxCount).toEqual(1); }); @@ -68,9 +69,10 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { await context.replicateSnapshot(); - const startRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; + const startRowCount = + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; const startTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; context.startStreaming(); @@ -80,9 +82,9 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { const data = await context.getBucketData('global[]'); expect(data).toMatchObject([putOp('test_DATA', { id: testId, description: 'test1' })]); - const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; const endTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; expect(endRowCount - startRowCount).toEqual(1); expect(endTxCount - startTxCount).toEqual(1); }); @@ -172,10 +174,15 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { const testId = uuid(); await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('${testId}','test1')`); + const startRowCount = + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; + await context.replicateSnapshot(); + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; const data = await context.getBucketData('global[]'); expect(data).toMatchObject([putOp('test_data', { id: testId, description: 'test1' })]); + expect(endRowCount - startRowCount).toEqual(1); }); test('snapshot with date values', async () => { @@ -227,9 +234,10 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { await context.replicateSnapshot(); - const startRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; + const startRowCount = + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; const startTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; context.startStreaming(); @@ -256,9 +264,9 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { timestamp: '2023-03-06T15:47:00.000Z' }) ]); - const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; const endTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; expect(endRowCount - startRowCount).toEqual(2); expect(endTxCount - startTxCount).toEqual(2); }); @@ -272,9 +280,10 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { await context.replicateSnapshot(); - const startRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; + const startRowCount = + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; const startTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; context.startStreaming(); @@ -282,9 +291,9 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { const data = await context.getBucketData('global[]'); expect(data).toMatchObject([]); - const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; const endTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; // There was a transaction, but we should not replicate any actual data expect(endRowCount - startRowCount).toEqual(0); diff --git a/modules/module-mysql/test/src/BinlogStreamUtils.ts b/modules/module-mysql/test/src/BinlogStreamUtils.ts index 8b4c331c..ffd71ba8 100644 --- a/modules/module-mysql/test/src/BinlogStreamUtils.ts +++ b/modules/module-mysql/test/src/BinlogStreamUtils.ts @@ -5,12 +5,14 @@ import { logger } from '@powersync/lib-services-framework'; import { ActiveCheckpoint, BucketStorageFactory, + createCoreReplicationMetrics, + initializeCoreReplicationMetrics, OpId, OplogEntry, storage, SyncRulesBucketStorage } from '@powersync/service-core'; -import { test_utils } from '@powersync/service-core-tests'; +import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; import mysqlPromise from 'mysql2/promise'; import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js'; @@ -43,7 +45,10 @@ export class BinlogStreamTestContext { constructor( public factory: BucketStorageFactory, public connectionManager: MySQLConnectionManager - ) {} + ) { + createCoreReplicationMetrics(METRICS_HELPER.metricsEngine); + initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine); + } async dispose() { this.abortController.abort(); @@ -96,6 +101,7 @@ export class BinlogStreamTestContext { } const options: BinLogStreamOptions = { storage: this.storage, + metrics: METRICS_HELPER.metricsEngine, connections: this.connectionManager, abortSignal: this.abortController.signal }; diff --git a/modules/module-mysql/test/src/setup.ts b/modules/module-mysql/test/src/setup.ts index 43946a1a..1524e547 100644 --- a/modules/module-mysql/test/src/setup.ts +++ b/modules/module-mysql/test/src/setup.ts @@ -1,13 +1,12 @@ import { container } from '@powersync/lib-services-framework'; -import { test_utils } from '@powersync/service-core-tests'; +import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; import { beforeAll, beforeEach } from 'vitest'; beforeAll(async () => { // Executes for every test file container.registerDefaults(); - await test_utils.initMetrics(); }); beforeEach(async () => { - await test_utils.resetMetrics(); + METRICS_HELPER.resetMetrics(); }); diff --git a/modules/module-postgres/test/src/large_batch.test.ts b/modules/module-postgres/test/src/large_batch.test.ts index 2806bad2..84be4501 100644 --- a/modules/module-postgres/test/src/large_batch.test.ts +++ b/modules/module-postgres/test/src/large_batch.test.ts @@ -1,4 +1,4 @@ -import { Metrics, storage } from '@powersync/service-core'; +import { ReplicationMetricType, storage } from '@powersync/service-core'; import * as timers from 'timers/promises'; import { describe, expect, test } from 'vitest'; import { populateData } from '../../dist/utils/populate_test_data.js'; @@ -9,6 +9,7 @@ import { TEST_CONNECTION_OPTIONS } from './util.js'; import { WalStreamTestContext } from './wal_stream_utils.js'; +import { METRICS_HELPER } from '@powersync/service-core-tests'; describe.skipIf(!env.TEST_MONGO_STORAGE)('batch replication tests - mongodb', { timeout: 120_000 }, function () { // These are slow but consistent tests. @@ -301,12 +302,13 @@ function defineBatchTests(factory: storage.TestStorageFactory) { let done = false; - const startRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; + const startRowCount = + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; try { (async () => { while (!done) { const count = - ((await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0) - + ((await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0) - startRowCount; if (count >= stopAfter) { diff --git a/modules/module-postgres/test/src/setup.ts b/modules/module-postgres/test/src/setup.ts index 43946a1a..8d0b885e 100644 --- a/modules/module-postgres/test/src/setup.ts +++ b/modules/module-postgres/test/src/setup.ts @@ -1,13 +1,12 @@ import { container } from '@powersync/lib-services-framework'; -import { test_utils } from '@powersync/service-core-tests'; +import { METRICS_HELPER } from '@powersync/service-core-tests'; import { beforeAll, beforeEach } from 'vitest'; beforeAll(async () => { // Executes for every test file container.registerDefaults(); - await test_utils.initMetrics(); }); beforeEach(async () => { - await test_utils.resetMetrics(); + METRICS_HELPER.resetMetrics(); }); diff --git a/modules/module-postgres/test/src/slow_tests.test.ts b/modules/module-postgres/test/src/slow_tests.test.ts index 438d3bb9..f6fcf76d 100644 --- a/modules/module-postgres/test/src/slow_tests.test.ts +++ b/modules/module-postgres/test/src/slow_tests.test.ts @@ -1,5 +1,5 @@ import * as bson from 'bson'; -import { afterEach, describe, expect, test } from 'vitest'; +import { afterEach, beforeAll, describe, expect, test } from 'vitest'; import { WalStream, WalStreamOptions } from '../../src/replication/WalStream.js'; import { env } from './env.js'; import { @@ -15,8 +15,8 @@ import * as pgwire from '@powersync/service-jpgwire'; import { SqliteRow } from '@powersync/service-sync-rules'; import { PgManager } from '@module/replication/PgManager.js'; -import { storage } from '@powersync/service-core'; -import { test_utils } from '@powersync/service-core-tests'; +import { createCoreReplicationMetrics, initializeCoreReplicationMetrics, storage } from '@powersync/service-core'; +import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; import * as mongo_storage from '@powersync/service-module-mongodb-storage'; import * as postgres_storage from '@powersync/service-module-postgres-storage'; import * as timers from 'node:timers/promises'; @@ -49,6 +49,11 @@ function defineSlowTests(factory: storage.TestStorageFactory) { let abortController: AbortController | undefined; let streamPromise: Promise | undefined; + beforeAll(async () => { + createCoreReplicationMetrics(METRICS_HELPER.metricsEngine); + initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine); + }); + afterEach(async () => { // This cleans up, similar to WalStreamTestContext.dispose(). // These tests are a little more complex than what is supported by WalStreamTestContext. @@ -106,7 +111,8 @@ bucket_definitions: const options: WalStreamOptions = { abort_signal: abortController.signal, connections, - storage: storage + storage: storage, + metrics: METRICS_HELPER.metricsEngine }; walStream = new WalStream(options); @@ -358,7 +364,8 @@ bucket_definitions: const options: WalStreamOptions = { abort_signal: abortController.signal, connections, - storage: storage + storage: storage, + metrics: METRICS_HELPER.metricsEngine }; walStream = new WalStream(options); diff --git a/modules/module-postgres/test/src/wal_stream.test.ts b/modules/module-postgres/test/src/wal_stream.test.ts index fea60664..2397272a 100644 --- a/modules/module-postgres/test/src/wal_stream.test.ts +++ b/modules/module-postgres/test/src/wal_stream.test.ts @@ -1,6 +1,6 @@ import { MissingReplicationSlotError } from '@module/replication/WalStream.js'; -import { Metrics, storage } from '@powersync/service-core'; -import { putOp, removeOp } from '@powersync/service-core-tests'; +import { ReplicationMetricType, storage } from '@powersync/service-core'; +import { METRICS_HELPER, putOp, removeOp } from '@powersync/service-core-tests'; import { pgwireRows } from '@powersync/service-jpgwire'; import * as crypto from 'crypto'; import { describe, expect, test } from 'vitest'; @@ -40,9 +40,10 @@ bucket_definitions: await context.replicateSnapshot(); - const startRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; + const startRowCount = + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; const startTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; context.startStreaming(); @@ -55,9 +56,9 @@ bucket_definitions: const data = await context.getBucketData('global[]'); expect(data).toMatchObject([putOp('test_data', { id: test_id, description: 'test1', num: 1152921504606846976n })]); - const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; const endTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; expect(endRowCount - startRowCount).toEqual(1); expect(endTxCount - startTxCount).toEqual(1); }); @@ -77,9 +78,10 @@ bucket_definitions: await context.replicateSnapshot(); - const startRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; + const startRowCount = + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; const startTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; context.startStreaming(); @@ -90,9 +92,9 @@ bucket_definitions: const data = await context.getBucketData('global[]'); expect(data).toMatchObject([putOp('test_DATA', { id: test_id, description: 'test1' })]); - const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; const endTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; expect(endRowCount - startRowCount).toEqual(1); expect(endTxCount - startTxCount).toEqual(1); }); @@ -274,9 +276,10 @@ bucket_definitions: await context.replicateSnapshot(); - const startRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; + const startRowCount = + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; const startTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; context.startStreaming(); @@ -287,9 +290,9 @@ bucket_definitions: const data = await context.getBucketData('global[]'); expect(data).toMatchObject([]); - const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; const endTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; // There was a transaction, but we should not replicate any actual data expect(endRowCount - startRowCount).toEqual(0); diff --git a/modules/module-postgres/test/src/wal_stream_utils.ts b/modules/module-postgres/test/src/wal_stream_utils.ts index 25af4347..600da5e2 100644 --- a/modules/module-postgres/test/src/wal_stream_utils.ts +++ b/modules/module-postgres/test/src/wal_stream_utils.ts @@ -1,7 +1,14 @@ import { PgManager } from '@module/replication/PgManager.js'; import { PUBLICATION_NAME, WalStream, WalStreamOptions } from '@module/replication/WalStream.js'; -import { BucketStorageFactory, OplogEntry, storage, SyncRulesBucketStorage } from '@powersync/service-core'; -import { test_utils } from '@powersync/service-core-tests'; +import { + BucketStorageFactory, + createCoreReplicationMetrics, + initializeCoreReplicationMetrics, + OplogEntry, + storage, + SyncRulesBucketStorage +} from '@powersync/service-core'; +import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; import * as pgwire from '@powersync/service-jpgwire'; import { clearTestDb, getClientCheckpoint, TEST_CONNECTION_OPTIONS } from './util.js'; @@ -35,7 +42,10 @@ export class WalStreamTestContext implements AsyncDisposable { constructor( public factory: BucketStorageFactory, public connectionManager: PgManager - ) {} + ) { + createCoreReplicationMetrics(METRICS_HELPER.metricsEngine); + initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine); + } async [Symbol.asyncDispose]() { await this.dispose(); @@ -96,6 +106,7 @@ export class WalStreamTestContext implements AsyncDisposable { } const options: WalStreamOptions = { storage: this.storage, + metrics: METRICS_HELPER.metricsEngine, connections: this.connectionManager, abort_signal: this.abortController.signal }; diff --git a/packages/service-core-tests/package.json b/packages/service-core-tests/package.json index 6a80d237..a3655236 100644 --- a/packages/service-core-tests/package.json +++ b/packages/service-core-tests/package.json @@ -23,6 +23,7 @@ "vitest": "^3.0.5" }, "devDependencies": { - "typescript": "^5.7.3" + "typescript": "^5.7.3", + "@opentelemetry/sdk-metrics": "^1.30.1" } } diff --git a/packages/service-core-tests/src/test-utils/MetricsHelper.ts b/packages/service-core-tests/src/test-utils/MetricsHelper.ts new file mode 100644 index 00000000..7e1553a1 --- /dev/null +++ b/packages/service-core-tests/src/test-utils/MetricsHelper.ts @@ -0,0 +1,54 @@ +import { MetricsEngine, MetricsFactory, OpenTelemetryMetricsFactory } from '@powersync/service-core'; +import { + AggregationTemporality, + InMemoryMetricExporter, + MeterProvider, + PeriodicExportingMetricReader +} from '@opentelemetry/sdk-metrics'; + +export class MetricsHelper { + public factory: MetricsFactory; + public metricsEngine: MetricsEngine; + private meterProvider: MeterProvider; + private exporter: InMemoryMetricExporter; + private metricReader: PeriodicExportingMetricReader; + + constructor() { + this.exporter = new InMemoryMetricExporter(AggregationTemporality.CUMULATIVE); + + this.metricReader = new PeriodicExportingMetricReader({ + exporter: this.exporter, + exportIntervalMillis: 100 // Export quickly for tests + }); + this.meterProvider = new MeterProvider({ + readers: [this.metricReader] + }); + const meter = this.meterProvider.getMeter('powersync-tests'); + this.factory = new OpenTelemetryMetricsFactory(meter); + this.metricsEngine = new MetricsEngine({ + factory: this.factory, + disable_telemetry_sharing: true + }); + } + + resetMetrics() { + this.exporter.reset(); + } + + async getMetricValueForTests(name: string): Promise { + await this.metricReader.forceFlush(); + const metrics = this.exporter.getMetrics(); + // Use the latest reported ResourceMetric + const scoped = metrics[metrics.length - 1]?.scopeMetrics?.[0]?.metrics; + const metric = scoped?.find((metric) => metric.descriptor.name == name); + if (metric == null) { + throw new Error( + `Cannot find metric ${name}. Options: ${scoped.map((metric) => metric.descriptor.name).join(',')}` + ); + } + const point = metric.dataPoints[metric.dataPoints.length - 1]; + return point?.value as number; + } +} + +export const METRICS_HELPER = new MetricsHelper(); diff --git a/packages/service-core-tests/src/test-utils/test-utils-index.ts b/packages/service-core-tests/src/test-utils/test-utils-index.ts index 1e4cbea0..1b174d84 100644 --- a/packages/service-core-tests/src/test-utils/test-utils-index.ts +++ b/packages/service-core-tests/src/test-utils/test-utils-index.ts @@ -1,4 +1,4 @@ export * from './bucket-validation.js'; export * from './general-utils.js'; -export * from './metrics-utils.js'; +export * from './MetricsHelper.js'; export * from './stream_utils.js'; diff --git a/packages/service-core-tests/src/tests/register-sync-tests.ts b/packages/service-core-tests/src/tests/register-sync-tests.ts index d96dc34a..d517c1f2 100644 --- a/packages/service-core-tests/src/tests/register-sync-tests.ts +++ b/packages/service-core-tests/src/tests/register-sync-tests.ts @@ -1,4 +1,4 @@ -import { storage, sync, utils } from '@powersync/service-core'; +import { createCoreAPIMetrics, storage, sync, utils } from '@powersync/service-core'; import { JSONBig } from '@powersync/service-jsonbig'; import { RequestParameters } from '@powersync/service-sync-rules'; import path from 'path'; @@ -6,6 +6,7 @@ import * as timers from 'timers/promises'; import { fileURLToPath } from 'url'; import { expect, test } from 'vitest'; import * as test_utils from '../test-utils/test-utils-index.js'; +import { METRICS_HELPER } from '../test-utils/test-utils-index.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -30,7 +31,8 @@ export const SYNC_SNAPSHOT_PATH = path.resolve(__dirname, '../__snapshots/sync.t * ``` */ export function registerSyncTests(factory: storage.TestStorageFactory) { - const tracker = new sync.RequestTracker(); + createCoreAPIMetrics(METRICS_HELPER.metricsEngine); + const tracker = new sync.RequestTracker(METRICS_HELPER.metricsEngine); test('sync global data', async () => { await using f = await factory(); @@ -220,7 +222,7 @@ bucket_definitions: }, afterReplicaId: 'highprio2' }); - + await batch.commit('0/2'); }); } From 8e9f75b82cfed298fb402c108c15f80e3b8f867e Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Mon, 3 Mar 2025 18:20:25 +0200 Subject: [PATCH 06/18] Updated Opentelemetry dependencies --- packages/service-core/package.json | 10 +- pnpm-lock.yaml | 327 ++++++++++++++++------------- 2 files changed, 191 insertions(+), 146 deletions(-) diff --git a/packages/service-core/package.json b/packages/service-core/package.json index 9f2ed28e..d2e9d16f 100644 --- a/packages/service-core/package.json +++ b/packages/service-core/package.json @@ -17,11 +17,11 @@ }, "dependencies": { "@js-sdsl/ordered-set": "^4.4.2", - "@opentelemetry/api": "~1.8.0", - "@opentelemetry/exporter-metrics-otlp-http": "^0.51.1", - "@opentelemetry/exporter-prometheus": "^0.51.1", - "@opentelemetry/resources": "^1.24.1", - "@opentelemetry/sdk-metrics": "1.24.1", + "@opentelemetry/api": "~1.9.0", + "@opentelemetry/exporter-metrics-otlp-http": "^0.57.2", + "@opentelemetry/exporter-prometheus": "^0.57.2", + "@opentelemetry/resources": "^1.30.1", + "@opentelemetry/sdk-metrics": "1.30.1", "@powersync/lib-services-framework": "workspace:*", "@powersync/service-jsonbig": "workspace:*", "@powersync/service-rsocket-router": "workspace:*", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8581c3f1..ae5fca63 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -464,20 +464,20 @@ importers: specifier: ^4.4.2 version: 4.4.2 '@opentelemetry/api': - specifier: ~1.8.0 - version: 1.8.0 + specifier: ~1.9.0 + version: 1.9.0 '@opentelemetry/exporter-metrics-otlp-http': - specifier: ^0.51.1 - version: 0.51.1(@opentelemetry/api@1.8.0) + specifier: ^0.57.2 + version: 0.57.2(@opentelemetry/api@1.9.0) '@opentelemetry/exporter-prometheus': - specifier: ^0.51.1 - version: 0.51.1(@opentelemetry/api@1.8.0) + specifier: ^0.57.2 + version: 0.57.2(@opentelemetry/api@1.9.0) '@opentelemetry/resources': - specifier: ^1.24.1 - version: 1.25.1(@opentelemetry/api@1.8.0) + specifier: ^1.30.1 + version: 1.30.1(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-metrics': - specifier: 1.24.1 - version: 1.24.1(@opentelemetry/api@1.8.0) + specifier: 1.30.1 + version: 1.30.1(@opentelemetry/api@1.9.0) '@powersync/lib-services-framework': specifier: workspace:* version: link:../../libs/lib-services @@ -576,6 +576,9 @@ importers: specifier: ^3.0.5 version: 3.0.5(@types/node@22.13.1)(yaml@2.5.0) devDependencies: + '@opentelemetry/sdk-metrics': + specifier: ^1.30.1 + version: 1.30.1(@opentelemetry/api@1.9.0) typescript: specifier: ^5.7.3 version: 5.7.3 @@ -1101,22 +1104,18 @@ packages: resolution: {integrity: sha512-NCcr1uQo1k5U+SYlnIrbAh3cxy+OQT1VtqiAbxdymSlptbzBb62AjH2xXgjNCoP073hoa1CfCAcwoZ8k96C4nA==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} - '@opentelemetry/api-logs@0.51.1': - resolution: {integrity: sha512-E3skn949Pk1z2XtXu/lxf6QAZpawuTM/IUEXcAzpiUkTd73Hmvw26FiN3cJuTmkpM5hZzHwkomVdtrh/n/zzwA==} - engines: {node: '>=14'} - '@opentelemetry/api-logs@0.52.1': resolution: {integrity: sha512-qnSqB2DQ9TPP96dl8cDubDvrUyWc0/sK81xHTK8eSUspzDM3bsewX903qclQFvVhgStjRWdC5bLb3kQqMkfV5A==} engines: {node: '>=14'} + '@opentelemetry/api-logs@0.57.2': + resolution: {integrity: sha512-uIX52NnTM0iBh84MShlpouI7UKqkZ7MrUszTmaypHBu4r7NofznSnQRfJ+uUeDtQDj6w8eFGg5KBLDAwAPz1+A==} + engines: {node: '>=14'} + '@opentelemetry/api@1.6.0': resolution: {integrity: sha512-OWlrQAnWn9577PhVgqjUvMr1pg57Bc4jv0iL4w0PRuOSRvq67rvHW9Ie/dZVMvCzhSCB+UxhcY/PmCmFj33Q+g==} engines: {node: '>=8.0.0'} - '@opentelemetry/api@1.8.0': - resolution: {integrity: sha512-I/s6F7yKUDdtMsoBWXJe8Qz40Tui5vsuKCWJEWVL+5q9sSWRzzx6v2KeNsOBEwd94j0eWkpWCH4yB6rZg9Mf0w==} - engines: {node: '>=8.0.0'} - '@opentelemetry/api@1.9.0': resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} engines: {node: '>=8.0.0'} @@ -1145,8 +1144,14 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/exporter-metrics-otlp-http@0.51.1': - resolution: {integrity: sha512-oFXvif9iksHUxrzG3P8ohMLt7xSrl+oDMqxD/3XXndU761RFAKSbRDpfrQs25U5D+A2aMV3qk+4kfUWdJhZ77g==} + '@opentelemetry/core@1.30.1': + resolution: {integrity: sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/exporter-metrics-otlp-http@0.57.2': + resolution: {integrity: sha512-ttb9+4iKw04IMubjm3t0EZsYRNWr3kg44uUuzfo9CaccYlOh8cDooe4QObDUkvx9d5qQUrbEckhrWKfJnKhemA==} engines: {node: '>=14'} peerDependencies: '@opentelemetry/api': ^1.3.0 @@ -1157,8 +1162,8 @@ packages: peerDependencies: '@opentelemetry/api': ^1.3.0 - '@opentelemetry/exporter-prometheus@0.51.1': - resolution: {integrity: sha512-c8TrTlLm9JJRIHW6MtFv6ESoZRgXBXD/YrTRYylWiyYBOVbYHo1c5Qaw/j/thXDhkmYOYAn4LAhJZpLl5gBFEQ==} + '@opentelemetry/exporter-prometheus@0.57.2': + resolution: {integrity: sha512-VqIqXnuxWMWE/1NatAGtB1PvsQipwxDcdG4RwA/umdBcW3/iOHp0uejvFHTRN2O78ZPged87ErJajyUBPUhlDQ==} engines: {node: '>=14'} peerDependencies: '@opentelemetry/api': ^1.3.0 @@ -1265,17 +1270,17 @@ packages: peerDependencies: '@opentelemetry/api': ^1.3.0 - '@opentelemetry/otlp-exporter-base@0.51.1': - resolution: {integrity: sha512-UYlnOYyDdzo1Gw559EHCzru0RwhvuXCwoH8jGo9J4gO1TE58GjnEmIjomMsKBCym3qWNJfIQXw+9SZCV0DdQNg==} + '@opentelemetry/otlp-exporter-base@0.57.2': + resolution: {integrity: sha512-XdxEzL23Urhidyebg5E6jZoaiW5ygP/mRjxLHixogbqwDy2Faduzb5N0o/Oi+XTIJu+iyxXdVORjXax+Qgfxag==} engines: {node: '>=14'} peerDependencies: - '@opentelemetry/api': ^1.0.0 + '@opentelemetry/api': ^1.3.0 - '@opentelemetry/otlp-transformer@0.51.1': - resolution: {integrity: sha512-OppYOXwV9LQqqtYUCywqoOqX/JT9LQ5/FMuPZ//eTkvuHdUC4ZMwz2c6uSoT2R90GWvvGnF1iEqTGyTT3xAt2Q==} + '@opentelemetry/otlp-transformer@0.57.2': + resolution: {integrity: sha512-48IIRj49gbQVK52jYsw70+Jv+JbahT8BqT2Th7C4H7RCM9d0gZ5sgNPoMpWldmfjvIsSgiGJtjfk9MeZvjhoig==} engines: {node: '>=14'} peerDependencies: - '@opentelemetry/api': '>=1.3.0 <1.9.0' + '@opentelemetry/api': ^1.3.0 '@opentelemetry/redis-common@0.36.2': resolution: {integrity: sha512-faYX1N0gpLhej/6nyp6bgRjzAKXn5GOEMYY7YhciSfCoITAktLUtQ36d24QEWNA1/WA1y6qQunCe0OhHRkVl9g==} @@ -1299,12 +1304,17 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/sdk-logs@0.51.1': - resolution: {integrity: sha512-ULQQtl82b673PpZc5/0EtH4V+BrwVOgKJZEB7tYZnGTG3I98tQVk89S9/JSixomDr++F4ih+LSJTCqIKBz+MQQ==} + '@opentelemetry/resources@1.30.1': + resolution: {integrity: sha512-5UxZqiAgLYGFjS4s9qm5mBVo433u+dSPUFWVWXmLAD4wB65oMCoXaJP1KJa9DIYYMeHu3z4BZcStG3LC593cWA==} engines: {node: '>=14'} peerDependencies: - '@opentelemetry/api': '>=1.4.0 <1.9.0' - '@opentelemetry/api-logs': '>=0.39.1' + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/sdk-logs@0.57.2': + resolution: {integrity: sha512-TXFHJ5c+BKggWbdEQ/inpgIzEmS2BGQowLE9UhsMd7YYlUfBQJ4uax0VF/B5NYigdM/75OoJGhAV3upEhK+3gg==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.4.0 <1.10.0' '@opentelemetry/sdk-metrics@1.17.0': resolution: {integrity: sha512-HlWM27yGmYuwCoVRe3yg2PqKnIsq0kEF0HQgvkeDWz2NYkq9fFaSspR6kvjxUTbghAlZrabiqbgyKoYpYaXS3w==} @@ -1318,11 +1328,11 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.3.0 <1.9.0' - '@opentelemetry/sdk-trace-base@1.24.1': - resolution: {integrity: sha512-zz+N423IcySgjihl2NfjBf0qw1RWe11XIAWVrTNOSSI6dtSPJiVom2zipFB2AEEtJWpv0Iz6DY6+TjnyTV5pWg==} + '@opentelemetry/sdk-metrics@1.30.1': + resolution: {integrity: sha512-q9zcZ0Okl8jRgmy7eNW3Ku1XSgg3sDLa5evHZpCwjspw7E8Is4K/haRPDJrBcX3YSn/Y7gUvFnByNYEKQNbNog==} engines: {node: '>=14'} peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.9.0' + '@opentelemetry/api': '>=1.3.0 <1.10.0' '@opentelemetry/sdk-trace-base@1.25.1': resolution: {integrity: sha512-C8k4hnEbc5FamuZQ92nTOp8X/diCY56XUTnMiv9UTuJitCzaNNHAVsdm5+HLCdI8SLQsLWIrG38tddMxLVoftw==} @@ -1330,6 +1340,12 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' + '@opentelemetry/sdk-trace-base@1.30.1': + resolution: {integrity: sha512-jVPgBbH1gCy2Lb7X0AVQ8XAfgg0pJ4nvl8/IiQA6nxOsPvS+0zMJaFSs2ltXe0J6C8dqjcnpyqINDJmU30+uOg==} + engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + '@opentelemetry/semantic-conventions@1.17.0': resolution: {integrity: sha512-+fguCd2d8d2qruk0H0DsCEy2CTK3t0Tugg7MhZ/UQMvmewbZLNnJ6heSYyzIZWG5IPfAXzoj4f4F/qpM7l4VBA==} engines: {node: '>=14'} @@ -1342,6 +1358,10 @@ packages: resolution: {integrity: sha512-ZDjMJJQRlyk8A1KZFCc+bCbsyrn1wTwdNt56F7twdfUfnHUZUq77/WfONCj8p72NZOyP7pNTdUWSTYC3GTbuuQ==} engines: {node: '>=14'} + '@opentelemetry/semantic-conventions@1.28.0': + resolution: {integrity: sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==} + engines: {node: '>=14'} + '@opentelemetry/sql-common@0.40.1': resolution: {integrity: sha512-nSDlnHSqzC3pXn/wZEZVLuAuJ1MYMXPBwtv2qAbCa3847SaHItdE7SzUq/Jtb0KZmh1zfAbNi3AAMjztTT4Ugg==} engines: {node: '>=14'} @@ -1374,6 +1394,36 @@ packages: '@prisma/instrumentation@5.16.1': resolution: {integrity: sha512-4m5gRFWnQb8s/yTyGbMZkL7A5uJgqOWcWJxapwcAD0T0kh5sGPEVSQl/zTQvE9aduXhFAxOtC3gO+R8Hb5xO1Q==} + '@protobufjs/aspromise@1.1.2': + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + + '@protobufjs/base64@1.1.2': + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + + '@protobufjs/codegen@2.0.4': + resolution: {integrity: sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==} + + '@protobufjs/eventemitter@1.1.0': + resolution: {integrity: sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==} + + '@protobufjs/fetch@1.1.0': + resolution: {integrity: sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==} + + '@protobufjs/float@1.0.2': + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + + '@protobufjs/inquire@1.1.0': + resolution: {integrity: sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==} + + '@protobufjs/path@1.1.2': + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + + '@protobufjs/pool@1.1.0': + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + + '@protobufjs/utf8@1.1.0': + resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} + '@rollup/rollup-android-arm-eabi@4.34.8': resolution: {integrity: sha512-q217OSE8DTp8AFHuNHXo0Y86e1wtlfVrXiAlwkIvGRQv9zbc6mE3sjIVfwI8sYUyNxwOg0j/Vm1RKM04JcWLJw==} cpu: [arm] @@ -3268,6 +3318,10 @@ packages: proto-list@1.2.4: resolution: {integrity: sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==} + protobufjs@7.4.0: + resolution: {integrity: sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==} + engines: {node: '>=12.0.0'} + proxy-addr@2.0.7: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} @@ -4508,18 +4562,16 @@ snapshots: - bluebird - supports-color - '@opentelemetry/api-logs@0.51.1': + '@opentelemetry/api-logs@0.52.1': dependencies: - '@opentelemetry/api': 1.8.0 + '@opentelemetry/api': 1.9.0 - '@opentelemetry/api-logs@0.52.1': + '@opentelemetry/api-logs@0.57.2': dependencies: - '@opentelemetry/api': 1.6.0 + '@opentelemetry/api': 1.9.0 '@opentelemetry/api@1.6.0': {} - '@opentelemetry/api@1.8.0': {} - '@opentelemetry/api@1.9.0': {} '@opentelemetry/context-async-hooks@1.25.1(@opentelemetry/api@1.9.0)': @@ -4536,34 +4588,24 @@ snapshots: '@opentelemetry/api': 1.6.0 '@opentelemetry/semantic-conventions': 1.24.1 - '@opentelemetry/core@1.24.1(@opentelemetry/api@1.8.0)': - dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/semantic-conventions': 1.24.1 - - '@opentelemetry/core@1.24.1(@opentelemetry/api@1.9.0)': + '@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/semantic-conventions': 1.24.1 - - '@opentelemetry/core@1.25.1(@opentelemetry/api@1.8.0)': - dependencies: - '@opentelemetry/api': 1.8.0 '@opentelemetry/semantic-conventions': 1.25.1 - '@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0)': + '@opentelemetry/core@1.30.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/semantic-conventions': 1.25.1 + '@opentelemetry/semantic-conventions': 1.28.0 - '@opentelemetry/exporter-metrics-otlp-http@0.51.1(@opentelemetry/api@1.8.0)': + '@opentelemetry/exporter-metrics-otlp-http@0.57.2(@opentelemetry/api@1.9.0)': dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/core': 1.24.1(@opentelemetry/api@1.8.0) - '@opentelemetry/otlp-exporter-base': 0.51.1(@opentelemetry/api@1.8.0) - '@opentelemetry/otlp-transformer': 0.51.1(@opentelemetry/api@1.8.0) - '@opentelemetry/resources': 1.24.1(@opentelemetry/api@1.8.0) - '@opentelemetry/sdk-metrics': 1.24.1(@opentelemetry/api@1.8.0) + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.57.2(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.57.2(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 1.30.1(@opentelemetry/api@1.9.0) '@opentelemetry/exporter-prometheus@0.43.0(@opentelemetry/api@1.6.0)': dependencies: @@ -4572,12 +4614,12 @@ snapshots: '@opentelemetry/resources': 1.17.0(@opentelemetry/api@1.6.0) '@opentelemetry/sdk-metrics': 1.17.0(@opentelemetry/api@1.6.0) - '@opentelemetry/exporter-prometheus@0.51.1(@opentelemetry/api@1.8.0)': + '@opentelemetry/exporter-prometheus@0.57.2(@opentelemetry/api@1.9.0)': dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/core': 1.24.1(@opentelemetry/api@1.8.0) - '@opentelemetry/resources': 1.24.1(@opentelemetry/api@1.8.0) - '@opentelemetry/sdk-metrics': 1.24.1(@opentelemetry/api@1.8.0) + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 1.30.1(@opentelemetry/api@1.9.0) '@opentelemetry/instrumentation-connect@0.38.0(@opentelemetry/api@1.9.0)': dependencies: @@ -4655,7 +4697,7 @@ snapshots: dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/instrumentation': 0.52.1(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-metrics': 1.24.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 1.30.1(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': 1.25.1 transitivePeerDependencies: - supports-color @@ -4727,18 +4769,6 @@ snapshots: - supports-color optional: true - '@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.8.0)': - dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/api-logs': 0.52.1 - '@types/shimmer': 1.2.0 - import-in-the-middle: 1.9.0 - require-in-the-middle: 7.3.0 - semver: 7.6.2 - shimmer: 1.2.1 - transitivePeerDependencies: - - supports-color - '@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -4751,20 +4781,22 @@ snapshots: transitivePeerDependencies: - supports-color - '@opentelemetry/otlp-exporter-base@0.51.1(@opentelemetry/api@1.8.0)': + '@opentelemetry/otlp-exporter-base@0.57.2(@opentelemetry/api@1.9.0)': dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/core': 1.24.1(@opentelemetry/api@1.8.0) + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.57.2(@opentelemetry/api@1.9.0) - '@opentelemetry/otlp-transformer@0.51.1(@opentelemetry/api@1.8.0)': + '@opentelemetry/otlp-transformer@0.57.2(@opentelemetry/api@1.9.0)': dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/api-logs': 0.51.1 - '@opentelemetry/core': 1.24.1(@opentelemetry/api@1.8.0) - '@opentelemetry/resources': 1.24.1(@opentelemetry/api@1.8.0) - '@opentelemetry/sdk-logs': 0.51.1(@opentelemetry/api-logs@0.51.1)(@opentelemetry/api@1.8.0) - '@opentelemetry/sdk-metrics': 1.24.1(@opentelemetry/api@1.8.0) - '@opentelemetry/sdk-trace-base': 1.24.1(@opentelemetry/api@1.8.0) + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.57.2 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-logs': 0.57.2(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 1.30.1(@opentelemetry/api@1.9.0) + protobufjs: 7.4.0 '@opentelemetry/redis-common@0.36.2': {} @@ -4780,36 +4812,24 @@ snapshots: '@opentelemetry/core': 1.24.1(@opentelemetry/api@1.6.0) '@opentelemetry/semantic-conventions': 1.24.1 - '@opentelemetry/resources@1.24.1(@opentelemetry/api@1.8.0)': - dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/core': 1.24.1(@opentelemetry/api@1.8.0) - '@opentelemetry/semantic-conventions': 1.24.1 - - '@opentelemetry/resources@1.24.1(@opentelemetry/api@1.9.0)': + '@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/core': 1.24.1(@opentelemetry/api@1.9.0) - '@opentelemetry/semantic-conventions': 1.24.1 - - '@opentelemetry/resources@1.25.1(@opentelemetry/api@1.8.0)': - dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/core': 1.25.1(@opentelemetry/api@1.8.0) + '@opentelemetry/core': 1.25.1(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': 1.25.1 - '@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0)': + '@opentelemetry/resources@1.30.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/core': 1.25.1(@opentelemetry/api@1.9.0) - '@opentelemetry/semantic-conventions': 1.25.1 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.28.0 - '@opentelemetry/sdk-logs@0.51.1(@opentelemetry/api-logs@0.51.1)(@opentelemetry/api@1.8.0)': + '@opentelemetry/sdk-logs@0.57.2(@opentelemetry/api@1.9.0)': dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/api-logs': 0.51.1 - '@opentelemetry/core': 1.24.1(@opentelemetry/api@1.8.0) - '@opentelemetry/resources': 1.24.1(@opentelemetry/api@1.8.0) + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.57.2 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-metrics@1.17.0(@opentelemetry/api@1.6.0)': dependencies: @@ -4825,33 +4845,11 @@ snapshots: '@opentelemetry/resources': 1.24.1(@opentelemetry/api@1.6.0) lodash.merge: 4.6.2 - '@opentelemetry/sdk-metrics@1.24.1(@opentelemetry/api@1.8.0)': - dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/core': 1.24.1(@opentelemetry/api@1.8.0) - '@opentelemetry/resources': 1.24.1(@opentelemetry/api@1.8.0) - lodash.merge: 4.6.2 - - '@opentelemetry/sdk-metrics@1.24.1(@opentelemetry/api@1.9.0)': + '@opentelemetry/sdk-metrics@1.30.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/core': 1.24.1(@opentelemetry/api@1.9.0) - '@opentelemetry/resources': 1.24.1(@opentelemetry/api@1.9.0) - lodash.merge: 4.6.2 - - '@opentelemetry/sdk-trace-base@1.24.1(@opentelemetry/api@1.8.0)': - dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/core': 1.24.1(@opentelemetry/api@1.8.0) - '@opentelemetry/resources': 1.24.1(@opentelemetry/api@1.8.0) - '@opentelemetry/semantic-conventions': 1.24.1 - - '@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.8.0)': - dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/core': 1.25.1(@opentelemetry/api@1.8.0) - '@opentelemetry/resources': 1.25.1(@opentelemetry/api@1.8.0) - '@opentelemetry/semantic-conventions': 1.25.1 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0)': dependencies: @@ -4860,12 +4858,21 @@ snapshots: '@opentelemetry/resources': 1.25.1(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': 1.25.1 + '@opentelemetry/sdk-trace-base@1.30.1(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.28.0 + '@opentelemetry/semantic-conventions@1.17.0': {} '@opentelemetry/semantic-conventions@1.24.1': {} '@opentelemetry/semantic-conventions@1.25.1': {} + '@opentelemetry/semantic-conventions@1.28.0': {} + '@opentelemetry/sql-common@0.40.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -4898,12 +4905,35 @@ snapshots: '@prisma/instrumentation@5.16.1': dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/instrumentation': 0.52.1(@opentelemetry/api@1.8.0) - '@opentelemetry/sdk-trace-base': 1.25.1(@opentelemetry/api@1.8.0) + '@opentelemetry/api': 1.9.0 + '@opentelemetry/instrumentation': 0.52.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 1.25.1(@opentelemetry/api@1.9.0) transitivePeerDependencies: - supports-color + '@protobufjs/aspromise@1.1.2': {} + + '@protobufjs/base64@1.1.2': {} + + '@protobufjs/codegen@2.0.4': {} + + '@protobufjs/eventemitter@1.1.0': {} + + '@protobufjs/fetch@1.1.0': + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/inquire': 1.1.0 + + '@protobufjs/float@1.0.2': {} + + '@protobufjs/inquire@1.1.0': {} + + '@protobufjs/path@1.1.2': {} + + '@protobufjs/pool@1.1.0': {} + + '@protobufjs/utf8@1.1.0': {} + '@rollup/rollup-android-arm-eabi@4.34.8': optional: true @@ -4987,7 +5017,7 @@ snapshots: '@opentelemetry/instrumentation-nestjs-core': 0.39.0(@opentelemetry/api@1.9.0) '@opentelemetry/instrumentation-pg': 0.43.0(@opentelemetry/api@1.9.0) '@opentelemetry/instrumentation-redis-4': 0.41.0(@opentelemetry/api@1.9.0) - '@opentelemetry/resources': 1.25.1(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-base': 1.25.1(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': 1.25.1 '@prisma/instrumentation': 5.16.1 @@ -6874,6 +6904,21 @@ snapshots: proto-list@1.2.4: {} + protobufjs@7.4.0: + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.4 + '@protobufjs/eventemitter': 1.1.0 + '@protobufjs/fetch': 1.1.0 + '@protobufjs/float': 1.0.2 + '@protobufjs/inquire': 1.1.0 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.0 + '@types/node': 22.13.1 + long: 5.2.3 + proxy-addr@2.0.7: dependencies: forwarded: 0.2.0 From 5dc75993dfc707e45c63b2680bacc3d947c334ce Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Mon, 3 Mar 2025 18:24:23 +0200 Subject: [PATCH 07/18] Remove unused import --- modules/module-mysql/test/src/setup.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/module-mysql/test/src/setup.ts b/modules/module-mysql/test/src/setup.ts index 1524e547..8d0b885e 100644 --- a/modules/module-mysql/test/src/setup.ts +++ b/modules/module-mysql/test/src/setup.ts @@ -1,5 +1,5 @@ import { container } from '@powersync/lib-services-framework'; -import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; +import { METRICS_HELPER } from '@powersync/service-core-tests'; import { beforeAll, beforeEach } from 'vitest'; beforeAll(async () => { From 9495d49539981460a905040e6f1968c129b7f632 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Mon, 3 Mar 2025 19:23:56 +0200 Subject: [PATCH 08/18] Branch update fixes --- .../module-mongodb/test/src/change_stream_utils.ts | 12 +++++++++--- modules/module-postgres-storage/test/src/setup.ts | 10 ++-------- packages/service-core/src/storage/storage-metrics.ts | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/modules/module-mongodb/test/src/change_stream_utils.ts b/modules/module-mongodb/test/src/change_stream_utils.ts index eb41a74e..2d8caf20 100644 --- a/modules/module-mongodb/test/src/change_stream_utils.ts +++ b/modules/module-mongodb/test/src/change_stream_utils.ts @@ -1,7 +1,13 @@ import { mongo } from '@powersync/lib-service-mongodb'; -import { BucketStorageFactory, OpId, createCoreReplicationMetrics, - initializeCoreReplicationMetrics, ReplicationCheckpoint, SyncRulesBucketStorage } from '@powersync/service-core'; -import { test_utils } from '@powersync/service-core-tests'; +import { + BucketStorageFactory, + OpId, + createCoreReplicationMetrics, + initializeCoreReplicationMetrics, + ReplicationCheckpoint, + SyncRulesBucketStorage +} from '@powersync/service-core'; +import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; import { ChangeStream, ChangeStreamOptions } from '@module/replication/ChangeStream.js'; import { MongoManager } from '@module/replication/MongoManager.js'; diff --git a/modules/module-postgres-storage/test/src/setup.ts b/modules/module-postgres-storage/test/src/setup.ts index 802007e8..25b983ea 100644 --- a/modules/module-postgres-storage/test/src/setup.ts +++ b/modules/module-postgres-storage/test/src/setup.ts @@ -1,16 +1,10 @@ import { container } from '@powersync/lib-services-framework'; -import { Metrics } from '@powersync/service-core'; import { beforeAll } from 'vitest'; +import { METRICS_HELPER } from '@powersync/service-core-tests'; beforeAll(async () => { // Executes for every test file container.registerDefaults(); - // The metrics need to be initialized before they can be used - await Metrics.initialise({ - disable_telemetry_sharing: true, - powersync_instance_id: 'test', - internal_metrics_endpoint: 'unused.for.tests.com' - }); - Metrics.getInstance().resetCounters(); + METRICS_HELPER.resetMetrics(); }); diff --git a/packages/service-core/src/storage/storage-metrics.ts b/packages/service-core/src/storage/storage-metrics.ts index 0f5af3bb..bed1817e 100644 --- a/packages/service-core/src/storage/storage-metrics.ts +++ b/packages/service-core/src/storage/storage-metrics.ts @@ -1,6 +1,6 @@ import { MetricsEngine } from '../metrics/MetricsEngine.js'; -import { BucketStorageFactory, StorageMetrics } from './BucketStorage.js'; import { logger } from '@powersync/lib-services-framework'; +import { BucketStorageFactory, StorageMetrics } from './BucketStorageFactory.js'; export enum StorageMetricType { // Size of current replication data stored in PowerSync From d239244e471cb9f468199441bccbb5c9f059fab0 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Tue, 4 Mar 2025 10:01:26 +0200 Subject: [PATCH 09/18] Added changeset --- .changeset/beige-camels-reply.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .changeset/beige-camels-reply.md diff --git a/.changeset/beige-camels-reply.md b/.changeset/beige-camels-reply.md new file mode 100644 index 00000000..1155d561 --- /dev/null +++ b/.changeset/beige-camels-reply.md @@ -0,0 +1,11 @@ +--- +'@powersync/service-module-postgres-storage': minor +'@powersync/service-module-mongodb-storage': minor +'@powersync/service-core-tests': minor +'@powersync/service-module-postgres': minor +'@powersync/service-module-mongodb': minor +'@powersync/service-core': minor +'@powersync/service-module-mysql': minor +--- + +Refactored Metrics to use a MetricsEngine which is telemetry framework agnostic. From d7fab45ccc87c930cc96dcacfbad427e9912dc99 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Thu, 6 Mar 2025 14:49:23 +0200 Subject: [PATCH 10/18] Moved metric enums to types package --- .../src/replication/ChangeStream.ts | 15 ++----- .../src/replication/BinLogStream.ts | 12 ++--- .../test/src/BinLogStream.test.ts | 44 +++++++++---------- .../src/module/PostgresModule.ts | 4 +- .../src/replication/WalStream.ts | 21 ++++----- .../test/src/large_batch.test.ts | 8 ++-- .../test/src/wal_stream.test.ts | 30 ++++++------- packages/service-core/src/api/api-metrics.ts | 18 +++----- .../src/replication/replication-metrics.ts | 28 ++++-------- .../src/routes/endpoints/socket-route.ts | 7 +-- .../src/routes/endpoints/sync-stream.ts | 9 ++-- .../src/storage/storage-metrics.ts | 22 +++------- .../service-core/src/sync/RequestTracker.ts | 7 +-- packages/types/src/index.ts | 1 + packages/types/src/metrics.ts | 28 ++++++++++++ 15 files changed, 121 insertions(+), 133 deletions(-) create mode 100644 packages/types/src/metrics.ts diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index 3c3080b5..9ec77057 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -8,15 +8,7 @@ import { ReplicationAssertionError, ServiceError } from '@powersync/lib-services-framework'; -import { - BSON_DESERIALIZE_DATA_OPTIONS, - MetricsEngine, - ReplicationMetricType, - SaveOperationTag, - SourceEntityDescriptor, - SourceTable, - storage -} from '@powersync/service-core'; +import { MetricsEngine, SaveOperationTag, SourceEntityDescriptor, SourceTable, storage } from '@powersync/service-core'; import { DatabaseInputRow, SqliteRow, SqlSyncRules, TablePattern } from '@powersync/service-sync-rules'; import { MongoLSN } from '../common/MongoLSN.js'; import { PostImagesOption } from '../types/types.js'; @@ -24,6 +16,7 @@ import { escapeRegExp } from '../utils.js'; import { MongoManager } from './MongoManager.js'; import { constructAfterRecord, createCheckpoint, getCacheIdentifier, getMongoRelation } from './MongoRelation.js'; import { CHECKPOINTS_COLLECTION } from './replication-utils.js'; +import { ReplicationMetric } from '@powersync/service-types'; export interface ChangeStreamOptions { connections: MongoManager; @@ -322,7 +315,7 @@ export class ChangeStream { } at += docBatch.length; - this.metrics.getCounter(ReplicationMetricType.ROWS_REPLICATED_TOTAL).add(docBatch.length); + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED_TOTAL).add(docBatch.length); const duration = performance.now() - lastBatch; lastBatch = performance.now(); logger.info( @@ -450,7 +443,7 @@ export class ChangeStream { return null; } - this.metrics.getCounter(ReplicationMetricType.ROWS_REPLICATED_TOTAL).add(1); + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED_TOTAL).add(1); if (change.operationType == 'insert') { const baseRecord = constructAfterRecord(change.fullDocument); return await batch.save({ diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index 9d3f5701..4fc678c0 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -7,7 +7,6 @@ import { framework, getUuidReplicaIdentityBson, MetricsEngine, - ReplicationMetricType, storage } from '@powersync/service-core'; import mysql, { FieldPacket } from 'mysql2'; @@ -19,6 +18,7 @@ import { isBinlogStillAvailable, ReplicatedGTID, toColumnDescriptors } from '../ import { createRandomServerId, escapeMysqlTableName } from '../utils/mysql-utils.js'; import { MySQLConnectionManager } from './MySQLConnectionManager.js'; import * as zongji_utils from './zongji/zongji-utils.js'; +import { ReplicationMetric } from '@powersync/service-types'; export interface BinLogStreamOptions { connections: MySQLConnectionManager; @@ -348,7 +348,7 @@ AND table_type = 'BASE TABLE';`, afterReplicaId: getUuidReplicaIdentityBson(record, table.replicaIdColumns) }); - this.metrics.getCounter(ReplicationMetricType.ROWS_REPLICATED_TOTAL).add(1); + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED_TOTAL).add(1); } await batch.flush(); } @@ -475,7 +475,7 @@ AND table_type = 'BASE TABLE';`, }); break; case zongji_utils.eventIsXid(evt): - this.metrics.getCounter(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL).add(1); + this.metrics.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL).add(1); // Need to commit with a replicated GTID with updated next position await batch.commit( new common.ReplicatedGTID({ @@ -617,7 +617,7 @@ AND table_type = 'BASE TABLE';`, ): Promise { switch (payload.type) { case storage.SaveOperationTag.INSERT: - this.metrics.getCounter(ReplicationMetricType.ROWS_REPLICATED_TOTAL).add(1); + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED_TOTAL).add(1); const record = common.toSQLiteRow(payload.data, payload.columns); return await batch.save({ tag: storage.SaveOperationTag.INSERT, @@ -628,7 +628,7 @@ AND table_type = 'BASE TABLE';`, afterReplicaId: getUuidReplicaIdentityBson(record, payload.sourceTable.replicaIdColumns) }); case storage.SaveOperationTag.UPDATE: - this.metrics.getCounter(ReplicationMetricType.ROWS_REPLICATED_TOTAL).add(1); + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED_TOTAL).add(1); // "before" may be null if the replica id columns are unchanged // It's fine to treat that the same as an insert. const beforeUpdated = payload.previous_data @@ -648,7 +648,7 @@ AND table_type = 'BASE TABLE';`, }); case storage.SaveOperationTag.DELETE: - this.metrics.getCounter(ReplicationMetricType.ROWS_REPLICATED_TOTAL).add(1); + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED_TOTAL).add(1); const beforeDeleted = common.toSQLiteRow(payload.data, payload.columns); return await batch.save({ diff --git a/modules/module-mysql/test/src/BinLogStream.test.ts b/modules/module-mysql/test/src/BinLogStream.test.ts index 4ab5e759..ab0b8b7d 100644 --- a/modules/module-mysql/test/src/BinLogStream.test.ts +++ b/modules/module-mysql/test/src/BinLogStream.test.ts @@ -1,10 +1,11 @@ -import { ReplicationMetricType, storage } from '@powersync/service-core'; +import { storage } from '@powersync/service-core'; import { METRICS_HELPER, putOp, removeOp } from '@powersync/service-core-tests'; import { v4 as uuid } from 'uuid'; import { describe, expect, test } from 'vitest'; import { BinlogStreamTestContext } from './BinlogStreamUtils.js'; import { env } from './env.js'; import { INITIALIZED_MONGO_STORAGE_FACTORY, INITIALIZED_POSTGRES_STORAGE_FACTORY } from './util.js'; +import { ReplicationMetric } from '@powersync/service-types'; const BASIC_SYNC_RULES = ` bucket_definitions: @@ -35,10 +36,9 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { await context.replicateSnapshot(); - const startRowCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; const startTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; context.startStreaming(); const testId = uuid(); @@ -48,9 +48,9 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { const data = await context.getBucketData('global[]'); expect(data).toMatchObject([putOp('test_data', { id: testId, description: 'test1', num: 1152921504606846976n })]); - const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; const endTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; expect(endRowCount - startRowCount).toEqual(1); expect(endTxCount - startTxCount).toEqual(1); }); @@ -69,10 +69,9 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { await context.replicateSnapshot(); - const startRowCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; const startTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; context.startStreaming(); @@ -82,9 +81,9 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { const data = await context.getBucketData('global[]'); expect(data).toMatchObject([putOp('test_DATA', { id: testId, description: 'test1' })]); - const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; const endTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; expect(endRowCount - startRowCount).toEqual(1); expect(endTxCount - startTxCount).toEqual(1); }); @@ -174,12 +173,11 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { const testId = uuid(); await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('${testId}','test1')`); - const startRowCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; await context.replicateSnapshot(); - const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; const data = await context.getBucketData('global[]'); expect(data).toMatchObject([putOp('test_data', { id: testId, description: 'test1' })]); expect(endRowCount - startRowCount).toEqual(1); @@ -234,10 +232,9 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { await context.replicateSnapshot(); - const startRowCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; const startTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; context.startStreaming(); @@ -264,9 +261,9 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { timestamp: '2023-03-06T15:47:00.000Z' }) ]); - const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; const endTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; expect(endRowCount - startRowCount).toEqual(2); expect(endTxCount - startTxCount).toEqual(2); }); @@ -280,10 +277,9 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { await context.replicateSnapshot(); - const startRowCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; const startTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; context.startStreaming(); @@ -291,9 +287,9 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { const data = await context.getBucketData('global[]'); expect(data).toMatchObject([]); - const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; const endTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; // There was a transaction, but we should not replicate any actual data expect(endRowCount - startRowCount).toEqual(0); diff --git a/modules/module-postgres/src/module/PostgresModule.ts b/modules/module-postgres/src/module/PostgresModule.ts index ee4f0e76..4dc3bd5f 100644 --- a/modules/module-postgres/src/module/PostgresModule.ts +++ b/modules/module-postgres/src/module/PostgresModule.ts @@ -5,7 +5,6 @@ import { ConnectionTestResult, modules, replication, - ReplicationMetricType, system } from '@powersync/service-core'; import * as jpgwire from '@powersync/service-jpgwire'; @@ -20,6 +19,7 @@ import { WalStreamReplicator } from '../replication/WalStreamReplicator.js'; import * as types from '../types/types.js'; import { PostgresConnectionConfig } from '../types/types.js'; import { baseUri, NormalizedBasePostgresConnectionConfig } from '@powersync/lib-service-postgres'; +import { ReplicationMetric } from '@powersync/service-types'; export class PostgresModule extends replication.ReplicationModule { constructor() { @@ -47,7 +47,7 @@ export class PostgresModule extends replication.ReplicationModule { while (!done) { const count = - ((await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0) - + ((await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0) - startRowCount; if (count >= stopAfter) { diff --git a/modules/module-postgres/test/src/wal_stream.test.ts b/modules/module-postgres/test/src/wal_stream.test.ts index 2397272a..5ec1f189 100644 --- a/modules/module-postgres/test/src/wal_stream.test.ts +++ b/modules/module-postgres/test/src/wal_stream.test.ts @@ -1,5 +1,5 @@ import { MissingReplicationSlotError } from '@module/replication/WalStream.js'; -import { ReplicationMetricType, storage } from '@powersync/service-core'; +import { storage } from '@powersync/service-core'; import { METRICS_HELPER, putOp, removeOp } from '@powersync/service-core-tests'; import { pgwireRows } from '@powersync/service-jpgwire'; import * as crypto from 'crypto'; @@ -7,6 +7,7 @@ import { describe, expect, test } from 'vitest'; import { env } from './env.js'; import { INITIALIZED_MONGO_STORAGE_FACTORY, INITIALIZED_POSTGRES_STORAGE_FACTORY } from './util.js'; import { WalStreamTestContext } from './wal_stream_utils.js'; +import { ReplicationMetric } from '@powersync/service-types'; const BASIC_SYNC_RULES = ` bucket_definitions: @@ -40,10 +41,9 @@ bucket_definitions: await context.replicateSnapshot(); - const startRowCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; const startTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; context.startStreaming(); @@ -56,9 +56,9 @@ bucket_definitions: const data = await context.getBucketData('global[]'); expect(data).toMatchObject([putOp('test_data', { id: test_id, description: 'test1', num: 1152921504606846976n })]); - const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; const endTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; expect(endRowCount - startRowCount).toEqual(1); expect(endTxCount - startTxCount).toEqual(1); }); @@ -78,10 +78,9 @@ bucket_definitions: await context.replicateSnapshot(); - const startRowCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; const startTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; context.startStreaming(); @@ -92,9 +91,9 @@ bucket_definitions: const data = await context.getBucketData('global[]'); expect(data).toMatchObject([putOp('test_DATA', { id: test_id, description: 'test1' })]); - const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; const endTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; expect(endRowCount - startRowCount).toEqual(1); expect(endTxCount - startTxCount).toEqual(1); }); @@ -276,10 +275,9 @@ bucket_definitions: await context.replicateSnapshot(); - const startRowCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; const startTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; context.startStreaming(); @@ -290,9 +288,9 @@ bucket_definitions: const data = await context.getBucketData('global[]'); expect(data).toMatchObject([]); - const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.ROWS_REPLICATED_TOTAL)) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; const endTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; // There was a transaction, but we should not replicate any actual data expect(endRowCount - startRowCount).toEqual(0); diff --git a/packages/service-core/src/api/api-metrics.ts b/packages/service-core/src/api/api-metrics.ts index 766aaf82..f1de26c1 100644 --- a/packages/service-core/src/api/api-metrics.ts +++ b/packages/service-core/src/api/api-metrics.ts @@ -1,13 +1,5 @@ import { MetricsEngine } from '../metrics/MetricsEngine.js'; - -export enum APIMetricType { - // Uncompressed size of synced data from PowerSync to Clients - DATA_SYNCED_BYTES = 'powersync_data_synced_bytes_total', - // Number of operations synced - OPERATIONS_SYNCED_TOTAL = 'powersync_operations_synced_total', - // Number of concurrent sync connections - CONCURRENT_CONNECTIONS = 'powersync_concurrent_connections' -} +import { APIMetric } from '@powersync/service-types'; /** * Create and register the core API metrics. @@ -15,18 +7,18 @@ export enum APIMetricType { */ export function createCoreAPIMetrics(engine: MetricsEngine): void { engine.createCounter({ - name: APIMetricType.DATA_SYNCED_BYTES, + name: APIMetric.DATA_SYNCED_BYTES, description: 'Uncompressed size of synced data', unit: 'bytes' }); engine.createCounter({ - name: APIMetricType.OPERATIONS_SYNCED_TOTAL, + name: APIMetric.OPERATIONS_SYNCED_TOTAL, description: 'Number of operations synced' }); engine.createUpDownCounter({ - name: APIMetricType.CONCURRENT_CONNECTIONS, + name: APIMetric.CONCURRENT_CONNECTIONS, description: 'Number of concurrent sync connections' }); } @@ -36,7 +28,7 @@ export function createCoreAPIMetrics(engine: MetricsEngine): void { * @param engine */ export function initializeCoreAPIMetrics(engine: MetricsEngine): void { - const concurrent_connections = engine.getUpDownCounter(APIMetricType.CONCURRENT_CONNECTIONS); + const concurrent_connections = engine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS); // Initialize the metric, so that it reports a value before connections have been opened. concurrent_connections.add(0); diff --git a/packages/service-core/src/replication/replication-metrics.ts b/packages/service-core/src/replication/replication-metrics.ts index 6c387fa3..3944a278 100644 --- a/packages/service-core/src/replication/replication-metrics.ts +++ b/packages/service-core/src/replication/replication-metrics.ts @@ -1,15 +1,5 @@ import { MetricsEngine } from '../metrics/metrics-index.js'; - -export enum ReplicationMetricType { - // Uncompressed size of replicated data from data source to PowerSync - DATA_REPLICATED_BYTES = 'powersync_data_replicated_bytes_total', - // Total number of replicated rows. Not used for pricing. - ROWS_REPLICATED_TOTAL = 'powersync_rows_replicated_total', - // Total number of replicated transactions. Not used for pricing. - TRANSACTIONS_REPLICATED_TOTAL = 'powersync_transactions_replicated_total', - // Total number of replication chunks. Not used for pricing. - CHUNKS_REPLICATED_TOTAL = 'powersync_chunks_replicated_total' -} +import { ReplicationMetric } from '@powersync/service-types'; /** * Create and register the core replication metrics. @@ -17,23 +7,23 @@ export enum ReplicationMetricType { */ export function createCoreReplicationMetrics(engine: MetricsEngine): void { engine.createCounter({ - name: ReplicationMetricType.DATA_REPLICATED_BYTES, + name: ReplicationMetric.DATA_REPLICATED_BYTES, description: 'Uncompressed size of replicated data', unit: 'bytes' }); engine.createCounter({ - name: ReplicationMetricType.ROWS_REPLICATED_TOTAL, + name: ReplicationMetric.ROWS_REPLICATED_TOTAL, description: 'Total number of replicated rows' }); engine.createCounter({ - name: ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL, + name: ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL, description: 'Total number of replicated transactions' }); engine.createCounter({ - name: ReplicationMetricType.CHUNKS_REPLICATED_TOTAL, + name: ReplicationMetric.CHUNKS_REPLICATED_TOTAL, description: 'Total number of replication chunks' }); } @@ -43,10 +33,10 @@ export function createCoreReplicationMetrics(engine: MetricsEngine): void { * @param engine */ export function initializeCoreReplicationMetrics(engine: MetricsEngine): void { - const data_replicated_bytes = engine.getCounter(ReplicationMetricType.DATA_REPLICATED_BYTES); - const rows_replicated_total = engine.getCounter(ReplicationMetricType.ROWS_REPLICATED_TOTAL); - const transactions_replicated_total = engine.getCounter(ReplicationMetricType.TRANSACTIONS_REPLICATED_TOTAL); - const chunks_replicated_total = engine.getCounter(ReplicationMetricType.CHUNKS_REPLICATED_TOTAL); + const data_replicated_bytes = engine.getCounter(ReplicationMetric.DATA_REPLICATED_BYTES); + const rows_replicated_total = engine.getCounter(ReplicationMetric.ROWS_REPLICATED_TOTAL); + const transactions_replicated_total = engine.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL); + const chunks_replicated_total = engine.getCounter(ReplicationMetric.CHUNKS_REPLICATED_TOTAL); data_replicated_bytes.add(0); rows_replicated_total.add(0); diff --git a/packages/service-core/src/routes/endpoints/socket-route.ts b/packages/service-core/src/routes/endpoints/socket-route.ts index 6a54368b..1712da46 100644 --- a/packages/service-core/src/routes/endpoints/socket-route.ts +++ b/packages/service-core/src/routes/endpoints/socket-route.ts @@ -6,7 +6,8 @@ import * as sync from '../../sync/sync-index.js'; import * as util from '../../util/util-index.js'; import { SocketRouteGenerator } from '../router-socket.js'; import { SyncRoutes } from './sync-stream.js'; -import { APIMetricType } from '../../api/api-metrics.js'; + +import { APIMetric } from '@powersync/service-types'; export const syncStreamReactive: SocketRouteGenerator = (router) => router.reactiveStream(SyncRoutes.STREAM, { @@ -69,7 +70,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => controller.abort(); }); - metricsEngine.getUpDownCounter(APIMetricType.CONCURRENT_CONNECTIONS).add(1); + metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1); const tracker = new sync.RequestTracker(metricsEngine); try { for await (const data of sync.streamResponse({ @@ -146,7 +147,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => operations_synced: tracker.operationsSynced, data_synced_bytes: tracker.dataSyncedBytes }); - metricsEngine.getUpDownCounter(APIMetricType.CONCURRENT_CONNECTIONS).add(-1); + metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); } } }); diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index c71d43ca..0877101f 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -7,7 +7,8 @@ import * as util from '../../util/util-index.js'; import { authUser } from '../auth.js'; import { routeDefinition } from '../router.js'; -import { APIMetricType } from '../../api/api-metrics.js'; + +import { APIMetric } from '@powersync/service-types'; export enum SyncRoutes { STREAM = '/sync/stream' @@ -51,7 +52,7 @@ export const syncStreamed = routeDefinition({ const controller = new AbortController(); const tracker = new sync.RequestTracker(metricsEngine); try { - metricsEngine.getUpDownCounter(APIMetricType.CONCURRENT_CONNECTIONS).add(1); + metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1); const stream = Readable.from( sync.transformToBytesTracked( sync.ndjson( @@ -95,7 +96,7 @@ export const syncStreamed = routeDefinition({ data: stream, afterSend: async () => { controller.abort(); - metricsEngine.getUpDownCounter(APIMetricType.CONCURRENT_CONNECTIONS).add(-1); + metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); logger.info(`Sync stream complete`, { user_id: syncParams.user_id, client_id: clientId, @@ -107,7 +108,7 @@ export const syncStreamed = routeDefinition({ }); } catch (ex) { controller.abort(); - metricsEngine.getUpDownCounter(APIMetricType.CONCURRENT_CONNECTIONS).add(-1); + metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); } } }); diff --git a/packages/service-core/src/storage/storage-metrics.ts b/packages/service-core/src/storage/storage-metrics.ts index bed1817e..5ce8f66c 100644 --- a/packages/service-core/src/storage/storage-metrics.ts +++ b/packages/service-core/src/storage/storage-metrics.ts @@ -1,40 +1,32 @@ import { MetricsEngine } from '../metrics/MetricsEngine.js'; import { logger } from '@powersync/lib-services-framework'; import { BucketStorageFactory, StorageMetrics } from './BucketStorageFactory.js'; - -export enum StorageMetricType { - // Size of current replication data stored in PowerSync - REPLICATION_SIZE_BYTES = 'powersync_replication_storage_size_bytes', - // Size of operations data stored in PowerSync - OPERATION_SIZE_BYTES = 'powersync_operation_storage_size_bytes', - // Size of parameter data stored in PowerSync - PARAMETER_SIZE_BYTES = 'powersync_parameter_storage_size_bytes' -} +import { StorageMetric } from '@powersync/service-types'; export function createCoreStorageMetrics(engine: MetricsEngine): void { engine.createObservableGauge({ - name: StorageMetricType.REPLICATION_SIZE_BYTES, + name: StorageMetric.REPLICATION_SIZE_BYTES, description: 'Size of current data stored in PowerSync', unit: 'bytes' }); engine.createObservableGauge({ - name: StorageMetricType.OPERATION_SIZE_BYTES, + name: StorageMetric.OPERATION_SIZE_BYTES, description: 'Size of operations stored in PowerSync', unit: 'bytes' }); engine.createObservableGauge({ - name: StorageMetricType.PARAMETER_SIZE_BYTES, + name: StorageMetric.PARAMETER_SIZE_BYTES, description: 'Size of parameter data stored in PowerSync', unit: 'bytes' }); } export function initializeCoreStorageMetrics(engine: MetricsEngine, storage: BucketStorageFactory): void { - const replication_storage_size_bytes = engine.getObservableGauge(StorageMetricType.REPLICATION_SIZE_BYTES); - const operation_storage_size_bytes = engine.getObservableGauge(StorageMetricType.OPERATION_SIZE_BYTES); - const parameter_storage_size_bytes = engine.getObservableGauge(StorageMetricType.PARAMETER_SIZE_BYTES); + const replication_storage_size_bytes = engine.getObservableGauge(StorageMetric.REPLICATION_SIZE_BYTES); + const operation_storage_size_bytes = engine.getObservableGauge(StorageMetric.OPERATION_SIZE_BYTES); + const parameter_storage_size_bytes = engine.getObservableGauge(StorageMetric.PARAMETER_SIZE_BYTES); const MINIMUM_INTERVAL = 60_000; diff --git a/packages/service-core/src/sync/RequestTracker.ts b/packages/service-core/src/sync/RequestTracker.ts index a4684f5e..27f8ac81 100644 --- a/packages/service-core/src/sync/RequestTracker.ts +++ b/packages/service-core/src/sync/RequestTracker.ts @@ -1,5 +1,6 @@ import { MetricsEngine } from '../metrics/MetricsEngine.js'; -import { APIMetricType } from '../api/api-metrics.js'; + +import { APIMetric } from '@powersync/service-types'; /** * Record sync stats per request stream. @@ -15,12 +16,12 @@ export class RequestTracker { addOperationsSynced(operations: number) { this.operationsSynced += operations; - this.metrics.getCounter(APIMetricType.OPERATIONS_SYNCED_TOTAL).add(operations); + this.metrics.getCounter(APIMetric.OPERATIONS_SYNCED_TOTAL).add(operations); } addDataSynced(bytes: number) { this.dataSyncedBytes += bytes; - this.metrics.getCounter(APIMetricType.DATA_SYNCED_BYTES).add(bytes); + this.metrics.getCounter(APIMetric.DATA_SYNCED_BYTES).add(bytes); } } diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 03d0aaaf..f8985772 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -2,3 +2,4 @@ export * as configFile from './config/PowerSyncConfig.js'; export * from './definitions.js'; export * as internal_routes from './routes.js'; +export * from './metrics.js'; diff --git a/packages/types/src/metrics.ts b/packages/types/src/metrics.ts new file mode 100644 index 00000000..5918ffdf --- /dev/null +++ b/packages/types/src/metrics.ts @@ -0,0 +1,28 @@ +export enum APIMetric { + // Uncompressed size of synced data from PowerSync to Clients + DATA_SYNCED_BYTES = 'powersync_data_synced_bytes_total', + // Number of operations synced + OPERATIONS_SYNCED_TOTAL = 'powersync_operations_synced_total', + // Number of concurrent sync connections + CONCURRENT_CONNECTIONS = 'powersync_concurrent_connections' +} + +export enum ReplicationMetric { + // Uncompressed size of replicated data from data source to PowerSync + DATA_REPLICATED_BYTES = 'powersync_data_replicated_bytes_total', + // Total number of replicated rows. Not used for pricing. + ROWS_REPLICATED_TOTAL = 'powersync_rows_replicated_total', + // Total number of replicated transactions. Not used for pricing. + TRANSACTIONS_REPLICATED_TOTAL = 'powersync_transactions_replicated_total', + // Total number of replication chunks. Not used for pricing. + CHUNKS_REPLICATED_TOTAL = 'powersync_chunks_replicated_total' +} + +export enum StorageMetric { + // Size of current replication data stored in PowerSync + REPLICATION_SIZE_BYTES = 'powersync_replication_storage_size_bytes', + // Size of operations data stored in PowerSync + OPERATION_SIZE_BYTES = 'powersync_operation_storage_size_bytes', + // Size of parameter data stored in PowerSync + PARAMETER_SIZE_BYTES = 'powersync_parameter_storage_size_bytes' +} From bf0e399d9b09408b6a7e9971dacd4a8ded9dc7b3 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Thu, 6 Mar 2025 14:52:30 +0200 Subject: [PATCH 11/18] Updated changeset --- .changeset/beige-camels-reply.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.changeset/beige-camels-reply.md b/.changeset/beige-camels-reply.md index 1155d561..baac818b 100644 --- a/.changeset/beige-camels-reply.md +++ b/.changeset/beige-camels-reply.md @@ -6,6 +6,7 @@ '@powersync/service-module-mongodb': minor '@powersync/service-core': minor '@powersync/service-module-mysql': minor +'@powersync/service-types': minor --- Refactored Metrics to use a MetricsEngine which is telemetry framework agnostic. From 04b3d4f442052c9cd5faedee00f140efdbedc119 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Fri, 7 Mar 2025 12:39:50 +0200 Subject: [PATCH 12/18] Export metric types as named export --- packages/types/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index f8985772..db2b7900 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -2,4 +2,4 @@ export * as configFile from './config/PowerSyncConfig.js'; export * from './definitions.js'; export * as internal_routes from './routes.js'; -export * from './metrics.js'; +export * as metric_types from './metrics.js'; From 60801d91a82e4fb80205014bd888b86fd9c308e6 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Fri, 7 Mar 2025 13:01:43 +0200 Subject: [PATCH 13/18] Keep original metrics type export as well --- packages/types/src/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index db2b7900..dca03e08 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -2,4 +2,5 @@ export * as configFile from './config/PowerSyncConfig.js'; export * from './definitions.js'; export * as internal_routes from './routes.js'; +export * from './metrics.js'; export * as metric_types from './metrics.js'; From ec896f3e01c05d556b6d24944e741d86af084138 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Fri, 7 Mar 2025 15:30:24 +0200 Subject: [PATCH 14/18] Made metric enum names consistent --- .../src/replication/ChangeStream.ts | 4 +- .../src/replication/BinLogStream.ts | 10 ++--- .../test/src/BinLogStream.test.ts | 44 ++++++++----------- .../src/replication/WalStream.ts | 12 ++--- .../test/src/large_batch.test.ts | 5 +-- .../test/src/wal_stream.test.ts | 30 +++++-------- packages/service-core/src/api/api-metrics.ts | 2 +- .../src/replication/replication-metrics.ts | 12 ++--- .../service-core/src/sync/RequestTracker.ts | 2 +- packages/types/src/metrics.ts | 8 ++-- 10 files changed, 57 insertions(+), 72 deletions(-) diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index 9ec77057..3a47ca43 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -315,7 +315,7 @@ export class ChangeStream { } at += docBatch.length; - this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED_TOTAL).add(docBatch.length); + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(docBatch.length); const duration = performance.now() - lastBatch; lastBatch = performance.now(); logger.info( @@ -443,7 +443,7 @@ export class ChangeStream { return null; } - this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED_TOTAL).add(1); + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); if (change.operationType == 'insert') { const baseRecord = constructAfterRecord(change.fullDocument); return await batch.save({ diff --git a/modules/module-mysql/src/replication/BinLogStream.ts b/modules/module-mysql/src/replication/BinLogStream.ts index 1d08f9e5..d42ddc97 100644 --- a/modules/module-mysql/src/replication/BinLogStream.ts +++ b/modules/module-mysql/src/replication/BinLogStream.ts @@ -348,7 +348,7 @@ AND table_type = 'BASE TABLE';`, afterReplicaId: getUuidReplicaIdentityBson(record, table.replicaIdColumns) }); - this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED_TOTAL).add(1); + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); } await batch.flush(); } @@ -475,7 +475,7 @@ AND table_type = 'BASE TABLE';`, }); break; case zongji_utils.eventIsXid(evt): - this.metrics.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL).add(1); + this.metrics.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED).add(1); // Need to commit with a replicated GTID with updated next position await batch.commit( new common.ReplicatedGTID({ @@ -620,7 +620,7 @@ AND table_type = 'BASE TABLE';`, ): Promise { switch (payload.type) { case storage.SaveOperationTag.INSERT: - this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED_TOTAL).add(1); + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); const record = common.toSQLiteRow(payload.data, payload.columns); return await batch.save({ tag: storage.SaveOperationTag.INSERT, @@ -631,7 +631,7 @@ AND table_type = 'BASE TABLE';`, afterReplicaId: getUuidReplicaIdentityBson(record, payload.sourceTable.replicaIdColumns) }); case storage.SaveOperationTag.UPDATE: - this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED_TOTAL).add(1); + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); // "before" may be null if the replica id columns are unchanged // It's fine to treat that the same as an insert. const beforeUpdated = payload.previous_data @@ -651,7 +651,7 @@ AND table_type = 'BASE TABLE';`, }); case storage.SaveOperationTag.DELETE: - this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED_TOTAL).add(1); + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); const beforeDeleted = common.toSQLiteRow(payload.data, payload.columns); return await batch.save({ diff --git a/modules/module-mysql/test/src/BinLogStream.test.ts b/modules/module-mysql/test/src/BinLogStream.test.ts index dc40e3d5..a0db934c 100644 --- a/modules/module-mysql/test/src/BinLogStream.test.ts +++ b/modules/module-mysql/test/src/BinLogStream.test.ts @@ -36,9 +36,8 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { await context.replicateSnapshot(); - const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; - const startTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; context.startStreaming(); const testId = uuid(); @@ -48,9 +47,8 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { const data = await context.getBucketData('global[]'); expect(data).toMatchObject([putOp('test_data', { id: testId, description: 'test1', num: 1152921504606846976n })]); - const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; - const endTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; expect(endRowCount - startRowCount).toEqual(1); expect(endTxCount - startTxCount).toEqual(1); }); @@ -69,9 +67,8 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { await context.replicateSnapshot(); - const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; - const startTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; context.startStreaming(); @@ -81,9 +78,8 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { const data = await context.getBucketData('global[]'); expect(data).toMatchObject([putOp('test_DATA', { id: testId, description: 'test1' })]); - const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; - const endTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; expect(endRowCount - startRowCount).toEqual(1); expect(endTxCount - startTxCount).toEqual(1); }); @@ -173,12 +169,12 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { const testId = uuid(); await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('${testId}','test1')`); - const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; await context.replicateSnapshot(); context.startStreaming(); - const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; const data = await context.getBucketData('global[]'); expect(data).toMatchObject([putOp('test_data', { id: testId, description: 'test1' })]); expect(endRowCount - startRowCount).toEqual(1); @@ -234,9 +230,8 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { await context.replicateSnapshot(); - const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; - const startTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; context.startStreaming(); @@ -263,9 +258,8 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { timestamp: '2023-03-06T15:47:00.000Z' }) ]); - const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; - const endTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; expect(endRowCount - startRowCount).toEqual(2); expect(endTxCount - startTxCount).toEqual(2); }); @@ -279,9 +273,8 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { await context.replicateSnapshot(); - const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; - const startTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; context.startStreaming(); @@ -289,9 +282,8 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { const data = await context.getBucketData('global[]'); expect(data).toMatchObject([]); - const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; - const endTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; // There was a transaction, but we should not replicate any actual data expect(endRowCount - startRowCount).toEqual(0); diff --git a/modules/module-postgres/src/replication/WalStream.ts b/modules/module-postgres/src/replication/WalStream.ts index 0496328b..ee6f6e4c 100644 --- a/modules/module-postgres/src/replication/WalStream.ts +++ b/modules/module-postgres/src/replication/WalStream.ts @@ -479,7 +479,7 @@ WHERE oid = $1::regclass`, } at += rows.length; - this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED_TOTAL).add(rows.length); + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(rows.length); await touch(); } @@ -570,7 +570,7 @@ WHERE oid = $1::regclass`, } if (msg.tag == 'insert') { - this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED_TOTAL).add(1); + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); const baseRecord = pg_utils.constructAfterRecord(msg); return await batch.save({ tag: storage.SaveOperationTag.INSERT, @@ -581,7 +581,7 @@ WHERE oid = $1::regclass`, afterReplicaId: getUuidReplicaIdentityBson(baseRecord, table.replicaIdColumns) }); } else if (msg.tag == 'update') { - this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED_TOTAL).add(1); + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); // "before" may be null if the replica id columns are unchanged // It's fine to treat that the same as an insert. const before = pg_utils.constructBeforeRecord(msg); @@ -595,7 +595,7 @@ WHERE oid = $1::regclass`, afterReplicaId: getUuidReplicaIdentityBson(after, table.replicaIdColumns) }); } else if (msg.tag == 'delete') { - this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED_TOTAL).add(1); + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); const before = pg_utils.constructBeforeRecord(msg)!; return await batch.save({ @@ -705,7 +705,7 @@ WHERE oid = $1::regclass`, } else if (msg.tag == 'begin') { inTx = true; } else if (msg.tag == 'commit') { - this.metrics.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL).add(1); + this.metrics.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED).add(1); inTx = false; await batch.commit(msg.lsn!, { createEmptyCheckpoints }); await this.ack(msg.lsn!, replicationStream); @@ -747,7 +747,7 @@ WHERE oid = $1::regclass`, await this.ack(chunkLastLsn, replicationStream); } - this.metrics.getCounter(ReplicationMetric.CHUNKS_REPLICATED_TOTAL).add(1); + this.metrics.getCounter(ReplicationMetric.CHUNKS_REPLICATED).add(1); } } ); diff --git a/modules/module-postgres/test/src/large_batch.test.ts b/modules/module-postgres/test/src/large_batch.test.ts index 64cb5ab8..79039489 100644 --- a/modules/module-postgres/test/src/large_batch.test.ts +++ b/modules/module-postgres/test/src/large_batch.test.ts @@ -303,13 +303,12 @@ function defineBatchTests(factory: storage.TestStorageFactory) { let done = false; - const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; try { (async () => { while (!done) { const count = - ((await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0) - - startRowCount; + ((await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0) - startRowCount; if (count >= stopAfter) { break; diff --git a/modules/module-postgres/test/src/wal_stream.test.ts b/modules/module-postgres/test/src/wal_stream.test.ts index 5ec1f189..109460ac 100644 --- a/modules/module-postgres/test/src/wal_stream.test.ts +++ b/modules/module-postgres/test/src/wal_stream.test.ts @@ -41,9 +41,8 @@ bucket_definitions: await context.replicateSnapshot(); - const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; - const startTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; context.startStreaming(); @@ -56,9 +55,8 @@ bucket_definitions: const data = await context.getBucketData('global[]'); expect(data).toMatchObject([putOp('test_data', { id: test_id, description: 'test1', num: 1152921504606846976n })]); - const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; - const endTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; expect(endRowCount - startRowCount).toEqual(1); expect(endTxCount - startTxCount).toEqual(1); }); @@ -78,9 +76,8 @@ bucket_definitions: await context.replicateSnapshot(); - const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; - const startTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; context.startStreaming(); @@ -91,9 +88,8 @@ bucket_definitions: const data = await context.getBucketData('global[]'); expect(data).toMatchObject([putOp('test_DATA', { id: test_id, description: 'test1' })]); - const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; - const endTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; expect(endRowCount - startRowCount).toEqual(1); expect(endTxCount - startTxCount).toEqual(1); }); @@ -275,9 +271,8 @@ bucket_definitions: await context.replicateSnapshot(); - const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; - const startTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; context.startStreaming(); @@ -288,9 +283,8 @@ bucket_definitions: const data = await context.getBucketData('global[]'); expect(data).toMatchObject([]); - const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED_TOTAL)) ?? 0; - const endTxCount = - (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL)) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; // There was a transaction, but we should not replicate any actual data expect(endRowCount - startRowCount).toEqual(0); diff --git a/packages/service-core/src/api/api-metrics.ts b/packages/service-core/src/api/api-metrics.ts index f1de26c1..5f57cfb1 100644 --- a/packages/service-core/src/api/api-metrics.ts +++ b/packages/service-core/src/api/api-metrics.ts @@ -13,7 +13,7 @@ export function createCoreAPIMetrics(engine: MetricsEngine): void { }); engine.createCounter({ - name: APIMetric.OPERATIONS_SYNCED_TOTAL, + name: APIMetric.OPERATIONS_SYNCED, description: 'Number of operations synced' }); diff --git a/packages/service-core/src/replication/replication-metrics.ts b/packages/service-core/src/replication/replication-metrics.ts index 3944a278..d589696e 100644 --- a/packages/service-core/src/replication/replication-metrics.ts +++ b/packages/service-core/src/replication/replication-metrics.ts @@ -13,17 +13,17 @@ export function createCoreReplicationMetrics(engine: MetricsEngine): void { }); engine.createCounter({ - name: ReplicationMetric.ROWS_REPLICATED_TOTAL, + name: ReplicationMetric.ROWS_REPLICATED, description: 'Total number of replicated rows' }); engine.createCounter({ - name: ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL, + name: ReplicationMetric.TRANSACTIONS_REPLICATED, description: 'Total number of replicated transactions' }); engine.createCounter({ - name: ReplicationMetric.CHUNKS_REPLICATED_TOTAL, + name: ReplicationMetric.CHUNKS_REPLICATED, description: 'Total number of replication chunks' }); } @@ -34,9 +34,9 @@ export function createCoreReplicationMetrics(engine: MetricsEngine): void { */ export function initializeCoreReplicationMetrics(engine: MetricsEngine): void { const data_replicated_bytes = engine.getCounter(ReplicationMetric.DATA_REPLICATED_BYTES); - const rows_replicated_total = engine.getCounter(ReplicationMetric.ROWS_REPLICATED_TOTAL); - const transactions_replicated_total = engine.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED_TOTAL); - const chunks_replicated_total = engine.getCounter(ReplicationMetric.CHUNKS_REPLICATED_TOTAL); + const rows_replicated_total = engine.getCounter(ReplicationMetric.ROWS_REPLICATED); + const transactions_replicated_total = engine.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED); + const chunks_replicated_total = engine.getCounter(ReplicationMetric.CHUNKS_REPLICATED); data_replicated_bytes.add(0); rows_replicated_total.add(0); diff --git a/packages/service-core/src/sync/RequestTracker.ts b/packages/service-core/src/sync/RequestTracker.ts index 27f8ac81..d6137e64 100644 --- a/packages/service-core/src/sync/RequestTracker.ts +++ b/packages/service-core/src/sync/RequestTracker.ts @@ -16,7 +16,7 @@ export class RequestTracker { addOperationsSynced(operations: number) { this.operationsSynced += operations; - this.metrics.getCounter(APIMetric.OPERATIONS_SYNCED_TOTAL).add(operations); + this.metrics.getCounter(APIMetric.OPERATIONS_SYNCED).add(operations); } addDataSynced(bytes: number) { diff --git a/packages/types/src/metrics.ts b/packages/types/src/metrics.ts index 5918ffdf..360c0f96 100644 --- a/packages/types/src/metrics.ts +++ b/packages/types/src/metrics.ts @@ -2,7 +2,7 @@ export enum APIMetric { // Uncompressed size of synced data from PowerSync to Clients DATA_SYNCED_BYTES = 'powersync_data_synced_bytes_total', // Number of operations synced - OPERATIONS_SYNCED_TOTAL = 'powersync_operations_synced_total', + OPERATIONS_SYNCED = 'powersync_operations_synced_total', // Number of concurrent sync connections CONCURRENT_CONNECTIONS = 'powersync_concurrent_connections' } @@ -11,11 +11,11 @@ export enum ReplicationMetric { // Uncompressed size of replicated data from data source to PowerSync DATA_REPLICATED_BYTES = 'powersync_data_replicated_bytes_total', // Total number of replicated rows. Not used for pricing. - ROWS_REPLICATED_TOTAL = 'powersync_rows_replicated_total', + ROWS_REPLICATED = 'powersync_rows_replicated_total', // Total number of replicated transactions. Not used for pricing. - TRANSACTIONS_REPLICATED_TOTAL = 'powersync_transactions_replicated_total', + TRANSACTIONS_REPLICATED = 'powersync_transactions_replicated_total', // Total number of replication chunks. Not used for pricing. - CHUNKS_REPLICATED_TOTAL = 'powersync_chunks_replicated_total' + CHUNKS_REPLICATED = 'powersync_chunks_replicated_total' } export enum StorageMetric { From 9c8ed976c129563f1dc6b8c4d1f387167e0c4607 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 26 Mar 2025 10:17:46 +0200 Subject: [PATCH 15/18] Move metrics registration to service-core --- .../service-core/src/metrics/metrics-index.ts | 1 + .../src/metrics/register-metrics.ts | 23 ++++++++----------- service/src/runners/server.ts | 5 ++-- service/src/runners/stream-worker.ts | 5 ++-- service/src/runners/unified-runner.ts | 5 ++-- 5 files changed, 17 insertions(+), 22 deletions(-) rename service/src/metrics.ts => packages/service-core/src/metrics/register-metrics.ts (63%) diff --git a/packages/service-core/src/metrics/metrics-index.ts b/packages/service-core/src/metrics/metrics-index.ts index 9c52331c..6f13dd7d 100644 --- a/packages/service-core/src/metrics/metrics-index.ts +++ b/packages/service-core/src/metrics/metrics-index.ts @@ -1,4 +1,5 @@ export * from './MetricsEngine.js'; export * from './metrics-interfaces.js'; +export * from './register-metrics.js'; export * from './open-telemetry/OpenTelemetryMetricsFactory.js'; export * from './open-telemetry/util.js'; diff --git a/service/src/metrics.ts b/packages/service-core/src/metrics/register-metrics.ts similarity index 63% rename from service/src/metrics.ts rename to packages/service-core/src/metrics/register-metrics.ts index 6e0e9ea0..5fef5a8f 100644 --- a/service/src/metrics.ts +++ b/packages/service-core/src/metrics/register-metrics.ts @@ -1,12 +1,9 @@ -import * as core from '@powersync/service-core'; -import { - createCoreAPIMetrics, - createCoreReplicationMetrics, - createCoreStorageMetrics, - initializeCoreAPIMetrics, - initializeCoreReplicationMetrics, - initializeCoreStorageMetrics -} from '@powersync/service-core'; +import { ServiceContextContainer } from '../system/ServiceContext.js'; +import { createOpenTelemetryMetricsFactory } from './open-telemetry/util.js'; +import { MetricsEngine } from './MetricsEngine.js'; +import { createCoreAPIMetrics, initializeCoreAPIMetrics } from '../api/api-metrics.js'; +import { createCoreReplicationMetrics, initializeCoreReplicationMetrics } from '../replication/replication-metrics.js'; +import { createCoreStorageMetrics, initializeCoreStorageMetrics } from '../storage/storage-metrics.js'; export enum MetricModes { API = 'api', @@ -15,19 +12,19 @@ export enum MetricModes { } export type MetricsRegistrationOptions = { - service_context: core.system.ServiceContextContainer; + service_context: ServiceContextContainer; modes: MetricModes[]; }; export const registerMetrics = async (options: MetricsRegistrationOptions) => { const { service_context, modes } = options; - const metricsFactory = core.metrics.createOpenTelemetryMetricsFactory(service_context); - const metricsEngine = new core.metrics.MetricsEngine({ + const metricsFactory = createOpenTelemetryMetricsFactory(service_context); + const metricsEngine = new MetricsEngine({ factory: metricsFactory, disable_telemetry_sharing: service_context.configuration.telemetry.disable_telemetry_sharing }); - service_context.register(core.metrics.MetricsEngine, metricsEngine); + service_context.register(MetricsEngine, metricsEngine); if (modes.includes(MetricModes.API)) { createCoreAPIMetrics(metricsEngine); diff --git a/service/src/runners/server.ts b/service/src/runners/server.ts index 1259b1e9..2420f2f7 100644 --- a/service/src/runners/server.ts +++ b/service/src/runners/server.ts @@ -5,7 +5,6 @@ import { container, logger } from '@powersync/lib-services-framework'; import * as core from '@powersync/service-core'; import { ReactiveSocketRouter } from '@powersync/service-rsocket-router'; -import { MetricModes, registerMetrics } from '../metrics.js'; /** * Configures the server portion on a {@link ServiceContext} @@ -82,9 +81,9 @@ export async function startServer(runnerConfig: core.utils.RunnerConfig) { registerServerServices(serviceContext); - await registerMetrics({ + await core.metrics.registerMetrics({ service_context: serviceContext, - modes: [MetricModes.API] + modes: [core.metrics.MetricModes.API] }); const moduleManager = container.getImplementation(core.modules.ModuleManager); diff --git a/service/src/runners/stream-worker.ts b/service/src/runners/stream-worker.ts index 42d5382a..4acda246 100644 --- a/service/src/runners/stream-worker.ts +++ b/service/src/runners/stream-worker.ts @@ -1,6 +1,5 @@ import { container, logger } from '@powersync/lib-services-framework'; import * as core from '@powersync/service-core'; -import { MetricModes, registerMetrics } from '../metrics.js'; /** * Configures the replication portion on a {@link serviceContext} @@ -26,9 +25,9 @@ export const startStreamRunner = async (runnerConfig: core.utils.RunnerConfig) = registerReplicationServices(serviceContext); - await registerMetrics({ + await core.metrics.registerMetrics({ service_context: serviceContext, - modes: [MetricModes.REPLICATION, MetricModes.STORAGE] + modes: [core.metrics.MetricModes.REPLICATION, core.metrics.MetricModes.STORAGE] }); const moduleManager = container.getImplementation(core.modules.ModuleManager); diff --git a/service/src/runners/unified-runner.ts b/service/src/runners/unified-runner.ts index b2fc684f..22671f52 100644 --- a/service/src/runners/unified-runner.ts +++ b/service/src/runners/unified-runner.ts @@ -1,7 +1,6 @@ import { container, logger } from '@powersync/lib-services-framework'; import * as core from '@powersync/service-core'; -import { MetricModes, registerMetrics } from '../metrics.js'; import { registerServerServices } from './server.js'; import { registerReplicationServices } from './stream-worker.js'; @@ -18,9 +17,9 @@ export const startUnifiedRunner = async (runnerConfig: core.utils.RunnerConfig) registerServerServices(serviceContext); registerReplicationServices(serviceContext); - await registerMetrics({ + await core.metrics.registerMetrics({ service_context: serviceContext, - modes: [MetricModes.API, MetricModes.REPLICATION, MetricModes.STORAGE] + modes: [core.metrics.MetricModes.API, core.metrics.MetricModes.REPLICATION, core.metrics.MetricModes.STORAGE] }); const moduleManager = container.getImplementation(core.modules.ModuleManager); From ed8e76e55c1e17588104fbfc04eee17fef537341 Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 26 Mar 2025 11:15:31 +0200 Subject: [PATCH 16/18] Removed unused dependencies in service package Consolidated some extra logging info from powersync cloud runners into service runners --- pnpm-lock.yaml | 181 +------------------------- service/package.json | 25 +--- service/src/runners/server.ts | 5 +- service/src/runners/stream-worker.ts | 4 +- service/src/runners/unified-runner.ts | 5 +- service/src/util/version.ts | 8 ++ 6 files changed, 20 insertions(+), 208 deletions(-) create mode 100644 service/src/util/version.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 53db405c..6ab9f4b2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -630,27 +630,12 @@ importers: '@fastify/cors': specifier: 8.4.1 version: 8.4.1 - '@opentelemetry/api': - specifier: ~1.6.0 - version: 1.6.0 - '@opentelemetry/exporter-prometheus': - specifier: ^0.43.0 - version: 0.43.0(@opentelemetry/api@1.6.0) - '@opentelemetry/sdk-metrics': - specifier: ^1.17.0 - version: 1.24.1(@opentelemetry/api@1.6.0) '@powersync/lib-services-framework': specifier: workspace:* version: link:../libs/lib-services '@powersync/service-core': specifier: workspace:* version: link:../packages/service-core - '@powersync/service-jpgwire': - specifier: workspace:* - version: link:../packages/jpgwire - '@powersync/service-jsonbig': - specifier: workspace:* - version: link:../packages/jsonbig '@powersync/service-module-mongodb': specifier: workspace:* version: link:../modules/module-mongodb @@ -669,70 +654,16 @@ importers: '@powersync/service-rsocket-router': specifier: workspace:* version: link:../packages/rsocket-router - '@powersync/service-sync-rules': - specifier: workspace:* - version: link:../packages/sync-rules - '@powersync/service-types': - specifier: workspace:* - version: link:../packages/types '@sentry/node': specifier: ^8.9.2 version: 8.17.0 - async-mutex: - specifier: ^0.5.0 - version: 0.5.0 - bson: - specifier: ^6.10.3 - version: 6.10.3 - commander: - specifier: ^12.0.0 - version: 12.1.0 - cors: - specifier: ^2.8.5 - version: 2.8.5 fastify: specifier: 4.23.2 version: 4.23.2 - ipaddr.js: - specifier: ^2.1.0 - version: 2.2.0 - ix: - specifier: ^5.0.0 - version: 5.0.0 - jose: - specifier: ^4.15.1 - version: 4.15.9 - lru-cache: - specifier: ^10.0.1 - version: 10.4.3 - mongodb: - specifier: ^6.14.1 - version: 6.14.2(socks@2.8.3) - node-fetch: - specifier: ^3.3.2 - version: 3.3.2 - pgwire: - specifier: github:kagis/pgwire#f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87 - version: https://codeload.github.com/kagis/pgwire/tar.gz/f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87 - ts-codec: - specifier: ^1.3.0 - version: 1.3.0 - uuid: - specifier: ^9.0.1 - version: 9.0.1 - winston: - specifier: ^3.13.0 - version: 3.13.1 - yaml: - specifier: ^2.3.2 - version: 2.4.5 devDependencies: '@sentry/types': specifier: ^8.9.2 version: 8.17.0 - '@types/uuid': - specifier: ^9.0.4 - version: 9.0.8 copyfiles: specifier: ^2.4.1 version: 2.4.1 @@ -1118,10 +1049,6 @@ packages: resolution: {integrity: sha512-uIX52NnTM0iBh84MShlpouI7UKqkZ7MrUszTmaypHBu4r7NofznSnQRfJ+uUeDtQDj6w8eFGg5KBLDAwAPz1+A==} engines: {node: '>=14'} - '@opentelemetry/api@1.6.0': - resolution: {integrity: sha512-OWlrQAnWn9577PhVgqjUvMr1pg57Bc4jv0iL4w0PRuOSRvq67rvHW9Ie/dZVMvCzhSCB+UxhcY/PmCmFj33Q+g==} - engines: {node: '>=8.0.0'} - '@opentelemetry/api@1.9.0': resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} engines: {node: '>=8.0.0'} @@ -1132,18 +1059,6 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/core@1.17.0': - resolution: {integrity: sha512-tfnl3h+UefCgx1aeN2xtrmr6BmdWGKXypk0pflQR0urFS40aE88trnkOMc2HTJZbMrqEEl4HsaBeFhwLVXsrJg==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.7.0' - - '@opentelemetry/core@1.24.1': - resolution: {integrity: sha512-wMSGfsdmibI88K9wB498zXY04yThPexo8jvwNNlm542HZB7XrrMRBbAyKJqG8qDRJwIBdBrPMi4V9ZPW/sqrcg==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.9.0' - '@opentelemetry/core@1.25.1': resolution: {integrity: sha512-GeT/l6rBYWVQ4XArluLVB6WWQ8flHbdb6r2FCHC3smtdOAbrJBIv35tpV/yp9bmYUJf+xmZpu9DRTIeJVhFbEQ==} engines: {node: '>=14'} @@ -1162,12 +1077,6 @@ packages: peerDependencies: '@opentelemetry/api': ^1.3.0 - '@opentelemetry/exporter-prometheus@0.43.0': - resolution: {integrity: sha512-tJeZVmzzeG98BMPssrnUYZ7AdMtZEYqgOL44z/bF4YWqGePQoelmxuTn8Do0tIyBURqr0Whi/7P5/XxWMK1zTw==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': ^1.3.0 - '@opentelemetry/exporter-prometheus@0.57.2': resolution: {integrity: sha512-VqIqXnuxWMWE/1NatAGtB1PvsQipwxDcdG4RwA/umdBcW3/iOHp0uejvFHTRN2O78ZPged87ErJajyUBPUhlDQ==} engines: {node: '>=14'} @@ -1292,18 +1201,6 @@ packages: resolution: {integrity: sha512-faYX1N0gpLhej/6nyp6bgRjzAKXn5GOEMYY7YhciSfCoITAktLUtQ36d24QEWNA1/WA1y6qQunCe0OhHRkVl9g==} engines: {node: '>=14'} - '@opentelemetry/resources@1.17.0': - resolution: {integrity: sha512-+u0ciVnj8lhuL/qGRBPeVYvk7fL+H/vOddfvmOeJaA1KC+5/3UED1c9KoZQlRsNT5Kw1FaK8LkY2NVLYfOVZQw==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.7.0' - - '@opentelemetry/resources@1.24.1': - resolution: {integrity: sha512-cyv0MwAaPF7O86x5hk3NNgenMObeejZFLJJDVuSeSMIsknlsj3oOZzRv3qSzlwYomXsICfBeFFlxwHQte5mGXQ==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.9.0' - '@opentelemetry/resources@1.25.1': resolution: {integrity: sha512-pkZT+iFYIZsVn6+GzM0kSX+u3MSLCY9md+lIJOoKl/P+gJFfxJte/60Usdp8Ce4rOs8GduUpSPNe1ddGyDT1sQ==} engines: {node: '>=14'} @@ -1322,18 +1219,6 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.4.0 <1.10.0' - '@opentelemetry/sdk-metrics@1.17.0': - resolution: {integrity: sha512-HlWM27yGmYuwCoVRe3yg2PqKnIsq0kEF0HQgvkeDWz2NYkq9fFaSspR6kvjxUTbghAlZrabiqbgyKoYpYaXS3w==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.3.0 <1.7.0' - - '@opentelemetry/sdk-metrics@1.24.1': - resolution: {integrity: sha512-FrAqCbbGao9iKI+Mgh+OsC9+U2YMoXnlDHe06yH7dvavCKzE3S892dGtX54+WhSFVxHR/TMRVJiK/CV93GR0TQ==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.3.0 <1.9.0' - '@opentelemetry/sdk-metrics@1.30.1': resolution: {integrity: sha512-q9zcZ0Okl8jRgmy7eNW3Ku1XSgg3sDLa5evHZpCwjspw7E8Is4K/haRPDJrBcX3YSn/Y7gUvFnByNYEKQNbNog==} engines: {node: '>=14'} @@ -1352,14 +1237,6 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/semantic-conventions@1.17.0': - resolution: {integrity: sha512-+fguCd2d8d2qruk0H0DsCEy2CTK3t0Tugg7MhZ/UQMvmewbZLNnJ6heSYyzIZWG5IPfAXzoj4f4F/qpM7l4VBA==} - engines: {node: '>=14'} - - '@opentelemetry/semantic-conventions@1.24.1': - resolution: {integrity: sha512-VkliWlS4/+GHLLW7J/rVBA00uXus1SWvwFvcUDxDwmFxYfg/2VI6ekwdXS28cjI8Qz2ky2BzG8OUHo+WeYIWqw==} - engines: {node: '>=14'} - '@opentelemetry/semantic-conventions@1.25.1': resolution: {integrity: sha512-ZDjMJJQRlyk8A1KZFCc+bCbsyrn1wTwdNt56F7twdfUfnHUZUq77/WfONCj8p72NZOyP7pNTdUWSTYC3GTbuuQ==} engines: {node: '>=14'} @@ -2750,9 +2627,6 @@ packages: resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} engines: {node: '>=10'} - lodash.merge@4.6.2: - resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} - lodash.startcase@4.4.0: resolution: {integrity: sha512-+WKqsK294HMSc2jEbNgpHpd0JfIBhp7rEV4aqXWqFr6AlXov+SlcgB1Fv01y2kGe3Gc8nMW7VA0SrGuSkRfIEg==} @@ -4576,24 +4450,12 @@ snapshots: dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/api@1.6.0': {} - '@opentelemetry/api@1.9.0': {} '@opentelemetry/context-async-hooks@1.25.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/core@1.17.0(@opentelemetry/api@1.6.0)': - dependencies: - '@opentelemetry/api': 1.6.0 - '@opentelemetry/semantic-conventions': 1.17.0 - - '@opentelemetry/core@1.24.1(@opentelemetry/api@1.6.0)': - dependencies: - '@opentelemetry/api': 1.6.0 - '@opentelemetry/semantic-conventions': 1.24.1 - '@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -4613,13 +4475,6 @@ snapshots: '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-metrics': 1.30.1(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-prometheus@0.43.0(@opentelemetry/api@1.6.0)': - dependencies: - '@opentelemetry/api': 1.6.0 - '@opentelemetry/core': 1.17.0(@opentelemetry/api@1.6.0) - '@opentelemetry/resources': 1.17.0(@opentelemetry/api@1.6.0) - '@opentelemetry/sdk-metrics': 1.17.0(@opentelemetry/api@1.6.0) - '@opentelemetry/exporter-prometheus@0.57.2(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -4806,18 +4661,6 @@ snapshots: '@opentelemetry/redis-common@0.36.2': {} - '@opentelemetry/resources@1.17.0(@opentelemetry/api@1.6.0)': - dependencies: - '@opentelemetry/api': 1.6.0 - '@opentelemetry/core': 1.17.0(@opentelemetry/api@1.6.0) - '@opentelemetry/semantic-conventions': 1.17.0 - - '@opentelemetry/resources@1.24.1(@opentelemetry/api@1.6.0)': - dependencies: - '@opentelemetry/api': 1.6.0 - '@opentelemetry/core': 1.24.1(@opentelemetry/api@1.6.0) - '@opentelemetry/semantic-conventions': 1.24.1 - '@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -4837,20 +4680,6 @@ snapshots: '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.0) '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-metrics@1.17.0(@opentelemetry/api@1.6.0)': - dependencies: - '@opentelemetry/api': 1.6.0 - '@opentelemetry/core': 1.17.0(@opentelemetry/api@1.6.0) - '@opentelemetry/resources': 1.17.0(@opentelemetry/api@1.6.0) - lodash.merge: 4.6.2 - - '@opentelemetry/sdk-metrics@1.24.1(@opentelemetry/api@1.6.0)': - dependencies: - '@opentelemetry/api': 1.6.0 - '@opentelemetry/core': 1.24.1(@opentelemetry/api@1.6.0) - '@opentelemetry/resources': 1.24.1(@opentelemetry/api@1.6.0) - lodash.merge: 4.6.2 - '@opentelemetry/sdk-metrics@1.30.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -4871,10 +4700,6 @@ snapshots: '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': 1.28.0 - '@opentelemetry/semantic-conventions@1.17.0': {} - - '@opentelemetry/semantic-conventions@1.24.1': {} - '@opentelemetry/semantic-conventions@1.25.1': {} '@opentelemetry/semantic-conventions@1.28.0': {} @@ -5028,7 +4853,7 @@ snapshots: '@opentelemetry/semantic-conventions': 1.25.1 '@prisma/instrumentation': 5.16.1 '@sentry/core': 8.17.0 - '@sentry/opentelemetry': 8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.25.1) + '@sentry/opentelemetry': 8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.25.1) '@sentry/types': 8.17.0 '@sentry/utils': 8.17.0 optionalDependencies: @@ -5036,7 +4861,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@sentry/opentelemetry@8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.25.1)': + '@sentry/opentelemetry@8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.25.1)': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/core': 1.25.1(@opentelemetry/api@1.9.0) @@ -6272,8 +6097,6 @@ snapshots: dependencies: p-locate: 5.0.0 - lodash.merge@4.6.2: {} - lodash.startcase@4.4.0: {} lodash@4.17.21: {} diff --git a/service/package.json b/service/package.json index a2c1b37b..cab0724f 100644 --- a/service/package.json +++ b/service/package.json @@ -11,9 +11,6 @@ }, "dependencies": { "@fastify/cors": "8.4.1", - "@opentelemetry/api": "~1.6.0", - "@opentelemetry/exporter-prometheus": "^0.43.0", - "@opentelemetry/sdk-metrics": "^1.17.0", "@powersync/service-core": "workspace:*", "@powersync/lib-services-framework": "workspace:*", "@powersync/service-module-postgres": "workspace:*", @@ -21,32 +18,12 @@ "@powersync/service-module-mongodb": "workspace:*", "@powersync/service-module-mongodb-storage": "workspace:*", "@powersync/service-module-mysql": "workspace:*", - "@powersync/service-jpgwire": "workspace:*", - "@powersync/service-jsonbig": "workspace:*", "@powersync/service-rsocket-router": "workspace:*", - "@powersync/service-sync-rules": "workspace:*", - "@powersync/service-types": "workspace:*", "@sentry/node": "^8.9.2", - "async-mutex": "^0.5.0", - "bson": "^6.10.3", - "commander": "^12.0.0", - "cors": "^2.8.5", - "fastify": "4.23.2", - "ipaddr.js": "^2.1.0", - "ix": "^5.0.0", - "jose": "^4.15.1", - "lru-cache": "^10.0.1", - "mongodb": "^6.14.1", - "node-fetch": "^3.3.2", - "pgwire": "github:kagis/pgwire#f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87", - "ts-codec": "^1.3.0", - "uuid": "^9.0.1", - "winston": "^3.13.0", - "yaml": "^2.3.2" + "fastify": "4.23.2" }, "devDependencies": { "@sentry/types": "^8.9.2", - "@types/uuid": "^9.0.4", "copyfiles": "^2.4.1", "nodemon": "^3.0.1", "npm-check-updates": "^16.14.4", diff --git a/service/src/runners/server.ts b/service/src/runners/server.ts index 2420f2f7..6eff0cfc 100644 --- a/service/src/runners/server.ts +++ b/service/src/runners/server.ts @@ -5,6 +5,7 @@ import { container, logger } from '@powersync/lib-services-framework'; import * as core from '@powersync/service-core'; import { ReactiveSocketRouter } from '@powersync/service-rsocket-router'; +import { logBooting } from '../util/version.js'; /** * Configures the server portion on a {@link ServiceContext} @@ -74,9 +75,10 @@ export function registerServerServices(serviceContext: core.system.ServiceContex * Starts an API server */ export async function startServer(runnerConfig: core.utils.RunnerConfig) { - logger.info('Booting'); + logBooting('API Container'); const config = await core.utils.loadConfig(runnerConfig); + core.utils.setTags(config.metadata); const serviceContext = new core.system.ServiceContextContainer(config); registerServerServices(serviceContext); @@ -90,7 +92,6 @@ export async function startServer(runnerConfig: core.utils.RunnerConfig) { await moduleManager.initialize(serviceContext); logger.info('Starting service...'); - await serviceContext.lifeCycleEngine.start(); logger.info('Service started.'); diff --git a/service/src/runners/stream-worker.ts b/service/src/runners/stream-worker.ts index 4acda246..ed27fef7 100644 --- a/service/src/runners/stream-worker.ts +++ b/service/src/runners/stream-worker.ts @@ -1,5 +1,6 @@ import { container, logger } from '@powersync/lib-services-framework'; import * as core from '@powersync/service-core'; +import { logBooting } from '../util/version.js'; /** * Configures the replication portion on a {@link serviceContext} @@ -16,9 +17,10 @@ export const registerReplicationServices = (serviceContext: core.system.ServiceC }; export const startStreamRunner = async (runnerConfig: core.utils.RunnerConfig) => { - logger.info('Booting'); + logBooting('Replication Container'); const config = await core.utils.loadConfig(runnerConfig); + core.utils.setTags(config.metadata); // Self-hosted version allows for automatic migrations const serviceContext = new core.system.ServiceContextContainer(config); diff --git a/service/src/runners/unified-runner.ts b/service/src/runners/unified-runner.ts index 22671f52..b5767b5a 100644 --- a/service/src/runners/unified-runner.ts +++ b/service/src/runners/unified-runner.ts @@ -3,15 +3,16 @@ import * as core from '@powersync/service-core'; import { registerServerServices } from './server.js'; import { registerReplicationServices } from './stream-worker.js'; +import { logBooting } from '../util/version.js'; /** * Starts an API server */ export const startUnifiedRunner = async (runnerConfig: core.utils.RunnerConfig) => { - logger.info('Booting'); + logBooting('Unified Container'); const config = await core.utils.loadConfig(runnerConfig); - + core.utils.setTags(config.metadata); const serviceContext = new core.system.ServiceContextContainer(config); registerServerServices(serviceContext); diff --git a/service/src/util/version.ts b/service/src/util/version.ts new file mode 100644 index 00000000..887b8692 --- /dev/null +++ b/service/src/util/version.ts @@ -0,0 +1,8 @@ +import { logger } from '@powersync/lib-services-framework'; + +export function logBooting(runner: string) { + const version = process.env.POWERSYNC_VERSION ?? '-dev'; + const isCloud = process.env.MICRO_SERVICE_NAME == 'powersync'; + const edition = isCloud ? 'Cloud Edition' : 'Enterprise Edition'; + logger.info(`Booting PowerSync Service v${version}, ${runner}, ${edition}`, { version, edition, runner }); +} From af146f614d9568c3b8e1636734b05815c1a0c0ea Mon Sep 17 00:00:00 2001 From: Roland Teichert Date: Wed, 26 Mar 2025 15:03:28 +0200 Subject: [PATCH 17/18] Only set up data replication metrics recorder for PostgresModule when it is started in a replication context. --- .../module-postgres/src/module/PostgresModule.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/modules/module-postgres/src/module/PostgresModule.ts b/modules/module-postgres/src/module/PostgresModule.ts index 4dc3bd5f..7dfb0610 100644 --- a/modules/module-postgres/src/module/PostgresModule.ts +++ b/modules/module-postgres/src/module/PostgresModule.ts @@ -44,12 +44,15 @@ export class PostgresModule extends replication.ReplicationModule Date: Wed, 9 Apr 2025 09:07:45 +0200 Subject: [PATCH 18/18] Moved the prometheus metrics port from an env variable to the collected powersync config --- modules/module-mongodb/test/src/env.ts | 2 +- .../src/metrics/open-telemetry/util.ts | 14 ++++++++++---- .../src/util/config/compound-config-collector.ts | 1 + packages/service-core/src/util/config/types.ts | 1 + packages/service-core/src/util/env.ts | 4 ---- packages/types/src/config/PowerSyncConfig.ts | 4 +++- 6 files changed, 16 insertions(+), 10 deletions(-) diff --git a/modules/module-mongodb/test/src/env.ts b/modules/module-mongodb/test/src/env.ts index 971bf241..ad0f171e 100644 --- a/modules/module-mongodb/test/src/env.ts +++ b/modules/module-mongodb/test/src/env.ts @@ -7,5 +7,5 @@ export const env = utils.collectEnvironmentVariables({ CI: utils.type.boolean.default('false'), SLOW_TESTS: utils.type.boolean.default('false'), TEST_MONGO_STORAGE: utils.type.boolean.default('true'), - TEST_POSTGRES_STORAGE: utils.type.boolean.default('false') + TEST_POSTGRES_STORAGE: utils.type.boolean.default('true') }); diff --git a/packages/service-core/src/metrics/open-telemetry/util.ts b/packages/service-core/src/metrics/open-telemetry/util.ts index 5396d795..af4af406 100644 --- a/packages/service-core/src/metrics/open-telemetry/util.ts +++ b/packages/service-core/src/metrics/open-telemetry/util.ts @@ -1,11 +1,11 @@ import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; -import { env } from '../../util/env.js'; import { PrometheusExporter } from '@opentelemetry/exporter-prometheus'; import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'; import { Resource } from '@opentelemetry/resources'; import { ServiceContext } from '../../system/ServiceContext.js'; import { OpenTelemetryMetricsFactory } from './OpenTelemetryMetricsFactory.js'; import { MetricsFactory } from '../metrics-interfaces.js'; +import { logger } from '@powersync/lib-services-framework'; export interface RuntimeMetadata { [key: string]: string | number | undefined; @@ -15,12 +15,18 @@ export function createOpenTelemetryMetricsFactory(context: ServiceContext): Metr const { configuration, lifeCycleEngine, storageEngine } = context; const configuredExporters: MetricReader[] = []; - if (env.METRICS_PORT) { - const prometheusExporter = new PrometheusExporter({ port: env.METRICS_PORT, preventServerStart: true }); + if (configuration.telemetry.prometheus_port) { + const prometheusExporter = new PrometheusExporter({ + port: configuration.telemetry.prometheus_port, + preventServerStart: true + }); configuredExporters.push(prometheusExporter); lifeCycleEngine.withLifecycle(prometheusExporter, { - start: () => prometheusExporter.startServer() + start: async () => { + await prometheusExporter.startServer(); + logger.info(`Prometheus metric export enabled on port:${configuration.telemetry.prometheus_port}`); + } }); } diff --git a/packages/service-core/src/util/config/compound-config-collector.ts b/packages/service-core/src/util/config/compound-config-collector.ts index fc9b53d7..dab753e0 100644 --- a/packages/service-core/src/util/config/compound-config-collector.ts +++ b/packages/service-core/src/util/config/compound-config-collector.ts @@ -154,6 +154,7 @@ export class CompoundConfigCollector { metadata: baseConfig.metadata ?? {}, migrations: baseConfig.migrations, telemetry: { + prometheus_port: baseConfig.telemetry?.prometheus_port, disable_telemetry_sharing: baseConfig.telemetry?.disable_telemetry_sharing ?? false, internal_service_endpoint: baseConfig.telemetry?.internal_service_endpoint ?? 'https://pulse.journeyapps.com/v1/metrics' diff --git a/packages/service-core/src/util/config/types.ts b/packages/service-core/src/util/config/types.ts index 0781a6ba..56b834ef 100644 --- a/packages/service-core/src/util/config/types.ts +++ b/packages/service-core/src/util/config/types.ts @@ -55,6 +55,7 @@ export type ResolvedPowerSyncConfig = { }; telemetry: { + prometheus_port?: number; disable_telemetry_sharing: boolean; internal_service_endpoint: string; }; diff --git a/packages/service-core/src/util/env.ts b/packages/service-core/src/util/env.ts index c373371b..f88267c6 100644 --- a/packages/service-core/src/util/env.ts +++ b/packages/service-core/src/util/env.ts @@ -19,10 +19,6 @@ export const env = utils.collectEnvironmentVariables({ * Runner to be started in this process */ PS_RUNNER_TYPE: utils.type.string.default(ServiceRunner.UNIFIED), - /** - * Port for metrics - */ - METRICS_PORT: utils.type.number.optional(), NODE_ENV: utils.type.string.optional() }); diff --git a/packages/types/src/config/PowerSyncConfig.ts b/packages/types/src/config/PowerSyncConfig.ts index a235ee58..9a8c70ce 100644 --- a/packages/types/src/config/PowerSyncConfig.ts +++ b/packages/types/src/config/PowerSyncConfig.ts @@ -104,7 +104,7 @@ export type StrictJwk = t.Decoded; export const BaseStorageConfig = t.object({ type: t.string, - // Maximum number of conncetions to the storage database, per process. + // Maximum number of connections to the storage database, per process. // Defaults to 8. max_pool_size: t.number.optional() }); @@ -200,6 +200,8 @@ export const powerSyncConfig = t.object({ telemetry: t .object({ + // When set, metrics will be available on this port for scraping by Prometheus. + prometheus_port: portCodec.optional(), disable_telemetry_sharing: t.boolean, internal_service_endpoint: t.string.optional() })