From bca401e2fd9634dc0f24ddef98bcab00e15684f1 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 16 Jul 2025 19:37:33 +0100 Subject: [PATCH 1/6] chore(nodejs): extract transport from sender, add back support for core http --- .gitignore | 1 - src/logging.ts | 11 +- src/options.ts | 91 +- src/sender.ts | 873 ++------- src/transport/http/base.ts | 114 ++ src/transport/http/legacy.ts | 165 ++ src/transport/http/undici.ts | 143 ++ src/transport/index.ts | 36 + src/transport/tcp.ts | 302 +++ src/utils.ts | 37 + test/logging.test.ts | 20 +- test/options.test.ts | 10 +- test/sender.buffer.test.ts | 803 ++++++++ test/sender.config.test.ts | 402 ++++ test/sender.integration.test.ts | 392 ++++ test/sender.test.ts | 2176 ---------------------- test/sender.transport.test.ts | 568 ++++++ test/testapp.ts | 9 +- test/{_utils_ => util}/mockhttp.ts | 36 +- test/{_utils_ => util}/mockproxy.ts | 19 +- test/{_utils_ => util}/proxy.ts | 12 +- test/{_utils_ => util}/proxyfunctions.ts | 24 +- 22 files changed, 3271 insertions(+), 2973 deletions(-) create mode 100644 src/transport/http/base.ts create mode 100644 src/transport/http/legacy.ts create mode 100644 src/transport/http/undici.ts create mode 100644 src/transport/index.ts create mode 100644 src/transport/tcp.ts create mode 100644 src/utils.ts create mode 100644 test/sender.buffer.test.ts create mode 100644 test/sender.config.test.ts create mode 100644 test/sender.integration.test.ts delete mode 100644 test/sender.test.ts create mode 100644 test/sender.transport.test.ts rename test/{_utils_ => util}/mockhttp.ts (74%) rename test/{_utils_ => util}/mockproxy.ts (78%) rename test/{_utils_ => util}/proxy.ts (78%) rename test/{_utils_ => util}/proxyfunctions.ts (66%) diff --git a/.gitignore b/.gitignore index 24018fa..b1870ec 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,4 @@ coverage .idea *.iml .DS_Store -certs dist \ No newline at end of file diff --git a/src/logging.ts b/src/logging.ts index 58b4ddf..48098b2 100644 --- a/src/logging.ts +++ b/src/logging.ts @@ -7,15 +7,20 @@ const LOG_LEVELS = { const DEFAULT_CRITICALITY = LOG_LEVELS.info.criticality; +type Logger = ( + level: "error" | "warn" | "info" | "debug", + message: string | Error, +) => void; + /** * Simple logger to write log messages to the console.
* Supported logging levels are `error`, `warn`, `info` and `debug`.
* Throws an error if logging level is invalid. * * @param {'error'|'warn'|'info'|'debug'} level - The log level of the message. - * @param {string} message - The log message. + * @param {string | Error} message - The log message. */ -function log(level: "error" | "warn" | "info" | "debug", message: string) { +function log(level: "error" | "warn" | "info" | "debug", message: string | Error) { const logLevel = LOG_LEVELS[level]; if (!logLevel) { throw new Error(`Invalid log level: '${level}'`); @@ -25,4 +30,4 @@ function log(level: "error" | "warn" | "info" | "debug", message: string) { } } -export { log }; +export { log, Logger }; diff --git a/src/options.ts b/src/options.ts index 9b32645..ee39178 100644 --- a/src/options.ts +++ b/src/options.ts @@ -1,5 +1,9 @@ import { PathOrFileDescriptor } from "fs"; import { Agent } from "undici"; +import http from "http"; +import https from "https"; + +import { Logger } from "./logging"; const HTTP_PORT = 9000; const TCP_PORT = 9009; @@ -13,6 +17,20 @@ const ON = "on"; const OFF = "off"; const UNSAFE_OFF = "unsafe_off"; +type ExtraOptions = { + log?: Logger; + agent?: Agent | http.Agent | https.Agent; +} + +type DeprecatedOptions = { + /** @deprecated */ + copy_buffer?: boolean; + /** @deprecated */ + copyBuffer?: boolean; + /** @deprecated */ + bufferSize?: number; +}; + /** @classdesc * Sender configuration options.
*
@@ -139,10 +157,10 @@ class SenderOptions { max_name_len?: number; - log?: - | ((level: "error" | "warn" | "info" | "debug", message: string) => void) - | null; - agent?: Agent; + log?: Logger; + agent?: Agent | http.Agent | https.Agent; + + legacy_http?: boolean; auth?: { username?: string; @@ -162,7 +180,7 @@ class SenderOptions { * - 'agent' is a custom http/https agent used by the Sender when http/https transport is used.
* A http.Agent or https.Agent object is expected. */ - constructor(configurationString: string, extraOptions = undefined) { + constructor(configurationString: string, extraOptions: ExtraOptions = undefined) { parseConfigurationString(this, configurationString); if (extraOptions) { @@ -171,13 +189,36 @@ class SenderOptions { } this.log = extraOptions.log; - if (extraOptions.agent && !(extraOptions.agent instanceof Agent)) { - throw new Error("Invalid http/https agent"); + if ( + extraOptions.agent + && !(extraOptions.agent instanceof Agent) + && !(extraOptions.agent instanceof http.Agent) + // @ts-ignore + && !(extraOptions.agent instanceof https.Agent) + ) { + throw new Error("Invalid HTTP agent"); } this.agent = extraOptions.agent; } } + static resolveDeprecated(options: SenderOptions & DeprecatedOptions, log: Logger) { + // deal with deprecated options + if (options.copy_buffer !== undefined) { + log("warn", `Option 'copy_buffer' is not supported anymore, please, remove it`); + options.copy_buffer = undefined; + } + if (options.copyBuffer !== undefined) { + log("warn", `Option 'copyBuffer' is not supported anymore, please, remove it`); + options.copyBuffer = undefined; + } + if (options.bufferSize !== undefined) { + log("warn", `Option 'bufferSize' is not supported anymore, please, replace it with 'init_buf_size'`); + options.init_buf_size = options.bufferSize; + options.bufferSize = undefined; + } + } + /** * Creates a Sender options object by parsing the provided configuration string. * @@ -190,16 +231,7 @@ class SenderOptions { * * @return {SenderOptions} A Sender configuration object initialized from the provided configuration string. */ - static fromConfig( - configurationString: string, - extraOptions: { - log?: ( - level: "error" | "warn" | "info" | "debug", - message: string, - ) => void; - agent?: Agent; - } = undefined, - ) { + static fromConfig(configurationString: string, extraOptions: ExtraOptions = undefined): SenderOptions { return new SenderOptions(configurationString, extraOptions); } @@ -214,15 +246,12 @@ class SenderOptions { * * @return {SenderOptions} A Sender configuration object initialized from the QDB_CLIENT_CONF environment variable. */ - static fromEnv(extraOptions = undefined) { + static fromEnv(extraOptions: ExtraOptions = undefined): SenderOptions { return SenderOptions.fromConfig(process.env.QDB_CLIENT_CONF, extraOptions); } } -function parseConfigurationString( - options: SenderOptions, - configString: string, -) { +function parseConfigurationString(options: SenderOptions, configString: string) { if (!configString) { throw new Error("Configuration string is missing or empty"); } @@ -235,6 +264,7 @@ function parseConfigurationString( parseTlsOptions(options); parseRequestTimeoutOptions(options); parseMaxNameLength(options); + parseLegacyTransport(options); } function parseSettings( @@ -244,10 +274,7 @@ function parseSettings( ) { let index = configString.indexOf(";", position); while (index > -1) { - if ( - index + 1 < configString.length && - configString.charAt(index + 1) === ";" - ) { + if (index + 1 < configString.length && configString.charAt(index + 1) === ";") { index = configString.indexOf(";", index + 2); continue; } @@ -297,6 +324,7 @@ const ValidConfigKeys = [ "max_buf_size", "max_name_len", "tls_verify", + "legacy_http", "tls_ca", "tls_roots", "tls_roots_password", @@ -322,10 +350,7 @@ function validateConfigValue(key: string, value: string) { } } -function parseProtocol( - options: SenderOptions, - configString: string | string[], -) { +function parseProtocol(options: SenderOptions, configString: string) { const index = configString.indexOf("::"); if (index < 0) { throw new Error( @@ -422,6 +447,10 @@ function parseMaxNameLength(options: SenderOptions) { parseInteger(options, "max_name_len", "max name length", 1); } +function parseLegacyTransport(options: SenderOptions) { + parseBoolean(options, "legacy_http", "legacy http"); +} + function parseBoolean( options: SenderOptions, property: string, @@ -466,4 +495,4 @@ function parseInteger( } } -export { SenderOptions, HTTP, HTTPS, TCP, TCPS }; +export { SenderOptions, ExtraOptions, HTTP, HTTPS, TCP, TCPS }; diff --git a/src/sender.ts b/src/sender.ts index 4cc6cb9..9ad0786 100644 --- a/src/sender.ts +++ b/src/sender.ts @@ -1,61 +1,19 @@ // @ts-check -import { readFileSync } from "node:fs"; import { Buffer } from "node:buffer"; -import net from "node:net"; -import tls from "node:tls"; -import crypto from "node:crypto"; -import { Agent, RetryAgent } from "undici"; -import { log } from "./logging"; +import { log, Logger } from "./logging"; import { validateColumnName, validateTableName } from "./validation"; -import { SenderOptions, HTTP, HTTPS, TCP, TCPS } from "./options"; +import { SenderOptions, ExtraOptions } from "./options"; +import { SenderTransport, createTransport } from "./transport"; +import { isBoolean, isInteger, timestampToMicros, timestampToNanos } from "./utils"; -const HTTP_NO_CONTENT = 204; // success - -const DEFAULT_HTTP_AUTO_FLUSH_ROWS = 75000; -const DEFAULT_TCP_AUTO_FLUSH_ROWS = 600; const DEFAULT_AUTO_FLUSH_INTERVAL = 1000; // 1 sec const DEFAULT_MAX_NAME_LENGTH = 127; -const DEFAULT_REQUEST_MIN_THROUGHPUT = 102400; // 100 KB/sec -const DEFAULT_REQUEST_TIMEOUT = 10000; // 10 sec -const DEFAULT_RETRY_TIMEOUT = 10000; // 10 sec - const DEFAULT_BUFFER_SIZE = 65536; // 64 KB const DEFAULT_MAX_BUFFER_SIZE = 104857600; // 100 MB -/** @type {Agent.Options} */ -const DEFAULT_HTTP_OPTIONS: Agent.Options = { - connect: { - keepAlive: true, - }, - pipelining: 1, - keepAliveTimeout: 60000, // 1 minute -}; -// an arbitrary public key, not used in authentication -// only used to construct a valid JWK token which is accepted by the crypto API -const PUBLIC_KEY = { - x: "aultdA0PjhD_cWViqKKyL5chm6H1n-BiZBo_48T-uqc", - y: "__ptaol41JWSpTTL525yVEfzmY8A6Vi_QrW1FjKcHMg", -}; - -/* -We are retrying on the following response codes (copied from the Rust client): -500: Internal Server Error -503: Service Unavailable -504: Gateway Timeout - -// Unofficial extensions -507: Insufficient Storage -509: Bandwidth Limit Exceeded -523: Origin is Unreachable -524: A Timeout Occurred -529: Site is overloaded -599: Network Connect Timeout Error -*/ -const RETRIABLE_STATUS_CODES = [500, 503, 504, 507, 509, 523, 524, 529, 599]; - /** @classdesc * The QuestDB client's API provides methods to connect to the database, ingest data, and close the connection. * The supported protocols are HTTP and TCP. HTTP is preferred as it provides feedback in the HTTP response.
@@ -101,48 +59,27 @@ const RETRIABLE_STATUS_CODES = [500, 503, 504, 507, 509, 523, 524, 529, 599]; *

*/ class Sender { - /** @private */ static DEFAULT_HTTP_AGENT; - /** @private */ static DEFAULT_HTTPS_AGENT; - - /** @private */ http; // true if the protocol is HTTP/HTTPS, false if it is TCP/TCPS - /** @private */ secure; // true if the protocol is HTTPS or TCPS, false otherwise - /** @private */ host; - /** @private */ port; - - /** @private */ socket; - - /** @private */ username; - /** @private */ password; - /** @private */ token; - - /** @private */ tlsVerify; - /** @private */ tlsCA; + private readonly transport: SenderTransport; - /** @private */ bufferSize; - /** @private */ maxBufferSize; - /** @private */ buffer; - /** @private */ position; - /** @private */ endOfLastRow; + private bufferSize: number; + private readonly maxBufferSize: number; + private buffer: Buffer; + private position: number; + private endOfLastRow: number; - /** @private */ autoFlush; - /** @private */ autoFlushRows; - /** @private */ autoFlushInterval; - /** @private */ lastFlushTime; - /** @private */ pendingRowCount; + private readonly autoFlush: boolean; + private readonly autoFlushRows: number; + private readonly autoFlushInterval: number; + private lastFlushTime: number; + private pendingRowCount: number; - /** @private */ requestMinThroughput; - /** @private */ requestTimeout; - /** @private */ retryTimeout; + private hasTable: boolean; + private hasSymbols: boolean; + private hasColumns: boolean; - /** @private */ hasTable; - /** @private */ hasSymbols; - /** @private */ hasColumns; + private readonly maxNameLength: number; - /** @private */ maxNameLength; - - /** @private */ log; - /** @private */ agent; - /** @private */ jwk; + private readonly log: Logger; /** * Creates an instance of Sender. @@ -151,80 +88,15 @@ class Sender { * See SenderOptions documentation for detailed description of configuration options.
*/ constructor(options: SenderOptions) { - if (!options || !options.protocol) { - throw new Error("The 'protocol' option is mandatory"); - } - this.log = typeof options.log === "function" ? options.log : log; - replaceDeprecatedOptions(options, this.log); - - switch (options.protocol) { - case HTTP: - this.http = true; - this.secure = false; - this.agent = - options.agent instanceof Agent - ? options.agent - : Sender.getDefaultHttpAgent(); - break; - case HTTPS: - this.http = true; - this.secure = true; - if (options.agent instanceof Agent) { - this.agent = options.agent; - } else { - // Create a new agent with instance-specific TLS options - this.agent = new Agent({ - ...DEFAULT_HTTP_OPTIONS, - connect: { - ...DEFAULT_HTTP_OPTIONS.connect, - requestCert: isBoolean(options.tls_verify) ? options.tls_verify : true, - rejectUnauthorized: isBoolean(options.tls_verify) ? options.tls_verify : true, - ca: options.tls_ca ? readFileSync(options.tls_ca) : undefined, - }, - }); - } - break; - case TCP: - this.http = false; - this.secure = false; - break; - case TCPS: - this.http = false; - this.secure = true; - break; - default: - throw new Error(`Invalid protocol: '${options.protocol}'`); - } + this.transport = createTransport(options); - if (this.http) { - this.username = options.username; - this.password = options.password; - this.token = options.token; - if (!options.port) { - options.port = 9000; - } - } else { - if (!options.auth && !options.jwk) { - constructAuth(options); - } - this.jwk = constructJwk(options); - if (!options.port) { - options.port = 9009; - } - } - - this.host = options.host; - this.port = options.port; - - this.tlsVerify = isBoolean(options.tls_verify) ? options.tls_verify : true; - this.tlsCA = options.tls_ca ? readFileSync(options.tls_ca) : undefined; + this.log = typeof options.log === "function" ? options.log : log; + SenderOptions.resolveDeprecated(options, this.log); this.autoFlush = isBoolean(options.auto_flush) ? options.auto_flush : true; this.autoFlushRows = isInteger(options.auto_flush_rows, 0) ? options.auto_flush_rows - : this.http - ? DEFAULT_HTTP_AUTO_FLUSH_ROWS - : DEFAULT_TCP_AUTO_FLUSH_ROWS; + : this.transport.getDefaultAutoFlushRows(); this.autoFlushInterval = isInteger(options.auto_flush_interval, 0) ? options.auto_flush_interval : DEFAULT_AUTO_FLUSH_INTERVAL; @@ -233,16 +105,6 @@ class Sender { ? options.max_name_len : DEFAULT_MAX_NAME_LENGTH; - this.requestMinThroughput = isInteger(options.request_min_throughput, 0) - ? options.request_min_throughput - : DEFAULT_REQUEST_MIN_THROUGHPUT; - this.requestTimeout = isInteger(options.request_timeout, 1) - ? options.request_timeout - : DEFAULT_REQUEST_TIMEOUT; - this.retryTimeout = isInteger(options.retry_timeout, 0) - ? options.retry_timeout - : DEFAULT_RETRY_TIMEOUT; - this.maxBufferSize = isInteger(options.max_buf_size, 1) ? options.max_buf_size : DEFAULT_MAX_BUFFER_SIZE; @@ -266,10 +128,7 @@ class Sender { * * @return {Sender} A Sender object initialized from the provided configuration string. */ - static fromConfig( - configurationString: string, - extraOptions: object = undefined, - ): Sender { + static fromConfig(configurationString: string, extraOptions: ExtraOptions = undefined): Sender { return new Sender( SenderOptions.fromConfig(configurationString, extraOptions), ); @@ -286,7 +145,7 @@ class Sender { * * @return {Sender} A Sender object initialized from the QDB_CLIENT_CONF environment variable. */ - static fromEnv(extraOptions: object = undefined): Sender { + static fromEnv(extraOptions: ExtraOptions = undefined): Sender { return new Sender( SenderOptions.fromConfig(process.env.QDB_CLIENT_CONF, extraOptions), ); @@ -299,11 +158,9 @@ class Sender { * * @param {number} bufferSize - New size of the buffer used by the sender, provided in bytes. */ - resize(bufferSize: number) { + private resize(bufferSize: number) { if (bufferSize > this.maxBufferSize) { - throw new Error( - `Max buffer size is ${this.maxBufferSize} bytes, requested buffer size: ${bufferSize}`, - ); + throw new Error(`Max buffer size is ${this.maxBufferSize} bytes, requested buffer size: ${bufferSize}`); } this.bufferSize = bufferSize; // Allocating an extra byte because Buffer.write() does not fail if the length of the data to be written is @@ -327,109 +184,17 @@ class Sender { this.position = 0; this.lastFlushTime = Date.now(); this.pendingRowCount = 0; - startNewRow(this); + this.startNewRow(); return this; } /** * Creates a TCP connection to the database. * - * @param {net.NetConnectOpts | tls.ConnectionOptions} connectOptions - Connection options, host and port are required. - * * @return {Promise} Resolves to true if the client is connected. */ - connect( - connectOptions: net.NetConnectOpts | tls.ConnectionOptions = undefined, - ): Promise { - if (this.http) { - throw new Error( - "'connect()' should be called only if the sender connects via TCP", - ); - } - - if (!connectOptions) { - connectOptions = { - host: this.host, - port: this.port, - ca: this.tlsCA, - }; - } - if (!(connectOptions as tls.ConnectionOptions).host) { - throw new Error("Hostname is not set"); - } - if (!(connectOptions as tls.ConnectionOptions).port) { - throw new Error("Port is not set"); - } - - return new Promise((resolve, reject) => { - if (this.socket) { - throw new Error("Sender connected already"); - } - - let authenticated: boolean = false; - let data; - - this.socket = !this.secure - ? net.connect(connectOptions as net.NetConnectOpts) - : tls.connect(connectOptions, () => { - if (authenticated) { - resolve(true); - } - }); - this.socket.setKeepAlive(true); - - this.socket - .on("data", async (raw) => { - data = !data ? raw : Buffer.concat([data, raw]); - if (!authenticated) { - authenticated = await authenticate(this, data); - if (authenticated) { - resolve(true); - } - } else { - this.log("warn", `Received unexpected data: ${data}`); - } - }) - .on("ready", async () => { - this.log( - "info", - `Successfully connected to ${(connectOptions as tls.ConnectionOptions).host}:${(connectOptions as tls.ConnectionOptions).port}`, - ); - if (this.jwk) { - this.log( - "info", - `Authenticating with ${(connectOptions as tls.ConnectionOptions).host}:${(connectOptions as tls.ConnectionOptions).port}`, - ); - await this.socket.write(`${this.jwk.kid}\n`, (err: Error) => { - if (err) { - reject(err); - } - }); - } else { - authenticated = true; - if (!this.secure || !this.tlsVerify) { - resolve(true); - } - } - }) - .on("error", (err) => { - this.log("error", err); - if (err.code !== "SELF_SIGNED_CERT_IN_CHAIN" || this.tlsVerify) { - reject(err); - } - }); - }); - } - - /** - * @ignore - * @return {Agent} Returns the default http agent. - */ - static getDefaultHttpAgent(): Agent { - if (!Sender.DEFAULT_HTTP_AGENT) { - Sender.DEFAULT_HTTP_AGENT = new Agent(DEFAULT_HTTP_OPTIONS); - } - return Sender.DEFAULT_HTTP_AGENT; + connect(): Promise { + return this.transport.connect(); } /** @@ -439,103 +204,12 @@ class Sender { * @return {Promise} Resolves to true when there was data in the buffer to send, and it was sent successfully. */ async flush(): Promise { - const dataToSend = this.toBufferNew(this.endOfLastRow); + const dataToSend: Buffer = this.toBufferNew(); if (!dataToSend) { return false; // Nothing to send } - try { - if (this.http) { - const { timeout: calculatedTimeoutMillis } = createRequestOptions(this, dataToSend); - const retryBegin = Date.now(); - const headers: Record = {}; - - const dispatcher = new RetryAgent(this.agent, { - maxRetries: Infinity, - minTimeout: 10, - maxTimeout: 1000, - timeoutFactor: 2, - retryAfter: true, - methods: ["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD"], - statusCodes: RETRIABLE_STATUS_CODES, - errorCodes: [ - "ECONNRESET", - "EAI_AGAIN", - "ECONNREFUSED", - "ETIMEDOUT", - "EPIPE", - "UND_ERR_CONNECT_TIMEOUT", - "UND_ERR_HEADERS_TIMEOUT", - "UND_ERR_BODY_TIMEOUT", - ], - retry: (err, context, callback) => { - const elapsed = Date.now() - retryBegin; - if (elapsed > this.retryTimeout) { - return callback(err); - } - return callback(null); - }, - }); - - if (this.token) { - headers["Authorization"] = "Bearer " + this.token; - } else if (this.username && this.password) { - headers["Authorization"] = - "Basic " + - Buffer.from(this.username + ":" + this.password).toString("base64"); - } - - const { statusCode, body } = await dispatcher.request({ - origin: `${this.secure ? "https" : "http"}://${this.host}:${this.port}`, - path: "/write?precision=n", - method: "POST", - headers, - body: dataToSend, - headersTimeout: this.requestTimeout, - bodyTimeout: calculatedTimeoutMillis, - }); - - const responseBody = await body.arrayBuffer(); - if (statusCode === HTTP_NO_CONTENT) { - if (responseBody.byteLength > 0) { - this.log( - "warn", - `Unexpected message from server: ${Buffer.from(responseBody).toString()}`, - ); - } - return true; - } else { - throw new Error( - `HTTP request failed, statusCode=${statusCode}, error=${Buffer.from(responseBody).toString()}`, - ); - } - } else { // TCP - if (!this.socket || this.socket.destroyed) { - throw new Error("Sender is not connected"); - } - return new Promise((resolve, reject) => { - this.socket.write(dataToSend, (err: Error) => { // Use the copied dataToSend - if (err) { - reject(err); - } else { - resolve(true); - } - }); - }); - } - } catch (err) { - // Log the error and then throw a new, standardized error - if (this.http && err.code === "UND_ERR_HEADERS_TIMEOUT") { - this.log("error", `HTTP request timeout, no response from server in time. Original error: ${err.message}`); - throw new Error(`HTTP request timeout, statusCode=${err.statusCode}, error=${err.message}`); - } else if (this.http) { - this.log("error", `HTTP request failed, statusCode=${err.statusCode || 'unknown'}, error=${err.message}`); - throw new Error(`HTTP request failed, statusCode=${err.statusCode || 'unknown'}, error=${err.message}`); - } else { // TCP - this.log("error", `TCP send failed: ${err.message}`); - throw new Error(`TCP send failed, error=${err.message}`); - } - } + await this.transport.send(dataToSend); } /** @@ -543,13 +217,7 @@ class Sender { * Data sitting in the Sender's buffer will be lost unless flush() is called before close(). */ async close() { - if (this.socket) { - const address = this.socket.remoteAddress; - const port = this.socket.remotePort; - this.socket.destroy(); - this.socket = null; - this.log("info", `Connection to ${address}:${port} is closed`); - } + return this.transport.close(); } /** @@ -558,7 +226,7 @@ class Sender { * The returned buffer is backed by the sender's buffer. * Used only in tests. */ - toBufferView(pos = this.position): Buffer { + toBufferView(pos = this.endOfLastRow): Buffer { return pos > 0 ? this.buffer.subarray(0, pos) : null; } @@ -568,11 +236,11 @@ class Sender { * The returned buffer is a copy of the sender's buffer. * It also compacts the Sender's buffer. */ - toBufferNew(pos = this.position): Buffer | null { + toBufferNew(pos = this.endOfLastRow): Buffer | null { if (pos > 0) { const data = Buffer.allocUnsafe(pos); this.buffer.copy(data, 0, 0, pos); - compact(this); + this.compact(); return data; } return null; @@ -592,8 +260,8 @@ class Sender { throw new Error("Table name has already been set"); } validateTableName(table, this.maxNameLength); - checkCapacity(this, [table]); - writeEscaped(this, table); + this.checkCapacity([table]); + this.writeEscaped(table); this.hasTable = true; return this; } @@ -610,17 +278,15 @@ class Sender { throw new Error(`Symbol name must be a string, received ${typeof name}`); } if (!this.hasTable || this.hasColumns) { - throw new Error( - "Symbol can be added only after table name is set and before any column added", - ); + throw new Error("Symbol can be added only after table name is set and before any column added"); } const valueStr = value.toString(); - checkCapacity(this, [name, valueStr], 2 + name.length + valueStr.length); - write(this, ","); + this.checkCapacity([name, valueStr], 2 + name.length + valueStr.length); + this.write(","); validateColumnName(name, this.maxNameLength); - writeEscaped(this, name); - write(this, "="); - writeEscaped(this, valueStr); + this.writeEscaped(name); + this.write("="); + this.writeEscaped(valueStr); this.hasSymbols = true; return this; } @@ -633,15 +299,14 @@ class Sender { * @return {Sender} Returns with a reference to this sender. */ stringColumn(name: string, value: string): Sender { - writeColumn( - this, + this.writeColumn( name, value, () => { - checkCapacity(this, [value], 2 + value.length); - write(this, '"'); - writeEscaped(this, value, true); - write(this, '"'); + this.checkCapacity([value], 2 + value.length); + this.write('"'); + this.writeEscaped(value, true); + this.write('"'); }, "string", ); @@ -656,13 +321,12 @@ class Sender { * @return {Sender} Returns with a reference to this sender. */ booleanColumn(name: string, value: boolean): Sender { - writeColumn( - this, + this.writeColumn( name, value, () => { - checkCapacity(this, [], 1); - write(this, value ? "t" : "f"); + this.checkCapacity([], 1); + this.write(value ? "t" : "f"); }, "boolean", ); @@ -677,14 +341,13 @@ class Sender { * @return {Sender} Returns with a reference to this sender. */ floatColumn(name: string, value: number): Sender { - writeColumn( - this, + this.writeColumn( name, value, () => { const valueStr = value.toString(); - checkCapacity(this, [valueStr], valueStr.length); - write(this, valueStr); + this.checkCapacity([valueStr], valueStr.length); + this.write(valueStr); }, "number", ); @@ -702,11 +365,11 @@ class Sender { if (!Number.isInteger(value)) { throw new Error(`Value must be an integer, received ${value}`); } - writeColumn(this, name, value, () => { + this.writeColumn(name, value, () => { const valueStr = value.toString(); - checkCapacity(this, [valueStr], 1 + valueStr.length); - write(this, valueStr); - write(this, "i"); + this.checkCapacity([valueStr], 1 + valueStr.length); + this.write(valueStr); + this.write("i"); }); return this; } @@ -727,12 +390,12 @@ class Sender { if (typeof value !== "bigint" && !Number.isInteger(value)) { throw new Error(`Value must be an integer or BigInt, received ${value}`); } - writeColumn(this, name, value, () => { + this.writeColumn(name, value, () => { const valueMicros = timestampToMicros(BigInt(value), unit); const valueStr = valueMicros.toString(); - checkCapacity(this, [valueStr], 1 + valueStr.length); - write(this, valueStr); - write(this, "t"); + this.checkCapacity([valueStr], 1 + valueStr.length); + this.write(valueStr); + this.write("t"); }); return this; } @@ -745,24 +408,20 @@ class Sender { */ async at(timestamp: number | bigint, unit: "ns" | "us" | "ms" = "us") { if (!this.hasSymbols && !this.hasColumns) { - throw new Error( - "The row must have a symbol or column set before it is closed", - ); + throw new Error("The row must have a symbol or column set before it is closed"); } if (typeof timestamp !== "bigint" && !Number.isInteger(timestamp)) { - throw new Error( - `Designated timestamp must be an integer or BigInt, received ${timestamp}`, - ); + throw new Error(`Designated timestamp must be an integer or BigInt, received ${timestamp}`); } const timestampNanos = timestampToNanos(BigInt(timestamp), unit); const timestampStr = timestampNanos.toString(); - checkCapacity(this, [], 2 + timestampStr.length); - write(this, " "); - write(this, timestampStr); - write(this, "\n"); + this.checkCapacity([], 2 + timestampStr.length); + this.write(" "); + this.write(timestampStr); + this.write("\n"); this.pendingRowCount++; - startNewRow(this); - await autoFlush(this); + this.startNewRow(); + await this.automaticFlush(); } /** @@ -771,310 +430,130 @@ class Sender { */ async atNow() { if (!this.hasSymbols && !this.hasColumns) { - throw new Error( - "The row must have a symbol or column set before it is closed", - ); + throw new Error("The row must have a symbol or column set before it is closed"); } - checkCapacity(this, [], 1); - write(this, "\n"); + this.checkCapacity([], 1); + this.write("\n"); this.pendingRowCount++; - startNewRow(this); - await autoFlush(this); - } -} - -function isBoolean(value: unknown): value is boolean { - return typeof value === "boolean"; -} - -function isInteger(value: unknown, lowerBound: number): value is number { - return ( - typeof value === "number" && Number.isInteger(value) && value >= lowerBound - ); -} - -async function authenticate( - sender: Sender, - challenge: Buffer, -): Promise { - // Check for trailing \n which ends the challenge - if (challenge.subarray(-1).readInt8() === 10) { - const keyObject = crypto.createPrivateKey({ - key: sender.jwk, - format: "jwk", - }); - const signature = crypto.sign( - "RSA-SHA256", - challenge.subarray(0, challenge.length - 1), - keyObject, - ); - - return new Promise((resolve, reject) => { - sender.socket.write( - `${Buffer.from(signature).toString("base64")}\n`, - (err: Error) => { - if (err) { - reject(err); - } else { - resolve(true); - } - }, - ); - }); - } - return false; -} - -function startNewRow(sender: Sender) { - sender.endOfLastRow = sender.position; - sender.hasTable = false; - sender.hasSymbols = false; - sender.hasColumns = false; -} - -type InternalHttpOptions = { - hostname: string; - port: number; - agent: Agent; - protocol: string; - path: string; - method: string; - timeout: number; -}; -function createRequestOptions( - sender: Sender, - data: Buffer, -): InternalHttpOptions { - const timeoutMillis = - (data.length / sender.requestMinThroughput) * 1000 + sender.requestTimeout; - return { - hostname: sender.host, - port: sender.port, - agent: sender.agent, - protocol: sender.secure ? "https" : "http", - path: "/write?precision=n", - method: "POST", - timeout: timeoutMillis, - }; -} - -async function autoFlush(sender: Sender) { - if ( - sender.autoFlush && - sender.pendingRowCount > 0 && - ((sender.autoFlushRows > 0 && - sender.pendingRowCount >= sender.autoFlushRows) || - (sender.autoFlushInterval > 0 && - Date.now() - sender.lastFlushTime >= sender.autoFlushInterval)) - ) { - await sender.flush(); - } -} - -function checkCapacity(sender: Sender, data: string[], base = 0) { - let length = base; - for (const str of data) { - length += Buffer.byteLength(str, "utf8"); - } - if (sender.position + length > sender.bufferSize) { - let newSize = sender.bufferSize; - do { - newSize += sender.bufferSize; - } while (sender.position + length > newSize); - sender.resize(newSize); - } -} - -function compact(sender: Sender) { - if (sender.endOfLastRow > 0) { - sender.buffer.copy(sender.buffer, 0, sender.endOfLastRow, sender.position); - sender.position = sender.position - sender.endOfLastRow; - sender.endOfLastRow = 0; - - sender.lastFlushTime = Date.now(); - sender.pendingRowCount = 0; - } -} - -function writeColumn( - sender: Sender, - name: string, - value: unknown, - writeValue: () => void, - valueType?: string | null, -) { - if (typeof name !== "string") { - throw new Error(`Column name must be a string, received ${typeof name}`); - } - if (valueType != null && typeof value !== valueType) { - throw new Error( - `Column value must be of type ${valueType}, received ${typeof value}`, - ); - } - if (!sender.hasTable) { - throw new Error("Column can be set only after table name is set"); - } - checkCapacity(sender, [name], 2 + name.length); - write(sender, sender.hasColumns ? "," : " "); - validateColumnName(name, sender.maxNameLength); - writeEscaped(sender, name); - write(sender, "="); - writeValue(); - sender.hasColumns = true; -} - -function write(sender: Sender, data: string) { - sender.position += sender.buffer.write(data, sender.position); - if (sender.position > sender.bufferSize) { - throw new Error( - `Buffer overflow [position=${sender.position}, bufferSize=${sender.bufferSize}]`, - ); + this.startNewRow(); + await this.automaticFlush(); + } + + private startNewRow() { + this.endOfLastRow = this.position; + this.hasTable = false; + this.hasSymbols = false; + this.hasColumns = false; + } + + private async automaticFlush() { + if ( + this.autoFlush && + this.pendingRowCount > 0 && + ((this.autoFlushRows > 0 && + this.pendingRowCount >= this.autoFlushRows) || + (this.autoFlushInterval > 0 && + Date.now() - this.lastFlushTime >= this.autoFlushInterval)) + ) { + await this.flush(); + } } -} -function writeEscaped(sender: Sender, data: string, quoted = false) { - for (const ch of data) { - if (ch > "\\") { - write(sender, ch); - continue; + private checkCapacity(data: string[], base = 0) { + let length = base; + for (const str of data) { + length += Buffer.byteLength(str, "utf8"); } - - switch (ch) { - case " ": - case ",": - case "=": - if (!quoted) { - write(sender, "\\"); - } - write(sender, ch); - break; - case "\n": - case "\r": - write(sender, "\\"); - write(sender, ch); - break; - case '"': - if (quoted) { - write(sender, "\\"); - } - write(sender, ch); - break; - case "\\": - write(sender, "\\\\"); - break; - default: - write(sender, ch); - break; + if (this.position + length > this.bufferSize) { + let newSize = this.bufferSize; + do { + newSize += this.bufferSize; + } while (this.position + length > newSize); + this.resize(newSize); } } -} - -function timestampToMicros(timestamp: bigint, unit: "ns" | "us" | "ms") { - switch (unit) { - case "ns": - return timestamp / 1000n; - case "us": - return timestamp; - case "ms": - return timestamp * 1000n; - default: - throw new Error("Unknown timestamp unit: " + unit); - } -} - -function timestampToNanos(timestamp: bigint, unit: "ns" | "us" | "ms") { - switch (unit) { - case "ns": - return timestamp; - case "us": - return timestamp * 1000n; - case "ms": - return timestamp * 1000_000n; - default: - throw new Error("Unknown timestamp unit: " + unit); - } -} -type DeprecatedOptions = { - /** @deprecated */ - copy_buffer?: boolean; - /** @deprecated */ - copyBuffer?: boolean; - /** @deprecated */ - bufferSize?: number; -}; -function replaceDeprecatedOptions( - options: SenderOptions & DeprecatedOptions, - log: (level: "error" | "warn" | "info" | "debug", message: string) => void -) { - // deal with deprecated options - if (options.copy_buffer !== undefined) { - log("warn", `Option 'copy_buffer' is not supported anymore, please, remove it`); - } - if (options.copyBuffer !== undefined) { - log("warn", `Option 'copyBuffer' is not supported anymore, please, remove it`); - } - if (options.bufferSize !== undefined) { - log("warn", `Option 'bufferSize' is not supported anymore, please, replace it with 'init_buf_size'`); - options.init_buf_size = options.bufferSize; - options.bufferSize = undefined; - } -} + private compact() { + if (this.endOfLastRow > 0) { + this.buffer.copy(this.buffer, 0, this.endOfLastRow, this.position); + this.position = this.position - this.endOfLastRow; + this.endOfLastRow = 0; -function constructAuth(options: SenderOptions) { - if (!options.username && !options.token && !options.password) { - // no intention to authenticate - return; - } - if (!options.username || !options.token) { - throw new Error( - "TCP transport requires a username and a private key for authentication, " + - "please, specify the 'username' and 'token' config options", - ); + this.lastFlushTime = Date.now(); + this.pendingRowCount = 0; + } } - options.auth = { - keyId: options.username, - token: options.token, - }; -} - -function constructJwk(options: SenderOptions) { - if (options.auth) { - if (!options.auth.keyId) { - throw new Error( - "Missing username, please, specify the 'keyId' property of the 'auth' config option. " + - "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", - ); + private writeColumn( + name: string, + value: unknown, + writeValue: () => void, + valueType?: string | null, + ) { + if (typeof name !== "string") { + throw new Error(`Column name must be a string, received ${typeof name}`); } - if (typeof options.auth.keyId !== "string") { + if (valueType != null && typeof value !== valueType) { throw new Error( - "Please, specify the 'keyId' property of the 'auth' config option as a string. " + - "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", + `Column value must be of type ${valueType}, received ${typeof value}`, ); } - if (!options.auth.token) { - throw new Error( - "Missing private key, please, specify the 'token' property of the 'auth' config option. " + - "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", - ); + if (!this.hasTable) { + throw new Error("Column can be set only after table name is set"); } - if (typeof options.auth.token !== "string") { + this.checkCapacity([name], 2 + name.length); + this.write(this.hasColumns ? "," : " "); + validateColumnName(name, this.maxNameLength); + this.writeEscaped(name); + this.write("="); + writeValue(); + this.hasColumns = true; + } + + private write(data: string) { + this.position += this.buffer.write(data, this.position); + if (this.position > this.bufferSize) { throw new Error( - "Please, specify the 'token' property of the 'auth' config option as a string. " + - "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", + `Buffer overflow [position=${this.position}, bufferSize=${this.bufferSize}]`, ); } + } + + private writeEscaped(data: string, quoted = false) { + for (const ch of data) { + if (ch > "\\") { + this.write(ch); + continue; + } - return { - kid: options.auth.keyId, - d: options.auth.token, - ...PUBLIC_KEY, - kty: "EC", - crv: "P-256", - }; - } else { - return options.jwk; + switch (ch) { + case " ": + case ",": + case "=": + if (!quoted) { + this.write("\\"); + } + this.write(ch); + break; + case "\n": + case "\r": + this.write("\\"); + this.write(ch); + break; + case '"': + if (quoted) { + this.write("\\"); + } + this.write(ch); + break; + case "\\": + this.write("\\\\"); + break; + default: + this.write(ch); + break; + } + } } } diff --git a/src/transport/http/base.ts b/src/transport/http/base.ts new file mode 100644 index 0000000..0d42b61 --- /dev/null +++ b/src/transport/http/base.ts @@ -0,0 +1,114 @@ +// @ts-check +import { readFileSync } from "node:fs"; +import { Buffer } from "node:buffer"; + +import { log, Logger } from "../../logging"; +import { SenderOptions, HTTP, HTTPS } from "../../options"; +import { SenderTransport } from "../index"; +import { isBoolean, isInteger } from "../../utils"; + +const HTTP_NO_CONTENT = 204; // success + +const DEFAULT_HTTP_AUTO_FLUSH_ROWS = 75000; + +const DEFAULT_REQUEST_MIN_THROUGHPUT = 102400; // 100 KB/sec +const DEFAULT_REQUEST_TIMEOUT = 10000; // 10 sec +const DEFAULT_RETRY_TIMEOUT = 10000; // 10 sec + +/* +We are retrying on the following response codes (copied from the Rust client): +500: Internal Server Error +503: Service Unavailable +504: Gateway Timeout + +// Unofficial extensions +507: Insufficient Storage +509: Bandwidth Limit Exceeded +523: Origin is Unreachable +524: A Timeout Occurred +529: Site is overloaded +599: Network Connect Timeout Error +*/ +const RETRIABLE_STATUS_CODES = [500, 503, 504, 507, 509, 523, 524, 529, 599]; + +/** @classdesc + * The QuestDB client's API provides methods to connect to the database, ingest data, and close the connection. + * The supported protocols are HTTP and TCP. HTTP is preferred as it provides feedback in the HTTP response.
+ */ +abstract class HttpTransportBase implements SenderTransport { + protected readonly secure: boolean; + protected readonly host: string; + protected readonly port: number; + + protected readonly username: string; + protected readonly password: string; + protected readonly token: string; + + protected readonly tlsVerify: boolean; + protected readonly tlsCA: Buffer; + + protected readonly requestMinThroughput: number; + protected readonly requestTimeout: number; + protected readonly retryTimeout: number; + + protected readonly log: Logger; + + protected constructor(options: SenderOptions) { + if (!options || !options.protocol) { + throw new Error("The 'protocol' option is mandatory"); + } + if (!options.host) { + throw new Error("The 'host' option is mandatory"); + } + this.log = typeof options.log === "function" ? options.log : log; + + this.tlsVerify = isBoolean(options.tls_verify) ? options.tls_verify : true; + this.tlsCA = options.tls_ca ? readFileSync(options.tls_ca) : undefined; + + this.username = options.username; + this.password = options.password; + this.token = options.token; + if (!options.port) { + options.port = 9000; + } + + this.host = options.host; + this.port = options.port; + + this.requestMinThroughput = isInteger(options.request_min_throughput, 0) + ? options.request_min_throughput + : DEFAULT_REQUEST_MIN_THROUGHPUT; + this.requestTimeout = isInteger(options.request_timeout, 1) + ? options.request_timeout + : DEFAULT_REQUEST_TIMEOUT; + this.retryTimeout = isInteger(options.retry_timeout, 0) + ? options.retry_timeout + : DEFAULT_RETRY_TIMEOUT; + + switch (options.protocol) { + case HTTP: + this.secure = false; + break; + case HTTPS: + this.secure = true; + break; + default: + throw new Error("The 'protocol' has to be 'http' or 'https' for the HTTP transport"); + } + } + + connect(): Promise { + throw new Error("'connect()' is not required for HTTP transport"); + } + + async close(): Promise { + } + + getDefaultAutoFlushRows(): number { + return DEFAULT_HTTP_AUTO_FLUSH_ROWS; + } + + abstract send(data: Buffer): Promise; +} + +export { HttpTransportBase, RETRIABLE_STATUS_CODES, HTTP_NO_CONTENT }; diff --git a/src/transport/http/legacy.ts b/src/transport/http/legacy.ts new file mode 100644 index 0000000..d62852f --- /dev/null +++ b/src/transport/http/legacy.ts @@ -0,0 +1,165 @@ +// @ts-check +import http from "http"; +import https from "https"; +import { Buffer } from "node:buffer"; + +import { SenderOptions, HTTP, HTTPS } from "../../options"; +import { HttpTransportBase, RETRIABLE_STATUS_CODES, HTTP_NO_CONTENT } from "./base"; + +// default options for HTTP agent +// - persistent connections with 1 minute idle timeout, server side has 5 minutes set by default +// - max open connections is set to 256, same as server side default +const DEFAULT_HTTP_AGENT_CONFIG = { + maxSockets: 256, + keepAlive: true, + timeout: 60000 // 1 min +} + +/** @classdesc + * The QuestDB client's API provides methods to connect to the database, ingest data, and close the connection. + * The supported protocols are HTTP and TCP. HTTP is preferred as it provides feedback in the HTTP response.
+ * Based on benchmarks HTTP also provides higher throughput, if configured to ingest data in bigger batches. + */ +class HttpTransport extends HttpTransportBase { + private static DEFAULT_HTTP_AGENT: http.Agent; + private static DEFAULT_HTTPS_AGENT: https.Agent; + + private readonly agent: http.Agent | https.Agent; + + /** + * Creates an instance of Sender. + * + * @param {SenderOptions} options - Sender configuration object.
+ * See SenderOptions documentation for detailed description of configuration options.
+ */ + constructor(options: SenderOptions) { + super(options); + + switch (options.protocol) { + case HTTP: + this.agent = options.agent instanceof http.Agent ? options.agent : HttpTransport.getDefaultHttpAgent(); + break; + case HTTPS: + this.agent = options.agent instanceof https.Agent ? options.agent : HttpTransport.getDefaultHttpsAgent(); + break; + default: + throw new Error("The 'protocol' has to be 'http' or 'https' for the HTTP transport"); + } + } + + send(data: Buffer, retryBegin = -1, retryInterval = -1): Promise { + const request = this.secure ? https.request : http.request; + + const timeoutMillis = (data.length / this.requestMinThroughput) * 1000 + this.requestTimeout; + const options = this.createRequestOptions(timeoutMillis); + + return new Promise((resolve, reject) => { + let statusCode = -1; + const req = request(options, response => { + statusCode = response.statusCode; + + const body = []; + response + .on("data", chunk => { + body.push(chunk); + }) + .on("error", err => { + this.log("error", `resp err=${err}`); + }); + + if (statusCode === HTTP_NO_CONTENT) { + response.on("end", () => { + if (body.length > 0) { + this.log("warn", `Unexpected message from server: ${Buffer.concat(body)}`); + } + resolve(true); + }); + } else { + req.destroy(new Error(`HTTP request failed, statusCode=${statusCode}, error=${Buffer.concat(body)}`)); + } + }); + + if (this.token) { + req.setHeader("Authorization", `Bearer ${this.token}`); + } else if (this.username && this.password) { + req.setHeader("Authorization", `Basic ${Buffer.from(`${this.username}:${this.password}`).toString("base64")}`); + } + + req.on("timeout", () => { + // set a retryable error code + statusCode = 524; + req.destroy(new Error("HTTP request timeout, no response from server in time")); + }); + req.on("error", err => { + // if the error is thrown while the request is sent, statusCode is -1 => no retry + // request timeout comes through with statusCode 524 => retry + // if the error is thrown while the response is processed, the statusCode is taken from the response => retry depends on statusCode + if (isRetryable(statusCode) && this.retryTimeout > 0) { + if (retryBegin < 0) { + retryBegin = Date.now(); + retryInterval = 10; + } else { + const elapsed = Date.now() - retryBegin; + if (elapsed > this.retryTimeout) { + reject(err); + return; + } + } + const jitter = Math.floor(Math.random() * 10) - 5; + setTimeout(() => { + retryInterval = Math.min(retryInterval * 2, 1000); + this.send(data, retryBegin, retryInterval) + .then(() => resolve(true)) + .catch(e => reject(e)); + }, retryInterval + jitter); + } else { + reject(err); + } + }); + req.write(data, err => err ? reject(err) : () => {}); + req.end(); + }); + } + + private createRequestOptions(timeoutMillis: number): http.RequestOptions | https.RequestOptions { + return { + //protocol: this.secure ? "https:" : "http:", + hostname: this.host, + port: this.port, + agent: this.agent, + path: "/write?precision=n", + method: "POST", + timeout: timeoutMillis, + rejectUnauthorized: this.secure && this.tlsVerify, + ca: this.secure ? this.tlsCA : undefined, + }; + } + + /** + * @ignore + * @return {http.Agent} Returns the default http agent. + */ + private static getDefaultHttpAgent(): http.Agent { + if (!HttpTransport.DEFAULT_HTTP_AGENT) { + HttpTransport.DEFAULT_HTTP_AGENT = new http.Agent(DEFAULT_HTTP_AGENT_CONFIG); + } + return HttpTransport.DEFAULT_HTTP_AGENT; + } + + /** + * @ignore + * @return {https.Agent} Returns the default https agent. + */ + private static getDefaultHttpsAgent(): https.Agent { + if (!HttpTransport.DEFAULT_HTTPS_AGENT) { + HttpTransport.DEFAULT_HTTPS_AGENT = new https.Agent(DEFAULT_HTTP_AGENT_CONFIG); + } + return HttpTransport.DEFAULT_HTTPS_AGENT; + } +} + +function isRetryable(statusCode: number) { + return RETRIABLE_STATUS_CODES.includes(statusCode); +} + +export { HttpTransport, HttpTransportBase }; diff --git a/src/transport/http/undici.ts b/src/transport/http/undici.ts new file mode 100644 index 0000000..43575fa --- /dev/null +++ b/src/transport/http/undici.ts @@ -0,0 +1,143 @@ +// @ts-check +import { Buffer } from "node:buffer"; +import { Agent, RetryAgent } from "undici"; +import Dispatcher from "undici/types/dispatcher"; + +import { SenderOptions, HTTP, HTTPS } from "../../options"; +import { HttpTransportBase, RETRIABLE_STATUS_CODES, HTTP_NO_CONTENT } from "./base"; + +const DEFAULT_HTTP_OPTIONS: Agent.Options = { + connect: { + keepAlive: true, + }, + pipelining: 1, + keepAliveTimeout: 60000, // 1 minute +}; + +/** @classdesc + * The QuestDB client's API provides methods to connect to the database, ingest data, and close the connection. + * The supported protocols are HTTP and TCP. HTTP is preferred as it provides feedback in the HTTP response.
+ * of undici.Agent. The Sender's own agent uses persistent connections with 1 minute idle timeout, pipelines requests default to 1. + *

+ */ +class UndiciTransport extends HttpTransportBase { + private static DEFAULT_HTTP_AGENT: Agent; + + private readonly agent: Dispatcher; + private readonly dispatcher : RetryAgent; + + /** + * Creates an instance of Sender. + * + * @param {SenderOptions} options - Sender configuration object.
+ * See SenderOptions documentation for detailed description of configuration options.
+ */ + constructor(options: SenderOptions) { + super(options); + + switch (options.protocol) { + case HTTP: + this.agent = options.agent instanceof Agent ? options.agent : UndiciTransport.getDefaultHttpAgent(); + break; + case HTTPS: + if (options.agent instanceof Agent) { + this.agent = options.agent; + } else { + // Create a new agent with instance-specific TLS options + this.agent = new Agent({ + ...DEFAULT_HTTP_OPTIONS, + connect: { + ...DEFAULT_HTTP_OPTIONS.connect, + requestCert: this.tlsVerify, + rejectUnauthorized: this.tlsVerify, + ca: this.tlsCA, + }, + }); + } + break; + default: + throw new Error("The 'protocol' has to be 'http' or 'https' for the Undici HTTP transport"); + } + + this.dispatcher = new RetryAgent(this.agent, { + maxRetries: Infinity, + minTimeout: 10, + maxTimeout: 1000, + timeoutFactor: 2, + retryAfter: true, + methods: ["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD"], + statusCodes: RETRIABLE_STATUS_CODES, + errorCodes: [ + "ECONNRESET", + "EAI_AGAIN", + "ECONNREFUSED", + "ETIMEDOUT", + "EPIPE", + "UND_ERR_CONNECT_TIMEOUT", + "UND_ERR_HEADERS_TIMEOUT", + "UND_ERR_BODY_TIMEOUT", + ], + }); + } + + async send(data: Buffer): Promise { + const headers: Record = {}; + if (this.token) { + headers["Authorization"] = `Bearer ${this.token}`; + } else if (this.username && this.password) { + headers["Authorization"] = `Basic ${Buffer.from(`${this.username}:${this.password}`).toString("base64")}`; + } + + const controller = new AbortController(); + const { signal } = controller; + setTimeout(() => controller.abort(), this.retryTimeout); + + let responseData: Dispatcher.ResponseData; + try { + const timeoutMillis = (data.length / this.requestMinThroughput) * 1000 + this.requestTimeout; + responseData = + await this.dispatcher.request({ + origin: `${this.secure ? "https" : "http"}://${this.host}:${this.port}`, + path: "/write?precision=n", + method: "POST", + headers, + body: data, + headersTimeout: this.requestTimeout, + bodyTimeout: timeoutMillis, + signal, + }); + } catch (err) { + if (err.name === "AbortError") { + throw new Error("HTTP request timeout, no response from server in time"); + } else { + throw err; + } + } + + const { statusCode} = responseData; + const body = await responseData.body.arrayBuffer(); + if (statusCode === HTTP_NO_CONTENT) { + if (body.byteLength > 0) { + this.log("warn", `Unexpected message from server: ${Buffer.from(body).toString()}`); + } + return true; + } else { + throw new Error( + `HTTP request failed, statusCode=${statusCode}, error=${Buffer.from(body).toString()}`, + ); + } + } + + /** + * @ignore + * @return {Agent} Returns the default http agent. + */ + private static getDefaultHttpAgent(): Agent { + if (!UndiciTransport.DEFAULT_HTTP_AGENT) { + UndiciTransport.DEFAULT_HTTP_AGENT = new Agent(DEFAULT_HTTP_OPTIONS); + } + return UndiciTransport.DEFAULT_HTTP_AGENT; + } +} + +export { UndiciTransport }; diff --git a/src/transport/index.ts b/src/transport/index.ts new file mode 100644 index 0000000..983af2c --- /dev/null +++ b/src/transport/index.ts @@ -0,0 +1,36 @@ +// @ts-check +import { Buffer } from "node:buffer"; + +import { SenderOptions, HTTP, HTTPS, TCP, TCPS } from "../options"; +import { UndiciTransport } from "./http/undici"; +import { TcpTransport } from "./tcp"; +import { HttpTransport } from "./http/legacy"; + +interface SenderTransport { + connect(): Promise; + send(data: Buffer): Promise; + close(): Promise; + getDefaultAutoFlushRows(): number; +} + +function createTransport(options: SenderOptions): SenderTransport { + if (!options || !options.protocol) { + throw new Error("The 'protocol' option is mandatory"); + } + if (!options.host) { + throw new Error("The 'host' option is mandatory"); + } + + switch (options.protocol) { + case HTTP: + case HTTPS: + return options.legacy_http ? new HttpTransport(options) : new UndiciTransport(options); + case TCP: + case TCPS: + return new TcpTransport(options); + default: + throw new Error(`Invalid protocol: '${options.protocol}'`); + } +} + +export { SenderTransport, createTransport } diff --git a/src/transport/tcp.ts b/src/transport/tcp.ts new file mode 100644 index 0000000..be6cf05 --- /dev/null +++ b/src/transport/tcp.ts @@ -0,0 +1,302 @@ +// @ts-check +import { readFileSync } from "node:fs"; +import { Buffer } from "node:buffer"; +import net from "node:net"; +import tls from "node:tls"; +import crypto from "node:crypto"; + +import { log, Logger } from "../logging"; +import { SenderOptions, TCP, TCPS } from "../options"; +import { SenderTransport } from "./index"; +import { isBoolean } from "../utils"; + +const DEFAULT_TCP_AUTO_FLUSH_ROWS = 600; + +// an arbitrary public key, not used in authentication +// only used to construct a valid JWK token which is accepted by the crypto API +const PUBLIC_KEY = { + x: "aultdA0PjhD_cWViqKKyL5chm6H1n-BiZBo_48T-uqc", + y: "__ptaol41JWSpTTL525yVEfzmY8A6Vi_QrW1FjKcHMg", +}; + +/** @classdesc + * The QuestDB client's API provides methods to connect to the database, ingest data, and close the connection. + * The supported protocols are HTTP and TCP. HTTP is preferred as it provides feedback in the HTTP response.
+ * Based on benchmarks HTTP also provides higher throughput, if configured to ingest data in bigger batches. + *

+ * The client supports authentication.
+ * Authentication details can be passed to the Sender in its configuration options.
+ * The client supports Basic username/password and Bearer token authentication methods when used with HTTP protocol, + * and JWK token authentication when ingesting data via TCP.
+ * Please, note that authentication is enabled by default in QuestDB Enterprise only.
+ * Details on how to configure authentication in the open source version of + * QuestDB: {@link https://questdb.io/docs/reference/api/ilp/authenticate} + *

+ *

+ * The client also supports TLS encryption for both, HTTP and TCP transports to provide a secure connection.
+ * Please, note that the open source version of QuestDB does not support TLS, and requires an external reverse-proxy, + * such as Nginx to enable encryption. + *

+ *

+ * The client uses a buffer to store data. It automatically flushes the buffer by sending its content to the server. + * Auto flushing can be disabled via configuration options to gain control over transactions. Initial and maximum + * buffer sizes can also be set. + *

+ *

+ * It is recommended that the Sender is created by using one of the static factory methods, + * Sender.fromConfig(configString, extraOptions) or Sender.fromEnv(extraOptions). + * If the Sender is created via its constructor, at least the SenderOptions configuration object should be + * initialized from a configuration string to make sure that the parameters are validated.
+ * Detailed description of the Sender's configuration options can be found in + * the SenderOptions documentation. + *

+ *

+ * Extra options can be provided to the Sender in the extraOptions configuration object.
+ * A custom logging function and a custom HTTP(S) agent can be passed to the Sender in this object.
+ * The logger implementation provides the option to direct log messages to the same place where the host application's + * log is saved. The default logger writes to the console.
+ * The custom HTTP(S) agent option becomes handy if there is a need to modify the default options set for the + * HTTP(S) connections. A popular setting would be disabling persistent connections, in this case an agent can be + * passed to the Sender with keepAlive set to false.
+ * For example: Sender.fromConfig(`http::addr=host:port`, { agent: new undici.Agent({ connect: { keepAlive: false } })})
+ * If no custom agent is configured, the Sender will use its own agent which overrides some default values + * of undici.Agent. The Sender's own agent uses persistent connections with 1 minute idle timeout, pipelines requests default to 1. + *

+ */ +class TcpTransport implements SenderTransport { + private readonly secure: boolean; + private readonly host: string; + private readonly port: number; + + private socket: net.Socket | tls.TLSSocket; + + private readonly tlsVerify: boolean; + private readonly tlsCA: Buffer; + + private readonly log: Logger; + private readonly jwk: Record; + + /** + * Creates an instance of Sender. + * + * @param {SenderOptions} options - Sender configuration object.
+ * See SenderOptions documentation for detailed description of configuration options.
+ */ + constructor(options: SenderOptions) { + if (!options || !options.protocol) { + throw new Error("The 'protocol' option is mandatory"); + } + if (!options.host) { + throw new Error("The 'host' option is mandatory"); + } + this.log = typeof options.log === "function" ? options.log : log; + + this.tlsVerify = isBoolean(options.tls_verify) ? options.tls_verify : true; + this.tlsCA = options.tls_ca ? readFileSync(options.tls_ca) : undefined; + + this.host = options.host; + this.port = options.port; + + switch (options.protocol) { + case TCP: + this.secure = false; + break; + case TCPS: + this.secure = true; + break; + default: + throw new Error("The 'protocol' has to be 'tcp' or 'tcps' for the TCP transport"); + } + + if (!options.auth && !options.jwk) { + constructAuth(options); + } + this.jwk = constructJwk(options); + if (!options.port) { + options.port = 9009; + } + } + + /** + * Creates a TCP connection to the database. + * + * @return {Promise} Resolves to true if the client is connected. + */ + connect(): Promise { + const connOptions: net.NetConnectOpts | tls.ConnectionOptions = { + host: this.host, + port: this.port, + ca: this.tlsCA, + }; + + return new Promise((resolve, reject) => { + if (this.socket) { + throw new Error("Sender connected already"); + } + + let authenticated: boolean = false; + let data: Buffer; + + this.socket = !this.secure + ? net.connect(connOptions as net.NetConnectOpts) + : tls.connect(connOptions as tls.ConnectionOptions, () => { + if (authenticated) { + resolve(true); + } + }); + this.socket.setKeepAlive(true); + + this.socket + .on("data", async (raw) => { + data = !data ? raw : Buffer.concat([data, raw]); + if (!authenticated) { + authenticated = await this.authenticate(data); + if (authenticated) { + resolve(true); + } + } else { + this.log("warn", `Received unexpected data: ${data}`); + } + }) + .on("ready", async () => { + this.log("info", `Successfully connected to ${connOptions.host}:${connOptions.port}`); + if (this.jwk) { + this.log("info", `Authenticating with ${connOptions.host}:${connOptions.port}`); + this.socket.write(`${this.jwk.kid}\n`, err => err ? reject(err) : () => {}); + } else { + authenticated = true; + if (!this.secure || !this.tlsVerify) { + resolve(true); + } + } + }) + .on("error", (err: Error & { code: string }) => { + this.log("error", err); + if (this.tlsVerify || !err.code || err.code !== "SELF_SIGNED_CERT_IN_CHAIN") { + reject(err); + } + }); + }); + } + + send(data: Buffer): Promise { + if (!this.socket || this.socket.destroyed) { + throw new Error("TCP transport is not connected"); + } + return new Promise((resolve, reject) => { + this.socket.write(data, (err: Error) => { + if (err) { + reject(err); + } else { + resolve(true); + } + }); + }); + } + + /** + * Closes the TCP connection to the database.
+ * Data sitting in the Sender's buffer will be lost unless flush() is called before close(). + */ + async close(): Promise { + if (this.socket) { + const address = this.socket.remoteAddress; + const port = this.socket.remotePort; + this.socket.destroy(); + this.socket = null; + this.log("info", `Connection to ${address}:${port} is closed`); + } + } + + getDefaultAutoFlushRows(): number { + return DEFAULT_TCP_AUTO_FLUSH_ROWS; + } + + private async authenticate(challenge: Buffer): Promise { + // Check for trailing \n which ends the challenge + if (challenge.subarray(-1).readInt8() === 10) { + const keyObject = crypto.createPrivateKey({ + key: this.jwk, + format: "jwk" + }); + const signature = crypto.sign( + "RSA-SHA256", + challenge.subarray(0, challenge.length - 1), + keyObject + ); + + return new Promise((resolve, reject) => { + this.socket.write( + `${Buffer.from(signature).toString("base64")}\n`, + (err: Error) => { + if (err) { + reject(err); + } else { + resolve(true); + } + } + ); + }); + } + return false; + } +} + +function constructAuth(options: SenderOptions) { + if (!options.username && !options.token && !options.password) { + // no intention to authenticate + return; + } + if (!options.username || !options.token) { + throw new Error( + "TCP transport requires a username and a private key for authentication, " + + "please, specify the 'username' and 'token' config options", + ); + } + + options.auth = { + keyId: options.username, + token: options.token, + }; +} + +function constructJwk(options: SenderOptions) { + if (options.auth) { + if (!options.auth.keyId) { + throw new Error( + "Missing username, please, specify the 'keyId' property of the 'auth' config option. " + + "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", + ); + } + if (typeof options.auth.keyId !== "string") { + throw new Error( + "Please, specify the 'keyId' property of the 'auth' config option as a string. " + + "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", + ); + } + if (!options.auth.token) { + throw new Error( + "Missing private key, please, specify the 'token' property of the 'auth' config option. " + + "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", + ); + } + if (typeof options.auth.token !== "string") { + throw new Error( + "Please, specify the 'token' property of the 'auth' config option as a string. " + + "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", + ); + } + + return { + kid: options.auth.keyId, + d: options.auth.token, + ...PUBLIC_KEY, + kty: "EC", + crv: "P-256", + }; + } else { + return options.jwk; + } +} + +export { TcpTransport }; diff --git a/src/utils.ts b/src/utils.ts new file mode 100644 index 0000000..dd82db6 --- /dev/null +++ b/src/utils.ts @@ -0,0 +1,37 @@ +function isBoolean(value: unknown): value is boolean { + return typeof value === "boolean"; +} + +function isInteger(value: unknown, lowerBound: number): value is number { + return ( + typeof value === "number" && Number.isInteger(value) && value >= lowerBound + ); +} + +function timestampToMicros(timestamp: bigint, unit: "ns" | "us" | "ms") { + switch (unit) { + case "ns": + return timestamp / 1000n; + case "us": + return timestamp; + case "ms": + return timestamp * 1000n; + default: + throw new Error("Unknown timestamp unit: " + unit); + } +} + +function timestampToNanos(timestamp: bigint, unit: "ns" | "us" | "ms") { + switch (unit) { + case "ns": + return timestamp; + case "us": + return timestamp * 1000n; + case "ms": + return timestamp * 1000_000n; + default: + throw new Error("Unknown timestamp unit: " + unit); + } +} + +export { isBoolean, isInteger, timestampToMicros, timestampToNanos }; diff --git a/test/logging.test.ts b/test/logging.test.ts index 4384e60..5997b78 100644 --- a/test/logging.test.ts +++ b/test/logging.test.ts @@ -1,22 +1,14 @@ -import { - describe, - it, - beforeAll, - afterAll, - afterEach, - expect, - vi, -} from "vitest"; +// @ts-check +import { describe, it, beforeAll, afterAll, afterEach, expect, vi, } from "vitest"; + +import { Logger } from "../src/logging"; describe("Default logging suite", function () { const error = vi.spyOn(console, "error").mockImplementation(() => { }); const warn = vi.spyOn(console, "warn").mockImplementation(() => { }); const info = vi.spyOn(console, "info").mockImplementation(() => { }); const debug = vi.spyOn(console, "debug").mockImplementation(() => { }); - let log: ( - level: "error" | "warn" | "info" | "debug", - message: string, - ) => void; + let log: Logger; beforeAll(async () => { log = (await import("../src/logging")).log; @@ -66,7 +58,7 @@ describe("Default logging suite", function () { expect(info).toHaveBeenCalledWith(testMessage); }); - it("cannot log debug level messages", function () { + it("cannot log debug level messages, default logging level is 'info'", function () { const testMessage = "DEBUG DEBUG DEBUG"; log("debug", testMessage); expect(error).toHaveBeenCalledTimes(0); diff --git a/test/options.test.ts b/test/options.test.ts index 576670c..e8be4e5 100644 --- a/test/options.test.ts +++ b/test/options.test.ts @@ -1,7 +1,9 @@ +// @ts-check import { describe, it, expect } from "vitest"; -import { SenderOptions } from "../src/options"; import { Agent } from "undici"; +import { SenderOptions } from "../src/options"; + describe("Configuration string parser suite", function () { it("can parse a basic config string", function () { const options = SenderOptions.fromConfig( @@ -688,14 +690,14 @@ describe("Configuration string parser suite", function () { // @ts-expect-error - Testing invalid input agent: { keepAlive: true }, }), - ).toThrow("Invalid http/https agent"); + ).toThrow("Invalid HTTP agent"); expect(() => // @ts-expect-error - Testing invalid input SenderOptions.fromConfig("http::addr=host:9000", { agent: 4567 }), - ).toThrow("Invalid http/https agent"); + ).toThrow("Invalid HTTP agent"); expect(() => // @ts-expect-error - Testing invalid input SenderOptions.fromConfig("http::addr=host:9000", { agent: "hopp" }), - ).toThrow("Invalid http/https agent"); + ).toThrow("Invalid HTTP agent"); }); }); diff --git a/test/sender.buffer.test.ts b/test/sender.buffer.test.ts new file mode 100644 index 0000000..9e7c745 --- /dev/null +++ b/test/sender.buffer.test.ts @@ -0,0 +1,803 @@ +// @ts-check +import { describe, it, expect } from "vitest"; +import { readFileSync } from "fs"; + +import { Sender } from "../src"; + +describe("Client interop test suite", function () { + it("runs client tests as per json test config", async function () { + const testCases = JSON.parse( + readFileSync("./questdb-client-test/ilp-client-interop-test.json").toString() + ); + + for (const testCase of testCases) { + console.info(`test name: ${testCase.testName}`); + + const sender = new Sender({ + protocol: "tcp", + host: "host", + auto_flush: false, + init_buf_size: 1024, + }); + + let errorMessage: string; + try { + sender.table(testCase.table); + for (const symbol of testCase.symbols) { + sender.symbol(symbol.name, symbol.value); + } + for (const column of testCase.columns) { + switch (column.type) { + case "STRING": + sender.stringColumn(column.name, column.value); + break; + case "LONG": + sender.intColumn(column.name, column.value); + break; + case "DOUBLE": + sender.floatColumn(column.name, column.value); + break; + case "BOOLEAN": + sender.booleanColumn(column.name, column.value); + break; + case "TIMESTAMP": + sender.timestampColumn(column.name, column.value); + break; + default: + errorMessage = "Unsupported column type"; + } + if (errorMessage) { + break; + } + } + await sender.atNow(); + } catch (e) { + if (testCase.result.status === "ERROR") { + // error is expected, continue to next test case + continue; + } + errorMessage = `Unexpected error: ${e.message}`; + } + + if (!errorMessage) { + const actualLine = sender.toBufferView().toString(); + + if (testCase.result.status === "SUCCESS") { + if (testCase.result.line) { + expect(actualLine).toBe(testCase.result.line + "\n"); + } else { + let foundMatch = false; + for (const expectedLine of testCase.result.anyLines) { + if (actualLine === expectedLine + "\n") { + foundMatch = true; + break; + } + } + if (!foundMatch) { + errorMessage = `Line is not matching any of the expected results: ${actualLine}`; + } + } + } else { + errorMessage = `Expected error missing, buffer's content: ${actualLine}`; + } + } + + await sender.close(); + expect(errorMessage).toBeUndefined(); + } + }); +}); + +describe("Sender message builder test suite (anything not covered in client interop test suite)", function () { + it("throws on invalid timestamp unit", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + auto_flush: false, + init_buf_size: 1024, + }); + + await expect(async () => + await sender + .table("tableName") + .booleanColumn("boolCol", true) + // @ts-expect-error - Testing invalid options + .timestampColumn("timestampCol", 1658484765000000, "foobar") + .atNow() + ).rejects.toThrow("Unknown timestamp unit: foobar"); + await sender.close(); + }); + + it("supports json object", async function () { + const pages: Array<{ + id: string; + gridId: string; + }>[] = []; + for (let i = 0; i < 4; i++) { + const pageProducts: Array<{ + id: string; + gridId: string; + }> = [ + { + id: "46022e96-076f-457f-b630-51b82b871618" + i, + gridId: "46022e96-076f-457f-b630-51b82b871618", + }, + { + id: "55615358-4af1-4179-9153-faaa57d71e55", + gridId: "55615358-4af1-4179-9153-faaa57d71e55", + }, + { + id: "365b9cdf-3d4e-4135-9cb0-f1a65601c840", + gridId: "365b9cdf-3d4e-4135-9cb0-f1a65601c840", + }, + { + id: "0b67ddf2-8e69-4482-bf0c-bb987ee5c280", + gridId: "0b67ddf2-8e69-4482-bf0c-bb987ee5c280" + i, + }, + ]; + pages.push(pageProducts); + } + + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 256, + }); + for (const p of pages) { + await sender + .table("tableName") + .stringColumn("page_products", JSON.stringify(p || [])) + .booleanColumn("boolCol", true) + .atNow(); + } + expect(sender.toBufferView().toString()).toBe( + 'tableName page_products="[{\\"id\\":\\"46022e96-076f-457f-b630-51b82b8716180\\",\\"gridId\\":\\"46022e96-076f-457f-b630-51b82b871618\\"},{\\"id\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\",\\"gridId\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\"},{\\"id\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\",\\"gridId\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\"},{\\"id\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c280\\",\\"gridId\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c2800\\"}]",boolCol=t\n' + + 'tableName page_products="[{\\"id\\":\\"46022e96-076f-457f-b630-51b82b8716181\\",\\"gridId\\":\\"46022e96-076f-457f-b630-51b82b871618\\"},{\\"id\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\",\\"gridId\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\"},{\\"id\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\",\\"gridId\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\"},{\\"id\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c280\\",\\"gridId\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c2801\\"}]",boolCol=t\n' + + 'tableName page_products="[{\\"id\\":\\"46022e96-076f-457f-b630-51b82b8716182\\",\\"gridId\\":\\"46022e96-076f-457f-b630-51b82b871618\\"},{\\"id\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\",\\"gridId\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\"},{\\"id\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\",\\"gridId\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\"},{\\"id\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c280\\",\\"gridId\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c2802\\"}]",boolCol=t\n' + + 'tableName page_products="[{\\"id\\":\\"46022e96-076f-457f-b630-51b82b8716183\\",\\"gridId\\":\\"46022e96-076f-457f-b630-51b82b871618\\"},{\\"id\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\",\\"gridId\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\"},{\\"id\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\",\\"gridId\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\"},{\\"id\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c280\\",\\"gridId\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c2803\\"}]",boolCol=t\n', + ); + await sender.close(); + }); + + it("supports timestamp field as number", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + await sender + .table("tableName") + .booleanColumn("boolCol", true) + .timestampColumn("timestampCol", 1658484765000000) + .atNow(); + expect(sender.toBufferView().toString()).toBe( + "tableName boolCol=t,timestampCol=1658484765000000t\n", + ); + await sender.close(); + }); + + it("supports timestamp field as ns number", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + await sender + .table("tableName") + .booleanColumn("boolCol", true) + .timestampColumn("timestampCol", 1658484765000000, "ns") + .atNow(); + expect(sender.toBufferView().toString()).toBe( + "tableName boolCol=t,timestampCol=1658484765000t\n", + ); + await sender.close(); + }); + + it("supports timestamp field as us number", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + await sender + .table("tableName") + .booleanColumn("boolCol", true) + .timestampColumn("timestampCol", 1658484765000000, "us") + .atNow(); + expect(sender.toBufferView().toString()).toBe( + "tableName boolCol=t,timestampCol=1658484765000000t\n", + ); + await sender.close(); + }); + + it("supports timestamp field as ms number", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + await sender + .table("tableName") + .booleanColumn("boolCol", true) + .timestampColumn("timestampCol", 1658484765000, "ms") + .atNow(); + expect(sender.toBufferView().toString()).toBe( + "tableName boolCol=t,timestampCol=1658484765000000t\n", + ); + await sender.close(); + }); + + it("supports timestamp field as BigInt", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + await sender + .table("tableName") + .booleanColumn("boolCol", true) + .timestampColumn("timestampCol", 1658484765000000n) + .atNow(); + expect(sender.toBufferView().toString()).toBe( + "tableName boolCol=t,timestampCol=1658484765000000t\n", + ); + await sender.close(); + }); + + it("supports timestamp field as ns BigInt", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + await sender + .table("tableName") + .booleanColumn("boolCol", true) + .timestampColumn("timestampCol", 1658484765000000000n, "ns") + .atNow(); + expect(sender.toBufferView().toString()).toBe( + "tableName boolCol=t,timestampCol=1658484765000000t\n", + ); + await sender.close(); + }); + + it("supports timestamp field as us BigInt", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + await sender + .table("tableName") + .booleanColumn("boolCol", true) + .timestampColumn("timestampCol", 1658484765000000n, "us") + .atNow(); + expect(sender.toBufferView().toString()).toBe( + "tableName boolCol=t,timestampCol=1658484765000000t\n", + ); + await sender.close(); + }); + + it("supports timestamp field as ms BigInt", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + await sender + .table("tableName") + .booleanColumn("boolCol", true) + .timestampColumn("timestampCol", 1658484765000n, "ms") + .atNow(); + expect(sender.toBufferView().toString()).toBe( + "tableName boolCol=t,timestampCol=1658484765000000t\n", + ); + await sender.close(); + }); + + it("throws on invalid designated timestamp unit", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + try { + await sender + .table("tableName") + .booleanColumn("boolCol", true) + .timestampColumn("timestampCol", 1658484765000000) + // @ts-expect-error - Testing invalid options + .at(1658484769000000, "foobar"); + } catch (err) { + expect(err.message).toBe("Unknown timestamp unit: foobar"); + } + await sender.close(); + }); + + it("supports setting designated us timestamp as number from client", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + await sender + .table("tableName") + .booleanColumn("boolCol", true) + .timestampColumn("timestampCol", 1658484765000000) + .at(1658484769000000, "us"); + expect(sender.toBufferView().toString()).toBe( + "tableName boolCol=t,timestampCol=1658484765000000t 1658484769000000000\n", + ); + await sender.close(); + }); + + it("supports setting designated ms timestamp as number from client", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + await sender + .table("tableName") + .booleanColumn("boolCol", true) + .timestampColumn("timestampCol", 1658484765000000) + .at(1658484769000, "ms"); + expect(sender.toBufferView().toString()).toBe( + "tableName boolCol=t,timestampCol=1658484765000000t 1658484769000000000\n", + ); + await sender.close(); + }); + + it("supports setting designated timestamp as BigInt from client", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + await sender + .table("tableName") + .booleanColumn("boolCol", true) + .timestampColumn("timestampCol", 1658484765000000) + .at(1658484769000000n); + expect(sender.toBufferView().toString()).toBe( + "tableName boolCol=t,timestampCol=1658484765000000t 1658484769000000000\n", + ); + await sender.close(); + }); + + it("supports setting designated ns timestamp as BigInt from client", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + await sender + .table("tableName") + .booleanColumn("boolCol", true) + .timestampColumn("timestampCol", 1658484765000000) + .at(1658484769000000123n, "ns"); + expect(sender.toBufferView().toString()).toBe( + "tableName boolCol=t,timestampCol=1658484765000000t 1658484769000000123\n", + ); + await sender.close(); + }); + + it("supports setting designated us timestamp as BigInt from client", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + await sender + .table("tableName") + .booleanColumn("boolCol", true) + .timestampColumn("timestampCol", 1658484765000000) + .at(1658484769000000n, "us"); + expect(sender.toBufferView().toString()).toBe( + "tableName boolCol=t,timestampCol=1658484765000000t 1658484769000000000\n", + ); + await sender.close(); + }); + + it("supports setting designated ms timestamp as BigInt from client", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + await sender + .table("tableName") + .booleanColumn("boolCol", true) + .timestampColumn("timestampCol", 1658484765000000) + .at(1658484769000n, "ms"); + expect(sender.toBufferView().toString()).toBe( + "tableName boolCol=t,timestampCol=1658484765000000t 1658484769000000000\n", + ); + await sender.close(); + }); + + it("throws exception if table name is not a string", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + // @ts-expect-error - Invalid options + expect(() => sender.table(23456)).toThrow( + "Table name must be a string, received number", + ); + await sender.close(); + }); + + it("throws exception if table name is too long", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + expect(() => + sender.table( + "123456789012345678901234567890123456789012345678901234567890" + + "12345678901234567890123456789012345678901234567890123456789012345678", + ), + ).toThrow("Table name is too long, max length is 127"); + await sender.close(); + }); + + it("throws exception if table name is set more times", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + expect(() => + sender.table("tableName").symbol("name", "value").table("newTableName"), + ).toThrow("Table name has already been set"); + await sender.close(); + }); + + it("throws exception if symbol name is not a string", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + // @ts-expect-error - Invalid options + expect(() => sender.table("tableName").symbol(12345.5656, "value")).toThrow( + "Symbol name must be a string, received number", + ); + await sender.close(); + }); + + it("throws exception if symbol name is empty string", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + expect(() => sender.table("tableName").symbol("", "value")).toThrow( + "Empty string is not allowed as column name", + ); + await sender.close(); + }); + + it("throws exception if column name is not a string", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + expect(() => + // @ts-expect-error - Invalid options + sender.table("tableName").stringColumn(12345.5656, "value"), + ).toThrow("Column name must be a string, received number"); + await sender.close(); + }); + + it("throws exception if column name is empty string", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + expect(() => sender.table("tableName").stringColumn("", "value")).toThrow( + "Empty string is not allowed as column name", + ); + await sender.close(); + }); + + it("throws exception if column name is too long", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + expect(() => + sender + .table("tableName") + .stringColumn( + "123456789012345678901234567890123456789012345678901234567890" + + "12345678901234567890123456789012345678901234567890123456789012345678", + "value", + ), + ).toThrow("Column name is too long, max length is 127"); + await sender.close(); + }); + + it("throws exception if column value is not the right type", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + expect(() => + // @ts-expect-error - Invalid options + sender.table("tableName").stringColumn("columnName", false), + ).toThrow("Column value must be of type string, received boolean"); + await sender.close(); + }); + + it("throws exception if adding column without setting table name", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + expect(() => sender.floatColumn("name", 12.459)).toThrow( + "Column can be set only after table name is set", + ); + await sender.close(); + }); + + it("throws exception if adding symbol without setting table name", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + expect(() => sender.symbol("name", "value")).toThrow( + "Symbol can be added only after table name is set and before any column added", + ); + await sender.close(); + }); + + it("throws exception if adding symbol after columns", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + expect(() => + sender + .table("tableName") + .stringColumn("name", "value") + .symbol("symbolName", "symbolValue"), + ).toThrow("Symbol can be added only after table name is set and before any column added"); + await sender.close(); + }); + + it("returns null if preparing an empty buffer for send", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + expect(sender.toBufferView()).toBe(null); + expect(sender.toBufferNew()).toBe(null); + await sender.close(); + }); + + it("leaves unfinished rows in the sender's buffer when preparing a copy of the buffer for send", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + sender.table("tableName").symbol("name", "value"); + await sender.at(1234567890n, "ns"); + sender.table("tableName").symbol("name", "value2"); + + // copy of the sender's buffer contains the finished row + expect(sender.toBufferNew().toString()).toBe( + "tableName,name=value 1234567890\n", + ); + // the sender's buffer is compacted, and contains only the unfinished row + // @ts-expect-error - Accessing private field + expect(sender.toBufferView(sender.position).toString()).toBe( + "tableName,name=value2", + ); + await sender.close(); + }); + + it("throws exception if a float is passed as integer field", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + expect(() => + sender.table("tableName").intColumn("intField", 123.222), + ).toThrow("Value must be an integer, received 123.222"); + await sender.close(); + }); + + it("throws exception if a float is passed as timestamp field", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + expect(() => + sender.table("tableName").timestampColumn("intField", 123.222), + ).toThrow("Value must be an integer or BigInt, received 123.222"); + await sender.close(); + }); + + it("throws exception if designated timestamp is not an integer or bigint", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + try { + await sender + .table("tableName") + .symbol("name", "value") + .at(23232322323.05); + } catch (e) { + expect(e.message).toEqual( + "Designated timestamp must be an integer or BigInt, received 23232322323.05", + ); + } + await sender.close(); + }); + + it("throws exception if designated timestamp is invalid", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + try { + // @ts-expect-error - Invalid options + await sender.table("tableName").symbol("name", "value").at("invalid_dts"); + } catch (e) { + expect(e.message).toEqual( + "Designated timestamp must be an integer or BigInt, received invalid_dts", + ); + } + await sender.close(); + }); + + it("throws exception if designated timestamp is set without any fields added", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + try { + await sender.table("tableName").at(12345678n, "ns"); + } catch (e) { + expect(e.message).toEqual( + "The row must have a symbol or column set before it is closed", + ); + } + await sender.close(); + }); + + it("extends the size of the buffer if data does not fit", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 8, + }); + // @ts-expect-error - Accessing private field + expect(sender.bufferSize).toBe(8); + // @ts-expect-error - Accessing private field + expect(sender.position).toBe(0); + sender.table("tableName"); + // @ts-expect-error - Accessing private field + expect(sender.bufferSize).toBe(16); + // @ts-expect-error - Accessing private field + expect(sender.position).toBe("tableName".length); + sender.intColumn("intField", 123); + // @ts-expect-error - Accessing private field + expect(sender.bufferSize).toBe(32); + // @ts-expect-error - Accessing private field + expect(sender.position).toBe("tableName intField=123i".length); + await sender.atNow(); + // @ts-expect-error - Accessing private field + expect(sender.bufferSize).toBe(32); + // @ts-expect-error - Accessing private field + expect(sender.position).toBe("tableName intField=123i\n".length); + expect(sender.toBufferView().toString()).toBe("tableName intField=123i\n"); + + await sender + .table("table2") + .intColumn("intField", 125) + .stringColumn("strField", "test") + .atNow(); + // @ts-expect-error - Accessing private field + expect(sender.bufferSize).toBe(64); + // @ts-expect-error - Accessing private field + expect(sender.position).toBe( + 'tableName intField=123i\ntable2 intField=125i,strField="test"\n'.length, + ); + expect(sender.toBufferView().toString()).toBe( + 'tableName intField=123i\ntable2 intField=125i,strField="test"\n', + ); + await sender.close(); + }); + + it("throws exception if tries to extend the size of the buffer above max buffer size", async function () { + const sender = Sender.fromConfig( + "tcp::addr=host;init_buf_size=8;max_buf_size=48;", + ); + // @ts-expect-error - Accessing private field + expect(sender.bufferSize).toBe(8); + // @ts-expect-error - Accessing private field + expect(sender.position).toBe(0); + sender.table("tableName"); + // @ts-expect-error - Accessing private field + expect(sender.bufferSize).toBe(16); + // @ts-expect-error - Accessing private field + expect(sender.position).toBe("tableName".length); + sender.intColumn("intField", 123); + // @ts-expect-error - Accessing private field + expect(sender.bufferSize).toBe(32); + // @ts-expect-error - Accessing private field + expect(sender.position).toBe("tableName intField=123i".length); + await sender.atNow(); + // @ts-expect-error - Accessing private field + expect(sender.bufferSize).toBe(32); + // @ts-expect-error - Accessing private field + expect(sender.position).toBe("tableName intField=123i\n".length); + expect(sender.toBufferView().toString()).toBe("tableName intField=123i\n"); + + try { + await sender + .table("table2") + .intColumn("intField", 125) + .stringColumn("strField", "test") + .atNow(); + } catch (err) { + expect(err.message).toBe( + "Max buffer size is 48 bytes, requested buffer size: 64", + ); + } + await sender.close(); + }); + + it("is possible to clear the buffer by calling reset()", async function () { + const sender = new Sender({ + protocol: "tcp", + host: "host", + init_buf_size: 1024, + }); + await sender + .table("tableName") + .booleanColumn("boolCol", true) + .timestampColumn("timestampCol", 1658484765000000) + .atNow(); + await sender + .table("tableName") + .booleanColumn("boolCol", false) + .timestampColumn("timestampCol", 1658484766000000) + .atNow(); + expect(sender.toBufferView().toString()).toBe( + "tableName boolCol=t,timestampCol=1658484765000000t\n" + + "tableName boolCol=f,timestampCol=1658484766000000t\n", + ); + + sender.reset(); + await sender + .table("tableName") + .floatColumn("floatCol", 1234567890) + .timestampColumn("timestampCol", 1658484767000000) + .atNow(); + expect(sender.toBufferView().toString()).toBe( + "tableName floatCol=1234567890,timestampCol=1658484767000000t\n", + ); + await sender.close(); + }); +}); diff --git a/test/sender.config.test.ts b/test/sender.config.test.ts new file mode 100644 index 0000000..6676d65 --- /dev/null +++ b/test/sender.config.test.ts @@ -0,0 +1,402 @@ +// @ts-check +import { describe, it, expect } from "vitest"; + +import { Sender, DEFAULT_BUFFER_SIZE, DEFAULT_MAX_BUFFER_SIZE } from "../src/sender"; +import { log } from "../src/logging"; + +describe("Sender configuration options suite", function () { + it("creates a sender from a configuration string", async function () { + await Sender.fromConfig("tcps::addr=hostname;").close(); + }); + + it("creates a sender from a configuration string picked up from env", async function () { + process.env.QDB_CLIENT_CONF = "https::addr=hostname;"; + await Sender.fromEnv().close(); + }); + + it("throws exception if the username or the token is missing when TCP transport is used", async function () { + await expect(async () => + await Sender.fromConfig("tcp::addr=hostname;username=bobo;").close() + ).rejects.toThrow( + "TCP transport requires a username and a private key for authentication, please, specify the 'username' and 'token' config options", + ); + + await expect(async () => + await Sender.fromConfig("tcp::addr=hostname;token=bobo_token;").close() + ).rejects.toThrow( + "TCP transport requires a username and a private key for authentication, please, specify the 'username' and 'token' config options", + ); + }); + + it("throws exception if tls_roots or tls_roots_password is used", async function () { + await expect(async () => + await Sender.fromConfig("tcps::addr=hostname;username=bobo;tls_roots=bla;").close() + ).rejects.toThrow( + "'tls_roots' and 'tls_roots_password' options are not supported, please, use the 'tls_ca' option or the NODE_EXTRA_CA_CERTS environment variable instead", + ); + + await expect(async () => + await Sender.fromConfig("tcps::addr=hostname;token=bobo_token;tls_roots_password=bla;").close() + ).rejects.toThrow( + "'tls_roots' and 'tls_roots_password' options are not supported, please, use the 'tls_ca' option or the NODE_EXTRA_CA_CERTS environment variable instead", + ); + }); + + it("throws exception if connect() is called when http transport is used", async function () { + let sender: Sender; + await expect(async () => { + sender = Sender.fromConfig("http::addr=hostname"); + await sender.connect(); + }).rejects.toThrow("'connect()' is not required for HTTP transport"); + await sender.close(); + }); +}); + +describe("Sender options test suite", function () { + it("fails if no options defined", async function () { + await expect(async () => + // @ts-expect-error - Testing invalid options + await new Sender().close() + ).rejects.toThrow("The 'protocol' option is mandatory"); + }); + + it("fails if options are null", async function () { + await expect(async () => + await new Sender(null).close() + ).rejects.toThrow("The 'protocol' option is mandatory"); + }); + + it("fails if options are undefined", async function () { + await expect(async () => + await new Sender(undefined).close() + ).rejects.toThrow("The 'protocol' option is mandatory"); + }); + + it("fails if options are empty", async function () { + await expect(async () => + // @ts-expect-error - Testing invalid options + await new Sender({}).close() + ).rejects.toThrow("The 'protocol' option is mandatory"); + }); + + it("fails if protocol option is missing", async function () { + await expect(async () => + // @ts-expect-error - Testing invalid options + await new Sender({ host: "host" }).close() + ).rejects.toThrow("The 'protocol' option is mandatory"); + }); + + it("fails if protocol option is invalid", async function () { + await expect(async () => + await new Sender({ protocol: "abcd", host: "hostname" }).close() + ).rejects.toThrow("Invalid protocol: 'abcd'"); + }); + + it("sets default buffer size if init_buf_size is not set", async function () { + const sender = new Sender({ + protocol: "http", + host: "host", + }); + // @ts-expect-error - Accessing private field + expect(sender.bufferSize).toBe(DEFAULT_BUFFER_SIZE); + await sender.close(); + }); + + it("sets the requested buffer size if init_buf_size is set", async function () { + const sender = new Sender({ + protocol: "http", + host: "host", + init_buf_size: 1024, + }); + // @ts-expect-error - Accessing private field + expect(sender.bufferSize).toBe(1024); + await sender.close(); + }); + + it("sets default buffer size if init_buf_size is set to null", async function () { + const sender = new Sender({ + protocol: "http", + host: "host", + init_buf_size: null, + }); + // @ts-expect-error - Accessing private field + expect(sender.bufferSize).toBe(DEFAULT_BUFFER_SIZE); + await sender.close(); + }); + + it("sets default buffer size if init_buf_size is set to undefined", async function () { + const sender = new Sender({ + protocol: "http", + host: "host", + init_buf_size: undefined, + }); + // @ts-expect-error - Accessing private field + expect(sender.bufferSize).toBe(DEFAULT_BUFFER_SIZE); + await sender.close(); + }); + + it("sets default buffer size if init_buf_size is not a number", async function () { + const sender = new Sender({ + protocol: "http", + host: "host", + // @ts-expect-error - Testing invalid options + init_buf_size: "1024", + }); + // @ts-expect-error - Accessing private field + expect(sender.bufferSize).toBe(DEFAULT_BUFFER_SIZE); + await sender.close(); + }); + + it("sets the requested buffer size if 'bufferSize' is set, but warns that it is deprecated", async function () { + const log = (level: "error" | "warn" | "info" | "debug", message: string | Error) => { + expect(level).toBe("warn"); + expect(message).toMatch("Option 'bufferSize' is not supported anymore, please, replace it with 'init_buf_size'"); + }; + const sender = new Sender({ + protocol: "http", + host: "host", + // @ts-expect-error - Testing deprecated option + bufferSize: 2048, + log: log, + }); + // @ts-expect-error - Accessing private field + expect(sender.bufferSize).toBe(2048); + await sender.close(); + }); + + it("warns about deprecated option 'copy_buffer'", async function () { + const log = (level: "error" | "warn" | "info" | "debug", message: string) => { + expect(level).toBe("warn"); + expect(message).toMatch("Option 'copy_buffer' is not supported anymore, please, remove it"); + }; + const sender = new Sender({ + protocol: "http", + host: "host", + // @ts-expect-error - Testing deprecated option + copy_buffer: false, + log: log, + }); + await sender.close(); + }); + + it("warns about deprecated option 'copyBuffer'", async function () { + const log = (level: "error" | "warn" | "info" | "debug", message: string) => { + expect(level).toBe("warn"); + expect(message).toMatch("Option 'copyBuffer' is not supported anymore, please, remove it"); + }; + const sender = new Sender({ + protocol: "http", + host: "host", + // @ts-expect-error - Testing deprecated option + copyBuffer: false, + log: log, + }); + await sender.close(); + }); + + it("sets default max buffer size if max_buf_size is not set", async function () { + const sender = new Sender({ protocol: "http", host: "host" }); + // @ts-expect-error - Accessing private field + expect(sender.maxBufferSize).toBe(DEFAULT_MAX_BUFFER_SIZE); + await sender.close(); + }); + + it("sets the requested max buffer size if max_buf_size is set", async function () { + const sender = new Sender({ + protocol: "http", + host: "host", + max_buf_size: 131072, + }); + // @ts-expect-error - Accessing private field + expect(sender.maxBufferSize).toBe(131072); + await sender.close(); + }); + + it("throws error if initial buffer size is greater than max_buf_size", async function () { + await expect(async () => + await new Sender({ + protocol: "http", + host: "host", + max_buf_size: 8192, + init_buf_size: 16384, + }).close() + ).rejects.toThrow("Max buffer size is 8192 bytes, requested buffer size: 16384") + }); + + it("sets default max buffer size if max_buf_size is set to null", async function () { + const sender = new Sender({ + protocol: "http", + host: "host", + max_buf_size: null, + }); + // @ts-expect-error - Accessing private field + expect(sender.maxBufferSize).toBe(DEFAULT_MAX_BUFFER_SIZE); + await sender.close(); + }); + + it("sets default max buffer size if max_buf_size is set to undefined", async function () { + const sender = new Sender({ + protocol: "http", + host: "host", + max_buf_size: undefined, + }); + // @ts-expect-error - Accessing private field + expect(sender.maxBufferSize).toBe(DEFAULT_MAX_BUFFER_SIZE); + await sender.close(); + }); + + it("sets default max buffer size if max_buf_size is not a number", async function () { + const sender = new Sender({ + protocol: "http", + host: "host", + // @ts-expect-error - Testing invalid value + max_buf_size: "1024", + }); + // @ts-expect-error - Accessing private field + expect(sender.maxBufferSize).toBe(DEFAULT_MAX_BUFFER_SIZE); + await sender.close(); + }); + + it("uses default logger if log function is not set", async function () { + const sender = new Sender({ protocol: "http", host: "host" }); + // @ts-expect-error - Accessing private field + expect(sender.log).toBe(log); + await sender.close(); + }); + + it("uses the required log function if it is set", async function () { + const testFunc = () => { }; + const sender = new Sender({ + protocol: "http", + host: "host", + log: testFunc, + }); + // @ts-expect-error - Accessing private field + expect(sender.log).toBe(testFunc); + await sender.close(); + }); + + it("uses default logger if log is set to null", async function () { + const sender = new Sender({ protocol: "http", host: "host", log: null }); + // @ts-expect-error - Accessing private field + expect(sender.log).toBe(log); + await sender.close(); + }); + + it("uses default logger if log is set to undefined", async function () { + const sender = new Sender({ + protocol: "http", + host: "host", + log: undefined, + }); + // @ts-expect-error - Accessing private field + expect(sender.log).toBe(log); + await sender.close(); + }); + + it("uses default logger if log is not a function", async function () { + // @ts-expect-error - Testing invalid options + const sender = new Sender({ protocol: "http", host: "host", log: "" }); + // @ts-expect-error - Accessing private field + expect(sender.log).toBe(log); + await sender.close(); + }); +}); + +describe("Sender auth config checks suite", function () { + it("requires a username for authentication", async function () { + await expect(async () => + await new Sender({ + protocol: "tcp", + host: "host", + auth: { + token: "privateKey", + }, + }).close() + ).rejects.toThrow( + "Missing username, please, specify the 'keyId' property of the 'auth' config option. " + + "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", + ); + }); + + it("requires a non-empty username", async function () { + await expect(async () => + await new Sender({ + protocol: "tcp", + host: "host", + auth: { + keyId: "", + token: "privateKey", + }, + }).close() + ).rejects.toThrow( + "Missing username, please, specify the 'keyId' property of the 'auth' config option. " + + "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", + ); + }); + + it("requires that the username is a string", async function () { + await expect(async () => + await new Sender({ + protocol: "tcp", + host: "host", + auth: { + // @ts-expect-error - Testing invalid options + keyId: 23, + token: "privateKey", + }, + }).close() + ).rejects.toThrow( + "Please, specify the 'keyId' property of the 'auth' config option as a string. " + + "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", + ); + }); + + it("requires a private key for authentication", async function () { + await expect(async () => + await new Sender({ + protocol: "tcp", + host: "host", + auth: { + keyId: "username", + }, + }).close() + ).rejects.toThrow( + "Missing private key, please, specify the 'token' property of the 'auth' config option. " + + "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", + ); + }); + + it("requires a non-empty private key", async function () { + await expect(async () => + await new Sender({ + protocol: "tcp", + host: "host", + auth: { + keyId: "username", + token: "", + }, + }).close() + ).rejects.toThrow( + "Missing private key, please, specify the 'token' property of the 'auth' config option. " + + "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", + ); + }); + + it("requires that the private key is a string", async function () { + await expect(async () => + await new Sender({ + protocol: "tcp", + host: "host", + auth: { + keyId: "username", + // @ts-expect-error - Testing invalid options + token: true, + }, + }).close() + ).rejects.toThrow( + "Please, specify the 'token' property of the 'auth' config option as a string. " + + "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", + ); + }); +}); diff --git a/test/sender.integration.test.ts b/test/sender.integration.test.ts new file mode 100644 index 0000000..cc9e410 --- /dev/null +++ b/test/sender.integration.test.ts @@ -0,0 +1,392 @@ +// @ts-check +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { GenericContainer } from "testcontainers"; +import http from "http"; + +import { Sender } from "../src"; + +const HTTP_OK = 200; + +const QUESTDB_HTTP_PORT = 9000; +const QUESTDB_ILP_PORT = 9009; + +async function sleep(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +describe("Sender tests with containerized QuestDB instance", () => { + let container: any; + + async function query(container: any, query: string) { + const options: http.RequestOptions = { + hostname: container.getHost(), + port: container.getMappedPort(QUESTDB_HTTP_PORT), + path: `/exec?query=${encodeURIComponent(query)}`, + method: "GET", + }; + + return new Promise((resolve, reject) => { + const req = http.request(options, (response) => { + if (response.statusCode === HTTP_OK) { + const body: Uint8Array[] = []; + response + .on("data", (data: Uint8Array) => { + body.push(data); + }) + .on("end", () => { + resolve(JSON.parse(Buffer.concat(body).toString())); + }); + } else { + reject(new Error(`HTTP request failed, statusCode=${response.statusCode}, query=${query}`)); + } + }); + + req.on("error", error => reject(error)); + req.end(); + }); + } + + async function runSelect(container: any, select: string, expectedCount: number, timeout = 60000) { + const interval = 500; + const num = timeout / interval; + let selectResult: any; + for (let i = 0; i < num; i++) { + selectResult = await query(container, select); + if (selectResult && selectResult.count >= expectedCount) { + return selectResult; + } + await sleep(interval); + } + throw new Error( + `Timed out while waiting for ${expectedCount} rows, select='${select}'`, + ); + } + + async function waitForTable(container: any, tableName: string, timeout = 30000) { + await runSelect(container, `tables() where table_name='${tableName}'`, 1, timeout); + } + + beforeAll(async () => { + container = await new GenericContainer("questdb/questdb:nightly") + .withExposedPorts(QUESTDB_HTTP_PORT, QUESTDB_ILP_PORT) + .start(); + + const stream = await container.logs(); + stream + .on("data", (line: string) => console.log(line)) + .on("err", (line: string) => console.error(line)) + .on("end", () => console.log("Stream closed")); + }, 3000000); + + afterAll(async () => { + await container.stop(); + }); + + it("can ingest data via TCP and run queries", async () => { + const sender = new Sender({ + protocol: "tcp", + host: container.getHost(), + port: container.getMappedPort(QUESTDB_ILP_PORT), + }); + await sender.connect(); + + const tableName = "test_tcp"; + const schema = [ + { name: "location", type: "SYMBOL" }, + { name: "temperature", type: "DOUBLE" }, + { name: "timestamp", type: "TIMESTAMP" }, + ]; + + // ingest via client + await sender + .table(tableName) + .symbol("location", "us") + .floatColumn("temperature", 17.1) + .at(1658484765000000000n, "ns"); + await sender.flush(); + + // wait for the table + await waitForTable(container, tableName) + + // query table + const select1Result = await runSelect(container, tableName, 1); + expect(select1Result.query).toBe(tableName); + expect(select1Result.count).toBe(1); + expect(select1Result.columns).toStrictEqual(schema); + expect(select1Result.dataset).toStrictEqual([ + ["us", 17.1, "2022-07-22T10:12:45.000000Z"], + ]); + + // ingest via client, add new column + await sender + .table(tableName) + .symbol("location", "us") + .floatColumn("temperature", 17.3) + .at(1658484765000666000n, "ns"); + await sender + .table(tableName) + .symbol("location", "emea") + .floatColumn("temperature", 17.4) + .at(1658484765000999000n, "ns"); + await sender + .table(tableName) + .symbol("location", "emea") + .symbol("city", "london") + .floatColumn("temperature", 18.8) + .at(1658484765001234000n, "ns"); + await sender.flush(); + + // query table + const select2Result = await runSelect(container, tableName, 4); + expect(select2Result.query).toBe(tableName); + expect(select2Result.count).toBe(4); + expect(select2Result.columns).toStrictEqual([ + { name: "location", type: "SYMBOL" }, + { name: "temperature", type: "DOUBLE" }, + { name: "timestamp", type: "TIMESTAMP" }, + { name: "city", type: "SYMBOL" }, + ]); + expect(select2Result.dataset).toStrictEqual([ + ["us", 17.1, "2022-07-22T10:12:45.000000Z", null], + ["us", 17.3, "2022-07-22T10:12:45.000666Z", null], + ["emea", 17.4, "2022-07-22T10:12:45.000999Z", null], + ["emea", 18.8, "2022-07-22T10:12:45.001234Z", "london"], + ]); + + await sender.close(); + }); + + it("can ingest data via HTTP with auto flush rows", async () => { + const tableName = "test_http_rows"; + const schema = [ + { name: "location", type: "SYMBOL" }, + { name: "temperature", type: "DOUBLE" }, + { name: "timestamp", type: "TIMESTAMP" }, + ]; + + const sender = Sender.fromConfig( + `http::addr=${container.getHost()}:${container.getMappedPort(QUESTDB_HTTP_PORT)};auto_flush_interval=0;auto_flush_rows=1`, + ); + + // ingest via client + await sender + .table(tableName) + .symbol("location", "us") + .floatColumn("temperature", 17.1) + .at(1658484765000000000n, "ns"); + + // wait for the table + await waitForTable(container, tableName) + + // query table + const select1Result = await runSelect(container, tableName, 1); + expect(select1Result.query).toBe(tableName); + expect(select1Result.count).toBe(1); + expect(select1Result.columns).toStrictEqual(schema); + expect(select1Result.dataset).toStrictEqual([ + ["us", 17.1, "2022-07-22T10:12:45.000000Z"], + ]); + + // ingest via client, add new column + await sender + .table(tableName) + .symbol("location", "us") + .floatColumn("temperature", 17.36) + .at(1658484765000666000n, "ns"); + await sender + .table(tableName) + .symbol("location", "emea") + .floatColumn("temperature", 17.41) + .at(1658484765000999000n, "ns"); + await sender + .table(tableName) + .symbol("location", "emea") + .symbol("city", "london") + .floatColumn("temperature", 18.81) + .at(1658484765001234000n, "ns"); + + // query table + const select2Result = await runSelect(container, tableName, 4); + expect(select2Result.query).toBe(tableName); + expect(select2Result.count).toBe(4); + expect(select2Result.columns).toStrictEqual([ + { name: "location", type: "SYMBOL" }, + { name: "temperature", type: "DOUBLE" }, + { name: "timestamp", type: "TIMESTAMP" }, + { name: "city", type: "SYMBOL" }, + ]); + expect(select2Result.dataset).toStrictEqual([ + ["us", 17.1, "2022-07-22T10:12:45.000000Z", null], + ["us", 17.36, "2022-07-22T10:12:45.000666Z", null], + ["emea", 17.41, "2022-07-22T10:12:45.000999Z", null], + ["emea", 18.81, "2022-07-22T10:12:45.001234Z", "london"], + ]); + + await sender.close(); + }); + + it("can ingest data via HTTP with auto flush interval", async () => { + const tableName = "test_http_interval"; + const schema = [ + { name: "location", type: "SYMBOL" }, + { name: "temperature", type: "DOUBLE" }, + { name: "timestamp", type: "TIMESTAMP" }, + ]; + + const sender = Sender.fromConfig( + `http::addr=${container.getHost()}:${container.getMappedPort(QUESTDB_HTTP_PORT)};auto_flush_interval=1;auto_flush_rows=0`, + ); + + // wait longer than the set auto flush interval to make sure there is a flush + await sleep(10); + + // ingest via client + await sender + .table(tableName) + .symbol("location", "us") + .floatColumn("temperature", 17.1) + .at(1658484765000000000n, "ns"); + + // wait for the table + await waitForTable(container, tableName) + + // query table + const select1Result = await runSelect(container, tableName, 1); + expect(select1Result.query).toBe(tableName); + expect(select1Result.count).toBe(1); + expect(select1Result.columns).toStrictEqual(schema); + expect(select1Result.dataset).toStrictEqual([ + ["us", 17.1, "2022-07-22T10:12:45.000000Z"], + ]); + + // ingest via client, add new column + await sleep(10); + await sender + .table(tableName) + .symbol("location", "us") + .floatColumn("temperature", 17.36) + .at(1658484765000666000n, "ns"); + await sleep(10); + await sender + .table(tableName) + .symbol("location", "emea") + .floatColumn("temperature", 17.41) + .at(1658484765000999000n, "ns"); + await sleep(10); + await sender + .table(tableName) + .symbol("location", "emea") + .symbol("city", "london") + .floatColumn("temperature", 18.81) + .at(1658484765001234000n, "ns"); + + // query table + const select2Result = await runSelect(container, tableName, 4); + expect(select2Result.query).toBe(tableName); + expect(select2Result.count).toBe(4); + expect(select2Result.columns).toStrictEqual([ + { name: "location", type: "SYMBOL" }, + { name: "temperature", type: "DOUBLE" }, + { name: "timestamp", type: "TIMESTAMP" }, + { name: "city", type: "SYMBOL" }, + ]); + expect(select2Result.dataset).toStrictEqual([ + ["us", 17.1, "2022-07-22T10:12:45.000000Z", null], + ["us", 17.36, "2022-07-22T10:12:45.000666Z", null], + ["emea", 17.41, "2022-07-22T10:12:45.000999Z", null], + ["emea", 18.81, "2022-07-22T10:12:45.001234Z", "london"], + ]); + + await sender.close(); + }); + + it("does not duplicate rows if await is missing when calling flush", async () => { + const sender = new Sender({ + protocol: "tcp", + host: container.getHost(), + port: container.getMappedPort(QUESTDB_ILP_PORT), + }); + await sender.connect(); + + const tableName = "test2"; + const schema = [ + { name: "location", type: "SYMBOL" }, + { name: "temperature", type: "DOUBLE" }, + { name: "timestamp", type: "TIMESTAMP" }, + ]; + + // ingest via client + const numOfRows = 100; + for (let i = 0; i < numOfRows; i++) { + const p1 = sender + .table(tableName) + .symbol("location", "us") + .floatColumn("temperature", i) + .at(1658484765000000000n, "ns"); + const p2 = sender.flush(); + // IMPORTANT: missing 'await' for p1 and p2 is intentional! + expect(p1).toBeTruthy(); + expect(p2).toBeTruthy(); + } + + // wait for the table + await waitForTable(container, tableName) + + // query table + const selectQuery = `${tableName} order by temperature`; + const selectResult = await runSelect(container, selectQuery, numOfRows); + expect(selectResult.query).toBe(selectQuery); + expect(selectResult.count).toBe(numOfRows); + expect(selectResult.columns).toStrictEqual(schema); + + const expectedData: (string | number)[][] = []; + for (let i = 0; i < numOfRows; i++) { + expectedData.push(["us", i, "2022-07-22T10:12:45.000000Z"]); + } + expect(selectResult.dataset).toStrictEqual(expectedData); + + await sender.close(); + }); + + it("ingests all data without loss under high load with auto-flush", async () => { + const sender = Sender.fromConfig( + `tcp::addr=${container.getHost()}:${container.getMappedPort(QUESTDB_ILP_PORT)};auto_flush_rows=5;auto_flush_interval=1`, + ); + await sender.connect(); + + const tableName = "test_high_load_autoflush"; + const numOfRows = 1000; + const promises: Promise[] = []; + + for (let i = 0; i < numOfRows; i++) { + // Not awaiting each .at() call individually to allow them to queue up + const p = sender + .table(tableName) + .intColumn("id", i) + .at(1658484765000000000n + BigInt(1000 * i), "ns"); // Unique timestamp for each row + promises.push(p); + } + + // Wait for all .at() calls to complete their processing (including triggering auto-flushes) + await Promise.all(promises); + + // Perform a final flush to ensure any data remaining in the buffer is sent. + // This will be queued correctly after any ongoing auto-flushes. + await sender.flush(); + + // Wait for the table + await waitForTable(container, tableName) + + // Query table and verify count + const selectQuery = `SELECT id FROM ${tableName}`; + const selectResult = await runSelect(container, selectQuery, numOfRows); + expect(selectResult.count).toBe(numOfRows); + + // Verify data integrity + for (let i = 0; i < numOfRows; i++) { + expect(selectResult.dataset[i][0]).toBe(i); + } + + await sender.close(); + }, 30000); // Increased test timeout for this specific test +}); diff --git a/test/sender.test.ts b/test/sender.test.ts deleted file mode 100644 index 57a60b5..0000000 --- a/test/sender.test.ts +++ /dev/null @@ -1,2176 +0,0 @@ -import { Sender } from "../src"; -import { describe, it, expect, beforeAll, afterAll } from "vitest"; -import { DEFAULT_BUFFER_SIZE, DEFAULT_MAX_BUFFER_SIZE } from "../src/sender"; -import { readFileSync } from "fs"; -import { MockProxy } from "./_utils_/mockproxy"; -import { MockHttp } from "./_utils_/mockhttp"; -import { GenericContainer } from "testcontainers"; -import http from "http"; -import { Agent } from "undici"; -import { SenderOptions } from "../src/options"; -import { fail } from "node:assert"; -import { log } from "../src/logging"; - -const HTTP_OK = 200; - -const QUESTDB_HTTP_PORT = 9000; -const QUESTDB_ILP_PORT = 9009; -const MOCK_HTTP_PORT = 9099; -const MOCK_HTTPS_PORT = 9098; -const PROXY_PORT = 9088; -const PROXY_HOST = "localhost"; - -const proxyOptions = { - key: readFileSync("test/certs/server/server.key"), - cert: readFileSync("test/certs/server/server.crt"), - ca: readFileSync("test/certs/ca/ca.crt"), -}; - -const USER_NAME = "testapp"; -const PRIVATE_KEY = "9b9x5WhJywDEuo1KGQWSPNxtX-6X6R2BRCKhYMMY6n8"; -const AUTH: SenderOptions["auth"] = { - keyId: USER_NAME, - token: PRIVATE_KEY, -}; - -async function sleep(ms: number) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -describe("Sender configuration options suite", function () { - it("creates a sender from a configuration string", async function () { - await Sender.fromConfig("tcps::addr=hostname;").close(); - }); - - it("creates a sender from a configuration string picked up from env", async function () { - process.env.QDB_CLIENT_CONF = "https::addr=hostname;"; - await Sender.fromEnv().close(); - }); - - it("throws exception if the username or the token is missing when TCP transport is used", async function () { - try { - await Sender.fromConfig("tcp::addr=hostname;username=bobo;").close(); - fail("Expected error is not thrown"); - } catch (err) { - expect(err.message).toBe( - "TCP transport requires a username and a private key for authentication, please, specify the 'username' and 'token' config options", - ); - } - - try { - await Sender.fromConfig("tcp::addr=hostname;token=bobo_token;").close(); - fail("Expected error is not thrown"); - } catch (err) { - expect(err.message).toBe( - "TCP transport requires a username and a private key for authentication, please, specify the 'username' and 'token' config options", - ); - } - }); - - it("throws exception if tls_roots or tls_roots_password is used", async function () { - try { - await Sender.fromConfig( - "tcps::addr=hostname;username=bobo;tls_roots=bla;", - ).close(); - fail("Expected error is not thrown"); - } catch (err) { - expect(err.message).toBe( - "'tls_roots' and 'tls_roots_password' options are not supported, please, use the 'tls_ca' option or the NODE_EXTRA_CA_CERTS environment variable instead", - ); - } - - try { - await Sender.fromConfig( - "tcps::addr=hostname;token=bobo_token;tls_roots_password=bla;", - ).close(); - fail("Expected error is not thrown"); - } catch (err) { - expect(err.message).toBe( - "'tls_roots' and 'tls_roots_password' options are not supported, please, use the 'tls_ca' option or the NODE_EXTRA_CA_CERTS environment variable instead", - ); - } - }); - - it("throws exception if connect() is called when http transport is used", async function () { - let sender: Sender; - try { - sender = Sender.fromConfig("http::addr=hostname"); - await sender.connect(); - fail("Expected error is not thrown"); - } catch (err) { - expect(err.message).toBe( - "'connect()' should be called only if the sender connects via TCP", - ); - } - await sender.close(); - }); -}); - -describe("Sender options test suite", function () { - it("fails if no options defined", async function () { - try { - // @ts-expect-error - Testing invalid options - await new Sender().close(); - fail("Expected error is not thrown"); - } catch (err) { - expect(err.message).toBe("The 'protocol' option is mandatory"); - } - }); - - it("fails if options are null", async function () { - try { - await new Sender(null).close(); - fail("Expected error is not thrown"); - } catch (err) { - expect(err.message).toBe("The 'protocol' option is mandatory"); - } - }); - - it("fails if options are undefined", async function () { - try { - await new Sender(undefined).close(); - fail("Expected error is not thrown"); - } catch (err) { - expect(err.message).toBe("The 'protocol' option is mandatory"); - } - }); - - it("fails if options are empty", async function () { - try { - // @ts-expect-error - Testing invalid options - await new Sender({}).close(); - fail("Expected error is not thrown"); - } catch (err) { - expect(err.message).toBe("The 'protocol' option is mandatory"); - } - }); - - it("fails if protocol option is missing", async function () { - try { - // @ts-expect-error - Testing invalid options - await new Sender({ host: "host" }).close(); - fail("Expected error is not thrown"); - } catch (err) { - expect(err.message).toBe("The 'protocol' option is mandatory"); - } - }); - - it("fails if protocol option is invalid", async function () { - try { - await new Sender({ protocol: "abcd" }).close(); - fail("Expected error is not thrown"); - } catch (err) { - expect(err.message).toBe("Invalid protocol: 'abcd'"); - } - }); - - it("sets default buffer size if init_buf_size is not set", async function () { - const sender = new Sender({ - protocol: "http", - host: "host", - }); - expect(sender.bufferSize).toBe(DEFAULT_BUFFER_SIZE); - await sender.close(); - }); - - it("sets the requested buffer size if init_buf_size is set", async function () { - const sender = new Sender({ - protocol: "http", - host: "host", - init_buf_size: 1024, - }); - expect(sender.bufferSize).toBe(1024); - await sender.close(); - }); - - it("sets default buffer size if init_buf_size is set to null", async function () { - const sender = new Sender({ - protocol: "http", - host: "host", - init_buf_size: null, - }); - expect(sender.bufferSize).toBe(DEFAULT_BUFFER_SIZE); - await sender.close(); - }); - - it("sets default buffer size if init_buf_size is set to undefined", async function () { - const sender = new Sender({ - protocol: "http", - host: "host", - init_buf_size: undefined, - }); - expect(sender.bufferSize).toBe(DEFAULT_BUFFER_SIZE); - await sender.close(); - }); - - it("sets default buffer size if init_buf_size is not a number", async function () { - const sender = new Sender({ - protocol: "http", - host: "host", - // @ts-expect-error - Testing invalid options - init_buf_size: "1024", - }); - expect(sender.bufferSize).toBe(DEFAULT_BUFFER_SIZE); - await sender.close(); - }); - - it("sets the requested buffer size if 'bufferSize' is set, but warns that it is deprecated", async function () { - const log = (level: "error" | "warn" | "info" | "debug", message: string) => { - expect(level).toBe("warn"); - expect(message).toMatch("Option 'bufferSize' is not supported anymore, please, replace it with 'init_buf_size'"); - }; - const sender = new Sender({ - protocol: "http", - host: "host", - // @ts-expect-error - Testing deprecated option - bufferSize: 2048, - log: log, - }); - expect(sender.bufferSize).toBe(2048); - await sender.close(); - }); - - it("warns about deprecated option 'copy_buffer'", async function () { - const log = (level: "error" | "warn" | "info" | "debug", message: string) => { - expect(level).toBe("warn"); - expect(message).toMatch("Option 'copy_buffer' is not supported anymore, please, remove it"); - }; - const sender = new Sender({ - protocol: "http", - host: "host", - // @ts-expect-error - Testing deprecated option - copy_buffer: false, - log: log, - }); - await sender.close(); - }); - - it("warns about deprecated option 'copyBuffer'", async function () { - const log = (level: "error" | "warn" | "info" | "debug", message: string) => { - expect(level).toBe("warn"); - expect(message).toMatch("Option 'copyBuffer' is not supported anymore, please, remove it"); - }; - const sender = new Sender({ - protocol: "http", - host: "host", - // @ts-expect-error - Testing deprecated option - copyBuffer: false, - log: log, - }); - await sender.close(); - }); - - it("sets default max buffer size if max_buf_size is not set", async function () { - const sender = new Sender({ protocol: "http", host: "host" }); - expect(sender.maxBufferSize).toBe(DEFAULT_MAX_BUFFER_SIZE); - await sender.close(); - }); - - it("sets the requested max buffer size if max_buf_size is set", async function () { - const sender = new Sender({ - protocol: "http", - host: "host", - max_buf_size: 131072, - }); - expect(sender.maxBufferSize).toBe(131072); - await sender.close(); - }); - - it("throws error if initial buffer size is greater than max_buf_size", async function () { - try { - await new Sender({ - protocol: "http", - host: "host", - max_buf_size: 8192, - init_buf_size: 16384, - }).close(); - fail('Expected error is not thrown'); - } catch (err) { - expect(err.message).toBe( - "Max buffer size is 8192 bytes, requested buffer size: 16384", - ); - } - }); - - it("sets default max buffer size if max_buf_size is set to null", async function () { - const sender = new Sender({ - protocol: "http", - host: "host", - max_buf_size: null, - }); - expect(sender.maxBufferSize).toBe(DEFAULT_MAX_BUFFER_SIZE); - await sender.close(); - }); - - it("sets default max buffer size if max_buf_size is set to undefined", async function () { - const sender = new Sender({ - protocol: "http", - host: "host", - max_buf_size: undefined, - }); - expect(sender.maxBufferSize).toBe(DEFAULT_MAX_BUFFER_SIZE); - await sender.close(); - }); - - it("sets default max buffer size if max_buf_size is not a number", async function () { - const sender = new Sender({ - protocol: "http", - host: "host", - // @ts-expect-error - Testing invalid vlaue - max_buf_size: "1024", - }); - expect(sender.maxBufferSize).toBe(DEFAULT_MAX_BUFFER_SIZE); - await sender.close(); - }); - - it("uses default logger if log function is not set", async function () { - const sender = new Sender({ protocol: "http", host: "host" }); - expect(sender.log).toBe(log); - await sender.close(); - }); - - it("uses the required log function if it is set", async function () { - const testFunc = () => { }; - const sender = new Sender({ - protocol: "http", - host: "host", - log: testFunc, - }); - expect(sender.log).toBe(testFunc); - await sender.close(); - }); - - it("uses default logger if log is set to null", async function () { - const sender = new Sender({ protocol: "http", host: "host", log: null }); - expect(sender.log).toBe(log); - await sender.close(); - }); - - it("uses default logger if log is set to undefined", async function () { - const sender = new Sender({ - protocol: "http", - host: "host", - log: undefined, - }); - expect(sender.log).toBe(log); - await sender.close(); - }); - - it("uses default logger if log is not a function", async function () { - // @ts-expect-error - Testing invalid options - const sender = new Sender({ protocol: "http", host: "host", log: "" }); - expect(sender.log).toBe(log); - await sender.close(); - }); -}); - -describe("Sender auth config checks suite", function () { - it("requires a username for authentication", async function () { - try { - await new Sender({ - protocol: "tcp", - host: "host", - auth: { - token: "privateKey", - }, - }).close(); - fail("it should not be able to create the sender"); - } catch (err) { - expect(err.message).toBe( - "Missing username, please, specify the 'keyId' property of the 'auth' config option. " + - "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", - ); - } - }); - - it("requires a non-empty username", async function () { - try { - await new Sender({ - protocol: "tcp", - host: "host", - auth: { - keyId: "", - token: "privateKey", - }, - }).close(); - fail("it should not be able to create the sender"); - } catch (err) { - expect(err.message).toBe( - "Missing username, please, specify the 'keyId' property of the 'auth' config option. " + - "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", - ); - } - }); - - it("requires that the username is a string", async function () { - try { - await new Sender({ - protocol: "tcp", - host: "host", - auth: { - // @ts-expect-error - Testing invalid options - keyId: 23, - token: "privateKey", - }, - }).close(); - fail("it should not be able to create the sender"); - } catch (err) { - expect(err.message).toBe( - "Please, specify the 'keyId' property of the 'auth' config option as a string. " + - "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", - ); - } - }); - - it("requires a private key for authentication", async function () { - try { - await new Sender({ - protocol: "tcp", - host: "host", - auth: { - keyId: "username", - }, - }).close(); - fail("it should not be able to create the sender"); - } catch (err) { - expect(err.message).toBe( - "Missing private key, please, specify the 'token' property of the 'auth' config option. " + - "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", - ); - } - }); - - it("requires a non-empty private key", async function () { - try { - await new Sender({ - protocol: "tcp", - host: "host", - auth: { - keyId: "username", - token: "", - }, - }).close(); - fail("it should not be able to create the sender"); - } catch (err) { - expect(err.message).toBe( - "Missing private key, please, specify the 'token' property of the 'auth' config option. " + - "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", - ); - } - }); - - it("requires that the private key is a string", async function () { - try { - await new Sender({ - protocol: "tcp", - host: "host", - auth: { - keyId: "username", - // @ts-expect-error - Testing invalid options - token: true, - }, - }).close(); - fail("it should not be able to create the sender"); - } catch (err) { - expect(err.message).toBe( - "Please, specify the 'token' property of the 'auth' config option as a string. " + - "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", - ); - } - }); -}); - -describe("Sender HTTP suite", function () { - async function sendData(sender: Sender) { - await sender - .table("test") - .symbol("location", "us") - .floatColumn("temperature", 17.1) - .at(1658484765000000000n, "ns"); - await sender.flush(); - } - - const mockHttp = new MockHttp(); - const mockHttps = new MockHttp(); - - beforeAll(async function () { - await mockHttp.start(MOCK_HTTP_PORT); - await mockHttps.start(MOCK_HTTPS_PORT, true, proxyOptions); - }); - - afterAll(async function () { - await mockHttp.stop(); - await mockHttps.stop(); - }); - - it("can ingest via HTTP", async function () { - mockHttp.reset(); - - const sender = Sender.fromConfig( - `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT}`, - ); - await sendData(sender); - expect(mockHttp.numOfRequests).toBe(1); - - await sender.close(); - }); - - it("supports custom http agent", async function () { - mockHttp.reset(); - const agent = new Agent({ pipelining: 3 }); - - const sender = Sender.fromConfig( - `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT}`, - { agent: agent }, - ); - await sendData(sender); - expect(mockHttp.numOfRequests).toBe(1); - - const symbols = Object.getOwnPropertySymbols(sender.agent); - expect(sender.agent[symbols[6]]).toEqual({ pipelining: 3 }); - - await sender.close(); - await agent.destroy(); - }); - - it("can ingest via HTTPS", async function () { - mockHttps.reset(); - - const senderCertCheckFail = Sender.fromConfig( - `https::addr=${PROXY_HOST}:${MOCK_HTTPS_PORT}`, - ); - await expect(sendData(senderCertCheckFail)).rejects.toThrowError( - "HTTP request failed, statusCode=unknown, error=self-signed certificate in certificate chain", - ); - await senderCertCheckFail.close(); - - const senderWithCA = Sender.fromConfig( - `https::addr=${PROXY_HOST}:${MOCK_HTTPS_PORT};tls_ca=test/certs/ca/ca.crt`, - ); - await sendData(senderWithCA); - expect(mockHttps.numOfRequests).toEqual(1); - await senderWithCA.close(); - - const senderVerifyOff = Sender.fromConfig( - `https::addr=${PROXY_HOST}:${MOCK_HTTPS_PORT};tls_verify=unsafe_off`, - ); - await sendData(senderVerifyOff); - expect(mockHttps.numOfRequests).toEqual(2); - await senderVerifyOff.close(); - }, 20000); - - it("can ingest via HTTP with basic auth", async function () { - mockHttp.reset({ username: "user1", password: "pwd" }); - - const sender = Sender.fromConfig( - `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};username=user1;password=pwd`, - ); - await sendData(sender); - expect(mockHttp.numOfRequests).toEqual(1); - await sender.close(); - - const senderFailPwd = Sender.fromConfig( - `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};username=user1;password=xyz`, - ); - await expect(sendData(senderFailPwd)).rejects.toThrowError( - "HTTP request failed, statusCode=401", - ); - await senderFailPwd.close(); - - const senderFailMissingPwd = Sender.fromConfig( - `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};username=user1z`, - ); - await expect(sendData(senderFailMissingPwd)).rejects.toThrowError( - "HTTP request failed, statusCode=401", - ); - await senderFailMissingPwd.close(); - - const senderFailUsername = Sender.fromConfig( - `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};username=xyz;password=pwd`, - ); - await expect(sendData(senderFailUsername)).rejects.toThrowError( - "HTTP request failed, statusCode=401", - ); - await senderFailUsername.close(); - - const senderFailMissingUsername = Sender.fromConfig( - `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};password=pwd`, - ); - await expect(sendData(senderFailMissingUsername)).rejects.toThrowError( - "HTTP request failed, statusCode=401", - ); - await senderFailMissingUsername.close(); - - const senderFailMissing = Sender.fromConfig( - `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT}`, - ); - await expect(sendData(senderFailMissing)).rejects.toThrowError( - "HTTP request failed, statusCode=401", - ); - await senderFailMissing.close(); - }); - - it("can ingest via HTTP with token auth", async function () { - mockHttp.reset({ token: "abcdefghijkl123" }); - - const sender = Sender.fromConfig( - `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};token=abcdefghijkl123`, - ); - await sendData(sender); - expect(mockHttp.numOfRequests).toBe(1); - await sender.close(); - - const senderFailToken = Sender.fromConfig( - `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};token=xyz`, - ); - await expect(sendData(senderFailToken)).rejects.toThrowError( - "HTTP request failed, statusCode=401", - ); - await senderFailToken.close(); - - const senderFailMissing = Sender.fromConfig( - `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT}`, - ); - await expect(sendData(senderFailMissing)).rejects.toThrowError( - "HTTP request failed, statusCode=401", - ); - await senderFailMissing.close(); - }); - - it("can retry via HTTP", async function () { - mockHttp.reset({ responseCodes: [204, 500, 523, 504, 500] }); - - const sender = Sender.fromConfig( - `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT}`, - ); - await sendData(sender); - expect(mockHttp.numOfRequests).toBe(5); - - await sender.close(); - }); - - it("fails when retry timeout expires", async function () { - // TODO: artificial delay (responseDelays) is the same as retry timeout, - // This should result in the request failing on the second try. - // However, with undici transport sometimes we reach the third request too. - // Investigate why, probably because of pipelining? - mockHttp.reset({ - responseCodes: [204, 500, 500], - responseDelays: [1000, 1000, 1000], - }); - - const sender = Sender.fromConfig( - `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};retry_timeout=1000`, - ); - await expect(sendData(sender)).rejects.toThrowError( - "HTTP request failed, statusCode=500, error=Request failed" - ); - await sender.close(); - }); - - it("fails when HTTP request times out", async function () { - // artificial delay (responseDelays) is greater than request timeout, and retry is switched off - // should result in the request failing with timeout - mockHttp.reset({ - responseCodes: [204], - responseDelays: [1000], - }); - - const sender = Sender.fromConfig( - `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};retry_timeout=0;request_timeout=100`, - ); - await expect(sendData(sender)).rejects.toThrowError( - "HTTP request timeout, statusCode=undefined, error=Headers Timeout Error", - ); - await sender.close(); - }); - - it("succeeds on the third request after two timeouts", async function () { - mockHttp.reset({ - responseCodes: [204, 504, 504], - responseDelays: [2000, 2000], - }); - - const sender = Sender.fromConfig( - `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};retry_timeout=30000;request_timeout=1000`, - ); - await sendData(sender); - - await sender.close(); - }); - - it("accepts custom http agent", async function () { - mockHttp.reset(); - const agent = new Agent({ connect: { keepAlive: false } }); - - const num = 300; - const senders: Sender[] = []; - const promises: Promise[] = []; - for (let i = 0; i < num; i++) { - const sender = Sender.fromConfig( - `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT}`, - { agent: agent }, - ); - senders.push(sender); - const promise = sendData(sender); - promises.push(promise); - } - await Promise.all(promises); - expect(mockHttp.numOfRequests).toBe(num); - - for (const sender of senders) { - await sender.close(); - } - await agent.destroy(); - }); -}); - -describe("Sender connection suite", function () { - async function createProxy( - auth = false, - tlsOptions?: Record, - ) { - const mockConfig = { auth: auth, assertions: true }; - const proxy = new MockProxy(mockConfig); - await proxy.start(PROXY_PORT, tlsOptions); - expect(proxy.mockConfig).toBe(mockConfig); - expect(proxy.dataSentToRemote).toStrictEqual([]); - return proxy; - } - - async function createSender(auth: SenderOptions["auth"], secure = false) { - const sender = new Sender({ - protocol: secure ? "tcps" : "tcp", - port: PROXY_PORT, - host: PROXY_HOST, - auth: auth, - tls_ca: "test/certs/ca/ca.crt", - }); - const connected = await sender.connect(); - expect(connected).toBe(true); - return sender; - } - - async function sendData(sender: Sender) { - await sender - .table("test") - .symbol("location", "us") - .floatColumn("temperature", 17.1) - .at(1658484765000000000n, "ns"); - await sender.flush(); - } - - async function assertSentData( - proxy: MockProxy, - authenticated: boolean, - expected: string, - timeout = 60000, - ) { - const interval = 100; - const num = timeout / interval; - let actual: string; - for (let i = 0; i < num; i++) { - const dataSentToRemote = proxy.getDataSentToRemote().join("").split("\n"); - if (authenticated) { - dataSentToRemote.splice(1, 1); - } - actual = dataSentToRemote.join("\n"); - if (actual === expected) { - return new Promise((resolve) => resolve(null)); - } - await sleep(interval); - } - return new Promise((resolve) => - resolve(`data assert failed [expected=${expected}, actual=${actual}]`), - ); - } - - it("can authenticate", async function () { - const proxy = await createProxy(true); - const sender = await createSender(AUTH); - await assertSentData(proxy, true, "testapp\n"); - await sender.close(); - await proxy.stop(); - }); - - it("can authenticate with a different private key", async function () { - const proxy = await createProxy(true); - const sender = await createSender({ - keyId: "user1", - token: "zhPiK3BkYMYJvRf5sqyrWNJwjDKHOWHnRbmQggUll6A", - }); - await assertSentData(proxy, true, "user1\n"); - await sender.close(); - await proxy.stop(); - }); - - it("is backwards compatible and still can authenticate with full JWK", async function () { - const JWK = { - x: "BtUXC_K3oAyGlsuPjTgkiwirMUJhuRQDfcUHeyoxFxU", - y: "R8SOup-rrNofB7wJagy4HrJhTVfrVKmj061lNRk3bF8", - kid: "user2", - kty: "EC", - d: "hsg6Zm4kSBlIEvKUWT3kif-2y2Wxw-iWaGrJxrPXQhs", - crv: "P-256", - }; - - const proxy = await createProxy(true); - const sender = new Sender({ - protocol: "tcp", - port: PROXY_PORT, - host: PROXY_HOST, - jwk: JWK, - }); - const connected = await sender.connect(); - expect(connected).toBe(true); - await assertSentData(proxy, true, "user2\n"); - await sender.close(); - await proxy.stop(); - }); - - it("can connect unauthenticated", async function () { - const proxy = await createProxy(); - // @ts-expect-error invalid options - const sender = await createSender(); - await assertSentData(proxy, false, ""); - await sender.close(); - await proxy.stop(); - }); - - it("can authenticate and send data to server", async function () { - const proxy = await createProxy(true); - const sender = await createSender(AUTH); - await sendData(sender); - await assertSentData( - proxy, - true, - "testapp\ntest,location=us temperature=17.1 1658484765000000000\n", - ); - await sender.close(); - await proxy.stop(); - }); - - it("can connect unauthenticated and send data to server", async function () { - const proxy = await createProxy(); - // @ts-expect-error invalid options - const sender = await createSender(); - await sendData(sender); - await assertSentData( - proxy, - false, - "test,location=us temperature=17.1 1658484765000000000\n", - ); - await sender.close(); - await proxy.stop(); - }); - - it("can authenticate and send data to server via secure connection", async function () { - const proxy = await createProxy(true, proxyOptions); - const sender = await createSender(AUTH, true); - await sendData(sender); - await assertSentData( - proxy, - true, - "testapp\ntest,location=us temperature=17.1 1658484765000000000\n", - ); - await sender.close(); - await proxy.stop(); - }); - - it("can connect unauthenticated and send data to server via secure connection", async function () { - const proxy = await createProxy(false, proxyOptions); - const sender = await createSender(null, true); - await sendData(sender); - await assertSentData( - proxy, - false, - "test,location=us temperature=17.1 1658484765000000000\n", - ); - await sender.close(); - await proxy.stop(); - }); - - it("fails to connect without hostname and port", async function () { - const sender = new Sender({ protocol: "tcp" }); - try { - await sender.connect(); - fail("it should not be able to connect"); - } catch (err) { - expect(err.message).toBe("Hostname is not set"); - } - await sender.close(); - }); - - it("fails to send data if not connected", async function () { - const sender = new Sender({ protocol: "tcp", host: "localhost" }); - try { - await sender.table("test").symbol("location", "us").atNow(); - await sender.flush(); - fail("it should not be able to send data"); - } catch (err) { - expect(err.message).toBe("TCP send failed, error=Sender is not connected"); - } - await sender.close(); - }); - - it("guards against multiple connect calls", async function () { - const proxy = await createProxy(true, proxyOptions); - const sender = await createSender(AUTH, true); - try { - await sender.connect(); - fail("it should not be able to connect again"); - } catch (err) { - expect(err.message).toBe("Sender connected already"); - } - await sender.close(); - await proxy.stop(); - }); - - it("guards against concurrent connect calls", async function () { - const proxy = await createProxy(true, proxyOptions); - const sender = new Sender({ - protocol: "tcps", - port: PROXY_PORT, - host: PROXY_HOST, - auth: AUTH, - tls_ca: "test/certs/ca/ca.crt", - }); - try { - await Promise.all([sender.connect(), sender.connect()]); - fail("it should not be able to connect twice"); - } catch (err) { - expect(err.message).toBe("Sender connected already"); - } - await sender.close(); - await proxy.stop(); - }); - - it("can disable the server certificate check", async function () { - const proxy = await createProxy(true, proxyOptions); - const senderCertCheckFail = Sender.fromConfig( - `tcps::addr=${PROXY_HOST}:${PROXY_PORT}`, - ); - try { - await senderCertCheckFail.connect(); - fail("it should not be able to connect"); - } catch (err) { - expect(err.message).toMatch( - /^self[ -]signed certificate in certificate chain$/, - ); - } - await senderCertCheckFail.close(); - - const senderCertCheckOn = Sender.fromConfig( - `tcps::addr=${PROXY_HOST}:${PROXY_PORT};tls_ca=test/certs/ca/ca.crt`, - ); - await senderCertCheckOn.connect(); - await senderCertCheckOn.close(); - - const senderCertCheckOff = Sender.fromConfig( - `tcps::addr=${PROXY_HOST}:${PROXY_PORT};tls_verify=unsafe_off`, - ); - await senderCertCheckOff.connect(); - await senderCertCheckOff.close(); - await proxy.stop(); - }); - - it("can handle unfinished rows during flush()", async function () { - const proxy = await createProxy(true, proxyOptions); - const sender = await createSender(AUTH, true); - sender.table("test").symbol("location", "us"); - const sent = await sender.flush(); - expect(sent).toBe(false); - await assertSentData(proxy, true, "testapp\n"); - await sender.close(); - await proxy.stop(); - }); - - it("supports custom logger", async function () { - const expectedMessages = [ - "Successfully connected to localhost:9088", - /^Connection to .*1:9088 is closed$/, - ]; - const log = (level: "error" | "warn" | "info" | "debug", message: string) => { - expect(level).toBe("info"); - expect(message).toMatch(expectedMessages.shift()); - }; - const proxy = await createProxy(); - const sender = new Sender({ - protocol: "tcp", - port: PROXY_PORT, - host: PROXY_HOST, - log: log, - }); - await sender.connect(); - await sendData(sender); - await assertSentData( - proxy, - false, - "test,location=us temperature=17.1 1658484765000000000\n", - ); - await sender.close(); - await proxy.stop(); - }); -}); - -describe("Client interop test suite", function () { - it("runs client tests as per json test config", async function () { - const testCases = JSON.parse( - readFileSync( - "./questdb-client-test/ilp-client-interop-test.json", - ).toString(), - ); - - loopTestCase: for (const testCase of testCases) { - console.info(`test name: ${testCase.testName}`); - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - try { - sender.table(testCase.table); - for (const symbol of testCase.symbols) { - sender.symbol(symbol.name, symbol.value); - } - for (const column of testCase.columns) { - switch (column.type) { - case "STRING": - sender.stringColumn(column.name, column.value); - break; - case "LONG": - sender.intColumn(column.name, column.value); - break; - case "DOUBLE": - sender.floatColumn(column.name, column.value); - break; - case "BOOLEAN": - sender.booleanColumn(column.name, column.value); - break; - case "TIMESTAMP": - sender.timestampColumn(column.name, column.value); - break; - default: - fail("Unsupported column type"); - } - } - await sender.atNow(); - } catch (e) { - if (testCase.result.status !== "ERROR") { - fail("Did not expect error: " + e.message); - } - await sender.close(); - continue; - } - - const buffer = sender.toBufferView(); - if (testCase.result.status === "SUCCESS") { - if (testCase.result.line) { - expect(buffer.toString()).toBe(testCase.result.line + "\n"); - } else { - for (const line of testCase.result.anyLines) { - if (buffer.toString() === line + "\n") { - // test passed - await sender.close(); - continue loopTestCase; - } - } - fail("Line is not matching any of the expected results: " + buffer.toString()); - } - } else { - fail("Expected error missing, instead we have a line: " + buffer.toString()); - } - - await sender.close(); - } - }); -}); - -describe("Sender message builder test suite (anything not covered in client interop test suite)", function () { - it("throws on invalid timestamp unit", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - try { - await sender - .table("tableName") - .booleanColumn("boolCol", true) - // @ts-expect-error - Testing invalid options - .timestampColumn("timestampCol", 1658484765000000, "foobar") - .atNow(); - fail("Expected error is not thrown"); - } catch (err) { - expect(err.message).toBe("Unknown timestamp unit: foobar"); - } - await sender.close(); - }); - - it("supports json object", async function () { - const pages: Array<{ - id: string; - gridId: string; - }>[] = []; - for (let i = 0; i < 4; i++) { - const pageProducts: Array<{ - id: string; - gridId: string; - }> = [ - { - id: "46022e96-076f-457f-b630-51b82b871618" + i, - gridId: "46022e96-076f-457f-b630-51b82b871618", - }, - { - id: "55615358-4af1-4179-9153-faaa57d71e55", - gridId: "55615358-4af1-4179-9153-faaa57d71e55", - }, - { - id: "365b9cdf-3d4e-4135-9cb0-f1a65601c840", - gridId: "365b9cdf-3d4e-4135-9cb0-f1a65601c840", - }, - { - id: "0b67ddf2-8e69-4482-bf0c-bb987ee5c280", - gridId: "0b67ddf2-8e69-4482-bf0c-bb987ee5c280" + i, - }, - ]; - pages.push(pageProducts); - } - - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 256, - }); - for (const p of pages) { - await sender - .table("tableName") - .stringColumn("page_products", JSON.stringify(p || [])) - .booleanColumn("boolCol", true) - .atNow(); - } - expect(sender.toBufferView().toString()).toBe( - 'tableName page_products="[{\\"id\\":\\"46022e96-076f-457f-b630-51b82b8716180\\",\\"gridId\\":\\"46022e96-076f-457f-b630-51b82b871618\\"},{\\"id\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\",\\"gridId\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\"},{\\"id\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\",\\"gridId\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\"},{\\"id\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c280\\",\\"gridId\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c2800\\"}]",boolCol=t\n' + - 'tableName page_products="[{\\"id\\":\\"46022e96-076f-457f-b630-51b82b8716181\\",\\"gridId\\":\\"46022e96-076f-457f-b630-51b82b871618\\"},{\\"id\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\",\\"gridId\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\"},{\\"id\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\",\\"gridId\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\"},{\\"id\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c280\\",\\"gridId\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c2801\\"}]",boolCol=t\n' + - 'tableName page_products="[{\\"id\\":\\"46022e96-076f-457f-b630-51b82b8716182\\",\\"gridId\\":\\"46022e96-076f-457f-b630-51b82b871618\\"},{\\"id\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\",\\"gridId\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\"},{\\"id\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\",\\"gridId\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\"},{\\"id\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c280\\",\\"gridId\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c2802\\"}]",boolCol=t\n' + - 'tableName page_products="[{\\"id\\":\\"46022e96-076f-457f-b630-51b82b8716183\\",\\"gridId\\":\\"46022e96-076f-457f-b630-51b82b871618\\"},{\\"id\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\",\\"gridId\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\"},{\\"id\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\",\\"gridId\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\"},{\\"id\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c280\\",\\"gridId\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c2803\\"}]",boolCol=t\n', - ); - await sender.close(); - }); - - it("supports timestamp field as number", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - await sender - .table("tableName") - .booleanColumn("boolCol", true) - .timestampColumn("timestampCol", 1658484765000000) - .atNow(); - expect(sender.toBufferView().toString()).toBe( - "tableName boolCol=t,timestampCol=1658484765000000t\n", - ); - await sender.close(); - }); - - it("supports timestamp field as ns number", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - await sender - .table("tableName") - .booleanColumn("boolCol", true) - .timestampColumn("timestampCol", 1658484765000000, "ns") - .atNow(); - expect(sender.toBufferView().toString()).toBe( - "tableName boolCol=t,timestampCol=1658484765000t\n", - ); - await sender.close(); - }); - - it("supports timestamp field as us number", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - await sender - .table("tableName") - .booleanColumn("boolCol", true) - .timestampColumn("timestampCol", 1658484765000000, "us") - .atNow(); - expect(sender.toBufferView().toString()).toBe( - "tableName boolCol=t,timestampCol=1658484765000000t\n", - ); - await sender.close(); - }); - - it("supports timestamp field as ms number", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - await sender - .table("tableName") - .booleanColumn("boolCol", true) - .timestampColumn("timestampCol", 1658484765000, "ms") - .atNow(); - expect(sender.toBufferView().toString()).toBe( - "tableName boolCol=t,timestampCol=1658484765000000t\n", - ); - await sender.close(); - }); - - it("supports timestamp field as BigInt", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - await sender - .table("tableName") - .booleanColumn("boolCol", true) - .timestampColumn("timestampCol", 1658484765000000n) - .atNow(); - expect(sender.toBufferView().toString()).toBe( - "tableName boolCol=t,timestampCol=1658484765000000t\n", - ); - await sender.close(); - }); - - it("supports timestamp field as ns BigInt", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - await sender - .table("tableName") - .booleanColumn("boolCol", true) - .timestampColumn("timestampCol", 1658484765000000000n, "ns") - .atNow(); - expect(sender.toBufferView().toString()).toBe( - "tableName boolCol=t,timestampCol=1658484765000000t\n", - ); - await sender.close(); - }); - - it("supports timestamp field as us BigInt", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - await sender - .table("tableName") - .booleanColumn("boolCol", true) - .timestampColumn("timestampCol", 1658484765000000n, "us") - .atNow(); - expect(sender.toBufferView().toString()).toBe( - "tableName boolCol=t,timestampCol=1658484765000000t\n", - ); - await sender.close(); - }); - - it("supports timestamp field as ms BigInt", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - await sender - .table("tableName") - .booleanColumn("boolCol", true) - .timestampColumn("timestampCol", 1658484765000n, "ms") - .atNow(); - expect(sender.toBufferView().toString()).toBe( - "tableName boolCol=t,timestampCol=1658484765000000t\n", - ); - await sender.close(); - }); - - it("throws on invalid designated timestamp unit", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - try { - await sender - .table("tableName") - .booleanColumn("boolCol", true) - .timestampColumn("timestampCol", 1658484765000000) - // @ts-expect-error - Testing invalid options - .at(1658484769000000, "foobar"); - } catch (err) { - expect(err.message).toBe("Unknown timestamp unit: foobar"); - } - await sender.close(); - }); - - it("supports setting designated us timestamp as number from client", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - await sender - .table("tableName") - .booleanColumn("boolCol", true) - .timestampColumn("timestampCol", 1658484765000000) - .at(1658484769000000, "us"); - expect(sender.toBufferView().toString()).toBe( - "tableName boolCol=t,timestampCol=1658484765000000t 1658484769000000000\n", - ); - await sender.close(); - }); - - it("supports setting designated ms timestamp as number from client", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - await sender - .table("tableName") - .booleanColumn("boolCol", true) - .timestampColumn("timestampCol", 1658484765000000) - .at(1658484769000, "ms"); - expect(sender.toBufferView().toString()).toBe( - "tableName boolCol=t,timestampCol=1658484765000000t 1658484769000000000\n", - ); - await sender.close(); - }); - - it("supports setting designated timestamp as BigInt from client", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - await sender - .table("tableName") - .booleanColumn("boolCol", true) - .timestampColumn("timestampCol", 1658484765000000) - .at(1658484769000000n); - expect(sender.toBufferView().toString()).toBe( - "tableName boolCol=t,timestampCol=1658484765000000t 1658484769000000000\n", - ); - await sender.close(); - }); - - it("supports setting designated ns timestamp as BigInt from client", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - await sender - .table("tableName") - .booleanColumn("boolCol", true) - .timestampColumn("timestampCol", 1658484765000000) - .at(1658484769000000123n, "ns"); - expect(sender.toBufferView().toString()).toBe( - "tableName boolCol=t,timestampCol=1658484765000000t 1658484769000000123\n", - ); - await sender.close(); - }); - - it("supports setting designated us timestamp as BigInt from client", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - await sender - .table("tableName") - .booleanColumn("boolCol", true) - .timestampColumn("timestampCol", 1658484765000000) - .at(1658484769000000n, "us"); - expect(sender.toBufferView().toString()).toBe( - "tableName boolCol=t,timestampCol=1658484765000000t 1658484769000000000\n", - ); - await sender.close(); - }); - - it("supports setting designated ms timestamp as BigInt from client", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - await sender - .table("tableName") - .booleanColumn("boolCol", true) - .timestampColumn("timestampCol", 1658484765000000) - .at(1658484769000n, "ms"); - expect(sender.toBufferView().toString()).toBe( - "tableName boolCol=t,timestampCol=1658484765000000t 1658484769000000000\n", - ); - await sender.close(); - }); - - it("throws exception if table name is not a string", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - // @ts-expect-error invalid options - expect(() => sender.table(23456)).toThrow( - "Table name must be a string, received number", - ); - await sender.close(); - }); - - it("throws exception if table name is too long", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - expect(() => - sender.table( - "123456789012345678901234567890123456789012345678901234567890" + - "12345678901234567890123456789012345678901234567890123456789012345678", - ), - ).toThrow("Table name is too long, max length is 127"); - await sender.close(); - }); - - it("throws exception if table name is set more times", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - expect(() => - sender.table("tableName").symbol("name", "value").table("newTableName"), - ).toThrow("Table name has already been set"); - await sender.close(); - }); - - it("throws exception if symbol name is not a string", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - // @ts-expect-error invalid options - expect(() => sender.table("tableName").symbol(12345.5656, "value")).toThrow( - "Symbol name must be a string, received number", - ); - await sender.close(); - }); - - it("throws exception if symbol name is empty string", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - expect(() => sender.table("tableName").symbol("", "value")).toThrow( - "Empty string is not allowed as column name", - ); - await sender.close(); - }); - - it("throws exception if column name is not a string", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - expect(() => - // @ts-expect-error invalid options - sender.table("tableName").stringColumn(12345.5656, "value"), - ).toThrow("Column name must be a string, received number"); - await sender.close(); - }); - - it("throws exception if column name is empty string", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - expect(() => sender.table("tableName").stringColumn("", "value")).toThrow( - "Empty string is not allowed as column name", - ); - await sender.close(); - }); - - it("throws exception if column name is too long", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - expect(() => - sender - .table("tableName") - .stringColumn( - "123456789012345678901234567890123456789012345678901234567890" + - "12345678901234567890123456789012345678901234567890123456789012345678", - "value", - ), - ).toThrow("Column name is too long, max length is 127"); - await sender.close(); - }); - - it("throws exception if column value is not the right type", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - expect(() => - // @ts-expect-error invalid options - sender.table("tableName").stringColumn("columnName", false), - ).toThrow("Column value must be of type string, received boolean"); - await sender.close(); - }); - - it("throws exception if adding column without setting table name", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - expect(() => sender.floatColumn("name", 12.459)).toThrow( - "Column can be set only after table name is set", - ); - await sender.close(); - }); - - it("throws exception if adding symbol without setting table name", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - expect(() => sender.symbol("name", "value")).toThrow( - "Symbol can be added only after table name is set and before any column added", - ); - await sender.close(); - }); - - it("throws exception if adding symbol after columns", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - expect(() => - sender - .table("tableName") - .stringColumn("name", "value") - .symbol("symbolName", "symbolValue"), - ).toThrow( - "Symbol can be added only after table name is set and before any column added", - ); - await sender.close(); - }); - - it("returns null if preparing an empty buffer for send", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - expect(sender.toBufferView()).toBe(null); - expect(sender.toBufferNew()).toBe(null); - await sender.close(); - }); - - it("leaves unfinished rows in the sender's buffer when preparing a copy of the buffer for send", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - sender.table("tableName").symbol("name", "value"); - await sender.at(1234567890n, "ns"); - sender.table("tableName").symbol("name", "value2"); - - // copy of the sender's buffer contains the finished row - expect(sender.toBufferNew(sender.endOfLastRow).toString()).toBe( - "tableName,name=value 1234567890\n", - ); - // the sender's buffer is compacted, and contains only the unfinished row - expect(sender.toBufferView().toString()).toBe( - "tableName,name=value2", - ); - await sender.close(); - }); - - it("throws exception if a float is passed as integer field", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - expect(() => - sender.table("tableName").intColumn("intField", 123.222), - ).toThrow("Value must be an integer, received 123.222"); - await sender.close(); - }); - - it("throws exception if a float is passed as timestamp field", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - expect(() => - sender.table("tableName").timestampColumn("intField", 123.222), - ).toThrow("Value must be an integer or BigInt, received 123.222"); - await sender.close(); - }); - - it("throws exception if designated timestamp is not an integer or bigint", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - try { - await sender - .table("tableName") - .symbol("name", "value") - .at(23232322323.05); - } catch (e) { - expect(e.message).toEqual( - "Designated timestamp must be an integer or BigInt, received 23232322323.05", - ); - } - await sender.close(); - }); - - it("throws exception if designated timestamp is invalid", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - try { - // @ts-expect-error invalid options - await sender.table("tableName").symbol("name", "value").at("invalid_dts"); - } catch (e) { - expect(e.message).toEqual( - "Designated timestamp must be an integer or BigInt, received invalid_dts", - ); - } - await sender.close(); - }); - - it("throws exception if designated timestamp is set without any fields added", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - try { - await sender.table("tableName").at(12345678n, "ns"); - } catch (e) { - expect(e.message).toEqual( - "The row must have a symbol or column set before it is closed", - ); - } - await sender.close(); - }); - - it("extends the size of the buffer if data does not fit", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 8, - }); - expect(sender.bufferSize).toBe(8); - expect(sender.position).toBe(0); - sender.table("tableName"); - expect(sender.bufferSize).toBe(16); - expect(sender.position).toBe("tableName".length); - sender.intColumn("intField", 123); - expect(sender.bufferSize).toBe(32); - expect(sender.position).toBe("tableName intField=123i".length); - await sender.atNow(); - expect(sender.bufferSize).toBe(32); - expect(sender.position).toBe("tableName intField=123i\n".length); - expect(sender.toBufferView().toString()).toBe("tableName intField=123i\n"); - - await sender - .table("table2") - .intColumn("intField", 125) - .stringColumn("strField", "test") - .atNow(); - expect(sender.bufferSize).toBe(64); - expect(sender.position).toBe( - 'tableName intField=123i\ntable2 intField=125i,strField="test"\n'.length, - ); - expect(sender.toBufferView().toString()).toBe( - 'tableName intField=123i\ntable2 intField=125i,strField="test"\n', - ); - await sender.close(); - }); - - it("throws exception if tries to extend the size of the buffer above max buffer size", async function () { - const sender = Sender.fromConfig( - "tcp::addr=host;init_buf_size=8;max_buf_size=48;", - ); - expect(sender.bufferSize).toBe(8); - expect(sender.position).toBe(0); - sender.table("tableName"); - expect(sender.bufferSize).toBe(16); - expect(sender.position).toBe("tableName".length); - sender.intColumn("intField", 123); - expect(sender.bufferSize).toBe(32); - expect(sender.position).toBe("tableName intField=123i".length); - await sender.atNow(); - expect(sender.bufferSize).toBe(32); - expect(sender.position).toBe("tableName intField=123i\n".length); - expect(sender.toBufferView().toString()).toBe("tableName intField=123i\n"); - - try { - await sender - .table("table2") - .intColumn("intField", 125) - .stringColumn("strField", "test") - .atNow(); - } catch (err) { - expect(err.message).toBe( - "Max buffer size is 48 bytes, requested buffer size: 64", - ); - } - await sender.close(); - }); - - it("is possible to clear the buffer by calling reset()", async function () { - const sender = new Sender({ - protocol: "tcp", - host: "host", - init_buf_size: 1024, - }); - await sender - .table("tableName") - .booleanColumn("boolCol", true) - .timestampColumn("timestampCol", 1658484765000000) - .atNow(); - await sender - .table("tableName") - .booleanColumn("boolCol", false) - .timestampColumn("timestampCol", 1658484766000000) - .atNow(); - expect(sender.toBufferView().toString()).toBe( - "tableName boolCol=t,timestampCol=1658484765000000t\n" + - "tableName boolCol=f,timestampCol=1658484766000000t\n", - ); - - sender.reset(); - await sender - .table("tableName") - .floatColumn("floatCol", 1234567890) - .timestampColumn("timestampCol", 1658484767000000) - .atNow(); - expect(sender.toBufferView().toString()).toBe( - "tableName floatCol=1234567890,timestampCol=1658484767000000t\n", - ); - await sender.close(); - }); -}); - -describe("Sender tests with containerized QuestDB instance", () => { - let container: any; - - async function query(container: any, query: string) { - const options = { - hostname: container.getHost(), - port: container.getMappedPort(QUESTDB_HTTP_PORT), - path: `/exec?query=${encodeURIComponent(query)}`, - method: "GET", - }; - - return new Promise((resolve, reject) => { - const req = http.request(options, (response) => { - if (response.statusCode === HTTP_OK) { - const body: Uint8Array[] = []; - response - .on("data", (data: Uint8Array) => { - body.push(data); - }) - .on("end", () => { - resolve(JSON.parse(Buffer.concat(body).toString())); - }); - } else { - reject( - new Error( - `HTTP request failed, statusCode=${response.statusCode}, query=${query}`, - ), - ); - } - }); - - req.on("error", (error) => { - reject(error); - }); - - req.end(); - }); - } - - async function runSelect(container: any, select: string, expectedCount: number, timeout = 60000) { - const interval = 500; - const num = timeout / interval; - let selectResult: any; - for (let i = 0; i < num; i++) { - selectResult = await query(container, select); - if (selectResult && selectResult.count >= expectedCount) { - return selectResult; - } - await sleep(interval); - } - throw new Error( - `Timed out while waiting for ${expectedCount} rows, select='${select}'`, - ); - } - - async function waitForTable(container: any, tableName: string, timeout = 30000) { - await runSelect(container, `tables() where table_name='${tableName}'`, 1, timeout); - } - - function getFieldsString(schema: any) { - let fields = ""; - for (const element of schema) { - fields += `${element.name} ${element.type}, `; - } - return fields.substring(0, fields.length - 2); - } - - beforeAll(async () => { - container = await new GenericContainer("questdb/questdb:nightly") - .withExposedPorts(QUESTDB_HTTP_PORT, QUESTDB_ILP_PORT) - .start(); - - const stream = await container.logs(); - stream - .on("data", (line: string) => console.log(line)) - .on("err", (line: string) => console.error(line)) - .on("end", () => console.log("Stream closed")); - }, 3000000); - - afterAll(async () => { - await container.stop(); - }); - - it("can ingest data via TCP and run queries", async () => { - const sender = new Sender({ - protocol: "tcp", - host: container.getHost(), - port: container.getMappedPort(QUESTDB_ILP_PORT), - }); - await sender.connect(); - - const tableName = "test_tcp"; - const schema = [ - { name: "location", type: "SYMBOL" }, - { name: "temperature", type: "DOUBLE" }, - { name: "timestamp", type: "TIMESTAMP" }, - ]; - - // ingest via client - await sender - .table(tableName) - .symbol("location", "us") - .floatColumn("temperature", 17.1) - .at(1658484765000000000n, "ns"); - await sender.flush(); - - // wait for the table - await waitForTable(container, tableName) - - // query table - const select1Result = await runSelect(container, tableName, 1); - expect(select1Result.query).toBe(tableName); - expect(select1Result.count).toBe(1); - expect(select1Result.columns).toStrictEqual(schema); - expect(select1Result.dataset).toStrictEqual([ - ["us", 17.1, "2022-07-22T10:12:45.000000Z"], - ]); - - // ingest via client, add new column - await sender - .table(tableName) - .symbol("location", "us") - .floatColumn("temperature", 17.3) - .at(1658484765000666000n, "ns"); - await sender - .table(tableName) - .symbol("location", "emea") - .floatColumn("temperature", 17.4) - .at(1658484765000999000n, "ns"); - await sender - .table(tableName) - .symbol("location", "emea") - .symbol("city", "london") - .floatColumn("temperature", 18.8) - .at(1658484765001234000n, "ns"); - await sender.flush(); - - // query table - const select2Result = await runSelect(container, tableName, 4); - expect(select2Result.query).toBe(tableName); - expect(select2Result.count).toBe(4); - expect(select2Result.columns).toStrictEqual([ - { name: "location", type: "SYMBOL" }, - { name: "temperature", type: "DOUBLE" }, - { name: "timestamp", type: "TIMESTAMP" }, - { name: "city", type: "SYMBOL" }, - ]); - expect(select2Result.dataset).toStrictEqual([ - ["us", 17.1, "2022-07-22T10:12:45.000000Z", null], - ["us", 17.3, "2022-07-22T10:12:45.000666Z", null], - ["emea", 17.4, "2022-07-22T10:12:45.000999Z", null], - ["emea", 18.8, "2022-07-22T10:12:45.001234Z", "london"], - ]); - - await sender.close(); - }); - - it("can ingest data via HTTP with auto flush rows", async () => { - const tableName = "test_http_rows"; - const schema = [ - { name: "location", type: "SYMBOL" }, - { name: "temperature", type: "DOUBLE" }, - { name: "timestamp", type: "TIMESTAMP" }, - ]; - - const sender = Sender.fromConfig( - `http::addr=${container.getHost()}:${container.getMappedPort(QUESTDB_HTTP_PORT)};auto_flush_interval=0;auto_flush_rows=1`, - ); - - // ingest via client - await sender - .table(tableName) - .symbol("location", "us") - .floatColumn("temperature", 17.1) - .at(1658484765000000000n, "ns"); - - // wait for the table - await waitForTable(container, tableName) - - // query table - const select1Result = await runSelect(container, tableName, 1); - expect(select1Result.query).toBe(tableName); - expect(select1Result.count).toBe(1); - expect(select1Result.columns).toStrictEqual(schema); - expect(select1Result.dataset).toStrictEqual([ - ["us", 17.1, "2022-07-22T10:12:45.000000Z"], - ]); - - // ingest via client, add new column - await sender - .table(tableName) - .symbol("location", "us") - .floatColumn("temperature", 17.36) - .at(1658484765000666000n, "ns"); - await sender - .table(tableName) - .symbol("location", "emea") - .floatColumn("temperature", 17.41) - .at(1658484765000999000n, "ns"); - await sender - .table(tableName) - .symbol("location", "emea") - .symbol("city", "london") - .floatColumn("temperature", 18.81) - .at(1658484765001234000n, "ns"); - - // query table - const select2Result = await runSelect(container, tableName, 4); - expect(select2Result.query).toBe(tableName); - expect(select2Result.count).toBe(4); - expect(select2Result.columns).toStrictEqual([ - { name: "location", type: "SYMBOL" }, - { name: "temperature", type: "DOUBLE" }, - { name: "timestamp", type: "TIMESTAMP" }, - { name: "city", type: "SYMBOL" }, - ]); - expect(select2Result.dataset).toStrictEqual([ - ["us", 17.1, "2022-07-22T10:12:45.000000Z", null], - ["us", 17.36, "2022-07-22T10:12:45.000666Z", null], - ["emea", 17.41, "2022-07-22T10:12:45.000999Z", null], - ["emea", 18.81, "2022-07-22T10:12:45.001234Z", "london"], - ]); - - await sender.close(); - }); - - it("can ingest data via HTTP with auto flush interval", async () => { - const tableName = "test_http_interval"; - const schema = [ - { name: "location", type: "SYMBOL" }, - { name: "temperature", type: "DOUBLE" }, - { name: "timestamp", type: "TIMESTAMP" }, - ]; - - const sender = Sender.fromConfig( - `http::addr=${container.getHost()}:${container.getMappedPort(QUESTDB_HTTP_PORT)};auto_flush_interval=1;auto_flush_rows=0`, - ); - - // wait longer than the set auto flush interval to make sure there is a flush - await sleep(10); - - // ingest via client - await sender - .table(tableName) - .symbol("location", "us") - .floatColumn("temperature", 17.1) - .at(1658484765000000000n, "ns"); - - // wait for the table - await waitForTable(container, tableName) - - // query table - const select1Result = await runSelect(container, tableName, 1); - expect(select1Result.query).toBe(tableName); - expect(select1Result.count).toBe(1); - expect(select1Result.columns).toStrictEqual(schema); - expect(select1Result.dataset).toStrictEqual([ - ["us", 17.1, "2022-07-22T10:12:45.000000Z"], - ]); - - // ingest via client, add new column - await sleep(10); - await sender - .table(tableName) - .symbol("location", "us") - .floatColumn("temperature", 17.36) - .at(1658484765000666000n, "ns"); - await sleep(10); - await sender - .table(tableName) - .symbol("location", "emea") - .floatColumn("temperature", 17.41) - .at(1658484765000999000n, "ns"); - await sleep(10); - await sender - .table(tableName) - .symbol("location", "emea") - .symbol("city", "london") - .floatColumn("temperature", 18.81) - .at(1658484765001234000n, "ns"); - - // query table - const select2Result = await runSelect(container, tableName, 4); - expect(select2Result.query).toBe(tableName); - expect(select2Result.count).toBe(4); - expect(select2Result.columns).toStrictEqual([ - { name: "location", type: "SYMBOL" }, - { name: "temperature", type: "DOUBLE" }, - { name: "timestamp", type: "TIMESTAMP" }, - { name: "city", type: "SYMBOL" }, - ]); - expect(select2Result.dataset).toStrictEqual([ - ["us", 17.1, "2022-07-22T10:12:45.000000Z", null], - ["us", 17.36, "2022-07-22T10:12:45.000666Z", null], - ["emea", 17.41, "2022-07-22T10:12:45.000999Z", null], - ["emea", 18.81, "2022-07-22T10:12:45.001234Z", "london"], - ]); - - await sender.close(); - }); - - it("does not duplicate rows if await is missing when calling flush", async () => { - // setting copyBuffer to make sure promises send data from their own local buffer - const sender = new Sender({ - protocol: "tcp", - host: container.getHost(), - port: container.getMappedPort(QUESTDB_ILP_PORT), - }); - await sender.connect(); - - const tableName = "test2"; - const schema = [ - { name: "location", type: "SYMBOL" }, - { name: "temperature", type: "DOUBLE" }, - { name: "timestamp", type: "TIMESTAMP" }, - ]; - - // ingest via client - const numOfRows = 100; - for (let i = 0; i < numOfRows; i++) { - await sender - .table(tableName) - .symbol("location", "us") - .floatColumn("temperature", i) - .at(1658484765000000000n, "ns"); - // missing await is intentional - await sender.flush(); - } - - // wait for the table - await waitForTable(container, tableName) - - // query table - const selectQuery = `${tableName} order by temperature`; - const selectResult = await runSelect(container, selectQuery, numOfRows); - expect(selectResult.query).toBe(selectQuery); - expect(selectResult.count).toBe(numOfRows); - expect(selectResult.columns).toStrictEqual(schema); - - const expectedData: (string | number)[][] = []; - for (let i = 0; i < numOfRows; i++) { - expectedData.push(["us", i, "2022-07-22T10:12:45.000000Z"]); - } - expect(selectResult.dataset).toStrictEqual(expectedData); - - await sender.close(); - }); - - it("ingests all data without loss under high load with auto-flush", async () => { - const sender = Sender.fromConfig( - `tcp::addr=${container.getHost()}:${container.getMappedPort(QUESTDB_ILP_PORT)};auto_flush_rows=5;auto_flush_interval=1`, - ); - await sender.connect(); - - const tableName = "test_high_load_autoflush"; - const numOfRows = 1000; - const promises: Promise[] = []; - - for (let i = 0; i < numOfRows; i++) { - // Not awaiting each .at() call individually to allow them to queue up - const p = sender - .table(tableName) - .intColumn("id", i) - .at(1658484765000000000n + BigInt(1000 * i), "ns"); // Unique timestamp for each row - promises.push(p); - } - - // Wait for all .at() calls to complete their processing (including triggering auto-flushes) - await Promise.all(promises); - - // Perform a final flush to ensure any data remaining in the buffer is sent. - // This will be queued correctly after any ongoing auto-flushes. - await sender.flush(); - - // Wait for the table - await waitForTable(container, tableName) - - // Query table and verify count - const selectQuery = `SELECT id FROM ${tableName}`; - const selectResult = await runSelect(container, selectQuery, numOfRows); - expect(selectResult.count).toBe(numOfRows); - - // Verify data integrity - for (let i = 0; i < numOfRows; i++) { - expect(selectResult.dataset[i][0]).toBe(i); - } - - await sender.close(); - }, 30000); // Increased test timeout for this specific test -}); diff --git a/test/sender.transport.test.ts b/test/sender.transport.test.ts new file mode 100644 index 0000000..3b32fcd --- /dev/null +++ b/test/sender.transport.test.ts @@ -0,0 +1,568 @@ +// @ts-check +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { readFileSync } from "fs"; +import { Agent } from "undici"; +import http from "http"; + +import { Sender } from "../src"; +import { SenderOptions } from "../src/options"; +import { UndiciTransport } from "../src/transport/http/undici"; +import { HttpTransport } from "../src/transport/http/legacy"; +import { MockProxy } from "./util/mockproxy"; +import { MockHttp } from "./util/mockhttp"; + +const MOCK_HTTP_PORT = 9099; +const MOCK_HTTPS_PORT = 9098; +const PROXY_PORT = 9088; +const PROXY_HOST = "localhost"; + +const proxyOptions = { + key: readFileSync("test/certs/server/server.key"), + cert: readFileSync("test/certs/server/server.crt"), + ca: readFileSync("test/certs/ca/ca.crt"), +}; + +const USER_NAME = "testapp"; +const PRIVATE_KEY = "9b9x5WhJywDEuo1KGQWSPNxtX-6X6R2BRCKhYMMY6n8"; +const AUTH: SenderOptions["auth"] = { + keyId: USER_NAME, + token: PRIVATE_KEY, +}; + +async function sleep(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +describe("Sender HTTP suite", function () { + async function sendData(sender: Sender) { + await sender + .table("test") + .symbol("location", "us") + .floatColumn("temperature", 17.1) + .at(1658484765000000000n, "ns"); + await sender.flush(); + } + + const mockHttp = new MockHttp(); + const mockHttps = new MockHttp(); + + beforeAll(async function () { + await mockHttp.start(MOCK_HTTP_PORT); + await mockHttps.start(MOCK_HTTPS_PORT, true, proxyOptions); + }); + + afterAll(async function () { + await mockHttp.stop(); + await mockHttps.stop(); + }); + + it("can ingest via HTTP", async function () { + mockHttp.reset(); + + const sender = Sender.fromConfig( + `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT}`, + ); + await sendData(sender); + expect(mockHttp.numOfRequests).toBe(1); + + await sender.close(); + }); + + it("can ingest via HTTPS", async function () { + mockHttps.reset(); + + const senderCertCheckFail = Sender.fromConfig( + `https::addr=${PROXY_HOST}:${MOCK_HTTPS_PORT}`, + ); + await expect(sendData(senderCertCheckFail)).rejects.toThrowError( + "self-signed certificate in certificate chain", + ); + await senderCertCheckFail.close(); + + const senderWithCA = Sender.fromConfig( + `https::addr=${PROXY_HOST}:${MOCK_HTTPS_PORT};tls_ca=test/certs/ca/ca.crt`, + ); + await sendData(senderWithCA); + expect(mockHttps.numOfRequests).toEqual(1); + await senderWithCA.close(); + + const senderVerifyOff = Sender.fromConfig( + `https::addr=${PROXY_HOST}:${MOCK_HTTPS_PORT};tls_verify=unsafe_off`, + ); + await sendData(senderVerifyOff); + expect(mockHttps.numOfRequests).toEqual(2); + await senderVerifyOff.close(); + }, 20000); + + it("can ingest via HTTP with basic auth", async function () { + mockHttp.reset({ username: "user1", password: "pwd" }); + + const sender = Sender.fromConfig( + `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};username=user1;password=pwd`, + ); + await sendData(sender); + expect(mockHttp.numOfRequests).toEqual(1); + await sender.close(); + + const senderFailPwd = Sender.fromConfig( + `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};username=user1;password=xyz`, + ); + await expect(sendData(senderFailPwd)).rejects.toThrowError( + "HTTP request failed, statusCode=401", + ); + await senderFailPwd.close(); + + const senderFailMissingPwd = Sender.fromConfig( + `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};username=user1z`, + ); + await expect(sendData(senderFailMissingPwd)).rejects.toThrowError( + "HTTP request failed, statusCode=401", + ); + await senderFailMissingPwd.close(); + + const senderFailUsername = Sender.fromConfig( + `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};username=xyz;password=pwd`, + ); + await expect(sendData(senderFailUsername)).rejects.toThrowError( + "HTTP request failed, statusCode=401", + ); + await senderFailUsername.close(); + + const senderFailMissingUsername = Sender.fromConfig( + `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};password=pwd`, + ); + await expect(sendData(senderFailMissingUsername)).rejects.toThrowError( + "HTTP request failed, statusCode=401", + ); + await senderFailMissingUsername.close(); + + const senderFailMissing = Sender.fromConfig( + `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT}`, + ); + await expect(sendData(senderFailMissing)).rejects.toThrowError( + "HTTP request failed, statusCode=401", + ); + await senderFailMissing.close(); + }); + + it("can ingest via HTTP with token auth", async function () { + mockHttp.reset({ token: "abcdefghijkl123" }); + + const sender = Sender.fromConfig( + `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};token=abcdefghijkl123`, + ); + await sendData(sender); + expect(mockHttp.numOfRequests).toBe(1); + await sender.close(); + + const senderFailToken = Sender.fromConfig( + `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};token=xyz`, + ); + await expect(sendData(senderFailToken)).rejects.toThrowError( + "HTTP request failed, statusCode=401", + ); + await senderFailToken.close(); + + const senderFailMissing = Sender.fromConfig( + `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT}`, + ); + await expect(sendData(senderFailMissing)).rejects.toThrowError( + "HTTP request failed, statusCode=401", + ); + await senderFailMissing.close(); + }); + + it("can retry via HTTP", async function () { + mockHttp.reset({ responseCodes: [204, 500, 523, 504, 500] }); + + const sender = Sender.fromConfig( + `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT}`, + ); + await sendData(sender); + expect(mockHttp.numOfRequests).toBe(5); + + await sender.close(); + }); + + it("fails when retry timeout expires", async function () { + // artificial delay (responseDelays) is the same as retry timeout, + // this should result in the request failing on the second try. + mockHttp.reset({ + responseCodes: [204, 500, 503], + responseDelays: [1000, 1000, 1000], + }); + + const sender = Sender.fromConfig( + `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};retry_timeout=1000`, + ); + await expect(sendData(sender)).rejects.toThrowError( + "HTTP request timeout, no response from server in time" + ); + await sender.close(); + }); + + it("fails when HTTP request times out", async function () { + // artificial delay (responseDelays) is greater than request timeout, and retry is switched off + // should result in the request failing with timeout + mockHttp.reset({ + responseCodes: [204], + responseDelays: [1000], + }); + + const sender = Sender.fromConfig( + `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};retry_timeout=0;request_timeout=100`, + ); + await expect(sendData(sender)).rejects.toThrowError( + "HTTP request timeout, no response from server in time", + ); + await sender.close(); + }); + + it("succeeds on the third request after two timeouts", async function () { + mockHttp.reset({ + responseCodes: [204, 504, 504], + responseDelays: [2000, 2000], + }); + + const sender = Sender.fromConfig( + `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};retry_timeout=600000;request_timeout=1000`, + ); + await sendData(sender); + + await sender.close(); + }); + + it("multiple senders can use a single HTTP agent", async function () { + mockHttp.reset(); + const agent = new Agent({ connect: { keepAlive: false } }); + + const num = 300; + const senders: Sender[] = []; + const promises: Promise[] = []; + for (let i = 0; i < num; i++) { + const sender = Sender.fromConfig( + `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT}`, + { agent: agent }, + ); + senders.push(sender); + const promise = sendData(sender); + promises.push(promise); + } + await Promise.all(promises); + expect(mockHttp.numOfRequests).toBe(num); + + for (const sender of senders) { + await sender.close(); + } + await agent.destroy(); + }); + + it("supports custom Undici HTTP agent", async function () { + mockHttp.reset(); + const agent = new Agent({ pipelining: 3 }); + + const sender = Sender.fromConfig( + `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT}`, + { agent: agent }, + ); + + await sendData(sender); + expect(mockHttp.numOfRequests).toBe(1); + + // @ts-expect-error - Accessing private field + const senderAgent = (sender.transport as UndiciTransport).agent; + const symbols = Object.getOwnPropertySymbols(senderAgent); + expect(senderAgent[symbols[6]]).toEqual({ pipelining: 3 }); + + await sender.close(); + await agent.destroy(); + }); + + it('supports custom legacy HTTP agent', async function () { + mockHttp.reset(); + const agent = new http.Agent({ maxSockets: 128 }); + + const sender = Sender.fromConfig( + `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};legacy_http=on`, + { agent: agent }, + ); + await sendData(sender); + expect(mockHttp.numOfRequests).toBe(1); + + // @ts-expect-error - Accessing private field + const senderAgent = (sender.transport as HttpTransport).agent; + expect(senderAgent.maxSockets).toBe(128); + + await sender.close(); + agent.destroy(); + }); +}); + +describe("Sender TCP suite", function () { + async function createProxy( + auth = false, + tlsOptions?: Record, + ) { + const mockConfig = { auth: auth, assertions: true }; + const proxy = new MockProxy(mockConfig); + await proxy.start(PROXY_PORT, tlsOptions); + expect(proxy.mockConfig).toBe(mockConfig); + expect(proxy.dataSentToRemote).toStrictEqual([]); + return proxy; + } + + async function createSender(auth: SenderOptions["auth"], secure = false) { + const sender = new Sender({ + protocol: secure ? "tcps" : "tcp", + port: PROXY_PORT, + host: PROXY_HOST, + auth: auth, + tls_ca: "test/certs/ca/ca.crt", + }); + const connected = await sender.connect(); + expect(connected).toBe(true); + return sender; + } + + async function sendData(sender: Sender) { + await sender + .table("test") + .symbol("location", "us") + .floatColumn("temperature", 17.1) + .at(1658484765000000000n, "ns"); + await sender.flush(); + } + + async function assertSentData( + proxy: MockProxy, + authenticated: boolean, + expected: string, + timeout = 60000, + ) { + const interval = 100; + const num = timeout / interval; + let actual: string; + for (let i = 0; i < num; i++) { + const dataSentToRemote = proxy.getDataSentToRemote().join("").split("\n"); + if (authenticated) { + dataSentToRemote.splice(1, 1); + } + actual = dataSentToRemote.join("\n"); + if (actual === expected) { + return new Promise((resolve) => resolve(null)); + } + await sleep(interval); + } + return new Promise((resolve) => + resolve(`data assert failed [expected=${expected}, actual=${actual}]`), + ); + } + + it("can authenticate", async function () { + const proxy = await createProxy(true); + const sender = await createSender(AUTH); + await assertSentData(proxy, true, "testapp\n"); + await sender.close(); + await proxy.stop(); + }); + + it("can authenticate with a different private key", async function () { + const proxy = await createProxy(true); + const sender = await createSender({ + keyId: "user1", + token: "zhPiK3BkYMYJvRf5sqyrWNJwjDKHOWHnRbmQggUll6A", + }); + await assertSentData(proxy, true, "user1\n"); + await sender.close(); + await proxy.stop(); + }); + + it("is backwards compatible and still can authenticate with full JWK", async function () { + const JWK = { + x: "BtUXC_K3oAyGlsuPjTgkiwirMUJhuRQDfcUHeyoxFxU", + y: "R8SOup-rrNofB7wJagy4HrJhTVfrVKmj061lNRk3bF8", + kid: "user2", + kty: "EC", + d: "hsg6Zm4kSBlIEvKUWT3kif-2y2Wxw-iWaGrJxrPXQhs", + crv: "P-256", + }; + + const proxy = await createProxy(true); + const sender = new Sender({ + protocol: "tcp", + port: PROXY_PORT, + host: PROXY_HOST, + jwk: JWK, + }); + const connected = await sender.connect(); + expect(connected).toBe(true); + await assertSentData(proxy, true, "user2\n"); + await sender.close(); + await proxy.stop(); + }); + + it("can connect unauthenticated", async function () { + const proxy = await createProxy(); + // @ts-expect-error - Invalid options + const sender = await createSender(); + await assertSentData(proxy, false, ""); + await sender.close(); + await proxy.stop(); + }); + + it("can authenticate and send data to server", async function () { + const proxy = await createProxy(true); + const sender = await createSender(AUTH); + await sendData(sender); + await assertSentData( + proxy, + true, + "testapp\ntest,location=us temperature=17.1 1658484765000000000\n", + ); + await sender.close(); + await proxy.stop(); + }); + + it("can connect unauthenticated and send data to server", async function () { + const proxy = await createProxy(); + // @ts-expect-error - Invalid options + const sender = await createSender(); + await sendData(sender); + await assertSentData( + proxy, + false, + "test,location=us temperature=17.1 1658484765000000000\n", + ); + await sender.close(); + await proxy.stop(); + }); + + it("can authenticate and send data to server via secure connection", async function () { + const proxy = await createProxy(true, proxyOptions); + const sender = await createSender(AUTH, true); + await sendData(sender); + await assertSentData( + proxy, + true, + "testapp\ntest,location=us temperature=17.1 1658484765000000000\n", + ); + await sender.close(); + await proxy.stop(); + }); + + it("can connect unauthenticated and send data to server via secure connection", async function () { + const proxy = await createProxy(false, proxyOptions); + const sender = await createSender(null, true); + await sendData(sender); + await assertSentData( + proxy, + false, + "test,location=us temperature=17.1 1658484765000000000\n", + ); + await sender.close(); + await proxy.stop(); + }); + + it("fails to connect without hostname and port", async function () { + await expect(async () => + await new Sender({ protocol: "tcp" }).close() + ).rejects.toThrow("The 'host' option is mandatory"); + }); + + it("fails to send data if not connected", async function () { + const sender = new Sender({ protocol: "tcp", host: "localhost" }); + await expect(async () => { + await sender.table("test").symbol("location", "us").atNow(); + await sender.flush(); + }).rejects.toThrow("TCP transport is not connected"); + await sender.close(); + }); + + it("guards against multiple connect calls", async function () { + const proxy = await createProxy(true, proxyOptions); + const sender = await createSender(AUTH, true); + await expect(async () => + await sender.connect() + ).rejects.toThrow("Sender connected already"); + await sender.close(); + await proxy.stop(); + }); + + it("guards against concurrent connect calls", async function () { + const proxy = await createProxy(true, proxyOptions); + const sender = new Sender({ + protocol: "tcps", + port: PROXY_PORT, + host: PROXY_HOST, + auth: AUTH, + tls_ca: "test/certs/ca/ca.crt", + }); + await expect(async () => + await Promise.all([sender.connect(), sender.connect()]) + ).rejects.toThrow("Sender connected already"); + await sender.close(); + await proxy.stop(); + }); + + it("can disable the server certificate check", async function () { + const proxy = await createProxy(true, proxyOptions); + const senderCertCheckFail = Sender.fromConfig( + `tcps::addr=${PROXY_HOST}:${PROXY_PORT}`, + ); + await expect(async () => + await senderCertCheckFail.connect() + ).rejects.toThrow("self-signed certificate in certificate chain"); + await senderCertCheckFail.close(); + + const senderCertCheckOn = Sender.fromConfig( + `tcps::addr=${PROXY_HOST}:${PROXY_PORT};tls_ca=test/certs/ca/ca.crt`, + ); + await senderCertCheckOn.connect(); + await senderCertCheckOn.close(); + + const senderCertCheckOff = Sender.fromConfig( + `tcps::addr=${PROXY_HOST}:${PROXY_PORT};tls_verify=unsafe_off`, + ); + await senderCertCheckOff.connect(); + await senderCertCheckOff.close(); + await proxy.stop(); + }); + + it("can handle unfinished rows during flush()", async function () { + const proxy = await createProxy(true, proxyOptions); + const sender = await createSender(AUTH, true); + sender.table("test").symbol("location", "us"); + const sent = await sender.flush(); + expect(sent).toBe(false); + await assertSentData(proxy, true, "testapp\n"); + await sender.close(); + await proxy.stop(); + }); + + it("supports custom logger", async function () { + const expectedMessages = [ + "Successfully connected to localhost:9088", + /^Connection to .*1:9088 is closed$/, + ]; + const log = (level: "error" | "warn" | "info" | "debug", message: string) => { + expect(level).toBe("info"); + expect(message).toMatch(expectedMessages.shift()); + }; + const proxy = await createProxy(); + const sender = new Sender({ + protocol: "tcp", + port: PROXY_PORT, + host: PROXY_HOST, + log: log, + }); + await sender.connect(); + await sendData(sender); + await assertSentData( + proxy, + false, + "test,location=us temperature=17.1 1658484765000000000\n", + ); + await sender.close(); + await proxy.stop(); + }); +}); diff --git a/test/testapp.ts b/test/testapp.ts index 9aedd9c..39f1267 100644 --- a/test/testapp.ts +++ b/test/testapp.ts @@ -1,6 +1,6 @@ import { readFileSync } from "node:fs"; -import { Proxy } from "./_utils_/proxy"; +import { Proxy } from "./util/proxy"; import { Sender } from "../src"; import { SenderOptions } from "../src/options"; @@ -8,13 +8,6 @@ const PROXY_PORT = 9099; const PORT = 9009; const HOST = "localhost"; -const USER_NAME = "testapp"; -const PRIVATE_KEY = "9b9x5WhJywDEuo1KGQWSPNxtX-6X6R2BRCKhYMMY6n8"; -const AUTH = { - kid: USER_NAME, - d: PRIVATE_KEY, -}; - const senderOptions: SenderOptions = { protocol: "tcps", host: HOST, diff --git a/test/_utils_/mockhttp.ts b/test/util/mockhttp.ts similarity index 74% rename from test/_utils_/mockhttp.ts rename to test/util/mockhttp.ts index 8487bc0..2341df6 100644 --- a/test/_utils_/mockhttp.ts +++ b/test/util/mockhttp.ts @@ -1,15 +1,17 @@ import http from "node:http"; import https from "node:https"; +type MockConfig = { + responseDelays?: number[], + responseCodes?: number[], + username?: string, + password?: string, + token?: string, +} + class MockHttp { server: http.Server | https.Server; - mockConfig: { - responseDelays?: number[], - responseCodes?: number[], - username?: string, - password?: string, - token?: string, - }; + mockConfig: MockConfig; numOfRequests: number; constructor() { @@ -21,10 +23,10 @@ class MockHttp { this.numOfRequests = 0; } - async start(listenPort: number, secure: boolean = false, options?: Record) { + async start(listenPort: number, secure: boolean = false, options?: Record): Promise { const serverCreator = secure ? https.createServer : http.createServer; // @ts-expect-error - Testing different options, so typing is not important - this.server = serverCreator(options, (req, res) => { + this.server = serverCreator(options, (req: http.IncomingMessage, res: http.ServerResponse) => { const authFailed = checkAuthHeader(this.mockConfig, req); const body: Uint8Array[] = []; @@ -56,8 +58,16 @@ class MockHttp { }); }); - this.server.listen(listenPort, () => { - console.info(`Server is running on port ${listenPort}`); + return new Promise((resolve, reject) => { + this.server.listen(listenPort, () => { + console.info(`Server is running on port ${listenPort}`); + resolve(true); + }); + + this.server.on("error", e => { + console.error(`server error: ${e}`); + reject(e); + }); }); } @@ -68,7 +78,7 @@ class MockHttp { } } -function checkAuthHeader(mockConfig, req) { +function checkAuthHeader(mockConfig: MockConfig, req: http.IncomingMessage) { let authFailed = false; const header = (req.headers.authorization || "").split(/\s+/); switch (header[0]) { @@ -92,7 +102,7 @@ function checkAuthHeader(mockConfig, req) { return authFailed; } -function sleep(ms) { +function sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)); } diff --git a/test/_utils_/mockproxy.ts b/test/util/mockproxy.ts similarity index 78% rename from test/_utils_/mockproxy.ts rename to test/util/mockproxy.ts index f26e404..ecc9c9c 100644 --- a/test/_utils_/mockproxy.ts +++ b/test/util/mockproxy.ts @@ -1,17 +1,22 @@ +import net, { Socket } from "node:net"; +import tls from "node:tls"; import { write, listen, shutdown } from "./proxyfunctions"; const CHALLENGE_LENGTH = 512; +type MockConfig = { + auth?: boolean; + assertions?: boolean; +} + class MockProxy { - mockConfig: { - auth?: boolean; - assertions?: boolean; - }; + mockConfig: MockConfig; dataSentToRemote: string[]; hasSentChallenge: boolean; - client: unknown; + client: Socket; + server: net.Server | tls.Server - constructor(mockConfig) { + constructor(mockConfig: MockConfig) { if (!mockConfig) { throw new Error("Missing mock config"); } @@ -19,7 +24,7 @@ class MockProxy { this.dataSentToRemote = []; } - async start(listenPort: number, tlsOptions?: Record) { + async start(listenPort: number, tlsOptions?: tls.TlsOptions) { await listen( this, listenPort, diff --git a/test/_utils_/proxy.ts b/test/util/proxy.ts similarity index 78% rename from test/_utils_/proxy.ts rename to test/util/proxy.ts index 00394bf..ca9a581 100644 --- a/test/_utils_/proxy.ts +++ b/test/util/proxy.ts @@ -1,17 +1,19 @@ -import { Socket } from "node:net"; +import net, { Socket } from "node:net"; +import tls from "node:tls"; import { write, listen, shutdown, connect, close } from "./proxyfunctions"; // handles only a single client // client -> server (Proxy) -> remote (QuestDB) // client <- server (Proxy) <- remote (QuestDB) class Proxy { - client: unknown; + client: Socket; remote: Socket; + server: net.Server | tls.Server constructor() { this.remote = new Socket(); - this.remote.on("data", async (data: unknown) => { + this.remote.on("data", async (data: string) => { console.info(`received from remote, forwarding to client: ${data}`); await write(this.client, data); }); @@ -25,14 +27,14 @@ class Proxy { }); } - async start(listenPort: unknown, remotePort: unknown, remoteHost: unknown, tlsOptions: Record) { + async start(listenPort: number, remotePort: number, remoteHost: string, tlsOptions: Record) { return new Promise((resolve) => { this.remote.on("ready", async () => { console.info("remote connection ready"); await listen( this, listenPort, - async (data: unknown) => { + async (data: string) => { console.info(`received from client, forwarding to remote: ${data}`); await write(this.remote, data); }, diff --git a/test/_utils_/proxyfunctions.ts b/test/util/proxyfunctions.ts similarity index 66% rename from test/_utils_/proxyfunctions.ts rename to test/util/proxyfunctions.ts index 0d5fab9..bc23f18 100644 --- a/test/_utils_/proxyfunctions.ts +++ b/test/util/proxyfunctions.ts @@ -1,23 +1,19 @@ -import net from "node:net"; -import tls from "node:tls"; +import net, { Socket } from "node:net"; +import tls, { TLSSocket } from "node:tls"; +import { Proxy } from "./proxy"; +import {MockProxy} from "./mockproxy"; const LOCALHOST = "localhost"; -async function write(socket, data) { +async function write(socket: Socket, data: string) { return new Promise((resolve, reject) => { - socket.write(data, "utf8", (err) => { - if (err) { - reject(err) - } else { - resolve(); - } - }); + socket.write(data, "utf8", (err: Error) => err ? reject(err): resolve()); }); } -async function listen(proxy, listenPort, dataHandler, tlsOptions) { +async function listen(proxy: Proxy | MockProxy, listenPort: number, dataHandler: (data: string) => void, tlsOptions: tls.TlsOptions) { return new Promise((resolve) => { - const clientConnHandler = (client) => { + const clientConnHandler = (client: Socket | TLSSocket) => { console.info("client connected"); if (proxy.client) { console.error("There is already a client connected"); @@ -43,7 +39,7 @@ async function listen(proxy, listenPort, dataHandler, tlsOptions) { }); } -async function shutdown(proxy, onServerClose = async () => { }) { +async function shutdown(proxy: Proxy | MockProxy, onServerClose = async () => {}) { console.info("closing proxy"); return new Promise((resolve) => { proxy.server.close(async () => { @@ -53,7 +49,7 @@ async function shutdown(proxy, onServerClose = async () => { }) { }); } -async function connect(proxy, remotePort, remoteHost) { +async function connect(proxy: Proxy, remotePort: number, remoteHost: string) { console.info(`opening remote connection to ${remoteHost}:${remotePort}`); return new Promise((resolve) => { proxy.remote.connect(remotePort, remoteHost, () => resolve()); From 7efb5ec72f18cd9a984cd05a6876a749eabb050a Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 16 Jul 2025 19:56:32 +0100 Subject: [PATCH 2/6] make linter happy --- src/options.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/options.ts b/src/options.ts index ee39178..2ee91f1 100644 --- a/src/options.ts +++ b/src/options.ts @@ -193,7 +193,7 @@ class SenderOptions { extraOptions.agent && !(extraOptions.agent instanceof Agent) && !(extraOptions.agent instanceof http.Agent) - // @ts-ignore + // @ts-expect-error - Not clear what the problem is, the two lines above have no issues && !(extraOptions.agent instanceof https.Agent) ) { throw new Error("Invalid HTTP agent"); From 0ddf668031019645baccb9dc095121c6ed67dd0e Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 16 Jul 2025 23:46:09 +0100 Subject: [PATCH 3/6] more type fixes --- src/options.ts | 6 +++--- src/sender.ts | 8 ++++---- test/sender.integration.test.ts | 10 +++++----- test/util/proxy.ts | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/options.ts b/src/options.ts index 2ee91f1..abb5e1a 100644 --- a/src/options.ts +++ b/src/options.ts @@ -180,7 +180,7 @@ class SenderOptions { * - 'agent' is a custom http/https agent used by the Sender when http/https transport is used.
* A http.Agent or https.Agent object is expected. */ - constructor(configurationString: string, extraOptions: ExtraOptions = undefined) { + constructor(configurationString: string, extraOptions?: ExtraOptions) { parseConfigurationString(this, configurationString); if (extraOptions) { @@ -231,7 +231,7 @@ class SenderOptions { * * @return {SenderOptions} A Sender configuration object initialized from the provided configuration string. */ - static fromConfig(configurationString: string, extraOptions: ExtraOptions = undefined): SenderOptions { + static fromConfig(configurationString: string, extraOptions?: ExtraOptions): SenderOptions { return new SenderOptions(configurationString, extraOptions); } @@ -246,7 +246,7 @@ class SenderOptions { * * @return {SenderOptions} A Sender configuration object initialized from the QDB_CLIENT_CONF environment variable. */ - static fromEnv(extraOptions: ExtraOptions = undefined): SenderOptions { + static fromEnv(extraOptions?: ExtraOptions): SenderOptions { return SenderOptions.fromConfig(process.env.QDB_CLIENT_CONF, extraOptions); } } diff --git a/src/sender.ts b/src/sender.ts index 9ad0786..fe90641 100644 --- a/src/sender.ts +++ b/src/sender.ts @@ -128,7 +128,7 @@ class Sender { * * @return {Sender} A Sender object initialized from the provided configuration string. */ - static fromConfig(configurationString: string, extraOptions: ExtraOptions = undefined): Sender { + static fromConfig(configurationString: string, extraOptions?: ExtraOptions): Sender { return new Sender( SenderOptions.fromConfig(configurationString, extraOptions), ); @@ -145,7 +145,7 @@ class Sender { * * @return {Sender} A Sender object initialized from the QDB_CLIENT_CONF environment variable. */ - static fromEnv(extraOptions: ExtraOptions = undefined): Sender { + static fromEnv(extraOptions?: ExtraOptions): Sender { return new Sender( SenderOptions.fromConfig(process.env.QDB_CLIENT_CONF, extraOptions), ); @@ -488,12 +488,12 @@ class Sender { name: string, value: unknown, writeValue: () => void, - valueType?: string | null, + valueType?: string ) { if (typeof name !== "string") { throw new Error(`Column name must be a string, received ${typeof name}`); } - if (valueType != null && typeof value !== valueType) { + if (valueType && typeof value !== valueType) { throw new Error( `Column value must be of type ${valueType}, received ${typeof value}`, ); diff --git a/test/sender.integration.test.ts b/test/sender.integration.test.ts index cc9e410..aa2b5ca 100644 --- a/test/sender.integration.test.ts +++ b/test/sender.integration.test.ts @@ -1,6 +1,6 @@ // @ts-check import { describe, it, expect, beforeAll, afterAll } from "vitest"; -import { GenericContainer } from "testcontainers"; +import { GenericContainer, StartedTestContainer } from "testcontainers"; import http from "http"; import { Sender } from "../src"; @@ -15,9 +15,9 @@ async function sleep(ms: number) { } describe("Sender tests with containerized QuestDB instance", () => { - let container: any; + let container: StartedTestContainer; - async function query(container: any, query: string) { + async function query(container: StartedTestContainer, query: string) { const options: http.RequestOptions = { hostname: container.getHost(), port: container.getMappedPort(QUESTDB_HTTP_PORT), @@ -46,7 +46,7 @@ describe("Sender tests with containerized QuestDB instance", () => { }); } - async function runSelect(container: any, select: string, expectedCount: number, timeout = 60000) { + async function runSelect(container: StartedTestContainer, select: string, expectedCount: number, timeout = 60000) { const interval = 500; const num = timeout / interval; let selectResult: any; @@ -62,7 +62,7 @@ describe("Sender tests with containerized QuestDB instance", () => { ); } - async function waitForTable(container: any, tableName: string, timeout = 30000) { + async function waitForTable(container: StartedTestContainer, tableName: string, timeout = 30000) { await runSelect(container, `tables() where table_name='${tableName}'`, 1, timeout); } diff --git a/test/util/proxy.ts b/test/util/proxy.ts index ca9a581..59c3076 100644 --- a/test/util/proxy.ts +++ b/test/util/proxy.ts @@ -22,7 +22,7 @@ class Proxy { console.info("remote connection closed"); }); - this.remote.on("error", (err: unknown) => { + this.remote.on("error", (err: Error) => { console.error(`remote connection: ${err}`); }); } From eb0d7ee02b1b16603d8cd5fb595e02c9f884a1e8 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 17 Jul 2025 03:10:46 +0100 Subject: [PATCH 4/6] extract buffer from sender --- src/buffer/index.ts | 414 ++++++++++++++++++++++++++++++++++ src/options.ts | 4 + src/sender.ts | 330 +++------------------------ test/sender.buffer.test.ts | 126 +++++------ test/sender.config.test.ts | 86 +++---- test/sender.transport.test.ts | 6 +- 6 files changed, 568 insertions(+), 398 deletions(-) create mode 100644 src/buffer/index.ts diff --git a/src/buffer/index.ts b/src/buffer/index.ts new file mode 100644 index 0000000..eb8f0a0 --- /dev/null +++ b/src/buffer/index.ts @@ -0,0 +1,414 @@ +// @ts-check +import { Buffer } from "node:buffer"; + +import { log, Logger } from "../logging"; +import { validateColumnName, validateTableName } from "../validation"; +import { SenderOptions } from "../options"; +import { isInteger, timestampToMicros, timestampToNanos } from "../utils"; + +const DEFAULT_MAX_NAME_LENGTH = 127; + +const DEFAULT_BUFFER_SIZE = 65536; // 64 KB +const DEFAULT_MAX_BUFFER_SIZE = 104857600; // 100 MB + +/** @classdesc + * The QuestDB client's API provides methods to connect to the database, ingest data, and close the connection. + * If no custom agent is configured, the Sender will use its own agent which overrides some default values + * of undici.Agent. The Sender's own agent uses persistent connections with 1 minute idle timeout, pipelines requests default to 1. + *

+ */ +class SenderBuffer { + private bufferSize: number; + private readonly maxBufferSize: number; + private buffer: Buffer; + private position: number; + private endOfLastRow: number; + + private hasTable: boolean; + private hasSymbols: boolean; + private hasColumns: boolean; + + private readonly maxNameLength: number; + + private readonly log: Logger; + + /** + * Creates an instance of Sender. + * + * @param {SenderOptions} options - Sender configuration object.
+ * See SenderOptions documentation for detailed description of configuration options.
+ */ + constructor(options: SenderOptions) { + this.log = options && typeof options.log === "function" ? options.log : log; + SenderOptions.resolveDeprecated(options, this.log); + + this.maxNameLength = options && isInteger(options.max_name_len, 1) + ? options.max_name_len + : DEFAULT_MAX_NAME_LENGTH; + + this.maxBufferSize = options && isInteger(options.max_buf_size, 1) + ? options.max_buf_size + : DEFAULT_MAX_BUFFER_SIZE; + this.resize( + options && isInteger(options.init_buf_size, 1) + ? options.init_buf_size + : DEFAULT_BUFFER_SIZE, + ); + + this.reset(); + } + + /** + * Extends the size of the sender's buffer.
+ * Can be used to increase the size of buffer if overflown. + * The buffer's content is copied into the new buffer. + * + * @param {number} bufferSize - New size of the buffer used by the sender, provided in bytes. + */ + private resize(bufferSize: number) { + if (bufferSize > this.maxBufferSize) { + throw new Error(`Max buffer size is ${this.maxBufferSize} bytes, requested buffer size: ${bufferSize}`); + } + this.bufferSize = bufferSize; + // Allocating an extra byte because Buffer.write() does not fail if the length of the data to be written is + // longer than the size of the buffer. It simply just writes whatever it can, and returns. + // If we can write into the extra byte, that indicates buffer overflow. + // See the check in the write() function. + const newBuffer = Buffer.alloc(this.bufferSize + 1, 0, "utf8"); + if (this.buffer) { + this.buffer.copy(newBuffer); + } + this.buffer = newBuffer; + } + + /** + * Resets the buffer, data added to the buffer will be lost.
+ * In other words it clears the buffer and sets the writing position to the beginning of the buffer. + * + * @return {Sender} Returns with a reference to this sender. + */ + reset(): SenderBuffer { + this.position = 0; + this.startNewRow(); + return this; + } + + private startNewRow() { + this.endOfLastRow = this.position; + this.hasTable = false; + this.hasSymbols = false; + this.hasColumns = false; + } + + /** + * @ignore + * @return {Buffer} Returns a cropped buffer, or null if there is nothing to send. + * The returned buffer is backed by the sender's buffer. + * Used only in tests. + */ + toBufferView(pos = this.endOfLastRow): Buffer { + return pos > 0 ? this.buffer.subarray(0, pos) : null; + } + + /** + * @ignore + * @return {Buffer|null} Returns a cropped buffer ready to send to the server, or null if there is nothing to send. + * The returned buffer is a copy of the sender's buffer. + * It also compacts the Sender's buffer. + */ + toBufferNew(pos = this.endOfLastRow): Buffer | null { + if (pos > 0) { + const data = Buffer.allocUnsafe(pos); + this.buffer.copy(data, 0, 0, pos); + this.compact(); + return data; + } + return null; + } + + /** + * Write the table name into the buffer of the sender. + * + * @param {string} table - Table name. + * @return {Sender} Returns with a reference to this sender. + */ + table(table: string): SenderBuffer { + if (typeof table !== "string") { + throw new Error(`Table name must be a string, received ${typeof table}`); + } + if (this.hasTable) { + throw new Error("Table name has already been set"); + } + validateTableName(table, this.maxNameLength); + this.checkCapacity([table], table.length); + this.writeEscaped(table); + this.hasTable = true; + return this; + } + + /** + * Write a symbol name and value into the buffer of the sender. + * + * @param {string} name - Symbol name. + * @param {unknown} value - Symbol value, toString() is called to extract the actual symbol value from the parameter. + * @return {Sender} Returns with a reference to this sender. + */ + symbol(name: string, value: unknown): SenderBuffer { + if (typeof name !== "string") { + throw new Error(`Symbol name must be a string, received ${typeof name}`); + } + if (!this.hasTable || this.hasColumns) { + throw new Error("Symbol can be added only after table name is set and before any column added"); + } + const valueStr = value.toString(); + this.checkCapacity([name, valueStr], 2 + name.length + valueStr.length); + this.write(","); + validateColumnName(name, this.maxNameLength); + this.writeEscaped(name); + this.write("="); + this.writeEscaped(valueStr); + this.hasSymbols = true; + return this; + } + + /** + * Write a string column with its value into the buffer of the sender. + * + * @param {string} name - Column name. + * @param {string} value - Column value, accepts only string values. + * @return {Sender} Returns with a reference to this sender. + */ + stringColumn(name: string, value: string): SenderBuffer { + this.writeColumn( + name, + value, + () => { + this.checkCapacity([value], 2 + value.length); + this.write('"'); + this.writeEscaped(value, true); + this.write('"'); + }, + "string", + ); + return this; + } + + /** + * Write a boolean column with its value into the buffer of the sender. + * + * @param {string} name - Column name. + * @param {boolean} value - Column value, accepts only boolean values. + * @return {Sender} Returns with a reference to this sender. + */ + booleanColumn(name: string, value: boolean): SenderBuffer { + this.writeColumn( + name, + value, + () => { + this.checkCapacity([], 1); + this.write(value ? "t" : "f"); + }, + "boolean", + ); + return this; + } + + /** + * Write a float column with its value into the buffer of the sender. + * + * @param {string} name - Column name. + * @param {number} value - Column value, accepts only number values. + * @return {Sender} Returns with a reference to this sender. + */ + floatColumn(name: string, value: number): SenderBuffer { + this.writeColumn( + name, + value, + () => { + const valueStr = value.toString(); + this.checkCapacity([valueStr]); + this.write(valueStr); + }, + "number", + ); + return this; + } + + /** + * Write an integer column with its value into the buffer of the sender. + * + * @param {string} name - Column name. + * @param {number} value - Column value, accepts only number values. + * @return {Sender} Returns with a reference to this sender. + */ + intColumn(name: string, value: number): SenderBuffer { + if (!Number.isInteger(value)) { + throw new Error(`Value must be an integer, received ${value}`); + } + this.writeColumn(name, value, () => { + const valueStr = value.toString(); + this.checkCapacity([valueStr], 1); + this.write(valueStr); + this.write("i"); + }); + return this; + } + + /** + * Write a timestamp column with its value into the buffer of the sender. + * + * @param {string} name - Column name. + * @param {number | bigint} value - Epoch timestamp, accepts numbers or BigInts. + * @param {string} [unit=us] - Timestamp unit. Supported values: 'ns' - nanoseconds, 'us' - microseconds, 'ms' - milliseconds. Defaults to 'us'. + * @return {Sender} Returns with a reference to this sender. + */ + timestampColumn( + name: string, + value: number | bigint, + unit: "ns" | "us" | "ms" = "us", + ): SenderBuffer { + if (typeof value !== "bigint" && !Number.isInteger(value)) { + throw new Error(`Value must be an integer or BigInt, received ${value}`); + } + this.writeColumn(name, value, () => { + const valueMicros = timestampToMicros(BigInt(value), unit); + const valueStr = valueMicros.toString(); + this.checkCapacity([valueStr], 1); + this.write(valueStr); + this.write("t"); + }); + return this; + } + + /** + * Closing the row after writing the designated timestamp into the buffer of the sender. + * + * @param {number | bigint} timestamp - Designated epoch timestamp, accepts numbers or BigInts. + * @param {string} [unit=us] - Timestamp unit. Supported values: 'ns' - nanoseconds, 'us' - microseconds, 'ms' - milliseconds. Defaults to 'us'. + */ + at(timestamp: number | bigint, unit: "ns" | "us" | "ms" = "us") { + if (!this.hasSymbols && !this.hasColumns) { + throw new Error("The row must have a symbol or column set before it is closed"); + } + if (typeof timestamp !== "bigint" && !Number.isInteger(timestamp)) { + throw new Error(`Designated timestamp must be an integer or BigInt, received ${timestamp}`); + } + const timestampNanos = timestampToNanos(BigInt(timestamp), unit); + const timestampStr = timestampNanos.toString(); + this.checkCapacity([timestampStr], 2); + this.write(" "); + this.write(timestampStr); + this.write("\n"); + this.startNewRow(); + } + + /** + * Closing the row without writing designated timestamp into the buffer of the sender.
+ * Designated timestamp will be populated by the server on this record. + */ + atNow() { + if (!this.hasSymbols && !this.hasColumns) { + throw new Error("The row must have a symbol or column set before it is closed"); + } + this.checkCapacity([], 1); + this.write("\n"); + this.startNewRow(); + } + + private checkCapacity(data: string[], base = 0) { + let length = base; + for (const str of data) { + length += Buffer.byteLength(str, "utf8"); + } + if (this.position + length > this.bufferSize) { + let newSize = this.bufferSize; + do { + newSize += this.bufferSize; + } while (this.position + length > newSize); + this.resize(newSize); + } + } + + private compact() { + if (this.endOfLastRow > 0) { + this.buffer.copy(this.buffer, 0, this.endOfLastRow, this.position); + this.position = this.position - this.endOfLastRow; + this.endOfLastRow = 0; + } + } + + private writeColumn( + name: string, + value: unknown, + writeValue: () => void, + valueType?: string + ) { + if (typeof name !== "string") { + throw new Error(`Column name must be a string, received ${typeof name}`); + } + if (valueType && typeof value !== valueType) { + throw new Error( + `Column value must be of type ${valueType}, received ${typeof value}`, + ); + } + if (!this.hasTable) { + throw new Error("Column can be set only after table name is set"); + } + this.checkCapacity([name], 2 + name.length); + this.write(this.hasColumns ? "," : " "); + validateColumnName(name, this.maxNameLength); + this.writeEscaped(name); + this.write("="); + writeValue(); + this.hasColumns = true; + } + + private write(data: string) { + this.position += this.buffer.write(data, this.position); + if (this.position > this.bufferSize) { + // should never happen, if checkCapacity() is correctly used + throw new Error( + `Buffer overflow [position=${this.position}, bufferSize=${this.bufferSize}]`, + ); + } + } + + private writeEscaped(data: string, quoted = false) { + for (const ch of data) { + if (ch > "\\") { + this.write(ch); + continue; + } + + switch (ch) { + case " ": + case ",": + case "=": + if (!quoted) { + this.write("\\"); + } + this.write(ch); + break; + case "\n": + case "\r": + this.write("\\"); + this.write(ch); + break; + case '"': + if (quoted) { + this.write("\\"); + } + this.write(ch); + break; + case "\\": + this.write("\\\\"); + break; + default: + this.write(ch); + break; + } + } + } +} + +export { SenderBuffer, DEFAULT_BUFFER_SIZE, DEFAULT_MAX_BUFFER_SIZE }; diff --git a/src/options.ts b/src/options.ts index abb5e1a..8fcb189 100644 --- a/src/options.ts +++ b/src/options.ts @@ -203,6 +203,10 @@ class SenderOptions { } static resolveDeprecated(options: SenderOptions & DeprecatedOptions, log: Logger) { + if (!options) { + return; + } + // deal with deprecated options if (options.copy_buffer !== undefined) { log("warn", `Option 'copy_buffer' is not supported anymore, please, remove it`); diff --git a/src/sender.ts b/src/sender.ts index fe90641..e174018 100644 --- a/src/sender.ts +++ b/src/sender.ts @@ -1,19 +1,12 @@ // @ts-check -import { Buffer } from "node:buffer"; - import { log, Logger } from "./logging"; -import { validateColumnName, validateTableName } from "./validation"; import { SenderOptions, ExtraOptions } from "./options"; import { SenderTransport, createTransport } from "./transport"; -import { isBoolean, isInteger, timestampToMicros, timestampToNanos } from "./utils"; +import { isBoolean, isInteger } from "./utils"; +import { SenderBuffer } from "./buffer"; const DEFAULT_AUTO_FLUSH_INTERVAL = 1000; // 1 sec -const DEFAULT_MAX_NAME_LENGTH = 127; - -const DEFAULT_BUFFER_SIZE = 65536; // 64 KB -const DEFAULT_MAX_BUFFER_SIZE = 104857600; // 100 MB - /** @classdesc * The QuestDB client's API provides methods to connect to the database, ingest data, and close the connection. * The supported protocols are HTTP and TCP. HTTP is preferred as it provides feedback in the HTTP response.
@@ -61,11 +54,7 @@ const DEFAULT_MAX_BUFFER_SIZE = 104857600; // 100 MB class Sender { private readonly transport: SenderTransport; - private bufferSize: number; - private readonly maxBufferSize: number; - private buffer: Buffer; - private position: number; - private endOfLastRow: number; + private buffer: SenderBuffer; private readonly autoFlush: boolean; private readonly autoFlushRows: number; @@ -73,12 +62,6 @@ class Sender { private lastFlushTime: number; private pendingRowCount: number; - private hasTable: boolean; - private hasSymbols: boolean; - private hasColumns: boolean; - - private readonly maxNameLength: number; - private readonly log: Logger; /** @@ -89,9 +72,9 @@ class Sender { */ constructor(options: SenderOptions) { this.transport = createTransport(options); + this.buffer = new SenderBuffer(options); this.log = typeof options.log === "function" ? options.log : log; - SenderOptions.resolveDeprecated(options, this.log); this.autoFlush = isBoolean(options.auto_flush) ? options.auto_flush : true; this.autoFlushRows = isInteger(options.auto_flush_rows, 0) @@ -101,18 +84,6 @@ class Sender { ? options.auto_flush_interval : DEFAULT_AUTO_FLUSH_INTERVAL; - this.maxNameLength = isInteger(options.max_name_len, 1) - ? options.max_name_len - : DEFAULT_MAX_NAME_LENGTH; - - this.maxBufferSize = isInteger(options.max_buf_size, 1) - ? options.max_buf_size - : DEFAULT_MAX_BUFFER_SIZE; - this.resize( - isInteger(options.init_buf_size, 1) - ? options.init_buf_size - : DEFAULT_BUFFER_SIZE, - ); this.reset(); } @@ -151,29 +122,6 @@ class Sender { ); } - /** - * Extends the size of the sender's buffer.
- * Can be used to increase the size of buffer if overflown. - * The buffer's content is copied into the new buffer. - * - * @param {number} bufferSize - New size of the buffer used by the sender, provided in bytes. - */ - private resize(bufferSize: number) { - if (bufferSize > this.maxBufferSize) { - throw new Error(`Max buffer size is ${this.maxBufferSize} bytes, requested buffer size: ${bufferSize}`); - } - this.bufferSize = bufferSize; - // Allocating an extra byte because Buffer.write() does not fail if the length of the data to be written is - // longer than the size of the buffer. It simply just writes whatever it can, and returns. - // If we can write into the extra byte, that indicates buffer overflow. - // See the check in our write() function. - const newBuffer = Buffer.alloc(this.bufferSize + 1, 0, "utf8"); - if (this.buffer) { - this.buffer.copy(newBuffer); - } - this.buffer = newBuffer; - } - /** * Resets the buffer, data added to the buffer will be lost.
* In other words it clears the buffer and sets the writing position to the beginning of the buffer. @@ -181,10 +129,8 @@ class Sender { * @return {Sender} Returns with a reference to this sender. */ reset(): Sender { - this.position = 0; - this.lastFlushTime = Date.now(); - this.pendingRowCount = 0; - this.startNewRow(); + this.buffer.reset(); + this.resetAutoFlush(); return this; } @@ -204,11 +150,14 @@ class Sender { * @return {Promise} Resolves to true when there was data in the buffer to send, and it was sent successfully. */ async flush(): Promise { - const dataToSend: Buffer = this.toBufferNew(); + const dataToSend: Buffer = this.buffer.toBufferNew(); if (!dataToSend) { return false; // Nothing to send } + this.log("debug", `Flushing, number of flushed rows: ${this.pendingRowCount}`); + this.resetAutoFlush(); + await this.transport.send(dataToSend); } @@ -220,32 +169,6 @@ class Sender { return this.transport.close(); } - /** - * @ignore - * @return {Buffer} Returns a cropped buffer, or null if there is nothing to send. - * The returned buffer is backed by the sender's buffer. - * Used only in tests. - */ - toBufferView(pos = this.endOfLastRow): Buffer { - return pos > 0 ? this.buffer.subarray(0, pos) : null; - } - - /** - * @ignore - * @return {Buffer|null} Returns a cropped buffer ready to send to the server, or null if there is nothing to send. - * The returned buffer is a copy of the sender's buffer. - * It also compacts the Sender's buffer. - */ - toBufferNew(pos = this.endOfLastRow): Buffer | null { - if (pos > 0) { - const data = Buffer.allocUnsafe(pos); - this.buffer.copy(data, 0, 0, pos); - this.compact(); - return data; - } - return null; - } - /** * Write the table name into the buffer of the sender. * @@ -253,16 +176,7 @@ class Sender { * @return {Sender} Returns with a reference to this sender. */ table(table: string): Sender { - if (typeof table !== "string") { - throw new Error(`Table name must be a string, received ${typeof table}`); - } - if (this.hasTable) { - throw new Error("Table name has already been set"); - } - validateTableName(table, this.maxNameLength); - this.checkCapacity([table]); - this.writeEscaped(table); - this.hasTable = true; + this.buffer.table(table); return this; } @@ -270,24 +184,11 @@ class Sender { * Write a symbol name and value into the buffer of the sender. * * @param {string} name - Symbol name. - * @param {any} value - Symbol value, toString() will be called to extract the actual symbol value from the parameter. + * @param {unknown} value - Symbol value, toString() is called to extract the actual symbol value from the parameter. * @return {Sender} Returns with a reference to this sender. */ - symbol(name: string, value: T): Sender { - if (typeof name !== "string") { - throw new Error(`Symbol name must be a string, received ${typeof name}`); - } - if (!this.hasTable || this.hasColumns) { - throw new Error("Symbol can be added only after table name is set and before any column added"); - } - const valueStr = value.toString(); - this.checkCapacity([name, valueStr], 2 + name.length + valueStr.length); - this.write(","); - validateColumnName(name, this.maxNameLength); - this.writeEscaped(name); - this.write("="); - this.writeEscaped(valueStr); - this.hasSymbols = true; + symbol(name: string, value: unknown): Sender { + this.buffer.symbol(name, value); return this; } @@ -299,17 +200,7 @@ class Sender { * @return {Sender} Returns with a reference to this sender. */ stringColumn(name: string, value: string): Sender { - this.writeColumn( - name, - value, - () => { - this.checkCapacity([value], 2 + value.length); - this.write('"'); - this.writeEscaped(value, true); - this.write('"'); - }, - "string", - ); + this.buffer.stringColumn(name, value); return this; } @@ -321,15 +212,7 @@ class Sender { * @return {Sender} Returns with a reference to this sender. */ booleanColumn(name: string, value: boolean): Sender { - this.writeColumn( - name, - value, - () => { - this.checkCapacity([], 1); - this.write(value ? "t" : "f"); - }, - "boolean", - ); + this.buffer.booleanColumn(name, value); return this; } @@ -341,16 +224,7 @@ class Sender { * @return {Sender} Returns with a reference to this sender. */ floatColumn(name: string, value: number): Sender { - this.writeColumn( - name, - value, - () => { - const valueStr = value.toString(); - this.checkCapacity([valueStr], valueStr.length); - this.write(valueStr); - }, - "number", - ); + this.buffer.floatColumn(name, value); return this; } @@ -362,15 +236,7 @@ class Sender { * @return {Sender} Returns with a reference to this sender. */ intColumn(name: string, value: number): Sender { - if (!Number.isInteger(value)) { - throw new Error(`Value must be an integer, received ${value}`); - } - this.writeColumn(name, value, () => { - const valueStr = value.toString(); - this.checkCapacity([valueStr], 1 + valueStr.length); - this.write(valueStr); - this.write("i"); - }); + this.buffer.intColumn(name, value); return this; } @@ -387,16 +253,7 @@ class Sender { value: number | bigint, unit: "ns" | "us" | "ms" = "us", ): Sender { - if (typeof value !== "bigint" && !Number.isInteger(value)) { - throw new Error(`Value must be an integer or BigInt, received ${value}`); - } - this.writeColumn(name, value, () => { - const valueMicros = timestampToMicros(BigInt(value), unit); - const valueStr = valueMicros.toString(); - this.checkCapacity([valueStr], 1 + valueStr.length); - this.write(valueStr); - this.write("t"); - }); + this.buffer.timestampColumn(name, value, unit); return this; } @@ -407,21 +264,10 @@ class Sender { * @param {string} [unit=us] - Timestamp unit. Supported values: 'ns' - nanoseconds, 'us' - microseconds, 'ms' - milliseconds. Defaults to 'us'. */ async at(timestamp: number | bigint, unit: "ns" | "us" | "ms" = "us") { - if (!this.hasSymbols && !this.hasColumns) { - throw new Error("The row must have a symbol or column set before it is closed"); - } - if (typeof timestamp !== "bigint" && !Number.isInteger(timestamp)) { - throw new Error(`Designated timestamp must be an integer or BigInt, received ${timestamp}`); - } - const timestampNanos = timestampToNanos(BigInt(timestamp), unit); - const timestampStr = timestampNanos.toString(); - this.checkCapacity([], 2 + timestampStr.length); - this.write(" "); - this.write(timestampStr); - this.write("\n"); + this.buffer.at(timestamp, unit); this.pendingRowCount++; - this.startNewRow(); - await this.automaticFlush(); + this.log("debug", `Pending row count: ${this.pendingRowCount}`); + await this.tryFlush(); } /** @@ -429,132 +275,30 @@ class Sender { * Designated timestamp will be populated by the server on this record. */ async atNow() { - if (!this.hasSymbols && !this.hasColumns) { - throw new Error("The row must have a symbol or column set before it is closed"); - } - this.checkCapacity([], 1); - this.write("\n"); + this.buffer.atNow(); this.pendingRowCount++; - this.startNewRow(); - await this.automaticFlush(); + this.log("debug", `Pending row count: ${this.pendingRowCount}`); + await this.tryFlush(); } - private startNewRow() { - this.endOfLastRow = this.position; - this.hasTable = false; - this.hasSymbols = false; - this.hasColumns = false; + private resetAutoFlush(): void { + this.lastFlushTime = Date.now(); + this.pendingRowCount = 0; + this.log("debug", `Pending row count: ${this.pendingRowCount}`); } - private async automaticFlush() { + private async tryFlush() { if ( - this.autoFlush && - this.pendingRowCount > 0 && - ((this.autoFlushRows > 0 && - this.pendingRowCount >= this.autoFlushRows) || - (this.autoFlushInterval > 0 && - Date.now() - this.lastFlushTime >= this.autoFlushInterval)) + this.autoFlush + && this.pendingRowCount > 0 + && ( + (this.autoFlushRows > 0 && this.pendingRowCount >= this.autoFlushRows) + || (this.autoFlushInterval > 0 && Date.now() - this.lastFlushTime >= this.autoFlushInterval) + ) ) { await this.flush(); } } - - private checkCapacity(data: string[], base = 0) { - let length = base; - for (const str of data) { - length += Buffer.byteLength(str, "utf8"); - } - if (this.position + length > this.bufferSize) { - let newSize = this.bufferSize; - do { - newSize += this.bufferSize; - } while (this.position + length > newSize); - this.resize(newSize); - } - } - - private compact() { - if (this.endOfLastRow > 0) { - this.buffer.copy(this.buffer, 0, this.endOfLastRow, this.position); - this.position = this.position - this.endOfLastRow; - this.endOfLastRow = 0; - - this.lastFlushTime = Date.now(); - this.pendingRowCount = 0; - } - } - - private writeColumn( - name: string, - value: unknown, - writeValue: () => void, - valueType?: string - ) { - if (typeof name !== "string") { - throw new Error(`Column name must be a string, received ${typeof name}`); - } - if (valueType && typeof value !== valueType) { - throw new Error( - `Column value must be of type ${valueType}, received ${typeof value}`, - ); - } - if (!this.hasTable) { - throw new Error("Column can be set only after table name is set"); - } - this.checkCapacity([name], 2 + name.length); - this.write(this.hasColumns ? "," : " "); - validateColumnName(name, this.maxNameLength); - this.writeEscaped(name); - this.write("="); - writeValue(); - this.hasColumns = true; - } - - private write(data: string) { - this.position += this.buffer.write(data, this.position); - if (this.position > this.bufferSize) { - throw new Error( - `Buffer overflow [position=${this.position}, bufferSize=${this.bufferSize}]`, - ); - } - } - - private writeEscaped(data: string, quoted = false) { - for (const ch of data) { - if (ch > "\\") { - this.write(ch); - continue; - } - - switch (ch) { - case " ": - case ",": - case "=": - if (!quoted) { - this.write("\\"); - } - this.write(ch); - break; - case "\n": - case "\r": - this.write("\\"); - this.write(ch); - break; - case '"': - if (quoted) { - this.write("\\"); - } - this.write(ch); - break; - case "\\": - this.write("\\\\"); - break; - default: - this.write(ch); - break; - } - } - } } -export { Sender, DEFAULT_BUFFER_SIZE, DEFAULT_MAX_BUFFER_SIZE }; +export { Sender }; diff --git a/test/sender.buffer.test.ts b/test/sender.buffer.test.ts index 9e7c745..5d36db7 100644 --- a/test/sender.buffer.test.ts +++ b/test/sender.buffer.test.ts @@ -60,7 +60,7 @@ describe("Client interop test suite", function () { } if (!errorMessage) { - const actualLine = sender.toBufferView().toString(); + const actualLine = bufferContent(sender); if (testCase.result.status === "SUCCESS") { if (testCase.result.line) { @@ -150,7 +150,7 @@ describe("Sender message builder test suite (anything not covered in client inte .booleanColumn("boolCol", true) .atNow(); } - expect(sender.toBufferView().toString()).toBe( + expect(bufferContent(sender)).toBe( 'tableName page_products="[{\\"id\\":\\"46022e96-076f-457f-b630-51b82b8716180\\",\\"gridId\\":\\"46022e96-076f-457f-b630-51b82b871618\\"},{\\"id\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\",\\"gridId\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\"},{\\"id\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\",\\"gridId\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\"},{\\"id\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c280\\",\\"gridId\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c2800\\"}]",boolCol=t\n' + 'tableName page_products="[{\\"id\\":\\"46022e96-076f-457f-b630-51b82b8716181\\",\\"gridId\\":\\"46022e96-076f-457f-b630-51b82b871618\\"},{\\"id\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\",\\"gridId\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\"},{\\"id\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\",\\"gridId\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\"},{\\"id\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c280\\",\\"gridId\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c2801\\"}]",boolCol=t\n' + 'tableName page_products="[{\\"id\\":\\"46022e96-076f-457f-b630-51b82b8716182\\",\\"gridId\\":\\"46022e96-076f-457f-b630-51b82b871618\\"},{\\"id\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\",\\"gridId\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\"},{\\"id\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\",\\"gridId\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\"},{\\"id\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c280\\",\\"gridId\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c2802\\"}]",boolCol=t\n' + @@ -170,7 +170,7 @@ describe("Sender message builder test suite (anything not covered in client inte .booleanColumn("boolCol", true) .timestampColumn("timestampCol", 1658484765000000) .atNow(); - expect(sender.toBufferView().toString()).toBe( + expect(bufferContent(sender)).toBe( "tableName boolCol=t,timestampCol=1658484765000000t\n", ); await sender.close(); @@ -187,7 +187,7 @@ describe("Sender message builder test suite (anything not covered in client inte .booleanColumn("boolCol", true) .timestampColumn("timestampCol", 1658484765000000, "ns") .atNow(); - expect(sender.toBufferView().toString()).toBe( + expect(bufferContent(sender)).toBe( "tableName boolCol=t,timestampCol=1658484765000t\n", ); await sender.close(); @@ -204,7 +204,7 @@ describe("Sender message builder test suite (anything not covered in client inte .booleanColumn("boolCol", true) .timestampColumn("timestampCol", 1658484765000000, "us") .atNow(); - expect(sender.toBufferView().toString()).toBe( + expect(bufferContent(sender)).toBe( "tableName boolCol=t,timestampCol=1658484765000000t\n", ); await sender.close(); @@ -221,7 +221,7 @@ describe("Sender message builder test suite (anything not covered in client inte .booleanColumn("boolCol", true) .timestampColumn("timestampCol", 1658484765000, "ms") .atNow(); - expect(sender.toBufferView().toString()).toBe( + expect(bufferContent(sender)).toBe( "tableName boolCol=t,timestampCol=1658484765000000t\n", ); await sender.close(); @@ -238,7 +238,7 @@ describe("Sender message builder test suite (anything not covered in client inte .booleanColumn("boolCol", true) .timestampColumn("timestampCol", 1658484765000000n) .atNow(); - expect(sender.toBufferView().toString()).toBe( + expect(bufferContent(sender)).toBe( "tableName boolCol=t,timestampCol=1658484765000000t\n", ); await sender.close(); @@ -255,7 +255,7 @@ describe("Sender message builder test suite (anything not covered in client inte .booleanColumn("boolCol", true) .timestampColumn("timestampCol", 1658484765000000000n, "ns") .atNow(); - expect(sender.toBufferView().toString()).toBe( + expect(bufferContent(sender)).toBe( "tableName boolCol=t,timestampCol=1658484765000000t\n", ); await sender.close(); @@ -272,7 +272,7 @@ describe("Sender message builder test suite (anything not covered in client inte .booleanColumn("boolCol", true) .timestampColumn("timestampCol", 1658484765000000n, "us") .atNow(); - expect(sender.toBufferView().toString()).toBe( + expect(bufferContent(sender)).toBe( "tableName boolCol=t,timestampCol=1658484765000000t\n", ); await sender.close(); @@ -289,7 +289,7 @@ describe("Sender message builder test suite (anything not covered in client inte .booleanColumn("boolCol", true) .timestampColumn("timestampCol", 1658484765000n, "ms") .atNow(); - expect(sender.toBufferView().toString()).toBe( + expect(bufferContent(sender)).toBe( "tableName boolCol=t,timestampCol=1658484765000000t\n", ); await sender.close(); @@ -325,7 +325,7 @@ describe("Sender message builder test suite (anything not covered in client inte .booleanColumn("boolCol", true) .timestampColumn("timestampCol", 1658484765000000) .at(1658484769000000, "us"); - expect(sender.toBufferView().toString()).toBe( + expect(bufferContent(sender)).toBe( "tableName boolCol=t,timestampCol=1658484765000000t 1658484769000000000\n", ); await sender.close(); @@ -342,7 +342,7 @@ describe("Sender message builder test suite (anything not covered in client inte .booleanColumn("boolCol", true) .timestampColumn("timestampCol", 1658484765000000) .at(1658484769000, "ms"); - expect(sender.toBufferView().toString()).toBe( + expect(bufferContent(sender)).toBe( "tableName boolCol=t,timestampCol=1658484765000000t 1658484769000000000\n", ); await sender.close(); @@ -359,7 +359,7 @@ describe("Sender message builder test suite (anything not covered in client inte .booleanColumn("boolCol", true) .timestampColumn("timestampCol", 1658484765000000) .at(1658484769000000n); - expect(sender.toBufferView().toString()).toBe( + expect(bufferContent(sender)).toBe( "tableName boolCol=t,timestampCol=1658484765000000t 1658484769000000000\n", ); await sender.close(); @@ -376,7 +376,7 @@ describe("Sender message builder test suite (anything not covered in client inte .booleanColumn("boolCol", true) .timestampColumn("timestampCol", 1658484765000000) .at(1658484769000000123n, "ns"); - expect(sender.toBufferView().toString()).toBe( + expect(bufferContent(sender)).toBe( "tableName boolCol=t,timestampCol=1658484765000000t 1658484769000000123\n", ); await sender.close(); @@ -393,7 +393,7 @@ describe("Sender message builder test suite (anything not covered in client inte .booleanColumn("boolCol", true) .timestampColumn("timestampCol", 1658484765000000) .at(1658484769000000n, "us"); - expect(sender.toBufferView().toString()).toBe( + expect(bufferContent(sender)).toBe( "tableName boolCol=t,timestampCol=1658484765000000t 1658484769000000000\n", ); await sender.close(); @@ -410,7 +410,7 @@ describe("Sender message builder test suite (anything not covered in client inte .booleanColumn("boolCol", true) .timestampColumn("timestampCol", 1658484765000000) .at(1658484769000n, "ms"); - expect(sender.toBufferView().toString()).toBe( + expect(bufferContent(sender)).toBe( "tableName boolCol=t,timestampCol=1658484765000000t 1658484769000000000\n", ); await sender.close(); @@ -582,8 +582,10 @@ describe("Sender message builder test suite (anything not covered in client inte host: "host", init_buf_size: 1024, }); - expect(sender.toBufferView()).toBe(null); - expect(sender.toBufferNew()).toBe(null); + // @ts-expect-error - Accessing private field + expect(sender.buffer.toBufferView()).toBe(null); + // @ts-expect-error - Accessing private field + expect(sender.buffer.toBufferNew()).toBe(null); await sender.close(); }); @@ -598,12 +600,13 @@ describe("Sender message builder test suite (anything not covered in client inte sender.table("tableName").symbol("name", "value2"); // copy of the sender's buffer contains the finished row - expect(sender.toBufferNew().toString()).toBe( + // @ts-expect-error - Accessing private field + expect(sender.buffer.toBufferNew().toString()).toBe( "tableName,name=value 1234567890\n", ); // the sender's buffer is compacted, and contains only the unfinished row // @ts-expect-error - Accessing private field - expect(sender.toBufferView(sender.position).toString()).toBe( + expect(sender.buffer.toBufferView(bufferPosition(sender)).toString()).toBe( "tableName,name=value2", ); await sender.close(); @@ -691,39 +694,29 @@ describe("Sender message builder test suite (anything not covered in client inte host: "host", init_buf_size: 8, }); - // @ts-expect-error - Accessing private field - expect(sender.bufferSize).toBe(8); - // @ts-expect-error - Accessing private field - expect(sender.position).toBe(0); + expect(bufferSize(sender)).toBe(8); + expect(bufferPosition(sender)).toBe(0); sender.table("tableName"); - // @ts-expect-error - Accessing private field - expect(sender.bufferSize).toBe(16); - // @ts-expect-error - Accessing private field - expect(sender.position).toBe("tableName".length); + expect(bufferSize(sender)).toBe(24); + expect(bufferPosition(sender)).toBe("tableName".length); sender.intColumn("intField", 123); - // @ts-expect-error - Accessing private field - expect(sender.bufferSize).toBe(32); - // @ts-expect-error - Accessing private field - expect(sender.position).toBe("tableName intField=123i".length); + expect(bufferSize(sender)).toBe(48); + expect(bufferPosition(sender)).toBe("tableName intField=123i".length); await sender.atNow(); - // @ts-expect-error - Accessing private field - expect(sender.bufferSize).toBe(32); - // @ts-expect-error - Accessing private field - expect(sender.position).toBe("tableName intField=123i\n".length); - expect(sender.toBufferView().toString()).toBe("tableName intField=123i\n"); + expect(bufferSize(sender)).toBe(48); + expect(bufferPosition(sender)).toBe("tableName intField=123i\n".length); + expect(bufferContent(sender)).toBe("tableName intField=123i\n"); await sender .table("table2") .intColumn("intField", 125) .stringColumn("strField", "test") .atNow(); - // @ts-expect-error - Accessing private field - expect(sender.bufferSize).toBe(64); - // @ts-expect-error - Accessing private field - expect(sender.position).toBe( + expect(bufferSize(sender)).toBe(96); + expect(bufferPosition(sender)).toBe( 'tableName intField=123i\ntable2 intField=125i,strField="test"\n'.length, ); - expect(sender.toBufferView().toString()).toBe( + expect(bufferContent(sender)).toBe( 'tableName intField=123i\ntable2 intField=125i,strField="test"\n', ); await sender.close(); @@ -731,28 +724,20 @@ describe("Sender message builder test suite (anything not covered in client inte it("throws exception if tries to extend the size of the buffer above max buffer size", async function () { const sender = Sender.fromConfig( - "tcp::addr=host;init_buf_size=8;max_buf_size=48;", + "tcp::addr=host;init_buf_size=8;max_buf_size=64;", ); - // @ts-expect-error - Accessing private field - expect(sender.bufferSize).toBe(8); - // @ts-expect-error - Accessing private field - expect(sender.position).toBe(0); + expect(bufferSize(sender)).toBe(8); + expect(bufferPosition(sender)).toBe(0); sender.table("tableName"); - // @ts-expect-error - Accessing private field - expect(sender.bufferSize).toBe(16); - // @ts-expect-error - Accessing private field - expect(sender.position).toBe("tableName".length); + expect(bufferSize(sender)).toBe(24); + expect(bufferPosition(sender)).toBe("tableName".length); sender.intColumn("intField", 123); - // @ts-expect-error - Accessing private field - expect(sender.bufferSize).toBe(32); - // @ts-expect-error - Accessing private field - expect(sender.position).toBe("tableName intField=123i".length); + expect(bufferSize(sender)).toBe(48); + expect(bufferPosition(sender)).toBe("tableName intField=123i".length); await sender.atNow(); - // @ts-expect-error - Accessing private field - expect(sender.bufferSize).toBe(32); - // @ts-expect-error - Accessing private field - expect(sender.position).toBe("tableName intField=123i\n".length); - expect(sender.toBufferView().toString()).toBe("tableName intField=123i\n"); + expect(bufferSize(sender)).toBe(48); + expect(bufferPosition(sender)).toBe("tableName intField=123i\n".length); + expect(bufferContent(sender)).toBe("tableName intField=123i\n"); try { await sender @@ -762,7 +747,7 @@ describe("Sender message builder test suite (anything not covered in client inte .atNow(); } catch (err) { expect(err.message).toBe( - "Max buffer size is 48 bytes, requested buffer size: 64", + "Max buffer size is 64 bytes, requested buffer size: 96", ); } await sender.close(); @@ -784,7 +769,7 @@ describe("Sender message builder test suite (anything not covered in client inte .booleanColumn("boolCol", false) .timestampColumn("timestampCol", 1658484766000000) .atNow(); - expect(sender.toBufferView().toString()).toBe( + expect(bufferContent(sender)).toBe( "tableName boolCol=t,timestampCol=1658484765000000t\n" + "tableName boolCol=f,timestampCol=1658484766000000t\n", ); @@ -795,9 +780,24 @@ describe("Sender message builder test suite (anything not covered in client inte .floatColumn("floatCol", 1234567890) .timestampColumn("timestampCol", 1658484767000000) .atNow(); - expect(sender.toBufferView().toString()).toBe( + expect(bufferContent(sender)).toBe( "tableName floatCol=1234567890,timestampCol=1658484767000000t\n", ); await sender.close(); }); }); + +function bufferContent(sender: Sender) { + // @ts-expect-error - Accessing private field + return sender.buffer.toBufferView().toString(); +} + +function bufferSize(sender: Sender) { + // @ts-expect-error - Accessing private field + return sender.buffer.bufferSize; +} + +function bufferPosition(sender: Sender) { + // @ts-expect-error - Accessing private field + return sender.buffer.position; +} diff --git a/test/sender.config.test.ts b/test/sender.config.test.ts index 6676d65..7c70a7c 100644 --- a/test/sender.config.test.ts +++ b/test/sender.config.test.ts @@ -1,7 +1,8 @@ // @ts-check import { describe, it, expect } from "vitest"; -import { Sender, DEFAULT_BUFFER_SIZE, DEFAULT_MAX_BUFFER_SIZE } from "../src/sender"; +import { Sender } from "../src"; +import { DEFAULT_BUFFER_SIZE, DEFAULT_MAX_BUFFER_SIZE } from "../src/buffer"; import { log } from "../src/logging"; describe("Sender configuration options suite", function () { @@ -55,7 +56,7 @@ describe("Sender configuration options suite", function () { describe("Sender options test suite", function () { it("fails if no options defined", async function () { await expect(async () => - // @ts-expect-error - Testing invalid options + // @ts-expect-error - Testing invalid options await new Sender().close() ).rejects.toThrow("The 'protocol' option is mandatory"); }); @@ -97,8 +98,7 @@ describe("Sender options test suite", function () { protocol: "http", host: "host", }); - // @ts-expect-error - Accessing private field - expect(sender.bufferSize).toBe(DEFAULT_BUFFER_SIZE); + expect(bufferSize(sender)).toBe(DEFAULT_BUFFER_SIZE); await sender.close(); }); @@ -108,8 +108,7 @@ describe("Sender options test suite", function () { host: "host", init_buf_size: 1024, }); - // @ts-expect-error - Accessing private field - expect(sender.bufferSize).toBe(1024); + expect(bufferSize(sender)).toBe(1024); await sender.close(); }); @@ -119,8 +118,7 @@ describe("Sender options test suite", function () { host: "host", init_buf_size: null, }); - // @ts-expect-error - Accessing private field - expect(sender.bufferSize).toBe(DEFAULT_BUFFER_SIZE); + expect(bufferSize(sender)).toBe(DEFAULT_BUFFER_SIZE); await sender.close(); }); @@ -130,8 +128,7 @@ describe("Sender options test suite", function () { host: "host", init_buf_size: undefined, }); - // @ts-expect-error - Accessing private field - expect(sender.bufferSize).toBe(DEFAULT_BUFFER_SIZE); + expect(bufferSize(sender)).toBe(DEFAULT_BUFFER_SIZE); await sender.close(); }); @@ -142,15 +139,16 @@ describe("Sender options test suite", function () { // @ts-expect-error - Testing invalid options init_buf_size: "1024", }); - // @ts-expect-error - Accessing private field - expect(sender.bufferSize).toBe(DEFAULT_BUFFER_SIZE); + expect(bufferSize(sender)).toBe(DEFAULT_BUFFER_SIZE); await sender.close(); }); it("sets the requested buffer size if 'bufferSize' is set, but warns that it is deprecated", async function () { const log = (level: "error" | "warn" | "info" | "debug", message: string | Error) => { - expect(level).toBe("warn"); - expect(message).toMatch("Option 'bufferSize' is not supported anymore, please, replace it with 'init_buf_size'"); + if (level !== "debug") { + expect(level).toBe("warn"); + expect(message).toMatch("Option 'bufferSize' is not supported anymore, please, replace it with 'init_buf_size'"); + } }; const sender = new Sender({ protocol: "http", @@ -159,15 +157,16 @@ describe("Sender options test suite", function () { bufferSize: 2048, log: log, }); - // @ts-expect-error - Accessing private field - expect(sender.bufferSize).toBe(2048); + expect(bufferSize(sender)).toBe(2048); await sender.close(); }); it("warns about deprecated option 'copy_buffer'", async function () { const log = (level: "error" | "warn" | "info" | "debug", message: string) => { - expect(level).toBe("warn"); - expect(message).toMatch("Option 'copy_buffer' is not supported anymore, please, remove it"); + if (level !== "debug") { + expect(level).toBe("warn"); + expect(message).toMatch("Option 'copy_buffer' is not supported anymore, please, remove it"); + } }; const sender = new Sender({ protocol: "http", @@ -181,8 +180,10 @@ describe("Sender options test suite", function () { it("warns about deprecated option 'copyBuffer'", async function () { const log = (level: "error" | "warn" | "info" | "debug", message: string) => { - expect(level).toBe("warn"); - expect(message).toMatch("Option 'copyBuffer' is not supported anymore, please, remove it"); + if (level !== "debug") { + expect(level).toBe("warn"); + expect(message).toMatch("Option 'copyBuffer' is not supported anymore, please, remove it"); + } }; const sender = new Sender({ protocol: "http", @@ -196,8 +197,7 @@ describe("Sender options test suite", function () { it("sets default max buffer size if max_buf_size is not set", async function () { const sender = new Sender({ protocol: "http", host: "host" }); - // @ts-expect-error - Accessing private field - expect(sender.maxBufferSize).toBe(DEFAULT_MAX_BUFFER_SIZE); + expect(maxBufferSize(sender)).toBe(DEFAULT_MAX_BUFFER_SIZE); await sender.close(); }); @@ -207,8 +207,7 @@ describe("Sender options test suite", function () { host: "host", max_buf_size: 131072, }); - // @ts-expect-error - Accessing private field - expect(sender.maxBufferSize).toBe(131072); + expect(maxBufferSize(sender)).toBe(131072); await sender.close(); }); @@ -229,8 +228,7 @@ describe("Sender options test suite", function () { host: "host", max_buf_size: null, }); - // @ts-expect-error - Accessing private field - expect(sender.maxBufferSize).toBe(DEFAULT_MAX_BUFFER_SIZE); + expect(maxBufferSize(sender)).toBe(DEFAULT_MAX_BUFFER_SIZE); await sender.close(); }); @@ -240,8 +238,7 @@ describe("Sender options test suite", function () { host: "host", max_buf_size: undefined, }); - // @ts-expect-error - Accessing private field - expect(sender.maxBufferSize).toBe(DEFAULT_MAX_BUFFER_SIZE); + expect(maxBufferSize(sender)).toBe(DEFAULT_MAX_BUFFER_SIZE); await sender.close(); }); @@ -252,15 +249,13 @@ describe("Sender options test suite", function () { // @ts-expect-error - Testing invalid value max_buf_size: "1024", }); - // @ts-expect-error - Accessing private field - expect(sender.maxBufferSize).toBe(DEFAULT_MAX_BUFFER_SIZE); + expect(maxBufferSize(sender)).toBe(DEFAULT_MAX_BUFFER_SIZE); await sender.close(); }); it("uses default logger if log function is not set", async function () { const sender = new Sender({ protocol: "http", host: "host" }); - // @ts-expect-error - Accessing private field - expect(sender.log).toBe(log); + expect(logger(sender)).toBe(log); await sender.close(); }); @@ -271,15 +266,13 @@ describe("Sender options test suite", function () { host: "host", log: testFunc, }); - // @ts-expect-error - Accessing private field - expect(sender.log).toBe(testFunc); + expect(logger(sender)).toBe(testFunc); await sender.close(); }); it("uses default logger if log is set to null", async function () { const sender = new Sender({ protocol: "http", host: "host", log: null }); - // @ts-expect-error - Accessing private field - expect(sender.log).toBe(log); + expect(logger(sender)).toBe(log); await sender.close(); }); @@ -289,16 +282,14 @@ describe("Sender options test suite", function () { host: "host", log: undefined, }); - // @ts-expect-error - Accessing private field - expect(sender.log).toBe(log); + expect(logger(sender)).toBe(log); await sender.close(); }); it("uses default logger if log is not a function", async function () { // @ts-expect-error - Testing invalid options const sender = new Sender({ protocol: "http", host: "host", log: "" }); - // @ts-expect-error - Accessing private field - expect(sender.log).toBe(log); + expect(logger(sender)).toBe(log); await sender.close(); }); }); @@ -400,3 +391,18 @@ describe("Sender auth config checks suite", function () { ); }); }); + +function bufferSize(sender: Sender) { + // @ts-expect-error - Accessing private field + return sender.buffer.bufferSize; +} + +function maxBufferSize(sender: Sender) { + // @ts-expect-error - Accessing private field + return sender.buffer.maxBufferSize; +} + +function logger(sender: Sender) { + // @ts-expect-error - Accessing private field + return sender.log; +} diff --git a/test/sender.transport.test.ts b/test/sender.transport.test.ts index 3b32fcd..39edd9d 100644 --- a/test/sender.transport.test.ts +++ b/test/sender.transport.test.ts @@ -545,8 +545,10 @@ describe("Sender TCP suite", function () { /^Connection to .*1:9088 is closed$/, ]; const log = (level: "error" | "warn" | "info" | "debug", message: string) => { - expect(level).toBe("info"); - expect(message).toMatch(expectedMessages.shift()); + if (level !== "debug") { + expect(level).toBe("info"); + expect(message).toMatch(expectedMessages.shift()); + } }; const proxy = await createProxy(); const sender = new Sender({ From 7f4d4548f3f3c631273f4e96ace226c2c6c3f53c Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 17 Jul 2025 03:17:55 +0100 Subject: [PATCH 5/6] TimestampUnit type --- src/buffer/index.ts | 10 +++------- src/sender.ts | 10 +++------- src/utils.ts | 12 +++++++----- 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/src/buffer/index.ts b/src/buffer/index.ts index eb8f0a0..3b2072a 100644 --- a/src/buffer/index.ts +++ b/src/buffer/index.ts @@ -4,7 +4,7 @@ import { Buffer } from "node:buffer"; import { log, Logger } from "../logging"; import { validateColumnName, validateTableName } from "../validation"; import { SenderOptions } from "../options"; -import { isInteger, timestampToMicros, timestampToNanos } from "../utils"; +import { isInteger, timestampToMicros, timestampToNanos, TimestampUnit } from "../utils"; const DEFAULT_MAX_NAME_LENGTH = 127; @@ -262,11 +262,7 @@ class SenderBuffer { * @param {string} [unit=us] - Timestamp unit. Supported values: 'ns' - nanoseconds, 'us' - microseconds, 'ms' - milliseconds. Defaults to 'us'. * @return {Sender} Returns with a reference to this sender. */ - timestampColumn( - name: string, - value: number | bigint, - unit: "ns" | "us" | "ms" = "us", - ): SenderBuffer { + timestampColumn(name: string, value: number | bigint, unit: TimestampUnit = "us"): SenderBuffer { if (typeof value !== "bigint" && !Number.isInteger(value)) { throw new Error(`Value must be an integer or BigInt, received ${value}`); } @@ -286,7 +282,7 @@ class SenderBuffer { * @param {number | bigint} timestamp - Designated epoch timestamp, accepts numbers or BigInts. * @param {string} [unit=us] - Timestamp unit. Supported values: 'ns' - nanoseconds, 'us' - microseconds, 'ms' - milliseconds. Defaults to 'us'. */ - at(timestamp: number | bigint, unit: "ns" | "us" | "ms" = "us") { + at(timestamp: number | bigint, unit: TimestampUnit = "us") { if (!this.hasSymbols && !this.hasColumns) { throw new Error("The row must have a symbol or column set before it is closed"); } diff --git a/src/sender.ts b/src/sender.ts index e174018..b94f969 100644 --- a/src/sender.ts +++ b/src/sender.ts @@ -2,7 +2,7 @@ import { log, Logger } from "./logging"; import { SenderOptions, ExtraOptions } from "./options"; import { SenderTransport, createTransport } from "./transport"; -import { isBoolean, isInteger } from "./utils"; +import { isBoolean, isInteger, TimestampUnit } from "./utils"; import { SenderBuffer } from "./buffer"; const DEFAULT_AUTO_FLUSH_INTERVAL = 1000; // 1 sec @@ -248,11 +248,7 @@ class Sender { * @param {string} [unit=us] - Timestamp unit. Supported values: 'ns' - nanoseconds, 'us' - microseconds, 'ms' - milliseconds. Defaults to 'us'. * @return {Sender} Returns with a reference to this sender. */ - timestampColumn( - name: string, - value: number | bigint, - unit: "ns" | "us" | "ms" = "us", - ): Sender { + timestampColumn(name: string, value: number | bigint, unit: TimestampUnit = "us"): Sender { this.buffer.timestampColumn(name, value, unit); return this; } @@ -263,7 +259,7 @@ class Sender { * @param {number | bigint} timestamp - Designated epoch timestamp, accepts numbers or BigInts. * @param {string} [unit=us] - Timestamp unit. Supported values: 'ns' - nanoseconds, 'us' - microseconds, 'ms' - milliseconds. Defaults to 'us'. */ - async at(timestamp: number | bigint, unit: "ns" | "us" | "ms" = "us") { + async at(timestamp: number | bigint, unit: TimestampUnit = "us") { this.buffer.at(timestamp, unit); this.pendingRowCount++; this.log("debug", `Pending row count: ${this.pendingRowCount}`); diff --git a/src/utils.ts b/src/utils.ts index dd82db6..1962242 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,3 +1,5 @@ +type TimestampUnit = "ns" | "us" | "ms"; + function isBoolean(value: unknown): value is boolean { return typeof value === "boolean"; } @@ -8,7 +10,7 @@ function isInteger(value: unknown, lowerBound: number): value is number { ); } -function timestampToMicros(timestamp: bigint, unit: "ns" | "us" | "ms") { +function timestampToMicros(timestamp: bigint, unit: TimestampUnit) { switch (unit) { case "ns": return timestamp / 1000n; @@ -17,11 +19,11 @@ function timestampToMicros(timestamp: bigint, unit: "ns" | "us" | "ms") { case "ms": return timestamp * 1000n; default: - throw new Error("Unknown timestamp unit: " + unit); + throw new Error(`Unknown timestamp unit: ${unit}`); } } -function timestampToNanos(timestamp: bigint, unit: "ns" | "us" | "ms") { +function timestampToNanos(timestamp: bigint, unit: TimestampUnit) { switch (unit) { case "ns": return timestamp; @@ -30,8 +32,8 @@ function timestampToNanos(timestamp: bigint, unit: "ns" | "us" | "ms") { case "ms": return timestamp * 1000_000n; default: - throw new Error("Unknown timestamp unit: " + unit); + throw new Error(`Unknown timestamp unit: ${unit}`); } } -export { isBoolean, isInteger, timestampToMicros, timestampToNanos }; +export { isBoolean, isInteger, timestampToMicros, timestampToNanos, TimestampUnit }; From f54ad6527ddd8ee3fb2146316498a4934a66bd7b Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 23 Jul 2025 23:25:03 +0100 Subject: [PATCH 6/6] code formatting --- package.json | 1 + src/buffer/index.ts | 61 ++++++--- src/logging.ts | 9 +- src/options.ts | 49 +++++-- src/sender.ts | 27 ++-- src/transport/http/base.ts | 19 +-- src/transport/http/legacy.ts | 81 +++++++---- src/transport/http/undici.ts | 57 +++++--- src/transport/index.ts | 6 +- src/transport/tcp.ts | 64 +++++---- src/utils.ts | 10 +- test/logging.test.ts | 18 ++- test/sender.buffer.test.ts | 69 +++++----- test/sender.config.test.ts | 234 ++++++++++++++++++-------------- test/sender.integration.test.ts | 38 ++++-- test/sender.transport.test.ts | 35 ++--- test/util/mockhttp.ts | 75 +++++----- test/util/mockproxy.ts | 4 +- test/util/proxy.ts | 9 +- test/util/proxyfunctions.ts | 16 ++- 20 files changed, 548 insertions(+), 334 deletions(-) diff --git a/package.json b/package.json index 4c10ee7..3b6b129 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "build": "bunchee", "eslint": "eslint src/**", "typecheck": "tsc --noEmit", + "format": "prettier --write '{src,test}/**/*.{ts,js,json}'", "docs": "pnpm run build && jsdoc ./dist/cjs/index.js README.md -d docs", "preview:docs": "serve docs" }, diff --git a/src/buffer/index.ts b/src/buffer/index.ts index 3b2072a..383f23f 100644 --- a/src/buffer/index.ts +++ b/src/buffer/index.ts @@ -4,7 +4,12 @@ import { Buffer } from "node:buffer"; import { log, Logger } from "../logging"; import { validateColumnName, validateTableName } from "../validation"; import { SenderOptions } from "../options"; -import { isInteger, timestampToMicros, timestampToNanos, TimestampUnit } from "../utils"; +import { + isInteger, + timestampToMicros, + timestampToNanos, + TimestampUnit, +} from "../utils"; const DEFAULT_MAX_NAME_LENGTH = 127; @@ -42,15 +47,17 @@ class SenderBuffer { this.log = options && typeof options.log === "function" ? options.log : log; SenderOptions.resolveDeprecated(options, this.log); - this.maxNameLength = options && isInteger(options.max_name_len, 1) - ? options.max_name_len - : DEFAULT_MAX_NAME_LENGTH; + this.maxNameLength = + options && isInteger(options.max_name_len, 1) + ? options.max_name_len + : DEFAULT_MAX_NAME_LENGTH; - this.maxBufferSize = options && isInteger(options.max_buf_size, 1) - ? options.max_buf_size - : DEFAULT_MAX_BUFFER_SIZE; + this.maxBufferSize = + options && isInteger(options.max_buf_size, 1) + ? options.max_buf_size + : DEFAULT_MAX_BUFFER_SIZE; this.resize( - options && isInteger(options.init_buf_size, 1) + options && isInteger(options.init_buf_size, 1) ? options.init_buf_size : DEFAULT_BUFFER_SIZE, ); @@ -67,7 +74,9 @@ class SenderBuffer { */ private resize(bufferSize: number) { if (bufferSize > this.maxBufferSize) { - throw new Error(`Max buffer size is ${this.maxBufferSize} bytes, requested buffer size: ${bufferSize}`); + throw new Error( + `Max buffer size is ${this.maxBufferSize} bytes, requested buffer size: ${bufferSize}`, + ); } this.bufferSize = bufferSize; // Allocating an extra byte because Buffer.write() does not fail if the length of the data to be written is @@ -158,7 +167,9 @@ class SenderBuffer { throw new Error(`Symbol name must be a string, received ${typeof name}`); } if (!this.hasTable || this.hasColumns) { - throw new Error("Symbol can be added only after table name is set and before any column added"); + throw new Error( + "Symbol can be added only after table name is set and before any column added", + ); } const valueStr = value.toString(); this.checkCapacity([name, valueStr], 2 + name.length + valueStr.length); @@ -262,7 +273,11 @@ class SenderBuffer { * @param {string} [unit=us] - Timestamp unit. Supported values: 'ns' - nanoseconds, 'us' - microseconds, 'ms' - milliseconds. Defaults to 'us'. * @return {Sender} Returns with a reference to this sender. */ - timestampColumn(name: string, value: number | bigint, unit: TimestampUnit = "us"): SenderBuffer { + timestampColumn( + name: string, + value: number | bigint, + unit: TimestampUnit = "us", + ): SenderBuffer { if (typeof value !== "bigint" && !Number.isInteger(value)) { throw new Error(`Value must be an integer or BigInt, received ${value}`); } @@ -284,10 +299,14 @@ class SenderBuffer { */ at(timestamp: number | bigint, unit: TimestampUnit = "us") { if (!this.hasSymbols && !this.hasColumns) { - throw new Error("The row must have a symbol or column set before it is closed"); + throw new Error( + "The row must have a symbol or column set before it is closed", + ); } if (typeof timestamp !== "bigint" && !Number.isInteger(timestamp)) { - throw new Error(`Designated timestamp must be an integer or BigInt, received ${timestamp}`); + throw new Error( + `Designated timestamp must be an integer or BigInt, received ${timestamp}`, + ); } const timestampNanos = timestampToNanos(BigInt(timestamp), unit); const timestampStr = timestampNanos.toString(); @@ -304,7 +323,9 @@ class SenderBuffer { */ atNow() { if (!this.hasSymbols && !this.hasColumns) { - throw new Error("The row must have a symbol or column set before it is closed"); + throw new Error( + "The row must have a symbol or column set before it is closed", + ); } this.checkCapacity([], 1); this.write("\n"); @@ -334,17 +355,17 @@ class SenderBuffer { } private writeColumn( - name: string, - value: unknown, - writeValue: () => void, - valueType?: string + name: string, + value: unknown, + writeValue: () => void, + valueType?: string, ) { if (typeof name !== "string") { throw new Error(`Column name must be a string, received ${typeof name}`); } if (valueType && typeof value !== valueType) { throw new Error( - `Column value must be of type ${valueType}, received ${typeof value}`, + `Column value must be of type ${valueType}, received ${typeof value}`, ); } if (!this.hasTable) { @@ -364,7 +385,7 @@ class SenderBuffer { if (this.position > this.bufferSize) { // should never happen, if checkCapacity() is correctly used throw new Error( - `Buffer overflow [position=${this.position}, bufferSize=${this.bufferSize}]`, + `Buffer overflow [position=${this.position}, bufferSize=${this.bufferSize}]`, ); } } diff --git a/src/logging.ts b/src/logging.ts index 48098b2..751629e 100644 --- a/src/logging.ts +++ b/src/logging.ts @@ -8,8 +8,8 @@ const LOG_LEVELS = { const DEFAULT_CRITICALITY = LOG_LEVELS.info.criticality; type Logger = ( - level: "error" | "warn" | "info" | "debug", - message: string | Error, + level: "error" | "warn" | "info" | "debug", + message: string | Error, ) => void; /** @@ -20,7 +20,10 @@ type Logger = ( * @param {'error'|'warn'|'info'|'debug'} level - The log level of the message. * @param {string | Error} message - The log message. */ -function log(level: "error" | "warn" | "info" | "debug", message: string | Error) { +function log( + level: "error" | "warn" | "info" | "debug", + message: string | Error, +) { const logLevel = LOG_LEVELS[level]; if (!logLevel) { throw new Error(`Invalid log level: '${level}'`); diff --git a/src/options.ts b/src/options.ts index 8fcb189..f7ec648 100644 --- a/src/options.ts +++ b/src/options.ts @@ -20,7 +20,7 @@ const UNSAFE_OFF = "unsafe_off"; type ExtraOptions = { log?: Logger; agent?: Agent | http.Agent | https.Agent; -} +}; type DeprecatedOptions = { /** @deprecated */ @@ -190,11 +190,11 @@ class SenderOptions { this.log = extraOptions.log; if ( - extraOptions.agent - && !(extraOptions.agent instanceof Agent) - && !(extraOptions.agent instanceof http.Agent) - // @ts-expect-error - Not clear what the problem is, the two lines above have no issues - && !(extraOptions.agent instanceof https.Agent) + extraOptions.agent && + !(extraOptions.agent instanceof Agent) && + !(extraOptions.agent instanceof http.Agent) && + // @ts-expect-error - Not clear what the problem is, the two lines above have no issues + !(extraOptions.agent instanceof https.Agent) ) { throw new Error("Invalid HTTP agent"); } @@ -202,22 +202,34 @@ class SenderOptions { } } - static resolveDeprecated(options: SenderOptions & DeprecatedOptions, log: Logger) { + static resolveDeprecated( + options: SenderOptions & DeprecatedOptions, + log: Logger, + ) { if (!options) { return; } // deal with deprecated options if (options.copy_buffer !== undefined) { - log("warn", `Option 'copy_buffer' is not supported anymore, please, remove it`); + log( + "warn", + `Option 'copy_buffer' is not supported anymore, please, remove it`, + ); options.copy_buffer = undefined; } if (options.copyBuffer !== undefined) { - log("warn", `Option 'copyBuffer' is not supported anymore, please, remove it`); + log( + "warn", + `Option 'copyBuffer' is not supported anymore, please, remove it`, + ); options.copyBuffer = undefined; } if (options.bufferSize !== undefined) { - log("warn", `Option 'bufferSize' is not supported anymore, please, replace it with 'init_buf_size'`); + log( + "warn", + `Option 'bufferSize' is not supported anymore, please, replace it with 'init_buf_size'`, + ); options.init_buf_size = options.bufferSize; options.bufferSize = undefined; } @@ -235,7 +247,10 @@ class SenderOptions { * * @return {SenderOptions} A Sender configuration object initialized from the provided configuration string. */ - static fromConfig(configurationString: string, extraOptions?: ExtraOptions): SenderOptions { + static fromConfig( + configurationString: string, + extraOptions?: ExtraOptions, + ): SenderOptions { return new SenderOptions(configurationString, extraOptions); } @@ -255,7 +270,10 @@ class SenderOptions { } } -function parseConfigurationString(options: SenderOptions, configString: string) { +function parseConfigurationString( + options: SenderOptions, + configString: string, +) { if (!configString) { throw new Error("Configuration string is missing or empty"); } @@ -278,7 +296,10 @@ function parseSettings( ) { let index = configString.indexOf(";", position); while (index > -1) { - if (index + 1 < configString.length && configString.charAt(index + 1) === ";") { + if ( + index + 1 < configString.length && + configString.charAt(index + 1) === ";" + ) { index = configString.indexOf(";", index + 2); continue; } @@ -436,7 +457,7 @@ function parseTlsOptions(options: SenderOptions) { if (options.tls_roots || options.tls_roots_password) { throw new Error( "'tls_roots' and 'tls_roots_password' options are not supported, please, " + - "use the 'tls_ca' option or the NODE_EXTRA_CA_CERTS environment variable instead", + "use the 'tls_ca' option or the NODE_EXTRA_CA_CERTS environment variable instead", ); } } diff --git a/src/sender.ts b/src/sender.ts index b94f969..62db289 100644 --- a/src/sender.ts +++ b/src/sender.ts @@ -99,7 +99,10 @@ class Sender { * * @return {Sender} A Sender object initialized from the provided configuration string. */ - static fromConfig(configurationString: string, extraOptions?: ExtraOptions): Sender { + static fromConfig( + configurationString: string, + extraOptions?: ExtraOptions, + ): Sender { return new Sender( SenderOptions.fromConfig(configurationString, extraOptions), ); @@ -155,7 +158,10 @@ class Sender { return false; // Nothing to send } - this.log("debug", `Flushing, number of flushed rows: ${this.pendingRowCount}`); + this.log( + "debug", + `Flushing, number of flushed rows: ${this.pendingRowCount}`, + ); this.resetAutoFlush(); await this.transport.send(dataToSend); @@ -248,7 +254,11 @@ class Sender { * @param {string} [unit=us] - Timestamp unit. Supported values: 'ns' - nanoseconds, 'us' - microseconds, 'ms' - milliseconds. Defaults to 'us'. * @return {Sender} Returns with a reference to this sender. */ - timestampColumn(name: string, value: number | bigint, unit: TimestampUnit = "us"): Sender { + timestampColumn( + name: string, + value: number | bigint, + unit: TimestampUnit = "us", + ): Sender { this.buffer.timestampColumn(name, value, unit); return this; } @@ -285,12 +295,11 @@ class Sender { private async tryFlush() { if ( - this.autoFlush - && this.pendingRowCount > 0 - && ( - (this.autoFlushRows > 0 && this.pendingRowCount >= this.autoFlushRows) - || (this.autoFlushInterval > 0 && Date.now() - this.lastFlushTime >= this.autoFlushInterval) - ) + this.autoFlush && + this.pendingRowCount > 0 && + ((this.autoFlushRows > 0 && this.pendingRowCount >= this.autoFlushRows) || + (this.autoFlushInterval > 0 && + Date.now() - this.lastFlushTime >= this.autoFlushInterval)) ) { await this.flush(); } diff --git a/src/transport/http/base.ts b/src/transport/http/base.ts index 0d42b61..cdaf5f3 100644 --- a/src/transport/http/base.ts +++ b/src/transport/http/base.ts @@ -76,14 +76,14 @@ abstract class HttpTransportBase implements SenderTransport { this.port = options.port; this.requestMinThroughput = isInteger(options.request_min_throughput, 0) - ? options.request_min_throughput - : DEFAULT_REQUEST_MIN_THROUGHPUT; + ? options.request_min_throughput + : DEFAULT_REQUEST_MIN_THROUGHPUT; this.requestTimeout = isInteger(options.request_timeout, 1) - ? options.request_timeout - : DEFAULT_REQUEST_TIMEOUT; + ? options.request_timeout + : DEFAULT_REQUEST_TIMEOUT; this.retryTimeout = isInteger(options.retry_timeout, 0) - ? options.retry_timeout - : DEFAULT_RETRY_TIMEOUT; + ? options.retry_timeout + : DEFAULT_RETRY_TIMEOUT; switch (options.protocol) { case HTTP: @@ -93,7 +93,9 @@ abstract class HttpTransportBase implements SenderTransport { this.secure = true; break; default: - throw new Error("The 'protocol' has to be 'http' or 'https' for the HTTP transport"); + throw new Error( + "The 'protocol' has to be 'http' or 'https' for the HTTP transport", + ); } } @@ -101,8 +103,7 @@ abstract class HttpTransportBase implements SenderTransport { throw new Error("'connect()' is not required for HTTP transport"); } - async close(): Promise { - } + async close(): Promise {} getDefaultAutoFlushRows(): number { return DEFAULT_HTTP_AUTO_FLUSH_ROWS; diff --git a/src/transport/http/legacy.ts b/src/transport/http/legacy.ts index d62852f..e51a88f 100644 --- a/src/transport/http/legacy.ts +++ b/src/transport/http/legacy.ts @@ -4,7 +4,11 @@ import https from "https"; import { Buffer } from "node:buffer"; import { SenderOptions, HTTP, HTTPS } from "../../options"; -import { HttpTransportBase, RETRIABLE_STATUS_CODES, HTTP_NO_CONTENT } from "./base"; +import { + HttpTransportBase, + RETRIABLE_STATUS_CODES, + HTTP_NO_CONTENT, +} from "./base"; // default options for HTTP agent // - persistent connections with 1 minute idle timeout, server side has 5 minutes set by default @@ -12,8 +16,8 @@ import { HttpTransportBase, RETRIABLE_STATUS_CODES, HTTP_NO_CONTENT } from "./ba const DEFAULT_HTTP_AGENT_CONFIG = { maxSockets: 256, keepAlive: true, - timeout: 60000 // 1 min -} + timeout: 60000, // 1 min +}; /** @classdesc * The QuestDB client's API provides methods to connect to the database, ingest data, and close the connection. @@ -37,60 +41,81 @@ class HttpTransport extends HttpTransportBase { switch (options.protocol) { case HTTP: - this.agent = options.agent instanceof http.Agent ? options.agent : HttpTransport.getDefaultHttpAgent(); + this.agent = + options.agent instanceof http.Agent + ? options.agent + : HttpTransport.getDefaultHttpAgent(); break; case HTTPS: - this.agent = options.agent instanceof https.Agent ? options.agent : HttpTransport.getDefaultHttpsAgent(); + this.agent = + options.agent instanceof https.Agent + ? options.agent + : HttpTransport.getDefaultHttpsAgent(); break; default: - throw new Error("The 'protocol' has to be 'http' or 'https' for the HTTP transport"); + throw new Error( + "The 'protocol' has to be 'http' or 'https' for the HTTP transport", + ); } } send(data: Buffer, retryBegin = -1, retryInterval = -1): Promise { const request = this.secure ? https.request : http.request; - const timeoutMillis = (data.length / this.requestMinThroughput) * 1000 + this.requestTimeout; + const timeoutMillis = + (data.length / this.requestMinThroughput) * 1000 + this.requestTimeout; const options = this.createRequestOptions(timeoutMillis); return new Promise((resolve, reject) => { let statusCode = -1; - const req = request(options, response => { + const req = request(options, (response) => { statusCode = response.statusCode; const body = []; response - .on("data", chunk => { - body.push(chunk); - }) - .on("error", err => { - this.log("error", `resp err=${err}`); - }); + .on("data", (chunk) => { + body.push(chunk); + }) + .on("error", (err) => { + this.log("error", `resp err=${err}`); + }); if (statusCode === HTTP_NO_CONTENT) { response.on("end", () => { if (body.length > 0) { - this.log("warn", `Unexpected message from server: ${Buffer.concat(body)}`); + this.log( + "warn", + `Unexpected message from server: ${Buffer.concat(body)}`, + ); } resolve(true); }); } else { - req.destroy(new Error(`HTTP request failed, statusCode=${statusCode}, error=${Buffer.concat(body)}`)); + req.destroy( + new Error( + `HTTP request failed, statusCode=${statusCode}, error=${Buffer.concat(body)}`, + ), + ); } }); if (this.token) { req.setHeader("Authorization", `Bearer ${this.token}`); } else if (this.username && this.password) { - req.setHeader("Authorization", `Basic ${Buffer.from(`${this.username}:${this.password}`).toString("base64")}`); + req.setHeader( + "Authorization", + `Basic ${Buffer.from(`${this.username}:${this.password}`).toString("base64")}`, + ); } req.on("timeout", () => { // set a retryable error code statusCode = 524; - req.destroy(new Error("HTTP request timeout, no response from server in time")); + req.destroy( + new Error("HTTP request timeout, no response from server in time"), + ); }); - req.on("error", err => { + req.on("error", (err) => { // if the error is thrown while the request is sent, statusCode is -1 => no retry // request timeout comes through with statusCode 524 => retry // if the error is thrown while the response is processed, the statusCode is taken from the response => retry depends on statusCode @@ -109,19 +134,21 @@ class HttpTransport extends HttpTransportBase { setTimeout(() => { retryInterval = Math.min(retryInterval * 2, 1000); this.send(data, retryBegin, retryInterval) - .then(() => resolve(true)) - .catch(e => reject(e)); + .then(() => resolve(true)) + .catch((e) => reject(e)); }, retryInterval + jitter); } else { reject(err); } }); - req.write(data, err => err ? reject(err) : () => {}); + req.write(data, (err) => (err ? reject(err) : () => {})); req.end(); }); } - private createRequestOptions(timeoutMillis: number): http.RequestOptions | https.RequestOptions { + private createRequestOptions( + timeoutMillis: number, + ): http.RequestOptions | https.RequestOptions { return { //protocol: this.secure ? "https:" : "http:", hostname: this.host, @@ -141,7 +168,9 @@ class HttpTransport extends HttpTransportBase { */ private static getDefaultHttpAgent(): http.Agent { if (!HttpTransport.DEFAULT_HTTP_AGENT) { - HttpTransport.DEFAULT_HTTP_AGENT = new http.Agent(DEFAULT_HTTP_AGENT_CONFIG); + HttpTransport.DEFAULT_HTTP_AGENT = new http.Agent( + DEFAULT_HTTP_AGENT_CONFIG, + ); } return HttpTransport.DEFAULT_HTTP_AGENT; } @@ -152,7 +181,9 @@ class HttpTransport extends HttpTransportBase { */ private static getDefaultHttpsAgent(): https.Agent { if (!HttpTransport.DEFAULT_HTTPS_AGENT) { - HttpTransport.DEFAULT_HTTPS_AGENT = new https.Agent(DEFAULT_HTTP_AGENT_CONFIG); + HttpTransport.DEFAULT_HTTPS_AGENT = new https.Agent( + DEFAULT_HTTP_AGENT_CONFIG, + ); } return HttpTransport.DEFAULT_HTTPS_AGENT; } diff --git a/src/transport/http/undici.ts b/src/transport/http/undici.ts index 43575fa..c6545cd 100644 --- a/src/transport/http/undici.ts +++ b/src/transport/http/undici.ts @@ -4,7 +4,11 @@ import { Agent, RetryAgent } from "undici"; import Dispatcher from "undici/types/dispatcher"; import { SenderOptions, HTTP, HTTPS } from "../../options"; -import { HttpTransportBase, RETRIABLE_STATUS_CODES, HTTP_NO_CONTENT } from "./base"; +import { + HttpTransportBase, + RETRIABLE_STATUS_CODES, + HTTP_NO_CONTENT, +} from "./base"; const DEFAULT_HTTP_OPTIONS: Agent.Options = { connect: { @@ -24,7 +28,7 @@ class UndiciTransport extends HttpTransportBase { private static DEFAULT_HTTP_AGENT: Agent; private readonly agent: Dispatcher; - private readonly dispatcher : RetryAgent; + private readonly dispatcher: RetryAgent; /** * Creates an instance of Sender. @@ -37,7 +41,10 @@ class UndiciTransport extends HttpTransportBase { switch (options.protocol) { case HTTP: - this.agent = options.agent instanceof Agent ? options.agent : UndiciTransport.getDefaultHttpAgent(); + this.agent = + options.agent instanceof Agent + ? options.agent + : UndiciTransport.getDefaultHttpAgent(); break; case HTTPS: if (options.agent instanceof Agent) { @@ -56,7 +63,9 @@ class UndiciTransport extends HttpTransportBase { } break; default: - throw new Error("The 'protocol' has to be 'http' or 'https' for the Undici HTTP transport"); + throw new Error( + "The 'protocol' has to be 'http' or 'https' for the Undici HTTP transport", + ); } this.dispatcher = new RetryAgent(this.agent, { @@ -85,7 +94,8 @@ class UndiciTransport extends HttpTransportBase { if (this.token) { headers["Authorization"] = `Bearer ${this.token}`; } else if (this.username && this.password) { - headers["Authorization"] = `Basic ${Buffer.from(`${this.username}:${this.password}`).toString("base64")}`; + headers["Authorization"] = + `Basic ${Buffer.from(`${this.username}:${this.password}`).toString("base64")}`; } const controller = new AbortController(); @@ -94,36 +104,41 @@ class UndiciTransport extends HttpTransportBase { let responseData: Dispatcher.ResponseData; try { - const timeoutMillis = (data.length / this.requestMinThroughput) * 1000 + this.requestTimeout; - responseData = - await this.dispatcher.request({ - origin: `${this.secure ? "https" : "http"}://${this.host}:${this.port}`, - path: "/write?precision=n", - method: "POST", - headers, - body: data, - headersTimeout: this.requestTimeout, - bodyTimeout: timeoutMillis, - signal, - }); + const timeoutMillis = + (data.length / this.requestMinThroughput) * 1000 + this.requestTimeout; + responseData = await this.dispatcher.request({ + origin: `${this.secure ? "https" : "http"}://${this.host}:${this.port}`, + path: "/write?precision=n", + method: "POST", + headers, + body: data, + headersTimeout: this.requestTimeout, + bodyTimeout: timeoutMillis, + signal, + }); } catch (err) { if (err.name === "AbortError") { - throw new Error("HTTP request timeout, no response from server in time"); + throw new Error( + "HTTP request timeout, no response from server in time", + ); } else { throw err; } } - const { statusCode} = responseData; + const { statusCode } = responseData; const body = await responseData.body.arrayBuffer(); if (statusCode === HTTP_NO_CONTENT) { if (body.byteLength > 0) { - this.log("warn", `Unexpected message from server: ${Buffer.from(body).toString()}`); + this.log( + "warn", + `Unexpected message from server: ${Buffer.from(body).toString()}`, + ); } return true; } else { throw new Error( - `HTTP request failed, statusCode=${statusCode}, error=${Buffer.from(body).toString()}`, + `HTTP request failed, statusCode=${statusCode}, error=${Buffer.from(body).toString()}`, ); } } diff --git a/src/transport/index.ts b/src/transport/index.ts index 983af2c..d0843f1 100644 --- a/src/transport/index.ts +++ b/src/transport/index.ts @@ -24,7 +24,9 @@ function createTransport(options: SenderOptions): SenderTransport { switch (options.protocol) { case HTTP: case HTTPS: - return options.legacy_http ? new HttpTransport(options) : new UndiciTransport(options); + return options.legacy_http + ? new HttpTransport(options) + : new UndiciTransport(options); case TCP: case TCPS: return new TcpTransport(options); @@ -33,4 +35,4 @@ function createTransport(options: SenderOptions): SenderTransport { } } -export { SenderTransport, createTransport } +export { SenderTransport, createTransport }; diff --git a/src/transport/tcp.ts b/src/transport/tcp.ts index be6cf05..b29e367 100644 --- a/src/transport/tcp.ts +++ b/src/transport/tcp.ts @@ -105,7 +105,9 @@ class TcpTransport implements SenderTransport { this.secure = true; break; default: - throw new Error("The 'protocol' has to be 'tcp' or 'tcps' for the TCP transport"); + throw new Error( + "The 'protocol' has to be 'tcp' or 'tcps' for the TCP transport", + ); } if (!options.auth && !options.jwk) { @@ -140,10 +142,10 @@ class TcpTransport implements SenderTransport { this.socket = !this.secure ? net.connect(connOptions as net.NetConnectOpts) : tls.connect(connOptions as tls.ConnectionOptions, () => { - if (authenticated) { - resolve(true); - } - }); + if (authenticated) { + resolve(true); + } + }); this.socket.setKeepAlive(true); this.socket @@ -159,10 +161,18 @@ class TcpTransport implements SenderTransport { } }) .on("ready", async () => { - this.log("info", `Successfully connected to ${connOptions.host}:${connOptions.port}`); + this.log( + "info", + `Successfully connected to ${connOptions.host}:${connOptions.port}`, + ); if (this.jwk) { - this.log("info", `Authenticating with ${connOptions.host}:${connOptions.port}`); - this.socket.write(`${this.jwk.kid}\n`, err => err ? reject(err) : () => {}); + this.log( + "info", + `Authenticating with ${connOptions.host}:${connOptions.port}`, + ); + this.socket.write(`${this.jwk.kid}\n`, (err) => + err ? reject(err) : () => {}, + ); } else { authenticated = true; if (!this.secure || !this.tlsVerify) { @@ -172,7 +182,11 @@ class TcpTransport implements SenderTransport { }) .on("error", (err: Error & { code: string }) => { this.log("error", err); - if (this.tlsVerify || !err.code || err.code !== "SELF_SIGNED_CERT_IN_CHAIN") { + if ( + this.tlsVerify || + !err.code || + err.code !== "SELF_SIGNED_CERT_IN_CHAIN" + ) { reject(err); } }); @@ -217,24 +231,24 @@ class TcpTransport implements SenderTransport { if (challenge.subarray(-1).readInt8() === 10) { const keyObject = crypto.createPrivateKey({ key: this.jwk, - format: "jwk" + format: "jwk", }); const signature = crypto.sign( - "RSA-SHA256", - challenge.subarray(0, challenge.length - 1), - keyObject + "RSA-SHA256", + challenge.subarray(0, challenge.length - 1), + keyObject, ); return new Promise((resolve, reject) => { this.socket.write( - `${Buffer.from(signature).toString("base64")}\n`, - (err: Error) => { - if (err) { - reject(err); - } else { - resolve(true); - } + `${Buffer.from(signature).toString("base64")}\n`, + (err: Error) => { + if (err) { + reject(err); + } else { + resolve(true); } + }, ); }); } @@ -250,7 +264,7 @@ function constructAuth(options: SenderOptions) { if (!options.username || !options.token) { throw new Error( "TCP transport requires a username and a private key for authentication, " + - "please, specify the 'username' and 'token' config options", + "please, specify the 'username' and 'token' config options", ); } @@ -265,25 +279,25 @@ function constructJwk(options: SenderOptions) { if (!options.auth.keyId) { throw new Error( "Missing username, please, specify the 'keyId' property of the 'auth' config option. " + - "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", + "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", ); } if (typeof options.auth.keyId !== "string") { throw new Error( "Please, specify the 'keyId' property of the 'auth' config option as a string. " + - "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", + "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", ); } if (!options.auth.token) { throw new Error( "Missing private key, please, specify the 'token' property of the 'auth' config option. " + - "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", + "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", ); } if (typeof options.auth.token !== "string") { throw new Error( "Please, specify the 'token' property of the 'auth' config option as a string. " + - "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", + "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", ); } diff --git a/src/utils.ts b/src/utils.ts index 1962242..e4e67fa 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -6,7 +6,7 @@ function isBoolean(value: unknown): value is boolean { function isInteger(value: unknown, lowerBound: number): value is number { return ( - typeof value === "number" && Number.isInteger(value) && value >= lowerBound + typeof value === "number" && Number.isInteger(value) && value >= lowerBound ); } @@ -36,4 +36,10 @@ function timestampToNanos(timestamp: bigint, unit: TimestampUnit) { } } -export { isBoolean, isInteger, timestampToMicros, timestampToNanos, TimestampUnit }; +export { + isBoolean, + isInteger, + timestampToMicros, + timestampToNanos, + TimestampUnit, +}; diff --git a/test/logging.test.ts b/test/logging.test.ts index 5997b78..6e17e34 100644 --- a/test/logging.test.ts +++ b/test/logging.test.ts @@ -1,13 +1,21 @@ // @ts-check -import { describe, it, beforeAll, afterAll, afterEach, expect, vi, } from "vitest"; +import { + describe, + it, + beforeAll, + afterAll, + afterEach, + expect, + vi, +} from "vitest"; import { Logger } from "../src/logging"; describe("Default logging suite", function () { - const error = vi.spyOn(console, "error").mockImplementation(() => { }); - const warn = vi.spyOn(console, "warn").mockImplementation(() => { }); - const info = vi.spyOn(console, "info").mockImplementation(() => { }); - const debug = vi.spyOn(console, "debug").mockImplementation(() => { }); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const info = vi.spyOn(console, "info").mockImplementation(() => {}); + const debug = vi.spyOn(console, "debug").mockImplementation(() => {}); let log: Logger; beforeAll(async () => { diff --git a/test/sender.buffer.test.ts b/test/sender.buffer.test.ts index 5d36db7..7f00f80 100644 --- a/test/sender.buffer.test.ts +++ b/test/sender.buffer.test.ts @@ -7,7 +7,9 @@ import { Sender } from "../src"; describe("Client interop test suite", function () { it("runs client tests as per json test config", async function () { const testCases = JSON.parse( - readFileSync("./questdb-client-test/ilp-client-interop-test.json").toString() + readFileSync( + "./questdb-client-test/ilp-client-interop-test.json", + ).toString(), ); for (const testCase of testCases) { @@ -97,13 +99,14 @@ describe("Sender message builder test suite (anything not covered in client inte init_buf_size: 1024, }); - await expect(async () => - await sender - .table("tableName") - .booleanColumn("boolCol", true) - // @ts-expect-error - Testing invalid options - .timestampColumn("timestampCol", 1658484765000000, "foobar") - .atNow() + await expect( + async () => + await sender + .table("tableName") + .booleanColumn("boolCol", true) + // @ts-expect-error - Testing invalid options + .timestampColumn("timestampCol", 1658484765000000, "foobar") + .atNow(), ).rejects.toThrow("Unknown timestamp unit: foobar"); await sender.close(); }); @@ -118,23 +121,23 @@ describe("Sender message builder test suite (anything not covered in client inte id: string; gridId: string; }> = [ - { - id: "46022e96-076f-457f-b630-51b82b871618" + i, - gridId: "46022e96-076f-457f-b630-51b82b871618", - }, - { - id: "55615358-4af1-4179-9153-faaa57d71e55", - gridId: "55615358-4af1-4179-9153-faaa57d71e55", - }, - { - id: "365b9cdf-3d4e-4135-9cb0-f1a65601c840", - gridId: "365b9cdf-3d4e-4135-9cb0-f1a65601c840", - }, - { - id: "0b67ddf2-8e69-4482-bf0c-bb987ee5c280", - gridId: "0b67ddf2-8e69-4482-bf0c-bb987ee5c280" + i, - }, - ]; + { + id: "46022e96-076f-457f-b630-51b82b871618" + i, + gridId: "46022e96-076f-457f-b630-51b82b871618", + }, + { + id: "55615358-4af1-4179-9153-faaa57d71e55", + gridId: "55615358-4af1-4179-9153-faaa57d71e55", + }, + { + id: "365b9cdf-3d4e-4135-9cb0-f1a65601c840", + gridId: "365b9cdf-3d4e-4135-9cb0-f1a65601c840", + }, + { + id: "0b67ddf2-8e69-4482-bf0c-bb987ee5c280", + gridId: "0b67ddf2-8e69-4482-bf0c-bb987ee5c280" + i, + }, + ]; pages.push(pageProducts); } @@ -152,9 +155,9 @@ describe("Sender message builder test suite (anything not covered in client inte } expect(bufferContent(sender)).toBe( 'tableName page_products="[{\\"id\\":\\"46022e96-076f-457f-b630-51b82b8716180\\",\\"gridId\\":\\"46022e96-076f-457f-b630-51b82b871618\\"},{\\"id\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\",\\"gridId\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\"},{\\"id\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\",\\"gridId\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\"},{\\"id\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c280\\",\\"gridId\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c2800\\"}]",boolCol=t\n' + - 'tableName page_products="[{\\"id\\":\\"46022e96-076f-457f-b630-51b82b8716181\\",\\"gridId\\":\\"46022e96-076f-457f-b630-51b82b871618\\"},{\\"id\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\",\\"gridId\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\"},{\\"id\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\",\\"gridId\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\"},{\\"id\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c280\\",\\"gridId\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c2801\\"}]",boolCol=t\n' + - 'tableName page_products="[{\\"id\\":\\"46022e96-076f-457f-b630-51b82b8716182\\",\\"gridId\\":\\"46022e96-076f-457f-b630-51b82b871618\\"},{\\"id\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\",\\"gridId\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\"},{\\"id\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\",\\"gridId\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\"},{\\"id\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c280\\",\\"gridId\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c2802\\"}]",boolCol=t\n' + - 'tableName page_products="[{\\"id\\":\\"46022e96-076f-457f-b630-51b82b8716183\\",\\"gridId\\":\\"46022e96-076f-457f-b630-51b82b871618\\"},{\\"id\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\",\\"gridId\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\"},{\\"id\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\",\\"gridId\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\"},{\\"id\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c280\\",\\"gridId\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c2803\\"}]",boolCol=t\n', + 'tableName page_products="[{\\"id\\":\\"46022e96-076f-457f-b630-51b82b8716181\\",\\"gridId\\":\\"46022e96-076f-457f-b630-51b82b871618\\"},{\\"id\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\",\\"gridId\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\"},{\\"id\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\",\\"gridId\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\"},{\\"id\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c280\\",\\"gridId\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c2801\\"}]",boolCol=t\n' + + 'tableName page_products="[{\\"id\\":\\"46022e96-076f-457f-b630-51b82b8716182\\",\\"gridId\\":\\"46022e96-076f-457f-b630-51b82b871618\\"},{\\"id\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\",\\"gridId\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\"},{\\"id\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\",\\"gridId\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\"},{\\"id\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c280\\",\\"gridId\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c2802\\"}]",boolCol=t\n' + + 'tableName page_products="[{\\"id\\":\\"46022e96-076f-457f-b630-51b82b8716183\\",\\"gridId\\":\\"46022e96-076f-457f-b630-51b82b871618\\"},{\\"id\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\",\\"gridId\\":\\"55615358-4af1-4179-9153-faaa57d71e55\\"},{\\"id\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\",\\"gridId\\":\\"365b9cdf-3d4e-4135-9cb0-f1a65601c840\\"},{\\"id\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c280\\",\\"gridId\\":\\"0b67ddf2-8e69-4482-bf0c-bb987ee5c2803\\"}]",boolCol=t\n', ); await sender.close(); }); @@ -438,7 +441,7 @@ describe("Sender message builder test suite (anything not covered in client inte expect(() => sender.table( "123456789012345678901234567890123456789012345678901234567890" + - "12345678901234567890123456789012345678901234567890123456789012345678", + "12345678901234567890123456789012345678901234567890123456789012345678", ), ).toThrow("Table name is too long, max length is 127"); await sender.close(); @@ -517,7 +520,7 @@ describe("Sender message builder test suite (anything not covered in client inte .table("tableName") .stringColumn( "123456789012345678901234567890123456789012345678901234567890" + - "12345678901234567890123456789012345678901234567890123456789012345678", + "12345678901234567890123456789012345678901234567890123456789012345678", "value", ), ).toThrow("Column name is too long, max length is 127"); @@ -572,7 +575,9 @@ describe("Sender message builder test suite (anything not covered in client inte .table("tableName") .stringColumn("name", "value") .symbol("symbolName", "symbolValue"), - ).toThrow("Symbol can be added only after table name is set and before any column added"); + ).toThrow( + "Symbol can be added only after table name is set and before any column added", + ); await sender.close(); }); @@ -771,7 +776,7 @@ describe("Sender message builder test suite (anything not covered in client inte .atNow(); expect(bufferContent(sender)).toBe( "tableName boolCol=t,timestampCol=1658484765000000t\n" + - "tableName boolCol=f,timestampCol=1658484766000000t\n", + "tableName boolCol=f,timestampCol=1658484766000000t\n", ); sender.reset(); diff --git a/test/sender.config.test.ts b/test/sender.config.test.ts index 7c70a7c..8b7d4d1 100644 --- a/test/sender.config.test.ts +++ b/test/sender.config.test.ts @@ -16,28 +16,36 @@ describe("Sender configuration options suite", function () { }); it("throws exception if the username or the token is missing when TCP transport is used", async function () { - await expect(async () => - await Sender.fromConfig("tcp::addr=hostname;username=bobo;").close() + await expect( + async () => + await Sender.fromConfig("tcp::addr=hostname;username=bobo;").close(), ).rejects.toThrow( - "TCP transport requires a username and a private key for authentication, please, specify the 'username' and 'token' config options", + "TCP transport requires a username and a private key for authentication, please, specify the 'username' and 'token' config options", ); - await expect(async () => - await Sender.fromConfig("tcp::addr=hostname;token=bobo_token;").close() + await expect( + async () => + await Sender.fromConfig("tcp::addr=hostname;token=bobo_token;").close(), ).rejects.toThrow( - "TCP transport requires a username and a private key for authentication, please, specify the 'username' and 'token' config options", + "TCP transport requires a username and a private key for authentication, please, specify the 'username' and 'token' config options", ); }); it("throws exception if tls_roots or tls_roots_password is used", async function () { - await expect(async () => - await Sender.fromConfig("tcps::addr=hostname;username=bobo;tls_roots=bla;").close() + await expect( + async () => + await Sender.fromConfig( + "tcps::addr=hostname;username=bobo;tls_roots=bla;", + ).close(), ).rejects.toThrow( "'tls_roots' and 'tls_roots_password' options are not supported, please, use the 'tls_ca' option or the NODE_EXTRA_CA_CERTS environment variable instead", ); - await expect(async () => - await Sender.fromConfig("tcps::addr=hostname;token=bobo_token;tls_roots_password=bla;").close() + await expect( + async () => + await Sender.fromConfig( + "tcps::addr=hostname;token=bobo_token;tls_roots_password=bla;", + ).close(), ).rejects.toThrow( "'tls_roots' and 'tls_roots_password' options are not supported, please, use the 'tls_ca' option or the NODE_EXTRA_CA_CERTS environment variable instead", ); @@ -55,41 +63,45 @@ describe("Sender configuration options suite", function () { describe("Sender options test suite", function () { it("fails if no options defined", async function () { - await expect(async () => - // @ts-expect-error - Testing invalid options - await new Sender().close() + await expect( + async () => + // @ts-expect-error - Testing invalid options + await new Sender().close(), ).rejects.toThrow("The 'protocol' option is mandatory"); }); it("fails if options are null", async function () { - await expect(async () => - await new Sender(null).close() - ).rejects.toThrow("The 'protocol' option is mandatory"); + await expect(async () => await new Sender(null).close()).rejects.toThrow( + "The 'protocol' option is mandatory", + ); }); it("fails if options are undefined", async function () { - await expect(async () => - await new Sender(undefined).close() + await expect( + async () => await new Sender(undefined).close(), ).rejects.toThrow("The 'protocol' option is mandatory"); }); it("fails if options are empty", async function () { - await expect(async () => + await expect( + async () => // @ts-expect-error - Testing invalid options - await new Sender({}).close() + await new Sender({}).close(), ).rejects.toThrow("The 'protocol' option is mandatory"); }); it("fails if protocol option is missing", async function () { - await expect(async () => + await expect( + async () => // @ts-expect-error - Testing invalid options - await new Sender({ host: "host" }).close() + await new Sender({ host: "host" }).close(), ).rejects.toThrow("The 'protocol' option is mandatory"); }); it("fails if protocol option is invalid", async function () { - await expect(async () => - await new Sender({ protocol: "abcd", host: "hostname" }).close() + await expect( + async () => + await new Sender({ protocol: "abcd", host: "hostname" }).close(), ).rejects.toThrow("Invalid protocol: 'abcd'"); }); @@ -144,10 +156,15 @@ describe("Sender options test suite", function () { }); it("sets the requested buffer size if 'bufferSize' is set, but warns that it is deprecated", async function () { - const log = (level: "error" | "warn" | "info" | "debug", message: string | Error) => { + const log = ( + level: "error" | "warn" | "info" | "debug", + message: string | Error, + ) => { if (level !== "debug") { expect(level).toBe("warn"); - expect(message).toMatch("Option 'bufferSize' is not supported anymore, please, replace it with 'init_buf_size'"); + expect(message).toMatch( + "Option 'bufferSize' is not supported anymore, please, replace it with 'init_buf_size'", + ); } }; const sender = new Sender({ @@ -162,10 +179,15 @@ describe("Sender options test suite", function () { }); it("warns about deprecated option 'copy_buffer'", async function () { - const log = (level: "error" | "warn" | "info" | "debug", message: string) => { + const log = ( + level: "error" | "warn" | "info" | "debug", + message: string, + ) => { if (level !== "debug") { expect(level).toBe("warn"); - expect(message).toMatch("Option 'copy_buffer' is not supported anymore, please, remove it"); + expect(message).toMatch( + "Option 'copy_buffer' is not supported anymore, please, remove it", + ); } }; const sender = new Sender({ @@ -179,10 +201,15 @@ describe("Sender options test suite", function () { }); it("warns about deprecated option 'copyBuffer'", async function () { - const log = (level: "error" | "warn" | "info" | "debug", message: string) => { + const log = ( + level: "error" | "warn" | "info" | "debug", + message: string, + ) => { if (level !== "debug") { expect(level).toBe("warn"); - expect(message).toMatch("Option 'copyBuffer' is not supported anymore, please, remove it"); + expect(message).toMatch( + "Option 'copyBuffer' is not supported anymore, please, remove it", + ); } }; const sender = new Sender({ @@ -212,14 +239,17 @@ describe("Sender options test suite", function () { }); it("throws error if initial buffer size is greater than max_buf_size", async function () { - await expect(async () => - await new Sender({ - protocol: "http", - host: "host", - max_buf_size: 8192, - init_buf_size: 16384, - }).close() - ).rejects.toThrow("Max buffer size is 8192 bytes, requested buffer size: 16384") + await expect( + async () => + await new Sender({ + protocol: "http", + host: "host", + max_buf_size: 8192, + init_buf_size: 16384, + }).close(), + ).rejects.toThrow( + "Max buffer size is 8192 bytes, requested buffer size: 16384", + ); }); it("sets default max buffer size if max_buf_size is set to null", async function () { @@ -260,7 +290,7 @@ describe("Sender options test suite", function () { }); it("uses the required log function if it is set", async function () { - const testFunc = () => { }; + const testFunc = () => {}; const sender = new Sender({ protocol: "http", host: "host", @@ -296,98 +326,104 @@ describe("Sender options test suite", function () { describe("Sender auth config checks suite", function () { it("requires a username for authentication", async function () { - await expect(async () => - await new Sender({ - protocol: "tcp", - host: "host", - auth: { - token: "privateKey", - }, - }).close() + await expect( + async () => + await new Sender({ + protocol: "tcp", + host: "host", + auth: { + token: "privateKey", + }, + }).close(), ).rejects.toThrow( "Missing username, please, specify the 'keyId' property of the 'auth' config option. " + - "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", + "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", ); }); it("requires a non-empty username", async function () { - await expect(async () => - await new Sender({ - protocol: "tcp", - host: "host", - auth: { - keyId: "", - token: "privateKey", - }, - }).close() + await expect( + async () => + await new Sender({ + protocol: "tcp", + host: "host", + auth: { + keyId: "", + token: "privateKey", + }, + }).close(), ).rejects.toThrow( "Missing username, please, specify the 'keyId' property of the 'auth' config option. " + - "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", + "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", ); }); it("requires that the username is a string", async function () { - await expect(async () => - await new Sender({ - protocol: "tcp", - host: "host", - auth: { - // @ts-expect-error - Testing invalid options - keyId: 23, - token: "privateKey", - }, - }).close() + await expect( + async () => + await new Sender({ + protocol: "tcp", + host: "host", + auth: { + // @ts-expect-error - Testing invalid options + keyId: 23, + token: "privateKey", + }, + }).close(), ).rejects.toThrow( "Please, specify the 'keyId' property of the 'auth' config option as a string. " + - "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", + "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", ); }); it("requires a private key for authentication", async function () { - await expect(async () => - await new Sender({ - protocol: "tcp", - host: "host", - auth: { - keyId: "username", - }, - }).close() + await expect( + async () => + await new Sender({ + protocol: "tcp", + host: "host", + auth: { + keyId: "username", + }, + }).close(), ).rejects.toThrow( "Missing private key, please, specify the 'token' property of the 'auth' config option. " + - "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", + "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", ); }); it("requires a non-empty private key", async function () { - await expect(async () => - await new Sender({ - protocol: "tcp", - host: "host", - auth: { - keyId: "username", - token: "", - }, - }).close() + await expect( + async () => + await new Sender({ + protocol: "tcp", + host: "host", + auth: { + keyId: "username", + token: "", + }, + }).close(), ).rejects.toThrow( "Missing private key, please, specify the 'token' property of the 'auth' config option. " + - "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", + "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", ); }); it("requires that the private key is a string", async function () { - await expect(async () => - await new Sender({ - protocol: "tcp", - host: "host", - auth: { - keyId: "username", - // @ts-expect-error - Testing invalid options - token: true, - }, - }).close() + await expect( + async () => + await new Sender({ + protocol: "tcp", + host: "host", + auth: { + keyId: "username", + // @ts-expect-error - Testing invalid options + token: true, + }, + }).close(), ).rejects.toThrow( "Please, specify the 'token' property of the 'auth' config option as a string. " + - "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", + "For example: new Sender({protocol: 'tcp', host: 'host', auth: {keyId: 'username', token: 'private key'}})", ); }); }); diff --git a/test/sender.integration.test.ts b/test/sender.integration.test.ts index aa2b5ca..eff352e 100644 --- a/test/sender.integration.test.ts +++ b/test/sender.integration.test.ts @@ -37,16 +37,25 @@ describe("Sender tests with containerized QuestDB instance", () => { resolve(JSON.parse(Buffer.concat(body).toString())); }); } else { - reject(new Error(`HTTP request failed, statusCode=${response.statusCode}, query=${query}`)); + reject( + new Error( + `HTTP request failed, statusCode=${response.statusCode}, query=${query}`, + ), + ); } }); - req.on("error", error => reject(error)); + req.on("error", (error) => reject(error)); req.end(); }); } - async function runSelect(container: StartedTestContainer, select: string, expectedCount: number, timeout = 60000) { + async function runSelect( + container: StartedTestContainer, + select: string, + expectedCount: number, + timeout = 60000, + ) { const interval = 500; const num = timeout / interval; let selectResult: any; @@ -62,8 +71,17 @@ describe("Sender tests with containerized QuestDB instance", () => { ); } - async function waitForTable(container: StartedTestContainer, tableName: string, timeout = 30000) { - await runSelect(container, `tables() where table_name='${tableName}'`, 1, timeout); + async function waitForTable( + container: StartedTestContainer, + tableName: string, + timeout = 30000, + ) { + await runSelect( + container, + `tables() where table_name='${tableName}'`, + 1, + timeout, + ); } beforeAll(async () => { @@ -106,7 +124,7 @@ describe("Sender tests with containerized QuestDB instance", () => { await sender.flush(); // wait for the table - await waitForTable(container, tableName) + await waitForTable(container, tableName); // query table const select1Result = await runSelect(container, tableName, 1); @@ -176,7 +194,7 @@ describe("Sender tests with containerized QuestDB instance", () => { .at(1658484765000000000n, "ns"); // wait for the table - await waitForTable(container, tableName) + await waitForTable(container, tableName); // query table const select1Result = await runSelect(container, tableName, 1); @@ -248,7 +266,7 @@ describe("Sender tests with containerized QuestDB instance", () => { .at(1658484765000000000n, "ns"); // wait for the table - await waitForTable(container, tableName) + await waitForTable(container, tableName); // query table const select1Result = await runSelect(container, tableName, 1); @@ -330,7 +348,7 @@ describe("Sender tests with containerized QuestDB instance", () => { } // wait for the table - await waitForTable(container, tableName) + await waitForTable(container, tableName); // query table const selectQuery = `${tableName} order by temperature`; @@ -375,7 +393,7 @@ describe("Sender tests with containerized QuestDB instance", () => { await sender.flush(); // Wait for the table - await waitForTable(container, tableName) + await waitForTable(container, tableName); // Query table and verify count const selectQuery = `SELECT id FROM ${tableName}`; diff --git a/test/sender.transport.test.ts b/test/sender.transport.test.ts index 39edd9d..ff22b96 100644 --- a/test/sender.transport.test.ts +++ b/test/sender.transport.test.ts @@ -196,7 +196,7 @@ describe("Sender HTTP suite", function () { `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};retry_timeout=1000`, ); await expect(sendData(sender)).rejects.toThrowError( - "HTTP request timeout, no response from server in time" + "HTTP request timeout, no response from server in time", ); await sender.close(); }); @@ -262,8 +262,8 @@ describe("Sender HTTP suite", function () { const agent = new Agent({ pipelining: 3 }); const sender = Sender.fromConfig( - `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT}`, - { agent: agent }, + `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT}`, + { agent: agent }, ); await sendData(sender); @@ -278,13 +278,13 @@ describe("Sender HTTP suite", function () { await agent.destroy(); }); - it('supports custom legacy HTTP agent', async function () { + it("supports custom legacy HTTP agent", async function () { mockHttp.reset(); const agent = new http.Agent({ maxSockets: 128 }); const sender = Sender.fromConfig( - `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};legacy_http=on`, - { agent: agent }, + `http::addr=${PROXY_HOST}:${MOCK_HTTP_PORT};legacy_http=on`, + { agent: agent }, ); await sendData(sender); expect(mockHttp.numOfRequests).toBe(1); @@ -464,8 +464,8 @@ describe("Sender TCP suite", function () { }); it("fails to connect without hostname and port", async function () { - await expect(async () => - await new Sender({ protocol: "tcp" }).close() + await expect( + async () => await new Sender({ protocol: "tcp" }).close(), ).rejects.toThrow("The 'host' option is mandatory"); }); @@ -481,9 +481,9 @@ describe("Sender TCP suite", function () { it("guards against multiple connect calls", async function () { const proxy = await createProxy(true, proxyOptions); const sender = await createSender(AUTH, true); - await expect(async () => - await sender.connect() - ).rejects.toThrow("Sender connected already"); + await expect(async () => await sender.connect()).rejects.toThrow( + "Sender connected already", + ); await sender.close(); await proxy.stop(); }); @@ -497,8 +497,8 @@ describe("Sender TCP suite", function () { auth: AUTH, tls_ca: "test/certs/ca/ca.crt", }); - await expect(async () => - await Promise.all([sender.connect(), sender.connect()]) + await expect( + async () => await Promise.all([sender.connect(), sender.connect()]), ).rejects.toThrow("Sender connected already"); await sender.close(); await proxy.stop(); @@ -509,8 +509,8 @@ describe("Sender TCP suite", function () { const senderCertCheckFail = Sender.fromConfig( `tcps::addr=${PROXY_HOST}:${PROXY_PORT}`, ); - await expect(async () => - await senderCertCheckFail.connect() + await expect( + async () => await senderCertCheckFail.connect(), ).rejects.toThrow("self-signed certificate in certificate chain"); await senderCertCheckFail.close(); @@ -544,7 +544,10 @@ describe("Sender TCP suite", function () { "Successfully connected to localhost:9088", /^Connection to .*1:9088 is closed$/, ]; - const log = (level: "error" | "warn" | "info" | "debug", message: string) => { + const log = ( + level: "error" | "warn" | "info" | "debug", + message: string, + ) => { if (level !== "debug") { expect(level).toBe("info"); expect(message).toMatch(expectedMessages.shift()); diff --git a/test/util/mockhttp.ts b/test/util/mockhttp.ts index 2341df6..45de985 100644 --- a/test/util/mockhttp.ts +++ b/test/util/mockhttp.ts @@ -2,12 +2,12 @@ import http from "node:http"; import https from "node:https"; type MockConfig = { - responseDelays?: number[], - responseCodes?: number[], - username?: string, - password?: string, - token?: string, -} + responseDelays?: number[]; + responseCodes?: number[]; + username?: string; + password?: string; + token?: string; +}; class MockHttp { server: http.Server | https.Server; @@ -23,40 +23,47 @@ class MockHttp { this.numOfRequests = 0; } - async start(listenPort: number, secure: boolean = false, options?: Record): Promise { + async start( + listenPort: number, + secure: boolean = false, + options?: Record, + ): Promise { const serverCreator = secure ? https.createServer : http.createServer; // @ts-expect-error - Testing different options, so typing is not important - this.server = serverCreator(options, (req: http.IncomingMessage, res: http.ServerResponse) => { - const authFailed = checkAuthHeader(this.mockConfig, req); + this.server = serverCreator( + options, + (req: http.IncomingMessage, res: http.ServerResponse) => { + const authFailed = checkAuthHeader(this.mockConfig, req); - const body: Uint8Array[] = []; - req.on("data", (chunk: Uint8Array) => { - body.push(chunk); - }); + const body: Uint8Array[] = []; + req.on("data", (chunk: Uint8Array) => { + body.push(chunk); + }); - req.on("end", async () => { - console.info(`Received data: ${Buffer.concat(body)}`); - this.numOfRequests++; + req.on("end", async () => { + console.info(`Received data: ${Buffer.concat(body)}`); + this.numOfRequests++; - const delay = - this.mockConfig.responseDelays && + const delay = + this.mockConfig.responseDelays && this.mockConfig.responseDelays.length > 0 - ? this.mockConfig.responseDelays.pop() - : undefined; - if (delay) { - await sleep(delay); - } + ? this.mockConfig.responseDelays.pop() + : undefined; + if (delay) { + await sleep(delay); + } - const responseCode = authFailed - ? 401 - : this.mockConfig.responseCodes && - this.mockConfig.responseCodes.length > 0 - ? this.mockConfig.responseCodes.pop() - : 204; - res.writeHead(responseCode); - res.end(); - }); - }); + const responseCode = authFailed + ? 401 + : this.mockConfig.responseCodes && + this.mockConfig.responseCodes.length > 0 + ? this.mockConfig.responseCodes.pop() + : 204; + res.writeHead(responseCode); + res.end(); + }); + }, + ); return new Promise((resolve, reject) => { this.server.listen(listenPort, () => { @@ -64,7 +71,7 @@ class MockHttp { resolve(true); }); - this.server.on("error", e => { + this.server.on("error", (e) => { console.error(`server error: ${e}`); reject(e); }); diff --git a/test/util/mockproxy.ts b/test/util/mockproxy.ts index ecc9c9c..0a270e5 100644 --- a/test/util/mockproxy.ts +++ b/test/util/mockproxy.ts @@ -7,14 +7,14 @@ const CHALLENGE_LENGTH = 512; type MockConfig = { auth?: boolean; assertions?: boolean; -} +}; class MockProxy { mockConfig: MockConfig; dataSentToRemote: string[]; hasSentChallenge: boolean; client: Socket; - server: net.Server | tls.Server + server: net.Server | tls.Server; constructor(mockConfig: MockConfig) { if (!mockConfig) { diff --git a/test/util/proxy.ts b/test/util/proxy.ts index 59c3076..9117813 100644 --- a/test/util/proxy.ts +++ b/test/util/proxy.ts @@ -8,7 +8,7 @@ import { write, listen, shutdown, connect, close } from "./proxyfunctions"; class Proxy { client: Socket; remote: Socket; - server: net.Server | tls.Server + server: net.Server | tls.Server; constructor() { this.remote = new Socket(); @@ -27,7 +27,12 @@ class Proxy { }); } - async start(listenPort: number, remotePort: number, remoteHost: string, tlsOptions: Record) { + async start( + listenPort: number, + remotePort: number, + remoteHost: string, + tlsOptions: Record, + ) { return new Promise((resolve) => { this.remote.on("ready", async () => { console.info("remote connection ready"); diff --git a/test/util/proxyfunctions.ts b/test/util/proxyfunctions.ts index bc23f18..31e33a6 100644 --- a/test/util/proxyfunctions.ts +++ b/test/util/proxyfunctions.ts @@ -1,17 +1,22 @@ import net, { Socket } from "node:net"; import tls, { TLSSocket } from "node:tls"; import { Proxy } from "./proxy"; -import {MockProxy} from "./mockproxy"; +import { MockProxy } from "./mockproxy"; const LOCALHOST = "localhost"; async function write(socket: Socket, data: string) { return new Promise((resolve, reject) => { - socket.write(data, "utf8", (err: Error) => err ? reject(err): resolve()); + socket.write(data, "utf8", (err: Error) => (err ? reject(err) : resolve())); }); } -async function listen(proxy: Proxy | MockProxy, listenPort: number, dataHandler: (data: string) => void, tlsOptions: tls.TlsOptions) { +async function listen( + proxy: Proxy | MockProxy, + listenPort: number, + dataHandler: (data: string) => void, + tlsOptions: tls.TlsOptions, +) { return new Promise((resolve) => { const clientConnHandler = (client: Socket | TLSSocket) => { console.info("client connected"); @@ -39,7 +44,10 @@ async function listen(proxy: Proxy | MockProxy, listenPort: number, dataHandler: }); } -async function shutdown(proxy: Proxy | MockProxy, onServerClose = async () => {}) { +async function shutdown( + proxy: Proxy | MockProxy, + onServerClose = async () => {}, +) { console.info("closing proxy"); return new Promise((resolve) => { proxy.server.close(async () => {