From 90965337c2b1967a371c89a68fe3f742b7e5551b Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 10 Oct 2025 15:47:38 +0200 Subject: [PATCH 01/65] feat: add decimal support to buffer --- questdb-rs-ffi/src/lib.rs | 6 + questdb-rs/Cargo.toml | 34 +- questdb-rs/src/error.rs | 3 + questdb-rs/src/ingress/buffer.rs | 19 +- questdb-rs/src/ingress/decimal.rs | 226 ++++++++++ questdb-rs/src/ingress/mod.rs | 4 + questdb-rs/src/tests/decimal.rs | 692 ++++++++++++++++++++++++++++++ questdb-rs/src/tests/mod.rs | 1 + 8 files changed, 978 insertions(+), 7 deletions(-) create mode 100644 questdb-rs/src/ingress/decimal.rs create mode 100644 questdb-rs/src/tests/decimal.rs diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 03be44a7..f255afbc 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -224,6 +224,9 @@ pub enum line_sender_error_code { /// Line sender protocol version error. line_sender_error_protocol_version_error, + + /// The supplied decimal is invalid. + line_sender_error_invalid_decimal, } impl From for line_sender_error_code { @@ -252,6 +255,9 @@ impl From for line_sender_error_code { ErrorCode::ProtocolVersionError => { line_sender_error_code::line_sender_error_protocol_version_error } + ErrorCode::InvalidDecimal => { + line_sender_error_code::line_sender_error_invalid_decimal + } } } } diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index 5212488b..5cc2aff8 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -28,18 +28,26 @@ itoa = "1.0" aws-lc-rs = { version = "1.13", optional = true } ring = { version = "0.17.14", optional = true } rustls-pki-types = "1.0.1" -rustls = { version = "0.23.25", default-features = false, features = ["logging", "std", "tls12"] } +rustls = { version = "0.23.25", default-features = false, features = [ + "logging", + "std", + "tls12", +] } rustls-native-certs = { version = "0.8.1", optional = true } webpki-roots = { version = "1.0.1", default-features = false, optional = true } chrono = { version = "0.4.40", optional = true } # We need to limit the `ureq` version to 3.0.x since we use # the `ureq::unversioned` module which does not respect semantic versioning. -ureq = { version = "3.0.10, <3.1.0", default-features = false, features = ["_tls"], optional = true } +ureq = { version = "3.0.10, <3.1.0", default-features = false, features = [ + "_tls", +], optional = true } serde_json = { version = "1", optional = true } questdb-confstr = "0.1.1" rand = { version = "0.9.0", optional = true } ndarray = { version = "0.16", optional = true } +rust_decimal = { version = "1.38.0", optional = true } +bigdecimal = { version = "0.4.8", optional = true } [target.'cfg(windows)'.dependencies] winapi = { version = "0.3.9", features = ["ws2def"] } @@ -68,7 +76,13 @@ sync-sender = ["sync-sender-tcp", "sync-sender-http"] sync-sender-tcp = ["_sync-sender", "_sender-tcp", "dep:socket2"] ## Sync ILP/HTTP -sync-sender-http = ["_sync-sender", "_sender-http", "dep:ureq", "dep:serde_json", "dep:rand"] +sync-sender-http = [ + "_sync-sender", + "_sender-http", + "dep:ureq", + "dep:serde_json", + "dep:rand", +] ## Allow use OS-provided root TLS certificates tls-native-certs = ["dep:rustls-native-certs"] @@ -91,6 +105,12 @@ json_tests = [] ## Enable methods to create timestamp objects from chrono::DateTime objects. chrono_timestamp = ["chrono"] +## Enable serialization of rust_decimal::Decimal in ILP +rust_decimal = ["dep:rust_decimal"] + +## Enable serialization of bigdecimal::BigDecimal in ILP +bigdecimal = ["dep:bigdecimal"] + # Hidden derived features, used in code to enable-disable code sections. Don't use directly. _sender-tcp = [] _sender-http = [] @@ -109,7 +129,9 @@ almost-all-features = [ "insecure-skip-verify", "json_tests", "chrono_timestamp", - "ndarray" + "ndarray", + "rust_decimal", + "bigdecimal", ] [[example]] @@ -126,8 +148,8 @@ required-features = ["chrono_timestamp"] [[example]] name = "http" -required-features = ["sync-sender-http", "ndarray"] +required-features = ["sync-sender-http", "ndarray", "rust_decimal"] [[example]] name = "protocol_version" -required-features = ["sync-sender-http", "ndarray"] +required-features = ["sync-sender-http", "ndarray", "bigdecimal"] diff --git a/questdb-rs/src/error.rs b/questdb-rs/src/error.rs index a28344b3..8b0c1c0e 100644 --- a/questdb-rs/src/error.rs +++ b/questdb-rs/src/error.rs @@ -78,6 +78,9 @@ pub enum ErrorCode { /// Validate protocol version error. ProtocolVersionError, + + /// The supplied decimal is invalid. + InvalidDecimal, } /// An error that occurred when using QuestDB client library. diff --git a/questdb-rs/src/ingress/buffer.rs b/questdb-rs/src/ingress/buffer.rs index 68edc67d..61f912e5 100644 --- a/questdb-rs/src/ingress/buffer.rs +++ b/questdb-rs/src/ingress/buffer.rs @@ -21,6 +21,7 @@ * limitations under the License. * ******************************************************************************/ +use crate::ingress::decimal::DecimalSerializer; use crate::ingress::ndarr::{check_and_get_array_bytes_size, ArrayElementSealed}; use crate::ingress::{ ndarr, ArrayElement, DebugBytes, NdArrayView, ProtocolVersion, Timestamp, TimestampNanos, @@ -71,7 +72,7 @@ where quoting_fn(output); } -fn must_escape_unquoted(c: u8) -> bool { +pub fn must_escape_unquoted(c: u8) -> bool { matches!(c, b' ' | b',' | b'=' | b'\n' | b'\r' | b'\\') } @@ -974,6 +975,22 @@ impl Buffer { Ok(self) } + /// Record a decimal value for the given column. + /// ``` + pub fn column_decimal<'a, N, S>(&mut self, name: N, value: S) -> crate::Result<&mut Self> + where + N: TryInto>, + S: DecimalSerializer, + Error: From, + { + self.write_column_key(name)?; + value.serialize( + &mut self.output, + self.protocol_version == ProtocolVersion::V2, + )?; + Ok(self) + } + /// Record a multidimensional array value for the given column. /// /// Supports arrays with up to [`MAX_ARRAY_DIMS`] dimensions. The array elements must diff --git a/questdb-rs/src/ingress/decimal.rs b/questdb-rs/src/ingress/decimal.rs new file mode 100644 index 00000000..2ec7fd1a --- /dev/null +++ b/questdb-rs/src/ingress/decimal.rs @@ -0,0 +1,226 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +use crate::{error, ingress::must_escape_unquoted, Result}; + +/// Trait for types that can be serialized as decimal values in the InfluxDB Line Protocol (ILP). +/// +/// Decimal values can be serialized in two formats: +/// +/// # Text Format +/// The decimal is written as a string representation followed by a `'d'` suffix. +/// +/// Example: `"123.45d"` or `"1.5e-3d"` +/// +/// Implementers must: +/// - Write the decimal's text representation to the output buffer +/// - Append the `'d'` suffix +/// - Ensure no ILP reserved characters are present (space, comma, equals, newline, carriage return, backslash) +/// +/// # Binary Format +/// A more compact binary encoding consisting of: +/// +/// 1. Binary format marker: `'='` (0x3D) +/// 2. Type identifier: [`DECIMAL_BINARY_FORMAT_TYPE`](crate::ingress::DECIMAL_BINARY_FORMAT_TYPE) byte +/// 3. Scale: 1 byte (0-76 inclusive) - number of decimal places +/// 4. Length: 1 byte - number of bytes in the unscaled value +/// 5. Unscaled value: variable-length byte array in two's complement format, big-endian +/// +/// Example: For decimal `123.45` with scale 2 and unscaled value 12345: +/// ```text +/// = [DECIMAL_BINARY_FORMAT_TYPE] [2] [2] [0x30] [0x39] +/// ``` +/// +/// # Binary Format Notes +/// - Binary format is only supported when `support_binary` is `true` (Protocol V2) +/// - The unscaled value must be encoded in two's complement big-endian format +/// - Maximum scale is 76 +/// - Length byte indicates how many bytes follow for the unscaled value +pub trait DecimalSerializer { + /// Serialize this value as a decimal in ILP format. + /// + /// # Parameters + /// + /// * `out` - The output buffer to write the serialized decimal to + /// * `support_binary` - If `true`, binary format may be used (Protocol V2). + /// If `false`, text format must be used (Protocol V1). + fn serialize(self, out: &mut Vec, support_binary: bool) -> Result<()>; +} + +/// Implementation for string slices containing decimal representations. +/// +/// This implementation always uses the text format, regardless of the `support_binary` parameter, +/// as it cannot parse the string to extract scale and unscaled value needed for binary encoding. +/// +/// # Format +/// The string is validated and written as-is, followed by the 'd' suffix. +/// +/// # Validation +/// The implementation performs **partial validation only**: +/// - Rejects ILP reserved characters (space, comma, equals, newline, carriage return, backslash) +/// - Does NOT validate the actual decimal syntax (e.g., "not-a-number" would pass) +/// +/// This is intentional: full parsing would add overhead. The QuestDB server performs complete +/// validation and will reject malformed decimals. +/// +/// # Examples +/// - `"123.45"` → `"123.45d"` +/// - `"1.5e-3"` → `"1.5e-3d"` +/// - `"-0.001"` → `"-0.001d"` +/// +/// # Errors +/// Returns [`Error`] with [`ErrorCode::InvalidDecimal`](crate::error::ErrorCode::InvalidDecimal) +/// if the string contains ILP reserved characters. +impl DecimalSerializer for &str { + fn serialize(self, out: &mut Vec, _support_binary: bool) -> Result<()> { + // Pre-allocate space for the string content plus the 'd' suffix + out.reserve(self.len() + 1); + + // Validate and copy each byte, rejecting ILP reserved characters + // that would break the protocol (space, comma, equals, newline, etc.) + for b in self.bytes() { + if must_escape_unquoted(b) { + return Err(error::fmt!( + InvalidDecimal, + "Unexpected character {:?} in decimal str", + b + )); + } + out.push(b); + } + + // Append the 'd' suffix to mark this as a decimal value + out.push(b'd'); + + Ok(()) + } +} + +use crate::ingress::DECIMAL_BINARY_FORMAT_TYPE; + +/// Helper to format decimal values directly to a byte buffer without heap allocation. +#[cfg(any(feature = "rust_decimal", feature = "bigdecimal"))] +struct DecimalWriter<'a> { + buf: &'a mut Vec, +} + +#[cfg(any(feature = "rust_decimal", feature = "bigdecimal"))] +impl<'a> std::fmt::Write for DecimalWriter<'a> { + fn write_str(&mut self, s: &str) -> std::fmt::Result { + self.buf.extend_from_slice(s.as_bytes()); + Ok(()) + } +} + +#[cfg(feature = "rust_decimal")] +impl DecimalSerializer for &rust_decimal::Decimal { + fn serialize(self, out: &mut Vec, support_binary: bool) -> Result<()> { + if !support_binary { + // Text format + use std::fmt::Write; + write!(DecimalWriter { buf: out }, "{}", self) + .map_err(|_| error::fmt!(InvalidDecimal, "Failed to format decimal value"))?; + out.push(b'd'); + return Ok(()); + } + + // Binary format: '=' marker + type + scale + length + mantissa bytes + out.push(b'='); + out.push(DECIMAL_BINARY_FORMAT_TYPE); + + // rust_decimal::Decimal guarantees: + // - MAX_SCALE is 28, which is within QuestDB's limit of 76 + // - Mantissa is always 96 bits (12 bytes), never exceeds this size + debug_assert!(rust_decimal::Decimal::MAX_SCALE <= 76); + debug_assert!( + rust_decimal::Decimal::MAX.mantissa() & 0x7FFF_FFFF_0000_0000_0000_0000_0000_0000i128 + == 0 + ); + + out.push(self.scale() as u8); + + // We skip the upper 3 bytes (which are sign-extended) and write the lower 13 bytes + let mantissa = self.mantissa(); + out.push(13); + out.extend_from_slice(&mantissa.to_be_bytes()[3..]); // Skip upper 4 bytes, write lower 12 + + Ok(()) + } +} + +#[cfg(feature = "bigdecimal")] +impl DecimalSerializer for &bigdecimal::BigDecimal { + fn serialize(self, out: &mut Vec, support_binary: bool) -> Result<()> { + if !support_binary { + // Text format + use std::fmt::Write; + write!(DecimalWriter { buf: out }, "{}", self) + .map_err(|_| error::fmt!(InvalidDecimal, "Failed to format decimal value"))?; + out.push(b'd'); + return Ok(()); + } + + // Binary format: '=' marker + type + scale + length + mantissa bytes + out.push(b'='); + out.push(DECIMAL_BINARY_FORMAT_TYPE); + + let (unscaled, mut scale) = self.as_bigint_and_scale(); + if scale > 76 { + return Err(error::fmt!( + InvalidDecimal, + "QuestDB ILP does not support scale greater than 76, got {}", + scale + )); + } + + // QuestDB binary ILP doesn't support negative scale, we need to upscale the + // unscaled value to be compliant + let bytes = if scale < 0 { + use bigdecimal::num_bigint; + let unscaled = + unscaled.into_owned() * num_bigint::BigInt::from(10).pow((-scale) as u32); + scale = 0; + unscaled.to_signed_bytes_be() + } else { + unscaled.to_signed_bytes_be() + }; + + if bytes.len() > i8::MAX as usize { + return Err(error::fmt!( + InvalidDecimal, + "QuestDB ILP does not support values greater than {} bytes, got {}", + i8::MAX, + bytes.len() + )); + } + + out.push(scale as u8); + + // Write length byte and mantissa bytes + out.push(bytes.len() as u8); + out.extend_from_slice(&bytes); + + Ok(()) + } +} diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 052774bc..8d09c849 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -62,6 +62,9 @@ pub use buffer::*; mod sender; pub use sender::*; +mod decimal; +pub use decimal::DecimalSerializer; + const MAX_NAME_LEN_DEFAULT: usize = 127; /// The maximum allowed dimensions for arrays. @@ -71,6 +74,7 @@ pub const MAX_ARRAY_DIM_LEN: usize = 0x0FFF_FFFF; // 1 << 28 - 1 pub(crate) const ARRAY_BINARY_FORMAT_TYPE: u8 = 14; pub(crate) const DOUBLE_BINARY_FORMAT_TYPE: u8 = 16; +pub(crate) const DECIMAL_BINARY_FORMAT_TYPE: u8 = 23; /// The version of InfluxDB Line Protocol used to communicate with the server. #[derive(Debug, Copy, Clone, PartialEq)] diff --git a/questdb-rs/src/tests/decimal.rs b/questdb-rs/src/tests/decimal.rs new file mode 100644 index 00000000..6bb051c6 --- /dev/null +++ b/questdb-rs/src/tests/decimal.rs @@ -0,0 +1,692 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +use crate::ingress::{Buffer, DecimalSerializer, ProtocolVersion, DECIMAL_BINARY_FORMAT_TYPE}; +use crate::tests::TestResult; +use crate::ErrorCode; + +// Helper function to serialize a decimal value and return the bytes +fn serialize_decimal( + value: D, + support_binary: bool, +) -> crate::Result> { + let mut out = Vec::new(); + value.serialize(&mut out, support_binary)?; + Ok(out) +} + +// ============================================================================ +// Tests for &str implementation +// ============================================================================ + +#[test] +fn test_str_positive_decimal() -> TestResult { + let result = serialize_decimal("123.45", false)?; + assert_eq!(result, b"123.45d"); + Ok(()) +} + +#[test] +fn test_str_negative_decimal() -> TestResult { + let result = serialize_decimal("-123.45", false)?; + assert_eq!(result, b"-123.45d"); + Ok(()) +} + +#[test] +fn test_str_zero() -> TestResult { + let result = serialize_decimal("0", false)?; + assert_eq!(result, b"0d"); + Ok(()) +} + +#[test] +fn test_str_scientific_notation() -> TestResult { + let result = serialize_decimal("1.5e-3", false)?; + assert_eq!(result, b"1.5e-3d"); + Ok(()) +} + +#[test] +fn test_str_large_decimal() -> TestResult { + let result = serialize_decimal("999999999999999999.123456789", false)?; + assert_eq!(result, b"999999999999999999.123456789d"); + Ok(()) +} + +#[test] +fn test_str_with_leading_zero() -> TestResult { + let result = serialize_decimal("0.001", false)?; + assert_eq!(result, b"0.001d"); + Ok(()) +} + +#[test] +fn test_str_ignores_binary_flag() -> TestResult { + // &str always uses text format, even with support_binary=true + let result = serialize_decimal("42.0", true)?; + assert_eq!(result, b"42.0d"); + Ok(()) +} + +#[test] +fn test_str_rejects_space() -> TestResult { + let result = serialize_decimal("12 3.45", false); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::InvalidDecimal); + assert!(err.msg().contains("Unexpected character")); + Ok(()) +} + +#[test] +fn test_str_rejects_comma() -> TestResult { + let result = serialize_decimal("1,234.56", false); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::InvalidDecimal); + Ok(()) +} + +#[test] +fn test_str_rejects_equals() -> TestResult { + let result = serialize_decimal("123=45", false); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::InvalidDecimal); + Ok(()) +} + +#[test] +fn test_str_rejects_newline() -> TestResult { + let result = serialize_decimal("123\n45", false); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::InvalidDecimal); + Ok(()) +} + +#[test] +fn test_str_rejects_backslash() -> TestResult { + let result = serialize_decimal("123\\45", false); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::InvalidDecimal); + Ok(()) +} + +/// Validates the binary format structure and extracts the components. +#[cfg(any(feature = "rust_decimal", feature = "bigdecimal"))] +fn parse_binary_decimal(bytes: &[u8]) -> (u8, i128) { + // Validate format markers + assert_eq!(bytes[0], b'=', "Missing binary format marker"); + assert_eq!( + bytes[1], DECIMAL_BINARY_FORMAT_TYPE, + "Invalid decimal type byte" + ); + + let scale = bytes[2]; + let length = bytes[3] as usize; + + assert!(scale <= 76, "Scale {} exceeds maximum of 76", scale); + assert_eq!( + bytes.len(), + 4 + length, + "Binary data length mismatch: expected {} bytes, got {}", + 4 + length, + bytes.len() + ); + + // Parse mantissa bytes as big-endian two's complement + let mantissa_bytes = &bytes[4..]; + + // Convert from big-endian bytes to i128 + // We need to sign-extend if the value is negative (high bit set) + let mut i128_bytes = [0u8; 16]; + let offset = 16 - length; + + // Copy mantissa bytes to the lower part of i128_bytes + i128_bytes[offset..].copy_from_slice(mantissa_bytes); + + // Sign extend if negative (check if high bit of mantissa is set) + if mantissa_bytes[0] & 0x80 != 0 { + // Fill upper bytes with 0xFF for negative numbers + i128_bytes[..offset].fill(0xFF); + } + + let unscaled = i128::from_be_bytes(i128_bytes); + + (scale, unscaled) +} + +// ============================================================================ +// Tests for rust_decimal::Decimal implementation +// ============================================================================ + +#[cfg(feature = "rust_decimal")] +mod rust_decimal_tests { + use super::*; + use rust_decimal::Decimal; + use std::str::FromStr; + + #[test] + fn test_decimal_text_format() -> TestResult { + let dec = Decimal::from_str("123.45")?; + let result = serialize_decimal(&dec, false)?; + assert_eq!(result, b"123.45d"); + Ok(()) + } + + #[test] + fn test_decimal_negative_text_format() -> TestResult { + let dec = Decimal::from_str("-123.45")?; + let result = serialize_decimal(&dec, false)?; + assert_eq!(result, b"-123.45d"); + Ok(()) + } + + #[test] + fn test_decimal_zero_text_format() -> TestResult { + let dec = Decimal::ZERO; + let result = serialize_decimal(&dec, false)?; + assert_eq!(result, b"0d"); + Ok(()) + } + + #[test] + fn test_decimal_binary_format_zero() -> TestResult { + let dec = Decimal::ZERO; + let result = serialize_decimal(&dec, true)?; + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 0, "Zero should have scale 0"); + assert_eq!(unscaled, 0, "Zero should have unscaled value 0"); + Ok(()) + } + + #[test] + fn test_decimal_binary_format_positive() -> TestResult { + let dec = Decimal::from_str("123.45")?; + let result = serialize_decimal(&dec, true)?; + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 2, "123.45 should have scale 2"); + assert_eq!(unscaled, 12345, "123.45 should have unscaled value 12345"); + Ok(()) + } + + #[test] + fn test_decimal_binary_format_negative() -> TestResult { + let dec = Decimal::from_str("-123.45")?; + let result = serialize_decimal(&dec, true)?; + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 2, "-123.45 should have scale 2"); + assert_eq!( + unscaled, -12345, + "-123.45 should have unscaled value -12345" + ); + Ok(()) + } + + #[test] + fn test_decimal_binary_format_one() -> TestResult { + let dec = Decimal::ONE; + let result = serialize_decimal(&dec, true)?; + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 0, "One should have scale 0"); + assert_eq!(unscaled, 1, "One should have unscaled value 1"); + Ok(()) + } + + #[test] + fn test_decimal_binary_format_max_scale() -> TestResult { + // Create a decimal with maximum scale (28 for rust_decimal) + let dec = Decimal::from_str("0.0000000000000000000000000001")?; + let result = serialize_decimal(&dec, true)?; + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 28, "Should have maximum scale of 28"); + assert_eq!(unscaled, 1, "Should have unscaled value 1"); + Ok(()) + } + + #[test] + fn test_decimal_binary_format_large_value() -> TestResult { + let dec = Decimal::MAX; + let result = serialize_decimal(&dec, true)?; + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 0, "Large integer should have scale 0"); + assert_eq!( + unscaled, 79228162514264337593543950335i128, + "Should have correct unscaled value" + ); + Ok(()) + } + + #[test] + fn test_decimal_binary_format_large_value2() -> TestResult { + let dec = Decimal::MIN; + let result = serialize_decimal(&dec, true)?; + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 0, "Large integer should have scale 0"); + assert_eq!( + unscaled, -79228162514264337593543950335i128, + "Should have correct unscaled value" + ); + Ok(()) + } + + #[test] + fn test_decimal_binary_format_small_negative() -> TestResult { + let dec = Decimal::from_str("-0.01")?; + let result = serialize_decimal(&dec, true)?; + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 2, "-0.01 should have scale 2"); + assert_eq!(unscaled, -1, "-0.01 should have unscaled value -1"); + Ok(()) + } + + #[test] + fn test_decimal_binary_format_trailing_zeros() -> TestResult { + let dec = Decimal::from_str("1.00")?; + let result = serialize_decimal(&dec, true)?; + + let (scale, unscaled) = parse_binary_decimal(&result); + // rust_decimal normalizes trailing zeros + assert_eq!(scale, 2, "1.00 should have scale 2"); + assert_eq!(unscaled, 100, "1.00 should have unscaled value 100"); + Ok(()) + } +} + +// ============================================================================ +// Tests for bigdecimal::BigDecimal implementation +// ============================================================================ + +#[cfg(feature = "bigdecimal")] +mod bigdecimal_tests { + use super::*; + use bigdecimal::BigDecimal; + use std::str::FromStr; + + #[test] + fn test_bigdecimal_text_format() -> TestResult { + let dec = BigDecimal::from_str("123.45")?; + let result = serialize_decimal(&dec, false)?; + assert_eq!(result, b"123.45d"); + Ok(()) + } + + #[test] + fn test_bigdecimal_negative_text_format() -> TestResult { + let dec = BigDecimal::from_str("-123.45")?; + let result = serialize_decimal(&dec, false)?; + assert_eq!(result, b"-123.45d"); + Ok(()) + } + + #[test] + fn test_bigdecimal_zero_text_format() -> TestResult { + let dec = BigDecimal::from_str("0")?; + let result = serialize_decimal(&dec, false)?; + assert_eq!(result, b"0d"); + Ok(()) + } + + #[test] + fn test_bigdecimal_binary_format_zero() -> TestResult { + let dec = BigDecimal::from_str("0")?; + let result = serialize_decimal(&dec, true)?; + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 0, "Zero should have scale 0"); + assert_eq!(unscaled, 0, "Zero should have unscaled value 0"); + Ok(()) + } + + #[test] + fn test_bigdecimal_binary_format_positive() -> TestResult { + let dec = BigDecimal::from_str("123.45")?; + let result = serialize_decimal(&dec, true)?; + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 2, "123.45 should have scale 2"); + assert_eq!(unscaled, 12345, "123.45 should have unscaled value 12345"); + Ok(()) + } + + #[test] + fn test_bigdecimal_binary_format_negative() -> TestResult { + let dec = BigDecimal::from_str("-123.45")?; + let result = serialize_decimal(&dec, true)?; + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 2, "-123.45 should have scale 2"); + assert_eq!( + unscaled, -12345, + "-123.45 should have unscaled value -12345" + ); + Ok(()) + } + + #[test] + fn test_bigdecimal_binary_format_one() -> TestResult { + let dec = BigDecimal::from_str("1")?; + let result = serialize_decimal(&dec, true)?; + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 0, "One should have scale 0"); + assert_eq!(unscaled, 1, "One should have unscaled value 1"); + Ok(()) + } + + #[test] + fn test_bigdecimal_binary_format_high_precision() -> TestResult { + // BigDecimal can handle arbitrary precision, test a value with many decimal places + let dec = BigDecimal::from_str("0.123456789012345678901234567890")?; + let result = serialize_decimal(&dec, true)?; + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 30, "Should preserve high precision scale"); + assert_eq!( + unscaled, 123456789012345678901234567890i128, + "Should have correct unscaled value" + ); + Ok(()) + } + + #[test] + fn test_bigdecimal_binary_format_large_value() -> TestResult { + // Test a very large value that BigDecimal can represent + let dec = BigDecimal::from_str("79228162514264337593543950335")?; + let result = serialize_decimal(&dec, true)?; + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 0, "Large integer should have scale 0"); + assert_eq!( + unscaled, 79228162514264337593543950335i128, + "Should have correct unscaled value" + ); + Ok(()) + } + + #[test] + fn test_bigdecimal_binary_format_large_negative() -> TestResult { + let dec = BigDecimal::from_str("-79228162514264337593543950335")?; + let result = serialize_decimal(&dec, true)?; + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 0, "Large negative integer should have scale 0"); + assert_eq!( + unscaled, -79228162514264337593543950335i128, + "Should have correct unscaled value" + ); + Ok(()) + } + + #[test] + fn test_bigdecimal_binary_format_small_negative() -> TestResult { + let dec = BigDecimal::from_str("-0.01")?; + let result = serialize_decimal(&dec, true)?; + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 2, "-0.01 should have scale 2"); + assert_eq!(unscaled, -1, "-0.01 should have unscaled value -1"); + Ok(()) + } + + #[test] + fn test_bigdecimal_binary_format_trailing_zeros() -> TestResult { + let dec = BigDecimal::from_str("1.00")?; + let result = serialize_decimal(&dec, true)?; + + let (scale, unscaled) = parse_binary_decimal(&result); + // BigDecimal may normalize trailing zeros differently than rust_decimal + assert_eq!(scale, 2, "1.00 should have scale 2"); + assert_eq!(unscaled, 100, "1.00 should have unscaled value 100"); + Ok(()) + } + + #[test] + fn test_bigdecimal_binary_format_max_scale() -> TestResult { + // Test with scale at QuestDB's limit of 76 + let dec = BigDecimal::from_str( + "0.0000000000000000000000000000000000000000000000000000000000000000000000000001", + )?; + let result = serialize_decimal(&dec, true)?; + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 76, "Should have maximum scale of 76"); + assert_eq!(unscaled, 1, "Should have unscaled value 1"); + Ok(()) + } + + #[test] + fn test_bigdecimal_binary_format_exceeds_max_scale() -> TestResult { + // Test that exceeding scale 76 returns an error + let dec = BigDecimal::from_str( + "0.00000000000000000000000000000000000000000000000000000000000000000000000000001", + )?; + let result = serialize_decimal(&dec, true); + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::InvalidDecimal); + assert!(err.msg().contains("scale greater than 76")); + Ok(()) + } + + #[test] + fn test_bigdecimal_binary_negative_scale() -> TestResult { + // Test with scale a negative scale + let dec = BigDecimal::from_str("1.23e12")?; + let result = serialize_decimal(&dec, true)?; + + let (scale, unscaled) = parse_binary_decimal(&result); + // QuestDB does not support negative scale, instead the value should be + // scaled properly + assert_eq!(scale, 0, "Should have scale of 0"); + assert_eq!( + unscaled, 1230000000000, + "Should have unscaled value 1230000000000" + ); + Ok(()) + } + + #[test] + fn test_bigdecimal_binary_value_too_large() -> TestResult { + // QuestDB cannot accept arrays that are larger than what an i8 can fit + let dec = BigDecimal::from_str("1e1000")?; + let result = serialize_decimal(&dec, true); + + assert!(result.is_err()); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::InvalidDecimal); + assert!(err.msg().contains("does not support values greater")); + Ok(()) + } +} + +// ============================================================================ +// Buffer integration tests +// ============================================================================ + +#[test] +fn test_buffer_column_decimal_str_v1() -> TestResult { + let mut buffer = Buffer::new(ProtocolVersion::V1); + buffer + .table("test")? + .symbol("sym", "val")? + .column_decimal("dec", "123.45")? + .at_now()?; + + let output = std::str::from_utf8(buffer.as_bytes())?; + assert!(output.starts_with("test,sym=val dec=123.45d")); + Ok(()) +} + +#[test] +fn test_buffer_column_decimal_str_v2() -> TestResult { + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer + .table("test")? + .symbol("sym", "val")? + .column_decimal("dec", "123.45")? + .at_now()?; + + // &str always uses text format, even in V2 + let output = std::str::from_utf8(buffer.as_bytes())?; + assert!(output.starts_with("test,sym=val dec=123.45d")); + Ok(()) +} + +#[cfg(feature = "rust_decimal")] +#[test] +fn test_buffer_column_decimal_rust_decimal_v1() -> TestResult { + use rust_decimal::Decimal; + use std::str::FromStr; + + let mut buffer = Buffer::new(ProtocolVersion::V1); + let dec = Decimal::from_str("123.45")?; + buffer + .table("test")? + .symbol("sym", "val")? + .column_decimal("dec", &dec)? + .at_now()?; + + let output = std::str::from_utf8(buffer.as_bytes())?; + assert!(output.starts_with("test,sym=val dec=123.45d")); + Ok(()) +} + +#[cfg(feature = "rust_decimal")] +#[test] +fn test_buffer_column_decimal_rust_decimal_v2() -> TestResult { + use rust_decimal::Decimal; + use std::str::FromStr; + + let mut buffer = Buffer::new(ProtocolVersion::V2); + let dec = Decimal::from_str("123.45")?; + buffer + .table("test")? + .symbol("sym", "val")? + .column_decimal("dec", &dec)? + .at_now()?; + + let bytes = buffer.as_bytes(); + // Should start with table name and symbol + assert!(bytes.starts_with(b"test,sym=val dec=")); + assert!(bytes.ends_with(b"\n")); + + // Skip the prefix and \n suffix + let dec_binary = &bytes[17..bytes.len() - 1]; + let (scale, unscaled) = parse_binary_decimal(dec_binary); + assert_eq!(scale, 2, "123.45 should have scale 2"); + assert_eq!(unscaled, 12345, "123.45 should have unscaled value 12345"); + Ok(()) +} + +#[test] +fn test_buffer_multiple_decimals() -> TestResult { + let mut buffer = Buffer::new(ProtocolVersion::V1); + buffer + .table("test")? + .column_decimal("dec1", "123.45")? + .column_decimal("dec2", "-67.89")? + .column_decimal("dec3", "0.001")? + .at_now()?; + + let output = std::str::from_utf8(buffer.as_bytes())?; + assert!(output.contains("dec1=123.45d")); + assert!(output.contains("dec2=-67.89d")); + assert!(output.contains("dec3=0.001d")); + Ok(()) +} + +#[test] +fn test_decimal_column_name_too_long() -> TestResult { + let mut buffer = Buffer::with_max_name_len(ProtocolVersion::V2, 4); + let name = "a name too long"; + let err = buffer + .table("tbl")? + .column_decimal(name, "123.45") + .unwrap_err(); + assert_eq!(err.code(), ErrorCode::InvalidName); + assert_eq!( + err.msg(), + r#"Bad name: "a name too long": Too long (max 4 characters)"# + ); + Ok(()) +} + +#[cfg(feature = "bigdecimal")] +#[test] +fn test_buffer_column_decimal_bigdecimal_v1() -> TestResult { + use bigdecimal::BigDecimal; + use std::str::FromStr; + + let mut buffer = Buffer::new(ProtocolVersion::V1); + let dec = BigDecimal::from_str("123.45")?; + buffer + .table("test")? + .symbol("sym", "val")? + .column_decimal("dec", &dec)? + .at_now()?; + + let output = std::str::from_utf8(buffer.as_bytes())?; + assert!(output.starts_with("test,sym=val dec=123.45d")); + Ok(()) +} + +#[cfg(feature = "bigdecimal")] +#[test] +fn test_buffer_column_decimal_bigdecimal_v2() -> TestResult { + use bigdecimal::BigDecimal; + use std::str::FromStr; + + let mut buffer = Buffer::new(ProtocolVersion::V2); + let dec = BigDecimal::from_str("123.45")?; + buffer + .table("test")? + .symbol("sym", "val")? + .column_decimal("dec", &dec)? + .at_now()?; + + let bytes = buffer.as_bytes(); + // Should start with table name and symbol + assert!(bytes.starts_with(b"test,sym=val dec=")); + assert!(bytes.ends_with(b"\n")); + + // Skip the prefix and \n suffix + let dec_binary = &bytes[17..bytes.len() - 1]; + let (scale, unscaled) = parse_binary_decimal(dec_binary); + assert_eq!(scale, 2, "123.45 should have scale 2"); + assert_eq!(unscaled, 12345, "123.45 should have unscaled value 12345"); + Ok(()) +} diff --git a/questdb-rs/src/tests/mod.rs b/questdb-rs/src/tests/mod.rs index 5611c74f..3abf3ab7 100644 --- a/questdb-rs/src/tests/mod.rs +++ b/questdb-rs/src/tests/mod.rs @@ -30,6 +30,7 @@ mod http; mod mock; mod sender; +mod decimal; mod ndarr; #[cfg(feature = "json_tests")] From 74f6f626431bd8779271e71d93f09c5d94e56559 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 10 Oct 2025 15:47:55 +0200 Subject: [PATCH 02/65] docs: add decimal in rust examples --- questdb-rs/examples/basic.rs | 4 ++-- questdb-rs/examples/http.rs | 10 ++++++++-- questdb-rs/examples/protocol_version.rs | 15 ++++++++++----- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/questdb-rs/examples/basic.rs b/questdb-rs/examples/basic.rs index 05c1f7c9..587c6dbd 100644 --- a/questdb-rs/examples/basic.rs +++ b/questdb-rs/examples/basic.rs @@ -8,7 +8,7 @@ use questdb::{ fn main() -> Result<()> { let host: String = std::env::args().nth(1).unwrap_or("localhost".to_string()); let port: &str = &std::env::args().nth(2).unwrap_or("9009".to_string()); - let mut sender = Sender::from_conf(format!("tcp::addr={host}:{port};"))?; + let mut sender = Sender::from_conf(format!("tcp::addr={host}:{port};protocol_version=2;"))?; let mut buffer = sender.new_buffer(); let designated_timestamp = TimestampNanos::from_datetime(Utc.with_ymd_and_hms(1997, 7, 4, 4, 56, 55).unwrap())?; @@ -16,7 +16,7 @@ fn main() -> Result<()> { .table("trades")? .symbol("symbol", "ETH-USD")? .symbol("side", "sell")? - .column_f64("price", 2615.54)? + .column_decimal("price", "2615.54")? .column_f64("amount", 0.00044)? // QuestDB server version 9.0.0 or later is required for array support. .column_arr("location", &arr1(&[100.0, 100.1, 100.2]).view())? diff --git a/questdb-rs/examples/http.rs b/questdb-rs/examples/http.rs index a3f4e8d7..c4f38002 100644 --- a/questdb-rs/examples/http.rs +++ b/questdb-rs/examples/http.rs @@ -1,17 +1,23 @@ +use std::str::FromStr; + use ndarray::arr1; use questdb::{ ingress::{Sender, TimestampNanos}, Result, }; +use rust_decimal::Decimal; fn main() -> Result<()> { - let mut sender = Sender::from_conf("https::addr=localhost:9000;username=foo;password=bar;")?; + let mut sender = Sender::from_conf( + "https::addr=localhost:9000;username=foo;password=bar;protocol_version=2;", + )?; let mut buffer = sender.new_buffer(); + let price = Decimal::from_str("2615.54").unwrap(); buffer .table("trades")? .symbol("symbol", "ETH-USD")? .symbol("side", "sell")? - .column_f64("price", 2615.54)? + .column_decimal("price", &price)? .column_f64("amount", 0.00044)? // QuestDB server version 9.0.0 or later is required for array support. .column_arr("location", &arr1(&[100.0, 100.1, 100.2]).view())? diff --git a/questdb-rs/examples/protocol_version.rs b/questdb-rs/examples/protocol_version.rs index e78fe6a6..6e7dfa23 100644 --- a/questdb-rs/examples/protocol_version.rs +++ b/questdb-rs/examples/protocol_version.rs @@ -1,3 +1,6 @@ +use std::str::FromStr; + +use bigdecimal::BigDecimal; use ndarray::arr1; use questdb::{ ingress::{Sender, TimestampNanos}, @@ -5,29 +8,31 @@ use questdb::{ }; fn main() -> Result<()> { + let price = BigDecimal::from_str("2615.54").unwrap(); + let mut sender = Sender::from_conf( - "https::addr=localhost:9000;username=foo;password=bar;protocol_version=1;", + "http::addr=localhost:9000;username=foo;password=bar;protocol_version=1;", )?; let mut buffer = sender.new_buffer(); buffer .table("trades_ilp_v1")? .symbol("symbol", "ETH-USD")? .symbol("side", "sell")? - .column_f64("price", 2615.54)? + .column_decimal("price", &price)? .column_f64("amount", 0.00044)? .at(TimestampNanos::now())?; sender.flush(&mut buffer)?; // QuestDB server version 9.0.0 or later is required for `protocol_version=2` support. let mut sender2 = Sender::from_conf( - "https::addr=localhost:9000;username=foo;password=bar;protocol_version=2;", + "http::addr=localhost:9000;username=foo;password=bar;protocol_version=2;", )?; - let mut buffer2 = sender.new_buffer(); + let mut buffer2 = sender2.new_buffer(); buffer2 .table("trades_ilp_v2")? .symbol("symbol", "ETH-USD")? .symbol("side", "sell")? - .column_f64("price", 2615.54)? + .column_decimal("price", &price)? .column_f64("amount", 0.00044)? .column_arr("location", &arr1(&[100.0, 100.1, 100.2]).view())? .at(TimestampNanos::now())?; From 3c87c76f51f2ea60f412943d83dde21517929078 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 10 Oct 2025 16:06:59 +0200 Subject: [PATCH 03/65] docs: improve doc for decimal --- questdb-rs/src/ingress/buffer.rs | 70 +++++++++++++++++++++++++++++++ questdb-rs/src/ingress/decimal.rs | 14 +++++-- 2 files changed, 81 insertions(+), 3 deletions(-) diff --git a/questdb-rs/src/ingress/buffer.rs b/questdb-rs/src/ingress/buffer.rs index 61f912e5..4398657e 100644 --- a/questdb-rs/src/ingress/buffer.rs +++ b/questdb-rs/src/ingress/buffer.rs @@ -976,6 +976,76 @@ impl Buffer { } /// Record a decimal value for the given column. + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?; + /// buffer.column_decimal("col_name", "123.45")?; + /// # Ok(()) + /// # } + /// ``` + /// + /// or + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// use questdb::ingress::ColumnName; + /// + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?; + /// let col_name = ColumnName::new("col_name")?; + /// buffer.column_decimal(col_name, "123.45")?; + /// # Ok(()) + /// # } + /// ``` + /// + /// With `rust_decimal` feature enabled: + /// + /// ```no_run + /// # #[cfg(feature = "rust_decimal")] + /// # { + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// use rust_decimal::Decimal; + /// use std::str::FromStr; + /// + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?; + /// let value = Decimal::from_str("123.45")?; + /// buffer.column_decimal("col_name", &value)?; + /// # Ok(()) + /// # } + /// # } + /// ``` + /// + /// With `bigdecimal` feature enabled: + /// + /// ```no_run + /// # #[cfg(feature = "bigdecimal")] + /// # { + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// use bigdecimal::BigDecimal; + /// use std::str::FromStr; + /// + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?; + /// let value = BigDecimal::from_str("0.123456789012345678901234567890")?; + /// buffer.column_decimal("col_name", &value)?; + /// # Ok(()) + /// # } + /// # } /// ``` pub fn column_decimal<'a, N, S>(&mut self, name: N, value: S) -> crate::Result<&mut Self> where diff --git a/questdb-rs/src/ingress/decimal.rs b/questdb-rs/src/ingress/decimal.rs index 2ec7fd1a..e34df66d 100644 --- a/questdb-rs/src/ingress/decimal.rs +++ b/questdb-rs/src/ingress/decimal.rs @@ -47,9 +47,17 @@ use crate::{error, ingress::must_escape_unquoted, Result}; /// 4. Length: 1 byte - number of bytes in the unscaled value /// 5. Unscaled value: variable-length byte array in two's complement format, big-endian /// -/// Example: For decimal `123.45` with scale 2 and unscaled value 12345: -/// ```text -/// = [DECIMAL_BINARY_FORMAT_TYPE] [2] [2] [0x30] [0x39] +/// Example: For decimal `123.45` with scale 2: +/// ``` +/// Unscaled value: 12345 +/// Binary representation: +/// = [23] [2] [2] [0x30] [0x39] +/// │ │ │ │ └───────────┘ +/// │ │ │ │ └─ Mantissa bytes (12345 in big-endian) +/// │ │ │ └─ Length: 2 bytes +/// │ │ └─ Scale: 2 +/// │ └─ Type: DECIMAL_BINARY_FORMAT_TYPE (23) +/// └─ Binary marker: '=' /// ``` /// /// # Binary Format Notes From 2883ea2fe95c8d04279a025c681ed08e638885fd Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 10 Oct 2025 16:08:31 +0200 Subject: [PATCH 04/65] fix: restrict visibility of must_escape_unquoted to the crate only --- questdb-rs/src/ingress/buffer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questdb-rs/src/ingress/buffer.rs b/questdb-rs/src/ingress/buffer.rs index 4398657e..c561030a 100644 --- a/questdb-rs/src/ingress/buffer.rs +++ b/questdb-rs/src/ingress/buffer.rs @@ -72,7 +72,7 @@ where quoting_fn(output); } -pub fn must_escape_unquoted(c: u8) -> bool { +pub(crate) fn must_escape_unquoted(c: u8) -> bool { matches!(c, b' ' | b',' | b'=' | b'\n' | b'\r' | b'\\') } From 457b77a678af4c70f60be7adb801d606ee5bde6a Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 10 Oct 2025 16:28:27 +0200 Subject: [PATCH 05/65] fix: improve binary compatibility check --- questdb-rs/src/ingress/buffer.rs | 2 +- questdb-rs/src/ingress/mod.rs | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/questdb-rs/src/ingress/buffer.rs b/questdb-rs/src/ingress/buffer.rs index c561030a..3fee7a85 100644 --- a/questdb-rs/src/ingress/buffer.rs +++ b/questdb-rs/src/ingress/buffer.rs @@ -1056,7 +1056,7 @@ impl Buffer { self.write_column_key(name)?; value.serialize( &mut self.output, - self.protocol_version == ProtocolVersion::V2, + self.protocol_version.supports_binary_encoding(), )?; Ok(self) } diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 8d09c849..152f3953 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -91,6 +91,23 @@ pub enum ProtocolVersion { V2 = 2, } +impl ProtocolVersion { + /// Returns `true` if this protocol version supports binary-encoded column values. + /// + /// # Examples + /// + /// ``` + /// use questdb::ingress::ProtocolVersion; + /// + /// assert_eq!(ProtocolVersion::V1.supports_binary_encoding(), false); + /// assert_eq!(ProtocolVersion::V2.supports_binary_encoding(), true); + /// ``` + #[inline] + pub fn supports_binary_encoding(self) -> bool { + self != ProtocolVersion::V1 + } +} + impl Display for ProtocolVersion { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { From 3d8645b1ab401aa4f14f93ded0925a8940dcbe9e Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 10 Oct 2025 16:37:33 +0200 Subject: [PATCH 06/65] fix: satisfy clippy with feature-gated usage of DECIMAL_BINARY_FORMAT_TYPE --- questdb-rs/src/ingress/decimal.rs | 6 +----- questdb-rs/src/ingress/mod.rs | 1 + questdb-rs/src/tests/decimal.rs | 4 +++- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/questdb-rs/src/ingress/decimal.rs b/questdb-rs/src/ingress/decimal.rs index e34df66d..5688c4e5 100644 --- a/questdb-rs/src/ingress/decimal.rs +++ b/questdb-rs/src/ingress/decimal.rs @@ -125,6 +125,7 @@ impl DecimalSerializer for &str { } } +#[cfg(any(feature = "rust_decimal", feature = "bigdecimal"))] use crate::ingress::DECIMAL_BINARY_FORMAT_TYPE; /// Helper to format decimal values directly to a byte buffer without heap allocation. @@ -160,11 +161,6 @@ impl DecimalSerializer for &rust_decimal::Decimal { // rust_decimal::Decimal guarantees: // - MAX_SCALE is 28, which is within QuestDB's limit of 76 // - Mantissa is always 96 bits (12 bytes), never exceeds this size - debug_assert!(rust_decimal::Decimal::MAX_SCALE <= 76); - debug_assert!( - rust_decimal::Decimal::MAX.mantissa() & 0x7FFF_FFFF_0000_0000_0000_0000_0000_0000i128 - == 0 - ); out.push(self.scale() as u8); diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 152f3953..8696acfc 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -74,6 +74,7 @@ pub const MAX_ARRAY_DIM_LEN: usize = 0x0FFF_FFFF; // 1 << 28 - 1 pub(crate) const ARRAY_BINARY_FORMAT_TYPE: u8 = 14; pub(crate) const DOUBLE_BINARY_FORMAT_TYPE: u8 = 16; +#[allow(dead_code)] pub(crate) const DECIMAL_BINARY_FORMAT_TYPE: u8 = 23; /// The version of InfluxDB Line Protocol used to communicate with the server. diff --git a/questdb-rs/src/tests/decimal.rs b/questdb-rs/src/tests/decimal.rs index 6bb051c6..b727c469 100644 --- a/questdb-rs/src/tests/decimal.rs +++ b/questdb-rs/src/tests/decimal.rs @@ -22,7 +22,7 @@ * ******************************************************************************/ -use crate::ingress::{Buffer, DecimalSerializer, ProtocolVersion, DECIMAL_BINARY_FORMAT_TYPE}; +use crate::ingress::{Buffer, DecimalSerializer, ProtocolVersion}; use crate::tests::TestResult; use crate::ErrorCode; @@ -140,6 +140,8 @@ fn test_str_rejects_backslash() -> TestResult { #[cfg(any(feature = "rust_decimal", feature = "bigdecimal"))] fn parse_binary_decimal(bytes: &[u8]) -> (u8, i128) { // Validate format markers + + use crate::ingress::DECIMAL_BINARY_FORMAT_TYPE; assert_eq!(bytes[0], b'=', "Missing binary format marker"); assert_eq!( bytes[1], DECIMAL_BINARY_FORMAT_TYPE, From 139e92d8a29ca71b402634bac7dec4d896de9e9a Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 10 Oct 2025 16:42:39 +0200 Subject: [PATCH 07/65] docs: set diagram comment as text --- questdb-rs/src/ingress/decimal.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questdb-rs/src/ingress/decimal.rs b/questdb-rs/src/ingress/decimal.rs index 5688c4e5..2fbce72d 100644 --- a/questdb-rs/src/ingress/decimal.rs +++ b/questdb-rs/src/ingress/decimal.rs @@ -48,7 +48,7 @@ use crate::{error, ingress::must_escape_unquoted, Result}; /// 5. Unscaled value: variable-length byte array in two's complement format, big-endian /// /// Example: For decimal `123.45` with scale 2: -/// ``` +/// ```text /// Unscaled value: 12345 /// Binary representation: /// = [23] [2] [2] [0x30] [0x39] From 4bf9bfc0e074d56d92cf795906968e3fde39af73 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 10 Oct 2025 16:43:55 +0200 Subject: [PATCH 08/65] docs: removed unnecessary comment --- questdb-rs/src/ingress/decimal.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questdb-rs/src/ingress/decimal.rs b/questdb-rs/src/ingress/decimal.rs index 2fbce72d..f7162e19 100644 --- a/questdb-rs/src/ingress/decimal.rs +++ b/questdb-rs/src/ingress/decimal.rs @@ -167,7 +167,7 @@ impl DecimalSerializer for &rust_decimal::Decimal { // We skip the upper 3 bytes (which are sign-extended) and write the lower 13 bytes let mantissa = self.mantissa(); out.push(13); - out.extend_from_slice(&mantissa.to_be_bytes()[3..]); // Skip upper 4 bytes, write lower 12 + out.extend_from_slice(&mantissa.to_be_bytes()[3..]); Ok(()) } From 17f699b2e94f6b7f5859131031063d9e9f3db724 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 10 Oct 2025 16:46:13 +0200 Subject: [PATCH 09/65] docs: fix comment typo --- questdb-rs/src/tests/decimal.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questdb-rs/src/tests/decimal.rs b/questdb-rs/src/tests/decimal.rs index b727c469..a310d804 100644 --- a/questdb-rs/src/tests/decimal.rs +++ b/questdb-rs/src/tests/decimal.rs @@ -506,7 +506,7 @@ mod bigdecimal_tests { #[test] fn test_bigdecimal_binary_negative_scale() -> TestResult { - // Test with scale a negative scale + // Test with a negative scale let dec = BigDecimal::from_str("1.23e12")?; let result = serialize_decimal(&dec, true)?; From a583839b3165e75b9b80b1e67ce962798117bd04 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 10 Oct 2025 17:00:16 +0200 Subject: [PATCH 10/65] docs: fix buffer decimal examples --- questdb-rs/src/ingress/buffer.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/questdb-rs/src/ingress/buffer.rs b/questdb-rs/src/ingress/buffer.rs index 3fee7a85..17ff7ff0 100644 --- a/questdb-rs/src/ingress/buffer.rs +++ b/questdb-rs/src/ingress/buffer.rs @@ -1020,7 +1020,7 @@ impl Buffer { /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; - /// let value = Decimal::from_str("123.45")?; + /// let value = Decimal::from_str("123.45").unwrap(); /// buffer.column_decimal("col_name", &value)?; /// # Ok(()) /// # } @@ -1041,7 +1041,7 @@ impl Buffer { /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; - /// let value = BigDecimal::from_str("0.123456789012345678901234567890")?; + /// let value = BigDecimal::from_str("0.123456789012345678901234567890").unwrap(); /// buffer.column_decimal("col_name", &value)?; /// # Ok(()) /// # } From c7f19afb49635ef4fd60e1a2ea2bd218bf50f47e Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 10 Oct 2025 18:17:20 +0200 Subject: [PATCH 11/65] feat: added support for C and C++ --- doc/C.md | 4 +- doc/CPP.md | 2 +- examples/line_sender_c_example.c | 4 +- examples/line_sender_c_example_auth.c | 4 +- examples/line_sender_c_example_auth_tls.c | 4 +- examples/line_sender_c_example_from_conf.c | 4 +- examples/line_sender_c_example_from_env.c | 4 +- examples/line_sender_c_example_http.c | 4 +- examples/line_sender_c_example_tls_ca.c | 4 +- examples/line_sender_cpp_example.cpp | 2 +- examples/line_sender_cpp_example_auth.cpp | 2 +- examples/line_sender_cpp_example_auth_tls.cpp | 2 +- .../line_sender_cpp_example_from_conf.cpp | 2 +- examples/line_sender_cpp_example_from_env.cpp | 2 +- examples/line_sender_cpp_example_http.cpp | 2 +- examples/line_sender_cpp_example_tls_ca.cpp | 2 +- include/questdb/ingress/line_sender.h | 16 ++++++++ include/questdb/ingress/line_sender.hpp | 35 ++++++++++++++++++ questdb-rs-ffi/src/lib.rs | 24 ++++++++++-- system_test/questdb_line_sender.py | 14 +++++++ system_test/test.py | 37 +++++++++++++++++-- 21 files changed, 152 insertions(+), 22 deletions(-) diff --git a/doc/C.md b/doc/C.md index 7956c96c..3f4222f1 100644 --- a/doc/C.md +++ b/doc/C.md @@ -90,7 +90,9 @@ line_sender_utf8 symbol_value = QDB_UTF8_LITERAL("ETH-USD"); if (!line_sender_buffer_symbol(buffer, symbol_name, symbol_value, &err)) goto on_error; -if (!line_sender_buffer_column_f64(buffer, price_name, 2615.54, &err)) +line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); +if (!line_sender_buffer_column_decimal_str( + buffer, price_name, price_value, &err)) goto on_error; if (!line_sender_buffer_at_nanos(buffer, line_sender_now_nanos(), &err)) diff --git a/doc/CPP.md b/doc/CPP.md index 9f2b1bf8..23b9cc53 100644 --- a/doc/CPP.md +++ b/doc/CPP.md @@ -76,7 +76,7 @@ questdb::ingress::line_sender_buffer buffer; buffer .table("trades") .symbol("symbol", "ETH-USD") - .column("price", 2615.54) + .column_decimal("price", "2615.54"_utf8) .at(timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. diff --git a/examples/line_sender_c_example.c b/examples/line_sender_c_example.c index 1dfbe77f..dbaffb5e 100644 --- a/examples/line_sender_c_example.c +++ b/examples/line_sender_c_example.c @@ -51,7 +51,9 @@ static bool example(const char* host, const char* port) if (!line_sender_buffer_symbol(buffer, side_name, side_value, &err)) goto on_error; - if (!line_sender_buffer_column_f64(buffer, price_name, 2615.54, &err)) + line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); + if (!line_sender_buffer_column_decimal_str( + buffer, price_name, price_value, &err)) goto on_error; if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) diff --git a/examples/line_sender_c_example_auth.c b/examples/line_sender_c_example_auth.c index 0be793df..4fac1691 100644 --- a/examples/line_sender_c_example_auth.c +++ b/examples/line_sender_c_example_auth.c @@ -60,7 +60,9 @@ static bool example(const char* host, const char* port) if (!line_sender_buffer_symbol(buffer, side_name, side_value, &err)) goto on_error; - if (!line_sender_buffer_column_f64(buffer, price_name, 2615.54, &err)) + line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); + if (!line_sender_buffer_column_decimal_str( + buffer, price_name, price_value, &err)) goto on_error; if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) diff --git a/examples/line_sender_c_example_auth_tls.c b/examples/line_sender_c_example_auth_tls.c index d7613d29..d060270d 100644 --- a/examples/line_sender_c_example_auth_tls.c +++ b/examples/line_sender_c_example_auth_tls.c @@ -61,7 +61,9 @@ static bool example(const char* host, const char* port) if (!line_sender_buffer_symbol(buffer, side_name, side_value, &err)) goto on_error; - if (!line_sender_buffer_column_f64(buffer, price_name, 2615.54, &err)) + line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); + if (!line_sender_buffer_column_decimal_str( + buffer, price_name, price_value, &err)) goto on_error; if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) diff --git a/examples/line_sender_c_example_from_conf.c b/examples/line_sender_c_example_from_conf.c index 757071d0..4bca3b8a 100644 --- a/examples/line_sender_c_example_from_conf.c +++ b/examples/line_sender_c_example_from_conf.c @@ -38,7 +38,9 @@ int main(int argc, const char* argv[]) if (!line_sender_buffer_symbol(buffer, side_name, side_value, &err)) goto on_error; - if (!line_sender_buffer_column_f64(buffer, price_name, 2615.54, &err)) + line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); + if (!line_sender_buffer_column_decimal_str( + buffer, price_name, price_value, &err)) goto on_error; if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) diff --git a/examples/line_sender_c_example_from_env.c b/examples/line_sender_c_example_from_env.c index 823c5928..8b531392 100644 --- a/examples/line_sender_c_example_from_env.c +++ b/examples/line_sender_c_example_from_env.c @@ -37,7 +37,9 @@ int main(int argc, const char* argv[]) if (!line_sender_buffer_symbol(buffer, side_name, side_value, &err)) goto on_error; - if (!line_sender_buffer_column_f64(buffer, price_name, 2615.54, &err)) + line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); + if (!line_sender_buffer_column_decimal_str( + buffer, price_name, price_value, &err)) goto on_error; if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) diff --git a/examples/line_sender_c_example_http.c b/examples/line_sender_c_example_http.c index 5b9a82cb..38f6f63d 100644 --- a/examples/line_sender_c_example_http.c +++ b/examples/line_sender_c_example_http.c @@ -49,7 +49,9 @@ static bool example(const char* host, const char* port) if (!line_sender_buffer_symbol(buffer, side_name, side_value, &err)) goto on_error; - if (!line_sender_buffer_column_f64(buffer, price_name, 2615.54, &err)) + line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); + if (!line_sender_buffer_column_decimal_str( + buffer, price_name, price_value, &err)) goto on_error; if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) diff --git a/examples/line_sender_c_example_tls_ca.c b/examples/line_sender_c_example_tls_ca.c index a5657ede..d217da1f 100644 --- a/examples/line_sender_c_example_tls_ca.c +++ b/examples/line_sender_c_example_tls_ca.c @@ -64,7 +64,9 @@ static bool example(const char* ca_path, const char* host, const char* port) if (!line_sender_buffer_symbol(buffer, side_name, side_value, &err)) goto on_error; - if (!line_sender_buffer_column_f64(buffer, price_name, 2615.54, &err)) + line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); + if (!line_sender_buffer_column_decimal_str( + buffer, price_name, price_value, &err)) goto on_error; if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) diff --git a/examples/line_sender_cpp_example.cpp b/examples/line_sender_cpp_example.cpp index 97943e74..92f4c5b0 100644 --- a/examples/line_sender_cpp_example.cpp +++ b/examples/line_sender_cpp_example.cpp @@ -25,7 +25,7 @@ static bool example(std::string_view host, std::string_view port) buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) - .column(price_name, 2615.54) + .column_decimal(price_name, "2615.54"_utf8) .column(amount_name, 0.00044) .at(questdb::ingress::timestamp_nanos::now()); diff --git a/examples/line_sender_cpp_example_auth.cpp b/examples/line_sender_cpp_example_auth.cpp index 9ec2f5e7..064ce8ad 100644 --- a/examples/line_sender_cpp_example_auth.cpp +++ b/examples/line_sender_cpp_example_auth.cpp @@ -29,7 +29,7 @@ static bool example(std::string_view host, std::string_view port) buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) - .column(price_name, 2615.54) + .column_decimal(price_name, "2615.54"_utf8) .column(amount_name, 0.00044) .at(questdb::ingress::timestamp_nanos::now()); diff --git a/examples/line_sender_cpp_example_auth_tls.cpp b/examples/line_sender_cpp_example_auth_tls.cpp index 3f08d8c7..e0064f0d 100644 --- a/examples/line_sender_cpp_example_auth_tls.cpp +++ b/examples/line_sender_cpp_example_auth_tls.cpp @@ -29,7 +29,7 @@ static bool example(std::string_view host, std::string_view port) buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) - .column(price_name, 2615.54) + .column_decimal(price_name, "2615.54"_utf8) .column(amount_name, 0.00044) .at(questdb::ingress::timestamp_nanos::now()); diff --git a/examples/line_sender_cpp_example_from_conf.cpp b/examples/line_sender_cpp_example_from_conf.cpp index 697850e0..3dce4449 100644 --- a/examples/line_sender_cpp_example_from_conf.cpp +++ b/examples/line_sender_cpp_example_from_conf.cpp @@ -24,7 +24,7 @@ int main(int argc, const char* argv[]) buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) - .column(price_name, 2615.54) + .column_decimal(price_name, "2615.54"_utf8) .column(amount_name, 0.00044) .at(questdb::ingress::timestamp_nanos::now()); diff --git a/examples/line_sender_cpp_example_from_env.cpp b/examples/line_sender_cpp_example_from_env.cpp index 3bf1c02a..0ff9f92d 100644 --- a/examples/line_sender_cpp_example_from_env.cpp +++ b/examples/line_sender_cpp_example_from_env.cpp @@ -23,7 +23,7 @@ int main(int argc, const char* argv[]) buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) - .column(price_name, 2615.54) + .column_decimal(price_name, "2615.54"_utf8) .column(amount_name, 0.00044) .at(questdb::ingress::timestamp_nanos::now()); diff --git a/examples/line_sender_cpp_example_http.cpp b/examples/line_sender_cpp_example_http.cpp index 1e675935..120b94ac 100644 --- a/examples/line_sender_cpp_example_http.cpp +++ b/examples/line_sender_cpp_example_http.cpp @@ -24,7 +24,7 @@ static bool example(std::string_view host, std::string_view port) buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) - .column(price_name, 2615.54) + .column_decimal(price_name, "2615.54"_utf8) .column(amount_name, 0.00044) .at(questdb::ingress::timestamp_nanos::now()); diff --git a/examples/line_sender_cpp_example_tls_ca.cpp b/examples/line_sender_cpp_example_tls_ca.cpp index 4e3d0f10..3f778a54 100644 --- a/examples/line_sender_cpp_example_tls_ca.cpp +++ b/examples/line_sender_cpp_example_tls_ca.cpp @@ -32,7 +32,7 @@ static bool example( buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) - .column(price_name, 2615.54) + .column_decimal(price_name, "2615.54"_utf8) .column(amount_name, 0.00044) .at(questdb::ingress::timestamp_nanos::now()); diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index b250d31c..de8baa56 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -496,6 +496,22 @@ bool line_sender_buffer_column_str( line_sender_utf8 value, line_sender_error** err_out); +/** + * Record a decimal string value for the given column. + * + * @param[in] buffer Line buffer object. + * @param[in] name Column name. + * @param[in] value Column value. + * @param[out] err_out Set on error. + * @return true on success, false on error. + */ +LINESENDER_API +bool line_sender_buffer_column_decimal_str( + line_sender_buffer* buffer, + line_sender_column_name name, + line_sender_utf8 value, + line_sender_error** err_out); + /** * Record a multidimensional array of `double` values in C-major order. * diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index fa08826c..79fe7654 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -1026,6 +1026,41 @@ class line_sender_buffer return column(name, utf8_view{value}); } + /** + * Record a decimal string value for the given column. + * @param name Column name. + * @param value Column value. + */ + line_sender_buffer& column_decimal(column_name_view name, utf8_view value) + { + may_init(); + line_sender_error::wrapped_call( + ::line_sender_buffer_column_decimal_str, + _impl, + name._impl, + value._impl); + return *this; + } + + template + line_sender_buffer& column_decimal( + column_name_view name, const char (&value)[N]) + { + return column_decimal(name, utf8_view{value}); + } + + line_sender_buffer& column_decimal( + column_name_view name, std::string_view value) + { + return column_decimal(name, utf8_view{value}); + } + + line_sender_buffer& column_decimal( + column_name_view name, const std::string& value) + { + return column_decimal(name, utf8_view{value}); + } + /** Record a nanosecond timestamp value for the given column. */ template line_sender_buffer& column( diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index f255afbc..6244090c 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -255,9 +255,7 @@ impl From for line_sender_error_code { ErrorCode::ProtocolVersionError => { line_sender_error_code::line_sender_error_protocol_version_error } - ErrorCode::InvalidDecimal => { - line_sender_error_code::line_sender_error_invalid_decimal - } + ErrorCode::InvalidDecimal => line_sender_error_code::line_sender_error_invalid_decimal, } } } @@ -940,6 +938,26 @@ pub unsafe extern "C" fn line_sender_buffer_column_str( true } +/// Record a decimal string value for the given column. +/// @param[in] buffer Line buffer object. +/// @param[in] name Column name. +/// @param[in] value Column value. +/// @param[out] err_out Set on error. +/// @return true on success, false on error. +#[no_mangle] +pub unsafe extern "C" fn line_sender_buffer_column_decimal_str( + buffer: *mut line_sender_buffer, + name: line_sender_column_name, + value: line_sender_utf8, + err_out: *mut *mut line_sender_error, +) -> bool { + let buffer = unwrap_buffer_mut(buffer); + let name = name.as_name(); + let value = value.as_str(); + bubble_err_to_c!(err_out, buffer.column_decimal(name, value)); + true +} + /// Records a float64 multidimensional array with **C-MAJOR memory layout**. /// /// @param[in] buffer Line buffer object. diff --git a/system_test/questdb_line_sender.py b/system_test/questdb_line_sender.py index 55e0fcb0..1b4e87dd 100644 --- a/system_test/questdb_line_sender.py +++ b/system_test/questdb_line_sender.py @@ -49,6 +49,7 @@ from datetime import datetime from functools import total_ordering from enum import Enum +from decimal import Decimal from ctypes import ( c_bool, @@ -290,6 +291,13 @@ def set_sig(fn, restype, *argtypes): c_line_sender_column_name, c_line_sender_utf8, c_line_sender_error_p_p) + set_sig( + dll.line_sender_buffer_column_decimal_str, + c_bool, + c_line_sender_buffer_p, + c_line_sender_column_name, + c_line_sender_utf8, + c_line_sender_error_p_p) set_sig( dll.line_sender_buffer_column_f64_arr_byte_strides, c_bool, @@ -703,6 +711,12 @@ def column( self._impl, _column_name(name), _utf8(value)) + elif isinstance(value, Decimal): + _error_wrapped_call( + _DLL.line_sender_buffer_column_decimal_str, + self._impl, + _column_name(name), + _utf8(str(value))) elif isinstance(value, TimestampMicros): _error_wrapped_call( _DLL.line_sender_buffer_column_ts_micros, diff --git a/system_test/test.py b/system_test/test.py index 9465d630..f6bd20e1 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -49,7 +49,7 @@ list_questdb_releases, AUTH) import subprocess -from collections import namedtuple +from decimal import Decimal QDB_FIXTURE: QuestDbFixtureBase = None TLS_PROXY_FIXTURE: TlsProxyFixture = None @@ -517,6 +517,37 @@ def test_timestamp_column(self): scrubbed_dataset = [row[:-1] for row in resp['dataset']] self.assertEqual(scrubbed_dataset, exp_dataset) + def test_decimal_column(self): + table_name = uuid.uuid4().hex + pending = None + decimals = [ + Decimal("12.99"), + Decimal("-12.34"), + Decimal("0.001"), + Decimal("10000000.0"), + Decimal("NaN"), + Decimal("Infinity"), + Decimal("0"), + Decimal("-0"), + Decimal("1e3") + ] + with self._mk_linesender() as sender: + for dec in decimals: + sender.table(table_name) + sender.column('dec', dec) + sender.at_now() + pending = sender.buffer.peek() + + resp = retry_check_table(table_name, min_rows=len(decimals), log_ctx=pending) + exp_columns = [ + {'name': 'dec', 'type': 'DECIMAL(18,3)'}, + {'name': 'timestamp', 'type': 'TIMESTAMP'}] + self.assertEqual(resp['columns'], exp_columns) + # By default, the decimal created as a scale of 3 + exp_dataset = [['12.990'], ['-12.340'], ['0.001'], ['10000000.000'], [None], [None], ['0.000'], ['0.000'], ['1000.000']] + scrubbed_dataset = [row[:-1] for row in resp['dataset']] + self.assertEqual(scrubbed_dataset, exp_dataset) + def test_f64_arr_column(self): if self.expected_protocol_version < qls.ProtocolVersion.V2: self.skipTest('communicating over old protocol which does not support arrays') @@ -753,14 +784,14 @@ def _test_example(self, bin_name, table_name, tls=False): exp_columns = [ {'name': 'symbol', 'type': 'SYMBOL'}, {'name': 'side', 'type': 'SYMBOL'}, - {'name': 'price', 'type': 'DOUBLE'}, + {'name': 'price', 'type': 'DECIMAL(18,3)'}, {'name': 'amount', 'type': 'DOUBLE'}, {'name': 'timestamp', 'type': 'TIMESTAMP'}] self.assertEqual(resp['columns'], exp_columns) exp_dataset = [['ETH-USD', 'sell', - 2615.54, + '2615.540', 0.00044]] # Comparison excludes timestamp column. scrubbed_dataset = [row[:-1] for row in resp['dataset']] From 6cc4237ff051c3111f2ab4661aea2d3e861a6c64 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 10 Oct 2025 18:24:29 +0200 Subject: [PATCH 12/65] fix: add missing error codes to C and C++ headers --- include/questdb/ingress/line_sender.h | 3 +++ include/questdb/ingress/line_sender.hpp | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index de8baa56..bca347d5 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -83,6 +83,9 @@ typedef enum line_sender_error_code /** Line sender protocol version error. */ line_sender_error_protocol_version_error, + + /** The supplied decimal is invalid. */ + line_sender_error_invalid_decimal, } line_sender_error_code; /** The protocol used to connect with. */ diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 79fe7654..416d5ba8 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -82,6 +82,15 @@ enum class line_sender_error_code /** Bad configuration. */ config_error, + + /** There was an error serializing an array. */ + array_error, + + /** Line sender protocol version error. */ + protocol_version_error, + + /** The supplied decimal is invalid. */ + invalid_decimal, }; /** The protocol used to connect with. */ From 4aa3b14c196d38cbbe34936722fd57db89b34c56 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 10 Oct 2025 18:25:03 +0200 Subject: [PATCH 13/65] fix: update type hints to include decimal in python test --- system_test/questdb_line_sender.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/system_test/questdb_line_sender.py b/system_test/questdb_line_sender.py index 1b4e87dd..ebf69399 100644 --- a/system_test/questdb_line_sender.py +++ b/system_test/questdb_line_sender.py @@ -734,7 +734,7 @@ def column( fqn = _fully_qual_name(value) raise ValueError( f'Bad field value of type {fqn}: Expected one of ' - '`bool`, `int`, `float` or `str`.') + '`bool`, `int`, `float`, `str`, `Decimal`, `TimestampMicros`, or `datetime`.') return self def column_f64_arr(self, name: str, @@ -916,7 +916,7 @@ def symbol(self, name: str, value: str): def column( self, name: str, - value: Union[bool, int, float, str, TimestampMicros, datetime]): + value: Union[bool, int, float, str, Decimal, TimestampMicros, datetime]): self._buffer.column(name, value) return self From fb8ac1e0dbed90b9dd46edb9bb0bd6aaf0158674 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 10 Oct 2025 19:06:04 +0200 Subject: [PATCH 14/65] ci: trigger From 2095903623fd976b7a991db1318873b1472b8a2e Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Mon, 13 Oct 2025 10:15:06 +0200 Subject: [PATCH 15/65] tests: add decimal support to ilp-client-interop-test --- questdb-rs/Cargo.toml | 2 +- questdb-rs/build.rs | 15 +++++++ .../interop/ilp-client-interop-test.json | 44 ++++++++++++++++++- 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index 5cc2aff8..35ca3040 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -100,7 +100,7 @@ ring-crypto = ["dep:ring", "rustls/ring"] insecure-skip-verify = [] ## Enable code-generation in `build.rs` for additional tests. -json_tests = [] +json_tests = ["bigdecimal"] ## Enable methods to create timestamp objects from chrono::DateTime objects. chrono_timestamp = ["chrono"] diff --git a/questdb-rs/build.rs b/questdb-rs/build.rs index df2d3000..f943d6b5 100644 --- a/questdb-rs/build.rs +++ b/questdb-rs/build.rs @@ -62,6 +62,12 @@ pub mod json_tests { value: bool, } + #[derive(Debug, Serialize, Deserialize)] + struct DecimalColumn { + name: String, + value: String, + } + #[derive(Debug, Serialize, Deserialize)] #[serde(tag = "type", rename_all = "UPPERCASE")] enum Column { @@ -69,6 +75,7 @@ pub mod json_tests { Long(LongColumn), Double(DoubleColumn), Boolean(BooleanColumn), + Decimal(DecimalColumn), } #[derive(Debug, Serialize, Deserialize)] @@ -126,6 +133,8 @@ pub mod json_tests { use base64ct::Base64; use base64ct::Encoding; use rstest::rstest; + use bigdecimal::BigDecimal; + use std::str::FromStr; fn matches_any_line(line: &[u8], expected: &[&str]) -> bool { for &exp in expected { @@ -133,6 +142,7 @@ pub mod json_tests { return true; } } + let line = String::from_utf8_lossy(line); eprintln!("Could not match:\n {line:?}\nTo any of: {expected:#?}"); false } @@ -191,6 +201,11 @@ pub mod json_tests { "{} .column_bool({:?}, {:?})?", indent, column.name, column.value )?, + Column::Decimal(column) => writeln!( + output, + "{} .column_decimal({:?}, &BigDecimal::from_str({:?}).unwrap())?", + indent, column.name, column.value + )?, } } writeln!(output, "{indent} .at_now()?;")?; diff --git a/questdb-rs/src/tests/interop/ilp-client-interop-test.json b/questdb-rs/src/tests/interop/ilp-client-interop-test.json index 0acedad7..6a76d007 100644 --- a/questdb-rs/src/tests/interop/ilp-client-interop-test.json +++ b/questdb-rs/src/tests/interop/ilp-client-interop-test.json @@ -1635,5 +1635,47 @@ "result": { "status": "ERROR" } + }, + { + "testName": "decimal", + "table": "decimals", + "symbols": [], + "columns": [ + { + "type": "DECIMAL", + "name": "zero", + "value": "0.0" + }, + { + "type": "DECIMAL", + "name": "neg_zero", + "value": "-0.0" + }, + { + "type": "DECIMAL", + "name": "one", + "value": "1.0" + }, + { + "type": "DECIMAL", + "name": "large", + "value": "99999999999999.999" + }, + { + "type": "DECIMAL", + "name": "small", + "value": "0.001" + }, + { + "type": "DECIMAL", + "name": "neg_small", + "value": "-0.001" + } + ], + "result": { + "status": "SUCCESS", + "line": "decimals zero=0d,neg_zero=0d,one=1.0d,large=99999999999999.999d,small=0.001d,neg_small=-0.001d", + "binaryBase64": "ZGVjaW1hbHMgemVybz09FwEBACxuZWdfemVybz09FwEBACxvbmU9PRcBAQosbGFyZ2U9PRcDCAFjRXhdif//LHNtYWxsPT0XAwEBLG5lZ19zbWFsbD09FwMB/wo=" + } } -] +] \ No newline at end of file From 984a236045759dc93dbad4b9b4f54f984b571857 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Tue, 14 Oct 2025 15:44:05 +0200 Subject: [PATCH 16/65] feat: improve cpp decimal integration to cpp --- CMakeLists.txt | 6 + doc/CPP.md | 2 +- examples/line_sender_cpp_example.cpp | 3 +- examples/line_sender_cpp_example_auth.cpp | 3 +- examples/line_sender_cpp_example_auth_tls.cpp | 3 +- ...line_sender_cpp_example_decimal_binary.cpp | 84 ++++++ ...line_sender_cpp_example_decimal_custom.cpp | 123 ++++++++ .../line_sender_cpp_example_from_conf.cpp | 3 +- examples/line_sender_cpp_example_from_env.cpp | 3 +- examples/line_sender_cpp_example_http.cpp | 3 +- examples/line_sender_cpp_example_tls_ca.cpp | 3 +- include/questdb/ingress/line_sender.h | 20 ++ include/questdb/ingress/line_sender.hpp | 271 ++++++++++++++++-- questdb-rs-ffi/src/decimal.rs | 146 ++++++++++ questdb-rs-ffi/src/lib.rs | 28 ++ questdb-rs/src/ingress/mod.rs | 2 +- 16 files changed, 672 insertions(+), 31 deletions(-) create mode 100644 examples/line_sender_cpp_example_decimal_binary.cpp create mode 100644 examples/line_sender_cpp_example_decimal_custom.cpp create mode 100644 questdb-rs-ffi/src/decimal.rs diff --git a/CMakeLists.txt b/CMakeLists.txt index bda9ee4e..d400ef13 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -165,6 +165,12 @@ if (QUESTDB_TESTS_AND_EXAMPLES) compile_example( line_sender_cpp_example_from_env examples/line_sender_cpp_example_from_env.cpp) + compile_example( + line_sender_cpp_example_decimal_custom + examples/line_sender_cpp_example_decimal_custom.cpp) + compile_example( + line_sender_cpp_example_decimal_binary + examples/line_sender_cpp_example_decimal_binary.cpp) # Include Rust tests as part of the tests run add_test( diff --git a/doc/CPP.md b/doc/CPP.md index 23b9cc53..922fdf26 100644 --- a/doc/CPP.md +++ b/doc/CPP.md @@ -76,7 +76,7 @@ questdb::ingress::line_sender_buffer buffer; buffer .table("trades") .symbol("symbol", "ETH-USD") - .column_decimal("price", "2615.54"_utf8) + .column("price", "2615.54"_decimal) .at(timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. diff --git a/examples/line_sender_cpp_example.cpp b/examples/line_sender_cpp_example.cpp index 92f4c5b0..1fc5c66f 100644 --- a/examples/line_sender_cpp_example.cpp +++ b/examples/line_sender_cpp_example.cpp @@ -3,6 +3,7 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; +using namespace questdb::ingress::decimal; static bool example(std::string_view host, std::string_view port) { @@ -25,7 +26,7 @@ static bool example(std::string_view host, std::string_view port) buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) - .column_decimal(price_name, "2615.54"_utf8) + .column(price_name, "2615.54"_decimal) .column(amount_name, 0.00044) .at(questdb::ingress::timestamp_nanos::now()); diff --git a/examples/line_sender_cpp_example_auth.cpp b/examples/line_sender_cpp_example_auth.cpp index 064ce8ad..d1bd1455 100644 --- a/examples/line_sender_cpp_example_auth.cpp +++ b/examples/line_sender_cpp_example_auth.cpp @@ -3,6 +3,7 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; +using namespace questdb::ingress::decimal; static bool example(std::string_view host, std::string_view port) { @@ -29,7 +30,7 @@ static bool example(std::string_view host, std::string_view port) buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) - .column_decimal(price_name, "2615.54"_utf8) + .column(price_name, "2615.54"_decimal) .column(amount_name, 0.00044) .at(questdb::ingress::timestamp_nanos::now()); diff --git a/examples/line_sender_cpp_example_auth_tls.cpp b/examples/line_sender_cpp_example_auth_tls.cpp index e0064f0d..5c26e3b4 100644 --- a/examples/line_sender_cpp_example_auth_tls.cpp +++ b/examples/line_sender_cpp_example_auth_tls.cpp @@ -3,6 +3,7 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; +using namespace questdb::ingress::decimal; static bool example(std::string_view host, std::string_view port) { @@ -29,7 +30,7 @@ static bool example(std::string_view host, std::string_view port) buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) - .column_decimal(price_name, "2615.54"_utf8) + .column(price_name, "2615.54"_decimal) .column(amount_name, 0.00044) .at(questdb::ingress::timestamp_nanos::now()); diff --git a/examples/line_sender_cpp_example_decimal_binary.cpp b/examples/line_sender_cpp_example_decimal_binary.cpp new file mode 100644 index 00000000..20e1a992 --- /dev/null +++ b/examples/line_sender_cpp_example_decimal_binary.cpp @@ -0,0 +1,84 @@ +#include +#include + +using namespace std::literals::string_view_literals; +using namespace questdb::ingress::literals; +using namespace questdb::ingress::decimal; + +static bool example(std::string_view host, std::string_view port) +{ + try + { + auto sender = questdb::ingress::line_sender::from_conf( + "tcp::addr=" + std::string{host} + ":" + std::string{port} + + ";protocol_version=2;"); + + // We prepare all our table names and column names in advance. + // If we're inserting multiple rows, this allows us to avoid + // re-validating the same strings over and over again. + const auto table_name = "cpp_trades"_tn; + const auto symbol_name = "symbol"_cn; + const auto side_name = "side"_cn; + const auto price_name = "price"_cn; + const auto amount_name = "amount"_cn; + const uint8_t price_unscaled_value[] = {123}; + // 123 with a scale of 1 gives a decimal of 12.3 + const auto price_value = binary_view(1, price_unscaled_value); + + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); + buffer.table(table_name) + .symbol(symbol_name, "ETH-USD"_utf8) + .symbol(side_name, "sell"_utf8) + .column(price_name, price_value) + .column(amount_name, 0.00044) + .at(questdb::ingress::timestamp_nanos::now()); + + // To insert more records, call `buffer.table(..)...` again. + + sender.flush(buffer); + + // It's recommended to keep a timer and/or maximum buffer size to flush + // the buffer periodically with any accumulated records. + + return true; + } + catch (const questdb::ingress::line_sender_error& err) + { + std::cerr << "Error running example: " << err.what() << std::endl; + + return false; + } +} + +static bool displayed_help(int argc, const char* argv[]) +{ + for (int index = 1; index < argc; ++index) + { + const std::string_view arg{argv[index]}; + if ((arg == "-h"sv) || (arg == "--help"sv)) + { + std::cerr << "Usage:\n" + << "line_sender_c_example: [HOST [PORT]]\n" + << " HOST: ILP host (defaults to \"localhost\").\n" + << " PORT: ILP port (defaults to \"9009\")." + << std::endl; + return true; + } + } + return false; +} + +int main(int argc, const char* argv[]) +{ + if (displayed_help(argc, argv)) + return 0; + + auto host = "localhost"sv; + if (argc >= 2) + host = std::string_view{argv[1]}; + auto port = "9009"sv; + if (argc >= 3) + port = std::string_view{argv[2]}; + + return !example(host, port); +} diff --git a/examples/line_sender_cpp_example_decimal_custom.cpp b/examples/line_sender_cpp_example_decimal_custom.cpp new file mode 100644 index 00000000..be5ca809 --- /dev/null +++ b/examples/line_sender_cpp_example_decimal_custom.cpp @@ -0,0 +1,123 @@ +#include +#include + +using namespace std::literals::string_view_literals; +using namespace questdb::ingress::literals; +using namespace questdb::ingress::decimal; + +namespace custom_decimal +{ +class Decimal32 +{ +public: + Decimal32(uint32_t scale, int32_t unscaled_value) + : _scale(scale) + , _unscaled_value(unscaled_value) + { + } + + int32_t unscaled_value() const + { + return _unscaled_value; + } + + uint32_t scale() const + { + return _scale; + } + + questdb::ingress::decimal::binary_view view() const + { + std::array data = { + // Big-Endiang bytes + static_cast(_unscaled_value >> 24), + static_cast(_unscaled_value >> 16), + static_cast(_unscaled_value >> 8), + static_cast(_unscaled_value >> 0), + }; + return {_scale, data}; + } + +private: + uint32_t _scale; + int32_t _unscaled_value; +}; +} + +static bool example(std::string_view host, std::string_view port) +{ + try + { + auto sender = questdb::ingress::line_sender::from_conf( + "tcp::addr=" + std::string{host} + ":" + std::string{port} + + ";protocol_version=2;"); + + // We prepare all our table names and column names in advance. + // If we're inserting multiple rows, this allows us to avoid + // re-validating the same strings over and over again. + const auto table_name = "cpp_trades"_tn; + const auto symbol_name = "symbol"_cn; + const auto side_name = "side"_cn; + const auto price_name = "price"_cn; + const auto amount_name = "amount"_cn; + + // 123 with a scale of 1 gives a decimal of 12.3 + const auto price_value = custom_decimal::Decimal32(1, 123); + + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); + buffer.table(table_name) + .symbol(symbol_name, "ETH-USD"_utf8) + .symbol(side_name, "sell"_utf8) + .column(price_name, price_value.view()) + .column(amount_name, 0.00044) + .at(questdb::ingress::timestamp_nanos::now()); + + // To insert more records, call `buffer.table(..)...` again. + + sender.flush(buffer); + + // It's recommended to keep a timer and/or maximum buffer size to flush + // the buffer periodically with any accumulated records. + + return true; + } + catch (const questdb::ingress::line_sender_error& err) + { + std::cerr << "Error running example: " << err.what() << std::endl; + + return false; + } +} + +static bool displayed_help(int argc, const char* argv[]) +{ + for (int index = 1; index < argc; ++index) + { + const std::string_view arg{argv[index]}; + if ((arg == "-h"sv) || (arg == "--help"sv)) + { + std::cerr << "Usage:\n" + << "line_sender_c_example: [HOST [PORT]]\n" + << " HOST: ILP host (defaults to \"localhost\").\n" + << " PORT: ILP port (defaults to \"9009\")." + << std::endl; + return true; + } + } + return false; +} + +int main(int argc, const char* argv[]) +{ + if (displayed_help(argc, argv)) + return 0; + + auto host = "localhost"sv; + if (argc >= 2) + host = std::string_view{argv[1]}; + auto port = "9009"sv; + if (argc >= 3) + port = std::string_view{argv[2]}; + + return !example(host, port); +} diff --git a/examples/line_sender_cpp_example_from_conf.cpp b/examples/line_sender_cpp_example_from_conf.cpp index 3dce4449..e7309ea2 100644 --- a/examples/line_sender_cpp_example_from_conf.cpp +++ b/examples/line_sender_cpp_example_from_conf.cpp @@ -3,6 +3,7 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; +using namespace questdb::ingress::decimal; int main(int argc, const char* argv[]) { @@ -24,7 +25,7 @@ int main(int argc, const char* argv[]) buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) - .column_decimal(price_name, "2615.54"_utf8) + .column(price_name, "2615.54"_decimal) .column(amount_name, 0.00044) .at(questdb::ingress::timestamp_nanos::now()); diff --git a/examples/line_sender_cpp_example_from_env.cpp b/examples/line_sender_cpp_example_from_env.cpp index 0ff9f92d..515a68a2 100644 --- a/examples/line_sender_cpp_example_from_env.cpp +++ b/examples/line_sender_cpp_example_from_env.cpp @@ -3,6 +3,7 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; +using namespace questdb::ingress::decimal; int main(int argc, const char* argv[]) { @@ -23,7 +24,7 @@ int main(int argc, const char* argv[]) buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) - .column_decimal(price_name, "2615.54"_utf8) + .column(price_name, "2615.54"_decimal) .column(amount_name, 0.00044) .at(questdb::ingress::timestamp_nanos::now()); diff --git a/examples/line_sender_cpp_example_http.cpp b/examples/line_sender_cpp_example_http.cpp index 120b94ac..9d4bfe2f 100644 --- a/examples/line_sender_cpp_example_http.cpp +++ b/examples/line_sender_cpp_example_http.cpp @@ -3,6 +3,7 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; +using namespace questdb::ingress::decimal; static bool example(std::string_view host, std::string_view port) { @@ -24,7 +25,7 @@ static bool example(std::string_view host, std::string_view port) buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) - .column_decimal(price_name, "2615.54"_utf8) + .column(price_name, "2615.54"_decimal) .column(amount_name, 0.00044) .at(questdb::ingress::timestamp_nanos::now()); diff --git a/examples/line_sender_cpp_example_tls_ca.cpp b/examples/line_sender_cpp_example_tls_ca.cpp index 3f778a54..7a36d0c7 100644 --- a/examples/line_sender_cpp_example_tls_ca.cpp +++ b/examples/line_sender_cpp_example_tls_ca.cpp @@ -3,6 +3,7 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; +using namespace questdb::ingress::decimal; static bool example( std::string_view ca_path, std::string_view host, std::string_view port) @@ -32,7 +33,7 @@ static bool example( buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) - .column_decimal(price_name, "2615.54"_utf8) + .column(price_name, "2615.54"_decimal) .column(amount_name, 0.00044) .at(questdb::ingress::timestamp_nanos::now()); diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index bca347d5..9a34e6ba 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -515,6 +515,26 @@ bool line_sender_buffer_column_decimal_str( line_sender_utf8 value, line_sender_error** err_out); +/** + * Record a decimal value for the given column. + * + * @param[in] buffer Line buffer object. + * @param[in] name Column name. + * @param[in] scale Number of digits after the decimal point + * @param[in] data Unscaled value in two's complement format, big-endian + * @param[in] data_len Length of the unscaled value array + * @param[out] err_out Set on error. + * @return true on success, false on error. + */ +LINESENDER_API +bool line_sender_buffer_column_decimal( + line_sender_buffer* buffer, + line_sender_column_name name, + const unsigned int scale, + const uint8_t* data, + size_t data_len, + line_sender_error** err_out); + /** * Record a multidimensional array of `double` values in C-major order. * diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 416d5ba8..75e62546 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -609,7 +609,7 @@ inline auto to_array_view_state_impl(const std::array& arr) } /** - * Customization point to enable serialization of additonal types as arrays. + * Customization point to enable serialization of additional types as arrays. * * Forwards to a namespace or ADL (König) lookup function. * The customized `to_array_view_state_impl` for your custom type can be placed @@ -637,6 +637,212 @@ inline constexpr to_array_view_state_fn to_array_view_state{}; } // namespace array +/** + * Types and utilities for working with arbitrary-precision decimal numbers. + * + * Decimals are represented as an unscaled integer value (mantissa) and a scale. + * For example, the decimal "123.45" with scale 2 is represented as: + * - Unscaled value: 12345 + * - Scale: 2 (meaning divide by 10^2 = 100) + * + * QuestDB supports decimal values with: + * - Maximum scale: 76 (QuestDB server limitation) + * - Maximum mantissa size: 127 bytes in binary format + * + * QuestDB server version 9.2.0 or later is required for decimal support. + */ +namespace decimal +{ + +/** + * A validated UTF-8 string view for text-based decimal representation. + * + * This is a wrapper around utf8_view that allows the compiler to distinguish + * between regular strings and decimal strings. + * + * Use this to send decimal values as strings (e.g., "123.456"). + * The string will be parsed by the QuestDB server as a decimal column type. + */ +class text_view +{ +public: + text_view(const char* buf, size_t len) + : _view{buf, len} + { + } + + template + text_view(const char (&buf)[N]) + : _view{buf} + { + } + + text_view(std::string_view s_view) + : _view{s_view} + { + } + + text_view(const std::string& s) + : _view{s} + { + } + + const utf8_view& view() const noexcept + { + return _view; + } + +private: + utf8_view _view; + + friend class line_sender_buffer; +}; + +/** + * Literal suffix to construct `text_view` objects from string literals. + * + * @code {.cpp} + * using namespace questdb::ingress::decimal; + * buffer.column("price"_cn, "123.456"_decimal); + * @endcode + */ +inline text_view operator"" _decimal(const char* buf, size_t len) +{ + return text_view{buf, len}; +} + +/** + * A view over a decimal number in binary format. + * + * The decimal is represented as: + * - A scale (number of decimal places) + * - An unscaled value (mantissa) encoded as bytes in two's complement + * big-endian format + * + * # Example + * + * To represent the decimal "123.45": + * - Scale: 2 + * - Unscaled value: 12345 = 0x3039 in big-endian format + * + * ```c++ + * // Represent 123.45 with scale 2 (unscaled value is 12345) + * uint8_t mantissa[] = {0x30, 0x39}; // 12345 in two's complement big-endian + * auto decimal = questdb::ingress::decimal::binary_view(2, mantissa, + * sizeof(mantissa)); buffer.column("price"_cn, decimal); + * ``` + * + * # Constraints + * + * - Maximum scale: 76 (QuestDB server limitation) + * - Maximum mantissa size: 127 bytes (protocol limitation) + */ +class binary_view +{ +public: + /** + * Construct a binary decimal view from raw bytes. + * + * @param scale Number of decimal places (must be ≤ 76) + * @param data Pointer to unscaled value in two's complement big-endian + * format + * @param data_size Number of bytes in the mantissa (must be ≤ 127) + */ + binary_view(uint32_t scale, const uint8_t* data, size_t data_size) + : _scale{scale} + , _data{data} + , _data_size{data_size} + { + } + + /** + * Construct a binary decimal view from a fixed-size array. + * + * @param scale Number of decimal places (must be ≤ 76) + * @param data Fixed-size array containing the unscaled value + */ + template + binary_view(uint32_t scale, const uint8_t (&data)[N]) + : _scale{scale} + , _data{data} + , _data_size{N} + { + } + + /** + * Construct a binary decimal view from a std::array. + * + * @param scale Number of decimal places (must be ≤ 76) + * @param data std::array containing the unscaled value + */ + template + binary_view(uint32_t scale, const std::array& data) + : _scale{scale} + , _data{data.data()} + , _data_size{N} + { + } + + /** + * Construct a binary decimal view from a std::vector. + * + * @param scale Number of decimal places (must be ≤ 76) + * @param vec Vector containing the unscaled value + */ + binary_view(uint32_t scale, const std::vector& vec) + : _scale{scale} + , _data{vec.data()} + , _data_size{vec.size()} + { + } + +#if __cplusplus >= 202002L + /** + * Construct a binary decimal view from a std::span (C++20). + * + * @param scale Number of decimal places (must be ≤ 76) + * @param span Span containing the unscaled value + */ + binary_view(uint32_t scale, const std::span& span) + : _scale{scale} + , _data{span.data()} + , _data_size{span.size()} + { + } +#endif + + /** Get the scale (number of decimal places). */ + uint32_t scale() const + { + return _scale; + } + + /** Get a pointer to the unscaled value bytes. */ + const uint8_t* data() const + { + return _data; + } + + /** Get the size of the unscaled value in bytes. */ + size_t data_size() const + { + return _data_size; + } + + /** Get a const reference to this view (for customization point + * compatibility). */ + const binary_view& view() const + { + return *this; + } + +private: + uint32_t _scale; + const uint8_t* _data; + size_t _data_size; +}; +} // namespace decimal + class line_sender_buffer { public: @@ -1036,38 +1242,59 @@ class line_sender_buffer } /** - * Record a decimal string value for the given column. - * @param name Column name. - * @param value Column value. + * Record an arbitrary-precision decimal value from a text representation. + * + * This sends the decimal as a string (e.g., "123.456") to be parsed by + * the QuestDB server. + * + * For better performance and precision control, consider using the binary + * format via `decimal::binary_view` instead. + * + * QuestDB server version 9.2.0 or later is required for decimal support. + * + * @param name Column name. + * @param value Decimal value as a validated UTF-8 string. */ - line_sender_buffer& column_decimal(column_name_view name, utf8_view value) + line_sender_buffer& column(column_name_view name, decimal::text_view value) { may_init(); line_sender_error::wrapped_call( ::line_sender_buffer_column_decimal_str, _impl, name._impl, - value._impl); + value.view()._impl); return *this; } - template - line_sender_buffer& column_decimal( - column_name_view name, const char (&value)[N]) - { - return column_decimal(name, utf8_view{value}); - } - - line_sender_buffer& column_decimal( - column_name_view name, std::string_view value) - { - return column_decimal(name, utf8_view{value}); - } - - line_sender_buffer& column_decimal( - column_name_view name, const std::string& value) + /** + * Record an arbitrary-precision decimal value in binary format. + * + * The decimal is represented as an unscaled integer (mantissa) and a scale. + * This provides precise control over the decimal representation and is more + * efficient than text-based serialization. + * + * QuestDB server version 9.2.0 or later is required for decimal support. + * + * # Constraints + * + * - Maximum scale: 76 (QuestDB server limitation) + * - Maximum mantissa size: 127 bytes (protocol limitation) + * + * @param name Column name. + * @param decimal Binary decimal view with scale and mantissa bytes. + */ + line_sender_buffer& column( + column_name_view name, const decimal::binary_view& decimal) { - return column_decimal(name, utf8_view{value}); + may_init(); + line_sender_error::wrapped_call( + ::line_sender_buffer_column_decimal, + _impl, + name._impl, + decimal.scale(), + decimal.data(), + decimal.data_size()); + return *this; } /** Record a nanosecond timestamp value for the given column. */ diff --git a/questdb-rs-ffi/src/decimal.rs b/questdb-rs-ffi/src/decimal.rs new file mode 100644 index 00000000..d93f44eb --- /dev/null +++ b/questdb-rs-ffi/src/decimal.rs @@ -0,0 +1,146 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +//! FFI-specific decimal number serialization for QuestDB ILP. +//! +//! This module provides decimal serialization support for the C FFI bindings. +//! Decimals are represented as arbitrary-precision numbers with a scale factor, +//! encoded in a binary format for transmission over the InfluxDB Line Protocol (ILP). + +use questdb::{ + ingress::{DecimalSerializer, DECIMAL_BINARY_FORMAT_TYPE}, + Result, +}; + +use crate::fmt_error; + +/// Represents a decimal number for binary serialization to QuestDB. +/// +/// A decimal consists of: +/// - An unscaled integer value (the mantissa), represented as raw bytes in big-endian format +/// - A scale indicating how many decimal places to shift (e.g., scale=2 means value/100) +/// +/// # Wire Format +/// +/// The binary serialization format is: +/// ```text +/// '=' marker (1 byte) + type ID (1 byte) + length (1 byte) + value bytes + scale (1 byte) +/// ``` +/// +/// # Constraints +/// +/// - Maximum scale: 76 (QuestDB server limitation) +/// - Maximum value size: 127 bytes (i8::MAX limitation from length field) +/// +/// # Example +/// +/// To represent the decimal `123.45` with scale 2: +/// - scale = 2 +/// - value = 12345 encoded as bytes [0x30, 0x39] (big-endian) +pub(super) struct Decimal<'a> { + /// The number of decimal places to shift. + /// For example, scale=2 means the value represents hundredths (divide by 100). + scale: u32, + + /// The unscaled integer value as raw bytes in big-endian format. + /// This represents the mantissa of the decimal number. + value: &'a [u8], +} + +impl<'a> Decimal<'a> { + /// Creates a new decimal number. + /// + /// # Arguments + /// + /// * `scale` - The number of decimal places (must be ≤ 76) + /// * `value` - The unscaled value as bytes in big-endian format (must be ≤ 127 bytes) + pub(super) fn new(scale: u32, value: &'a [u8]) -> Self { + Self { scale, value } + } +} + +impl<'a> DecimalSerializer for Decimal<'a> { + /// Serializes the decimal value into the QuestDB ILP binary format. + /// + /// # Wire Format Layout + /// + /// The serialization produces the following byte sequence: + /// 1. `'='` (0x3D) - Binary encoding marker + /// 2. Type ID (23) - Identifies this as a decimal type + /// 3. Length byte - Number of bytes in the value (max 127) + /// 4. Value bytes - The unscaled integer in big-endian format + /// 5. Scale byte - Number of decimal places (implicitly written after value) + /// + /// # Arguments + /// + /// * `out` - The output buffer to write the serialized decimal to + /// * `support_binary` - Whether the protocol version supports binary encoding + /// + /// # Errors + /// + /// Returns an error if: + /// - Binary encoding is not supported by the protocol version + /// - Scale exceeds 76 (QuestDB server maximum) + /// - Value size exceeds 127 bytes (protocol limitation) + fn serialize(self, out: &mut Vec, support_binary: bool) -> Result<()> { + // Binary encoding is required for decimal types (only available in protocol v2+) + if !support_binary { + return Err(fmt_error!( + ProtocolVersionError, + "Protocol version does not support binary encoding" + )); + } + + // Validate scale constraint (QuestDB server limitation) + // The server's decimal implementation supports a maximum scale of 76 + if self.scale > 76 { + return Err(fmt_error!( + InvalidDecimal, + "QuestDB ILP does not support scale greater than 76, got {}", + self.scale + )); + } + + // Write binary format header + out.push(b'='); // Binary encoding marker + out.push(DECIMAL_BINARY_FORMAT_TYPE); // Type ID = 23 + + // Validate value size constraint (protocol limitation) + // The length field is a single byte (i8), limiting value size to 127 bytes + if self.value.len() > i8::MAX as usize { + return Err(fmt_error!( + InvalidDecimal, + "QuestDB ILP does not support values greater than {} bytes, got {}", + i8::MAX, + self.value.len() + )); + } + + out.push(self.scale as u8); + out.push(self.value.len() as u8); + out.extend_from_slice(self.value); + + Ok(()) + } +} diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 6244090c..fa3052a8 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -44,6 +44,7 @@ use questdb::{ mod ndarr; use ndarr::StrideArrayView; +mod decimal; macro_rules! bubble_err_to_c { ($err_out:expr, $expression:expr) => { @@ -939,6 +940,7 @@ pub unsafe extern "C" fn line_sender_buffer_column_str( } /// Record a decimal string value for the given column. +/// /// @param[in] buffer Line buffer object. /// @param[in] name Column name. /// @param[in] value Column value. @@ -958,6 +960,31 @@ pub unsafe extern "C" fn line_sender_buffer_column_decimal_str( true } +/// Record a decimal value for the given column. +/// +/// @param[in] buffer Line buffer object. +/// @param[in] name Column name. +/// @param[in] scale Number of digits after the decimal point +/// @param[in] data Unscaled value in two's complement format, big-endian +/// @param[in] data_len Length of the unscaled value array +/// @param[out] err_out Set on error. +/// @return true on success, false on error. +#[no_mangle] +pub unsafe extern "C" fn line_sender_buffer_column_decimal( + buffer: *mut line_sender_buffer, + name: line_sender_column_name, + scale: u32, + data: *const u8, + data_len: size_t, + err_out: *mut *mut line_sender_error, +) -> bool { + let buffer = unwrap_buffer_mut(buffer); + let name = name.as_name(); + let decimal = Decimal::new(scale, slice::from_raw_parts(data, data_len)); + bubble_err_to_c!(err_out, buffer.column_decimal(name, decimal)); + true +} + /// Records a float64 multidimensional array with **C-MAJOR memory layout**. /// /// @param[in] buffer Line buffer object. @@ -1773,6 +1800,7 @@ pub unsafe extern "C" fn line_sender_now_micros() -> i64 { TimestampMicros::now().as_i64() } +use crate::decimal::Decimal; use crate::ndarr::CMajorArrayView; #[cfg(feature = "confstr-ffi")] use questdb_confstr_ffi::questdb_conf_str_parse_err; diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 8696acfc..27b1020b 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -75,7 +75,7 @@ pub const MAX_ARRAY_DIM_LEN: usize = 0x0FFF_FFFF; // 1 << 28 - 1 pub(crate) const ARRAY_BINARY_FORMAT_TYPE: u8 = 14; pub(crate) const DOUBLE_BINARY_FORMAT_TYPE: u8 = 16; #[allow(dead_code)] -pub(crate) const DECIMAL_BINARY_FORMAT_TYPE: u8 = 23; +pub const DECIMAL_BINARY_FORMAT_TYPE: u8 = 23; /// The version of InfluxDB Line Protocol used to communicate with the server. #[derive(Debug, Copy, Clone, PartialEq)] From ce0fe1752886e100c71f3407919774db246cd047 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Tue, 14 Oct 2025 15:54:38 +0200 Subject: [PATCH 17/65] typo: fix invalid comments --- examples/line_sender_cpp_example_decimal_custom.cpp | 2 +- questdb-rs-ffi/src/decimal.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/line_sender_cpp_example_decimal_custom.cpp b/examples/line_sender_cpp_example_decimal_custom.cpp index be5ca809..c3f76a4a 100644 --- a/examples/line_sender_cpp_example_decimal_custom.cpp +++ b/examples/line_sender_cpp_example_decimal_custom.cpp @@ -29,7 +29,7 @@ class Decimal32 questdb::ingress::decimal::binary_view view() const { std::array data = { - // Big-Endiang bytes + // Big-Endian bytes static_cast(_unscaled_value >> 24), static_cast(_unscaled_value >> 16), static_cast(_unscaled_value >> 8), diff --git a/questdb-rs-ffi/src/decimal.rs b/questdb-rs-ffi/src/decimal.rs index d93f44eb..5a93300d 100644 --- a/questdb-rs-ffi/src/decimal.rs +++ b/questdb-rs-ffi/src/decimal.rs @@ -88,9 +88,9 @@ impl<'a> DecimalSerializer for Decimal<'a> { /// The serialization produces the following byte sequence: /// 1. `'='` (0x3D) - Binary encoding marker /// 2. Type ID (23) - Identifies this as a decimal type - /// 3. Length byte - Number of bytes in the value (max 127) - /// 4. Value bytes - The unscaled integer in big-endian format - /// 5. Scale byte - Number of decimal places (implicitly written after value) + /// 3. Scale byte - Number of decimal places + /// 4. Length byte - Number of bytes in the value (max 127) + /// 5. Value bytes - The unscaled integer in big-endian format /// /// # Arguments /// From d2ee26c284dab63341ec14e72da6444e8f1bb88d Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Tue, 14 Oct 2025 17:29:33 +0200 Subject: [PATCH 18/65] feat: add protocol version 3 --- doc/C.md | 2 +- examples/line_sender_c_example.c | 2 +- examples/line_sender_c_example_auth.c | 2 +- examples/line_sender_c_example_auth_tls.c | 2 +- examples/line_sender_c_example_from_conf.c | 2 +- examples/line_sender_c_example_from_env.c | 2 +- examples/line_sender_c_example_http.c | 2 +- examples/line_sender_c_example_tls_ca.c | 2 +- include/questdb/ingress/line_sender.h | 16 +- include/questdb/ingress/line_sender.hpp | 4 +- questdb-rs-ffi/src/decimal.rs | 11 +- questdb-rs-ffi/src/lib.rs | 16 +- questdb-rs/build.rs | 35 ++- questdb-rs/examples/basic.rs | 2 +- questdb-rs/examples/http.rs | 2 +- questdb-rs/examples/protocol_version.rs | 4 +- questdb-rs/src/ingress/buffer.rs | 28 ++- questdb-rs/src/ingress/decimal.rs | 42 +--- questdb-rs/src/ingress/mod.rs | 18 +- questdb-rs/src/ingress/sender/http.rs | 1 + questdb-rs/src/tests/decimal.rs | 226 +++++------------- questdb-rs/src/tests/http.rs | 2 +- .../interop/ilp-client-interop-test.json | 1 + questdb-rs/src/tests/sender.rs | 13 +- system_test/questdb_line_sender.py | 5 +- system_test/test.py | 10 +- 26 files changed, 188 insertions(+), 264 deletions(-) diff --git a/doc/C.md b/doc/C.md index 3f4222f1..8898b276 100644 --- a/doc/C.md +++ b/doc/C.md @@ -91,7 +91,7 @@ if (!line_sender_buffer_symbol(buffer, symbol_name, symbol_value, &err)) goto on_error; line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); -if (!line_sender_buffer_column_decimal_str( +if (!line_sender_buffer_column_dec_str( buffer, price_name, price_value, &err)) goto on_error; diff --git a/examples/line_sender_c_example.c b/examples/line_sender_c_example.c index dbaffb5e..50ce854a 100644 --- a/examples/line_sender_c_example.c +++ b/examples/line_sender_c_example.c @@ -52,7 +52,7 @@ static bool example(const char* host, const char* port) goto on_error; line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); - if (!line_sender_buffer_column_decimal_str( + if (!line_sender_buffer_column_dec_str( buffer, price_name, price_value, &err)) goto on_error; diff --git a/examples/line_sender_c_example_auth.c b/examples/line_sender_c_example_auth.c index 4fac1691..130941eb 100644 --- a/examples/line_sender_c_example_auth.c +++ b/examples/line_sender_c_example_auth.c @@ -61,7 +61,7 @@ static bool example(const char* host, const char* port) goto on_error; line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); - if (!line_sender_buffer_column_decimal_str( + if (!line_sender_buffer_column_dec_str( buffer, price_name, price_value, &err)) goto on_error; diff --git a/examples/line_sender_c_example_auth_tls.c b/examples/line_sender_c_example_auth_tls.c index d060270d..e3ef6f3e 100644 --- a/examples/line_sender_c_example_auth_tls.c +++ b/examples/line_sender_c_example_auth_tls.c @@ -62,7 +62,7 @@ static bool example(const char* host, const char* port) goto on_error; line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); - if (!line_sender_buffer_column_decimal_str( + if (!line_sender_buffer_column_dec_str( buffer, price_name, price_value, &err)) goto on_error; diff --git a/examples/line_sender_c_example_from_conf.c b/examples/line_sender_c_example_from_conf.c index 4bca3b8a..ead11a2f 100644 --- a/examples/line_sender_c_example_from_conf.c +++ b/examples/line_sender_c_example_from_conf.c @@ -39,7 +39,7 @@ int main(int argc, const char* argv[]) goto on_error; line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); - if (!line_sender_buffer_column_decimal_str( + if (!line_sender_buffer_column_dec_str( buffer, price_name, price_value, &err)) goto on_error; diff --git a/examples/line_sender_c_example_from_env.c b/examples/line_sender_c_example_from_env.c index 8b531392..6ab78c3b 100644 --- a/examples/line_sender_c_example_from_env.c +++ b/examples/line_sender_c_example_from_env.c @@ -38,7 +38,7 @@ int main(int argc, const char* argv[]) goto on_error; line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); - if (!line_sender_buffer_column_decimal_str( + if (!line_sender_buffer_column_dec_str( buffer, price_name, price_value, &err)) goto on_error; diff --git a/examples/line_sender_c_example_http.c b/examples/line_sender_c_example_http.c index 38f6f63d..0d12d602 100644 --- a/examples/line_sender_c_example_http.c +++ b/examples/line_sender_c_example_http.c @@ -50,7 +50,7 @@ static bool example(const char* host, const char* port) goto on_error; line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); - if (!line_sender_buffer_column_decimal_str( + if (!line_sender_buffer_column_dec_str( buffer, price_name, price_value, &err)) goto on_error; diff --git a/examples/line_sender_c_example_tls_ca.c b/examples/line_sender_c_example_tls_ca.c index d217da1f..a0540487 100644 --- a/examples/line_sender_c_example_tls_ca.c +++ b/examples/line_sender_c_example_tls_ca.c @@ -65,7 +65,7 @@ static bool example(const char* ca_path, const char* host, const char* port) goto on_error; line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); - if (!line_sender_buffer_column_decimal_str( + if (!line_sender_buffer_column_dec_str( buffer, price_name, price_value, &err)) goto on_error; diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 9a34e6ba..10b9e332 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -122,6 +122,15 @@ typedef enum line_sender_protocol_version * `line_sender_protocol_version_2` support. */ line_sender_protocol_version_2 = 2, + + /** + * Version 3 of InfluxDB Line Protocol. + * Supports the decimal data type in text and binary formats. + * This version is specific to QuestDB and not compatible with InfluxDB. + * QuestDB server version 9.2.0 or later is required for + * `line_sender_protocol_version_3` support. + */ + line_sender_protocol_version_3 = 3, } line_sender_protocol_version; /** Possible sources of the root certificates used to validate the server's @@ -509,7 +518,7 @@ bool line_sender_buffer_column_str( * @return true on success, false on error. */ LINESENDER_API -bool line_sender_buffer_column_decimal_str( +bool line_sender_buffer_column_dec_str( line_sender_buffer* buffer, line_sender_column_name name, line_sender_utf8 value, @@ -527,7 +536,7 @@ bool line_sender_buffer_column_decimal_str( * @return true on success, false on error. */ LINESENDER_API -bool line_sender_buffer_column_decimal( +bool line_sender_buffer_column_dec( line_sender_buffer* buffer, line_sender_column_name name, const unsigned int scale, @@ -881,6 +890,9 @@ bool line_sender_opts_token_y( * * QuestDB server version 9.0.0 or later is required for * `line_sender_protocol_version_2` support. + * + * QuestDB server version 9.2.0 or later is required for + * `line_sender_protocol_version_3` support. */ LINESENDER_API bool line_sender_opts_protocol_version( diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 75e62546..2e803ff0 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -1259,7 +1259,7 @@ class line_sender_buffer { may_init(); line_sender_error::wrapped_call( - ::line_sender_buffer_column_decimal_str, + ::line_sender_buffer_column_dec_str, _impl, name._impl, value.view()._impl); @@ -1288,7 +1288,7 @@ class line_sender_buffer { may_init(); line_sender_error::wrapped_call( - ::line_sender_buffer_column_decimal, + ::line_sender_buffer_column_dec, _impl, name._impl, decimal.scale(), diff --git a/questdb-rs-ffi/src/decimal.rs b/questdb-rs-ffi/src/decimal.rs index 5a93300d..c4fd0753 100644 --- a/questdb-rs-ffi/src/decimal.rs +++ b/questdb-rs-ffi/src/decimal.rs @@ -100,18 +100,9 @@ impl<'a> DecimalSerializer for Decimal<'a> { /// # Errors /// /// Returns an error if: - /// - Binary encoding is not supported by the protocol version /// - Scale exceeds 76 (QuestDB server maximum) /// - Value size exceeds 127 bytes (protocol limitation) - fn serialize(self, out: &mut Vec, support_binary: bool) -> Result<()> { - // Binary encoding is required for decimal types (only available in protocol v2+) - if !support_binary { - return Err(fmt_error!( - ProtocolVersionError, - "Protocol version does not support binary encoding" - )); - } - + fn serialize(self, out: &mut Vec) -> Result<()> { // Validate scale constraint (QuestDB server limitation) // The server's decimal implementation supports a maximum scale of 76 if self.scale > 76 { diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index fa3052a8..7d3bb99b 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -314,6 +314,12 @@ pub enum ProtocolVersion { /// This version is specific to QuestDB and is not compatible with InfluxDB. /// QuestDB server version 9.0.0 or later is required for `V2` supported. V2 = 2, + + /// Version 3 of InfluxDB Line Protocol. + /// Supports the decimal data type in text and binary formats. + /// This version is specific to QuestDB and is not compatible with InfluxDB. + /// QuestDB server version 9.2.0 or later is required for `V3` supported. + V3 = 3, } impl From for ingress::ProtocolVersion { @@ -321,6 +327,7 @@ impl From for ingress::ProtocolVersion { match version { ProtocolVersion::V1 => ingress::ProtocolVersion::V1, ProtocolVersion::V2 => ingress::ProtocolVersion::V2, + ProtocolVersion::V3 => ingress::ProtocolVersion::V3, } } } @@ -330,6 +337,7 @@ impl From for ProtocolVersion { match version { ingress::ProtocolVersion::V1 => ProtocolVersion::V1, ingress::ProtocolVersion::V2 => ProtocolVersion::V2, + ingress::ProtocolVersion::V3 => ProtocolVersion::V3, } } } @@ -947,7 +955,7 @@ pub unsafe extern "C" fn line_sender_buffer_column_str( /// @param[out] err_out Set on error. /// @return true on success, false on error. #[no_mangle] -pub unsafe extern "C" fn line_sender_buffer_column_decimal_str( +pub unsafe extern "C" fn line_sender_buffer_column_dec_str( buffer: *mut line_sender_buffer, name: line_sender_column_name, value: line_sender_utf8, @@ -956,7 +964,7 @@ pub unsafe extern "C" fn line_sender_buffer_column_decimal_str( let buffer = unwrap_buffer_mut(buffer); let name = name.as_name(); let value = value.as_str(); - bubble_err_to_c!(err_out, buffer.column_decimal(name, value)); + bubble_err_to_c!(err_out, buffer.column_dec(name, value)); true } @@ -970,7 +978,7 @@ pub unsafe extern "C" fn line_sender_buffer_column_decimal_str( /// @param[out] err_out Set on error. /// @return true on success, false on error. #[no_mangle] -pub unsafe extern "C" fn line_sender_buffer_column_decimal( +pub unsafe extern "C" fn line_sender_buffer_column_dec( buffer: *mut line_sender_buffer, name: line_sender_column_name, scale: u32, @@ -981,7 +989,7 @@ pub unsafe extern "C" fn line_sender_buffer_column_decimal( let buffer = unwrap_buffer_mut(buffer); let name = name.as_name(); let decimal = Decimal::new(scale, slice::from_raw_parts(data, data_len)); - bubble_err_to_c!(err_out, buffer.column_decimal(name, decimal)); + bubble_err_to_c!(err_out, buffer.column_dec(name, decimal)); true } diff --git a/questdb-rs/build.rs b/questdb-rs/build.rs index f943d6b5..a4cc94cc 100644 --- a/questdb-rs/build.rs +++ b/questdb-rs/build.rs @@ -95,10 +95,19 @@ pub mod json_tests { Error, } + fn default_minimum_protocol_version() -> u32 { + 1 + } + #[derive(Debug, Serialize, Deserialize)] struct TestSpec { #[serde(rename = "testName")] test_name: String, + #[serde( + rename = "minimumProtocolVersion", + default = "default_minimum_protocol_version" + )] + minimum_protocol_version: u32, table: String, symbols: Vec, columns: Vec, @@ -150,18 +159,24 @@ pub mod json_tests { )?; for (index, spec) in specs.iter().enumerate() { - writeln!(output, "/// {}", spec.test_name)?; - // for line in serde_json::to_string_pretty(&spec).unwrap().split("\n") { - // writeln!(output, "/// {}", line)?; - // } - writeln!(output, "#[rstest]")?; - writeln!( + write!( output, - "fn test_{:03}_{}(\n #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion,\n) -> TestResult {{", + indoc! {r#" + /// {} + #[rstest] + fn test_{:03}_{}( + #[values(ProtocolVersion::V1, ProtocolVersion::V2, ProtocolVersion::V3)] version: ProtocolVersion + ) -> TestResult {{ + if (version as u8) < {} {{ + return Ok(()); + }} + let mut buffer = Buffer::new(version); + "#}, + spec.test_name, index, - slugify!(&spec.test_name, separator = "_") + slugify!(&spec.test_name, separator = "_"), + spec.minimum_protocol_version )?; - writeln!(output, " let mut buffer = Buffer::new(version);")?; let (expected, indent) = match &spec.result { Outcome::Success(line) => (Some(line), ""), @@ -203,7 +218,7 @@ pub mod json_tests { )?, Column::Decimal(column) => writeln!( output, - "{} .column_decimal({:?}, &BigDecimal::from_str({:?}).unwrap())?", + "{} .column_dec({:?}, &BigDecimal::from_str({:?}).unwrap())?", indent, column.name, column.value )?, } diff --git a/questdb-rs/examples/basic.rs b/questdb-rs/examples/basic.rs index 587c6dbd..f68c7bac 100644 --- a/questdb-rs/examples/basic.rs +++ b/questdb-rs/examples/basic.rs @@ -16,7 +16,7 @@ fn main() -> Result<()> { .table("trades")? .symbol("symbol", "ETH-USD")? .symbol("side", "sell")? - .column_decimal("price", "2615.54")? + .column_dec("price", "2615.54")? .column_f64("amount", 0.00044)? // QuestDB server version 9.0.0 or later is required for array support. .column_arr("location", &arr1(&[100.0, 100.1, 100.2]).view())? diff --git a/questdb-rs/examples/http.rs b/questdb-rs/examples/http.rs index c4f38002..0eaea654 100644 --- a/questdb-rs/examples/http.rs +++ b/questdb-rs/examples/http.rs @@ -17,7 +17,7 @@ fn main() -> Result<()> { .table("trades")? .symbol("symbol", "ETH-USD")? .symbol("side", "sell")? - .column_decimal("price", &price)? + .column_dec("price", &price)? .column_f64("amount", 0.00044)? // QuestDB server version 9.0.0 or later is required for array support. .column_arr("location", &arr1(&[100.0, 100.1, 100.2]).view())? diff --git a/questdb-rs/examples/protocol_version.rs b/questdb-rs/examples/protocol_version.rs index 6e7dfa23..6682c5a0 100644 --- a/questdb-rs/examples/protocol_version.rs +++ b/questdb-rs/examples/protocol_version.rs @@ -18,7 +18,7 @@ fn main() -> Result<()> { .table("trades_ilp_v1")? .symbol("symbol", "ETH-USD")? .symbol("side", "sell")? - .column_decimal("price", &price)? + .column_dec("price", &price)? .column_f64("amount", 0.00044)? .at(TimestampNanos::now())?; sender.flush(&mut buffer)?; @@ -32,7 +32,7 @@ fn main() -> Result<()> { .table("trades_ilp_v2")? .symbol("symbol", "ETH-USD")? .symbol("side", "sell")? - .column_decimal("price", &price)? + .column_dec("price", &price)? .column_f64("amount", 0.00044)? .column_arr("location", &arr1(&[100.0, 100.1, 100.2]).view())? .at(TimestampNanos::now())?; diff --git a/questdb-rs/src/ingress/buffer.rs b/questdb-rs/src/ingress/buffer.rs index 17ff7ff0..16cd542f 100644 --- a/questdb-rs/src/ingress/buffer.rs +++ b/questdb-rs/src/ingress/buffer.rs @@ -984,7 +984,7 @@ impl Buffer { /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; - /// buffer.column_decimal("col_name", "123.45")?; + /// buffer.column_dec("col_name", "123.45")?; /// # Ok(()) /// # } /// ``` @@ -1001,7 +1001,7 @@ impl Buffer { /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// let col_name = ColumnName::new("col_name")?; - /// buffer.column_decimal(col_name, "123.45")?; + /// buffer.column_dec(col_name, "123.45")?; /// # Ok(()) /// # } /// ``` @@ -1021,7 +1021,7 @@ impl Buffer { /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// let value = Decimal::from_str("123.45").unwrap(); - /// buffer.column_decimal("col_name", &value)?; + /// buffer.column_dec("col_name", &value)?; /// # Ok(()) /// # } /// # } @@ -1042,22 +1042,27 @@ impl Buffer { /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// let value = BigDecimal::from_str("0.123456789012345678901234567890").unwrap(); - /// buffer.column_decimal("col_name", &value)?; + /// buffer.column_dec("col_name", &value)?; /// # Ok(()) /// # } /// # } /// ``` - pub fn column_decimal<'a, N, S>(&mut self, name: N, value: S) -> crate::Result<&mut Self> + pub fn column_dec<'a, N, S>(&mut self, name: N, value: S) -> crate::Result<&mut Self> where N: TryInto>, S: DecimalSerializer, Error: From, { + if !self.protocol_version.supports(ProtocolVersion::V3) { + return Err(error::fmt!( + ProtocolVersionError, + "Protocol version {} does not support the decimal datatype", + self.protocol_version + )); + } + self.write_column_key(name)?; - value.serialize( - &mut self.output, - self.protocol_version.supports_binary_encoding(), - )?; + value.serialize(&mut self.output)?; Ok(self) } @@ -1116,10 +1121,11 @@ impl Buffer { D: ArrayElement + ArrayElementSealed, Error: From, { - if self.protocol_version == ProtocolVersion::V1 { + if !self.protocol_version.supports(ProtocolVersion::V2) { return Err(error::fmt!( ProtocolVersionError, - "Protocol version v1 does not support array datatype", + "Protocol version {} does not support array datatype", + self.protocol_version )); } let ndim = view.ndim(); diff --git a/questdb-rs/src/ingress/decimal.rs b/questdb-rs/src/ingress/decimal.rs index f7162e19..b6c1e495 100644 --- a/questdb-rs/src/ingress/decimal.rs +++ b/questdb-rs/src/ingress/decimal.rs @@ -71,9 +71,7 @@ pub trait DecimalSerializer { /// # Parameters /// /// * `out` - The output buffer to write the serialized decimal to - /// * `support_binary` - If `true`, binary format may be used (Protocol V2). - /// If `false`, text format must be used (Protocol V1). - fn serialize(self, out: &mut Vec, support_binary: bool) -> Result<()>; + fn serialize(self, out: &mut Vec) -> Result<()>; } /// Implementation for string slices containing decimal representations. @@ -101,7 +99,7 @@ pub trait DecimalSerializer { /// Returns [`Error`] with [`ErrorCode::InvalidDecimal`](crate::error::ErrorCode::InvalidDecimal) /// if the string contains ILP reserved characters. impl DecimalSerializer for &str { - fn serialize(self, out: &mut Vec, _support_binary: bool) -> Result<()> { + fn serialize(self, out: &mut Vec) -> Result<()> { // Pre-allocate space for the string content plus the 'd' suffix out.reserve(self.len() + 1); @@ -128,32 +126,9 @@ impl DecimalSerializer for &str { #[cfg(any(feature = "rust_decimal", feature = "bigdecimal"))] use crate::ingress::DECIMAL_BINARY_FORMAT_TYPE; -/// Helper to format decimal values directly to a byte buffer without heap allocation. -#[cfg(any(feature = "rust_decimal", feature = "bigdecimal"))] -struct DecimalWriter<'a> { - buf: &'a mut Vec, -} - -#[cfg(any(feature = "rust_decimal", feature = "bigdecimal"))] -impl<'a> std::fmt::Write for DecimalWriter<'a> { - fn write_str(&mut self, s: &str) -> std::fmt::Result { - self.buf.extend_from_slice(s.as_bytes()); - Ok(()) - } -} - #[cfg(feature = "rust_decimal")] impl DecimalSerializer for &rust_decimal::Decimal { - fn serialize(self, out: &mut Vec, support_binary: bool) -> Result<()> { - if !support_binary { - // Text format - use std::fmt::Write; - write!(DecimalWriter { buf: out }, "{}", self) - .map_err(|_| error::fmt!(InvalidDecimal, "Failed to format decimal value"))?; - out.push(b'd'); - return Ok(()); - } - + fn serialize(self, out: &mut Vec) -> Result<()> { // Binary format: '=' marker + type + scale + length + mantissa bytes out.push(b'='); out.push(DECIMAL_BINARY_FORMAT_TYPE); @@ -175,16 +150,7 @@ impl DecimalSerializer for &rust_decimal::Decimal { #[cfg(feature = "bigdecimal")] impl DecimalSerializer for &bigdecimal::BigDecimal { - fn serialize(self, out: &mut Vec, support_binary: bool) -> Result<()> { - if !support_binary { - // Text format - use std::fmt::Write; - write!(DecimalWriter { buf: out }, "{}", self) - .map_err(|_| error::fmt!(InvalidDecimal, "Failed to format decimal value"))?; - out.push(b'd'); - return Ok(()); - } - + fn serialize(self, out: &mut Vec) -> Result<()> { // Binary format: '=' marker + type + scale + length + mantissa bytes out.push(b'='); out.push(DECIMAL_BINARY_FORMAT_TYPE); diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 27b1020b..e24fe9b7 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -90,22 +90,29 @@ pub enum ProtocolVersion { /// This version is specific to QuestDB and is not compatible with InfluxDB. /// QuestDB server version 9.0.0 or later is required for `V2` supported. V2 = 2, + + /// Version 3 of InfluxDB Line Protocol. + /// Supports the decimal data type in text and binary formats. + /// This version is specific to QuestDB and is not compatible with InfluxDB. + /// QuestDB server version 9.2.0 or later is required for `V3` supported. + V3 = 3, } impl ProtocolVersion { - /// Returns `true` if this protocol version supports binary-encoded column values. + /// Returns `true` if this protocol version supports the given protocol version. /// /// # Examples /// /// ``` /// use questdb::ingress::ProtocolVersion; /// - /// assert_eq!(ProtocolVersion::V1.supports_binary_encoding(), false); - /// assert_eq!(ProtocolVersion::V2.supports_binary_encoding(), true); + /// assert_eq!(ProtocolVersion::V1.supports(ProtocolVersion::V2), false); + /// assert_eq!(ProtocolVersion::V2.supports(ProtocolVersion::V1), true); /// ``` #[inline] - pub fn supports_binary_encoding(self) -> bool { - self != ProtocolVersion::V1 + pub fn supports(self, version: ProtocolVersion) -> bool { + // Protocol versions are backward compatible + self as u32 >= version as u32 } } @@ -114,6 +121,7 @@ impl Display for ProtocolVersion { match self { ProtocolVersion::V1 => write!(f, "v1"), ProtocolVersion::V2 => write!(f, "v2"), + ProtocolVersion::V3 => write!(f, "v3"), } } } diff --git a/questdb-rs/src/ingress/sender/http.rs b/questdb-rs/src/ingress/sender/http.rs index 3a99be8e..9ac400af 100644 --- a/questdb-rs/src/ingress/sender/http.rs +++ b/questdb-rs/src/ingress/sender/http.rs @@ -466,6 +466,7 @@ pub(crate) fn read_server_settings( match v { 1 => support_versions.push(ProtocolVersion::V1), 2 => support_versions.push(ProtocolVersion::V2), + 3 => support_versions.push(ProtocolVersion::V3), _ => {} } } diff --git a/questdb-rs/src/tests/decimal.rs b/questdb-rs/src/tests/decimal.rs index a310d804..f8a1b2a9 100644 --- a/questdb-rs/src/tests/decimal.rs +++ b/questdb-rs/src/tests/decimal.rs @@ -25,14 +25,12 @@ use crate::ingress::{Buffer, DecimalSerializer, ProtocolVersion}; use crate::tests::TestResult; use crate::ErrorCode; +use rstest::rstest; // Helper function to serialize a decimal value and return the bytes -fn serialize_decimal( - value: D, - support_binary: bool, -) -> crate::Result> { +fn serialize_decimal(value: D) -> crate::Result> { let mut out = Vec::new(); - value.serialize(&mut out, support_binary)?; + value.serialize(&mut out)?; Ok(out) } @@ -42,57 +40,49 @@ fn serialize_decimal( #[test] fn test_str_positive_decimal() -> TestResult { - let result = serialize_decimal("123.45", false)?; + let result = serialize_decimal("123.45")?; assert_eq!(result, b"123.45d"); Ok(()) } #[test] fn test_str_negative_decimal() -> TestResult { - let result = serialize_decimal("-123.45", false)?; + let result = serialize_decimal("-123.45")?; assert_eq!(result, b"-123.45d"); Ok(()) } #[test] fn test_str_zero() -> TestResult { - let result = serialize_decimal("0", false)?; + let result = serialize_decimal("0")?; assert_eq!(result, b"0d"); Ok(()) } #[test] fn test_str_scientific_notation() -> TestResult { - let result = serialize_decimal("1.5e-3", false)?; + let result = serialize_decimal("1.5e-3")?; assert_eq!(result, b"1.5e-3d"); Ok(()) } #[test] fn test_str_large_decimal() -> TestResult { - let result = serialize_decimal("999999999999999999.123456789", false)?; + let result = serialize_decimal("999999999999999999.123456789")?; assert_eq!(result, b"999999999999999999.123456789d"); Ok(()) } #[test] fn test_str_with_leading_zero() -> TestResult { - let result = serialize_decimal("0.001", false)?; + let result = serialize_decimal("0.001")?; assert_eq!(result, b"0.001d"); Ok(()) } -#[test] -fn test_str_ignores_binary_flag() -> TestResult { - // &str always uses text format, even with support_binary=true - let result = serialize_decimal("42.0", true)?; - assert_eq!(result, b"42.0d"); - Ok(()) -} - #[test] fn test_str_rejects_space() -> TestResult { - let result = serialize_decimal("12 3.45", false); + let result = serialize_decimal("12 3.45"); assert!(result.is_err()); let err = result.unwrap_err(); assert_eq!(err.code(), ErrorCode::InvalidDecimal); @@ -102,7 +92,7 @@ fn test_str_rejects_space() -> TestResult { #[test] fn test_str_rejects_comma() -> TestResult { - let result = serialize_decimal("1,234.56", false); + let result = serialize_decimal("1,234.56"); assert!(result.is_err()); let err = result.unwrap_err(); assert_eq!(err.code(), ErrorCode::InvalidDecimal); @@ -111,7 +101,7 @@ fn test_str_rejects_comma() -> TestResult { #[test] fn test_str_rejects_equals() -> TestResult { - let result = serialize_decimal("123=45", false); + let result = serialize_decimal("123=45"); assert!(result.is_err()); let err = result.unwrap_err(); assert_eq!(err.code(), ErrorCode::InvalidDecimal); @@ -120,7 +110,7 @@ fn test_str_rejects_equals() -> TestResult { #[test] fn test_str_rejects_newline() -> TestResult { - let result = serialize_decimal("123\n45", false); + let result = serialize_decimal("123\n45"); assert!(result.is_err()); let err = result.unwrap_err(); assert_eq!(err.code(), ErrorCode::InvalidDecimal); @@ -129,7 +119,7 @@ fn test_str_rejects_newline() -> TestResult { #[test] fn test_str_rejects_backslash() -> TestResult { - let result = serialize_decimal("123\\45", false); + let result = serialize_decimal("123\\45"); assert!(result.is_err()); let err = result.unwrap_err(); assert_eq!(err.code(), ErrorCode::InvalidDecimal); @@ -192,34 +182,10 @@ mod rust_decimal_tests { use rust_decimal::Decimal; use std::str::FromStr; - #[test] - fn test_decimal_text_format() -> TestResult { - let dec = Decimal::from_str("123.45")?; - let result = serialize_decimal(&dec, false)?; - assert_eq!(result, b"123.45d"); - Ok(()) - } - - #[test] - fn test_decimal_negative_text_format() -> TestResult { - let dec = Decimal::from_str("-123.45")?; - let result = serialize_decimal(&dec, false)?; - assert_eq!(result, b"-123.45d"); - Ok(()) - } - - #[test] - fn test_decimal_zero_text_format() -> TestResult { - let dec = Decimal::ZERO; - let result = serialize_decimal(&dec, false)?; - assert_eq!(result, b"0d"); - Ok(()) - } - #[test] fn test_decimal_binary_format_zero() -> TestResult { let dec = Decimal::ZERO; - let result = serialize_decimal(&dec, true)?; + let result = serialize_decimal(&dec)?; let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 0, "Zero should have scale 0"); @@ -230,7 +196,7 @@ mod rust_decimal_tests { #[test] fn test_decimal_binary_format_positive() -> TestResult { let dec = Decimal::from_str("123.45")?; - let result = serialize_decimal(&dec, true)?; + let result = serialize_decimal(&dec)?; let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 2, "123.45 should have scale 2"); @@ -241,7 +207,7 @@ mod rust_decimal_tests { #[test] fn test_decimal_binary_format_negative() -> TestResult { let dec = Decimal::from_str("-123.45")?; - let result = serialize_decimal(&dec, true)?; + let result = serialize_decimal(&dec)?; let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 2, "-123.45 should have scale 2"); @@ -255,7 +221,7 @@ mod rust_decimal_tests { #[test] fn test_decimal_binary_format_one() -> TestResult { let dec = Decimal::ONE; - let result = serialize_decimal(&dec, true)?; + let result = serialize_decimal(&dec)?; let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 0, "One should have scale 0"); @@ -267,7 +233,7 @@ mod rust_decimal_tests { fn test_decimal_binary_format_max_scale() -> TestResult { // Create a decimal with maximum scale (28 for rust_decimal) let dec = Decimal::from_str("0.0000000000000000000000000001")?; - let result = serialize_decimal(&dec, true)?; + let result = serialize_decimal(&dec)?; let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 28, "Should have maximum scale of 28"); @@ -278,7 +244,7 @@ mod rust_decimal_tests { #[test] fn test_decimal_binary_format_large_value() -> TestResult { let dec = Decimal::MAX; - let result = serialize_decimal(&dec, true)?; + let result = serialize_decimal(&dec)?; let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 0, "Large integer should have scale 0"); @@ -292,7 +258,7 @@ mod rust_decimal_tests { #[test] fn test_decimal_binary_format_large_value2() -> TestResult { let dec = Decimal::MIN; - let result = serialize_decimal(&dec, true)?; + let result = serialize_decimal(&dec)?; let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 0, "Large integer should have scale 0"); @@ -306,7 +272,7 @@ mod rust_decimal_tests { #[test] fn test_decimal_binary_format_small_negative() -> TestResult { let dec = Decimal::from_str("-0.01")?; - let result = serialize_decimal(&dec, true)?; + let result = serialize_decimal(&dec)?; let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 2, "-0.01 should have scale 2"); @@ -317,7 +283,7 @@ mod rust_decimal_tests { #[test] fn test_decimal_binary_format_trailing_zeros() -> TestResult { let dec = Decimal::from_str("1.00")?; - let result = serialize_decimal(&dec, true)?; + let result = serialize_decimal(&dec)?; let (scale, unscaled) = parse_binary_decimal(&result); // rust_decimal normalizes trailing zeros @@ -337,34 +303,10 @@ mod bigdecimal_tests { use bigdecimal::BigDecimal; use std::str::FromStr; - #[test] - fn test_bigdecimal_text_format() -> TestResult { - let dec = BigDecimal::from_str("123.45")?; - let result = serialize_decimal(&dec, false)?; - assert_eq!(result, b"123.45d"); - Ok(()) - } - - #[test] - fn test_bigdecimal_negative_text_format() -> TestResult { - let dec = BigDecimal::from_str("-123.45")?; - let result = serialize_decimal(&dec, false)?; - assert_eq!(result, b"-123.45d"); - Ok(()) - } - - #[test] - fn test_bigdecimal_zero_text_format() -> TestResult { - let dec = BigDecimal::from_str("0")?; - let result = serialize_decimal(&dec, false)?; - assert_eq!(result, b"0d"); - Ok(()) - } - #[test] fn test_bigdecimal_binary_format_zero() -> TestResult { let dec = BigDecimal::from_str("0")?; - let result = serialize_decimal(&dec, true)?; + let result = serialize_decimal(&dec)?; let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 0, "Zero should have scale 0"); @@ -375,7 +317,7 @@ mod bigdecimal_tests { #[test] fn test_bigdecimal_binary_format_positive() -> TestResult { let dec = BigDecimal::from_str("123.45")?; - let result = serialize_decimal(&dec, true)?; + let result = serialize_decimal(&dec)?; let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 2, "123.45 should have scale 2"); @@ -386,7 +328,7 @@ mod bigdecimal_tests { #[test] fn test_bigdecimal_binary_format_negative() -> TestResult { let dec = BigDecimal::from_str("-123.45")?; - let result = serialize_decimal(&dec, true)?; + let result = serialize_decimal(&dec)?; let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 2, "-123.45 should have scale 2"); @@ -400,7 +342,7 @@ mod bigdecimal_tests { #[test] fn test_bigdecimal_binary_format_one() -> TestResult { let dec = BigDecimal::from_str("1")?; - let result = serialize_decimal(&dec, true)?; + let result = serialize_decimal(&dec)?; let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 0, "One should have scale 0"); @@ -412,7 +354,7 @@ mod bigdecimal_tests { fn test_bigdecimal_binary_format_high_precision() -> TestResult { // BigDecimal can handle arbitrary precision, test a value with many decimal places let dec = BigDecimal::from_str("0.123456789012345678901234567890")?; - let result = serialize_decimal(&dec, true)?; + let result = serialize_decimal(&dec)?; let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 30, "Should preserve high precision scale"); @@ -427,7 +369,7 @@ mod bigdecimal_tests { fn test_bigdecimal_binary_format_large_value() -> TestResult { // Test a very large value that BigDecimal can represent let dec = BigDecimal::from_str("79228162514264337593543950335")?; - let result = serialize_decimal(&dec, true)?; + let result = serialize_decimal(&dec)?; let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 0, "Large integer should have scale 0"); @@ -441,7 +383,7 @@ mod bigdecimal_tests { #[test] fn test_bigdecimal_binary_format_large_negative() -> TestResult { let dec = BigDecimal::from_str("-79228162514264337593543950335")?; - let result = serialize_decimal(&dec, true)?; + let result = serialize_decimal(&dec)?; let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 0, "Large negative integer should have scale 0"); @@ -455,7 +397,7 @@ mod bigdecimal_tests { #[test] fn test_bigdecimal_binary_format_small_negative() -> TestResult { let dec = BigDecimal::from_str("-0.01")?; - let result = serialize_decimal(&dec, true)?; + let result = serialize_decimal(&dec)?; let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 2, "-0.01 should have scale 2"); @@ -466,7 +408,7 @@ mod bigdecimal_tests { #[test] fn test_bigdecimal_binary_format_trailing_zeros() -> TestResult { let dec = BigDecimal::from_str("1.00")?; - let result = serialize_decimal(&dec, true)?; + let result = serialize_decimal(&dec)?; let (scale, unscaled) = parse_binary_decimal(&result); // BigDecimal may normalize trailing zeros differently than rust_decimal @@ -481,7 +423,7 @@ mod bigdecimal_tests { let dec = BigDecimal::from_str( "0.0000000000000000000000000000000000000000000000000000000000000000000000000001", )?; - let result = serialize_decimal(&dec, true)?; + let result = serialize_decimal(&dec)?; let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 76, "Should have maximum scale of 76"); @@ -495,7 +437,7 @@ mod bigdecimal_tests { let dec = BigDecimal::from_str( "0.00000000000000000000000000000000000000000000000000000000000000000000000000001", )?; - let result = serialize_decimal(&dec, true); + let result = serialize_decimal(&dec); assert!(result.is_err()); let err = result.unwrap_err(); @@ -508,7 +450,7 @@ mod bigdecimal_tests { fn test_bigdecimal_binary_negative_scale() -> TestResult { // Test with a negative scale let dec = BigDecimal::from_str("1.23e12")?; - let result = serialize_decimal(&dec, true)?; + let result = serialize_decimal(&dec)?; let (scale, unscaled) = parse_binary_decimal(&result); // QuestDB does not support negative scale, instead the value should be @@ -525,7 +467,7 @@ mod bigdecimal_tests { fn test_bigdecimal_binary_value_too_large() -> TestResult { // QuestDB cannot accept arrays that are larger than what an i8 can fit let dec = BigDecimal::from_str("1e1000")?; - let result = serialize_decimal(&dec, true); + let result = serialize_decimal(&dec); assert!(result.is_err()); let err = result.unwrap_err(); @@ -539,13 +481,15 @@ mod bigdecimal_tests { // Buffer integration tests // ============================================================================ -#[test] -fn test_buffer_column_decimal_str_v1() -> TestResult { - let mut buffer = Buffer::new(ProtocolVersion::V1); +#[rstest] +fn test_buffer_column_decimal_str( + #[values(ProtocolVersion::V3)] version: ProtocolVersion, +) -> TestResult { + let mut buffer = Buffer::new(version); buffer .table("test")? .symbol("sym", "val")? - .column_decimal("dec", "123.45")? + .column_dec("dec", "123.45")? .at_now()?; let output = std::str::from_utf8(buffer.as_bytes())?; @@ -553,52 +497,34 @@ fn test_buffer_column_decimal_str_v1() -> TestResult { Ok(()) } -#[test] -fn test_buffer_column_decimal_str_v2() -> TestResult { - let mut buffer = Buffer::new(ProtocolVersion::V2); - buffer +#[rstest] +fn test_buffer_column_decimal_str_unsupported( + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, +) -> TestResult { + let mut buffer = Buffer::new(version); + let result = buffer .table("test")? .symbol("sym", "val")? - .column_decimal("dec", "123.45")? - .at_now()?; - - // &str always uses text format, even in V2 - let output = std::str::from_utf8(buffer.as_bytes())?; - assert!(output.starts_with("test,sym=val dec=123.45d")); - Ok(()) -} - -#[cfg(feature = "rust_decimal")] -#[test] -fn test_buffer_column_decimal_rust_decimal_v1() -> TestResult { - use rust_decimal::Decimal; - use std::str::FromStr; - - let mut buffer = Buffer::new(ProtocolVersion::V1); - let dec = Decimal::from_str("123.45")?; - buffer - .table("test")? - .symbol("sym", "val")? - .column_decimal("dec", &dec)? - .at_now()?; - - let output = std::str::from_utf8(buffer.as_bytes())?; - assert!(output.starts_with("test,sym=val dec=123.45d")); + .column_dec("dec", "123.45"); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ProtocolVersionError); + assert!(err.msg().contains("does not support the decimal datatype")); Ok(()) } #[cfg(feature = "rust_decimal")] #[test] -fn test_buffer_column_decimal_rust_decimal_v2() -> TestResult { +fn test_buffer_column_decimal_rust_decimal() -> TestResult { use rust_decimal::Decimal; use std::str::FromStr; - let mut buffer = Buffer::new(ProtocolVersion::V2); + let mut buffer = Buffer::new(ProtocolVersion::V3); let dec = Decimal::from_str("123.45")?; buffer .table("test")? .symbol("sym", "val")? - .column_decimal("dec", &dec)? + .column_dec("dec", &dec)? .at_now()?; let bytes = buffer.as_bytes(); @@ -616,12 +542,12 @@ fn test_buffer_column_decimal_rust_decimal_v2() -> TestResult { #[test] fn test_buffer_multiple_decimals() -> TestResult { - let mut buffer = Buffer::new(ProtocolVersion::V1); + let mut buffer = Buffer::new(ProtocolVersion::V3); buffer .table("test")? - .column_decimal("dec1", "123.45")? - .column_decimal("dec2", "-67.89")? - .column_decimal("dec3", "0.001")? + .column_dec("dec1", "123.45")? + .column_dec("dec2", "-67.89")? + .column_dec("dec3", "0.001")? .at_now()?; let output = std::str::from_utf8(buffer.as_bytes())?; @@ -633,12 +559,9 @@ fn test_buffer_multiple_decimals() -> TestResult { #[test] fn test_decimal_column_name_too_long() -> TestResult { - let mut buffer = Buffer::with_max_name_len(ProtocolVersion::V2, 4); + let mut buffer = Buffer::with_max_name_len(ProtocolVersion::V3, 4); let name = "a name too long"; - let err = buffer - .table("tbl")? - .column_decimal(name, "123.45") - .unwrap_err(); + let err = buffer.table("tbl")?.column_dec(name, "123.45").unwrap_err(); assert_eq!(err.code(), ErrorCode::InvalidName); assert_eq!( err.msg(), @@ -649,35 +572,16 @@ fn test_decimal_column_name_too_long() -> TestResult { #[cfg(feature = "bigdecimal")] #[test] -fn test_buffer_column_decimal_bigdecimal_v1() -> TestResult { - use bigdecimal::BigDecimal; - use std::str::FromStr; - - let mut buffer = Buffer::new(ProtocolVersion::V1); - let dec = BigDecimal::from_str("123.45")?; - buffer - .table("test")? - .symbol("sym", "val")? - .column_decimal("dec", &dec)? - .at_now()?; - - let output = std::str::from_utf8(buffer.as_bytes())?; - assert!(output.starts_with("test,sym=val dec=123.45d")); - Ok(()) -} - -#[cfg(feature = "bigdecimal")] -#[test] -fn test_buffer_column_decimal_bigdecimal_v2() -> TestResult { +fn test_buffer_column_decimal_bigdecimal() -> TestResult { use bigdecimal::BigDecimal; use std::str::FromStr; - let mut buffer = Buffer::new(ProtocolVersion::V2); + let mut buffer = Buffer::new(ProtocolVersion::V3); let dec = BigDecimal::from_str("123.45")?; buffer .table("test")? .symbol("sym", "val")? - .column_decimal("dec", &dec)? + .column_dec("dec", &dec)? .at_now()?; let bytes = buffer.as_bytes(); diff --git a/questdb-rs/src/tests/http.rs b/questdb-rs/src/tests/http.rs index d66242db..9ddcf3f6 100644 --- a/questdb-rs/src/tests/http.rs +++ b/questdb-rs/src/tests/http.rs @@ -777,7 +777,7 @@ fn test_sender_auto_protocol_version_only_v2() -> TestResult { #[test] fn test_sender_auto_protocol_version_unsupported_client() -> TestResult { - let mut server = MockServer::new()?.configure_settings_response(&[3, 4], 127); + let mut server = MockServer::new()?.configure_settings_response(&[4, 5], 127); let sender_builder = server.lsb_http(); let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; diff --git a/questdb-rs/src/tests/interop/ilp-client-interop-test.json b/questdb-rs/src/tests/interop/ilp-client-interop-test.json index 6a76d007..2b3bdd62 100644 --- a/questdb-rs/src/tests/interop/ilp-client-interop-test.json +++ b/questdb-rs/src/tests/interop/ilp-client-interop-test.json @@ -1638,6 +1638,7 @@ }, { "testName": "decimal", + "minimumProtocolVersion": 3, "table": "decimals", "symbols": [], "columns": [ diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index 0a62e582..241f7a8e 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -234,7 +234,8 @@ fn test_array_f64_for_ndarray() -> TestResult { #[cfg(feature = "sync-sender-tcp")] #[rstest] fn test_max_buf_size( - #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, + #[values(ProtocolVersion::V1, ProtocolVersion::V2, ProtocolVersion::V3)] + version: ProtocolVersion, ) -> TestResult { let max = 1024; let mut server = MockServer::new()?; @@ -264,7 +265,7 @@ fn test_max_buf_size( "Could not flush buffer: Buffer size of 1026 exceeds maximum configured allowed size of 1024 bytes." ); } - ProtocolVersion::V2 => { + ProtocolVersion::V2 | ProtocolVersion::V3 => { assert_eq!( err.msg(), "Could not flush buffer: Buffer size of 1025 exceeds maximum configured allowed size of 1024 bytes." @@ -612,7 +613,8 @@ fn test_arr_column_name_too_long() -> TestResult { #[cfg(feature = "sync-sender-tcp")] #[rstest] fn test_tls_with_file_ca( - #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, + #[values(ProtocolVersion::V1, ProtocolVersion::V2, ProtocolVersion::V3)] + version: ProtocolVersion, ) -> TestResult { let mut ca_path = certs_dir(); ca_path.push("server_rootCA.pem"); @@ -721,7 +723,8 @@ fn test_plain_to_tls_server() -> TestResult { #[cfg(feature = "insecure-skip-verify")] #[rstest] fn test_tls_insecure_skip_verify( - #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, + #[values(ProtocolVersion::V1, ProtocolVersion::V2, ProtocolVersion::V3)] + version: ProtocolVersion, ) -> TestResult { let server = MockServer::new()?; let lsb = server @@ -802,7 +805,7 @@ pub(crate) fn f64_to_bytes(name: &str, value: f64, version: ProtocolVersion) -> let mut ser = F64Serializer::new(value); buf.extend_from_slice(ser.as_str().as_bytes()); } - ProtocolVersion::V2 => { + ProtocolVersion::V2 | ProtocolVersion::V3 => { buf.push(b'='); buf.push(DOUBLE_BINARY_FORMAT_TYPE); buf.extend_from_slice(&value.to_le_bytes()); diff --git a/system_test/questdb_line_sender.py b/system_test/questdb_line_sender.py index ebf69399..5202ea32 100644 --- a/system_test/questdb_line_sender.py +++ b/system_test/questdb_line_sender.py @@ -102,6 +102,7 @@ class CertificateAuthority(Enum): class ProtocolVersion(Enum): V1 = (c_protocol_version(1), '1') V2 = (c_protocol_version(2), '2') + V3 = (c_protocol_version(3), '3') @classmethod def from_int(cls, value: c_protocol_version): @@ -292,7 +293,7 @@ def set_sig(fn, restype, *argtypes): c_line_sender_utf8, c_line_sender_error_p_p) set_sig( - dll.line_sender_buffer_column_decimal_str, + dll.line_sender_buffer_column_dec_str, c_bool, c_line_sender_buffer_p, c_line_sender_column_name, @@ -713,7 +714,7 @@ def column( _utf8(value)) elif isinstance(value, Decimal): _error_wrapped_call( - _DLL.line_sender_buffer_column_decimal_str, + _DLL.line_sender_buffer_column_dec_str, self._impl, _column_name(name), _utf8(str(value))) diff --git a/system_test/test.py b/system_test/test.py index f6bd20e1..59b29bfa 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -57,6 +57,7 @@ # The first QuestDB version that supports array types. FIRST_ARRAYS_RELEASE = (8, 3, 3) +DECIMAL_RELEASE = (9, 2, 0) def retry_check_table(*args, **kwargs): @@ -120,9 +121,13 @@ def expected_protocol_version(self) -> qls.ProtocolVersion: if not QDB_FIXTURE.http: return qls.ProtocolVersion.V1 + print('Here got =========================') if QDB_FIXTURE.version >= FIRST_ARRAYS_RELEASE: return qls.ProtocolVersion.V2 + if QDB_FIXTURE.version >= DECIMAL_RELEASE: + return qls.ProtocolVersion.V3 + return qls.ProtocolVersion.V1 return QDB_FIXTURE.protocol_version @@ -518,6 +523,9 @@ def test_timestamp_column(self): self.assertEqual(scrubbed_dataset, exp_dataset) def test_decimal_column(self): + if self.expected_protocol_version < qls.ProtocolVersion.V3: + self.skipTest('communicating over old protocol which does not support decimals') + table_name = uuid.uuid4().hex pending = None decimals = [ @@ -1172,7 +1180,7 @@ def run_with_existing(args): (999, 999, 999), True, False, - qls.ProtocolVersion.V2 + qls.ProtocolVersion.V3 ) unittest.main() From 4c4b9afdd862aadb0feafebb44f51fd7450158f3 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Tue, 14 Oct 2025 17:44:56 +0200 Subject: [PATCH 19/65] revert: revert removal of implicit coercion of decimal binary views --- ...line_sender_cpp_example_decimal_custom.cpp | 41 ++++++++---- include/questdb/ingress/line_sender.hpp | 63 +++++++++++++++++++ 2 files changed, 91 insertions(+), 13 deletions(-) diff --git a/examples/line_sender_cpp_example_decimal_custom.cpp b/examples/line_sender_cpp_example_decimal_custom.cpp index c3f76a4a..e727c640 100644 --- a/examples/line_sender_cpp_example_decimal_custom.cpp +++ b/examples/line_sender_cpp_example_decimal_custom.cpp @@ -5,6 +5,16 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; using namespace questdb::ingress::decimal; +struct ViewHolder +{ + std::array data; + uint32_t scale; + questdb::ingress::decimal::binary_view view() const + { + return {scale, data}; + } +}; + namespace custom_decimal { class Decimal32 @@ -26,22 +36,27 @@ class Decimal32 return _scale; } - questdb::ingress::decimal::binary_view view() const - { - std::array data = { - // Big-Endian bytes - static_cast(_unscaled_value >> 24), - static_cast(_unscaled_value >> 16), - static_cast(_unscaled_value >> 8), - static_cast(_unscaled_value >> 0), - }; - return {_scale, data}; - } - private: uint32_t _scale; int32_t _unscaled_value; }; + +// Customization point for QuestDB decimal API (discovered via König lookup) +// If you need to support a 3rd party type, put this function in the namespace +// of the type in question or in the `questdb::ingress::decimal` namespace +inline auto to_decimal_view_state_impl(const Decimal32& d) +{ + int32_t unscaled_value = d.unscaled_value(); + return ViewHolder{ + { + // Big-Endiang bytes + static_cast(unscaled_value >> 24), + static_cast(unscaled_value >> 16), + static_cast(unscaled_value >> 8), + static_cast(unscaled_value >> 0), + }, + d.scale()}; +} } static bool example(std::string_view host, std::string_view port) @@ -68,7 +83,7 @@ static bool example(std::string_view host, std::string_view port) buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) - .column(price_name, price_value.view()) + .column(price_name, price_value) .column(amount_name, 0.00044) .at(questdb::ingress::timestamp_nanos::now()); diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 2e803ff0..d133e9c6 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -841,6 +841,34 @@ class binary_view const uint8_t* _data; size_t _data_size; }; +/** + * Customization point to enable serialization of additional types as decimals. + * + * This allows you to support custom decimal types by implementing a conversion + * function. The customized `to_decimal_view_state_impl` for your type can be + * placed in either: + * - The namespace of the type in question (ADL/Koenig lookup) + * - The `questdb::ingress::decimal` namespace + * + * The function can either: + * - Return a `binary_view` object directly, or + * - Return an object with a `.view()` method that returns `const binary_view&` + * (useful if you need to store temporary data like shape/strides on the +stack) + */ +struct to_decimal_view_state_fn +{ + template + auto operator()(const T& decimal) const + { + // Implement your own `to_decimal_view_state_impl` as needed. + // ADL lookup for user-defined to_decimal_view_state_impl + return to_decimal_view_state_impl(decimal); + } +}; + +inline constexpr to_decimal_view_state_fn to_decimal_view_state{}; + } // namespace decimal class line_sender_buffer @@ -1297,6 +1325,41 @@ class line_sender_buffer return *this; } + /** + * Record a decimal value using a custom type via a customization point. + * + * This overload allows you to serialize custom decimal types by + * implementing a `to_decimal_view_state_impl` function for your type. + * + * QuestDB server version 9.2.0 or later is required for decimal support. + * + * # Customization + * + * To support your custom decimal type, implement + * `to_decimal_view_state_impl` in either: + * - The namespace of your type (ADL/Koenig lookup) + * - The `questdb::ingress::decimal` namespace + * + * The function should return either: + * - A `decimal::binary_view` directly, or + * - An object with a `.view()` method returning `const + * decimal::binary_view&` + * + * Include your customization point before including `line_sender.hpp`. + * + * @tparam ToDecimalViewT Type convertible to decimal::binary_view. + * @param name Column name. + * @param decimal Custom decimal value. + */ + template + line_sender_buffer& column(column_name_view name, ToDecimalViewT decimal) + { + may_init(); + const auto decimal_view_state = + questdb::ingress::decimal::to_decimal_view_state(decimal); + return column(name, decimal_view_state.view()); + } + /** Record a nanosecond timestamp value for the given column. */ template line_sender_buffer& column( From 4a6a07cdb3c4cfe7e2bf2d6384551c8d9fe58dd8 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Tue, 14 Oct 2025 18:00:05 +0200 Subject: [PATCH 20/65] feat: generalize array customization point --- .../line_sender_cpp_example_array_custom.cpp | 2 +- ...line_sender_cpp_example_decimal_custom.cpp | 2 +- include/questdb/ingress/line_sender.hpp | 86 ++++++------------- 3 files changed, 27 insertions(+), 63 deletions(-) diff --git a/examples/line_sender_cpp_example_array_custom.cpp b/examples/line_sender_cpp_example_array_custom.cpp index 5009eabc..2be6d31d 100644 --- a/examples/line_sender_cpp_example_array_custom.cpp +++ b/examples/line_sender_cpp_example_array_custom.cpp @@ -96,7 +96,7 @@ class Matrix // Customization point for QuestDB array API (discovered via König lookup) // If you need to support a 3rd party type, put this function in the namespace // of the type in question or in the `questdb::ingress::array` namespace -inline auto to_array_view_state_impl(const Matrix& m) +inline auto to_view_state_impl(const Matrix& m) { return ViewHolder{ {static_cast(m.rows()), static_cast(m.cols())}, diff --git a/examples/line_sender_cpp_example_decimal_custom.cpp b/examples/line_sender_cpp_example_decimal_custom.cpp index e727c640..25451d38 100644 --- a/examples/line_sender_cpp_example_decimal_custom.cpp +++ b/examples/line_sender_cpp_example_decimal_custom.cpp @@ -44,7 +44,7 @@ class Decimal32 // Customization point for QuestDB decimal API (discovered via König lookup) // If you need to support a 3rd party type, put this function in the namespace // of the type in question or in the `questdb::ingress::decimal` namespace -inline auto to_decimal_view_state_impl(const Decimal32& d) +inline auto to_view_state_impl(const Decimal32& d) { int32_t unscaled_value = d.unscaled_value(); return ViewHolder{ diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index d133e9c6..1b36dee7 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -585,57 +585,57 @@ struct row_major_1d_holder return {1, shape, data, size}; } }; +} // namespace array template -inline auto to_array_view_state_impl(const std::vector& vec) +inline auto to_view_state_impl(const std::vector& vec) { - return row_major_1d_holder::type>( + return array::row_major_1d_holder::type>( vec.data(), vec.size()); } #if __cplusplus >= 202002L template -inline auto to_array_view_state_impl(const std::span& span) +inline auto to_view_state_impl(const std::span& span) { - return row_major_1d_holder::type>( + return array::row_major_1d_holder::type>( span.data(), span.size()); } #endif template -inline auto to_array_view_state_impl(const std::array& arr) +inline auto to_view_state_impl(const std::array& arr) { - return row_major_1d_holder::type>(arr.data(), N); + return array::row_major_1d_holder::type>( + arr.data(), N); } /** - * Customization point to enable serialization of additional types as arrays. + * Customization point to enable serialization of additional types. * * Forwards to a namespace or ADL (König) lookup function. - * The customized `to_array_view_state_impl` for your custom type can be placed + * The customized `to_view_state_impl` for your custom type can be placed * in either: * * The namespace of the type in question. - * * In the `questdb::ingress::array` namespace. + * * In the `questdb::ingress` namespace. /// * The function can either return a view object directly (either - * `row_major_view` or `strided_view`), or, if you need to place some fields on - * the stack, an object with a `.view()` method which returns a `const&` to one - * "materialize" shape or strides information into contiguous memory. - * of the two view types. Returning an object may be useful if you need to + * `array::row_major_view`, `array::strided_view` or `decimal::binary_view`), + * or, if you need to place some fields on the stack, an object with a `.view()` + * method which returns a `const&` to one "materialize" shape or strides + * information into contiguous memory. */ -struct to_array_view_state_fn +struct to_view_state_fn { template auto operator()(const T& array) const { - // Implement your own `to_array_view_state_impl` as needed. - return to_array_view_state_impl(array); + // Implement your own `to_view_state_impl` as needed. + return to_view_state_impl(array); } }; -inline constexpr to_array_view_state_fn to_array_view_state{}; - -} // namespace array +inline constexpr to_view_state_fn to_view_state{}; /** * Types and utilities for working with arbitrary-precision decimal numbers. @@ -1221,23 +1221,22 @@ class line_sender_buffer * * This overload uses a customization point to support additional types: * If you need to support your additional types you may implement a - * `to_array_view_state_impl` function in the object's namespace (via ADL) + * `to_view_state_impl` function in the object's namespace (via ADL) * or in the `questdb::ingress::array` namespace. * Ensure that any additional customization points are included before * `line_sender.hpp`. * - * @tparam ToArrayViewT Type convertible to a custom object instance which + * @tparam ToViewT Type convertible to a custom object instance which * can be converted to an array view. * @param name Column name. * @param array Multi-dimensional array. */ - template - line_sender_buffer& column(column_name_view name, ToArrayViewT array) + template + line_sender_buffer& column(column_name_view name, ToViewT array) { may_init(); - const auto array_view_state = - questdb::ingress::array::to_array_view_state(array); - return column(name, array_view_state.view()); + const auto view_state = questdb::ingress::to_view_state(array); + return column(name, view_state.view()); } /** @@ -1325,41 +1324,6 @@ class line_sender_buffer return *this; } - /** - * Record a decimal value using a custom type via a customization point. - * - * This overload allows you to serialize custom decimal types by - * implementing a `to_decimal_view_state_impl` function for your type. - * - * QuestDB server version 9.2.0 or later is required for decimal support. - * - * # Customization - * - * To support your custom decimal type, implement - * `to_decimal_view_state_impl` in either: - * - The namespace of your type (ADL/Koenig lookup) - * - The `questdb::ingress::decimal` namespace - * - * The function should return either: - * - A `decimal::binary_view` directly, or - * - An object with a `.view()` method returning `const - * decimal::binary_view&` - * - * Include your customization point before including `line_sender.hpp`. - * - * @tparam ToDecimalViewT Type convertible to decimal::binary_view. - * @param name Column name. - * @param decimal Custom decimal value. - */ - template - line_sender_buffer& column(column_name_view name, ToDecimalViewT decimal) - { - may_init(); - const auto decimal_view_state = - questdb::ingress::decimal::to_decimal_view_state(decimal); - return column(name, decimal_view_state.view()); - } - /** Record a nanosecond timestamp value for the given column. */ template line_sender_buffer& column( From cf425b5282c38fa39887d5729a1acaaa19d456eb Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Tue, 14 Oct 2025 18:00:31 +0200 Subject: [PATCH 21/65] typo: remove garbage --- system_test/test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/system_test/test.py b/system_test/test.py index 59b29bfa..fc4090a7 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -121,7 +121,6 @@ def expected_protocol_version(self) -> qls.ProtocolVersion: if not QDB_FIXTURE.http: return qls.ProtocolVersion.V1 - print('Here got =========================') if QDB_FIXTURE.version >= FIRST_ARRAYS_RELEASE: return qls.ProtocolVersion.V2 From 2180a9f2d1e77041328e0b750725b45947d99bef Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Tue, 14 Oct 2025 18:13:35 +0200 Subject: [PATCH 22/65] docs: remove no longer true comment --- questdb-rs/src/ingress/decimal.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/questdb-rs/src/ingress/decimal.rs b/questdb-rs/src/ingress/decimal.rs index b6c1e495..de506292 100644 --- a/questdb-rs/src/ingress/decimal.rs +++ b/questdb-rs/src/ingress/decimal.rs @@ -61,7 +61,6 @@ use crate::{error, ingress::must_escape_unquoted, Result}; /// ``` /// /// # Binary Format Notes -/// - Binary format is only supported when `support_binary` is `true` (Protocol V2) /// - The unscaled value must be encoded in two's complement big-endian format /// - Maximum scale is 76 /// - Length byte indicates how many bytes follow for the unscaled value From 14b54eb5db1a54d3474b5f89d5de09bbf07c1445 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Wed, 15 Oct 2025 10:01:43 +0200 Subject: [PATCH 23/65] feat: improved decimal string validation --- questdb-rs/src/ingress/buffer.rs | 2 +- questdb-rs/src/ingress/decimal.rs | 30 ++++++++++--------- questdb-rs/src/tests/decimal.rs | 48 +++++++++++-------------------- 3 files changed, 34 insertions(+), 46 deletions(-) diff --git a/questdb-rs/src/ingress/buffer.rs b/questdb-rs/src/ingress/buffer.rs index 16cd542f..2a4c757a 100644 --- a/questdb-rs/src/ingress/buffer.rs +++ b/questdb-rs/src/ingress/buffer.rs @@ -72,7 +72,7 @@ where quoting_fn(output); } -pub(crate) fn must_escape_unquoted(c: u8) -> bool { +fn must_escape_unquoted(c: u8) -> bool { matches!(c, b' ' | b',' | b'=' | b'\n' | b'\r' | b'\\') } diff --git a/questdb-rs/src/ingress/decimal.rs b/questdb-rs/src/ingress/decimal.rs index de506292..872160c1 100644 --- a/questdb-rs/src/ingress/decimal.rs +++ b/questdb-rs/src/ingress/decimal.rs @@ -22,7 +22,7 @@ * ******************************************************************************/ -use crate::{error, ingress::must_escape_unquoted, Result}; +use crate::{error, Result}; /// Trait for types that can be serialized as decimal values in the InfluxDB Line Protocol (ILP). /// @@ -79,12 +79,13 @@ pub trait DecimalSerializer { /// as it cannot parse the string to extract scale and unscaled value needed for binary encoding. /// /// # Format -/// The string is validated and written as-is, followed by the 'd' suffix. +/// The string is validated and written as-is, followed by the 'd' suffix. Thousand separators +/// (commas) are not allowed and the decimal point must be a dot (`.`). /// /// # Validation /// The implementation performs **partial validation only**: -/// - Rejects ILP reserved characters (space, comma, equals, newline, carriage return, backslash) -/// - Does NOT validate the actual decimal syntax (e.g., "not-a-number" would pass) +/// - Rejects non-numerical characters (not -/+, 0-9, e/E, .) +/// - Does NOT validate the actual decimal syntax (e.g., "e2e" would pass) /// /// This is intentional: full parsing would add overhead. The QuestDB server performs complete /// validation and will reject malformed decimals. @@ -96,23 +97,24 @@ pub trait DecimalSerializer { /// /// # Errors /// Returns [`Error`] with [`ErrorCode::InvalidDecimal`](crate::error::ErrorCode::InvalidDecimal) -/// if the string contains ILP reserved characters. +/// if the string contains non-numerical characters. impl DecimalSerializer for &str { fn serialize(self, out: &mut Vec) -> Result<()> { // Pre-allocate space for the string content plus the 'd' suffix out.reserve(self.len() + 1); - // Validate and copy each byte, rejecting ILP reserved characters - // that would break the protocol (space, comma, equals, newline, etc.) + // Validate and copy each byte, rejecting non-numeric characters for b in self.bytes() { - if must_escape_unquoted(b) { - return Err(error::fmt!( - InvalidDecimal, - "Unexpected character {:?} in decimal str", - b - )); + match b { + b'0'..=b'9' | b'.' | b'-' | b'+' | b'e' | b'E' => out.push(b), + _ => { + return Err(error::fmt!( + InvalidDecimal, + "Invalid character {:?} in decimal str", + b as char + )); + } } - out.push(b); } // Append the 'd' suffix to mark this as a decimal value diff --git a/questdb-rs/src/tests/decimal.rs b/questdb-rs/src/tests/decimal.rs index f8a1b2a9..6abb4bf9 100644 --- a/questdb-rs/src/tests/decimal.rs +++ b/questdb-rs/src/tests/decimal.rs @@ -23,7 +23,7 @@ ******************************************************************************/ use crate::ingress::{Buffer, DecimalSerializer, ProtocolVersion}; -use crate::tests::TestResult; +use crate::tests::{assert_err_contains, TestResult}; use crate::ErrorCode; use rstest::rstest; @@ -83,46 +83,35 @@ fn test_str_with_leading_zero() -> TestResult { #[test] fn test_str_rejects_space() -> TestResult { let result = serialize_decimal("12 3.45"); - assert!(result.is_err()); - let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::InvalidDecimal); - assert!(err.msg().contains("Unexpected character")); + assert_err_contains(result, ErrorCode::InvalidDecimal, "Invalid character"); Ok(()) } #[test] fn test_str_rejects_comma() -> TestResult { let result = serialize_decimal("1,234.56"); - assert!(result.is_err()); - let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::InvalidDecimal); + assert_err_contains(result, ErrorCode::InvalidDecimal, "Invalid character"); Ok(()) } #[test] fn test_str_rejects_equals() -> TestResult { let result = serialize_decimal("123=45"); - assert!(result.is_err()); - let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::InvalidDecimal); + assert_err_contains(result, ErrorCode::InvalidDecimal, "Invalid character"); Ok(()) } #[test] fn test_str_rejects_newline() -> TestResult { let result = serialize_decimal("123\n45"); - assert!(result.is_err()); - let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::InvalidDecimal); + assert_err_contains(result, ErrorCode::InvalidDecimal, "Invalid character"); Ok(()) } #[test] fn test_str_rejects_backslash() -> TestResult { let result = serialize_decimal("123\\45"); - assert!(result.is_err()); - let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::InvalidDecimal); + assert_err_contains(result, ErrorCode::InvalidDecimal, "Invalid character"); Ok(()) } @@ -438,11 +427,7 @@ mod bigdecimal_tests { "0.00000000000000000000000000000000000000000000000000000000000000000000000000001", )?; let result = serialize_decimal(&dec); - - assert!(result.is_err()); - let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::InvalidDecimal); - assert!(err.msg().contains("scale greater than 76")); + assert_err_contains(result, ErrorCode::InvalidDecimal, "scale greater than 76"); Ok(()) } @@ -468,11 +453,11 @@ mod bigdecimal_tests { // QuestDB cannot accept arrays that are larger than what an i8 can fit let dec = BigDecimal::from_str("1e1000")?; let result = serialize_decimal(&dec); - - assert!(result.is_err()); - let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::InvalidDecimal); - assert!(err.msg().contains("does not support values greater")); + assert_err_contains( + result, + ErrorCode::InvalidDecimal, + "does not support values greater", + ); Ok(()) } } @@ -506,10 +491,11 @@ fn test_buffer_column_decimal_str_unsupported( .table("test")? .symbol("sym", "val")? .column_dec("dec", "123.45"); - assert!(result.is_err()); - let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ProtocolVersionError); - assert!(err.msg().contains("does not support the decimal datatype")); + assert_err_contains( + result, + ErrorCode::ProtocolVersionError, + "does not support the decimal datatype", + ); Ok(()) } From fe7cceb4c88aa092b98c90faa9dbd7e17ac6dd85 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Wed, 15 Oct 2025 10:30:30 +0200 Subject: [PATCH 24/65] feat: add customization point for decimal --- .../line_sender_cpp_example_array_custom.cpp | 2 +- ...line_sender_cpp_example_decimal_custom.cpp | 4 +- include/questdb/ingress/line_sender.hpp | 288 +++++------------- include/questdb/ingress/line_sender_array.hpp | 269 ++++++++++++++++ 4 files changed, 340 insertions(+), 223 deletions(-) create mode 100644 include/questdb/ingress/line_sender_array.hpp diff --git a/examples/line_sender_cpp_example_array_custom.cpp b/examples/line_sender_cpp_example_array_custom.cpp index 2be6d31d..5009eabc 100644 --- a/examples/line_sender_cpp_example_array_custom.cpp +++ b/examples/line_sender_cpp_example_array_custom.cpp @@ -96,7 +96,7 @@ class Matrix // Customization point for QuestDB array API (discovered via König lookup) // If you need to support a 3rd party type, put this function in the namespace // of the type in question or in the `questdb::ingress::array` namespace -inline auto to_view_state_impl(const Matrix& m) +inline auto to_array_view_state_impl(const Matrix& m) { return ViewHolder{ {static_cast(m.rows()), static_cast(m.cols())}, diff --git a/examples/line_sender_cpp_example_decimal_custom.cpp b/examples/line_sender_cpp_example_decimal_custom.cpp index 25451d38..cec604cd 100644 --- a/examples/line_sender_cpp_example_decimal_custom.cpp +++ b/examples/line_sender_cpp_example_decimal_custom.cpp @@ -44,12 +44,12 @@ class Decimal32 // Customization point for QuestDB decimal API (discovered via König lookup) // If you need to support a 3rd party type, put this function in the namespace // of the type in question or in the `questdb::ingress::decimal` namespace -inline auto to_view_state_impl(const Decimal32& d) +inline auto to_decimal_view_state_impl(const Decimal32& d) { int32_t unscaled_value = d.unscaled_value(); return ViewHolder{ { - // Big-Endiang bytes + // Big-Endian bytes static_cast(unscaled_value >> 24), static_cast(unscaled_value >> 16), static_cast(unscaled_value >> 8), diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 1b36dee7..a2ba6a45 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -26,6 +26,8 @@ #include "line_sender.h" +#include "line_sender_array.hpp" + #include #include #include @@ -423,220 +425,6 @@ class buffer_view final }; #endif -namespace array -{ -enum class strides_mode -{ - /** Strides are provided in bytes */ - bytes, - - /** Strides are provided in elements */ - elements, -}; - -/** - * A view over a multi-dimensional array with custom strides. - * - * The strides can be expressed as bytes offsets or as element counts. - * The `rank` is the number of dimensions in the array, and the `shape` - * describes the size of each dimension. - * - * If the data is stored in a row-major order, it may be more convenient and - * efficient to use the `row_major_view` instead of `strided_view`. - * - * The `data` pointer must point to a contiguous block of memory that contains - * the array data. - */ -template -class strided_view -{ -public: - using element_type = T; - static constexpr strides_mode stride_size_mode = M; - - strided_view( - size_t rank, - const uintptr_t* shape, - const intptr_t* strides, - const T* data, - size_t data_size) - : _rank{rank} - , _shape{shape} - , _strides{strides} - , _data{data} - , _data_size{data_size} - { - } - - size_t rank() const - { - return _rank; - } - - const uintptr_t* shape() const - { - return _shape; - } - - const intptr_t* strides() const - { - return _strides; - } - - const T* data() const - { - return _data; - } - - size_t data_size() const - { - return _data_size; - } - - const strided_view& view() const - { - return *this; - } - -private: - size_t _rank; - const uintptr_t* _shape; - const intptr_t* _strides; - const T* _data; - size_t _data_size; -}; - -/** - * A view over a multi-dimensional array in row-major order. - * - * The `rank` is the number of dimensions in the array, and the `shape` - * describes the size of each dimension. - * - * The `data` pointer must point to a contiguous block of memory that contains - * the array data. - * - * If the source array is not stored in a row-major order, you may express - * the strides explicitly using the `strided_view` class. - * - * This class provides a simpler and more efficient interface for row-major - * arrays. - */ -template -class row_major_view -{ -public: - using element_type = T; - - row_major_view( - size_t rank, const uintptr_t* shape, const T* data, size_t data_size) - : _rank{rank} - , _shape{shape} - , _data{data} - , _data_size{data_size} - { - } - - size_t rank() const - { - return _rank; - } - const uintptr_t* shape() const - { - return _shape; - } - const T* data() const - { - return _data; - } - - size_t data_size() const - { - return _data_size; - } - - const row_major_view& view() const - { - return *this; - } - -private: - size_t _rank; - const uintptr_t* _shape; - const T* _data; - size_t _data_size; -}; - -template -struct row_major_1d_holder -{ - uintptr_t shape[1]; - const T* data; - size_t size; - - row_major_1d_holder(const T* d, size_t s) - : data(d) - , size(s) - { - shape[0] = static_cast(s); - } - - array::row_major_view view() const - { - return {1, shape, data, size}; - } -}; -} // namespace array - -template -inline auto to_view_state_impl(const std::vector& vec) -{ - return array::row_major_1d_holder::type>( - vec.data(), vec.size()); -} - -#if __cplusplus >= 202002L -template -inline auto to_view_state_impl(const std::span& span) -{ - return array::row_major_1d_holder::type>( - span.data(), span.size()); -} -#endif - -template -inline auto to_view_state_impl(const std::array& arr) -{ - return array::row_major_1d_holder::type>( - arr.data(), N); -} - -/** - * Customization point to enable serialization of additional types. - * - * Forwards to a namespace or ADL (König) lookup function. - * The customized `to_view_state_impl` for your custom type can be placed - * in either: - * * The namespace of the type in question. - * * In the `questdb::ingress` namespace. -/// - * The function can either return a view object directly (either - * `array::row_major_view`, `array::strided_view` or `decimal::binary_view`), - * or, if you need to place some fields on the stack, an object with a `.view()` - * method which returns a `const&` to one "materialize" shape or strides - * information into contiguous memory. - */ -struct to_view_state_fn -{ - template - auto operator()(const T& array) const - { - // Implement your own `to_view_state_impl` as needed. - return to_view_state_impl(array); - } -}; - -inline constexpr to_view_state_fn to_view_state{}; - /** * Types and utilities for working with arbitrary-precision decimal numbers. * @@ -869,6 +657,22 @@ struct to_decimal_view_state_fn inline constexpr to_decimal_view_state_fn to_decimal_view_state{}; +template +struct has_decimal_view_state : std::false_type +{ +}; + +template +struct has_decimal_view_state< + T, + std::void_t()))>> + : std::true_type +{ +}; + +template +inline constexpr bool has_decimal_view_state_v = + has_decimal_view_state::value; } // namespace decimal class line_sender_buffer @@ -1221,22 +1025,27 @@ class line_sender_buffer * * This overload uses a customization point to support additional types: * If you need to support your additional types you may implement a - * `to_view_state_impl` function in the object's namespace (via ADL) + * `to_array_view_state_impl` function in the object's namespace (via ADL) * or in the `questdb::ingress::array` namespace. * Ensure that any additional customization points are included before * `line_sender.hpp`. * - * @tparam ToViewT Type convertible to a custom object instance which + * @tparam ToArrayViewT Type convertible to a custom object instance which * can be converted to an array view. * @param name Column name. * @param array Multi-dimensional array. */ - template - line_sender_buffer& column(column_name_view name, ToViewT array) + template < + typename ToArrayViewT, + std::enable_if_t< + questdb::ingress::array::has_array_view_state_v, + int> = 0> + line_sender_buffer& column(column_name_view name, ToArrayViewT array) { may_init(); - const auto view_state = questdb::ingress::to_view_state(array); - return column(name, view_state.view()); + const auto array_view_state = + questdb::ingress::array::to_array_view_state(array); + return column(name, array_view_state.view()); } /** @@ -1324,6 +1133,45 @@ class line_sender_buffer return *this; } + /** + * Record a decimal value using a custom type via a customization point. + * + * This overload allows you to serialize custom decimal types by + * implementing a `to_decimal_view_state_impl` function for your type. + * + * QuestDB server version 9.2.0 or later is required for decimal support. + * + * # Customization + * + * To support your custom decimal type, implement + * `to_decimal_view_state_impl` in either: + * - The namespace of your type (ADL/Koenig lookup) + * - The `questdb::ingress::decimal` namespace + * + * The function should return either: + * - A `decimal::binary_view` directly, or + * - An object with a `.view()` method returning `const + * decimal::binary_view&` + * + * Include your customization point before including `line_sender.hpp`. + * + * @tparam ToDecimalViewT Type convertible to decimal::binary_view. + * @param name Column name. + * @param decimal Custom decimal value. + */ + template < + typename ToDecimalViewT, + std::enable_if_t< + questdb::ingress::decimal::has_decimal_view_state_v, + int> = 0> + line_sender_buffer& column(column_name_view name, ToDecimalViewT decimal) + { + may_init(); + const auto decimal_view_state = + questdb::ingress::decimal::to_decimal_view_state(decimal); + return column(name, decimal_view_state.view()); + } + /** Record a nanosecond timestamp value for the given column. */ template line_sender_buffer& column( diff --git a/include/questdb/ingress/line_sender_array.hpp b/include/questdb/ingress/line_sender_array.hpp new file mode 100644 index 00000000..1bd10932 --- /dev/null +++ b/include/questdb/ingress/line_sender_array.hpp @@ -0,0 +1,269 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +#pragma once + +#include "line_sender.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if __cplusplus >= 202002L +# include +#endif + +namespace questdb::ingress::array +{ +enum class strides_mode +{ + /** Strides are provided in bytes */ + bytes, + + /** Strides are provided in elements */ + elements, +}; + +/** + * A view over a multi-dimensional array with custom strides. + * + * The strides can be expressed as bytes offsets or as element counts. + * The `rank` is the number of dimensions in the array, and the `shape` + * describes the size of each dimension. + * + * If the data is stored in a row-major order, it may be more convenient and + * efficient to use the `row_major_view` instead of `strided_view`. + * + * The `data` pointer must point to a contiguous block of memory that contains + * the array data. + */ +template +class strided_view +{ +public: + using element_type = T; + static constexpr strides_mode stride_size_mode = M; + + strided_view( + size_t rank, + const uintptr_t* shape, + const intptr_t* strides, + const T* data, + size_t data_size) + : _rank{rank} + , _shape{shape} + , _strides{strides} + , _data{data} + , _data_size{data_size} + { + } + + size_t rank() const + { + return _rank; + } + + const uintptr_t* shape() const + { + return _shape; + } + + const intptr_t* strides() const + { + return _strides; + } + + const T* data() const + { + return _data; + } + + size_t data_size() const + { + return _data_size; + } + + const strided_view& view() const + { + return *this; + } + +private: + size_t _rank; + const uintptr_t* _shape; + const intptr_t* _strides; + const T* _data; + size_t _data_size; +}; + +/** + * A view over a multi-dimensional array in row-major order. + * + * The `rank` is the number of dimensions in the array, and the `shape` + * describes the size of each dimension. + * + * The `data` pointer must point to a contiguous block of memory that contains + * the array data. + * + * If the source array is not stored in a row-major order, you may express + * the strides explicitly using the `strided_view` class. + * + * This class provides a simpler and more efficient interface for row-major + * arrays. + */ +template +class row_major_view +{ +public: + using element_type = T; + + row_major_view( + size_t rank, const uintptr_t* shape, const T* data, size_t data_size) + : _rank{rank} + , _shape{shape} + , _data{data} + , _data_size{data_size} + { + } + + size_t rank() const + { + return _rank; + } + const uintptr_t* shape() const + { + return _shape; + } + const T* data() const + { + return _data; + } + + size_t data_size() const + { + return _data_size; + } + + const row_major_view& view() const + { + return *this; + } + +private: + size_t _rank; + const uintptr_t* _shape; + const T* _data; + size_t _data_size; +}; + +template +struct row_major_1d_holder +{ + uintptr_t shape[1]; + const T* data; + size_t size; + + row_major_1d_holder(const T* d, size_t s) + : data(d) + , size(s) + { + shape[0] = static_cast(s); + } + + array::row_major_view view() const + { + return {1, shape, data, size}; + } +}; + +template +inline auto to_array_view_state_impl(const std::vector& vec) +{ + return row_major_1d_holder::type>( + vec.data(), vec.size()); +} + +#if __cplusplus >= 202002L +template +inline auto to_array_view_state_impl(const std::span& span) +{ + return row_major_1d_holder::type>( + span.data(), span.size()); +} +#endif + +template +inline auto to_array_view_state_impl(const std::array& arr) +{ + return row_major_1d_holder::type>(arr.data(), N); +} + +/** + * Customization point to enable serialization of additional types as arrays. + * + * Forwards to a namespace or ADL (König) lookup function. + * The customized `to_array_view_state_impl` for your custom type can be placed + * in either: + * * The namespace of the type in question. + * * In the `questdb::ingress::array` namespace. +/// + * The function can either return a view object directly (either + * `row_major_view` or `strided_view`), or, if you need to place some fields on + * the stack, an object with a `.view()` method which returns a `const&` to one + * "materialize" shape or strides information into contiguous memory. + * of the two view types. Returning an object may be useful if you need to + */ +struct to_array_view_state_fn +{ + template + auto operator()(const T& array) const + { + // Implement your own `to_array_view_state_impl` as needed. + return to_array_view_state_impl(array); + } +}; + +inline constexpr to_array_view_state_fn to_array_view_state{}; + +template +struct has_array_view_state : std::false_type +{ +}; + +template +struct has_array_view_state< + T, + std::void_t()))>> + : std::true_type +{ +}; + +template +inline constexpr bool has_array_view_state_v = has_array_view_state::value; +} // namespace questdb::ingress::array From 44fe751c6362fd10ba0e68f612af055412bb695b Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Wed, 15 Oct 2025 14:33:06 +0200 Subject: [PATCH 25/65] fix: accept more characters in decimal serializer for str to allow nan/inf. --- questdb-rs/src/ingress/decimal.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/questdb-rs/src/ingress/decimal.rs b/questdb-rs/src/ingress/decimal.rs index 872160c1..fdadd007 100644 --- a/questdb-rs/src/ingress/decimal.rs +++ b/questdb-rs/src/ingress/decimal.rs @@ -84,7 +84,7 @@ pub trait DecimalSerializer { /// /// # Validation /// The implementation performs **partial validation only**: -/// - Rejects non-numerical characters (not -/+, 0-9, e/E, .) +/// - Rejects non-numerical characters (not -/+, 0-9, a-z/A-Z, .) /// - Does NOT validate the actual decimal syntax (e.g., "e2e" would pass) /// /// This is intentional: full parsing would add overhead. The QuestDB server performs complete @@ -106,7 +106,7 @@ impl DecimalSerializer for &str { // Validate and copy each byte, rejecting non-numeric characters for b in self.bytes() { match b { - b'0'..=b'9' | b'.' | b'-' | b'+' | b'e' | b'E' => out.push(b), + b'0'..=b'9' | b'.' | b'-' | b'+' | b'a'..=b'z' | b'A'..=b'Z' => out.push(b), _ => { return Err(error::fmt!( InvalidDecimal, From 1b6999413474bf993f0d1eb45a5e2361ace1867a Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Wed, 15 Oct 2025 17:45:44 +0200 Subject: [PATCH 26/65] tests: update test to use protocol version 3 --- cpp_test/test_line_sender.cpp | 61 +++++---- examples/line_sender_c_example.c | 2 +- ...line_sender_c_example_array_byte_strides.c | 2 +- .../line_sender_c_example_array_c_major.c | 2 +- ...line_sender_c_example_array_elem_strides.c | 2 +- examples/line_sender_c_example_auth.c | 2 +- examples/line_sender_c_example_auth_tls.c | 2 +- .../line_sender_c_example_decimal_binary.c | 125 ++++++++++++++++++ examples/line_sender_c_example_from_conf.c | 2 +- examples/line_sender_c_example_tls_ca.c | 2 +- examples/line_sender_cpp_example.cpp | 2 +- ..._sender_cpp_example_array_byte_strides.cpp | 2 +- .../line_sender_cpp_example_array_c_major.cpp | 2 +- ..._sender_cpp_example_array_elem_strides.cpp | 2 +- examples/line_sender_cpp_example_auth.cpp | 2 +- examples/line_sender_cpp_example_auth_tls.cpp | 2 +- ...line_sender_cpp_example_decimal_binary.cpp | 2 +- ...line_sender_cpp_example_decimal_custom.cpp | 2 +- .../line_sender_cpp_example_from_conf.cpp | 2 +- examples/line_sender_cpp_example_tls_ca.cpp | 2 +- 20 files changed, 175 insertions(+), 47 deletions(-) create mode 100644 examples/line_sender_c_example_decimal_binary.c diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index f6ed3ebd..a0d0e5ee 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -196,36 +196,39 @@ TEST_CASE("line_sender c api basics") 2.7, 48121.5, 4.3}; - CHECK(::line_sender_buffer_column_f64_arr_byte_strides( - buffer, - arr_name, - rank, - shape, - strides, - arr_data.data(), - arr_data.size(), - &err)); + CHECK( + ::line_sender_buffer_column_f64_arr_byte_strides( + buffer, + arr_name, + rank, + shape, + strides, + arr_data.data(), + arr_data.size(), + &err)); line_sender_column_name arr_name2 = QDB_COLUMN_NAME_LITERAL("a2"); intptr_t elem_strides[] = {6, 2, 1}; - CHECK(::line_sender_buffer_column_f64_arr_elem_strides( - buffer, - arr_name2, - rank, - shape, - elem_strides, - arr_data.data(), - arr_data.size(), - &err)); + CHECK( + ::line_sender_buffer_column_f64_arr_elem_strides( + buffer, + arr_name2, + rank, + shape, + elem_strides, + arr_data.data(), + arr_data.size(), + &err)); line_sender_column_name arr_name3 = QDB_COLUMN_NAME_LITERAL("a3"); - CHECK(::line_sender_buffer_column_f64_arr_c_major( - buffer, - arr_name3, - rank, - shape, - arr_data.data(), - arr_data.size(), - &err)); + CHECK( + ::line_sender_buffer_column_f64_arr_c_major( + buffer, + arr_name3, + rank, + shape, + arr_data.data(), + arr_data.size(), + &err)); CHECK(::line_sender_buffer_at_nanos(buffer, 10000000, &err)); CHECK(server.recv() == 0); CHECK(::line_sender_buffer_size(buffer) == 382); @@ -442,7 +445,7 @@ TEST_CASE("test multiple lines") questdb::ingress::test::mock_server server; std::string conf_str = "tcp::addr=127.0.0.1:" + std::to_string(server.port()) + - ";protocol_version=2;"; + ";protocol_version=3;"; questdb::ingress::line_sender sender = questdb::ingress::line_sender::from_conf(conf_str); CHECK_FALSE(sender.must_close()); @@ -1110,12 +1113,12 @@ TEST_CASE("HTTP basics") questdb::ingress::protocol::http, "127.0.0.1", 1}; questdb::ingress::opts opts1conf = questdb::ingress::opts::from_conf( "http::addr=127.0.0.1:1;username=user;password=pass;request_timeout=" - "5000;retry_timeout=5;protocol_version=2;"); + "5000;retry_timeout=5;protocol_version=3;"); questdb::ingress::opts opts2{ questdb::ingress::protocol::https, "localhost", "1"}; questdb::ingress::opts opts2conf = questdb::ingress::opts::from_conf( "http::addr=127.0.0.1:1;token=token;request_min_throughput=1000;retry_" - "timeout=0;protocol_version=2;"); + "timeout=0;protocol_version=3;"); opts1.protocol_version(questdb::ingress::protocol_version::v2) .username("user") .password("pass") diff --git a/examples/line_sender_c_example.c b/examples/line_sender_c_example.c index 50ce854a..fed0923e 100644 --- a/examples/line_sender_c_example.c +++ b/examples/line_sender_c_example.c @@ -10,7 +10,7 @@ static bool example(const char* host, const char* port) line_sender* sender = NULL; line_sender_buffer* buffer = NULL; char* conf_str = - concat("tcp::addr=", host, ":", port, ";protocol_version=2;"); + concat("tcp::addr=", host, ":", port, ";protocol_version=3;"); if (!conf_str) { fprintf(stderr, "Could not concatenate configuration string.\n"); diff --git a/examples/line_sender_c_example_array_byte_strides.c b/examples/line_sender_c_example_array_byte_strides.c index efe65f33..e2dbaf4e 100644 --- a/examples/line_sender_c_example_array_byte_strides.c +++ b/examples/line_sender_c_example_array_byte_strides.c @@ -13,7 +13,7 @@ static bool example(const char* host, const char* port) line_sender* sender = NULL; line_sender_buffer* buffer = NULL; char* conf_str = - concat("tcp::addr=", host, ":", port, ";protocol_version=2;"); + concat("tcp::addr=", host, ":", port, ";protocol_version=3;"); if (!conf_str) { fprintf(stderr, "Could not concatenate configuration string.\n"); diff --git a/examples/line_sender_c_example_array_c_major.c b/examples/line_sender_c_example_array_c_major.c index 798b13ba..62b36524 100644 --- a/examples/line_sender_c_example_array_c_major.c +++ b/examples/line_sender_c_example_array_c_major.c @@ -13,7 +13,7 @@ static bool example(const char* host, const char* port) line_sender* sender = NULL; line_sender_buffer* buffer = NULL; char* conf_str = - concat("tcp::addr=", host, ":", port, ";protocol_version=2;"); + concat("tcp::addr=", host, ":", port, ";protocol_version=3;"); if (!conf_str) { fprintf(stderr, "Could not concatenate configuration string.\n"); diff --git a/examples/line_sender_c_example_array_elem_strides.c b/examples/line_sender_c_example_array_elem_strides.c index f2a4272e..c55673ca 100644 --- a/examples/line_sender_c_example_array_elem_strides.c +++ b/examples/line_sender_c_example_array_elem_strides.c @@ -13,7 +13,7 @@ static bool example(const char* host, const char* port) line_sender* sender = NULL; line_sender_buffer* buffer = NULL; char* conf_str = - concat("tcp::addr=", host, ":", port, ";protocol_version=2;"); + concat("tcp::addr=", host, ":", port, ";protocol_version=3;"); if (!conf_str) { fprintf(stderr, "Could not concatenate configuration string.\n"); diff --git a/examples/line_sender_c_example_auth.c b/examples/line_sender_c_example_auth.c index 130941eb..04101584 100644 --- a/examples/line_sender_c_example_auth.c +++ b/examples/line_sender_c_example_auth.c @@ -15,7 +15,7 @@ static bool example(const char* host, const char* port) ":", port, ";" - "protocol_version=2;" + "protocol_version=3;" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" diff --git a/examples/line_sender_c_example_auth_tls.c b/examples/line_sender_c_example_auth_tls.c index e3ef6f3e..ff761692 100644 --- a/examples/line_sender_c_example_auth_tls.c +++ b/examples/line_sender_c_example_auth_tls.c @@ -15,7 +15,7 @@ static bool example(const char* host, const char* port) ":", port, ";" - "protocol_version=2;" + "protocol_version=3;" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" diff --git a/examples/line_sender_c_example_decimal_binary.c b/examples/line_sender_c_example_decimal_binary.c new file mode 100644 index 00000000..295afd53 --- /dev/null +++ b/examples/line_sender_c_example_decimal_binary.c @@ -0,0 +1,125 @@ +#include +#include +#include +#include +#include "concat.h" + +static bool example(const char* host, const char* port) +{ + line_sender_error* err = NULL; + line_sender* sender = NULL; + line_sender_buffer* buffer = NULL; + char* conf_str = + concat("tcp::addr=", host, ":", port, ";protocol_version=3;"); + if (!conf_str) + { + fprintf(stderr, "Could not concatenate configuration string.\n"); + return false; + } + line_sender_utf8 conf_str_utf8 = {0, NULL}; + if (!line_sender_utf8_init( + &conf_str_utf8, strlen(conf_str), conf_str, &err)) + goto on_error; + + sender = line_sender_from_conf(conf_str_utf8, &err); + if (!sender) + goto on_error; + + free(conf_str); + conf_str = NULL; + + buffer = line_sender_buffer_new_for_sender(sender); + line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. + + // We prepare all our table names and column names in advance. + // If we're inserting multiple rows, this allows us to avoid + // re-validating the same strings over and over again. + line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("c_trades"); + line_sender_column_name symbol_name = QDB_COLUMN_NAME_LITERAL("symbol"); + line_sender_column_name side_name = QDB_COLUMN_NAME_LITERAL("side"); + line_sender_column_name price_name = QDB_COLUMN_NAME_LITERAL("price"); + line_sender_column_name amount_name = QDB_COLUMN_NAME_LITERAL("amount"); + + if (!line_sender_buffer_table(buffer, table_name, &err)) + goto on_error; + + line_sender_utf8 symbol_value = QDB_UTF8_LITERAL("ETH-USD"); + if (!line_sender_buffer_symbol(buffer, symbol_name, symbol_value, &err)) + goto on_error; + + line_sender_utf8 side_value = QDB_UTF8_LITERAL("sell"); + if (!line_sender_buffer_symbol(buffer, side_name, side_value, &err)) + goto on_error; + + line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); + // 123 with a scale of 1 gives a decimal of 12.3 + const uint8_t price_unscaled_value[] = {123}; + if (!line_sender_buffer_column_dec( + buffer, price_name, 1, price_unscaled_value, 1, &err)) + goto on_error; + + if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) + goto on_error; + + // 1997-07-04 04:56:55 UTC + int64_t designated_timestamp = 867992215000000000; + if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) + goto on_error; + + //// If you want to get the current system timestamp as nanos, call: + // if (!line_sender_buffer_at_nanos(buffer, line_sender_now_nanos(), &err)) + // goto on_error; + + // To insert more records, call `line_sender_buffer_table(..)...` again. + // It's recommended to keep a timer and/or maximum buffer size to flush + // the buffer periodically with any accumulated records. + if (!line_sender_flush(sender, buffer, &err)) + goto on_error; + + line_sender_close(sender); + + return true; + +on_error:; + size_t err_len = 0; + const char* err_msg = line_sender_error_msg(err, &err_len); + fprintf(stderr, "Error running example: %.*s\n", (int)err_len, err_msg); + free(conf_str); + line_sender_error_free(err); + line_sender_buffer_free(buffer); + line_sender_close(sender); + return false; +} + +static bool displayed_help(int argc, const char* argv[]) +{ + for (int index = 1; index < argc; ++index) + { + const char* arg = argv[index]; + if ((strncmp(arg, "-h", 2) == 0) || (strncmp(arg, "--help", 6) == 0)) + { + fprintf(stderr, "Usage:\n"); + fprintf(stderr, "line_sender_c_example: [HOST [PORT]]\n"); + fprintf( + stderr, " HOST: ILP host (defaults to \"localhost\").\n"); + fprintf(stderr, " PORT: ILP port (defaults to \"9009\").\n"); + return true; + } + } + return false; +} + +int main(int argc, const char* argv[]) +{ + if (displayed_help(argc, argv)) + return 0; + + const char* host = "localhost"; + if (argc >= 2) + host = argv[1]; + const char* port = "9009"; + if (argc >= 3) + port = argv[2]; + + return !example(host, port); +} diff --git a/examples/line_sender_c_example_from_conf.c b/examples/line_sender_c_example_from_conf.c index ead11a2f..971d0c36 100644 --- a/examples/line_sender_c_example_from_conf.c +++ b/examples/line_sender_c_example_from_conf.c @@ -9,7 +9,7 @@ int main(int argc, const char* argv[]) line_sender_buffer* buffer = NULL; line_sender_utf8 conf = - QDB_UTF8_LITERAL("tcp::addr=localhost:9009;protocol_version=2;"); + QDB_UTF8_LITERAL("tcp::addr=localhost:9009;protocol_version=3;"); line_sender* sender = line_sender_from_conf(conf, &err); if (!sender) goto on_error; diff --git a/examples/line_sender_c_example_tls_ca.c b/examples/line_sender_c_example_tls_ca.c index a0540487..5928f1bf 100644 --- a/examples/line_sender_c_example_tls_ca.c +++ b/examples/line_sender_c_example_tls_ca.c @@ -15,7 +15,7 @@ static bool example(const char* ca_path, const char* host, const char* port) ":", port, ";", - "protocol_version=2;" + "protocol_version=3;" "tls_roots=", ca_path, ";", diff --git a/examples/line_sender_cpp_example.cpp b/examples/line_sender_cpp_example.cpp index 1fc5c66f..a1e2f1a2 100644 --- a/examples/line_sender_cpp_example.cpp +++ b/examples/line_sender_cpp_example.cpp @@ -11,7 +11,7 @@ static bool example(std::string_view host, std::string_view port) { auto sender = questdb::ingress::line_sender::from_conf( "tcp::addr=" + std::string{host} + ":" + std::string{port} + - ";protocol_version=2;"); + ";protocol_version=3;"); // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid diff --git a/examples/line_sender_cpp_example_array_byte_strides.cpp b/examples/line_sender_cpp_example_array_byte_strides.cpp index 792cead8..7df061a0 100644 --- a/examples/line_sender_cpp_example_array_byte_strides.cpp +++ b/examples/line_sender_cpp_example_array_byte_strides.cpp @@ -14,7 +14,7 @@ static bool array_example(std::string_view host, std::string_view port) { auto sender = questdb::ingress::line_sender::from_conf( "tcp::addr=" + std::string{host} + ":" + std::string{port} + - ";protocol_version=2;"); + ";protocol_version=3;"); const auto table_name = "cpp_market_orders_byte_strides"_tn; const auto symbol_col = "symbol"_cn; diff --git a/examples/line_sender_cpp_example_array_c_major.cpp b/examples/line_sender_cpp_example_array_c_major.cpp index 01bb869a..122c437f 100644 --- a/examples/line_sender_cpp_example_array_c_major.cpp +++ b/examples/line_sender_cpp_example_array_c_major.cpp @@ -14,7 +14,7 @@ static bool array_example(std::string_view host, std::string_view port) { auto sender = questdb::ingress::line_sender::from_conf( "tcp::addr=" + std::string{host} + ":" + std::string{port} + - ";protocol_version=2;"); + ";protocol_version=3;"); const auto table_name = "cpp_market_orders_c_major"_tn; const auto symbol_col = "symbol"_cn; diff --git a/examples/line_sender_cpp_example_array_elem_strides.cpp b/examples/line_sender_cpp_example_array_elem_strides.cpp index 7598de48..8864f2fb 100644 --- a/examples/line_sender_cpp_example_array_elem_strides.cpp +++ b/examples/line_sender_cpp_example_array_elem_strides.cpp @@ -14,7 +14,7 @@ static bool array_example(std::string_view host, std::string_view port) { auto sender = questdb::ingress::line_sender::from_conf( "tcp::addr=" + std::string{host} + ":" + std::string{port} + - ";protocol_version=2;"); + ";protocol_version=3;"); const auto table_name = "cpp_market_orders_elem_strides"_tn; const auto symbol_col = "symbol"_cn; diff --git a/examples/line_sender_cpp_example_auth.cpp b/examples/line_sender_cpp_example_auth.cpp index d1bd1455..07cb7c6c 100644 --- a/examples/line_sender_cpp_example_auth.cpp +++ b/examples/line_sender_cpp_example_auth.cpp @@ -11,7 +11,7 @@ static bool example(std::string_view host, std::string_view port) { auto sender = questdb::ingress::line_sender::from_conf( "tcp::addr=" + std::string{host} + ":" + std::string{port} + - ";protocol_version=2;" + ";protocol_version=3;" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" diff --git a/examples/line_sender_cpp_example_auth_tls.cpp b/examples/line_sender_cpp_example_auth_tls.cpp index 5c26e3b4..dc1ba25f 100644 --- a/examples/line_sender_cpp_example_auth_tls.cpp +++ b/examples/line_sender_cpp_example_auth_tls.cpp @@ -11,7 +11,7 @@ static bool example(std::string_view host, std::string_view port) { auto sender = questdb::ingress::line_sender::from_conf( "tcps::addr=" + std::string{host} + ":" + std::string{port} + - ";protocol_version=2;" + ";protocol_version=3;" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" diff --git a/examples/line_sender_cpp_example_decimal_binary.cpp b/examples/line_sender_cpp_example_decimal_binary.cpp index 20e1a992..2c7b3994 100644 --- a/examples/line_sender_cpp_example_decimal_binary.cpp +++ b/examples/line_sender_cpp_example_decimal_binary.cpp @@ -11,7 +11,7 @@ static bool example(std::string_view host, std::string_view port) { auto sender = questdb::ingress::line_sender::from_conf( "tcp::addr=" + std::string{host} + ":" + std::string{port} + - ";protocol_version=2;"); + ";protocol_version=3;"); // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid diff --git a/examples/line_sender_cpp_example_decimal_custom.cpp b/examples/line_sender_cpp_example_decimal_custom.cpp index cec604cd..71a3a1ed 100644 --- a/examples/line_sender_cpp_example_decimal_custom.cpp +++ b/examples/line_sender_cpp_example_decimal_custom.cpp @@ -65,7 +65,7 @@ static bool example(std::string_view host, std::string_view port) { auto sender = questdb::ingress::line_sender::from_conf( "tcp::addr=" + std::string{host} + ":" + std::string{port} + - ";protocol_version=2;"); + ";protocol_version=3;"); // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid diff --git a/examples/line_sender_cpp_example_from_conf.cpp b/examples/line_sender_cpp_example_from_conf.cpp index e7309ea2..142efb27 100644 --- a/examples/line_sender_cpp_example_from_conf.cpp +++ b/examples/line_sender_cpp_example_from_conf.cpp @@ -10,7 +10,7 @@ int main(int argc, const char* argv[]) try { auto sender = questdb::ingress::line_sender::from_conf( - "tcp::addr=localhost:9009;protocol_version=2;"); + "tcp::addr=localhost:9009;protocol_version=3;"); // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid diff --git a/examples/line_sender_cpp_example_tls_ca.cpp b/examples/line_sender_cpp_example_tls_ca.cpp index 7a36d0c7..c7a847ee 100644 --- a/examples/line_sender_cpp_example_tls_ca.cpp +++ b/examples/line_sender_cpp_example_tls_ca.cpp @@ -12,7 +12,7 @@ static bool example( { auto sender = questdb::ingress::line_sender::from_conf( "tcps::addr=" + std::string{host} + ":" + std::string{port} + - ";protocol_version=2;" + ";protocol_version=3;" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" From 36f534e7b27321c26c97f2443eff14a34b3d1629 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Wed, 15 Oct 2025 17:46:05 +0200 Subject: [PATCH 27/65] docs: add decimal to c/cpp docs --- doc/C.md | 3 +++ doc/CPP.md | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/doc/C.md b/doc/C.md index 8898b276..f57faf8a 100644 --- a/doc/C.md +++ b/doc/C.md @@ -29,6 +29,9 @@ - [Array with element strides](../examples/line_sender_c_example_array_elem_strides.c) - [Array in C-major order](../examples/line_sender_c_example_array_c_major.c) +**Decimal** +- [Decimal in binary format](../examples/line_sender_c_example_decimal_binary.c) + ## API Overview ### Header diff --git a/doc/CPP.md b/doc/CPP.md index 922fdf26..d1a8f494 100644 --- a/doc/CPP.md +++ b/doc/CPP.md @@ -30,6 +30,10 @@ - [Array in C-major order](../examples/line_sender_cpp_example_array_c_major.cpp) - [Custom array type integration](../examples/line_sender_cpp_example_array_custom.cpp) +**Decimal** +- [Decimal in binary format](../examples/line_sender_cpp_example_decimal_binary.cpp) +- [Custom decimal type integration](../examples/line_sender_cpp_example_decimal_custom.cpp) + ## API Overview ### Header From b11912abf02ffb7b71a0453ec1904d18124bc06c Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Wed, 15 Oct 2025 17:46:46 +0200 Subject: [PATCH 28/65] tests: update rust examples to use protocol version 3 --- questdb-rs/examples/basic.rs | 2 +- questdb-rs/examples/http.rs | 2 +- questdb-rs/examples/protocol_version.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/questdb-rs/examples/basic.rs b/questdb-rs/examples/basic.rs index f68c7bac..17cfa325 100644 --- a/questdb-rs/examples/basic.rs +++ b/questdb-rs/examples/basic.rs @@ -8,7 +8,7 @@ use questdb::{ fn main() -> Result<()> { let host: String = std::env::args().nth(1).unwrap_or("localhost".to_string()); let port: &str = &std::env::args().nth(2).unwrap_or("9009".to_string()); - let mut sender = Sender::from_conf(format!("tcp::addr={host}:{port};protocol_version=2;"))?; + let mut sender = Sender::from_conf(format!("tcp::addr={host}:{port};protocol_version=3;"))?; let mut buffer = sender.new_buffer(); let designated_timestamp = TimestampNanos::from_datetime(Utc.with_ymd_and_hms(1997, 7, 4, 4, 56, 55).unwrap())?; diff --git a/questdb-rs/examples/http.rs b/questdb-rs/examples/http.rs index 0eaea654..b938097f 100644 --- a/questdb-rs/examples/http.rs +++ b/questdb-rs/examples/http.rs @@ -9,7 +9,7 @@ use rust_decimal::Decimal; fn main() -> Result<()> { let mut sender = Sender::from_conf( - "https::addr=localhost:9000;username=foo;password=bar;protocol_version=2;", + "https::addr=localhost:9000;username=foo;password=bar;protocol_version=3;", )?; let mut buffer = sender.new_buffer(); let price = Decimal::from_str("2615.54").unwrap(); diff --git a/questdb-rs/examples/protocol_version.rs b/questdb-rs/examples/protocol_version.rs index 6682c5a0..0fed2ab4 100644 --- a/questdb-rs/examples/protocol_version.rs +++ b/questdb-rs/examples/protocol_version.rs @@ -25,7 +25,7 @@ fn main() -> Result<()> { // QuestDB server version 9.0.0 or later is required for `protocol_version=2` support. let mut sender2 = Sender::from_conf( - "http::addr=localhost:9000;username=foo;password=bar;protocol_version=2;", + "http::addr=localhost:9000;username=foo;password=bar;protocol_version=3;", )?; let mut buffer2 = sender2.new_buffer(); buffer2 From 147bab4ce5b3be3424f2bb78672a16202ff372df Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Wed, 15 Oct 2025 17:47:13 +0200 Subject: [PATCH 29/65] feat: add protocol version 3 to supported protocol versions --- questdb-rs/src/ingress/mod.rs | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index e24fe9b7..4bc7d3fe 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -98,6 +98,13 @@ pub enum ProtocolVersion { V3 = 3, } +/// List of supported protocol versions, in order of preference (highest to lowest). +const SUPPORTED_PROTOCOL_VERSIONS: [ProtocolVersion; 3] = [ + ProtocolVersion::V3, + ProtocolVersion::V2, + ProtocolVersion::V1, +]; + impl ProtocolVersion { /// Returns `true` if this protocol version supports the given protocol version. /// @@ -447,11 +454,12 @@ impl SenderBuilder { "protocol_version" => match val { "1" => builder.protocol_version(ProtocolVersion::V1)?, "2" => builder.protocol_version(ProtocolVersion::V2)?, + "3" => builder.protocol_version(ProtocolVersion::V3)?, "auto" => builder, invalid => { return Err(error::fmt!( ConfigError, - "invalid \"protocol_version\" [value={invalid}, allowed-values=[auto, 1, 2]]]\"]" + "invalid \"protocol_version\" [value={invalid}, allowed-values=[auto, 1, 2, 3]]" )) } }, @@ -1163,22 +1171,27 @@ impl SenderBuilder { let (protocol_versions, server_max_name_len) = read_server_settings(http_state, settings_url, max_name_len)?; max_name_len = server_max_name_len; - if protocol_versions.contains(&ProtocolVersion::V2) { - ProtocolVersion::V2 - } else if protocol_versions.contains(&ProtocolVersion::V1) { - ProtocolVersion::V1 - } else { - return Err(fmt!( - ProtocolVersionError, - "Server does not support current client" - )); - } + SUPPORTED_PROTOCOL_VERSIONS + .iter() + .find(|version| protocol_versions.contains(version)) + .copied() + .ok_or_else(|| { + fmt!( + ProtocolVersionError, + "Server does not support any of the client protocol versions: {:?}", + SUPPORTED_PROTOCOL_VERSIONS + ) + })? } else { unreachable!("HTTP handler should be used for HTTP protocol"); } } }, }; + eprintln!( + "Using protocol version {:?} with max_name_len={}", + protocol_version, max_name_len + ); if auth.is_some() { descr.push_str("auth=on]"); From 77520efcfbf23874aa72c2e47c6cc09ada5bc8d4 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Wed, 15 Oct 2025 17:47:45 +0200 Subject: [PATCH 30/65] docs: add a simple description about the decimal datatype to ingress --- questdb-rs/src/ingress/mod.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/questdb-rs/src/ingress/mod.md b/questdb-rs/src/ingress/mod.md index 97fa5ee6..9d0c23d3 100644 --- a/questdb-rs/src/ingress/mod.md +++ b/questdb-rs/src/ingress/mod.md @@ -326,6 +326,21 @@ buffer.table(table_name)?.column_f64(price_name, 39269.98)?.at(TimestampNanos::n # } ``` +## Decimal Datatype + +The [`Buffer::column_dec`](Buffer::column_dec) method supports efficient ingestion of decimal values using several convenient types: + +- native Rust String slices +- decimals from the [`rust_decimal`](https://docs.rs/rust_decimal) crate +- decimals from the [`bigdecimal`](https://docs.rs/bigdecimal) crate + +You must use protocol version 3 to ingest decimals. HTTP transport will +automatically enable it as long as you're connecting to an up-to-date QuestDB +server (version 9.2.0 or later), but with TCP you must explicitly specify it in +the configuration string: `protocol_version=3;`. + +**Note**: QuestDB server version 9.2.0 or later is required for decimal support. + ## Check out the CONSIDERATIONS Document The [Library From d9ef50d221fdda9463566efa3b91d5de4fedebee Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Wed, 15 Oct 2025 17:55:15 +0200 Subject: [PATCH 31/65] fix: change port variable type to String for consistency --- questdb-rs/examples/basic.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questdb-rs/examples/basic.rs b/questdb-rs/examples/basic.rs index 17cfa325..4cbba7f7 100644 --- a/questdb-rs/examples/basic.rs +++ b/questdb-rs/examples/basic.rs @@ -7,7 +7,7 @@ use questdb::{ fn main() -> Result<()> { let host: String = std::env::args().nth(1).unwrap_or("localhost".to_string()); - let port: &str = &std::env::args().nth(2).unwrap_or("9009".to_string()); + let port: String = std::env::args().nth(2).unwrap_or("9009".to_string()); let mut sender = Sender::from_conf(format!("tcp::addr={host}:{port};protocol_version=3;"))?; let mut buffer = sender.new_buffer(); let designated_timestamp = From 6d19bd9508465d033e21f44354ed09e08e80b496 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Wed, 15 Oct 2025 17:55:26 +0200 Subject: [PATCH 32/65] fix: change to_array_view_state_impl argument to const reference --- include/questdb/ingress/line_sender_array.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/questdb/ingress/line_sender_array.hpp b/include/questdb/ingress/line_sender_array.hpp index 1bd10932..2bba3d4d 100644 --- a/include/questdb/ingress/line_sender_array.hpp +++ b/include/questdb/ingress/line_sender_array.hpp @@ -259,7 +259,7 @@ struct has_array_view_state : std::false_type template struct has_array_view_state< T, - std::void_t()))>> + std::void_t()))>> : std::true_type { }; From f47ee4fcaca5eaa83b360391790016f73c9b985a Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Wed, 15 Oct 2025 17:58:13 +0200 Subject: [PATCH 33/65] fix: update usage message to line_sender_c_example_decimal_custom --- examples/line_sender_cpp_example_decimal_custom.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/line_sender_cpp_example_decimal_custom.cpp b/examples/line_sender_cpp_example_decimal_custom.cpp index 71a3a1ed..7b58ef90 100644 --- a/examples/line_sender_cpp_example_decimal_custom.cpp +++ b/examples/line_sender_cpp_example_decimal_custom.cpp @@ -112,7 +112,7 @@ static bool displayed_help(int argc, const char* argv[]) if ((arg == "-h"sv) || (arg == "--help"sv)) { std::cerr << "Usage:\n" - << "line_sender_c_example: [HOST [PORT]]\n" + << "line_sender_c_example_decimal_custom: [HOST [PORT]]\n" << " HOST: ILP host (defaults to \"localhost\").\n" << " PORT: ILP port (defaults to \"9009\")." << std::endl; From 23d51b239f190f01fd8e60747fdc8d4fee3648ea Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Wed, 15 Oct 2025 17:59:01 +0200 Subject: [PATCH 34/65] fix: remove unused includes from line_sender_array.hpp --- include/questdb/ingress/line_sender_array.hpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/include/questdb/ingress/line_sender_array.hpp b/include/questdb/ingress/line_sender_array.hpp index 2bba3d4d..d4042b25 100644 --- a/include/questdb/ingress/line_sender_array.hpp +++ b/include/questdb/ingress/line_sender_array.hpp @@ -27,13 +27,10 @@ #include "line_sender.h" #include -#include #include #include -#include -#include -#include #include +#include #include #if __cplusplus >= 202002L # include From 1f889e7e8b388f2b511043af0bf483bdec083c02 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Wed, 15 Oct 2025 18:13:33 +0200 Subject: [PATCH 35/65] fix: update scale validation in DecimalSerializer to allow negative scales --- questdb-rs/src/ingress/decimal.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questdb-rs/src/ingress/decimal.rs b/questdb-rs/src/ingress/decimal.rs index fdadd007..5983ab69 100644 --- a/questdb-rs/src/ingress/decimal.rs +++ b/questdb-rs/src/ingress/decimal.rs @@ -157,7 +157,7 @@ impl DecimalSerializer for &bigdecimal::BigDecimal { out.push(DECIMAL_BINARY_FORMAT_TYPE); let (unscaled, mut scale) = self.as_bigint_and_scale(); - if scale > 76 { + if scale < -76 || scale > 76 { return Err(error::fmt!( InvalidDecimal, "QuestDB ILP does not support scale greater than 76, got {}", From 466b26de9aaf333a7ef7eac3046d0c7b5533759a Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Wed, 15 Oct 2025 18:15:48 +0200 Subject: [PATCH 36/65] fix: update protocol version to v3 in line_sender tests and header --- cpp_test/test_line_sender.cpp | 20 ++++++++++---------- include/questdb/ingress/line_sender.hpp | 7 +++++++ 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index a0d0e5ee..7e824cfc 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -284,7 +284,7 @@ TEST_CASE("line_sender c++ api basics") questdb::ingress::protocol::tcp, std::string("127.0.0.1"), std::to_string(server.port())}; - opts.protocol_version(questdb::ingress::protocol_version::v2); + opts.protocol_version(questdb::ingress::protocol_version::v3); questdb::ingress::line_sender sender{opts}; CHECK_FALSE(sender.must_close()); server.accept(); @@ -354,7 +354,7 @@ TEST_CASE("line_sender array vector API") questdb::ingress::protocol::tcp, std::string("127.0.0.1"), std::to_string(server.port())}; - opts.protocol_version(questdb::ingress::protocol_version::v2); + opts.protocol_version(questdb::ingress::protocol_version::v3); questdb::ingress::line_sender sender{opts}; CHECK_FALSE(sender.must_close()); server.accept(); @@ -398,7 +398,7 @@ TEST_CASE("line_sender array span API") questdb::ingress::protocol::tcp, std::string("127.0.0.1"), std::to_string(server.port())}; - opts.protocol_version(questdb::ingress::protocol_version::v2); + opts.protocol_version(questdb::ingress::protocol_version::v3); questdb::ingress::line_sender sender{opts}; CHECK_FALSE(sender.must_close()); server.accept(); @@ -1063,21 +1063,21 @@ TEST_CASE("Moved View") TEST_CASE("Empty Buffer") { questdb::ingress::line_sender_buffer b1{ - questdb::ingress::protocol_version::v2}; + questdb::ingress::protocol_version::v3}; CHECK(b1.size() == 0); questdb::ingress::line_sender_buffer b2{std::move(b1)}; CHECK(b1.size() == 0); CHECK(b2.size() == 0); questdb::ingress::line_sender_buffer b3{ - questdb::ingress::protocol_version::v2}; + questdb::ingress::protocol_version::v3}; b3 = std::move(b2); CHECK(b2.size() == 0); CHECK(b3.size() == 0); questdb::ingress::line_sender_buffer b4{ - questdb::ingress::protocol_version::v2}; + questdb::ingress::protocol_version::v3}; b4.table("test").symbol("a", "b").at_now(); questdb::ingress::line_sender_buffer b5{ - questdb::ingress::protocol_version::v2}; + questdb::ingress::protocol_version::v3}; b5 = std::move(b4); CHECK(b4.size() == 0); CHECK(b5.size() == 9); @@ -1119,13 +1119,13 @@ TEST_CASE("HTTP basics") questdb::ingress::opts opts2conf = questdb::ingress::opts::from_conf( "http::addr=127.0.0.1:1;token=token;request_min_throughput=1000;retry_" "timeout=0;protocol_version=3;"); - opts1.protocol_version(questdb::ingress::protocol_version::v2) + opts1.protocol_version(questdb::ingress::protocol_version::v3) .username("user") .password("pass") .max_buf_size(1000000) .request_timeout(5000) .retry_timeout(5); - opts2.protocol_version(questdb::ingress::protocol_version::v2) + opts2.protocol_version(questdb::ingress::protocol_version::v3) .token("token") .request_min_throughput(1000) .retry_timeout(0); @@ -1182,7 +1182,7 @@ TEST_CASE("line sender protocol version v2") questdb::ingress::protocol::tcp, std::string("127.0.0.1"), std::to_string(server.port())}; - opts.protocol_version(questdb::ingress::protocol_version::v2); + opts.protocol_version(questdb::ingress::protocol_version::v3); questdb::ingress::line_sender sender{opts}; CHECK_FALSE(sender.must_close()); server.accept(); diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index a2ba6a45..88cb0ccc 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -122,6 +122,13 @@ enum class protocol_version * `v2` support. */ v2 = 2, + + /** + * InfluxDB Line Protocol v3. + * QuestDB server version 9.2.0 or later is required for + * `v3` support. + */ + v3 = 3, }; /* Possible sources of the root certificates used to validate the server's TLS From 8dad86615cb11d2495e797c1bb257fb1873340c8 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Wed, 15 Oct 2025 18:20:29 +0200 Subject: [PATCH 37/65] fix: remove unused price_value assignment in line_sender_c_example_decimal_binary --- examples/line_sender_c_example_decimal_binary.c | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/line_sender_c_example_decimal_binary.c b/examples/line_sender_c_example_decimal_binary.c index 295afd53..0228d7fb 100644 --- a/examples/line_sender_c_example_decimal_binary.c +++ b/examples/line_sender_c_example_decimal_binary.c @@ -51,7 +51,6 @@ static bool example(const char* host, const char* port) if (!line_sender_buffer_symbol(buffer, side_name, side_value, &err)) goto on_error; - line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); // 123 with a scale of 1 gives a decimal of 12.3 const uint8_t price_unscaled_value[] = {123}; if (!line_sender_buffer_column_dec( From de1bba3f053d92a1eab1202781b2987c3cb5974e Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Wed, 15 Oct 2025 18:20:43 +0200 Subject: [PATCH 38/65] fix: update usage message to reflect correct example name in line_sender_c_example_decimal_binary --- examples/line_sender_c_example_decimal_binary.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/line_sender_c_example_decimal_binary.c b/examples/line_sender_c_example_decimal_binary.c index 0228d7fb..5e5c6fec 100644 --- a/examples/line_sender_c_example_decimal_binary.c +++ b/examples/line_sender_c_example_decimal_binary.c @@ -98,7 +98,9 @@ static bool displayed_help(int argc, const char* argv[]) if ((strncmp(arg, "-h", 2) == 0) || (strncmp(arg, "--help", 6) == 0)) { fprintf(stderr, "Usage:\n"); - fprintf(stderr, "line_sender_c_example: [HOST [PORT]]\n"); + fprintf( + stderr, + "line_sender_c_example_decimal_binary: [HOST [PORT]]\n"); fprintf( stderr, " HOST: ILP host (defaults to \"localhost\").\n"); fprintf(stderr, " PORT: ILP port (defaults to \"9009\").\n"); From 12882be6e2a27669a836eeebe4f76f75354903b4 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Wed, 15 Oct 2025 18:23:02 +0200 Subject: [PATCH 39/65] fix: update price column type and correct table name in protocol_version example --- questdb-rs/examples/protocol_version.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/questdb-rs/examples/protocol_version.rs b/questdb-rs/examples/protocol_version.rs index 0fed2ab4..4887f187 100644 --- a/questdb-rs/examples/protocol_version.rs +++ b/questdb-rs/examples/protocol_version.rs @@ -8,8 +8,6 @@ use questdb::{ }; fn main() -> Result<()> { - let price = BigDecimal::from_str("2615.54").unwrap(); - let mut sender = Sender::from_conf( "http::addr=localhost:9000;username=foo;password=bar;protocol_version=1;", )?; @@ -18,18 +16,19 @@ fn main() -> Result<()> { .table("trades_ilp_v1")? .symbol("symbol", "ETH-USD")? .symbol("side", "sell")? - .column_dec("price", &price)? + .column_f64("price", 2615.54)? .column_f64("amount", 0.00044)? .at(TimestampNanos::now())?; sender.flush(&mut buffer)?; - // QuestDB server version 9.0.0 or later is required for `protocol_version=2` support. + // QuestDB server version 9.2.0 or later is required for `protocol_version=3` support. let mut sender2 = Sender::from_conf( "http::addr=localhost:9000;username=foo;password=bar;protocol_version=3;", )?; + let price = BigDecimal::from_str("2615.54").unwrap(); let mut buffer2 = sender2.new_buffer(); buffer2 - .table("trades_ilp_v2")? + .table("trades_ilp_v3")? .symbol("symbol", "ETH-USD")? .symbol("side", "sell")? .column_dec("price", &price)? From f1624df09ee65b98e5b0e8baf6d6cf559d75b40d Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Thu, 16 Oct 2025 17:13:53 +0200 Subject: [PATCH 40/65] fix: free line sender buffer after flushing in example files --- doc/C.md | 1 + examples/line_sender_c_example.c | 1 + examples/line_sender_c_example_array_byte_strides.c | 1 + examples/line_sender_c_example_array_c_major.c | 1 + examples/line_sender_c_example_array_elem_strides.c | 1 + examples/line_sender_c_example_auth.c | 1 + examples/line_sender_c_example_auth_tls.c | 1 + examples/line_sender_c_example_decimal_binary.c | 1 + examples/line_sender_c_example_from_conf.c | 1 + examples/line_sender_c_example_from_env.c | 1 + examples/line_sender_c_example_http.c | 1 + examples/line_sender_c_example_tls_ca.c | 1 + 12 files changed, 12 insertions(+) diff --git a/doc/C.md b/doc/C.md index f57faf8a..123c8346 100644 --- a/doc/C.md +++ b/doc/C.md @@ -106,6 +106,7 @@ if (!line_sender_buffer_at_nanos(buffer, line_sender_now_nanos(), &err)) if (!line_sender_flush(sender, buffer, &err)) goto on_error; +line_sender_buffer_free(buffer); line_sender_close(sender); ``` diff --git a/examples/line_sender_c_example.c b/examples/line_sender_c_example.c index fed0923e..6f711ded 100644 --- a/examples/line_sender_c_example.c +++ b/examples/line_sender_c_example.c @@ -74,6 +74,7 @@ static bool example(const char* host, const char* port) if (!line_sender_flush(sender, buffer, &err)) goto on_error; + line_sender_buffer_free(buffer); line_sender_close(sender); return true; diff --git a/examples/line_sender_c_example_array_byte_strides.c b/examples/line_sender_c_example_array_byte_strides.c index e2dbaf4e..6439f9d6 100644 --- a/examples/line_sender_c_example_array_byte_strides.c +++ b/examples/line_sender_c_example_array_byte_strides.c @@ -82,6 +82,7 @@ static bool example(const char* host, const char* port) if (!line_sender_flush(sender, buffer, &err)) goto on_error; + line_sender_buffer_free(buffer); line_sender_close(sender); return true; diff --git a/examples/line_sender_c_example_array_c_major.c b/examples/line_sender_c_example_array_c_major.c index 62b36524..7c9764f8 100644 --- a/examples/line_sender_c_example_array_c_major.c +++ b/examples/line_sender_c_example_array_c_major.c @@ -79,6 +79,7 @@ static bool example(const char* host, const char* port) if (!line_sender_flush(sender, buffer, &err)) goto on_error; + line_sender_buffer_free(buffer); line_sender_close(sender); return true; diff --git a/examples/line_sender_c_example_array_elem_strides.c b/examples/line_sender_c_example_array_elem_strides.c index c55673ca..f5ef14f4 100644 --- a/examples/line_sender_c_example_array_elem_strides.c +++ b/examples/line_sender_c_example_array_elem_strides.c @@ -82,6 +82,7 @@ static bool example(const char* host, const char* port) if (!line_sender_flush(sender, buffer, &err)) goto on_error; + line_sender_buffer_free(buffer); line_sender_close(sender); return true; diff --git a/examples/line_sender_c_example_auth.c b/examples/line_sender_c_example_auth.c index 04101584..e13e0715 100644 --- a/examples/line_sender_c_example_auth.c +++ b/examples/line_sender_c_example_auth.c @@ -83,6 +83,7 @@ static bool example(const char* host, const char* port) if (!line_sender_flush(sender, buffer, &err)) goto on_error; + line_sender_buffer_free(buffer); line_sender_close(sender); return true; diff --git a/examples/line_sender_c_example_auth_tls.c b/examples/line_sender_c_example_auth_tls.c index ff761692..4b67117f 100644 --- a/examples/line_sender_c_example_auth_tls.c +++ b/examples/line_sender_c_example_auth_tls.c @@ -84,6 +84,7 @@ static bool example(const char* host, const char* port) if (!line_sender_flush(sender, buffer, &err)) goto on_error; + line_sender_buffer_free(buffer); line_sender_close(sender); return true; diff --git a/examples/line_sender_c_example_decimal_binary.c b/examples/line_sender_c_example_decimal_binary.c index 5e5c6fec..d2b87f2b 100644 --- a/examples/line_sender_c_example_decimal_binary.c +++ b/examples/line_sender_c_example_decimal_binary.c @@ -75,6 +75,7 @@ static bool example(const char* host, const char* port) if (!line_sender_flush(sender, buffer, &err)) goto on_error; + line_sender_buffer_free(buffer); line_sender_close(sender); return true; diff --git a/examples/line_sender_c_example_from_conf.c b/examples/line_sender_c_example_from_conf.c index 971d0c36..04a46b31 100644 --- a/examples/line_sender_c_example_from_conf.c +++ b/examples/line_sender_c_example_from_conf.c @@ -61,6 +61,7 @@ int main(int argc, const char* argv[]) if (!line_sender_flush(sender, buffer, &err)) goto on_error; + line_sender_buffer_free(buffer); line_sender_close(sender); return 0; diff --git a/examples/line_sender_c_example_from_env.c b/examples/line_sender_c_example_from_env.c index 6ab78c3b..d927bac7 100644 --- a/examples/line_sender_c_example_from_env.c +++ b/examples/line_sender_c_example_from_env.c @@ -60,6 +60,7 @@ int main(int argc, const char* argv[]) if (!line_sender_flush(sender, buffer, &err)) goto on_error; + line_sender_buffer_free(buffer); line_sender_close(sender); return 0; diff --git a/examples/line_sender_c_example_http.c b/examples/line_sender_c_example_http.c index 0d12d602..349f5e9d 100644 --- a/examples/line_sender_c_example_http.c +++ b/examples/line_sender_c_example_http.c @@ -72,6 +72,7 @@ static bool example(const char* host, const char* port) if (!line_sender_flush(sender, buffer, &err)) goto on_error; + line_sender_buffer_free(buffer); line_sender_close(sender); return true; diff --git a/examples/line_sender_c_example_tls_ca.c b/examples/line_sender_c_example_tls_ca.c index 5928f1bf..a8591ca2 100644 --- a/examples/line_sender_c_example_tls_ca.c +++ b/examples/line_sender_c_example_tls_ca.c @@ -87,6 +87,7 @@ static bool example(const char* ca_path, const char* host, const char* port) if (!line_sender_flush(sender, buffer, &err)) goto on_error; + line_sender_buffer_free(buffer); line_sender_close(sender); return true; From 9d564a80af6c95f5ae0439ed1b19be107c891814 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Thu, 16 Oct 2025 17:18:27 +0200 Subject: [PATCH 41/65] fix: qualify binary_view with questdb::ingress::decimal namespace --- examples/line_sender_cpp_example_decimal_binary.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/line_sender_cpp_example_decimal_binary.cpp b/examples/line_sender_cpp_example_decimal_binary.cpp index 2c7b3994..a0f52342 100644 --- a/examples/line_sender_cpp_example_decimal_binary.cpp +++ b/examples/line_sender_cpp_example_decimal_binary.cpp @@ -3,7 +3,6 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; -using namespace questdb::ingress::decimal; static bool example(std::string_view host, std::string_view port) { @@ -23,7 +22,8 @@ static bool example(std::string_view host, std::string_view port) const auto amount_name = "amount"_cn; const uint8_t price_unscaled_value[] = {123}; // 123 with a scale of 1 gives a decimal of 12.3 - const auto price_value = binary_view(1, price_unscaled_value); + const auto price_value = + questdb::ingress::decimal::binary_view(1, price_unscaled_value); questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table(table_name) From e63da895872ebdac3d3e3d7ab5b972b6d0b72a63 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Thu, 16 Oct 2025 17:20:29 +0200 Subject: [PATCH 42/65] fix: update protocol version to 2 in line sender examples --- examples/line_sender_c_example_array_byte_strides.c | 2 +- examples/line_sender_cpp_example_array_byte_strides.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/line_sender_c_example_array_byte_strides.c b/examples/line_sender_c_example_array_byte_strides.c index 6439f9d6..4f44db75 100644 --- a/examples/line_sender_c_example_array_byte_strides.c +++ b/examples/line_sender_c_example_array_byte_strides.c @@ -13,7 +13,7 @@ static bool example(const char* host, const char* port) line_sender* sender = NULL; line_sender_buffer* buffer = NULL; char* conf_str = - concat("tcp::addr=", host, ":", port, ";protocol_version=3;"); + concat("tcp::addr=", host, ":", port, ";protocol_version=2;"); if (!conf_str) { fprintf(stderr, "Could not concatenate configuration string.\n"); diff --git a/examples/line_sender_cpp_example_array_byte_strides.cpp b/examples/line_sender_cpp_example_array_byte_strides.cpp index 7df061a0..792cead8 100644 --- a/examples/line_sender_cpp_example_array_byte_strides.cpp +++ b/examples/line_sender_cpp_example_array_byte_strides.cpp @@ -14,7 +14,7 @@ static bool array_example(std::string_view host, std::string_view port) { auto sender = questdb::ingress::line_sender::from_conf( "tcp::addr=" + std::string{host} + ":" + std::string{port} + - ";protocol_version=3;"); + ";protocol_version=2;"); const auto table_name = "cpp_market_orders_byte_strides"_tn; const auto symbol_col = "symbol"_cn; From ed83ff2e9ec43f8b89d630086090b631836d1afb Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 09:42:43 +0200 Subject: [PATCH 43/65] fix: update protocol version checks to use comparison operators --- questdb-rs/src/ingress/buffer.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/questdb-rs/src/ingress/buffer.rs b/questdb-rs/src/ingress/buffer.rs index 76155dca..4562512e 100644 --- a/questdb-rs/src/ingress/buffer.rs +++ b/questdb-rs/src/ingress/buffer.rs @@ -1054,7 +1054,7 @@ impl Buffer { S: DecimalSerializer, Error: From, { - if !self.protocol_version.supports(ProtocolVersion::V3) { + if self.protocol_version < ProtocolVersion::V3 { return Err(error::fmt!( ProtocolVersionError, "Protocol version {} does not support the decimal datatype", @@ -1122,7 +1122,7 @@ impl Buffer { D: ArrayElement + ArrayElementSealed, Error: From, { - if !self.protocol_version.supports(ProtocolVersion::V2) { + if self.protocol_version < ProtocolVersion::V2 { return Err(error::fmt!( ProtocolVersionError, "Protocol version {} does not support array datatype", From ed7a9842400dab98e780e0a9ca27e54d34b7b642 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 09:44:25 +0200 Subject: [PATCH 44/65] fix: add PartialOrd to ProtocolVersion --- questdb-rs/build.rs | 2 +- questdb-rs/src/ingress/mod.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/questdb-rs/build.rs b/questdb-rs/build.rs index a4cc94cc..cada1599 100644 --- a/questdb-rs/build.rs +++ b/questdb-rs/build.rs @@ -167,7 +167,7 @@ pub mod json_tests { fn test_{:03}_{}( #[values(ProtocolVersion::V1, ProtocolVersion::V2, ProtocolVersion::V3)] version: ProtocolVersion ) -> TestResult {{ - if (version as u8) < {} {{ + if version < {} {{ return Ok(()); }} let mut buffer = Buffer::new(version); diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 4bc7d3fe..3fb2900f 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -78,7 +78,7 @@ pub(crate) const DOUBLE_BINARY_FORMAT_TYPE: u8 = 16; pub const DECIMAL_BINARY_FORMAT_TYPE: u8 = 23; /// The version of InfluxDB Line Protocol used to communicate with the server. -#[derive(Debug, Copy, Clone, PartialEq)] +#[derive(Debug, Copy, Clone, PartialEq, PartialOrd)] pub enum ProtocolVersion { /// Version 1 of Line Protocol. /// Full-text protocol. From 727cf0940c187689a9b6e0ead9116a2422c1283f Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 09:44:42 +0200 Subject: [PATCH 45/65] fix: rename binary_view to decimal_view in decimal namespace --- ...line_sender_cpp_example_decimal_binary.cpp | 2 +- ...line_sender_cpp_example_decimal_custom.cpp | 2 +- include/questdb/ingress/line_sender.hpp | 49 ++++++++++--------- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/examples/line_sender_cpp_example_decimal_binary.cpp b/examples/line_sender_cpp_example_decimal_binary.cpp index a0f52342..dcf9ee60 100644 --- a/examples/line_sender_cpp_example_decimal_binary.cpp +++ b/examples/line_sender_cpp_example_decimal_binary.cpp @@ -23,7 +23,7 @@ static bool example(std::string_view host, std::string_view port) const uint8_t price_unscaled_value[] = {123}; // 123 with a scale of 1 gives a decimal of 12.3 const auto price_value = - questdb::ingress::decimal::binary_view(1, price_unscaled_value); + questdb::ingress::decimal::decimal_view(1, price_unscaled_value); questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table(table_name) diff --git a/examples/line_sender_cpp_example_decimal_custom.cpp b/examples/line_sender_cpp_example_decimal_custom.cpp index 7b58ef90..d6186555 100644 --- a/examples/line_sender_cpp_example_decimal_custom.cpp +++ b/examples/line_sender_cpp_example_decimal_custom.cpp @@ -9,7 +9,7 @@ struct ViewHolder { std::array data; uint32_t scale; - questdb::ingress::decimal::binary_view view() const + questdb::ingress::decimal::decimal_view view() const { return {scale, data}; } diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 88cb0ccc..83f39b18 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -458,26 +458,26 @@ namespace decimal * Use this to send decimal values as strings (e.g., "123.456"). * The string will be parsed by the QuestDB server as a decimal column type. */ -class text_view +class decimal_str_view { public: - text_view(const char* buf, size_t len) + decimal_str_view(const char* buf, size_t len) : _view{buf, len} { } template - text_view(const char (&buf)[N]) + decimal_str_view(const char (&buf)[N]) : _view{buf} { } - text_view(std::string_view s_view) + decimal_str_view(std::string_view s_view) : _view{s_view} { } - text_view(const std::string& s) + decimal_str_view(const std::string& s) : _view{s} { } @@ -494,16 +494,16 @@ class text_view }; /** - * Literal suffix to construct `text_view` objects from string literals. + * Literal suffix to construct `decimal_str_view` objects from string literals. * * @code {.cpp} * using namespace questdb::ingress::decimal; * buffer.column("price"_cn, "123.456"_decimal); * @endcode */ -inline text_view operator"" _decimal(const char* buf, size_t len) +inline decimal_str_view operator"" _decimal(const char* buf, size_t len) { - return text_view{buf, len}; + return decimal_str_view{buf, len}; } /** @@ -523,7 +523,7 @@ inline text_view operator"" _decimal(const char* buf, size_t len) * ```c++ * // Represent 123.45 with scale 2 (unscaled value is 12345) * uint8_t mantissa[] = {0x30, 0x39}; // 12345 in two's complement big-endian - * auto decimal = questdb::ingress::decimal::binary_view(2, mantissa, + * auto decimal = questdb::ingress::decimal::decimal_view(2, mantissa, * sizeof(mantissa)); buffer.column("price"_cn, decimal); * ``` * @@ -532,7 +532,7 @@ inline text_view operator"" _decimal(const char* buf, size_t len) * - Maximum scale: 76 (QuestDB server limitation) * - Maximum mantissa size: 127 bytes (protocol limitation) */ -class binary_view +class decimal_view { public: /** @@ -543,7 +543,7 @@ class binary_view * format * @param data_size Number of bytes in the mantissa (must be ≤ 127) */ - binary_view(uint32_t scale, const uint8_t* data, size_t data_size) + decimal_view(uint32_t scale, const uint8_t* data, size_t data_size) : _scale{scale} , _data{data} , _data_size{data_size} @@ -557,7 +557,7 @@ class binary_view * @param data Fixed-size array containing the unscaled value */ template - binary_view(uint32_t scale, const uint8_t (&data)[N]) + decimal_view(uint32_t scale, const uint8_t (&data)[N]) : _scale{scale} , _data{data} , _data_size{N} @@ -571,7 +571,7 @@ class binary_view * @param data std::array containing the unscaled value */ template - binary_view(uint32_t scale, const std::array& data) + decimal_view(uint32_t scale, const std::array& data) : _scale{scale} , _data{data.data()} , _data_size{N} @@ -584,7 +584,7 @@ class binary_view * @param scale Number of decimal places (must be ≤ 76) * @param vec Vector containing the unscaled value */ - binary_view(uint32_t scale, const std::vector& vec) + decimal_view(uint32_t scale, const std::vector& vec) : _scale{scale} , _data{vec.data()} , _data_size{vec.size()} @@ -598,7 +598,7 @@ class binary_view * @param scale Number of decimal places (must be ≤ 76) * @param span Span containing the unscaled value */ - binary_view(uint32_t scale, const std::span& span) + decimal_view(uint32_t scale, const std::span& span) : _scale{scale} , _data{span.data()} , _data_size{span.size()} @@ -626,7 +626,7 @@ class binary_view /** Get a const reference to this view (for customization point * compatibility). */ - const binary_view& view() const + const decimal_view& view() const { return *this; } @@ -646,8 +646,8 @@ class binary_view * - The `questdb::ingress::decimal` namespace * * The function can either: - * - Return a `binary_view` object directly, or - * - Return an object with a `.view()` method that returns `const binary_view&` + * - Return a `decimal_view` object directly, or + * - Return an object with a `.view()` method that returns `const decimal_view&` * (useful if you need to store temporary data like shape/strides on the stack) */ @@ -1091,14 +1091,15 @@ class line_sender_buffer * the QuestDB server. * * For better performance and precision control, consider using the binary - * format via `decimal::binary_view` instead. + * format via `decimal::decimal_view` instead. * * QuestDB server version 9.2.0 or later is required for decimal support. * * @param name Column name. * @param value Decimal value as a validated UTF-8 string. */ - line_sender_buffer& column(column_name_view name, decimal::text_view value) + line_sender_buffer& column( + column_name_view name, decimal::decimal_str_view value) { may_init(); line_sender_error::wrapped_call( @@ -1127,7 +1128,7 @@ class line_sender_buffer * @param decimal Binary decimal view with scale and mantissa bytes. */ line_sender_buffer& column( - column_name_view name, const decimal::binary_view& decimal) + column_name_view name, const decimal::decimal_view& decimal) { may_init(); line_sender_error::wrapped_call( @@ -1156,13 +1157,13 @@ class line_sender_buffer * - The `questdb::ingress::decimal` namespace * * The function should return either: - * - A `decimal::binary_view` directly, or + * - A `decimal::decimal_view` directly, or * - An object with a `.view()` method returning `const - * decimal::binary_view&` + * decimal::decimal_view&` * * Include your customization point before including `line_sender.hpp`. * - * @tparam ToDecimalViewT Type convertible to decimal::binary_view. + * @tparam ToDecimalViewT Type convertible to decimal::decimal_view. * @param name Column name. * @param decimal Custom decimal value. */ From d730bade13ee3f885329f6066c939d0938ad7d2c Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 09:45:24 +0200 Subject: [PATCH 46/65] fix: remove supports method from ProtocolVersion implementation --- questdb-rs/src/ingress/mod.rs | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 3fb2900f..4b735afe 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -105,24 +105,6 @@ const SUPPORTED_PROTOCOL_VERSIONS: [ProtocolVersion; 3] = [ ProtocolVersion::V1, ]; -impl ProtocolVersion { - /// Returns `true` if this protocol version supports the given protocol version. - /// - /// # Examples - /// - /// ``` - /// use questdb::ingress::ProtocolVersion; - /// - /// assert_eq!(ProtocolVersion::V1.supports(ProtocolVersion::V2), false); - /// assert_eq!(ProtocolVersion::V2.supports(ProtocolVersion::V1), true); - /// ``` - #[inline] - pub fn supports(self, version: ProtocolVersion) -> bool { - // Protocol versions are backward compatible - self as u32 >= version as u32 - } -} - impl Display for ProtocolVersion { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { From e6140e373e395d9b1daf9bbfc7bf9974dcbf2647 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 10:07:37 +0200 Subject: [PATCH 47/65] fix: clarify error messages for decimal scale and byte length limits --- questdb-rs/src/ingress/decimal.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/questdb-rs/src/ingress/decimal.rs b/questdb-rs/src/ingress/decimal.rs index 5983ab69..76dca883 100644 --- a/questdb-rs/src/ingress/decimal.rs +++ b/questdb-rs/src/ingress/decimal.rs @@ -160,7 +160,7 @@ impl DecimalSerializer for &bigdecimal::BigDecimal { if scale < -76 || scale > 76 { return Err(error::fmt!( InvalidDecimal, - "QuestDB ILP does not support scale greater than 76, got {}", + "QuestDB ILP does not support decimal scale greater than 76, got {}", scale )); } @@ -180,7 +180,7 @@ impl DecimalSerializer for &bigdecimal::BigDecimal { if bytes.len() > i8::MAX as usize { return Err(error::fmt!( InvalidDecimal, - "QuestDB ILP does not support values greater than {} bytes, got {}", + "QuestDB ILP does not support decimal longer than {} bytes, got {}", i8::MAX, bytes.len() )); From 1ebc6e6ae79d0fc135d3a479b0ac1710d2d602cf Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 10:09:19 +0200 Subject: [PATCH 48/65] typo: add missing definite article in ingress documentation --- questdb-rs/src/ingress/mod.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/questdb-rs/src/ingress/mod.md b/questdb-rs/src/ingress/mod.md index 9d0c23d3..16806795 100644 --- a/questdb-rs/src/ingress/mod.md +++ b/questdb-rs/src/ingress/mod.md @@ -263,7 +263,7 @@ arrays using several convenient types: - native Rust vectors (up to 3-dimensional) - arrays from the [`ndarray`](https://docs.rs/ndarray) crate -You must use protocol version 2 to ingest arrays. HTTP transport will +You must use protocol version 2 to ingest arrays. The HTTP transport will automatically enable it as long as you're connecting to an up-to-date QuestDB server (version 9.0.0 or later), but with TCP you must explicitly specify it in the configuration string: `protocol_version=2;`. @@ -334,7 +334,7 @@ The [`Buffer::column_dec`](Buffer::column_dec) method supports efficient ingesti - decimals from the [`rust_decimal`](https://docs.rs/rust_decimal) crate - decimals from the [`bigdecimal`](https://docs.rs/bigdecimal) crate -You must use protocol version 3 to ingest decimals. HTTP transport will +You must use protocol version 3 to ingest decimals. The HTTP transport will automatically enable it as long as you're connecting to an up-to-date QuestDB server (version 9.2.0 or later), but with TCP you must explicitly specify it in the configuration string: `protocol_version=3;`. From 07648464b4acda37fa0972b743dff21b98cb2948 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 10:20:04 +0200 Subject: [PATCH 49/65] fix: expose decimal_view in line_sender.hpp --- include/questdb/ingress/line_sender.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 83f39b18..0e082448 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -682,6 +682,8 @@ inline constexpr bool has_decimal_view_state_v = has_decimal_view_state::value; } // namespace decimal +using decimal::decimal_view; + class line_sender_buffer { public: From e2fa2114c816ddd3d4ed1057aa88e3cd98e29282 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 10:25:51 +0200 Subject: [PATCH 50/65] fix: cast ProtocolVersion to u8 for comparison in json_tests --- questdb-rs/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questdb-rs/build.rs b/questdb-rs/build.rs index cada1599..a4cc94cc 100644 --- a/questdb-rs/build.rs +++ b/questdb-rs/build.rs @@ -167,7 +167,7 @@ pub mod json_tests { fn test_{:03}_{}( #[values(ProtocolVersion::V1, ProtocolVersion::V2, ProtocolVersion::V3)] version: ProtocolVersion ) -> TestResult {{ - if version < {} {{ + if (version as u8) < {} {{ return Ok(()); }} let mut buffer = Buffer::new(version); From 3bc2e3ab23b410e139cea00eefd9be89b734c101 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 10:32:54 +0200 Subject: [PATCH 51/65] refactor: split headers --- include/questdb/ingress/line_sender.hpp | 657 +----------------- include/questdb/ingress/line_sender_core.hpp | 432 ++++++++++++ .../questdb/ingress/line_sender_decimal.hpp | 282 ++++++++ 3 files changed, 715 insertions(+), 656 deletions(-) create mode 100644 include/questdb/ingress/line_sender_core.hpp create mode 100644 include/questdb/ingress/line_sender_decimal.hpp diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 0e082448..a5c492f0 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -24,666 +24,11 @@ #pragma once -#include "line_sender.h" - #include "line_sender_array.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if __cplusplus >= 202002L -# include -#endif +#include "line_sender_decimal.hpp" namespace questdb::ingress { -constexpr const char* inaddr_any = "0.0.0.0"; - -class line_sender; -class line_sender_buffer; -class opts; - -/** Category of error. */ -enum class line_sender_error_code -{ - /** The host, port, or interface was incorrect. */ - could_not_resolve_addr, - - /** Called methods in the wrong order. E.g. `symbol` after `column`. */ - invalid_api_call, - - /** A network error connecting or flushing data out. */ - socket_error, - - /** The string or symbol field is not encoded in valid UTF-8. */ - invalid_utf8, - - /** The table name or column name contains bad characters. */ - invalid_name, - - /** The supplied timestamp is invalid. */ - invalid_timestamp, - - /** Error during the authentication process. */ - auth_error, - - /** Error during TLS handshake. */ - tls_error, - - /** The server does not support ILP over HTTP. */ - http_not_supported, - - /** Error sent back from the server during flush. */ - server_flush_error, - - /** Bad configuration. */ - config_error, - - /** There was an error serializing an array. */ - array_error, - - /** Line sender protocol version error. */ - protocol_version_error, - - /** The supplied decimal is invalid. */ - invalid_decimal, -}; - -/** The protocol used to connect with. */ -enum class protocol -{ - /** InfluxDB Line Protocol over TCP. */ - tcp, - - /** InfluxDB Line Protocol over TCP with TLS. */ - tcps, - - /** InfluxDB Line Protocol over HTTP. */ - http, - - /** InfluxDB Line Protocol over HTTP with TLS. */ - https, -}; - -enum class protocol_version -{ - /** InfluxDB Line Protocol v1. */ - v1 = 1, - - /** - * InfluxDB Line Protocol v2. - * QuestDB server version 9.0.0 or later is required for - * `v2` support. - */ - v2 = 2, - - /** - * InfluxDB Line Protocol v3. - * QuestDB server version 9.2.0 or later is required for - * `v3` support. - */ - v3 = 3, -}; - -/* Possible sources of the root certificates used to validate the server's TLS - * certificate. */ -enum class ca -{ - /** Use the set of root certificates provided by the `webpki` crate. */ - webpki_roots, - - /** Use the set of root certificates provided by the operating system. */ - os_roots, - - /** Combine the set of root certificates provided by the `webpki` crate and - * the operating system. */ - webpki_and_os_roots, - - /** Use the root certificates provided in a PEM-encoded file. */ - pem_file, -}; - -/** - * An error that occurred when using the line sender. - * - * Call `.what()` to obtain the ASCII-encoded error message. - */ -class line_sender_error : public std::runtime_error -{ -public: - line_sender_error(line_sender_error_code code, const std::string& what) - : std::runtime_error{what} - , _code{code} - { - } - - /** Error code categorizing the error. */ - line_sender_error_code code() const noexcept - { - return _code; - } - -private: - inline static line_sender_error from_c(::line_sender_error* c_err) - { - line_sender_error_code code = static_cast( - static_cast(::line_sender_error_get_code(c_err))); - size_t c_len{0}; - const char* c_msg{::line_sender_error_msg(c_err, &c_len)}; - std::string msg{c_msg, c_len}; - line_sender_error err{code, msg}; - ::line_sender_error_free(c_err); - return err; - } - - template - inline static auto wrapped_call(F&& f, Args&&... args) - { - ::line_sender_error* c_err{nullptr}; - auto obj = f(std::forward(args)..., &c_err); - if (obj) - return obj; - else - throw from_c(c_err); - } - - friend class line_sender; - friend class line_sender_buffer; - friend class opts; - - template < - typename T, - bool (*F)(T*, size_t, const char*, ::line_sender_error**)> - friend class basic_view; - - line_sender_error_code _code; -}; - -/** - * Non-owning validated string. - * - * See `table_name_view`, `column_name_view` and `utf8_view` along with the - * `_utf8`, `_tn` and `_cn` literal suffixes in the `literals` namespace. - */ -template -class basic_view -{ -public: - basic_view(const char* buf, size_t len) - : _impl{0, nullptr} - { - line_sender_error::wrapped_call(F, &_impl, len, buf); - } - - template - basic_view(const char (&buf)[N]) - : basic_view{buf, N - 1} - { - } - - basic_view(std::string_view s_view) - : basic_view{s_view.data(), s_view.size()} - { - } - - basic_view(const std::string& s) - : basic_view{s.data(), s.size()} - { - } - - size_t size() const noexcept - { - return _impl.len; - } - - const char* data() const noexcept - { - return _impl.buf; - } - - std::string_view to_string_view() const noexcept - { - return std::string_view{_impl.buf, _impl.len}; - } - -private: - T _impl; - - friend class line_sender; - friend class line_sender_buffer; - friend class opts; -}; - -using utf8_view = basic_view<::line_sender_utf8, ::line_sender_utf8_init>; - -using table_name_view = - basic_view<::line_sender_table_name, ::line_sender_table_name_init>; - -using column_name_view = - basic_view<::line_sender_column_name, ::line_sender_column_name_init>; - -namespace literals -{ -/** - * Utility to construct `utf8_view` objects from string literals. - * @code {.cpp} - * auto validated = "A UTF-8 encoded string"_utf8; - * @endcode - */ -inline utf8_view operator"" _utf8(const char* buf, size_t len) -{ - return utf8_view{buf, len}; -} - -/** - * Utility to construct `table_name_view` objects from string literals. - * @code {.cpp} - * auto table_name = "events"_tn; - * @endcode - */ -inline table_name_view operator"" _tn(const char* buf, size_t len) -{ - return table_name_view{buf, len}; -} - -/** - * Utility to construct `column_name_view` objects from string literals. - * @code {.cpp} - * auto column_name = "events"_cn; - * @endcode - */ -inline column_name_view operator"" _cn(const char* buf, size_t len) -{ - return column_name_view{buf, len}; -} -} // namespace literals - -class timestamp_micros -{ -public: - template - explicit timestamp_micros(std::chrono::time_point tp) - : _ts{std::chrono::duration_cast( - tp.time_since_epoch()) - .count()} - { - } - - explicit timestamp_micros(int64_t ts) noexcept - : _ts{ts} - { - } - - int64_t as_micros() const noexcept - { - return _ts; - } - - static inline timestamp_micros now() noexcept - { - return timestamp_micros{::line_sender_now_micros()}; - } - -private: - int64_t _ts; -}; - -class timestamp_nanos -{ -public: - template - explicit timestamp_nanos(std::chrono::time_point tp) - : _ts{std::chrono::duration_cast( - tp.time_since_epoch()) - .count()} - { - } - - explicit timestamp_nanos(int64_t ts) noexcept - : _ts{ts} - { - } - - int64_t as_nanos() const noexcept - { - return _ts; - } - - static inline timestamp_nanos now() noexcept - { - return timestamp_nanos{::line_sender_now_nanos()}; - } - -private: - int64_t _ts; -}; - -#if __cplusplus < 202002L -class buffer_view final -{ -public: - /** - * Default constructor. Creates an empty buffer view. - */ - buffer_view() noexcept = default; - - /** - * Construct a buffer view from raw byte data. - * @param data Pointer to the underlying byte array (may be nullptr if - * length=0). - * @param length Number of bytes in the array. - */ - constexpr buffer_view(const std::byte* data, size_t length) noexcept - : buf(data) - , len(length) - { - } - - /** - * Obtain a pointer to the underlying byte array. - * - * @return Const pointer to the data (may be nullptr if empty()). - */ - constexpr const std::byte* data() const noexcept - { - return buf; - } - - /** - * Obtain the number of bytes in the view. - * - * @return Size of the view in bytes. - */ - constexpr size_t size() const noexcept - { - return len; - } - - /** - * Check if the buffer view is empty. - * @return true if the view has no bytes (size() == 0). - */ - constexpr bool empty() const noexcept - { - return len == 0; - } - - /** - * Check byte-wise if two buffer views are equal. - * @return true if both views have the same size and - * the same byte content. - */ - friend bool operator==( - const buffer_view& lhs, const buffer_view& rhs) noexcept - { - return lhs.size() == rhs.size() && - std::equal(lhs.buf, lhs.buf + lhs.len, rhs.buf); - } - -private: - const std::byte* buf{nullptr}; - size_t len{0}; -}; -#endif - -/** - * Types and utilities for working with arbitrary-precision decimal numbers. - * - * Decimals are represented as an unscaled integer value (mantissa) and a scale. - * For example, the decimal "123.45" with scale 2 is represented as: - * - Unscaled value: 12345 - * - Scale: 2 (meaning divide by 10^2 = 100) - * - * QuestDB supports decimal values with: - * - Maximum scale: 76 (QuestDB server limitation) - * - Maximum mantissa size: 127 bytes in binary format - * - * QuestDB server version 9.2.0 or later is required for decimal support. - */ -namespace decimal -{ - -/** - * A validated UTF-8 string view for text-based decimal representation. - * - * This is a wrapper around utf8_view that allows the compiler to distinguish - * between regular strings and decimal strings. - * - * Use this to send decimal values as strings (e.g., "123.456"). - * The string will be parsed by the QuestDB server as a decimal column type. - */ -class decimal_str_view -{ -public: - decimal_str_view(const char* buf, size_t len) - : _view{buf, len} - { - } - - template - decimal_str_view(const char (&buf)[N]) - : _view{buf} - { - } - - decimal_str_view(std::string_view s_view) - : _view{s_view} - { - } - - decimal_str_view(const std::string& s) - : _view{s} - { - } - - const utf8_view& view() const noexcept - { - return _view; - } - -private: - utf8_view _view; - - friend class line_sender_buffer; -}; - -/** - * Literal suffix to construct `decimal_str_view` objects from string literals. - * - * @code {.cpp} - * using namespace questdb::ingress::decimal; - * buffer.column("price"_cn, "123.456"_decimal); - * @endcode - */ -inline decimal_str_view operator"" _decimal(const char* buf, size_t len) -{ - return decimal_str_view{buf, len}; -} - -/** - * A view over a decimal number in binary format. - * - * The decimal is represented as: - * - A scale (number of decimal places) - * - An unscaled value (mantissa) encoded as bytes in two's complement - * big-endian format - * - * # Example - * - * To represent the decimal "123.45": - * - Scale: 2 - * - Unscaled value: 12345 = 0x3039 in big-endian format - * - * ```c++ - * // Represent 123.45 with scale 2 (unscaled value is 12345) - * uint8_t mantissa[] = {0x30, 0x39}; // 12345 in two's complement big-endian - * auto decimal = questdb::ingress::decimal::decimal_view(2, mantissa, - * sizeof(mantissa)); buffer.column("price"_cn, decimal); - * ``` - * - * # Constraints - * - * - Maximum scale: 76 (QuestDB server limitation) - * - Maximum mantissa size: 127 bytes (protocol limitation) - */ -class decimal_view -{ -public: - /** - * Construct a binary decimal view from raw bytes. - * - * @param scale Number of decimal places (must be ≤ 76) - * @param data Pointer to unscaled value in two's complement big-endian - * format - * @param data_size Number of bytes in the mantissa (must be ≤ 127) - */ - decimal_view(uint32_t scale, const uint8_t* data, size_t data_size) - : _scale{scale} - , _data{data} - , _data_size{data_size} - { - } - - /** - * Construct a binary decimal view from a fixed-size array. - * - * @param scale Number of decimal places (must be ≤ 76) - * @param data Fixed-size array containing the unscaled value - */ - template - decimal_view(uint32_t scale, const uint8_t (&data)[N]) - : _scale{scale} - , _data{data} - , _data_size{N} - { - } - - /** - * Construct a binary decimal view from a std::array. - * - * @param scale Number of decimal places (must be ≤ 76) - * @param data std::array containing the unscaled value - */ - template - decimal_view(uint32_t scale, const std::array& data) - : _scale{scale} - , _data{data.data()} - , _data_size{N} - { - } - - /** - * Construct a binary decimal view from a std::vector. - * - * @param scale Number of decimal places (must be ≤ 76) - * @param vec Vector containing the unscaled value - */ - decimal_view(uint32_t scale, const std::vector& vec) - : _scale{scale} - , _data{vec.data()} - , _data_size{vec.size()} - { - } - -#if __cplusplus >= 202002L - /** - * Construct a binary decimal view from a std::span (C++20). - * - * @param scale Number of decimal places (must be ≤ 76) - * @param span Span containing the unscaled value - */ - decimal_view(uint32_t scale, const std::span& span) - : _scale{scale} - , _data{span.data()} - , _data_size{span.size()} - { - } -#endif - - /** Get the scale (number of decimal places). */ - uint32_t scale() const - { - return _scale; - } - - /** Get a pointer to the unscaled value bytes. */ - const uint8_t* data() const - { - return _data; - } - - /** Get the size of the unscaled value in bytes. */ - size_t data_size() const - { - return _data_size; - } - - /** Get a const reference to this view (for customization point - * compatibility). */ - const decimal_view& view() const - { - return *this; - } - -private: - uint32_t _scale; - const uint8_t* _data; - size_t _data_size; -}; -/** - * Customization point to enable serialization of additional types as decimals. - * - * This allows you to support custom decimal types by implementing a conversion - * function. The customized `to_decimal_view_state_impl` for your type can be - * placed in either: - * - The namespace of the type in question (ADL/Koenig lookup) - * - The `questdb::ingress::decimal` namespace - * - * The function can either: - * - Return a `decimal_view` object directly, or - * - Return an object with a `.view()` method that returns `const decimal_view&` - * (useful if you need to store temporary data like shape/strides on the -stack) - */ -struct to_decimal_view_state_fn -{ - template - auto operator()(const T& decimal) const - { - // Implement your own `to_decimal_view_state_impl` as needed. - // ADL lookup for user-defined to_decimal_view_state_impl - return to_decimal_view_state_impl(decimal); - } -}; - -inline constexpr to_decimal_view_state_fn to_decimal_view_state{}; - -template -struct has_decimal_view_state : std::false_type -{ -}; - -template -struct has_decimal_view_state< - T, - std::void_t()))>> - : std::true_type -{ -}; - -template -inline constexpr bool has_decimal_view_state_v = - has_decimal_view_state::value; -} // namespace decimal - -using decimal::decimal_view; - class line_sender_buffer { public: diff --git a/include/questdb/ingress/line_sender_core.hpp b/include/questdb/ingress/line_sender_core.hpp new file mode 100644 index 00000000..8fd3685b --- /dev/null +++ b/include/questdb/ingress/line_sender_core.hpp @@ -0,0 +1,432 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +#pragma once + +#include "line_sender.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#if __cplusplus >= 202002L +# include +#endif + +namespace questdb::ingress +{ +constexpr const char* inaddr_any = "0.0.0.0"; + +class line_sender; +class line_sender_buffer; +class opts; + +/** Category of error. */ +enum class line_sender_error_code +{ + /** The host, port, or interface was incorrect. */ + could_not_resolve_addr, + + /** Called methods in the wrong order. E.g. `symbol` after `column`. */ + invalid_api_call, + + /** A network error connecting or flushing data out. */ + socket_error, + + /** The string or symbol field is not encoded in valid UTF-8. */ + invalid_utf8, + + /** The table name or column name contains bad characters. */ + invalid_name, + + /** The supplied timestamp is invalid. */ + invalid_timestamp, + + /** Error during the authentication process. */ + auth_error, + + /** Error during TLS handshake. */ + tls_error, + + /** The server does not support ILP over HTTP. */ + http_not_supported, + + /** Error sent back from the server during flush. */ + server_flush_error, + + /** Bad configuration. */ + config_error, + + /** There was an error serializing an array. */ + array_error, + + /** Line sender protocol version error. */ + protocol_version_error, + + /** The supplied decimal is invalid. */ + invalid_decimal, +}; + +/** The protocol used to connect with. */ +enum class protocol +{ + /** InfluxDB Line Protocol over TCP. */ + tcp, + + /** InfluxDB Line Protocol over TCP with TLS. */ + tcps, + + /** InfluxDB Line Protocol over HTTP. */ + http, + + /** InfluxDB Line Protocol over HTTP with TLS. */ + https, +}; + +enum class protocol_version +{ + /** InfluxDB Line Protocol v1. */ + v1 = 1, + + /** + * InfluxDB Line Protocol v2. + * QuestDB server version 9.0.0 or later is required for + * `v2` support. + */ + v2 = 2, + + /** + * InfluxDB Line Protocol v3. + * QuestDB server version 9.2.0 or later is required for + * `v3` support. + */ + v3 = 3, +}; + +/* Possible sources of the root certificates used to validate the server's TLS + * certificate. */ +enum class ca +{ + /** Use the set of root certificates provided by the `webpki` crate. */ + webpki_roots, + + /** Use the set of root certificates provided by the operating system. */ + os_roots, + + /** Combine the set of root certificates provided by the `webpki` crate and + * the operating system. */ + webpki_and_os_roots, + + /** Use the root certificates provided in a PEM-encoded file. */ + pem_file, +}; + +/** + * An error that occurred when using the line sender. + * + * Call `.what()` to obtain the ASCII-encoded error message. + */ +class line_sender_error : public std::runtime_error +{ +public: + line_sender_error(line_sender_error_code code, const std::string& what) + : std::runtime_error{what} + , _code{code} + { + } + + /** Error code categorizing the error. */ + line_sender_error_code code() const noexcept + { + return _code; + } + +private: + inline static line_sender_error from_c(::line_sender_error* c_err) + { + line_sender_error_code code = static_cast( + static_cast(::line_sender_error_get_code(c_err))); + size_t c_len{0}; + const char* c_msg{::line_sender_error_msg(c_err, &c_len)}; + std::string msg{c_msg, c_len}; + line_sender_error err{code, msg}; + ::line_sender_error_free(c_err); + return err; + } + + template + inline static auto wrapped_call(F&& f, Args&&... args) + { + ::line_sender_error* c_err{nullptr}; + auto obj = f(std::forward(args)..., &c_err); + if (obj) + return obj; + else + throw from_c(c_err); + } + + friend class line_sender; + friend class line_sender_buffer; + friend class opts; + + template < + typename T, + bool (*F)(T*, size_t, const char*, ::line_sender_error**)> + friend class basic_view; + + line_sender_error_code _code; +}; + +/** + * Non-owning validated string. + * + * See `table_name_view`, `column_name_view` and `utf8_view` along with the + * `_utf8`, `_tn` and `_cn` literal suffixes in the `literals` namespace. + */ +template +class basic_view +{ +public: + basic_view(const char* buf, size_t len) + : _impl{0, nullptr} + { + line_sender_error::wrapped_call(F, &_impl, len, buf); + } + + template + basic_view(const char (&buf)[N]) + : basic_view{buf, N - 1} + { + } + + basic_view(std::string_view s_view) + : basic_view{s_view.data(), s_view.size()} + { + } + + basic_view(const std::string& s) + : basic_view{s.data(), s.size()} + { + } + + size_t size() const noexcept + { + return _impl.len; + } + + const char* data() const noexcept + { + return _impl.buf; + } + + std::string_view to_string_view() const noexcept + { + return std::string_view{_impl.buf, _impl.len}; + } + +private: + T _impl; + + friend class line_sender; + friend class line_sender_buffer; + friend class opts; +}; + +using utf8_view = basic_view<::line_sender_utf8, ::line_sender_utf8_init>; + +using table_name_view = + basic_view<::line_sender_table_name, ::line_sender_table_name_init>; + +using column_name_view = + basic_view<::line_sender_column_name, ::line_sender_column_name_init>; + +namespace literals +{ +/** + * Utility to construct `utf8_view` objects from string literals. + * @code {.cpp} + * auto validated = "A UTF-8 encoded string"_utf8; + * @endcode + */ +inline utf8_view operator"" _utf8(const char* buf, size_t len) +{ + return utf8_view{buf, len}; +} + +/** + * Utility to construct `table_name_view` objects from string literals. + * @code {.cpp} + * auto table_name = "events"_tn; + * @endcode + */ +inline table_name_view operator"" _tn(const char* buf, size_t len) +{ + return table_name_view{buf, len}; +} + +/** + * Utility to construct `column_name_view` objects from string literals. + * @code {.cpp} + * auto column_name = "events"_cn; + * @endcode + */ +inline column_name_view operator"" _cn(const char* buf, size_t len) +{ + return column_name_view{buf, len}; +} +} // namespace literals + +class timestamp_micros +{ +public: + template + explicit timestamp_micros(std::chrono::time_point tp) + : _ts{std::chrono::duration_cast( + tp.time_since_epoch()) + .count()} + { + } + + explicit timestamp_micros(int64_t ts) noexcept + : _ts{ts} + { + } + + int64_t as_micros() const noexcept + { + return _ts; + } + + static inline timestamp_micros now() noexcept + { + return timestamp_micros{::line_sender_now_micros()}; + } + +private: + int64_t _ts; +}; + +class timestamp_nanos +{ +public: + template + explicit timestamp_nanos(std::chrono::time_point tp) + : _ts{std::chrono::duration_cast( + tp.time_since_epoch()) + .count()} + { + } + + explicit timestamp_nanos(int64_t ts) noexcept + : _ts{ts} + { + } + + int64_t as_nanos() const noexcept + { + return _ts; + } + + static inline timestamp_nanos now() noexcept + { + return timestamp_nanos{::line_sender_now_nanos()}; + } + +private: + int64_t _ts; +}; + +#if __cplusplus < 202002L +class buffer_view final +{ +public: + /** + * Default constructor. Creates an empty buffer view. + */ + buffer_view() noexcept = default; + + /** + * Construct a buffer view from raw byte data. + * @param data Pointer to the underlying byte array (may be nullptr if + * length=0). + * @param length Number of bytes in the array. + */ + constexpr buffer_view(const std::byte* data, size_t length) noexcept + : buf(data) + , len(length) + { + } + + /** + * Obtain a pointer to the underlying byte array. + * + * @return Const pointer to the data (may be nullptr if empty()). + */ + constexpr const std::byte* data() const noexcept + { + return buf; + } + + /** + * Obtain the number of bytes in the view. + * + * @return Size of the view in bytes. + */ + constexpr size_t size() const noexcept + { + return len; + } + + /** + * Check if the buffer view is empty. + * @return true if the view has no bytes (size() == 0). + */ + constexpr bool empty() const noexcept + { + return len == 0; + } + + /** + * Check byte-wise if two buffer views are equal. + * @return true if both views have the same size and + * the same byte content. + */ + friend bool operator==( + const buffer_view& lhs, const buffer_view& rhs) noexcept + { + return lhs.size() == rhs.size() && + std::equal(lhs.buf, lhs.buf + lhs.len, rhs.buf); + } + +private: + const std::byte* buf{nullptr}; + size_t len{0}; +}; +#endif + +} // namespace questdb::ingress \ No newline at end of file diff --git a/include/questdb/ingress/line_sender_decimal.hpp b/include/questdb/ingress/line_sender_decimal.hpp new file mode 100644 index 00000000..c6501e24 --- /dev/null +++ b/include/questdb/ingress/line_sender_decimal.hpp @@ -0,0 +1,282 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +#pragma once + +#include "line_sender_core.hpp" + +/** + * Types and utilities for working with arbitrary-precision decimal numbers. + * + * Decimals are represented as an unscaled integer value (mantissa) and a scale. + * For example, the decimal "123.45" with scale 2 is represented as: + * - Unscaled value: 12345 + * - Scale: 2 (meaning divide by 10^2 = 100) + * + * QuestDB supports decimal values with: + * - Maximum scale: 76 (QuestDB server limitation) + * - Maximum mantissa size: 127 bytes in binary format + * + * QuestDB server version 9.2.0 or later is required for decimal support. + */ +namespace questdb::ingress::decimal +{ + +/** + * A validated UTF-8 string view for text-based decimal representation. + * + * This is a wrapper around utf8_view that allows the compiler to distinguish + * between regular strings and decimal strings. + * + * Use this to send decimal values as strings (e.g., "123.456"). + * The string will be parsed by the QuestDB server as a decimal column type. + */ +class decimal_str_view +{ +public: + decimal_str_view(const char* buf, size_t len) + : _view{buf, len} + { + } + + template + decimal_str_view(const char (&buf)[N]) + : _view{buf} + { + } + + decimal_str_view(std::string_view s_view) + : _view{s_view} + { + } + + decimal_str_view(const std::string& s) + : _view{s} + { + } + + const utf8_view& view() const noexcept + { + return _view; + } + +private: + utf8_view _view; + + friend class line_sender_buffer; +}; + +/** + * Literal suffix to construct `decimal_str_view` objects from string literals. + * + * @code {.cpp} + * using namespace questdb::ingress::decimal; + * buffer.column("price"_cn, "123.456"_decimal); + * @endcode + */ +inline decimal_str_view operator"" _decimal(const char* buf, size_t len) +{ + return decimal_str_view{buf, len}; +} + +/** + * A view over a decimal number in binary format. + * + * The decimal is represented as: + * - A scale (number of decimal places) + * - An unscaled value (mantissa) encoded as bytes in two's complement + * big-endian format + * + * # Example + * + * To represent the decimal "123.45": + * - Scale: 2 + * - Unscaled value: 12345 = 0x3039 in big-endian format + * + * ```c++ + * // Represent 123.45 with scale 2 (unscaled value is 12345) + * uint8_t mantissa[] = {0x30, 0x39}; // 12345 in two's complement big-endian + * auto decimal = questdb::ingress::decimal::decimal_view(2, mantissa, + * sizeof(mantissa)); buffer.column("price"_cn, decimal); + * ``` + * + * # Constraints + * + * - Maximum scale: 76 (QuestDB server limitation) + * - Maximum mantissa size: 127 bytes (protocol limitation) + */ +class decimal_view +{ +public: + /** + * Construct a binary decimal view from raw bytes. + * + * @param scale Number of decimal places (must be ≤ 76) + * @param data Pointer to unscaled value in two's complement big-endian + * format + * @param data_size Number of bytes in the mantissa (must be ≤ 127) + */ + decimal_view(uint32_t scale, const uint8_t* data, size_t data_size) + : _scale{scale} + , _data{data} + , _data_size{data_size} + { + } + + /** + * Construct a binary decimal view from a fixed-size array. + * + * @param scale Number of decimal places (must be ≤ 76) + * @param data Fixed-size array containing the unscaled value + */ + template + decimal_view(uint32_t scale, const uint8_t (&data)[N]) + : _scale{scale} + , _data{data} + , _data_size{N} + { + } + + /** + * Construct a binary decimal view from a std::array. + * + * @param scale Number of decimal places (must be ≤ 76) + * @param data std::array containing the unscaled value + */ + template + decimal_view(uint32_t scale, const std::array& data) + : _scale{scale} + , _data{data.data()} + , _data_size{N} + { + } + + /** + * Construct a binary decimal view from a std::vector. + * + * @param scale Number of decimal places (must be ≤ 76) + * @param vec Vector containing the unscaled value + */ + decimal_view(uint32_t scale, const std::vector& vec) + : _scale{scale} + , _data{vec.data()} + , _data_size{vec.size()} + { + } + +#if __cplusplus >= 202002L + /** + * Construct a binary decimal view from a std::span (C++20). + * + * @param scale Number of decimal places (must be ≤ 76) + * @param span Span containing the unscaled value + */ + decimal_view(uint32_t scale, const std::span& span) + : _scale{scale} + , _data{span.data()} + , _data_size{span.size()} + { + } +#endif + + /** Get the scale (number of decimal places). */ + uint32_t scale() const + { + return _scale; + } + + /** Get a pointer to the unscaled value bytes. */ + const uint8_t* data() const + { + return _data; + } + + /** Get the size of the unscaled value in bytes. */ + size_t data_size() const + { + return _data_size; + } + + /** Get a const reference to this view (for customization point + * compatibility). */ + const decimal_view& view() const + { + return *this; + } + +private: + uint32_t _scale; + const uint8_t* _data; + size_t _data_size; +}; +/** + * Customization point to enable serialization of additional types as decimals. + * + * This allows you to support custom decimal types by implementing a conversion + * function. The customized `to_decimal_view_state_impl` for your type can be + * placed in either: + * - The namespace of the type in question (ADL/Koenig lookup) + * - The `questdb::ingress::decimal` namespace + * + * The function can either: + * - Return a `decimal_view` object directly, or + * - Return an object with a `.view()` method that returns `const decimal_view&` + * (useful if you need to store temporary data like shape/strides on the +stack) + */ +struct to_decimal_view_state_fn +{ + template + auto operator()(const T& decimal) const + { + // Implement your own `to_decimal_view_state_impl` as needed. + // ADL lookup for user-defined to_decimal_view_state_impl + return to_decimal_view_state_impl(decimal); + } +}; + +inline constexpr to_decimal_view_state_fn to_decimal_view_state{}; + +template +struct has_decimal_view_state : std::false_type +{ +}; + +template +struct has_decimal_view_state< + T, + std::void_t()))>> + : std::true_type +{ +}; + +template +inline constexpr bool has_decimal_view_state_v = + has_decimal_view_state::value; +} // namespace questdb::ingress::decimal + +namespace questdb::ingress +{ +using decimal::decimal_view; +} // namespace questdb::ingress \ No newline at end of file From 17f2427da65907a5d6b04641f1810325693698ab Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 10:36:15 +0200 Subject: [PATCH 52/65] fix: improve error message for unsupported client protocol version --- questdb-rs/src/tests/http.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questdb-rs/src/tests/http.rs b/questdb-rs/src/tests/http.rs index 2822fabf..e7302511 100644 --- a/questdb-rs/src/tests/http.rs +++ b/questdb-rs/src/tests/http.rs @@ -793,7 +793,7 @@ fn test_sender_auto_protocol_version_unsupported_client() -> TestResult { assert_err_contains( sender_builder.build(), ErrorCode::ProtocolVersionError, - "Server does not support current client", + "Server does not support any of the client protocol versions", ); // We keep the server around til the end of the test to ensure that the response is fully received. From 948941401779e9c5a4a1a7c4eb45e313331cbab2 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 11:37:07 +0200 Subject: [PATCH 53/65] fix: clarify implementation details for DecimalSerializer --- questdb-rs-ffi/src/decimal.rs | 1 - questdb-rs/src/ingress/decimal.rs | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/questdb-rs-ffi/src/decimal.rs b/questdb-rs-ffi/src/decimal.rs index a2a4167d..f2a74481 100644 --- a/questdb-rs-ffi/src/decimal.rs +++ b/questdb-rs-ffi/src/decimal.rs @@ -95,7 +95,6 @@ impl<'a> DecimalSerializer for Decimal<'a> { /// # Arguments /// /// * `out` - The output buffer to write the serialized decimal to - /// * `support_binary` - Whether the protocol version supports binary encoding /// /// # Errors /// diff --git a/questdb-rs/src/ingress/decimal.rs b/questdb-rs/src/ingress/decimal.rs index 66f685ea..e85a8152 100644 --- a/questdb-rs/src/ingress/decimal.rs +++ b/questdb-rs/src/ingress/decimal.rs @@ -75,8 +75,7 @@ pub trait DecimalSerializer { /// Implementation for string slices containing decimal representations. /// -/// This implementation always uses the text format, regardless of the `support_binary` parameter, -/// as it cannot parse the string to extract scale and unscaled value needed for binary encoding. +/// This implementation uses the text format. /// /// # Format /// The string is validated and written as-is, followed by the 'd' suffix. Thousand separators From ba6744c5ea0d3ac434cc6b5e3fb17ac3ba5b63fa Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 15:46:22 +0200 Subject: [PATCH 54/65] fix: correct wording in protocol version documentation for clarity --- questdb-rs-ffi/src/lib.rs | 4 ++-- questdb-rs/src/ingress/mod.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index c05ce3f6..ececf121 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -311,13 +311,13 @@ pub enum ProtocolVersion { /// Version 2 of InfluxDB Line Protocol. /// Uses binary format serialization for f64, and supports the array data type. /// This version is specific to QuestDB and is not compatible with InfluxDB. - /// QuestDB server version 9.0.0 or later is required for `V2` supported. + /// QuestDB server version 9.0.0 or later is required for `V2` support. V2 = 2, /// Version 3 of InfluxDB Line Protocol. /// Supports the decimal data type in text and binary formats. /// This version is specific to QuestDB and is not compatible with InfluxDB. - /// QuestDB server version 9.2.0 or later is required for `V3` supported. + /// QuestDB server version 9.2.0 or later is required for `V3` support. V3 = 3, } diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 5db29e9e..c76b4f72 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -88,13 +88,13 @@ pub enum ProtocolVersion { /// Version 2 of InfluxDB Line Protocol. /// Uses binary format serialization for f64, and supports the array data type. /// This version is specific to QuestDB and is not compatible with InfluxDB. - /// QuestDB server version 9.0.0 or later is required for `V2` supported. + /// QuestDB server version 9.0.0 or later is required for `V2` support. V2 = 2, /// Version 3 of InfluxDB Line Protocol. /// Supports the decimal data type in text and binary formats. /// This version is specific to QuestDB and is not compatible with InfluxDB. - /// QuestDB server version 9.2.0 or later is required for `V3` supported. + /// QuestDB server version 9.2.0 or later is required for `V3` support. V3 = 3, } From 61f16a7f212b9d2ab0bb9be56e82813f482dc967 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 15:47:37 +0200 Subject: [PATCH 55/65] fix: improve error messages for decimal scale and value length constraints --- questdb-rs-ffi/src/decimal.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/questdb-rs-ffi/src/decimal.rs b/questdb-rs-ffi/src/decimal.rs index f2a74481..d9b62b24 100644 --- a/questdb-rs-ffi/src/decimal.rs +++ b/questdb-rs-ffi/src/decimal.rs @@ -107,7 +107,7 @@ impl<'a> DecimalSerializer for Decimal<'a> { if self.scale > 76 { return Err(fmt_error!( InvalidDecimal, - "QuestDB ILP does not support scale greater than 76, got {}", + "QuestDB ILP does not support decimal scale greater than 76, got {}", self.scale )); } @@ -121,7 +121,7 @@ impl<'a> DecimalSerializer for Decimal<'a> { if self.value.len() > i8::MAX as usize { return Err(fmt_error!( InvalidDecimal, - "QuestDB ILP does not support values greater than {} bytes, got {}", + "QuestDB ILP does not support decimal longer than {} bytes, got {}", i8::MAX, self.value.len() )); From 61ff7ec791a7a72256ddfadfcff0b52335480759 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 15:48:16 +0200 Subject: [PATCH 56/65] fix: correct order of fields in binary serialization format documentation --- questdb-rs-ffi/src/decimal.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questdb-rs-ffi/src/decimal.rs b/questdb-rs-ffi/src/decimal.rs index d9b62b24..1a8e44e9 100644 --- a/questdb-rs-ffi/src/decimal.rs +++ b/questdb-rs-ffi/src/decimal.rs @@ -45,7 +45,7 @@ use crate::fmt_error; /// /// The binary serialization format is: /// ```text -/// '=' marker (1 byte) + type ID (1 byte) + length (1 byte) + value bytes + scale (1 byte) +/// '=' marker (1 byte) + type ID (1 byte) + scale (1 byte) + length (1 byte) + value bytes /// ``` /// /// # Constraints From b2cedfd5dc261d63622e6c52ed33fe5a83480c23 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 16:28:24 +0200 Subject: [PATCH 57/65] fix: handle null data in line_sender_buffer_column_dec function --- questdb-rs-ffi/src/lib.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index ececf121..ed73e9fa 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -1044,9 +1044,14 @@ pub unsafe extern "C" fn line_sender_buffer_column_dec( err_out: *mut *mut line_sender_error, ) -> bool { unsafe { + let data = if data.is_null() { + &[] + } else { + slice::from_raw_parts(data, data_len) + }; let buffer = unwrap_buffer_mut(buffer); let name = name.as_name(); - let decimal = Decimal::new(scale, slice::from_raw_parts(data, data_len)); + let decimal = Decimal::new(scale, data); bubble_err_to_c!(err_out, buffer.column_dec(name, decimal)); } true From 1ede6d73031da5053c76f0bee453fb5cb6eadf38 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 16:29:11 +0200 Subject: [PATCH 58/65] fix: remove debug output for protocol version in SenderBuilder --- questdb-rs/src/ingress/mod.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index c76b4f72..b05ccd67 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -1183,10 +1183,6 @@ impl SenderBuilder { } }, }; - eprintln!( - "Using protocol version {:?} with max_name_len={}", - protocol_version, max_name_len - ); if auth.is_some() { descr.push_str("auth=on]"); From e86a18d482c01b4e19ce8d3a5f0c67f52a600cd5 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 17:10:18 +0200 Subject: [PATCH 59/65] fix: update usage message to include program name in help output --- examples/line_sender_cpp_example_decimal_binary.cpp | 2 +- examples/line_sender_cpp_example_decimal_custom.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/line_sender_cpp_example_decimal_binary.cpp b/examples/line_sender_cpp_example_decimal_binary.cpp index dcf9ee60..7008a607 100644 --- a/examples/line_sender_cpp_example_decimal_binary.cpp +++ b/examples/line_sender_cpp_example_decimal_binary.cpp @@ -58,7 +58,7 @@ static bool displayed_help(int argc, const char* argv[]) if ((arg == "-h"sv) || (arg == "--help"sv)) { std::cerr << "Usage:\n" - << "line_sender_c_example: [HOST [PORT]]\n" + << " " << argv[0] << ": [HOST [PORT]]\n" << " HOST: ILP host (defaults to \"localhost\").\n" << " PORT: ILP port (defaults to \"9009\")." << std::endl; diff --git a/examples/line_sender_cpp_example_decimal_custom.cpp b/examples/line_sender_cpp_example_decimal_custom.cpp index d6186555..52f8cc73 100644 --- a/examples/line_sender_cpp_example_decimal_custom.cpp +++ b/examples/line_sender_cpp_example_decimal_custom.cpp @@ -112,7 +112,7 @@ static bool displayed_help(int argc, const char* argv[]) if ((arg == "-h"sv) || (arg == "--help"sv)) { std::cerr << "Usage:\n" - << "line_sender_c_example_decimal_custom: [HOST [PORT]]\n" + << " " << argv[0] << ": [HOST [PORT]]\n" << " HOST: ILP host (defaults to \"localhost\").\n" << " PORT: ILP port (defaults to \"9009\")." << std::endl; From 5b177158a96a69144dccd168aadc55a4878f813d Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 17:36:28 +0200 Subject: [PATCH 60/65] fix: use http instead of tcp in decimal examples --- CMakeLists.txt | 4 ++++ .../line_sender_c_example_decimal_binary.c | 14 ++++++++------ .../line_sender_cpp_example_decimal_binary.cpp | 18 +++++++++--------- .../line_sender_cpp_example_decimal_custom.cpp | 9 ++++----- examples/line_sender_cpp_example_http.cpp | 13 +++++++------ 5 files changed, 32 insertions(+), 26 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 59f06b14..e86e5ce8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -132,6 +132,10 @@ if (QUESTDB_TESTS_AND_EXAMPLES) compile_example( line_sender_c_example_from_env examples/line_sender_c_example_from_env.c) + compile_example( + line_sender_c_example_decimal_binary + examples/concat.c + examples/line_sender_c_example_decimal_binary.c) compile_example( line_sender_cpp_example examples/line_sender_cpp_example.cpp) diff --git a/examples/line_sender_c_example_decimal_binary.c b/examples/line_sender_c_example_decimal_binary.c index d2b87f2b..a4f325c4 100644 --- a/examples/line_sender_c_example_decimal_binary.c +++ b/examples/line_sender_c_example_decimal_binary.c @@ -9,8 +9,7 @@ static bool example(const char* host, const char* port) line_sender_error* err = NULL; line_sender* sender = NULL; line_sender_buffer* buffer = NULL; - char* conf_str = - concat("tcp::addr=", host, ":", port, ";protocol_version=3;"); + char* conf_str = concat("http::addr=", host, ":", port, ";"); if (!conf_str) { fprintf(stderr, "Could not concatenate configuration string.\n"); @@ -34,7 +33,8 @@ static bool example(const char* host, const char* port) // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid // re-validating the same strings over and over again. - line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("c_trades"); + line_sender_table_name table_name = + QDB_TABLE_NAME_LITERAL("c_trades_decimal"); line_sender_column_name symbol_name = QDB_COLUMN_NAME_LITERAL("symbol"); line_sender_column_name side_name = QDB_COLUMN_NAME_LITERAL("side"); line_sender_column_name price_name = QDB_COLUMN_NAME_LITERAL("price"); @@ -103,8 +103,10 @@ static bool displayed_help(int argc, const char* argv[]) stderr, "line_sender_c_example_decimal_binary: [HOST [PORT]]\n"); fprintf( - stderr, " HOST: ILP host (defaults to \"localhost\").\n"); - fprintf(stderr, " PORT: ILP port (defaults to \"9009\").\n"); + stderr, + " HOST: ILP/HTTP host (defaults to \"localhost\").\n"); + fprintf( + stderr, " PORT: ILP/HTTP port (defaults to \"9000\").\n"); return true; } } @@ -119,7 +121,7 @@ int main(int argc, const char* argv[]) const char* host = "localhost"; if (argc >= 2) host = argv[1]; - const char* port = "9009"; + const char* port = "9000"; if (argc >= 3) port = argv[2]; diff --git a/examples/line_sender_cpp_example_decimal_binary.cpp b/examples/line_sender_cpp_example_decimal_binary.cpp index 7008a607..7115265d 100644 --- a/examples/line_sender_cpp_example_decimal_binary.cpp +++ b/examples/line_sender_cpp_example_decimal_binary.cpp @@ -9,13 +9,12 @@ static bool example(std::string_view host, std::string_view port) try { auto sender = questdb::ingress::line_sender::from_conf( - "tcp::addr=" + std::string{host} + ":" + std::string{port} + - ";protocol_version=3;"); + "http::addr=" + std::string{host} + ":" + std::string{port} + ";"); // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid // re-validating the same strings over and over again. - const auto table_name = "cpp_trades"_tn; + const auto table_name = "cpp_trades_decimal"_tn; const auto symbol_name = "symbol"_cn; const auto side_name = "side"_cn; const auto price_name = "price"_cn; @@ -57,11 +56,12 @@ static bool displayed_help(int argc, const char* argv[]) const std::string_view arg{argv[index]}; if ((arg == "-h"sv) || (arg == "--help"sv)) { - std::cerr << "Usage:\n" - << " " << argv[0] << ": [HOST [PORT]]\n" - << " HOST: ILP host (defaults to \"localhost\").\n" - << " PORT: ILP port (defaults to \"9009\")." - << std::endl; + std::cerr + << "Usage:\n" + << " " << argv[0] << ": [HOST [PORT]]\n" + << " HOST: ILP/HTTP host (defaults to \"localhost\").\n" + << " PORT: ILP/HTTP port (defaults to \"9000\")." + << std::endl; return true; } } @@ -76,7 +76,7 @@ int main(int argc, const char* argv[]) auto host = "localhost"sv; if (argc >= 2) host = std::string_view{argv[1]}; - auto port = "9009"sv; + auto port = "9000"sv; if (argc >= 3) port = std::string_view{argv[2]}; diff --git a/examples/line_sender_cpp_example_decimal_custom.cpp b/examples/line_sender_cpp_example_decimal_custom.cpp index 52f8cc73..b71931ea 100644 --- a/examples/line_sender_cpp_example_decimal_custom.cpp +++ b/examples/line_sender_cpp_example_decimal_custom.cpp @@ -64,13 +64,12 @@ static bool example(std::string_view host, std::string_view port) try { auto sender = questdb::ingress::line_sender::from_conf( - "tcp::addr=" + std::string{host} + ":" + std::string{port} + - ";protocol_version=3;"); + "http::addr=" + std::string{host} + ":" + std::string{port} + ";"); // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid // re-validating the same strings over and over again. - const auto table_name = "cpp_trades"_tn; + const auto table_name = "cpp_trades_decimal"_tn; const auto symbol_name = "symbol"_cn; const auto side_name = "side"_cn; const auto price_name = "price"_cn; @@ -114,7 +113,7 @@ static bool displayed_help(int argc, const char* argv[]) std::cerr << "Usage:\n" << " " << argv[0] << ": [HOST [PORT]]\n" << " HOST: ILP host (defaults to \"localhost\").\n" - << " PORT: ILP port (defaults to \"9009\")." + << " PORT: ILP port (defaults to \"9000\")." << std::endl; return true; } @@ -130,7 +129,7 @@ int main(int argc, const char* argv[]) auto host = "localhost"sv; if (argc >= 2) host = std::string_view{argv[1]}; - auto port = "9009"sv; + auto port = "9000"sv; if (argc >= 3) port = std::string_view{argv[2]}; diff --git a/examples/line_sender_cpp_example_http.cpp b/examples/line_sender_cpp_example_http.cpp index 9d4bfe2f..80969151 100644 --- a/examples/line_sender_cpp_example_http.cpp +++ b/examples/line_sender_cpp_example_http.cpp @@ -53,11 +53,12 @@ static bool displayed_help(int argc, const char* argv[]) const std::string_view arg{argv[index]}; if ((arg == "-h"sv) || (arg == "--help"sv)) { - std::cerr << "Usage:\n" - << "line_sender_c_example: [HOST [PORT]]\n" - << " HOST: ILP host (defaults to \"localhost\").\n" - << " PORT: ILP port (defaults to \"9009\")." - << std::endl; + std::cerr + << "Usage:\n" + << " " << argv[0] << ": [HOST [PORT]]\n" + << " HOST: ILP/HTTP host (defaults to \"localhost\").\n" + << " PORT: ILP/HTTP port (defaults to \"9000\")." + << std::endl; return true; } } @@ -72,7 +73,7 @@ int main(int argc, const char* argv[]) auto host = "localhost"sv; if (argc >= 2) host = std::string_view{argv[1]}; - auto port = "9009"sv; + auto port = "9000"sv; if (argc >= 3) port = std::string_view{argv[2]}; From 76104aac13bca81d41929fc4dc8d8858b2d69fc5 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Tue, 21 Oct 2025 10:48:19 +0200 Subject: [PATCH 61/65] refactor: replace DecimalSerializer with DecimalView --- questdb-rs-ffi/src/decimal.rs | 136 -------------------- questdb-rs-ffi/src/lib.rs | 5 +- questdb-rs/src/ingress/buffer.rs | 8 +- questdb-rs/src/ingress/decimal.rs | 202 ++++++++++++++++++------------ questdb-rs/src/ingress/mod.rs | 2 +- questdb-rs/src/tests/decimal.rs | 121 +++++++++++------- 6 files changed, 205 insertions(+), 269 deletions(-) delete mode 100644 questdb-rs-ffi/src/decimal.rs diff --git a/questdb-rs-ffi/src/decimal.rs b/questdb-rs-ffi/src/decimal.rs deleted file mode 100644 index 1a8e44e9..00000000 --- a/questdb-rs-ffi/src/decimal.rs +++ /dev/null @@ -1,136 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2025 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - -//! FFI-specific decimal number serialization for QuestDB ILP. -//! -//! This module provides decimal serialization support for the C FFI bindings. -//! Decimals are represented as arbitrary-precision numbers with a scale factor, -//! encoded in a binary format for transmission over the InfluxDB Line Protocol (ILP). - -use questdb::{ - Result, - ingress::{DECIMAL_BINARY_FORMAT_TYPE, DecimalSerializer}, -}; - -use crate::fmt_error; - -/// Represents a decimal number for binary serialization to QuestDB. -/// -/// A decimal consists of: -/// - An unscaled integer value (the mantissa), represented as raw bytes in big-endian format -/// - A scale indicating how many decimal places to shift (e.g., scale=2 means value/100) -/// -/// # Wire Format -/// -/// The binary serialization format is: -/// ```text -/// '=' marker (1 byte) + type ID (1 byte) + scale (1 byte) + length (1 byte) + value bytes -/// ``` -/// -/// # Constraints -/// -/// - Maximum scale: 76 (QuestDB server limitation) -/// - Maximum value size: 127 bytes (i8::MAX limitation from length field) -/// -/// # Example -/// -/// To represent the decimal `123.45` with scale 2: -/// - scale = 2 -/// - value = 12345 encoded as bytes [0x30, 0x39] (big-endian) -pub(super) struct Decimal<'a> { - /// The number of decimal places to shift. - /// For example, scale=2 means the value represents hundredths (divide by 100). - scale: u32, - - /// The unscaled integer value as raw bytes in big-endian format. - /// This represents the mantissa of the decimal number. - value: &'a [u8], -} - -impl<'a> Decimal<'a> { - /// Creates a new decimal number. - /// - /// # Arguments - /// - /// * `scale` - The number of decimal places (must be ≤ 76) - /// * `value` - The unscaled value as bytes in big-endian format (must be ≤ 127 bytes) - pub(super) fn new(scale: u32, value: &'a [u8]) -> Self { - Self { scale, value } - } -} - -impl<'a> DecimalSerializer for Decimal<'a> { - /// Serializes the decimal value into the QuestDB ILP binary format. - /// - /// # Wire Format Layout - /// - /// The serialization produces the following byte sequence: - /// 1. `'='` (0x3D) - Binary encoding marker - /// 2. Type ID (23) - Identifies this as a decimal type - /// 3. Scale byte - Number of decimal places - /// 4. Length byte - Number of bytes in the value (max 127) - /// 5. Value bytes - The unscaled integer in big-endian format - /// - /// # Arguments - /// - /// * `out` - The output buffer to write the serialized decimal to - /// - /// # Errors - /// - /// Returns an error if: - /// - Scale exceeds 76 (QuestDB server maximum) - /// - Value size exceeds 127 bytes (protocol limitation) - fn serialize(self, out: &mut Vec) -> Result<()> { - // Validate scale constraint (QuestDB server limitation) - // The server's decimal implementation supports a maximum scale of 76 - if self.scale > 76 { - return Err(fmt_error!( - InvalidDecimal, - "QuestDB ILP does not support decimal scale greater than 76, got {}", - self.scale - )); - } - - // Write binary format header - out.push(b'='); // Binary encoding marker - out.push(DECIMAL_BINARY_FORMAT_TYPE); // Type ID = 23 - - // Validate value size constraint (protocol limitation) - // The length field is a single byte (i8), limiting value size to 127 bytes - if self.value.len() > i8::MAX as usize { - return Err(fmt_error!( - InvalidDecimal, - "QuestDB ILP does not support decimal longer than {} bytes, got {}", - i8::MAX, - self.value.len() - )); - } - - out.push(self.scale as u8); - out.push(self.value.len() as u8); - out.extend_from_slice(self.value); - - Ok(()) - } -} diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index ed73e9fa..d7b76c98 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -25,6 +25,7 @@ #![allow(non_camel_case_types, clippy::missing_safety_doc)] use libc::{c_char, size_t}; +use questdb::ingress::DecimalView; use std::ascii; use std::boxed::Box; use std::convert::{From, Into}; @@ -43,7 +44,6 @@ use questdb::{ mod ndarr; use ndarr::StrideArrayView; -mod decimal; macro_rules! bubble_err_to_c { ($err_out:expr, $expression:expr) => { @@ -1051,7 +1051,7 @@ pub unsafe extern "C" fn line_sender_buffer_column_dec( }; let buffer = unwrap_buffer_mut(buffer); let name = name.as_name(); - let decimal = Decimal::new(scale, data); + let decimal = bubble_err_to_c!(err_out, DecimalView::try_new_scaled(scale, data)); bubble_err_to_c!(err_out, buffer.column_dec(name, decimal)); } true @@ -1924,7 +1924,6 @@ pub unsafe extern "C" fn line_sender_now_micros() -> i64 { TimestampMicros::now().as_i64() } -use crate::decimal::Decimal; use crate::ndarr::CMajorArrayView; #[cfg(feature = "confstr-ffi")] use questdb_confstr_ffi::questdb_conf_str_parse_err; diff --git a/questdb-rs/src/ingress/buffer.rs b/questdb-rs/src/ingress/buffer.rs index 3ae3a30c..5cee2497 100644 --- a/questdb-rs/src/ingress/buffer.rs +++ b/questdb-rs/src/ingress/buffer.rs @@ -22,7 +22,7 @@ * ******************************************************************************/ -use crate::ingress::decimal::DecimalSerializer; +use crate::ingress::decimal::DecimalView; use crate::ingress::ndarr::{ArrayElementSealed, check_and_get_array_bytes_size}; use crate::ingress::{ ARRAY_BINARY_FORMAT_TYPE, ArrayElement, DOUBLE_BINARY_FORMAT_TYPE, DebugBytes, MAX_ARRAY_DIMS, @@ -1052,8 +1052,9 @@ impl Buffer { pub fn column_dec<'a, N, S>(&mut self, name: N, value: S) -> crate::Result<&mut Self> where N: TryInto>, - S: DecimalSerializer, + S: TryInto>, Error: From, + Error: From, { if self.protocol_version < ProtocolVersion::V3 { return Err(error::fmt!( @@ -1063,8 +1064,9 @@ impl Buffer { )); } + let value: DecimalView = value.try_into()?; self.write_column_key(name)?; - value.serialize(&mut self.output)?; + value.serialize(&mut self.output); Ok(self) } diff --git a/questdb-rs/src/ingress/decimal.rs b/questdb-rs/src/ingress/decimal.rs index e85a8152..4edc1371 100644 --- a/questdb-rs/src/ingress/decimal.rs +++ b/questdb-rs/src/ingress/decimal.rs @@ -23,12 +23,13 @@ ******************************************************************************/ use crate::{Result, error}; +use std::borrow::Cow; -/// Trait for types that can be serialized as decimal values in the InfluxDB Line Protocol (ILP). +/// A decimal value backed by either a string representation or a scaled mantissa. /// /// Decimal values can be serialized in two formats: /// -/// # Text Format +/// ### Text Format /// The decimal is written as a string representation followed by a `'d'` suffix. /// /// Example: `"123.45d"` or `"1.5e-3d"` @@ -38,7 +39,7 @@ use crate::{Result, error}; /// - Append the `'d'` suffix /// - Ensure no ILP reserved characters are present (space, comma, equals, newline, carriage return, backslash) /// -/// # Binary Format +/// ### Binary Format /// A more compact binary encoding consisting of: /// /// 1. Binary format marker: `'='` (0x3D) @@ -60,17 +61,87 @@ use crate::{Result, error}; /// └─ Binary marker: '=' /// ``` /// -/// # Binary Format Notes +/// #### Binary Format Notes /// - The unscaled value must be encoded in two's complement big-endian format /// - Maximum scale is 76 /// - Length byte indicates how many bytes follow for the unscaled value -pub trait DecimalSerializer { - /// Serialize this value as a decimal in ILP format. - /// - /// # Parameters - /// - /// * `out` - The output buffer to write the serialized decimal to - fn serialize(self, out: &mut Vec) -> Result<()>; +#[derive(Debug)] +pub enum DecimalView<'a> { + String { value: &'a str }, + Scaled { scale: u8, value: Cow<'a, [u8]> }, +} + +impl<'a> DecimalView<'a> { + pub fn try_new_scaled(scale: u32, value: T) -> Result + where + T: Into>, + { + if scale > 76 { + return Err(error::fmt!( + InvalidDecimal, + "QuestDB ILP does not support decimal scale greater than 76, got {}", + scale + )); + } + let value: Cow<'a, [u8]> = value.into(); + if value.len() > i8::MAX as usize { + return Err(error::fmt!( + InvalidDecimal, + "QuestDB ILP does not support decimal longer than {} bytes, got {}", + i8::MAX, + value.len() + )); + } + Ok(DecimalView::Scaled { + scale: scale as u8, + value, + }) + } + + pub fn try_new_string(value: &'a str) -> Result { + // Basic validation: ensure no ILP reserved characters are present + for b in value.bytes() { + match b { + b' ' | b',' | b'=' | b'\n' | b'\r' | b'\\' => { + return Err(error::fmt!( + InvalidDecimal, + "Decimal string contains ILP reserved character {:?}", + b as char + )); + } + _ => {} + } + } + Ok(DecimalView::String { value }) + } + + pub(crate) fn serialize(&self, out: &mut Vec) { + match self { + DecimalView::String { value } => Self::serialize_string(value, out), + DecimalView::Scaled { scale, value } => { + Self::serialize_scaled(*scale, value.as_ref(), out) + } + } + } + + fn serialize_string(value: &str, out: &mut Vec) { + // Pre-allocate space for the string content plus the 'd' suffix + out.reserve(value.len() + 1); + + out.extend_from_slice(value.as_bytes()); + + // Append the 'd' suffix to mark this as a decimal value + out.push(b'd'); + } + + fn serialize_scaled(scale: u8, value: &[u8], out: &mut Vec) { + // Write binary format: '=' marker + type + scale + length + mantissa bytes + out.push(b'='); + out.push(crate::ingress::DECIMAL_BINARY_FORMAT_TYPE); + out.push(scale); + out.push(value.len() as u8); + out.extend_from_slice(value); + } } /// Implementation for string slices containing decimal representations. @@ -97,72 +168,31 @@ pub trait DecimalSerializer { /// # Errors /// Returns [`Error`] with [`ErrorCode::InvalidDecimal`](crate::error::ErrorCode::InvalidDecimal) /// if the string contains non-numerical characters. -impl DecimalSerializer for &str { - fn serialize(self, out: &mut Vec) -> Result<()> { - // Pre-allocate space for the string content plus the 'd' suffix - out.reserve(self.len() + 1); +impl<'a> TryInto> for &'a str { + type Error = crate::Error; - // Validate and copy each byte, rejecting non-numeric characters - for b in self.bytes() { - match b { - b'0'..=b'9' | b'.' | b'-' | b'+' | b'a'..=b'z' | b'A'..=b'Z' => out.push(b), - _ => { - return Err(error::fmt!( - InvalidDecimal, - "Invalid character {:?} in decimal str", - b as char - )); - } - } - } - - // Append the 'd' suffix to mark this as a decimal value - out.push(b'd'); - - Ok(()) + fn try_into(self) -> Result> { + DecimalView::try_new_string(self) } } -#[cfg(any(feature = "rust_decimal", feature = "bigdecimal"))] -use crate::ingress::DECIMAL_BINARY_FORMAT_TYPE; - #[cfg(feature = "rust_decimal")] -impl DecimalSerializer for &rust_decimal::Decimal { - fn serialize(self, out: &mut Vec) -> Result<()> { - // Binary format: '=' marker + type + scale + length + mantissa bytes - out.push(b'='); - out.push(DECIMAL_BINARY_FORMAT_TYPE); - - // rust_decimal::Decimal guarantees: - // - MAX_SCALE is 28, which is within QuestDB's limit of 76 - // - Mantissa is always 96 bits (12 bytes), never exceeds this size - - out.push(self.scale() as u8); - - // We skip the upper 3 bytes (which are sign-extended) and write the lower 13 bytes - let mantissa = self.mantissa(); - out.push(13); - out.extend_from_slice(&mantissa.to_be_bytes()[3..]); +impl<'a> TryInto> for &'a rust_decimal::Decimal { + type Error = crate::Error; - Ok(()) + fn try_into(self) -> Result> { + let raw = self.mantissa().to_be_bytes(); + let bytes = trim_leading_sign_bytes(&raw); + DecimalView::try_new_scaled(self.scale() as u32, bytes) } } #[cfg(feature = "bigdecimal")] -impl DecimalSerializer for &bigdecimal::BigDecimal { - fn serialize(self, out: &mut Vec) -> Result<()> { - // Binary format: '=' marker + type + scale + length + mantissa bytes - out.push(b'='); - out.push(DECIMAL_BINARY_FORMAT_TYPE); +impl<'a> TryInto> for &'a bigdecimal::BigDecimal { + type Error = crate::Error; + fn try_into(self) -> Result> { let (unscaled, mut scale) = self.as_bigint_and_scale(); - if scale < -76 || scale > 76 { - return Err(error::fmt!( - InvalidDecimal, - "QuestDB ILP does not support decimal scale greater than 76, got {}", - scale - )); - } // QuestDB binary ILP doesn't support negative scale, we need to upscale the // unscaled value to be compliant @@ -176,21 +206,37 @@ impl DecimalSerializer for &bigdecimal::BigDecimal { unscaled.to_signed_bytes_be() }; - if bytes.len() > i8::MAX as usize { - return Err(error::fmt!( - InvalidDecimal, - "QuestDB ILP does not support decimal longer than {} bytes, got {}", - i8::MAX, - bytes.len() - )); - } + let bytes = trim_leading_sign_bytes(&bytes); + + DecimalView::try_new_scaled(scale as u32, bytes) + } +} + +#[cfg(any(feature = "rust_decimal", feature = "bigdecimal"))] +fn trim_leading_sign_bytes(bytes: &[u8]) -> Vec { + if bytes.is_empty() { + return vec![0]; + } + + let negative = bytes[0] & 0x80 != 0; + let mut keep_from = 0usize; - out.push(scale as u8); + while keep_from < bytes.len() - 1 { + let current = bytes[keep_from]; + let next = bytes[keep_from + 1]; - // Write length byte and mantissa bytes - out.push(bytes.len() as u8); - out.extend_from_slice(&bytes); + let should_trim = if negative { + current == 0xFF && (next & 0x80) == 0x80 + } else { + current == 0x00 && (next & 0x80) == 0x00 + }; - Ok(()) + if should_trim { + keep_from += 1; + } else { + break; + } } + + bytes[keep_from..].to_vec() } diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index b05ccd67..d8b73b89 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -63,7 +63,7 @@ mod sender; pub use sender::*; mod decimal; -pub use decimal::DecimalSerializer; +pub use decimal::DecimalView; const MAX_NAME_LEN_DEFAULT: usize = 127; diff --git a/questdb-rs/src/tests/decimal.rs b/questdb-rs/src/tests/decimal.rs index e8752e66..2cf14ff9 100644 --- a/questdb-rs/src/tests/decimal.rs +++ b/questdb-rs/src/tests/decimal.rs @@ -23,15 +23,14 @@ ******************************************************************************/ use crate::ErrorCode; -use crate::ingress::{Buffer, DecimalSerializer, ProtocolVersion}; +use crate::ingress::{Buffer, DecimalView, ProtocolVersion}; use crate::tests::{TestResult, assert_err_contains}; use rstest::rstest; -// Helper function to serialize a decimal value and return the bytes -fn serialize_decimal(value: D) -> crate::Result> { +fn serialize_decimal(decimal: DecimalView) -> Vec { let mut out = Vec::new(); - value.serialize(&mut out)?; - Ok(out) + decimal.serialize(&mut out); + out } // ============================================================================ @@ -40,78 +39,84 @@ fn serialize_decimal(value: D) -> crate::Result> { #[test] fn test_str_positive_decimal() -> TestResult { - let result = serialize_decimal("123.45")?; + let decimal = DecimalView::try_new_string("123.45")?; + let result = serialize_decimal(decimal); assert_eq!(result, b"123.45d"); Ok(()) } #[test] fn test_str_negative_decimal() -> TestResult { - let result = serialize_decimal("-123.45")?; + let decimal = DecimalView::try_new_string("-123.45")?; + let result = serialize_decimal(decimal); assert_eq!(result, b"-123.45d"); Ok(()) } #[test] fn test_str_zero() -> TestResult { - let result = serialize_decimal("0")?; + let decimal = DecimalView::try_new_string("0")?; + let result = serialize_decimal(decimal); assert_eq!(result, b"0d"); Ok(()) } #[test] fn test_str_scientific_notation() -> TestResult { - let result = serialize_decimal("1.5e-3")?; + let decimal = DecimalView::try_new_string("1.5e-3")?; + let result = serialize_decimal(decimal); assert_eq!(result, b"1.5e-3d"); Ok(()) } #[test] fn test_str_large_decimal() -> TestResult { - let result = serialize_decimal("999999999999999999.123456789")?; + let decimal = DecimalView::try_new_string("999999999999999999.123456789")?; + let result = serialize_decimal(decimal); assert_eq!(result, b"999999999999999999.123456789d"); Ok(()) } #[test] fn test_str_with_leading_zero() -> TestResult { - let result = serialize_decimal("0.001")?; + let decimal = DecimalView::try_new_string("0.001")?; + let result = serialize_decimal(decimal); assert_eq!(result, b"0.001d"); Ok(()) } #[test] fn test_str_rejects_space() -> TestResult { - let result = serialize_decimal("12 3.45"); - assert_err_contains(result, ErrorCode::InvalidDecimal, "Invalid character"); + let result = DecimalView::try_new_string("12 3.45"); + assert_err_contains(result, ErrorCode::InvalidDecimal, "reserved character"); Ok(()) } #[test] fn test_str_rejects_comma() -> TestResult { - let result = serialize_decimal("1,234.56"); - assert_err_contains(result, ErrorCode::InvalidDecimal, "Invalid character"); + let result = DecimalView::try_new_string("1,234.56"); + assert_err_contains(result, ErrorCode::InvalidDecimal, "reserved character"); Ok(()) } #[test] fn test_str_rejects_equals() -> TestResult { - let result = serialize_decimal("123=45"); - assert_err_contains(result, ErrorCode::InvalidDecimal, "Invalid character"); + let result = DecimalView::try_new_string("123=45"); + assert_err_contains(result, ErrorCode::InvalidDecimal, "reserved character"); Ok(()) } #[test] fn test_str_rejects_newline() -> TestResult { - let result = serialize_decimal("123\n45"); - assert_err_contains(result, ErrorCode::InvalidDecimal, "Invalid character"); + let result = DecimalView::try_new_string("123\n45"); + assert_err_contains(result, ErrorCode::InvalidDecimal, "reserved character"); Ok(()) } #[test] fn test_str_rejects_backslash() -> TestResult { - let result = serialize_decimal("123\\45"); - assert_err_contains(result, ErrorCode::InvalidDecimal, "Invalid character"); + let result = DecimalView::try_new_string("123\\45"); + assert_err_contains(result, ErrorCode::InvalidDecimal, "reserved character"); Ok(()) } @@ -168,13 +173,16 @@ fn parse_binary_decimal(bytes: &[u8]) -> (u8, i128) { #[cfg(feature = "rust_decimal")] mod rust_decimal_tests { use super::*; + use crate::ingress::DecimalView; use rust_decimal::Decimal; + use std::convert::TryInto; use std::str::FromStr; #[test] fn test_decimal_binary_format_zero() -> TestResult { let dec = Decimal::ZERO; - let result = serialize_decimal(&dec)?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 0, "Zero should have scale 0"); @@ -185,7 +193,8 @@ mod rust_decimal_tests { #[test] fn test_decimal_binary_format_positive() -> TestResult { let dec = Decimal::from_str("123.45")?; - let result = serialize_decimal(&dec)?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 2, "123.45 should have scale 2"); @@ -196,7 +205,8 @@ mod rust_decimal_tests { #[test] fn test_decimal_binary_format_negative() -> TestResult { let dec = Decimal::from_str("-123.45")?; - let result = serialize_decimal(&dec)?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 2, "-123.45 should have scale 2"); @@ -210,7 +220,8 @@ mod rust_decimal_tests { #[test] fn test_decimal_binary_format_one() -> TestResult { let dec = Decimal::ONE; - let result = serialize_decimal(&dec)?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 0, "One should have scale 0"); @@ -222,7 +233,8 @@ mod rust_decimal_tests { fn test_decimal_binary_format_max_scale() -> TestResult { // Create a decimal with maximum scale (28 for rust_decimal) let dec = Decimal::from_str("0.0000000000000000000000000001")?; - let result = serialize_decimal(&dec)?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 28, "Should have maximum scale of 28"); @@ -233,7 +245,8 @@ mod rust_decimal_tests { #[test] fn test_decimal_binary_format_large_value() -> TestResult { let dec = Decimal::MAX; - let result = serialize_decimal(&dec)?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 0, "Large integer should have scale 0"); @@ -247,7 +260,8 @@ mod rust_decimal_tests { #[test] fn test_decimal_binary_format_large_value2() -> TestResult { let dec = Decimal::MIN; - let result = serialize_decimal(&dec)?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 0, "Large integer should have scale 0"); @@ -261,7 +275,8 @@ mod rust_decimal_tests { #[test] fn test_decimal_binary_format_small_negative() -> TestResult { let dec = Decimal::from_str("-0.01")?; - let result = serialize_decimal(&dec)?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 2, "-0.01 should have scale 2"); @@ -272,7 +287,8 @@ mod rust_decimal_tests { #[test] fn test_decimal_binary_format_trailing_zeros() -> TestResult { let dec = Decimal::from_str("1.00")?; - let result = serialize_decimal(&dec)?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); let (scale, unscaled) = parse_binary_decimal(&result); // rust_decimal normalizes trailing zeros @@ -289,13 +305,16 @@ mod rust_decimal_tests { #[cfg(feature = "bigdecimal")] mod bigdecimal_tests { use super::*; + use crate::ingress::DecimalView; use bigdecimal::BigDecimal; + use std::convert::TryInto; use std::str::FromStr; #[test] fn test_bigdecimal_binary_format_zero() -> TestResult { let dec = BigDecimal::from_str("0")?; - let result = serialize_decimal(&dec)?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 0, "Zero should have scale 0"); @@ -306,7 +325,8 @@ mod bigdecimal_tests { #[test] fn test_bigdecimal_binary_format_positive() -> TestResult { let dec = BigDecimal::from_str("123.45")?; - let result = serialize_decimal(&dec)?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 2, "123.45 should have scale 2"); @@ -317,7 +337,8 @@ mod bigdecimal_tests { #[test] fn test_bigdecimal_binary_format_negative() -> TestResult { let dec = BigDecimal::from_str("-123.45")?; - let result = serialize_decimal(&dec)?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 2, "-123.45 should have scale 2"); @@ -331,7 +352,8 @@ mod bigdecimal_tests { #[test] fn test_bigdecimal_binary_format_one() -> TestResult { let dec = BigDecimal::from_str("1")?; - let result = serialize_decimal(&dec)?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 0, "One should have scale 0"); @@ -343,7 +365,8 @@ mod bigdecimal_tests { fn test_bigdecimal_binary_format_high_precision() -> TestResult { // BigDecimal can handle arbitrary precision, test a value with many decimal places let dec = BigDecimal::from_str("0.123456789012345678901234567890")?; - let result = serialize_decimal(&dec)?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 30, "Should preserve high precision scale"); @@ -358,7 +381,8 @@ mod bigdecimal_tests { fn test_bigdecimal_binary_format_large_value() -> TestResult { // Test a very large value that BigDecimal can represent let dec = BigDecimal::from_str("79228162514264337593543950335")?; - let result = serialize_decimal(&dec)?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 0, "Large integer should have scale 0"); @@ -372,7 +396,8 @@ mod bigdecimal_tests { #[test] fn test_bigdecimal_binary_format_large_negative() -> TestResult { let dec = BigDecimal::from_str("-79228162514264337593543950335")?; - let result = serialize_decimal(&dec)?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 0, "Large negative integer should have scale 0"); @@ -386,7 +411,8 @@ mod bigdecimal_tests { #[test] fn test_bigdecimal_binary_format_small_negative() -> TestResult { let dec = BigDecimal::from_str("-0.01")?; - let result = serialize_decimal(&dec)?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 2, "-0.01 should have scale 2"); @@ -397,7 +423,8 @@ mod bigdecimal_tests { #[test] fn test_bigdecimal_binary_format_trailing_zeros() -> TestResult { let dec = BigDecimal::from_str("1.00")?; - let result = serialize_decimal(&dec)?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); let (scale, unscaled) = parse_binary_decimal(&result); // BigDecimal may normalize trailing zeros differently than rust_decimal @@ -412,7 +439,8 @@ mod bigdecimal_tests { let dec = BigDecimal::from_str( "0.0000000000000000000000000000000000000000000000000000000000000000000000000001", )?; - let result = serialize_decimal(&dec)?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); let (scale, unscaled) = parse_binary_decimal(&result); assert_eq!(scale, 76, "Should have maximum scale of 76"); @@ -426,7 +454,7 @@ mod bigdecimal_tests { let dec = BigDecimal::from_str( "0.00000000000000000000000000000000000000000000000000000000000000000000000000001", )?; - let result = serialize_decimal(&dec); + let result: crate::Result = (&dec).try_into(); assert_err_contains(result, ErrorCode::InvalidDecimal, "scale greater than 76"); Ok(()) } @@ -435,7 +463,8 @@ mod bigdecimal_tests { fn test_bigdecimal_binary_negative_scale() -> TestResult { // Test with a negative scale let dec = BigDecimal::from_str("1.23e12")?; - let result = serialize_decimal(&dec)?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); let (scale, unscaled) = parse_binary_decimal(&result); // QuestDB does not support negative scale, instead the value should be @@ -452,12 +481,8 @@ mod bigdecimal_tests { fn test_bigdecimal_binary_value_too_large() -> TestResult { // QuestDB cannot accept arrays that are larger than what an i8 can fit let dec = BigDecimal::from_str("1e1000")?; - let result = serialize_decimal(&dec); - assert_err_contains( - result, - ErrorCode::InvalidDecimal, - "does not support decimal scale greater than 76", - ); + let result: crate::Result = (&dec).try_into(); + assert_err_contains(result, ErrorCode::InvalidDecimal, "decimal longer than"); Ok(()) } } From fec21647279d9fd1fd8d3dd9c855b00962a13f74 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Tue, 21 Oct 2025 10:57:16 +0200 Subject: [PATCH 62/65] test: add tests for NaN and Infinity decimal representations --- questdb-rs/src/ingress/decimal.rs | 53 +++++++++++++++++++++++++------ questdb-rs/src/tests/decimal.rs | 24 ++++++++++++++ 2 files changed, 67 insertions(+), 10 deletions(-) diff --git a/questdb-rs/src/ingress/decimal.rs b/questdb-rs/src/ingress/decimal.rs index 4edc1371..ee0ebc4a 100644 --- a/questdb-rs/src/ingress/decimal.rs +++ b/questdb-rs/src/ingress/decimal.rs @@ -34,11 +34,6 @@ use std::borrow::Cow; /// /// Example: `"123.45d"` or `"1.5e-3d"` /// -/// Implementers must: -/// - Write the decimal's text representation to the output buffer -/// - Append the `'d'` suffix -/// - Ensure no ILP reserved characters are present (space, comma, equals, newline, carriage return, backslash) -/// /// ### Binary Format /// A more compact binary encoding consisting of: /// @@ -72,6 +67,14 @@ pub enum DecimalView<'a> { } impl<'a> DecimalView<'a> { + /// Creates a [`DecimalView::Scaled`] from a mantissa buffer and scale. + /// + /// Validates that: + /// - `scale` does not exceed the QuestDB maximum of 76 decimal places. + /// - The mantissa fits into at most 127 bytes (ILP binary limit). + /// + /// Returns an [`error::ErrorCode::InvalidDecimal`](crate::error::ErrorCode::InvalidDecimal) + /// error if either constraint is violated. pub fn try_new_scaled(scale: u32, value: T) -> Result where T: Into>, @@ -98,23 +101,50 @@ impl<'a> DecimalView<'a> { }) } + /// Creates a [`DecimalView::String`] from a textual decimal representation. + /// + /// Thousand separators (commas) are not allowed and the decimal point must be a dot (`.`). + /// + /// Performs lightweight validation and rejects values containing ILP-reserved characters. + /// Accepts plain decimals, optional `+/-` prefixes, `NaN`, `Inf[inity]`, and scientific + /// notation (`e`/`E`). + /// + /// Returns [`error::ErrorCode::InvalidDecimal`](crate::error::ErrorCode::InvalidDecimal) + /// if disallowed characters are encountered. pub fn try_new_string(value: &'a str) -> Result { - // Basic validation: ensure no ILP reserved characters are present - for b in value.bytes() { + // Basic validation: ensure only numerical characters are present (accepts NaN, Inf[inity], and e-notation) + for b in value.chars() { match b { - b' ' | b',' | b'=' | b'\n' | b'\r' | b'\\' => { + '0'..='9' + | '.' + | '-' + | '+' + | 'e' + | 'E' + | 'N' + | 'a' + | 'I' + | 'n' + | 'f' + | 'i' + | 't' + | 'y' => {} + _ => { return Err(error::fmt!( InvalidDecimal, "Decimal string contains ILP reserved character {:?}", b as char )); } - _ => {} } } Ok(DecimalView::String { value }) } + /// Serializes the decimal view into the provided output buffer using the ILP encoding. + /// + /// Delegates to [`serialize_string`] for textual representations and [`serialize_scaled`] for + /// the compact binary format. pub(crate) fn serialize(&self, out: &mut Vec) { match self { DecimalView::String { value } => Self::serialize_string(value, out), @@ -124,6 +154,7 @@ impl<'a> DecimalView<'a> { } } + /// Serializes a textual decimal by copying the string and appending the `d` suffix. fn serialize_string(value: &str, out: &mut Vec) { // Pre-allocate space for the string content plus the 'd' suffix out.reserve(value.len() + 1); @@ -134,6 +165,8 @@ impl<'a> DecimalView<'a> { out.push(b'd'); } + /// Serializes a scaled decimal into the binary ILP format, writing the marker, type tag, + /// scale, mantissa length, and mantissa bytes. fn serialize_scaled(scale: u8, value: &[u8], out: &mut Vec) { // Write binary format: '=' marker + type + scale + length + mantissa bytes out.push(b'='); @@ -154,7 +187,7 @@ impl<'a> DecimalView<'a> { /// /// # Validation /// The implementation performs **partial validation only**: -/// - Rejects non-numerical characters (not -/+, 0-9, a-z/A-Z, .) +/// - Rejects non-numerical characters (not -/+, 0-9, ., Inf, NaN, e/E) /// - Does NOT validate the actual decimal syntax (e.g., "e2e" would pass) /// /// This is intentional: full parsing would add overhead. The QuestDB server performs complete diff --git a/questdb-rs/src/tests/decimal.rs b/questdb-rs/src/tests/decimal.rs index 2cf14ff9..4356a45d 100644 --- a/questdb-rs/src/tests/decimal.rs +++ b/questdb-rs/src/tests/decimal.rs @@ -61,6 +61,30 @@ fn test_str_zero() -> TestResult { Ok(()) } +#[test] +fn test_str_nan() -> TestResult { + let decimal = DecimalView::try_new_string("NaN")?; + let result = serialize_decimal(decimal); + assert_eq!(result, b"NaNd"); + Ok(()) +} + +#[test] +fn test_str_inf() -> TestResult { + let decimal = DecimalView::try_new_string("Infinity")?; + let result = serialize_decimal(decimal); + assert_eq!(result, b"Infinityd"); + Ok(()) +} + +#[test] +fn test_str_negative_infinity() -> TestResult { + let decimal = DecimalView::try_new_string("-Infinity")?; + let result = serialize_decimal(decimal); + assert_eq!(result, b"-Infinityd"); + Ok(()) +} + #[test] fn test_str_scientific_notation() -> TestResult { let decimal = DecimalView::try_new_string("1.5e-3")?; From dfb9fd4e4485e4c58d28063105586a7141ca8536 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Tue, 21 Oct 2025 11:03:21 +0200 Subject: [PATCH 63/65] fix: correct spelling of 'Infinity' in validation comments --- questdb-rs/src/ingress/decimal.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/questdb-rs/src/ingress/decimal.rs b/questdb-rs/src/ingress/decimal.rs index ee0ebc4a..88b48c41 100644 --- a/questdb-rs/src/ingress/decimal.rs +++ b/questdb-rs/src/ingress/decimal.rs @@ -106,7 +106,7 @@ impl<'a> DecimalView<'a> { /// Thousand separators (commas) are not allowed and the decimal point must be a dot (`.`). /// /// Performs lightweight validation and rejects values containing ILP-reserved characters. - /// Accepts plain decimals, optional `+/-` prefixes, `NaN`, `Inf[inity]`, and scientific + /// Accepts plain decimals, optional `+/-` prefixes, `NaN`, `Infinity`, and scientific /// notation (`e`/`E`). /// /// Returns [`error::ErrorCode::InvalidDecimal`](crate::error::ErrorCode::InvalidDecimal) @@ -187,7 +187,7 @@ impl<'a> DecimalView<'a> { /// /// # Validation /// The implementation performs **partial validation only**: -/// - Rejects non-numerical characters (not -/+, 0-9, ., Inf, NaN, e/E) +/// - Rejects non-numerical characters (not -/+, 0-9, ., Infinity, NaN, e/E) /// - Does NOT validate the actual decimal syntax (e.g., "e2e" would pass) /// /// This is intentional: full parsing would add overhead. The QuestDB server performs complete From 23329b000313bf18269a9d4d0496512853c02783 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Tue, 21 Oct 2025 11:23:41 +0200 Subject: [PATCH 64/65] fix: improve error message for invalid decimal strings --- questdb-rs/src/ingress/decimal.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questdb-rs/src/ingress/decimal.rs b/questdb-rs/src/ingress/decimal.rs index 88b48c41..7eeabf56 100644 --- a/questdb-rs/src/ingress/decimal.rs +++ b/questdb-rs/src/ingress/decimal.rs @@ -133,7 +133,7 @@ impl<'a> DecimalView<'a> { return Err(error::fmt!( InvalidDecimal, "Decimal string contains ILP reserved character {:?}", - b as char + b )); } } From 73d9f0d88a6f2440744ccd2acf687d920f89eb23 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Tue, 21 Oct 2025 12:13:28 +0200 Subject: [PATCH 65/65] fix: correct scale type in DecimalView conversion --- questdb-rs/src/ingress/decimal.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questdb-rs/src/ingress/decimal.rs b/questdb-rs/src/ingress/decimal.rs index 7eeabf56..74159da3 100644 --- a/questdb-rs/src/ingress/decimal.rs +++ b/questdb-rs/src/ingress/decimal.rs @@ -216,7 +216,7 @@ impl<'a> TryInto> for &'a rust_decimal::Decimal { fn try_into(self) -> Result> { let raw = self.mantissa().to_be_bytes(); let bytes = trim_leading_sign_bytes(&raw); - DecimalView::try_new_scaled(self.scale() as u32, bytes) + DecimalView::try_new_scaled(self.scale(), bytes) } }