From 87aa2407a5c1f0de68ab5019ad90234d90d2469e Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 1 Apr 2025 23:02:30 +0800 Subject: [PATCH 001/119] c client nd array begin. --- questdb-rs/src/ingress/mod.rs | 1 + questdb-rs/src/ingress/ndarr.rs | 109 ++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 questdb-rs/src/ingress/ndarr.rs diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index f3d33d8a..b429ef79 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -2691,3 +2691,4 @@ use http::*; #[cfg(test)] mod tests; +mod ndarr; diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs new file mode 100644 index 00000000..41f912f0 --- /dev/null +++ b/questdb-rs/src/ingress/ndarr.rs @@ -0,0 +1,109 @@ +use std::mem::ManuallyDrop; +use std::ptr::NonNull; +use std::slice; + +pub struct Shape(Vec); + +impl From> for Shape { + fn from(vec: Vec) -> Self { + Self(vec) + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct OwnedRepr +{ + ptr: NonNull, + len: usize, + capacity: usize, +} + +impl OwnedRepr { + pub fn from(v: Vec) -> Self + { + let mut v = ManuallyDrop::new(v); + let len = v.len(); + let capacity = v.capacity(); + unsafe { + let ptr = NonNull::new_unchecked(v.as_mut_ptr()); + Self { ptr, len, capacity } + } + } + + pub(crate) fn into_vec(self) -> Vec + { + ManuallyDrop::new(self).take_as_vec() + } + + pub(crate) fn as_slice(&self) -> &[A] + { + unsafe { slice::from_raw_parts(self.ptr.as_ptr(), self.len) } + } + + pub(crate) fn len(&self) -> usize + { + self.len + } + + pub(crate) fn as_ptr(&self) -> *const A + { + self.ptr.as_ptr() + } + + pub(crate) fn as_nonnull_mut(&mut self) -> NonNull + { + self.ptr + } + + fn take_as_vec(&mut self) -> Vec + { + let capacity = self.capacity; + let len = self.len; + self.len = 0; + self.capacity = 0; + unsafe { Vec::from_raw_parts(self.ptr.as_ptr(), len, capacity) } + } +} + +pub struct Array +where + T: ArrayElement, +{ + shape: Shape, + data: OwnedRepr, +} + +impl Array { + fn ndim(&self) -> usize { + self.shape.0.len() + } + + fn dim(&self, index: usize) -> Option { + if index < self.shape.0.len() { + Some(self.shape.0[index]) + } else { + None + } + } + + fn from(v: Vec) -> Self + { + Self::from_shape_vec_unchecked(vec![v.len()], v) + } + + pub fn from_shape_vec_unchecked(shape: Sh, v: Vec) -> Self + where + Sh: Into, + { + Array { + shape: shape.into(), + data: OwnedRepr::from(v), + } + } +} + +pub trait ArrayElement: Copy + 'static {} +impl ArrayElement for f64 {} + + From 6bd3223512e8453300e1f7149fd59991c3f86ec1 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 2 Apr 2025 19:26:35 +0800 Subject: [PATCH 002/119] Ndarray support for rust client. --- questdb-rs/Cargo.toml | 1 + questdb-rs/src/error.rs | 9 ++ questdb-rs/src/ingress/mod.rs | 111 ++++++++++++++++++- questdb-rs/src/ingress/ndarr.rs | 186 ++++++++++++++++++-------------- 4 files changed, 225 insertions(+), 82 deletions(-) diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index 8db1d4da..fee87082 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -35,6 +35,7 @@ ureq = { version = "=2.9", optional = true } serde_json = { version = "1.0.108", optional = true } questdb-confstr = "0.1.0" rand = { version = "0.8.5", optional = true } +ndarray = { version = "0.16.1", optional = true } no-panic = { version = "0.1", optional = true } [target.'cfg(windows)'.dependencies] diff --git a/questdb-rs/src/error.rs b/questdb-rs/src/error.rs index 6848280a..af39b78b 100644 --- a/questdb-rs/src/error.rs +++ b/questdb-rs/src/error.rs @@ -48,6 +48,15 @@ pub enum ErrorCode { /// Bad configuration. ConfigError, + + /// Array has too many dims + ArrayHasTooManyDims, + + /// Array error + ArrayViewError, + + /// Buffer outOfMemory + BufferOutOfMemory, } /// An error that occurred when using QuestDB client library. diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index b429ef79..e0c63ba8 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -39,6 +39,7 @@ use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; +use crate::ingress::ndarr::{ArrayElement, NdArrayView, MAX_DIMS}; use base64ct::{Base64, Base64UrlUnpadded, Encoding}; use ring::rand::SystemRandom; use ring::signature::{EcdsaKeyPair, ECDSA_P256_SHA256_FIXED_SIGNING}; @@ -993,6 +994,112 @@ impl Buffer { Ok(self) } + /// Record a multidimensional array value for the given column. + /// + /// Supports arrays with up to [`MAX_DIMS`] dimensions. The array elements must + /// implement [`ArrayElement`] trait which provides type-to-[`ElemDataType`] mapping. + /// + /// # Examples + /// + /// Basic usage with direct dimension specification: + /// + /// ``` + /// # #[cfg(feature = "ndarray")] + /// # { + /// # use questdb::Result; + /// # use questdb::ingress::Buffer; + /// # use ndarray::array; + /// # fn main() -> Result<()> { + /// # let mut buffer = Buffer::new(); + /// # buffer.table("x")?; + /// // Record a 2D array of f64 values + /// let array_2d = array![[1.1, 2.2], [3.3, 4.4]]; + /// buffer.column_arr("array_col", &array_2d.view())?; + /// # Ok(()) + /// # } + /// # } + /// + /// ``` + /// + /// Using [`ColumnName`] for validated column names: + /// + /// ``` + /// # #[cfg(feature = "ndarray")] + /// # { + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, ColumnName}; + /// # use ndarray::Array3; + /// # fn main() -> Result<()> { + /// # let mut buffer = Buffer::new(); + /// # buffer.table("x1")?; + /// // Record a 3D array of f64 values + /// let array_3d = Array3::from_elem((2, 3, 4), 42f64); + /// let col_name = ColumnName::new("col1")?; + /// buffer.column_arr(col_name, &array_3d.view())?; + /// # Ok(()) + /// # } + /// # } + /// ``` + /// # Errors + /// + /// Returns [`Error`] if: + /// - Array dimensions exceed [`MAX_DIMS`] + /// - Failed to get dimension sizes + /// - Memory allocation fails for array data + /// - Column name validation fails + pub fn column_arr<'a, N, T, D>(&mut self, name: N, view: &T) -> Result<&mut Self> + where + N: TryInto>, + T: NdArrayView, + D: ArrayElement, + Error: From, + { + self.write_column_key(name)?; + + // check dimension less equal than max dims + if MAX_DIMS < view.ndim() { + return Err(error::fmt!( + ArrayHasTooManyDims, + "Array dimension mismatch: expected at most {} dimensions, but got {}", + MAX_DIMS, + view.ndim() + )); + } + + // binary format flag '=' + self.output.push(b'='); + // binary format entity type + self.output.push(ARRAY_BINARY_FORMAT_TYPE); + // ndarr dims + self.output.push(view.ndim() as u8); + // ndarr shapes + self.output.push(D::elem_type().into()); + + let mut reserve_size = size_of::(); + for i in 0..view.ndim() { + let d = view.dim(i).ok_or_else(|| { + error::fmt!(ArrayViewError, "Can not get correct dimensions for dim {}", i) + })?; + + self.output + .extend_from_slice((d as i32).to_le_bytes().as_slice()); + reserve_size = reserve_size + .checked_mul(d) + .ok_or(error::fmt!(ArrayViewError, "Array total elem size overflow"))? + } + + let index = self.output.len(); + let reserve_size = reserve_size + .checked_add(index) + .ok_or(error::fmt!(ArrayViewError, "Array total elem size overflow"))?; + self.output + .try_reserve(reserve_size) + .map_err(|_| error::fmt!(BufferOutOfMemory, "Buffer out of memory"))?; + unsafe { self.output.set_len(reserve_size) } + view.write_row_major_buf(&mut self.output[index..]); + Ok(self) + } + /// Record a timestamp value for the given column. /// /// ``` @@ -2680,6 +2787,8 @@ impl Sender { } } +const ARRAY_BINARY_FORMAT_TYPE: u8 = 14; + mod conf; mod timestamp; @@ -2689,6 +2798,6 @@ mod http; #[cfg(feature = "ilp-over-http")] use http::*; +mod ndarr; #[cfg(test)] mod tests; -mod ndarr; diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 41f912f0..5713ee46 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -1,109 +1,133 @@ -use std::mem::ManuallyDrop; -use std::ptr::NonNull; -use std::slice; +pub(crate) const MAX_DIMS: usize = 32; -pub struct Shape(Vec); +pub trait NdArrayView +where + T: ArrayElement, +{ + /// Returns the number of dimensions (rank) of the array. + fn ndim(&self) -> usize; + + /// Returns the size of the specified dimension. + fn dim(&self, index: usize) -> Option; + + /// Writes array data to buffer in row-major order. + /// + /// # Important Notes + /// - Buffer must be pre-allocated with exact required size + /// - No alignment assumptions should be made about buffer start + /// - Handles both contiguous and non-contiguous memory layouts + fn write_row_major_buf(&self, buff: &mut [u8]); +} -impl From> for Shape { - fn from(vec: Vec) -> Self { - Self(vec) - } +/// Marker trait for valid array element types. +/// +/// Implemented for primitive types that can be stored in arrays. +/// Combines type information with data type classification. +pub trait ArrayElement: Copy + 'static { + /// Returns the corresponding data type classification. + /// + /// This enables runtime type identification while maintaining + /// compile-time type safety. + fn elem_type() -> ElemDataType; } +#[repr(u8)] #[derive(Debug)] -#[repr(C)] -pub struct OwnedRepr -{ - ptr: NonNull, - len: usize, - capacity: usize, +pub enum ElemDataType { + /// Uninitialized/placeholder type + Undefined = 0, + /// Boolean values (true/false) + Boolean, + /// 8-bit signed integer + Byte, + /// 16-bit signed integer + Short, + /// UTF-16 character + Char, + /// 32-bit signed integer + Int, + /// 64-bit signed integer + Long, + /// Date type (days since epoch) + Date, + /// Microsecond-precision timestamp + Timestamp, + /// 32-bit floating point + Float, + /// 64-bit floating point + Double, + /// UTF-8 string data + String, + /// Interned string symbol + Symbol, + /// 256-bit integer value + Long256, + /// Geospatial byte coordinates + GeoByte, + /// Geospatial short coordinates + GeoShort, + /// Geospatial integer coordinates + GeoInt, + /// Geospatial long coordinates + GeoLong, + /// Binary large object + Binary, + /// UUID values + Uuid, } -impl OwnedRepr { - pub fn from(v: Vec) -> Self - { - let mut v = ManuallyDrop::new(v); - let len = v.len(); - let capacity = v.capacity(); - unsafe { - let ptr = NonNull::new_unchecked(v.as_mut_ptr()); - Self { ptr, len, capacity } - } - } - - pub(crate) fn into_vec(self) -> Vec - { - ManuallyDrop::new(self).take_as_vec() - } - - pub(crate) fn as_slice(&self) -> &[A] - { - unsafe { slice::from_raw_parts(self.ptr.as_ptr(), self.len) } - } - - pub(crate) fn len(&self) -> usize - { - self.len - } - - pub(crate) fn as_ptr(&self) -> *const A - { - self.ptr.as_ptr() - } - - pub(crate) fn as_nonnull_mut(&mut self) -> NonNull - { - self.ptr +impl From for u8 { + fn from(val: ElemDataType) -> Self { + val as u8 } +} - fn take_as_vec(&mut self) -> Vec - { - let capacity = self.capacity; - let len = self.len; - self.len = 0; - self.capacity = 0; - unsafe { Vec::from_raw_parts(self.ptr.as_ptr(), len, capacity) } +impl ArrayElement for f64 { + /// Identifies f64 as Double type in QuestDB's type system. + fn elem_type() -> ElemDataType { + ElemDataType::Double } } -pub struct Array +#[cfg(feature = "ndarray")] +use ndarray::{ArrayView, Axis, Dimension}; + +#[cfg(feature = "ndarray")] +impl NdArrayView for ArrayView<'_, T, D> where T: ArrayElement, + D: Dimension, { - shape: Shape, - data: OwnedRepr, -} - -impl Array { fn ndim(&self) -> usize { - self.shape.0.len() + self.ndim() } fn dim(&self, index: usize) -> Option { - if index < self.shape.0.len() { - Some(self.shape.0[index]) + let len = self.len(); + if index < len { + Some(self.len_of(Axis(index))) } else { None } } - fn from(v: Vec) -> Self - { - Self::from_shape_vec_unchecked(vec![v.len()], v) - } + fn write_row_major_buf(&self, buf: &mut [u8]) { + let elem_size = size_of::(); + + if let Some(slice) = self.as_slice() { + let byte_len = size_of_val(slice); + let bytes = + unsafe { std::slice::from_raw_parts(slice.as_ptr() as *const u8, byte_len) }; + buf[..byte_len].copy_from_slice(bytes); + return; + } - pub fn from_shape_vec_unchecked(shape: Sh, v: Vec) -> Self - where - Sh: Into, - { - Array { - shape: shape.into(), - data: OwnedRepr::from(v), + let mut bytes_written = 0; + for &element in self.iter() { + let element_bytes = + unsafe { std::slice::from_raw_parts(&element as *const T as *const _, elem_size) }; + buf[bytes_written..bytes_written + elem_size].copy_from_slice(element_bytes); + bytes_written += elem_size; } } } - -pub trait ArrayElement: Copy + 'static {} -impl ArrayElement for f64 {} - - From b518a9988480766fbe239a8cef5b66bd5bc2e9ad Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 2 Apr 2025 19:30:30 +0800 Subject: [PATCH 003/119] optimize doc. --- questdb-rs/src/error.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/questdb-rs/src/error.rs b/questdb-rs/src/error.rs index af39b78b..1405b61a 100644 --- a/questdb-rs/src/error.rs +++ b/questdb-rs/src/error.rs @@ -49,13 +49,13 @@ pub enum ErrorCode { /// Bad configuration. ConfigError, - /// Array has too many dims + /// Array has too many dims. Currently, only arrays with a maximum [`crate::ingress::ndarr::MAX_DIMS`] dimensions are supported. ArrayHasTooManyDims, - /// Array error + /// Array view internal error. ArrayViewError, - /// Buffer outOfMemory + /// Buffer outOfMemory. BufferOutOfMemory, } From 53a430e0f4fe3c4a5551bd69f04d5272418f0583 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 2 Apr 2025 20:55:32 +0800 Subject: [PATCH 004/119] add tests. --- questdb-rs/src/ingress/mod.rs | 2 +- questdb-rs/src/ingress/ndarr.rs | 4 +- questdb-rs/src/tests/mod.rs | 3 ++ questdb-rs/src/tests/ndarr.rs | 79 +++++++++++++++++++++++++++++++++ questdb-rs/src/tests/sender.rs | 7 +++ 5 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 questdb-rs/src/tests/ndarr.rs diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index e0c63ba8..83d6365b 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -2798,6 +2798,6 @@ mod http; #[cfg(feature = "ilp-over-http")] use http::*; -mod ndarr; +pub(crate) mod ndarr; #[cfg(test)] mod tests; diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 5713ee46..222db184 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -32,7 +32,7 @@ pub trait ArrayElement: Copy + 'static { } #[repr(u8)] -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub enum ElemDataType { /// Uninitialized/placeholder type Undefined = 0, @@ -103,7 +103,7 @@ where } fn dim(&self, index: usize) -> Option { - let len = self.len(); + let len = self.ndim(); if index < len { Some(self.len_of(Axis(index))) } else { diff --git a/questdb-rs/src/tests/mod.rs b/questdb-rs/src/tests/mod.rs index 50266006..60b811a8 100644 --- a/questdb-rs/src/tests/mod.rs +++ b/questdb-rs/src/tests/mod.rs @@ -29,6 +29,9 @@ mod http; mod mock; mod sender; +#[cfg(feature = "ndarray")] +mod ndarr; + #[cfg(feature = "json_tests")] mod json_tests { include!(concat!(env!("OUT_DIR"), "/json_tests.rs")); diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs new file mode 100644 index 00000000..31448131 --- /dev/null +++ b/questdb-rs/src/tests/ndarr.rs @@ -0,0 +1,79 @@ +use crate::ingress::ndarr; +use crate::ingress::ndarr::NdArrayView; +use ndarray::{arr1, arr2, arr3, s}; + +#[test] +fn test_f64_element_type() { + assert_eq!(::elem_type(), ndarr::ElemDataType::Double); + assert_eq!(u8::from(ndarr::ElemDataType::Double), 10); +} + +#[test] +fn test_1d_contiguous_buffer() { + let array = arr1(&[1.0, 2.0, 3.0, 4.0]); + let view = array.view(); + let mut buf = vec![0u8; 4 * size_of::()]; + view.write_row_major_buf(&mut buf); + let expected: Vec = array + .iter() + .flat_map(|&x| x.to_ne_bytes().to_vec()) + .collect(); + assert_eq!(buf, expected); +} + +#[test] +fn test_2d_non_contiguous_buffer() { + let array = arr2(&[[1.0, 2.0], [3.0, 4.0]]); + let transposed = array.view().reversed_axes(); + assert!(!transposed.is_standard_layout()); + let mut buf = vec![0u8; 4 * size_of::()]; + transposed.write_row_major_buf(&mut buf); + let expected = [1.0f64, 3.0, 2.0, 4.0] + .iter() + .flat_map(|&x| x.to_ne_bytes()) + .collect::>(); + assert_eq!(buf, expected); +} + +#[test] +fn test_strided_layout() { + let array = arr2(&[ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + [13.0, 14.0, 15.0, 16.0] + ]); + let strided_view = array.slice(s![1..;2, 1..;2]); + assert_eq!(strided_view.dim(), (2, 2)); + let mut buf = vec![0u8; 4 * size_of::()]; + strided_view.write_row_major_buf(&mut buf); + + // expect：6.0, 8.0, 14.0, 16.0 + let expected = [6.0f64, 8.0, 14.0, 16.0] + .iter() + .flat_map(|&x| x.to_ne_bytes()) + .collect::>(); + + assert_eq!(buf, expected); +} + +#[test] +fn test_1d_dimension_info() { + let array = arr1(&[1.0, 2.0, 3.0]); + let view = array.view(); + + assert_eq!(NdArrayView::ndim(&view), 1); + assert_eq!(NdArrayView::dim(&view, 0), Some(3)); + assert_eq!(NdArrayView::dim(&view, 1), None); +} + +#[test] +fn test_complex_dimensions() { + let array = arr3(&[[[1.0], [2.0]], [[3.0], [4.0]]]); + let view = array.view(); + + assert_eq!(NdArrayView::ndim(&view), 3); + assert_eq!(NdArrayView::dim(&view, 0), Some(2)); + assert_eq!(NdArrayView::dim(&view, 1), Some(2)); + assert_eq!(NdArrayView::dim(&view, 2), Some(1)); +} diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index cc0f02fe..1fda0df6 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -35,6 +35,8 @@ use crate::tests::{ }; use core::time::Duration; +#[cfg(feature = "ndarray")] +use ndarray::arr1; use std::{io, time::SystemTime}; #[test] @@ -351,6 +353,11 @@ fn test_str_column_name_too_long() -> TestResult { column_name_too_long_test_impl!(column_str, "value") } +#[cfg(feature = "ndarray")] +fn test_arr_column_name_too_long() -> TestResult { + column_name_too_long_test_impl!(column_arr, &arr1(&[1.0, 2.0, 3.0]).view()) +} + #[test] fn test_tls_with_file_ca() -> TestResult { let mut ca_path = certs_dir(); From b1ebd1dd0b01df742e26b1926fb45267da049d38 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 2 Apr 2025 21:00:34 +0800 Subject: [PATCH 005/119] fix comments. --- questdb-rs/src/ingress/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 83d6365b..223253c7 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -1072,7 +1072,7 @@ impl Buffer { self.output.push(ARRAY_BINARY_FORMAT_TYPE); // ndarr dims self.output.push(view.ndim() as u8); - // ndarr shapes + // ndarr datatype self.output.push(D::elem_type().into()); let mut reserve_size = size_of::(); @@ -1081,6 +1081,7 @@ impl Buffer { error::fmt!(ArrayViewError, "Can not get correct dimensions for dim {}", i) })?; + // ndarr shapes self.output .extend_from_slice((d as i32).to_le_bytes().as_slice()); reserve_size = reserve_size @@ -1096,6 +1097,8 @@ impl Buffer { .try_reserve(reserve_size) .map_err(|_| error::fmt!(BufferOutOfMemory, "Buffer out of memory"))?; unsafe { self.output.set_len(reserve_size) } + + // ndarr data view.write_row_major_buf(&mut self.output[index..]); Ok(self) } From 006af982b8731ab6ab74675910a31d5713890faa Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 3 Apr 2025 09:35:38 +0800 Subject: [PATCH 006/119] fix docs. --- questdb-rs/src/ingress/mod.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 223253c7..03d4b447 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -498,6 +498,7 @@ impl BufferState { /// [`column_i64`](Buffer::column_i64), /// [`column_f64`](Buffer::column_f64), /// [`column_str`](Buffer::column_str), +/// [`column_arr`](Buffer::column_arr), /// [`column_ts`](Buffer::column_ts)). /// * Symbols must appear before columns. /// * A row must be terminated with either [`at`](Buffer::at) or @@ -516,6 +517,7 @@ impl BufferState { /// | [`column_i64`](Buffer::column_i64) | [`INTEGER`](https://questdb.io/docs/reference/api/ilp/columnset-types#integer) | /// | [`column_f64`](Buffer::column_f64) | [`FLOAT`](https://questdb.io/docs/reference/api/ilp/columnset-types#float) | /// | [`column_str`](Buffer::column_str) | [`STRING`](https://questdb.io/docs/reference/api/ilp/columnset-types#string) | +/// | [`column_arr`](Buffer::column_arr) | [`ARRAY`](https://questdb.io/docs/reference/api/ilp/columnset-types#array) | /// | [`column_ts`](Buffer::column_ts) | [`TIMESTAMP`](https://questdb.io/docs/reference/api/ilp/columnset-types#timestamp) | /// /// QuestDB supports both `STRING` and `SYMBOL` column types. @@ -1070,10 +1072,10 @@ impl Buffer { self.output.push(b'='); // binary format entity type self.output.push(ARRAY_BINARY_FORMAT_TYPE); - // ndarr dims - self.output.push(view.ndim() as u8); // ndarr datatype self.output.push(D::elem_type().into()); + // ndarr dims + self.output.push(view.ndim() as u8); let mut reserve_size = size_of::(); for i in 0..view.ndim() { From a31716afaf3dc94267350b4dbb6945a1de149590 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 3 Apr 2025 12:14:02 +0800 Subject: [PATCH 007/119] fix tests and compile error in ffi. --- questdb-rs-ffi/src/lib.rs | 12 +++++ questdb-rs/Cargo.toml | 4 +- questdb-rs/examples/basic.rs | 2 + questdb-rs/examples/http.rs | 2 + questdb-rs/src/ingress/mod.rs | 24 +++++---- questdb-rs/src/tests/mock.rs | 18 ++----- questdb-rs/src/tests/ndarr.rs | 63 ++++++++++++++++++++++-- questdb-rs/src/tests/sender.rs | 90 +++++++++++++++++++++++++++++++--- 8 files changed, 177 insertions(+), 38 deletions(-) diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 63904f66..3a5658ae 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -135,6 +135,15 @@ pub enum line_sender_error_code { /// Bad configuration. line_sender_error_config_error, + + /// Currently, only arrays with a maximum 32 dimensions are supported. + line_sender_error_array_large_dim, + + /// ArrayView internal error, such as failure to get the size of a valid dimension. + line_sender_error_array_view_internal_error, + + /// Buffer Out Of Memory + line_sender_error_buffer_out_of_memory, } impl From for line_sender_error_code { @@ -159,6 +168,9 @@ impl From for line_sender_error_code { line_sender_error_code::line_sender_error_server_flush_error } ErrorCode::ConfigError => line_sender_error_code::line_sender_error_config_error, + ErrorCode::ArrayHasTooManyDims => line_sender_error_code::line_sender_error_array_large_dim, + ErrorCode::ArrayViewError => line_sender_error_code::line_sender_error_array_view_internal_error, + ErrorCode::BufferOutOfMemory => line_sender_error_code::line_sender_error_buffer_out_of_memory, } } } diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index fee87082..53d2e90b 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -75,7 +75,7 @@ chrono_timestamp = ["chrono"] [[example]] name = "basic" -required-features = ["chrono_timestamp"] +required-features = ["chrono_timestamp", "ndarray"] [[example]] name = "auth" @@ -87,4 +87,4 @@ required-features = ["chrono_timestamp"] [[example]] name = "http" -required-features = ["ilp-over-http"] +required-features = ["ilp-over-http", "ndarray"] diff --git a/questdb-rs/examples/basic.rs b/questdb-rs/examples/basic.rs index 6fa665d3..5df32873 100644 --- a/questdb-rs/examples/basic.rs +++ b/questdb-rs/examples/basic.rs @@ -1,4 +1,5 @@ use chrono::{TimeZone, Utc}; +use ndarray::arr1; use questdb::{ ingress::{Buffer, Sender, TimestampNanos}, Result, @@ -17,6 +18,7 @@ fn main() -> Result<()> { .symbol("side", "sell")? .column_f64("price", 2615.54)? .column_f64("amount", 0.00044)? + .column_arr("location", &arr1(&[100.0, 100.1, 100.2]).view())? .at(designated_timestamp)?; //// If you want to pass the current system timestamp, replace with: diff --git a/questdb-rs/examples/http.rs b/questdb-rs/examples/http.rs index 74b2f3e9..3e11dd0f 100644 --- a/questdb-rs/examples/http.rs +++ b/questdb-rs/examples/http.rs @@ -1,3 +1,4 @@ +use ndarray::arr1; use questdb::{ ingress::{Buffer, Sender, TimestampNanos}, Result, @@ -12,6 +13,7 @@ fn main() -> Result<()> { .symbol("side", "sell")? .column_f64("price", 2615.54)? .column_f64("amount", 0.00044)? + .column_arr("location", &arr1(&[100.0, 100.1, 100.2]).view())? .at(TimestampNanos::now())?; sender.flush(&mut buffer)?; Ok(()) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 03d4b447..79df1f6d 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -1080,26 +1080,32 @@ impl Buffer { let mut reserve_size = size_of::(); for i in 0..view.ndim() { let d = view.dim(i).ok_or_else(|| { - error::fmt!(ArrayViewError, "Can not get correct dimensions for dim {}", i) + error::fmt!( + ArrayViewError, + "Can not get correct dimensions for dim {}", + i + ) })?; // ndarr shapes self.output .extend_from_slice((d as i32).to_le_bytes().as_slice()); - reserve_size = reserve_size - .checked_mul(d) - .ok_or(error::fmt!(ArrayViewError, "Array total elem size overflow"))? + reserve_size = reserve_size.checked_mul(d).ok_or(error::fmt!( + ArrayViewError, + "Array total elem size overflow" + ))? } let index = self.output.len(); - let reserve_size = reserve_size - .checked_add(index) - .ok_or(error::fmt!(ArrayViewError, "Array total elem size overflow"))?; + let reserve_size = reserve_size.checked_add(index).ok_or(error::fmt!( + ArrayViewError, + "Array total elem size overflow" + ))?; self.output .try_reserve(reserve_size) .map_err(|_| error::fmt!(BufferOutOfMemory, "Buffer out of memory"))?; unsafe { self.output.set_len(reserve_size) } - + // ndarr data view.write_row_major_buf(&mut self.output[index..]); Ok(self) @@ -2792,7 +2798,7 @@ impl Sender { } } -const ARRAY_BINARY_FORMAT_TYPE: u8 = 14; +pub(crate) const ARRAY_BINARY_FORMAT_TYPE: u8 = 14; mod conf; mod timestamp; diff --git a/questdb-rs/src/tests/mock.rs b/questdb-rs/src/tests/mock.rs index d673a2d1..7f207be5 100644 --- a/questdb-rs/src/tests/mock.rs +++ b/questdb-rs/src/tests/mock.rs @@ -50,7 +50,7 @@ pub struct MockServer { tls_conn: Option, pub host: &'static str, pub port: u16, - pub msgs: Vec, + pub msgs: Vec, } pub fn certs_dir() -> std::path::PathBuf { @@ -519,20 +519,8 @@ impl MockServer { } } - let mut received_count = 0usize; - let mut head = 0usize; - for index in 1..accum.len() { - let last = accum[index]; - let prev = accum[index - 1]; - if (last == b'\n') && (prev != b'\\') { - let tail = index + 1; - let msg = std::str::from_utf8(&accum[head..tail]).unwrap(); - self.msgs.push(msg.to_owned()); - head = tail; - received_count += 1; - } - } - Ok(received_count) + self.msgs = accum; + Ok(self.msgs.len()) } pub fn recv_q(&mut self) -> io::Result { diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index 31448131..f7dab239 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -1,10 +1,16 @@ -use crate::ingress::ndarr; -use crate::ingress::ndarr::NdArrayView; -use ndarray::{arr1, arr2, arr3, s}; +use crate::ingress::ndarr::{ElemDataType, NdArrayView, MAX_DIMS}; +use crate::ingress::{ndarr, Buffer}; +use crate::tests::TestResult; +use crate::{ingress, ErrorCode}; +use ndarray::{arr1, arr2, arr3, s, ArrayD}; +use std::iter; #[test] fn test_f64_element_type() { - assert_eq!(::elem_type(), ndarr::ElemDataType::Double); + assert_eq!( + ::elem_type(), + ndarr::ElemDataType::Double + ); assert_eq!(u8::from(ndarr::ElemDataType::Double), 10); } @@ -41,7 +47,7 @@ fn test_strided_layout() { [1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [9.0, 10.0, 11.0, 12.0], - [13.0, 14.0, 15.0, 16.0] + [13.0, 14.0, 15.0, 16.0], ]); let strided_view = array.slice(s![1..;2, 1..;2]); assert_eq!(strided_view.dim(), (2, 2)); @@ -77,3 +83,50 @@ fn test_complex_dimensions() { assert_eq!(NdArrayView::dim(&view, 1), Some(2)); assert_eq!(NdArrayView::dim(&view, 2), Some(1)); } + +#[test] +fn test_buffer_array_write() -> TestResult { + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + let array_2d = arr2(&[[1.1, 2.2], [3.3, 4.4]]); + buffer.column_arr("temperature", &array_2d.view())?; + + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); + assert_eq!( + &data[19..24], + &[ + b'=', + b'=', + ingress::ARRAY_BINARY_FORMAT_TYPE, + ElemDataType::Double.into(), + 2u8 + ] + ); + assert_eq!( + &data[24..32], + [2i32.to_le_bytes().as_slice(), 2i32.to_le_bytes().as_slice()].concat() + ); + Ok(()) +} + +#[test] +fn test_buffer_write_max_dimensions() -> TestResult { + let mut buffer = Buffer::new(); + buffer.table("nd_test")?; + let shape: Vec = iter::repeat(1).take(MAX_DIMS).collect(); + let array = ArrayD::::zeros(shape.clone()); + buffer.column_arr("max_dim", &array.view())?; + let data = buffer.as_bytes(); + assert_eq!(data[19], MAX_DIMS as u8); + + // 33 dims error + let shape_invalid: Vec<_> = iter::repeat(1).take(MAX_DIMS + 1).collect(); + let array_invalid = ArrayD::::zeros(shape_invalid); + let result = buffer.column_arr("invalid", &array_invalid.view()); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayHasTooManyDims); + Ok(()) +} diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index 1fda0df6..b69e3a0b 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -23,6 +23,7 @@ ******************************************************************************/ use crate::{ + ingress, ingress::{ Buffer, CertificateAuthority, Sender, TableName, Timestamp, TimestampMicros, TimestampNanos, }, @@ -34,9 +35,12 @@ use crate::tests::{ TestResult, }; +#[cfg(feature = "ndarray")] +use crate::ingress::ndarr::{ElemDataType, NdArrayView}; use core::time::Duration; #[cfg(feature = "ndarray")] -use ndarray::arr1; +use ndarray::{arr1, arr2, ArrayD}; + use std::{io, time::SystemTime}; #[test] @@ -79,8 +83,79 @@ fn test_basics() -> TestResult { sender.flush(&mut buffer)?; assert_eq!(buffer.len(), 0); assert_eq!(buffer.as_bytes(), b""); - assert_eq!(server.recv_q()?, 1); - assert_eq!(server.msgs[0].as_bytes(), exp_byte); + assert_eq!(server.recv_q()?, exp_byte.len()); + assert_eq!(server.msgs, exp_byte); + Ok(()) +} + +#[cfg(feature = "ndarray")] +#[test] +fn test_array_basic() -> TestResult { + let mut server = MockServer::new()?; + let mut sender = server.lsb_tcp().build()?; + server.accept()?; + + let ts = SystemTime::now(); + let ts_nanos_num = ts.duration_since(SystemTime::UNIX_EPOCH)?.as_nanos() as i64; + let ts_nanos = TimestampNanos::from_systemtime(ts)?; + let array_2d = arr2(&[[1.1, 2.2], [3.3, 4.4]]); + let array_3d = ArrayD::::ones(vec![2, 3, 4]); + + let mut buffer = Buffer::new(); + buffer + .table("my_table")? + .symbol("device", "A001")? + .column_f64("f1", 25.5)? + .column_arr("arr2d", &array_2d.view())? + .column_arr("arr3d", &array_3d.view())? + .at(ts_nanos)?; + + assert_eq!(server.recv_q()?, 0); + + let array_header2d = &[ + &[b'='][..], + &[ingress::ARRAY_BINARY_FORMAT_TYPE], + &[ElemDataType::Double as u8], + &[2u8], + &2i32.to_le_bytes(), + &2i32.to_le_bytes(), + ] + .concat(); + let mut array_data2d = vec![0u8; 4 * size_of::()]; + array_2d.view().write_row_major_buf(&mut array_data2d); + + let array_header3d = &[ + &[b'='][..], + &[ingress::ARRAY_BINARY_FORMAT_TYPE], + &[ElemDataType::Double as u8], + &[3u8], + &2i32.to_le_bytes(), + &3i32.to_le_bytes(), + &4i32.to_le_bytes(), + ] + .concat(); + let mut array_data3d = vec![0u8; 24 * size_of::()]; + array_3d.view().write_row_major_buf(&mut array_data3d); + + let exp = &[ + "my_table,device=A001 f1=25.5".as_bytes(), + ",arr2d=".as_bytes(), + array_header2d, + array_data2d.as_slice(), + ",arr3d=".as_bytes(), + array_header3d, + array_data3d.as_slice(), + format!(" {}\n", ts_nanos_num).as_bytes(), + ] + .concat(); + + assert_eq!(buffer.as_bytes(), exp); + assert_eq!(buffer.len(), exp.len()); + sender.flush(&mut buffer)?; + assert_eq!(buffer.len(), 0); + assert_eq!(buffer.as_bytes(), b""); + assert_eq!(server.recv_q()?, exp.len()); + assert_eq!(server.msgs.as_slice(), exp); Ok(()) } @@ -354,6 +429,7 @@ fn test_str_column_name_too_long() -> TestResult { } #[cfg(feature = "ndarray")] +#[test] fn test_arr_column_name_too_long() -> TestResult { column_name_too_long_test_impl!(column_arr, &arr1(&[1.0, 2.0, 3.0]).view()) } @@ -381,8 +457,8 @@ fn test_tls_with_file_ca() -> TestResult { assert_eq!(buffer.as_bytes(), exp); assert_eq!(buffer.len(), exp.len()); sender.flush(&mut buffer)?; - assert_eq!(server.recv_q()?, 1); - assert_eq!(server.msgs[0].as_bytes(), exp); + assert_eq!(server.recv_q()?, exp.len()); + assert_eq!(server.msgs.as_slice(), exp); Ok(()) } @@ -474,8 +550,8 @@ fn test_tls_insecure_skip_verify() -> TestResult { assert_eq!(buffer.as_bytes(), exp); assert_eq!(buffer.len(), exp.len()); sender.flush(&mut buffer)?; - assert_eq!(server.recv_q()?, 1); - assert_eq!(server.msgs[0].as_bytes(), exp); + assert_eq!(server.recv_q()?, exp.len()); + assert_eq!(server.msgs.as_slice(), exp); Ok(()) } From 463a2d3a1752f824d15ed60811e9f7715b9e0975 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 3 Apr 2025 14:21:57 +0800 Subject: [PATCH 008/119] format code. --- questdb-rs-ffi/src/lib.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 3a5658ae..73f1928e 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -168,9 +168,15 @@ impl From for line_sender_error_code { line_sender_error_code::line_sender_error_server_flush_error } ErrorCode::ConfigError => line_sender_error_code::line_sender_error_config_error, - ErrorCode::ArrayHasTooManyDims => line_sender_error_code::line_sender_error_array_large_dim, - ErrorCode::ArrayViewError => line_sender_error_code::line_sender_error_array_view_internal_error, - ErrorCode::BufferOutOfMemory => line_sender_error_code::line_sender_error_buffer_out_of_memory, + ErrorCode::ArrayHasTooManyDims => { + line_sender_error_code::line_sender_error_array_large_dim + } + ErrorCode::ArrayViewError => { + line_sender_error_code::line_sender_error_array_view_internal_error + } + ErrorCode::BufferOutOfMemory => { + line_sender_error_code::line_sender_error_buffer_out_of_memory + } } } } From 44fae08a1ae82d7afdc51c08a840cb567b15d118 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 3 Apr 2025 16:01:12 +0800 Subject: [PATCH 009/119] ffi supported --- questdb-rs-ffi/src/lib.rs | 93 +++++++++++++++++++++++++++++++++-- questdb-rs/src/ingress/mod.rs | 5 +- 2 files changed, 92 insertions(+), 6 deletions(-) diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 73f1928e..20e30a17 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -24,7 +24,7 @@ #![allow(non_camel_case_types, clippy::missing_safety_doc)] -use libc::{c_char, size_t}; +use libc::{c_char, c_int, size_t}; use std::ascii; use std::boxed::Box; use std::convert::{From, Into}; @@ -35,8 +35,8 @@ use std::str; use questdb::{ ingress::{ - Buffer, CertificateAuthority, ColumnName, Protocol, Sender, SenderBuilder, TableName, - TimestampMicros, TimestampNanos, + ArrayElement, Buffer, CertificateAuthority, ColumnName, NdArrayView, Protocol, Sender, + SenderBuilder, TableName, TimestampMicros, TimestampNanos, }, Error, ErrorCode, }; @@ -309,6 +309,56 @@ impl line_sender_utf8 { } } +#[derive(Debug, Copy, Clone)] +struct line_sender_array { + dims: size_t, + shapes: *const i32, + buf_len: size_t, + buf: *const u8, +} + +impl NdArrayView for line_sender_array +where + T: ArrayElement, +{ + fn ndim(&self) -> usize { + self.dims + } + + fn dim(&self, index: usize) -> Option { + if index >= self.dims { + return None; + } + + unsafe { + if self.shapes.is_null() { + return None; + } + + let dim_size = *self.shapes.add(index); + if dim_size < 0 { + None + } else { + Some(dim_size as usize) + } + } + } + + fn write_row_major_buf(&self, buff: &mut [u8]) { + let elem_size = size_of::(); + let total_elements: usize = (0..self.dims) + .filter_map(|i| >::dim(self, i)) + .product(); + assert_eq!( + self.buf_len, + total_elements * elem_size, + "Buffer length mismatch" + ); + let bytes = unsafe { slice::from_raw_parts(self.buf, self.buf_len) }; + buff[..self.buf_len].copy_from_slice(bytes); + } +} + /// An ASCII-safe description of a binary buffer. Trimmed if too long. fn describe_buf(buf: &[u8]) -> String { let max_len = 100usize; @@ -822,6 +872,43 @@ pub unsafe extern "C" fn line_sender_buffer_column_str( true } +/// Record a float multidimensional array value for the given column. +/// @param[in] buffer Line buffer object. +/// @param[in] name Column name. +/// @param[in] rank Array dims. +/// @param[in] shape Array shapes. +/// @param[in] data_buffer Array data memory ptr. +/// @param[in] data_buffer_len Array data memory length. +/// @param[out] err_out Set on error. +/// # Safety +/// - All pointer parameters must be valid and non-null +/// - shape must point to an array of `rank` integers +/// - data_buffer must point to a buffer of size `data_buffer_len` bytes +#[no_mangle] +pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( + buffer: *mut line_sender_buffer, + name: line_sender_column_name, + rank: size_t, + shape: *const c_int, // C array of dimension sizes + data_buffer: *const u8, // Raw array data + data_buffer_len: size_t, // Total bytes length + err_out: *mut *mut line_sender_error, +) -> bool { + let buffer = unwrap_buffer_mut(buffer); + let name = name.as_name(); + let view = line_sender_array { + dims: rank, + shapes: shape, + buf_len: data_buffer_len, + buf: data_buffer, + }; + bubble_err_to_c!( + err_out, + buffer.column_arr::, line_sender_array, f64>(name, &view) + ); + true +} + /// Record a nanosecond timestamp value for the given column. /// @param[in] buffer Line buffer object. /// @param[in] name Column name. diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 79df1f6d..52185ff9 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -24,8 +24,8 @@ #![doc = include_str!("mod.md")] +pub use self::ndarr::*; pub use self::timestamp::*; - use crate::error::{self, Error, Result}; use crate::gai; use crate::ingress::conf::ConfigSetting; @@ -39,7 +39,6 @@ use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; -use crate::ingress::ndarr::{ArrayElement, NdArrayView, MAX_DIMS}; use base64ct::{Base64, Base64UrlUnpadded, Encoding}; use ring::rand::SystemRandom; use ring::signature::{EcdsaKeyPair, ECDSA_P256_SHA256_FIXED_SIGNING}; @@ -2802,6 +2801,7 @@ pub(crate) const ARRAY_BINARY_FORMAT_TYPE: u8 = 14; mod conf; mod timestamp; +pub(crate) mod ndarr; #[cfg(feature = "ilp-over-http")] mod http; @@ -2809,6 +2809,5 @@ mod http; #[cfg(feature = "ilp-over-http")] use http::*; -pub(crate) mod ndarr; #[cfg(test)] mod tests; From 93c6f9decbc3514654c33bcaef1b255433fa5cff Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 3 Apr 2025 16:57:30 +0800 Subject: [PATCH 010/119] ndarray c api --- include/questdb/ingress/line_sender.h | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index cbfa225b..2bd1439b 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -461,6 +461,32 @@ bool line_sender_buffer_column_str( line_sender_utf8 value, line_sender_error** err_out); +/** + * Record a multidimensional array of double for the given column. + * The array data must be stored in row-major order (C-style contiguous layout). + * + * @param[in] buffer Line buffer object. + * @param[in] name Column name. + * @param[in] rank Number of dimensions of the array. + * @param[in] shapes Array of dimension sizes (length = `rank`). + * Each element must be a positive integer. + * @param[in] data_buffer Raw bytes of the array data in little-endian format. + * Size must be `sizeof(double) * (shapes[0] * ... * + * shapes[rank-1])`. + * @param[in] data_buffer_len Byte length of the data buffer. + * @param[out] err_out Set to an error object on failure (if non-NULL). + * @return true on success, false on error. + */ +LINESENDER_API +bool line_sender_buffer_column_f64_arr( + line_sender_buffer* buffer, + line_sender_column_name name, + size_t rank, + const int32_t* shapes, + const uint8_t* data_buffer, + size_t data_buffer_len, + line_sender_error** err_out); + /** * Record a nanosecond timestamp value for the given column. * @param[in] buffer Line buffer object. From e9da55875bbdd3406ad5febe73523a1dc8893934 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 3 Apr 2025 17:05:10 +0800 Subject: [PATCH 011/119] format --- questdb-rs/src/ingress/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 52185ff9..6bdc9fc2 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -2800,8 +2800,8 @@ impl Sender { pub(crate) const ARRAY_BINARY_FORMAT_TYPE: u8 = 14; mod conf; -mod timestamp; pub(crate) mod ndarr; +mod timestamp; #[cfg(feature = "ilp-over-http")] mod http; From 0e8a9e5f7f1ff8ff4157cd8a98a6e902dbce17f1 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 3 Apr 2025 21:36:58 +0800 Subject: [PATCH 012/119] c++ api support. --- include/questdb/ingress/line_sender.h | 2 +- include/questdb/ingress/line_sender.hpp | 34 +++++++++++++++++++++++++ questdb-rs-ffi/src/lib.rs | 12 +++------ 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 2bd1439b..0bd00ec8 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -482,7 +482,7 @@ bool line_sender_buffer_column_f64_arr( line_sender_buffer* buffer, line_sender_column_name name, size_t rank, - const int32_t* shapes, + const uint32_t* shapes, const uint8_t* data_buffer, size_t data_buffer_len, line_sender_error** err_out); diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 19ee2e15..5c2063e6 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -26,6 +26,7 @@ #include "line_sender.h" +#include #include #include #include @@ -624,6 +625,39 @@ class line_sender_buffer return *this; } + /** + * Record a multidimensional double-precision array for the given column. + * + * @param name Column name. + * @param shape Array dimensions (e.g., [2,3] for a 2x3 matrix). + * @param data Array data in row-major order. Size must match product of + * dimensions. + * + * @note Data is stored contiguously in row-major (C-style) order. + * Example: shape [2,3] expects 6 elements ordered as: + * [a11, a12, a13, a21, a22, a23] + */ + template + line_sender_buffer& column( + column_name_view name, + const std::vector& shape, + const std::array& data) + { + static_assert( + std::is_same_v, + "Only double types are supported for arrays"); + may_init(); + line_sender_error::wrapped_call( + ::line_sender_buffer_column_f64_arr, + _impl, + name._impl, + shape.size(), + shape.data(), + reinterpret_cast(data.data()), + sizeof(double) * N); + return *this; + } + /** * Record a string value for the given column. * @param name Column name. diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 20e30a17..1378405d 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -24,7 +24,7 @@ #![allow(non_camel_case_types, clippy::missing_safety_doc)] -use libc::{c_char, c_int, size_t}; +use libc::{c_char, c_uint, size_t}; use std::ascii; use std::boxed::Box; use std::convert::{From, Into}; @@ -312,7 +312,7 @@ impl line_sender_utf8 { #[derive(Debug, Copy, Clone)] struct line_sender_array { dims: size_t, - shapes: *const i32, + shapes: *const u32, buf_len: size_t, buf: *const u8, } @@ -336,11 +336,7 @@ where } let dim_size = *self.shapes.add(index); - if dim_size < 0 { - None - } else { - Some(dim_size as usize) - } + Some(dim_size as usize) } } @@ -889,7 +885,7 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( buffer: *mut line_sender_buffer, name: line_sender_column_name, rank: size_t, - shape: *const c_int, // C array of dimension sizes + shape: *const c_uint, // C array of dimension sizes data_buffer: *const u8, // Raw array data data_buffer_len: size_t, // Total bytes length err_out: *mut *mut line_sender_error, From 261163cb05dd37fbab1561f351ddc5e59dc483cc Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 3 Apr 2025 23:50:34 +0800 Subject: [PATCH 013/119] add include. --- include/questdb/ingress/line_sender.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 5c2063e6..19945589 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -34,6 +34,7 @@ #include #include #include +#include #if __cplusplus >= 202002L # include #endif From a565109a850766f6b5e8f3a041744cf26c80174c Mon Sep 17 00:00:00 2001 From: victor Date: Fri, 4 Apr 2025 00:37:32 +0800 Subject: [PATCH 014/119] code format. --- questdb-rs-ffi/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 1378405d..3b0a2476 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -885,7 +885,7 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( buffer: *mut line_sender_buffer, name: line_sender_column_name, rank: size_t, - shape: *const c_uint, // C array of dimension sizes + shape: *const c_uint, // C array of dimension sizes data_buffer: *const u8, // Raw array data data_buffer_len: size_t, // Total bytes length err_out: *mut *mut line_sender_error, From e161b1daf146062fb4033add6081ff58c7055420 Mon Sep 17 00:00:00 2001 From: victor Date: Sun, 6 Apr 2025 23:01:23 +0800 Subject: [PATCH 015/119] conflict resolve --- questdb-rs/Cargo.toml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index 90ace7f3..6601d9b3 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -39,6 +39,7 @@ ureq = { version = "3.0.10, <3.1.0", default-features = false, features = ["rust serde_json = { version = "1", optional = true } questdb-confstr = "0.1.1" rand = { version = "0.9.0", optional = true } +ndarray = { version = "0.16", optional = true } no-panic = { version = "0.1", optional = true } [target.'cfg(windows)'.dependencies] @@ -93,12 +94,13 @@ almost-all-features = [ "aws-lc-crypto", "insecure-skip-verify", "json_tests", - "chrono_timestamp" + "chrono_timestamp", + "ndarray" ] [[example]] name = "basic" -required-features = ["chrono_timestamp"] +required-features = ["chrono_timestamp", "ndarray"] [[example]] name = "auth" @@ -110,4 +112,4 @@ required-features = ["chrono_timestamp"] [[example]] name = "http" -required-features = ["ilp-over-http"] +required-features = ["ilp-over-http", "ndarray"] From 4a59ec045e7b386b6ffb309f2d46bfcd1ee8160a Mon Sep 17 00:00:00 2001 From: victor Date: Mon, 7 Apr 2025 15:22:46 +0800 Subject: [PATCH 016/119] code review. --- questdb-rs-ffi/Cargo.lock | 81 +++++++++ questdb-rs-ffi/src/lib.rs | 22 +-- questdb-rs/Cargo.toml | 3 +- questdb-rs/src/error.rs | 6 +- questdb-rs/src/ingress/mod.rs | 41 +++-- questdb-rs/src/ingress/ndarr.rs | 80 +++------ questdb-rs/src/tests/mod.rs | 1 - questdb-rs/src/tests/ndarr.rs | 297 ++++++++++++++++++++++++++++++-- questdb-rs/src/tests/sender.rs | 8 +- 9 files changed, 428 insertions(+), 111 deletions(-) diff --git a/questdb-rs-ffi/Cargo.lock b/questdb-rs-ffi/Cargo.lock index 5666057c..248ab0c9 100644 --- a/questdb-rs-ffi/Cargo.lock +++ b/questdb-rs-ffi/Cargo.lock @@ -11,6 +11,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + [[package]] name = "aws-lc-rs" version = "1.13.0" @@ -374,6 +380,16 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +[[package]] +name = "matrixmultiply" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9380b911e3e96d10c1f415da0876389aaf1b56759054eeb0de7df940c456ba1a" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "memchr" version = "2.7.4" @@ -386,6 +402,21 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "ndarray" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + [[package]] name = "nom" version = "7.1.3" @@ -396,6 +427,33 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -414,6 +472,21 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "portable-atomic" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -467,6 +540,8 @@ dependencies = [ "indoc", "itoa", "libc", + "log", + "ndarray", "questdb-confstr", "rand", "rustls", @@ -538,6 +613,12 @@ dependencies = [ "getrandom 0.3.2", ] +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "regex" version = "1.11.1" diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index d78c2e86..3d929e6c 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -142,8 +142,8 @@ pub enum line_sender_error_code { /// ArrayView internal error, such as failure to get the size of a valid dimension. line_sender_error_array_view_internal_error, - /// Buffer Out Of Memory - line_sender_error_buffer_out_of_memory, + /// Write arrayView to sender buffer error. + line_sender_error_array_view_write_to_buffer_error } impl From for line_sender_error_code { @@ -174,8 +174,8 @@ impl From for line_sender_error_code { ErrorCode::ArrayViewError => { line_sender_error_code::line_sender_error_array_view_internal_error } - ErrorCode::BufferOutOfMemory => { - line_sender_error_code::line_sender_error_buffer_out_of_memory + ErrorCode::ArrayWriteToBufferError => { + line_sender_error_code::line_sender_error_array_view_write_to_buffer_error } } } @@ -340,18 +340,10 @@ where } } - fn write_row_major_buf(&self, buff: &mut [u8]) { - let elem_size = size_of::(); - let total_elements: usize = (0..self.dims) - .filter_map(|i| >::dim(self, i)) - .product(); - assert_eq!( - self.buf_len, - total_elements * elem_size, - "Buffer length mismatch" - ); + fn write_row_major(&self, writer: &mut W) -> std::io::Result<()> { let bytes = unsafe { slice::from_raw_parts(self.buf, self.buf_len) }; - buff[..self.buf_len].copy_from_slice(bytes); + writer.write_all(bytes)?; + Ok(()) } } diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index 6601d9b3..50649d37 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -41,6 +41,7 @@ questdb-confstr = "0.1.1" rand = { version = "0.9.0", optional = true } ndarray = { version = "0.16", optional = true } no-panic = { version = "0.1", optional = true } +log = "0.4.27" [target.'cfg(windows)'.dependencies] winapi = { version = "0.3.9", features = ["ws2def"] } @@ -58,7 +59,7 @@ tempfile = "3" webpki-roots = "0.26.8" [features] -default = ["tls-webpki-certs", "ilp-over-http", "aws-lc-crypto"] +default = ["tls-webpki-certs", "ilp-over-http", "aws-lc-crypto", "ndarray"] # Include support for ILP over HTTP. ilp-over-http = ["dep:ureq", "dep:serde_json", "dep:rand"] diff --git a/questdb-rs/src/error.rs b/questdb-rs/src/error.rs index 2958c16d..58f31e74 100644 --- a/questdb-rs/src/error.rs +++ b/questdb-rs/src/error.rs @@ -49,14 +49,14 @@ pub enum ErrorCode { /// Bad configuration. ConfigError, - /// Array has too many dims. Currently, only arrays with a maximum [`crate::ingress::ndarr::MAX_DIMS`] dimensions are supported. + /// Array has too many dims. Currently, only arrays with a maximum [`crate::ingress::MAX_DIMS`] dimensions are supported. ArrayHasTooManyDims, /// Array view internal error. ArrayViewError, - /// Buffer outOfMemory. - BufferOutOfMemory, + /// Array write to buffer error + ArrayWriteToBufferError, } /// An error that occurred when using QuestDB client library. diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 50ed4d23..cc7d98fe 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -24,7 +24,7 @@ #![doc = include_str!("mod.md")] -pub use self::ndarr::*; +pub use self::ndarr::{ArrayElement, ElemDataType, NdArrayView}; pub use self::timestamp::*; use crate::error::{self, Error, Result}; use crate::gai; @@ -37,9 +37,10 @@ use socket2::{Domain, Protocol as SockProtocol, SockAddr, Socket, Type}; use std::collections::HashMap; use std::convert::Infallible; use std::fmt::{Debug, Display, Formatter, Write}; -use std::io::{self, BufRead, BufReader, ErrorKind, Write as IoWrite}; +use std::io::{self, BufRead, BufReader, Cursor, ErrorKind, Write as IoWrite}; use std::ops::Deref; use std::path::PathBuf; +use std::slice::from_raw_parts_mut; use std::str::FromStr; use std::sync::Arc; @@ -55,6 +56,9 @@ use ring::{ signature::{EcdsaKeyPair, ECDSA_P256_SHA256_FIXED_SIGNING}, }; +/// Defines the maximum allowed dimensions for array data in binary serialization protocols. +pub const MAX_DIMS: usize = 32; + #[derive(Debug, Copy, Clone)] enum Op { Table = 1, @@ -1055,7 +1059,6 @@ impl Buffer { /// Returns [`Error`] if: /// - Array dimensions exceed [`MAX_DIMS`] /// - Failed to get dimension sizes - /// - Memory allocation fails for array data /// - Column name validation fails pub fn column_arr<'a, N, T, D>(&mut self, name: N, view: &T) -> Result<&mut Self> where @@ -1104,18 +1107,30 @@ impl Buffer { ))? } + self.output.reserve(reserve_size); let index = self.output.len(); - let reserve_size = reserve_size.checked_add(index).ok_or(error::fmt!( - ArrayViewError, - "Array total elem size overflow" - ))?; - self.output - .try_reserve(reserve_size) - .map_err(|_| error::fmt!(BufferOutOfMemory, "Buffer out of memory"))?; - unsafe { self.output.set_len(reserve_size) } - + let writeable = + unsafe { from_raw_parts_mut(self.output.as_mut_ptr().add(index), reserve_size) }; + let mut cursor = Cursor::new(writeable); // ndarr data - view.write_row_major_buf(&mut self.output[index..]); + if let Err(e) = view.write_row_major(&mut cursor) { + return Err(error::fmt!( + ArrayWriteToBufferError, + "Can not write row major to writer: {}", + e + )); + } + + if cursor.position() != (reserve_size as u64) { + return Err(error::fmt!( + ArrayWriteToBufferError, + "Array write buffer length mismatch (actual: {}, expected: {})", + cursor.position(), + reserve_size + )); + } + + unsafe { self.output.set_len(reserve_size + index) } Ok(self) } diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 222db184..2c0fd0ad 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -1,5 +1,3 @@ -pub(crate) const MAX_DIMS: usize = 32; - pub trait NdArrayView where T: ArrayElement, @@ -10,13 +8,13 @@ where /// Returns the size of the specified dimension. fn dim(&self, index: usize) -> Option; - /// Writes array data to buffer in row-major order. + /// Writes array data to writer in row-major order. /// /// # Important Notes - /// - Buffer must be pre-allocated with exact required size - /// - No alignment assumptions should be made about buffer start + /// - Writer must be pre-allocated with exact required size + /// - No alignment assumptions should be made about writer start /// - Handles both contiguous and non-contiguous memory layouts - fn write_row_major_buf(&self, buff: &mut [u8]); + fn write_row_major(&self, writer: &mut W) -> std::io::Result<()>; } /// Marker trait for valid array element types. @@ -31,49 +29,13 @@ pub trait ArrayElement: Copy + 'static { fn elem_type() -> ElemDataType; } +/// Defines binary format identifiers for array element types compatible with +/// QuestDB's [`ColumnType`]: https://github.com/questdb/questdb/blob/e1853db56ae586d923ca77de01a487cad44093b9/core/src/main/java/io/questdb/cairo/ColumnType.java#L67-L89. #[repr(u8)] -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone, Copy)] pub enum ElemDataType { - /// Uninitialized/placeholder type - Undefined = 0, - /// Boolean values (true/false) - Boolean, - /// 8-bit signed integer - Byte, - /// 16-bit signed integer - Short, - /// UTF-16 character - Char, - /// 32-bit signed integer - Int, - /// 64-bit signed integer - Long, - /// Date type (days since epoch) - Date, - /// Microsecond-precision timestamp - Timestamp, - /// 32-bit floating point - Float, /// 64-bit floating point - Double, - /// UTF-8 string data - String, - /// Interned string symbol - Symbol, - /// 256-bit integer value - Long256, - /// Geospatial byte coordinates - GeoByte, - /// Geospatial short coordinates - GeoShort, - /// Geospatial integer coordinates - GeoInt, - /// Geospatial long coordinates - GeoLong, - /// Binary large object - Binary, - /// UUID values - Uuid, + Double = 0x0A, } impl From for u8 { @@ -111,23 +73,23 @@ where } } - fn write_row_major_buf(&self, buf: &mut [u8]) { - let elem_size = size_of::(); - - if let Some(slice) = self.as_slice() { - let byte_len = size_of_val(slice); - let bytes = - unsafe { std::slice::from_raw_parts(slice.as_ptr() as *const u8, byte_len) }; - buf[..byte_len].copy_from_slice(bytes); - return; + fn write_row_major(&self, writer: &mut W) -> std::io::Result<()> { + if let Some(contiguous) = self.as_slice() { + let bytes = unsafe { + std::slice::from_raw_parts( + contiguous.as_ptr() as *const u8, + size_of_val(contiguous), + ) + }; + return writer.write_all(bytes); } - let mut bytes_written = 0; + let elem_size = size_of::(); for &element in self.iter() { - let element_bytes = + let bytes = unsafe { std::slice::from_raw_parts(&element as *const T as *const _, elem_size) }; - buf[bytes_written..bytes_written + elem_size].copy_from_slice(element_bytes); - bytes_written += elem_size; + writer.write_all(bytes)?; } + Ok(()) } } diff --git a/questdb-rs/src/tests/mod.rs b/questdb-rs/src/tests/mod.rs index f06dba8c..29c1b3f4 100644 --- a/questdb-rs/src/tests/mod.rs +++ b/questdb-rs/src/tests/mod.rs @@ -29,7 +29,6 @@ mod http; mod mock; mod sender; -#[cfg(feature = "ndarray")] mod ndarr; #[cfg(feature = "json_tests")] diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index f7dab239..886bde63 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -1,48 +1,306 @@ -use crate::ingress::ndarr::{ElemDataType, NdArrayView, MAX_DIMS}; -use crate::ingress::{ndarr, Buffer}; +use crate::ingress::{ + ArrayElement, Buffer, ElemDataType, NdArrayView, ARRAY_BINARY_FORMAT_TYPE, MAX_DIMS, +}; use crate::tests::TestResult; -use crate::{ingress, ErrorCode}; +use crate::ErrorCode; + +#[cfg(feature = "ndarray")] use ndarray::{arr1, arr2, arr3, s, ArrayD}; +#[cfg(feature = "ndarray")] use std::iter; +use std::marker::PhantomData; #[test] fn test_f64_element_type() { + assert_eq!(::elem_type(), ElemDataType::Double); + assert_eq!(u8::from(ElemDataType::Double), 10); +} + +struct Array2D { + data: Vec, + rows: usize, + cols: usize, + contiguous: bool, +} + +impl Array2D { + fn new(data: Vec, rows: usize, cols: usize, contiguous: bool) -> Self { + Self { + data, + rows, + cols, + contiguous, + } + } +} + +impl NdArrayView for Array2D { + fn ndim(&self) -> usize { + 2 + } + + fn dim(&self, index: usize) -> Option { + match index { + 0 => Some(self.rows), + 1 => Some(self.cols), + _ => None, + } + } + + fn write_row_major(&self, writer: &mut W) -> std::io::Result<()> { + if self.contiguous { + let bytes = unsafe { + std::slice::from_raw_parts( + self.data.as_ptr() as *const u8, + self.data.len() * std::mem::size_of::(), + ) + }; + return writer.write_all(bytes); + } + + for chunk in self.data.chunks(self.cols) { + let bytes = unsafe { + std::slice::from_raw_parts( + chunk.as_ptr() as *const u8, + chunk.len() * std::mem::size_of::(), + ) + }; + writer.write_all(bytes)?; + } + Ok(()) + } +} + +fn to_bytes(data: &[T]) -> Vec { + data.iter() + .flat_map(|x| { + let bytes = + unsafe { std::slice::from_raw_parts(x as *const T as *const u8, size_of::()) }; + bytes.to_vec() + }) + .collect() +} + +#[test] +fn test_basic_array_view() { + // contiguous layout + let test_data = vec![1.0f64, 2.0, 3.0, 4.0, 5.0, 6.0]; + let array = Array2D::new(test_data.clone(), 2, 3, true); + assert_eq!(array.ndim(), 2); + assert_eq!(array.dim(0), Some(2)); + assert_eq!(array.dim(1), Some(3)); + assert_eq!(array.dim(2), None); + let mut buf = vec![]; + array.write_row_major(&mut buf).unwrap(); + let expected = to_bytes(&test_data); + assert_eq!(buf, expected); + + // non-contiguous layout + let test_data = vec![vec![1.0, 2.0], vec![3.0, 4.0], vec![5.0, 6.0]] + .into_iter() + .flatten() + .collect(); + let array_non_contig = Array2D::new(test_data, 3, 2, false); + let mut buf_non_contig = vec![]; + array_non_contig + .write_row_major(&mut buf_non_contig) + .unwrap(); + let expected_non_contig = to_bytes(&[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]); + assert_eq!(buf_non_contig, expected_non_contig); +} + +#[test] +fn test_basic_edge_cases() { + // empty array + let empty_array = Array2D::::new(vec![], 0, 0, false); + assert_eq!(empty_array.ndim(), 2); + assert_eq!(empty_array.dim(0), Some(0)); + assert_eq!(empty_array.dim(1), Some(0)); + + // single element array + let single = Array2D::new(vec![42.0], 1, 1, true); + let mut buf = vec![]; + single.write_row_major(&mut buf).unwrap(); + assert_eq!(buf, 42.0f64.to_ne_bytes().to_vec()); +} + +#[test] +fn test_buffer_basic_write() -> TestResult { + let test_data = vec![1.1f64, 2.2, 3.3, 4.4]; + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + let array_2d = Array2D::::new(test_data, 2, 2, true); + buffer.column_arr("temperature", &array_2d)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); assert_eq!( - ::elem_type(), - ndarr::ElemDataType::Double + &data[19..24], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ElemDataType::Double.into(), + 2u8 + ] ); - assert_eq!(u8::from(ndarr::ElemDataType::Double), 10); + assert_eq!( + &data[24..32], + [2i32.to_le_bytes().as_slice(), 2i32.to_le_bytes().as_slice()].concat() + ); + Ok(()) } #[test] -fn test_1d_contiguous_buffer() { +fn test_invalid_dimension() -> TestResult { + struct InvalidDimArray; + impl NdArrayView for InvalidDimArray { + fn ndim(&self) -> usize { + 2 + } + fn dim(&self, _: usize) -> Option { + None + } + fn write_row_major(&self, _: &mut W) -> std::io::Result<()> { + todo!() + } + } + + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + let result = buffer.column_arr("arr1", &InvalidDimArray); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert!(err + .msg() + .contains("Can not get correct dimensions for dim 0")); + Ok(()) +} + +#[test] +fn test_size_overflow() -> TestResult { + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + let data = vec![1.0f64]; + let arr = Array2D:: { + data, + rows: usize::MAX, + cols: usize::MAX, + contiguous: false, + }; + + let result = buffer.column_arr("arr1", &arr); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert!(err.msg().contains("Array total elem size overflow")); + Ok(()) +} + +#[test] +fn test_write_failure() -> TestResult { + struct FaultyArray(PhantomData); + impl NdArrayView for FaultyArray { + fn ndim(&self) -> usize { + 2 + } + fn dim(&self, _: usize) -> Option { + Some(1) + } + fn write_row_major(&self, _: &mut W) -> std::io::Result<()> { + Err(std::io::Error::new( + std::io::ErrorKind::Other, + "mock write error", + )) + } + } + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + let result = buffer.column_arr("arr1", &FaultyArray(PhantomData::)); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayWriteToBufferError); + assert!(err + .msg() + .contains("Can not write row major to writer: mock write error")); + Ok(()) +} + +#[test] +fn test_array_length_mismatch() -> TestResult { + // actual data length is larger than shapes + let test_data = vec![1.1f64, 2.2, 3.3, 4.4]; + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + let array_2d = Array2D::::new(test_data, 1, 2, true); + let result = buffer.column_arr("arr1", &array_2d); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayWriteToBufferError); + assert!(err + .msg() + .contains("Can not write row major to writer: failed to write whole buffer")); + buffer.clear(); + + // actual data length is less than shapes + let test_data = vec![1.1f64]; + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + let array_2d = Array2D::::new(test_data, 1, 2, true); + let result = buffer.column_arr("arr1", &array_2d); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayWriteToBufferError); + assert!(err + .msg() + .contains("Array write buffer length mismatch (actual: 8, expected: 16)")); + buffer.clear(); + + // non-contiguous layout + let test_data = vec![1.1f64]; + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + let array_2d = Array2D::::new(test_data, 1, 2, false); + let result = buffer.column_arr("arr1", &array_2d); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayWriteToBufferError); + assert!(err + .msg() + .contains("Array write buffer length mismatch (actual: 8, expected: 16)")); + Ok(()) +} + +#[cfg(feature = "ndarray")] +#[test] +fn test_1d_contiguous_ndarray_buffer() -> TestResult { let array = arr1(&[1.0, 2.0, 3.0, 4.0]); let view = array.view(); let mut buf = vec![0u8; 4 * size_of::()]; - view.write_row_major_buf(&mut buf); + view.write_row_major(&mut &mut buf[0..])?; let expected: Vec = array .iter() .flat_map(|&x| x.to_ne_bytes().to_vec()) .collect(); assert_eq!(buf, expected); + Ok(()) } +#[cfg(feature = "ndarray")] #[test] -fn test_2d_non_contiguous_buffer() { +fn test_2d_non_contiguous_ndarray_buffer() -> TestResult { let array = arr2(&[[1.0, 2.0], [3.0, 4.0]]); let transposed = array.view().reversed_axes(); assert!(!transposed.is_standard_layout()); let mut buf = vec![0u8; 4 * size_of::()]; - transposed.write_row_major_buf(&mut buf); + transposed.write_row_major(&mut &mut buf[0..])?; let expected = [1.0f64, 3.0, 2.0, 4.0] .iter() .flat_map(|&x| x.to_ne_bytes()) .collect::>(); assert_eq!(buf, expected); + Ok(()) } +#[cfg(feature = "ndarray")] #[test] -fn test_strided_layout() { +fn test_strided_ndarray_layout() -> TestResult { let array = arr2(&[ [1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], @@ -52,7 +310,7 @@ fn test_strided_layout() { let strided_view = array.slice(s![1..;2, 1..;2]); assert_eq!(strided_view.dim(), (2, 2)); let mut buf = vec![0u8; 4 * size_of::()]; - strided_view.write_row_major_buf(&mut buf); + strided_view.write_row_major(&mut &mut buf[0..])?; // expect：6.0, 8.0, 14.0, 16.0 let expected = [6.0f64, 8.0, 14.0, 16.0] @@ -61,10 +319,12 @@ fn test_strided_layout() { .collect::>(); assert_eq!(buf, expected); + Ok(()) } +#[cfg(feature = "ndarray")] #[test] -fn test_1d_dimension_info() { +fn test_1d_dimension_ndarray_info() { let array = arr1(&[1.0, 2.0, 3.0]); let view = array.view(); @@ -73,8 +333,9 @@ fn test_1d_dimension_info() { assert_eq!(NdArrayView::dim(&view, 1), None); } +#[cfg(feature = "ndarray")] #[test] -fn test_complex_dimensions() { +fn test_complex_ndarray_dimensions() { let array = arr3(&[[[1.0], [2.0]], [[3.0], [4.0]]]); let view = array.view(); @@ -84,8 +345,9 @@ fn test_complex_dimensions() { assert_eq!(NdArrayView::dim(&view, 2), Some(1)); } +#[cfg(feature = "ndarray")] #[test] -fn test_buffer_array_write() -> TestResult { +fn test_buffer_ndarray_write() -> TestResult { let mut buffer = Buffer::new(); buffer.table("my_test")?; let array_2d = arr2(&[[1.1, 2.2], [3.3, 4.4]]); @@ -99,7 +361,7 @@ fn test_buffer_array_write() -> TestResult { &[ b'=', b'=', - ingress::ARRAY_BINARY_FORMAT_TYPE, + ARRAY_BINARY_FORMAT_TYPE, ElemDataType::Double.into(), 2u8 ] @@ -111,8 +373,9 @@ fn test_buffer_array_write() -> TestResult { Ok(()) } +#[cfg(feature = "ndarray")] #[test] -fn test_buffer_write_max_dimensions() -> TestResult { +fn test_buffer_write_ndarray_max_dimensions() -> TestResult { let mut buffer = Buffer::new(); buffer.table("nd_test")?; let shape: Vec = iter::repeat(1).take(MAX_DIMS).collect(); diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index 66b8cefd..665bc449 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -122,7 +122,9 @@ fn test_array_basic() -> TestResult { ] .concat(); let mut array_data2d = vec![0u8; 4 * size_of::()]; - array_2d.view().write_row_major_buf(&mut array_data2d); + array_2d + .view() + .write_row_major(&mut &mut array_data2d[0..])?; let array_header3d = &[ &[b'='][..], @@ -135,7 +137,9 @@ fn test_array_basic() -> TestResult { ] .concat(); let mut array_data3d = vec![0u8; 24 * size_of::()]; - array_3d.view().write_row_major_buf(&mut array_data3d); + array_3d + .view() + .write_row_major(&mut &mut array_data3d[0..])?; let exp = &[ "my_table,device=A001 f1=25.5".as_bytes(), From 0c513b1498853370583e848c137e648918c7da30 Mon Sep 17 00:00:00 2001 From: victor Date: Mon, 7 Apr 2025 16:06:33 +0800 Subject: [PATCH 017/119] code format. --- questdb-rs/src/tests/ndarr.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index 886bde63..d6945e69 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -62,7 +62,7 @@ impl NdArrayView for Array2D { let bytes = unsafe { std::slice::from_raw_parts( chunk.as_ptr() as *const u8, - chunk.len() * std::mem::size_of::(), + size_of_val(chunk) ) }; writer.write_all(bytes)?; From 4a5834688eabc79f2b1f84c800ad07101b8e6a70 Mon Sep 17 00:00:00 2001 From: victor Date: Mon, 7 Apr 2025 16:36:23 +0800 Subject: [PATCH 018/119] c example --- cpp_test/mock_server.cpp | 5 ++-- cpp_test/test_line_sender.cpp | 27 ++++++++++++++++++++++ examples/line_sender_c_example.c | 27 ++++++++++++++++++++++ examples/line_sender_c_example_auth.c | 27 ++++++++++++++++++++++ examples/line_sender_c_example_auth_tls.c | 27 ++++++++++++++++++++++ examples/line_sender_c_example_from_conf.c | 27 ++++++++++++++++++++++ examples/line_sender_c_example_from_env.c | 27 ++++++++++++++++++++++ examples/line_sender_c_example_http.c | 27 ++++++++++++++++++++++ examples/line_sender_c_example_tls_ca.c | 27 ++++++++++++++++++++++ 9 files changed, 219 insertions(+), 2 deletions(-) diff --git a/cpp_test/mock_server.cpp b/cpp_test/mock_server.cpp index 0e8a1e85..ad9cf83a 100644 --- a/cpp_test/mock_server.cpp +++ b/cpp_test/mock_server.cpp @@ -24,6 +24,7 @@ #include "mock_server.hpp" +#include #include #if defined(PLATFORM_UNIX) @@ -195,9 +196,9 @@ size_t mock_server::recv(double wait_timeout_sec) if (!wait_for_data(wait_timeout_sec)) return 0; - char chunk[1024]; + std::byte chunk[1024]; size_t chunk_len{sizeof(chunk)}; - std::vector accum; + std::vector accum; for (;;) { wait_for_data(); diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 1510aef2..6f8ca1fc 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -95,6 +95,33 @@ TEST_CASE("line_sender c api basics") CHECK(::line_sender_buffer_table(buffer, table_name, &err)); CHECK(::line_sender_buffer_symbol(buffer, t1_name, v1_utf8, &err)); CHECK(::line_sender_buffer_column_f64(buffer, f1_name, 0.5, &err)); + + line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("order_book"); + // 3D array of doubles + size_t rank = 3; + uint32_t shapes[] = {2, 3, 2}; + double arr_data[] = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + CHECK( + ::line_sender_buffer_column_f64_arr( + buffer, + arr_name, + rank, + shapes, + reinterpret_cast(arr_data), + sizeof(arr_data), + &err)); CHECK(::line_sender_buffer_at_nanos(buffer, 10000000, &err)); CHECK(server.recv() == 0); CHECK(::line_sender_buffer_size(buffer) == 27); diff --git a/examples/line_sender_c_example.c b/examples/line_sender_c_example.c index 0f6805cc..59190e53 100644 --- a/examples/line_sender_c_example.c +++ b/examples/line_sender_c_example.c @@ -55,6 +55,33 @@ static bool example(const char* host, const char* port) if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) goto on_error; + line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("order_book"); + // 3D array of doubles + size_t rank = 3; + uint32_t shapes[] = {2, 3, 2}; + double arr_data[] = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + if (!line_sender_buffer_column_f64_arr( + buffer, + arr_name, + rank, + shapes, + (const uint8_t*)arr_data, + sizeof(arr_data), + &err)) + goto on_error; + // 1997-07-04 04:56:55 UTC int64_t designated_timestamp = 867992215000000000; if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) diff --git a/examples/line_sender_c_example_auth.c b/examples/line_sender_c_example_auth.c index 59a99421..1564da88 100644 --- a/examples/line_sender_c_example_auth.c +++ b/examples/line_sender_c_example_auth.c @@ -60,6 +60,33 @@ static bool example(const char* host, const char* port) if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) goto on_error; + line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("order_book"); + // 3D array of doubles + size_t rank = 3; + uint32_t shapes[] = {2, 3, 2}; + double arr_data[] = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + if (!line_sender_buffer_column_f64_arr( + buffer, + arr_name, + rank, + shapes, + (const uint8_t*)arr_data, + sizeof(arr_data), + &err)) + goto on_error; + // 1997-07-04 04:56:55 UTC int64_t designated_timestamp = 867992215000000000; if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) diff --git a/examples/line_sender_c_example_auth_tls.c b/examples/line_sender_c_example_auth_tls.c index e7454d8d..523e91d1 100644 --- a/examples/line_sender_c_example_auth_tls.c +++ b/examples/line_sender_c_example_auth_tls.c @@ -60,6 +60,33 @@ static bool example(const char* host, const char* port) if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) goto on_error; + line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("order_book"); + // 3D array of doubles + size_t rank = 3; + uint32_t shapes[] = {2, 3, 2}; + double arr_data[] = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + if (!line_sender_buffer_column_f64_arr( + buffer, + arr_name, + rank, + shapes, + (const uint8_t*)arr_data, + sizeof(arr_data), + &err)) + goto on_error; + // 1997-07-04 04:56:55 UTC int64_t designated_timestamp = 867992215000000000; if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) diff --git a/examples/line_sender_c_example_from_conf.c b/examples/line_sender_c_example_from_conf.c index 6f1d4e43..ec3a217f 100644 --- a/examples/line_sender_c_example_from_conf.c +++ b/examples/line_sender_c_example_from_conf.c @@ -44,6 +44,33 @@ int main(int argc, const char* argv[]) if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) goto on_error; + line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("order_book"); + // 3D array of doubles + size_t rank = 3; + uint32_t shapes[] = {2, 3, 2}; + double arr_data[] = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + if (!line_sender_buffer_column_f64_arr( + buffer, + arr_name, + rank, + shapes, + (const uint8_t*)arr_data, + sizeof(arr_data), + &err)) + goto on_error; + // 1997-07-04 04:56:55 UTC int64_t designated_timestamp = 867992215000000000; if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) diff --git a/examples/line_sender_c_example_from_env.c b/examples/line_sender_c_example_from_env.c index e67dbe59..f949b843 100644 --- a/examples/line_sender_c_example_from_env.c +++ b/examples/line_sender_c_example_from_env.c @@ -43,6 +43,33 @@ int main(int argc, const char* argv[]) if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) goto on_error; + line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("order_book"); + // 3D array of doubles + size_t rank = 3; + uint32_t shapes[] = {2, 3, 2}; + double arr_data[] = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + if (!line_sender_buffer_column_f64_arr( + buffer, + arr_name, + rank, + shapes, + (const uint8_t*)arr_data, + sizeof(arr_data), + &err)) + goto on_error; + // 1997-07-04 04:56:55 UTC int64_t designated_timestamp = 867992215000000000; if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) diff --git a/examples/line_sender_c_example_http.c b/examples/line_sender_c_example_http.c index 427ab705..e4dcf662 100644 --- a/examples/line_sender_c_example_http.c +++ b/examples/line_sender_c_example_http.c @@ -54,6 +54,33 @@ static bool example(const char* host, const char* port) if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) goto on_error; + line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("order_book"); + // 3D array of doubles + size_t rank = 3; + uint32_t shapes[] = {2, 3, 2}; + double arr_data[] = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + if (!line_sender_buffer_column_f64_arr( + buffer, + arr_name, + rank, + shapes, + (const uint8_t*)arr_data, + sizeof(arr_data), + &err)) + goto on_error; + // 1997-07-04 04:56:55 UTC int64_t designated_timestamp = 867992215000000000; if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) diff --git a/examples/line_sender_c_example_tls_ca.c b/examples/line_sender_c_example_tls_ca.c index 63151945..d52901fe 100644 --- a/examples/line_sender_c_example_tls_ca.c +++ b/examples/line_sender_c_example_tls_ca.c @@ -61,6 +61,33 @@ static bool example(const char* ca_path, const char* host, const char* port) if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) goto on_error; + line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("order_book"); + // 3D array of doubles + size_t rank = 3; + uint32_t shapes[] = {2, 3, 2}; + double arr_data[] = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + if (!line_sender_buffer_column_f64_arr( + buffer, + arr_name, + rank, + shapes, + (const uint8_t*)arr_data, + sizeof(arr_data), + &err)) + goto on_error; + // 1997-07-04 04:56:55 UTC int64_t designated_timestamp = 867992215000000000; if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) From 1a6cf6f2d2b5d8a3d3222c87e3c10889a6488a6e Mon Sep 17 00:00:00 2001 From: victor Date: Mon, 7 Apr 2025 20:28:09 +0800 Subject: [PATCH 019/119] make mock.rs recv works. --- cpp_test/mock_server.cpp | 5 +-- questdb-rs-ffi/Cargo.lock | 80 --------------------------------- questdb-rs/Cargo.toml | 2 +- questdb-rs/src/ingress/mod.rs | 2 + questdb-rs/src/ingress/ndarr.rs | 11 +++++ questdb-rs/src/tests/mock.rs | 56 +++++++++++++++++++++-- questdb-rs/src/tests/ndarr.rs | 11 +++-- questdb-rs/src/tests/sender.rs | 28 +++++------- 8 files changed, 85 insertions(+), 110 deletions(-) diff --git a/cpp_test/mock_server.cpp b/cpp_test/mock_server.cpp index ad9cf83a..0e8a1e85 100644 --- a/cpp_test/mock_server.cpp +++ b/cpp_test/mock_server.cpp @@ -24,7 +24,6 @@ #include "mock_server.hpp" -#include #include #if defined(PLATFORM_UNIX) @@ -196,9 +195,9 @@ size_t mock_server::recv(double wait_timeout_sec) if (!wait_for_data(wait_timeout_sec)) return 0; - std::byte chunk[1024]; + char chunk[1024]; size_t chunk_len{sizeof(chunk)}; - std::vector accum; + std::vector accum; for (;;) { wait_for_data(); diff --git a/questdb-rs-ffi/Cargo.lock b/questdb-rs-ffi/Cargo.lock index 248ab0c9..a23526e2 100644 --- a/questdb-rs-ffi/Cargo.lock +++ b/questdb-rs-ffi/Cargo.lock @@ -11,12 +11,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "autocfg" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" - [[package]] name = "aws-lc-rs" version = "1.13.0" @@ -380,16 +374,6 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" -[[package]] -name = "matrixmultiply" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9380b911e3e96d10c1f415da0876389aaf1b56759054eeb0de7df940c456ba1a" -dependencies = [ - "autocfg", - "rawpointer", -] - [[package]] name = "memchr" version = "2.7.4" @@ -402,21 +386,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "ndarray" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" -dependencies = [ - "matrixmultiply", - "num-complex", - "num-integer", - "num-traits", - "portable-atomic", - "portable-atomic-util", - "rawpointer", -] - [[package]] name = "nom" version = "7.1.3" @@ -427,33 +396,6 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "num-complex" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - [[package]] name = "once_cell" version = "1.21.3" @@ -472,21 +414,6 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" -[[package]] -name = "portable-atomic" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" - -[[package]] -name = "portable-atomic-util" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" -dependencies = [ - "portable-atomic", -] - [[package]] name = "ppv-lite86" version = "0.2.21" @@ -541,7 +468,6 @@ dependencies = [ "itoa", "libc", "log", - "ndarray", "questdb-confstr", "rand", "rustls", @@ -613,12 +539,6 @@ dependencies = [ "getrandom 0.3.2", ] -[[package]] -name = "rawpointer" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" - [[package]] name = "regex" version = "1.11.1" diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index 50649d37..90461b53 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -59,7 +59,7 @@ tempfile = "3" webpki-roots = "0.26.8" [features] -default = ["tls-webpki-certs", "ilp-over-http", "aws-lc-crypto", "ndarray"] +default = ["tls-webpki-certs", "ilp-over-http", "aws-lc-crypto"] # Include support for ILP over HTTP. ilp-over-http = ["dep:ureq", "dep:serde_json", "dep:rand"] diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index cc7d98fe..8699e572 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -2851,6 +2851,8 @@ impl Sender { } pub(crate) const ARRAY_BINARY_FORMAT_TYPE: u8 = 14; +#[allow(dead_code)] +pub(crate) const DOUBLE_BINARY_FORMAT_TYPE: u8 = 16; mod conf; pub(crate) mod ndarr; diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 2c0fd0ad..a66bd342 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -44,6 +44,17 @@ impl From for u8 { } } +impl TryFrom for ElemDataType { + type Error = String; + + fn try_from(value: u8) -> Result { + match value { + 0x0A => Ok(ElemDataType::Double), + _ => Err(format!("Unknown element type: {}", value)), + } + } +} + impl ArrayElement for f64 { /// Identifies f64 as Double type in QuestDB's type system. fn elem_type() -> ElemDataType { diff --git a/questdb-rs/src/tests/mock.rs b/questdb-rs/src/tests/mock.rs index 5b96d123..49bf150a 100644 --- a/questdb-rs/src/tests/mock.rs +++ b/questdb-rs/src/tests/mock.rs @@ -22,7 +22,7 @@ * ******************************************************************************/ -use crate::ingress::{Protocol, SenderBuilder}; +use crate::ingress::{ElemDataType, Protocol, SenderBuilder}; use core::time::Duration; use mio::event::Event; @@ -36,6 +36,7 @@ use std::net::SocketAddr; use std::sync::Arc; use std::time::Instant; +use crate::ingress; #[cfg(feature = "ilp-over-http")] use std::io::Write; @@ -50,7 +51,7 @@ pub struct MockServer { tls_conn: Option, pub host: &'static str, pub port: u16, - pub msgs: Vec, + pub msgs: Vec>, } pub fn certs_dir() -> std::path::PathBuf { @@ -519,8 +520,55 @@ impl MockServer { } } - self.msgs = accum; - Ok(self.msgs.len()) + let mut received_count = 0usize; + let mut head = 0usize; + let binary_length = 0usize; + let mut index = 1; + + while index < accum.len() { + let last = accum[index]; + let prev = accum[index - 1]; + if last == b'=' && prev == b'=' { + index += 1; + // calc binary length + let binary_type = accum[index]; + if binary_type == ingress::DOUBLE_BINARY_FORMAT_TYPE { + index += size_of::(); + } else if binary_type == ingress::ARRAY_BINARY_FORMAT_TYPE { + index += 1; + let element_type = match ElemDataType::try_from(accum[index]) { + Ok(t) => t, + Err(e) => { + return Err(io::Error::new(io::ErrorKind::Other, e)); + } + }; + let mut elems_size = match element_type { + ElemDataType::Double => size_of::(), + }; + index += 1; + let dims = accum[index] as usize; + index += 1; + for _ in 0..dims { + elems_size = elems_size + * i32::from_le_bytes( + accum[index..index + size_of::()].try_into().unwrap(), + ) as usize; + index += size_of::(); + } + index += elems_size; + } + } else if (last == b'\n') && (prev != b'\\' && binary_length == 0) { + let tail = index + 1; + let msg = &accum[head..tail]; + self.msgs.push(msg.to_vec()); + head = tail; + received_count += 1; + index = tail; + } else { + index += 1; + } + } + Ok(received_count) } pub fn recv_q(&mut self) -> io::Result { diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index d6945e69..ec6d4a08 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -1,5 +1,7 @@ +#[cfg(feature = "ndarray")] +use crate::ingress::MAX_DIMS; use crate::ingress::{ - ArrayElement, Buffer, ElemDataType, NdArrayView, ARRAY_BINARY_FORMAT_TYPE, MAX_DIMS, + ArrayElement, Buffer, ElemDataType, NdArrayView, ARRAY_BINARY_FORMAT_TYPE }; use crate::tests::TestResult; use crate::ErrorCode; @@ -60,10 +62,7 @@ impl NdArrayView for Array2D { for chunk in self.data.chunks(self.cols) { let bytes = unsafe { - std::slice::from_raw_parts( - chunk.as_ptr() as *const u8, - size_of_val(chunk) - ) + std::slice::from_raw_parts(chunk.as_ptr() as *const u8, size_of_val(chunk)) }; writer.write_all(bytes)?; } @@ -162,7 +161,7 @@ fn test_invalid_dimension() -> TestResult { None } fn write_row_major(&self, _: &mut W) -> std::io::Result<()> { - todo!() + Ok(()) } } diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index 665bc449..b3809507 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -22,13 +22,9 @@ * ******************************************************************************/ -use crate::{ - ingress, - ingress::{ - Buffer, CertificateAuthority, Sender, TableName, Timestamp, TimestampMicros, TimestampNanos, - }, - Error, ErrorCode, -}; +use crate::{ingress::{ + Buffer, CertificateAuthority, Sender, TableName, Timestamp, TimestampMicros, TimestampNanos, +}, Error, ErrorCode}; use crate::tests::{ mock::{certs_dir, MockServer}, @@ -36,7 +32,7 @@ use crate::tests::{ }; #[cfg(feature = "ndarray")] -use crate::ingress::ndarr::{ElemDataType, NdArrayView}; +use crate::{ingress, ingress::{ElemDataType, NdArrayView}}; use core::time::Duration; #[cfg(feature = "ndarray")] use ndarray::{arr1, arr2, ArrayD}; @@ -83,8 +79,8 @@ fn test_basics() -> TestResult { sender.flush(&mut buffer)?; assert_eq!(buffer.len(), 0); assert_eq!(buffer.as_bytes(), b""); - assert_eq!(server.recv_q()?, exp_byte.len()); - assert_eq!(server.msgs, exp_byte); + assert_eq!(server.recv_q()?, 1); + assert_eq!(server.msgs[0], exp_byte); Ok(()) } @@ -158,8 +154,8 @@ fn test_array_basic() -> TestResult { sender.flush(&mut buffer)?; assert_eq!(buffer.len(), 0); assert_eq!(buffer.as_bytes(), b""); - assert_eq!(server.recv_q()?, exp.len()); - assert_eq!(server.msgs.as_slice(), exp); + assert_eq!(server.recv_q()?, 1); + assert_eq!(server.msgs[0].as_slice(), exp); Ok(()) } @@ -467,8 +463,8 @@ fn test_tls_with_file_ca() -> TestResult { assert_eq!(buffer.as_bytes(), exp); assert_eq!(buffer.len(), exp.len()); sender.flush(&mut buffer)?; - assert_eq!(server.recv_q()?, exp.len()); - assert_eq!(server.msgs.as_slice(), exp); + assert_eq!(server.recv_q()?, 1); + assert_eq!(server.msgs[0].as_slice(), exp); Ok(()) } @@ -560,8 +556,8 @@ fn test_tls_insecure_skip_verify() -> TestResult { assert_eq!(buffer.as_bytes(), exp); assert_eq!(buffer.len(), exp.len()); sender.flush(&mut buffer)?; - assert_eq!(server.recv_q()?, exp.len()); - assert_eq!(server.msgs.as_slice(), exp); + assert_eq!(server.recv_q()?, 1); + assert_eq!(server.msgs[0].as_slice(), exp); Ok(()) } From 9caed7ad268e95f32907edd77423bab39dae47ef Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Mon, 7 Apr 2025 15:58:26 +0100 Subject: [PATCH 020/119] easier pre-ci 'proj' script to build lint and test changes --- cpp_test/test_line_sender.cpp | 17 ++- proj | 2 + proj.bat | 2 + proj.ps1 | 1 + proj.py | 220 +++++++++++++++++++++++++++++++++ questdb-rs/src/tests/mock.rs | 7 +- questdb-rs/src/tests/ndarr.rs | 4 +- questdb-rs/src/tests/sender.rs | 14 ++- 8 files changed, 247 insertions(+), 20 deletions(-) create mode 100755 proj create mode 100644 proj.bat create mode 100644 proj.ps1 create mode 100755 proj.py diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 6f8ca1fc..80cd72da 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -113,15 +113,14 @@ TEST_CASE("line_sender c api basics") 2.7, 48121.5, 4.3}; - CHECK( - ::line_sender_buffer_column_f64_arr( - buffer, - arr_name, - rank, - shapes, - reinterpret_cast(arr_data), - sizeof(arr_data), - &err)); + CHECK(::line_sender_buffer_column_f64_arr( + buffer, + arr_name, + rank, + shapes, + reinterpret_cast(arr_data), + sizeof(arr_data), + &err)); CHECK(::line_sender_buffer_at_nanos(buffer, 10000000, &err)); CHECK(server.recv() == 0); CHECK(::line_sender_buffer_size(buffer) == 27); diff --git a/proj b/proj new file mode 100755 index 00000000..d4c80095 --- /dev/null +++ b/proj @@ -0,0 +1,2 @@ +#!/bin/bash +python3 proj.py "$@" diff --git a/proj.bat b/proj.bat new file mode 100644 index 00000000..027f5468 --- /dev/null +++ b/proj.bat @@ -0,0 +1,2 @@ +@echo off +python3 proj.py %* diff --git a/proj.ps1 b/proj.ps1 new file mode 100644 index 00000000..7f9e830e --- /dev/null +++ b/proj.ps1 @@ -0,0 +1 @@ +python3 proj.py $args diff --git a/proj.py b/proj.py new file mode 100755 index 00000000..bd36a449 --- /dev/null +++ b/proj.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 + +import sys +sys.dont_write_bytecode = True +import pathlib +import shutil +import shlex +import subprocess +import os + + +PROJ_ROOT = pathlib.Path(__file__).parent + + +def _run(*args, env=None, cwd=None): + """ + Log and run a command within the build dir. + On error, exit with child's return code. + """ + args = [str(arg) for arg in args] + cwd = cwd or PROJ_ROOT + sys.stderr.write('[CMD] ') + if env is not None: + env_str = ' '.join(f'{k}={shlex.quote(v)}' for k, v in env.items()) + sys.stderr.write(f'{env_str} ') + env = {**os.environ, **env} + escaped_cmd = ' '.join(shlex.quote(arg) for arg in args) + sys.stderr.write(f'{escaped_cmd}\n') + ret_code = subprocess.run(args, cwd=str(cwd), env=env).returncode + if ret_code != 0: + sys.exit(ret_code) + + +def _rm(path: pathlib.Path, pattern: str): + paths = path.glob(pattern) + for path in paths: + sys.stderr.write(f'[RM] {path}\n') + path.unlink() + + +def _rmtree(path: pathlib.Path): + if not path.exists(): + return + sys.stderr.write(f'[RMTREE] {path}\n') + shutil.rmtree(path, ignore_errors=True) + + +def _has_command(command: str) -> bool: + """ + Check if a command is available in the system. + """ + return shutil.which(command) is not None + + +COMMANDS = [] + + +def command(fn): + COMMANDS.append(fn.__name__) + return fn + + +@command +def clean(): + _rmtree(PROJ_ROOT / 'build') + _rmtree(PROJ_ROOT / 'build_CXX20') + _rmtree(PROJ_ROOT / 'questdb-rs' / 'target') + _rmtree(PROJ_ROOT / 'questdb-rs-ffi' / 'target') + + +@command +def cmake_cxx17(): + _rmtree(PROJ_ROOT / 'build') + cmd = [ + 'cmake', + '-S', '.', + '-B', 'build', + '-DCMAKE_BUILD_TYPE=Release', + '-DQUESTDB_TESTS_AND_EXAMPLES=ON'] + if _has_command('ninja'): + cmd.insert(1, '-G') + cmd.insert(2, 'Ninja') + _run(*cmd) + + +@command +def cmake_cxx20(): + _rmtree(PROJ_ROOT / 'build_CXX20') + cmd = [ + 'cmake', + '-S', '.', + '-B', 'build_CXX20', + '-DCMAKE_BUILD_TYPE=Release', + '-DQUESTDB_TESTS_AND_EXAMPLES=ON', + '-DCMAKE_CXX_STANDARD=20'] + if _has_command('ninja'): + cmd.insert(1, '-G') + cmd.insert(2, 'Ninja') + _run(*cmd) + + +@command +def build_cxx17(): + if not (PROJ_ROOT / 'build').exists(): + cmake_cxx17() + _run('cmake', '--build', 'build') + + +@command +def build_cxx20(): + if not (PROJ_ROOT / 'build_CXX20').exists(): + cmake_cxx20() + _run('cmake', '--build', 'build_CXX20') + + +@command +def build(): + build_cxx17() + build_cxx20() + + +@command +def lint_rust(): + questdb_rs_path = PROJ_ROOT / 'questdb-rs' + questdb_rs_ffi_path = PROJ_ROOT / 'questdb-rs-ffi' + _run('cargo', 'fmt', '--all', '--', '--check', cwd=questdb_rs_path) + _run('cargo', 'clippy', '--all-targets', '--features', 'almost-all-features', '--', '-D', 'warnings', cwd=questdb_rs_path) + _run('cargo', 'fmt', '--all', '--', '--check', cwd=questdb_rs_ffi_path) + _run('cargo', 'clippy', '--all-targets', '--all-features', '--', '-D', 'warnings', cwd=questdb_rs_ffi_path) + + +@command +def lint_cpp(): + try: + _run( + sys.executable, + PROJ_ROOT / 'ci' / 'format_cpp.py', + '--check') + except subprocess.CalledProcessError: + sys.stderr.write('REMINDER: To fix any C++ formatting issues, run: ./proj format_cpp\n') + raise + + +@command +def lint(): + lint_rust() + lint_cpp() + + +@command +def format_rust(): + questdb_rs_path = PROJ_ROOT / 'questdb-rs' + questdb_rs_ffi_path = PROJ_ROOT / 'questdb-rs-ffi' + _run('cargo', 'fmt', '--all', cwd=questdb_rs_path) + _run('cargo', 'fmt', '--all', cwd=questdb_rs_ffi_path) + + +@command +def format_cpp(): + _run( + sys.executable, + PROJ_ROOT / 'ci' / 'format_cpp.py') + + +@command +def test(): + build() + _run( + sys.executable, + PROJ_ROOT / 'ci' / 'run_all_tests.py') + + +@command +def build_latest_questdb(branch='master'): + questdb_path = PROJ_ROOT / 'questdb' + if not questdb_path.exists(): + _run('git', 'clone', 'https://github.com/questdb/questdb.git') + _run('git', 'fetch', 'origin', branch, cwd=questdb_path) + _run('git', 'switch', branch=questdb_path) + _run('git', 'pull', 'origin', branch=questdb_path) + _run('git', 'submodule', 'update', '--init', '--recursive', cwd=questdb_path) + _run('mvn', 'clean', 'package', '-DskipTests', '-Pbuild-web-console', cwd=questdb_path) + + +@command +def test_vs_latest_questdb(): + questdb_path = PROJ_ROOT / 'questdb' + if not questdb_path.exists(): + build_latest_questdb() + _run( + sys.executable, + PROJ_ROOT / 'system_test' / 'test.py', + '--repo', PROJ_ROOT / 'questdb', + '-v') + + +@command +def all(): + clean() + build() + lint() + test() + test_vs_latest_questdb() + + +def main(): + if len(sys.argv) < 2: + sys.stderr.write('Usage: python3 proj.py \n') + sys.stderr.write('Commands:\n') + for command in COMMANDS: + sys.stderr.write(f' {command}\n') + sys.stderr.write('\n') + sys.exit(0) + fn = sys.argv[1] + args = list(sys.argv)[2:] + globals()[fn](*args) + + +if __name__ == '__main__': + main() diff --git a/questdb-rs/src/tests/mock.rs b/questdb-rs/src/tests/mock.rs index 49bf150a..1d1f438a 100644 --- a/questdb-rs/src/tests/mock.rs +++ b/questdb-rs/src/tests/mock.rs @@ -549,10 +549,9 @@ impl MockServer { let dims = accum[index] as usize; index += 1; for _ in 0..dims { - elems_size = elems_size - * i32::from_le_bytes( - accum[index..index + size_of::()].try_into().unwrap(), - ) as usize; + elems_size *= i32::from_le_bytes( + accum[index..index + size_of::()].try_into().unwrap(), + ) as usize; index += size_of::(); } index += elems_size; diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index ec6d4a08..2d7ee97b 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -1,8 +1,6 @@ #[cfg(feature = "ndarray")] use crate::ingress::MAX_DIMS; -use crate::ingress::{ - ArrayElement, Buffer, ElemDataType, NdArrayView, ARRAY_BINARY_FORMAT_TYPE -}; +use crate::ingress::{ArrayElement, Buffer, ElemDataType, NdArrayView, ARRAY_BINARY_FORMAT_TYPE}; use crate::tests::TestResult; use crate::ErrorCode; diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index b3809507..1f308e25 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -22,9 +22,12 @@ * ******************************************************************************/ -use crate::{ingress::{ - Buffer, CertificateAuthority, Sender, TableName, Timestamp, TimestampMicros, TimestampNanos, -}, Error, ErrorCode}; +use crate::{ + ingress::{ + Buffer, CertificateAuthority, Sender, TableName, Timestamp, TimestampMicros, TimestampNanos, + }, + Error, ErrorCode, +}; use crate::tests::{ mock::{certs_dir, MockServer}, @@ -32,7 +35,10 @@ use crate::tests::{ }; #[cfg(feature = "ndarray")] -use crate::{ingress, ingress::{ElemDataType, NdArrayView}}; +use crate::{ + ingress, + ingress::{ElemDataType, NdArrayView}, +}; use core::time::Duration; #[cfg(feature = "ndarray")] use ndarray::{arr1, arr2, ArrayD}; From c2583f1dbe0aaff7aebf72eefcffc26f19fc28fc Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 8 Apr 2025 01:03:04 +0800 Subject: [PATCH 021/119] c api add strides. --- include/questdb/ingress/line_sender.hpp | 7 +- questdb-rs-ffi/src/lib.rs | 69 ++------- questdb-rs/src/ingress/mod.rs | 3 +- questdb-rs/src/ingress/ndarr.rs | 191 +++++++++++++++++++++++- 4 files changed, 209 insertions(+), 61 deletions(-) diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index aa87a711..1f2ed06e 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -641,7 +641,8 @@ class line_sender_buffer template line_sender_buffer& column( column_name_view name, - const std::vector& shape, + const std::vector& shapes, + const std::vector& strides, const std::array& data) { static_assert( @@ -652,8 +653,8 @@ class line_sender_buffer ::line_sender_buffer_column_f64_arr, _impl, name._impl, - shape.size(), - shape.data(), + shapes.data(), + strides.data(), reinterpret_cast(data.data()), sizeof(double) * N); return *this; diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 3d929e6c..193fe34b 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -24,7 +24,7 @@ #![allow(non_camel_case_types, clippy::missing_safety_doc)] -use libc::{c_char, c_uint, size_t}; +use libc::{c_char, size_t}; use std::ascii; use std::boxed::Box; use std::convert::{From, Into}; @@ -33,13 +33,10 @@ use std::ptr; use std::slice; use std::str; -use questdb::{ - ingress::{ - ArrayElement, Buffer, CertificateAuthority, ColumnName, NdArrayView, Protocol, Sender, - SenderBuilder, TableName, TimestampMicros, TimestampNanos, - }, - Error, ErrorCode, -}; +use questdb::{ingress, ingress::{ + Buffer, CertificateAuthority, ColumnName, Protocol, Sender, + SenderBuilder, TableName, TimestampMicros, TimestampNanos, +}, Error, ErrorCode}; macro_rules! bubble_err_to_c { ($err_out:expr, $expression:expr) => { @@ -309,44 +306,6 @@ impl line_sender_utf8 { } } -#[derive(Debug, Copy, Clone)] -struct line_sender_array { - dims: size_t, - shapes: *const u32, - buf_len: size_t, - buf: *const u8, -} - -impl NdArrayView for line_sender_array -where - T: ArrayElement, -{ - fn ndim(&self) -> usize { - self.dims - } - - fn dim(&self, index: usize) -> Option { - if index >= self.dims { - return None; - } - - unsafe { - if self.shapes.is_null() { - return None; - } - - let dim_size = *self.shapes.add(index); - Some(dim_size as usize) - } - } - - fn write_row_major(&self, writer: &mut W) -> std::io::Result<()> { - let bytes = unsafe { slice::from_raw_parts(self.buf, self.buf_len) }; - writer.write_all(bytes)?; - Ok(()) - } -} - /// An ASCII-safe description of a binary buffer. Trimmed if too long. fn describe_buf(buf: &[u8]) -> String { let max_len = 100usize; @@ -877,22 +836,24 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( buffer: *mut line_sender_buffer, name: line_sender_column_name, rank: size_t, - shape: *const c_uint, // C array of dimension sizes + shapes: *const size_t, // C array of shapes + strides: *const i64, // C array of strides data_buffer: *const u8, // Raw array data data_buffer_len: size_t, // Total bytes length err_out: *mut *mut line_sender_error, ) -> bool { let buffer = unwrap_buffer_mut(buffer); let name = name.as_name(); - let view = line_sender_array { - dims: rank, - shapes: shape, - buf_len: data_buffer_len, - buf: data_buffer, - }; + let view = ingress::ArrayViewWithStrides::::new( + rank, + shapes, + strides as *const isize, + data_buffer, + data_buffer_len, + ); bubble_err_to_c!( err_out, - buffer.column_arr::, line_sender_array, f64>(name, &view) + buffer.column_arr::, ingress::ArrayViewWithStrides<'_, f64>, f64>(name, &view) ); true } diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 8699e572..b9690323 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -24,7 +24,7 @@ #![doc = include_str!("mod.md")] -pub use self::ndarr::{ArrayElement, ElemDataType, NdArrayView}; +pub use self::ndarr::{ArrayElement, ArrayViewWithStrides, ElemDataType, NdArrayView}; pub use self::timestamp::*; use crate::error::{self, Error, Result}; use crate::gai; @@ -1112,6 +1112,7 @@ impl Buffer { let writeable = unsafe { from_raw_parts_mut(self.output.as_mut_ptr().add(index), reserve_size) }; let mut cursor = Cursor::new(writeable); + // ndarr data if let Err(e) = view.write_row_major(&mut cursor) { return Err(error::fmt!( diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index a66bd342..3bb02cb6 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -62,8 +62,180 @@ impl ArrayElement for f64 { } } +#[derive(Debug)] +pub struct ArrayViewWithStrides<'a, T> { + dims: usize, + shapes: &'a [usize], + strides: &'a [isize], + buf_len: usize, + buf: *const u8, + _marker: std::marker::PhantomData, +} + +impl NdArrayView for ArrayViewWithStrides<'_, T> +where + T: ArrayElement, +{ + fn ndim(&self) -> usize { + self.dims + } + + fn dim(&self, index: usize) -> Option { + if index >= self.dims { + return None; + } + + Some(self.shapes[index]) + } + + fn write_row_major(&self, writer: &mut W) -> std::io::Result<()> { + if self.is_c_major() { + let bytes = unsafe { slice::from_raw_parts(self.buf, self.buf_len) }; + writer.write_all(bytes)?; + Ok(()) + } else { + let mut io_slices = Vec::new(); + for element in self.iter() { + io_slices.push(std::io::IoSlice::new(element)); + } + + let mut io_slices: &mut [std::io::IoSlice<'_>] = io_slices.as_mut_slice(); + while !io_slices.is_empty() { + let written = writer.write_vectored(io_slices)?; + if written == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::WriteZero, + "Failed to write all bytes", + )); + } + io_slices = &mut io_slices[written..]; + } + Ok(()) + } + } +} + +impl ArrayViewWithStrides<'_, T> +where + T: ArrayElement, +{ + /// # Safety + /// + /// todo + pub unsafe fn new( + dims: usize, + shapes: *const usize, + strides: *const isize, + data: *const u8, + data_len: usize, + ) -> Self { + let shapes = slice::from_raw_parts(shapes, dims); + let strides = slice::from_raw_parts(strides, dims); + Self { + dims, + shapes, + strides, + buf_len: data_len, + buf: data, + _marker: std::marker::PhantomData::, + } + } + + fn is_c_major(&self) -> bool { + let mut expected_stride = size_of::() as isize; + self.strides + .iter() + .rev() + .skip(1) + .zip(self.shapes.iter().rev()) + .all(|(&actual, &dim)| { + let expected = expected_stride; + expected_stride *= dim as isize; + actual.abs() == expected.abs() + }) + } + + fn iter(&self) -> CMajorIterWithStrides { + let mut dim_products = Vec::with_capacity(self.dims); + let mut product = 1; + for &dim in self.shapes.iter().rev() { + dim_products.push(product); + product *= dim; + } + dim_products.reverse(); + + // consider minus strides + let base_ptr = self + .strides + .iter() + .enumerate() + .fold(self.buf, |ptr, (dim, &stride)| { + if stride < 0 { + let dim_size = self.shapes[dim] as isize; + unsafe { ptr.offset(stride * (dim_size - 1)) } + } else { + ptr + } + }); + CMajorIterWithStrides { + base_ptr, + array: self, + dim_products, + current_linear: 0, + total_elements: self.shapes.iter().product(), + } + } +} + +struct CMajorIterWithStrides<'a, T> { + base_ptr: *const u8, + array: &'a ArrayViewWithStrides<'a, T>, + dim_products: Vec, + current_linear: usize, + total_elements: usize, +} + +impl CMajorIterWithStrides<'_, T> { + fn is_ptr_valid(&self, ptr: *const u8) -> bool { + let start = self.array.buf; + let end = unsafe { start.add(self.array.buf_len) }; + ptr >= start && ptr < end + } +} + +impl<'a, T> Iterator for CMajorIterWithStrides<'a, T> +where + T: ArrayElement, +{ + type Item = &'a [u8]; + fn next(&mut self) -> Option { + if self.current_linear >= self.total_elements { + return None; + } + let mut index = self.current_linear; + let mut offset = 0isize; + + for (dim, &prod) in self.dim_products.iter().enumerate() { + let coord = index / prod; + offset += self.array.strides[dim] * coord as isize; + index %= prod; + } + + self.current_linear += 1; + unsafe { + let ptr = self.base_ptr.offset(offset); + if self.is_ptr_valid(ptr) { + Some(slice::from_raw_parts(ptr, size_of::())) + } else { + None + } + } + } +} + #[cfg(feature = "ndarray")] use ndarray::{ArrayView, Axis, Dimension}; +use std::slice; #[cfg(feature = "ndarray")] impl NdArrayView for ArrayView<'_, T, D> @@ -96,10 +268,23 @@ where } let elem_size = size_of::(); - for &element in self.iter() { + let mut io_slices = Vec::new(); + for element in self.iter() { let bytes = - unsafe { std::slice::from_raw_parts(&element as *const T as *const _, elem_size) }; - writer.write_all(bytes)?; + unsafe { std::slice::from_raw_parts(element as *const T as *const _, elem_size) }; + io_slices.push(std::io::IoSlice::new(bytes)); + } + + let mut io_slices: &mut [std::io::IoSlice<'_>] = io_slices.as_mut_slice(); + while !io_slices.is_empty() { + let written = writer.write_vectored(io_slices)?; + if written == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::WriteZero, + "Failed to write all bytes", + )); + } + io_slices = &mut io_slices[written..]; } Ok(()) } From 0476faf4a594ab3aebd320035670cbd108c7172c Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 8 Apr 2025 16:49:11 +0800 Subject: [PATCH 022/119] add strideArrayView for rust array ingress api --- questdb-rs-ffi/src/lib.rs | 4 +- questdb-rs/src/ingress/mod.rs | 11 +- questdb-rs/src/ingress/ndarr.rs | 296 ++++++++++++++++++----------- questdb-rs/src/tests/ndarr.rs | 318 ++++++++++++++------------------ questdb-rs/src/tests/sender.rs | 11 +- 5 files changed, 340 insertions(+), 300 deletions(-) diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 193fe34b..3ffade93 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -844,7 +844,7 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( ) -> bool { let buffer = unwrap_buffer_mut(buffer); let name = name.as_name(); - let view = ingress::ArrayViewWithStrides::::new( + let view = ingress::StridedArrayView::::new( rank, shapes, strides as *const isize, @@ -853,7 +853,7 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( ); bubble_err_to_c!( err_out, - buffer.column_arr::, ingress::ArrayViewWithStrides<'_, f64>, f64>(name, &view) + buffer.column_arr::, ingress::StridedArrayView<'_, f64>, f64>(name, &view) ); true } diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index b9690323..da093ca6 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -24,7 +24,7 @@ #![doc = include_str!("mod.md")] -pub use self::ndarr::{ArrayElement, ArrayViewWithStrides, ElemDataType, NdArrayView}; +pub use self::ndarr::{ArrayElement, ElemDataType, NdArrayView, StridedArrayView}; pub use self::timestamp::*; use crate::error::{self, Error, Result}; use crate::gai; @@ -1079,6 +1079,7 @@ impl Buffer { )); } + let reserve_size = view.check_data_buf()?; // binary format flag '=' self.output.push(b'='); // binary format entity type @@ -1088,7 +1089,6 @@ impl Buffer { // ndarr dims self.output.push(view.ndim() as u8); - let mut reserve_size = size_of::(); for i in 0..view.ndim() { let d = view.dim(i).ok_or_else(|| { error::fmt!( @@ -1097,14 +1097,9 @@ impl Buffer { i ) })?; - // ndarr shapes self.output .extend_from_slice((d as i32).to_le_bytes().as_slice()); - reserve_size = reserve_size.checked_mul(d).ok_or(error::fmt!( - ArrayViewError, - "Array total elem size overflow" - ))? } self.output.reserve(reserve_size); @@ -1114,7 +1109,7 @@ impl Buffer { let mut cursor = Cursor::new(writeable); // ndarr data - if let Err(e) = view.write_row_major(&mut cursor) { + if let Err(e) = ndarr::write_array_data(view, &mut cursor) { return Err(error::fmt!( ArrayWriteToBufferError, "Can not write row major to writer: {}", diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 3bb02cb6..6be47f69 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -2,19 +2,70 @@ pub trait NdArrayView where T: ArrayElement, { + type Iter<'a>: Iterator + where + Self: 'a, + T: 'a; + /// Returns the number of dimensions (rank) of the array. fn ndim(&self) -> usize; /// Returns the size of the specified dimension. fn dim(&self, index: usize) -> Option; - /// Writes array data to writer in row-major order. + /// Return the array’s data as a slice, if it is c-major-layout. + /// Return `None` otherwise. + fn as_slice(&self) -> Option<&[T]>; + + /// Return an iterator of references to the elements of the array. + /// Iterator element type is `&T`. + fn iter(&self) -> Self::Iter<'_>; + + /// Validates the data buffer size of array is consistency of shapes. /// - /// # Important Notes - /// - Writer must be pre-allocated with exact required size - /// - No alignment assumptions should be made about writer start - /// - Handles both contiguous and non-contiguous memory layouts - fn write_row_major(&self, writer: &mut W) -> std::io::Result<()>; + /// # Returns + /// - `Ok(usize)`: Expected buffer size in bytes if valid + /// - `Err(Error)`: Currently never returned, but reserved for future validation logic + fn check_data_buf(&self) -> Result; +} + +pub(crate) fn write_array_data, T>( + array: &A, + writer: &mut W, +) -> std::io::Result<()> +where + T: ArrayElement, +{ + // First optimization path: write contiguous memory directly + if let Some(contiguous) = array.as_slice() { + let bytes = unsafe { + slice::from_raw_parts(contiguous.as_ptr() as *const u8, size_of_val(contiguous)) + }; + return writer.write_all(bytes); + } + + // Fallback path: non-contiguous memory handling + let elem_size = size_of::(); + let mut io_slices = Vec::new(); + for element in array.iter() { + let bytes = unsafe { slice::from_raw_parts(element as *const T as *const _, elem_size) }; + io_slices.push(std::io::IoSlice::new(bytes)); + } + + let mut io_slices: &mut [IoSlice<'_>] = io_slices.as_mut_slice(); + IoSlice::advance_slices(&mut io_slices, 0); + + while !io_slices.is_empty() { + let written = writer.write_vectored(io_slices)?; + if written == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::WriteZero, + "Failed to write all bytes", + )); + } + IoSlice::advance_slices(&mut io_slices, written); + } + Ok(()) } /// Marker trait for valid array element types. @@ -62,8 +113,9 @@ impl ArrayElement for f64 { } } +/// A view into a multi-dimensional array with custom memory strides. #[derive(Debug)] -pub struct ArrayViewWithStrides<'a, T> { +pub struct StridedArrayView<'a, T> { dims: usize, shapes: &'a [usize], strides: &'a [isize], @@ -72,10 +124,16 @@ pub struct ArrayViewWithStrides<'a, T> { _marker: std::marker::PhantomData, } -impl NdArrayView for ArrayViewWithStrides<'_, T> +impl NdArrayView for StridedArrayView<'_, T> where T: ArrayElement, { + type Iter<'b> + = RowMajorIter<'b, T> + where + Self: 'b, + T: 'b; + fn ndim(&self) -> usize { self.dims } @@ -88,40 +146,82 @@ where Some(self.shapes[index]) } - fn write_row_major(&self, writer: &mut W) -> std::io::Result<()> { + fn as_slice(&self) -> Option<&[T]> { if self.is_c_major() { - let bytes = unsafe { slice::from_raw_parts(self.buf, self.buf_len) }; - writer.write_all(bytes)?; - Ok(()) + Some(unsafe { slice::from_raw_parts(self.buf as *const T, self.buf_len / size_of::()) }) } else { - let mut io_slices = Vec::new(); - for element in self.iter() { - io_slices.push(std::io::IoSlice::new(element)); - } + None + } + } + + fn iter(&self) -> Self::Iter<'_> { + let mut dim_products = Vec::with_capacity(self.dims); + let mut product = 1; + for &dim in self.shapes.iter().rev() { + dim_products.push(product); + product *= dim; + } + dim_products.reverse(); - let mut io_slices: &mut [std::io::IoSlice<'_>] = io_slices.as_mut_slice(); - while !io_slices.is_empty() { - let written = writer.write_vectored(io_slices)?; - if written == 0 { - return Err(std::io::Error::new( - std::io::ErrorKind::WriteZero, - "Failed to write all bytes", - )); + // consider minus strides + let base_ptr = self + .strides + .iter() + .enumerate() + .fold(self.buf, |ptr, (dim, &stride)| { + if stride < 0 { + let dim_size = self.shapes[dim] as isize; + unsafe { ptr.offset(stride * (dim_size - 1)) } + } else { + ptr } - io_slices = &mut io_slices[written..]; - } - Ok(()) + }); + RowMajorIter { + base_ptr, + array: self, + dim_products, + current_linear: 0, + total_elements: self.shapes.iter().product(), } } + + fn check_data_buf(&self) -> Result { + let mut size = size_of::(); + for i in 0..self.dims { + let d = self.shapes[i]; + size = size.checked_mul(d).ok_or(error::fmt!( + ArrayViewError, + "Array total elem size overflow" + ))? + } + if size != self.buf_len { + return Err(error::fmt!( + ArrayWriteToBufferError, + "Array buffer length mismatch (actual: {}, expected: {})", + self.buf_len, + size + )); + } + Ok(size) + } } -impl ArrayViewWithStrides<'_, T> +impl StridedArrayView<'_, T> where T: ArrayElement, { - /// # Safety + /// Creates a new strided array view from raw components (unsafe constructor). /// - /// todo + /// # Safety + /// Caller must ensure all the following conditions: + /// - `shapes` points to a valid array of at least `dims` elements + /// - `strides` points to a valid array of at least `dims` elements + /// - `data` points to a valid memory block of at least `data_len` bytes + /// - Memory layout must satisfy: + /// 1. `data_len ≥ (shape[0]-1)*abs(strides[0]) + ... + (shape[n-1]-1)*abs(strides[n-1]) + size_of::()` + /// 2. All calculated offsets stay within `[0, data_len - size_of::()]` + /// - Lifetime `'a` must outlive the view's usage + /// - Strides are measured in bytes (not elements) pub unsafe fn new( dims: usize, shapes: *const usize, @@ -141,61 +241,50 @@ where } } + /// Verifies if the array follows C-style row-major memory layout. fn is_c_major(&self) -> bool { - let mut expected_stride = size_of::() as isize; - self.strides - .iter() - .rev() - .skip(1) - .zip(self.shapes.iter().rev()) - .all(|(&actual, &dim)| { - let expected = expected_stride; - expected_stride *= dim as isize; - actual.abs() == expected.abs() - }) - } + if self.buf.is_null() || self.buf_len == 0 { + return false; + } - fn iter(&self) -> CMajorIterWithStrides { - let mut dim_products = Vec::with_capacity(self.dims); - let mut product = 1; - for &dim in self.shapes.iter().rev() { - dim_products.push(product); - product *= dim; + if self.dims == 1 { + return self.strides[0] == 1 || self.shapes[0] <= 1; } - dim_products.reverse(); - // consider minus strides - let base_ptr = self - .strides - .iter() - .enumerate() - .fold(self.buf, |ptr, (dim, &stride)| { - if stride < 0 { - let dim_size = self.shapes[dim] as isize; - unsafe { ptr.offset(stride * (dim_size - 1)) } - } else { - ptr + for &d in self.shapes { + if d == 0 { + return true; + } + } + + let mut contig_stride = size_of::(); + for (dim, stride) in self.shapes.iter() + .rev() + .zip(self.strides.iter().rev()) + { + if *dim != 1 { + let s = *stride; + if s.abs() != contig_stride as isize { + return false; } - }); - CMajorIterWithStrides { - base_ptr, - array: self, - dim_products, - current_linear: 0, - total_elements: self.shapes.iter().product(), + contig_stride *= *dim; + } } + + true } } -struct CMajorIterWithStrides<'a, T> { +/// Iterator for traversing a strided array in row-major (C-style) order. +pub struct RowMajorIter<'a, T> { base_ptr: *const u8, - array: &'a ArrayViewWithStrides<'a, T>, + array: &'a StridedArrayView<'a, T>, dim_products: Vec, current_linear: usize, total_elements: usize, } -impl CMajorIterWithStrides<'_, T> { +impl RowMajorIter<'_, T> { fn is_ptr_valid(&self, ptr: *const u8) -> bool { let start = self.array.buf; let end = unsafe { start.add(self.array.buf_len) }; @@ -203,29 +292,35 @@ impl CMajorIterWithStrides<'_, T> { } } -impl<'a, T> Iterator for CMajorIterWithStrides<'a, T> +impl<'a, T> Iterator for RowMajorIter<'a, T> where T: ArrayElement, { - type Item = &'a [u8]; + type Item = &'a T; fn next(&mut self) -> Option { if self.current_linear >= self.total_elements { return None; } - let mut index = self.current_linear; + let mut remaining_index = self.current_linear; let mut offset = 0isize; - for (dim, &prod) in self.dim_products.iter().enumerate() { - let coord = index / prod; - offset += self.array.strides[dim] * coord as isize; - index %= prod; + for (dim, &dim_factor) in self.dim_products.iter().enumerate() { + let coord = remaining_index / dim_factor; + remaining_index %= dim_factor; + let stride = self.array.strides[dim]; + let actual_coord = if stride >= 0 { + coord + } else { + self.array.shapes[dim] - 1 - coord + }; + offset += (actual_coord as isize) * stride; } self.current_linear += 1; unsafe { let ptr = self.base_ptr.offset(offset); if self.is_ptr_valid(ptr) { - Some(slice::from_raw_parts(ptr, size_of::())) + Some(&*(ptr as *const T)) } else { None } @@ -233,8 +328,10 @@ where } } +use crate::{error, Error}; #[cfg(feature = "ndarray")] use ndarray::{ArrayView, Axis, Dimension}; +use std::io::IoSlice; use std::slice; #[cfg(feature = "ndarray")] @@ -243,6 +340,12 @@ where T: ArrayElement, D: Dimension, { + type Iter<'a> + = ndarray::iter::Iter<'a, T, D> + where + Self: 'a, + T: 'a; + fn ndim(&self) -> usize { self.ndim() } @@ -256,36 +359,15 @@ where } } - fn write_row_major(&self, writer: &mut W) -> std::io::Result<()> { - if let Some(contiguous) = self.as_slice() { - let bytes = unsafe { - std::slice::from_raw_parts( - contiguous.as_ptr() as *const u8, - size_of_val(contiguous), - ) - }; - return writer.write_all(bytes); - } + fn iter(&self) -> Self::Iter<'_> { + self.iter() + } - let elem_size = size_of::(); - let mut io_slices = Vec::new(); - for element in self.iter() { - let bytes = - unsafe { std::slice::from_raw_parts(element as *const T as *const _, elem_size) }; - io_slices.push(std::io::IoSlice::new(bytes)); - } + fn as_slice(&self) -> Option<&[T]> { + self.as_slice() + } - let mut io_slices: &mut [std::io::IoSlice<'_>] = io_slices.as_mut_slice(); - while !io_slices.is_empty() { - let written = writer.write_vectored(io_slices)?; - if written == 0 { - return Err(std::io::Error::new( - std::io::ErrorKind::WriteZero, - "Failed to write all bytes", - )); - } - io_slices = &mut io_slices[written..]; - } - Ok(()) + fn check_data_buf(&self) -> Result { + Ok(self.len() * size_of::()) } } diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index 2d7ee97b..d667b9fb 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -1,14 +1,17 @@ #[cfg(feature = "ndarray")] use crate::ingress::MAX_DIMS; -use crate::ingress::{ArrayElement, Buffer, ElemDataType, NdArrayView, ARRAY_BINARY_FORMAT_TYPE}; +use crate::ingress::{ + ArrayElement, Buffer, ElemDataType, NdArrayView, StridedArrayView, ARRAY_BINARY_FORMAT_TYPE, +}; use crate::tests::TestResult; use crate::ErrorCode; +use crate::ingress::ndarr::write_array_data; #[cfg(feature = "ndarray")] use ndarray::{arr1, arr2, arr3, s, ArrayD}; #[cfg(feature = "ndarray")] use std::iter; -use std::marker::PhantomData; +use std::ptr; #[test] fn test_f64_element_type() { @@ -16,58 +19,6 @@ fn test_f64_element_type() { assert_eq!(u8::from(ElemDataType::Double), 10); } -struct Array2D { - data: Vec, - rows: usize, - cols: usize, - contiguous: bool, -} - -impl Array2D { - fn new(data: Vec, rows: usize, cols: usize, contiguous: bool) -> Self { - Self { - data, - rows, - cols, - contiguous, - } - } -} - -impl NdArrayView for Array2D { - fn ndim(&self) -> usize { - 2 - } - - fn dim(&self, index: usize) -> Option { - match index { - 0 => Some(self.rows), - 1 => Some(self.cols), - _ => None, - } - } - - fn write_row_major(&self, writer: &mut W) -> std::io::Result<()> { - if self.contiguous { - let bytes = unsafe { - std::slice::from_raw_parts( - self.data.as_ptr() as *const u8, - self.data.len() * std::mem::size_of::(), - ) - }; - return writer.write_all(bytes); - } - - for chunk in self.data.chunks(self.cols) { - let bytes = unsafe { - std::slice::from_raw_parts(chunk.as_ptr() as *const u8, size_of_val(chunk)) - }; - writer.write_all(bytes)?; - } - Ok(()) - } -} - fn to_bytes(data: &[T]) -> Vec { data.iter() .flat_map(|x| { @@ -79,55 +30,115 @@ fn to_bytes(data: &[T]) -> Vec { } #[test] -fn test_basic_array_view() { +fn test_strided_array_view() -> TestResult { // contiguous layout - let test_data = vec![1.0f64, 2.0, 3.0, 4.0, 5.0, 6.0]; - let array = Array2D::new(test_data.clone(), 2, 3, true); + let test_data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; + let shapes = [2, 3]; + let strides = [ + (shapes[1] * size_of::()) as isize, + size_of::() as isize, + ]; + let array = unsafe { + StridedArrayView::::new( + shapes.len(), + shapes.as_ptr(), + strides.as_ptr(), + test_data.as_ptr() as *const u8, + test_data.len() * size_of::(), + ) + }; + assert_eq!(array.ndim(), 2); assert_eq!(array.dim(0), Some(2)); assert_eq!(array.dim(1), Some(3)); assert_eq!(array.dim(2), None); + assert!(array.as_slice().is_some()); let mut buf = vec![]; - array.write_row_major(&mut buf).unwrap(); + write_array_data(&array, &mut buf).unwrap(); let expected = to_bytes(&test_data); assert_eq!(buf, expected); + Ok(()) +} - // non-contiguous layout - let test_data = vec![vec![1.0, 2.0], vec![3.0, 4.0], vec![5.0, 6.0]] - .into_iter() - .flatten() - .collect(); - let array_non_contig = Array2D::new(test_data, 3, 2, false); - let mut buf_non_contig = vec![]; - array_non_contig - .write_row_major(&mut buf_non_contig) - .unwrap(); - let expected_non_contig = to_bytes(&[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]); - assert_eq!(buf_non_contig, expected_non_contig); +#[test] +fn test_strided_non_contiguous() -> TestResult { + let elem_size = size_of::(); + let col_major_data = vec![1.0, 3.0, 5.0, 2.0, 4.0, 6.0]; + let shapes = [3, 2]; + let strides = [elem_size as isize, (shapes[0] * elem_size) as isize]; + + let array_view: StridedArrayView<'_, f64> = unsafe { + StridedArrayView::new( + shapes.len(), + shapes.as_ptr(), + strides.as_ptr(), + col_major_data.as_ptr() as *const u8, + col_major_data.len() * elem_size, + ) + }; + + assert_eq!(array_view.ndim(), 2); + assert_eq!(array_view.dim(0), Some(3)); + assert_eq!(array_view.dim(1), Some(2)); + assert_eq!(array_view.dim(2), None); + assert!(array_view.as_slice().is_none()); + let mut buffer = Vec::new(); + write_array_data(&array_view, &mut buffer)?; + + let expected_data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; + let expected_bytes = unsafe { + std::slice::from_raw_parts( + expected_data.as_ptr() as *const u8, + expected_data.len() * elem_size, + ) + }; + assert_eq!(buffer, expected_bytes); + Ok(()) } #[test] fn test_basic_edge_cases() { // empty array - let empty_array = Array2D::::new(vec![], 0, 0, false); - assert_eq!(empty_array.ndim(), 2); - assert_eq!(empty_array.dim(0), Some(0)); - assert_eq!(empty_array.dim(1), Some(0)); + let elem_size = std::mem::size_of::(); + let empty_view: StridedArrayView<'_, f64> = + unsafe { StridedArrayView::new(2, [0, 0].as_ptr(), [0, 0].as_ptr(), ptr::null(), 0) }; + assert_eq!(empty_view.ndim(), 2); + assert_eq!(empty_view.dim(0), Some(0)); + assert_eq!(empty_view.dim(1), Some(0)); // single element array - let single = Array2D::new(vec![42.0], 1, 1, true); + let single_data = vec![42.0]; + let single_view: StridedArrayView<'_, f64> = unsafe { + StridedArrayView::new( + 2, + [1, 1].as_ptr(), // 1x1 矩阵 + [elem_size as isize, elem_size as isize].as_ptr(), + single_data.as_ptr() as *const u8, + elem_size, + ) + }; let mut buf = vec![]; - single.write_row_major(&mut buf).unwrap(); - assert_eq!(buf, 42.0f64.to_ne_bytes().to_vec()); + write_array_data(&single_view, &mut buf).unwrap(); + assert_eq!(buf, 42.0f64.to_ne_bytes()); } #[test] fn test_buffer_basic_write() -> TestResult { - let test_data = vec![1.1f64, 2.2, 3.3, 4.4]; + let elem_size = std::mem::size_of::(); + + let test_data = vec![1.1, 2.2, 3.3, 4.4]; + let array_view: StridedArrayView<'_, f64> = unsafe { + StridedArrayView::new( + 2, + [2, 2].as_ptr(), + [2 * elem_size as isize, elem_size as isize].as_ptr(), + test_data.as_ptr() as *const u8, + test_data.len() * elem_size, + ) + }; let mut buffer = Buffer::new(); buffer.table("my_test")?; - let array_2d = Array2D::::new(test_data, 2, 2, true); - buffer.column_arr("temperature", &array_2d)?; + buffer.column_arr("temperature", &array_view)?; let data = buffer.as_bytes(); assert_eq!(&data[0..7], b"my_test"); assert_eq!(&data[8..19], b"temperature"); @@ -143,124 +154,79 @@ fn test_buffer_basic_write() -> TestResult { ); assert_eq!( &data[24..32], - [2i32.to_le_bytes().as_slice(), 2i32.to_le_bytes().as_slice()].concat() + [2i32.to_le_bytes(), 2i32.to_le_bytes()].concat() + ); + assert_eq!( + &data[32..64], + &[ + 1.1f64.to_ne_bytes(), + 2.2f64.to_le_bytes(), + 3.3f64.to_le_bytes(), + 4.4f64.to_le_bytes(), + ].concat() ); - Ok(()) -} - -#[test] -fn test_invalid_dimension() -> TestResult { - struct InvalidDimArray; - impl NdArrayView for InvalidDimArray { - fn ndim(&self) -> usize { - 2 - } - fn dim(&self, _: usize) -> Option { - None - } - fn write_row_major(&self, _: &mut W) -> std::io::Result<()> { - Ok(()) - } - } - - let mut buffer = Buffer::new(); - buffer.table("my_test")?; - let result = buffer.column_arr("arr1", &InvalidDimArray); - assert!(result.is_err()); - let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ArrayViewError); - assert!(err - .msg() - .contains("Can not get correct dimensions for dim 0")); Ok(()) } #[test] fn test_size_overflow() -> TestResult { - let mut buffer = Buffer::new(); - buffer.table("my_test")?; - let data = vec![1.0f64]; - let arr = Array2D:: { - data, - rows: usize::MAX, - cols: usize::MAX, - contiguous: false, + let overflow_view = unsafe { + StridedArrayView::::new( + 2, + [usize::MAX, usize::MAX].as_ptr(), + [8, 8].as_ptr(), + ptr::null(), + 0, + ) }; - let result = buffer.column_arr("arr1", &arr); - let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ArrayViewError); - assert!(err.msg().contains("Array total elem size overflow")); - Ok(()) -} - -#[test] -fn test_write_failure() -> TestResult { - struct FaultyArray(PhantomData); - impl NdArrayView for FaultyArray { - fn ndim(&self) -> usize { - 2 - } - fn dim(&self, _: usize) -> Option { - Some(1) - } - fn write_row_major(&self, _: &mut W) -> std::io::Result<()> { - Err(std::io::Error::new( - std::io::ErrorKind::Other, - "mock write error", - )) - } - } let mut buffer = Buffer::new(); buffer.table("my_test")?; - let result = buffer.column_arr("arr1", &FaultyArray(PhantomData::)); + let result = buffer.column_arr("arr1", &overflow_view); let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ArrayWriteToBufferError); - assert!(err - .msg() - .contains("Can not write row major to writer: mock write error")); + assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert!(err.msg().contains("Array total elem size overflow")); Ok(()) } #[test] fn test_array_length_mismatch() -> TestResult { - // actual data length is larger than shapes - let test_data = vec![1.1f64, 2.2, 3.3, 4.4]; - let mut buffer = Buffer::new(); - buffer.table("my_test")?; - let array_2d = Array2D::::new(test_data, 1, 2, true); - let result = buffer.column_arr("arr1", &array_2d); - let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ArrayWriteToBufferError); - assert!(err - .msg() - .contains("Can not write row major to writer: failed to write whole buffer")); - buffer.clear(); + let elem_size = std::mem::size_of::(); + let under_data = vec![1.1]; + let under_view: StridedArrayView<'_, f64> = unsafe { + StridedArrayView::new( + 2, + [1, 2].as_ptr(), + [elem_size as isize, elem_size as isize].as_ptr(), + under_data.as_ptr() as *const u8, + under_data.len() * elem_size, + ) + }; - // actual data length is less than shapes - let test_data = vec![1.1f64]; let mut buffer = Buffer::new(); buffer.table("my_test")?; - let array_2d = Array2D::::new(test_data, 1, 2, true); - let result = buffer.column_arr("arr1", &array_2d); + let result = buffer.column_arr("arr1", &under_view); let err = result.unwrap_err(); assert_eq!(err.code(), ErrorCode::ArrayWriteToBufferError); - assert!(err - .msg() - .contains("Array write buffer length mismatch (actual: 8, expected: 16)")); - buffer.clear(); + assert!(err.msg().contains("Array buffer length mismatch (actual: 8, expected: 16)")); + + let over_data = vec![1.1, 2.2, 3.3]; + let over_view: StridedArrayView<'_, f64> = unsafe { + StridedArrayView::new( + 2, + [1, 2].as_ptr(), + [elem_size as isize, elem_size as isize].as_ptr(), + over_data.as_ptr() as *const u8, + over_data.len() * elem_size, + ) + }; - // non-contiguous layout - let test_data = vec![1.1f64]; - let mut buffer = Buffer::new(); + buffer.clear(); buffer.table("my_test")?; - let array_2d = Array2D::::new(test_data, 1, 2, false); - let result = buffer.column_arr("arr1", &array_2d); + let result = buffer.column_arr("arr1", &over_view); let err = result.unwrap_err(); assert_eq!(err.code(), ErrorCode::ArrayWriteToBufferError); - assert!(err - .msg() - .contains("Array write buffer length mismatch (actual: 8, expected: 16)")); + assert!(err.msg().contains("Array buffer length mismatch (actual: 24, expected: 16)")); Ok(()) } @@ -270,7 +236,7 @@ fn test_1d_contiguous_ndarray_buffer() -> TestResult { let array = arr1(&[1.0, 2.0, 3.0, 4.0]); let view = array.view(); let mut buf = vec![0u8; 4 * size_of::()]; - view.write_row_major(&mut &mut buf[0..])?; + write_array_data(&view, &mut &mut buf[0..])?; let expected: Vec = array .iter() .flat_map(|&x| x.to_ne_bytes().to_vec()) @@ -286,7 +252,7 @@ fn test_2d_non_contiguous_ndarray_buffer() -> TestResult { let transposed = array.view().reversed_axes(); assert!(!transposed.is_standard_layout()); let mut buf = vec![0u8; 4 * size_of::()]; - transposed.write_row_major(&mut &mut buf[0..])?; + write_array_data(&transposed, &mut &mut buf[0..])?; let expected = [1.0f64, 3.0, 2.0, 4.0] .iter() .flat_map(|&x| x.to_ne_bytes()) @@ -307,7 +273,7 @@ fn test_strided_ndarray_layout() -> TestResult { let strided_view = array.slice(s![1..;2, 1..;2]); assert_eq!(strided_view.dim(), (2, 2)); let mut buf = vec![0u8; 4 * size_of::()]; - strided_view.write_row_major(&mut &mut buf[0..])?; + write_array_data(&strided_view, &mut &mut buf[0..])?; // expect：6.0, 8.0, 14.0, 16.0 let expected = [6.0f64, 8.0, 14.0, 16.0] diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index 1f308e25..9f61e4cd 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -37,12 +37,13 @@ use crate::tests::{ #[cfg(feature = "ndarray")] use crate::{ ingress, - ingress::{ElemDataType, NdArrayView}, + ingress::ElemDataType, }; use core::time::Duration; #[cfg(feature = "ndarray")] use ndarray::{arr1, arr2, ArrayD}; +use crate::ingress::ndarr; use std::{io, time::SystemTime}; #[test] @@ -124,9 +125,7 @@ fn test_array_basic() -> TestResult { ] .concat(); let mut array_data2d = vec![0u8; 4 * size_of::()]; - array_2d - .view() - .write_row_major(&mut &mut array_data2d[0..])?; + ndarr::write_array_data(&array_2d.view(), &mut &mut array_data2d[0..])?; let array_header3d = &[ &[b'='][..], @@ -139,9 +138,7 @@ fn test_array_basic() -> TestResult { ] .concat(); let mut array_data3d = vec![0u8; 24 * size_of::()]; - array_3d - .view() - .write_row_major(&mut &mut array_data3d[0..])?; + ndarr::write_array_data(&array_3d.view(), &mut &mut array_data3d[0..])?; let exp = &[ "my_table,device=A001 f1=25.5".as_bytes(), From 31e57c56d4fc9b6bb9db24a84e92ac13a7ceb93b Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 8 Apr 2025 16:55:35 +0800 Subject: [PATCH 023/119] add strideArrayView for rust array ingress api --- questdb-rs/src/ingress/ndarr.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 6be47f69..d5a71901 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -21,7 +21,7 @@ where /// Iterator element type is `&T`. fn iter(&self) -> Self::Iter<'_>; - /// Validates the data buffer size of array is consistency of shapes. + /// Validates the data buffer size of array is consistency with array [`Shapes`]. /// /// # Returns /// - `Ok(usize)`: Expected buffer size in bytes if valid From 533224a4340ca4de44cae90ba3b75e1835f4adb9 Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 8 Apr 2025 21:07:44 +0800 Subject: [PATCH 024/119] add benchmark test. --- include/questdb/ingress/line_sender.h | 3 +- questdb-rs-ffi/Cargo.lock | 1 - questdb-rs-ffi/src/lib.rs | 18 ++++--- questdb-rs/Cargo.toml | 9 +++- questdb-rs/benches/ndarr.rs | 69 +++++++++++++++++++++++++++ questdb-rs/src/ingress/mod.rs | 50 +++++++++++++++++++ questdb-rs/src/ingress/ndarr.rs | 41 ++++++++++++---- questdb-rs/src/tests/ndarr.rs | 11 +++-- questdb-rs/src/tests/sender.rs | 6 +-- 9 files changed, 181 insertions(+), 27 deletions(-) create mode 100644 questdb-rs/benches/ndarr.rs diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 77877f3d..8e2ad20e 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -482,7 +482,8 @@ bool line_sender_buffer_column_f64_arr( line_sender_buffer* buffer, line_sender_column_name name, size_t rank, - const uint32_t* shapes, + const size_t* shapes, + const int64_t* strides, const uint8_t* data_buffer, size_t data_buffer_len, line_sender_error** err_out); diff --git a/questdb-rs-ffi/Cargo.lock b/questdb-rs-ffi/Cargo.lock index a23526e2..5666057c 100644 --- a/questdb-rs-ffi/Cargo.lock +++ b/questdb-rs-ffi/Cargo.lock @@ -467,7 +467,6 @@ dependencies = [ "indoc", "itoa", "libc", - "log", "questdb-confstr", "rand", "rustls", diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 3ffade93..26e8a760 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -33,10 +33,14 @@ use std::ptr; use std::slice; use std::str; -use questdb::{ingress, ingress::{ - Buffer, CertificateAuthority, ColumnName, Protocol, Sender, - SenderBuilder, TableName, TimestampMicros, TimestampNanos, -}, Error, ErrorCode}; +use questdb::{ + ingress, + ingress::{ + Buffer, CertificateAuthority, ColumnName, Protocol, Sender, SenderBuilder, TableName, + TimestampMicros, TimestampNanos, + }, + Error, ErrorCode, +}; macro_rules! bubble_err_to_c { ($err_out:expr, $expression:expr) => { @@ -140,7 +144,7 @@ pub enum line_sender_error_code { line_sender_error_array_view_internal_error, /// Write arrayView to sender buffer error. - line_sender_error_array_view_write_to_buffer_error + line_sender_error_array_view_write_to_buffer_error, } impl From for line_sender_error_code { @@ -836,8 +840,8 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( buffer: *mut line_sender_buffer, name: line_sender_column_name, rank: size_t, - shapes: *const size_t, // C array of shapes - strides: *const i64, // C array of strides + shapes: *const size_t, // C array of shapes + strides: *const i64, // C array of strides data_buffer: *const u8, // Raw array data data_buffer_len: size_t, // Total bytes length err_out: *mut *mut line_sender_error, diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index 90461b53..ba2a5680 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -41,7 +41,6 @@ questdb-confstr = "0.1.1" rand = { version = "0.9.0", optional = true } ndarray = { version = "0.16", optional = true } no-panic = { version = "0.1", optional = true } -log = "0.4.27" [target.'cfg(windows)'.dependencies] winapi = { version = "0.3.9", features = ["ws2def"] } @@ -57,6 +56,7 @@ mio = { version = "1", features = ["os-poll", "net"] } chrono = "0.4.31" tempfile = "3" webpki-roots = "0.26.8" +criterion = "0.5" [features] default = ["tls-webpki-certs", "ilp-over-http", "aws-lc-crypto"] @@ -85,6 +85,8 @@ json_tests = [] # Enable methods to create timestamp objects from chrono::DateTime objects. chrono_timestamp = ["chrono"] +benchmark = [] + # The `aws-lc-crypto` and `ring-crypto` features are mutually exclusive, # thus compiling with `--all-features` will not work. # Instead compile with `--features almost-all-features`. @@ -99,6 +101,11 @@ almost-all-features = [ "ndarray" ] +[[bench]] +name = "ndarr" +harness = false +required-features = ["benchmark", "ndarray"] + [[example]] name = "basic" required-features = ["chrono_timestamp", "ndarray"] diff --git a/questdb-rs/benches/ndarr.rs b/questdb-rs/benches/ndarr.rs new file mode 100644 index 00000000..babac254 --- /dev/null +++ b/questdb-rs/benches/ndarr.rs @@ -0,0 +1,69 @@ +use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use ndarray::{Array, Array2}; +use questdb::ingress::{Buffer, ColumnName}; + +/// run with +/// ```shell +/// cargo bench --bench ndarr --features="benchmark, ndarray" +/// ``` +fn bench_write_array_data(c: &mut Criterion) { + let mut group = c.benchmark_group("write_array_data"); + let contiguous_array: Array2 = Array::zeros((1000, 1000)); + let non_contiguous_array = contiguous_array.t(); + assert!(contiguous_array.is_standard_layout()); + assert!(!non_contiguous_array.is_standard_layout()); + + let col_name = ColumnName::new("col1").unwrap(); + // Case 1 + group.bench_function("contiguous_writer", |b| { + let mut buffer = Buffer::new(); + buffer.table("x1").unwrap(); + b.iter(|| { + buffer + .column_arr(col_name, black_box(&contiguous_array.view())) + .unwrap(); + }); + buffer.clear(); + }); + + // Case 2 + group.bench_function("contiguous_raw_buffer", |b| { + let mut buffer = Buffer::new(); + buffer.table("x1").unwrap(); + b.iter(|| { + buffer + .column_arr_use_raw_buffer(col_name, black_box(&contiguous_array.view())) + .unwrap(); + }); + buffer.clear(); + }); + + // Case 3 + group.bench_function("non_contiguous_writer", |b| { + let mut buffer = Buffer::new(); + buffer.table("x1").unwrap(); + b.iter(|| { + buffer + .column_arr(col_name, black_box(&non_contiguous_array.view())) + .unwrap(); + }); + buffer.clear(); + }); + + // Case 4 + group.bench_function("non_contiguous_raw_buffer", |b| { + let mut buffer = Buffer::new(); + buffer.table("x1").unwrap(); + b.iter(|| { + buffer + .column_arr_use_raw_buffer(col_name, black_box(&non_contiguous_array.view())) + .unwrap(); + }); + buffer.clear(); + }); + + group.finish(); +} + +criterion_group!(benches, bench_write_array_data); +criterion_main!(benches); diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index da093ca6..2374a16d 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -1130,6 +1130,56 @@ impl Buffer { Ok(self) } + #[cfg(feature = "benchmark")] + pub fn column_arr_use_raw_buffer<'a, N, T, D>(&mut self, name: N, view: &T) -> Result<&mut Self> + where + N: TryInto>, + T: NdArrayView, + D: ArrayElement, + Error: From, + { + self.write_column_key(name)?; + + // check dimension less equal than max dims + if MAX_DIMS < view.ndim() { + return Err(error::fmt!( + ArrayHasTooManyDims, + "Array dimension mismatch: expected at most {} dimensions, but got {}", + MAX_DIMS, + view.ndim() + )); + } + + let reserve_size = view.check_data_buf()?; + // binary format flag '=' + self.output.push(b'='); + // binary format entity type + self.output.push(ARRAY_BINARY_FORMAT_TYPE); + // ndarr datatype + self.output.push(D::elem_type().into()); + // ndarr dims + self.output.push(view.ndim() as u8); + + for i in 0..view.ndim() { + let d = view.dim(i).ok_or_else(|| { + error::fmt!( + ArrayViewError, + "Can not get correct dimensions for dim {}", + i + ) + })?; + // ndarr shapes + self.output + .extend_from_slice((d as i32).to_le_bytes().as_slice()); + } + + self.output.reserve(reserve_size); + let index = self.output.len(); + unsafe { self.output.set_len(reserve_size + index) } + ndarr::write_array_data_use_raw_buffer(&mut self.output[index..], view); + Ok(self) + } + /// Record a timestamp value for the given column. /// /// ``` diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index d5a71901..9d4ca801 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -2,7 +2,7 @@ pub trait NdArrayView where T: ArrayElement, { - type Iter<'a>: Iterator + type Iter<'a>: Iterator where Self: 'a, T: 'a; @@ -29,7 +29,7 @@ where fn check_data_buf(&self) -> Result; } -pub(crate) fn write_array_data, T>( +pub fn write_array_data, T>( array: &A, writer: &mut W, ) -> std::io::Result<()> @@ -68,6 +68,30 @@ where Ok(()) } +#[cfg(feature = "benchmark")] +pub(crate) fn write_array_data_use_raw_buffer, T>(buf: &mut [u8], array: &A) +where + T: ArrayElement, +{ + // First optimization path: write contiguous memory directly + if let Some(contiguous) = array.as_slice() { + let byte_len = size_of_val(contiguous); + let bytes = + unsafe { std::slice::from_raw_parts(contiguous.as_ptr() as *const u8, byte_len) }; + buf[..byte_len].copy_from_slice(bytes); + } + + // Fallback path: non-contiguous memory handling + let mut bytes_written = 0; + let elem_size = size_of::(); + for &element in array.iter() { + let element_bytes = + unsafe { std::slice::from_raw_parts(&element as *const T as *const _, elem_size) }; + buf[bytes_written..bytes_written + elem_size].copy_from_slice(element_bytes); + bytes_written += elem_size; + } +} + /// Marker trait for valid array element types. /// /// Implemented for primitive types that can be stored in arrays. @@ -129,7 +153,7 @@ where T: ArrayElement, { type Iter<'b> - = RowMajorIter<'b, T> + = RowMajorIter<'b, T> where Self: 'b, T: 'b; @@ -148,7 +172,9 @@ where fn as_slice(&self) -> Option<&[T]> { if self.is_c_major() { - Some(unsafe { slice::from_raw_parts(self.buf as *const T, self.buf_len / size_of::()) }) + Some(unsafe { + slice::from_raw_parts(self.buf as *const T, self.buf_len / size_of::()) + }) } else { None } @@ -258,10 +284,7 @@ where } let mut contig_stride = size_of::(); - for (dim, stride) in self.shapes.iter() - .rev() - .zip(self.strides.iter().rev()) - { + for (dim, stride) in self.shapes.iter().rev().zip(self.strides.iter().rev()) { if *dim != 1 { let s = *stride; if s.abs() != contig_stride as isize { @@ -341,7 +364,7 @@ where D: Dimension, { type Iter<'a> - = ndarray::iter::Iter<'a, T, D> + = ndarray::iter::Iter<'a, T, D> where Self: 'a, T: 'a; diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index d667b9fb..e0dd25f4 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -163,7 +163,8 @@ fn test_buffer_basic_write() -> TestResult { 2.2f64.to_le_bytes(), 3.3f64.to_le_bytes(), 4.4f64.to_le_bytes(), - ].concat() + ] + .concat() ); Ok(()) } @@ -208,7 +209,9 @@ fn test_array_length_mismatch() -> TestResult { let result = buffer.column_arr("arr1", &under_view); let err = result.unwrap_err(); assert_eq!(err.code(), ErrorCode::ArrayWriteToBufferError); - assert!(err.msg().contains("Array buffer length mismatch (actual: 8, expected: 16)")); + assert!(err + .msg() + .contains("Array buffer length mismatch (actual: 8, expected: 16)")); let over_data = vec![1.1, 2.2, 3.3]; let over_view: StridedArrayView<'_, f64> = unsafe { @@ -226,7 +229,9 @@ fn test_array_length_mismatch() -> TestResult { let result = buffer.column_arr("arr1", &over_view); let err = result.unwrap_err(); assert_eq!(err.code(), ErrorCode::ArrayWriteToBufferError); - assert!(err.msg().contains("Array buffer length mismatch (actual: 24, expected: 16)")); + assert!(err + .msg() + .contains("Array buffer length mismatch (actual: 24, expected: 16)")); Ok(()) } diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index 9f61e4cd..ede64bc6 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -35,15 +35,11 @@ use crate::tests::{ }; #[cfg(feature = "ndarray")] -use crate::{ - ingress, - ingress::ElemDataType, -}; +use crate::{ingress, ingress::ElemDataType}; use core::time::Duration; #[cfg(feature = "ndarray")] use ndarray::{arr1, arr2, ArrayD}; -use crate::ingress::ndarr; use std::{io, time::SystemTime}; #[test] From 8d30e57e22a0965b1822cb48511171239fa0e41b Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 9 Apr 2025 11:59:35 +0800 Subject: [PATCH 025/119] fix tests --- questdb-rs/src/ingress/ndarr.rs | 42 ++++++++++----------------------- questdb-rs/src/tests/ndarr.rs | 29 +++++++++++++++++++++++ questdb-rs/src/tests/sender.rs | 5 ++-- 3 files changed, 44 insertions(+), 32 deletions(-) diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 9d4ca801..27b873b7 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -273,27 +273,21 @@ where return false; } - if self.dims == 1 { - return self.strides[0] == 1 || self.shapes[0] <= 1; + if self.shapes.iter().any(|&d| d == 0) { + return true; } - - for &d in self.shapes { - if d == 0 { - return true; - } + let elem_size = size_of::() as isize; + if self.dims == 1 { + return self.strides[0] == elem_size || self.shapes[0] == 1; } - let mut contig_stride = size_of::(); - for (dim, stride) in self.shapes.iter().rev().zip(self.strides.iter().rev()) { - if *dim != 1 { - let s = *stride; - if s.abs() != contig_stride as isize { - return false; - } - contig_stride *= *dim; + let mut expected_stride = elem_size; + for (dim, &stride) in self.shapes.iter().zip(self.strides).rev() { + if *dim > 1 && stride != expected_stride { + return false; } + expected_stride *= *dim as isize; } - true } } @@ -307,14 +301,6 @@ pub struct RowMajorIter<'a, T> { total_elements: usize, } -impl RowMajorIter<'_, T> { - fn is_ptr_valid(&self, ptr: *const u8) -> bool { - let start = self.array.buf; - let end = unsafe { start.add(self.array.buf_len) }; - ptr >= start && ptr < end - } -} - impl<'a, T> Iterator for RowMajorIter<'a, T> where T: ArrayElement, @@ -336,17 +322,13 @@ where } else { self.array.shapes[dim] - 1 - coord }; - offset += (actual_coord as isize) * stride; + offset += (actual_coord as isize) * stride.abs(); } self.current_linear += 1; unsafe { let ptr = self.base_ptr.offset(offset); - if self.is_ptr_valid(ptr) { - Some(&*(ptr as *const T)) - } else { - None - } + Some(&*(ptr as *const T)) } } } diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index e0dd25f4..c179e089 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -96,6 +96,35 @@ fn test_strided_non_contiguous() -> TestResult { Ok(()) } +#[test] +fn test_negative_strides() -> TestResult { + let elem_size = size_of::(); + let data = vec![1f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]; + let view = unsafe { + StridedArrayView::::new( + 2, + &[3usize, 3usize] as *const usize, + &[-24isize, 8] as *const isize, + (data.as_ptr() as *const u8).add(48), + data.len() * elem_size, + ) + }; + let collected: Vec<_> = view.iter().copied().collect(); + assert!(view.as_slice().is_none()); + let expected_data = vec![7.0, 8.0, 9.0, 4.0, 5.0, 6.0, 1.0, 2.0, 3.0]; + assert_eq!(collected, expected_data); + let mut buffer = Vec::new(); + write_array_data(&view, &mut buffer)?; + let expected_bytes = unsafe { + std::slice::from_raw_parts( + expected_data.as_ptr() as *const u8, + expected_data.len() * elem_size, + ) + }; + assert_eq!(buffer, expected_bytes); + Ok(()) +} + #[test] fn test_basic_edge_cases() { // empty array diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index ede64bc6..f475b2fd 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -40,6 +40,7 @@ use core::time::Duration; #[cfg(feature = "ndarray")] use ndarray::{arr1, arr2, ArrayD}; +use crate::ingress::ndarr::write_array_data; use std::{io, time::SystemTime}; #[test] @@ -121,7 +122,7 @@ fn test_array_basic() -> TestResult { ] .concat(); let mut array_data2d = vec![0u8; 4 * size_of::()]; - ndarr::write_array_data(&array_2d.view(), &mut &mut array_data2d[0..])?; + write_array_data(&array_2d.view(), &mut &mut array_data2d[0..])?; let array_header3d = &[ &[b'='][..], @@ -134,7 +135,7 @@ fn test_array_basic() -> TestResult { ] .concat(); let mut array_data3d = vec![0u8; 24 * size_of::()]; - ndarr::write_array_data(&array_3d.view(), &mut &mut array_data3d[0..])?; + write_array_data(&array_3d.view(), &mut &mut array_data3d[0..])?; let exp = &[ "my_table,device=A001 f1=25.5".as_bytes(), From 36203a6c5ef1b18f104257ccb32c3ba45ba078df Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 9 Apr 2025 16:18:10 +0800 Subject: [PATCH 026/119] better memory copy. --- questdb-rs/src/ingress/ndarr.rs | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 27b873b7..325435d9 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -76,19 +76,25 @@ where // First optimization path: write contiguous memory directly if let Some(contiguous) = array.as_slice() { let byte_len = size_of_val(contiguous); - let bytes = - unsafe { std::slice::from_raw_parts(contiguous.as_ptr() as *const u8, byte_len) }; - buf[..byte_len].copy_from_slice(bytes); + unsafe { + ptr::copy_nonoverlapping( + contiguous.as_ptr() as *const u8, + buf.as_mut_ptr(), + byte_len, + ) + } } // Fallback path: non-contiguous memory handling - let mut bytes_written = 0; let elem_size = size_of::(); - for &element in array.iter() { - let element_bytes = - unsafe { std::slice::from_raw_parts(&element as *const T as *const _, elem_size) }; - buf[bytes_written..bytes_written + elem_size].copy_from_slice(element_bytes); - bytes_written += elem_size; + for (i, &element) in array.iter().enumerate() { + unsafe { + ptr::copy_nonoverlapping( + &element as *const T as *const u8, + buf.as_mut_ptr().add(i * elem_size), + elem_size, + ) + } } } @@ -337,7 +343,7 @@ use crate::{error, Error}; #[cfg(feature = "ndarray")] use ndarray::{ArrayView, Axis, Dimension}; use std::io::IoSlice; -use std::slice; +use std::{ptr, slice}; #[cfg(feature = "ndarray")] impl NdArrayView for ArrayView<'_, T, D> From feaf5be25830c007386c34a1f1149d28f6dd4d0c Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 9 Apr 2025 16:22:35 +0800 Subject: [PATCH 027/119] code format --- questdb-rs/src/ingress/ndarr.rs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 325435d9..701ecb89 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -77,11 +77,7 @@ where if let Some(contiguous) = array.as_slice() { let byte_len = size_of_val(contiguous); unsafe { - ptr::copy_nonoverlapping( - contiguous.as_ptr() as *const u8, - buf.as_mut_ptr(), - byte_len, - ) + std::ptr::copy_nonoverlapping(contiguous.as_ptr() as *const u8, buf.as_mut_ptr(), byte_len) } } @@ -89,7 +85,7 @@ where let elem_size = size_of::(); for (i, &element) in array.iter().enumerate() { unsafe { - ptr::copy_nonoverlapping( + std::ptr::copy_nonoverlapping( &element as *const T as *const u8, buf.as_mut_ptr().add(i * elem_size), elem_size, @@ -279,9 +275,6 @@ where return false; } - if self.shapes.iter().any(|&d| d == 0) { - return true; - } let elem_size = size_of::() as isize; if self.dims == 1 { return self.strides[0] == elem_size || self.shapes[0] == 1; @@ -343,7 +336,7 @@ use crate::{error, Error}; #[cfg(feature = "ndarray")] use ndarray::{ArrayView, Axis, Dimension}; use std::io::IoSlice; -use std::{ptr, slice}; +use std::slice; #[cfg(feature = "ndarray")] impl NdArrayView for ArrayView<'_, T, D> From c4bed101735f9f98900ff227c45862bc337f628d Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 9 Apr 2025 17:21:46 +0800 Subject: [PATCH 028/119] make c api works. --- cpp_test/test_line_sender.cpp | 19 +++++----- examples/line_sender_c_example.c | 2 ++ include/questdb/ingress/line_sender.h | 10 +++--- include/questdb/ingress/line_sender.hpp | 12 +++---- questdb-rs-ffi/src/lib.rs | 15 ++++---- questdb-rs/src/ingress/ndarr.rs | 48 ++++++++++++------------- questdb-rs/src/tests/ndarr.rs | 46 ++++++++++++------------ questdb-rs/src/tests/sender.rs | 4 +-- 8 files changed, 79 insertions(+), 77 deletions(-) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 80cd72da..198f13cf 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -100,6 +100,7 @@ TEST_CASE("line_sender c api basics") // 3D array of doubles size_t rank = 3; uint32_t shapes[] = {2, 3, 2}; + int32_t strides[] = {48, 16, 8}; double arr_data[] = { 48123.5, 2.4, @@ -113,14 +114,16 @@ TEST_CASE("line_sender c api basics") 2.7, 48121.5, 4.3}; - CHECK(::line_sender_buffer_column_f64_arr( - buffer, - arr_name, - rank, - shapes, - reinterpret_cast(arr_data), - sizeof(arr_data), - &err)); + CHECK( + ::line_sender_buffer_column_f64_arr( + buffer, + arr_name, + rank, + shapes, + strides, + reinterpret_cast(arr_data), + sizeof(arr_data), + &err)); CHECK(::line_sender_buffer_at_nanos(buffer, 10000000, &err)); CHECK(server.recv() == 0); CHECK(::line_sender_buffer_size(buffer) == 27); diff --git a/examples/line_sender_c_example.c b/examples/line_sender_c_example.c index 59190e53..64807fe9 100644 --- a/examples/line_sender_c_example.c +++ b/examples/line_sender_c_example.c @@ -59,6 +59,7 @@ static bool example(const char* host, const char* port) // 3D array of doubles size_t rank = 3; uint32_t shapes[] = {2, 3, 2}; + int32_t strides[] = {48, 16, 8}; double arr_data[] = { 48123.5, 2.4, @@ -77,6 +78,7 @@ static bool example(const char* host, const char* port) arr_name, rank, shapes, + strides, (const uint8_t*)arr_data, sizeof(arr_data), &err)) diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 8e2ad20e..122897d5 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -470,10 +470,8 @@ bool line_sender_buffer_column_str( * @param[in] rank Number of dimensions of the array. * @param[in] shapes Array of dimension sizes (length = `rank`). * Each element must be a positive integer. - * @param[in] data_buffer Raw bytes of the array data in little-endian format. - * Size must be `sizeof(double) * (shapes[0] * ... * - * shapes[rank-1])`. - * @param[in] data_buffer_len Byte length of the data buffer. + * @param[in] data_buffer First array element data. + * @param[in] data_buffer_len Bytes length of the array data. * @param[out] err_out Set to an error object on failure (if non-NULL). * @return true on success, false on error. */ @@ -482,8 +480,8 @@ bool line_sender_buffer_column_f64_arr( line_sender_buffer* buffer, line_sender_column_name name, size_t rank, - const size_t* shapes, - const int64_t* strides, + const uint32_t* shapes, + const int32_t* strides, const uint8_t* data_buffer, size_t data_buffer_len, line_sender_error** err_out); diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 1f2ed06e..1e9adf82 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -631,18 +631,15 @@ class line_sender_buffer * * @param name Column name. * @param shape Array dimensions (e.g., [2,3] for a 2x3 matrix). - * @param data Array data in row-major order. Size must match product of + * @param data Array first element data. Size must match product of * dimensions. - * - * @note Data is stored contiguously in row-major (C-style) order. - * Example: shape [2,3] expects 6 elements ordered as: - * [a11, a12, a13, a21, a22, a23] */ template line_sender_buffer& column( column_name_view name, - const std::vector& shapes, - const std::vector& strides, + const size_t rank, + const std::vector& shapes, + const std::vector& strides, const std::array& data) { static_assert( @@ -653,6 +650,7 @@ class line_sender_buffer ::line_sender_buffer_column_f64_arr, _impl, name._impl, + rank, shapes.data(), strides.data(), reinterpret_cast(data.data()), diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 26e8a760..8f28914b 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -828,7 +828,8 @@ pub unsafe extern "C" fn line_sender_buffer_column_str( /// @param[in] name Column name. /// @param[in] rank Array dims. /// @param[in] shape Array shapes. -/// @param[in] data_buffer Array data memory ptr. +/// @param[in] strides Array strides. +/// @param[in] data_buffer Array **first element** data memory ptr. /// @param[in] data_buffer_len Array data memory length. /// @param[out] err_out Set on error. /// # Safety @@ -840,18 +841,18 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( buffer: *mut line_sender_buffer, name: line_sender_column_name, rank: size_t, - shapes: *const size_t, // C array of shapes - strides: *const i64, // C array of strides - data_buffer: *const u8, // Raw array data - data_buffer_len: size_t, // Total bytes length + shape: *const u32, + strides: *const i32, + data_buffer: *const u8, + data_buffer_len: size_t, err_out: *mut *mut line_sender_error, ) -> bool { let buffer = unwrap_buffer_mut(buffer); let name = name.as_name(); let view = ingress::StridedArrayView::::new( rank, - shapes, - strides as *const isize, + shape, + strides, data_buffer, data_buffer_len, ); diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 701ecb89..4f619358 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -143,8 +143,8 @@ impl ArrayElement for f64 { #[derive(Debug)] pub struct StridedArrayView<'a, T> { dims: usize, - shapes: &'a [usize], - strides: &'a [isize], + shape: &'a [u32], + strides: &'a [i32], buf_len: usize, buf: *const u8, _marker: std::marker::PhantomData, @@ -169,7 +169,7 @@ where return None; } - Some(self.shapes[index]) + Some(self.shape[index] as usize) } fn as_slice(&self) -> Option<&[T]> { @@ -185,7 +185,7 @@ where fn iter(&self) -> Self::Iter<'_> { let mut dim_products = Vec::with_capacity(self.dims); let mut product = 1; - for &dim in self.shapes.iter().rev() { + for &dim in self.shape.iter().rev() { dim_products.push(product); product *= dim; } @@ -198,8 +198,8 @@ where .enumerate() .fold(self.buf, |ptr, (dim, &stride)| { if stride < 0 { - let dim_size = self.shapes[dim] as isize; - unsafe { ptr.offset(stride * (dim_size - 1)) } + let dim_size = self.shape[dim] as isize; + unsafe { ptr.offset(stride as isize * (dim_size - 1)) } } else { ptr } @@ -209,15 +209,15 @@ where array: self, dim_products, current_linear: 0, - total_elements: self.shapes.iter().product(), + total_elements: self.shape.iter().product(), } } fn check_data_buf(&self) -> Result { let mut size = size_of::(); for i in 0..self.dims { - let d = self.shapes[i]; - size = size.checked_mul(d).ok_or(error::fmt!( + let d = self.shape[i]; + size = size.checked_mul(d as usize).ok_or(error::fmt!( ArrayViewError, "Array total elem size overflow" ))? @@ -252,16 +252,16 @@ where /// - Strides are measured in bytes (not elements) pub unsafe fn new( dims: usize, - shapes: *const usize, - strides: *const isize, + shape: *const u32, + strides: *const i32, data: *const u8, data_len: usize, ) -> Self { - let shapes = slice::from_raw_parts(shapes, dims); + let shapes = slice::from_raw_parts(shape, dims); let strides = slice::from_raw_parts(strides, dims); Self { dims, - shapes, + shape: shapes, strides, buf_len: data_len, buf: data, @@ -275,17 +275,17 @@ where return false; } - let elem_size = size_of::() as isize; + let elem_size = size_of::() as i32; if self.dims == 1 { - return self.strides[0] == elem_size || self.shapes[0] == 1; + return self.strides[0] == elem_size || self.shape[0] == 1; } let mut expected_stride = elem_size; - for (dim, &stride) in self.shapes.iter().zip(self.strides).rev() { + for (dim, &stride) in self.shape.iter().zip(self.strides).rev() { if *dim > 1 && stride != expected_stride { return false; } - expected_stride *= *dim as isize; + expected_stride *= *dim as i32; } true } @@ -295,9 +295,9 @@ where pub struct RowMajorIter<'a, T> { base_ptr: *const u8, array: &'a StridedArrayView<'a, T>, - dim_products: Vec, - current_linear: usize, - total_elements: usize, + dim_products: Vec, + current_linear: u32, + total_elements: u32, } impl<'a, T> Iterator for RowMajorIter<'a, T> @@ -310,7 +310,7 @@ where return None; } let mut remaining_index = self.current_linear; - let mut offset = 0isize; + let mut offset = 0; for (dim, &dim_factor) in self.dim_products.iter().enumerate() { let coord = remaining_index / dim_factor; @@ -319,14 +319,14 @@ where let actual_coord = if stride >= 0 { coord } else { - self.array.shapes[dim] - 1 - coord + self.array.shape[dim] - 1 - coord }; - offset += (actual_coord as isize) * stride.abs(); + offset += actual_coord * stride.unsigned_abs(); } self.current_linear += 1; unsafe { - let ptr = self.base_ptr.offset(offset); + let ptr = self.base_ptr.offset(offset as isize); Some(&*(ptr as *const T)) } } diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index c179e089..308c5131 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -33,10 +33,10 @@ fn to_bytes(data: &[T]) -> Vec { fn test_strided_array_view() -> TestResult { // contiguous layout let test_data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; - let shapes = [2, 3]; + let shapes = [2u32, 3]; let strides = [ - (shapes[1] * size_of::()) as isize, - size_of::() as isize, + (shapes[1] * size_of::() as u32) as i32, + size_of::() as i32, ]; let array = unsafe { StridedArrayView::::new( @@ -62,10 +62,10 @@ fn test_strided_array_view() -> TestResult { #[test] fn test_strided_non_contiguous() -> TestResult { - let elem_size = size_of::(); + let elem_size = size_of::() as i32; let col_major_data = vec![1.0, 3.0, 5.0, 2.0, 4.0, 6.0]; - let shapes = [3, 2]; - let strides = [elem_size as isize, (shapes[0] * elem_size) as isize]; + let shapes = [3u32, 2]; + let strides = [elem_size, shapes[0] as i32 * elem_size]; let array_view: StridedArrayView<'_, f64> = unsafe { StridedArrayView::new( @@ -73,7 +73,7 @@ fn test_strided_non_contiguous() -> TestResult { shapes.as_ptr(), strides.as_ptr(), col_major_data.as_ptr() as *const u8, - col_major_data.len() * elem_size, + col_major_data.len() * elem_size as usize, ) }; @@ -89,7 +89,7 @@ fn test_strided_non_contiguous() -> TestResult { let expected_bytes = unsafe { std::slice::from_raw_parts( expected_data.as_ptr() as *const u8, - expected_data.len() * elem_size, + expected_data.len() * elem_size as usize, ) }; assert_eq!(buffer, expected_bytes); @@ -103,8 +103,8 @@ fn test_negative_strides() -> TestResult { let view = unsafe { StridedArrayView::::new( 2, - &[3usize, 3usize] as *const usize, - &[-24isize, 8] as *const isize, + &[3u32, 3] as *const u32, + &[-24i32, 8] as *const i32, (data.as_ptr() as *const u8).add(48), data.len() * elem_size, ) @@ -128,7 +128,7 @@ fn test_negative_strides() -> TestResult { #[test] fn test_basic_edge_cases() { // empty array - let elem_size = std::mem::size_of::(); + let elem_size = std::mem::size_of::() as i32; let empty_view: StridedArrayView<'_, f64> = unsafe { StridedArrayView::new(2, [0, 0].as_ptr(), [0, 0].as_ptr(), ptr::null(), 0) }; assert_eq!(empty_view.ndim(), 2); @@ -140,10 +140,10 @@ fn test_basic_edge_cases() { let single_view: StridedArrayView<'_, f64> = unsafe { StridedArrayView::new( 2, - [1, 1].as_ptr(), // 1x1 矩阵 - [elem_size as isize, elem_size as isize].as_ptr(), + [1, 1].as_ptr(), + [elem_size, elem_size].as_ptr(), single_data.as_ptr() as *const u8, - elem_size, + elem_size as usize, ) }; let mut buf = vec![]; @@ -153,16 +153,16 @@ fn test_basic_edge_cases() { #[test] fn test_buffer_basic_write() -> TestResult { - let elem_size = std::mem::size_of::(); + let elem_size = std::mem::size_of::() as i32; let test_data = vec![1.1, 2.2, 3.3, 4.4]; let array_view: StridedArrayView<'_, f64> = unsafe { StridedArrayView::new( 2, [2, 2].as_ptr(), - [2 * elem_size as isize, elem_size as isize].as_ptr(), + [2 * elem_size, elem_size].as_ptr(), test_data.as_ptr() as *const u8, - test_data.len() * elem_size, + test_data.len() * elem_size as usize, ) }; let mut buffer = Buffer::new(); @@ -203,7 +203,7 @@ fn test_size_overflow() -> TestResult { let overflow_view = unsafe { StridedArrayView::::new( 2, - [usize::MAX, usize::MAX].as_ptr(), + [u32::MAX, u32::MAX].as_ptr(), [8, 8].as_ptr(), ptr::null(), 0, @@ -221,15 +221,15 @@ fn test_size_overflow() -> TestResult { #[test] fn test_array_length_mismatch() -> TestResult { - let elem_size = std::mem::size_of::(); + let elem_size = std::mem::size_of::() as i32; let under_data = vec![1.1]; let under_view: StridedArrayView<'_, f64> = unsafe { StridedArrayView::new( 2, [1, 2].as_ptr(), - [elem_size as isize, elem_size as isize].as_ptr(), + [elem_size, elem_size].as_ptr(), under_data.as_ptr() as *const u8, - under_data.len() * elem_size, + under_data.len() * elem_size as usize, ) }; @@ -247,9 +247,9 @@ fn test_array_length_mismatch() -> TestResult { StridedArrayView::new( 2, [1, 2].as_ptr(), - [elem_size as isize, elem_size as isize].as_ptr(), + [elem_size, elem_size].as_ptr(), over_data.as_ptr() as *const u8, - over_data.len() * elem_size, + over_data.len() * elem_size as usize, ) }; diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index f475b2fd..3edc0eb8 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -34,13 +34,13 @@ use crate::tests::{ TestResult, }; +#[cfg(feature = "ndarray")] +use crate::ingress::ndarr::write_array_data; #[cfg(feature = "ndarray")] use crate::{ingress, ingress::ElemDataType}; use core::time::Duration; #[cfg(feature = "ndarray")] use ndarray::{arr1, arr2, ArrayD}; - -use crate::ingress::ndarr::write_array_data; use std::{io, time::SystemTime}; #[test] From f3011e120b477a10ec6be63a1aeac2d66da1f3f8 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 9 Apr 2025 17:22:21 +0800 Subject: [PATCH 029/119] make c api works. --- examples/line_sender_c_example_auth.c | 2 ++ examples/line_sender_c_example_auth_tls.c | 2 ++ examples/line_sender_c_example_from_conf.c | 2 ++ examples/line_sender_c_example_from_env.c | 2 ++ examples/line_sender_c_example_http.c | 2 ++ examples/line_sender_c_example_tls_ca.c | 2 ++ 6 files changed, 12 insertions(+) diff --git a/examples/line_sender_c_example_auth.c b/examples/line_sender_c_example_auth.c index 1564da88..c6905a9e 100644 --- a/examples/line_sender_c_example_auth.c +++ b/examples/line_sender_c_example_auth.c @@ -64,6 +64,7 @@ static bool example(const char* host, const char* port) // 3D array of doubles size_t rank = 3; uint32_t shapes[] = {2, 3, 2}; + int32_t strides[] = {48, 16, 8}; double arr_data[] = { 48123.5, 2.4, @@ -82,6 +83,7 @@ static bool example(const char* host, const char* port) arr_name, rank, shapes, + strides, (const uint8_t*)arr_data, sizeof(arr_data), &err)) diff --git a/examples/line_sender_c_example_auth_tls.c b/examples/line_sender_c_example_auth_tls.c index 523e91d1..16abb263 100644 --- a/examples/line_sender_c_example_auth_tls.c +++ b/examples/line_sender_c_example_auth_tls.c @@ -64,6 +64,7 @@ static bool example(const char* host, const char* port) // 3D array of doubles size_t rank = 3; uint32_t shapes[] = {2, 3, 2}; + int32_t strides[] = {48, 16, 8}; double arr_data[] = { 48123.5, 2.4, @@ -82,6 +83,7 @@ static bool example(const char* host, const char* port) arr_name, rank, shapes, + strides, (const uint8_t*)arr_data, sizeof(arr_data), &err)) diff --git a/examples/line_sender_c_example_from_conf.c b/examples/line_sender_c_example_from_conf.c index ec3a217f..6b077e58 100644 --- a/examples/line_sender_c_example_from_conf.c +++ b/examples/line_sender_c_example_from_conf.c @@ -48,6 +48,7 @@ int main(int argc, const char* argv[]) // 3D array of doubles size_t rank = 3; uint32_t shapes[] = {2, 3, 2}; + int32_t strides[] = {48, 16, 8}; double arr_data[] = { 48123.5, 2.4, @@ -66,6 +67,7 @@ int main(int argc, const char* argv[]) arr_name, rank, shapes, + strides, (const uint8_t*)arr_data, sizeof(arr_data), &err)) diff --git a/examples/line_sender_c_example_from_env.c b/examples/line_sender_c_example_from_env.c index f949b843..5121bf70 100644 --- a/examples/line_sender_c_example_from_env.c +++ b/examples/line_sender_c_example_from_env.c @@ -47,6 +47,7 @@ int main(int argc, const char* argv[]) // 3D array of doubles size_t rank = 3; uint32_t shapes[] = {2, 3, 2}; + int32_t strides[] = {48, 16, 8}; double arr_data[] = { 48123.5, 2.4, @@ -65,6 +66,7 @@ int main(int argc, const char* argv[]) arr_name, rank, shapes, + strides, (const uint8_t*)arr_data, sizeof(arr_data), &err)) diff --git a/examples/line_sender_c_example_http.c b/examples/line_sender_c_example_http.c index e4dcf662..7d504a7e 100644 --- a/examples/line_sender_c_example_http.c +++ b/examples/line_sender_c_example_http.c @@ -58,6 +58,7 @@ static bool example(const char* host, const char* port) // 3D array of doubles size_t rank = 3; uint32_t shapes[] = {2, 3, 2}; + int32_t strides[] = {48, 16, 8}; double arr_data[] = { 48123.5, 2.4, @@ -76,6 +77,7 @@ static bool example(const char* host, const char* port) arr_name, rank, shapes, + strides, (const uint8_t*)arr_data, sizeof(arr_data), &err)) diff --git a/examples/line_sender_c_example_tls_ca.c b/examples/line_sender_c_example_tls_ca.c index d52901fe..3b8fb48a 100644 --- a/examples/line_sender_c_example_tls_ca.c +++ b/examples/line_sender_c_example_tls_ca.c @@ -65,6 +65,7 @@ static bool example(const char* ca_path, const char* host, const char* port) // 3D array of doubles size_t rank = 3; uint32_t shapes[] = {2, 3, 2}; + int32_t strides[] = {48, 16, 8}; double arr_data[] = { 48123.5, 2.4, @@ -83,6 +84,7 @@ static bool example(const char* ca_path, const char* host, const char* port) arr_name, rank, shapes, + strides, (const uint8_t*)arr_data, sizeof(arr_data), &err)) From 70eaa24a8f68dccfa1aa4b339a2bfaf22412f859 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 9 Apr 2025 17:24:35 +0800 Subject: [PATCH 030/119] code format --- questdb-rs-ffi/src/lib.rs | 9 ++------- questdb-rs/src/ingress/ndarr.rs | 6 +++++- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 8f28914b..b877e31c 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -849,13 +849,8 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( ) -> bool { let buffer = unwrap_buffer_mut(buffer); let name = name.as_name(); - let view = ingress::StridedArrayView::::new( - rank, - shape, - strides, - data_buffer, - data_buffer_len, - ); + let view = + ingress::StridedArrayView::::new(rank, shape, strides, data_buffer, data_buffer_len); bubble_err_to_c!( err_out, buffer.column_arr::, ingress::StridedArrayView<'_, f64>, f64>(name, &view) diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 4f619358..61626171 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -77,7 +77,11 @@ where if let Some(contiguous) = array.as_slice() { let byte_len = size_of_val(contiguous); unsafe { - std::ptr::copy_nonoverlapping(contiguous.as_ptr() as *const u8, buf.as_mut_ptr(), byte_len) + std::ptr::copy_nonoverlapping( + contiguous.as_ptr() as *const u8, + buf.as_mut_ptr(), + byte_len, + ) } } From 526e8ae7fece238e0e83b406ebb13869d478cb5c Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 9 Apr 2025 21:11:59 +0800 Subject: [PATCH 031/119] make c++ tests work. --- cpp_test/mock_server.cpp | 57 +++++++++++++---- cpp_test/mock_server.hpp | 8 ++- cpp_test/test_line_sender.cpp | 116 ++++++++++++++++++++-------------- questdb-rs/src/tests/mock.rs | 2 +- 4 files changed, 120 insertions(+), 63 deletions(-) diff --git a/cpp_test/mock_server.cpp b/cpp_test/mock_server.cpp index 0e8a1e85..bfc67e0f 100644 --- a/cpp_test/mock_server.cpp +++ b/cpp_test/mock_server.cpp @@ -190,14 +190,21 @@ bool mock_server::wait_for_data(std::optional wait_timeout_sec) return !!count; } +int32_t bytes_to_int32_le(const std::byte* bytes) +{ + return static_cast( + (bytes[0] << 0) | (bytes[1] << 8) | (bytes[2] << 16) | + (bytes[3] << 24)); +} + size_t mock_server::recv(double wait_timeout_sec) { if (!wait_for_data(wait_timeout_sec)) return 0; - char chunk[1024]; + std::byte chunk[1024]; size_t chunk_len{sizeof(chunk)}; - std::vector accum; + std::vector accum; for (;;) { wait_for_data(); @@ -209,24 +216,52 @@ size_t mock_server::recv(double wait_timeout_sec) accum.insert(accum.end(), chunk, chunk + u_count); if (accum.size() < 2) continue; - if ((accum[accum.size() - 1] == '\n') && - (accum[accum.size() - 2] != '\\')) + if ((accum[accum.size() - 1] == std::byte('\n')) && + (accum[accum.size() - 2] != std::byte('\\'))) break; } size_t received_count{0}; - const char* head{&accum[0]}; - for (size_t index = 1; index < accum.size(); ++index) + const std::byte* head{&accum[0]}; + size_t index{1}; + while (index < accum.size()) { - const char& last = accum[index]; - const char& prev = accum[index - 1]; - if ((last == '\n') && (prev != '\\')) + const std::byte& last = accum[index]; + const std::byte& prev = accum[index - 1]; + if (last == std::byte('=') && prev == std::byte('=')) + { + index++; + std::byte& binary_type = accum[index]; + if (binary_type == std::byte(16)) // DOUBLE_BINARY_FORMAT_TYPE + index += sizeof(double) + 1; + else if (binary_type == std::byte(14)) // ARRAY_BINARY_FORMAT_TYPE + { + index++; + const std::byte& array_elem_type = accum[index]; + if (array_elem_type == std::byte(10)) + { + index++; + const size_t dims = size_t(accum[index]); + index++; + size_t data_size{sizeof(double)}; + for (size_t i = 0; i < dims; i++) + { + data_size *= bytes_to_int32_le(&accum[index]); + index += sizeof(int32_t); + } + index += data_size; + } + } + continue; + } + else if ((last == std::byte('\n')) && (prev != std::byte('\\'))) { - const char* tail{&last + 1}; - _msgs.emplace_back(head, tail - head); + const std::byte* tail{&last + 1}; + _msgs.emplace_back(head, tail); head = tail; ++received_count; } + index++; } return received_count; } diff --git a/cpp_test/mock_server.hpp b/cpp_test/mock_server.hpp index ba66efb0..dff8ff4e 100644 --- a/cpp_test/mock_server.hpp +++ b/cpp_test/mock_server.hpp @@ -31,6 +31,7 @@ #include #include "build_env.h" +#include "questdb/ingress/line_sender.hpp" #if defined(PLATFORM_UNIX) typedef int socketfd_t; @@ -60,9 +61,10 @@ class mock_server size_t recv(double wait_timeout_sec = 0.1); - const std::vector& msgs() const + const buffer_view msgs(size_t index) const { - return _msgs; + assert(index < _msgs.size()); + return buffer_view{_msgs[index].data(), _msgs[index].size()}; } void close(); @@ -75,7 +77,7 @@ class mock_server socketfd_t _listen_fd; socketfd_t _conn_fd; uint16_t _port; - std::vector _msgs; + std::vector> _msgs; }; } // namespace questdb::ingress::test diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 198f13cf..7036d268 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -56,6 +56,60 @@ class on_scope_exit F _f; }; +#if __cplusplus >= 202002L +template +bool operator==(std::span lhs, const char (&rhs)[N]) +{ + constexpr size_t bytelen = N - 1; // Exclude null terminator + const std::span rhs_span{ + reinterpret_cast(rhs), bytelen}; + return lhs.size() == bytelen && std::ranges::equal(lhs, rhs_span); +} + +bool operator==(std::span lhs, const std::string& rhs) +{ + const std::span rhs_span{ + reinterpret_cast(rhs.data()), rhs.size()}; + return lhs.size() == rhs.size() && std::ranges::equal(lhs, rhs_span); +} +#else +template +bool operator==( + const questdb::ingress::buffer_view lhs_view, const char (&rhs)[N]) +{ + constexpr size_t bytelen = N - 1; // Exclude null terminator + const questdb::ingress::buffer_view rhs_view{ + reinterpret_cast(rhs), bytelen}; + return lhs_view == rhs_view; +} + +bool operator==( + const questdb::ingress::buffer_view lhs_view, const std::string& rhs) +{ + const questdb::ingress::buffer_view rhs_view{ + reinterpret_cast(rhs.data()), rhs.size()}; + return lhs_view == rhs_view; +} +#endif + +template +void push_double_arr_to_buffer( + std::string& buffer, + std::array data, + size_t rank, + uint32_t* shapes) +{ + buffer.push_back(14); + buffer.push_back(10); + buffer.push_back(static_cast(rank)); + for (size_t i = 0; i < rank; ++i) + buffer.append( + reinterpret_cast(&shapes[i]), sizeof(uint32_t)); + buffer.append( + reinterpret_cast(data.data()), + data.size() * sizeof(double)); +} + TEST_CASE("line_sender c api basics") { questdb::ingress::test::mock_server server; @@ -96,12 +150,12 @@ TEST_CASE("line_sender c api basics") CHECK(::line_sender_buffer_symbol(buffer, t1_name, v1_utf8, &err)); CHECK(::line_sender_buffer_column_f64(buffer, f1_name, 0.5, &err)); - line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("order_book"); + line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("a1"); // 3D array of doubles size_t rank = 3; uint32_t shapes[] = {2, 3, 2}; int32_t strides[] = {48, 16, 8}; - double arr_data[] = { + std::array arr_data = { 48123.5, 2.4, 48124.0, @@ -121,16 +175,19 @@ TEST_CASE("line_sender c api basics") rank, shapes, strides, - reinterpret_cast(arr_data), + reinterpret_cast(arr_data.data()), sizeof(arr_data), &err)); CHECK(::line_sender_buffer_at_nanos(buffer, 10000000, &err)); CHECK(server.recv() == 0); - CHECK(::line_sender_buffer_size(buffer) == 27); + CHECK(::line_sender_buffer_size(buffer) == 143); CHECK(::line_sender_flush(sender, buffer, &err)); ::line_sender_buffer_free(buffer); CHECK(server.recv() == 1); - CHECK(server.msgs().front() == "test,t1=v1 f1=0.5 10000000\n"); + std::string expect{"test,t1=v1 f1=0.5,a1=="}; + push_double_arr_to_buffer(expect, arr_data, 3, shapes); + expect.append(" 10000000\n"); + CHECK(server.msgs(0) == expect); } TEST_CASE("Opts service API tests") @@ -189,7 +246,7 @@ TEST_CASE("line_sender c++ api basics") CHECK(buffer.size() == 31); sender.flush(buffer); CHECK(server.recv() == 1); - CHECK(server.msgs().front() == "test,t1=v1,t2= f1=0.5 10000000\n"); + CHECK(server.msgs(0) == "test,t1=v1,t2= f1=0.5 10000000\n"); } TEST_CASE("test multiple lines") @@ -226,12 +283,11 @@ TEST_CASE("test multiple lines") sender.flush(buffer); CHECK(server.recv() == 2); CHECK( - server.msgs()[0] == + server.msgs(0) == ("metric1,t1=val1,t2=val2 f1=t,f2=12345i," "f3=10.75,f4=\"val3\",f5=\"val4\",f6=\"val5\" 111222233333\n")); CHECK( - server.msgs()[1] == - "metric1,tag3=value\\ 3,tag\\ 4=value:4 field5=f\n"); + server.msgs(1) == "metric1,tag3=value\\ 3,tag\\ 4=value:4 field5=f\n"); } TEST_CASE("State machine testing -- flush without data.") @@ -272,7 +328,7 @@ TEST_CASE("One symbol only - flush before server accept") // but the server hasn't actually accepted the client connection yet. server.accept(); CHECK(server.recv() == 1); - CHECK(server.msgs()[0] == "test,t1=v1\n"); + CHECK(server.msgs(0) == "test,t1=v1\n"); } TEST_CASE("One column only - server.accept() after flush, before close") @@ -292,7 +348,7 @@ TEST_CASE("One column only - server.accept() after flush, before close") sender.close(); CHECK(server.recv() == 1); - CHECK(server.msgs()[0] == "test t1=\"v1\"\n"); + CHECK(server.msgs(0) == "test t1=\"v1\"\n"); } TEST_CASE("Symbol after column") @@ -415,42 +471,6 @@ TEST_CASE("Validation of bad chars in key names.") } } -#if __cplusplus >= 202002L -template -bool operator==(std::span lhs, const char (&rhs)[N]) -{ - constexpr size_t bytelen = N - 1; // Exclude null terminator - const std::span rhs_span{ - reinterpret_cast(rhs), bytelen}; - return lhs.size() == bytelen && std::ranges::equal(lhs, rhs_span); -} - -bool operator==(std::span lhs, const std::string& rhs) -{ - const std::span rhs_span{ - reinterpret_cast(rhs.data()), rhs.size()}; - return lhs.size() == rhs.size() && std::ranges::equal(lhs, rhs_span); -} -#else -template -bool operator==( - const questdb::ingress::buffer_view lhs_view, const char (&rhs)[N]) -{ - constexpr size_t bytelen = N - 1; // Exclude null terminator - const questdb::ingress::buffer_view rhs_view{ - reinterpret_cast(rhs), bytelen}; - return lhs_view == rhs_view; -} - -bool operator==( - const questdb::ingress::buffer_view lhs_view, const std::string& rhs) -{ - const questdb::ingress::buffer_view rhs_view{ - reinterpret_cast(rhs.data()), rhs.size()}; - return lhs_view == rhs_view; -} -#endif - TEST_CASE("Buffer move and copy ctor testing") { const size_t init_buf_size = 128; @@ -744,7 +764,7 @@ TEST_CASE("Test timestamp column.") sender.close(); CHECK(server.recv() == 1); - CHECK(server.msgs()[0] == exp); + CHECK(server.msgs(0) == exp); } TEST_CASE("test timestamp_micros and timestamp_nanos::now()") diff --git a/questdb-rs/src/tests/mock.rs b/questdb-rs/src/tests/mock.rs index 1d1f438a..03fcf527 100644 --- a/questdb-rs/src/tests/mock.rs +++ b/questdb-rs/src/tests/mock.rs @@ -533,7 +533,7 @@ impl MockServer { // calc binary length let binary_type = accum[index]; if binary_type == ingress::DOUBLE_BINARY_FORMAT_TYPE { - index += size_of::(); + index += size_of::() + 1; } else if binary_type == ingress::ARRAY_BINARY_FORMAT_TYPE { index += 1; let element_type = match ElemDataType::try_from(accum[index]) { From 9c16d74fd85d017322dd0a6e5ba5fab4a6abde6d Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 9 Apr 2025 21:28:59 +0800 Subject: [PATCH 032/119] fix compile error. --- cpp_test/mock_server.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp_test/mock_server.hpp b/cpp_test/mock_server.hpp index dff8ff4e..ea6009a2 100644 --- a/cpp_test/mock_server.hpp +++ b/cpp_test/mock_server.hpp @@ -24,8 +24,8 @@ #pragma once +#include #include -#include #include #include #include From 5dae9d0e3ea951f05ab6c3bfc5cc41e1c7c88819 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 9 Apr 2025 21:48:23 +0800 Subject: [PATCH 033/119] make c++20 build happy --- cpp_test/mock_server.hpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/cpp_test/mock_server.hpp b/cpp_test/mock_server.hpp index ea6009a2..8adb6ad7 100644 --- a/cpp_test/mock_server.hpp +++ b/cpp_test/mock_server.hpp @@ -29,9 +29,12 @@ #include #include #include - #include "build_env.h" -#include "questdb/ingress/line_sender.hpp" +#if __cplusplus < 202002L +# include "questdb/ingress/line_sender.hpp" +#else +# include +#endif #if defined(PLATFORM_UNIX) typedef int socketfd_t; @@ -61,10 +64,14 @@ class mock_server size_t recv(double wait_timeout_sec = 0.1); - const buffer_view msgs(size_t index) const +#if __cplusplus >= 202002L + using buffer_view = std::span; +#endif + + buffer_view msgs(size_t index) const { assert(index < _msgs.size()); - return buffer_view{_msgs[index].data(), _msgs[index].size()}; + return {_msgs[index].data(), _msgs[index].size()}; } void close(); From ea522934d56717694af69950647b476fc9738b1b Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 9 Apr 2025 22:12:20 +0800 Subject: [PATCH 034/119] code format --- questdb-rs/src/ingress/ndarr.rs | 4 ++-- questdb-rs/src/tests/ndarr.rs | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 61626171..ff81d034 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -21,7 +21,7 @@ where /// Iterator element type is `&T`. fn iter(&self) -> Self::Iter<'_>; - /// Validates the data buffer size of array is consistency with array [`Shapes`]. + /// Validates the data buffer size of array is consistency with array shapes. /// /// # Returns /// - `Ok(usize)`: Expected buffer size in bytes if valid @@ -111,7 +111,7 @@ pub trait ArrayElement: Copy + 'static { } /// Defines binary format identifiers for array element types compatible with -/// QuestDB's [`ColumnType`]: https://github.com/questdb/questdb/blob/e1853db56ae586d923ca77de01a487cad44093b9/core/src/main/java/io/questdb/cairo/ColumnType.java#L67-L89. +/// QuestDB's ColumnType: . #[repr(u8)] #[derive(Debug, PartialEq, Clone, Copy)] pub enum ElemDataType { diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index 308c5131..99c92222 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -63,7 +63,7 @@ fn test_strided_array_view() -> TestResult { #[test] fn test_strided_non_contiguous() -> TestResult { let elem_size = size_of::() as i32; - let col_major_data = vec![1.0, 3.0, 5.0, 2.0, 4.0, 6.0]; + let col_major_data = [1.0, 3.0, 5.0, 2.0, 4.0, 6.0]; let shapes = [3u32, 2]; let strides = [elem_size, shapes[0] as i32 * elem_size]; @@ -85,7 +85,7 @@ fn test_strided_non_contiguous() -> TestResult { let mut buffer = Vec::new(); write_array_data(&array_view, &mut buffer)?; - let expected_data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; + let expected_data = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; let expected_bytes = unsafe { std::slice::from_raw_parts( expected_data.as_ptr() as *const u8, @@ -99,7 +99,7 @@ fn test_strided_non_contiguous() -> TestResult { #[test] fn test_negative_strides() -> TestResult { let elem_size = size_of::(); - let data = vec![1f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]; + let data = [1f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]; let view = unsafe { StridedArrayView::::new( 2, @@ -136,7 +136,7 @@ fn test_basic_edge_cases() { assert_eq!(empty_view.dim(1), Some(0)); // single element array - let single_data = vec![42.0]; + let single_data = [42.0]; let single_view: StridedArrayView<'_, f64> = unsafe { StridedArrayView::new( 2, @@ -155,7 +155,7 @@ fn test_basic_edge_cases() { fn test_buffer_basic_write() -> TestResult { let elem_size = std::mem::size_of::() as i32; - let test_data = vec![1.1, 2.2, 3.3, 4.4]; + let test_data = [1.1, 2.2, 3.3, 4.4]; let array_view: StridedArrayView<'_, f64> = unsafe { StridedArrayView::new( 2, @@ -221,8 +221,8 @@ fn test_size_overflow() -> TestResult { #[test] fn test_array_length_mismatch() -> TestResult { - let elem_size = std::mem::size_of::() as i32; - let under_data = vec![1.1]; + let elem_size = size_of::() as i32; + let under_data = [1.1]; let under_view: StridedArrayView<'_, f64> = unsafe { StridedArrayView::new( 2, @@ -242,7 +242,7 @@ fn test_array_length_mismatch() -> TestResult { .msg() .contains("Array buffer length mismatch (actual: 8, expected: 16)")); - let over_data = vec![1.1, 2.2, 3.3]; + let over_data = [1.1, 2.2, 3.3]; let over_view: StridedArrayView<'_, f64> = unsafe { StridedArrayView::new( 2, From 193b887af056a4aab2ea264eee562964a7dc61bf Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 10 Apr 2025 22:02:20 +0800 Subject: [PATCH 035/119] fix cpp problem on windows. --- cpp_test/mock_server.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cpp_test/mock_server.cpp b/cpp_test/mock_server.cpp index bfc67e0f..f2bd8bc5 100644 --- a/cpp_test/mock_server.cpp +++ b/cpp_test/mock_server.cpp @@ -208,8 +208,11 @@ size_t mock_server::recv(double wait_timeout_sec) for (;;) { wait_for_data(); - sock_ssize_t count = - ::recv(_conn_fd, &chunk[0], static_cast(chunk_len), 0); + sock_ssize_t count = ::recv( + _conn_fd, + reinterpret_cast(&chunk[0]), + static_cast(chunk_len), + 0); if (count == -1) throw std::runtime_error{"Bad `recv()`."}; const size_t u_count = static_cast(count); From 523a855e2f6928b3002150cb33e37c35657372c7 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Thu, 10 Apr 2025 17:36:27 +0100 Subject: [PATCH 036/119] lint issues --- questdb-rs/src/ingress/mod.rs | 12 ++++++------ questdb-rs/src/tests/ndarr.rs | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 2374a16d..fc376a6e 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -506,12 +506,12 @@ impl BufferState { /// * A row always starts with [`table`](Buffer::table). /// * A row must contain at least one [`symbol`](Buffer::symbol) or /// column ( -/// [`column_bool`](Buffer::column_bool), -/// [`column_i64`](Buffer::column_i64), -/// [`column_f64`](Buffer::column_f64), -/// [`column_str`](Buffer::column_str), -/// [`column_arr`](Buffer::column_arr), -/// [`column_ts`](Buffer::column_ts)). +/// [`column_bool`](Buffer::column_bool), +/// [`column_i64`](Buffer::column_i64), +/// [`column_f64`](Buffer::column_f64), +/// [`column_str`](Buffer::column_str), +/// [`column_arr`](Buffer::column_arr), +/// [`column_ts`](Buffer::column_ts)). /// * Symbols must appear before columns. /// * A row must be terminated with either [`at`](Buffer::at) or /// [`at_now`](Buffer::at_now). diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index 99c92222..56417d10 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -375,14 +375,14 @@ fn test_buffer_ndarray_write() -> TestResult { fn test_buffer_write_ndarray_max_dimensions() -> TestResult { let mut buffer = Buffer::new(); buffer.table("nd_test")?; - let shape: Vec = iter::repeat(1).take(MAX_DIMS).collect(); + let shape: Vec = iter::repeat_n(1, MAX_DIMS).collect(); let array = ArrayD::::zeros(shape.clone()); buffer.column_arr("max_dim", &array.view())?; let data = buffer.as_bytes(); assert_eq!(data[19], MAX_DIMS as u8); // 33 dims error - let shape_invalid: Vec<_> = iter::repeat(1).take(MAX_DIMS + 1).collect(); + let shape_invalid: Vec<_> = iter::repeat_n(1, MAX_DIMS + 1).collect(); let array_invalid = ArrayD::::zeros(shape_invalid); let result = buffer.column_arr("invalid", &array_invalid.view()); assert!(result.is_err()); From 3442cbea0db68106214efb1acfcca36c93224e1e Mon Sep 17 00:00:00 2001 From: victorgao Date: Mon, 14 Apr 2025 11:11:43 +0800 Subject: [PATCH 037/119] introduce line protocol version --- ci/compile.yaml | 4 + ci/run_all_tests.py | 5 +- ci/run_tests_pipeline.yaml | 13 +- cpp_test/test_line_sender.cpp | 60 ++-- examples/line_sender_cpp_example.cpp | 24 +- examples/line_sender_cpp_example_auth.cpp | 25 +- examples/line_sender_cpp_example_auth_tls.cpp | 26 +- .../line_sender_cpp_example_from_conf.cpp | 18 ++ examples/line_sender_cpp_example_from_env.cpp | 18 ++ examples/line_sender_cpp_example_http.cpp | 24 +- examples/line_sender_cpp_example_tls_ca.cpp | 28 +- include/questdb/ingress/line_sender.h | 55 ++++ include/questdb/ingress/line_sender.hpp | 93 +++++- questdb-rs-ffi/src/lib.rs | 73 +++++ questdb-rs/Cargo.toml | 7 +- questdb-rs/benches/ndarr.rs | 51 ++- questdb-rs/build.rs | 57 +++- questdb-rs/examples/line_protocol_version.rs | 32 ++ questdb-rs/src/error.rs | 5 +- questdb-rs/src/ingress/http.rs | 194 +++++++++++ questdb-rs/src/ingress/mod.rs | 221 +++++++++++-- questdb-rs/src/ingress/ndarr.rs | 3 +- questdb-rs/src/ingress/tests.rs | 2 + questdb-rs/src/tests/http.rs | 301 ++++++++++++------ .../interop/ilp-client-interop-test.json | 2 + questdb-rs/src/tests/mock.rs | 34 ++ questdb-rs/src/tests/mod.rs | 1 + questdb-rs/src/tests/sender.rs | 120 +++++-- system_test/fixture.py | 3 +- system_test/questdb_line_sender.py | 55 +++- system_test/test.py | 197 +++++++----- 31 files changed, 1467 insertions(+), 284 deletions(-) create mode 100644 questdb-rs/examples/line_protocol_version.rs diff --git a/ci/compile.yaml b/ci/compile.yaml index aa9e1059..4c892ce4 100644 --- a/ci/compile.yaml +++ b/ci/compile.yaml @@ -4,6 +4,10 @@ steps: rustup default $(toolchain) condition: ne(variables['toolchain'], '') displayName: "Update and set Rust toolchain" + - script: | + python -m pip install --upgrade pip + pip install numpy + displayName: 'Install Python Dependencies' - script: cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DQUESTDB_TESTS_AND_EXAMPLES=ON env: JAVA_HOME: $(JAVA_HOME_11_X64) diff --git a/ci/run_all_tests.py b/ci/run_all_tests.py index b4e4dc4a..ee261896 100644 --- a/ci/run_all_tests.py +++ b/ci/run_all_tests.py @@ -41,7 +41,7 @@ def main(): build_cxx20_dir.glob(f'**/test_line_sender{exe_suffix}'))) system_test_path = pathlib.Path('system_test') / 'test.py' - qdb_v = '8.2.3' # The version of QuestDB we'll test against. + #qdb_v = '8.2.3' # The version of QuestDB we'll test against. run_cmd('cargo', 'test', '--', '--nocapture', cwd='questdb-rs') @@ -53,7 +53,8 @@ def main(): '--', '--nocapture', cwd='questdb-rs') run_cmd(str(test_line_sender_path)) run_cmd(str(test_line_sender_path_CXX20)) - run_cmd('python3', str(system_test_path), 'run', '--versions', qdb_v, '-v') + #run_cmd('python3', str(system_test_path), 'run', '--versions', qdb_v, '-v') + run_cmd('python3', str(system_test_path), 'run', '--repo', './questdb_nd_arr', '-v') if __name__ == '__main__': diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml index 1fced6be..25434444 100644 --- a/ci/run_tests_pipeline.yaml +++ b/ci/run_tests_pipeline.yaml @@ -54,6 +54,17 @@ stages: cd questdb-rs cargo build --examples --features almost-all-features displayName: "Build Rust examples" + ############################# temp for test begin ##################### + - script: | + git clone -b nd_arr --depth 1 https://github.com/questdb/questdb.git ./questdb_nd_arr + displayName: git clone questdb + - task: Maven@3 + displayName: "Compile QuestDB" + inputs: + mavenPOMFile: 'questdb_nd_arr/pom.xml' + jdkVersionOption: '1.11' + options: "-DskipTests -Pbuild-web-console" + ############################# temp for test end ##################### - script: python3 ci/run_all_tests.py env: JAVA_HOME: $(JAVA_HOME_11_X64) @@ -115,7 +126,7 @@ stages: submodules: false - template: compile.yaml - script: | - git clone --depth 1 https://github.com/questdb/questdb.git + git clone -b nd_arr --depth 1 https://github.com/questdb/questdb.git displayName: git clone questdb - task: Maven@3 displayName: "Compile QuestDB" diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 7036d268..c1cadd22 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -93,7 +93,7 @@ bool operator==( #endif template -void push_double_arr_to_buffer( +std::string& push_double_arr_to_buffer( std::string& buffer, std::array data, size_t rank, @@ -108,6 +108,14 @@ void push_double_arr_to_buffer( buffer.append( reinterpret_cast(data.data()), data.size() * sizeof(double)); + return buffer; +} + +std::string& push_double_to_buffer(std::string& buffer, double data) +{ + buffer.push_back(16); + buffer.append(reinterpret_cast(&data), sizeof(double)); + return buffer; } TEST_CASE("line_sender c api basics") @@ -180,13 +188,14 @@ TEST_CASE("line_sender c api basics") &err)); CHECK(::line_sender_buffer_at_nanos(buffer, 10000000, &err)); CHECK(server.recv() == 0); - CHECK(::line_sender_buffer_size(buffer) == 143); + CHECK(::line_sender_buffer_size(buffer) == 150); CHECK(::line_sender_flush(sender, buffer, &err)); ::line_sender_buffer_free(buffer); CHECK(server.recv() == 1); - std::string expect{"test,t1=v1 f1=0.5,a1=="}; - push_double_arr_to_buffer(expect, arr_data, 3, shapes); - expect.append(" 10000000\n"); + std::string expect{"test,t1=v1 f1=="}; + push_double_to_buffer(expect, 0.5).append(",a1=="); + push_double_arr_to_buffer(expect, arr_data, 3, shapes) + .append(" 10000000\n"); CHECK(server.msgs(0) == expect); } @@ -243,10 +252,12 @@ TEST_CASE("line_sender c++ api basics") .at(questdb::ingress::timestamp_nanos{10000000}); CHECK(server.recv() == 0); - CHECK(buffer.size() == 31); + CHECK(buffer.size() == 38); sender.flush(buffer); CHECK(server.recv() == 1); - CHECK(server.msgs(0) == "test,t1=v1,t2= f1=0.5 10000000\n"); + std::string expect{"test,t1=v1,t2= f1=="}; + push_double_to_buffer(expect, 0.5).append(" 10000000\n"); + CHECK(server.msgs(0) == expect); } TEST_CASE("test multiple lines") @@ -279,13 +290,13 @@ TEST_CASE("test multiple lines") .at_now(); CHECK(server.recv() == 0); - CHECK(buffer.size() == 137); + CHECK(buffer.size() == 142); sender.flush(buffer); CHECK(server.recv() == 2); - CHECK( - server.msgs(0) == - ("metric1,t1=val1,t2=val2 f1=t,f2=12345i," - "f3=10.75,f4=\"val3\",f5=\"val4\",f6=\"val5\" 111222233333\n")); + std::string expect{"metric1,t1=val1,t2=val2 f1=t,f2=12345i,f3=="}; + push_double_to_buffer(expect, 10.75) + .append(",f4=\"val3\",f5=\"val4\",f6=\"val5\" 111222233333\n"); + CHECK(server.msgs(0) == expect); CHECK( server.msgs(1) == "metric1,tag3=value\\ 3,tag\\ 4=value:4 field5=f\n"); } @@ -683,13 +694,19 @@ TEST_CASE("os certs") { questdb::ingress::opts opts{ - questdb::ingress::protocol::https, "localhost", server.port()}; + questdb::ingress::protocol::https, + "localhost", + server.port(), + true}; opts.tls_ca(questdb::ingress::ca::os_roots); } { questdb::ingress::opts opts{ - questdb::ingress::protocol::https, "localhost", server.port()}; + questdb::ingress::protocol::https, + "localhost", + server.port(), + true}; opts.tls_ca(questdb::ingress::ca::webpki_and_os_roots); } } @@ -718,9 +735,12 @@ TEST_CASE("Opts copy ctor, assignment and move testing.") { questdb::ingress::opts opts1{ - questdb::ingress::protocol::https, "localhost", "9009"}; + questdb::ingress::protocol::https, "localhost", "9009", true}; questdb::ingress::opts opts2{ - questdb::ingress::protocol::https, "altavista.digital.com", "9009"}; + questdb::ingress::protocol::https, + "altavista.digital.com", + "9009", + true}; opts1 = opts2; } } @@ -890,15 +910,15 @@ TEST_CASE("Opts from conf") TEST_CASE("HTTP basics") { questdb::ingress::opts opts1{ - questdb::ingress::protocol::http, "localhost", 1}; + questdb::ingress::protocol::http, "localhost", 1, true}; questdb::ingress::opts opts1conf = questdb::ingress::opts::from_conf( "http::addr=localhost:1;username=user;password=pass;request_timeout=" - "5000;retry_timeout=5;"); + "5000;retry_timeout=5;disable_line_protocol_validation=on;"); questdb::ingress::opts opts2{ - questdb::ingress::protocol::https, "localhost", "1"}; + questdb::ingress::protocol::https, "localhost", "1", true}; questdb::ingress::opts opts2conf = questdb::ingress::opts::from_conf( "http::addr=localhost:1;token=token;request_min_throughput=1000;retry_" - "timeout=0;"); + "timeout=0;disable_line_protocol_validation=on;"); opts1.username("user") .password("pass") .max_buf_size(1000000) diff --git a/examples/line_sender_cpp_example.cpp b/examples/line_sender_cpp_example.cpp index 2d90812f..5e3b12d1 100644 --- a/examples/line_sender_cpp_example.cpp +++ b/examples/line_sender_cpp_example.cpp @@ -19,6 +19,23 @@ static bool example(std::string_view host, std::string_view port) const auto side_name = "side"_cn; const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; + const auto order_book_name = "order_book"_cn; + size_t rank = 3; + std::vector shape{2, 3, 2}; + std::vector strides{48, 16, 8}; + std::array arr_data = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; questdb::ingress::line_sender_buffer buffer; buffer @@ -27,6 +44,7 @@ static bool example(std::string_view host, std::string_view port) .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) .column(amount_name, 0.00044) + .column(order_book_name, 3, shape, strides, arr_data) .at(questdb::ingress::timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. @@ -59,9 +77,9 @@ static bool displayed_help(int argc, const char* argv[]) std::cerr << "Usage:\n" << "line_sender_c_example: [HOST [PORT]]\n" - << " HOST: ILP host (defaults to \"localhost\").\n" - << " PORT: ILP port (defaults to \"9009\")." - << std::endl; + << " HOST: ILP host (defaults to \"localhost\").\n" + << " PORT: ILP port (defaults to \"9009\")." + << std::endl; return true; } } diff --git a/examples/line_sender_cpp_example_auth.cpp b/examples/line_sender_cpp_example_auth.cpp index 85e0d6e1..b4cf2b70 100644 --- a/examples/line_sender_cpp_example_auth.cpp +++ b/examples/line_sender_cpp_example_auth.cpp @@ -23,6 +23,24 @@ static bool example(std::string_view host, std::string_view port) const auto side_name = "side"_cn; const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; + const auto order_book_name = "order_book"_cn; + + size_t rank = 3; + std::vector shape{2, 3, 2}; + std::vector strides{48, 16, 8}; + std::array arr_data = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; questdb::ingress::line_sender_buffer buffer; buffer @@ -31,6 +49,7 @@ static bool example(std::string_view host, std::string_view port) .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) .column(amount_name, 0.00044) + .column(order_book_name, 3, shape, strides, arr_data) .at(questdb::ingress::timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. @@ -63,9 +82,9 @@ static bool displayed_help(int argc, const char* argv[]) std::cerr << "Usage:\n" << "line_sender_c_example: [HOST [PORT]]\n" - << " HOST: ILP host (defaults to \"localhost\").\n" - << " PORT: ILP port (defaults to \"9009\")." - << std::endl; + << " HOST: ILP host (defaults to \"localhost\").\n" + << " PORT: ILP port (defaults to \"9009\")." + << std::endl; return true; } } diff --git a/examples/line_sender_cpp_example_auth_tls.cpp b/examples/line_sender_cpp_example_auth_tls.cpp index f100dd04..90e2a46b 100644 --- a/examples/line_sender_cpp_example_auth_tls.cpp +++ b/examples/line_sender_cpp_example_auth_tls.cpp @@ -25,6 +25,23 @@ static bool example( const auto side_name = "side"_cn; const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; + const auto order_book_name = "order_book"_cn; + size_t rank = 3; + std::vector shape{2, 3, 2}; + std::vector strides{48, 16, 8}; + std::array arr_data = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; questdb::ingress::line_sender_buffer buffer; buffer @@ -33,6 +50,7 @@ static bool example( .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) .column(amount_name, 0.00044) + .column(order_book_name, 3, shape, strides, arr_data) .at(questdb::ingress::timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. @@ -64,10 +82,10 @@ static bool displayed_help(int argc, const char* argv[]) { std::cerr << "Usage:\n" - << "line_sender_c_example: CA_PATH [HOST [PORT]]\n" - << " HOST: ILP host (defaults to \"localhost\").\n" - << " PORT: ILP port (defaults to \"9009\")." - << std::endl; + << "line_sender_c_example: CA_PATH [HOST [PORT]]\n" + << " HOST: ILP host (defaults to \"localhost\").\n" + << " PORT: ILP port (defaults to \"9009\")." + << std::endl; return true; } } diff --git a/examples/line_sender_cpp_example_from_conf.cpp b/examples/line_sender_cpp_example_from_conf.cpp index bb71c6e3..d0d1e470 100644 --- a/examples/line_sender_cpp_example_from_conf.cpp +++ b/examples/line_sender_cpp_example_from_conf.cpp @@ -19,6 +19,23 @@ int main(int argc, const char* argv[]) const auto side_name = "side"_cn; const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; + const auto order_book_name = "order_book"_cn; + size_t rank = 3; + std::vector shape{2, 3, 2}; + std::vector strides{48, 16, 8}; + std::array arr_data = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; questdb::ingress::line_sender_buffer buffer; buffer @@ -27,6 +44,7 @@ int main(int argc, const char* argv[]) .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) .column(amount_name, 0.00044) + .column(order_book_name, 3, shape, strides, arr_data) .at(questdb::ingress::timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. diff --git a/examples/line_sender_cpp_example_from_env.cpp b/examples/line_sender_cpp_example_from_env.cpp index 63e99b26..069e4e22 100644 --- a/examples/line_sender_cpp_example_from_env.cpp +++ b/examples/line_sender_cpp_example_from_env.cpp @@ -18,6 +18,23 @@ int main(int argc, const char* argv[]) const auto side_name = "side"_cn; const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; + const auto order_book_name = "order_book"_cn; + size_t rank = 3; + std::vector shape{2, 3, 2}; + std::vector strides{48, 16, 8}; + std::array arr_data = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; questdb::ingress::line_sender_buffer buffer; buffer @@ -26,6 +43,7 @@ int main(int argc, const char* argv[]) .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) .column(amount_name, 0.00044) + .column(order_book_name, 3, shape, strides, arr_data) .at(questdb::ingress::timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. diff --git a/examples/line_sender_cpp_example_http.cpp b/examples/line_sender_cpp_example_http.cpp index 217845a2..43b4bdd5 100644 --- a/examples/line_sender_cpp_example_http.cpp +++ b/examples/line_sender_cpp_example_http.cpp @@ -19,6 +19,23 @@ static bool example(std::string_view host, std::string_view port) const auto side_name = "side"_cn; const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; + const auto order_book_name = "order_book"_cn; + size_t rank = 3; + std::vector shape{2, 3, 2}; + std::vector strides{48, 16, 8}; + std::array arr_data = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; questdb::ingress::line_sender_buffer buffer; buffer @@ -27,6 +44,7 @@ static bool example(std::string_view host, std::string_view port) .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) .column(amount_name, 0.00044) + .column(order_book_name, 3, shape, strides, arr_data) .at(questdb::ingress::timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. @@ -59,9 +77,9 @@ static bool displayed_help(int argc, const char* argv[]) std::cerr << "Usage:\n" << "line_sender_c_example: [HOST [PORT]]\n" - << " HOST: ILP host (defaults to \"localhost\").\n" - << " PORT: ILP port (defaults to \"9009\")." - << std::endl; + << " HOST: ILP host (defaults to \"localhost\").\n" + << " PORT: ILP port (defaults to \"9009\")." + << std::endl; return true; } } diff --git a/examples/line_sender_cpp_example_tls_ca.cpp b/examples/line_sender_cpp_example_tls_ca.cpp index c0327e96..27a91f81 100644 --- a/examples/line_sender_cpp_example_tls_ca.cpp +++ b/examples/line_sender_cpp_example_tls_ca.cpp @@ -27,6 +27,23 @@ static bool example( const auto side_name = "side"_cn; const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; + const auto order_book_name = "order_book"_cn; + size_t rank = 3; + std::vector shape{2, 3, 2}; + std::vector strides{48, 16, 8}; + std::array arr_data = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; questdb::ingress::line_sender_buffer buffer; buffer @@ -35,6 +52,7 @@ static bool example( .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) .column(amount_name, 0.00044) + .column(order_book_name, 3, shape, strides, arr_data) .at(questdb::ingress::timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. @@ -66,11 +84,11 @@ static bool displayed_help(int argc, const char* argv[]) { std::cerr << "Usage:\n" - << "line_sender_c_example: CA_PATH [HOST [PORT]]\n" - << " CA_PATH: Certificate authority pem file.\n" - << " HOST: ILP host (defaults to \"localhost\").\n" - << " PORT: ILP port (defaults to \"9009\")." - << std::endl; + << "line_sender_c_example: CA_PATH [HOST [PORT]]\n" + << " CA_PATH: Certificate authority pem file.\n" + << " HOST: ILP host (defaults to \"localhost\").\n" + << " PORT: ILP port (defaults to \"9009\")." + << std::endl; return true; } } diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 122897d5..96c2b607 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -95,6 +95,18 @@ typedef enum line_sender_protocol line_sender_protocol_https, } line_sender_protocol; +/** The line protocol version used to write data to buffer. */ +typedef enum line_protocol_version +{ + /** Version 1 of InfluxDB Line Protocol. + Uses text format serialization for f64. */ + line_protocol_version_1, + + /** Version 2 of InfluxDB Line Protocol. + Uses binary format serialization for f64, and support array data type.*/ + line_protocol_version_2, +} line_protocol_version; + /** Possible sources of the root certificates used to validate the server's * TLS certificate. */ typedef enum line_sender_ca @@ -296,6 +308,23 @@ line_sender_buffer* line_sender_buffer_new(); LINESENDER_API line_sender_buffer* line_sender_buffer_with_max_name_len(size_t max_name_len); +/** + * Sets the Line Protocol version for line_sender_buffer. + * + * The buffer defaults is line_protocol_version_2 which uses + * binary format f64 serialization and support array data type. Call this to + * switch to version 1 (text format f64) when connecting to servers that don't + * support line_protocol_version_2(under 8.3.2). + * + * Must be called before adding any data to the buffer. Protocol version cannot + * be changed after the buffer contains data. + */ +LINESENDER_API +line_sender_buffer* line_sender_buffer_set_line_protocol_version( + line_sender_buffer* buffer, + line_protocol_version version, + line_sender_error** err_out); + /** Release the `line_sender_buffer` object. */ LINESENDER_API void line_sender_buffer_free(line_sender_buffer* buffer); @@ -718,6 +747,13 @@ bool line_sender_opts_token_y( line_sender_utf8 token_y, line_sender_error** err_out); +/** + * Set the ECDSA public key Y for TCP authentication. + */ +LINESENDER_API +bool line_sender_opts_disable_line_protocol_validation( + line_sender_opts* opts, line_sender_error** err_out); + /** * Configure how long to wait for messages from the QuestDB server during * the TLS handshake and authentication process. @@ -877,6 +913,25 @@ line_sender* line_sender_from_conf( LINESENDER_API line_sender* line_sender_from_env(line_sender_error** err_out); +/** + * Returns the client's recommended default line protocol version. + * Will be used to [`line_sender_buffer_set_line_protocol_version`] + * + * The version selection follows these rules: + * 1. TCP/TCPS Protocol: Always returns [`LineProtocolVersion::V2`] + * 2. HTTP/HTTPS Protocol: + * - If line protocol auto-detection is disabled + * [`line_sender_opts_disable_line_protocol_validation`], returns + * [`LineProtocolVersion::V2`] + * - If line protocol auto-detection is enabled: + * - Uses the server's default version if supported by the client + * - Otherwise uses the highest mutually supported version from the + * intersection of client and server compatible versions. + */ +LINESENDER_API +line_protocol_version line_sender_default_line_protocol_version( + line_sender* sender); + /** * Tell whether the sender is no longer usable and must be closed. * This happens when there was an earlier failure. diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 1e9adf82..34f14cd8 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -401,10 +401,22 @@ class line_sender_buffer { } + line_sender_buffer( + size_t init_buf_size, + size_t max_name_len, + line_protocol_version version) noexcept + : _impl{nullptr} + , _init_buf_size{init_buf_size} + , _max_name_len{max_name_len} + , _line_protocol_version{version} + { + } + line_sender_buffer(const line_sender_buffer& other) noexcept : _impl{::line_sender_buffer_clone(other._impl)} , _init_buf_size{other._init_buf_size} , _max_name_len{other._max_name_len} + , _line_protocol_version{other._line_protocol_version} { } @@ -412,6 +424,7 @@ class line_sender_buffer : _impl{other._impl} , _init_buf_size{other._init_buf_size} , _max_name_len{other._max_name_len} + , _line_protocol_version{other._line_protocol_version} { other._impl = nullptr; } @@ -427,6 +440,7 @@ class line_sender_buffer _impl = nullptr; _init_buf_size = other._init_buf_size; _max_name_len = other._max_name_len; + _line_protocol_version = other._line_protocol_version; } return *this; } @@ -439,11 +453,32 @@ class line_sender_buffer _impl = other._impl; _init_buf_size = other._init_buf_size; _max_name_len = other._max_name_len; + _line_protocol_version = other._line_protocol_version; other._impl = nullptr; } return *this; } + /** + * Sets the Line Protocol version for line_sender_buffer. + * + * The buffer defaults is line_protocol_version_2 which uses + * binary format f64 serialization and support array data type. Call this to + * switch to version 1 (text format f64) when connecting to servers that + * don't support line_protocol_version_2(under 8.3.2). + * + * Must be called before adding any data to the buffer. Protocol version + * cannot be changed after the buffer contains data. + */ + line_sender_buffer& set_line_protocol_version(line_protocol_version v) + { + may_init(); + line_sender_error::wrapped_call( + ::line_sender_buffer_set_line_protocol_version, _impl, v); + _line_protocol_version = v; + return *this; + } + /** * Pre-allocate to ensure the buffer has enough capacity for at least * the specified additional byte count. This may be rounded up. @@ -803,12 +838,17 @@ class line_sender_buffer { _impl = ::line_sender_buffer_with_max_name_len(_max_name_len); ::line_sender_buffer_reserve(_impl, _init_buf_size); + line_sender_error::wrapped_call( + line_sender_buffer_set_line_protocol_version, + _impl, + _line_protocol_version); } } ::line_sender_buffer* _impl; size_t _init_buf_size; size_t _max_name_len; + line_protocol_version _line_protocol_version{::line_protocol_version_2}; friend class line_sender; }; @@ -868,13 +908,24 @@ class opts * @param[in] protocol The protocol to use. * @param[in] host The QuestDB database host. * @param[in] port The QuestDB tcp or http port. + * @param[in] disable_line_protocol_validation disable line protocol version + * validation. */ - opts(protocol protocol, utf8_view host, uint16_t port) noexcept + opts( + protocol protocol, + utf8_view host, + uint16_t port, + bool disable_line_protocol_validation = false) noexcept : _impl{::line_sender_opts_new( static_cast<::line_sender_protocol>(protocol), host._impl, port)} { line_sender_error::wrapped_call( ::line_sender_opts_user_agent, _impl, _user_agent::name()); + if (disable_line_protocol_validation) + { + line_sender_error::wrapped_call( + ::line_sender_opts_disable_line_protocol_validation, _impl); + } } /** @@ -883,8 +934,13 @@ class opts * @param[in] protocol The protocol to use. * @param[in] host The QuestDB database host. * @param[in] port The QuestDB tcp or http port as service name. + * @param[in] disable_line_protocol_validation disable line protocol version */ - opts(protocol protocol, utf8_view host, utf8_view port) noexcept + opts( + protocol protocol, + utf8_view host, + utf8_view port, + bool disable_line_protocol_validation = false) noexcept : _impl{::line_sender_opts_new_service( static_cast<::line_sender_protocol>(protocol), host._impl, @@ -892,6 +948,11 @@ class opts { line_sender_error::wrapped_call( ::line_sender_opts_user_agent, _impl, _user_agent::name()); + if (disable_line_protocol_validation) + { + line_sender_error::wrapped_call( + ::line_sender_opts_disable_line_protocol_validation, _impl); + } } opts(const opts& other) noexcept @@ -998,6 +1059,16 @@ class opts return *this; } + /** + * Disable the validation of the line protocol version. + */ + opts& disable_line_protocol_validation() + { + line_sender_error::wrapped_call( + ::line_sender_opts_disable_line_protocol_validation, _impl); + return *this; + } + /** * Configure how long to wait for messages from the QuestDB server during * the TLS handshake and authentication process. @@ -1184,13 +1255,23 @@ class line_sender return {opts::from_env()}; } - line_sender(protocol protocol, utf8_view host, uint16_t port) - : line_sender{opts{protocol, host, port}} + line_sender( + protocol protocol, + utf8_view host, + uint16_t port, + bool disable_line_protocol_validation = false) + : line_sender{ + opts{protocol, host, port, disable_line_protocol_validation}} { } - line_sender(protocol protocol, utf8_view host, utf8_view port) - : line_sender{opts{protocol, host, port}} + line_sender( + protocol protocol, + utf8_view host, + utf8_view port, + bool disable_line_protocol_validation = false) + : line_sender{ + opts{protocol, host, port, disable_line_protocol_validation}} { } diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index b877e31c..d2b47d1d 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -145,6 +145,9 @@ pub enum line_sender_error_code { /// Write arrayView to sender buffer error. line_sender_error_array_view_write_to_buffer_error, + + /// Line sender protocol version error. + line_sender_error_line_protocol_version_error, } impl From for line_sender_error_code { @@ -178,6 +181,9 @@ impl From for line_sender_error_code { ErrorCode::ArrayWriteToBufferError => { line_sender_error_code::line_sender_error_array_view_write_to_buffer_error } + ErrorCode::LineProtocolVersionError => { + line_sender_error_code::line_sender_error_line_protocol_version_error + } } } } @@ -221,6 +227,37 @@ impl From for Protocol { } } +/// The version of Line Protocol used for [`Buffer`]. +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub enum LineProtocolVersion { + /// Version 1 of Line Protocol. + /// Uses text format serialization for f64. + V1, + + /// Version 2 of InfluxDB Line Protocol. + /// Uses binary format serialization for f64, and support array data type. + V2, +} + +impl From for ingress::LineProtocolVersion { + fn from(version: LineProtocolVersion) -> Self { + match version { + LineProtocolVersion::V1 => ingress::LineProtocolVersion::V1, + LineProtocolVersion::V2 => ingress::LineProtocolVersion::V2, + } + } +} + +impl From for LineProtocolVersion { + fn from(version: ingress::LineProtocolVersion) -> Self { + match version { + ingress::LineProtocolVersion::V1 => LineProtocolVersion::V1, + ingress::LineProtocolVersion::V2 => LineProtocolVersion::V2, + } + } +} + /// Possible sources of the root certificates used to validate the server's TLS certificate. #[repr(C)] #[derive(Debug, Copy, Clone)] @@ -580,6 +617,17 @@ pub unsafe extern "C" fn line_sender_buffer_with_max_name_len( Box::into_raw(Box::new(line_sender_buffer(buffer))) } +#[no_mangle] +pub unsafe extern "C" fn line_sender_buffer_set_line_protocol_version( + buffer: *mut line_sender_buffer, + version: LineProtocolVersion, + err_out: *mut *mut line_sender_error, +) -> bool { + let buffer = unwrap_buffer_mut(buffer); + bubble_err_to_c!(err_out, buffer.set_line_proto_version(version.into())); + true +} + /// Release the `line_sender_buffer` object. #[no_mangle] pub unsafe extern "C" fn line_sender_buffer_free(buffer: *mut line_sender_buffer) { @@ -1120,6 +1168,15 @@ pub unsafe extern "C" fn line_sender_opts_token_y( upd_opts!(opts, err_out, token_y, token_y.as_str()) } +/// Disable the line protocol validation. +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_disable_line_protocol_validation( + opts: *mut line_sender_opts, + err_out: *mut *mut line_sender_error, +) -> bool { + upd_opts!(opts, err_out, disable_line_protocol_validation) +} + /// Configure how long to wait for messages from the QuestDB server during /// the TLS handshake and authentication process. /// The value is in milliseconds, and the default is 15 seconds. @@ -1349,6 +1406,22 @@ unsafe fn unwrap_sender_mut<'a>(sender: *mut line_sender) -> &'a mut Sender { &mut (*sender).0 } +/// Returns the client's recommended default line protocol version. +/// Will be used to [`line_sender_buffer_set_line_protocol_version`] +/// +/// The version selection follows these rules: +/// 1. **TCP/TCPS Protocol**: Always returns [`LineProtocolVersion::V2`] +/// 2. **HTTP/HTTPS Protocol**: +/// - If line protocol auto-detection is disabled [`line_sender_opts_disable_line_protocol_validation`], returns [`LineProtocolVersion::V2`] +/// - If line protocol auto-detection is enabled: +/// - Uses the server's default version if supported by the client +/// - Otherwise uses the highest mutually supported version from the intersection +/// of client and server compatible versions +#[no_mangle] +pub unsafe extern "C" fn line_sender_default_line_protocol_version(sender: *mut line_sender) -> LineProtocolVersion { + unwrap_sender(sender).default_line_protocol_version().into() +} + /// Tell whether the sender is no longer usable and must be closed. /// This happens when there was an earlier failure. /// This fuction is specific to TCP and is not relevant for HTTP. diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index ba2a5680..15b3b1f3 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -23,7 +23,7 @@ socket2 = "0.5.5" dns-lookup = "2.0.4" base64ct = { version = "1.7", features = ["alloc"] } rustls-pemfile = "2.0.0" -ryu = "1.0" +ryu = { version = "1.0" } itoa = "1.0" aws-lc-rs = { version = "1.13", optional = true } ring = { version = "0.17.14", optional = true } @@ -57,6 +57,7 @@ chrono = "0.4.31" tempfile = "3" webpki-roots = "0.26.8" criterion = "0.5" +rstest = "0.25.0" [features] default = ["tls-webpki-certs", "ilp-over-http", "aws-lc-crypto"] @@ -121,3 +122,7 @@ required-features = ["chrono_timestamp"] [[example]] name = "http" required-features = ["ilp-over-http", "ndarray"] + +[[example]] +name = "line_protocol_version" +required-features = ["ilp-over-http", "ndarray"] diff --git a/questdb-rs/benches/ndarr.rs b/questdb-rs/benches/ndarr.rs index babac254..a5b8b704 100644 --- a/questdb-rs/benches/ndarr.rs +++ b/questdb-rs/benches/ndarr.rs @@ -1,6 +1,6 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; use ndarray::{Array, Array2}; -use questdb::ingress::{Buffer, ColumnName}; +use questdb::ingress::{Buffer, ColumnName, StridedArrayView}; /// run with /// ```shell @@ -65,5 +65,52 @@ fn bench_write_array_data(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, bench_write_array_data); +// bench NdArrayView and StridedArrayView write performance. +fn bench_array_view(c: &mut Criterion) { + let mut group = c.benchmark_group("write_array_view"); + let col_name = ColumnName::new("col1").unwrap(); + let array: Array2 = Array::ones((1000, 1000)); + let transposed_view = array.t(); + + // Case 1 + group.bench_function("ndarray_view", |b| { + let mut buffer = Buffer::new(); + buffer.table("x1").unwrap(); + b.iter(|| { + buffer + .column_arr(col_name, black_box(&transposed_view)) + .unwrap(); + }); + buffer.clear(); + }); + + let shape: Vec = transposed_view.shape().iter().map(|&d| d as u32).collect(); + let elem_size = size_of::() as i32; + let strides: Vec = transposed_view + .strides() + .iter() + .map(|&s| s as i32 * elem_size) // 转换为字节步长 + .collect(); + let view2: StridedArrayView<'_, f64> = unsafe { + StridedArrayView::new( + transposed_view.ndim(), + shape.as_ptr(), + strides.as_ptr(), + transposed_view.as_ptr() as *const u8, + transposed_view.len() * elem_size as usize, + ) + }; + + // Case 2 + group.bench_function("strides_view", |b| { + let mut buffer = Buffer::new(); + buffer.table("x1").unwrap(); + b.iter(|| { + buffer.column_arr(col_name, black_box(&view2)).unwrap(); + }); + buffer.clear(); + }); +} + +criterion_group!(benches, bench_write_array_data, bench_array_view); criterion_main!(benches); diff --git a/questdb-rs/build.rs b/questdb-rs/build.rs index 636f55e8..99d0cf10 100644 --- a/questdb-rs/build.rs +++ b/questdb-rs/build.rs @@ -50,6 +50,8 @@ pub mod json_tests { #[derive(Debug, Serialize, Deserialize)] struct Expected { line: Option, + #[serde(rename = "binaryBase64")] + binary_base64: Option, #[serde(rename = "anyLines")] any_lines: Option>, @@ -95,8 +97,11 @@ pub mod json_tests { indoc! {r#" // This file is auto-generated by build.rs. - use crate::{Result, ingress::{Buffer}}; + use crate::{Result, ingress::{Buffer, LineProtocolVersion}}; use crate::tests::{TestResult}; + use base64ct::Base64; + use base64ct::Encoding; + use rstest::rstest; fn matches_any_line(line: &[u8], expected: &[&str]) -> bool { for &exp in expected { @@ -117,14 +122,17 @@ pub mod json_tests { // for line in serde_json::to_string_pretty(&spec).unwrap().split("\n") { // writeln!(output, "/// {}", line)?; // } - writeln!(output, "#[test]")?; + writeln!(output, "#[rstest]")?; writeln!( output, - "fn test_{:03}_{}() -> TestResult {{", + "fn test_{:03}_{}(\n #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion,\n) -> TestResult {{", index, slugify!(&spec.test_name, separator = "_") )?; - writeln!(output, " let mut buffer = Buffer::new();")?; + writeln!( + output, + " let mut buffer = Buffer::new().with_line_proto_version(version)?;" + )?; let (expected, indent) = match &spec.result { Outcome::Success(line) => (Some(line), ""), @@ -168,7 +176,46 @@ pub mod json_tests { } writeln!(output, "{} .at_now()?;", indent)?; if let Some(expected) = expected { - if let Some(ref line) = expected.line { + if let Some(ref base64) = expected.binary_base64 { + writeln!(output, " if version != LineProtocolVersion::V1 {{")?; + writeln!( + output, + " let exp = Base64::decode_vec(\"{}\").unwrap();", + base64 + )?; + writeln!( + output, + " assert_eq!(buffer.as_bytes(), exp.as_slice());" + )?; + writeln!(output, " }} else {{")?; + if let Some(ref line) = expected.line { + let exp_ln = format!("{}\n", line); + writeln!(output, " let exp = {:?};", exp_ln)?; + writeln!( + output, + " assert_eq!(buffer.as_bytes(), exp.as_bytes());" + )?; + } else { + // 处理 V1 版本的 any_lines + let any: Vec = expected + .any_lines + .as_ref() + .unwrap() + .iter() + .map(|line| format!("{}\n", line)) + .collect(); + writeln!(output, " let any = [")?; + for line in any.iter() { + writeln!(output, " {:?},", line)?; + } + writeln!(output, " ];")?; + writeln!( + output, + " assert!(matches_any_line(buffer.as_bytes(), &any));" + )?; + } + writeln!(output, " }}")?; + } else if let Some(ref line) = expected.line { let exp_ln = format!("{}\n", line); writeln!(output, " let exp = {:?};", exp_ln)?; writeln!(output, " assert_eq!(buffer.as_bytes(), exp.as_bytes());")?; diff --git a/questdb-rs/examples/line_protocol_version.rs b/questdb-rs/examples/line_protocol_version.rs new file mode 100644 index 00000000..d9944991 --- /dev/null +++ b/questdb-rs/examples/line_protocol_version.rs @@ -0,0 +1,32 @@ +use ndarray::arr1; +use questdb::ingress::LineProtocolVersion; +use questdb::{ + ingress::{Buffer, Sender, TimestampNanos}, + Result, +}; + +fn main() -> Result<()> { + let mut sender = Sender::from_conf("https::addr=localhost:9000;username=foo;password=bar;")?; + let mut buffer = Buffer::new().with_line_proto_version(LineProtocolVersion::V1)?; + buffer + .table("trades_ilp_v1")? + .symbol("symbol", "ETH-USD")? + .symbol("side", "sell")? + .column_f64("price", 2615.54)? + .column_f64("amount", 0.00044)? + .at(TimestampNanos::now())?; + sender.flush(&mut buffer)?; + + let mut sender2 = Sender::from_conf("https::addr=localhost:9000;username=foo;password=bar;")?; + let mut buffer2 = Buffer::new().with_line_proto_version(LineProtocolVersion::V1)?; + buffer2 + .table("trades_ilp_v2")? + .symbol("symbol", "ETH-USD")? + .symbol("side", "sell")? + .column_f64("price", 2615.54)? + .column_f64("amount", 0.00044)? + .column_arr("location", &arr1(&[100.0, 100.1, 100.2]).view())? + .at(TimestampNanos::now())?; + sender2.flush(&mut buffer2)?; + Ok(()) +} diff --git a/questdb-rs/src/error.rs b/questdb-rs/src/error.rs index 58f31e74..cadb0971 100644 --- a/questdb-rs/src/error.rs +++ b/questdb-rs/src/error.rs @@ -55,8 +55,11 @@ pub enum ErrorCode { /// Array view internal error. ArrayViewError, - /// Array write to buffer error + /// Array write to buffer error. ArrayWriteToBufferError, + + /// Validate line protocol version error. + LineProtocolVersionError, } /// An error that occurred when using QuestDB client library. diff --git a/questdb-rs/src/ingress/http.rs b/questdb-rs/src/ingress/http.rs index 323a4a92..26e519b7 100644 --- a/questdb-rs/src/ingress/http.rs +++ b/questdb-rs/src/ingress/http.rs @@ -1,4 +1,5 @@ use super::conf::ConfigSetting; +use crate::error::fmt; use crate::{error, Error}; use base64ct::Base64; use base64ct::Encoding; @@ -16,6 +17,7 @@ use ureq::unversioned::transport::{ Buffers, Connector, LazyBuffers, NextTimeout, Transport, TransportAdapter, }; +use crate::ingress::LineProtocolVersion; use ureq::unversioned::*; use ureq::Error::*; use ureq::{http, Body}; @@ -57,6 +59,7 @@ pub(super) struct HttpConfig { pub(super) user_agent: String, pub(super) retry_timeout: ConfigSetting, pub(super) request_timeout: ConfigSetting, + pub(super) disable_line_proto_validation: ConfigSetting, } impl Default for HttpConfig { @@ -66,6 +69,7 @@ impl Default for HttpConfig { user_agent: concat!("questdb/rust/", env!("CARGO_PKG_VERSION")).to_string(), retry_timeout: ConfigSetting::new_default(Duration::from_secs(10)), request_timeout: ConfigSetting::new_default(Duration::from_secs(10)), + disable_line_proto_validation: ConfigSetting::new_default(false), } } } @@ -109,6 +113,24 @@ impl HttpHandlerState { Err(err) => (need_retry(Err(err)), response), } } + + pub(crate) fn get_request( + &self, + url: &str, + request_timeout: Duration, + ) -> (bool, Result, ureq::Error>) { + let request = self + .agent + .get(url) + .config() + .timeout_per_call(Some(request_timeout)) + .build(); + let response = request.call(); + match &response { + Ok(res) => (need_retry(Ok(res.status())), response), + Err(err) => (need_retry(Err(err)), response), + } + } } #[derive(Debug)] @@ -383,3 +405,175 @@ pub(super) fn http_send_with_retries( retry_http_send(state, buf, request_timeout, retry_timeout, last_rep) } + +pub(super) fn get_line_protocol_version( + state: &HttpHandlerState, + settings_url: &str, +) -> Result<(Option>, LineProtocolVersion), Error> { + let mut support_versions: Option> = None; + let mut default_version = LineProtocolVersion::V1; + + let response = match http_get_with_retries( + state, + settings_url, + *state.config.request_timeout, + Duration::from_secs(1), + ) { + Ok(res) => { + if res.status().is_client_error() || res.status().is_server_error() { + if res.status().as_u16() == 404 { + return Ok((support_versions, default_version)); + } + return Err(fmt!( + LineProtocolVersionError, + "Failed to detect server's line protocol version, settings url: {}, status code: {}.", + settings_url, + res.status() + )); + } else { + res + } + } + Err(err) => { + let e = match err { + ureq::Error::StatusCode(code) => { + if code == 404 { + return Ok((support_versions, default_version)); + } else { + fmt!( + LineProtocolVersionError, + "Failed to detect server's line protocol version, settings url: {}, err: {}.", + settings_url, + err + ) + } + } + e => { + fmt!( + LineProtocolVersionError, + "Failed to detect server's line protocol version, settings url: {}, err: {}.", + settings_url, + e + ) + } + }; + return Err(e); + } + }; + + let (_, body) = response.into_parts(); + let body_content = body.into_with_config().lossy_utf8(true).read_to_string(); + + if let Ok(msg) = body_content { + let json: serde_json::Value = serde_json::from_str(&msg).map_err(|_| { + error::fmt!( + LineProtocolVersionError, + "Malformed server response, settings url: {}, err: response is not valid JSON.", + settings_url, + ) + })?; + + if let Some(serde_json::Value::Array(ref values)) = json.get("line.proto.support.versions") + { + let mut versions = Vec::new(); + for value in values.iter() { + if let Some(v) = value.as_u64() { + match v { + 1 => versions.push(LineProtocolVersion::V1), + 2 => versions.push(LineProtocolVersion::V2), + _ => {} + } + } + } + support_versions = Some(versions); + } + + if let Some(serde_json::Value::Number(ref v)) = json.get("line.proto.default.version") { + default_version = match v.as_u64() { + Some(vu64) => match vu64 { + 1 => LineProtocolVersion::V1, + 2 => LineProtocolVersion::V2, + _ => { + if let Some(ref versions) = support_versions { + if versions.contains(&LineProtocolVersion::V2) { + LineProtocolVersion::V2 + } else if versions.contains(&LineProtocolVersion::V1) { + LineProtocolVersion::V1 + } else { + return Err(error::fmt!( + LineProtocolVersionError, + "Server does not support current client." + )); + } + } else { + return Err(error::fmt!( + LineProtocolVersionError, + "Unexpected response version content." + )); + } + } + }, + None => { + return Err(error::fmt!( + LineProtocolVersionError, + "Not a valid int for line.proto.default.version in response." + )) + } + }; + } + } else { + return Err(error::fmt!( + LineProtocolVersionError, + "Malformed server response, settings url: {}, err: failed to read response body as UTF-8", settings_url + )); + } + Ok((support_versions, default_version)) +} + +#[allow(clippy::result_large_err)] // `ureq::Error` is large enough to cause this warning. +fn retry_http_get( + state: &HttpHandlerState, + url: &str, + request_timeout: Duration, + retry_timeout: Duration, + mut last_rep: Result, ureq::Error>, +) -> Result, ureq::Error> { + let mut rng = rand::rng(); + let retry_end = std::time::Instant::now() + retry_timeout; + let mut retry_interval_ms = 10; + let mut need_retry; + loop { + let jitter_ms = rng.random_range(-5i32..5); + let to_sleep_ms = retry_interval_ms + jitter_ms; + let to_sleep = Duration::from_millis(to_sleep_ms as u64); + if (std::time::Instant::now() + to_sleep) > retry_end { + return last_rep; + } + sleep(to_sleep); + if let Ok(last_rep) = last_rep { + // Actively consume the reader to return the connection to the connection pool. + // see https://github.com/algesten/ureq/issues/94 + _ = last_rep.into_body().read_to_vec(); + } + (need_retry, last_rep) = state.get_request(url, request_timeout); + if !need_retry { + return last_rep; + } + retry_interval_ms = (retry_interval_ms * 2).min(1000); + } +} + +#[allow(clippy::result_large_err)] // `ureq::Error` is large enough to cause this warning. +fn http_get_with_retries( + state: &HttpHandlerState, + url: &str, + request_timeout: Duration, + retry_timeout: Duration, +) -> Result, ureq::Error> { + let (need_retry, last_rep) = state.get_request(url, request_timeout); + if !need_retry || retry_timeout.is_zero() { + return last_rep; + } + + retry_http_get(state, url, request_timeout, retry_timeout, last_rep) +} diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index fc376a6e..14ef5374 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -59,6 +59,22 @@ use ring::{ /// Defines the maximum allowed dimensions for array data in binary serialization protocols. pub const MAX_DIMS: usize = 32; +/// Line Protocol Version supported by current client. +#[derive(Debug, Copy, Clone, PartialEq)] +pub enum LineProtocolVersion { + V1 = 1, + V2 = 2, +} + +impl std::fmt::Display for LineProtocolVersion { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + LineProtocolVersion::V1 => write!(f, "v1"), + LineProtocolVersion::V2 => write!(f, "v2"), + } + } +} + #[derive(Debug, Copy, Clone)] enum Op { Table = 1, @@ -468,6 +484,8 @@ impl BufferState { } } +pub trait Buffer1 {} + /// A reusable buffer to prepare a batch of ILP messages. /// /// # Example @@ -561,6 +579,8 @@ pub struct Buffer { state: BufferState, marker: Option<(usize, BufferState)>, max_name_len: usize, + f64serializer: fn(&mut Vec, f64), + version: LineProtocolVersion, } impl Buffer { @@ -572,6 +592,8 @@ impl Buffer { state: BufferState::new(), marker: None, max_name_len: 127, + f64serializer: f64_binary_series, + version: LineProtocolVersion::V2, } } @@ -588,6 +610,36 @@ impl Buffer { buf } + pub fn with_line_proto_version(mut self, version: LineProtocolVersion) -> Result { + if self.state.op_case != OpCase::Init { + return Err(error::fmt!( + LineProtocolVersionError, + "Line protocol version must be set before adding any data." + )); + } + self.f64serializer = match version { + LineProtocolVersion::V1 => f64_text_series, + LineProtocolVersion::V2 => f64_binary_series, + }; + self.version = version; + Ok(self) + } + + pub fn set_line_proto_version(&mut self, version: LineProtocolVersion) -> Result<&mut Self> { + if self.state.op_case != OpCase::Init { + return Err(error::fmt!( + LineProtocolVersionError, + "Line protocol version must be set before adding any data." + )); + } + self.f64serializer = match version { + LineProtocolVersion::V1 => f64_text_series, + LineProtocolVersion::V2 => f64_binary_series, + }; + self.version = version; + Ok(self) + } + /// Pre-allocate to ensure the buffer has enough capacity for at least the /// specified additional byte count. This may be rounded up. /// This does not allocate if such additional capacity is already satisfied. @@ -950,8 +1002,7 @@ impl Buffer { Error: From, { self.write_column_key(name)?; - let mut ser = F64Serializer::new(value); - self.output.extend_from_slice(ser.as_str().as_bytes()); + (self.f64serializer)(&mut self.output, value); Ok(self) } @@ -1109,24 +1160,27 @@ impl Buffer { let mut cursor = Cursor::new(writeable); // ndarr data - if let Err(e) = ndarr::write_array_data(view, &mut cursor) { - return Err(error::fmt!( - ArrayWriteToBufferError, - "Can not write row major to writer: {}", - e - )); - } - - if cursor.position() != (reserve_size as u64) { - return Err(error::fmt!( - ArrayWriteToBufferError, - "Array write buffer length mismatch (actual: {}, expected: {})", - cursor.position(), - reserve_size - )); + if view.as_slice().is_some() { + if let Err(e) = ndarr::write_array_data(view, &mut cursor) { + return Err(error::fmt!( + ArrayWriteToBufferError, + "Can not write row major to writer: {}", + e + )); + } + if cursor.position() != (reserve_size as u64) { + return Err(error::fmt!( + ArrayWriteToBufferError, + "Array write buffer length mismatch (actual: {}, expected: {})", + cursor.position(), + reserve_size + )); + } + unsafe { self.output.set_len(reserve_size + index) } + } else { + unsafe { self.output.set_len(reserve_size + index) } + ndarr::write_array_data_use_raw_buffer(&mut self.output[index..], view); } - - unsafe { self.output.set_len(reserve_size + index) } Ok(self) } @@ -1368,6 +1422,9 @@ pub struct Sender { handler: ProtocolHandler, connected: bool, max_buf_size: usize, + default_line_protocol_version: LineProtocolVersion, + #[cfg(feature = "ilp-over-http")] + supported_line_protocol_versions: Option>, } impl std::fmt::Debug for Sender { @@ -1991,6 +2048,19 @@ impl SenderBuilder { "retry_timeout" => { builder.retry_timeout(Duration::from_millis(parse_conf_value(key, val)?))? } + + #[cfg(feature = "ilp-over-http")] + "disable_line_protocol_validation" => { + if val == "on" { + builder.disable_line_protocol_validation()? + } else if val != "off" { + return Err(error::fmt!( + ConfigError, "invalid \"disable_line_protocol_validation\" [value={val}, allowed-values=[on, off]]]\"]")); + } else { + builder + } + } + // Ignore other parameters. // We don't want to fail on unknown keys as this would require releasing different // library implementations in lock step as soon as a new parameter is added to any of them, @@ -2261,6 +2331,22 @@ impl SenderBuilder { Ok(self) } + #[cfg(feature = "ilp-over-http")] + /// Disables automatic line protocol version validation for ILP-over-HTTP. + /// + /// - When set to `"off"`: Skips the initial server version handshake and disables protocol validation. + /// - When set to `"on"`: Keeps default validation behavior (recommended). + /// + /// Please ensure client's default version ([`LINE_PROTOCOL_VERSION_V2`]) or + /// explicitly set protocol version exactly matches server expectation. + pub fn disable_line_protocol_validation(mut self) -> Result { + if let Some(http) = &mut self.http { + http.disable_line_proto_validation + .set_specified("disable_line_protocol_validation", true)?; + } + Ok(self) + } + #[cfg(feature = "ilp-over-http")] /// Internal API, do not use. /// This is exposed exclusively for the Python client. @@ -2543,23 +2629,51 @@ impl SenderBuilder { agent, url, auth, - config: self.http.as_ref().unwrap().clone(), }) } }; + let mut default_line_protocol_version = LineProtocolVersion::V2; + #[cfg(feature = "ilp-over-http")] + let mut supported_line_protocol_versions: Option> = None; + + #[cfg(feature = "ilp-over-http")] + match self.protocol { + Protocol::Tcp | Protocol::Tcps => {} + Protocol::Http | Protocol::Https => { + let http_config = self.http.as_ref().unwrap(); + if !*http_config.disable_line_proto_validation.deref() { + if let ProtocolHandler::Http(http_state) = &handler { + let settings_url = &format!( + "http://{}:{}/settings", + self.host.deref(), + self.port.deref() + ); + ( + supported_line_protocol_versions, + default_line_protocol_version, + ) = get_line_protocol_version(http_state, settings_url)?; + } else { + default_line_protocol_version = LineProtocolVersion::V1; + } + } + } + }; + if auth.is_some() { descr.push_str("auth=on]"); } else { descr.push_str("auth=off]"); } - let sender = Sender { descr, handler, connected: true, max_buf_size: *self.max_buf_size, + default_line_protocol_version, + #[cfg(feature = "ilp-over-http")] + supported_line_protocol_versions, }; Ok(sender) @@ -2678,6 +2792,17 @@ fn parse_key_pair(auth: &EcdsaAuthParams) -> Result { }) } +fn f64_text_series(vec: &mut Vec, value: f64) { + let mut ser = F64Serializer::new(value); + vec.extend_from_slice(ser.as_str().as_bytes()) +} + +fn f64_binary_series(vec: &mut Vec, value: f64) { + vec.push(b'='); + vec.push(DOUBLE_BINARY_FORMAT_TYPE); + vec.extend_from_slice(&value.to_le_bytes()) +} + pub(crate) struct F64Serializer { buf: ryu::Buffer, n: f64, @@ -2774,6 +2899,9 @@ impl Sender { )); } + #[cfg(feature = "ilp-over-http")] + self.check_line_protocol_version(buf.version)?; + let bytes = buf.as_bytes(); if bytes.is_empty() { return Ok(()); @@ -2894,10 +3022,59 @@ impl Sender { pub fn must_close(&self) -> bool { !self.connected } + + /// Returns the client's recommended default line protocol version. + /// Will be used to [`Buffer::with_line_proto_version`] + /// + /// The version selection follows these rules: + /// 1. **TCP/TCPS Protocol**: Always returns [`LineProtocolVersion::V2`] + /// 2. **HTTP/HTTPS Protocol**: + /// - If line protocol auto-detection is disabled [`SenderBuilder::disable_line_protocol_validation`], returns [`LineProtocolVersion::V2`] + /// - If line protocol auto-detection is enabled: + /// - Uses the server's default version if supported by the client + /// - Otherwise uses the highest mutually supported version from the intersection + /// of client and server compatible versions + pub fn default_line_protocol_version(&self) -> LineProtocolVersion { + self.default_line_protocol_version + } + + #[cfg(feature = "ilp-over-http")] + #[inline(always)] + fn check_line_protocol_version(&self, version: LineProtocolVersion) -> Result<()> { + match &self.handler { + ProtocolHandler::Socket(_) => Ok(()), + #[cfg(feature = "ilp-over-http")] + ProtocolHandler::Http(http) => { + if *http.config.disable_line_proto_validation.deref() { + Ok(()) + } else { + match self.supported_line_protocol_versions { + Some(ref supported_line_protocols) => { + if supported_line_protocols.contains(&version) { + Ok(()) + } else { + Err(error::fmt!( + LineProtocolVersionError, + "Line protocol version {} is not supported by current QuestDB Server", version)) + } + } + None => { + if version == LineProtocolVersion::V1 { + Ok(()) + } else { + Err(error::fmt!( + LineProtocolVersionError, + "Line protocol version {} is not supported by current QuestDB Server", version)) + } + } + } + } + } + } + } } pub(crate) const ARRAY_BINARY_FORMAT_TYPE: u8 = 14; -#[allow(dead_code)] pub(crate) const DOUBLE_BINARY_FORMAT_TYPE: u8 = 16; mod conf; diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index ff81d034..3a324ecd 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -25,7 +25,7 @@ where /// /// # Returns /// - `Ok(usize)`: Expected buffer size in bytes if valid - /// - `Err(Error)`: Currently never returned, but reserved for future validation logic + /// - `Err(Error)`: Otherwise fn check_data_buf(&self) -> Result; } @@ -68,7 +68,6 @@ where Ok(()) } -#[cfg(feature = "benchmark")] pub(crate) fn write_array_data_use_raw_buffer, T>(buf: &mut [u8], array: &A) where T: ArrayElement, diff --git a/questdb-rs/src/ingress/tests.rs b/questdb-rs/src/ingress/tests.rs index 8d83a7b0..01a06f51 100644 --- a/questdb-rs/src/ingress/tests.rs +++ b/questdb-rs/src/ingress/tests.rs @@ -447,6 +447,8 @@ fn connect_timeout_uses_request_timeout() { let builder = SenderBuilder::new(Protocol::Http, "127.0.0.2", "1111") .request_timeout(request_timeout) .unwrap() + .disable_line_protocol_validation() + .unwrap() .retry_timeout(Duration::from_millis(10)) .unwrap() .request_min_throughput(0) diff --git a/questdb-rs/src/tests/http.rs b/questdb-rs/src/tests/http.rs index 90ec1222..c2079450 100644 --- a/questdb-rs/src/tests/http.rs +++ b/questdb-rs/src/tests/http.rs @@ -22,18 +22,20 @@ * ******************************************************************************/ -use crate::ingress::{Buffer, Protocol, SenderBuilder, TimestampNanos}; +use crate::ingress::{Buffer, LineProtocolVersion, Protocol, SenderBuilder, TimestampNanos}; use crate::tests::mock::{certs_dir, HttpResponse, MockServer}; +use crate::tests::TestResult; use crate::ErrorCode; +use rstest::rstest; use std::io; use std::io::ErrorKind; use std::time::Duration; -use crate::tests::TestResult; - -#[test] -fn test_two_lines() -> TestResult { - let mut buffer = Buffer::new(); +#[rstest] +fn test_two_lines( + #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, +) -> TestResult { + let mut buffer = Buffer::new().with_line_proto_version(version)?; buffer .table("test")? .symbol("sym", "bol")? @@ -46,11 +48,19 @@ fn test_two_lines() -> TestResult { .at_now()?; let buffer2 = buffer.clone(); - let mut server = MockServer::new()?; - let mut sender = server.lsb_http().build()?; + let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let sender_builder = server.lsb_http(); let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; + let req = server.recv_http_q()?; + assert_eq!(req.method(), "GET"); + assert_eq!(req.path(), "/settings"); + assert_eq!( + req.header("user-agent"), + Some(concat!("questdb/rust/", env!("CARGO_PKG_VERSION"))) + ); + server.send_settings_response()?; let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); @@ -66,6 +76,7 @@ fn test_two_lines() -> TestResult { Ok(()) }); + let mut sender = sender_builder.build()?; let res = sender.flush(&mut buffer); server_thread.join().unwrap()?; @@ -77,9 +88,11 @@ fn test_two_lines() -> TestResult { Ok(()) } -#[test] -fn test_text_plain_error() -> TestResult { - let mut buffer = Buffer::new(); +#[rstest] +fn test_text_plain_error( + #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, +) -> TestResult { + let mut buffer = Buffer::new().with_line_proto_version(version)?; buffer .table("test")? .symbol("sym", "bol")? @@ -87,12 +100,20 @@ fn test_text_plain_error() -> TestResult { .at_now()?; buffer.table("test")?.column_f64("sym", 2.0)?.at_now()?; - let mut server = MockServer::new()?; - let mut sender = server.lsb_http().build()?; + let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let sender_builder = server.lsb_http(); let buffer2 = buffer.clone(); let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; + let req = server.recv_http_q()?; + assert_eq!(req.method(), "GET"); + assert_eq!(req.path(), "/settings"); + assert_eq!( + req.header("user-agent"), + Some(concat!("questdb/rust/", env!("CARGO_PKG_VERSION"))) + ); + server.send_settings_response()?; let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); @@ -109,6 +130,7 @@ fn test_text_plain_error() -> TestResult { Ok(()) }); + let mut sender = sender_builder.build()?; let res = sender.flush(&mut buffer); server_thread.join().unwrap()?; @@ -123,9 +145,11 @@ fn test_text_plain_error() -> TestResult { Ok(()) } -#[test] -fn test_bad_json_error() -> TestResult { - let mut buffer = Buffer::new(); +#[rstest] +fn test_bad_json_error( + #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, +) -> TestResult { + let mut buffer = Buffer::new().with_line_proto_version(version)?; buffer .table("test")? .symbol("sym", "bol")? @@ -133,12 +157,20 @@ fn test_bad_json_error() -> TestResult { .at_now()?; buffer.table("test")?.column_f64("sym", 2.0)?.at_now()?; - let mut server = MockServer::new()?; - let mut sender = server.lsb_http().build()?; + let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let sender_builder = server.lsb_http(); let buffer2 = buffer.clone(); let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; + let req = server.recv_http_q()?; + assert_eq!(req.method(), "GET"); + assert_eq!(req.path(), "/settings"); + assert_eq!( + req.header("user-agent"), + Some(concat!("questdb/rust/", env!("CARGO_PKG_VERSION"))) + ); + server.send_settings_response()?; let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); @@ -156,6 +188,7 @@ fn test_bad_json_error() -> TestResult { Ok(()) }); + let mut sender = sender_builder.build()?; let res = sender.flush_and_keep(&buffer); server_thread.join().unwrap()?; @@ -171,9 +204,11 @@ fn test_bad_json_error() -> TestResult { Ok(()) } -#[test] -fn test_json_error() -> TestResult { - let mut buffer = Buffer::new(); +#[rstest] +fn test_json_error( + #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, +) -> TestResult { + let mut buffer = Buffer::new().with_line_proto_version(version)?; buffer .table("test")? .symbol("sym", "bol")? @@ -181,12 +216,20 @@ fn test_json_error() -> TestResult { .at_now()?; buffer.table("test")?.column_f64("sym", 2.0)?.at_now()?; - let mut server = MockServer::new()?; - let mut sender = server.lsb_http().build()?; + let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let sender_builder = server.lsb_http(); let buffer2 = buffer.clone(); let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; + let req = server.recv_http_q()?; + assert_eq!(req.method(), "GET"); + assert_eq!(req.path(), "/settings"); + assert_eq!( + req.header("user-agent"), + Some(concat!("questdb/rust/", env!("CARGO_PKG_VERSION"))) + ); + server.send_settings_response()?; let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); @@ -207,7 +250,7 @@ fn test_json_error() -> TestResult { Ok(()) }); - let res = sender.flush_and_keep(&buffer); + let res = sender_builder.build()?.flush_and_keep(&buffer); server_thread.join().unwrap()?; @@ -222,16 +265,20 @@ fn test_json_error() -> TestResult { Ok(()) } -#[test] -fn test_no_connection() -> TestResult { - let mut buffer = Buffer::new(); +#[rstest] +fn test_no_connection( + #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, +) -> TestResult { + let mut buffer = Buffer::new().with_line_proto_version(version)?; buffer .table("test")? .symbol("sym", "bol")? .column_f64("x", 1.0)? .at_now()?; - let mut sender = SenderBuilder::new(Protocol::Http, "127.0.0.1", 1).build()?; + let mut sender = SenderBuilder::new(Protocol::Http, "127.0.0.1", 1) + .disable_line_protocol_validation()? + .build()?; let res = sender.flush_and_keep(&buffer); assert!(res.is_err()); let err = res.unwrap_err(); @@ -242,21 +289,31 @@ fn test_no_connection() -> TestResult { Ok(()) } -#[test] -fn test_old_server_without_ilp_http_support() -> TestResult { - let mut buffer = Buffer::new(); +#[rstest] +fn test_old_server_without_ilp_http_support( + #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, +) -> TestResult { + let mut buffer = Buffer::new().with_line_proto_version(version)?; buffer .table("test")? .symbol("sym", "bol")? .column_f64("x", 1.0)? .at_now()?; - let mut server = MockServer::new()?; - let mut sender = server.lsb_http().build()?; + let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let sender_builder = server.lsb_http(); let buffer2 = buffer.clone(); let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; + let req = server.recv_http_q()?; + assert_eq!(req.method(), "GET"); + assert_eq!(req.path(), "/settings"); + assert_eq!( + req.header("user-agent"), + Some(concat!("questdb/rust/", env!("CARGO_PKG_VERSION"))) + ); + server.send_settings_response()?; let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); @@ -273,7 +330,7 @@ fn test_old_server_without_ilp_http_support() -> TestResult { Ok(()) }); - let res = sender.flush_and_keep(&buffer); + let res = sender_builder.build()?.flush_and_keep(&buffer); server_thread.join().unwrap()?; @@ -288,25 +345,34 @@ fn test_old_server_without_ilp_http_support() -> TestResult { Ok(()) } -#[test] -fn test_http_basic_auth() -> TestResult { - let mut buffer = Buffer::new(); +#[rstest] +fn test_http_basic_auth( + #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, +) -> TestResult { + let mut buffer = Buffer::new().with_line_proto_version(version)?; buffer .table("test")? .symbol("sym", "bol")? .column_f64("x", 1.0)? .at_now()?; - let mut server = MockServer::new()?; - let mut sender = server + let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let sender_builder = server .lsb_http() .username("Aladdin")? - .password("OpenSesame")? - .build()?; + .password("OpenSesame")?; let buffer2 = buffer.clone(); let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; + let req = server.recv_http_q()?; + assert_eq!(req.method(), "GET"); + assert_eq!(req.path(), "/settings"); + assert_eq!( + req.header("user-agent"), + Some(concat!("questdb/rust/", env!("CARGO_PKG_VERSION"))) + ); + server.send_settings_response()?; let req = server.recv_http_q()?; @@ -323,7 +389,7 @@ fn test_http_basic_auth() -> TestResult { Ok(()) }); - let res = sender.flush(&mut buffer); + let res = sender_builder.build()?.flush(&mut buffer); server_thread.join().unwrap()?; @@ -334,21 +400,31 @@ fn test_http_basic_auth() -> TestResult { Ok(()) } -#[test] -fn test_unauthenticated() -> TestResult { - let mut buffer = Buffer::new(); +#[rstest] +fn test_unauthenticated( + #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, +) -> TestResult { + let mut buffer = Buffer::new().with_line_proto_version(version)?; buffer .table("test")? .symbol("sym", "bol")? .column_f64("x", 1.0)? .at_now()?; - let mut server = MockServer::new()?; - let mut sender = server.lsb_http().build()?; + let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let sender_builder = server.lsb_http(); let buffer2 = buffer.clone(); let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; + let req = server.recv_http_q()?; + assert_eq!(req.method(), "GET"); + assert_eq!(req.path(), "/settings"); + assert_eq!( + req.header("user-agent"), + Some(concat!("questdb/rust/", env!("CARGO_PKG_VERSION"))) + ); + server.send_settings_response()?; let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); @@ -365,7 +441,7 @@ fn test_unauthenticated() -> TestResult { Ok(()) }); - let res = sender.flush(&mut buffer); + let res = sender_builder.build()?.flush(&mut buffer); server_thread.join().unwrap()?; @@ -382,21 +458,31 @@ fn test_unauthenticated() -> TestResult { Ok(()) } -#[test] -fn test_token_auth() -> TestResult { - let mut buffer = Buffer::new(); +#[rstest] +fn test_token_auth( + #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, +) -> TestResult { + let mut buffer = Buffer::new().with_line_proto_version(version)?; buffer .table("test")? .symbol("sym", "bol")? .column_f64("x", 1.0)? .at_now()?; - let mut server = MockServer::new()?; - let mut sender = server.lsb_http().token("0123456789")?.build()?; + let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let sender_builder = server.lsb_http().token("0123456789")?; let buffer2 = buffer.clone(); let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; + let req = server.recv_http_q()?; + assert_eq!(req.method(), "GET"); + assert_eq!(req.path(), "/settings"); + assert_eq!( + req.header("user-agent"), + Some(concat!("questdb/rust/", env!("CARGO_PKG_VERSION"))) + ); + server.send_settings_response()?; let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); @@ -409,7 +495,7 @@ fn test_token_auth() -> TestResult { Ok(()) }); - let res = sender.flush(&mut buffer); + let res = sender_builder.build()?.flush(&mut buffer); server_thread.join().unwrap()?; @@ -418,9 +504,11 @@ fn test_token_auth() -> TestResult { Ok(()) } -#[test] -fn test_request_timeout() -> TestResult { - let mut buffer = Buffer::new(); +#[rstest] +fn test_request_timeout( + #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, +) -> TestResult { + let mut buffer = Buffer::new().with_line_proto_version(version)?; buffer .table("test")? .symbol("sym", "bol")? @@ -428,12 +516,13 @@ fn test_request_timeout() -> TestResult { .at_now()?; // Here we use a mock (tcp) server instead and don't send a response back. - let server = MockServer::new()?; + let server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); let request_timeout = Duration::from_millis(50); let time_start = std::time::Instant::now(); let mut sender = server .lsb_http() + .disable_line_protocol_validation()? .request_timeout(request_timeout)? .build()?; let res = sender.flush_and_keep(&buffer); @@ -446,12 +535,14 @@ fn test_request_timeout() -> TestResult { Ok(()) } -#[test] -fn test_tls() -> TestResult { +#[rstest] +fn test_tls( + #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, +) -> TestResult { let mut ca_path = certs_dir(); ca_path.push("server_rootCA.pem"); - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new().with_line_proto_version(version)?; buffer .table("test")? .symbol("t1", "v1")? @@ -459,8 +550,12 @@ fn test_tls() -> TestResult { .at(TimestampNanos::new(10000000))?; let buffer2 = buffer.clone(); - let mut server = MockServer::new()?; - let mut sender = server.lsb_https().tls_roots(ca_path)?.build()?; + let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let mut sender = server + .lsb_https() + .tls_roots(ca_path)? + .disable_line_protocol_validation()? + .build()?; let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept_tls_sync()?; @@ -484,9 +579,11 @@ fn test_tls() -> TestResult { Ok(()) } -#[test] -fn test_user_agent() -> TestResult { - let mut buffer = Buffer::new(); +#[rstest] +fn test_user_agent( + #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, +) -> TestResult { + let mut buffer = Buffer::new().with_line_proto_version(version)?; buffer .table("test")? .symbol("t1", "v1")? @@ -494,11 +591,15 @@ fn test_user_agent() -> TestResult { .at(TimestampNanos::new(10000000))?; let buffer2 = buffer.clone(); - let mut server = MockServer::new()?; - let mut sender = server.lsb_http().user_agent("wallabies/1.2.99")?.build()?; + let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let sender_builder = server.lsb_http().user_agent("wallabies/1.2.99")?; let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; + let req = server.recv_http_q()?; + assert_eq!(req.method(), "GET"); + assert_eq!(req.path(), "/settings"); + server.send_settings_response()?; let req = server.recv_http_q()?; assert_eq!(req.header("user-agent"), Some("wallabies/1.2.99")); @@ -509,7 +610,7 @@ fn test_user_agent() -> TestResult { Ok(()) }); - let res = sender.flush_and_keep(&buffer); + let res = sender_builder.build()?.flush_and_keep(&buffer); server_thread.join().unwrap()?; @@ -519,11 +620,13 @@ fn test_user_agent() -> TestResult { Ok(()) } -#[test] -fn test_two_retries() -> TestResult { +#[rstest] +fn test_two_retries( + #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, +) -> TestResult { // Note: This also tests that the _same_ connection is being reused, i.e. tests keepalive. - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new().with_line_proto_version(version)?; buffer .table("test")? .symbol("t1", "v1")? @@ -531,14 +634,19 @@ fn test_two_retries() -> TestResult { .at(TimestampNanos::new(10000000))?; let buffer2 = buffer.clone(); - let mut server = MockServer::new()?; - let mut sender = server - .lsb_http() - .retry_timeout(Duration::from_secs(30))? - .build()?; + let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let sender_builder = server.lsb_http().retry_timeout(Duration::from_secs(30))?; let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; + let req = server.recv_http_q()?; + assert_eq!(req.method(), "GET"); + assert_eq!(req.path(), "/settings"); + assert_eq!( + req.header("user-agent"), + Some(concat!("questdb/rust/", env!("CARGO_PKG_VERSION"))) + ); + server.send_settings_response()?; let req = server.recv_http_q()?; assert_eq!(req.body(), buffer2.as_bytes()); @@ -574,7 +682,7 @@ fn test_two_retries() -> TestResult { Ok(()) }); - let res = sender.flush_and_keep(&buffer); + let res = sender_builder.build()?.flush_and_keep(&buffer); server_thread.join().unwrap()?; @@ -584,9 +692,11 @@ fn test_two_retries() -> TestResult { Ok(()) } -#[test] -fn test_one_retry() -> TestResult { - let mut buffer = Buffer::new(); +#[rstest] +fn test_one_retry( + #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, +) -> TestResult { + let mut buffer = Buffer::new().with_line_proto_version(version)?; buffer .table("test")? .symbol("t1", "v1")? @@ -594,15 +704,16 @@ fn test_one_retry() -> TestResult { .at(TimestampNanos::new(10000000))?; let buffer2 = buffer.clone(); - let mut server = MockServer::new()?; + let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); let mut sender = server .lsb_http() .retry_timeout(Duration::from_millis(19))? + .disable_line_protocol_validation() + .unwrap() .build()?; let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; - let req = server.recv_http_q()?; assert_eq!(req.body(), buffer2.as_bytes()); @@ -648,10 +759,12 @@ fn test_one_retry() -> TestResult { Ok(()) } -#[test] -fn test_transactional() -> TestResult { +#[rstest] +fn test_transactional( + #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, +) -> TestResult { // A buffer with a two tables. - let mut buffer1 = Buffer::new(); + let mut buffer1 = Buffer::new().with_line_proto_version(version)?; buffer1 .table("tab1")? .symbol("t1", "v1")? @@ -665,7 +778,7 @@ fn test_transactional() -> TestResult { assert!(!buffer1.transactional()); // A buffer with a single table. - let mut buffer2 = Buffer::new(); + let mut buffer2 = Buffer::new().with_line_proto_version(version)?; buffer2 .table("test")? .symbol("t1", "v1")? @@ -674,11 +787,19 @@ fn test_transactional() -> TestResult { let buffer3 = buffer2.clone(); assert!(buffer2.transactional()); - let mut server = MockServer::new()?; - let mut sender = server.lsb_http().build()?; + let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let sender_builder = server.lsb_http(); let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; + let req = server.recv_http_q()?; + assert_eq!(req.method(), "GET"); + assert_eq!(req.path(), "/settings"); + assert_eq!( + req.header("user-agent"), + Some(concat!("questdb/rust/", env!("CARGO_PKG_VERSION"))) + ); + server.send_settings_response()?; let req = server.recv_http_q()?; assert_eq!(req.body(), buffer3.as_bytes()); @@ -688,6 +809,8 @@ fn test_transactional() -> TestResult { Ok(()) }); + let mut sender = sender_builder.build()?; + let res = sender.flush_and_keep_with_flags(&buffer1, true); assert!(res.is_err()); let err = res.unwrap_err(); diff --git a/questdb-rs/src/tests/interop/ilp-client-interop-test.json b/questdb-rs/src/tests/interop/ilp-client-interop-test.json index d3e0e259..0acedad7 100644 --- a/questdb-rs/src/tests/interop/ilp-client-interop-test.json +++ b/questdb-rs/src/tests/interop/ilp-client-interop-test.json @@ -32,6 +32,7 @@ ], "result": { "status": "SUCCESS", + "binaryBase64": "dGVzdF90YWJsZSxzeW1fY29sPXN5bV92YWwgc3RyX2NvbD0iZm9vIGJhciBiYXoiLGxvbmdfY29sPTQyaSxkb3VibGVfY29sPT0QAAAAAABARUAsYm9vbF9jb2w9dAo=", "line": "test_table,sym_col=sym_val str_col=\"foo bar baz\",long_col=42i,double_col=42.5,bool_col=t" } }, @@ -73,6 +74,7 @@ ], "result": { "status": "SUCCESS", + "binaryBase64": "ZG91YmxlcyBkMD09EAAAAAAAAAAALGRtMD09EAAAAAAAAACALGQxPT0QAAAAAAAA8D8sZEUxMDA9PRB9w5QlrUmyVCxkMDAwMDAwMT09EI3ttaD3xrA+LGROMDAwMDAwMT09EI3ttaD3xrC+Cg==", "anyLines": [ "doubles d0=0,dm0=-0,d1=1,dE100=1E+100,d0000001=1E-06,dN0000001=-1E-06", "doubles d0=0.0,dm0=-0.0,d1=1.0,dE100=1e100,d0000001=1e-6,dN0000001=-1e-6" diff --git a/questdb-rs/src/tests/mock.rs b/questdb-rs/src/tests/mock.rs index 03fcf527..57e13c8f 100644 --- a/questdb-rs/src/tests/mock.rs +++ b/questdb-rs/src/tests/mock.rs @@ -52,6 +52,8 @@ pub struct MockServer { pub host: &'static str, pub port: u16, pub msgs: Vec>, + #[cfg(feature = "ilp-over-http")] + settings_response: serde_json::Value, } pub fn certs_dir() -> std::path::PathBuf { @@ -207,6 +209,8 @@ impl MockServer { host: "localhost", port, msgs: Vec::new(), + #[cfg(feature = "ilp-over-http")] + settings_response: serde_json::Value::Null, }) } @@ -303,6 +307,19 @@ impl MockServer { } } + #[cfg(feature = "ilp-over-http")] + pub fn configure_settings_response( + mut self, + default_version: u16, + supported_versions: &[u16], + ) -> Self { + self.settings_response = serde_json::json!({ + "line.proto.default.version": default_version, + "line.proto.support.versions": supported_versions + }); + self + } + #[cfg(feature = "ilp-over-http")] fn do_write(&mut self, buf: &[u8]) -> io::Result { let client = self.client.as_mut().unwrap(); @@ -455,6 +472,15 @@ impl MockServer { Ok(()) } + #[cfg(feature = "ilp-over-http")] + pub fn send_settings_response(&mut self) -> io::Result<()> { + let response = HttpResponse::empty() + .with_status(200, "OK") + .with_body_json(&self.settings_response); + self.send_http_response(response, Some(2.0))?; + Ok(()) + } + #[cfg(feature = "ilp-over-http")] pub fn send_http_response_q(&mut self, response: HttpResponse) -> io::Result<()> { self.send_http_response(response, Some(5.0)) @@ -466,6 +492,14 @@ impl MockServer { let deadline = Instant::now() + Duration::from_secs_f64(wait_timeout_sec); let (pos, method, path) = self.recv_http_method(&mut accum, deadline)?; let (pos, headers) = self.recv_http_headers(pos, &mut accum, deadline)?; + if &method == "GET" { + return Ok(HttpRequest { + method, + path, + headers, + body: vec![], + }); + } let content_length = headers .get("content-length") .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Missing Content-Length"))? diff --git a/questdb-rs/src/tests/mod.rs b/questdb-rs/src/tests/mod.rs index 29c1b3f4..d68dede9 100644 --- a/questdb-rs/src/tests/mod.rs +++ b/questdb-rs/src/tests/mod.rs @@ -21,6 +21,7 @@ * limitations under the License. * ******************************************************************************/ + mod f64_serializer; #[cfg(feature = "ilp-over-http")] diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index 3edc0eb8..81957e26 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -29,22 +29,25 @@ use crate::{ Error, ErrorCode, }; +#[cfg(feature = "ndarray")] +use crate::ingress::ndarr::write_array_data; +use crate::ingress::LineProtocolVersion; use crate::tests::{ mock::{certs_dir, MockServer}, TestResult, }; - -#[cfg(feature = "ndarray")] -use crate::ingress::ndarr::write_array_data; #[cfg(feature = "ndarray")] use crate::{ingress, ingress::ElemDataType}; use core::time::Duration; #[cfg(feature = "ndarray")] use ndarray::{arr1, arr2, ArrayD}; +use rstest::rstest; use std::{io, time::SystemTime}; -#[test] -fn test_basics() -> TestResult { +#[rstest] +fn test_basics( + #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, +) -> TestResult { let mut server = MockServer::new()?; let mut sender = server.lsb_tcp().build()?; assert!(!sender.must_close()); @@ -60,7 +63,7 @@ fn test_basics() -> TestResult { let ts_nanos = TimestampNanos::from_systemtime(ts)?; assert_eq!(ts_nanos.as_i64(), ts_nanos_num); - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new().with_line_proto_version(version)?; buffer .table("test")? .symbol("t1", "v1")? @@ -71,20 +74,25 @@ fn test_basics() -> TestResult { .at(ts_nanos)?; assert_eq!(server.recv_q()?, 0); - let exp = format!( - "test,t1=v1 f1=0.5,ts1=12345t,ts2={}t,ts3={}t {}\n", - ts_micros_num, - ts_nanos_num / 1000i64, - ts_nanos_num - ); - let exp_byte = exp.as_bytes(); - assert_eq!(buffer.as_bytes(), exp_byte); + let exp = &[ + "test,t1=v1 ".as_bytes(), + f64_to_bytes("f1", 0.5, version).as_slice(), + format!( + ",ts1=12345t,ts2={}t,ts3={}t {}\n", + ts_micros_num, + ts_nanos_num / 1000i64, + ts_nanos_num + ) + .as_bytes(), + ] + .concat(); + assert_eq!(buffer.as_bytes(), exp); assert_eq!(buffer.len(), exp.len()); sender.flush(&mut buffer)?; assert_eq!(buffer.len(), 0); assert_eq!(buffer.as_bytes(), b""); assert_eq!(server.recv_q()?, 1); - assert_eq!(server.msgs[0], exp_byte); + assert_eq!(server.msgs[0], *exp); Ok(()) } @@ -101,7 +109,8 @@ fn test_array_basic() -> TestResult { let array_2d = arr2(&[[1.1, 2.2], [3.3, 4.4]]); let array_3d = ArrayD::::ones(vec![2, 3, 4]); - let mut buffer = Buffer::new(); + let mut buffer = + Buffer::new().with_line_proto_version(sender.default_line_protocol_version())?; buffer .table("my_table")? .symbol("device", "A001")? @@ -138,7 +147,8 @@ fn test_array_basic() -> TestResult { write_array_data(&array_3d.view(), &mut &mut array_data3d[0..])?; let exp = &[ - "my_table,device=A001 f1=25.5".as_bytes(), + "my_table,device=A001 ".as_bytes(), + f64_to_bytes("f1", 25.5, LineProtocolVersion::V2).as_slice(), ",arr2d=".as_bytes(), array_header2d, array_data2d.as_slice(), @@ -159,15 +169,16 @@ fn test_array_basic() -> TestResult { Ok(()) } -#[test] -fn test_max_buf_size() -> TestResult { +#[rstest] +fn test_max_buf_size( + #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, +) -> TestResult { let max = 1024; let mut server = MockServer::new()?; let mut sender = server.lsb_tcp().max_buf_size(max)?.build()?; assert!(!sender.must_close()); server.accept()?; - - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new().with_line_proto_version(version)?; while buffer.len() < max { buffer @@ -179,10 +190,20 @@ fn test_max_buf_size() -> TestResult { let err = sender.flush(&mut buffer).unwrap_err(); assert_eq!(err.code(), ErrorCode::InvalidApiCall); - assert_eq!( - err.msg(), - "Could not flush buffer: Buffer size of 1026 exceeds maximum configured allowed size of 1024 bytes." - ); + match version { + LineProtocolVersion::V1 => { + assert_eq!( + err.msg(), + "Could not flush buffer: Buffer size of 1026 exceeds maximum configured allowed size of 1024 bytes." + ); + } + LineProtocolVersion::V2 => { + assert_eq!( + err.msg(), + "Could not flush buffer: Buffer size of 1025 exceeds maximum configured allowed size of 1024 bytes." + ); + } + } Ok(()) } @@ -440,8 +461,10 @@ fn test_arr_column_name_too_long() -> TestResult { column_name_too_long_test_impl!(column_arr, &arr1(&[1.0, 2.0, 3.0]).view()) } -#[test] -fn test_tls_with_file_ca() -> TestResult { +#[rstest] +fn test_tls_with_file_ca( + #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, +) -> TestResult { let mut ca_path = certs_dir(); ca_path.push("server_rootCA.pem"); @@ -451,7 +474,7 @@ fn test_tls_with_file_ca() -> TestResult { let mut sender = lsb.build()?; let mut server: MockServer = server_jh.join().unwrap()?; - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new().with_line_proto_version(version)?; buffer .table("test")? .symbol("t1", "v1")? @@ -459,7 +482,12 @@ fn test_tls_with_file_ca() -> TestResult { .at(TimestampNanos::new(10000000))?; assert_eq!(server.recv_q()?, 0); - let exp = b"test,t1=v1 f1=0.5 10000000\n"; + let exp = &[ + "test,t1=v1 ".as_bytes(), + f64_to_bytes("f1", 0.5, version).as_slice(), + " 10000000\n".as_bytes(), + ] + .concat(); assert_eq!(buffer.as_bytes(), exp); assert_eq!(buffer.len(), exp.len()); sender.flush(&mut buffer)?; @@ -536,15 +564,17 @@ fn test_plain_to_tls_server() -> TestResult { } #[cfg(feature = "insecure-skip-verify")] -#[test] -fn test_tls_insecure_skip_verify() -> TestResult { +#[rstest] +fn test_tls_insecure_skip_verify( + #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, +) -> TestResult { let server = MockServer::new()?; let lsb = server.lsb_tcps().tls_verify(false)?; let server_jh = server.accept_tls(); let mut sender = lsb.build()?; let mut server: MockServer = server_jh.join().unwrap()?; - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new().with_line_proto_version(version)?; buffer .table("test")? .symbol("t1", "v1")? @@ -552,7 +582,12 @@ fn test_tls_insecure_skip_verify() -> TestResult { .at(TimestampNanos::new(10000000))?; assert_eq!(server.recv_q()?, 0); - let exp = b"test,t1=v1 f1=0.5 10000000\n"; + let exp = &[ + "test,t1=v1 ".as_bytes(), + f64_to_bytes("f1", 0.5, version).as_slice(), + " 10000000\n".as_bytes(), + ] + .concat(); assert_eq!(buffer.as_bytes(), exp); assert_eq!(buffer.len(), exp.len()); sender.flush(&mut buffer)?; @@ -578,3 +613,22 @@ fn bad_uppercase_addr() { assert!(err.code() == ErrorCode::ConfigError); assert!(err.msg() == "Missing \"addr\" parameter in config string"); } + +fn f64_to_bytes(name: &str, value: f64, version: LineProtocolVersion) -> Vec { + let mut buf = Vec::new(); + buf.extend_from_slice(name.as_bytes()); + buf.push(b'='); + + match version { + LineProtocolVersion::V1 => { + let mut ser = crate::ingress::F64Serializer::new(value); + buf.extend_from_slice(ser.as_str().as_bytes()); + } + LineProtocolVersion::V2 => { + buf.push(b'='); + buf.push(crate::ingress::DOUBLE_BINARY_FORMAT_TYPE); + buf.extend_from_slice(&value.to_le_bytes()); + } + } + buf +} diff --git a/system_test/fixture.py b/system_test/fixture.py index 40dca629..19d6a50e 100644 --- a/system_test/fixture.py +++ b/system_test/fixture.py @@ -414,7 +414,8 @@ def check_table(): except TimeoutError as toe: if log: if log_ctx: - log_ctx = f'\n{textwrap.indent(log_ctx, " ")}\n' + log_ctx_str = log_ctx.decode('utf-8', errors='replace') + log_ctx = f'\n{textwrap.indent(log_ctx_str, " ")}\n' sys.stderr.write( f'Timed out after {timeout_sec} seconds ' + f'waiting for query {sql_query!r}. ' + diff --git a/system_test/questdb_line_sender.py b/system_test/questdb_line_sender.py index 5d4eca85..38416a4c 100644 --- a/system_test/questdb_line_sender.py +++ b/system_test/questdb_line_sender.py @@ -40,6 +40,8 @@ from ast import arg import sys +import numpy + sys.dont_write_bytecode = True import pathlib @@ -53,10 +55,12 @@ c_size_t, c_char_p, c_int, + c_int32, c_int64, c_double, c_uint8, c_uint16, + c_uint32, c_uint64, c_void_p, c_ssize_t) @@ -98,6 +102,10 @@ class c_line_sender_error(ctypes.Structure): c_line_sender_opts_p = ctypes.POINTER(c_line_sender_opts) c_line_sender_error_p = ctypes.POINTER(c_line_sender_error) c_line_sender_error_p_p = ctypes.POINTER(c_line_sender_error_p) +c_int32_p = ctypes.POINTER(c_int32) +c_uint8_p = ctypes.POINTER(c_uint8) +c_uint32_p = ctypes.POINTER(c_uint32) + class c_line_sender_utf8(ctypes.Structure): _fields_ = [("len", c_size_t), ("buf", c_char_p)] @@ -107,7 +115,7 @@ class c_line_sender_table_name(ctypes.Structure): ("buf", c_char_p)] class line_sender_buffer_view(ctypes.Structure): _fields_ = [("len", c_size_t), - ("buf", ctypes.POINTER(c_uint8))] + ("buf", c_uint8_p)] c_line_sender_table_name_p = ctypes.POINTER(c_line_sender_table_name) class c_line_sender_column_name(ctypes.Structure): @@ -237,6 +245,17 @@ def set_sig(fn, restype, *argtypes): c_line_sender_column_name, c_line_sender_utf8, c_line_sender_error_p_p) + set_sig( + dll.line_sender_buffer_column_f64_arr, + c_bool, + c_line_sender_buffer_p, + c_line_sender_column_name, + c_size_t, + c_uint32_p, + c_int32_p, + c_uint8_p, + c_size_t, + c_line_sender_error_p_p) set_sig( dll.line_sender_buffer_column_ts_nanos, c_bool, @@ -627,6 +646,34 @@ def column( '`bool`, `int`, `float` or `str`.') return self + def column_f64_arr(self, name: str, + rank: int, + shapes: tuple[int, ...], + strides: tuple[int, ...], + data: c_void_p, + length: int): + def _convert_tuple(tpl: tuple[int, ...], c_type: type, name: str) -> ctypes.POINTER: + arr_type = c_type * len(tpl) + try: + return arr_type(*[c_type(v) for v in tpl]) + except OverflowError as e: + raise ValueError( + f"{name} value exceeds {c_type.__name__} range" + ) from e + + c_shapes = _convert_tuple(shapes, c_uint32, "shapes") + c_strides = _convert_tuple(strides, c_int32, "strides") + _error_wrapped_call( + _DLL.line_sender_buffer_column_f64_arr, + self._impl, + _column_name(name), + c_size_t(rank), + c_shapes, + c_strides, + ctypes.cast(data, c_uint8_p), + c_size_t(length) + ) + def at_now(self): _error_wrapped_call( _DLL.line_sender_buffer_at_now, @@ -736,6 +783,12 @@ def column( self._buffer.column(name, value) return self + def column_f64_arr( + self, name: str, + array: numpy.ndarray): + self._buffer.column_f64_arr(name, array.ndim, array.shape, array.strides, array.ctypes.data, array.nbytes) + return self + def at_now(self): self._buffer.at_now() diff --git a/system_test/test.py b/system_test/test.py index c28ccd6d..d2860d9d 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -25,14 +25,15 @@ ################################################################################ import sys + sys.dont_write_bytecode = True import os - import pathlib import math import datetime import argparse import unittest +import numpy as np import time import questdb_line_sender as qls import uuid @@ -47,7 +48,6 @@ import subprocess from collections import namedtuple - QDB_FIXTURE: QuestDbFixture = None TLS_PROXY_FIXTURE: TlsProxyFixture = None BUILD_MODE = None @@ -72,7 +72,6 @@ def ns_to_qdb_date(at_ts_ns): token_x="-nSHz3evuPl-rGLIlbIZjwOJeWao0rbk53Cll6XEgak", token_y="9iYksF4L5mfmArupv0CMoyVAWjQ4gNIoupdg6N5noG8") - # Bad malformed key AUTH_MALFORMED1 = dict( username="testUser3", @@ -80,7 +79,6 @@ def ns_to_qdb_date(at_ts_ns): token_x="-nSHz3evuPl-rGLIlbIZjwOJeWao0rbk53Cll6XEgak", token_y="9iYksF4L6mfmArupv0CMoyVAWjQ4gNIoupdg6N5noG8") - # Another malformed key where the keys invalid base 64. AUTH_MALFORMED2 = dict( username="testUser4", @@ -88,7 +86,6 @@ def ns_to_qdb_date(at_ts_ns): token_x="-nSHz3evuPl-rGLIlbIZjwOJeWao0rbk5XEgak", token_y="9iYksF4L6mfmArupv0CMoyVAWjQ4gNIou5noG8") - # All the keys are valid, but the username is wrong. AUTH_MALFORMED3 = dict( username="wrongUser", @@ -115,9 +112,9 @@ def _expect_eventual_disconnect(self, sender): for _ in range(1000): time.sleep(0.1) (sender - .table(table_name) - .symbol('s1', 'v1') - .at_now()) + .table(table_name) + .symbol('s1', 'v1') + .at_now()) sender.flush() def test_insert_three_rows(self): @@ -126,13 +123,13 @@ def test_insert_three_rows(self): with self._mk_linesender() as sender: for _ in range(3): (sender - .table(table_name) - .symbol('name_a', 'val_a') - .column('name_b', True) - .column('name_c', 42) - .column('name_d', 2.5) - .column('name_e', 'val_b') - .at_now()) + .table(table_name) + .symbol('name_a', 'val_a') + .column('name_b', True) + .column('name_c', 42) + .column('name_d', 2.5) + .column('name_e', 'val_b') + .at_now()) pending = sender.buffer.peek() sender.flush() @@ -161,12 +158,12 @@ def test_repeated_symbol_and_column_names(self): pending = None with self._mk_linesender() as sender: (sender - .table(table_name) - .symbol('a', 'A') - .symbol('a', 'B') - .column('b', False) - .column('b', 'C') - .at_now()) + .table(table_name) + .symbol('a', 'A') + .symbol('a', 'B') + .column('b', False) + .column('b', 'C') + .at_now()) pending = sender.buffer.peek() resp = retry_check_table(table_name, log_ctx=pending) @@ -188,10 +185,10 @@ def test_same_symbol_and_col_name(self): pending = None with self._mk_linesender() as sender: (sender - .table(table_name) - .symbol('a', 'A') - .column('a', 'B') - .at_now()) + .table(table_name) + .symbol('a', 'A') + .column('a', 'B') + .at_now()) pending = sender.buffer.peek() resp = retry_check_table(table_name, log_ctx=pending) @@ -209,9 +206,9 @@ def _test_single_symbol_impl(self, sender): pending = None with sender: (sender - .table(table_name) - .symbol('a', 'A') - .at_now()) + .table(table_name) + .symbol('a', 'A') + .at_now()) pending = sender.buffer.peek() resp = retry_check_table(table_name, log_ctx=pending) @@ -232,10 +229,10 @@ def test_two_columns(self): pending = None with self._mk_linesender() as sender: (sender - .table(table_name) - .column('a', 'A') - .column('b', 'B') - .at_now()) + .table(table_name) + .column('a', 'A') + .column('b', 'B') + .at_now()) pending = sender.buffer.peek() resp = retry_check_table(table_name, log_ctx=pending) @@ -254,13 +251,13 @@ def test_mismatched_types_across_rows(self): pending = None with self._mk_linesender() as sender: (sender - .table(table_name) - .column('a', 1) # LONG - .at_now()) + .table(table_name) + .column('a', 1) # LONG + .at_now()) (sender - .table(table_name) - .symbol('a', 'B') # SYMBOL - .at_now()) + .table(table_name) + .symbol('a', 'B') # SYMBOL + .at_now()) pending = sender.buffer.peek() @@ -304,9 +301,9 @@ def test_at(self): pending = None with self._mk_linesender() as sender: (sender - .table(table_name) - .symbol('a', 'A') - .at(at_ts_ns)) + .table(table_name) + .symbol('a', 'A') + .at(at_ts_ns)) pending = sender.buffer.peek() resp = retry_check_table(table_name, log_ctx=pending) exp_dataset = [['A', ns_to_qdb_date(at_ts_ns)]] @@ -322,9 +319,9 @@ def test_neg_at(self): with self._mk_linesender() as sender: with self.assertRaisesRegex(qls.SenderError, r'.*Timestamp .* is negative.*'): (sender - .table(table_name) - .symbol('a', 'A') - .at(at_ts_ns)) + .table(table_name) + .symbol('a', 'A') + .at(at_ts_ns)) def test_timestamp_col(self): if QDB_FIXTURE.version <= (6, 0, 7, 1): @@ -334,13 +331,13 @@ def test_timestamp_col(self): pending = None with self._mk_linesender() as sender: (sender - .table(table_name) - .column('a', qls.TimestampMicros(-1000000)) - .at_now()) + .table(table_name) + .column('a', qls.TimestampMicros(-1000000)) + .at_now()) (sender - .table(table_name) - .column('a', qls.TimestampMicros(1000000)) - .at_now()) + .table(table_name) + .column('a', qls.TimestampMicros(1000000)) + .at_now()) pending = sender.buffer.peek() resp = retry_check_table(table_name, log_ctx=pending) @@ -353,16 +350,15 @@ def test_timestamp_col(self): scrubbed_dataset = [row[:-1] for row in resp['dataset']] self.assertEqual(scrubbed_dataset, exp_dataset) - def test_underscores(self): table_name = f'_{uuid.uuid4().hex}_' pending = None with self._mk_linesender() as sender: (sender - .table(table_name) - .symbol('_a_b_c_', 'A') - .column('_d_e_f_', True) - .at_now()) + .table(table_name) + .symbol('_a_b_c_', 'A') + .column('_d_e_f_', True) + .at_now()) pending = sender.buffer.peek() resp = retry_check_table(table_name, log_ctx=pending) @@ -422,16 +418,16 @@ def test_floats(self): 1.23456789012, 1000000000000000000000000.0, -1000000000000000000000000.0, - float("nan"), # Converted to `None`. - float("inf"), # Converted to `None`. + float("nan"), # Converted to `None`. + float("inf"), # Converted to `None`. float("-inf")] # Converted to `None`. - # These values below do not round-trip properly: QuestDB limitation. - # 1.2345678901234567, - # 2.2250738585072014e-308, - # -2.2250738585072014e-308, - # 1.7976931348623157e+308, - # -1.7976931348623157e+308] + # These values below do not round-trip properly: QuestDB limitation. + # 1.2345678901234567, + # 2.2250738585072014e-308, + # -2.2250738585072014e-308, + # 1.7976931348623157e+308, + # -1.7976931348623157e+308] table_name = uuid.uuid4().hex pending = None with self._mk_linesender() as sender: @@ -469,9 +465,9 @@ def test_timestamp_column(self): ts = qls.TimestampMicros(3600000000) # One hour past epoch. with self._mk_linesender() as sender: (sender - .table(table_name) - .column('ts1', ts) - .at_now()) + .table(table_name) + .column('ts1', ts) + .at_now()) pending = sender.buffer.peek() resp = retry_check_table(table_name, log_ctx=pending) @@ -483,6 +479,38 @@ def test_timestamp_column(self): scrubbed_dataset = [row[:-1] for row in resp['dataset']] self.assertEqual(scrubbed_dataset, exp_dataset) + def test_f64_arr_column(self): + table_name = uuid.uuid4().hex + array1 = np.array( + [ + [[1.1, 2.2], [3.3, 4.4]], + [[5.5, 6.6], [7.7, 8.8]] + ], + dtype=np.float64 + ) + array2 = array1.T + array3 = array1[::-1, ::-1] + + with self._mk_linesender() as sender: + (sender + .table(table_name) + .column_f64_arr('f64_arr1', array1) + .column_f64_arr('f64_arr2', array2) + .column_f64_arr('f64_arr3', array3) + .at_now()) + + resp = retry_check_table(table_name) + exp_columns = [{'dim': 3, 'elemType': 'DOUBLE', 'name': 'f64_arr1', 'type': 'ARRAY'}, + {'dim': 3, 'elemType': 'DOUBLE', 'name': 'f64_arr2', 'type': 'ARRAY'}, + {'dim': 3, 'elemType': 'DOUBLE', 'name': 'f64_arr3', 'type': 'ARRAY'}, + {'name': 'timestamp', 'type': 'TIMESTAMP'}] + self.assertEqual(resp['columns'], exp_columns) + expected_data = [[[[[1.1, 2.2], [3.3, 4.4]], [[5.5, 6.6], [7.7, 8.8]]], + [[[1.1, 5.5], [3.3, 7.7]], [[2.2, 6.6], [4.4, 8.8]]], + [[[7.7, 8.8], [5.5, 6.6]], [[3.3, 4.4], [1.1, 2.2]]]]] + scrubbed_data = [row[:-1] for row in resp['dataset']] + self.assertEqual(scrubbed_data, expected_data) + def _test_example(self, bin_name, table_name, tls=False): if BUILD_MODE != qls.BuildMode.API: self.skipTest('BuildMode.API-only test') @@ -511,10 +539,17 @@ def _test_example(self, bin_name, table_name, tls=False): {'name': 'side', 'type': 'SYMBOL'}, {'name': 'price', 'type': 'DOUBLE'}, {'name': 'amount', 'type': 'DOUBLE'}, + {'dim': 3, 'elemType': 'DOUBLE', 'name': 'order_book', 'type': 'ARRAY'}, {'name': 'timestamp', 'type': 'TIMESTAMP'}] self.assertEqual(resp['columns'], exp_columns) - exp_dataset = [['ETH-USD', 'sell', 2615.54, 0.00044]] # Comparison excludes timestamp column. + exp_dataset = [['ETH-USD', + 'sell', + 2615.54, + 0.00044, + [[[48123.5, 2.4], [48124.0, 1.8], [48124.5, 0.9]], + [[48122.5, 3.1], [48122.0, 2.7], [48121.5, 4.3]]]]] + # Comparison excludes timestamp column. scrubbed_dataset = [row[:-1] for row in resp['dataset']] self.assertEqual(scrubbed_dataset, exp_dataset) @@ -570,9 +605,9 @@ def test_opposite_auth(self): # The sending the first line will not fail. (sender - .table(table_name) - .symbol('s1', 'v1') - .at_now()) + .table(table_name) + .symbol('s1', 'v1') + .at_now()) sender.flush() self._expect_eventual_disconnect(sender) @@ -663,7 +698,7 @@ def test_tls_insecure_skip_verify(self): def test_tls_roots(self): protocol = qls.Protocol.HTTPS if QDB_FIXTURE.http else qls.Protocol.TCPS - auth = auth=AUTH if QDB_FIXTURE.auth else {} + auth = auth = AUTH if QDB_FIXTURE.auth else {} sender = qls.Sender( BUILD_MODE, protocol, @@ -679,7 +714,7 @@ def _test_tls_ca(self, tls_ca): try: os.environ['SSL_CERT_FILE'] = str( Project().tls_certs_dir / 'server_rootCA.pem') - auth = auth=AUTH if QDB_FIXTURE.auth else {} + auth = auth = AUTH if QDB_FIXTURE.auth else {} sender = qls.Sender( BUILD_MODE, protocol, @@ -803,14 +838,15 @@ def run_with_existing(args): global QDB_FIXTURE MockFixture = namedtuple( 'MockFixture', - ('host', 'line_tcp_port', 'http_server_port', 'version', 'http')) + ('host', 'line_tcp_port', 'http_server_port', 'version', 'http', "auth")) host, line_tcp_port, http_server_port = args.existing.split(':') QDB_FIXTURE = MockFixture( host, int(line_tcp_port), int(http_server_port), (999, 999, 999), - True) + True, + False) unittest.main() @@ -831,11 +867,11 @@ def iter_versions(args): if versions_args: versions = { version: ( - 'https://github.com/questdb/questdb/releases/download/' + - version + - '/questdb-' + - version + - '-no-jre-bin.tar.gz') + 'https://github.com/questdb/questdb/releases/download/' + + version + + '/questdb-' + + version + + '-no-jre-bin.tar.gz') for version in versions_args} else: last_n = getattr(args, 'last_n', None) or 1 @@ -857,7 +893,8 @@ def run_with_fixtures(args): for auth in (False, True): for http in (False, True): for build_mode in list(qls.BuildMode): - print(f'Running tests [questdb_dir={questdb_dir}, auth={auth}, http={http}, build_mode={build_mode}]') + print( + f'Running tests [questdb_dir={questdb_dir}, auth={auth}, http={http}, build_mode={build_mode}]') if http and last_version <= (7, 3, 7): print('Skipping ILP/HTTP tests for versions <= 7.3.7') continue From 2b927ae0c8c13da95064556a19ba40b7adf9f422 Mon Sep 17 00:00:00 2001 From: victorgao Date: Mon, 5 May 2025 11:07:21 +0800 Subject: [PATCH 038/119] support https --- questdb-rs/src/ingress/mod.rs | 3 ++- system_test/questdb_line_sender.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 14ef5374..90e378db 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -2646,7 +2646,8 @@ impl SenderBuilder { if !*http_config.disable_line_proto_validation.deref() { if let ProtocolHandler::Http(http_state) = &handler { let settings_url = &format!( - "http://{}:{}/settings", + "{}://{}:{}/settings", + self.protocol.schema(), self.host.deref(), self.port.deref() ); diff --git a/system_test/questdb_line_sender.py b/system_test/questdb_line_sender.py index 38416a4c..437aa48d 100644 --- a/system_test/questdb_line_sender.py +++ b/system_test/questdb_line_sender.py @@ -90,6 +90,12 @@ class CertificateAuthority(Enum): WEBPKI_AND_OS_ROOTS = (c_line_sender_ca(2), 'webpki_and_os_roots') PEM_FILE = (c_line_sender_ca(3), 'pem_file') +c_line_protocol_version = ctypes.c_int + +class ProtocolVersion(Enum): + TCP = (c_line_protocol_version(0), 'v1') + TCPS = (c_line_protocol_version(1), 'v2') + class c_line_sender_opts(ctypes.Structure): pass @@ -184,6 +190,10 @@ def set_sig(fn, restype, *argtypes): dll.line_sender_buffer_with_max_name_len, c_line_sender_buffer_p, c_size_t) + set_sig( + dll.line_sender_buffer_set_line_protocol_version, + c_line_protocol_version, + c_line_sender_error_p_p) set_sig( dll.line_sender_buffer_free, None, @@ -335,6 +345,11 @@ def set_sig(fn, restype, *argtypes): c_line_sender_opts_p, c_line_sender_utf8, c_line_sender_error_p_p) + set_sig( + dll.line_sender_opts_disable_line_protocol_validation, + c_bool, + c_line_sender_opts_p, + c_line_sender_error_p_p) set_sig( dll.line_sender_opts_auth_timeout, c_bool, @@ -405,6 +420,10 @@ def set_sig(fn, restype, *argtypes): dll.line_sender_from_env, c_line_sender_p, c_line_sender_error_p_p) + set_sig( + dll.line_sender_default_line_protocol_version, + c_line_protocol_version, + c_line_sender_p) set_sig( dll.line_sender_must_close, None, From edb40fce4347db2cc200f5d5658baa5bb3ef21e1 Mon Sep 17 00:00:00 2001 From: victorgao Date: Mon, 5 May 2025 18:33:43 +0800 Subject: [PATCH 039/119] code format. --- questdb-rs-ffi/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index d2b47d1d..fb4e8e35 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -1418,7 +1418,9 @@ unsafe fn unwrap_sender_mut<'a>(sender: *mut line_sender) -> &'a mut Sender { /// - Otherwise uses the highest mutually supported version from the intersection /// of client and server compatible versions #[no_mangle] -pub unsafe extern "C" fn line_sender_default_line_protocol_version(sender: *mut line_sender) -> LineProtocolVersion { +pub unsafe extern "C" fn line_sender_default_line_protocol_version( + sender: *mut line_sender, +) -> LineProtocolVersion { unwrap_sender(sender).default_line_protocol_version().into() } From c9f1476cf506cd44cb6fd76c1de5539090696a26 Mon Sep 17 00:00:00 2001 From: victorgao Date: Tue, 6 May 2025 09:54:18 +0800 Subject: [PATCH 040/119] implement NDArrayView for common built-in array, vector and slices. --- questdb-rs/src/ingress/ndarr.rs | 307 ++++++++++++++++++++++++++++++++ 1 file changed, 307 insertions(+) diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 3a324ecd..2d731fde 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -335,6 +335,313 @@ where } } +/// impl NdArrayView for one dimension vector +impl NdArrayView for Vec { + type Iter<'a> + = std::slice::Iter<'a, T> + where + T: 'a; + + fn ndim(&self) -> usize { + 1 + } + + fn dim(&self, idx: usize) -> Option { + (idx == 0).then_some(self.len()) + } + + fn as_slice(&self) -> Option<&[T]> { + Some(self.as_slice()) + } + + fn iter(&self) -> Self::Iter<'_> { + self.as_slice().iter() + } + + fn check_data_buf(&self) -> Result { + Ok(self.len() * std::mem::size_of::()) + } +} + +/// impl NdArrayView for one dimension array +impl NdArrayView for [T; N] { + type Iter<'a> + = std::slice::Iter<'a, T> + where + T: 'a; + + fn ndim(&self) -> usize { + 1 + } + + fn dim(&self, idx: usize) -> Option { + (idx == 0).then_some(N) + } + + fn as_slice(&self) -> Option<&[T]> { + Some(self) + } + + fn iter(&self) -> Self::Iter<'_> { + self.as_slice().iter() + } + + fn check_data_buf(&self) -> Result { + Ok(N * std::mem::size_of::()) + } +} + +/// impl NdArrayView for one dimension slice +impl NdArrayView for &[T] { + type Iter<'a> + = std::slice::Iter<'a, T> + where + Self: 'a, + T: 'a; + + fn ndim(&self) -> usize { + 1 + } + + fn dim(&self, idx: usize) -> Option { + (idx == 0).then_some(self.len()) + } + + fn as_slice(&self) -> Option<&[T]> { + Some(self) + } + + fn iter(&self) -> Self::Iter<'_> { + <[T]>::iter(self) + } + + fn check_data_buf(&self) -> Result { + Ok(std::mem::size_of_val(*self)) + } +} + +/// impl NdArrayView for two dimensions vector +impl NdArrayView for Vec> { + type Iter<'a> + = std::iter::Flatten>> + where + T: 'a; + + fn ndim(&self) -> usize { + 2 + } + + fn dim(&self, idx: usize) -> Option { + match idx { + 0 => Some(self.len()), + 1 => self.first().map(|v| v.len()), + _ => None, + } + } + + fn as_slice(&self) -> Option<&[T]> { + None + } + + fn iter(&self) -> Self::Iter<'_> { + self.as_slice().iter().flatten() + } + + fn check_data_buf(&self) -> Result { + let row_len = self.first().map_or(0, |v| v.len()); + if self.as_slice().iter().any(|v| v.len() != row_len) { + return Err(error::fmt!(ArrayViewError, "Irregular array shape")); + } + Ok(self.len() * row_len * std::mem::size_of::()) + } +} + +/// impl NdArrayView for two dimensions array +impl NdArrayView for [[T; M]; N] { + type Iter<'a> + = std::iter::Flatten> + where + T: 'a; + + fn ndim(&self) -> usize { + 2 + } + + fn dim(&self, idx: usize) -> Option { + match idx { + 0 => Some(N), + 1 => Some(M), + _ => None, + } + } + + fn as_slice(&self) -> Option<&[T]> { + Some(unsafe { std::slice::from_raw_parts(self.as_ptr() as *const T, N * M) }) + } + + fn iter(&self) -> Self::Iter<'_> { + self.as_slice().iter().flatten() + } + + fn check_data_buf(&self) -> Result { + Ok(N * M * std::mem::size_of::()) + } +} + +/// impl NdArrayView for two dimensions slices +impl NdArrayView for &[[T; M]] { + type Iter<'a> + = std::iter::Flatten> + where + Self: 'a, + T: 'a; + + fn ndim(&self) -> usize { + 2 + } + + fn dim(&self, idx: usize) -> Option { + match idx { + 0 => Some(self.len()), + 1 => Some(M), + _ => None, + } + } + + fn as_slice(&self) -> Option<&[T]> { + Some(unsafe { std::slice::from_raw_parts(self.as_ptr() as *const T, self.len() * M) }) + } + + fn iter(&self) -> Self::Iter<'_> { + <[[T; M]]>::iter(self).flatten() + } + + fn check_data_buf(&self) -> Result { + Ok(self.len() * M * std::mem::size_of::()) + } +} + +/// impl NdArrayView for three dimensions vector +impl NdArrayView for Vec>> { + type Iter<'a> + = std::iter::Flatten>>>> + where + T: 'a; + + fn ndim(&self) -> usize { + 3 + } + + fn dim(&self, idx: usize) -> Option { + match idx { + 0 => Some(self.len()), + 1 => self.first().map(|v| v.len()), + 2 => self.first().and_then(|v2| v2.first()).map(|v3| v3.len()), + _ => None, + } + } + + fn as_slice(&self) -> Option<&[T]> { + None + } + + fn iter(&self) -> Self::Iter<'_> { + self.as_slice().iter().flatten().flatten() + } + + fn check_data_buf(&self) -> Result { + let dim1 = self.first().map_or(0, |v| v.len()); + + if self.as_slice().iter().any(|v2| v2.len() != dim1) { + return Err(error::fmt!(ArrayViewError, "Irregular array shape")); + } + + let dim2 = self + .first() + .and_then(|v2| v2.first()) + .map_or(0, |v3| v3.len()); + + if self + .as_slice() + .iter() + .flat_map(|v2| v2.as_slice().iter()) + .any(|v3| v3.len() != dim2) + { + return Err(error::fmt!(ArrayViewError, "Irregular array shape")); + } + + Ok(self.len() * dim1 * dim2 * std::mem::size_of::()) + } +} + +/// impl NdArrayView for three dimensions array +impl NdArrayView + for [[[T; M]; N]; L] +{ + type Iter<'a> + = std::iter::Flatten>> + where + T: 'a; + + fn ndim(&self) -> usize { + 3 + } + + fn dim(&self, idx: usize) -> Option { + match idx { + 0 => Some(L), + 1 => Some(N), + 2 => Some(M), + _ => None, + } + } + + fn as_slice(&self) -> Option<&[T]> { + Some(unsafe { std::slice::from_raw_parts(self.as_ptr() as *const T, L * N * M) }) + } + + fn iter(&self) -> Self::Iter<'_> { + self.as_slice().iter().flatten().flatten() + } + + fn check_data_buf(&self) -> Result { + Ok(L * N * M * std::mem::size_of::()) + } +} + +impl NdArrayView for &[[[T; M]; N]] { + type Iter<'a> + = std::iter::Flatten>> + where + Self: 'a, + T: 'a; + + fn ndim(&self) -> usize { + 3 + } + + fn dim(&self, idx: usize) -> Option { + match idx { + 0 => Some(self.len()), + 1 => Some(N), + 2 => Some(M), + _ => None, + } + } + + fn as_slice(&self) -> Option<&[T]> { + Some(unsafe { std::slice::from_raw_parts(self.as_ptr() as *const T, self.len() * N * M) }) + } + + fn iter(&self) -> Self::Iter<'_> { + <[[[T; M]; N]]>::iter(self).flatten().flatten() + } + + fn check_data_buf(&self) -> Result { + Ok(self.len() * N * M * std::mem::size_of::()) + } +} + use crate::{error, Error}; #[cfg(feature = "ndarray")] use ndarray::{ArrayView, Axis, Dimension}; From a88311208408aee072f01b76cc16280fad8e00d7 Mon Sep 17 00:00:00 2001 From: victorgao Date: Tue, 6 May 2025 10:11:32 +0800 Subject: [PATCH 041/119] code format --- questdb-rs/src/tests/http.rs | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/questdb-rs/src/tests/http.rs b/questdb-rs/src/tests/http.rs index c2079450..79070d09 100644 --- a/questdb-rs/src/tests/http.rs +++ b/questdb-rs/src/tests/http.rs @@ -48,7 +48,7 @@ fn test_two_lines( .at_now()?; let buffer2 = buffer.clone(); - let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); let sender_builder = server.lsb_http(); let server_thread = std::thread::spawn(move || -> io::Result<()> { @@ -100,7 +100,7 @@ fn test_text_plain_error( .at_now()?; buffer.table("test")?.column_f64("sym", 2.0)?.at_now()?; - let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); let sender_builder = server.lsb_http(); let buffer2 = buffer.clone(); @@ -157,7 +157,7 @@ fn test_bad_json_error( .at_now()?; buffer.table("test")?.column_f64("sym", 2.0)?.at_now()?; - let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); let sender_builder = server.lsb_http(); let buffer2 = buffer.clone(); @@ -216,7 +216,7 @@ fn test_json_error( .at_now()?; buffer.table("test")?.column_f64("sym", 2.0)?.at_now()?; - let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); let sender_builder = server.lsb_http(); let buffer2 = buffer.clone(); @@ -300,7 +300,7 @@ fn test_old_server_without_ilp_http_support( .column_f64("x", 1.0)? .at_now()?; - let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); let sender_builder = server.lsb_http(); let buffer2 = buffer.clone(); @@ -356,7 +356,7 @@ fn test_http_basic_auth( .column_f64("x", 1.0)? .at_now()?; - let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); let sender_builder = server .lsb_http() .username("Aladdin")? @@ -411,7 +411,7 @@ fn test_unauthenticated( .column_f64("x", 1.0)? .at_now()?; - let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); let sender_builder = server.lsb_http(); let buffer2 = buffer.clone(); @@ -469,7 +469,7 @@ fn test_token_auth( .column_f64("x", 1.0)? .at_now()?; - let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); let sender_builder = server.lsb_http().token("0123456789")?; let buffer2 = buffer.clone(); @@ -516,7 +516,7 @@ fn test_request_timeout( .at_now()?; // Here we use a mock (tcp) server instead and don't send a response back. - let server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let server = MockServer::new()?.configure_settings_response(2, &[1, 2]); let request_timeout = Duration::from_millis(50); let time_start = std::time::Instant::now(); @@ -550,7 +550,7 @@ fn test_tls( .at(TimestampNanos::new(10000000))?; let buffer2 = buffer.clone(); - let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); let mut sender = server .lsb_https() .tls_roots(ca_path)? @@ -591,7 +591,7 @@ fn test_user_agent( .at(TimestampNanos::new(10000000))?; let buffer2 = buffer.clone(); - let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); let sender_builder = server.lsb_http().user_agent("wallabies/1.2.99")?; let server_thread = std::thread::spawn(move || -> io::Result<()> { @@ -634,7 +634,7 @@ fn test_two_retries( .at(TimestampNanos::new(10000000))?; let buffer2 = buffer.clone(); - let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); let sender_builder = server.lsb_http().retry_timeout(Duration::from_secs(30))?; let server_thread = std::thread::spawn(move || -> io::Result<()> { @@ -704,7 +704,7 @@ fn test_one_retry( .at(TimestampNanos::new(10000000))?; let buffer2 = buffer.clone(); - let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); let mut sender = server .lsb_http() .retry_timeout(Duration::from_millis(19))? @@ -787,7 +787,7 @@ fn test_transactional( let buffer3 = buffer2.clone(); assert!(buffer2.transactional()); - let mut server = MockServer::new()?.configure_settings_response(2, &vec![1, 2]); + let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); let sender_builder = server.lsb_http(); let server_thread = std::thread::spawn(move || -> io::Result<()> { From f3e8681db5800ba8f8a0ba345e69931e42737c1b Mon Sep 17 00:00:00 2001 From: victorgao Date: Tue, 6 May 2025 12:03:03 +0800 Subject: [PATCH 042/119] add tests for build-in array,vector and slice. --- questdb-rs/examples/http.rs | 3 +- questdb-rs/examples/line_protocol_version.rs | 3 +- questdb-rs/src/tests/ndarr.rs | 728 +++++++++++++++++++ 3 files changed, 732 insertions(+), 2 deletions(-) diff --git a/questdb-rs/examples/http.rs b/questdb-rs/examples/http.rs index 3e11dd0f..ead1efde 100644 --- a/questdb-rs/examples/http.rs +++ b/questdb-rs/examples/http.rs @@ -6,7 +6,8 @@ use questdb::{ fn main() -> Result<()> { let mut sender = Sender::from_conf("https::addr=localhost:9000;username=foo;password=bar;")?; - let mut buffer = Buffer::new(); + let mut buffer = + Buffer::new().with_line_proto_version(sender.default_line_protocol_version())?; buffer .table("trades")? .symbol("symbol", "ETH-USD")? diff --git a/questdb-rs/examples/line_protocol_version.rs b/questdb-rs/examples/line_protocol_version.rs index d9944991..656db6b8 100644 --- a/questdb-rs/examples/line_protocol_version.rs +++ b/questdb-rs/examples/line_protocol_version.rs @@ -18,7 +18,8 @@ fn main() -> Result<()> { sender.flush(&mut buffer)?; let mut sender2 = Sender::from_conf("https::addr=localhost:9000;username=foo;password=bar;")?; - let mut buffer2 = Buffer::new().with_line_proto_version(LineProtocolVersion::V1)?; + let mut buffer2 = + Buffer::new().with_line_proto_version(sender2.default_line_protocol_version())?; buffer2 .table("trades_ilp_v2")? .symbol("symbol", "ETH-USD")? diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index 56417d10..4ead055e 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -264,6 +264,734 @@ fn test_array_length_mismatch() -> TestResult { Ok(()) } +#[test] +fn test_build_in_1d_array_normal() -> TestResult { + let arr = [1.0f64, 2.0, 3.0, 4.0]; + assert_eq!(arr.ndim(), 1); + assert_eq!(arr.dim(0), Some(4)); + assert_eq!(arr.dim(1), None); + assert_eq!(NdArrayView::as_slice(&arr), Some(&[1.0, 2.0, 3.0, 4.0][..])); + let collected: Vec<_> = NdArrayView::iter(&arr).copied().collect(); + assert_eq!(collected, vec![1.0, 2.0, 3.0, 4.0]); + assert_eq!(arr.check_data_buf(), Ok(32)); + + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + buffer.column_arr("temperature", &arr)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); + assert_eq!( + &data[19..24], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ElemDataType::Double.into(), + 1u8 + ] + ); + assert_eq!(&data[24..28], [4i32.to_le_bytes()].concat()); + assert_eq!( + &data[28..60], + &[ + 1.0f64.to_ne_bytes(), + 2.0f64.to_le_bytes(), + 3.0f64.to_le_bytes(), + 4.0f64.to_le_bytes(), + ] + .concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_1d_array_empty() -> TestResult { + let arr: [f64; 0] = []; + assert_eq!(arr.ndim(), 1); + assert_eq!(arr.dim(0), Some(0)); + assert_eq!(NdArrayView::as_slice(&arr), Some(&[][..])); + assert_eq!(arr.check_data_buf(), Ok(0)); + + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + buffer.column_arr("temperature", &arr)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); + assert_eq!( + &data[19..24], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ElemDataType::Double.into(), + 1u8 + ] + ); + assert_eq!(&data[24..28], [0i32.to_le_bytes()].concat()); + Ok(()) +} + +#[test] +fn test_build_in_1d_vec_normal() -> TestResult { + let vec = vec![5.0f64, 6.0, 7.0]; + assert_eq!(vec.ndim(), 1); + assert_eq!(vec.dim(0), Some(3)); + assert_eq!(NdArrayView::as_slice(&vec), Some(&[5.0, 6.0, 7.0][..])); + let collected: Vec<_> = NdArrayView::iter(&vec).copied().collect(); + assert_eq!(collected, vec![5.0, 6.0, 7.0]); + assert_eq!(vec.check_data_buf(), Ok(24)); + + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + buffer.column_arr("temperature", &vec)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); + assert_eq!( + &data[19..24], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ElemDataType::Double.into(), + 1u8 + ] + ); + assert_eq!(&data[24..28], [3i32.to_le_bytes()].concat()); + assert_eq!( + &data[28..52], + &[ + 5.0f64.to_le_bytes(), + 6.0f64.to_le_bytes(), + 7.0f64.to_le_bytes(), + ] + .concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_1d_vec_empty() -> TestResult { + let vec: Vec = Vec::new(); + assert_eq!(vec.ndim(), 1); + assert_eq!(vec.dim(0), Some(0)); + assert_eq!(NdArrayView::as_slice(&vec), Some(&[][..])); + assert_eq!(vec.check_data_buf(), Ok(0)); + + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + buffer.column_arr("temperature", &vec)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); + assert_eq!( + &data[19..24], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ElemDataType::Double.into(), + 1u8 + ] + ); + assert_eq!(&data[24..28], [0i32.to_le_bytes()].concat()); + Ok(()) +} + +#[test] +fn test_build_in_1d_slice_normal() -> TestResult { + let data = [10.0f64, 20.0, 30.0, 40.0]; + let slice = &data[1..3]; + assert_eq!(slice.ndim(), 1); + assert_eq!(slice.dim(0), Some(2)); + assert_eq!(NdArrayView::as_slice(&slice), Some(&[20.0, 30.0][..])); + assert_eq!(slice.check_data_buf(), Ok(16)); + + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + buffer.column_arr("temperature", &slice)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); + assert_eq!( + &data[19..24], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ElemDataType::Double.into(), + 1u8 + ] + ); + assert_eq!(&data[24..28], [2i32.to_le_bytes()].concat()); + assert_eq!( + &data[28..44], + &[20.0f64.to_le_bytes(), 30.0f64.to_le_bytes(),].concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_1d_slice_empty() -> TestResult { + let data = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; + let slice = &data[2..2]; + assert_eq!(slice.ndim(), 1); + assert_eq!(slice.dim(0), Some(0)); + assert_eq!(NdArrayView::as_slice(&slice), Some(&[][..])); + assert_eq!(slice.check_data_buf(), Ok(0)); + + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + buffer.column_arr("temperature", &slice)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); + assert_eq!( + &data[19..24], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ElemDataType::Double.into(), + 1u8 + ] + ); + assert_eq!(&data[24..28], [0i32.to_le_bytes()].concat()); + Ok(()) +} + +#[test] +fn test_build_in_2d_array_normal() -> TestResult { + let arr = [[1.0f64, 2.0], [3.0, 4.0], [5.0, 6.0]]; + assert_eq!(arr.ndim(), 2); + assert_eq!(arr.dim(0), Some(3)); + assert_eq!(arr.dim(1), Some(2)); + assert_eq!( + NdArrayView::as_slice(&arr), + Some(&[1.0, 2.0, 3.0, 4.0, 5.0, 6.0][..]) + ); + let collected: Vec<_> = NdArrayView::iter(&arr).copied().collect(); + assert_eq!(collected, vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]); + assert_eq!(arr.check_data_buf(), Ok(48)); + + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + buffer.column_arr("2darray", &arr)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..15], b"2darray"); + assert_eq!( + &data[15..20], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ElemDataType::Double.into(), + 2u8 + ] + ); + assert_eq!( + &data[20..28], + [3i32.to_le_bytes(), 2i32.to_le_bytes()].concat() + ); + assert_eq!( + &data[28..76], + &[ + 1.0f64.to_le_bytes(), + 2.0f64.to_le_bytes(), + 3.0f64.to_le_bytes(), + 4.0f64.to_le_bytes(), + 5.0f64.to_le_bytes(), + 6.0f64.to_le_bytes(), + ] + .concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_2d_array_empty() -> TestResult { + let arr: [[f64; 0]; 0] = []; + assert_eq!(arr.ndim(), 2); + assert_eq!(arr.dim(0), Some(0)); + assert_eq!(arr.dim(1), Some(0)); + assert_eq!(NdArrayView::as_slice(&arr), Some(&[][..])); + assert_eq!(arr.check_data_buf(), Ok(0)); + + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + buffer.column_arr("2darray", &arr)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..15], b"2darray"); + assert_eq!( + &data[15..20], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ElemDataType::Double.into(), + 2u8 + ] + ); + assert_eq!( + &data[20..28], + [0i32.to_le_bytes(), 0i32.to_le_bytes()].concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_2d_vec_normal() -> TestResult { + let vec = vec![vec![1.0f64, 2.0], vec![3.0, 4.0], vec![5.0, 6.0]]; + assert_eq!(vec.ndim(), 2); + assert_eq!(vec.dim(0), Some(3)); + assert_eq!(vec.dim(1), Some(2)); + assert!(NdArrayView::as_slice(&vec).is_none()); + let collected: Vec<_> = NdArrayView::iter(&vec).copied().collect(); + assert_eq!(collected, vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]); + assert_eq!(vec.check_data_buf(), Ok(48)); + + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + buffer.column_arr("2darray", &vec)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..15], b"2darray"); + assert_eq!( + &data[15..20], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ElemDataType::Double.into(), + 2u8 + ] + ); + assert_eq!( + &data[20..28], + [3i32.to_le_bytes(), 2i32.to_le_bytes()].concat() + ); + assert_eq!( + &data[28..76], + &[ + 1.0f64.to_le_bytes(), + 2.0f64.to_le_bytes(), + 3.0f64.to_le_bytes(), + 4.0f64.to_le_bytes(), + 5.0f64.to_le_bytes(), + 6.0f64.to_le_bytes(), + ] + .concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_2d_vec_irregular_shape() -> TestResult { + let irregular_vec = vec![vec![1.0, 2.0], vec![3.0], vec![4.0, 5.0]]; + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + let result = buffer.column_arr("arr", &irregular_vec); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert!(err.msg().contains("Irregular array shape")); + Ok(()) +} + +#[test] +fn test_build_in_2d_vec_empty() -> TestResult { + let vec: Vec> = vec![vec![], vec![], vec![]]; + assert_eq!(vec.ndim(), 2); + assert_eq!(vec.dim(0), Some(3)); + assert_eq!(vec.dim(1), Some(0)); + assert_eq!(vec.check_data_buf(), Ok(0)); + + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + buffer.column_arr("2darray", &vec)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..15], b"2darray"); + assert_eq!( + &data[15..20], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ElemDataType::Double.into(), + 2u8 + ] + ); + assert_eq!( + &data[20..28], + [3i32.to_le_bytes(), 0i32.to_le_bytes()].concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_2d_slice_normal() -> TestResult { + let data = [[1.0f64, 2.0], [3.0, 4.0], [5.0, 6.0]]; + let slice = &data[..2]; + assert_eq!(slice.ndim(), 2); + assert_eq!(slice.dim(0), Some(2)); + assert_eq!(slice.dim(1), Some(2)); + assert_eq!( + NdArrayView::as_slice(&slice), + Some(&[1.0, 2.0, 3.0, 4.0][..]) + ); + assert_eq!(slice.check_data_buf(), Ok(32)); + + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + buffer.column_arr("2darray", &slice)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..15], b"2darray"); + assert_eq!( + &data[15..20], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ElemDataType::Double.into(), + 2u8 + ] + ); + assert_eq!( + &data[20..28], + [2i32.to_le_bytes(), 2i32.to_le_bytes()].concat() + ); + assert_eq!( + &data[28..60], + &[ + 1.0f64.to_le_bytes(), + 2.0f64.to_le_bytes(), + 3.0f64.to_le_bytes(), + 4.0f64.to_le_bytes(), + ] + .concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_2d_slice_empty() -> TestResult { + let data = [[1.0f64, 2.0], [3.0, 4.0], [5.0, 6.0]]; + let slice = &data[2..2]; + assert_eq!(slice.ndim(), 2); + assert_eq!(slice.dim(0), Some(0)); + assert_eq!(slice.dim(1), Some(2)); + assert_eq!(NdArrayView::as_slice(&slice), Some(&[][..])); + assert_eq!(slice.check_data_buf(), Ok(0)); + + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + buffer.column_arr("2darray", &slice)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..15], b"2darray"); + assert_eq!( + &data[15..20], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ElemDataType::Double.into(), + 2u8 + ] + ); + assert_eq!( + &data[20..28], + [0i32.to_le_bytes(), 2i32.to_le_bytes()].concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_3d_array_normal() -> TestResult { + let arr = [[[1.0f64, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]]; + assert_eq!(arr.ndim(), 3); + assert_eq!(arr.dim(0), Some(2)); + assert_eq!(arr.dim(1), Some(2)); + assert_eq!(arr.dim(2), Some(2)); + assert_eq!( + NdArrayView::as_slice(&arr), + Some(&[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0][..]) + ); + let collected: Vec<_> = NdArrayView::iter(&arr).copied().collect(); + assert_eq!(collected, vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]); + assert_eq!(arr.check_data_buf(), Ok(64)); + + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + buffer.column_arr("3darray", &arr)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..15], b"3darray"); + assert_eq!( + &data[15..20], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ElemDataType::Double.into(), + 3u8 + ] + ); + assert_eq!( + &data[20..32], + [2i32.to_le_bytes(), 2i32.to_le_bytes(), 2i32.to_le_bytes()].concat() + ); + assert_eq!( + &data[32..96], + &[ + 1.0f64.to_le_bytes(), + 2.0f64.to_le_bytes(), + 3.0f64.to_le_bytes(), + 4.0f64.to_le_bytes(), + 5.0f64.to_le_bytes(), + 6.0f64.to_le_bytes(), + 7.0f64.to_le_bytes(), + 8.0f64.to_le_bytes() + ] + .concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_3d_array_empty() -> TestResult { + let arr: [[[f64; 2]; 0]; 0] = []; + assert_eq!(arr.ndim(), 3); + assert_eq!(arr.dim(0), Some(0)); + assert_eq!(arr.dim(1), Some(0)); + assert_eq!(arr.dim(2), Some(2)); + assert_eq!(NdArrayView::as_slice(&arr), Some(&[][..])); + assert_eq!(arr.check_data_buf(), Ok(0)); + + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + buffer.column_arr("3darray", &arr)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..15], b"3darray"); + assert_eq!( + &data[15..20], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ElemDataType::Double.into(), + 3u8 + ] + ); + assert_eq!( + &data[20..32], + [0i32.to_le_bytes(), 0i32.to_le_bytes(), 2i32.to_le_bytes()].concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_3d_vec_normal() -> TestResult { + let vec = vec![ + vec![vec![1.0, 2.0, 3.0], vec![4.0, 5.0, 6.0]], + vec![vec![7.0, 8.0, 9.0], vec![10.0, 11.0, 12.0]], + ]; + assert_eq!(vec.ndim(), 3); + assert_eq!(vec.dim(0), Some(2)); + assert_eq!(vec.dim(1), Some(2)); + assert_eq!(vec.dim(2), Some(3)); + assert!(NdArrayView::as_slice(&vec).is_none()); + let collected: Vec<_> = NdArrayView::iter(&vec).copied().collect(); + assert_eq!( + collected, + vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0] + ); + assert_eq!(vec.check_data_buf(), Ok(96)); + + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + buffer.column_arr("3darray", &vec)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..15], b"3darray"); + assert_eq!( + &data[15..20], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ElemDataType::Double.into(), + 3u8 + ] + ); + assert_eq!( + &data[20..32], + [2i32.to_le_bytes(), 2i32.to_le_bytes(), 3i32.to_le_bytes()].concat() + ); + assert_eq!( + &data[32..128], + &[ + 1.0f64.to_le_bytes(), + 2.0f64.to_le_bytes(), + 3.0f64.to_le_bytes(), + 4.0f64.to_le_bytes(), + 5.0f64.to_le_bytes(), + 6.0f64.to_le_bytes(), + 7.0f64.to_le_bytes(), + 8.0f64.to_le_bytes(), + 9.0f64.to_le_bytes(), + 10.0f64.to_le_bytes(), + 11.0f64.to_le_bytes(), + 12.0f64.to_le_bytes(), + ] + .concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_3d_vec_empty() -> TestResult { + let vec: Vec>> = vec![vec![vec![], vec![]], vec![vec![], vec![]]]; + assert_eq!(vec.ndim(), 3); + assert_eq!(vec.dim(0), Some(2)); + assert_eq!(vec.dim(1), Some(2)); + assert_eq!(vec.dim(2), Some(0)); + assert!(NdArrayView::as_slice(&vec).is_none()); + assert_eq!(vec.check_data_buf(), Ok(0)); + + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + buffer.column_arr("3darray", &vec)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..15], b"3darray"); + assert_eq!( + &data[15..20], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ElemDataType::Double.into(), + 3u8 + ] + ); + assert_eq!( + &data[20..32], + [2i32.to_le_bytes(), 2i32.to_le_bytes(), 0i32.to_le_bytes()].concat() + ); + Ok(()) +} + +#[test] +fn test_build_in_3d_vec_irregular_shape() -> TestResult { + let irregular1 = vec![vec![vec![1.0, 2.0], vec![3.0, 4.0]], vec![vec![5.0, 6.0]]]; + + let irregular2 = vec![ + vec![vec![1.0, 2.0], vec![3.0, 4.0, 5.0]], + vec![vec![6.0, 7.0], vec![8.0, 9.0]], + ]; + + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + let result = buffer.column_arr("arr", &irregular1); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert!(err.msg().contains("Irregular array shape")); + + let result = buffer.column_arr("arr", &irregular2); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert!(err.msg().contains("Irregular array shape")); + Ok(()) +} + +#[test] +fn test_3d_slice_normal() -> TestResult { + let data = [[[1f64, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]]; + let slice = &data[..1]; + assert_eq!(slice.ndim(), 3); + assert_eq!(slice.dim(0), Some(1)); + assert_eq!(slice.dim(1), Some(2)); + assert_eq!(slice.dim(2), Some(2)); + assert_eq!( + NdArrayView::as_slice(&slice), + Some(&[1.0, 2.0, 3.0, 4.0][..]) + ); + assert_eq!(slice.check_data_buf(), Ok(32)); + + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + buffer.column_arr("3darray", &slice)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..15], b"3darray"); + assert_eq!( + &data[15..20], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ElemDataType::Double.into(), + 3u8 + ] + ); + assert_eq!( + &data[20..32], + [1i32.to_le_bytes(), 2i32.to_le_bytes(), 2i32.to_le_bytes()].concat() + ); + assert_eq!( + &data[32..64], + &[ + 1.0f64.to_le_bytes(), + 2.0f64.to_le_bytes(), + 3.0f64.to_le_bytes(), + 4.0f64.to_le_bytes(), + ] + .concat() + ); + Ok(()) +} + +#[test] +fn test_3d_slice_empty() -> TestResult { + let data = [[[1f64, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]]; + let slice = &data[1..1]; + assert_eq!(slice.ndim(), 3); + assert_eq!(slice.dim(0), Some(0)); + assert_eq!(slice.dim(1), Some(2)); + assert_eq!(slice.dim(2), Some(2)); + assert_eq!(NdArrayView::as_slice(&slice), Some(&[][..])); + assert_eq!(slice.check_data_buf(), Ok(0)); + + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + buffer.column_arr("3darray", &slice)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..15], b"3darray"); + assert_eq!( + &data[15..20], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ElemDataType::Double.into(), + 3u8 + ] + ); + assert_eq!( + &data[20..32], + [0i32.to_le_bytes(), 2i32.to_le_bytes(), 2i32.to_le_bytes()].concat() + ); + Ok(()) +} + #[cfg(feature = "ndarray")] #[test] fn test_1d_contiguous_ndarray_buffer() -> TestResult { From 1ee3b5827143b89126a39fe0873087903bb41ab0 Mon Sep 17 00:00:00 2001 From: victorgao Date: Tue, 6 May 2025 16:25:56 +0800 Subject: [PATCH 043/119] add line protocol version tests for rust client. --- questdb-rs/src/ingress/mod.rs | 12 +++ questdb-rs/src/tests/http.rs | 142 ++++++++++++++++++++++++++++++++++ questdb-rs/src/tests/mock.rs | 12 ++- 3 files changed, 162 insertions(+), 4 deletions(-) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 90e378db..164aedbd 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -1118,6 +1118,12 @@ impl Buffer { D: ArrayElement, Error: From, { + if self.version == LineProtocolVersion::V1 { + return Err(error::fmt!( + LineProtocolVersionError, + "line protocol version v1 does not support array datatype", + )); + } self.write_column_key(name)?; // check dimension less equal than max dims @@ -3039,6 +3045,12 @@ impl Sender { self.default_line_protocol_version } + #[cfg(feature = "ilp-over-http")] + #[cfg(test)] + pub(crate) fn support_line_protocol_versions(&self) -> Option> { + self.supported_line_protocol_versions.clone() + } + #[cfg(feature = "ilp-over-http")] #[inline(always)] fn check_line_protocol_version(&self, version: LineProtocolVersion) -> Result<()> { diff --git a/questdb-rs/src/tests/http.rs b/questdb-rs/src/tests/http.rs index 79070d09..d1d2c719 100644 --- a/questdb-rs/src/tests/http.rs +++ b/questdb-rs/src/tests/http.rs @@ -830,3 +830,145 @@ fn test_transactional( Ok(()) } + +#[test] +fn test_sender_line_protocol_version() -> TestResult { + let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); + let sender_builder = server.lsb_http(); + let server_thread = std::thread::spawn(move || -> io::Result<()> { + server.accept()?; + let req = server.recv_http_q()?; + assert_eq!(req.method(), "GET"); + assert_eq!(req.path(), "/settings"); + assert_eq!( + req.header("user-agent"), + Some(concat!("questdb/rust/", env!("CARGO_PKG_VERSION"))) + ); + server.send_settings_response()?; + Ok(()) + }); + let sender = sender_builder.build()?; + assert_eq!( + sender.default_line_protocol_version(), + LineProtocolVersion::V2 + ); + assert_eq!( + sender.support_line_protocol_versions().unwrap(), + vec![LineProtocolVersion::V1, LineProtocolVersion::V2] + ); + server_thread.join().unwrap()?; + Ok(()) +} + +#[test] +fn test_sender_line_protocol_version_old_server1() -> TestResult { + let mut server = MockServer::new()?.configure_settings_response(0, &[1, 2]); + let sender_builder = server.lsb_http(); + let server_thread = std::thread::spawn(move || -> io::Result<()> { + server.accept()?; + let req = server.recv_http_q()?; + assert_eq!(req.method(), "GET"); + assert_eq!(req.path(), "/settings"); + assert_eq!( + req.header("user-agent"), + Some(concat!("questdb/rust/", env!("CARGO_PKG_VERSION"))) + ); + server.send_settings_response()?; + Ok(()) + }); + let sender = sender_builder.build()?; + assert_eq!( + sender.default_line_protocol_version(), + LineProtocolVersion::V1 + ); + assert!(sender.support_line_protocol_versions().is_none()); + server_thread.join().unwrap()?; + Ok(()) +} + +#[test] +fn test_sender_line_protocol_version_old_server2() -> TestResult { + let mut server = MockServer::new()?.configure_settings_response(0, &[1, 2]); + let sender_builder = server.lsb_http(); + let server_thread = std::thread::spawn(move || -> io::Result<()> { + server.accept()?; + server.send_http_response_q( + HttpResponse::empty() + .with_status(404, "Not Found") + .with_header("content-type", "text/plain") + .with_body_str("Not Found"), + )?; + Ok(()) + }); + let sender = sender_builder.build()?; + assert_eq!( + sender.default_line_protocol_version(), + LineProtocolVersion::V1 + ); + assert!(sender.support_line_protocol_versions().is_none()); + server_thread.join().unwrap()?; + Ok(()) +} + +#[test] +fn test_sender_line_protocol_version_unsupported_client() -> TestResult { + let mut server = MockServer::new()?.configure_settings_response(3, &[3, 4]); + let sender_builder = server.lsb_http(); + let server_thread = std::thread::spawn(move || -> io::Result<()> { + server.accept()?; + server.send_settings_response()?; + Ok(()) + }); + let res1 = sender_builder.build(); + assert!(res1.is_err()); + let e1 = res1.err().unwrap(); + assert_eq!(e1.code(), ErrorCode::LineProtocolVersionError); + assert!(e1.msg().contains("Server does not support current client.")); + server_thread.join().unwrap()?; + Ok(()) +} + +#[test] +fn test_sender_disable_line_protocol_version_validation() -> TestResult { + let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); + let mut sender = server + .lsb_http() + .disable_line_protocol_validation()? + .build()?; + let mut buffer = + Buffer::new().with_line_proto_version(sender.default_line_protocol_version())?; + buffer + .table("test")? + .symbol("sym", "bol")? + .column_f64("x", 1.0)? + .at_now()?; + let buffer2 = buffer.clone(); + + let server_thread = std::thread::spawn(move || -> io::Result<()> { + server.accept()?; + let req = server.recv_http_q()?; + assert_eq!(req.body(), buffer2.as_bytes()); + server.send_http_response_q(HttpResponse::empty())?; + Ok(()) + }); + + sender.flush(&mut buffer)?; + server_thread.join().unwrap()?; + Ok(()) +} + +#[test] +fn test_sender_line_protocol_version1_not_support_array() -> TestResult { + let mut buffer = Buffer::new().with_line_proto_version(LineProtocolVersion::V1)?; + let res = buffer + .table("test")? + .symbol("sym", "bol")? + .column_arr("x", &[1.0f64, 2.0]); + assert!(res.is_err()); + let e1 = res.as_ref().err().unwrap(); + assert_eq!(e1.code(), ErrorCode::LineProtocolVersionError); + assert!(e1 + .msg() + .contains("line protocol version v1 does not support array datatype")); + Ok(()) +} diff --git a/questdb-rs/src/tests/mock.rs b/questdb-rs/src/tests/mock.rs index 57e13c8f..ade56414 100644 --- a/questdb-rs/src/tests/mock.rs +++ b/questdb-rs/src/tests/mock.rs @@ -313,10 +313,14 @@ impl MockServer { default_version: u16, supported_versions: &[u16], ) -> Self { - self.settings_response = serde_json::json!({ - "line.proto.default.version": default_version, - "line.proto.support.versions": supported_versions - }); + if default_version == 0 { + self.settings_response = serde_json::json!({"version": "8.1.2"}); + } else { + self.settings_response = serde_json::json!({ + "line.proto.default.version": default_version, + "line.proto.support.versions": supported_versions + }); + } self } From d34c6512eecd57d315c74aca8a3272e18b11c1b7 Mon Sep 17 00:00:00 2001 From: victorgao Date: Tue, 6 May 2025 19:42:55 +0800 Subject: [PATCH 044/119] make line protocol version c++ interface more ideamatic --- examples/line_sender_c_example_http.c | 4 ++++ examples/line_sender_cpp_example_http.cpp | 3 ++- include/questdb/ingress/line_sender.hpp | 15 +++++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/examples/line_sender_c_example_http.c b/examples/line_sender_c_example_http.c index 7d504a7e..382a0bfd 100644 --- a/examples/line_sender_c_example_http.c +++ b/examples/line_sender_c_example_http.c @@ -29,6 +29,10 @@ static bool example(const char* host, const char* port) buffer = line_sender_buffer_new(); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. + line_sender_buffer_set_line_protocol_version( + buffer, line_sender_default_line_protocol_version(sender), &err); + if (!sender) + goto on_error; line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("c_trades_http"); line_sender_column_name symbol_name = QDB_COLUMN_NAME_LITERAL("symbol"); diff --git a/examples/line_sender_cpp_example_http.cpp b/examples/line_sender_cpp_example_http.cpp index 43b4bdd5..6c6b82f9 100644 --- a/examples/line_sender_cpp_example_http.cpp +++ b/examples/line_sender_cpp_example_http.cpp @@ -37,7 +37,8 @@ static bool example(std::string_view host, std::string_view port) 48121.5, 4.3}; - questdb::ingress::line_sender_buffer buffer; + questdb::ingress::line_sender_buffer buffer{ + sender.default_line_protocol_version()}; buffer .table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 34f14cd8..9f7fde58 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -412,6 +412,11 @@ class line_sender_buffer { } + line_sender_buffer(line_protocol_version version) noexcept + : line_sender_buffer{64 * 1024, 127, version} + { + } + line_sender_buffer(const line_sender_buffer& other) noexcept : _impl{::line_sender_buffer_clone(other._impl)} , _init_buf_size{other._init_buf_size} @@ -1361,6 +1366,16 @@ class line_sender } } + /** + * Returns the client's recommended default line protocol version. + */ + line_protocol_version default_line_protocol_version() + { + ensure_impl(); + return line_sender_error::wrapped_call( + ::line_sender_default_line_protocol_version, _impl); + } + /** * Check if an error occurred previously and the sender must be closed. * This happens when there was an earlier failure. From 9c78839451bbb2d95ce984b65c28f8360c2c4855 Mon Sep 17 00:00:00 2001 From: victorgao Date: Tue, 6 May 2025 21:17:52 +0800 Subject: [PATCH 045/119] fix build failure. --- include/questdb/ingress/line_sender.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 9f7fde58..31a5e45e 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -1372,8 +1372,7 @@ class line_sender line_protocol_version default_line_protocol_version() { ensure_impl(); - return line_sender_error::wrapped_call( - ::line_sender_default_line_protocol_version, _impl); + return line_sender_default_line_protocol_version(_impl); } /** From ced8e05c1cba0a1b2d9889b99d9643c9d4a3f43f Mon Sep 17 00:00:00 2001 From: victorgao Date: Tue, 6 May 2025 23:04:20 +0800 Subject: [PATCH 046/119] fix little comment --- include/questdb/ingress/line_sender.h | 2 +- include/questdb/ingress/line_sender.hpp | 2 +- questdb-rs/src/ingress/mod.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 96c2b607..ce4512db 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -914,7 +914,7 @@ LINESENDER_API line_sender* line_sender_from_env(line_sender_error** err_out); /** - * Returns the client's recommended default line protocol version. + * Returns the QuestDB server's recommended default line protocol version. * Will be used to [`line_sender_buffer_set_line_protocol_version`] * * The version selection follows these rules: diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 31a5e45e..99d67afa 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -1367,7 +1367,7 @@ class line_sender } /** - * Returns the client's recommended default line protocol version. + * Returns the QuestDB server's recommended default line protocol version. */ line_protocol_version default_line_protocol_version() { diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 164aedbd..29fff680 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -3030,7 +3030,7 @@ impl Sender { !self.connected } - /// Returns the client's recommended default line protocol version. + /// Returns the QuestDB server's recommended default line protocol version. /// Will be used to [`Buffer::with_line_proto_version`] /// /// The version selection follows these rules: From ef8ab61c8489f0f6adef2183543926780a9408da Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 7 May 2025 13:44:36 +0800 Subject: [PATCH 047/119] add system_test. --- cpp_test/test_line_sender.cpp | 45 +++++++++++---- examples/line_sender_c_example_http.c | 5 +- include/questdb/ingress/line_sender.h | 2 +- questdb-rs-ffi/src/lib.rs | 4 +- questdb-rs/src/ingress/mod.rs | 5 +- system_test/fixture.py | 33 +++++------ system_test/questdb_line_sender.py | 79 ++++++++++++++++++++++----- system_test/test.py | 66 +++++++++++++++++++++- 8 files changed, 187 insertions(+), 52 deletions(-) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index c1cadd22..5d8ad52a 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -176,16 +176,15 @@ TEST_CASE("line_sender c api basics") 2.7, 48121.5, 4.3}; - CHECK( - ::line_sender_buffer_column_f64_arr( - buffer, - arr_name, - rank, - shapes, - strides, - reinterpret_cast(arr_data.data()), - sizeof(arr_data), - &err)); + CHECK(::line_sender_buffer_column_f64_arr( + buffer, + arr_name, + rank, + shapes, + strides, + reinterpret_cast(arr_data.data()), + sizeof(arr_data), + &err)); CHECK(::line_sender_buffer_at_nanos(buffer, 10000000, &err)); CHECK(server.recv() == 0); CHECK(::line_sender_buffer_size(buffer) == 150); @@ -942,4 +941,30 @@ TEST_CASE("HTTP basics") questdb::ingress::opts::from_conf( "http::addr=localhost:1;bind_interface=0.0.0.0;"), questdb::ingress::line_sender_error); +} + +TEST_CASE("line sender protocol version v1") +{ + questdb::ingress::test::mock_server server; + questdb::ingress::line_sender sender{ + questdb::ingress::protocol::tcp, + std::string("localhost"), + std::to_string(server.port())}; + CHECK_FALSE(sender.must_close()); + server.accept(); + CHECK(server.recv() == 0); + + questdb::ingress::line_sender_buffer buffer{line_protocol_version_1}; + buffer.table("test") + .symbol("t1", "v1") + .symbol("t2", "") + .column("f1", 0.5) + .at(questdb::ingress::timestamp_nanos{10000000}); + + CHECK(server.recv() == 0); + CHECK(buffer.size() == 31); + sender.flush(buffer); + CHECK(server.recv() == 1); + std::string expect{"test,t1=v1,t2= f1=0.5 10000000\n"}; + CHECK(server.msgs(0) == expect); } \ No newline at end of file diff --git a/examples/line_sender_c_example_http.c b/examples/line_sender_c_example_http.c index 382a0bfd..5b848d51 100644 --- a/examples/line_sender_c_example_http.c +++ b/examples/line_sender_c_example_http.c @@ -29,9 +29,8 @@ static bool example(const char* host, const char* port) buffer = line_sender_buffer_new(); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. - line_sender_buffer_set_line_protocol_version( - buffer, line_sender_default_line_protocol_version(sender), &err); - if (!sender) + if (!line_sender_buffer_set_line_protocol_version( + buffer, line_sender_default_line_protocol_version(sender), &err)) goto on_error; line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("c_trades_http"); diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index ce4512db..22ec686a 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -320,7 +320,7 @@ line_sender_buffer* line_sender_buffer_with_max_name_len(size_t max_name_len); * be changed after the buffer contains data. */ LINESENDER_API -line_sender_buffer* line_sender_buffer_set_line_protocol_version( +bool line_sender_buffer_set_line_protocol_version( line_sender_buffer* buffer, line_protocol_version version, line_sender_error** err_out); diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index fb4e8e35..35b2d9de 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -233,11 +233,11 @@ impl From for Protocol { pub enum LineProtocolVersion { /// Version 1 of Line Protocol. /// Uses text format serialization for f64. - V1, + V1 = 1, /// Version 2 of InfluxDB Line Protocol. /// Uses binary format serialization for f64, and support array data type. - V2, + V2 = 2, } impl From for ingress::LineProtocolVersion { diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 29fff680..435c3747 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -2347,8 +2347,9 @@ impl SenderBuilder { /// explicitly set protocol version exactly matches server expectation. pub fn disable_line_protocol_validation(mut self) -> Result { if let Some(http) = &mut self.http { - http.disable_line_proto_validation - .set_specified("disable_line_protocol_validation", true)?; + // ignore "already specified" error + let _ = http.disable_line_proto_validation + .set_specified("disable_line_protocol_validation", true); } Ok(self) } diff --git a/system_test/fixture.py b/system_test/fixture.py index 19d6a50e..c992127a 100644 --- a/system_test/fixture.py +++ b/system_test/fixture.py @@ -23,12 +23,12 @@ ################################################################################ import sys + sys.dont_write_bytecode = True import os import re import pathlib -import textwrap import json import tarfile import shutil @@ -42,11 +42,9 @@ import urllib.error from pprint import pformat - AUTH_TXT = """admin ec-p-256-sha256 fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac # [key/user id] [key type] {keyX keyY}""" - # Valid keys as registered with the QuestDB fixture. AUTH = dict( username="admin", @@ -54,18 +52,17 @@ token_x="fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU", token_y="Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac") - CA_PATH = (pathlib.Path(__file__).parent.parent / - 'tls_certs' / 'server_rootCA.pem') + 'tls_certs' / 'server_rootCA.pem') def retry( - predicate_task, - timeout_sec=30, - every=0.05, - msg='Timed out retrying', - backoff_till=5.0, - lead_sleep=0.001): + predicate_task, + timeout_sec=30, + every=0.05, + msg='Timed out retrying', + backoff_till=5.0, + lead_sleep=0.001): """ Repeat task every `interval` until it returns a truthy value or times out. """ @@ -121,8 +118,8 @@ def __init__(self): def list_questdb_releases(max_results=1): url = ( - 'https://api.github.com/repos/questdb/questdb/releases?' + - urllib.parse.urlencode({'per_page': max_results})) + 'https://api.github.com/repos/questdb/questdb/releases?' + + urllib.parse.urlencode({'per_page': max_results})) req = urllib.request.Request( url, headers={ @@ -351,8 +348,8 @@ def check_http_up(): def http_sql_query(self, sql_query): url = ( - f'http://{self.host}:{self.http_server_port}/exec?' + - urllib.parse.urlencode({'query': sql_query})) + f'http://{self.host}:{self.http_server_port}/exec?' + + urllib.parse.urlencode({'query': sql_query})) buf = None try: resp = urllib.request.urlopen(url, timeout=5) @@ -370,7 +367,7 @@ def http_sql_query(self, sql_query): if 'error' in data: raise QueryError(data['error']) return data - + def query_version(self): try: res = self.http_sql_query('select build') @@ -397,6 +394,7 @@ def retry_check_table( log_ctx=None): sql_query = f"select * from '{table_name}'" http_response_log = [] + def check_table(): try: resp = self.http_sql_query(sql_query) @@ -494,7 +492,7 @@ def check_started(): self.listen_port = retry( check_started, timeout_sec=180, # Longer to include time to compile. - msg='Timed out waiting for `tls_proxy` to start.',) + msg='Timed out waiting for `tls_proxy` to start.', ) def connect_to_listening_port(): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -519,4 +517,3 @@ def stop(self): if self._log_file: self._log_file.close() self._log_file = None - diff --git a/system_test/questdb_line_sender.py b/system_test/questdb_line_sender.py index 437aa48d..7e50610a 100644 --- a/system_test/questdb_line_sender.py +++ b/system_test/questdb_line_sender.py @@ -37,9 +37,8 @@ """ - -from ast import arg import sys + import numpy sys.dont_write_bytecode = True @@ -65,43 +64,60 @@ c_void_p, c_ssize_t) -from typing import Optional, Tuple, Union +from typing import Optional, Union class c_line_sender(ctypes.Structure): pass + class c_line_sender_buffer(ctypes.Structure): pass + c_line_sender_protocol = ctypes.c_int + class Protocol(Enum): TCP = (c_line_sender_protocol(0), 'tcp') TCPS = (c_line_sender_protocol(1), 'tcps') HTTP = (c_line_sender_protocol(2), 'http') HTTPS = (c_line_sender_protocol(3), 'https') + c_line_sender_ca = ctypes.c_int + class CertificateAuthority(Enum): WEBPKI_ROOTS = (c_line_sender_ca(0), 'webpki_roots') OS_ROOTS = (c_line_sender_ca(1), 'os_roots') WEBPKI_AND_OS_ROOTS = (c_line_sender_ca(2), 'webpki_and_os_roots') PEM_FILE = (c_line_sender_ca(3), 'pem_file') + c_line_protocol_version = ctypes.c_int -class ProtocolVersion(Enum): - TCP = (c_line_protocol_version(0), 'v1') - TCPS = (c_line_protocol_version(1), 'v2') + +class LineProtocolVersion(Enum): + V1 = (c_line_protocol_version(1), 'v1') + V2 = (c_line_protocol_version(2), 'v2') + + @classmethod + def from_int(cls, value: c_line_protocol_version): + for member in cls: + if member.value[0].value == value: + return member + raise ValueError(f"invalid protocol version: {value}") + class c_line_sender_opts(ctypes.Structure): pass + class c_line_sender_error(ctypes.Structure): pass + c_size_t_p = ctypes.POINTER(c_size_t) c_line_sender_p = ctypes.POINTER(c_line_sender) c_line_sender_buffer_p = ctypes.POINTER(c_line_sender_buffer) @@ -112,21 +128,33 @@ class c_line_sender_error(ctypes.Structure): c_uint8_p = ctypes.POINTER(c_uint8) c_uint32_p = ctypes.POINTER(c_uint32) + class c_line_sender_utf8(ctypes.Structure): _fields_ = [("len", c_size_t), ("buf", c_char_p)] + + c_line_sender_utf8_p = ctypes.POINTER(c_line_sender_utf8) + + class c_line_sender_table_name(ctypes.Structure): _fields_ = [("len", c_size_t), ("buf", c_char_p)] + + class line_sender_buffer_view(ctypes.Structure): _fields_ = [("len", c_size_t), ("buf", c_uint8_p)] + c_line_sender_table_name_p = ctypes.POINTER(c_line_sender_table_name) + + class c_line_sender_column_name(ctypes.Structure): _fields_ = [("len", c_size_t), ("buf", c_char_p)] + + c_line_sender_column_name_p = ctypes.POINTER(c_line_sender_column_name) @@ -143,7 +171,7 @@ def _setup_cdll(): 'darwin': 'dylib', 'win32': 'dll'}[sys.platform] dll_path = next( - build_dir.glob(f'**/*questdb_client*.{dll_ext}')) + build_dir.glob(f'**/*questdb_client*.{dll_ext}')) dll = ctypes.CDLL(str(dll_path)) @@ -192,6 +220,8 @@ def set_sig(fn, restype, *argtypes): c_size_t) set_sig( dll.line_sender_buffer_set_line_protocol_version, + c_bool, + c_line_sender_buffer_p, c_line_protocol_version, c_line_sender_error_p_p) set_sig( @@ -551,7 +581,10 @@ def __init__(self, host, port, protocol=Protocol.TCP): def __getattr__(self, name: str): fn = getattr(_DLL, 'line_sender_opts_' + name) + def wrapper(*args): + if name == 'disable_line_protocol_validation': + return _error_wrapped_call(fn, self.impl) mapped_args = [ (_utf8(arg) if isinstance(arg, str) else arg) for arg in args] @@ -559,6 +592,7 @@ def wrapper(*args): return _error_wrapped_call(fn, self.impl, *mapped_args) else: return fn(self.impl, *mapped_args) + return wrapper def __del__(self): @@ -571,16 +605,20 @@ def __init__(self, micros: int): class Buffer: - def __init__(self, init_buf_size=65536, max_name_len=127): + def __init__(self, init_buf_size=65536, max_name_len=127, line_protocol_version=LineProtocolVersion.V2): self._impl = _DLL.line_sender_buffer_with_max_name_len( c_size_t(max_name_len)) _DLL.line_sender_buffer_reserve(self._impl, c_size_t(init_buf_size)) + _error_wrapped_call( + _DLL.line_sender_buffer_set_line_protocol_version, + self._impl, + line_protocol_version.value[0]) def __len__(self): return _DLL.line_sender_buffer_size(self._impl) def peek(self) -> str: - # This is a hacky way of doing it because it copies the whole buffer. + #  This is a hacky way of doing it because it copies the whole buffer. # Instead the `buffer` should be made to support the buffer protocol: # https://docs.python.org/3/c-api/buffer.html # This way we would not need to `bytes(..)` the object to keep it alive. @@ -592,6 +630,12 @@ def peek(self) -> str: else: return '' + def set_line_protocol_version(self, version: LineProtocolVersion): + _error_wrapped_call( + _DLL.line_sender_buffer_set_line_protocol_version, + self._impl, + version.value[0]) + def reserve(self, additional): _DLL.line_sender_buffer_reserve(self._impl, c_size_t(additional)) @@ -692,7 +736,7 @@ def _convert_tuple(tpl: tuple[int, ...], c_type: type, name: str) -> ctypes.POIN ctypes.cast(data, c_uint8_p), c_size_t(length) ) - + def at_now(self): _error_wrapped_call( _DLL.line_sender_buffer_at_now, @@ -737,7 +781,7 @@ def __init__( host: str, port: Union[str, int], **kwargs): - + self._build_mode = build_mode self._impl = None self._conf = [ @@ -745,7 +789,6 @@ def __init__( '::', f'addr={host}:{port};'] self._opts = None - self._buffer = Buffer() opts = _Opts(host, port, protocol) for key, value in kwargs.items(): # Build the config string param pair. @@ -782,12 +825,18 @@ def connect(self): def __enter__(self): self.connect() + self._buffer = Buffer( + line_protocol_version=LineProtocolVersion.from_int(self.line_sender_default_line_protocol_version())) return self def _check_connected(self): if not self._impl: raise SenderError('Not connected.') + def line_sender_default_line_protocol_version(self): + self._check_connected() + return _DLL.line_sender_default_line_protocol_version(self._impl) + def table(self, table: str): self._buffer.table(table) return self @@ -803,8 +852,8 @@ def column( return self def column_f64_arr( - self, name: str, - array: numpy.ndarray): + self, name: str, + array: numpy.ndarray): self._buffer.column_f64_arr(name, array.ndim, array.shape, array.strides, array.ctypes.data, array.nbytes) return self @@ -814,7 +863,7 @@ def at_now(self): def at(self, timestamp: int): self._buffer.at(timestamp) - def flush(self, buffer: Optional[Buffer]=None, clear=True, transactional=None): + def flush(self, buffer: Optional[Buffer] = None, clear=True, transactional=None): if (buffer is None) and not clear: raise ValueError( 'Clear flag must be True when using internal buffer') diff --git a/system_test/test.py b/system_test/test.py index d2860d9d..dc30f020 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -95,9 +95,11 @@ def ns_to_qdb_date(at_ts_ns): class TestSender(unittest.TestCase): - def _mk_linesender(self): + def _mk_linesender(self, disable_line_protocol_validation=False): # N.B.: We never connect with TLS here. auth = AUTH if QDB_FIXTURE.auth else {} + if disable_line_protocol_validation: + auth["disable_line_protocol_validation"] = "on" return qls.Sender( BUILD_MODE, qls.Protocol.HTTP if QDB_FIXTURE.http else qls.Protocol.TCP, @@ -511,6 +513,68 @@ def test_f64_arr_column(self): scrubbed_data = [row[:-1] for row in resp['dataset']] self.assertEqual(scrubbed_data, expected_data) + def test_line_protocol_version_v1(self): + if QDB_FIXTURE.version <= (6, 1, 2): + self.skipTest('Float issues support') + numbers = [ + 0.0, + -0.0, + 1.0, + -1.0] # Converted to `None`. + + table_name = uuid.uuid4().hex + pending = None + with self._mk_linesender() as sender: + sender.buffer.set_line_protocol_version(qls.LineProtocolVersion.V1) + for num in numbers: + sender.table(table_name) + sender.column('n', num) + sender.at_now() + pending = sender.buffer.peek() + + resp = retry_check_table( + table_name, + min_rows=len(numbers), + log_ctx=pending) + exp_columns = [ + {'name': 'n', 'type': 'DOUBLE'}, + {'name': 'timestamp', 'type': 'TIMESTAMP'}] + self.assertEqual(resp['columns'], exp_columns) + + def massage(num): + if math.isnan(num) or math.isinf(num): + return None + elif num == -0.0: + return 0.0 + else: + return num + + # Comparison excludes timestamp column. + exp_dataset = [[massage(num)] for num in numbers] + scrubbed_dataset = [row[:-1] for row in resp['dataset']] + self.assertEqual(scrubbed_dataset, exp_dataset) + + def test_line_protocol_version_v1_array_unsupported(self): + if QDB_FIXTURE.version <= (8, 3, 1): + self.skipTest('array unsupported') + + array1 = np.array( + [ + [[1.1, 2.2], [3.3, 4.4]], + [[5.5, 6.6], [7.7, 8.8]] + ], + dtype=np.float64 + ) + table_name = uuid.uuid4().hex + try: + with self._mk_linesender(True) as sender: + sender.buffer.set_line_protocol_version(qls.LineProtocolVersion.V1) + sender.table(table_name) + sender.column_f64_arr('f64_arr1', array1) + sender.at_now() + except qls.SenderError as e: + self.assertIn('line protocol version v1 does not support array datatype', str(e)) + def _test_example(self, bin_name, table_name, tls=False): if BUILD_MODE != qls.BuildMode.API: self.skipTest('BuildMode.API-only test') From ed3fc06c3093b7f58914ac42062eebc2e24ad098 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 7 May 2025 18:00:01 +0800 Subject: [PATCH 048/119] add more system array tests --- cpp_test/test_line_sender.cpp | 2 +- include/questdb/ingress/line_sender.h | 4 +- questdb-rs/src/ingress/mod.rs | 7 +++ system_test/questdb_line_sender.py | 2 + system_test/test.py | 85 +++++++++++++++++++++++++++ 5 files changed, 97 insertions(+), 3 deletions(-) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 5d8ad52a..5516ac40 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -967,4 +967,4 @@ TEST_CASE("line sender protocol version v1") CHECK(server.recv() == 1); std::string expect{"test,t1=v1,t2= f1=0.5 10000000\n"}; CHECK(server.msgs(0) == expect); -} \ No newline at end of file +} diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 22ec686a..e1df8cbf 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -100,11 +100,11 @@ typedef enum line_protocol_version { /** Version 1 of InfluxDB Line Protocol. Uses text format serialization for f64. */ - line_protocol_version_1, + line_protocol_version_1 = 1, /** Version 2 of InfluxDB Line Protocol. Uses binary format serialization for f64, and support array data type.*/ - line_protocol_version_2, + line_protocol_version_2 = 2, } line_protocol_version; /** Possible sources of the root certificates used to validate the server's diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 435c3747..bab37369 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -1124,6 +1124,13 @@ impl Buffer { "line protocol version v1 does not support array datatype", )); } + if view.ndim() == 0 { + return Err(error::fmt!( + ArrayViewError, + "Zero-dimensional arrays are not supported", + )); + } + self.write_column_key(name)?; // check dimension less equal than max dims diff --git a/system_test/questdb_line_sender.py b/system_test/questdb_line_sender.py index 7e50610a..1ca0573a 100644 --- a/system_test/questdb_line_sender.py +++ b/system_test/questdb_line_sender.py @@ -854,6 +854,8 @@ def column( def column_f64_arr( self, name: str, array: numpy.ndarray): + if array.dtype != numpy.float64: + raise ValueError('expect float64 array') self._buffer.column_f64_arr(name, array.ndim, array.shape, array.strides, array.ctypes.data, array.nbytes) return self diff --git a/system_test/test.py b/system_test/test.py index dc30f020..c2cffb72 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -482,6 +482,9 @@ def test_timestamp_column(self): self.assertEqual(scrubbed_dataset, exp_dataset) def test_f64_arr_column(self): + if QDB_FIXTURE.version < (8, 3, 1): + self.skipTest('array issues support') + table_name = uuid.uuid4().hex array1 = np.array( [ @@ -513,6 +516,88 @@ def test_f64_arr_column(self): scrubbed_data = [row[:-1] for row in resp['dataset']] self.assertEqual(scrubbed_data, expected_data) + def test_f64_arr_empty(self): + if QDB_FIXTURE.version < (8, 3, 1): + self.skipTest('array issues support') + + table_name = uuid.uuid4().hex + empty_array = np.array([], dtype=np.float64).reshape(0, 0, 0) + with self._mk_linesender() as sender: + (sender.table(table_name) + .column_f64_arr('empty', empty_array) + .at_now()) + + resp = retry_check_table(table_name) + exp_columns = [{'dim': 3, 'elemType': 'DOUBLE', 'name': 'empty', 'type': 'ARRAY'}, + {'name': 'timestamp', 'type': 'TIMESTAMP'}] + self.assertEqual(exp_columns, resp['columns']) + self.assertEqual(resp['dataset'][0][0], []) + + def test_f64_arr_non_contiguous(self): + if QDB_FIXTURE.version < (8, 3, 1): + self.skipTest('array issues support') + + table_name = uuid.uuid4().hex + array = np.array([[1.1, 2.2], [3.3, 4.4]], dtype=np.float64)[:, ::2] + with self._mk_linesender() as sender: + (sender.table(table_name) + .column_f64_arr('non_contiguous', array) + .at_now()) + + resp = retry_check_table(table_name) + exp_columns = [{'dim': 2, 'elemType': 'DOUBLE', 'name': 'non_contiguous', 'type': 'ARRAY'}, + {'name': 'timestamp', 'type': 'TIMESTAMP'}] + self.assertEqual(exp_columns, resp['columns']) + self.assertEqual(resp['dataset'][0][0], [[1.1], [3.3]]) + + def test_f64_arr_zero_dimensional(self): + if QDB_FIXTURE.version < (8, 3, 1): + self.skipTest('array issues support') + + table_name = uuid.uuid4().hex + array = np.array(42.0, dtype=np.float64) + try: + with self._mk_linesender() as sender: + (sender.table(table_name) + .column_f64_arr('scalar', array) + .at_now()) + except qls.SenderError as e: + self.assertIn('Zero-dimensional arrays are not supported', str(e)) + + def test_f64_arr_wrong_datatype(self): + if QDB_FIXTURE.version < (8, 3, 1): + self.skipTest('array issues support') + + table_name = uuid.uuid4().hex + array = np.array([1, 2], dtype=np.int32) + try: + with self._mk_linesender() as sender: + (sender.table(table_name) + .column_f64_arr('wrong', array) + .at_now()) + except ValueError as e: + self.assertIn('expect float64 array', str(e)) + + def test_f64_arr_mix_dims(self): + if QDB_FIXTURE.version < (8, 3, 1): + self.skipTest('array issues support') + + array_2d = np.array([[1.1, 2.2], [3.3, 4.4]], dtype=np.float64) + array_1d = np.array([1.1], dtype=np.float64) + table_name = uuid.uuid4().hex + try: + with self._mk_linesender() as sender: + (sender.table(table_name) + .column_f64_arr('array', array_2d) + .at_now() + ) + (sender.table(table_name) + .column_f64_arr('array', array_1d) + .at_now() + ) + except qls.SenderError as e: + self.assertIn('cast error from protocol type: DOUBLE[] to column type: DOUBLE[][]', str(e)) + def test_line_protocol_version_v1(self): if QDB_FIXTURE.version <= (6, 1, 2): self.skipTest('Float issues support') From 3839403bc9a72a728d3dcc0176fb4b80a9cb5321 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 7 May 2025 18:13:49 +0800 Subject: [PATCH 049/119] code format --- questdb-rs/src/ingress/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index bab37369..d6ceef70 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -2355,7 +2355,8 @@ impl SenderBuilder { pub fn disable_line_protocol_validation(mut self) -> Result { if let Some(http) = &mut self.http { // ignore "already specified" error - let _ = http.disable_line_proto_validation + let _ = http + .disable_line_proto_validation .set_specified("disable_line_protocol_validation", true); } Ok(self) From 33767f341e89b5d9002a82fa07cf5fe59e961156 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 7 May 2025 19:09:21 +0800 Subject: [PATCH 050/119] better c/cpp array examples. --- CMakeLists.txt | 7 ++ examples/line_sender_c_example.c | 29 ------ examples/line_sender_c_example_array.c | 99 +++++++++++++++++++ examples/line_sender_c_example_auth.c | 29 ------ examples/line_sender_c_example_auth_tls.c | 29 ------ examples/line_sender_c_example_from_conf.c | 29 ------ examples/line_sender_c_example_from_env.c | 29 ------ examples/line_sender_c_example_http.c | 29 ------ examples/line_sender_c_example_tls_ca.c | 29 ------ examples/line_sender_cpp_example.cpp | 31 +----- examples/line_sender_cpp_example_array.cpp | 61 ++++++++++++ examples/line_sender_cpp_example_auth.cpp | 35 ++----- examples/line_sender_cpp_example_auth_tls.cpp | 36 ++----- .../line_sender_cpp_example_from_conf.cpp | 26 +---- examples/line_sender_cpp_example_from_env.cpp | 26 +---- examples/line_sender_cpp_example_http.cpp | 31 +----- examples/line_sender_cpp_example_tls_ca.cpp | 39 ++------ 17 files changed, 199 insertions(+), 395 deletions(-) create mode 100644 examples/line_sender_c_example_array.c create mode 100644 examples/line_sender_cpp_example_array.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 8de0cb5e..0fd094fa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -98,6 +98,10 @@ if (QUESTDB_TESTS_AND_EXAMPLES) line_sender_c_example examples/concat.c examples/line_sender_c_example.c) + compile_example( + line_sender_c_example_array + examples/concat.c + examples/line_sender_c_example_array.c) compile_example( line_sender_c_example_auth examples/concat.c @@ -123,6 +127,9 @@ if (QUESTDB_TESTS_AND_EXAMPLES) compile_example( line_sender_cpp_example examples/line_sender_cpp_example.cpp) + compile_example( + line_sender_cpp_example_array + examples/line_sender_cpp_example_array.cpp) compile_example( line_sender_cpp_example_auth examples/line_sender_cpp_example_auth.cpp) diff --git a/examples/line_sender_c_example.c b/examples/line_sender_c_example.c index 64807fe9..0f6805cc 100644 --- a/examples/line_sender_c_example.c +++ b/examples/line_sender_c_example.c @@ -55,35 +55,6 @@ static bool example(const char* host, const char* port) if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) goto on_error; - line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("order_book"); - // 3D array of doubles - size_t rank = 3; - uint32_t shapes[] = {2, 3, 2}; - int32_t strides[] = {48, 16, 8}; - double arr_data[] = { - 48123.5, - 2.4, - 48124.0, - 1.8, - 48124.5, - 0.9, - 48122.5, - 3.1, - 48122.0, - 2.7, - 48121.5, - 4.3}; - if (!line_sender_buffer_column_f64_arr( - buffer, - arr_name, - rank, - shapes, - strides, - (const uint8_t*)arr_data, - sizeof(arr_data), - &err)) - goto on_error; - // 1997-07-04 04:56:55 UTC int64_t designated_timestamp = 867992215000000000; if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) diff --git a/examples/line_sender_c_example_array.c b/examples/line_sender_c_example_array.c new file mode 100644 index 00000000..56bc69bf --- /dev/null +++ b/examples/line_sender_c_example_array.c @@ -0,0 +1,99 @@ +#include +#include +#include +#include +#include "concat.h" + +static bool example(const char* host, const char* port) +{ + line_sender_error* err = NULL; + line_sender* sender = NULL; + line_sender_buffer* buffer = NULL; + char* conf_str = concat("tcp::addr=", host, ":", port, ";"); + if (!conf_str) + { + fprintf(stderr, "Could not concatenate configuration string.\n"); + return false; + } + + line_sender_utf8 conf_str_utf8 = {0, NULL}; + if (!line_sender_utf8_init( + &conf_str_utf8, strlen(conf_str), conf_str, &err)) + goto on_error; + + sender = line_sender_from_conf(conf_str_utf8, &err); + if (!sender) + goto on_error; + + free(conf_str); + conf_str = NULL; + + buffer = line_sender_buffer_new(); + line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB 初始缓冲 + + line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("market_orders"); + line_sender_column_name symbol_col = QDB_COLUMN_NAME_LITERAL("symbol"); + line_sender_column_name book_col = QDB_COLUMN_NAME_LITERAL("order_book"); + + if (!line_sender_buffer_table(buffer, table_name, &err)) + goto on_error; + + line_sender_utf8 symbol_val = QDB_UTF8_LITERAL("BTC-USD"); + if (!line_sender_buffer_symbol(buffer, symbol_col, symbol_val, &err)) + goto on_error; + + size_t array_rank = 3; + uint32_t array_shapes[] = {2, 3, 2}; + int32_t array_strides[] = {48, 16, 8}; + + double array_data[] = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + + if (!line_sender_buffer_column_f64_arr( + buffer, + book_col, + array_rank, + array_shapes, + array_strides, + (const uint8_t*)array_data, + sizeof(array_data), + &err)) + goto on_error; + + if (!line_sender_buffer_at_nanos(buffer, line_sender_now_nanos(), &err)) + goto on_error; + + if (!line_sender_flush(sender, buffer, &err)) + goto on_error; + + line_sender_close(sender); + return true; + +on_error:; + size_t err_len = 0; + const char* err_msg = line_sender_error_msg(err, &err_len); + fprintf(stderr, "Error: %.*s\n", (int)err_len, err_msg); + free(conf_str); + line_sender_error_free(err); + line_sender_buffer_free(buffer); + line_sender_close(sender); + return false; +} + +int main(int argc, const char* argv[]) +{ + const char* host = (argc >= 2) ? argv[1] : "localhost"; + const char* port = (argc >= 3) ? argv[2] : "9009"; + return !example(host, port); +} diff --git a/examples/line_sender_c_example_auth.c b/examples/line_sender_c_example_auth.c index c6905a9e..59a99421 100644 --- a/examples/line_sender_c_example_auth.c +++ b/examples/line_sender_c_example_auth.c @@ -60,35 +60,6 @@ static bool example(const char* host, const char* port) if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) goto on_error; - line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("order_book"); - // 3D array of doubles - size_t rank = 3; - uint32_t shapes[] = {2, 3, 2}; - int32_t strides[] = {48, 16, 8}; - double arr_data[] = { - 48123.5, - 2.4, - 48124.0, - 1.8, - 48124.5, - 0.9, - 48122.5, - 3.1, - 48122.0, - 2.7, - 48121.5, - 4.3}; - if (!line_sender_buffer_column_f64_arr( - buffer, - arr_name, - rank, - shapes, - strides, - (const uint8_t*)arr_data, - sizeof(arr_data), - &err)) - goto on_error; - // 1997-07-04 04:56:55 UTC int64_t designated_timestamp = 867992215000000000; if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) diff --git a/examples/line_sender_c_example_auth_tls.c b/examples/line_sender_c_example_auth_tls.c index 16abb263..e7454d8d 100644 --- a/examples/line_sender_c_example_auth_tls.c +++ b/examples/line_sender_c_example_auth_tls.c @@ -60,35 +60,6 @@ static bool example(const char* host, const char* port) if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) goto on_error; - line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("order_book"); - // 3D array of doubles - size_t rank = 3; - uint32_t shapes[] = {2, 3, 2}; - int32_t strides[] = {48, 16, 8}; - double arr_data[] = { - 48123.5, - 2.4, - 48124.0, - 1.8, - 48124.5, - 0.9, - 48122.5, - 3.1, - 48122.0, - 2.7, - 48121.5, - 4.3}; - if (!line_sender_buffer_column_f64_arr( - buffer, - arr_name, - rank, - shapes, - strides, - (const uint8_t*)arr_data, - sizeof(arr_data), - &err)) - goto on_error; - // 1997-07-04 04:56:55 UTC int64_t designated_timestamp = 867992215000000000; if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) diff --git a/examples/line_sender_c_example_from_conf.c b/examples/line_sender_c_example_from_conf.c index 6b077e58..6f1d4e43 100644 --- a/examples/line_sender_c_example_from_conf.c +++ b/examples/line_sender_c_example_from_conf.c @@ -44,35 +44,6 @@ int main(int argc, const char* argv[]) if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) goto on_error; - line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("order_book"); - // 3D array of doubles - size_t rank = 3; - uint32_t shapes[] = {2, 3, 2}; - int32_t strides[] = {48, 16, 8}; - double arr_data[] = { - 48123.5, - 2.4, - 48124.0, - 1.8, - 48124.5, - 0.9, - 48122.5, - 3.1, - 48122.0, - 2.7, - 48121.5, - 4.3}; - if (!line_sender_buffer_column_f64_arr( - buffer, - arr_name, - rank, - shapes, - strides, - (const uint8_t*)arr_data, - sizeof(arr_data), - &err)) - goto on_error; - // 1997-07-04 04:56:55 UTC int64_t designated_timestamp = 867992215000000000; if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) diff --git a/examples/line_sender_c_example_from_env.c b/examples/line_sender_c_example_from_env.c index 5121bf70..e67dbe59 100644 --- a/examples/line_sender_c_example_from_env.c +++ b/examples/line_sender_c_example_from_env.c @@ -43,35 +43,6 @@ int main(int argc, const char* argv[]) if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) goto on_error; - line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("order_book"); - // 3D array of doubles - size_t rank = 3; - uint32_t shapes[] = {2, 3, 2}; - int32_t strides[] = {48, 16, 8}; - double arr_data[] = { - 48123.5, - 2.4, - 48124.0, - 1.8, - 48124.5, - 0.9, - 48122.5, - 3.1, - 48122.0, - 2.7, - 48121.5, - 4.3}; - if (!line_sender_buffer_column_f64_arr( - buffer, - arr_name, - rank, - shapes, - strides, - (const uint8_t*)arr_data, - sizeof(arr_data), - &err)) - goto on_error; - // 1997-07-04 04:56:55 UTC int64_t designated_timestamp = 867992215000000000; if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) diff --git a/examples/line_sender_c_example_http.c b/examples/line_sender_c_example_http.c index 5b848d51..f3bd3248 100644 --- a/examples/line_sender_c_example_http.c +++ b/examples/line_sender_c_example_http.c @@ -57,35 +57,6 @@ static bool example(const char* host, const char* port) if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) goto on_error; - line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("order_book"); - // 3D array of doubles - size_t rank = 3; - uint32_t shapes[] = {2, 3, 2}; - int32_t strides[] = {48, 16, 8}; - double arr_data[] = { - 48123.5, - 2.4, - 48124.0, - 1.8, - 48124.5, - 0.9, - 48122.5, - 3.1, - 48122.0, - 2.7, - 48121.5, - 4.3}; - if (!line_sender_buffer_column_f64_arr( - buffer, - arr_name, - rank, - shapes, - strides, - (const uint8_t*)arr_data, - sizeof(arr_data), - &err)) - goto on_error; - // 1997-07-04 04:56:55 UTC int64_t designated_timestamp = 867992215000000000; if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) diff --git a/examples/line_sender_c_example_tls_ca.c b/examples/line_sender_c_example_tls_ca.c index 3b8fb48a..63151945 100644 --- a/examples/line_sender_c_example_tls_ca.c +++ b/examples/line_sender_c_example_tls_ca.c @@ -61,35 +61,6 @@ static bool example(const char* ca_path, const char* host, const char* port) if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) goto on_error; - line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("order_book"); - // 3D array of doubles - size_t rank = 3; - uint32_t shapes[] = {2, 3, 2}; - int32_t strides[] = {48, 16, 8}; - double arr_data[] = { - 48123.5, - 2.4, - 48124.0, - 1.8, - 48124.5, - 0.9, - 48122.5, - 3.1, - 48122.0, - 2.7, - 48121.5, - 4.3}; - if (!line_sender_buffer_column_f64_arr( - buffer, - arr_name, - rank, - shapes, - strides, - (const uint8_t*)arr_data, - sizeof(arr_data), - &err)) - goto on_error; - // 1997-07-04 04:56:55 UTC int64_t designated_timestamp = 867992215000000000; if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) diff --git a/examples/line_sender_cpp_example.cpp b/examples/line_sender_cpp_example.cpp index 5e3b12d1..d65ecc31 100644 --- a/examples/line_sender_cpp_example.cpp +++ b/examples/line_sender_cpp_example.cpp @@ -19,32 +19,13 @@ static bool example(std::string_view host, std::string_view port) const auto side_name = "side"_cn; const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; - const auto order_book_name = "order_book"_cn; - size_t rank = 3; - std::vector shape{2, 3, 2}; - std::vector strides{48, 16, 8}; - std::array arr_data = { - 48123.5, - 2.4, - 48124.0, - 1.8, - 48124.5, - 0.9, - 48122.5, - 3.1, - 48122.0, - 2.7, - 48121.5, - 4.3}; questdb::ingress::line_sender_buffer buffer; - buffer - .table(table_name) + buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) .column(amount_name, 0.00044) - .column(order_book_name, 3, shape, strides, arr_data) .at(questdb::ingress::timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. @@ -58,10 +39,7 @@ static bool example(std::string_view host, std::string_view port) } catch (const questdb::ingress::line_sender_error& err) { - std::cerr - << "Error running example: " - << err.what() - << std::endl; + std::cerr << "Error running example: " << err.what() << std::endl; return false; } @@ -74,9 +52,8 @@ static bool displayed_help(int argc, const char* argv[]) const std::string_view arg{argv[index]}; if ((arg == "-h"sv) || (arg == "--help"sv)) { - std::cerr - << "Usage:\n" - << "line_sender_c_example: [HOST [PORT]]\n" + std::cerr << "Usage:\n" + << "line_sender_c_example: [HOST [PORT]]\n" << " HOST: ILP host (defaults to \"localhost\").\n" << " PORT: ILP port (defaults to \"9009\")." << std::endl; diff --git a/examples/line_sender_cpp_example_array.cpp b/examples/line_sender_cpp_example_array.cpp new file mode 100644 index 00000000..608b8642 --- /dev/null +++ b/examples/line_sender_cpp_example_array.cpp @@ -0,0 +1,61 @@ +#include +#include +#include + +using namespace std::literals::string_view_literals; +using namespace questdb::ingress::literals; + +static bool array_example(std::string_view host, std::string_view port) +{ + try + { + auto sender = questdb::ingress::line_sender::from_conf( + "tcp::addr=" + std::string{host} + ":" + std::string{port} + ";"); + + const auto table_name = "cpp_market_orders"_tn; + const auto symbol_col = "symbol"_cn; + const auto book_col = "order_book"_cn; + size_t rank = 3; + std::vector shape{2, 3, 2}; + std::vector strides{48, 16, 8}; + std::array arr_data = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + + questdb::ingress::line_sender_buffer buffer; + buffer.table(table_name) + .symbol(symbol_col, "BTC-USD"_utf8) + .column(book_col, 3, shape, strides, arr_data) + .at(questdb::ingress::timestamp_nanos::now()); + sender.flush(buffer); + return true; + } + catch (const questdb::ingress::line_sender_error& err) + { + std::cerr << "[ERROR] " << err.what() << std::endl; + return false; + } +} + +int main(int argc, const char* argv[]) +{ + auto host = "localhost"sv; + if (argc >= 2) + host = std::string_view{argv[1]}; + + auto port = "9009"sv; + if (argc >= 3) + port = std::string_view{argv[2]}; + + return !array_example(host, port); +} diff --git a/examples/line_sender_cpp_example_auth.cpp b/examples/line_sender_cpp_example_auth.cpp index b4cf2b70..4c229617 100644 --- a/examples/line_sender_cpp_example_auth.cpp +++ b/examples/line_sender_cpp_example_auth.cpp @@ -9,7 +9,8 @@ static bool example(std::string_view host, std::string_view port) try { auto sender = questdb::ingress::line_sender::from_conf( - "tcp::addr=" + std::string{host} + ":" + std::string{port} + ";" + "tcp::addr=" + std::string{host} + ":" + std::string{port} + + ";" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" @@ -23,33 +24,13 @@ static bool example(std::string_view host, std::string_view port) const auto side_name = "side"_cn; const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; - const auto order_book_name = "order_book"_cn; - - size_t rank = 3; - std::vector shape{2, 3, 2}; - std::vector strides{48, 16, 8}; - std::array arr_data = { - 48123.5, - 2.4, - 48124.0, - 1.8, - 48124.5, - 0.9, - 48122.5, - 3.1, - 48122.0, - 2.7, - 48121.5, - 4.3}; questdb::ingress::line_sender_buffer buffer; - buffer - .table(table_name) + buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) .column(amount_name, 0.00044) - .column(order_book_name, 3, shape, strides, arr_data) .at(questdb::ingress::timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. @@ -63,10 +44,7 @@ static bool example(std::string_view host, std::string_view port) } catch (const questdb::ingress::line_sender_error& err) { - std::cerr - << "Error running example: " - << err.what() - << std::endl; + std::cerr << "Error running example: " << err.what() << std::endl; return false; } @@ -79,9 +57,8 @@ static bool displayed_help(int argc, const char* argv[]) const std::string_view arg{argv[index]}; if ((arg == "-h"sv) || (arg == "--help"sv)) { - std::cerr - << "Usage:\n" - << "line_sender_c_example: [HOST [PORT]]\n" + std::cerr << "Usage:\n" + << "line_sender_c_example: [HOST [PORT]]\n" << " HOST: ILP host (defaults to \"localhost\").\n" << " PORT: ILP port (defaults to \"9009\")." << std::endl; diff --git a/examples/line_sender_cpp_example_auth_tls.cpp b/examples/line_sender_cpp_example_auth_tls.cpp index 90e2a46b..f202fc75 100644 --- a/examples/line_sender_cpp_example_auth_tls.cpp +++ b/examples/line_sender_cpp_example_auth_tls.cpp @@ -4,14 +4,13 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; -static bool example( - std::string_view host, - std::string_view port) +static bool example(std::string_view host, std::string_view port) { try { auto sender = questdb::ingress::line_sender::from_conf( - "tcps::addr=" + std::string{host} + ":" + std::string{port} + ";" + "tcps::addr=" + std::string{host} + ":" + std::string{port} + + ";" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" @@ -25,32 +24,13 @@ static bool example( const auto side_name = "side"_cn; const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; - const auto order_book_name = "order_book"_cn; - size_t rank = 3; - std::vector shape{2, 3, 2}; - std::vector strides{48, 16, 8}; - std::array arr_data = { - 48123.5, - 2.4, - 48124.0, - 1.8, - 48124.5, - 0.9, - 48122.5, - 3.1, - 48122.0, - 2.7, - 48121.5, - 4.3}; questdb::ingress::line_sender_buffer buffer; - buffer - .table(table_name) + buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) .column(amount_name, 0.00044) - .column(order_book_name, 3, shape, strides, arr_data) .at(questdb::ingress::timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. @@ -64,10 +44,7 @@ static bool example( } catch (const questdb::ingress::line_sender_error& err) { - std::cerr - << "Error running example: " - << err.what() - << std::endl; + std::cerr << "Error running example: " << err.what() << std::endl; return false; } @@ -80,8 +57,7 @@ static bool displayed_help(int argc, const char* argv[]) const std::string_view arg{argv[index]}; if ((arg == "-h"sv) || (arg == "--help"sv)) { - std::cerr - << "Usage:\n" + std::cerr << "Usage:\n" << "line_sender_c_example: CA_PATH [HOST [PORT]]\n" << " HOST: ILP host (defaults to \"localhost\").\n" << " PORT: ILP port (defaults to \"9009\")." diff --git a/examples/line_sender_cpp_example_from_conf.cpp b/examples/line_sender_cpp_example_from_conf.cpp index d0d1e470..2a2fe510 100644 --- a/examples/line_sender_cpp_example_from_conf.cpp +++ b/examples/line_sender_cpp_example_from_conf.cpp @@ -19,32 +19,13 @@ int main(int argc, const char* argv[]) const auto side_name = "side"_cn; const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; - const auto order_book_name = "order_book"_cn; - size_t rank = 3; - std::vector shape{2, 3, 2}; - std::vector strides{48, 16, 8}; - std::array arr_data = { - 48123.5, - 2.4, - 48124.0, - 1.8, - 48124.5, - 0.9, - 48122.5, - 3.1, - 48122.0, - 2.7, - 48121.5, - 4.3}; questdb::ingress::line_sender_buffer buffer; - buffer - .table(table_name) + buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) .column(amount_name, 0.00044) - .column(order_book_name, 3, shape, strides, arr_data) .at(questdb::ingress::timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. @@ -58,10 +39,7 @@ int main(int argc, const char* argv[]) } catch (const questdb::ingress::line_sender_error& err) { - std::cerr - << "Error running example: " - << err.what() - << std::endl; + std::cerr << "Error running example: " << err.what() << std::endl; return 1; } diff --git a/examples/line_sender_cpp_example_from_env.cpp b/examples/line_sender_cpp_example_from_env.cpp index 069e4e22..54acd658 100644 --- a/examples/line_sender_cpp_example_from_env.cpp +++ b/examples/line_sender_cpp_example_from_env.cpp @@ -18,32 +18,13 @@ int main(int argc, const char* argv[]) const auto side_name = "side"_cn; const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; - const auto order_book_name = "order_book"_cn; - size_t rank = 3; - std::vector shape{2, 3, 2}; - std::vector strides{48, 16, 8}; - std::array arr_data = { - 48123.5, - 2.4, - 48124.0, - 1.8, - 48124.5, - 0.9, - 48122.5, - 3.1, - 48122.0, - 2.7, - 48121.5, - 4.3}; questdb::ingress::line_sender_buffer buffer; - buffer - .table(table_name) + buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) .column(amount_name, 0.00044) - .column(order_book_name, 3, shape, strides, arr_data) .at(questdb::ingress::timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. @@ -57,10 +38,7 @@ int main(int argc, const char* argv[]) } catch (const questdb::ingress::line_sender_error& err) { - std::cerr - << "Error running example: " - << err.what() - << std::endl; + std::cerr << "Error running example: " << err.what() << std::endl; return 1; } diff --git a/examples/line_sender_cpp_example_http.cpp b/examples/line_sender_cpp_example_http.cpp index 6c6b82f9..800e11aa 100644 --- a/examples/line_sender_cpp_example_http.cpp +++ b/examples/line_sender_cpp_example_http.cpp @@ -19,33 +19,14 @@ static bool example(std::string_view host, std::string_view port) const auto side_name = "side"_cn; const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; - const auto order_book_name = "order_book"_cn; - size_t rank = 3; - std::vector shape{2, 3, 2}; - std::vector strides{48, 16, 8}; - std::array arr_data = { - 48123.5, - 2.4, - 48124.0, - 1.8, - 48124.5, - 0.9, - 48122.5, - 3.1, - 48122.0, - 2.7, - 48121.5, - 4.3}; questdb::ingress::line_sender_buffer buffer{ sender.default_line_protocol_version()}; - buffer - .table(table_name) + buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) .column(amount_name, 0.00044) - .column(order_book_name, 3, shape, strides, arr_data) .at(questdb::ingress::timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. @@ -59,10 +40,7 @@ static bool example(std::string_view host, std::string_view port) } catch (const questdb::ingress::line_sender_error& err) { - std::cerr - << "Error running example: " - << err.what() - << std::endl; + std::cerr << "Error running example: " << err.what() << std::endl; return false; } @@ -75,9 +53,8 @@ static bool displayed_help(int argc, const char* argv[]) const std::string_view arg{argv[index]}; if ((arg == "-h"sv) || (arg == "--help"sv)) { - std::cerr - << "Usage:\n" - << "line_sender_c_example: [HOST [PORT]]\n" + std::cerr << "Usage:\n" + << "line_sender_c_example: [HOST [PORT]]\n" << " HOST: ILP host (defaults to \"localhost\").\n" << " PORT: ILP port (defaults to \"9009\")." << std::endl; diff --git a/examples/line_sender_cpp_example_tls_ca.cpp b/examples/line_sender_cpp_example_tls_ca.cpp index 27a91f81..ac4d4743 100644 --- a/examples/line_sender_cpp_example_tls_ca.cpp +++ b/examples/line_sender_cpp_example_tls_ca.cpp @@ -5,19 +5,19 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; static bool example( - std::string_view ca_path, - std::string_view host, - std::string_view port) + std::string_view ca_path, std::string_view host, std::string_view port) { try { auto sender = questdb::ingress::line_sender::from_conf( - "tcps::addr=" + std::string{host} + ":" + std::string{port} + ";" + "tcps::addr=" + std::string{host} + ":" + std::string{port} + + ";" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" "token_y=Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac;" - "tls_roots=" + std::string{ca_path} + ";"); // path to custom `.pem` file. + "tls_roots=" + + std::string{ca_path} + ";"); // path to custom `.pem` file. // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid @@ -27,32 +27,13 @@ static bool example( const auto side_name = "side"_cn; const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; - const auto order_book_name = "order_book"_cn; - size_t rank = 3; - std::vector shape{2, 3, 2}; - std::vector strides{48, 16, 8}; - std::array arr_data = { - 48123.5, - 2.4, - 48124.0, - 1.8, - 48124.5, - 0.9, - 48122.5, - 3.1, - 48122.0, - 2.7, - 48121.5, - 4.3}; questdb::ingress::line_sender_buffer buffer; - buffer - .table(table_name) + buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) .column(amount_name, 0.00044) - .column(order_book_name, 3, shape, strides, arr_data) .at(questdb::ingress::timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. @@ -66,10 +47,7 @@ static bool example( } catch (const questdb::ingress::line_sender_error& err) { - std::cerr - << "Error running example: " - << err.what() - << std::endl; + std::cerr << "Error running example: " << err.what() << std::endl; return false; } @@ -82,8 +60,7 @@ static bool displayed_help(int argc, const char* argv[]) const std::string_view arg{argv[index]}; if ((arg == "-h"sv) || (arg == "--help"sv)) { - std::cerr - << "Usage:\n" + std::cerr << "Usage:\n" << "line_sender_c_example: CA_PATH [HOST [PORT]]\n" << " CA_PATH: Certificate authority pem file.\n" << " HOST: ILP host (defaults to \"localhost\").\n" From 6e8d77e13b311316d8119a8707ad24d56b25d9a2 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 7 May 2025 19:09:34 +0800 Subject: [PATCH 051/119] fix test.py --- system_test/test.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/system_test/test.py b/system_test/test.py index c2cffb72..6b198858 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -640,7 +640,7 @@ def massage(num): self.assertEqual(scrubbed_dataset, exp_dataset) def test_line_protocol_version_v1_array_unsupported(self): - if QDB_FIXTURE.version <= (8, 3, 1): + if QDB_FIXTURE.version < (8, 3, 1): self.skipTest('array unsupported') array1 = np.array( @@ -688,16 +688,13 @@ def _test_example(self, bin_name, table_name, tls=False): {'name': 'side', 'type': 'SYMBOL'}, {'name': 'price', 'type': 'DOUBLE'}, {'name': 'amount', 'type': 'DOUBLE'}, - {'dim': 3, 'elemType': 'DOUBLE', 'name': 'order_book', 'type': 'ARRAY'}, {'name': 'timestamp', 'type': 'TIMESTAMP'}] self.assertEqual(resp['columns'], exp_columns) exp_dataset = [['ETH-USD', 'sell', 2615.54, - 0.00044, - [[[48123.5, 2.4], [48124.0, 1.8], [48124.5, 0.9]], - [[48122.5, 3.1], [48122.0, 2.7], [48121.5, 4.3]]]]] + 0.00044]] # Comparison excludes timestamp column. scrubbed_dataset = [row[:-1] for row in resp['dataset']] self.assertEqual(scrubbed_dataset, exp_dataset) From 6168e7f6f53b76fb7056a698f99449036e76f417 Mon Sep 17 00:00:00 2001 From: victorgao Date: Wed, 7 May 2025 22:08:55 +0800 Subject: [PATCH 052/119] add array example tests. --- questdb-rs/src/ingress/http.rs | 2 +- questdb-rs/src/tests/http.rs | 2 +- system_test/test.py | 42 ++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/questdb-rs/src/ingress/http.rs b/questdb-rs/src/ingress/http.rs index 26e519b7..92c867bf 100644 --- a/questdb-rs/src/ingress/http.rs +++ b/questdb-rs/src/ingress/http.rs @@ -502,7 +502,7 @@ pub(super) fn get_line_protocol_version( } else { return Err(error::fmt!( LineProtocolVersionError, - "Server does not support current client." + "Server does not support current client" )); } } else { diff --git a/questdb-rs/src/tests/http.rs b/questdb-rs/src/tests/http.rs index d1d2c719..170182ab 100644 --- a/questdb-rs/src/tests/http.rs +++ b/questdb-rs/src/tests/http.rs @@ -923,7 +923,7 @@ fn test_sender_line_protocol_version_unsupported_client() -> TestResult { assert!(res1.is_err()); let e1 = res1.err().unwrap(); assert_eq!(e1.code(), ErrorCode::LineProtocolVersionError); - assert!(e1.msg().contains("Server does not support current client.")); + assert!(e1.msg().contains("Server does not support current client")); server_thread.join().unwrap()?; Ok(()) } diff --git a/system_test/test.py b/system_test/test.py index 6b198858..19dc3c0c 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -725,6 +725,48 @@ def test_cpp_tls_example(self): 'cpp_trades_tls_ca', tls=True) + def test_cpp_array_example(self): + self._test_array_example( + 'line_sender_cpp_example_array', + 'cpp_market_orders') + + def test_c_array_example(self): + self._test_array_example( + 'line_sender_c_example_array', + 'market_orders') + + def _test_array_example(self, bin_name, table_name): + if QDB_FIXTURE.version < (8, 3, 1): + self.skipTest('array unsupported') + if QDB_FIXTURE.http: + self.skipTest('TCP-only test') + if BUILD_MODE != qls.BuildMode.API: + self.skipTest('BuildMode.API-only test') + if QDB_FIXTURE.auth: + self.skipTest('auth') + + proj = Project() + ext = '.exe' if sys.platform == 'win32' else '' + try: + bin_path = next(proj.build_dir.glob(f'**/{bin_name}{ext}')) + except StopIteration: + raise RuntimeError(f'Could not find {bin_name}{ext} in {proj.build_dir}') + port = QDB_FIXTURE.line_tcp_port + args = [str(bin_path)] + args.extend(['localhost', str(port)]) + subprocess.check_call(args, cwd=bin_path.parent) + resp = retry_check_table(table_name) + exp_columns = [ + {'name': 'symbol', 'type': 'SYMBOL'}, + {'dim': 3, 'elemType': 'DOUBLE', 'name': 'order_book', 'type': 'ARRAY'}, + {'name': 'timestamp', 'type': 'TIMESTAMP'}] + self.assertEqual(resp['columns'], exp_columns) + exp_dataset = [['BTC-USD', + [[[48123.5, 2.4], [48124.0, 1.8], [48124.5, 0.9]], + [[48122.5, 3.1], [48122.0, 2.7], [48121.5, 4.3]]]]] + scrubbed_dataset = [row[:-1] for row in resp['dataset']] + self.assertEqual(scrubbed_dataset, exp_dataset) + def test_opposite_auth(self): """ We simulate incorrectly connecting either: From d758e6ddd51fbece6353953008d9ec97fdabf6b4 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 8 May 2025 11:59:12 +0800 Subject: [PATCH 053/119] c error code. --- include/questdb/ingress/line_sender.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index e1df8cbf..2185bdf5 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -77,6 +77,19 @@ typedef enum line_sender_error_code /** Bad configuration. */ line_sender_error_config_error, + + /** Currently, only arrays with a maximum 32 dimensions are supported. */ + line_sender_error_array_large_dim, + + /** ArrayView internal error, such as failure to get the size of a valid + * dimension. */ + line_sender_error_array_view_internal_error, + + /** Write arrayView to sender buffer error. */ + line_sender_error_array_view_write_to_buffer_error, + + /** Line sender protocol version error. */ + line_sender_error_line_protocol_version_error, } line_sender_error_code; /** The protocol used to connect with. */ @@ -499,6 +512,7 @@ bool line_sender_buffer_column_str( * @param[in] rank Number of dimensions of the array. * @param[in] shapes Array of dimension sizes (length = `rank`). * Each element must be a positive integer. + * @param[in] strides Array strides. * @param[in] data_buffer First array element data. * @param[in] data_buffer_len Bytes length of the array data. * @param[out] err_out Set to an error object on failure (if non-NULL). From bf8ae373d00699ecd02748d29d9f234a86400667 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 8 May 2025 12:13:22 +0800 Subject: [PATCH 054/119] remove unnecessary mut. --- include/questdb/ingress/line_sender.h | 2 +- questdb-rs-ffi/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 2185bdf5..a3eeb8b2 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -944,7 +944,7 @@ line_sender* line_sender_from_env(line_sender_error** err_out); */ LINESENDER_API line_protocol_version line_sender_default_line_protocol_version( - line_sender* sender); + const line_sender* sender); /** * Tell whether the sender is no longer usable and must be closed. diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 35b2d9de..a6801cc3 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -1419,7 +1419,7 @@ unsafe fn unwrap_sender_mut<'a>(sender: *mut line_sender) -> &'a mut Sender { /// of client and server compatible versions #[no_mangle] pub unsafe extern "C" fn line_sender_default_line_protocol_version( - sender: *mut line_sender, + sender: *const line_sender, ) -> LineProtocolVersion { unwrap_sender(sender).default_line_protocol_version().into() } From fa6f04d8af1423726effe1a8495cc4f358d4fda1 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 8 May 2025 14:20:43 +0800 Subject: [PATCH 055/119] make c++20 happy --- cpp_test/mock_server.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp_test/mock_server.cpp b/cpp_test/mock_server.cpp index f2bd8bc5..6a93666c 100644 --- a/cpp_test/mock_server.cpp +++ b/cpp_test/mock_server.cpp @@ -24,7 +24,7 @@ #include "mock_server.hpp" -#include +#include #if defined(PLATFORM_UNIX) # include From 3ce862e74ebc56ffa966c1851c672a4c7e94d513 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 8 May 2025 21:14:26 +0800 Subject: [PATCH 056/119] more ideamatic python array interface --- cpp_test/test_line_sender.cpp | 6 +-- examples/line_sender_c_example_array.c | 4 +- examples/line_sender_cpp_example_array.cpp | 4 +- include/questdb/ingress/line_sender.h | 4 +- include/questdb/ingress/line_sender.hpp | 4 +- questdb-rs-ffi/src/lib.rs | 8 +-- questdb-rs/benches/ndarr.rs | 15 +++--- questdb-rs/src/ingress/mod.rs | 8 ++- questdb-rs/src/ingress/ndarr.rs | 34 ++++++------- questdb-rs/src/tests/ndarr.rs | 58 +++++++++++----------- system_test/questdb_line_sender.py | 13 ++--- 11 files changed, 80 insertions(+), 78 deletions(-) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 5516ac40..e6f4f663 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -97,7 +97,7 @@ std::string& push_double_arr_to_buffer( std::string& buffer, std::array data, size_t rank, - uint32_t* shapes) + uintptr_t* shapes) { buffer.push_back(14); buffer.push_back(10); @@ -161,8 +161,8 @@ TEST_CASE("line_sender c api basics") line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("a1"); // 3D array of doubles size_t rank = 3; - uint32_t shapes[] = {2, 3, 2}; - int32_t strides[] = {48, 16, 8}; + uintptr_t shapes[] = {2, 3, 2}; + intptr_t strides[] = {48, 16, 8}; std::array arr_data = { 48123.5, 2.4, diff --git a/examples/line_sender_c_example_array.c b/examples/line_sender_c_example_array.c index 56bc69bf..6f5650c7 100644 --- a/examples/line_sender_c_example_array.c +++ b/examples/line_sender_c_example_array.c @@ -43,8 +43,8 @@ static bool example(const char* host, const char* port) goto on_error; size_t array_rank = 3; - uint32_t array_shapes[] = {2, 3, 2}; - int32_t array_strides[] = {48, 16, 8}; + uintptr_t array_shapes[] = {2, 3, 2}; + intptr_t array_strides[] = {48, 16, 8}; double array_data[] = { 48123.5, diff --git a/examples/line_sender_cpp_example_array.cpp b/examples/line_sender_cpp_example_array.cpp index 608b8642..e07cd21b 100644 --- a/examples/line_sender_cpp_example_array.cpp +++ b/examples/line_sender_cpp_example_array.cpp @@ -16,8 +16,8 @@ static bool array_example(std::string_view host, std::string_view port) const auto symbol_col = "symbol"_cn; const auto book_col = "order_book"_cn; size_t rank = 3; - std::vector shape{2, 3, 2}; - std::vector strides{48, 16, 8}; + std::vector shape{2, 3, 2}; + std::vector strides{48, 16, 8}; std::array arr_data = { 48123.5, 2.4, diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index a3eeb8b2..959fd149 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -523,8 +523,8 @@ bool line_sender_buffer_column_f64_arr( line_sender_buffer* buffer, line_sender_column_name name, size_t rank, - const uint32_t* shapes, - const int32_t* strides, + const uintptr_t* shape, + const intptr_t* strides, const uint8_t* data_buffer, size_t data_buffer_len, line_sender_error** err_out); diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 99d67afa..21906aa9 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -678,8 +678,8 @@ class line_sender_buffer line_sender_buffer& column( column_name_view name, const size_t rank, - const std::vector& shapes, - const std::vector& strides, + const std::vector& shapes, + const std::vector& strides, const std::array& data) { static_assert( diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index a6801cc3..c0815d5d 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -889,8 +889,8 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( buffer: *mut line_sender_buffer, name: line_sender_column_name, rank: size_t, - shape: *const u32, - strides: *const i32, + shape: *const usize, + strides: *const isize, data_buffer: *const u8, data_buffer_len: size_t, err_out: *mut *mut line_sender_error, @@ -898,10 +898,10 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( let buffer = unwrap_buffer_mut(buffer); let name = name.as_name(); let view = - ingress::StridedArrayView::::new(rank, shape, strides, data_buffer, data_buffer_len); + ingress::StrideArrayView::::new(rank, shape, strides, data_buffer, data_buffer_len); bubble_err_to_c!( err_out, - buffer.column_arr::, ingress::StridedArrayView<'_, f64>, f64>(name, &view) + buffer.column_arr::, ingress::StrideArrayView<'_, f64>, f64>(name, &view) ); true } diff --git a/questdb-rs/benches/ndarr.rs b/questdb-rs/benches/ndarr.rs index a5b8b704..9491644c 100644 --- a/questdb-rs/benches/ndarr.rs +++ b/questdb-rs/benches/ndarr.rs @@ -1,6 +1,6 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; use ndarray::{Array, Array2}; -use questdb::ingress::{Buffer, ColumnName, StridedArrayView}; +use questdb::ingress::{Buffer, ColumnName, StrideArrayView}; /// run with /// ```shell @@ -84,17 +84,16 @@ fn bench_array_view(c: &mut Criterion) { buffer.clear(); }); - let shape: Vec = transposed_view.shape().iter().map(|&d| d as u32).collect(); - let elem_size = size_of::() as i32; - let strides: Vec = transposed_view + let elem_size = size_of::() as isize; + let strides: Vec = transposed_view .strides() .iter() - .map(|&s| s as i32 * elem_size) // 转换为字节步长 + .map(|&s| s * elem_size) // 转换为字节步长 .collect(); - let view2: StridedArrayView<'_, f64> = unsafe { - StridedArrayView::new( + let view2: StrideArrayView<'_, f64> = unsafe { + StrideArrayView::new( transposed_view.ndim(), - shape.as_ptr(), + transposed_view.shape().as_ptr(), strides.as_ptr(), transposed_view.as_ptr() as *const u8, transposed_view.len() * elem_size as usize, diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index d6ceef70..3113d14a 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -24,7 +24,7 @@ #![doc = include_str!("mod.md")] -pub use self::ndarr::{ArrayElement, ElemDataType, NdArrayView, StridedArrayView}; +pub use self::ndarr::{ArrayElement, ElemDataType, NdArrayView, StrideArrayView}; pub use self::timestamp::*; use crate::error::{self, Error, Result}; use crate::gai; @@ -1144,6 +1144,12 @@ impl Buffer { } let reserve_size = view.check_data_buf()?; + i32::try_from(reserve_size).map_err( + |_| error::fmt!( + ArrayViewError, + "Array total elem size overflow" + ) + )?; // binary format flag '=' self.output.push(b'='); // binary format entity type diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 2d731fde..d449deec 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -144,16 +144,16 @@ impl ArrayElement for f64 { /// A view into a multi-dimensional array with custom memory strides. #[derive(Debug)] -pub struct StridedArrayView<'a, T> { +pub struct StrideArrayView<'a, T> { dims: usize, - shape: &'a [u32], - strides: &'a [i32], + shape: &'a [usize], + strides: &'a [isize], buf_len: usize, buf: *const u8, _marker: std::marker::PhantomData, } -impl NdArrayView for StridedArrayView<'_, T> +impl NdArrayView for StrideArrayView<'_, T> where T: ArrayElement, { @@ -172,7 +172,7 @@ where return None; } - Some(self.shape[index] as usize) + Some(self.shape[index]) } fn as_slice(&self) -> Option<&[T]> { @@ -202,7 +202,7 @@ where .fold(self.buf, |ptr, (dim, &stride)| { if stride < 0 { let dim_size = self.shape[dim] as isize; - unsafe { ptr.offset(stride as isize * (dim_size - 1)) } + unsafe { ptr.offset(stride * (dim_size - 1)) } } else { ptr } @@ -220,7 +220,7 @@ where let mut size = size_of::(); for i in 0..self.dims { let d = self.shape[i]; - size = size.checked_mul(d as usize).ok_or(error::fmt!( + size = size.checked_mul(d).ok_or(error::fmt!( ArrayViewError, "Array total elem size overflow" ))? @@ -237,7 +237,7 @@ where } } -impl StridedArrayView<'_, T> +impl StrideArrayView<'_, T> where T: ArrayElement, { @@ -255,8 +255,8 @@ where /// - Strides are measured in bytes (not elements) pub unsafe fn new( dims: usize, - shape: *const u32, - strides: *const i32, + shape: *const usize, + strides: *const isize, data: *const u8, data_len: usize, ) -> Self { @@ -278,7 +278,7 @@ where return false; } - let elem_size = size_of::() as i32; + let elem_size = size_of::() as isize; if self.dims == 1 { return self.strides[0] == elem_size || self.shape[0] == 1; } @@ -288,7 +288,7 @@ where if *dim > 1 && stride != expected_stride { return false; } - expected_stride *= *dim as i32; + expected_stride *= *dim as isize; } true } @@ -297,10 +297,10 @@ where /// Iterator for traversing a strided array in row-major (C-style) order. pub struct RowMajorIter<'a, T> { base_ptr: *const u8, - array: &'a StridedArrayView<'a, T>, - dim_products: Vec, - current_linear: u32, - total_elements: u32, + array: &'a StrideArrayView<'a, T>, + dim_products: Vec, + current_linear: usize, + total_elements: usize, } impl<'a, T> Iterator for RowMajorIter<'a, T> @@ -329,7 +329,7 @@ where self.current_linear += 1; unsafe { - let ptr = self.base_ptr.offset(offset as isize); + let ptr = self.base_ptr.add(offset); Some(&*(ptr as *const T)) } } diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index 4ead055e..fd39f371 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -1,7 +1,7 @@ #[cfg(feature = "ndarray")] use crate::ingress::MAX_DIMS; use crate::ingress::{ - ArrayElement, Buffer, ElemDataType, NdArrayView, StridedArrayView, ARRAY_BINARY_FORMAT_TYPE, + ArrayElement, Buffer, ElemDataType, NdArrayView, StrideArrayView, ARRAY_BINARY_FORMAT_TYPE, }; use crate::tests::TestResult; use crate::ErrorCode; @@ -30,16 +30,16 @@ fn to_bytes(data: &[T]) -> Vec { } #[test] -fn test_strided_array_view() -> TestResult { +fn test_stride_array_view() -> TestResult { // contiguous layout let test_data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; - let shapes = [2u32, 3]; + let shapes = [2usize, 3]; let strides = [ - (shapes[1] * size_of::() as u32) as i32, - size_of::() as i32, + (shapes[1] * size_of::()) as isize, + size_of::() as isize, ]; let array = unsafe { - StridedArrayView::::new( + StrideArrayView::::new( shapes.len(), shapes.as_ptr(), strides.as_ptr(), @@ -62,13 +62,13 @@ fn test_strided_array_view() -> TestResult { #[test] fn test_strided_non_contiguous() -> TestResult { - let elem_size = size_of::() as i32; + let elem_size = size_of::() as isize; let col_major_data = [1.0, 3.0, 5.0, 2.0, 4.0, 6.0]; - let shapes = [3u32, 2]; - let strides = [elem_size, shapes[0] as i32 * elem_size]; + let shapes = [3usize, 2]; + let strides = [elem_size, shapes[0] as isize * elem_size]; - let array_view: StridedArrayView<'_, f64> = unsafe { - StridedArrayView::new( + let array_view: StrideArrayView<'_, f64> = unsafe { + StrideArrayView::new( shapes.len(), shapes.as_ptr(), strides.as_ptr(), @@ -101,10 +101,10 @@ fn test_negative_strides() -> TestResult { let elem_size = size_of::(); let data = [1f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]; let view = unsafe { - StridedArrayView::::new( + StrideArrayView::::new( 2, - &[3u32, 3] as *const u32, - &[-24i32, 8] as *const i32, + &[3usize, 3] as *const usize, + &[-24isize, 8] as *const isize, (data.as_ptr() as *const u8).add(48), data.len() * elem_size, ) @@ -128,17 +128,17 @@ fn test_negative_strides() -> TestResult { #[test] fn test_basic_edge_cases() { // empty array - let elem_size = std::mem::size_of::() as i32; - let empty_view: StridedArrayView<'_, f64> = - unsafe { StridedArrayView::new(2, [0, 0].as_ptr(), [0, 0].as_ptr(), ptr::null(), 0) }; + let elem_size = std::mem::size_of::() as isize; + let empty_view: StrideArrayView<'_, f64> = + unsafe { StrideArrayView::new(2, [0, 0].as_ptr(), [0, 0].as_ptr(), ptr::null(), 0) }; assert_eq!(empty_view.ndim(), 2); assert_eq!(empty_view.dim(0), Some(0)); assert_eq!(empty_view.dim(1), Some(0)); // single element array let single_data = [42.0]; - let single_view: StridedArrayView<'_, f64> = unsafe { - StridedArrayView::new( + let single_view: StrideArrayView<'_, f64> = unsafe { + StrideArrayView::new( 2, [1, 1].as_ptr(), [elem_size, elem_size].as_ptr(), @@ -153,11 +153,11 @@ fn test_basic_edge_cases() { #[test] fn test_buffer_basic_write() -> TestResult { - let elem_size = std::mem::size_of::() as i32; + let elem_size = std::mem::size_of::() as isize; let test_data = [1.1, 2.2, 3.3, 4.4]; - let array_view: StridedArrayView<'_, f64> = unsafe { - StridedArrayView::new( + let array_view: StrideArrayView<'_, f64> = unsafe { + StrideArrayView::new( 2, [2, 2].as_ptr(), [2 * elem_size, elem_size].as_ptr(), @@ -201,9 +201,9 @@ fn test_buffer_basic_write() -> TestResult { #[test] fn test_size_overflow() -> TestResult { let overflow_view = unsafe { - StridedArrayView::::new( + StrideArrayView::::new( 2, - [u32::MAX, u32::MAX].as_ptr(), + [u32::MAX as usize, u32::MAX as usize].as_ptr(), [8, 8].as_ptr(), ptr::null(), 0, @@ -221,10 +221,10 @@ fn test_size_overflow() -> TestResult { #[test] fn test_array_length_mismatch() -> TestResult { - let elem_size = size_of::() as i32; + let elem_size = size_of::() as isize; let under_data = [1.1]; - let under_view: StridedArrayView<'_, f64> = unsafe { - StridedArrayView::new( + let under_view: StrideArrayView<'_, f64> = unsafe { + StrideArrayView::new( 2, [1, 2].as_ptr(), [elem_size, elem_size].as_ptr(), @@ -243,8 +243,8 @@ fn test_array_length_mismatch() -> TestResult { .contains("Array buffer length mismatch (actual: 8, expected: 16)")); let over_data = [1.1, 2.2, 3.3]; - let over_view: StridedArrayView<'_, f64> = unsafe { - StridedArrayView::new( + let over_view: StrideArrayView<'_, f64> = unsafe { + StrideArrayView::new( 2, [1, 2].as_ptr(), [elem_size, elem_size].as_ptr(), diff --git a/system_test/questdb_line_sender.py b/system_test/questdb_line_sender.py index 1ca0573a..595049fd 100644 --- a/system_test/questdb_line_sender.py +++ b/system_test/questdb_line_sender.py @@ -54,12 +54,10 @@ c_size_t, c_char_p, c_int, - c_int32, c_int64, c_double, c_uint8, c_uint16, - c_uint32, c_uint64, c_void_p, c_ssize_t) @@ -119,14 +117,13 @@ class c_line_sender_error(ctypes.Structure): c_size_t_p = ctypes.POINTER(c_size_t) +c_ssize_t_p = ctypes.POINTER(c_ssize_t) c_line_sender_p = ctypes.POINTER(c_line_sender) c_line_sender_buffer_p = ctypes.POINTER(c_line_sender_buffer) c_line_sender_opts_p = ctypes.POINTER(c_line_sender_opts) c_line_sender_error_p = ctypes.POINTER(c_line_sender_error) c_line_sender_error_p_p = ctypes.POINTER(c_line_sender_error_p) -c_int32_p = ctypes.POINTER(c_int32) c_uint8_p = ctypes.POINTER(c_uint8) -c_uint32_p = ctypes.POINTER(c_uint32) class c_line_sender_utf8(ctypes.Structure): @@ -291,8 +288,8 @@ def set_sig(fn, restype, *argtypes): c_line_sender_buffer_p, c_line_sender_column_name, c_size_t, - c_uint32_p, - c_int32_p, + c_size_t_p, + c_ssize_t_p, c_uint8_p, c_size_t, c_line_sender_error_p_p) @@ -724,8 +721,8 @@ def _convert_tuple(tpl: tuple[int, ...], c_type: type, name: str) -> ctypes.POIN f"{name} value exceeds {c_type.__name__} range" ) from e - c_shapes = _convert_tuple(shapes, c_uint32, "shapes") - c_strides = _convert_tuple(strides, c_int32, "strides") + c_shapes = _convert_tuple(shapes, c_size_t, "shapes") + c_strides = _convert_tuple(strides, c_ssize_t, "strides") _error_wrapped_call( _DLL.line_sender_buffer_column_f64_arr, self._impl, From 6678679d52c151d260dcff3826be4e100d1a7ed5 Mon Sep 17 00:00:00 2001 From: victorgao Date: Thu, 8 May 2025 23:10:47 +0800 Subject: [PATCH 057/119] code format and cpp compile fix. --- cpp_test/mock_server.cpp | 1 + questdb-rs/src/ingress/mod.rs | 8 ++------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/cpp_test/mock_server.cpp b/cpp_test/mock_server.cpp index 6a93666c..e21d77d0 100644 --- a/cpp_test/mock_server.cpp +++ b/cpp_test/mock_server.cpp @@ -24,6 +24,7 @@ #include "mock_server.hpp" +#include #include #if defined(PLATFORM_UNIX) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 3113d14a..71e4e517 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -1144,12 +1144,8 @@ impl Buffer { } let reserve_size = view.check_data_buf()?; - i32::try_from(reserve_size).map_err( - |_| error::fmt!( - ArrayViewError, - "Array total elem size overflow" - ) - )?; + i32::try_from(reserve_size) + .map_err(|_| error::fmt!(ArrayViewError, "Array total elem size overflow"))?; // binary format flag '=' self.output.push(b'='); // binary format entity type From 6f1e7e3d07940f826e9e67f76b051931dcf9e68d Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Thu, 8 May 2025 22:25:02 +0100 Subject: [PATCH 058/119] fixed encapsulation issue (we were leaking internal types to the public api) --- questdb-rs/src/ingress/mod.rs | 16 +++---- questdb-rs/src/ingress/ndarr.rs | 53 ++++++++-------------- questdb-rs/src/tests/mock.rs | 10 ++--- questdb-rs/src/tests/ndarr.rs | 78 +++++++++++++++++++++------------ questdb-rs/src/tests/sender.rs | 10 +++-- 5 files changed, 88 insertions(+), 79 deletions(-) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 3113d14a..90f16ea9 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -24,13 +24,14 @@ #![doc = include_str!("mod.md")] -pub use self::ndarr::{ArrayElement, ElemDataType, NdArrayView, StrideArrayView}; +pub use self::ndarr::{ArrayElement, NdArrayView, StrideArrayView}; pub use self::timestamp::*; use crate::error::{self, Error, Result}; use crate::gai; use crate::ingress::conf::ConfigSetting; use base64ct::{Base64, Base64UrlUnpadded, Encoding}; use core::time::Duration; +use ndarr::ArrayElementSealed; use rustls::{ClientConnection, RootCertStore, StreamOwned}; use rustls_pki_types::ServerName; use socket2::{Domain, Protocol as SockProtocol, SockAddr, Socket, Type}; @@ -1111,11 +1112,12 @@ impl Buffer { /// - Array dimensions exceed [`MAX_DIMS`] /// - Failed to get dimension sizes /// - Column name validation fails + #[allow(private_bounds)] pub fn column_arr<'a, N, T, D>(&mut self, name: N, view: &T) -> Result<&mut Self> where N: TryInto>, T: NdArrayView, - D: ArrayElement, + D: ArrayElement + ArrayElementSealed, Error: From, { if self.version == LineProtocolVersion::V1 { @@ -1144,18 +1146,14 @@ impl Buffer { } let reserve_size = view.check_data_buf()?; - i32::try_from(reserve_size).map_err( - |_| error::fmt!( - ArrayViewError, - "Array total elem size overflow" - ) - )?; + i32::try_from(reserve_size) + .map_err(|_| error::fmt!(ArrayViewError, "Array total elem size overflow"))?; // binary format flag '=' self.output.push(b'='); // binary format entity type self.output.push(ARRAY_BINARY_FORMAT_TYPE); // ndarr datatype - self.output.push(D::elem_type().into()); + self.output.push(D::type_tag()); // ndarr dims self.output.push(view.ndim() as u8); diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index d449deec..62bb6f1a 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -101,44 +101,19 @@ where /// /// Implemented for primitive types that can be stored in arrays. /// Combines type information with data type classification. -pub trait ArrayElement: Copy + 'static { - /// Returns the corresponding data type classification. - /// - /// This enables runtime type identification while maintaining - /// compile-time type safety. - fn elem_type() -> ElemDataType; -} +pub trait ArrayElement: Copy + 'static {} -/// Defines binary format identifiers for array element types compatible with -/// QuestDB's ColumnType: . -#[repr(u8)] -#[derive(Debug, PartialEq, Clone, Copy)] -pub enum ElemDataType { - /// 64-bit floating point - Double = 0x0A, +pub(crate) trait ArrayElementSealed { + /// Returns the binary format identifier for array element types compatible + /// with QuestDB's io.questdb.cairo.ColumnType numeric type constants. + fn type_tag() -> u8; } -impl From for u8 { - fn from(val: ElemDataType) -> Self { - val as u8 - } -} +impl ArrayElement for f64 {} -impl TryFrom for ElemDataType { - type Error = String; - - fn try_from(value: u8) -> Result { - match value { - 0x0A => Ok(ElemDataType::Double), - _ => Err(format!("Unknown element type: {}", value)), - } - } -} - -impl ArrayElement for f64 { - /// Identifies f64 as Double type in QuestDB's type system. - fn elem_type() -> ElemDataType { - ElemDataType::Double +impl ArrayElementSealed for f64 { + fn type_tag() -> u8 { + 10 // Double } } @@ -685,3 +660,13 @@ where Ok(self.len() * size_of::()) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_f64_element_type() { + assert_eq!(::type_tag(), 10); + } +} diff --git a/questdb-rs/src/tests/mock.rs b/questdb-rs/src/tests/mock.rs index ade56414..71d63c29 100644 --- a/questdb-rs/src/tests/mock.rs +++ b/questdb-rs/src/tests/mock.rs @@ -22,7 +22,7 @@ * ******************************************************************************/ -use crate::ingress::{ElemDataType, Protocol, SenderBuilder}; +use crate::ingress::{Protocol, SenderBuilder}; use core::time::Duration; use mio::event::Event; @@ -40,6 +40,8 @@ use crate::ingress; #[cfg(feature = "ilp-over-http")] use std::io::Write; +use super::ndarr::ArrayColumnTypeTag; + const CLIENT: Token = Token(0); #[derive(Debug)] @@ -574,15 +576,13 @@ impl MockServer { index += size_of::() + 1; } else if binary_type == ingress::ARRAY_BINARY_FORMAT_TYPE { index += 1; - let element_type = match ElemDataType::try_from(accum[index]) { + let element_type = match ArrayColumnTypeTag::try_from(accum[index]) { Ok(t) => t, Err(e) => { return Err(io::Error::new(io::ErrorKind::Other, e)); } }; - let mut elems_size = match element_type { - ElemDataType::Double => size_of::(), - }; + let mut elems_size = element_type.size(); index += 1; let dims = accum[index] as usize; index += 1; diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index fd39f371..ab0b984b 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -1,8 +1,6 @@ #[cfg(feature = "ndarray")] use crate::ingress::MAX_DIMS; -use crate::ingress::{ - ArrayElement, Buffer, ElemDataType, NdArrayView, StrideArrayView, ARRAY_BINARY_FORMAT_TYPE, -}; +use crate::ingress::{Buffer, NdArrayView, StrideArrayView, ARRAY_BINARY_FORMAT_TYPE}; use crate::tests::TestResult; use crate::ErrorCode; @@ -13,10 +11,36 @@ use ndarray::{arr1, arr2, arr3, s, ArrayD}; use std::iter; use std::ptr; -#[test] -fn test_f64_element_type() { - assert_eq!(::elem_type(), ElemDataType::Double); - assert_eq!(u8::from(ElemDataType::Double), 10); +/// QuestDB column type tags that are supported as array element types. +#[derive(Clone, Copy)] +#[repr(u8)] +pub enum ArrayColumnTypeTag { + Double = 10, +} + +impl ArrayColumnTypeTag { + pub fn size(&self) -> usize { + match self { + ArrayColumnTypeTag::Double => std::mem::size_of::(), + } + } +} + +impl From for u8 { + fn from(tag: ArrayColumnTypeTag) -> Self { + tag as u8 + } +} + +impl TryFrom for ArrayColumnTypeTag { + type Error = String; + + fn try_from(value: u8) -> Result { + match value { + 10 => Ok(ArrayColumnTypeTag::Double), + _ => Err(format!("Unsupported column type tag {} for arrays", value)), + } + } } fn to_bytes(data: &[T]) -> Vec { @@ -177,7 +201,7 @@ fn test_buffer_basic_write() -> TestResult { b'=', b'=', ARRAY_BINARY_FORMAT_TYPE, - ElemDataType::Double.into(), + ArrayColumnTypeTag::Double.into(), 2u8 ] ); @@ -287,7 +311,7 @@ fn test_build_in_1d_array_normal() -> TestResult { b'=', b'=', ARRAY_BINARY_FORMAT_TYPE, - ElemDataType::Double.into(), + ArrayColumnTypeTag::Double.into(), 1u8 ] ); @@ -325,7 +349,7 @@ fn test_build_in_1d_array_empty() -> TestResult { b'=', b'=', ARRAY_BINARY_FORMAT_TYPE, - ElemDataType::Double.into(), + ArrayColumnTypeTag::Double.into(), 1u8 ] ); @@ -355,7 +379,7 @@ fn test_build_in_1d_vec_normal() -> TestResult { b'=', b'=', ARRAY_BINARY_FORMAT_TYPE, - ElemDataType::Double.into(), + ArrayColumnTypeTag::Double.into(), 1u8 ] ); @@ -392,7 +416,7 @@ fn test_build_in_1d_vec_empty() -> TestResult { b'=', b'=', ARRAY_BINARY_FORMAT_TYPE, - ElemDataType::Double.into(), + ArrayColumnTypeTag::Double.into(), 1u8 ] ); @@ -421,7 +445,7 @@ fn test_build_in_1d_slice_normal() -> TestResult { b'=', b'=', ARRAY_BINARY_FORMAT_TYPE, - ElemDataType::Double.into(), + ArrayColumnTypeTag::Double.into(), 1u8 ] ); @@ -454,7 +478,7 @@ fn test_build_in_1d_slice_empty() -> TestResult { b'=', b'=', ARRAY_BINARY_FORMAT_TYPE, - ElemDataType::Double.into(), + ArrayColumnTypeTag::Double.into(), 1u8 ] ); @@ -488,7 +512,7 @@ fn test_build_in_2d_array_normal() -> TestResult { b'=', b'=', ARRAY_BINARY_FORMAT_TYPE, - ElemDataType::Double.into(), + ArrayColumnTypeTag::Double.into(), 2u8 ] ); @@ -532,7 +556,7 @@ fn test_build_in_2d_array_empty() -> TestResult { b'=', b'=', ARRAY_BINARY_FORMAT_TYPE, - ElemDataType::Double.into(), + ArrayColumnTypeTag::Double.into(), 2u8 ] ); @@ -566,7 +590,7 @@ fn test_build_in_2d_vec_normal() -> TestResult { b'=', b'=', ARRAY_BINARY_FORMAT_TYPE, - ElemDataType::Double.into(), + ArrayColumnTypeTag::Double.into(), 2u8 ] ); @@ -621,7 +645,7 @@ fn test_build_in_2d_vec_empty() -> TestResult { b'=', b'=', ARRAY_BINARY_FORMAT_TYPE, - ElemDataType::Double.into(), + ArrayColumnTypeTag::Double.into(), 2u8 ] ); @@ -657,7 +681,7 @@ fn test_build_in_2d_slice_normal() -> TestResult { b'=', b'=', ARRAY_BINARY_FORMAT_TYPE, - ElemDataType::Double.into(), + ArrayColumnTypeTag::Double.into(), 2u8 ] ); @@ -700,7 +724,7 @@ fn test_build_in_2d_slice_empty() -> TestResult { b'=', b'=', ARRAY_BINARY_FORMAT_TYPE, - ElemDataType::Double.into(), + ArrayColumnTypeTag::Double.into(), 2u8 ] ); @@ -738,7 +762,7 @@ fn test_build_in_3d_array_normal() -> TestResult { b'=', b'=', ARRAY_BINARY_FORMAT_TYPE, - ElemDataType::Double.into(), + ArrayColumnTypeTag::Double.into(), 3u8 ] ); @@ -785,7 +809,7 @@ fn test_build_in_3d_array_empty() -> TestResult { b'=', b'=', ARRAY_BINARY_FORMAT_TYPE, - ElemDataType::Double.into(), + ArrayColumnTypeTag::Double.into(), 3u8 ] ); @@ -826,7 +850,7 @@ fn test_build_in_3d_vec_normal() -> TestResult { b'=', b'=', ARRAY_BINARY_FORMAT_TYPE, - ElemDataType::Double.into(), + ArrayColumnTypeTag::Double.into(), 3u8 ] ); @@ -877,7 +901,7 @@ fn test_build_in_3d_vec_empty() -> TestResult { b'=', b'=', ARRAY_BINARY_FORMAT_TYPE, - ElemDataType::Double.into(), + ArrayColumnTypeTag::Double.into(), 3u8 ] ); @@ -937,7 +961,7 @@ fn test_3d_slice_normal() -> TestResult { b'=', b'=', ARRAY_BINARY_FORMAT_TYPE, - ElemDataType::Double.into(), + ArrayColumnTypeTag::Double.into(), 3u8 ] ); @@ -981,7 +1005,7 @@ fn test_3d_slice_empty() -> TestResult { b'=', b'=', ARRAY_BINARY_FORMAT_TYPE, - ElemDataType::Double.into(), + ArrayColumnTypeTag::Double.into(), 3u8 ] ); @@ -1087,7 +1111,7 @@ fn test_buffer_ndarray_write() -> TestResult { b'=', b'=', ARRAY_BINARY_FORMAT_TYPE, - ElemDataType::Double.into(), + ArrayColumnTypeTag::Double.into(), 2u8 ] ); diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index 81957e26..5a6bf910 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -29,6 +29,8 @@ use crate::{ Error, ErrorCode, }; +#[cfg(feature = "ndarray")] +use crate::ingress; #[cfg(feature = "ndarray")] use crate::ingress::ndarr::write_array_data; use crate::ingress::LineProtocolVersion; @@ -36,8 +38,6 @@ use crate::tests::{ mock::{certs_dir, MockServer}, TestResult, }; -#[cfg(feature = "ndarray")] -use crate::{ingress, ingress::ElemDataType}; use core::time::Duration; #[cfg(feature = "ndarray")] use ndarray::{arr1, arr2, ArrayD}; @@ -99,6 +99,8 @@ fn test_basics( #[cfg(feature = "ndarray")] #[test] fn test_array_basic() -> TestResult { + use crate::tests::ndarr::ArrayColumnTypeTag; + let mut server = MockServer::new()?; let mut sender = server.lsb_tcp().build()?; server.accept()?; @@ -124,7 +126,7 @@ fn test_array_basic() -> TestResult { let array_header2d = &[ &[b'='][..], &[ingress::ARRAY_BINARY_FORMAT_TYPE], - &[ElemDataType::Double as u8], + &[ArrayColumnTypeTag::Double.into()], &[2u8], &2i32.to_le_bytes(), &2i32.to_le_bytes(), @@ -136,7 +138,7 @@ fn test_array_basic() -> TestResult { let array_header3d = &[ &[b'='][..], &[ingress::ARRAY_BINARY_FORMAT_TYPE], - &[ElemDataType::Double as u8], + &[ArrayColumnTypeTag::Double.into()], &[3u8], &2i32.to_le_bytes(), &3i32.to_le_bytes(), From 8d739f1cf895bcf44b22db95eb0f5b50a2eebeb4 Mon Sep 17 00:00:00 2001 From: victor Date: Fri, 9 May 2025 09:05:54 +0800 Subject: [PATCH 059/119] typo fixed. --- questdb-rs/src/ingress/ndarr.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 62bb6f1a..829a917f 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -269,7 +269,7 @@ where } } -/// Iterator for traversing a strided array in row-major (C-style) order. +/// Iterator for traversing a stride array in row-major (C-style) order. pub struct RowMajorIter<'a, T> { base_ptr: *const u8, array: &'a StrideArrayView<'a, T>, From 710a69f063c3acef99fb1922bf211541799f87b9 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Fri, 9 May 2025 17:36:20 +0100 Subject: [PATCH 060/119] cleanup and added todo comments --- questdb-rs/src/ingress/mod.rs | 75 ++++++++++++++++++++++++--------- questdb-rs/src/ingress/ndarr.rs | 2 + questdb-rs/src/tests/sender.rs | 65 +++++++++++++++++++++++----- 3 files changed, 113 insertions(+), 29 deletions(-) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 90f16ea9..59b4b796 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -1126,7 +1126,8 @@ impl Buffer { "line protocol version v1 does not support array datatype", )); } - if view.ndim() == 0 { + let ndim = view.ndim(); + if ndim == 0 { return Err(error::fmt!( ArrayViewError, "Zero-dimensional arrays are not supported", @@ -1136,18 +1137,31 @@ impl Buffer { self.write_column_key(name)?; // check dimension less equal than max dims - if MAX_DIMS < view.ndim() { + if MAX_DIMS < ndim { return Err(error::fmt!( ArrayHasTooManyDims, "Array dimension mismatch: expected at most {} dimensions, but got {}", MAX_DIMS, - view.ndim() + ndim + )); + } + + // TODO: Remove `check_data_buf` this from the trait. + // It's private impl details that can be coded generically + let array_buf_size = view.check_data_buf()?; + if array_buf_size > i32::MAX as usize { + // TODO: We should probably agree on a significantly + // _smaller_ limit here, since there's no way + // we've ever tested anything that big. + // My gut feeling is that the maximum array buffer should be + // in the order of 100MB or so. + return Err(error::fmt!( + ArrayViewError, + "Array buffer size too big: {}", + array_buf_size )); } - let reserve_size = view.check_data_buf()?; - i32::try_from(reserve_size) - .map_err(|_| error::fmt!(ArrayViewError, "Array total elem size overflow"))?; // binary format flag '=' self.output.push(b'='); // binary format entity type @@ -1155,27 +1169,50 @@ impl Buffer { // ndarr datatype self.output.push(D::type_tag()); // ndarr dims - self.output.push(view.ndim() as u8); + self.output.push(ndim as u8); - for i in 0..view.ndim() { - let d = view.dim(i).ok_or_else(|| { + let dim_header_size = size_of::() * ndim; + self.output.reserve(dim_header_size + array_buf_size); + + for i in 0..ndim { + let dim = view.dim(i).ok_or_else(|| { error::fmt!( ArrayViewError, - "Can not get correct dimensions for dim {}", + "Cannot get correct dimensions for dim {}", i ) })?; + + // TODO: check that the dimension is not past + // the maximum size that the java impl will accept. + // I seem to remember that it's 2^28-1 or something like that. + // Must check Java impl. + // ndarr shapes self.output - .extend_from_slice((d as i32).to_le_bytes().as_slice()); + .extend_from_slice((dim as u32).to_le_bytes().as_slice()); } - self.output.reserve(reserve_size); let index = self.output.len(); let writeable = - unsafe { from_raw_parts_mut(self.output.as_mut_ptr().add(index), reserve_size) }; + unsafe { from_raw_parts_mut(self.output.as_mut_ptr().add(index), array_buf_size) }; let mut cursor = Cursor::new(writeable); + // TODO: The next section needs a bit of a rewrite. + // It also needs clear comments that explain the design decisions. + // + // I'd be expecting two code paths here: + // 1. The array is row-major contiguous + // 2. The data needs to be written out via the strides. + // + // The code here seems to do something a bit different and + // is worth explaining. + // I see two code paths that I honestly don't understand, + // the `ndarr::write_array_data` and the `ndarr::write_array_data_use_raw_buffer` + // functions both seem to handle both cases (why?) and then seem + // to construct a vectored IoSlice buffer (why??) before writing + // the strided data out. + // ndarr data if view.as_slice().is_some() { if let Err(e) = ndarr::write_array_data(view, &mut cursor) { @@ -1185,17 +1222,17 @@ impl Buffer { e )); } - if cursor.position() != (reserve_size as u64) { + if cursor.position() != (array_buf_size as u64) { return Err(error::fmt!( ArrayWriteToBufferError, "Array write buffer length mismatch (actual: {}, expected: {})", cursor.position(), - reserve_size + array_buf_size )); } - unsafe { self.output.set_len(reserve_size + index) } + unsafe { self.output.set_len(array_buf_size + index) } } else { - unsafe { self.output.set_len(reserve_size + index) } + unsafe { self.output.set_len(array_buf_size + index) } ndarr::write_array_data_use_raw_buffer(&mut self.output[index..], view); } Ok(self) @@ -1300,7 +1337,7 @@ impl Buffer { /// or you can also pass in a `TimestampNanos`. /// /// Note that both `TimestampMicros` and `TimestampNanos` can be constructed - /// easily from either `chrono::DateTime` and `std::time::SystemTime`. + /// easily from either `std::time::SystemTime` or `chrono::DateTime`. /// /// This last option requires the `chrono_timestamp` feature. pub fn column_ts<'a, N, T>(&mut self, name: N, value: T) -> Result<&mut Self> @@ -1354,7 +1391,7 @@ impl Buffer { /// You can also pass in a `TimestampMicros`. /// /// Note that both `TimestampMicros` and `TimestampNanos` can be constructed - /// easily from either `chrono::DateTime` and `std::time::SystemTime`. + /// easily from either `std::time::SystemTime` or `chrono::DateTime`. /// pub fn at(&mut self, timestamp: T) -> Result<()> where diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 62bb6f1a..4be5f0c7 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -123,6 +123,8 @@ pub struct StrideArrayView<'a, T> { dims: usize, shape: &'a [usize], strides: &'a [isize], + + // TODO: Why a pointer and len? Shouldn't it be a `&'a [u8]` slice? buf_len: usize, buf: *const u8, _marker: std::marker::PhantomData, diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index 5a6bf910..1b812b5b 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -29,25 +29,27 @@ use crate::{ Error, ErrorCode, }; -#[cfg(feature = "ndarray")] use crate::ingress; #[cfg(feature = "ndarray")] use crate::ingress::ndarr::write_array_data; use crate::ingress::LineProtocolVersion; use crate::tests::{ mock::{certs_dir, MockServer}, + ndarr::ArrayColumnTypeTag, TestResult, }; use core::time::Duration; #[cfg(feature = "ndarray")] use ndarray::{arr1, arr2, ArrayD}; use rstest::rstest; -use std::{io, time::SystemTime}; +use std::io; #[rstest] fn test_basics( #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, ) -> TestResult { + use std::time::SystemTime; + let mut server = MockServer::new()?; let mut sender = server.lsb_tcp().build()?; assert!(!sender.must_close()); @@ -96,18 +98,59 @@ fn test_basics( Ok(()) } -#[cfg(feature = "ndarray")] #[test] -fn test_array_basic() -> TestResult { - use crate::tests::ndarr::ArrayColumnTypeTag; +fn test_array_f64_basic() -> TestResult { + let mut server = MockServer::new()?; + let mut sender = server.lsb_tcp().build()?; + server.accept()?; + let ts = TimestampNanos::now(); + + let mut buffer = + Buffer::new().with_line_proto_version(sender.default_line_protocol_version())?; + buffer + .table("my_table")? + .symbol("device", "A001")? + .column_f64("f1", 25.5)? + .column_arr("arr1d", &[1.0, 2.0, 3.0])? + .at(ts)?; + + assert_eq!(server.recv_q()?, 0); + + let exp = &[ + b"my_table,device=A001 ", + f64_to_bytes("f1", 25.5, LineProtocolVersion::V2).as_slice(), + b",arr1d=", + b"=", // binary field + &[ingress::ARRAY_BINARY_FORMAT_TYPE], + &[ArrayColumnTypeTag::Double.into()], + &[1u8], // 1D array + &3u32.to_le_bytes(), // 3 elements + &1.0f64.to_le_bytes(), + &2.0f64.to_le_bytes(), + &3.0f64.to_le_bytes(), + format!(" {}\n", ts.as_i64()).as_bytes(), + ] + .concat(); + + assert_eq!(buffer.as_bytes(), exp); + assert_eq!(buffer.len(), exp.len()); + sender.flush(&mut buffer)?; + assert_eq!(buffer.len(), 0); + assert_eq!(buffer.as_bytes(), b""); + assert_eq!(server.recv_q()?, 1); + assert_eq!(server.msgs[0].as_slice(), exp); + Ok(()) +} + +#[cfg(feature = "ndarray")] +#[test] +fn test_array_f64_from_ndarray() -> TestResult { let mut server = MockServer::new()?; let mut sender = server.lsb_tcp().build()?; server.accept()?; - let ts = SystemTime::now(); - let ts_nanos_num = ts.duration_since(SystemTime::UNIX_EPOCH)?.as_nanos() as i64; - let ts_nanos = TimestampNanos::from_systemtime(ts)?; + let ts = TimestampNanos::now(); let array_2d = arr2(&[[1.1, 2.2], [3.3, 4.4]]); let array_3d = ArrayD::::ones(vec![2, 3, 4]); @@ -119,12 +162,12 @@ fn test_array_basic() -> TestResult { .column_f64("f1", 25.5)? .column_arr("arr2d", &array_2d.view())? .column_arr("arr3d", &array_3d.view())? - .at(ts_nanos)?; + .at(ts)?; assert_eq!(server.recv_q()?, 0); let array_header2d = &[ - &[b'='][..], + &[b'='], &[ingress::ARRAY_BINARY_FORMAT_TYPE], &[ArrayColumnTypeTag::Double.into()], &[2u8], @@ -362,6 +405,8 @@ fn test_bad_key( #[test] fn test_timestamp_overloads() -> TestResult { + use std::time::SystemTime; + let tbl_name = TableName::new("tbl_name")?; let mut buffer = Buffer::new(); From 75d01aeb715d08abeaa7b17c4e601aa0050321f0 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Fri, 9 May 2025 17:37:28 +0100 Subject: [PATCH 061/119] removed slow build debug artifacts upload which we generally don't need --- ci/run_tests_pipeline.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml index 25434444..03b8d129 100644 --- a/ci/run_tests_pipeline.yaml +++ b/ci/run_tests_pipeline.yaml @@ -69,10 +69,10 @@ stages: env: JAVA_HOME: $(JAVA_HOME_11_X64) displayName: "Tests" - - task: PublishBuildArtifacts@1 - inputs: - pathToPublish: ./build - displayName: "Publish build directory" + # - task: PublishBuildArtifacts@1 + # inputs: + # pathToPublish: ./build + # displayName: "Publish build directory" - job: FormatAndLinting displayName: "cargo fmt and clippy" pool: From b091bf62cede3153a74bb3f09ad104026b5b1e7f Mon Sep 17 00:00:00 2001 From: victor Date: Mon, 12 May 2025 16:34:32 +0800 Subject: [PATCH 062/119] fix failed tests. --- questdb-rs/src/tests/sender.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index 1b812b5b..4ff58023 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -167,7 +167,7 @@ fn test_array_f64_from_ndarray() -> TestResult { assert_eq!(server.recv_q()?, 0); let array_header2d = &[ - &[b'='], + &[b'='][..], &[ingress::ARRAY_BINARY_FORMAT_TYPE], &[ArrayColumnTypeTag::Double.into()], &[2u8], @@ -200,7 +200,7 @@ fn test_array_f64_from_ndarray() -> TestResult { ",arr3d=".as_bytes(), array_header3d, array_data3d.as_slice(), - format!(" {}\n", ts_nanos_num).as_bytes(), + format!(" {}\n", ts.as_i64()).as_bytes(), ] .concat(); From ca4f47c18ccbb3f7c536deac707769a663a3e5ee Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 13 May 2025 19:50:26 +0800 Subject: [PATCH 063/119] optimize internal implementation --- questdb-rs/benches/ndarr.rs | 1 + questdb-rs/src/error.rs | 2 +- questdb-rs/src/ingress/mod.rs | 58 ++--- questdb-rs/src/ingress/ndarr.rs | 400 ++++++++++++++++++-------------- questdb-rs/src/tests/ndarr.rs | 168 ++++++-------- 5 files changed, 319 insertions(+), 310 deletions(-) diff --git a/questdb-rs/benches/ndarr.rs b/questdb-rs/benches/ndarr.rs index 9491644c..c6643ab5 100644 --- a/questdb-rs/benches/ndarr.rs +++ b/questdb-rs/benches/ndarr.rs @@ -98,6 +98,7 @@ fn bench_array_view(c: &mut Criterion) { transposed_view.as_ptr() as *const u8, transposed_view.len() * elem_size as usize, ) + .unwrap() }; // Case 2 diff --git a/questdb-rs/src/error.rs b/questdb-rs/src/error.rs index cadb0971..680a6b50 100644 --- a/questdb-rs/src/error.rs +++ b/questdb-rs/src/error.rs @@ -49,7 +49,7 @@ pub enum ErrorCode { /// Bad configuration. ConfigError, - /// Array has too many dims. Currently, only arrays with a maximum [`crate::ingress::MAX_DIMS`] dimensions are supported. + /// Array has too many dims. Currently, only arrays with a maximum [`crate::ingress::MAX_ARRAY_DIMS`] dimensions are supported. ArrayHasTooManyDims, /// Array view internal error. diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index c04576ff..89382ca3 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -58,7 +58,7 @@ use ring::{ }; /// Defines the maximum allowed dimensions for array data in binary serialization protocols. -pub const MAX_DIMS: usize = 32; +pub const MAX_ARRAY_DIMS: usize = 32; /// Line Protocol Version supported by current client. #[derive(Debug, Copy, Clone, PartialEq)] @@ -1071,7 +1071,7 @@ impl Buffer { /// Record a multidimensional array value for the given column. /// - /// Supports arrays with up to [`MAX_DIMS`] dimensions. The array elements must + /// Supports arrays with up to [`MAX_ARRAY_DIMS`] dimensions. The array elements must /// implement [`ArrayElement`] trait which provides type-to-[`ElemDataType`] mapping. /// /// # Examples @@ -1118,7 +1118,7 @@ impl Buffer { /// # Errors /// /// Returns [`Error`] if: - /// - Array dimensions exceed [`MAX_DIMS`] + /// - Array dimensions exceed [`MAX_ARRAY_DIMS`] /// - Failed to get dimension sizes /// - Column name validation fails #[allow(private_bounds)] @@ -1143,34 +1143,18 @@ impl Buffer { )); } - self.write_column_key(name)?; - // check dimension less equal than max dims - if MAX_DIMS < ndim { + if MAX_ARRAY_DIMS < ndim { return Err(error::fmt!( ArrayHasTooManyDims, "Array dimension mismatch: expected at most {} dimensions, but got {}", - MAX_DIMS, + MAX_ARRAY_DIMS, ndim )); } - // TODO: Remove `check_data_buf` this from the trait. - // It's private impl details that can be coded generically - let array_buf_size = view.check_data_buf()?; - if array_buf_size > i32::MAX as usize { - // TODO: We should probably agree on a significantly - // _smaller_ limit here, since there's no way - // we've ever tested anything that big. - // My gut feeling is that the maximum array buffer should be - // in the order of 100MB or so. - return Err(error::fmt!( - ArrayViewError, - "Array buffer size too big: {}", - array_buf_size - )); - } - + let array_buf_size = get_and_check_array_bytes_size(view)?; + self.write_column_key(name)?; // binary format flag '=' self.output.push(b'='); // binary format entity type @@ -1184,19 +1168,16 @@ impl Buffer { self.output.reserve(dim_header_size + array_buf_size); for i in 0..ndim { - let dim = view.dim(i).ok_or_else(|| { - error::fmt!( + let dim = view.dim(i)?; + if dim > MAX_ARRAY_DIM_LEN { + return Err(error::fmt!( ArrayViewError, - "Cannot get correct dimensions for dim {}", - i - ) - })?; - - // TODO: check that the dimension is not past - // the maximum size that the java impl will accept. - // I seem to remember that it's 2^28-1 or something like that. - // Must check Java impl. - + "dimension length out of range: dim {}, dim length {}, max length {}", + i, + dim, + MAX_ARRAY_DIM_LEN + )); + } // ndarr shapes self.output .extend_from_slice((dim as u32).to_le_bytes().as_slice()); @@ -1258,16 +1239,16 @@ impl Buffer { self.write_column_key(name)?; // check dimension less equal than max dims - if MAX_DIMS < view.ndim() { + if MAX_ARRAY_DIMS < view.ndim() { return Err(error::fmt!( ArrayHasTooManyDims, "Array dimension mismatch: expected at most {} dimensions, but got {}", - MAX_DIMS, + MAX_ARRAY_DIMS, view.ndim() )); } - let reserve_size = view.check_data_buf()?; + let reserve_size = get_and_check_array_bytes_size(view)?; // binary format flag '=' self.output.push(b'='); // binary format entity type @@ -3156,6 +3137,7 @@ mod timestamp; #[cfg(feature = "ilp-over-http")] mod http; +use crate::ingress::ndarr::{get_and_check_array_bytes_size, MAX_ARRAY_DIM_LEN}; #[cfg(feature = "ilp-over-http")] use http::*; diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index a4a1a14b..6a3a07f1 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -11,7 +11,7 @@ where fn ndim(&self) -> usize; /// Returns the size of the specified dimension. - fn dim(&self, index: usize) -> Option; + fn dim(&self, index: usize) -> Result; /// Return the array’s data as a slice, if it is c-major-layout. /// Return `None` otherwise. @@ -20,16 +20,17 @@ where /// Return an iterator of references to the elements of the array. /// Iterator element type is `&T`. fn iter(&self) -> Self::Iter<'_>; - - /// Validates the data buffer size of array is consistency with array shapes. - /// - /// # Returns - /// - `Ok(usize)`: Expected buffer size in bytes if valid - /// - `Err(Error)`: Otherwise - fn check_data_buf(&self) -> Result; } -pub fn write_array_data, T>( +// TODO: We should probably agree on a significantly +// _smaller_ limit here, since there's no way +// we've ever tested anything that big. +// My gut feeling is that the maximum array buffer should be +// in the order of 100MB or so. +const MAX_ARRAY_BUFFER_SIZE: usize = i32::MAX as usize; +pub(crate) const MAX_ARRAY_DIM_LEN: usize = 0x0FFF_FFFF; // 1 << 28 - 1 + +pub(crate) fn write_array_data, T>( array: &A, writer: &mut W, ) -> std::io::Result<()> @@ -97,6 +98,25 @@ where } } +pub(crate) fn get_and_check_array_bytes_size, T>( + array: &A, +) -> Result +where + T: ArrayElement, +{ + (0..array.ndim()) + .try_fold(std::mem::size_of::(), |acc, i| Ok(acc * array.dim(i)?)) + .and_then(|p| match p <= MAX_ARRAY_BUFFER_SIZE { + true => Ok(p), + false => Err(error::fmt!( + ArrayViewError, + "Array buffer size too big: {}, maximum: {}", + p, + MAX_ARRAY_BUFFER_SIZE + )), + }) +} + /// Marker trait for valid array element types. /// /// Implemented for primitive types that can be stored in arrays. @@ -123,10 +143,7 @@ pub struct StrideArrayView<'a, T> { dims: usize, shape: &'a [usize], strides: &'a [isize], - - // TODO: Why a pointer and len? Shouldn't it be a `&'a [u8]` slice? - buf_len: usize, - buf: *const u8, + data: Option<&'a [u8]>, _marker: std::marker::PhantomData, } @@ -144,21 +161,26 @@ where self.dims } - fn dim(&self, index: usize) -> Option { + fn dim(&self, index: usize) -> Result { if index >= self.dims { - return None; + return Err(error::fmt!( + ArrayViewError, + "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", + index, + self.dims + )); } - - Some(self.shape[index]) + Ok(self.shape[index]) } fn as_slice(&self) -> Option<&[T]> { - if self.is_c_major() { - Some(unsafe { - slice::from_raw_parts(self.buf as *const T, self.buf_len / size_of::()) - }) - } else { - None + unsafe { + self.is_c_major().then_some(self.data.and_then(|data| { + Some(slice::from_raw_parts( + data.as_ptr() as *const T, + data.len() / size_of::(), + )) + })?) } } @@ -172,18 +194,23 @@ where dim_products.reverse(); // consider minus strides - let base_ptr = self - .strides - .iter() - .enumerate() - .fold(self.buf, |ptr, (dim, &stride)| { - if stride < 0 { - let dim_size = self.shape[dim] as isize; - unsafe { ptr.offset(stride * (dim_size - 1)) } - } else { - ptr - } - }); + let base_ptr = match self.data { + None => std::ptr::null(), + Some(data) => { + self.strides + .iter() + .enumerate() + .fold(data.as_ptr(), |ptr, (dim, &stride)| { + if stride < 0 { + let dim_size = self.shape[dim] as isize; + unsafe { ptr.offset(stride * (dim_size - 1)) } + } else { + ptr + } + }) + } + }; + RowMajorIter { base_ptr, array: self, @@ -192,26 +219,6 @@ where total_elements: self.shape.iter().product(), } } - - fn check_data_buf(&self) -> Result { - let mut size = size_of::(); - for i in 0..self.dims { - let d = self.shape[i]; - size = size.checked_mul(d).ok_or(error::fmt!( - ArrayViewError, - "Array total elem size overflow" - ))? - } - if size != self.buf_len { - return Err(error::fmt!( - ArrayWriteToBufferError, - "Array buffer length mismatch (actual: {}, expected: {})", - self.buf_len, - size - )); - } - Ok(size) - } } impl StrideArrayView<'_, T> @@ -236,38 +243,60 @@ where strides: *const isize, data: *const u8, data_len: usize, - ) -> Self { + ) -> Result { let shapes = slice::from_raw_parts(shape, dims); + let size = shapes + .iter() + .try_fold(std::mem::size_of::(), |acc, &dim| { + acc.checked_mul(dim) + .ok_or_else(|| error::fmt!(ArrayViewError, "Array total elem size overflow")) + })?; + if size != data_len { + return Err(error::fmt!( + ArrayViewError, + "Array buffer length mismatch (actual: {}, expected: {})", + data_len, + size + )); + } let strides = slice::from_raw_parts(strides, dims); - Self { + let mut slice = None; + if data_len != 0 { + slice = Some(slice::from_raw_parts(data, data_len)); + } + Ok(Self { dims, shape: shapes, strides, - buf_len: data_len, - buf: data, + data: slice, _marker: std::marker::PhantomData::, - } + }) } /// Verifies if the array follows C-style row-major memory layout. fn is_c_major(&self) -> bool { - if self.buf.is_null() || self.buf_len == 0 { - return false; - } + match self.data { + None => false, + Some(data) => { + if data.is_empty() { + return false; + } - let elem_size = size_of::() as isize; - if self.dims == 1 { - return self.strides[0] == elem_size || self.shape[0] == 1; - } + let elem_size = size_of::() as isize; + if self.dims == 1 { + return self.strides[0] == elem_size || self.shape[0] == 1; + } - let mut expected_stride = elem_size; - for (dim, &stride) in self.shape.iter().zip(self.strides).rev() { - if *dim > 1 && stride != expected_stride { - return false; + let mut expected_stride = elem_size; + for (dim, &stride) in self.shape.iter().zip(self.strides).rev() { + if *dim > 1 && stride != expected_stride { + return false; + } + expected_stride *= *dim as isize; + } + true } - expected_stride *= *dim as isize; } - true } } @@ -323,8 +352,17 @@ impl NdArrayView for Vec { 1 } - fn dim(&self, idx: usize) -> Option { - (idx == 0).then_some(self.len()) + fn dim(&self, idx: usize) -> Result { + if idx == 0 { + Ok(self.len()) + } else { + Err(error::fmt!( + ArrayViewError, + "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", + idx, + 1 + )) + } } fn as_slice(&self) -> Option<&[T]> { @@ -334,10 +372,6 @@ impl NdArrayView for Vec { fn iter(&self) -> Self::Iter<'_> { self.as_slice().iter() } - - fn check_data_buf(&self) -> Result { - Ok(self.len() * std::mem::size_of::()) - } } /// impl NdArrayView for one dimension array @@ -351,8 +385,17 @@ impl NdArrayView for [T; N] { 1 } - fn dim(&self, idx: usize) -> Option { - (idx == 0).then_some(N) + fn dim(&self, idx: usize) -> Result { + if idx == 0 { + Ok(N) + } else { + Err(error::fmt!( + ArrayViewError, + "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", + idx, + 1 + )) + } } fn as_slice(&self) -> Option<&[T]> { @@ -362,10 +405,6 @@ impl NdArrayView for [T; N] { fn iter(&self) -> Self::Iter<'_> { self.as_slice().iter() } - - fn check_data_buf(&self) -> Result { - Ok(N * std::mem::size_of::()) - } } /// impl NdArrayView for one dimension slice @@ -380,8 +419,17 @@ impl NdArrayView for &[T] { 1 } - fn dim(&self, idx: usize) -> Option { - (idx == 0).then_some(self.len()) + fn dim(&self, idx: usize) -> Result { + if idx == 0 { + Ok(self.len()) + } else { + Err(error::fmt!( + ArrayViewError, + "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", + idx, + 1 + )) + } } fn as_slice(&self) -> Option<&[T]> { @@ -391,10 +439,6 @@ impl NdArrayView for &[T] { fn iter(&self) -> Self::Iter<'_> { <[T]>::iter(self) } - - fn check_data_buf(&self) -> Result { - Ok(std::mem::size_of_val(*self)) - } } /// impl NdArrayView for two dimensions vector @@ -408,11 +452,22 @@ impl NdArrayView for Vec> { 2 } - fn dim(&self, idx: usize) -> Option { + fn dim(&self, idx: usize) -> Result { match idx { - 0 => Some(self.len()), - 1 => self.first().map(|v| v.len()), - _ => None, + 0 => Ok(self.len()), + 1 => { + let dim1 = self.first().map_or(0, |v| v.len()); + if self.as_slice().iter().any(|v2| v2.len() != dim1) { + return Err(error::fmt!(ArrayViewError, "Irregular array shape")); + } + Ok(dim1) + } + _ => Err(error::fmt!( + ArrayViewError, + "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", + idx, + 2 + )), } } @@ -423,14 +478,6 @@ impl NdArrayView for Vec> { fn iter(&self) -> Self::Iter<'_> { self.as_slice().iter().flatten() } - - fn check_data_buf(&self) -> Result { - let row_len = self.first().map_or(0, |v| v.len()); - if self.as_slice().iter().any(|v| v.len() != row_len) { - return Err(error::fmt!(ArrayViewError, "Irregular array shape")); - } - Ok(self.len() * row_len * std::mem::size_of::()) - } } /// impl NdArrayView for two dimensions array @@ -444,11 +491,16 @@ impl NdArrayView for [[T; M] 2 } - fn dim(&self, idx: usize) -> Option { + fn dim(&self, idx: usize) -> Result { match idx { - 0 => Some(N), - 1 => Some(M), - _ => None, + 0 => Ok(N), + 1 => Ok(M), + _ => Err(error::fmt!( + ArrayViewError, + "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", + idx, + 2 + )), } } @@ -459,10 +511,6 @@ impl NdArrayView for [[T; M] fn iter(&self) -> Self::Iter<'_> { self.as_slice().iter().flatten() } - - fn check_data_buf(&self) -> Result { - Ok(N * M * std::mem::size_of::()) - } } /// impl NdArrayView for two dimensions slices @@ -477,11 +525,16 @@ impl NdArrayView for &[[T; M]] { 2 } - fn dim(&self, idx: usize) -> Option { + fn dim(&self, idx: usize) -> Result { match idx { - 0 => Some(self.len()), - 1 => Some(M), - _ => None, + 0 => Ok(self.len()), + 1 => Ok(M), + _ => Err(error::fmt!( + ArrayViewError, + "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", + idx, + 2 + )), } } @@ -492,10 +545,6 @@ impl NdArrayView for &[[T; M]] { fn iter(&self) -> Self::Iter<'_> { <[[T; M]]>::iter(self).flatten() } - - fn check_data_buf(&self) -> Result { - Ok(self.len() * M * std::mem::size_of::()) - } } /// impl NdArrayView for three dimensions vector @@ -509,12 +558,38 @@ impl NdArrayView for Vec>> { 3 } - fn dim(&self, idx: usize) -> Option { + fn dim(&self, idx: usize) -> Result { match idx { - 0 => Some(self.len()), - 1 => self.first().map(|v| v.len()), - 2 => self.first().and_then(|v2| v2.first()).map(|v3| v3.len()), - _ => None, + 0 => Ok(self.len()), + 1 => { + let dim1 = self.first().map_or(0, |v| v.len()); + if self.as_slice().iter().any(|v2| v2.len() != dim1) { + return Err(error::fmt!(ArrayViewError, "Irregular array shape")); + } + Ok(dim1) + } + 2 => { + let dim2 = self + .first() + .and_then(|v2| v2.first()) + .map_or(0, |v3| v3.len()); + + if self + .as_slice() + .iter() + .flat_map(|v2| v2.as_slice().iter()) + .any(|v3| v3.len() != dim2) + { + return Err(error::fmt!(ArrayViewError, "Irregular array shape")); + } + Ok(dim2) + } + _ => Err(error::fmt!( + ArrayViewError, + "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", + idx, + 3 + )), } } @@ -525,30 +600,6 @@ impl NdArrayView for Vec>> { fn iter(&self) -> Self::Iter<'_> { self.as_slice().iter().flatten().flatten() } - - fn check_data_buf(&self) -> Result { - let dim1 = self.first().map_or(0, |v| v.len()); - - if self.as_slice().iter().any(|v2| v2.len() != dim1) { - return Err(error::fmt!(ArrayViewError, "Irregular array shape")); - } - - let dim2 = self - .first() - .and_then(|v2| v2.first()) - .map_or(0, |v3| v3.len()); - - if self - .as_slice() - .iter() - .flat_map(|v2| v2.as_slice().iter()) - .any(|v3| v3.len() != dim2) - { - return Err(error::fmt!(ArrayViewError, "Irregular array shape")); - } - - Ok(self.len() * dim1 * dim2 * std::mem::size_of::()) - } } /// impl NdArrayView for three dimensions array @@ -564,12 +615,17 @@ impl NdArrayVie 3 } - fn dim(&self, idx: usize) -> Option { + fn dim(&self, idx: usize) -> Result { match idx { - 0 => Some(L), - 1 => Some(N), - 2 => Some(M), - _ => None, + 0 => Ok(L), + 1 => Ok(N), + 2 => Ok(M), + _ => Err(error::fmt!( + ArrayViewError, + "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", + idx, + 3 + )), } } @@ -580,10 +636,6 @@ impl NdArrayVie fn iter(&self) -> Self::Iter<'_> { self.as_slice().iter().flatten().flatten() } - - fn check_data_buf(&self) -> Result { - Ok(L * N * M * std::mem::size_of::()) - } } impl NdArrayView for &[[[T; M]; N]] { @@ -597,12 +649,17 @@ impl NdArrayView for &[[[T; 3 } - fn dim(&self, idx: usize) -> Option { + fn dim(&self, idx: usize) -> Result { match idx { - 0 => Some(self.len()), - 1 => Some(N), - 2 => Some(M), - _ => None, + 0 => Ok(self.len()), + 1 => Ok(N), + 2 => Ok(M), + _ => Err(error::fmt!( + ArrayViewError, + "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", + idx, + 3 + )), } } @@ -613,10 +670,6 @@ impl NdArrayView for &[[[T; fn iter(&self) -> Self::Iter<'_> { <[[[T; M]; N]]>::iter(self).flatten().flatten() } - - fn check_data_buf(&self) -> Result { - Ok(self.len() * N * M * std::mem::size_of::()) - } } use crate::{error, Error}; @@ -641,12 +694,17 @@ where self.ndim() } - fn dim(&self, index: usize) -> Option { + fn dim(&self, index: usize) -> Result { let len = self.ndim(); if index < len { - Some(self.len_of(Axis(index))) + Ok(self.len_of(Axis(index))) } else { - None + Err(error::fmt!( + ArrayViewError, + "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", + index, + 3 + )) } } @@ -657,10 +715,6 @@ where fn as_slice(&self) -> Option<&[T]> { self.as_slice() } - - fn check_data_buf(&self) -> Result { - Ok(self.len() * size_of::()) - } } #[cfg(test)] diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index ab0b984b..ecc093a3 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -1,8 +1,8 @@ #[cfg(feature = "ndarray")] -use crate::ingress::MAX_DIMS; +use crate::ingress::MAX_ARRAY_DIMS; use crate::ingress::{Buffer, NdArrayView, StrideArrayView, ARRAY_BINARY_FORMAT_TYPE}; use crate::tests::TestResult; -use crate::ErrorCode; +use crate::{Error, ErrorCode}; use crate::ingress::ndarr::write_array_data; #[cfg(feature = "ndarray")] @@ -70,12 +70,12 @@ fn test_stride_array_view() -> TestResult { test_data.as_ptr() as *const u8, test_data.len() * size_of::(), ) - }; + }?; assert_eq!(array.ndim(), 2); - assert_eq!(array.dim(0), Some(2)); - assert_eq!(array.dim(1), Some(3)); - assert_eq!(array.dim(2), None); + assert_eq!(array.dim(0), Ok(2)); + assert_eq!(array.dim(1), Ok(3)); + assert!(array.dim(2).is_err()); assert!(array.as_slice().is_some()); let mut buf = vec![]; write_array_data(&array, &mut buf).unwrap(); @@ -99,12 +99,12 @@ fn test_strided_non_contiguous() -> TestResult { col_major_data.as_ptr() as *const u8, col_major_data.len() * elem_size as usize, ) - }; + }?; assert_eq!(array_view.ndim(), 2); - assert_eq!(array_view.dim(0), Some(3)); - assert_eq!(array_view.dim(1), Some(2)); - assert_eq!(array_view.dim(2), None); + assert_eq!(array_view.dim(0), Ok(3)); + assert_eq!(array_view.dim(1), Ok(2)); + assert!(array_view.dim(2).is_err()); assert!(array_view.as_slice().is_none()); let mut buffer = Vec::new(); write_array_data(&array_view, &mut buffer)?; @@ -132,7 +132,7 @@ fn test_negative_strides() -> TestResult { (data.as_ptr() as *const u8).add(48), data.len() * elem_size, ) - }; + }?; let collected: Vec<_> = view.iter().copied().collect(); assert!(view.as_slice().is_none()); let expected_data = vec![7.0, 8.0, 9.0, 4.0, 5.0, 6.0, 1.0, 2.0, 3.0]; @@ -150,14 +150,14 @@ fn test_negative_strides() -> TestResult { } #[test] -fn test_basic_edge_cases() { +fn test_basic_edge_cases() -> TestResult { // empty array let elem_size = std::mem::size_of::() as isize; let empty_view: StrideArrayView<'_, f64> = - unsafe { StrideArrayView::new(2, [0, 0].as_ptr(), [0, 0].as_ptr(), ptr::null(), 0) }; + unsafe { StrideArrayView::new(2, [0, 0].as_ptr(), [0, 0].as_ptr(), ptr::null(), 0)? }; assert_eq!(empty_view.ndim(), 2); - assert_eq!(empty_view.dim(0), Some(0)); - assert_eq!(empty_view.dim(1), Some(0)); + assert_eq!(empty_view.dim(0), Ok(0)); + assert_eq!(empty_view.dim(1), Ok(0)); // single element array let single_data = [42.0]; @@ -169,10 +169,11 @@ fn test_basic_edge_cases() { single_data.as_ptr() as *const u8, elem_size as usize, ) - }; + }?; let mut buf = vec![]; write_array_data(&single_view, &mut buf).unwrap(); assert_eq!(buf, 42.0f64.to_ne_bytes()); + Ok(()) } #[test] @@ -188,7 +189,7 @@ fn test_buffer_basic_write() -> TestResult { test_data.as_ptr() as *const u8, test_data.len() * elem_size as usize, ) - }; + }?; let mut buffer = Buffer::new(); buffer.table("my_test")?; buffer.column_arr("temperature", &array_view)?; @@ -223,8 +224,8 @@ fn test_buffer_basic_write() -> TestResult { } #[test] -fn test_size_overflow() -> TestResult { - let overflow_view = unsafe { +fn test_stride_array_size_overflow() -> TestResult { + let result = unsafe { StrideArrayView::::new( 2, [u32::MAX as usize, u32::MAX as usize].as_ptr(), @@ -233,10 +234,6 @@ fn test_size_overflow() -> TestResult { 0, ) }; - - let mut buffer = Buffer::new(); - buffer.table("my_test")?; - let result = buffer.column_arr("arr1", &overflow_view); let err = result.unwrap_err(); assert_eq!(err.code(), ErrorCode::ArrayViewError); assert!(err.msg().contains("Array total elem size overflow")); @@ -244,10 +241,10 @@ fn test_size_overflow() -> TestResult { } #[test] -fn test_array_length_mismatch() -> TestResult { +fn test_stride_view_length_mismatch() -> TestResult { let elem_size = size_of::() as isize; let under_data = [1.1]; - let under_view: StrideArrayView<'_, f64> = unsafe { + let result: Result, Error> = unsafe { StrideArrayView::new( 2, [1, 2].as_ptr(), @@ -256,18 +253,14 @@ fn test_array_length_mismatch() -> TestResult { under_data.len() * elem_size as usize, ) }; - - let mut buffer = Buffer::new(); - buffer.table("my_test")?; - let result = buffer.column_arr("arr1", &under_view); let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ArrayWriteToBufferError); + assert_eq!(err.code(), ErrorCode::ArrayViewError); assert!(err .msg() .contains("Array buffer length mismatch (actual: 8, expected: 16)")); let over_data = [1.1, 2.2, 3.3]; - let over_view: StrideArrayView<'_, f64> = unsafe { + let result: Result, Error> = unsafe { StrideArrayView::new( 2, [1, 2].as_ptr(), @@ -277,11 +270,8 @@ fn test_array_length_mismatch() -> TestResult { ) }; - buffer.clear(); - buffer.table("my_test")?; - let result = buffer.column_arr("arr1", &over_view); let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ArrayWriteToBufferError); + assert_eq!(err.code(), ErrorCode::ArrayViewError); assert!(err .msg() .contains("Array buffer length mismatch (actual: 24, expected: 16)")); @@ -292,12 +282,11 @@ fn test_array_length_mismatch() -> TestResult { fn test_build_in_1d_array_normal() -> TestResult { let arr = [1.0f64, 2.0, 3.0, 4.0]; assert_eq!(arr.ndim(), 1); - assert_eq!(arr.dim(0), Some(4)); - assert_eq!(arr.dim(1), None); + assert_eq!(arr.dim(0), Ok(4)); + assert!(arr.dim(1).is_err()); assert_eq!(NdArrayView::as_slice(&arr), Some(&[1.0, 2.0, 3.0, 4.0][..])); let collected: Vec<_> = NdArrayView::iter(&arr).copied().collect(); assert_eq!(collected, vec![1.0, 2.0, 3.0, 4.0]); - assert_eq!(arr.check_data_buf(), Ok(32)); let mut buffer = Buffer::new(); buffer.table("my_test")?; @@ -333,9 +322,8 @@ fn test_build_in_1d_array_normal() -> TestResult { fn test_build_in_1d_array_empty() -> TestResult { let arr: [f64; 0] = []; assert_eq!(arr.ndim(), 1); - assert_eq!(arr.dim(0), Some(0)); + assert_eq!(arr.dim(0), Ok(0)); assert_eq!(NdArrayView::as_slice(&arr), Some(&[][..])); - assert_eq!(arr.check_data_buf(), Ok(0)); let mut buffer = Buffer::new(); buffer.table("my_test")?; @@ -361,11 +349,10 @@ fn test_build_in_1d_array_empty() -> TestResult { fn test_build_in_1d_vec_normal() -> TestResult { let vec = vec![5.0f64, 6.0, 7.0]; assert_eq!(vec.ndim(), 1); - assert_eq!(vec.dim(0), Some(3)); + assert_eq!(vec.dim(0), Ok(3)); assert_eq!(NdArrayView::as_slice(&vec), Some(&[5.0, 6.0, 7.0][..])); let collected: Vec<_> = NdArrayView::iter(&vec).copied().collect(); assert_eq!(collected, vec![5.0, 6.0, 7.0]); - assert_eq!(vec.check_data_buf(), Ok(24)); let mut buffer = Buffer::new(); buffer.table("my_test")?; @@ -400,9 +387,8 @@ fn test_build_in_1d_vec_normal() -> TestResult { fn test_build_in_1d_vec_empty() -> TestResult { let vec: Vec = Vec::new(); assert_eq!(vec.ndim(), 1); - assert_eq!(vec.dim(0), Some(0)); + assert_eq!(vec.dim(0), Ok(0)); assert_eq!(NdArrayView::as_slice(&vec), Some(&[][..])); - assert_eq!(vec.check_data_buf(), Ok(0)); let mut buffer = Buffer::new(); buffer.table("my_test")?; @@ -429,9 +415,8 @@ fn test_build_in_1d_slice_normal() -> TestResult { let data = [10.0f64, 20.0, 30.0, 40.0]; let slice = &data[1..3]; assert_eq!(slice.ndim(), 1); - assert_eq!(slice.dim(0), Some(2)); + assert_eq!(slice.dim(0), Ok(2)); assert_eq!(NdArrayView::as_slice(&slice), Some(&[20.0, 30.0][..])); - assert_eq!(slice.check_data_buf(), Ok(16)); let mut buffer = Buffer::new(); buffer.table("my_test")?; @@ -462,9 +447,8 @@ fn test_build_in_1d_slice_empty() -> TestResult { let data = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; let slice = &data[2..2]; assert_eq!(slice.ndim(), 1); - assert_eq!(slice.dim(0), Some(0)); + assert_eq!(slice.dim(0), Ok(0)); assert_eq!(NdArrayView::as_slice(&slice), Some(&[][..])); - assert_eq!(slice.check_data_buf(), Ok(0)); let mut buffer = Buffer::new(); buffer.table("my_test")?; @@ -490,15 +474,14 @@ fn test_build_in_1d_slice_empty() -> TestResult { fn test_build_in_2d_array_normal() -> TestResult { let arr = [[1.0f64, 2.0], [3.0, 4.0], [5.0, 6.0]]; assert_eq!(arr.ndim(), 2); - assert_eq!(arr.dim(0), Some(3)); - assert_eq!(arr.dim(1), Some(2)); + assert_eq!(arr.dim(0), Ok(3)); + assert_eq!(arr.dim(1), Ok(2)); assert_eq!( NdArrayView::as_slice(&arr), Some(&[1.0, 2.0, 3.0, 4.0, 5.0, 6.0][..]) ); let collected: Vec<_> = NdArrayView::iter(&arr).copied().collect(); assert_eq!(collected, vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]); - assert_eq!(arr.check_data_buf(), Ok(48)); let mut buffer = Buffer::new(); buffer.table("my_test")?; @@ -539,10 +522,9 @@ fn test_build_in_2d_array_normal() -> TestResult { fn test_build_in_2d_array_empty() -> TestResult { let arr: [[f64; 0]; 0] = []; assert_eq!(arr.ndim(), 2); - assert_eq!(arr.dim(0), Some(0)); - assert_eq!(arr.dim(1), Some(0)); + assert_eq!(arr.dim(0), Ok(0)); + assert_eq!(arr.dim(1), Ok(0)); assert_eq!(NdArrayView::as_slice(&arr), Some(&[][..])); - assert_eq!(arr.check_data_buf(), Ok(0)); let mut buffer = Buffer::new(); buffer.table("my_test")?; @@ -571,12 +553,11 @@ fn test_build_in_2d_array_empty() -> TestResult { fn test_build_in_2d_vec_normal() -> TestResult { let vec = vec![vec![1.0f64, 2.0], vec![3.0, 4.0], vec![5.0, 6.0]]; assert_eq!(vec.ndim(), 2); - assert_eq!(vec.dim(0), Some(3)); - assert_eq!(vec.dim(1), Some(2)); + assert_eq!(vec.dim(0), Ok(3)); + assert_eq!(vec.dim(1), Ok(2)); assert!(NdArrayView::as_slice(&vec).is_none()); let collected: Vec<_> = NdArrayView::iter(&vec).copied().collect(); assert_eq!(collected, vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]); - assert_eq!(vec.check_data_buf(), Ok(48)); let mut buffer = Buffer::new(); buffer.table("my_test")?; @@ -629,9 +610,8 @@ fn test_build_in_2d_vec_irregular_shape() -> TestResult { fn test_build_in_2d_vec_empty() -> TestResult { let vec: Vec> = vec![vec![], vec![], vec![]]; assert_eq!(vec.ndim(), 2); - assert_eq!(vec.dim(0), Some(3)); - assert_eq!(vec.dim(1), Some(0)); - assert_eq!(vec.check_data_buf(), Ok(0)); + assert_eq!(vec.dim(0), Ok(3)); + assert_eq!(vec.dim(1), Ok(0)); let mut buffer = Buffer::new(); buffer.table("my_test")?; @@ -661,13 +641,12 @@ fn test_build_in_2d_slice_normal() -> TestResult { let data = [[1.0f64, 2.0], [3.0, 4.0], [5.0, 6.0]]; let slice = &data[..2]; assert_eq!(slice.ndim(), 2); - assert_eq!(slice.dim(0), Some(2)); - assert_eq!(slice.dim(1), Some(2)); + assert_eq!(slice.dim(0), Ok(2)); + assert_eq!(slice.dim(1), Ok(2)); assert_eq!( NdArrayView::as_slice(&slice), Some(&[1.0, 2.0, 3.0, 4.0][..]) ); - assert_eq!(slice.check_data_buf(), Ok(32)); let mut buffer = Buffer::new(); buffer.table("my_test")?; @@ -707,10 +686,9 @@ fn test_build_in_2d_slice_empty() -> TestResult { let data = [[1.0f64, 2.0], [3.0, 4.0], [5.0, 6.0]]; let slice = &data[2..2]; assert_eq!(slice.ndim(), 2); - assert_eq!(slice.dim(0), Some(0)); - assert_eq!(slice.dim(1), Some(2)); + assert_eq!(slice.dim(0), Ok(0)); + assert_eq!(slice.dim(1), Ok(2)); assert_eq!(NdArrayView::as_slice(&slice), Some(&[][..])); - assert_eq!(slice.check_data_buf(), Ok(0)); let mut buffer = Buffer::new(); buffer.table("my_test")?; @@ -739,16 +717,15 @@ fn test_build_in_2d_slice_empty() -> TestResult { fn test_build_in_3d_array_normal() -> TestResult { let arr = [[[1.0f64, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]]; assert_eq!(arr.ndim(), 3); - assert_eq!(arr.dim(0), Some(2)); - assert_eq!(arr.dim(1), Some(2)); - assert_eq!(arr.dim(2), Some(2)); + assert_eq!(arr.dim(0), Ok(2)); + assert_eq!(arr.dim(1), Ok(2)); + assert_eq!(arr.dim(2), Ok(2)); assert_eq!( NdArrayView::as_slice(&arr), Some(&[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0][..]) ); let collected: Vec<_> = NdArrayView::iter(&arr).copied().collect(); assert_eq!(collected, vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]); - assert_eq!(arr.check_data_buf(), Ok(64)); let mut buffer = Buffer::new(); buffer.table("my_test")?; @@ -791,11 +768,10 @@ fn test_build_in_3d_array_normal() -> TestResult { fn test_build_in_3d_array_empty() -> TestResult { let arr: [[[f64; 2]; 0]; 0] = []; assert_eq!(arr.ndim(), 3); - assert_eq!(arr.dim(0), Some(0)); - assert_eq!(arr.dim(1), Some(0)); - assert_eq!(arr.dim(2), Some(2)); + assert_eq!(arr.dim(0), Ok(0)); + assert_eq!(arr.dim(1), Ok(0)); + assert_eq!(arr.dim(2), Ok(2)); assert_eq!(NdArrayView::as_slice(&arr), Some(&[][..])); - assert_eq!(arr.check_data_buf(), Ok(0)); let mut buffer = Buffer::new(); buffer.table("my_test")?; @@ -827,16 +803,15 @@ fn test_build_in_3d_vec_normal() -> TestResult { vec![vec![7.0, 8.0, 9.0], vec![10.0, 11.0, 12.0]], ]; assert_eq!(vec.ndim(), 3); - assert_eq!(vec.dim(0), Some(2)); - assert_eq!(vec.dim(1), Some(2)); - assert_eq!(vec.dim(2), Some(3)); + assert_eq!(vec.dim(0), Ok(2)); + assert_eq!(vec.dim(1), Ok(2)); + assert_eq!(vec.dim(2), Ok(3)); assert!(NdArrayView::as_slice(&vec).is_none()); let collected: Vec<_> = NdArrayView::iter(&vec).copied().collect(); assert_eq!( collected, vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0] ); - assert_eq!(vec.check_data_buf(), Ok(96)); let mut buffer = Buffer::new(); buffer.table("my_test")?; @@ -883,11 +858,10 @@ fn test_build_in_3d_vec_normal() -> TestResult { fn test_build_in_3d_vec_empty() -> TestResult { let vec: Vec>> = vec![vec![vec![], vec![]], vec![vec![], vec![]]]; assert_eq!(vec.ndim(), 3); - assert_eq!(vec.dim(0), Some(2)); - assert_eq!(vec.dim(1), Some(2)); - assert_eq!(vec.dim(2), Some(0)); + assert_eq!(vec.dim(0), Ok(2)); + assert_eq!(vec.dim(1), Ok(2)); + assert_eq!(vec.dim(2), Ok(0)); assert!(NdArrayView::as_slice(&vec).is_none()); - assert_eq!(vec.check_data_buf(), Ok(0)); let mut buffer = Buffer::new(); buffer.table("my_test")?; @@ -940,14 +914,13 @@ fn test_3d_slice_normal() -> TestResult { let data = [[[1f64, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]]; let slice = &data[..1]; assert_eq!(slice.ndim(), 3); - assert_eq!(slice.dim(0), Some(1)); - assert_eq!(slice.dim(1), Some(2)); - assert_eq!(slice.dim(2), Some(2)); + assert_eq!(slice.dim(0), Ok(1)); + assert_eq!(slice.dim(1), Ok(2)); + assert_eq!(slice.dim(2), Ok(2)); assert_eq!( NdArrayView::as_slice(&slice), Some(&[1.0, 2.0, 3.0, 4.0][..]) ); - assert_eq!(slice.check_data_buf(), Ok(32)); let mut buffer = Buffer::new(); buffer.table("my_test")?; @@ -987,11 +960,10 @@ fn test_3d_slice_empty() -> TestResult { let data = [[[1f64, 2.0], [3.0, 4.0]], [[5.0, 6.0], [7.0, 8.0]]]; let slice = &data[1..1]; assert_eq!(slice.ndim(), 3); - assert_eq!(slice.dim(0), Some(0)); - assert_eq!(slice.dim(1), Some(2)); - assert_eq!(slice.dim(2), Some(2)); + assert_eq!(slice.dim(0), Ok(0)); + assert_eq!(slice.dim(1), Ok(2)); + assert_eq!(slice.dim(2), Ok(2)); assert_eq!(NdArrayView::as_slice(&slice), Some(&[][..])); - assert_eq!(slice.check_data_buf(), Ok(0)); let mut buffer = Buffer::new(); buffer.table("my_test")?; @@ -1078,8 +1050,8 @@ fn test_1d_dimension_ndarray_info() { let view = array.view(); assert_eq!(NdArrayView::ndim(&view), 1); - assert_eq!(NdArrayView::dim(&view, 0), Some(3)); - assert_eq!(NdArrayView::dim(&view, 1), None); + assert_eq!(NdArrayView::dim(&view, 0), Ok(3)); + assert!(NdArrayView::dim(&view, 1).is_err()); } #[cfg(feature = "ndarray")] @@ -1089,9 +1061,9 @@ fn test_complex_ndarray_dimensions() { let view = array.view(); assert_eq!(NdArrayView::ndim(&view), 3); - assert_eq!(NdArrayView::dim(&view, 0), Some(2)); - assert_eq!(NdArrayView::dim(&view, 1), Some(2)); - assert_eq!(NdArrayView::dim(&view, 2), Some(1)); + assert_eq!(NdArrayView::dim(&view, 0), Ok(2)); + assert_eq!(NdArrayView::dim(&view, 1), Ok(2)); + assert_eq!(NdArrayView::dim(&view, 2), Ok(1)); } #[cfg(feature = "ndarray")] @@ -1127,14 +1099,14 @@ fn test_buffer_ndarray_write() -> TestResult { fn test_buffer_write_ndarray_max_dimensions() -> TestResult { let mut buffer = Buffer::new(); buffer.table("nd_test")?; - let shape: Vec = iter::repeat_n(1, MAX_DIMS).collect(); + let shape: Vec = iter::repeat_n(1, MAX_ARRAY_DIMS).collect(); let array = ArrayD::::zeros(shape.clone()); buffer.column_arr("max_dim", &array.view())?; let data = buffer.as_bytes(); - assert_eq!(data[19], MAX_DIMS as u8); + assert_eq!(data[19], MAX_ARRAY_DIMS as u8); // 33 dims error - let shape_invalid: Vec<_> = iter::repeat_n(1, MAX_DIMS + 1).collect(); + let shape_invalid: Vec<_> = iter::repeat_n(1, MAX_ARRAY_DIMS + 1).collect(); let array_invalid = ArrayD::::zeros(shape_invalid); let result = buffer.column_arr("invalid", &array_invalid.view()); assert!(result.is_err()); From 68d89d323f96bd65f2ab33ab150baeed07b7b663 Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 13 May 2025 20:21:10 +0800 Subject: [PATCH 064/119] fix benchmark --- questdb-rs/src/ingress/mod.rs | 12 +++--------- questdb-rs/src/ingress/ndarr.rs | 6 ++---- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 89382ca3..becfc59f 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -1233,7 +1233,7 @@ impl Buffer { where N: TryInto>, T: NdArrayView, - D: ArrayElement, + D: ArrayElement + ArrayElementSealed, Error: From, { self.write_column_key(name)?; @@ -1254,18 +1254,12 @@ impl Buffer { // binary format entity type self.output.push(ARRAY_BINARY_FORMAT_TYPE); // ndarr datatype - self.output.push(D::elem_type().into()); + self.output.push(D::type_tag()); // ndarr dims self.output.push(view.ndim() as u8); for i in 0..view.ndim() { - let d = view.dim(i).ok_or_else(|| { - error::fmt!( - ArrayViewError, - "Can not get correct dimensions for dim {}", - i - ) - })?; + let d = view.dim(i)?; // ndarr shapes self.output .extend_from_slice((d as i32).to_le_bytes().as_slice()); diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 6a3a07f1..148abf3d 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -175,12 +175,10 @@ where fn as_slice(&self) -> Option<&[T]> { unsafe { - self.is_c_major().then_some(self.data.and_then(|data| { - Some(slice::from_raw_parts( + self.is_c_major().then_some(self.data.map(|data| slice::from_raw_parts( data.as_ptr() as *const T, data.len() / size_of::(), - )) - })?) + ))?) } } From 47cdde669ad81b11a37f38c566cd4ae673988e5d Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 13 May 2025 22:16:06 +0800 Subject: [PATCH 065/119] optimize array internal implementation --- questdb-rs/src/ingress/mod.rs | 42 ++--------------- questdb-rs/src/ingress/ndarr.rs | 80 ++++++++++++++++++++++----------- questdb-rs/src/tests/ndarr.rs | 28 ++++++------ questdb-rs/src/tests/sender.rs | 8 +++- 4 files changed, 78 insertions(+), 80 deletions(-) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index becfc59f..ef3307ad 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -38,7 +38,7 @@ use socket2::{Domain, Protocol as SockProtocol, SockAddr, Socket, Type}; use std::collections::HashMap; use std::convert::Infallible; use std::fmt::{Debug, Display, Formatter, Write}; -use std::io::{self, BufRead, BufReader, Cursor, ErrorKind, Write as IoWrite}; +use std::io::{self, BufRead, BufReader, ErrorKind, Write as IoWrite}; use std::ops::Deref; use std::path::PathBuf; use std::slice::from_raw_parts_mut; @@ -1186,48 +1186,14 @@ impl Buffer { let index = self.output.len(); let writeable = unsafe { from_raw_parts_mut(self.output.as_mut_ptr().add(index), array_buf_size) }; - let mut cursor = Cursor::new(writeable); - - // TODO: The next section needs a bit of a rewrite. - // It also needs clear comments that explain the design decisions. - // - // I'd be expecting two code paths here: - // 1. The array is row-major contiguous - // 2. The data needs to be written out via the strides. - // - // The code here seems to do something a bit different and - // is worth explaining. - // I see two code paths that I honestly don't understand, - // the `ndarr::write_array_data` and the `ndarr::write_array_data_use_raw_buffer` - // functions both seem to handle both cases (why?) and then seem - // to construct a vectored IoSlice buffer (why??) before writing - // the strided data out. // ndarr data - if view.as_slice().is_some() { - if let Err(e) = ndarr::write_array_data(view, &mut cursor) { - return Err(error::fmt!( - ArrayWriteToBufferError, - "Can not write row major to writer: {}", - e - )); - } - if cursor.position() != (array_buf_size as u64) { - return Err(error::fmt!( - ArrayWriteToBufferError, - "Array write buffer length mismatch (actual: {}, expected: {})", - cursor.position(), - array_buf_size - )); - } - unsafe { self.output.set_len(array_buf_size + index) } - } else { - unsafe { self.output.set_len(array_buf_size + index) } - ndarr::write_array_data_use_raw_buffer(&mut self.output[index..], view); - } + ndarr::write_array_data(view, writeable, array_buf_size)?; + unsafe { self.output.set_len(array_buf_size + index) } Ok(self) } + #[allow(private_bounds)] #[cfg(feature = "benchmark")] pub fn column_arr_use_raw_buffer<'a, N, T, D>(&mut self, name: N, view: &T) -> Result<&mut Self> where diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 148abf3d..257b18ae 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -30,45 +30,71 @@ where const MAX_ARRAY_BUFFER_SIZE: usize = i32::MAX as usize; pub(crate) const MAX_ARRAY_DIM_LEN: usize = 0x0FFF_FFFF; // 1 << 28 - 1 -pub(crate) fn write_array_data, T>( +pub(crate) fn write_array_data, T>( array: &A, - writer: &mut W, -) -> std::io::Result<()> + buf: &mut [u8], + expect_size: usize, +) -> Result<(), Error> where T: ArrayElement, { // First optimization path: write contiguous memory directly + // When working with contiguous layout. Benchmark shows `copy_from_slice` has better performance than + // `std::ptr::copy_nonoverlapping` on both arm(Macos) and x86(Linux) platform. + // This may because `copy_from_slice` benefits more from compiler. if let Some(contiguous) = array.as_slice() { let bytes = unsafe { slice::from_raw_parts(contiguous.as_ptr() as *const u8, size_of_val(contiguous)) }; - return writer.write_all(bytes); + + if bytes.len() != expect_size { + return Err(error::fmt!( + ArrayWriteToBufferError, + "Array write buffer length mismatch (actual: {}, expected: {})", + expect_size, + bytes.len() + )); + } + + if buf.len() < bytes.len() { + return Err(error::fmt!( + ArrayWriteToBufferError, + "Buffer capacity {} < required {}", + buf.len(), + bytes.len() + )); + } + + buf[..bytes.len()].copy_from_slice(bytes); + return Ok(()); } // Fallback path: non-contiguous memory handling + // For non-contiguous memory layouts, direct raw pointer operations are preferred. let elem_size = size_of::(); - let mut io_slices = Vec::new(); - for element in array.iter() { - let bytes = unsafe { slice::from_raw_parts(element as *const T as *const _, elem_size) }; - io_slices.push(std::io::IoSlice::new(bytes)); - } - - let mut io_slices: &mut [IoSlice<'_>] = io_slices.as_mut_slice(); - IoSlice::advance_slices(&mut io_slices, 0); - - while !io_slices.is_empty() { - let written = writer.write_vectored(io_slices)?; - if written == 0 { - return Err(std::io::Error::new( - std::io::ErrorKind::WriteZero, - "Failed to write all bytes", - )); + let mut total_len = 0; + for (i, &element) in array.iter().enumerate() { + unsafe { + std::ptr::copy_nonoverlapping( + &element as *const T as *const u8, + buf.as_mut_ptr().add(i * elem_size), + elem_size, + ) } - IoSlice::advance_slices(&mut io_slices, written); + total_len += elem_size; + } + if total_len != expect_size { + return Err(error::fmt!( + ArrayWriteToBufferError, + "Array write buffer length mismatch (actual: {}, expected: {})", + total_len, + expect_size + )); } Ok(()) } +#[cfg(feature = "benchmark")] pub(crate) fn write_array_data_use_raw_buffer, T>(buf: &mut [u8], array: &A) where T: ArrayElement, @@ -138,6 +164,10 @@ impl ArrayElementSealed for f64 { } /// A view into a multi-dimensional array with custom memory strides. +// TODO: We are currently evaluating whether to use StrideArrayView or ndarray's view. +// Current benchmarks show that StrideArrayView's iter implementation underperforms(2x) +// compared to ndarray's view. If we proceed with StrideArrayView, we need to +// optimize the iter traversal pattern #[derive(Debug)] pub struct StrideArrayView<'a, T> { dims: usize, @@ -175,10 +205,9 @@ where fn as_slice(&self) -> Option<&[T]> { unsafe { - self.is_c_major().then_some(self.data.map(|data| slice::from_raw_parts( - data.as_ptr() as *const T, - data.len() / size_of::(), - ))?) + self.is_c_major().then_some(self.data.map(|data| { + slice::from_raw_parts(data.as_ptr() as *const T, data.len() / size_of::()) + })?) } } @@ -673,7 +702,6 @@ impl NdArrayView for &[[[T; use crate::{error, Error}; #[cfg(feature = "ndarray")] use ndarray::{ArrayView, Axis, Dimension}; -use std::io::IoSlice; use std::slice; #[cfg(feature = "ndarray")] diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index ecc093a3..79b24e66 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -77,8 +77,8 @@ fn test_stride_array_view() -> TestResult { assert_eq!(array.dim(1), Ok(3)); assert!(array.dim(2).is_err()); assert!(array.as_slice().is_some()); - let mut buf = vec![]; - write_array_data(&array, &mut buf).unwrap(); + let mut buf = vec![0u8; 48]; + write_array_data(&array, &mut buf, 48).unwrap(); let expected = to_bytes(&test_data); assert_eq!(buf, expected); Ok(()) @@ -106,8 +106,8 @@ fn test_strided_non_contiguous() -> TestResult { assert_eq!(array_view.dim(1), Ok(2)); assert!(array_view.dim(2).is_err()); assert!(array_view.as_slice().is_none()); - let mut buffer = Vec::new(); - write_array_data(&array_view, &mut buffer)?; + let mut buffer = vec![0u8; 48]; + write_array_data(&array_view, &mut buffer, 48)?; let expected_data = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; let expected_bytes = unsafe { @@ -137,8 +137,8 @@ fn test_negative_strides() -> TestResult { assert!(view.as_slice().is_none()); let expected_data = vec![7.0, 8.0, 9.0, 4.0, 5.0, 6.0, 1.0, 2.0, 3.0]; assert_eq!(collected, expected_data); - let mut buffer = Vec::new(); - write_array_data(&view, &mut buffer)?; + let mut buffer = vec![0u8; 72]; + write_array_data(&view, &mut buffer, 72)?; let expected_bytes = unsafe { std::slice::from_raw_parts( expected_data.as_ptr() as *const u8, @@ -163,15 +163,15 @@ fn test_basic_edge_cases() -> TestResult { let single_data = [42.0]; let single_view: StrideArrayView<'_, f64> = unsafe { StrideArrayView::new( - 2, - [1, 1].as_ptr(), - [elem_size, elem_size].as_ptr(), + 1, + [1].as_ptr(), + [elem_size].as_ptr(), single_data.as_ptr() as *const u8, elem_size as usize, ) }?; - let mut buf = vec![]; - write_array_data(&single_view, &mut buf).unwrap(); + let mut buf = vec![0u8; 8]; + write_array_data(&single_view, &mut buf, 8).unwrap(); assert_eq!(buf, 42.0f64.to_ne_bytes()); Ok(()) } @@ -994,7 +994,7 @@ fn test_1d_contiguous_ndarray_buffer() -> TestResult { let array = arr1(&[1.0, 2.0, 3.0, 4.0]); let view = array.view(); let mut buf = vec![0u8; 4 * size_of::()]; - write_array_data(&view, &mut &mut buf[0..])?; + write_array_data(&view, &mut buf[0..], 32)?; let expected: Vec = array .iter() .flat_map(|&x| x.to_ne_bytes().to_vec()) @@ -1010,7 +1010,7 @@ fn test_2d_non_contiguous_ndarray_buffer() -> TestResult { let transposed = array.view().reversed_axes(); assert!(!transposed.is_standard_layout()); let mut buf = vec![0u8; 4 * size_of::()]; - write_array_data(&transposed, &mut &mut buf[0..])?; + write_array_data(&transposed, &mut buf[0..], 32)?; let expected = [1.0f64, 3.0, 2.0, 4.0] .iter() .flat_map(|&x| x.to_ne_bytes()) @@ -1031,7 +1031,7 @@ fn test_strided_ndarray_layout() -> TestResult { let strided_view = array.slice(s![1..;2, 1..;2]); assert_eq!(strided_view.dim(), (2, 2)); let mut buf = vec![0u8; 4 * size_of::()]; - write_array_data(&strided_view, &mut &mut buf[0..])?; + write_array_data(&strided_view, &mut buf[0..], 32)?; // expect：6.0, 8.0, 14.0, 16.0 let expected = [6.0f64, 8.0, 14.0, 16.0] diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index 4ff58023..e8d4c239 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -176,7 +176,7 @@ fn test_array_f64_from_ndarray() -> TestResult { ] .concat(); let mut array_data2d = vec![0u8; 4 * size_of::()]; - write_array_data(&array_2d.view(), &mut &mut array_data2d[0..])?; + write_array_data(&array_2d.view(), &mut &mut array_data2d[0..], 32)?; let array_header3d = &[ &[b'='][..], @@ -189,7 +189,11 @@ fn test_array_f64_from_ndarray() -> TestResult { ] .concat(); let mut array_data3d = vec![0u8; 24 * size_of::()]; - write_array_data(&array_3d.view(), &mut &mut array_data3d[0..])?; + write_array_data( + &array_3d.view(), + &mut &mut array_data3d[0..], + 24 * size_of::(), + )?; let exp = &[ "my_table,device=A001 ".as_bytes(), From 542da0307ce5ddb0b96a406de75ade789499f698 Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 13 May 2025 23:08:00 +0800 Subject: [PATCH 066/119] fix compile error. --- questdb-rs-ffi/src/lib.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index c0815d5d..594a2c97 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -897,8 +897,20 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( ) -> bool { let buffer = unwrap_buffer_mut(buffer); let name = name.as_name(); - let view = - ingress::StrideArrayView::::new(rank, shape, strides, data_buffer, data_buffer_len); + let view = match ingress::StrideArrayView::::new( + rank, + shape, + strides, + data_buffer, + data_buffer_len, + ) { + Ok(value) => value, + Err(err) => { + let err_ptr = Box::into_raw(Box::new(line_sender_error(err))); + *err_out = err_ptr; + return false; + } + }; bubble_err_to_c!( err_out, buffer.column_arr::, ingress::StrideArrayView<'_, f64>, f64>(name, &view) From 175d895baead14d5c903cdd48c108726ccba8c7a Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 13 May 2025 23:17:48 +0800 Subject: [PATCH 067/119] test format --- questdb-rs/src/tests/sender.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index e8d4c239..2945d634 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -176,7 +176,7 @@ fn test_array_f64_from_ndarray() -> TestResult { ] .concat(); let mut array_data2d = vec![0u8; 4 * size_of::()]; - write_array_data(&array_2d.view(), &mut &mut array_data2d[0..], 32)?; + write_array_data(&array_2d.view(), &mut array_data2d[0..], 32)?; let array_header3d = &[ &[b'='][..], @@ -191,7 +191,7 @@ fn test_array_f64_from_ndarray() -> TestResult { let mut array_data3d = vec![0u8; 24 * size_of::()]; write_array_data( &array_3d.view(), - &mut &mut array_data3d[0..], + &mut array_data3d[0..], 24 * size_of::(), )?; From b89788c20c1fe57796f76b992e99432c7b5fbb43 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 14 May 2025 21:19:42 +0800 Subject: [PATCH 068/119] code review. --- cpp_test/test_line_sender.cpp | 6 +- examples/line_sender_c_example_http.c | 4 +- examples/line_sender_cpp_example_http.cpp | 2 +- include/questdb/ingress/line_sender.h | 72 +-- include/questdb/ingress/line_sender.hpp | 74 +-- questdb-rs-ffi/src/lib.rs | 95 ++-- questdb-rs/Cargo.toml | 10 +- questdb-rs/README.md | 2 +- questdb-rs/benches/ndarr.rs | 116 ----- questdb-rs/build.rs | 11 +- questdb-rs/examples/auth.rs | 4 +- questdb-rs/examples/auth_tls.rs | 4 +- questdb-rs/examples/basic.rs | 4 +- questdb-rs/examples/from_conf.rs | 4 +- questdb-rs/examples/from_env.rs | 4 +- questdb-rs/examples/http.rs | 5 +- ...rotocol_version.rs => protocol_version.rs} | 16 +- questdb-rs/src/error.rs | 4 +- questdb-rs/src/ingress/http.rs | 57 ++- questdb-rs/src/ingress/mod.md | 6 +- questdb-rs/src/ingress/mod.rs | 454 +++++++++--------- questdb-rs/src/ingress/ndarr.rs | 48 +- questdb-rs/src/ingress/tests.rs | 4 +- questdb-rs/src/lib.rs | 1 - questdb-rs/src/tests/http.rs | 121 +++-- questdb-rs/src/tests/ndarr.rs | 50 +- questdb-rs/src/tests/sender.rs | 84 ++-- system_test/questdb_line_sender.py | 38 +- system_test/test.py | 14 +- 29 files changed, 587 insertions(+), 727 deletions(-) delete mode 100644 questdb-rs/benches/ndarr.rs rename questdb-rs/examples/{line_protocol_version.rs => protocol_version.rs} (58%) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index e6f4f663..179a661d 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -912,12 +912,12 @@ TEST_CASE("HTTP basics") questdb::ingress::protocol::http, "localhost", 1, true}; questdb::ingress::opts opts1conf = questdb::ingress::opts::from_conf( "http::addr=localhost:1;username=user;password=pass;request_timeout=" - "5000;retry_timeout=5;disable_line_protocol_validation=on;"); + "5000;retry_timeout=5;disable_protocol_validation=on;"); questdb::ingress::opts opts2{ questdb::ingress::protocol::https, "localhost", "1", true}; questdb::ingress::opts opts2conf = questdb::ingress::opts::from_conf( "http::addr=localhost:1;token=token;request_min_throughput=1000;retry_" - "timeout=0;disable_line_protocol_validation=on;"); + "timeout=0;disable_protocol_validation=on;"); opts1.username("user") .password("pass") .max_buf_size(1000000) @@ -954,7 +954,7 @@ TEST_CASE("line sender protocol version v1") server.accept(); CHECK(server.recv() == 0); - questdb::ingress::line_sender_buffer buffer{line_protocol_version_1}; + questdb::ingress::line_sender_buffer buffer{protocol_version_1}; buffer.table("test") .symbol("t1", "v1") .symbol("t2", "") diff --git a/examples/line_sender_c_example_http.c b/examples/line_sender_c_example_http.c index f3bd3248..b0b448bb 100644 --- a/examples/line_sender_c_example_http.c +++ b/examples/line_sender_c_example_http.c @@ -29,8 +29,8 @@ static bool example(const char* host, const char* port) buffer = line_sender_buffer_new(); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. - if (!line_sender_buffer_set_line_protocol_version( - buffer, line_sender_default_line_protocol_version(sender), &err)) + if (!line_sender_buffer_set_protocol_version( + buffer, line_sender_default_protocol_version(sender), &err)) goto on_error; line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("c_trades_http"); diff --git a/examples/line_sender_cpp_example_http.cpp b/examples/line_sender_cpp_example_http.cpp index 800e11aa..f36901c1 100644 --- a/examples/line_sender_cpp_example_http.cpp +++ b/examples/line_sender_cpp_example_http.cpp @@ -21,7 +21,7 @@ static bool example(std::string_view host, std::string_view port) const auto amount_name = "amount"_cn; questdb::ingress::line_sender_buffer buffer{ - sender.default_line_protocol_version()}; + sender.default_protocol_version()}; buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 959fd149..b4421cee 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -89,7 +89,7 @@ typedef enum line_sender_error_code line_sender_error_array_view_write_to_buffer_error, /** Line sender protocol version error. */ - line_sender_error_line_protocol_version_error, + line_sender_error_protocol_version_error, } line_sender_error_code; /** The protocol used to connect with. */ @@ -109,16 +109,16 @@ typedef enum line_sender_protocol } line_sender_protocol; /** The line protocol version used to write data to buffer. */ -typedef enum line_protocol_version +typedef enum protocol_version { /** Version 1 of InfluxDB Line Protocol. Uses text format serialization for f64. */ - line_protocol_version_1 = 1, + protocol_version_1 = 1, /** Version 2 of InfluxDB Line Protocol. Uses binary format serialization for f64, and support array data type.*/ - line_protocol_version_2 = 2, -} line_protocol_version; + protocol_version_2 = 2, +} protocol_version; /** Possible sources of the root certificates used to validate the server's * TLS certificate. */ @@ -309,7 +309,7 @@ typedef struct line_sender_buffer line_sender_buffer; * the same as the QuestDB server default. */ LINESENDER_API -line_sender_buffer* line_sender_buffer_new(); +line_sender_buffer* line_sender_buffer_new(protocol_version version); /** * Construct a `line_sender_buffer` with a custom maximum length for table @@ -319,24 +319,8 @@ line_sender_buffer* line_sender_buffer_new(); * `line_sender_buffer_new()` instead. */ LINESENDER_API -line_sender_buffer* line_sender_buffer_with_max_name_len(size_t max_name_len); - -/** - * Sets the Line Protocol version for line_sender_buffer. - * - * The buffer defaults is line_protocol_version_2 which uses - * binary format f64 serialization and support array data type. Call this to - * switch to version 1 (text format f64) when connecting to servers that don't - * support line_protocol_version_2(under 8.3.2). - * - * Must be called before adding any data to the buffer. Protocol version cannot - * be changed after the buffer contains data. - */ -LINESENDER_API -bool line_sender_buffer_set_line_protocol_version( - line_sender_buffer* buffer, - line_protocol_version version, - line_sender_error** err_out); +line_sender_buffer* line_sender_buffer_with_max_name_len( + size_t max_name_len, protocol_version version); /** Release the `line_sender_buffer` object. */ LINESENDER_API @@ -762,12 +746,21 @@ bool line_sender_opts_token_y( line_sender_error** err_out); /** - * Set the ECDSA public key Y for TCP authentication. + * Disable the line protocol validation. */ LINESENDER_API -bool line_sender_opts_disable_line_protocol_validation( +bool line_sender_opts_disable_protocol_validation( line_sender_opts* opts, line_sender_error** err_out); +/** + * set the line protocol version. + */ +LINESENDER_API +bool line_sender_opts_protocol_version( + line_sender_opts* opts, + protocol_version version, + line_sender_error** err_out); + /** * Configure how long to wait for messages from the QuestDB server during * the TLS handshake and authentication process. @@ -929,23 +922,40 @@ line_sender* line_sender_from_env(line_sender_error** err_out); /** * Returns the QuestDB server's recommended default line protocol version. - * Will be used to [`line_sender_buffer_set_line_protocol_version`] + * Will be used to [`line_sender_buffer_set_protocol_version`] * * The version selection follows these rules: - * 1. TCP/TCPS Protocol: Always returns [`LineProtocolVersion::V2`] + * 1. TCP/TCPS Protocol: Always returns [`ProtocolVersion::V2`] * 2. HTTP/HTTPS Protocol: * - If line protocol auto-detection is disabled - * [`line_sender_opts_disable_line_protocol_validation`], returns - * [`LineProtocolVersion::V2`] + * [`line_sender_opts_disable_protocol_validation`], returns + * [`ProtocolVersion::V2`] * - If line protocol auto-detection is enabled: * - Uses the server's default version if supported by the client * - Otherwise uses the highest mutually supported version from the * intersection of client and server compatible versions. */ LINESENDER_API -line_protocol_version line_sender_default_line_protocol_version( +protocol_version line_sender_default_protocol_version( const line_sender* sender); +/** + * Construct a `line_sender_buffer` with a `max_name_len` of `127` and sender's + * default protocol version + * which is the same as the QuestDB server default. + */ +line_sender_buffer* line_sender_new_buffer(const line_sender* sender); + +/** + * Construct a `line_sender_buffer` with sender's default protocol version and + * a custom maximum length for table and column names. This should match the + * `cairo.max.file.name.length` setting of the QuestDB server you're + * connecting to. If the server does not configure it, the default is `127`, + * and you can call `line_sender_new_buffer()` instead. + */ +line_sender_buffer* line_sender_new_buffer_with_max_name_len( + const line_sender* sender, size_t max_name_len); + /** * Tell whether the sender is no longer usable and must be closed. * This happens when there was an earlier failure. diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 21906aa9..d036d125 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -404,15 +404,15 @@ class line_sender_buffer line_sender_buffer( size_t init_buf_size, size_t max_name_len, - line_protocol_version version) noexcept + protocol_version version) noexcept : _impl{nullptr} , _init_buf_size{init_buf_size} , _max_name_len{max_name_len} - , _line_protocol_version{version} + , _protocol_version{version} { } - line_sender_buffer(line_protocol_version version) noexcept + line_sender_buffer(protocol_version version) noexcept : line_sender_buffer{64 * 1024, 127, version} { } @@ -421,7 +421,7 @@ class line_sender_buffer : _impl{::line_sender_buffer_clone(other._impl)} , _init_buf_size{other._init_buf_size} , _max_name_len{other._max_name_len} - , _line_protocol_version{other._line_protocol_version} + , _protocol_version{other._protocol_version} { } @@ -429,7 +429,7 @@ class line_sender_buffer : _impl{other._impl} , _init_buf_size{other._init_buf_size} , _max_name_len{other._max_name_len} - , _line_protocol_version{other._line_protocol_version} + , _protocol_version{other._protocol_version} { other._impl = nullptr; } @@ -445,7 +445,7 @@ class line_sender_buffer _impl = nullptr; _init_buf_size = other._init_buf_size; _max_name_len = other._max_name_len; - _line_protocol_version = other._line_protocol_version; + _protocol_version = other._protocol_version; } return *this; } @@ -458,32 +458,12 @@ class line_sender_buffer _impl = other._impl; _init_buf_size = other._init_buf_size; _max_name_len = other._max_name_len; - _line_protocol_version = other._line_protocol_version; + _protocol_version = other._protocol_version; other._impl = nullptr; } return *this; } - /** - * Sets the Line Protocol version for line_sender_buffer. - * - * The buffer defaults is line_protocol_version_2 which uses - * binary format f64 serialization and support array data type. Call this to - * switch to version 1 (text format f64) when connecting to servers that - * don't support line_protocol_version_2(under 8.3.2). - * - * Must be called before adding any data to the buffer. Protocol version - * cannot be changed after the buffer contains data. - */ - line_sender_buffer& set_line_protocol_version(line_protocol_version v) - { - may_init(); - line_sender_error::wrapped_call( - ::line_sender_buffer_set_line_protocol_version, _impl, v); - _line_protocol_version = v; - return *this; - } - /** * Pre-allocate to ensure the buffer has enough capacity for at least * the specified additional byte count. This may be rounded up. @@ -844,16 +824,16 @@ class line_sender_buffer _impl = ::line_sender_buffer_with_max_name_len(_max_name_len); ::line_sender_buffer_reserve(_impl, _init_buf_size); line_sender_error::wrapped_call( - line_sender_buffer_set_line_protocol_version, + line_sender_buffer_set_protocol_version, _impl, - _line_protocol_version); + _protocol_version); } } ::line_sender_buffer* _impl; size_t _init_buf_size; size_t _max_name_len; - line_protocol_version _line_protocol_version{::line_protocol_version_2}; + protocol_version _protocol_version{::protocol_version_2}; friend class line_sender; }; @@ -913,23 +893,23 @@ class opts * @param[in] protocol The protocol to use. * @param[in] host The QuestDB database host. * @param[in] port The QuestDB tcp or http port. - * @param[in] disable_line_protocol_validation disable line protocol version + * @param[in] disable_protocol_validation disable line protocol version * validation. */ opts( protocol protocol, utf8_view host, uint16_t port, - bool disable_line_protocol_validation = false) noexcept + bool disable_protocol_validation = false) noexcept : _impl{::line_sender_opts_new( static_cast<::line_sender_protocol>(protocol), host._impl, port)} { line_sender_error::wrapped_call( ::line_sender_opts_user_agent, _impl, _user_agent::name()); - if (disable_line_protocol_validation) + if (disable_protocol_validation) { line_sender_error::wrapped_call( - ::line_sender_opts_disable_line_protocol_validation, _impl); + ::line_sender_opts_disable_protocol_validation, _impl); } } @@ -939,13 +919,13 @@ class opts * @param[in] protocol The protocol to use. * @param[in] host The QuestDB database host. * @param[in] port The QuestDB tcp or http port as service name. - * @param[in] disable_line_protocol_validation disable line protocol version + * @param[in] disable_protocol_validation disable line protocol version */ opts( protocol protocol, utf8_view host, utf8_view port, - bool disable_line_protocol_validation = false) noexcept + bool disable_protocol_validation = false) noexcept : _impl{::line_sender_opts_new_service( static_cast<::line_sender_protocol>(protocol), host._impl, @@ -953,10 +933,10 @@ class opts { line_sender_error::wrapped_call( ::line_sender_opts_user_agent, _impl, _user_agent::name()); - if (disable_line_protocol_validation) + if (disable_protocol_validation) { line_sender_error::wrapped_call( - ::line_sender_opts_disable_line_protocol_validation, _impl); + ::line_sender_opts_disable_protocol_validation, _impl); } } @@ -1067,10 +1047,10 @@ class opts /** * Disable the validation of the line protocol version. */ - opts& disable_line_protocol_validation() + opts& disable_protocol_validation() { line_sender_error::wrapped_call( - ::line_sender_opts_disable_line_protocol_validation, _impl); + ::line_sender_opts_disable_protocol_validation, _impl); return *this; } @@ -1264,9 +1244,8 @@ class line_sender protocol protocol, utf8_view host, uint16_t port, - bool disable_line_protocol_validation = false) - : line_sender{ - opts{protocol, host, port, disable_line_protocol_validation}} + bool disable_protocol_validation = false) + : line_sender{opts{protocol, host, port, disable_protocol_validation}} { } @@ -1274,9 +1253,8 @@ class line_sender protocol protocol, utf8_view host, utf8_view port, - bool disable_line_protocol_validation = false) - : line_sender{ - opts{protocol, host, port, disable_line_protocol_validation}} + bool disable_protocol_validation = false) + : line_sender{opts{protocol, host, port, disable_protocol_validation}} { } @@ -1369,10 +1347,10 @@ class line_sender /** * Returns the QuestDB server's recommended default line protocol version. */ - line_protocol_version default_line_protocol_version() + protocol_version default_protocol_version() { ensure_impl(); - return line_sender_default_line_protocol_version(_impl); + return line_sender_default_protocol_version(_impl); } /** diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 594a2c97..8a93a817 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -147,7 +147,7 @@ pub enum line_sender_error_code { line_sender_error_array_view_write_to_buffer_error, /// Line sender protocol version error. - line_sender_error_line_protocol_version_error, + line_sender_error_protocol_version_error, } impl From for line_sender_error_code { @@ -181,8 +181,8 @@ impl From for line_sender_error_code { ErrorCode::ArrayWriteToBufferError => { line_sender_error_code::line_sender_error_array_view_write_to_buffer_error } - ErrorCode::LineProtocolVersionError => { - line_sender_error_code::line_sender_error_line_protocol_version_error + ErrorCode::ProtocolVersionError => { + line_sender_error_code::line_sender_error_protocol_version_error } } } @@ -230,7 +230,7 @@ impl From for Protocol { /// The version of Line Protocol used for [`Buffer`]. #[repr(C)] #[derive(Debug, Copy, Clone)] -pub enum LineProtocolVersion { +pub enum ProtocolVersion { /// Version 1 of Line Protocol. /// Uses text format serialization for f64. V1 = 1, @@ -240,20 +240,20 @@ pub enum LineProtocolVersion { V2 = 2, } -impl From for ingress::LineProtocolVersion { - fn from(version: LineProtocolVersion) -> Self { +impl From for ingress::ProtocolVersion { + fn from(version: ProtocolVersion) -> Self { match version { - LineProtocolVersion::V1 => ingress::LineProtocolVersion::V1, - LineProtocolVersion::V2 => ingress::LineProtocolVersion::V2, + ProtocolVersion::V1 => ingress::ProtocolVersion::V1, + ProtocolVersion::V2 => ingress::ProtocolVersion::V2, } } } -impl From for LineProtocolVersion { - fn from(version: ingress::LineProtocolVersion) -> Self { +impl From for ProtocolVersion { + fn from(version: ingress::ProtocolVersion) -> Self { match version { - ingress::LineProtocolVersion::V1 => LineProtocolVersion::V1, - ingress::LineProtocolVersion::V2 => LineProtocolVersion::V2, + ingress::ProtocolVersion::V1 => ProtocolVersion::V1, + ingress::ProtocolVersion::V2 => ProtocolVersion::V2, } } } @@ -599,8 +599,10 @@ pub struct line_sender_buffer(Buffer); /// Construct a `line_sender_buffer` with a `max_name_len` of `127`, which is the /// same as the QuestDB server default. #[no_mangle] -pub unsafe extern "C" fn line_sender_buffer_new() -> *mut line_sender_buffer { - let buffer = Buffer::new(); +pub unsafe extern "C" fn line_sender_buffer_new( + version: ProtocolVersion, +) -> *mut line_sender_buffer { + let buffer = Buffer::new(version.into()); Box::into_raw(Box::new(line_sender_buffer(buffer))) } @@ -612,22 +614,12 @@ pub unsafe extern "C" fn line_sender_buffer_new() -> *mut line_sender_buffer { #[no_mangle] pub unsafe extern "C" fn line_sender_buffer_with_max_name_len( max_name_len: size_t, + version: ProtocolVersion, ) -> *mut line_sender_buffer { - let buffer = Buffer::with_max_name_len(max_name_len); + let buffer = Buffer::with_max_name_len(max_name_len, version.into()); Box::into_raw(Box::new(line_sender_buffer(buffer))) } -#[no_mangle] -pub unsafe extern "C" fn line_sender_buffer_set_line_protocol_version( - buffer: *mut line_sender_buffer, - version: LineProtocolVersion, - err_out: *mut *mut line_sender_error, -) -> bool { - let buffer = unwrap_buffer_mut(buffer); - bubble_err_to_c!(err_out, buffer.set_line_proto_version(version.into())); - true -} - /// Release the `line_sender_buffer` object. #[no_mangle] pub unsafe extern "C" fn line_sender_buffer_free(buffer: *mut line_sender_buffer) { @@ -1182,11 +1174,21 @@ pub unsafe extern "C" fn line_sender_opts_token_y( /// Disable the line protocol validation. #[no_mangle] -pub unsafe extern "C" fn line_sender_opts_disable_line_protocol_validation( +pub unsafe extern "C" fn line_sender_opts_disable_protocol_validation( + opts: *mut line_sender_opts, + err_out: *mut *mut line_sender_error, +) -> bool { + upd_opts!(opts, err_out, disable_protocol_validation) +} + +/// set the line protocol version. +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_protocol_version( opts: *mut line_sender_opts, + version: ProtocolVersion, err_out: *mut *mut line_sender_error, ) -> bool { - upd_opts!(opts, err_out, disable_line_protocol_validation) + upd_opts!(opts, err_out, protocol_version, version.into()) } /// Configure how long to wait for messages from the QuestDB server during @@ -1419,21 +1421,46 @@ unsafe fn unwrap_sender_mut<'a>(sender: *mut line_sender) -> &'a mut Sender { } /// Returns the client's recommended default line protocol version. -/// Will be used to [`line_sender_buffer_set_line_protocol_version`] /// /// The version selection follows these rules: -/// 1. **TCP/TCPS Protocol**: Always returns [`LineProtocolVersion::V2`] +/// 1. **TCP/TCPS Protocol**: Always returns [`ProtocolVersion::V2`] /// 2. **HTTP/HTTPS Protocol**: -/// - If line protocol auto-detection is disabled [`line_sender_opts_disable_line_protocol_validation`], returns [`LineProtocolVersion::V2`] +/// - If line protocol auto-detection is disabled [`line_sender_opts_disable_protocol_validation`], returns [`ProtocolVersion::V2`] /// - If line protocol auto-detection is enabled: /// - Uses the server's default version if supported by the client /// - Otherwise uses the highest mutually supported version from the intersection /// of client and server compatible versions #[no_mangle] -pub unsafe extern "C" fn line_sender_default_line_protocol_version( +pub unsafe extern "C" fn line_sender_default_protocol_version( sender: *const line_sender, -) -> LineProtocolVersion { - unwrap_sender(sender).default_line_protocol_version().into() +) -> ProtocolVersion { + unwrap_sender(sender).default_protocol_version().into() +} + +/// Construct a `line_sender_buffer` with a `max_name_len` of `127` and sender's default protocol version +/// which is the same as the QuestDB server default. +#[no_mangle] +pub unsafe extern "C" fn line_sender_new_buffer( + sender: *const line_sender, +) -> *mut line_sender_buffer { + let sender = unwrap_sender(sender); + let buffer = sender.new_buffer(); + Box::into_raw(Box::new(line_sender_buffer(buffer))) +} + +/// Construct a `line_sender_buffer` with sender's default protocol version and a custom maximum +/// length for table and column names. This should match the `cairo.max.file.name.length` setting of +/// the QuestDB server you're connecting to. +/// If the server does not configure it, the default is `127`, and you can +/// call `line_sender_new_buffer()` instead. +#[no_mangle] +pub unsafe extern "C" fn line_sender_new_buffer_with_max_name_len( + sender: *const line_sender, + max_name_len: size_t, +) -> *mut line_sender_buffer { + let sender = unwrap_sender(sender); + let buffer = sender.new_buffer_with_max_name_len(max_name_len); + Box::into_raw(Box::new(line_sender_buffer(buffer))) } /// Tell whether the sender is no longer usable and must be closed. diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index 15b3b1f3..67133d08 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -56,7 +56,6 @@ mio = { version = "1", features = ["os-poll", "net"] } chrono = "0.4.31" tempfile = "3" webpki-roots = "0.26.8" -criterion = "0.5" rstest = "0.25.0" [features] @@ -86,8 +85,6 @@ json_tests = [] # Enable methods to create timestamp objects from chrono::DateTime objects. chrono_timestamp = ["chrono"] -benchmark = [] - # The `aws-lc-crypto` and `ring-crypto` features are mutually exclusive, # thus compiling with `--all-features` will not work. # Instead compile with `--features almost-all-features`. @@ -102,11 +99,6 @@ almost-all-features = [ "ndarray" ] -[[bench]] -name = "ndarr" -harness = false -required-features = ["benchmark", "ndarray"] - [[example]] name = "basic" required-features = ["chrono_timestamp", "ndarray"] @@ -124,5 +116,5 @@ name = "http" required-features = ["ilp-over-http", "ndarray"] [[example]] -name = "line_protocol_version" +name = "protocol_version" required-features = ["ilp-over-http", "ndarray"] diff --git a/questdb-rs/README.md b/questdb-rs/README.md index 093fd6de..4d868d35 100644 --- a/questdb-rs/README.md +++ b/questdb-rs/README.md @@ -30,7 +30,7 @@ use questdb::{ fn main() -> Result<()> { let mut sender = Sender::from_conf("http::addr=localhost:9000;")?; - let mut buffer = Buffer::new(); + let mut buffer = sender.new_buffer(); buffer .table("trades")? .symbol("symbol", "ETH-USD")? diff --git a/questdb-rs/benches/ndarr.rs b/questdb-rs/benches/ndarr.rs deleted file mode 100644 index c6643ab5..00000000 --- a/questdb-rs/benches/ndarr.rs +++ /dev/null @@ -1,116 +0,0 @@ -use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use ndarray::{Array, Array2}; -use questdb::ingress::{Buffer, ColumnName, StrideArrayView}; - -/// run with -/// ```shell -/// cargo bench --bench ndarr --features="benchmark, ndarray" -/// ``` -fn bench_write_array_data(c: &mut Criterion) { - let mut group = c.benchmark_group("write_array_data"); - let contiguous_array: Array2 = Array::zeros((1000, 1000)); - let non_contiguous_array = contiguous_array.t(); - assert!(contiguous_array.is_standard_layout()); - assert!(!non_contiguous_array.is_standard_layout()); - - let col_name = ColumnName::new("col1").unwrap(); - // Case 1 - group.bench_function("contiguous_writer", |b| { - let mut buffer = Buffer::new(); - buffer.table("x1").unwrap(); - b.iter(|| { - buffer - .column_arr(col_name, black_box(&contiguous_array.view())) - .unwrap(); - }); - buffer.clear(); - }); - - // Case 2 - group.bench_function("contiguous_raw_buffer", |b| { - let mut buffer = Buffer::new(); - buffer.table("x1").unwrap(); - b.iter(|| { - buffer - .column_arr_use_raw_buffer(col_name, black_box(&contiguous_array.view())) - .unwrap(); - }); - buffer.clear(); - }); - - // Case 3 - group.bench_function("non_contiguous_writer", |b| { - let mut buffer = Buffer::new(); - buffer.table("x1").unwrap(); - b.iter(|| { - buffer - .column_arr(col_name, black_box(&non_contiguous_array.view())) - .unwrap(); - }); - buffer.clear(); - }); - - // Case 4 - group.bench_function("non_contiguous_raw_buffer", |b| { - let mut buffer = Buffer::new(); - buffer.table("x1").unwrap(); - b.iter(|| { - buffer - .column_arr_use_raw_buffer(col_name, black_box(&non_contiguous_array.view())) - .unwrap(); - }); - buffer.clear(); - }); - - group.finish(); -} - -// bench NdArrayView and StridedArrayView write performance. -fn bench_array_view(c: &mut Criterion) { - let mut group = c.benchmark_group("write_array_view"); - let col_name = ColumnName::new("col1").unwrap(); - let array: Array2 = Array::ones((1000, 1000)); - let transposed_view = array.t(); - - // Case 1 - group.bench_function("ndarray_view", |b| { - let mut buffer = Buffer::new(); - buffer.table("x1").unwrap(); - b.iter(|| { - buffer - .column_arr(col_name, black_box(&transposed_view)) - .unwrap(); - }); - buffer.clear(); - }); - - let elem_size = size_of::() as isize; - let strides: Vec = transposed_view - .strides() - .iter() - .map(|&s| s * elem_size) // 转换为字节步长 - .collect(); - let view2: StrideArrayView<'_, f64> = unsafe { - StrideArrayView::new( - transposed_view.ndim(), - transposed_view.shape().as_ptr(), - strides.as_ptr(), - transposed_view.as_ptr() as *const u8, - transposed_view.len() * elem_size as usize, - ) - .unwrap() - }; - - // Case 2 - group.bench_function("strides_view", |b| { - let mut buffer = Buffer::new(); - buffer.table("x1").unwrap(); - b.iter(|| { - buffer.column_arr(col_name, black_box(&view2)).unwrap(); - }); - buffer.clear(); - }); -} - -criterion_group!(benches, bench_write_array_data, bench_array_view); -criterion_main!(benches); diff --git a/questdb-rs/build.rs b/questdb-rs/build.rs index 99d0cf10..9a1350f3 100644 --- a/questdb-rs/build.rs +++ b/questdb-rs/build.rs @@ -97,7 +97,7 @@ pub mod json_tests { indoc! {r#" // This file is auto-generated by build.rs. - use crate::{Result, ingress::{Buffer, LineProtocolVersion}}; + use crate::{Result, ingress::{Buffer, ProtocolVersion}}; use crate::tests::{TestResult}; use base64ct::Base64; use base64ct::Encoding; @@ -125,14 +125,11 @@ pub mod json_tests { writeln!(output, "#[rstest]")?; writeln!( output, - "fn test_{:03}_{}(\n #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion,\n) -> TestResult {{", + "fn test_{:03}_{}(\n #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion,\n) -> TestResult {{", index, slugify!(&spec.test_name, separator = "_") )?; - writeln!( - output, - " let mut buffer = Buffer::new().with_line_proto_version(version)?;" - )?; + writeln!(output, " let mut buffer = Buffer::new(version);")?; let (expected, indent) = match &spec.result { Outcome::Success(line) => (Some(line), ""), @@ -177,7 +174,7 @@ pub mod json_tests { writeln!(output, "{} .at_now()?;", indent)?; if let Some(expected) = expected { if let Some(ref base64) = expected.binary_base64 { - writeln!(output, " if version != LineProtocolVersion::V1 {{")?; + writeln!(output, " if version != ProtocolVersion::V1 {{")?; writeln!( output, " let exp = Base64::decode_vec(\"{}\").unwrap();", diff --git a/questdb-rs/examples/auth.rs b/questdb-rs/examples/auth.rs index 2dd6e66d..10eb20be 100644 --- a/questdb-rs/examples/auth.rs +++ b/questdb-rs/examples/auth.rs @@ -1,6 +1,6 @@ use chrono::{TimeZone, Utc}; use questdb::{ - ingress::{Buffer, Sender, TimestampNanos}, + ingress::{Sender, TimestampNanos}, Result, }; @@ -17,7 +17,7 @@ fn main() -> Result<()> { ), host, port ))?; - let mut buffer = Buffer::new(); + let mut buffer = sender.new_buffer(); let designated_timestamp = TimestampNanos::from_datetime(Utc.with_ymd_and_hms(1997, 7, 4, 4, 56, 55).unwrap())?; buffer diff --git a/questdb-rs/examples/auth_tls.rs b/questdb-rs/examples/auth_tls.rs index 19225027..8355ebd5 100644 --- a/questdb-rs/examples/auth_tls.rs +++ b/questdb-rs/examples/auth_tls.rs @@ -1,6 +1,6 @@ use chrono::{TimeZone, Utc}; use questdb::{ - ingress::{Buffer, Sender, TimestampNanos}, + ingress::{Sender, TimestampNanos}, Result, }; @@ -17,7 +17,7 @@ fn main() -> Result<()> { ), host, port ))?; - let mut buffer = Buffer::new(); + let mut buffer = sender.new_buffer(); let designated_timestamp = TimestampNanos::from_datetime(Utc.with_ymd_and_hms(1997, 7, 4, 4, 56, 55).unwrap())?; buffer diff --git a/questdb-rs/examples/basic.rs b/questdb-rs/examples/basic.rs index 5df32873..853003b9 100644 --- a/questdb-rs/examples/basic.rs +++ b/questdb-rs/examples/basic.rs @@ -1,7 +1,7 @@ use chrono::{TimeZone, Utc}; use ndarray::arr1; use questdb::{ - ingress::{Buffer, Sender, TimestampNanos}, + ingress::{Sender, TimestampNanos}, Result, }; @@ -9,7 +9,7 @@ fn main() -> Result<()> { let host: String = std::env::args().nth(1).unwrap_or("localhost".to_string()); let port: &str = &std::env::args().nth(2).unwrap_or("9009".to_string()); let mut sender = Sender::from_conf(format!("tcp::addr={host}:{port};"))?; - let mut buffer = Buffer::new(); + let mut buffer = sender.new_buffer(); let designated_timestamp = TimestampNanos::from_datetime(Utc.with_ymd_and_hms(1997, 7, 4, 4, 56, 55).unwrap())?; buffer diff --git a/questdb-rs/examples/from_conf.rs b/questdb-rs/examples/from_conf.rs index d328c31c..2cdbaed0 100644 --- a/questdb-rs/examples/from_conf.rs +++ b/questdb-rs/examples/from_conf.rs @@ -1,11 +1,11 @@ use questdb::{ - ingress::{Buffer, Sender, TimestampNanos}, + ingress::{Sender, TimestampNanos}, Result, }; fn main() -> Result<()> { let mut sender = Sender::from_conf("tcp::addr=localhost:9009;")?; - let mut buffer = Buffer::new(); + let mut buffer = sender.new_buffer(); buffer .table("trades")? .symbol("symbol", "ETH-USD")? diff --git a/questdb-rs/examples/from_env.rs b/questdb-rs/examples/from_env.rs index ca338851..8b2b3d8b 100644 --- a/questdb-rs/examples/from_env.rs +++ b/questdb-rs/examples/from_env.rs @@ -1,12 +1,12 @@ use questdb::{ - ingress::{Buffer, Sender, TimestampNanos}, + ingress::{Sender, TimestampNanos}, Result, }; fn main() -> Result<()> { // Read configuration string from the `QDB_CLIENT_CONF` environment variable. let mut sender = Sender::from_env()?; - let mut buffer = Buffer::new(); + let mut buffer = sender.new_buffer(); buffer .table("trades")? .symbol("symbol", "ETH-USD")? diff --git a/questdb-rs/examples/http.rs b/questdb-rs/examples/http.rs index ead1efde..f9e5954e 100644 --- a/questdb-rs/examples/http.rs +++ b/questdb-rs/examples/http.rs @@ -1,13 +1,12 @@ use ndarray::arr1; use questdb::{ - ingress::{Buffer, Sender, TimestampNanos}, + ingress::{Sender, TimestampNanos}, Result, }; fn main() -> Result<()> { let mut sender = Sender::from_conf("https::addr=localhost:9000;username=foo;password=bar;")?; - let mut buffer = - Buffer::new().with_line_proto_version(sender.default_line_protocol_version())?; + let mut buffer = sender.new_buffer(); buffer .table("trades")? .symbol("symbol", "ETH-USD")? diff --git a/questdb-rs/examples/line_protocol_version.rs b/questdb-rs/examples/protocol_version.rs similarity index 58% rename from questdb-rs/examples/line_protocol_version.rs rename to questdb-rs/examples/protocol_version.rs index 656db6b8..0d184d90 100644 --- a/questdb-rs/examples/line_protocol_version.rs +++ b/questdb-rs/examples/protocol_version.rs @@ -1,13 +1,14 @@ use ndarray::arr1; -use questdb::ingress::LineProtocolVersion; use questdb::{ - ingress::{Buffer, Sender, TimestampNanos}, + ingress::{Sender, TimestampNanos}, Result, }; fn main() -> Result<()> { - let mut sender = Sender::from_conf("https::addr=localhost:9000;username=foo;password=bar;")?; - let mut buffer = Buffer::new().with_line_proto_version(LineProtocolVersion::V1)?; + let mut sender = Sender::from_conf( + "https::addr=localhost:9000;username=foo;password=bar;protocol_version=1;", + )?; + let mut buffer = sender.new_buffer(); buffer .table("trades_ilp_v1")? .symbol("symbol", "ETH-USD")? @@ -17,9 +18,10 @@ fn main() -> Result<()> { .at(TimestampNanos::now())?; sender.flush(&mut buffer)?; - let mut sender2 = Sender::from_conf("https::addr=localhost:9000;username=foo;password=bar;")?; - let mut buffer2 = - Buffer::new().with_line_proto_version(sender2.default_line_protocol_version())?; + let mut sender2 = Sender::from_conf( + "https::addr=localhost:9000;username=foo;password=bar;protocol_version=1;", + )?; + let mut buffer2 = sender.new_buffer(); buffer2 .table("trades_ilp_v2")? .symbol("symbol", "ETH-USD")? diff --git a/questdb-rs/src/error.rs b/questdb-rs/src/error.rs index 680a6b50..a6fecf23 100644 --- a/questdb-rs/src/error.rs +++ b/questdb-rs/src/error.rs @@ -58,8 +58,8 @@ pub enum ErrorCode { /// Array write to buffer error. ArrayWriteToBufferError, - /// Validate line protocol version error. - LineProtocolVersionError, + /// Validate protocol version error. + ProtocolVersionError, } /// An error that occurred when using QuestDB client library. diff --git a/questdb-rs/src/ingress/http.rs b/questdb-rs/src/ingress/http.rs index 92c867bf..ac36b615 100644 --- a/questdb-rs/src/ingress/http.rs +++ b/questdb-rs/src/ingress/http.rs @@ -17,7 +17,7 @@ use ureq::unversioned::transport::{ Buffers, Connector, LazyBuffers, NextTimeout, Transport, TransportAdapter, }; -use crate::ingress::LineProtocolVersion; +use crate::ingress::ProtocolVersion; use ureq::unversioned::*; use ureq::Error::*; use ureq::{http, Body}; @@ -59,7 +59,7 @@ pub(super) struct HttpConfig { pub(super) user_agent: String, pub(super) retry_timeout: ConfigSetting, pub(super) request_timeout: ConfigSetting, - pub(super) disable_line_proto_validation: ConfigSetting, + pub(super) disable_protocol_validation: ConfigSetting, } impl Default for HttpConfig { @@ -69,7 +69,7 @@ impl Default for HttpConfig { user_agent: concat!("questdb/rust/", env!("CARGO_PKG_VERSION")).to_string(), retry_timeout: ConfigSetting::new_default(Duration::from_secs(10)), request_timeout: ConfigSetting::new_default(Duration::from_secs(10)), - disable_line_proto_validation: ConfigSetting::new_default(false), + disable_protocol_validation: ConfigSetting::new_default(false), } } } @@ -406,12 +406,25 @@ pub(super) fn http_send_with_retries( retry_http_send(state, buf, request_timeout, retry_timeout, last_rep) } -pub(super) fn get_line_protocol_version( +/// Determines the server's default and all-supported protocol versions. +/// +/// Returns a tuple containing: +/// - `Option>`: List of all protocol versions supported by the server. +/// - `Some(versions)`: When server explicitly provides supported versions (modern servers). +/// - `None`: When server doesn't provide version info (legacy servers or 404 response). +/// - `ProtocolVersion`: The server-recommended default protocol version +/// (Here we introduce a new field, rather than use the implicit max value of supported versions). +/// +/// When protocol version is auto-detection mode (no explicit set by use user), +/// client will use the server's `default_version` as protocol version. +/// When user explicitly specifies a `protocol_version`, client will +/// validate against `support_versions`, Returns error if specified version not in supported list. +pub(super) fn get_protocol_version( state: &HttpHandlerState, settings_url: &str, -) -> Result<(Option>, LineProtocolVersion), Error> { +) -> Result<(Option>, ProtocolVersion), Error> { let mut support_versions: Option> = None; - let mut default_version = LineProtocolVersion::V1; + let mut default_version = ProtocolVersion::V1; let response = match http_get_with_retries( state, @@ -425,7 +438,7 @@ pub(super) fn get_line_protocol_version( return Ok((support_versions, default_version)); } return Err(fmt!( - LineProtocolVersionError, + ProtocolVersionError, "Failed to detect server's line protocol version, settings url: {}, status code: {}.", settings_url, res.status() @@ -441,7 +454,7 @@ pub(super) fn get_line_protocol_version( return Ok((support_versions, default_version)); } else { fmt!( - LineProtocolVersionError, + ProtocolVersionError, "Failed to detect server's line protocol version, settings url: {}, err: {}.", settings_url, err @@ -450,7 +463,7 @@ pub(super) fn get_line_protocol_version( } e => { fmt!( - LineProtocolVersionError, + ProtocolVersionError, "Failed to detect server's line protocol version, settings url: {}, err: {}.", settings_url, e @@ -467,7 +480,7 @@ pub(super) fn get_line_protocol_version( if let Ok(msg) = body_content { let json: serde_json::Value = serde_json::from_str(&msg).map_err(|_| { error::fmt!( - LineProtocolVersionError, + ProtocolVersionError, "Malformed server response, settings url: {}, err: response is not valid JSON.", settings_url, ) @@ -479,8 +492,8 @@ pub(super) fn get_line_protocol_version( for value in values.iter() { if let Some(v) = value.as_u64() { match v { - 1 => versions.push(LineProtocolVersion::V1), - 2 => versions.push(LineProtocolVersion::V2), + 1 => versions.push(ProtocolVersion::V1), + 2 => versions.push(ProtocolVersion::V2), _ => {} } } @@ -491,23 +504,23 @@ pub(super) fn get_line_protocol_version( if let Some(serde_json::Value::Number(ref v)) = json.get("line.proto.default.version") { default_version = match v.as_u64() { Some(vu64) => match vu64 { - 1 => LineProtocolVersion::V1, - 2 => LineProtocolVersion::V2, + 1 => ProtocolVersion::V1, + 2 => ProtocolVersion::V2, _ => { if let Some(ref versions) = support_versions { - if versions.contains(&LineProtocolVersion::V2) { - LineProtocolVersion::V2 - } else if versions.contains(&LineProtocolVersion::V1) { - LineProtocolVersion::V1 + if versions.contains(&ProtocolVersion::V2) { + ProtocolVersion::V2 + } else if versions.contains(&ProtocolVersion::V1) { + ProtocolVersion::V1 } else { return Err(error::fmt!( - LineProtocolVersionError, + ProtocolVersionError, "Server does not support current client" )); } } else { return Err(error::fmt!( - LineProtocolVersionError, + ProtocolVersionError, "Unexpected response version content." )); } @@ -515,7 +528,7 @@ pub(super) fn get_line_protocol_version( }, None => { return Err(error::fmt!( - LineProtocolVersionError, + ProtocolVersionError, "Not a valid int for line.proto.default.version in response." )) } @@ -523,7 +536,7 @@ pub(super) fn get_line_protocol_version( } } else { return Err(error::fmt!( - LineProtocolVersionError, + ProtocolVersionError, "Malformed server response, settings url: {}, err: failed to read response body as UTF-8", settings_url )); } diff --git a/questdb-rs/src/ingress/mod.md b/questdb-rs/src/ingress/mod.md index aed342af..4a1a78cb 100644 --- a/questdb-rs/src/ingress/mod.md +++ b/questdb-rs/src/ingress/mod.md @@ -19,7 +19,7 @@ use questdb::{ TimestampNanos}}; fn main() -> Result<()> { let mut sender = Sender::from_conf("http::addr=localhost:9000;")?; - let mut buffer = Buffer::new(); + let mut buffer = sender.new_buffer(); buffer .table("trades")? .symbol("symbol", "ETH-USD")? @@ -297,9 +297,11 @@ use questdb::ingress::{ TableName, ColumnName, Buffer, + SenderBuilder, TimestampNanos}; # fn main() -> Result<()> { -let mut buffer = Buffer::new(); +let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; +let mut buffer = sender.new_buffer(); let table_name = TableName::new("trades")?; let price_name = ColumnName::new("price")?; buffer.table(table_name)?.column_f64(price_name, 2615.54)?.at(TimestampNanos::now())?; diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index ef3307ad..953fc883 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -62,16 +62,16 @@ pub const MAX_ARRAY_DIMS: usize = 32; /// Line Protocol Version supported by current client. #[derive(Debug, Copy, Clone, PartialEq)] -pub enum LineProtocolVersion { +pub enum ProtocolVersion { V1 = 1, V2 = 2, } -impl std::fmt::Display for LineProtocolVersion { +impl std::fmt::Display for ProtocolVersion { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - LineProtocolVersion::V1 => write!(f, "v1"), - LineProtocolVersion::V2 => write!(f, "v2"), + ProtocolVersion::V1 => write!(f, "v1"), + ProtocolVersion::V2 => write!(f, "v2"), } } } @@ -485,18 +485,19 @@ impl BufferState { } } -pub trait Buffer1 {} - /// A reusable buffer to prepare a batch of ILP messages. /// /// # Example /// -/// ``` +/// ```no_run /// # use questdb::Result; -/// use questdb::ingress::{Buffer, TimestampMicros, TimestampNanos}; +/// # use questdb::ingress::SenderBuilder; /// /// # fn main() -> Result<()> { -/// let mut buffer = Buffer::new(); +/// # let mut sender = SenderBuilder::from_conf("http::addr=localhost:9000;")?.build()?; +/// # use questdb::Result; +/// use questdb::ingress::{Buffer, TimestampMicros, TimestampNanos}; +/// let mut buffer = sender.new_buffer(); /// /// // first row /// buffer @@ -580,67 +581,43 @@ pub struct Buffer { state: BufferState, marker: Option<(usize, BufferState)>, max_name_len: usize, - f64serializer: fn(&mut Vec, f64), - version: LineProtocolVersion, + version: ProtocolVersion, } impl Buffer { - /// Construct a `Buffer` with a `max_name_len` of `127`, which is the same as the - /// QuestDB server default. - pub fn new() -> Self { + /// Creates a new [`Buffer`] with default parameters. + /// + /// - Uses the specified protocol version + /// - Sets maximum name length to **127 characters** (QuestDB server default) + /// + /// This is equivalent to [`Sender::new_buffer`] when using the sender's + /// protocol version. For custom name lengths, use + /// [`Sender::new_buffer_with_max_name_len`]. + pub fn new(protocol_version: ProtocolVersion) -> Self { Self { output: Vec::new(), state: BufferState::new(), marker: None, max_name_len: 127, - f64serializer: f64_binary_series, - version: LineProtocolVersion::V2, + version: protocol_version, } } - /// Construct a `Buffer` with a custom maximum length for table and column names. + /// Creates a new [`Buffer`] with a custom maximum name length. /// - /// This should match the `cairo.max.file.name.length` setting of the - /// QuestDB instance you're connecting to. + /// - `max_name_len`: Maximum allowed length for table/column names, must match + /// your QuestDB server's `cairo.max.file.name.length` configuration + /// - `protocol_version`: Protocol version to use /// - /// If the server does not configure it, the default is `127` and you can simply - /// call [`new`](Buffer::new). - pub fn with_max_name_len(max_name_len: usize) -> Self { - let mut buf = Self::new(); + /// This is equivalent to [`Sender::new_buffer_with_max_name_len`] when using + /// the sender's protocol version. For the default name length (127), + /// use [`new`](Self::new) or [`Sender::new_buffer`]. + pub fn with_max_name_len(max_name_len: usize, protocol_version: ProtocolVersion) -> Self { + let mut buf = Self::new(protocol_version); buf.max_name_len = max_name_len; buf } - pub fn with_line_proto_version(mut self, version: LineProtocolVersion) -> Result { - if self.state.op_case != OpCase::Init { - return Err(error::fmt!( - LineProtocolVersionError, - "Line protocol version must be set before adding any data." - )); - } - self.f64serializer = match version { - LineProtocolVersion::V1 => f64_text_series, - LineProtocolVersion::V2 => f64_binary_series, - }; - self.version = version; - Ok(self) - } - - pub fn set_line_proto_version(&mut self, version: LineProtocolVersion) -> Result<&mut Self> { - if self.state.op_case != OpCase::Init { - return Err(error::fmt!( - LineProtocolVersionError, - "Line protocol version must be set before adding any data." - )); - } - self.f64serializer = match version { - LineProtocolVersion::V1 => f64_text_series, - LineProtocolVersion::V2 => f64_binary_series, - }; - self.version = version; - Ok(self) - } - /// Pre-allocate to ensure the buffer has enough capacity for at least the /// specified additional byte count. This may be rounded up. /// This does not allocate if such additional capacity is already satisfied. @@ -772,11 +749,12 @@ impl Buffer { /// Begin recording a new row for the given table. /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// buffer.table("table_name")?; /// # Ok(()) /// # } @@ -784,13 +762,14 @@ impl Buffer { /// /// or /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// use questdb::ingress::TableName; /// /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// let table_name = TableName::new("table_name")?; /// buffer.table(table_name)?; /// # Ok(()) @@ -821,11 +800,12 @@ impl Buffer { /// Record a symbol for the given column. /// Make sure you record all symbol columns before any other column type. /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// buffer.symbol("col_name", "value")?; /// # Ok(()) @@ -834,11 +814,12 @@ impl Buffer { /// /// or /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// let value: String = "value".to_owned(); /// buffer.symbol("col_name", value)?; @@ -848,13 +829,14 @@ impl Buffer { /// /// or /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// use questdb::ingress::ColumnName; /// /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// let col_name = ColumnName::new("col_name")?; /// buffer.symbol(col_name, "value")?; @@ -901,11 +883,12 @@ impl Buffer { /// Record a boolean value for the given column. /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// buffer.column_bool("col_name", true)?; /// # Ok(()) @@ -914,13 +897,14 @@ impl Buffer { /// /// or /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// use questdb::ingress::ColumnName; /// /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// let col_name = ColumnName::new("col_name")?; /// buffer.column_bool(col_name, true)?; @@ -939,11 +923,12 @@ impl Buffer { /// Record an integer value for the given column. /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// buffer.column_i64("col_name", 42)?; /// # Ok(()) @@ -952,13 +937,14 @@ impl Buffer { /// /// or /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// use questdb::ingress::ColumnName; /// /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// let col_name = ColumnName::new("col_name")?; /// buffer.column_i64(col_name, 42); @@ -980,11 +966,12 @@ impl Buffer { /// Record a floating point value for the given column. /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// buffer.column_f64("col_name", 3.14)?; /// # Ok(()) @@ -993,13 +980,14 @@ impl Buffer { /// /// or /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// use questdb::ingress::ColumnName; /// /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// let col_name = ColumnName::new("col_name")?; /// buffer.column_f64(col_name, 3.14)?; @@ -1012,17 +1000,25 @@ impl Buffer { Error: From, { self.write_column_key(name)?; - (self.f64serializer)(&mut self.output, value); + if !matches!(self.version, ProtocolVersion::V1) { + self.output.push(b'='); + self.output.push(DOUBLE_BINARY_FORMAT_TYPE); + self.output.extend_from_slice(&value.to_le_bytes()) + } else { + let mut ser = F64Serializer::new(value); + self.output.extend_from_slice(ser.as_str().as_bytes()) + } Ok(self) } /// Record a string value for the given column. /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// buffer.column_str("col_name", "value")?; /// # Ok(()) @@ -1031,11 +1027,12 @@ impl Buffer { /// /// or /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// let value: String = "value".to_owned(); /// buffer.column_str("col_name", value)?; @@ -1045,13 +1042,14 @@ impl Buffer { /// /// or /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// use questdb::ingress::ColumnName; /// /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// let col_name = ColumnName::new("col_name")?; /// buffer.column_str(col_name, "value")?; @@ -1072,55 +1070,49 @@ impl Buffer { /// Record a multidimensional array value for the given column. /// /// Supports arrays with up to [`MAX_ARRAY_DIMS`] dimensions. The array elements must - /// implement [`ArrayElement`] trait which provides type-to-[`ElemDataType`] mapping. + /// be of type `f64`, which is currently the only supported data type. /// /// # Examples /// - /// Basic usage with direct dimension specification: + /// Recording a 2D array using slices: /// - /// ``` - /// # #[cfg(feature = "ndarray")] - /// # { + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; - /// # use ndarray::array; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; - /// // Record a 2D array of f64 values - /// let array_2d = array![[1.1, 2.2], [3.3, 4.4]]; - /// buffer.column_arr("array_col", &array_2d.view())?; + /// let array_2d = vec![vec![1.1, 2.2], vec![3.3, 4.4]]; + /// buffer.column_arr("array_col", &array_2d)?; /// # Ok(()) /// # } - /// # } - /// /// ``` /// - /// Using [`ColumnName`] for validated column names: + /// Recording a 3D array using vectors: /// - /// ``` - /// # #[cfg(feature = "ndarray")] - /// # { + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::{Buffer, ColumnName}; - /// # use ndarray::Array3; + /// # use questdb::ingress::{Buffer, ColumnName, SenderBuilder}; /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x1")?; - /// // Record a 3D array of f64 values - /// let array_3d = Array3::from_elem((2, 3, 4), 42f64); + /// let array_3d = vec![vec![vec![42.0; 4]; 3]; 2]; /// let col_name = ColumnName::new("col1")?; - /// buffer.column_arr(col_name, &array_3d.view())?; + /// buffer.column_arr(col_name, &array_3d)?; /// # Ok(()) /// # } - /// # } /// ``` + /// /// # Errors /// /// Returns [`Error`] if: /// - Array dimensions exceed [`MAX_ARRAY_DIMS`] /// - Failed to get dimension sizes /// - Column name validation fails + /// - Protocol version v1 is used (arrays require v2+) + /// - Array contains unsupported element types (non-f64) #[allow(private_bounds)] pub fn column_arr<'a, N, T, D>(&mut self, name: N, view: &T) -> Result<&mut Self> where @@ -1129,9 +1121,9 @@ impl Buffer { D: ArrayElement + ArrayElementSealed, Error: From, { - if self.version == LineProtocolVersion::V1 { + if self.version == ProtocolVersion::V1 { return Err(error::fmt!( - LineProtocolVersionError, + ProtocolVersionError, "line protocol version v1 does not support array datatype", )); } @@ -1193,59 +1185,15 @@ impl Buffer { Ok(self) } - #[allow(private_bounds)] - #[cfg(feature = "benchmark")] - pub fn column_arr_use_raw_buffer<'a, N, T, D>(&mut self, name: N, view: &T) -> Result<&mut Self> - where - N: TryInto>, - T: NdArrayView, - D: ArrayElement + ArrayElementSealed, - Error: From, - { - self.write_column_key(name)?; - - // check dimension less equal than max dims - if MAX_ARRAY_DIMS < view.ndim() { - return Err(error::fmt!( - ArrayHasTooManyDims, - "Array dimension mismatch: expected at most {} dimensions, but got {}", - MAX_ARRAY_DIMS, - view.ndim() - )); - } - - let reserve_size = get_and_check_array_bytes_size(view)?; - // binary format flag '=' - self.output.push(b'='); - // binary format entity type - self.output.push(ARRAY_BINARY_FORMAT_TYPE); - // ndarr datatype - self.output.push(D::type_tag()); - // ndarr dims - self.output.push(view.ndim() as u8); - - for i in 0..view.ndim() { - let d = view.dim(i)?; - // ndarr shapes - self.output - .extend_from_slice((d as i32).to_le_bytes().as_slice()); - } - - self.output.reserve(reserve_size); - let index = self.output.len(); - unsafe { self.output.set_len(reserve_size + index) } - ndarr::write_array_data_use_raw_buffer(&mut self.output[index..], view); - Ok(self) - } - /// Record a timestamp value for the given column. /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// use questdb::ingress::TimestampMicros; /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// buffer.column_ts("col_name", TimestampMicros::now())?; /// # Ok(()) @@ -1254,13 +1202,14 @@ impl Buffer { /// /// or /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// use questdb::ingress::TimestampMicros; /// /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// buffer.column_ts("col_name", TimestampMicros::new(1659548204354448))?; /// # Ok(()) @@ -1269,14 +1218,15 @@ impl Buffer { /// /// or /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// use questdb::ingress::TimestampMicros; /// use questdb::ingress::ColumnName; /// /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?; /// let col_name = ColumnName::new("col_name")?; /// buffer.column_ts(col_name, TimestampMicros::now())?; @@ -1311,12 +1261,13 @@ impl Buffer { /// start recording the next row by calling [Buffer::table] again, or you can send /// the accumulated batch by calling [Sender::flush] or one of its variants. /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// use questdb::ingress::TimestampNanos; /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?.symbol("a", "b")?; /// buffer.at(TimestampNanos::now())?; /// # Ok(()) @@ -1325,13 +1276,14 @@ impl Buffer { /// /// or /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// use questdb::ingress::TimestampNanos; /// /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?.symbol("a", "b")?; /// buffer.at(TimestampNanos::new(1659548315647406592))?; /// # Ok(()) @@ -1391,11 +1343,12 @@ impl Buffer { /// again, or you can send the accumulated batch by calling [Sender::flush] or one of /// its variants. /// - /// ``` + /// ```no_run /// # use questdb::Result; - /// # use questdb::ingress::Buffer; + /// # use questdb::ingress::{Buffer, SenderBuilder}; /// # fn main() -> Result<()> { - /// # let mut buffer = Buffer::new(); + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); /// # buffer.table("x")?.symbol("a", "b")?; /// buffer.at_now()?; /// # Ok(()) @@ -1410,12 +1363,6 @@ impl Buffer { } } -impl Default for Buffer { - fn default() -> Self { - Self::new() - } -} - /// Connects to a QuestDB instance and inserts data via the ILP protocol. /// /// * To construct an instance, use [`Sender::from_conf`] or the [`SenderBuilder`]. @@ -1426,9 +1373,14 @@ pub struct Sender { handler: ProtocolHandler, connected: bool, max_buf_size: usize, - default_line_protocol_version: LineProtocolVersion, + default_protocol_version: ProtocolVersion, #[cfg(feature = "ilp-over-http")] - supported_line_protocol_versions: Option>, + /// List of protocol versions supported by the server. + /// This is an `Option>` because: + /// - `Some(versions)`: The server explicitly returned supported protocol versions + /// during handshake (with `/settings` endpoint). + /// - `None`: The server didn't provide protocol version information (legacy servers). + supported_protocol_versions: Option>, } impl std::fmt::Debug for Sender { @@ -1884,6 +1836,7 @@ pub struct SenderBuilder { token: ConfigSetting>, token_x: ConfigSetting>, token_y: ConfigSetting>, + protocol_version: ConfigSetting>, #[cfg(feature = "insecure-skip-verify")] tls_verify: ConfigSetting, @@ -1953,6 +1906,17 @@ impl SenderBuilder { "token_x" => builder.token_x(val)?, "token_y" => builder.token_y(val)?, "bind_interface" => builder.bind_interface(val)?, + "protocol_version" => match val { + "1" => builder.protocol_version(ProtocolVersion::V1)?, + "2" => builder.protocol_version(ProtocolVersion::V2)?, + "auto" => builder, + invalid => { + return Err(error::fmt!( + ConfigError, + "invalid \"protocol_version\" [value={invalid}, allowed-values=[auto, 1, 2]]]\"]" + )) + } + }, "init_buf_size" => { return Err(error::fmt!( @@ -2054,12 +2018,12 @@ impl SenderBuilder { } #[cfg(feature = "ilp-over-http")] - "disable_line_protocol_validation" => { + "disable_protocol_validation" => { if val == "on" { - builder.disable_line_protocol_validation()? + builder.disable_protocol_validation()? } else if val != "off" { return Err(error::fmt!( - ConfigError, "invalid \"disable_line_protocol_validation\" [value={val}, allowed-values=[on, off]]]\"]")); + ConfigError, "invalid \"disable_protocol_validation\" [value={val}, allowed-values=[on, off]]]\"]")); } else { builder } @@ -2126,6 +2090,7 @@ impl SenderBuilder { token: ConfigSetting::new_default(None), token_x: ConfigSetting::new_default(None), token_y: ConfigSetting::new_default(None), + protocol_version: ConfigSetting::new_default(None), #[cfg(feature = "insecure-skip-verify")] tls_verify: ConfigSetting::new_default(true), @@ -2198,6 +2163,13 @@ impl SenderBuilder { Ok(self) } + /// Set the line protocol version. + pub fn protocol_version(mut self, protocol_version: ProtocolVersion) -> Result { + self.protocol_version + .set_specified("protocol_version", Some(protocol_version))?; + Ok(self) + } + /// Configure how long to wait for messages from the QuestDB server during /// the TLS handshake and authentication process. This only applies to TCP. /// The default is 15 seconds. @@ -2341,14 +2313,14 @@ impl SenderBuilder { /// - When set to `"off"`: Skips the initial server version handshake and disables protocol validation. /// - When set to `"on"`: Keeps default validation behavior (recommended). /// - /// Please ensure client's default version ([`LINE_PROTOCOL_VERSION_V2`]) or + /// Please ensure client's default version ([`ProtocolVersion::V1`]) or /// explicitly set protocol version exactly matches server expectation. - pub fn disable_line_protocol_validation(mut self) -> Result { + pub fn disable_protocol_validation(mut self) -> Result { if let Some(http) = &mut self.http { // ignore "already specified" error let _ = http - .disable_line_proto_validation - .set_specified("disable_line_protocol_validation", true); + .disable_protocol_validation + .set_specified("disable_protocol_validation", true); } Ok(self) } @@ -2640,16 +2612,20 @@ impl SenderBuilder { } }; - let mut default_line_protocol_version = LineProtocolVersion::V2; + let mut default_protocol_version = match self.protocol_version.deref() { + None => ProtocolVersion::V1, + Some(v) => *v, + }; + #[cfg(feature = "ilp-over-http")] - let mut supported_line_protocol_versions: Option> = None; + let mut supported_protocol_versions: Option> = None; #[cfg(feature = "ilp-over-http")] match self.protocol { Protocol::Tcp | Protocol::Tcps => {} Protocol::Http | Protocol::Https => { let http_config = self.http.as_ref().unwrap(); - if !*http_config.disable_line_proto_validation.deref() { + if !*http_config.disable_protocol_validation.deref() { if let ProtocolHandler::Http(http_state) = &handler { let settings_url = &format!( "{}://{}:{}/settings", @@ -2657,12 +2633,10 @@ impl SenderBuilder { self.host.deref(), self.port.deref() ); - ( - supported_line_protocol_versions, - default_line_protocol_version, - ) = get_line_protocol_version(http_state, settings_url)?; + (supported_protocol_versions, default_protocol_version) = + get_protocol_version(http_state, settings_url)?; } else { - default_line_protocol_version = LineProtocolVersion::V1; + default_protocol_version = ProtocolVersion::V1; } } } @@ -2678,9 +2652,9 @@ impl SenderBuilder { handler, connected: true, max_buf_size: *self.max_buf_size, - default_line_protocol_version, + default_protocol_version, #[cfg(feature = "ilp-over-http")] - supported_line_protocol_versions, + supported_protocol_versions, }; Ok(sender) @@ -2799,17 +2773,6 @@ fn parse_key_pair(auth: &EcdsaAuthParams) -> Result { }) } -fn f64_text_series(vec: &mut Vec, value: f64) { - let mut ser = F64Serializer::new(value); - vec.extend_from_slice(ser.as_str().as_bytes()) -} - -fn f64_binary_series(vec: &mut Vec, value: f64) { - vec.push(b'='); - vec.push(DOUBLE_BINARY_FORMAT_TYPE); - vec.extend_from_slice(&value.to_le_bytes()) -} - pub(crate) struct F64Serializer { buf: ryu::Buffer, n: f64, @@ -2887,6 +2850,28 @@ impl Sender { SenderBuilder::from_env()?.build() } + /// Creates a new [`Buffer`] with default parameters. + /// + /// This initializes a buffer using the sender's protocol version and + /// the QuestDB server's default maximum name length of 127 characters. + /// For custom name lengths, use [`new_buffer_with_max_name_len`](Self::new_buffer_with_max_name_len) + /// + /// The default `max_name_len` matches the QuestDB server's `cairo.max.file.name.length` setting. + pub fn new_buffer(&self) -> Buffer { + Buffer::new(self.default_protocol_version) + } + + /// Creates a new [`Buffer`] with a custom maximum name length. + /// + /// This initializes a buffer using the sender's protocol version and + /// a specified maximum length for table/column names. The value should match + /// your QuestDB server's `cairo.max.file.name.length` configuration. + /// + /// For the default name length (127), use [`new_buffer`](Self::new_buffer) + pub fn new_buffer_with_max_name_len(&self, max_name_len: usize) -> Buffer { + Buffer::with_max_name_len(max_name_len, self.default_protocol_version) + } + #[allow(unused_variables)] fn flush_impl(&mut self, buf: &Buffer, transactional: bool) -> Result<()> { if !self.connected { @@ -2907,7 +2892,7 @@ impl Sender { } #[cfg(feature = "ilp-over-http")] - self.check_line_protocol_version(buf.version)?; + self.check_protocol_version(buf.version)?; let bytes = buf.as_bytes(); if bytes.is_empty() { @@ -3031,52 +3016,51 @@ impl Sender { } /// Returns the QuestDB server's recommended default line protocol version. - /// Will be used to [`Buffer::with_line_proto_version`] /// /// The version selection follows these rules: - /// 1. **TCP/TCPS Protocol**: Always returns [`LineProtocolVersion::V2`] + /// 1. **TCP/TCPS Protocol**: Always returns [`ProtocolVersion::V2`] /// 2. **HTTP/HTTPS Protocol**: - /// - If line protocol auto-detection is disabled [`SenderBuilder::disable_line_protocol_validation`], returns [`LineProtocolVersion::V2`] + /// - If line protocol auto-detection is disabled [`SenderBuilder::disable_protocol_validation`], returns [`ProtocolVersion::V2`] /// - If line protocol auto-detection is enabled: /// - Uses the server's default version if supported by the client /// - Otherwise uses the highest mutually supported version from the intersection /// of client and server compatible versions - pub fn default_line_protocol_version(&self) -> LineProtocolVersion { - self.default_line_protocol_version + pub fn default_protocol_version(&self) -> ProtocolVersion { + self.default_protocol_version } #[cfg(feature = "ilp-over-http")] #[cfg(test)] - pub(crate) fn support_line_protocol_versions(&self) -> Option> { - self.supported_line_protocol_versions.clone() + pub(crate) fn support_protocol_versions(&self) -> Option> { + self.supported_protocol_versions.clone() } #[cfg(feature = "ilp-over-http")] #[inline(always)] - fn check_line_protocol_version(&self, version: LineProtocolVersion) -> Result<()> { + fn check_protocol_version(&self, version: ProtocolVersion) -> Result<()> { match &self.handler { ProtocolHandler::Socket(_) => Ok(()), #[cfg(feature = "ilp-over-http")] ProtocolHandler::Http(http) => { - if *http.config.disable_line_proto_validation.deref() { + if *http.config.disable_protocol_validation.deref() { Ok(()) } else { - match self.supported_line_protocol_versions { + match self.supported_protocol_versions { Some(ref supported_line_protocols) => { if supported_line_protocols.contains(&version) { Ok(()) } else { Err(error::fmt!( - LineProtocolVersionError, + ProtocolVersionError, "Line protocol version {} is not supported by current QuestDB Server", version)) } } None => { - if version == LineProtocolVersion::V1 { + if version == ProtocolVersion::V1 { Ok(()) } else { Err(error::fmt!( - LineProtocolVersionError, + ProtocolVersionError, "Line protocol version {} is not supported by current QuestDB Server", version)) } } diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 257b18ae..6cb6e404 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -94,53 +94,25 @@ where Ok(()) } -#[cfg(feature = "benchmark")] -pub(crate) fn write_array_data_use_raw_buffer, T>(buf: &mut [u8], array: &A) -where - T: ArrayElement, -{ - // First optimization path: write contiguous memory directly - if let Some(contiguous) = array.as_slice() { - let byte_len = size_of_val(contiguous); - unsafe { - std::ptr::copy_nonoverlapping( - contiguous.as_ptr() as *const u8, - buf.as_mut_ptr(), - byte_len, - ) - } - } - - // Fallback path: non-contiguous memory handling - let elem_size = size_of::(); - for (i, &element) in array.iter().enumerate() { - unsafe { - std::ptr::copy_nonoverlapping( - &element as *const T as *const u8, - buf.as_mut_ptr().add(i * elem_size), - elem_size, - ) - } - } -} - pub(crate) fn get_and_check_array_bytes_size, T>( array: &A, ) -> Result where T: ArrayElement, { - (0..array.ndim()) - .try_fold(std::mem::size_of::(), |acc, i| Ok(acc * array.dim(i)?)) - .and_then(|p| match p <= MAX_ARRAY_BUFFER_SIZE { - true => Ok(p), - false => Err(error::fmt!( + let mut size = std::mem::size_of::(); + for dim_index in 0..array.ndim() { + size *= array.dim(dim_index)?; + if size > MAX_ARRAY_BUFFER_SIZE { + return Err(error::fmt!( ArrayViewError, "Array buffer size too big: {}, maximum: {}", - p, + size, MAX_ARRAY_BUFFER_SIZE - )), - }) + )); + } + } + Ok(size) } /// Marker trait for valid array element types. diff --git a/questdb-rs/src/ingress/tests.rs b/questdb-rs/src/ingress/tests.rs index 01a06f51..ee4defab 100644 --- a/questdb-rs/src/ingress/tests.rs +++ b/questdb-rs/src/ingress/tests.rs @@ -447,14 +447,14 @@ fn connect_timeout_uses_request_timeout() { let builder = SenderBuilder::new(Protocol::Http, "127.0.0.2", "1111") .request_timeout(request_timeout) .unwrap() - .disable_line_protocol_validation() + .disable_protocol_validation() .unwrap() .retry_timeout(Duration::from_millis(10)) .unwrap() .request_min_throughput(0) .unwrap(); let mut sender = builder.build().unwrap(); - let mut buf = Buffer::new(); + let mut buf = sender.new_buffer(); buf.table("x") .unwrap() .symbol("x", "x") diff --git a/questdb-rs/src/lib.rs b/questdb-rs/src/lib.rs index dc18bd8c..d46a5333 100644 --- a/questdb-rs/src/lib.rs +++ b/questdb-rs/src/lib.rs @@ -21,7 +21,6 @@ * limitations under the License. * ******************************************************************************/ - #![doc = include_str!("../README.md")] mod error; diff --git a/questdb-rs/src/tests/http.rs b/questdb-rs/src/tests/http.rs index 170182ab..d6698987 100644 --- a/questdb-rs/src/tests/http.rs +++ b/questdb-rs/src/tests/http.rs @@ -22,7 +22,7 @@ * ******************************************************************************/ -use crate::ingress::{Buffer, LineProtocolVersion, Protocol, SenderBuilder, TimestampNanos}; +use crate::ingress::{Buffer, Protocol, ProtocolVersion, SenderBuilder, TimestampNanos}; use crate::tests::mock::{certs_dir, HttpResponse, MockServer}; use crate::tests::TestResult; use crate::ErrorCode; @@ -33,9 +33,9 @@ use std::time::Duration; #[rstest] fn test_two_lines( - #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { - let mut buffer = Buffer::new().with_line_proto_version(version)?; + let mut buffer = Buffer::new(version); buffer .table("test")? .symbol("sym", "bol")? @@ -90,9 +90,9 @@ fn test_two_lines( #[rstest] fn test_text_plain_error( - #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { - let mut buffer = Buffer::new().with_line_proto_version(version)?; + let mut buffer = Buffer::new(version); buffer .table("test")? .symbol("sym", "bol")? @@ -147,9 +147,9 @@ fn test_text_plain_error( #[rstest] fn test_bad_json_error( - #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { - let mut buffer = Buffer::new().with_line_proto_version(version)?; + let mut buffer = Buffer::new(version); buffer .table("test")? .symbol("sym", "bol")? @@ -206,9 +206,9 @@ fn test_bad_json_error( #[rstest] fn test_json_error( - #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { - let mut buffer = Buffer::new().with_line_proto_version(version)?; + let mut buffer = Buffer::new(version); buffer .table("test")? .symbol("sym", "bol")? @@ -267,9 +267,9 @@ fn test_json_error( #[rstest] fn test_no_connection( - #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { - let mut buffer = Buffer::new().with_line_proto_version(version)?; + let mut buffer = Buffer::new(version); buffer .table("test")? .symbol("sym", "bol")? @@ -277,7 +277,7 @@ fn test_no_connection( .at_now()?; let mut sender = SenderBuilder::new(Protocol::Http, "127.0.0.1", 1) - .disable_line_protocol_validation()? + .disable_protocol_validation()? .build()?; let res = sender.flush_and_keep(&buffer); assert!(res.is_err()); @@ -291,9 +291,9 @@ fn test_no_connection( #[rstest] fn test_old_server_without_ilp_http_support( - #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { - let mut buffer = Buffer::new().with_line_proto_version(version)?; + let mut buffer = Buffer::new(version); buffer .table("test")? .symbol("sym", "bol")? @@ -347,9 +347,9 @@ fn test_old_server_without_ilp_http_support( #[rstest] fn test_http_basic_auth( - #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { - let mut buffer = Buffer::new().with_line_proto_version(version)?; + let mut buffer = Buffer::new(version); buffer .table("test")? .symbol("sym", "bol")? @@ -402,9 +402,9 @@ fn test_http_basic_auth( #[rstest] fn test_unauthenticated( - #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { - let mut buffer = Buffer::new().with_line_proto_version(version)?; + let mut buffer = Buffer::new(version); buffer .table("test")? .symbol("sym", "bol")? @@ -460,9 +460,9 @@ fn test_unauthenticated( #[rstest] fn test_token_auth( - #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { - let mut buffer = Buffer::new().with_line_proto_version(version)?; + let mut buffer = Buffer::new(version); buffer .table("test")? .symbol("sym", "bol")? @@ -506,9 +506,9 @@ fn test_token_auth( #[rstest] fn test_request_timeout( - #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { - let mut buffer = Buffer::new().with_line_proto_version(version)?; + let mut buffer = Buffer::new(version); buffer .table("test")? .symbol("sym", "bol")? @@ -522,7 +522,7 @@ fn test_request_timeout( let time_start = std::time::Instant::now(); let mut sender = server .lsb_http() - .disable_line_protocol_validation()? + .disable_protocol_validation()? .request_timeout(request_timeout)? .build()?; let res = sender.flush_and_keep(&buffer); @@ -537,12 +537,12 @@ fn test_request_timeout( #[rstest] fn test_tls( - #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { let mut ca_path = certs_dir(); ca_path.push("server_rootCA.pem"); - let mut buffer = Buffer::new().with_line_proto_version(version)?; + let mut buffer = Buffer::new(version); buffer .table("test")? .symbol("t1", "v1")? @@ -554,7 +554,7 @@ fn test_tls( let mut sender = server .lsb_https() .tls_roots(ca_path)? - .disable_line_protocol_validation()? + .disable_protocol_validation()? .build()?; let server_thread = std::thread::spawn(move || -> io::Result<()> { @@ -581,9 +581,9 @@ fn test_tls( #[rstest] fn test_user_agent( - #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { - let mut buffer = Buffer::new().with_line_proto_version(version)?; + let mut buffer = Buffer::new(version); buffer .table("test")? .symbol("t1", "v1")? @@ -622,11 +622,11 @@ fn test_user_agent( #[rstest] fn test_two_retries( - #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { // Note: This also tests that the _same_ connection is being reused, i.e. tests keepalive. - let mut buffer = Buffer::new().with_line_proto_version(version)?; + let mut buffer = Buffer::new(version); buffer .table("test")? .symbol("t1", "v1")? @@ -694,9 +694,9 @@ fn test_two_retries( #[rstest] fn test_one_retry( - #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { - let mut buffer = Buffer::new().with_line_proto_version(version)?; + let mut buffer = Buffer::new(version); buffer .table("test")? .symbol("t1", "v1")? @@ -708,7 +708,7 @@ fn test_one_retry( let mut sender = server .lsb_http() .retry_timeout(Duration::from_millis(19))? - .disable_line_protocol_validation() + .disable_protocol_validation() .unwrap() .build()?; @@ -761,10 +761,10 @@ fn test_one_retry( #[rstest] fn test_transactional( - #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { // A buffer with a two tables. - let mut buffer1 = Buffer::new().with_line_proto_version(version)?; + let mut buffer1 = Buffer::new(version); buffer1 .table("tab1")? .symbol("t1", "v1")? @@ -778,7 +778,7 @@ fn test_transactional( assert!(!buffer1.transactional()); // A buffer with a single table. - let mut buffer2 = Buffer::new().with_line_proto_version(version)?; + let mut buffer2 = Buffer::new(version); buffer2 .table("test")? .symbol("t1", "v1")? @@ -832,7 +832,7 @@ fn test_transactional( } #[test] -fn test_sender_line_protocol_version() -> TestResult { +fn test_sender_protocol_version() -> TestResult { let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); let sender_builder = server.lsb_http(); let server_thread = std::thread::spawn(move || -> io::Result<()> { @@ -848,20 +848,17 @@ fn test_sender_line_protocol_version() -> TestResult { Ok(()) }); let sender = sender_builder.build()?; + assert_eq!(sender.default_protocol_version(), ProtocolVersion::V2); assert_eq!( - sender.default_line_protocol_version(), - LineProtocolVersion::V2 - ); - assert_eq!( - sender.support_line_protocol_versions().unwrap(), - vec![LineProtocolVersion::V1, LineProtocolVersion::V2] + sender.support_protocol_versions().unwrap(), + vec![ProtocolVersion::V1, ProtocolVersion::V2] ); server_thread.join().unwrap()?; Ok(()) } #[test] -fn test_sender_line_protocol_version_old_server1() -> TestResult { +fn test_sender_protocol_version_old_server1() -> TestResult { let mut server = MockServer::new()?.configure_settings_response(0, &[1, 2]); let sender_builder = server.lsb_http(); let server_thread = std::thread::spawn(move || -> io::Result<()> { @@ -877,17 +874,14 @@ fn test_sender_line_protocol_version_old_server1() -> TestResult { Ok(()) }); let sender = sender_builder.build()?; - assert_eq!( - sender.default_line_protocol_version(), - LineProtocolVersion::V1 - ); - assert!(sender.support_line_protocol_versions().is_none()); + assert_eq!(sender.default_protocol_version(), ProtocolVersion::V1); + assert!(sender.support_protocol_versions().is_none()); server_thread.join().unwrap()?; Ok(()) } #[test] -fn test_sender_line_protocol_version_old_server2() -> TestResult { +fn test_sender_protocol_version_old_server2() -> TestResult { let mut server = MockServer::new()?.configure_settings_response(0, &[1, 2]); let sender_builder = server.lsb_http(); let server_thread = std::thread::spawn(move || -> io::Result<()> { @@ -901,17 +895,14 @@ fn test_sender_line_protocol_version_old_server2() -> TestResult { Ok(()) }); let sender = sender_builder.build()?; - assert_eq!( - sender.default_line_protocol_version(), - LineProtocolVersion::V1 - ); - assert!(sender.support_line_protocol_versions().is_none()); + assert_eq!(sender.default_protocol_version(), ProtocolVersion::V1); + assert!(sender.support_protocol_versions().is_none()); server_thread.join().unwrap()?; Ok(()) } #[test] -fn test_sender_line_protocol_version_unsupported_client() -> TestResult { +fn test_sender_protocol_version_unsupported_client() -> TestResult { let mut server = MockServer::new()?.configure_settings_response(3, &[3, 4]); let sender_builder = server.lsb_http(); let server_thread = std::thread::spawn(move || -> io::Result<()> { @@ -922,21 +913,17 @@ fn test_sender_line_protocol_version_unsupported_client() -> TestResult { let res1 = sender_builder.build(); assert!(res1.is_err()); let e1 = res1.err().unwrap(); - assert_eq!(e1.code(), ErrorCode::LineProtocolVersionError); + assert_eq!(e1.code(), ErrorCode::ProtocolVersionError); assert!(e1.msg().contains("Server does not support current client")); server_thread.join().unwrap()?; Ok(()) } #[test] -fn test_sender_disable_line_protocol_version_validation() -> TestResult { +fn test_sender_disable_protocol_version_validation() -> TestResult { let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); - let mut sender = server - .lsb_http() - .disable_line_protocol_validation()? - .build()?; - let mut buffer = - Buffer::new().with_line_proto_version(sender.default_line_protocol_version())?; + let mut sender = server.lsb_http().disable_protocol_validation()?.build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("sym", "bol")? @@ -958,15 +945,15 @@ fn test_sender_disable_line_protocol_version_validation() -> TestResult { } #[test] -fn test_sender_line_protocol_version1_not_support_array() -> TestResult { - let mut buffer = Buffer::new().with_line_proto_version(LineProtocolVersion::V1)?; +fn test_sender_protocol_version1_not_support_array() -> TestResult { + let mut buffer = Buffer::new(ProtocolVersion::V1); let res = buffer .table("test")? .symbol("sym", "bol")? .column_arr("x", &[1.0f64, 2.0]); assert!(res.is_err()); let e1 = res.as_ref().err().unwrap(); - assert_eq!(e1.code(), ErrorCode::LineProtocolVersionError); + assert_eq!(e1.code(), ErrorCode::ProtocolVersionError); assert!(e1 .msg() .contains("line protocol version v1 does not support array datatype")); diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index 79b24e66..a7b49001 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -1,6 +1,8 @@ #[cfg(feature = "ndarray")] use crate::ingress::MAX_ARRAY_DIMS; -use crate::ingress::{Buffer, NdArrayView, StrideArrayView, ARRAY_BINARY_FORMAT_TYPE}; +use crate::ingress::{ + Buffer, NdArrayView, ProtocolVersion, StrideArrayView, ARRAY_BINARY_FORMAT_TYPE, +}; use crate::tests::TestResult; use crate::{Error, ErrorCode}; @@ -190,7 +192,7 @@ fn test_buffer_basic_write() -> TestResult { test_data.len() * elem_size as usize, ) }?; - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("my_test")?; buffer.column_arr("temperature", &array_view)?; let data = buffer.as_bytes(); @@ -288,7 +290,7 @@ fn test_build_in_1d_array_normal() -> TestResult { let collected: Vec<_> = NdArrayView::iter(&arr).copied().collect(); assert_eq!(collected, vec![1.0, 2.0, 3.0, 4.0]); - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("my_test")?; buffer.column_arr("temperature", &arr)?; let data = buffer.as_bytes(); @@ -325,7 +327,7 @@ fn test_build_in_1d_array_empty() -> TestResult { assert_eq!(arr.dim(0), Ok(0)); assert_eq!(NdArrayView::as_slice(&arr), Some(&[][..])); - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("my_test")?; buffer.column_arr("temperature", &arr)?; let data = buffer.as_bytes(); @@ -354,7 +356,7 @@ fn test_build_in_1d_vec_normal() -> TestResult { let collected: Vec<_> = NdArrayView::iter(&vec).copied().collect(); assert_eq!(collected, vec![5.0, 6.0, 7.0]); - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("my_test")?; buffer.column_arr("temperature", &vec)?; let data = buffer.as_bytes(); @@ -390,7 +392,7 @@ fn test_build_in_1d_vec_empty() -> TestResult { assert_eq!(vec.dim(0), Ok(0)); assert_eq!(NdArrayView::as_slice(&vec), Some(&[][..])); - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("my_test")?; buffer.column_arr("temperature", &vec)?; let data = buffer.as_bytes(); @@ -418,7 +420,7 @@ fn test_build_in_1d_slice_normal() -> TestResult { assert_eq!(slice.dim(0), Ok(2)); assert_eq!(NdArrayView::as_slice(&slice), Some(&[20.0, 30.0][..])); - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("my_test")?; buffer.column_arr("temperature", &slice)?; let data = buffer.as_bytes(); @@ -450,7 +452,7 @@ fn test_build_in_1d_slice_empty() -> TestResult { assert_eq!(slice.dim(0), Ok(0)); assert_eq!(NdArrayView::as_slice(&slice), Some(&[][..])); - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("my_test")?; buffer.column_arr("temperature", &slice)?; let data = buffer.as_bytes(); @@ -483,7 +485,7 @@ fn test_build_in_2d_array_normal() -> TestResult { let collected: Vec<_> = NdArrayView::iter(&arr).copied().collect(); assert_eq!(collected, vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]); - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("my_test")?; buffer.column_arr("2darray", &arr)?; let data = buffer.as_bytes(); @@ -526,7 +528,7 @@ fn test_build_in_2d_array_empty() -> TestResult { assert_eq!(arr.dim(1), Ok(0)); assert_eq!(NdArrayView::as_slice(&arr), Some(&[][..])); - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("my_test")?; buffer.column_arr("2darray", &arr)?; let data = buffer.as_bytes(); @@ -559,7 +561,7 @@ fn test_build_in_2d_vec_normal() -> TestResult { let collected: Vec<_> = NdArrayView::iter(&vec).copied().collect(); assert_eq!(collected, vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]); - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("my_test")?; buffer.column_arr("2darray", &vec)?; let data = buffer.as_bytes(); @@ -597,7 +599,7 @@ fn test_build_in_2d_vec_normal() -> TestResult { #[test] fn test_build_in_2d_vec_irregular_shape() -> TestResult { let irregular_vec = vec![vec![1.0, 2.0], vec![3.0], vec![4.0, 5.0]]; - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("my_test")?; let result = buffer.column_arr("arr", &irregular_vec); let err = result.unwrap_err(); @@ -613,7 +615,7 @@ fn test_build_in_2d_vec_empty() -> TestResult { assert_eq!(vec.dim(0), Ok(3)); assert_eq!(vec.dim(1), Ok(0)); - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("my_test")?; buffer.column_arr("2darray", &vec)?; let data = buffer.as_bytes(); @@ -648,7 +650,7 @@ fn test_build_in_2d_slice_normal() -> TestResult { Some(&[1.0, 2.0, 3.0, 4.0][..]) ); - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("my_test")?; buffer.column_arr("2darray", &slice)?; let data = buffer.as_bytes(); @@ -690,7 +692,7 @@ fn test_build_in_2d_slice_empty() -> TestResult { assert_eq!(slice.dim(1), Ok(2)); assert_eq!(NdArrayView::as_slice(&slice), Some(&[][..])); - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("my_test")?; buffer.column_arr("2darray", &slice)?; let data = buffer.as_bytes(); @@ -727,7 +729,7 @@ fn test_build_in_3d_array_normal() -> TestResult { let collected: Vec<_> = NdArrayView::iter(&arr).copied().collect(); assert_eq!(collected, vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]); - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("my_test")?; buffer.column_arr("3darray", &arr)?; let data = buffer.as_bytes(); @@ -773,7 +775,7 @@ fn test_build_in_3d_array_empty() -> TestResult { assert_eq!(arr.dim(2), Ok(2)); assert_eq!(NdArrayView::as_slice(&arr), Some(&[][..])); - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("my_test")?; buffer.column_arr("3darray", &arr)?; let data = buffer.as_bytes(); @@ -813,7 +815,7 @@ fn test_build_in_3d_vec_normal() -> TestResult { vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0] ); - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("my_test")?; buffer.column_arr("3darray", &vec)?; let data = buffer.as_bytes(); @@ -863,7 +865,7 @@ fn test_build_in_3d_vec_empty() -> TestResult { assert_eq!(vec.dim(2), Ok(0)); assert!(NdArrayView::as_slice(&vec).is_none()); - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("my_test")?; buffer.column_arr("3darray", &vec)?; let data = buffer.as_bytes(); @@ -895,7 +897,7 @@ fn test_build_in_3d_vec_irregular_shape() -> TestResult { vec![vec![6.0, 7.0], vec![8.0, 9.0]], ]; - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("my_test")?; let result = buffer.column_arr("arr", &irregular1); let err = result.unwrap_err(); @@ -922,7 +924,7 @@ fn test_3d_slice_normal() -> TestResult { Some(&[1.0, 2.0, 3.0, 4.0][..]) ); - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("my_test")?; buffer.column_arr("3darray", &slice)?; let data = buffer.as_bytes(); @@ -965,7 +967,7 @@ fn test_3d_slice_empty() -> TestResult { assert_eq!(slice.dim(2), Ok(2)); assert_eq!(NdArrayView::as_slice(&slice), Some(&[][..])); - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("my_test")?; buffer.column_arr("3darray", &slice)?; let data = buffer.as_bytes(); @@ -1069,7 +1071,7 @@ fn test_complex_ndarray_dimensions() { #[cfg(feature = "ndarray")] #[test] fn test_buffer_ndarray_write() -> TestResult { - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("my_test")?; let array_2d = arr2(&[[1.1, 2.2], [3.3, 4.4]]); buffer.column_arr("temperature", &array_2d.view())?; @@ -1097,7 +1099,7 @@ fn test_buffer_ndarray_write() -> TestResult { #[cfg(feature = "ndarray")] #[test] fn test_buffer_write_ndarray_max_dimensions() -> TestResult { - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("nd_test")?; let shape: Vec = iter::repeat_n(1, MAX_ARRAY_DIMS).collect(); let array = ArrayD::::zeros(shape.clone()); diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index 2945d634..b40cce8e 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -32,7 +32,7 @@ use crate::{ use crate::ingress; #[cfg(feature = "ndarray")] use crate::ingress::ndarr::write_array_data; -use crate::ingress::LineProtocolVersion; +use crate::ingress::ProtocolVersion; use crate::tests::{ mock::{certs_dir, MockServer}, ndarr::ArrayColumnTypeTag, @@ -40,18 +40,18 @@ use crate::tests::{ }; use core::time::Duration; #[cfg(feature = "ndarray")] -use ndarray::{arr1, arr2, ArrayD}; +use ndarray::{arr2, ArrayD}; use rstest::rstest; use std::io; #[rstest] fn test_basics( - #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { use std::time::SystemTime; let mut server = MockServer::new()?; - let mut sender = server.lsb_tcp().build()?; + let mut sender = server.lsb_tcp().protocol_version(version)?.build()?; assert!(!sender.must_close()); server.accept()?; @@ -65,7 +65,7 @@ fn test_basics( let ts_nanos = TimestampNanos::from_systemtime(ts)?; assert_eq!(ts_nanos.as_i64(), ts_nanos_num); - let mut buffer = Buffer::new().with_line_proto_version(version)?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("t1", "v1")? @@ -101,13 +101,14 @@ fn test_basics( #[test] fn test_array_f64_basic() -> TestResult { let mut server = MockServer::new()?; - let mut sender = server.lsb_tcp().build()?; + let mut sender = server + .lsb_tcp() + .protocol_version(ProtocolVersion::V2)? + .build()?; server.accept()?; let ts = TimestampNanos::now(); - - let mut buffer = - Buffer::new().with_line_proto_version(sender.default_line_protocol_version())?; + let mut buffer = sender.new_buffer(); buffer .table("my_table")? .symbol("device", "A001")? @@ -119,7 +120,7 @@ fn test_array_f64_basic() -> TestResult { let exp = &[ b"my_table,device=A001 ", - f64_to_bytes("f1", 25.5, LineProtocolVersion::V2).as_slice(), + f64_to_bytes("f1", 25.5, ProtocolVersion::V2).as_slice(), b",arr1d=", b"=", // binary field &[ingress::ARRAY_BINARY_FORMAT_TYPE], @@ -147,15 +148,17 @@ fn test_array_f64_basic() -> TestResult { #[test] fn test_array_f64_from_ndarray() -> TestResult { let mut server = MockServer::new()?; - let mut sender = server.lsb_tcp().build()?; + let mut sender = server + .lsb_tcp() + .protocol_version(ProtocolVersion::V2)? + .build()?; server.accept()?; let ts = TimestampNanos::now(); let array_2d = arr2(&[[1.1, 2.2], [3.3, 4.4]]); let array_3d = ArrayD::::ones(vec![2, 3, 4]); - let mut buffer = - Buffer::new().with_line_proto_version(sender.default_line_protocol_version())?; + let mut buffer = sender.new_buffer(); buffer .table("my_table")? .symbol("device", "A001")? @@ -197,7 +200,7 @@ fn test_array_f64_from_ndarray() -> TestResult { let exp = &[ "my_table,device=A001 ".as_bytes(), - f64_to_bytes("f1", 25.5, LineProtocolVersion::V2).as_slice(), + f64_to_bytes("f1", 25.5, ProtocolVersion::V2).as_slice(), ",arr2d=".as_bytes(), array_header2d, array_data2d.as_slice(), @@ -220,14 +223,18 @@ fn test_array_f64_from_ndarray() -> TestResult { #[rstest] fn test_max_buf_size( - #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { let max = 1024; let mut server = MockServer::new()?; - let mut sender = server.lsb_tcp().max_buf_size(max)?.build()?; + let mut sender = server + .lsb_tcp() + .protocol_version(version)? + .max_buf_size(max)? + .build()?; assert!(!sender.must_close()); server.accept()?; - let mut buffer = Buffer::new().with_line_proto_version(version)?; + let mut buffer = sender.new_buffer(); while buffer.len() < max { buffer @@ -240,13 +247,13 @@ fn test_max_buf_size( let err = sender.flush(&mut buffer).unwrap_err(); assert_eq!(err.code(), ErrorCode::InvalidApiCall); match version { - LineProtocolVersion::V1 => { + ProtocolVersion::V1 => { assert_eq!( err.msg(), "Could not flush buffer: Buffer size of 1026 exceeds maximum configured allowed size of 1024 bytes." ); } - LineProtocolVersion::V2 => { + ProtocolVersion::V2 => { assert_eq!( err.msg(), "Could not flush buffer: Buffer size of 1025 exceeds maximum configured allowed size of 1024 bytes." @@ -258,7 +265,7 @@ fn test_max_buf_size( #[test] fn test_table_name_too_long() -> TestResult { - let mut buffer = Buffer::with_max_name_len(4); + let mut buffer = Buffer::with_max_name_len(4, ProtocolVersion::V2); let name = "a name too long"; let err = buffer.table(name).unwrap_err(); assert_eq!(err.code(), ErrorCode::InvalidName); @@ -271,7 +278,7 @@ fn test_table_name_too_long() -> TestResult { #[test] fn test_row_count() -> TestResult { - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); assert_eq!(buffer.row_count(), 0); buffer.table("x")?.symbol("y", "z1")?.at_now()?; @@ -413,7 +420,7 @@ fn test_timestamp_overloads() -> TestResult { let tbl_name = TableName::new("tbl_name")?; - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer .table(tbl_name)? .column_ts("a", TimestampMicros::new(12345))? @@ -458,7 +465,7 @@ fn test_chrono_timestamp() -> TestResult { let ts: DateTime = Utc.with_ymd_and_hms(1970, 1, 1, 0, 0, 1).unwrap(); let ts = TimestampNanos::from_datetime(ts)?; - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table(tbl_name)?.column_ts("a", ts)?.at(ts)?; let exp = b"tbl_name a=1000000t 1000000000\n"; @@ -469,7 +476,7 @@ fn test_chrono_timestamp() -> TestResult { macro_rules! column_name_too_long_test_impl { ($column_fn:ident, $value:expr) => {{ - let mut buffer = Buffer::with_max_name_len(4); + let mut buffer = Buffer::with_max_name_len(4, ProtocolVersion::V2); let name = "a name too long"; let err = buffer.table("tbl")?.$column_fn(name, $value).unwrap_err(); assert_eq!(err.code(), ErrorCode::InvalidName); @@ -506,26 +513,28 @@ fn test_str_column_name_too_long() -> TestResult { column_name_too_long_test_impl!(column_str, "value") } -#[cfg(feature = "ndarray")] #[test] fn test_arr_column_name_too_long() -> TestResult { - column_name_too_long_test_impl!(column_arr, &arr1(&[1.0, 2.0, 3.0]).view()) + column_name_too_long_test_impl!(column_arr, &[1.0, 2.0, 3.0]) } #[rstest] fn test_tls_with_file_ca( - #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { let mut ca_path = certs_dir(); ca_path.push("server_rootCA.pem"); let server = MockServer::new()?; - let lsb = server.lsb_tcps().tls_roots(ca_path)?; + let lsb = server + .lsb_tcps() + .protocol_version(version)? + .tls_roots(ca_path)?; let server_jh = server.accept_tls(); let mut sender = lsb.build()?; let mut server: MockServer = server_jh.join().unwrap()?; - let mut buffer = Buffer::new().with_line_proto_version(version)?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("t1", "v1")? @@ -581,7 +590,7 @@ fn expect_eventual_disconnect(sender: &mut Sender) { let mut retry = || { for _ in 0..1000 { std::thread::sleep(Duration::from_millis(100)); - let mut buffer = Buffer::new(); + let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("test_table")?.symbol("s1", "v1")?.at_now()?; sender.flush(&mut buffer)?; } @@ -617,15 +626,18 @@ fn test_plain_to_tls_server() -> TestResult { #[cfg(feature = "insecure-skip-verify")] #[rstest] fn test_tls_insecure_skip_verify( - #[values(LineProtocolVersion::V1, LineProtocolVersion::V2)] version: LineProtocolVersion, + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { let server = MockServer::new()?; - let lsb = server.lsb_tcps().tls_verify(false)?; + let lsb = server + .lsb_tcps() + .protocol_version(version)? + .tls_verify(false)?; let server_jh = server.accept_tls(); let mut sender = lsb.build()?; let mut server: MockServer = server_jh.join().unwrap()?; - let mut buffer = Buffer::new().with_line_proto_version(version)?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("t1", "v1")? @@ -665,17 +677,17 @@ fn bad_uppercase_addr() { assert!(err.msg() == "Missing \"addr\" parameter in config string"); } -fn f64_to_bytes(name: &str, value: f64, version: LineProtocolVersion) -> Vec { +fn f64_to_bytes(name: &str, value: f64, version: ProtocolVersion) -> Vec { let mut buf = Vec::new(); buf.extend_from_slice(name.as_bytes()); buf.push(b'='); match version { - LineProtocolVersion::V1 => { + ProtocolVersion::V1 => { let mut ser = crate::ingress::F64Serializer::new(value); buf.extend_from_slice(ser.as_str().as_bytes()); } - LineProtocolVersion::V2 => { + ProtocolVersion::V2 => { buf.push(b'='); buf.push(crate::ingress::DOUBLE_BINARY_FORMAT_TYPE); buf.extend_from_slice(&value.to_le_bytes()); diff --git a/system_test/questdb_line_sender.py b/system_test/questdb_line_sender.py index 595049fd..654118dd 100644 --- a/system_test/questdb_line_sender.py +++ b/system_test/questdb_line_sender.py @@ -93,15 +93,15 @@ class CertificateAuthority(Enum): PEM_FILE = (c_line_sender_ca(3), 'pem_file') -c_line_protocol_version = ctypes.c_int +c_protocol_version = ctypes.c_int -class LineProtocolVersion(Enum): - V1 = (c_line_protocol_version(1), 'v1') - V2 = (c_line_protocol_version(2), 'v2') +class ProtocolVersion(Enum): + V1 = (c_protocol_version(1), 'v1') + V2 = (c_protocol_version(2), 'v2') @classmethod - def from_int(cls, value: c_line_protocol_version): + def from_int(cls, value: c_protocol_version): for member in cls: if member.value[0].value == value: return member @@ -216,10 +216,10 @@ def set_sig(fn, restype, *argtypes): c_line_sender_buffer_p, c_size_t) set_sig( - dll.line_sender_buffer_set_line_protocol_version, + dll.line_sender_buffer_set_protocol_version, c_bool, c_line_sender_buffer_p, - c_line_protocol_version, + c_protocol_version, c_line_sender_error_p_p) set_sig( dll.line_sender_buffer_free, @@ -373,7 +373,7 @@ def set_sig(fn, restype, *argtypes): c_line_sender_utf8, c_line_sender_error_p_p) set_sig( - dll.line_sender_opts_disable_line_protocol_validation, + dll.line_sender_opts_disable_protocol_validation, c_bool, c_line_sender_opts_p, c_line_sender_error_p_p) @@ -448,8 +448,8 @@ def set_sig(fn, restype, *argtypes): c_line_sender_p, c_line_sender_error_p_p) set_sig( - dll.line_sender_default_line_protocol_version, - c_line_protocol_version, + dll.line_sender_default_protocol_version, + c_protocol_version, c_line_sender_p) set_sig( dll.line_sender_must_close, @@ -580,7 +580,7 @@ def __getattr__(self, name: str): fn = getattr(_DLL, 'line_sender_opts_' + name) def wrapper(*args): - if name == 'disable_line_protocol_validation': + if name == 'disable_protocol_validation': return _error_wrapped_call(fn, self.impl) mapped_args = [ (_utf8(arg) if isinstance(arg, str) else arg) @@ -602,14 +602,14 @@ def __init__(self, micros: int): class Buffer: - def __init__(self, init_buf_size=65536, max_name_len=127, line_protocol_version=LineProtocolVersion.V2): + def __init__(self, init_buf_size=65536, max_name_len=127, protocol_version=ProtocolVersion.V2): self._impl = _DLL.line_sender_buffer_with_max_name_len( c_size_t(max_name_len)) _DLL.line_sender_buffer_reserve(self._impl, c_size_t(init_buf_size)) _error_wrapped_call( - _DLL.line_sender_buffer_set_line_protocol_version, + _DLL.line_sender_buffer_set_protocol_version, self._impl, - line_protocol_version.value[0]) + protocol_version.value[0]) def __len__(self): return _DLL.line_sender_buffer_size(self._impl) @@ -627,9 +627,9 @@ def peek(self) -> str: else: return '' - def set_line_protocol_version(self, version: LineProtocolVersion): + def set_protocol_version(self, version: ProtocolVersion): _error_wrapped_call( - _DLL.line_sender_buffer_set_line_protocol_version, + _DLL.line_sender_buffer_set_protocol_version, self._impl, version.value[0]) @@ -823,16 +823,16 @@ def connect(self): def __enter__(self): self.connect() self._buffer = Buffer( - line_protocol_version=LineProtocolVersion.from_int(self.line_sender_default_line_protocol_version())) + protocol_version=ProtocolVersion.from_int(self.line_sender_default_protocol_version())) return self def _check_connected(self): if not self._impl: raise SenderError('Not connected.') - def line_sender_default_line_protocol_version(self): + def line_sender_default_protocol_version(self): self._check_connected() - return _DLL.line_sender_default_line_protocol_version(self._impl) + return _DLL.line_sender_default_protocol_version(self._impl) def table(self, table: str): self._buffer.table(table) diff --git a/system_test/test.py b/system_test/test.py index 19dc3c0c..bf3bc795 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -95,11 +95,11 @@ def ns_to_qdb_date(at_ts_ns): class TestSender(unittest.TestCase): - def _mk_linesender(self, disable_line_protocol_validation=False): + def _mk_linesender(self, disable_protocol_validation=False): # N.B.: We never connect with TLS here. auth = AUTH if QDB_FIXTURE.auth else {} - if disable_line_protocol_validation: - auth["disable_line_protocol_validation"] = "on" + if disable_protocol_validation: + auth["disable_protocol_validation"] = "on" return qls.Sender( BUILD_MODE, qls.Protocol.HTTP if QDB_FIXTURE.http else qls.Protocol.TCP, @@ -598,7 +598,7 @@ def test_f64_arr_mix_dims(self): except qls.SenderError as e: self.assertIn('cast error from protocol type: DOUBLE[] to column type: DOUBLE[][]', str(e)) - def test_line_protocol_version_v1(self): + def test_protocol_version_v1(self): if QDB_FIXTURE.version <= (6, 1, 2): self.skipTest('Float issues support') numbers = [ @@ -610,7 +610,7 @@ def test_line_protocol_version_v1(self): table_name = uuid.uuid4().hex pending = None with self._mk_linesender() as sender: - sender.buffer.set_line_protocol_version(qls.LineProtocolVersion.V1) + sender.buffer.set_protocol_version(qls.ProtocolVersion.V1) for num in numbers: sender.table(table_name) sender.column('n', num) @@ -639,7 +639,7 @@ def massage(num): scrubbed_dataset = [row[:-1] for row in resp['dataset']] self.assertEqual(scrubbed_dataset, exp_dataset) - def test_line_protocol_version_v1_array_unsupported(self): + def test_protocol_version_v1_array_unsupported(self): if QDB_FIXTURE.version < (8, 3, 1): self.skipTest('array unsupported') @@ -653,7 +653,7 @@ def test_line_protocol_version_v1_array_unsupported(self): table_name = uuid.uuid4().hex try: with self._mk_linesender(True) as sender: - sender.buffer.set_line_protocol_version(qls.LineProtocolVersion.V1) + sender.buffer.set_protocol_version(qls.ProtocolVersion.V1) sender.table(table_name) sender.column_f64_arr('f64_arr1', array1) sender.at_now() From 58c6539a12a39142c885bb063b10e8cb5f25e7b3 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 15 May 2025 09:31:24 +0800 Subject: [PATCH 069/119] fix c api --- cpp_test/test_line_sender.cpp | 70 +++++++--- examples/line_sender_c_example.c | 2 +- examples/line_sender_c_example_array.c | 2 +- examples/line_sender_c_example_auth_tls.c | 2 +- examples/line_sender_c_example_from_conf.c | 2 +- examples/line_sender_c_example_from_env.c | 2 +- examples/line_sender_c_example_http.c | 7 +- examples/line_sender_c_example_tls_ca.c | 2 +- examples/line_sender_cpp_example.cpp | 2 +- examples/line_sender_cpp_example_array.cpp | 9 +- examples/line_sender_cpp_example_auth.cpp | 2 +- examples/line_sender_cpp_example_auth_tls.cpp | 2 +- .../line_sender_cpp_example_from_conf.cpp | 2 +- examples/line_sender_cpp_example_from_env.cpp | 2 +- examples/line_sender_cpp_example_http.cpp | 3 +- examples/line_sender_cpp_example_tls_ca.cpp | 2 +- include/questdb/ingress/line_sender.hpp | 128 ++++++++++++++---- 17 files changed, 172 insertions(+), 69 deletions(-) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 179a661d..9a45cfbf 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -152,7 +152,7 @@ TEST_CASE("line_sender c api basics") CHECK(::line_sender_utf8_init(&v1_utf8, 2, "v1", &err)); ::line_sender_column_name f1_name{0, nullptr}; CHECK(::line_sender_column_name_init(&f1_name, 2, "f1", &err)); - ::line_sender_buffer* buffer = line_sender_buffer_new(); + ::line_sender_buffer* buffer = line_sender_new_buffer(sender); CHECK(buffer != nullptr); CHECK(::line_sender_buffer_table(buffer, table_name, &err)); CHECK(::line_sender_buffer_symbol(buffer, t1_name, v1_utf8, &err)); @@ -243,7 +243,7 @@ TEST_CASE("line_sender c++ api basics") server.accept(); CHECK(server.recv() == 0); - questdb::ingress::line_sender_buffer buffer; + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table("test") .symbol("t1", "v1") .symbol("t2", "") @@ -271,7 +271,7 @@ TEST_CASE("test multiple lines") CHECK(server.recv() == 0); const auto table_name = "metric1"_tn; - questdb::ingress::line_sender_buffer buffer; + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table(table_name) .symbol("t1"_cn, "val1"_utf8) .symbol("t2"_cn, "val2"_utf8) @@ -308,7 +308,7 @@ TEST_CASE("State machine testing -- flush without data.") std::string_view{"localhost"}, std::to_string(server.port())}; - questdb::ingress::line_sender_buffer buffer; + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); CHECK(buffer.size() == 0); CHECK_THROWS_WITH_AS( sender.flush(buffer), @@ -327,7 +327,7 @@ TEST_CASE("One symbol only - flush before server accept") server.port()}; // Does not raise - this is unlike InfluxDB spec that disallows this. - questdb::ingress::line_sender_buffer buffer; + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table("test").symbol("t1", std::string{"v1"}).at_now(); CHECK(!sender.must_close()); CHECK(buffer.size() == 11); @@ -348,7 +348,7 @@ TEST_CASE("One column only - server.accept() after flush, before close") questdb::ingress::protocol::tcp, "localhost", server.port()}; // Does not raise - this is unlike InfluxDB spec that disallows this. - questdb::ingress::line_sender_buffer buffer; + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table("test").column("t1", "v1").at_now(); CHECK(!sender.must_close()); CHECK(buffer.size() == 13); @@ -367,7 +367,7 @@ TEST_CASE("Symbol after column") questdb::ingress::line_sender sender{ questdb::ingress::protocol::tcp, "localhost", server.port()}; - questdb::ingress::line_sender_buffer buffer; + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table("test").column("t1", "v1"); CHECK_THROWS_AS( @@ -485,15 +485,18 @@ TEST_CASE("Buffer move and copy ctor testing") { const size_t init_buf_size = 128; - questdb::ingress::line_sender_buffer buffer1{init_buf_size}; + questdb::ingress::line_sender_buffer buffer1{ + protocol_version_1, init_buf_size}; buffer1.table("buffer1"); CHECK(buffer1.peek() == "buffer1"); - questdb::ingress::line_sender_buffer buffer2{2 * init_buf_size}; + questdb::ingress::line_sender_buffer buffer2{ + protocol_version_1, 2 * init_buf_size}; buffer2.table("buffer2"); CHECK(buffer2.peek() == "buffer2"); - questdb::ingress::line_sender_buffer buffer3{3 * init_buf_size}; + questdb::ingress::line_sender_buffer buffer3{ + protocol_version_1, 3 * init_buf_size}; buffer3.table("buffer3"); CHECK(buffer3.peek() == "buffer3"); @@ -532,7 +535,7 @@ TEST_CASE("Sender move testing.") questdb::ingress::line_sender sender1{ questdb::ingress::protocol::tcp, host_ref, server1.port()}; - questdb::ingress::line_sender_buffer buffer; + questdb::ingress::line_sender_buffer buffer = sender1.new_buffer(); buffer.table("test").column("t1", "v1").at_now(); server1.close(); @@ -762,7 +765,7 @@ TEST_CASE("Test timestamp column.") const auto now_nanos_ts = questdb::ingress::timestamp_nanos{now_nanos}; const auto now_micros_ts = questdb::ingress::timestamp_micros{now_micros}; - questdb::ingress::line_sender_buffer buffer; + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table("test") .column("ts1", questdb::ingress::timestamp_micros{12345}) .column("ts2", now_micros_ts) @@ -809,7 +812,7 @@ TEST_CASE("test timestamp_micros and timestamp_nanos::now()") TEST_CASE("Test Marker") { - questdb::ingress::line_sender_buffer buffer; + questdb::ingress::line_sender_buffer buffer{protocol_version_1}; buffer.clear_marker(); buffer.clear_marker(); @@ -865,18 +868,18 @@ TEST_CASE("Moved View") TEST_CASE("Empty Buffer") { - questdb::ingress::line_sender_buffer b1; + questdb::ingress::line_sender_buffer b1{protocol_version_2}; CHECK(b1.size() == 0); questdb::ingress::line_sender_buffer b2{std::move(b1)}; CHECK(b1.size() == 0); CHECK(b2.size() == 0); - questdb::ingress::line_sender_buffer b3; + questdb::ingress::line_sender_buffer b3{protocol_version_2}; b3 = std::move(b2); CHECK(b2.size() == 0); CHECK(b3.size() == 0); - questdb::ingress::line_sender_buffer b4; + questdb::ingress::line_sender_buffer b4{protocol_version_2}; b4.table("test").symbol("a", "b").at_now(); - questdb::ingress::line_sender_buffer b5; + questdb::ingress::line_sender_buffer b5{protocol_version_2}; b5 = std::move(b4); CHECK(b4.size() == 0); CHECK(b5.size() == 9); @@ -912,7 +915,8 @@ TEST_CASE("HTTP basics") questdb::ingress::protocol::http, "localhost", 1, true}; questdb::ingress::opts opts1conf = questdb::ingress::opts::from_conf( "http::addr=localhost:1;username=user;password=pass;request_timeout=" - "5000;retry_timeout=5;disable_protocol_validation=on;"); + "5000;retry_timeout=5;disable_protocol_validation=on;protocol_version=" + "2;"); questdb::ingress::opts opts2{ questdb::ingress::protocol::https, "localhost", "1", true}; questdb::ingress::opts opts2conf = questdb::ingress::opts::from_conf( @@ -929,7 +933,7 @@ TEST_CASE("HTTP basics") questdb::ingress::line_sender sender2{opts2}; questdb::ingress::line_sender sender2conf{opts2conf}; - questdb::ingress::line_sender_buffer b1; + questdb::ingress::line_sender_buffer b1 = sender1.new_buffer(); b1.table("test").symbol("a", "b").at_now(); CHECK_THROWS_AS(sender1.flush(b1), questdb::ingress::line_sender_error); @@ -954,7 +958,33 @@ TEST_CASE("line sender protocol version v1") server.accept(); CHECK(server.recv() == 0); - questdb::ingress::line_sender_buffer buffer{protocol_version_1}; + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); + buffer.table("test") + .symbol("t1", "v1") + .symbol("t2", "") + .column("f1", 0.5) + .at(questdb::ingress::timestamp_nanos{10000000}); + + CHECK(server.recv() == 0); + CHECK(buffer.size() == 31); + sender.flush(buffer); + CHECK(server.recv() == 1); + std::string expect{"test,t1=v1,t2= f1=0.5 10000000\n"}; + CHECK(server.msgs(0) == expect); +} + +TEST_CASE("line sender protocol version v2") +{ + questdb::ingress::test::mock_server server; + questdb::ingress::line_sender sender{ + questdb::ingress::protocol::tcp, + std::string("localhost"), + std::to_string(server.port())}; + CHECK_FALSE(sender.must_close()); + server.accept(); + CHECK(server.recv() == 0); + + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table("test") .symbol("t1", "v1") .symbol("t2", "") diff --git a/examples/line_sender_c_example.c b/examples/line_sender_c_example.c index 0f6805cc..243062db 100644 --- a/examples/line_sender_c_example.c +++ b/examples/line_sender_c_example.c @@ -25,7 +25,7 @@ static bool example(const char* host, const char* port) free(conf_str); conf_str = NULL; - buffer = line_sender_buffer_new(); + buffer = line_sender_new_buffer(sender); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. // We prepare all our table names and column names in advance. diff --git a/examples/line_sender_c_example_array.c b/examples/line_sender_c_example_array.c index 6f5650c7..cf02e654 100644 --- a/examples/line_sender_c_example_array.c +++ b/examples/line_sender_c_example_array.c @@ -28,7 +28,7 @@ static bool example(const char* host, const char* port) free(conf_str); conf_str = NULL; - buffer = line_sender_buffer_new(); + buffer = line_sender_new_buffer(sender); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB 初始缓冲 line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("market_orders"); diff --git a/examples/line_sender_c_example_auth_tls.c b/examples/line_sender_c_example_auth_tls.c index e7454d8d..0960d1bc 100644 --- a/examples/line_sender_c_example_auth_tls.c +++ b/examples/line_sender_c_example_auth_tls.c @@ -30,7 +30,7 @@ static bool example(const char* host, const char* port) free(conf_str); conf_str = NULL; - buffer = line_sender_buffer_new(); + buffer = line_sender_new_buffer(sender); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. // We prepare all our table names and column names in advance. diff --git a/examples/line_sender_c_example_from_conf.c b/examples/line_sender_c_example_from_conf.c index 6f1d4e43..bcd8f1f1 100644 --- a/examples/line_sender_c_example_from_conf.c +++ b/examples/line_sender_c_example_from_conf.c @@ -14,7 +14,7 @@ int main(int argc, const char* argv[]) if (!sender) goto on_error; - buffer = line_sender_buffer_new(); + buffer = line_sender_new_buffer(sender); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. // We prepare all our table names and column names in advance. diff --git a/examples/line_sender_c_example_from_env.c b/examples/line_sender_c_example_from_env.c index e67dbe59..eed0912a 100644 --- a/examples/line_sender_c_example_from_env.c +++ b/examples/line_sender_c_example_from_env.c @@ -13,7 +13,7 @@ int main(int argc, const char* argv[]) if (!sender) goto on_error; - buffer = line_sender_buffer_new(); + buffer = line_sender_new_buffer(sender); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. // We prepare all our table names and column names in advance. diff --git a/examples/line_sender_c_example_http.c b/examples/line_sender_c_example_http.c index b0b448bb..c8d7fc52 100644 --- a/examples/line_sender_c_example_http.c +++ b/examples/line_sender_c_example_http.c @@ -27,11 +27,8 @@ static bool example(const char* host, const char* port) free(conf_str); conf_str = NULL; - buffer = line_sender_buffer_new(); - line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. - if (!line_sender_buffer_set_protocol_version( - buffer, line_sender_default_protocol_version(sender), &err)) - goto on_error; + buffer = line_sender_new_buffer(sender); + line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("c_trades_http"); line_sender_column_name symbol_name = QDB_COLUMN_NAME_LITERAL("symbol"); diff --git a/examples/line_sender_c_example_tls_ca.c b/examples/line_sender_c_example_tls_ca.c index 63151945..08d6bbf7 100644 --- a/examples/line_sender_c_example_tls_ca.c +++ b/examples/line_sender_c_example_tls_ca.c @@ -31,7 +31,7 @@ static bool example(const char* ca_path, const char* host, const char* port) free(conf_str); conf_str = NULL; - buffer = line_sender_buffer_new(); + buffer = line_sender_new_buffer(sender); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. // We prepare all our table names and column names in advance. diff --git a/examples/line_sender_cpp_example.cpp b/examples/line_sender_cpp_example.cpp index d65ecc31..173e7b7b 100644 --- a/examples/line_sender_cpp_example.cpp +++ b/examples/line_sender_cpp_example.cpp @@ -20,7 +20,7 @@ static bool example(std::string_view host, std::string_view port) const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; - questdb::ingress::line_sender_buffer buffer; + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) diff --git a/examples/line_sender_cpp_example_array.cpp b/examples/line_sender_cpp_example_array.cpp index e07cd21b..f6b1ad75 100644 --- a/examples/line_sender_cpp_example_array.cpp +++ b/examples/line_sender_cpp_example_array.cpp @@ -10,14 +10,15 @@ static bool array_example(std::string_view host, std::string_view port) try { auto sender = questdb::ingress::line_sender::from_conf( - "tcp::addr=" + std::string{host} + ":" + std::string{port} + ";"); + "tcp::addr=" + std::string{host} + ":" + std::string{port} + + ";protocol_version=2;"); const auto table_name = "cpp_market_orders"_tn; const auto symbol_col = "symbol"_cn; const auto book_col = "order_book"_cn; size_t rank = 3; - std::vector shape{2, 3, 2}; - std::vector strides{48, 16, 8}; + std::vector shape{2, 3, 2}; + std::vector strides{48, 16, 8}; std::array arr_data = { 48123.5, 2.4, @@ -32,7 +33,7 @@ static bool array_example(std::string_view host, std::string_view port) 48121.5, 4.3}; - questdb::ingress::line_sender_buffer buffer; + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table(table_name) .symbol(symbol_col, "BTC-USD"_utf8) .column(book_col, 3, shape, strides, arr_data) diff --git a/examples/line_sender_cpp_example_auth.cpp b/examples/line_sender_cpp_example_auth.cpp index 4c229617..2e780c4d 100644 --- a/examples/line_sender_cpp_example_auth.cpp +++ b/examples/line_sender_cpp_example_auth.cpp @@ -25,7 +25,7 @@ static bool example(std::string_view host, std::string_view port) const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; - questdb::ingress::line_sender_buffer buffer; + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) diff --git a/examples/line_sender_cpp_example_auth_tls.cpp b/examples/line_sender_cpp_example_auth_tls.cpp index f202fc75..766d08a9 100644 --- a/examples/line_sender_cpp_example_auth_tls.cpp +++ b/examples/line_sender_cpp_example_auth_tls.cpp @@ -25,7 +25,7 @@ static bool example(std::string_view host, std::string_view port) const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; - questdb::ingress::line_sender_buffer buffer; + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) diff --git a/examples/line_sender_cpp_example_from_conf.cpp b/examples/line_sender_cpp_example_from_conf.cpp index 2a2fe510..36e27455 100644 --- a/examples/line_sender_cpp_example_from_conf.cpp +++ b/examples/line_sender_cpp_example_from_conf.cpp @@ -20,7 +20,7 @@ int main(int argc, const char* argv[]) const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; - questdb::ingress::line_sender_buffer buffer; + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) diff --git a/examples/line_sender_cpp_example_from_env.cpp b/examples/line_sender_cpp_example_from_env.cpp index 54acd658..3bf1c02a 100644 --- a/examples/line_sender_cpp_example_from_env.cpp +++ b/examples/line_sender_cpp_example_from_env.cpp @@ -19,7 +19,7 @@ int main(int argc, const char* argv[]) const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; - questdb::ingress::line_sender_buffer buffer; + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) diff --git a/examples/line_sender_cpp_example_http.cpp b/examples/line_sender_cpp_example_http.cpp index f36901c1..1e675935 100644 --- a/examples/line_sender_cpp_example_http.cpp +++ b/examples/line_sender_cpp_example_http.cpp @@ -20,8 +20,7 @@ static bool example(std::string_view host, std::string_view port) const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; - questdb::ingress::line_sender_buffer buffer{ - sender.default_protocol_version()}; + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) diff --git a/examples/line_sender_cpp_example_tls_ca.cpp b/examples/line_sender_cpp_example_tls_ca.cpp index ac4d4743..734f0bc0 100644 --- a/examples/line_sender_cpp_example_tls_ca.cpp +++ b/examples/line_sender_cpp_example_tls_ca.cpp @@ -28,7 +28,7 @@ static bool example( const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; - questdb::ingress::line_sender_buffer buffer; + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index d036d125..4c55e2a2 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -389,47 +389,39 @@ class buffer_view final class line_sender_buffer { public: - explicit line_sender_buffer(size_t init_buf_size = 64 * 1024) noexcept - : line_sender_buffer{init_buf_size, 127} - { - } - - line_sender_buffer(size_t init_buf_size, size_t max_name_len) noexcept - : _impl{nullptr} - , _init_buf_size{init_buf_size} - , _max_name_len{max_name_len} + explicit line_sender_buffer( + protocol_version protocol_version, + size_t init_buf_size = 64 * 1024) noexcept + : line_sender_buffer{protocol_version, init_buf_size, 127} { } line_sender_buffer( + protocol_version version, size_t init_buf_size, - size_t max_name_len, - protocol_version version) noexcept + size_t max_name_len) noexcept : _impl{nullptr} + , _protocol_version(version) , _init_buf_size{init_buf_size} , _max_name_len{max_name_len} - , _protocol_version{version} - { - } - - line_sender_buffer(protocol_version version) noexcept - : line_sender_buffer{64 * 1024, 127, version} { } line_sender_buffer(const line_sender_buffer& other) noexcept : _impl{::line_sender_buffer_clone(other._impl)} + , _protocol_version{other._protocol_version} , _init_buf_size{other._init_buf_size} , _max_name_len{other._max_name_len} - , _protocol_version{other._protocol_version} + { } line_sender_buffer(line_sender_buffer&& other) noexcept : _impl{other._impl} + , _protocol_version{other._protocol_version} , _init_buf_size{other._init_buf_size} , _max_name_len{other._max_name_len} - , _protocol_version{other._protocol_version} + { other._impl = nullptr; } @@ -821,19 +813,16 @@ class line_sender_buffer { if (!_impl) { - _impl = ::line_sender_buffer_with_max_name_len(_max_name_len); + _impl = ::line_sender_buffer_with_max_name_len( + _max_name_len, _protocol_version); ::line_sender_buffer_reserve(_impl, _init_buf_size); - line_sender_error::wrapped_call( - line_sender_buffer_set_protocol_version, - _impl, - _protocol_version); } } ::line_sender_buffer* _impl; + protocol_version _protocol_version; size_t _init_buf_size; size_t _max_name_len; - protocol_version _protocol_version{::protocol_version_2}; friend class line_sender; }; @@ -913,6 +902,34 @@ class opts } } + /** + * Create a new `opts` instance with the given protocol, hostname and port. + * @param[in] protocol The protocol to use. + * @param[in] host The QuestDB database host. + * @param[in] port The QuestDB tcp or http port. + * @param[in] disable_protocol_validation disable line protocol version + * validation. + */ + opts( + protocol protocol, + utf8_view host, + uint16_t port, + protocol_version version, + bool disable_protocol_validation = false) noexcept + : _impl{::line_sender_opts_new( + static_cast<::line_sender_protocol>(protocol), host._impl, port)} + { + line_sender_error::wrapped_call( + ::line_sender_opts_user_agent, _impl, _user_agent::name()); + line_sender_error::wrapped_call( + ::line_sender_opts_protocol_version, _impl, version); + if (disable_protocol_validation) + { + line_sender_error::wrapped_call( + ::line_sender_opts_disable_protocol_validation, _impl); + } + } + /** * Create a new `opts` instance with the given protocol, hostname and * service name. @@ -940,6 +957,36 @@ class opts } } + /** + * Create a new `opts` instance with the given protocol, hostname and + * service name. + * @param[in] protocol The protocol to use. + * @param[in] host The QuestDB database host. + * @param[in] port The QuestDB tcp or http port as service name. + * @param[in] disable_protocol_validation disable line protocol version + */ + opts( + protocol protocol, + utf8_view host, + utf8_view port, + protocol_version version, + bool disable_protocol_validation = false) noexcept + : _impl{::line_sender_opts_new_service( + static_cast<::line_sender_protocol>(protocol), + host._impl, + port._impl)} + { + line_sender_error::wrapped_call( + ::line_sender_opts_user_agent, _impl, _user_agent::name()); + line_sender_error::wrapped_call( + ::line_sender_opts_protocol_version, _impl, version); + if (disable_protocol_validation) + { + line_sender_error::wrapped_call( + ::line_sender_opts_disable_protocol_validation, _impl); + } + } + opts(const opts& other) noexcept : _impl{::line_sender_opts_clone(other._impl)} { @@ -1258,6 +1305,28 @@ class line_sender { } + line_sender( + protocol protocol, + utf8_view host, + uint16_t port, + protocol_version version, + bool disable_protocol_validation = false) + : line_sender{ + opts{protocol, host, port, version, disable_protocol_validation}} + { + } + + line_sender( + protocol protocol, + utf8_view host, + utf8_view port, + protocol_version version, + bool disable_protocol_validation = false) + : line_sender{ + opts{protocol, host, port, version, disable_protocol_validation}} + { + } + line_sender(const opts& opts) : _impl{ line_sender_error::wrapped_call(::line_sender_build, opts._impl)} @@ -1285,6 +1354,13 @@ class line_sender return *this; } + line_sender_buffer new_buffer( + size_t init_buf_size = 64 * 1024, size_t max_name_len = 127) noexcept + { + return line_sender_buffer{ + default_protocol_version(), init_buf_size, max_name_len}; + } + /** * Send the given buffer of rows to the QuestDB server, clearing the buffer. * @@ -1337,7 +1413,7 @@ class line_sender } else { - line_sender_buffer buffer2{0}; + line_sender_buffer buffer2{default_protocol_version(), 0}; buffer2.may_init(); line_sender_error::wrapped_call( ::line_sender_flush_and_keep, _impl, buffer2._impl); From 8fb0c4a7876c47dcfd3a1a25e22e7b419506d143 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 15 May 2025 17:02:55 +0800 Subject: [PATCH 070/119] code reviews. --- cpp_test/test_line_sender.cpp | 36 +++-- examples/line_sender_c_example.c | 2 +- examples/line_sender_c_example_array.c | 2 +- examples/line_sender_c_example_auth.c | 3 +- examples/line_sender_c_example_auth_tls.c | 1 + examples/line_sender_c_example_from_conf.c | 2 +- examples/line_sender_c_example_tls_ca.c | 1 + examples/line_sender_cpp_example.cpp | 3 +- examples/line_sender_cpp_example_array.cpp | 4 +- examples/line_sender_cpp_example_auth.cpp | 2 +- examples/line_sender_cpp_example_auth_tls.cpp | 2 +- .../line_sender_cpp_example_from_conf.cpp | 2 +- examples/line_sender_cpp_example_tls_ca.cpp | 2 +- include/questdb/ingress/line_sender.h | 26 +--- include/questdb/ingress/line_sender.hpp | 82 ++---------- questdb-rs-ffi/src/lib.rs | 23 +--- questdb-rs/src/ingress/http.rs | 75 +++-------- questdb-rs/src/ingress/mod.rs | 125 ++++++------------ questdb-rs/src/ingress/tests.rs | 2 +- questdb-rs/src/tests/http.rs | 57 ++++---- questdb-rs/src/tests/mock.rs | 9 +- 21 files changed, 143 insertions(+), 318 deletions(-) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 9a45cfbf..39d3f16d 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -131,6 +131,7 @@ TEST_CASE("line_sender c api basics") ::line_sender_opts* opts = ::line_sender_opts_new(::line_sender_protocol_tcp, host, server.port()); CHECK_NE(opts, nullptr); + line_sender_opts_protocol_version(opts, protocol_version_2, &err); ::line_sender* sender = ::line_sender_build(opts, &err); line_sender_opts_free(opts); CHECK_NE(sender, nullptr); @@ -238,7 +239,8 @@ TEST_CASE("line_sender c++ api basics") questdb::ingress::line_sender sender{ questdb::ingress::protocol::tcp, std::string("localhost"), - std::to_string(server.port())}; + std::to_string(server.port()), + protocol_version_2}; CHECK_FALSE(sender.must_close()); server.accept(); CHECK(server.recv() == 0); @@ -263,7 +265,8 @@ TEST_CASE("test multiple lines") { questdb::ingress::test::mock_server server; std::string conf_str = - "tcp::addr=localhost:" + std::to_string(server.port()) + ";"; + "tcp::addr=localhost:" + std::to_string(server.port()) + + ";protocol_version=2;"; questdb::ingress::line_sender sender = questdb::ingress::line_sender::from_conf(conf_str); CHECK_FALSE(sender.must_close()); @@ -696,19 +699,13 @@ TEST_CASE("os certs") { questdb::ingress::opts opts{ - questdb::ingress::protocol::https, - "localhost", - server.port(), - true}; + questdb::ingress::protocol::https, "localhost", server.port()}; opts.tls_ca(questdb::ingress::ca::os_roots); } { questdb::ingress::opts opts{ - questdb::ingress::protocol::https, - "localhost", - server.port(), - true}; + questdb::ingress::protocol::https, "localhost", server.port()}; opts.tls_ca(questdb::ingress::ca::webpki_and_os_roots); } } @@ -737,12 +734,9 @@ TEST_CASE("Opts copy ctor, assignment and move testing.") { questdb::ingress::opts opts1{ - questdb::ingress::protocol::https, "localhost", "9009", true}; + questdb::ingress::protocol::https, "localhost", "9009"}; questdb::ingress::opts opts2{ - questdb::ingress::protocol::https, - "altavista.digital.com", - "9009", - true}; + questdb::ingress::protocol::https, "altavista.digital.com", "9009"}; opts1 = opts2; } } @@ -912,16 +906,18 @@ TEST_CASE("Opts from conf") TEST_CASE("HTTP basics") { questdb::ingress::opts opts1{ - questdb::ingress::protocol::http, "localhost", 1, true}; + questdb::ingress::protocol::http, "localhost", 1, protocol_version_2}; questdb::ingress::opts opts1conf = questdb::ingress::opts::from_conf( "http::addr=localhost:1;username=user;password=pass;request_timeout=" - "5000;retry_timeout=5;disable_protocol_validation=on;protocol_version=" - "2;"); + "5000;retry_timeout=5;protocol_version=2;"); questdb::ingress::opts opts2{ - questdb::ingress::protocol::https, "localhost", "1", true}; + questdb::ingress::protocol::https, + "localhost", + "1", + protocol_version_2}; questdb::ingress::opts opts2conf = questdb::ingress::opts::from_conf( "http::addr=localhost:1;token=token;request_min_throughput=1000;retry_" - "timeout=0;disable_protocol_validation=on;"); + "timeout=0;protocol_version=2;"); opts1.username("user") .password("pass") .max_buf_size(1000000) diff --git a/examples/line_sender_c_example.c b/examples/line_sender_c_example.c index 243062db..df1ee8b3 100644 --- a/examples/line_sender_c_example.c +++ b/examples/line_sender_c_example.c @@ -9,7 +9,7 @@ static bool example(const char* host, const char* port) line_sender_error* err = NULL; line_sender* sender = NULL; line_sender_buffer* buffer = NULL; - char* conf_str = concat("tcp::addr=", host, ":", port, ";"); + char* conf_str = concat("tcp::addr=", host, ":", port, ";protocol_version=2;"); if (!conf_str) { fprintf(stderr, "Could not concatenate configuration string.\n"); return false; diff --git a/examples/line_sender_c_example_array.c b/examples/line_sender_c_example_array.c index cf02e654..b25fffaa 100644 --- a/examples/line_sender_c_example_array.c +++ b/examples/line_sender_c_example_array.c @@ -9,7 +9,7 @@ static bool example(const char* host, const char* port) line_sender_error* err = NULL; line_sender* sender = NULL; line_sender_buffer* buffer = NULL; - char* conf_str = concat("tcp::addr=", host, ":", port, ";"); + char* conf_str = concat("tcp::addr=", host, ":", port, ";protocol_version=2;"); if (!conf_str) { fprintf(stderr, "Could not concatenate configuration string.\n"); diff --git a/examples/line_sender_c_example_auth.c b/examples/line_sender_c_example_auth.c index 59a99421..2f0df546 100644 --- a/examples/line_sender_c_example_auth.c +++ b/examples/line_sender_c_example_auth.c @@ -11,6 +11,7 @@ static bool example(const char* host, const char* port) line_sender_buffer* buffer = NULL; char* conf_str = concat( "tcp::addr=", host, ":", port, ";" + "protocol_version=2" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" @@ -30,7 +31,7 @@ static bool example(const char* host, const char* port) free(conf_str); conf_str = NULL; - buffer = line_sender_buffer_new(); + buffer = line_sender_new_buffer(sender); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. // We prepare all our table names and column names in advance. diff --git a/examples/line_sender_c_example_auth_tls.c b/examples/line_sender_c_example_auth_tls.c index 0960d1bc..79c044e5 100644 --- a/examples/line_sender_c_example_auth_tls.c +++ b/examples/line_sender_c_example_auth_tls.c @@ -11,6 +11,7 @@ static bool example(const char* host, const char* port) line_sender_buffer* buffer = NULL; char* conf_str = concat( "tcps::addr=", host, ":", port, ";" + "protocol_version=2;" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" diff --git a/examples/line_sender_c_example_from_conf.c b/examples/line_sender_c_example_from_conf.c index bcd8f1f1..487fce62 100644 --- a/examples/line_sender_c_example_from_conf.c +++ b/examples/line_sender_c_example_from_conf.c @@ -9,7 +9,7 @@ int main(int argc, const char* argv[]) line_sender_buffer* buffer = NULL; line_sender_utf8 conf = QDB_UTF8_LITERAL( - "tcp::addr=localhost:9009;"); + "tcp::addr=localhost:9009;protocol_version=2;"); line_sender* sender = line_sender_from_conf(conf, &err); if (!sender) goto on_error; diff --git a/examples/line_sender_c_example_tls_ca.c b/examples/line_sender_c_example_tls_ca.c index 08d6bbf7..72aaeb74 100644 --- a/examples/line_sender_c_example_tls_ca.c +++ b/examples/line_sender_c_example_tls_ca.c @@ -11,6 +11,7 @@ static bool example(const char* ca_path, const char* host, const char* port) line_sender_buffer* buffer = NULL; char* conf_str = concat( "tcps::addr=", host, ":", port, ";", + "protocol_version=2;" "tls_roots=", ca_path, ";", "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" diff --git a/examples/line_sender_cpp_example.cpp b/examples/line_sender_cpp_example.cpp index 173e7b7b..97943e74 100644 --- a/examples/line_sender_cpp_example.cpp +++ b/examples/line_sender_cpp_example.cpp @@ -9,7 +9,8 @@ static bool example(std::string_view host, std::string_view port) try { auto sender = questdb::ingress::line_sender::from_conf( - "tcp::addr=" + std::string{host} + ":" + std::string{port} + ";"); + "tcp::addr=" + std::string{host} + ":" + std::string{port} + + ";protocol_version=2;"); // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid diff --git a/examples/line_sender_cpp_example_array.cpp b/examples/line_sender_cpp_example_array.cpp index f6b1ad75..73b56c62 100644 --- a/examples/line_sender_cpp_example_array.cpp +++ b/examples/line_sender_cpp_example_array.cpp @@ -17,8 +17,8 @@ static bool array_example(std::string_view host, std::string_view port) const auto symbol_col = "symbol"_cn; const auto book_col = "order_book"_cn; size_t rank = 3; - std::vector shape{2, 3, 2}; - std::vector strides{48, 16, 8}; + std::vector shape{2, 3, 2}; + std::vector strides{48, 16, 8}; std::array arr_data = { 48123.5, 2.4, diff --git a/examples/line_sender_cpp_example_auth.cpp b/examples/line_sender_cpp_example_auth.cpp index 2e780c4d..9ec2f5e7 100644 --- a/examples/line_sender_cpp_example_auth.cpp +++ b/examples/line_sender_cpp_example_auth.cpp @@ -10,7 +10,7 @@ static bool example(std::string_view host, std::string_view port) { auto sender = questdb::ingress::line_sender::from_conf( "tcp::addr=" + std::string{host} + ":" + std::string{port} + - ";" + ";protocol_version=2;" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" diff --git a/examples/line_sender_cpp_example_auth_tls.cpp b/examples/line_sender_cpp_example_auth_tls.cpp index 766d08a9..3f08d8c7 100644 --- a/examples/line_sender_cpp_example_auth_tls.cpp +++ b/examples/line_sender_cpp_example_auth_tls.cpp @@ -10,7 +10,7 @@ static bool example(std::string_view host, std::string_view port) { auto sender = questdb::ingress::line_sender::from_conf( "tcps::addr=" + std::string{host} + ":" + std::string{port} + - ";" + ";protocol_version=2;" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" diff --git a/examples/line_sender_cpp_example_from_conf.cpp b/examples/line_sender_cpp_example_from_conf.cpp index 36e27455..697850e0 100644 --- a/examples/line_sender_cpp_example_from_conf.cpp +++ b/examples/line_sender_cpp_example_from_conf.cpp @@ -9,7 +9,7 @@ int main(int argc, const char* argv[]) try { auto sender = questdb::ingress::line_sender::from_conf( - "tcp::addr=localhost:9009;"); + "tcp::addr=localhost:9009;protocol_version=2;"); // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid diff --git a/examples/line_sender_cpp_example_tls_ca.cpp b/examples/line_sender_cpp_example_tls_ca.cpp index 734f0bc0..4e3d0f10 100644 --- a/examples/line_sender_cpp_example_tls_ca.cpp +++ b/examples/line_sender_cpp_example_tls_ca.cpp @@ -11,7 +11,7 @@ static bool example( { auto sender = questdb::ingress::line_sender::from_conf( "tcps::addr=" + std::string{host} + ":" + std::string{port} + - ";" + ";protocol_version=2;" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index b4421cee..2880eac0 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -745,13 +745,6 @@ bool line_sender_opts_token_y( line_sender_utf8 token_y, line_sender_error** err_out); -/** - * Disable the line protocol validation. - */ -LINESENDER_API -bool line_sender_opts_disable_protocol_validation( - line_sender_opts* opts, line_sender_error** err_out); - /** * set the line protocol version. */ @@ -920,21 +913,10 @@ line_sender* line_sender_from_conf( LINESENDER_API line_sender* line_sender_from_env(line_sender_error** err_out); -/** - * Returns the QuestDB server's recommended default line protocol version. - * Will be used to [`line_sender_buffer_set_protocol_version`] - * - * The version selection follows these rules: - * 1. TCP/TCPS Protocol: Always returns [`ProtocolVersion::V2`] - * 2. HTTP/HTTPS Protocol: - * - If line protocol auto-detection is disabled - * [`line_sender_opts_disable_protocol_validation`], returns - * [`ProtocolVersion::V2`] - * - If line protocol auto-detection is enabled: - * - Uses the server's default version if supported by the client - * - Otherwise uses the highest mutually supported version from the - * intersection of client and server compatible versions. - */ +/// Returns sender's default protocol version. +/// 1. User-set value via [`line_sender_opts_protocol_version`] +/// 2. V1 for TCP/TCPS (legacy protocol) +/// 3. Auto-detected version for HTTP/HTTPS LINESENDER_API protocol_version line_sender_default_protocol_version( const line_sender* sender); diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 4c55e2a2..e80192d4 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -882,24 +882,14 @@ class opts * @param[in] protocol The protocol to use. * @param[in] host The QuestDB database host. * @param[in] port The QuestDB tcp or http port. - * @param[in] disable_protocol_validation disable line protocol version * validation. */ - opts( - protocol protocol, - utf8_view host, - uint16_t port, - bool disable_protocol_validation = false) noexcept + opts(protocol protocol, utf8_view host, uint16_t port) noexcept : _impl{::line_sender_opts_new( static_cast<::line_sender_protocol>(protocol), host._impl, port)} { line_sender_error::wrapped_call( ::line_sender_opts_user_agent, _impl, _user_agent::name()); - if (disable_protocol_validation) - { - line_sender_error::wrapped_call( - ::line_sender_opts_disable_protocol_validation, _impl); - } } /** @@ -907,15 +897,14 @@ class opts * @param[in] protocol The protocol to use. * @param[in] host The QuestDB database host. * @param[in] port The QuestDB tcp or http port. - * @param[in] disable_protocol_validation disable line protocol version + * @param[in] version The protocol version to use. * validation. */ opts( protocol protocol, utf8_view host, uint16_t port, - protocol_version version, - bool disable_protocol_validation = false) noexcept + protocol_version version) noexcept : _impl{::line_sender_opts_new( static_cast<::line_sender_protocol>(protocol), host._impl, port)} { @@ -923,11 +912,6 @@ class opts ::line_sender_opts_user_agent, _impl, _user_agent::name()); line_sender_error::wrapped_call( ::line_sender_opts_protocol_version, _impl, version); - if (disable_protocol_validation) - { - line_sender_error::wrapped_call( - ::line_sender_opts_disable_protocol_validation, _impl); - } } /** @@ -936,13 +920,8 @@ class opts * @param[in] protocol The protocol to use. * @param[in] host The QuestDB database host. * @param[in] port The QuestDB tcp or http port as service name. - * @param[in] disable_protocol_validation disable line protocol version */ - opts( - protocol protocol, - utf8_view host, - utf8_view port, - bool disable_protocol_validation = false) noexcept + opts(protocol protocol, utf8_view host, utf8_view port) noexcept : _impl{::line_sender_opts_new_service( static_cast<::line_sender_protocol>(protocol), host._impl, @@ -950,11 +929,6 @@ class opts { line_sender_error::wrapped_call( ::line_sender_opts_user_agent, _impl, _user_agent::name()); - if (disable_protocol_validation) - { - line_sender_error::wrapped_call( - ::line_sender_opts_disable_protocol_validation, _impl); - } } /** @@ -963,14 +937,13 @@ class opts * @param[in] protocol The protocol to use. * @param[in] host The QuestDB database host. * @param[in] port The QuestDB tcp or http port as service name. - * @param[in] disable_protocol_validation disable line protocol version + * @param[in] version The protocol version to use. */ opts( protocol protocol, utf8_view host, utf8_view port, - protocol_version version, - bool disable_protocol_validation = false) noexcept + protocol_version version) noexcept : _impl{::line_sender_opts_new_service( static_cast<::line_sender_protocol>(protocol), host._impl, @@ -980,11 +953,6 @@ class opts ::line_sender_opts_user_agent, _impl, _user_agent::name()); line_sender_error::wrapped_call( ::line_sender_opts_protocol_version, _impl, version); - if (disable_protocol_validation) - { - line_sender_error::wrapped_call( - ::line_sender_opts_disable_protocol_validation, _impl); - } } opts(const opts& other) noexcept @@ -1091,16 +1059,6 @@ class opts return *this; } - /** - * Disable the validation of the line protocol version. - */ - opts& disable_protocol_validation() - { - line_sender_error::wrapped_call( - ::line_sender_opts_disable_protocol_validation, _impl); - return *this; - } - /** * Configure how long to wait for messages from the QuestDB server during * the TLS handshake and authentication process. @@ -1287,21 +1245,13 @@ class line_sender return {opts::from_env()}; } - line_sender( - protocol protocol, - utf8_view host, - uint16_t port, - bool disable_protocol_validation = false) - : line_sender{opts{protocol, host, port, disable_protocol_validation}} + line_sender(protocol protocol, utf8_view host, uint16_t port) + : line_sender{opts{protocol, host, port}} { } - line_sender( - protocol protocol, - utf8_view host, - utf8_view port, - bool disable_protocol_validation = false) - : line_sender{opts{protocol, host, port, disable_protocol_validation}} + line_sender(protocol protocol, utf8_view host, utf8_view port) + : line_sender{opts{protocol, host, port}} { } @@ -1309,10 +1259,8 @@ class line_sender protocol protocol, utf8_view host, uint16_t port, - protocol_version version, - bool disable_protocol_validation = false) - : line_sender{ - opts{protocol, host, port, version, disable_protocol_validation}} + protocol_version version) + : line_sender{opts{protocol, host, port, version}} { } @@ -1320,10 +1268,8 @@ class line_sender protocol protocol, utf8_view host, utf8_view port, - protocol_version version, - bool disable_protocol_validation = false) - : line_sender{ - opts{protocol, host, port, version, disable_protocol_validation}} + protocol_version version) + : line_sender{opts{protocol, host, port, version}} { } diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 8a93a817..921f6731 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -1172,15 +1172,6 @@ pub unsafe extern "C" fn line_sender_opts_token_y( upd_opts!(opts, err_out, token_y, token_y.as_str()) } -/// Disable the line protocol validation. -#[no_mangle] -pub unsafe extern "C" fn line_sender_opts_disable_protocol_validation( - opts: *mut line_sender_opts, - err_out: *mut *mut line_sender_error, -) -> bool { - upd_opts!(opts, err_out, disable_protocol_validation) -} - /// set the line protocol version. #[no_mangle] pub unsafe extern "C" fn line_sender_opts_protocol_version( @@ -1420,16 +1411,10 @@ unsafe fn unwrap_sender_mut<'a>(sender: *mut line_sender) -> &'a mut Sender { &mut (*sender).0 } -/// Returns the client's recommended default line protocol version. -/// -/// The version selection follows these rules: -/// 1. **TCP/TCPS Protocol**: Always returns [`ProtocolVersion::V2`] -/// 2. **HTTP/HTTPS Protocol**: -/// - If line protocol auto-detection is disabled [`line_sender_opts_disable_protocol_validation`], returns [`ProtocolVersion::V2`] -/// - If line protocol auto-detection is enabled: -/// - Uses the server's default version if supported by the client -/// - Otherwise uses the highest mutually supported version from the intersection -/// of client and server compatible versions +/// Returns sender's default protocol version. +/// 1. User-set value via [`line_sender_opts_protocol_version`] +/// 2. V1 for TCP/TCPS (legacy protocol) +/// 3. Auto-detected version for HTTP/HTTPS #[no_mangle] pub unsafe extern "C" fn line_sender_default_protocol_version( sender: *const line_sender, diff --git a/questdb-rs/src/ingress/http.rs b/questdb-rs/src/ingress/http.rs index ac36b615..9dcf5ada 100644 --- a/questdb-rs/src/ingress/http.rs +++ b/questdb-rs/src/ingress/http.rs @@ -59,7 +59,6 @@ pub(super) struct HttpConfig { pub(super) user_agent: String, pub(super) retry_timeout: ConfigSetting, pub(super) request_timeout: ConfigSetting, - pub(super) disable_protocol_validation: ConfigSetting, } impl Default for HttpConfig { @@ -69,7 +68,6 @@ impl Default for HttpConfig { user_agent: concat!("questdb/rust/", env!("CARGO_PKG_VERSION")).to_string(), retry_timeout: ConfigSetting::new_default(Duration::from_secs(10)), request_timeout: ConfigSetting::new_default(Duration::from_secs(10)), - disable_protocol_validation: ConfigSetting::new_default(false), } } } @@ -406,25 +404,14 @@ pub(super) fn http_send_with_retries( retry_http_send(state, buf, request_timeout, retry_timeout, last_rep) } -/// Determines the server's default and all-supported protocol versions. -/// -/// Returns a tuple containing: -/// - `Option>`: List of all protocol versions supported by the server. -/// - `Some(versions)`: When server explicitly provides supported versions (modern servers). -/// - `None`: When server doesn't provide version info (legacy servers or 404 response). -/// - `ProtocolVersion`: The server-recommended default protocol version -/// (Here we introduce a new field, rather than use the implicit max value of supported versions). -/// -/// When protocol version is auto-detection mode (no explicit set by use user), -/// client will use the server's `default_version` as protocol version. -/// When user explicitly specifies a `protocol_version`, client will -/// validate against `support_versions`, Returns error if specified version not in supported list. -pub(super) fn get_protocol_version( +/// Return and the server's all supported protocol versions. +/// - For modern servers: Returns explicit version list from `line.proto.support.versions` JSON field(/settings endpoint) +/// - For legacy servers (404 response or missing version field): Automatically falls back to [`ProtocolVersion::V1`] +pub(super) fn get_supported_protocol_versions( state: &HttpHandlerState, settings_url: &str, -) -> Result<(Option>, ProtocolVersion), Error> { - let mut support_versions: Option> = None; - let mut default_version = ProtocolVersion::V1; +) -> Result, Error> { + let mut support_versions: Vec = vec![]; let response = match http_get_with_retries( state, @@ -435,7 +422,8 @@ pub(super) fn get_protocol_version( Ok(res) => { if res.status().is_client_error() || res.status().is_server_error() { if res.status().as_u16() == 404 { - return Ok((support_versions, default_version)); + support_versions.push(ProtocolVersion::V1); + return Ok(support_versions); } return Err(fmt!( ProtocolVersionError, @@ -451,7 +439,8 @@ pub(super) fn get_protocol_version( let e = match err { ureq::Error::StatusCode(code) => { if code == 404 { - return Ok((support_versions, default_version)); + support_versions.push(ProtocolVersion::V1); + return Ok(support_versions); } else { fmt!( ProtocolVersionError, @@ -488,51 +477,17 @@ pub(super) fn get_protocol_version( if let Some(serde_json::Value::Array(ref values)) = json.get("line.proto.support.versions") { - let mut versions = Vec::new(); for value in values.iter() { if let Some(v) = value.as_u64() { match v { - 1 => versions.push(ProtocolVersion::V1), - 2 => versions.push(ProtocolVersion::V2), + 1 => support_versions.push(ProtocolVersion::V1), + 2 => support_versions.push(ProtocolVersion::V2), _ => {} } } } - support_versions = Some(versions); - } - - if let Some(serde_json::Value::Number(ref v)) = json.get("line.proto.default.version") { - default_version = match v.as_u64() { - Some(vu64) => match vu64 { - 1 => ProtocolVersion::V1, - 2 => ProtocolVersion::V2, - _ => { - if let Some(ref versions) = support_versions { - if versions.contains(&ProtocolVersion::V2) { - ProtocolVersion::V2 - } else if versions.contains(&ProtocolVersion::V1) { - ProtocolVersion::V1 - } else { - return Err(error::fmt!( - ProtocolVersionError, - "Server does not support current client" - )); - } - } else { - return Err(error::fmt!( - ProtocolVersionError, - "Unexpected response version content." - )); - } - } - }, - None => { - return Err(error::fmt!( - ProtocolVersionError, - "Not a valid int for line.proto.default.version in response." - )) - } - }; + } else { + support_versions.push(ProtocolVersion::V1); } } else { return Err(error::fmt!( @@ -540,7 +495,7 @@ pub(super) fn get_protocol_version( "Malformed server response, settings url: {}, err: failed to read response body as UTF-8", settings_url )); } - Ok((support_versions, default_version)) + Ok(support_versions) } #[allow(clippy::result_large_err)] // `ureq::Error` is large enough to cause this warning. diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 953fc883..6c95657f 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -591,8 +591,8 @@ impl Buffer { /// - Sets maximum name length to **127 characters** (QuestDB server default) /// /// This is equivalent to [`Sender::new_buffer`] when using the sender's - /// protocol version. For custom name lengths, use - /// [`Sender::new_buffer_with_max_name_len`]. + /// protocol version. For custom name lengths, use [`with_max_name_len`](Self::with_max_name_len) + /// or [`Sender::new_buffer_with_max_name_len`]. pub fn new(protocol_version: ProtocolVersion) -> Self { Self { output: Vec::new(), @@ -1376,10 +1376,9 @@ pub struct Sender { default_protocol_version: ProtocolVersion, #[cfg(feature = "ilp-over-http")] /// List of protocol versions supported by the server. - /// This is an `Option>` because: - /// - `Some(versions)`: The server explicitly returned supported protocol versions - /// during handshake (with `/settings` endpoint). - /// - `None`: The server didn't provide protocol version information (legacy servers). + /// It will be `None` when user explicitly sets `protocol_version` (no first http round trip). + /// Note that when connecting to older servers (responding with 404 or missing version data), + /// it will be init as [`ProtocolVersion::V1`] instead of leaving `None` supported_protocol_versions: Option>, } @@ -2017,18 +2016,6 @@ impl SenderBuilder { builder.retry_timeout(Duration::from_millis(parse_conf_value(key, val)?))? } - #[cfg(feature = "ilp-over-http")] - "disable_protocol_validation" => { - if val == "on" { - builder.disable_protocol_validation()? - } else if val != "off" { - return Err(error::fmt!( - ConfigError, "invalid \"disable_protocol_validation\" [value={val}, allowed-values=[on, off]]]\"]")); - } else { - builder - } - } - // Ignore other parameters. // We don't want to fail on unknown keys as this would require releasing different // library implementations in lock step as soon as a new parameter is added to any of them, @@ -2307,24 +2294,6 @@ impl SenderBuilder { Ok(self) } - #[cfg(feature = "ilp-over-http")] - /// Disables automatic line protocol version validation for ILP-over-HTTP. - /// - /// - When set to `"off"`: Skips the initial server version handshake and disables protocol validation. - /// - When set to `"on"`: Keeps default validation behavior (recommended). - /// - /// Please ensure client's default version ([`ProtocolVersion::V1`]) or - /// explicitly set protocol version exactly matches server expectation. - pub fn disable_protocol_validation(mut self) -> Result { - if let Some(http) = &mut self.http { - // ignore "already specified" error - let _ = http - .disable_protocol_validation - .set_specified("disable_protocol_validation", true); - } - Ok(self) - } - #[cfg(feature = "ilp-over-http")] /// Internal API, do not use. /// This is exposed exclusively for the Python client. @@ -2612,20 +2581,20 @@ impl SenderBuilder { } }; - let mut default_protocol_version = match self.protocol_version.deref() { - None => ProtocolVersion::V1, - Some(v) => *v, - }; - + let default_protocol_version; #[cfg(feature = "ilp-over-http")] let mut supported_protocol_versions: Option> = None; - #[cfg(feature = "ilp-over-http")] - match self.protocol { - Protocol::Tcp | Protocol::Tcps => {} - Protocol::Http | Protocol::Https => { - let http_config = self.http.as_ref().unwrap(); - if !*http_config.disable_protocol_validation.deref() { + match self.protocol_version.deref() { + Some(v) => { + default_protocol_version = *v; + } + None => match self.protocol { + Protocol::Tcp | Protocol::Tcps => { + default_protocol_version = ProtocolVersion::V1; + } + #[cfg(feature = "ilp-over-http")] + Protocol::Http | Protocol::Https => { if let ProtocolHandler::Http(http_state) = &handler { let settings_url = &format!( "{}://{}:{}/settings", @@ -2633,14 +2602,24 @@ impl SenderBuilder { self.host.deref(), self.port.deref() ); - (supported_protocol_versions, default_protocol_version) = - get_protocol_version(http_state, settings_url)?; + let versions = get_supported_protocol_versions(http_state, settings_url)?; + if versions.contains(&ProtocolVersion::V2) { + default_protocol_version = ProtocolVersion::V2; + } else if versions.contains(&ProtocolVersion::V1) { + default_protocol_version = ProtocolVersion::V1; + } else { + return Err(error::fmt!( + ProtocolVersionError, + "Server does not support current client" + )); + } + supported_protocol_versions = Some(versions); } else { default_protocol_version = ProtocolVersion::V1; } } - } - }; + }, + } if auth.is_some() { descr.push_str("auth=on]"); @@ -2855,8 +2834,6 @@ impl Sender { /// This initializes a buffer using the sender's protocol version and /// the QuestDB server's default maximum name length of 127 characters. /// For custom name lengths, use [`new_buffer_with_max_name_len`](Self::new_buffer_with_max_name_len) - /// - /// The default `max_name_len` matches the QuestDB server's `cairo.max.file.name.length` setting. pub fn new_buffer(&self) -> Buffer { Buffer::new(self.default_protocol_version) } @@ -3015,16 +2992,10 @@ impl Sender { !self.connected } - /// Returns the QuestDB server's recommended default line protocol version. - /// - /// The version selection follows these rules: - /// 1. **TCP/TCPS Protocol**: Always returns [`ProtocolVersion::V2`] - /// 2. **HTTP/HTTPS Protocol**: - /// - If line protocol auto-detection is disabled [`SenderBuilder::disable_protocol_validation`], returns [`ProtocolVersion::V2`] - /// - If line protocol auto-detection is enabled: - /// - Uses the server's default version if supported by the client - /// - Otherwise uses the highest mutually supported version from the intersection - /// of client and server compatible versions + /// Returns sender's default protocol version. + /// 1. User-set value via [`SenderBuilder::protocol_version`] + /// 2. V1 for TCP/TCPS (legacy protocol) + /// 3. Auto-detected version for HTTP/HTTPS pub fn default_protocol_version(&self) -> ProtocolVersion { self.default_protocol_version } @@ -3041,30 +3012,18 @@ impl Sender { match &self.handler { ProtocolHandler::Socket(_) => Ok(()), #[cfg(feature = "ilp-over-http")] - ProtocolHandler::Http(http) => { - if *http.config.disable_protocol_validation.deref() { - Ok(()) - } else { - match self.supported_protocol_versions { - Some(ref supported_line_protocols) => { - if supported_line_protocols.contains(&version) { - Ok(()) - } else { - Err(error::fmt!( + ProtocolHandler::Http(_) => { + match self.supported_protocol_versions { + Some(ref supported_line_protocols) => { + if supported_line_protocols.contains(&version) { + Ok(()) + } else { + Err(error::fmt!( ProtocolVersionError, "Line protocol version {} is not supported by current QuestDB Server", version)) - } - } - None => { - if version == ProtocolVersion::V1 { - Ok(()) - } else { - Err(error::fmt!( - ProtocolVersionError, - "Line protocol version {} is not supported by current QuestDB Server", version)) - } } } + None => Ok(()), } } } diff --git a/questdb-rs/src/ingress/tests.rs b/questdb-rs/src/ingress/tests.rs index ee4defab..599b5ac9 100644 --- a/questdb-rs/src/ingress/tests.rs +++ b/questdb-rs/src/ingress/tests.rs @@ -447,7 +447,7 @@ fn connect_timeout_uses_request_timeout() { let builder = SenderBuilder::new(Protocol::Http, "127.0.0.2", "1111") .request_timeout(request_timeout) .unwrap() - .disable_protocol_validation() + .protocol_version(ProtocolVersion::V2) .unwrap() .retry_timeout(Duration::from_millis(10)) .unwrap() diff --git a/questdb-rs/src/tests/http.rs b/questdb-rs/src/tests/http.rs index d6698987..1700f01f 100644 --- a/questdb-rs/src/tests/http.rs +++ b/questdb-rs/src/tests/http.rs @@ -48,7 +48,7 @@ fn test_two_lines( .at_now()?; let buffer2 = buffer.clone(); - let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); + let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); let sender_builder = server.lsb_http(); let server_thread = std::thread::spawn(move || -> io::Result<()> { @@ -100,7 +100,7 @@ fn test_text_plain_error( .at_now()?; buffer.table("test")?.column_f64("sym", 2.0)?.at_now()?; - let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); + let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); let sender_builder = server.lsb_http(); let buffer2 = buffer.clone(); @@ -157,7 +157,7 @@ fn test_bad_json_error( .at_now()?; buffer.table("test")?.column_f64("sym", 2.0)?.at_now()?; - let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); + let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); let sender_builder = server.lsb_http(); let buffer2 = buffer.clone(); @@ -216,7 +216,7 @@ fn test_json_error( .at_now()?; buffer.table("test")?.column_f64("sym", 2.0)?.at_now()?; - let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); + let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); let sender_builder = server.lsb_http(); let buffer2 = buffer.clone(); @@ -277,7 +277,7 @@ fn test_no_connection( .at_now()?; let mut sender = SenderBuilder::new(Protocol::Http, "127.0.0.1", 1) - .disable_protocol_validation()? + .protocol_version(ProtocolVersion::V2)? .build()?; let res = sender.flush_and_keep(&buffer); assert!(res.is_err()); @@ -300,7 +300,7 @@ fn test_old_server_without_ilp_http_support( .column_f64("x", 1.0)? .at_now()?; - let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); + let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); let sender_builder = server.lsb_http(); let buffer2 = buffer.clone(); @@ -356,7 +356,7 @@ fn test_http_basic_auth( .column_f64("x", 1.0)? .at_now()?; - let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); + let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); let sender_builder = server .lsb_http() .username("Aladdin")? @@ -411,7 +411,7 @@ fn test_unauthenticated( .column_f64("x", 1.0)? .at_now()?; - let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); + let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); let sender_builder = server.lsb_http(); let buffer2 = buffer.clone(); @@ -469,7 +469,7 @@ fn test_token_auth( .column_f64("x", 1.0)? .at_now()?; - let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); + let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); let sender_builder = server.lsb_http().token("0123456789")?; let buffer2 = buffer.clone(); @@ -516,13 +516,13 @@ fn test_request_timeout( .at_now()?; // Here we use a mock (tcp) server instead and don't send a response back. - let server = MockServer::new()?.configure_settings_response(2, &[1, 2]); + let server = MockServer::new()?.configure_settings_response(&[1, 2]); let request_timeout = Duration::from_millis(50); let time_start = std::time::Instant::now(); let mut sender = server .lsb_http() - .disable_protocol_validation()? + .protocol_version(ProtocolVersion::V2)? .request_timeout(request_timeout)? .build()?; let res = sender.flush_and_keep(&buffer); @@ -550,11 +550,11 @@ fn test_tls( .at(TimestampNanos::new(10000000))?; let buffer2 = buffer.clone(); - let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); + let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); let mut sender = server .lsb_https() .tls_roots(ca_path)? - .disable_protocol_validation()? + .protocol_version(ProtocolVersion::V2)? .build()?; let server_thread = std::thread::spawn(move || -> io::Result<()> { @@ -591,7 +591,7 @@ fn test_user_agent( .at(TimestampNanos::new(10000000))?; let buffer2 = buffer.clone(); - let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); + let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); let sender_builder = server.lsb_http().user_agent("wallabies/1.2.99")?; let server_thread = std::thread::spawn(move || -> io::Result<()> { @@ -634,7 +634,7 @@ fn test_two_retries( .at(TimestampNanos::new(10000000))?; let buffer2 = buffer.clone(); - let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); + let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); let sender_builder = server.lsb_http().retry_timeout(Duration::from_secs(30))?; let server_thread = std::thread::spawn(move || -> io::Result<()> { @@ -704,11 +704,11 @@ fn test_one_retry( .at(TimestampNanos::new(10000000))?; let buffer2 = buffer.clone(); - let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); + let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); let mut sender = server .lsb_http() .retry_timeout(Duration::from_millis(19))? - .disable_protocol_validation() + .protocol_version(ProtocolVersion::V2) .unwrap() .build()?; @@ -787,7 +787,7 @@ fn test_transactional( let buffer3 = buffer2.clone(); assert!(buffer2.transactional()); - let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); + let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); let sender_builder = server.lsb_http(); let server_thread = std::thread::spawn(move || -> io::Result<()> { @@ -833,7 +833,7 @@ fn test_transactional( #[test] fn test_sender_protocol_version() -> TestResult { - let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); + let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); let sender_builder = server.lsb_http(); let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; @@ -859,7 +859,7 @@ fn test_sender_protocol_version() -> TestResult { #[test] fn test_sender_protocol_version_old_server1() -> TestResult { - let mut server = MockServer::new()?.configure_settings_response(0, &[1, 2]); + let mut server = MockServer::new()?.configure_settings_response(&[]); let sender_builder = server.lsb_http(); let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; @@ -875,14 +875,14 @@ fn test_sender_protocol_version_old_server1() -> TestResult { }); let sender = sender_builder.build()?; assert_eq!(sender.default_protocol_version(), ProtocolVersion::V1); - assert!(sender.support_protocol_versions().is_none()); + assert_eq!(sender.support_protocol_versions(), Some(vec![ProtocolVersion::V1])); server_thread.join().unwrap()?; Ok(()) } #[test] fn test_sender_protocol_version_old_server2() -> TestResult { - let mut server = MockServer::new()?.configure_settings_response(0, &[1, 2]); + let mut server = MockServer::new()?.configure_settings_response(&[]); let sender_builder = server.lsb_http(); let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; @@ -896,14 +896,14 @@ fn test_sender_protocol_version_old_server2() -> TestResult { }); let sender = sender_builder.build()?; assert_eq!(sender.default_protocol_version(), ProtocolVersion::V1); - assert!(sender.support_protocol_versions().is_none()); + assert_eq!(sender.support_protocol_versions(), Some(vec![ProtocolVersion::V1])); server_thread.join().unwrap()?; Ok(()) } #[test] fn test_sender_protocol_version_unsupported_client() -> TestResult { - let mut server = MockServer::new()?.configure_settings_response(3, &[3, 4]); + let mut server = MockServer::new()?.configure_settings_response(&[3, 4]); let sender_builder = server.lsb_http(); let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; @@ -920,9 +920,12 @@ fn test_sender_protocol_version_unsupported_client() -> TestResult { } #[test] -fn test_sender_disable_protocol_version_validation() -> TestResult { - let mut server = MockServer::new()?.configure_settings_response(2, &[1, 2]); - let mut sender = server.lsb_http().disable_protocol_validation()?.build()?; +fn test_sender_explicit_set_protocol_version_v2() -> TestResult { + let mut server = MockServer::new()?.configure_settings_response(&[]); + let mut sender = server + .lsb_http() + .protocol_version(ProtocolVersion::V2)? + .build()?; let mut buffer = sender.new_buffer(); buffer .table("test")? diff --git a/questdb-rs/src/tests/mock.rs b/questdb-rs/src/tests/mock.rs index 71d63c29..59e98515 100644 --- a/questdb-rs/src/tests/mock.rs +++ b/questdb-rs/src/tests/mock.rs @@ -310,16 +310,11 @@ impl MockServer { } #[cfg(feature = "ilp-over-http")] - pub fn configure_settings_response( - mut self, - default_version: u16, - supported_versions: &[u16], - ) -> Self { - if default_version == 0 { + pub fn configure_settings_response(mut self, supported_versions: &[u16]) -> Self { + if supported_versions.is_empty() { self.settings_response = serde_json::json!({"version": "8.1.2"}); } else { self.settings_response = serde_json::json!({ - "line.proto.default.version": default_version, "line.proto.support.versions": supported_versions }); } From 56e349adab6aa91d04b477699f5882232604cd38 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 15 May 2025 17:17:06 +0800 Subject: [PATCH 071/119] rename c api `new_buffer()` name. --- cpp_test/test_line_sender.cpp | 2 +- examples/line_sender_c_example.c | 2 +- examples/line_sender_c_example_array.c | 2 +- examples/line_sender_c_example_auth.c | 2 +- examples/line_sender_c_example_auth_tls.c | 2 +- examples/line_sender_c_example_from_conf.c | 2 +- examples/line_sender_c_example_from_env.c | 2 +- examples/line_sender_c_example_http.c | 2 +- examples/line_sender_c_example_tls_ca.c | 2 +- include/questdb/ingress/line_sender.h | 7 ++++--- questdb-rs-ffi/src/lib.rs | 6 +++--- questdb-rs/src/tests/http.rs | 10 ++++++++-- system_test/questdb_line_sender.py | 6 ------ 13 files changed, 24 insertions(+), 23 deletions(-) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 39d3f16d..8816ea74 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -153,7 +153,7 @@ TEST_CASE("line_sender c api basics") CHECK(::line_sender_utf8_init(&v1_utf8, 2, "v1", &err)); ::line_sender_column_name f1_name{0, nullptr}; CHECK(::line_sender_column_name_init(&f1_name, 2, "f1", &err)); - ::line_sender_buffer* buffer = line_sender_new_buffer(sender); + ::line_sender_buffer* buffer = line_sender_buffer_new_for_sender(sender); CHECK(buffer != nullptr); CHECK(::line_sender_buffer_table(buffer, table_name, &err)); CHECK(::line_sender_buffer_symbol(buffer, t1_name, v1_utf8, &err)); diff --git a/examples/line_sender_c_example.c b/examples/line_sender_c_example.c index df1ee8b3..01796664 100644 --- a/examples/line_sender_c_example.c +++ b/examples/line_sender_c_example.c @@ -25,7 +25,7 @@ static bool example(const char* host, const char* port) free(conf_str); conf_str = NULL; - buffer = line_sender_new_buffer(sender); + buffer = line_sender_buffer_new_for_sender(sender); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. // We prepare all our table names and column names in advance. diff --git a/examples/line_sender_c_example_array.c b/examples/line_sender_c_example_array.c index b25fffaa..dabf8b3c 100644 --- a/examples/line_sender_c_example_array.c +++ b/examples/line_sender_c_example_array.c @@ -28,7 +28,7 @@ static bool example(const char* host, const char* port) free(conf_str); conf_str = NULL; - buffer = line_sender_new_buffer(sender); + buffer = line_sender_buffer_new_for_sender(sender); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB 初始缓冲 line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("market_orders"); diff --git a/examples/line_sender_c_example_auth.c b/examples/line_sender_c_example_auth.c index 2f0df546..11823162 100644 --- a/examples/line_sender_c_example_auth.c +++ b/examples/line_sender_c_example_auth.c @@ -31,7 +31,7 @@ static bool example(const char* host, const char* port) free(conf_str); conf_str = NULL; - buffer = line_sender_new_buffer(sender); + buffer = line_sender_buffer_new_for_sender(sender); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. // We prepare all our table names and column names in advance. diff --git a/examples/line_sender_c_example_auth_tls.c b/examples/line_sender_c_example_auth_tls.c index 79c044e5..4594afe6 100644 --- a/examples/line_sender_c_example_auth_tls.c +++ b/examples/line_sender_c_example_auth_tls.c @@ -31,7 +31,7 @@ static bool example(const char* host, const char* port) free(conf_str); conf_str = NULL; - buffer = line_sender_new_buffer(sender); + buffer = line_sender_buffer_new_for_sender(sender); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. // We prepare all our table names and column names in advance. diff --git a/examples/line_sender_c_example_from_conf.c b/examples/line_sender_c_example_from_conf.c index 487fce62..122d0853 100644 --- a/examples/line_sender_c_example_from_conf.c +++ b/examples/line_sender_c_example_from_conf.c @@ -14,7 +14,7 @@ int main(int argc, const char* argv[]) if (!sender) goto on_error; - buffer = line_sender_new_buffer(sender); + buffer = line_sender_buffer_new_for_sender(sender); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. // We prepare all our table names and column names in advance. diff --git a/examples/line_sender_c_example_from_env.c b/examples/line_sender_c_example_from_env.c index eed0912a..75904bc3 100644 --- a/examples/line_sender_c_example_from_env.c +++ b/examples/line_sender_c_example_from_env.c @@ -13,7 +13,7 @@ int main(int argc, const char* argv[]) if (!sender) goto on_error; - buffer = line_sender_new_buffer(sender); + buffer = line_sender_buffer_new_for_sender(sender); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. // We prepare all our table names and column names in advance. diff --git a/examples/line_sender_c_example_http.c b/examples/line_sender_c_example_http.c index c8d7fc52..09a7fe93 100644 --- a/examples/line_sender_c_example_http.c +++ b/examples/line_sender_c_example_http.c @@ -27,7 +27,7 @@ static bool example(const char* host, const char* port) free(conf_str); conf_str = NULL; - buffer = line_sender_new_buffer(sender); + buffer = line_sender_buffer_new_for_sender(sender); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("c_trades_http"); diff --git a/examples/line_sender_c_example_tls_ca.c b/examples/line_sender_c_example_tls_ca.c index 72aaeb74..058ee223 100644 --- a/examples/line_sender_c_example_tls_ca.c +++ b/examples/line_sender_c_example_tls_ca.c @@ -32,7 +32,7 @@ static bool example(const char* ca_path, const char* host, const char* port) free(conf_str); conf_str = NULL; - buffer = line_sender_new_buffer(sender); + buffer = line_sender_buffer_new_for_sender(sender); line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. // We prepare all our table names and column names in advance. diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 2880eac0..03aa1762 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -926,16 +926,17 @@ protocol_version line_sender_default_protocol_version( * default protocol version * which is the same as the QuestDB server default. */ -line_sender_buffer* line_sender_new_buffer(const line_sender* sender); +line_sender_buffer* line_sender_buffer_new_for_sender( + const line_sender* sender); /** * Construct a `line_sender_buffer` with sender's default protocol version and * a custom maximum length for table and column names. This should match the * `cairo.max.file.name.length` setting of the QuestDB server you're * connecting to. If the server does not configure it, the default is `127`, - * and you can call `line_sender_new_buffer()` instead. + * and you can call `line_sender_buffer_new_for_sender()` instead. */ -line_sender_buffer* line_sender_new_buffer_with_max_name_len( +line_sender_buffer* line_sender_buffer_new_with_max_name_len_for_sender( const line_sender* sender, size_t max_name_len); /** diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 921f6731..6b256074 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -1425,7 +1425,7 @@ pub unsafe extern "C" fn line_sender_default_protocol_version( /// Construct a `line_sender_buffer` with a `max_name_len` of `127` and sender's default protocol version /// which is the same as the QuestDB server default. #[no_mangle] -pub unsafe extern "C" fn line_sender_new_buffer( +pub unsafe extern "C" fn line_sender_buffer_new_for_sender( sender: *const line_sender, ) -> *mut line_sender_buffer { let sender = unwrap_sender(sender); @@ -1437,9 +1437,9 @@ pub unsafe extern "C" fn line_sender_new_buffer( /// length for table and column names. This should match the `cairo.max.file.name.length` setting of /// the QuestDB server you're connecting to. /// If the server does not configure it, the default is `127`, and you can -/// call `line_sender_new_buffer()` instead. +/// call `line_sender_buffer_new_for_sender()` instead. #[no_mangle] -pub unsafe extern "C" fn line_sender_new_buffer_with_max_name_len( +pub unsafe extern "C" fn line_sender_buffer_new_with_max_name_len_for_sender( sender: *const line_sender, max_name_len: size_t, ) -> *mut line_sender_buffer { diff --git a/questdb-rs/src/tests/http.rs b/questdb-rs/src/tests/http.rs index 1700f01f..e17a1ba1 100644 --- a/questdb-rs/src/tests/http.rs +++ b/questdb-rs/src/tests/http.rs @@ -875,7 +875,10 @@ fn test_sender_protocol_version_old_server1() -> TestResult { }); let sender = sender_builder.build()?; assert_eq!(sender.default_protocol_version(), ProtocolVersion::V1); - assert_eq!(sender.support_protocol_versions(), Some(vec![ProtocolVersion::V1])); + assert_eq!( + sender.support_protocol_versions(), + Some(vec![ProtocolVersion::V1]) + ); server_thread.join().unwrap()?; Ok(()) } @@ -896,7 +899,10 @@ fn test_sender_protocol_version_old_server2() -> TestResult { }); let sender = sender_builder.build()?; assert_eq!(sender.default_protocol_version(), ProtocolVersion::V1); - assert_eq!(sender.support_protocol_versions(), Some(vec![ProtocolVersion::V1])); + assert_eq!( + sender.support_protocol_versions(), + Some(vec![ProtocolVersion::V1]) + ); server_thread.join().unwrap()?; Ok(()) } diff --git a/system_test/questdb_line_sender.py b/system_test/questdb_line_sender.py index 654118dd..82486c9b 100644 --- a/system_test/questdb_line_sender.py +++ b/system_test/questdb_line_sender.py @@ -215,12 +215,6 @@ def set_sig(fn, restype, *argtypes): dll.line_sender_buffer_with_max_name_len, c_line_sender_buffer_p, c_size_t) - set_sig( - dll.line_sender_buffer_set_protocol_version, - c_bool, - c_line_sender_buffer_p, - c_protocol_version, - c_line_sender_error_p_p) set_sig( dll.line_sender_buffer_free, None, From f163a31cf4e56b10c758b41c73bcb9140ae97636 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 15 May 2025 18:08:34 +0800 Subject: [PATCH 072/119] fix system.tests --- include/questdb/ingress/line_sender.h | 2 +- questdb-rs-ffi/src/lib.rs | 2 +- system_test/questdb_line_sender.py | 27 +++++++++++---------------- system_test/test.py | 11 +++++++---- 4 files changed, 20 insertions(+), 22 deletions(-) diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 03aa1762..002acc9c 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -936,7 +936,7 @@ line_sender_buffer* line_sender_buffer_new_for_sender( * connecting to. If the server does not configure it, the default is `127`, * and you can call `line_sender_buffer_new_for_sender()` instead. */ -line_sender_buffer* line_sender_buffer_new_with_max_name_len_for_sender( +line_sender_buffer* line_sender_buffer_with_max_name_len_for_sender( const line_sender* sender, size_t max_name_len); /** diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 6b256074..5bad85a7 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -1439,7 +1439,7 @@ pub unsafe extern "C" fn line_sender_buffer_new_for_sender( /// If the server does not configure it, the default is `127`, and you can /// call `line_sender_buffer_new_for_sender()` instead. #[no_mangle] -pub unsafe extern "C" fn line_sender_buffer_new_with_max_name_len_for_sender( +pub unsafe extern "C" fn line_sender_buffer_with_max_name_len_for_sender( sender: *const line_sender, max_name_len: size_t, ) -> *mut line_sender_buffer { diff --git a/system_test/questdb_line_sender.py b/system_test/questdb_line_sender.py index 82486c9b..7f2e22bf 100644 --- a/system_test/questdb_line_sender.py +++ b/system_test/questdb_line_sender.py @@ -214,7 +214,8 @@ def set_sig(fn, restype, *argtypes): set_sig( dll.line_sender_buffer_with_max_name_len, c_line_sender_buffer_p, - c_size_t) + c_size_t, + c_protocol_version) set_sig( dll.line_sender_buffer_free, None, @@ -367,9 +368,10 @@ def set_sig(fn, restype, *argtypes): c_line_sender_utf8, c_line_sender_error_p_p) set_sig( - dll.line_sender_opts_disable_protocol_validation, + dll.line_sender_opts_protocol_version, c_bool, c_line_sender_opts_p, + c_protocol_version, c_line_sender_error_p_p) set_sig( dll.line_sender_opts_auth_timeout, @@ -445,6 +447,11 @@ def set_sig(fn, restype, *argtypes): dll.line_sender_default_protocol_version, c_protocol_version, c_line_sender_p) + set_sig( + dll.line_sender_buffer_with_max_name_len_for_sender, + c_line_sender_buffer_p, + c_line_sender_p, + c_size_t) set_sig( dll.line_sender_must_close, None, @@ -574,8 +581,6 @@ def __getattr__(self, name: str): fn = getattr(_DLL, 'line_sender_opts_' + name) def wrapper(*args): - if name == 'disable_protocol_validation': - return _error_wrapped_call(fn, self.impl) mapped_args = [ (_utf8(arg) if isinstance(arg, str) else arg) for arg in args] @@ -596,14 +601,10 @@ def __init__(self, micros: int): class Buffer: - def __init__(self, init_buf_size=65536, max_name_len=127, protocol_version=ProtocolVersion.V2): + def __init__(self, protocol_version: ProtocolVersion, init_buf_size=65536, max_name_len=127, ): self._impl = _DLL.line_sender_buffer_with_max_name_len( - c_size_t(max_name_len)) + c_size_t(max_name_len), protocol_version.value[0]) _DLL.line_sender_buffer_reserve(self._impl, c_size_t(init_buf_size)) - _error_wrapped_call( - _DLL.line_sender_buffer_set_protocol_version, - self._impl, - protocol_version.value[0]) def __len__(self): return _DLL.line_sender_buffer_size(self._impl) @@ -621,12 +622,6 @@ def peek(self) -> str: else: return '' - def set_protocol_version(self, version: ProtocolVersion): - _error_wrapped_call( - _DLL.line_sender_buffer_set_protocol_version, - self._impl, - version.value[0]) - def reserve(self, additional): _DLL.line_sender_buffer_reserve(self._impl, c_size_t(additional)) diff --git a/system_test/test.py b/system_test/test.py index bf3bc795..67111ebc 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -95,11 +95,11 @@ def ns_to_qdb_date(at_ts_ns): class TestSender(unittest.TestCase): - def _mk_linesender(self, disable_protocol_validation=False): + def _mk_linesender(self): # N.B.: We never connect with TLS here. auth = AUTH if QDB_FIXTURE.auth else {} - if disable_protocol_validation: - auth["disable_protocol_validation"] = "on" + if not QDB_FIXTURE.http and not QDB_FIXTURE.version < (8, 3, 1): + auth["protocol_version"] = "2" return qls.Sender( BUILD_MODE, qls.Protocol.HTTP if QDB_FIXTURE.http else qls.Protocol.TCP, @@ -952,7 +952,10 @@ def test_tcp_transactions(self): self.skipTest('TCP-only test') if QDB_FIXTURE.version <= (7, 3, 7): self.skipTest('No ILP/HTTP support') - buf = qls.Buffer() + version = qls.ProtocolVersion.V2 + if QDB_FIXTURE.version <= (8, 3, 1): + version = qls.ProtocolVersion.V1 + buf = qls.Buffer(version) buf.table('t1').column('c1', 'v1').at(time.time_ns()) with self.assertRaisesRegex(qls.SenderError, r'.*Transactional .* not supported.*'): with self._mk_linesender() as sender: From 0c5e5091c099fa29518886a533ef0c2e010ecc94 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Thu, 15 May 2025 12:01:37 +0100 Subject: [PATCH 073/119] renamed Influx to Ingestion in various places to avoid being misleading, since we are no longer compatible --- README.md | 6 +++++- cpp_test/test_line_sender.cpp | 2 +- include/questdb/ingress/line_sender.h | 23 ++++++++++++++--------- include/questdb/ingress/line_sender.hpp | 10 +++++----- questdb-rs-ffi/src/lib.rs | 12 ++++++------ questdb-rs/README.md | 8 ++++++-- questdb-rs/src/ingress/mod.md | 12 ++++++------ questdb-rs/src/ingress/mod.rs | 3 ++- 8 files changed, 45 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index f4f1931c..122612d1 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,13 @@ This library makes it easy to insert data into [QuestDB](https://questdb.io/). -This client library implements the [InfluxDB Line Protocol]( +This client library implements the QuestDB [Ingestion Line Protocol]( https://questdb.io/docs/reference/api/ilp/overview/) (ILP) over HTTP and TCP. +When connecting to QuestDB over HTTP, the library will auto-detect the server's +latest supported version and use it. Version 1 is compatible with +the [InfluxDB Line Protocol](https://docs.influxdata.com/influxdb/v2/reference/syntax/line-protocol/). + * Implementation is in Rust, with no additional [run-time or link-time dependencies](doc/BUILD.md#pre-requisites-and-dependencies) on the C++ standard library or other libraries. diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 8816ea74..92654d63 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -350,7 +350,7 @@ TEST_CASE("One column only - server.accept() after flush, before close") questdb::ingress::line_sender sender{ questdb::ingress::protocol::tcp, "localhost", server.port()}; - // Does not raise - this is unlike InfluxDB spec that disallows this. + // Does not raise - this is unlike the InfluxDB spec that disallows this. questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table("test").column("t1", "v1").at_now(); CHECK(!sender.must_close()); diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 002acc9c..79ed839c 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -95,28 +95,33 @@ typedef enum line_sender_error_code /** The protocol used to connect with. */ typedef enum line_sender_protocol { - /** InfluxDB Line Protocol over TCP. */ + /** Ingestion Line Protocol over TCP. */ line_sender_protocol_tcp, - /** InfluxDB Line Protocol over TCP with TLS. */ + /** Ingestion Line Protocol over TCP with TLS. */ line_sender_protocol_tcps, - /** InfluxDB Line Protocol over HTTP. */ + /** Ingestion Line Protocol over HTTP. */ line_sender_protocol_http, - /** InfluxDB Line Protocol over HTTP with TLS. */ + /** Ingestion Line Protocol over HTTP with TLS. */ line_sender_protocol_https, } line_sender_protocol; /** The line protocol version used to write data to buffer. */ typedef enum protocol_version { - /** Version 1 of InfluxDB Line Protocol. - Uses text format serialization for f64. */ + /** + * Version 1 of Ingestion Line Protocol. + * This version is compatible with InfluxDB line protocol. + */ protocol_version_1 = 1, - /** Version 2 of InfluxDB Line Protocol. - Uses binary format serialization for f64, and support array data type.*/ + /** + * Version 2 of Ingestion Line Protocol. + * Uses a binary format serialization for f64, and supports + * the array data type. + */ protocol_version_2 = 2, } protocol_version; @@ -618,7 +623,7 @@ bool line_sender_buffer_at_now( /////////// Connecting, sending and disconnecting. /** - * Inserts data into QuestDB via the InfluxDB Line Protocol. + * Inserts data into QuestDB via the Ingestion Line Protocol. * * Batch up rows in a `line_sender_buffer`, then call `line_sender_flush()` * or one of its variants with this object to send them. diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index e80192d4..35988e0f 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -87,16 +87,16 @@ enum class line_sender_error_code /** The protocol used to connect with. */ enum class protocol { - /** InfluxDB Line Protocol over TCP. */ + /** Ingestion Line Protocol over TCP. */ tcp, - /** InfluxDB Line Protocol over TCP with TLS. */ + /** Ingestion Line Protocol over TCP with TLS. */ tcps, - /** InfluxDB Line Protocol over HTTP. */ + /** Ingestion Line Protocol over HTTP. */ http, - /** InfluxDB Line Protocol over HTTP with TLS. */ + /** Ingestion Line Protocol over HTTP with TLS. */ https, }; @@ -1192,7 +1192,7 @@ class opts }; /** - * Inserts data into QuestDB via the InfluxDB Line Protocol. + * Inserts data into QuestDB via the Ingestion Line Protocol. * * Batch up rows in a `line_sender_buffer` object, then call * `.flush()` or one of its variants to send. diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 5bad85a7..c9e936ee 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -192,16 +192,16 @@ impl From for line_sender_error_code { #[repr(C)] #[derive(Debug, Copy, Clone)] pub enum line_sender_protocol { - /// InfluxDB Line Protocol over TCP. + /// Ingestion Line Protocol over TCP. line_sender_protocol_tcp, - /// InfluxDB Line Protocol over TCP with TLS. + /// Ingestion Line Protocol over TCP with TLS. line_sender_protocol_tcps, - /// InfluxDB Line Protocol over HTTP. + /// Ingestion Line Protocol over HTTP. line_sender_protocol_http, - /// InfluxDB Line Protocol over HTTP with TLS. + /// Ingestion Line Protocol over HTTP with TLS. line_sender_protocol_https, } @@ -235,7 +235,7 @@ pub enum ProtocolVersion { /// Uses text format serialization for f64. V1 = 1, - /// Version 2 of InfluxDB Line Protocol. + /// Version 2 of Ingestion Line Protocol. /// Uses binary format serialization for f64, and support array data type. V2 = 2, } @@ -1321,7 +1321,7 @@ pub unsafe extern "C" fn line_sender_opts_free(opts: *mut line_sender_opts) { } } -/// Inserts data into QuestDB via the InfluxDB Line Protocol. +/// Inserts data into QuestDB via the Ingestion Line Protocol. /// /// Batch up rows in a `line_sender_buffer`, then call `line_sender_flush()` or /// one of its variants with this object to send them. diff --git a/questdb-rs/README.md b/questdb-rs/README.md index 4d868d35..c44c9dcb 100644 --- a/questdb-rs/README.md +++ b/questdb-rs/README.md @@ -4,10 +4,14 @@ Official Rust client for [QuestDB](https://questdb.io/), an open-source SQL database designed to process time-series data, faster. The client library is designed for fast ingestion of data into QuestDB via the -InfluxDB Line Protocol (ILP). +Ingestion Line Protocol (ILP) over either HTTP (recommended) or TCP. * [QuestDB Database docs](https://questdb.io/docs/) -* [Docs on InfluxDB Line Protocol](https://questdb.io/docs/reference/api/ilp/overview/) +* [Docs on Ingestion Line Protocol](https://questdb.io/docs/reference/api/ilp/overview/) + +When connecting to QuestDB over HTTP, the library will auto-detect the server's +latest supported version and use it. Version 1 is compatible with +the [InfluxDB Line Protocol](https://docs.influxdata.com/influxdb/v2/reference/syntax/line-protocol/). ## Quick Start diff --git a/questdb-rs/src/ingress/mod.md b/questdb-rs/src/ingress/mod.md index 4a1a78cb..464713d9 100644 --- a/questdb-rs/src/ingress/mod.md +++ b/questdb-rs/src/ingress/mod.md @@ -1,7 +1,7 @@ # Fast Ingestion of Data into QuestDB -The `ingress` module implements QuestDB's variant of the -[InfluxDB Line Protocol](https://questdb.io/docs/reference/api/ilp/overview/) +The `ingress` module implements QuestDB's +[Ingestion Line Protocol](https://questdb.io/docs/reference/api/ilp/overview/) (ILP). To get started: @@ -93,8 +93,8 @@ error as appropriate and continue using it. # Health Check The QuestDB server has a "ping" endpoint you can access to see if it's alive, -and confirm the version of InfluxDB Line Protocol with which you are -interacting: +and confirm the version of the InfluxDB that it is compatible with at a protocol +level. ```shell curl -I http://localhost:9000/ping @@ -256,7 +256,7 @@ with a high-latency network connection. ### Timestamp Column Name -InfluxDB Line Protocol (ILP) does not give a name to the designated timestamp, +The Ingestion Line Protocol (ILP) does not give a name to the designated timestamp, so if you let this client auto-create the table, it will have the default `timestamp` name. To use a custom name, say `my_ts`, pre-create the table with the desired timestamp column name: @@ -317,7 +317,7 @@ considerations](https://github.com/questdb/c-questdb-client/blob/main/doc/CONSID document covers these topics: * Threading -* Differences between the InfluxDB Line Protocol and QuestDB Data Types +* Differences between the Ingestion Line Protocol and QuestDB Data Types * Data Quality * Client-side checks and server errors * Flushing diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 29b440f4..ece65649 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -1693,7 +1693,8 @@ pub enum Protocol { Tcps, #[cfg(feature = "ilp-over-http")] - /// ILP over HTTP (request-response, InfluxDB-compatible). + /// ILP over HTTP (request-response) + /// Version 1 is compatible with the InfluxDB Line Protocol. Http, #[cfg(feature = "ilp-over-http")] From 6e1e68d5b6c2cee810044a65213685d3bdcce026 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 15 May 2025 20:28:27 +0800 Subject: [PATCH 074/119] fix systemtests --- system_test/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system_test/test.py b/system_test/test.py index 67111ebc..5d6b337e 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -99,7 +99,7 @@ def _mk_linesender(self): # N.B.: We never connect with TLS here. auth = AUTH if QDB_FIXTURE.auth else {} if not QDB_FIXTURE.http and not QDB_FIXTURE.version < (8, 3, 1): - auth["protocol_version"] = "2" + auth["protocol_version"] = qls.ProtocolVersion.V2.value[0] return qls.Sender( BUILD_MODE, qls.Protocol.HTTP if QDB_FIXTURE.http else qls.Protocol.TCP, From a461bfdc2a3a11412ac89723ce51e8409e1e8933 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Thu, 15 May 2025 15:40:15 +0100 Subject: [PATCH 075/119] reworked the python system tests for cross version testing --- system_test/fixture.py | 3 +- system_test/questdb_line_sender.py | 30 ++++-- system_test/test.py | 144 ++++++++++++++++++----------- 3 files changed, 113 insertions(+), 64 deletions(-) diff --git a/system_test/fixture.py b/system_test/fixture.py index c992127a..d25aa335 100644 --- a/system_test/fixture.py +++ b/system_test/fixture.py @@ -230,7 +230,7 @@ class QueryError(Exception): class QuestDbFixture: - def __init__(self, root_dir: pathlib.Path, auth=False, wrap_tls=False, http=False): + def __init__(self, root_dir: pathlib.Path, auth=False, wrap_tls=False, http=False, protocol_version=None): self._root_dir = root_dir self.version = _parse_version(self._root_dir.name) self._data_dir = self._root_dir / 'data' @@ -255,6 +255,7 @@ def __init__(self, root_dir: pathlib.Path, auth=False, wrap_tls=False, http=Fals with open(auth_txt_path, 'w', encoding='utf-8') as auth_file: auth_file.write(AUTH_TXT) self.http = http + self.protocol_version = protocol_version def print_log(self): with open(self._log_path, 'r', encoding='utf-8') as log_file: diff --git a/system_test/questdb_line_sender.py b/system_test/questdb_line_sender.py index 7f2e22bf..a058181f 100644 --- a/system_test/questdb_line_sender.py +++ b/system_test/questdb_line_sender.py @@ -47,6 +47,7 @@ import ctypes import os from datetime import datetime +from functools import total_ordering from enum import Enum from ctypes import ( @@ -96,9 +97,10 @@ class CertificateAuthority(Enum): c_protocol_version = ctypes.c_int +@total_ordering class ProtocolVersion(Enum): - V1 = (c_protocol_version(1), 'v1') - V2 = (c_protocol_version(2), 'v2') + V1 = (c_protocol_version(1), '1') + V2 = (c_protocol_version(2), '2') @classmethod def from_int(cls, value: c_protocol_version): @@ -107,6 +109,16 @@ def from_int(cls, value: c_protocol_version): return member raise ValueError(f"invalid protocol version: {value}") + def __lt__(self, other): + if not isinstance(other, ProtocolVersion): + return NotImplemented + return self.value[0].value < other.value[0].value + + def __eq__(self, other): + if not isinstance(other, ProtocolVersion): + return NotImplemented + return self.value[0].value == other.value[0].value + class c_line_sender_opts(ctypes.Structure): pass @@ -753,8 +765,8 @@ def _map_value(key, value): return (value, 'on' if value else 'unsafe_off') else: return (value, 'on' if value else 'off') - elif isinstance(value, CertificateAuthority): - return value.value # a tuple of `(c_line_sender_ca, str)` + elif isinstance(value, (CertificateAuthority, ProtocolVersion)): + return value.value # a tuple of `(c enum value, str)` else: return (value, f'{value}') @@ -795,9 +807,13 @@ def connect(self): if self._impl: raise SenderError('Already connected') if self._build_mode == BuildMode.CONF: - self._impl = _error_wrapped_call( - _DLL.line_sender_from_conf, - _utf8(self._conf)) + try: + self._impl = _error_wrapped_call( + _DLL.line_sender_from_conf, + _utf8(self._conf)) + except SenderError as e: + raise SenderError( + f'Failed to connect to QuestDB with conf `{self._conf}`: {e}') from e elif self._build_mode == BuildMode.ENV: env_var = 'QDB_CLIENT_CONF' os.environ[env_var] = self._conf diff --git a/system_test/test.py b/system_test/test.py index 5d6b337e..f9fb5a98 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -33,6 +33,7 @@ import datetime import argparse import unittest +import itertools import numpy as np import time import questdb_line_sender as qls @@ -52,6 +53,9 @@ TLS_PROXY_FIXTURE: TlsProxyFixture = None BUILD_MODE = None +# The first QuestDB version that supports array types. +FIRST_ARRAYS_RELEASE = (8, 3, 1) + def retry_check_table(*args, **kwargs): return QDB_FIXTURE.retry_check_table(*args, **kwargs) @@ -97,15 +101,29 @@ def ns_to_qdb_date(at_ts_ns): class TestSender(unittest.TestCase): def _mk_linesender(self): # N.B.: We never connect with TLS here. - auth = AUTH if QDB_FIXTURE.auth else {} - if not QDB_FIXTURE.http and not QDB_FIXTURE.version < (8, 3, 1): - auth["protocol_version"] = qls.ProtocolVersion.V2.value[0] + kwargs = AUTH if QDB_FIXTURE.auth else {} + if QDB_FIXTURE.protocol_version: + kwargs["protocol_version"] = QDB_FIXTURE.protocol_version return qls.Sender( BUILD_MODE, qls.Protocol.HTTP if QDB_FIXTURE.http else qls.Protocol.TCP, QDB_FIXTURE.host, QDB_FIXTURE.http_server_port if QDB_FIXTURE.http else QDB_FIXTURE.line_tcp_port, - **auth) + **kwargs) + + @property + def expected_protocol_version(self) -> qls.ProtocolVersion: + """The protocol version that we expect to be handling.""" + if QDB_FIXTURE.protocol_version is None: + if not QDB_FIXTURE.http: + return qls.ProtocolVersion.V1 + + if QDB_FIXTURE.version >= FIRST_ARRAYS_RELEASE: + return qls.ProtocolVersion.V2 + + return qls.ProtocolVersion.V1 + + return QDB_FIXTURE.protocol_version def _expect_eventual_disconnect(self, sender): with self.assertRaisesRegex( @@ -482,8 +500,8 @@ def test_timestamp_column(self): self.assertEqual(scrubbed_dataset, exp_dataset) def test_f64_arr_column(self): - if QDB_FIXTURE.version < (8, 3, 1): - self.skipTest('array issues support') + if self.expected_protocol_version < qls.ProtocolVersion.V2: + self.skipTest('communicating over old protocol which does not support arrays') table_name = uuid.uuid4().hex array1 = np.array( @@ -517,8 +535,8 @@ def test_f64_arr_column(self): self.assertEqual(scrubbed_data, expected_data) def test_f64_arr_empty(self): - if QDB_FIXTURE.version < (8, 3, 1): - self.skipTest('array issues support') + if self.expected_protocol_version < qls.ProtocolVersion.V2: + self.skipTest('communicating over old protocol which does not support arrays') table_name = uuid.uuid4().hex empty_array = np.array([], dtype=np.float64).reshape(0, 0, 0) @@ -534,8 +552,8 @@ def test_f64_arr_empty(self): self.assertEqual(resp['dataset'][0][0], []) def test_f64_arr_non_contiguous(self): - if QDB_FIXTURE.version < (8, 3, 1): - self.skipTest('array issues support') + if self.expected_protocol_version < qls.ProtocolVersion.V2: + self.skipTest('communicating over old protocol which does not support arrays') table_name = uuid.uuid4().hex array = np.array([[1.1, 2.2], [3.3, 4.4]], dtype=np.float64)[:, ::2] @@ -551,8 +569,8 @@ def test_f64_arr_non_contiguous(self): self.assertEqual(resp['dataset'][0][0], [[1.1], [3.3]]) def test_f64_arr_zero_dimensional(self): - if QDB_FIXTURE.version < (8, 3, 1): - self.skipTest('array issues support') + if self.expected_protocol_version < qls.ProtocolVersion.V2: + self.skipTest('communicating over old protocol which does not support arrays') table_name = uuid.uuid4().hex array = np.array(42.0, dtype=np.float64) @@ -565,8 +583,8 @@ def test_f64_arr_zero_dimensional(self): self.assertIn('Zero-dimensional arrays are not supported', str(e)) def test_f64_arr_wrong_datatype(self): - if QDB_FIXTURE.version < (8, 3, 1): - self.skipTest('array issues support') + if self.expected_protocol_version < qls.ProtocolVersion.V2: + self.skipTest('communicating over old protocol which does not support arrays') table_name = uuid.uuid4().hex array = np.array([1, 2], dtype=np.int32) @@ -579,8 +597,8 @@ def test_f64_arr_wrong_datatype(self): self.assertIn('expect float64 array', str(e)) def test_f64_arr_mix_dims(self): - if QDB_FIXTURE.version < (8, 3, 1): - self.skipTest('array issues support') + if self.expected_protocol_version < qls.ProtocolVersion.V2: + self.skipTest('communicating over old protocol which does not support arrays') array_2d = np.array([[1.1, 2.2], [3.3, 4.4]], dtype=np.float64) array_1d = np.array([1.1], dtype=np.float64) @@ -599,6 +617,8 @@ def test_f64_arr_mix_dims(self): self.assertIn('cast error from protocol type: DOUBLE[] to column type: DOUBLE[][]', str(e)) def test_protocol_version_v1(self): + if self.expected_protocol_version >= qls.ProtocolVersion.V2: + self.skipTest('we are only validating the older protocol here') if QDB_FIXTURE.version <= (6, 1, 2): self.skipTest('Float issues support') numbers = [ @@ -610,7 +630,6 @@ def test_protocol_version_v1(self): table_name = uuid.uuid4().hex pending = None with self._mk_linesender() as sender: - sender.buffer.set_protocol_version(qls.ProtocolVersion.V1) for num in numbers: sender.table(table_name) sender.column('n', num) @@ -640,8 +659,8 @@ def massage(num): self.assertEqual(scrubbed_dataset, exp_dataset) def test_protocol_version_v1_array_unsupported(self): - if QDB_FIXTURE.version < (8, 3, 1): - self.skipTest('array unsupported') + if self.expected_protocol_version >= qls.ProtocolVersion.V2: + self.skipTest('communicating over a newer protocl that DOES support arrays') array1 = np.array( [ @@ -652,8 +671,7 @@ def test_protocol_version_v1_array_unsupported(self): ) table_name = uuid.uuid4().hex try: - with self._mk_linesender(True) as sender: - sender.buffer.set_protocol_version(qls.ProtocolVersion.V1) + with self._mk_linesender() as sender: sender.table(table_name) sender.column_f64_arr('f64_arr1', array1) sender.at_now() @@ -736,8 +754,8 @@ def test_c_array_example(self): 'market_orders') def _test_array_example(self, bin_name, table_name): - if QDB_FIXTURE.version < (8, 3, 1): - self.skipTest('array unsupported') + if self.expected_protocol_version < qls.ProtocolVersion.V2: + self.skipTest('communicating over old protocol which does not support arrays') if QDB_FIXTURE.http: self.skipTest('TCP-only test') if BUILD_MODE != qls.BuildMode.API: @@ -1029,7 +1047,14 @@ def run_with_existing(args): global QDB_FIXTURE MockFixture = namedtuple( 'MockFixture', - ('host', 'line_tcp_port', 'http_server_port', 'version', 'http', "auth")) + ( + 'host', + 'line_tcp_port', + 'http_server_port', + 'version', + 'http', + 'auth', + 'protocol_version')) host, line_tcp_port, http_server_port = args.existing.split(':') QDB_FIXTURE = MockFixture( host, @@ -1080,38 +1105,45 @@ def run_with_fixtures(args): global TLS_PROXY_FIXTURE global BUILD_MODE last_version = None - for questdb_dir in iter_versions(args): - for auth in (False, True): - for http in (False, True): - for build_mode in list(qls.BuildMode): - print( - f'Running tests [questdb_dir={questdb_dir}, auth={auth}, http={http}, build_mode={build_mode}]') - if http and last_version <= (7, 3, 7): - print('Skipping ILP/HTTP tests for versions <= 7.3.7') - continue - if http and auth: - print('Skipping auth for ILP/HTTP tests') - continue - QDB_FIXTURE = QuestDbFixture(questdb_dir, auth=auth, http=http) - TLS_PROXY_FIXTURE = None - BUILD_MODE = build_mode - try: - QDB_FIXTURE.start() - # Read the version _after_ a first start so it can rely - # on the live one from the `select build` query. - last_version = QDB_FIXTURE.version - port_to_proxy = QDB_FIXTURE.http_server_port \ - if http else QDB_FIXTURE.line_tcp_port - TLS_PROXY_FIXTURE = TlsProxyFixture(port_to_proxy) - TLS_PROXY_FIXTURE.start() - - test_prog = unittest.TestProgram(exit=False) - if not test_prog.result.wasSuccessful(): - sys.exit(1) - finally: - if TLS_PROXY_FIXTURE: - TLS_PROXY_FIXTURE.stop() - QDB_FIXTURE.stop() + + for questdb_dir, auth, http, protocol_version, build_mode in itertools.product( + iter_versions(args), + (False, True), # auth + (False, True), # http + [None] + list(qls.ProtocolVersion), # None is for `auto` + list(qls.BuildMode)): + print( + f'Running tests [questdb_dir={questdb_dir}, auth={auth}, http={http}, build_mode={build_mode}, protocol_version={protocol_version}]') + if http and last_version <= (7, 3, 7): + print('Skipping ILP/HTTP tests for versions <= 7.3.7') + continue + if http and auth: + print('Skipping auth for ILP/HTTP tests') + continue + QDB_FIXTURE = QuestDbFixture( + questdb_dir, + auth=auth, + http=http, + protocol_version=protocol_version) + TLS_PROXY_FIXTURE = None + BUILD_MODE = build_mode + try: + QDB_FIXTURE.start() + # Read the version _after_ a first start so it can rely + # on the live one from the `select build` query. + last_version = QDB_FIXTURE.version + port_to_proxy = QDB_FIXTURE.http_server_port \ + if http else QDB_FIXTURE.line_tcp_port + TLS_PROXY_FIXTURE = TlsProxyFixture(port_to_proxy) + TLS_PROXY_FIXTURE.start() + + test_prog = unittest.TestProgram(exit=False) + if not test_prog.result.wasSuccessful(): + sys.exit(1) + finally: + if TLS_PROXY_FIXTURE: + TLS_PROXY_FIXTURE.stop() + QDB_FIXTURE.stop() def run(args, show_help=False): From bf419f1c6a458a5eb065908510ed2f5d41dcc8d1 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 15 May 2025 23:10:09 +0800 Subject: [PATCH 076/119] enrich rust tests. --- questdb-rs/src/ingress/http.rs | 8 +- questdb-rs/src/ingress/mod.rs | 13 +- questdb-rs/src/ingress/ndarr.rs | 12 +- questdb-rs/src/tests/http.rs | 397 +++++++++++--------------------- questdb-rs/src/tests/sender.rs | 4 +- 5 files changed, 146 insertions(+), 288 deletions(-) diff --git a/questdb-rs/src/ingress/http.rs b/questdb-rs/src/ingress/http.rs index 9dcf5ada..9d19a0fa 100644 --- a/questdb-rs/src/ingress/http.rs +++ b/questdb-rs/src/ingress/http.rs @@ -405,7 +405,7 @@ pub(super) fn http_send_with_retries( } /// Return and the server's all supported protocol versions. -/// - For modern servers: Returns explicit version list from `line.proto.support.versions` JSON field(/settings endpoint) +/// - For modern servers: Returns explicit version list from QuestDB's server's `/settings` endpoint response /// - For legacy servers (404 response or missing version field): Automatically falls back to [`ProtocolVersion::V1`] pub(super) fn get_supported_protocol_versions( state: &HttpHandlerState, @@ -421,7 +421,9 @@ pub(super) fn get_supported_protocol_versions( ) { Ok(res) => { if res.status().is_client_error() || res.status().is_server_error() { - if res.status().as_u16() == 404 { + let status = res.status(); + let _ = res.into_body(); + if status.as_u16() == 404 { support_versions.push(ProtocolVersion::V1); return Ok(support_versions); } @@ -429,7 +431,7 @@ pub(super) fn get_supported_protocol_versions( ProtocolVersionError, "Failed to detect server's line protocol version, settings url: {}, status code: {}.", settings_url, - res.status() + status )); } else { res diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index ece65649..75200fa9 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -1112,7 +1112,6 @@ impl Buffer { /// - Failed to get dimension sizes /// - Column name validation fails /// - Protocol version v1 is used (arrays require v2+) - /// - Array contains unsupported element types (non-f64) #[allow(private_bounds)] pub fn column_arr<'a, N, T, D>(&mut self, name: N, view: &T) -> Result<&mut Self> where @@ -1145,7 +1144,7 @@ impl Buffer { )); } - let array_buf_size = get_and_check_array_bytes_size(view)?; + let array_buf_size = check_and_get_array_bytes_size(view)?; self.write_column_key(name)?; // binary format flag '=' self.output.push(b'='); @@ -3001,18 +3000,11 @@ impl Sender { self.default_protocol_version } - #[cfg(feature = "ilp-over-http")] - #[cfg(test)] - pub(crate) fn support_protocol_versions(&self) -> Option> { - self.supported_protocol_versions.clone() - } - #[cfg(feature = "ilp-over-http")] #[inline(always)] fn check_protocol_version(&self, version: ProtocolVersion) -> Result<()> { match &self.handler { ProtocolHandler::Socket(_) => Ok(()), - #[cfg(feature = "ilp-over-http")] ProtocolHandler::Http(_) => { match self.supported_protocol_versions { Some(ref supported_line_protocols) => { @@ -3024,6 +3016,7 @@ impl Sender { "Line protocol version {} is not supported by current QuestDB Server", version)) } } + // `None` implies user set protocol_version explicitly None => Ok(()), } } @@ -3041,7 +3034,7 @@ mod timestamp; #[cfg(feature = "ilp-over-http")] mod http; -use crate::ingress::ndarr::{get_and_check_array_bytes_size, MAX_ARRAY_DIM_LEN}; +use crate::ingress::ndarr::{check_and_get_array_bytes_size, MAX_ARRAY_DIM_LEN}; #[cfg(feature = "ilp-over-http")] use http::*; diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 6cb6e404..e5005ece 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -38,9 +38,8 @@ pub(crate) fn write_array_data, T>( where T: ArrayElement, { - // First optimization path: write contiguous memory directly - // When working with contiguous layout. Benchmark shows `copy_from_slice` has better performance than - // `std::ptr::copy_nonoverlapping` on both arm(Macos) and x86(Linux) platform. + // When working with contiguous layout, benchmark shows `copy_from_slice` has better performance than + // `std::ptr::copy_nonoverlapping` on both Arm(Macos) and x86(Linux) platform. // This may because `copy_from_slice` benefits more from compiler. if let Some(contiguous) = array.as_slice() { let bytes = unsafe { @@ -69,7 +68,6 @@ where return Ok(()); } - // Fallback path: non-contiguous memory handling // For non-contiguous memory layouts, direct raw pointer operations are preferred. let elem_size = size_of::(); let mut total_len = 0; @@ -94,7 +92,7 @@ where Ok(()) } -pub(crate) fn get_and_check_array_bytes_size, T>( +pub(crate) fn check_and_get_array_bytes_size, T>( array: &A, ) -> Result where @@ -135,10 +133,10 @@ impl ArrayElementSealed for f64 { } } -/// A view into a multi-dimensional array with custom memory strides. +/// A view into a multidimensional array with custom memory strides. // TODO: We are currently evaluating whether to use StrideArrayView or ndarray's view. // Current benchmarks show that StrideArrayView's iter implementation underperforms(2x) -// compared to ndarray's view. If we proceed with StrideArrayView, we need to +// compared to ndarray's. If we proceed with StrideArrayView, we need to // optimize the iter traversal pattern #[derive(Debug)] pub struct StrideArrayView<'a, T> { diff --git a/questdb-rs/src/tests/http.rs b/questdb-rs/src/tests/http.rs index e17a1ba1..42af1eae 100644 --- a/questdb-rs/src/tests/http.rs +++ b/questdb-rs/src/tests/http.rs @@ -35,7 +35,9 @@ use std::time::Duration; fn test_two_lines( #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { - let mut buffer = Buffer::new(version); + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().protocol_version(version)?.build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("sym", "bol")? @@ -48,20 +50,8 @@ fn test_two_lines( .at_now()?; let buffer2 = buffer.clone(); - let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); - let sender_builder = server.lsb_http(); - let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; - let req = server.recv_http_q()?; - assert_eq!(req.method(), "GET"); - assert_eq!(req.path(), "/settings"); - assert_eq!( - req.header("user-agent"), - Some(concat!("questdb/rust/", env!("CARGO_PKG_VERSION"))) - ); - server.send_settings_response()?; - let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); assert_eq!(req.path(), "/write?precision=n"); @@ -76,7 +66,6 @@ fn test_two_lines( Ok(()) }); - let mut sender = sender_builder.build()?; let res = sender.flush(&mut buffer); server_thread.join().unwrap()?; @@ -92,29 +81,18 @@ fn test_two_lines( fn test_text_plain_error( #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { - let mut buffer = Buffer::new(version); + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().protocol_version(version)?.build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("sym", "bol")? .column_f64("x", 1.0)? .at_now()?; buffer.table("test")?.column_f64("sym", 2.0)?.at_now()?; - - let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); - let sender_builder = server.lsb_http(); - let buffer2 = buffer.clone(); let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; - let req = server.recv_http_q()?; - assert_eq!(req.method(), "GET"); - assert_eq!(req.path(), "/settings"); - assert_eq!( - req.header("user-agent"), - Some(concat!("questdb/rust/", env!("CARGO_PKG_VERSION"))) - ); - server.send_settings_response()?; - let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); assert_eq!(req.path(), "/write?precision=n"); @@ -130,7 +108,6 @@ fn test_text_plain_error( Ok(()) }); - let mut sender = sender_builder.build()?; let res = sender.flush(&mut buffer); server_thread.join().unwrap()?; @@ -149,7 +126,9 @@ fn test_text_plain_error( fn test_bad_json_error( #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { - let mut buffer = Buffer::new(version); + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().protocol_version(version)?.build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("sym", "bol")? @@ -157,21 +136,9 @@ fn test_bad_json_error( .at_now()?; buffer.table("test")?.column_f64("sym", 2.0)?.at_now()?; - let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); - let sender_builder = server.lsb_http(); - let buffer2 = buffer.clone(); let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; - let req = server.recv_http_q()?; - assert_eq!(req.method(), "GET"); - assert_eq!(req.path(), "/settings"); - assert_eq!( - req.header("user-agent"), - Some(concat!("questdb/rust/", env!("CARGO_PKG_VERSION"))) - ); - server.send_settings_response()?; - let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); assert_eq!(req.path(), "/write?precision=n"); @@ -188,7 +155,6 @@ fn test_bad_json_error( Ok(()) }); - let mut sender = sender_builder.build()?; let res = sender.flush_and_keep(&buffer); server_thread.join().unwrap()?; @@ -208,7 +174,9 @@ fn test_bad_json_error( fn test_json_error( #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { - let mut buffer = Buffer::new(version); + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().protocol_version(version)?.build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("sym", "bol")? @@ -216,21 +184,9 @@ fn test_json_error( .at_now()?; buffer.table("test")?.column_f64("sym", 2.0)?.at_now()?; - let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); - let sender_builder = server.lsb_http(); - let buffer2 = buffer.clone(); let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; - let req = server.recv_http_q()?; - assert_eq!(req.method(), "GET"); - assert_eq!(req.path(), "/settings"); - assert_eq!( - req.header("user-agent"), - Some(concat!("questdb/rust/", env!("CARGO_PKG_VERSION"))) - ); - server.send_settings_response()?; - let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); assert_eq!(req.path(), "/write?precision=n"); @@ -250,7 +206,7 @@ fn test_json_error( Ok(()) }); - let res = sender_builder.build()?.flush_and_keep(&buffer); + let res = sender.flush_and_keep(&buffer); server_thread.join().unwrap()?; @@ -269,16 +225,15 @@ fn test_json_error( fn test_no_connection( #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { - let mut buffer = Buffer::new(version); + let mut sender = SenderBuilder::new(Protocol::Http, "127.0.0.1", 1) + .protocol_version(version)? + .build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("sym", "bol")? .column_f64("x", 1.0)? .at_now()?; - - let mut sender = SenderBuilder::new(Protocol::Http, "127.0.0.1", 1) - .protocol_version(ProtocolVersion::V2)? - .build()?; let res = sender.flush_and_keep(&buffer); assert!(res.is_err()); let err = res.unwrap_err(); @@ -293,28 +248,18 @@ fn test_no_connection( fn test_old_server_without_ilp_http_support( #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { - let mut buffer = Buffer::new(version); + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().protocol_version(version)?.build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("sym", "bol")? .column_f64("x", 1.0)? .at_now()?; - let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); - let sender_builder = server.lsb_http(); - let buffer2 = buffer.clone(); let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; - let req = server.recv_http_q()?; - assert_eq!(req.method(), "GET"); - assert_eq!(req.path(), "/settings"); - assert_eq!( - req.header("user-agent"), - Some(concat!("questdb/rust/", env!("CARGO_PKG_VERSION"))) - ); - server.send_settings_response()?; - let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); assert_eq!(req.path(), "/write?precision=n"); @@ -330,7 +275,7 @@ fn test_old_server_without_ilp_http_support( Ok(()) }); - let res = sender_builder.build()?.flush_and_keep(&buffer); + let res = sender.flush_and_keep(&buffer); server_thread.join().unwrap()?; @@ -349,33 +294,24 @@ fn test_old_server_without_ilp_http_support( fn test_http_basic_auth( #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { - let mut buffer = Buffer::new(version); + let mut server = MockServer::new()?; + let mut sender = server + .lsb_http() + .protocol_version(version)? + .username("Aladdin")? + .password("OpenSesame")? + .build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("sym", "bol")? .column_f64("x", 1.0)? .at_now()?; - let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); - let sender_builder = server - .lsb_http() - .username("Aladdin")? - .password("OpenSesame")?; - let buffer2 = buffer.clone(); let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; let req = server.recv_http_q()?; - assert_eq!(req.method(), "GET"); - assert_eq!(req.path(), "/settings"); - assert_eq!( - req.header("user-agent"), - Some(concat!("questdb/rust/", env!("CARGO_PKG_VERSION"))) - ); - server.send_settings_response()?; - - let req = server.recv_http_q()?; - assert_eq!(req.method(), "POST"); assert_eq!(req.path(), "/write?precision=n"); assert_eq!( @@ -389,7 +325,7 @@ fn test_http_basic_auth( Ok(()) }); - let res = sender_builder.build()?.flush(&mut buffer); + let res = sender.flush(&mut buffer); server_thread.join().unwrap()?; @@ -404,28 +340,18 @@ fn test_http_basic_auth( fn test_unauthenticated( #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { - let mut buffer = Buffer::new(version); + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().protocol_version(version)?.build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("sym", "bol")? .column_f64("x", 1.0)? .at_now()?; - let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); - let sender_builder = server.lsb_http(); - let buffer2 = buffer.clone(); let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; - let req = server.recv_http_q()?; - assert_eq!(req.method(), "GET"); - assert_eq!(req.path(), "/settings"); - assert_eq!( - req.header("user-agent"), - Some(concat!("questdb/rust/", env!("CARGO_PKG_VERSION"))) - ); - server.send_settings_response()?; - let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); assert_eq!(req.path(), "/write?precision=n"); @@ -441,7 +367,7 @@ fn test_unauthenticated( Ok(()) }); - let res = sender_builder.build()?.flush(&mut buffer); + let res = sender.flush(&mut buffer); server_thread.join().unwrap()?; @@ -462,28 +388,18 @@ fn test_unauthenticated( fn test_token_auth( #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { - let mut buffer = Buffer::new(version); + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().protocol_version(version)?.token("0123456789")?.build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("sym", "bol")? .column_f64("x", 1.0)? .at_now()?; - let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); - let sender_builder = server.lsb_http().token("0123456789")?; - let buffer2 = buffer.clone(); let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; - let req = server.recv_http_q()?; - assert_eq!(req.method(), "GET"); - assert_eq!(req.path(), "/settings"); - assert_eq!( - req.header("user-agent"), - Some(concat!("questdb/rust/", env!("CARGO_PKG_VERSION"))) - ); - server.send_settings_response()?; - let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); assert_eq!(req.path(), "/write?precision=n"); @@ -495,7 +411,7 @@ fn test_token_auth( Ok(()) }); - let res = sender_builder.build()?.flush(&mut buffer); + let res = sender.flush(&mut buffer); server_thread.join().unwrap()?; @@ -508,7 +424,14 @@ fn test_token_auth( fn test_request_timeout( #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { - let mut buffer = Buffer::new(version); + let server = MockServer::new()?; + let request_timeout = Duration::from_millis(50); + let mut sender = server + .lsb_http() + .protocol_version(version)? + .request_timeout(request_timeout)? + .build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("sym", "bol")? @@ -516,15 +439,7 @@ fn test_request_timeout( .at_now()?; // Here we use a mock (tcp) server instead and don't send a response back. - let server = MockServer::new()?.configure_settings_response(&[1, 2]); - - let request_timeout = Duration::from_millis(50); let time_start = std::time::Instant::now(); - let mut sender = server - .lsb_http() - .protocol_version(ProtocolVersion::V2)? - .request_timeout(request_timeout)? - .build()?; let res = sender.flush_and_keep(&buffer); let time_elapsed = time_start.elapsed(); assert!(res.is_err()); @@ -541,22 +456,20 @@ fn test_tls( ) -> TestResult { let mut ca_path = certs_dir(); ca_path.push("server_rootCA.pem"); + let mut server = MockServer::new()?; + let mut sender = server + .lsb_https() + .tls_roots(ca_path)? + .protocol_version(version)? + .build()?; - let mut buffer = Buffer::new(version); + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("t1", "v1")? .column_f64("f1", 0.5)? .at(TimestampNanos::new(10000000))?; let buffer2 = buffer.clone(); - - let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); - let mut sender = server - .lsb_https() - .tls_roots(ca_path)? - .protocol_version(ProtocolVersion::V2)? - .build()?; - let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept_tls_sync()?; let req = server.recv_http_q()?; @@ -583,24 +496,17 @@ fn test_tls( fn test_user_agent( #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { - let mut buffer = Buffer::new(version); + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().user_agent("wallabies/1.2.99")?.protocol_version(version)?.build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("t1", "v1")? .column_f64("f1", 0.5)? .at(TimestampNanos::new(10000000))?; let buffer2 = buffer.clone(); - - let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); - let sender_builder = server.lsb_http().user_agent("wallabies/1.2.99")?; - let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; - let req = server.recv_http_q()?; - assert_eq!(req.method(), "GET"); - assert_eq!(req.path(), "/settings"); - server.send_settings_response()?; - let req = server.recv_http_q()?; assert_eq!(req.header("user-agent"), Some("wallabies/1.2.99")); assert_eq!(req.body(), buffer2.as_bytes()); @@ -610,7 +516,7 @@ fn test_user_agent( Ok(()) }); - let res = sender_builder.build()?.flush_and_keep(&buffer); + let res = sender.flush_and_keep(&buffer); server_thread.join().unwrap()?; @@ -625,29 +531,17 @@ fn test_two_retries( #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { // Note: This also tests that the _same_ connection is being reused, i.e. tests keepalive. - - let mut buffer = Buffer::new(version); + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().protocol_version(version)?.retry_timeout(Duration::from_secs(30))?.build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("t1", "v1")? .column_f64("f1", 0.5)? .at(TimestampNanos::new(10000000))?; let buffer2 = buffer.clone(); - - let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); - let sender_builder = server.lsb_http().retry_timeout(Duration::from_secs(30))?; - let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; - let req = server.recv_http_q()?; - assert_eq!(req.method(), "GET"); - assert_eq!(req.path(), "/settings"); - assert_eq!( - req.header("user-agent"), - Some(concat!("questdb/rust/", env!("CARGO_PKG_VERSION"))) - ); - server.send_settings_response()?; - let req = server.recv_http_q()?; assert_eq!(req.body(), buffer2.as_bytes()); @@ -682,7 +576,7 @@ fn test_two_retries( Ok(()) }); - let res = sender_builder.build()?.flush_and_keep(&buffer); + let res = sender.flush_and_keep(&buffer); server_thread.join().unwrap()?; @@ -696,7 +590,13 @@ fn test_two_retries( fn test_one_retry( #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { - let mut buffer = Buffer::new(version); + let mut server = MockServer::new()?; + let mut sender = server + .lsb_http() + .retry_timeout(Duration::from_millis(19))? + .protocol_version(version)? + .build()?; + let mut buffer = sender.new_buffer(); buffer .table("test")? .symbol("t1", "v1")? @@ -704,14 +604,6 @@ fn test_one_retry( .at(TimestampNanos::new(10000000))?; let buffer2 = buffer.clone(); - let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); - let mut sender = server - .lsb_http() - .retry_timeout(Duration::from_millis(19))? - .protocol_version(ProtocolVersion::V2) - .unwrap() - .build()?; - let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; let req = server.recv_http_q()?; @@ -763,8 +655,10 @@ fn test_one_retry( fn test_transactional( #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { + let mut server = MockServer::new()?; + let mut sender = server.lsb_http().protocol_version(version)?.build()?; // A buffer with a two tables. - let mut buffer1 = Buffer::new(version); + let mut buffer1 = sender.new_buffer(); buffer1 .table("tab1")? .symbol("t1", "v1")? @@ -778,7 +672,7 @@ fn test_transactional( assert!(!buffer1.transactional()); // A buffer with a single table. - let mut buffer2 = Buffer::new(version); + let mut buffer2 = sender.new_buffer(); buffer2 .table("test")? .symbol("t1", "v1")? @@ -787,20 +681,8 @@ fn test_transactional( let buffer3 = buffer2.clone(); assert!(buffer2.transactional()); - let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); - let sender_builder = server.lsb_http(); - let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; - let req = server.recv_http_q()?; - assert_eq!(req.method(), "GET"); - assert_eq!(req.path(), "/settings"); - assert_eq!( - req.header("user-agent"), - Some(concat!("questdb/rust/", env!("CARGO_PKG_VERSION"))) - ); - server.send_settings_response()?; - let req = server.recv_http_q()?; assert_eq!(req.body(), buffer3.as_bytes()); @@ -809,8 +691,6 @@ fn test_transactional( Ok(()) }); - let mut sender = sender_builder.build()?; - let res = sender.flush_and_keep_with_flags(&buffer1, true); assert!(res.is_err()); let err = res.unwrap_err(); @@ -831,60 +711,51 @@ fn test_transactional( Ok(()) } -#[test] -fn test_sender_protocol_version() -> TestResult { - let mut server = MockServer::new()?.configure_settings_response(&[1, 2]); +fn _test_sender_auto_detect_protocol_version(supported_versions: Option>, expect_version: ProtocolVersion) -> TestResult { + let supported_versions1 = supported_versions.clone(); + let mut server = MockServer::new()?.configure_settings_response(supported_versions.as_deref().unwrap_or(&[])); let sender_builder = server.lsb_http(); - let server_thread = std::thread::spawn(move || -> io::Result<()> { - server.accept()?; - let req = server.recv_http_q()?; - assert_eq!(req.method(), "GET"); - assert_eq!(req.path(), "/settings"); - assert_eq!( - req.header("user-agent"), - Some(concat!("questdb/rust/", env!("CARGO_PKG_VERSION"))) - ); - server.send_settings_response()?; - Ok(()) - }); - let sender = sender_builder.build()?; - assert_eq!(sender.default_protocol_version(), ProtocolVersion::V2); - assert_eq!( - sender.support_protocol_versions().unwrap(), - vec![ProtocolVersion::V1, ProtocolVersion::V2] - ); - server_thread.join().unwrap()?; - Ok(()) -} -#[test] -fn test_sender_protocol_version_old_server1() -> TestResult { - let mut server = MockServer::new()?.configure_settings_response(&[]); - let sender_builder = server.lsb_http(); let server_thread = std::thread::spawn(move || -> io::Result<()> { server.accept()?; let req = server.recv_http_q()?; assert_eq!(req.method(), "GET"); assert_eq!(req.path(), "/settings"); - assert_eq!( - req.header("user-agent"), - Some(concat!("questdb/rust/", env!("CARGO_PKG_VERSION"))) - ); - server.send_settings_response()?; + match supported_versions1 { + None => server.send_http_response_q( + HttpResponse::empty() + .with_status(404, "Not Found") + .with_header("content-type", "text/plain") + .with_body_str("Not Found"), + )?, + Some(_) => server.send_settings_response()? + } + let exp = &[ + b"test,t1=v1 ", + crate::tests::sender::f64_to_bytes("f1", 0.5, expect_version).as_slice(), + b" 10000000\n" + ].concat(); + let req = server.recv_http_q()?; + assert_eq!(req.body(), exp); + server.send_http_response_q(HttpResponse::empty())?; Ok(()) }); - let sender = sender_builder.build()?; - assert_eq!(sender.default_protocol_version(), ProtocolVersion::V1); - assert_eq!( - sender.support_protocol_versions(), - Some(vec![ProtocolVersion::V1]) - ); + + let mut sender = sender_builder.build()?; + assert_eq!(sender.default_protocol_version(), expect_version); + let mut buffer = sender.new_buffer(); + buffer + .table("test")? + .symbol("t1", "v1")? + .column_f64("f1", 0.5)? + .at(TimestampNanos::new(10000000))?; + let res = sender.flush(&mut buffer); server_thread.join().unwrap()?; + res?; Ok(()) } - #[test] -fn test_sender_protocol_version_old_server2() -> TestResult { +fn test_sender_protocol_version_old_server21() -> TestResult { let mut server = MockServer::new()?.configure_settings_response(&[]); let sender_builder = server.lsb_http(); let server_thread = std::thread::spawn(move || -> io::Result<()> { @@ -899,13 +770,35 @@ fn test_sender_protocol_version_old_server2() -> TestResult { }); let sender = sender_builder.build()?; assert_eq!(sender.default_protocol_version(), ProtocolVersion::V1); - assert_eq!( - sender.support_protocol_versions(), - Some(vec![ProtocolVersion::V1]) - ); server_thread.join().unwrap()?; Ok(()) } +#[test] +fn test_sender_protocol_version() -> TestResult { + _test_sender_auto_detect_protocol_version(Some(vec![1, 2]), ProtocolVersion::V2) +} + +#[test] +fn test_sender_protocol_version_old_server1() -> TestResult { + _test_sender_auto_detect_protocol_version(Some(vec![]), ProtocolVersion::V1) +} + +// todo test hang +// #[test] +// fn test_sender_protocol_version_old_server2() -> TestResult { +// _test_sender_auto_detect_protocol_version(None, ProtocolVersion::V1) +// } + +#[test] +fn test_sender_protocol_version_only_v1() -> TestResult { + _test_sender_auto_detect_protocol_version(Some(vec![1]), ProtocolVersion::V1) +} + +#[test] +fn test_sender_protocol_version_only_v2() -> TestResult { + _test_sender_auto_detect_protocol_version(Some(vec![2]), ProtocolVersion::V2) +} + #[test] fn test_sender_protocol_version_unsupported_client() -> TestResult { @@ -926,35 +819,7 @@ fn test_sender_protocol_version_unsupported_client() -> TestResult { } #[test] -fn test_sender_explicit_set_protocol_version_v2() -> TestResult { - let mut server = MockServer::new()?.configure_settings_response(&[]); - let mut sender = server - .lsb_http() - .protocol_version(ProtocolVersion::V2)? - .build()?; - let mut buffer = sender.new_buffer(); - buffer - .table("test")? - .symbol("sym", "bol")? - .column_f64("x", 1.0)? - .at_now()?; - let buffer2 = buffer.clone(); - - let server_thread = std::thread::spawn(move || -> io::Result<()> { - server.accept()?; - let req = server.recv_http_q()?; - assert_eq!(req.body(), buffer2.as_bytes()); - server.send_http_response_q(HttpResponse::empty())?; - Ok(()) - }); - - sender.flush(&mut buffer)?; - server_thread.join().unwrap()?; - Ok(()) -} - -#[test] -fn test_sender_protocol_version1_not_support_array() -> TestResult { +fn test_buffer_protocol_version1_not_support_array() -> TestResult { let mut buffer = Buffer::new(ProtocolVersion::V1); let res = buffer .table("test")? diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index b40cce8e..ae2632c2 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -146,7 +146,7 @@ fn test_array_f64_basic() -> TestResult { #[cfg(feature = "ndarray")] #[test] -fn test_array_f64_from_ndarray() -> TestResult { +fn test_array_f64_for_ndarray() -> TestResult { let mut server = MockServer::new()?; let mut sender = server .lsb_tcp() @@ -677,7 +677,7 @@ fn bad_uppercase_addr() { assert!(err.msg() == "Missing \"addr\" parameter in config string"); } -fn f64_to_bytes(name: &str, value: f64, version: ProtocolVersion) -> Vec { +pub(crate) fn f64_to_bytes(name: &str, value: f64, version: ProtocolVersion) -> Vec { let mut buf = Vec::new(); buf.extend_from_slice(name.as_bytes()); buf.push(b'='); From 01bfd914d4544e6ca00f098c995ceba8944d5768 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Thu, 15 May 2025 16:10:52 +0100 Subject: [PATCH 077/119] clearing all tables before each test run --- system_test/fixture.py | 20 ++++++++++++++++++++ system_test/test.py | 1 + 2 files changed, 21 insertions(+) diff --git a/system_test/fixture.py b/system_test/fixture.py index d25aa335..ec145e44 100644 --- a/system_test/fixture.py +++ b/system_test/fixture.py @@ -424,6 +424,26 @@ def check_table(): f'\nQuestDB log:\n') self.print_log() raise toe + + def show_tables(self): + """Return a list of tables in the database.""" + sql_query = "show tables" + try: + resp = self.http_sql_query(sql_query) + return [row[0] for row in resp['dataset']] + except QueryError as qe: + raise qe + + def drop_table(self, table_name): + self.http_sql_query(f"drop table '{table_name}'") + + def drop_all_tables(self): + """Drop all tables in the database.""" + all_tables = self.show_tables() + # if all_tables: + # print(f'Dropping {len(all_tables)} tables: {all_tables!r}') + for table_name in all_tables: + self.drop_table(table_name) def __enter__(self): self.start() diff --git a/system_test/test.py b/system_test/test.py index f9fb5a98..351a0327 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -1137,6 +1137,7 @@ def run_with_fixtures(args): TLS_PROXY_FIXTURE = TlsProxyFixture(port_to_proxy) TLS_PROXY_FIXTURE.start() + QDB_FIXTURE.drop_all_tables() test_prog = unittest.TestProgram(exit=False) if not test_prog.result.wasSuccessful(): sys.exit(1) From 8c1d9b94f5abc75afa4a0c87fc1353c7337cde24 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Thu, 15 May 2025 16:18:26 +0100 Subject: [PATCH 078/119] fixed broken C example --- examples/line_sender_c_example_auth.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/line_sender_c_example_auth.c b/examples/line_sender_c_example_auth.c index 11823162..5f4fb8c8 100644 --- a/examples/line_sender_c_example_auth.c +++ b/examples/line_sender_c_example_auth.c @@ -11,7 +11,7 @@ static bool example(const char* host, const char* port) line_sender_buffer* buffer = NULL; char* conf_str = concat( "tcp::addr=", host, ":", port, ";" - "protocol_version=2" + "protocol_version=2;" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" From 36f4bd5b6364003def26a33bfccdc84e29064fee Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Thu, 15 May 2025 17:06:58 +0100 Subject: [PATCH 079/119] minor fixes --- proj.py | 5 +++-- questdb-rs/src/tests/http.rs | 34 +++++++++++++++++++++++++--------- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/proj.py b/proj.py index bd36a449..609df248 100755 --- a/proj.py +++ b/proj.py @@ -176,8 +176,8 @@ def build_latest_questdb(branch='master'): if not questdb_path.exists(): _run('git', 'clone', 'https://github.com/questdb/questdb.git') _run('git', 'fetch', 'origin', branch, cwd=questdb_path) - _run('git', 'switch', branch=questdb_path) - _run('git', 'pull', 'origin', branch=questdb_path) + _run('git', 'switch', branch, cwd=questdb_path) + _run('git', 'pull', 'origin', branch, cwd=questdb_path) _run('git', 'submodule', 'update', '--init', '--recursive', cwd=questdb_path) _run('mvn', 'clean', 'package', '-DskipTests', '-Pbuild-web-console', cwd=questdb_path) @@ -190,6 +190,7 @@ def test_vs_latest_questdb(): _run( sys.executable, PROJ_ROOT / 'system_test' / 'test.py', + 'run', '--repo', PROJ_ROOT / 'questdb', '-v') diff --git a/questdb-rs/src/tests/http.rs b/questdb-rs/src/tests/http.rs index 42af1eae..1908662c 100644 --- a/questdb-rs/src/tests/http.rs +++ b/questdb-rs/src/tests/http.rs @@ -389,7 +389,11 @@ fn test_token_auth( #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { let mut server = MockServer::new()?; - let mut sender = server.lsb_http().protocol_version(version)?.token("0123456789")?.build()?; + let mut sender = server + .lsb_http() + .protocol_version(version)? + .token("0123456789")? + .build()?; let mut buffer = sender.new_buffer(); buffer .table("test")? @@ -497,7 +501,11 @@ fn test_user_agent( #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, ) -> TestResult { let mut server = MockServer::new()?; - let mut sender = server.lsb_http().user_agent("wallabies/1.2.99")?.protocol_version(version)?.build()?; + let mut sender = server + .lsb_http() + .user_agent("wallabies/1.2.99")? + .protocol_version(version)? + .build()?; let mut buffer = sender.new_buffer(); buffer .table("test")? @@ -532,7 +540,11 @@ fn test_two_retries( ) -> TestResult { // Note: This also tests that the _same_ connection is being reused, i.e. tests keepalive. let mut server = MockServer::new()?; - let mut sender = server.lsb_http().protocol_version(version)?.retry_timeout(Duration::from_secs(30))?.build()?; + let mut sender = server + .lsb_http() + .protocol_version(version)? + .retry_timeout(Duration::from_secs(30))? + .build()?; let mut buffer = sender.new_buffer(); buffer .table("test")? @@ -711,9 +723,13 @@ fn test_transactional( Ok(()) } -fn _test_sender_auto_detect_protocol_version(supported_versions: Option>, expect_version: ProtocolVersion) -> TestResult { +fn _test_sender_auto_detect_protocol_version( + supported_versions: Option>, + expect_version: ProtocolVersion, +) -> TestResult { let supported_versions1 = supported_versions.clone(); - let mut server = MockServer::new()?.configure_settings_response(supported_versions.as_deref().unwrap_or(&[])); + let mut server = MockServer::new()? + .configure_settings_response(supported_versions.as_deref().unwrap_or(&[])); let sender_builder = server.lsb_http(); let server_thread = std::thread::spawn(move || -> io::Result<()> { @@ -728,13 +744,14 @@ fn _test_sender_auto_detect_protocol_version(supported_versions: Option .with_header("content-type", "text/plain") .with_body_str("Not Found"), )?, - Some(_) => server.send_settings_response()? + Some(_) => server.send_settings_response()?, } let exp = &[ b"test,t1=v1 ", crate::tests::sender::f64_to_bytes("f1", 0.5, expect_version).as_slice(), - b" 10000000\n" - ].concat(); + b" 10000000\n", + ] + .concat(); let req = server.recv_http_q()?; assert_eq!(req.body(), exp); server.send_http_response_q(HttpResponse::empty())?; @@ -799,7 +816,6 @@ fn test_sender_protocol_version_only_v2() -> TestResult { _test_sender_auto_detect_protocol_version(Some(vec![2]), ProtocolVersion::V2) } - #[test] fn test_sender_protocol_version_unsupported_client() -> TestResult { let mut server = MockServer::new()?.configure_settings_response(&[3, 4]); From 339262708cae09250febe5bb705f01c62c524af1 Mon Sep 17 00:00:00 2001 From: victor Date: Fri, 16 May 2025 09:50:39 +0800 Subject: [PATCH 080/119] fix abnormal tests hang. --- questdb-rs/src/ingress/http.rs | 2 +- questdb-rs/src/tests/http.rs | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/questdb-rs/src/ingress/http.rs b/questdb-rs/src/ingress/http.rs index 9d19a0fa..a9c281dd 100644 --- a/questdb-rs/src/ingress/http.rs +++ b/questdb-rs/src/ingress/http.rs @@ -422,7 +422,7 @@ pub(super) fn get_supported_protocol_versions( Ok(res) => { if res.status().is_client_error() || res.status().is_server_error() { let status = res.status(); - let _ = res.into_body(); + _ = res.into_body().read_to_vec(); if status.as_u16() == 404 { support_versions.push(ProtocolVersion::V1); return Ok(support_versions); diff --git a/questdb-rs/src/tests/http.rs b/questdb-rs/src/tests/http.rs index 1908662c..e6f3ffbf 100644 --- a/questdb-rs/src/tests/http.rs +++ b/questdb-rs/src/tests/http.rs @@ -790,34 +790,34 @@ fn test_sender_protocol_version_old_server21() -> TestResult { server_thread.join().unwrap()?; Ok(()) } + #[test] -fn test_sender_protocol_version() -> TestResult { +fn test_sender_auto_protocol_version_basic() -> TestResult { _test_sender_auto_detect_protocol_version(Some(vec![1, 2]), ProtocolVersion::V2) } #[test] -fn test_sender_protocol_version_old_server1() -> TestResult { +fn test_sender_auto_protocol_version_old_server1() -> TestResult { _test_sender_auto_detect_protocol_version(Some(vec![]), ProtocolVersion::V1) } -// todo test hang -// #[test] -// fn test_sender_protocol_version_old_server2() -> TestResult { -// _test_sender_auto_detect_protocol_version(None, ProtocolVersion::V1) -// } +#[test] +fn test_sender_auto_protocol_version_old_server2() -> TestResult { + _test_sender_auto_detect_protocol_version(None, ProtocolVersion::V1) +} #[test] -fn test_sender_protocol_version_only_v1() -> TestResult { +fn test_sender_auto_protocol_version_only_v1() -> TestResult { _test_sender_auto_detect_protocol_version(Some(vec![1]), ProtocolVersion::V1) } #[test] -fn test_sender_protocol_version_only_v2() -> TestResult { +fn test_sender_auto_protocol_version_only_v2() -> TestResult { _test_sender_auto_detect_protocol_version(Some(vec![2]), ProtocolVersion::V2) } #[test] -fn test_sender_protocol_version_unsupported_client() -> TestResult { +fn test_sender_auto_protocol_version_unsupported_client() -> TestResult { let mut server = MockServer::new()?.configure_settings_response(&[3, 4]); let sender_builder = server.lsb_http(); let server_thread = std::thread::spawn(move || -> io::Result<()> { From d4d6bb1b267a0c6d4fb925c81a13b4b39f1a1532 Mon Sep 17 00:00:00 2001 From: victor Date: Fri, 16 May 2025 11:06:54 +0800 Subject: [PATCH 081/119] enrich tests --- cpp_test/test_line_sender.cpp | 86 +++++++++++++++++++------ examples/line_sender_c_example_array.c | 4 +- include/questdb/ingress/line_sender.h | 2 +- include/questdb/ingress/line_sender.hpp | 4 +- questdb-rs-ffi/src/lib.rs | 2 +- questdb-rs/src/ingress/mod.rs | 16 +---- questdb-rs/src/ingress/ndarr.rs | 40 +++++++++--- questdb-rs/src/tests/http.rs | 19 ------ questdb-rs/src/tests/ndarr.rs | 18 +++--- system_test/questdb_line_sender.py | 6 +- system_test/test.py | 58 ++++++++++++++--- 11 files changed, 166 insertions(+), 89 deletions(-) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 92654d63..8424c133 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -97,14 +97,14 @@ std::string& push_double_arr_to_buffer( std::string& buffer, std::array data, size_t rank, - uintptr_t* shapes) + uintptr_t* shape) { buffer.push_back(14); buffer.push_back(10); buffer.push_back(static_cast(rank)); for (size_t i = 0; i < rank; ++i) buffer.append( - reinterpret_cast(&shapes[i]), sizeof(uint32_t)); + reinterpret_cast(&shape[i]), sizeof(uint32_t)); buffer.append( reinterpret_cast(data.data()), data.size() * sizeof(double)); @@ -162,7 +162,7 @@ TEST_CASE("line_sender c api basics") line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("a1"); // 3D array of doubles size_t rank = 3; - uintptr_t shapes[] = {2, 3, 2}; + uintptr_t shape[] = {2, 3, 2}; intptr_t strides[] = {48, 16, 8}; std::array arr_data = { 48123.5, @@ -177,15 +177,16 @@ TEST_CASE("line_sender c api basics") 2.7, 48121.5, 4.3}; - CHECK(::line_sender_buffer_column_f64_arr( - buffer, - arr_name, - rank, - shapes, - strides, - reinterpret_cast(arr_data.data()), - sizeof(arr_data), - &err)); + CHECK( + ::line_sender_buffer_column_f64_arr( + buffer, + arr_name, + rank, + shape, + strides, + reinterpret_cast(arr_data.data()), + sizeof(arr_data), + &err)); CHECK(::line_sender_buffer_at_nanos(buffer, 10000000, &err)); CHECK(server.recv() == 0); CHECK(::line_sender_buffer_size(buffer) == 150); @@ -194,8 +195,7 @@ TEST_CASE("line_sender c api basics") CHECK(server.recv() == 1); std::string expect{"test,t1=v1 f1=="}; push_double_to_buffer(expect, 0.5).append(",a1=="); - push_double_arr_to_buffer(expect, arr_data, 3, shapes) - .append(" 10000000\n"); + push_double_arr_to_buffer(expect, arr_data, 3, shape).append(" 10000000\n"); CHECK(server.msgs(0) == expect); } @@ -246,18 +246,38 @@ TEST_CASE("line_sender c++ api basics") CHECK(server.recv() == 0); questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); + // 3D array of doubles + size_t rank = 3; + std::vector shape{2, 3, 2}; + std::vector strides{48, 16, 8}; + std::array arr_data = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; buffer.table("test") .symbol("t1", "v1") .symbol("t2", "") .column("f1", 0.5) + .column("a1", rank, shape, strides, arr_data) .at(questdb::ingress::timestamp_nanos{10000000}); CHECK(server.recv() == 0); - CHECK(buffer.size() == 38); + CHECK(buffer.size() == 154); sender.flush(buffer); CHECK(server.recv() == 1); std::string expect{"test,t1=v1,t2= f1=="}; - push_double_to_buffer(expect, 0.5).append(" 10000000\n"); + push_double_to_buffer(expect, 0.5).append(",a1=="); + push_double_arr_to_buffer(expect, arr_data, 3, shape.data()) + .append(" 10000000\n"); CHECK(server.msgs(0) == expect); } @@ -943,7 +963,7 @@ TEST_CASE("HTTP basics") questdb::ingress::line_sender_error); } -TEST_CASE("line sender protocol version v1") +TEST_CASE("line sender protocol version default v1 for tcp") { questdb::ingress::test::mock_server server; questdb::ingress::line_sender sender{ @@ -961,6 +981,7 @@ TEST_CASE("line sender protocol version v1") .column("f1", 0.5) .at(questdb::ingress::timestamp_nanos{10000000}); + CHECK(sender.default_protocol_version() == protocol_version_1); CHECK(server.recv() == 0); CHECK(buffer.size() == 31); sender.flush(buffer); @@ -975,7 +996,8 @@ TEST_CASE("line sender protocol version v2") questdb::ingress::line_sender sender{ questdb::ingress::protocol::tcp, std::string("localhost"), - std::to_string(server.port())}; + std::to_string(server.port()), + protocol_version_2}; CHECK_FALSE(sender.must_close()); server.accept(); CHECK(server.recv() == 0); @@ -988,9 +1010,33 @@ TEST_CASE("line sender protocol version v2") .at(questdb::ingress::timestamp_nanos{10000000}); CHECK(server.recv() == 0); - CHECK(buffer.size() == 31); + CHECK(buffer.size() == 38); sender.flush(buffer); CHECK(server.recv() == 1); - std::string expect{"test,t1=v1,t2= f1=0.5 10000000\n"}; + std::string expect{"test,t1=v1,t2= f1=="}; + push_double_to_buffer(expect, 0.5).append(" 10000000\n"); CHECK(server.msgs(0) == expect); } + +TEST_CASE("Http auto detect line protocol version failed") +{ + try + { + questdb::ingress::opts opts{ + questdb::ingress::protocol::http, "localhost", 1}; + questdb::ingress::line_sender sender1{opts}; + CHECK_MESSAGE(false, "Expected exception"); + } + catch (const questdb::ingress::line_sender_error& se) + { + std::string msg{se.what()}; + CHECK_MESSAGE( + msg.rfind("Failed to detect server's line protocol version", 0) == + 0, + msg); + } + catch (...) + { + CHECK_MESSAGE(false, "Other exception raised."); + } +} diff --git a/examples/line_sender_c_example_array.c b/examples/line_sender_c_example_array.c index dabf8b3c..f152b2ea 100644 --- a/examples/line_sender_c_example_array.c +++ b/examples/line_sender_c_example_array.c @@ -43,7 +43,7 @@ static bool example(const char* host, const char* port) goto on_error; size_t array_rank = 3; - uintptr_t array_shapes[] = {2, 3, 2}; + uintptr_t array_shape[] = {2, 3, 2}; intptr_t array_strides[] = {48, 16, 8}; double array_data[] = { @@ -64,7 +64,7 @@ static bool example(const char* host, const char* port) buffer, book_col, array_rank, - array_shapes, + array_shape, array_strides, (const uint8_t*)array_data, sizeof(array_data), diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 79ed839c..30a9ce02 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -499,7 +499,7 @@ bool line_sender_buffer_column_str( * @param[in] buffer Line buffer object. * @param[in] name Column name. * @param[in] rank Number of dimensions of the array. - * @param[in] shapes Array of dimension sizes (length = `rank`). + * @param[in] shape Array of dimension sizes (length = `rank`). * Each element must be a positive integer. * @param[in] strides Array strides. * @param[in] data_buffer First array element data. diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 35988e0f..901e8012 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -650,7 +650,7 @@ class line_sender_buffer line_sender_buffer& column( column_name_view name, const size_t rank, - const std::vector& shapes, + const std::vector& shape, const std::vector& strides, const std::array& data) { @@ -663,7 +663,7 @@ class line_sender_buffer _impl, name._impl, rank, - shapes.data(), + shape.data(), strides.data(), reinterpret_cast(data.data()), sizeof(double) * N); diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index c9e936ee..86bcd275 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -867,7 +867,7 @@ pub unsafe extern "C" fn line_sender_buffer_column_str( /// @param[in] buffer Line buffer object. /// @param[in] name Column name. /// @param[in] rank Array dims. -/// @param[in] shape Array shapes. +/// @param[in] shape Array shape. /// @param[in] strides Array strides. /// @param[in] data_buffer Array **first element** data memory ptr. /// @param[in] data_buffer_len Array data memory length. diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 75200fa9..18f2c546 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -1159,19 +1159,9 @@ impl Buffer { self.output.reserve(dim_header_size + array_buf_size); for i in 0..ndim { - let dim = view.dim(i)?; - if dim > MAX_ARRAY_DIM_LEN { - return Err(error::fmt!( - ArrayViewError, - "dimension length out of range: dim {}, dim length {}, max length {}", - i, - dim, - MAX_ARRAY_DIM_LEN - )); - } - // ndarr shapes + // ndarr shape self.output - .extend_from_slice((dim as u32).to_le_bytes().as_slice()); + .extend_from_slice((view.dim(i)? as u32).to_le_bytes().as_slice()); } let index = self.output.len(); @@ -3034,7 +3024,7 @@ mod timestamp; #[cfg(feature = "ilp-over-http")] mod http; -use crate::ingress::ndarr::{check_and_get_array_bytes_size, MAX_ARRAY_DIM_LEN}; +use crate::ingress::ndarr::check_and_get_array_bytes_size; #[cfg(feature = "ilp-over-http")] use http::*; diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index e5005ece..d2f6345b 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -100,15 +100,27 @@ where { let mut size = std::mem::size_of::(); for dim_index in 0..array.ndim() { - size *= array.dim(dim_index)?; - if size > MAX_ARRAY_BUFFER_SIZE { + let dim = array.dim(dim_index)?; + if dim > MAX_ARRAY_DIM_LEN { return Err(error::fmt!( ArrayViewError, - "Array buffer size too big: {}, maximum: {}", - size, - MAX_ARRAY_BUFFER_SIZE + "dimension length out of range: dim {}, dim length {}, max length {}", + dim_index, + dim, + MAX_ARRAY_DIM_LEN )); } + // following dimension's length may be zero, so check the size in out of loop + size *= dim; + } + + if size > MAX_ARRAY_BUFFER_SIZE { + return Err(error::fmt!( + ArrayViewError, + "Array buffer size too big: {}, maximum: {}", + size, + MAX_ARRAY_BUFFER_SIZE + )); } Ok(size) } @@ -226,7 +238,7 @@ where /// /// # Safety /// Caller must ensure all the following conditions: - /// - `shapes` points to a valid array of at least `dims` elements + /// - `shape` points to a valid array of at least `dims` elements /// - `strides` points to a valid array of at least `dims` elements /// - `data` points to a valid memory block of at least `data_len` bytes /// - Memory layout must satisfy: @@ -241,12 +253,20 @@ where data: *const u8, data_len: usize, ) -> Result { - let shapes = slice::from_raw_parts(shape, dims); - let size = shapes + if data_len > MAX_ARRAY_BUFFER_SIZE { + return Err(error::fmt!( + ArrayViewError, + "Array buffer size too big: {}, maximum: {}", + data_len, + MAX_ARRAY_BUFFER_SIZE + )); + } + let shape = slice::from_raw_parts(shape, dims); + let size = shape .iter() .try_fold(std::mem::size_of::(), |acc, &dim| { acc.checked_mul(dim) - .ok_or_else(|| error::fmt!(ArrayViewError, "Array total elem size overflow")) + .ok_or_else(|| error::fmt!(ArrayViewError, "Array buffer size too big")) })?; if size != data_len { return Err(error::fmt!( @@ -263,7 +283,7 @@ where } Ok(Self { dims, - shape: shapes, + shape, strides, data: slice, _marker: std::marker::PhantomData::, diff --git a/questdb-rs/src/tests/http.rs b/questdb-rs/src/tests/http.rs index e6f3ffbf..dd8480b3 100644 --- a/questdb-rs/src/tests/http.rs +++ b/questdb-rs/src/tests/http.rs @@ -771,25 +771,6 @@ fn _test_sender_auto_detect_protocol_version( res?; Ok(()) } -#[test] -fn test_sender_protocol_version_old_server21() -> TestResult { - let mut server = MockServer::new()?.configure_settings_response(&[]); - let sender_builder = server.lsb_http(); - let server_thread = std::thread::spawn(move || -> io::Result<()> { - server.accept()?; - server.send_http_response_q( - HttpResponse::empty() - .with_status(404, "Not Found") - .with_header("content-type", "text/plain") - .with_body_str("Not Found"), - )?; - Ok(()) - }); - let sender = sender_builder.build()?; - assert_eq!(sender.default_protocol_version(), ProtocolVersion::V1); - server_thread.join().unwrap()?; - Ok(()) -} #[test] fn test_sender_auto_protocol_version_basic() -> TestResult { diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index a7b49001..2f27fb3e 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -59,15 +59,15 @@ fn to_bytes(data: &[T]) -> Vec { fn test_stride_array_view() -> TestResult { // contiguous layout let test_data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; - let shapes = [2usize, 3]; + let shape = [2usize, 3]; let strides = [ - (shapes[1] * size_of::()) as isize, + (shape[1] * size_of::()) as isize, size_of::() as isize, ]; let array = unsafe { StrideArrayView::::new( - shapes.len(), - shapes.as_ptr(), + shape.len(), + shape.as_ptr(), strides.as_ptr(), test_data.as_ptr() as *const u8, test_data.len() * size_of::(), @@ -90,13 +90,13 @@ fn test_stride_array_view() -> TestResult { fn test_strided_non_contiguous() -> TestResult { let elem_size = size_of::() as isize; let col_major_data = [1.0, 3.0, 5.0, 2.0, 4.0, 6.0]; - let shapes = [3usize, 2]; - let strides = [elem_size, shapes[0] as isize * elem_size]; + let shape = [3usize, 2]; + let strides = [elem_size, shape[0] as isize * elem_size]; let array_view: StrideArrayView<'_, f64> = unsafe { StrideArrayView::new( - shapes.len(), - shapes.as_ptr(), + shape.len(), + shape.as_ptr(), strides.as_ptr(), col_major_data.as_ptr() as *const u8, col_major_data.len() * elem_size as usize, @@ -238,7 +238,7 @@ fn test_stride_array_size_overflow() -> TestResult { }; let err = result.unwrap_err(); assert_eq!(err.code(), ErrorCode::ArrayViewError); - assert!(err.msg().contains("Array total elem size overflow")); + assert!(err.msg().contains("Array buffer size too big")); Ok(()) } diff --git a/system_test/questdb_line_sender.py b/system_test/questdb_line_sender.py index a058181f..291a3834 100644 --- a/system_test/questdb_line_sender.py +++ b/system_test/questdb_line_sender.py @@ -709,7 +709,7 @@ def column( def column_f64_arr(self, name: str, rank: int, - shapes: tuple[int, ...], + shape: tuple[int, ...], strides: tuple[int, ...], data: c_void_p, length: int): @@ -722,14 +722,14 @@ def _convert_tuple(tpl: tuple[int, ...], c_type: type, name: str) -> ctypes.POIN f"{name} value exceeds {c_type.__name__} range" ) from e - c_shapes = _convert_tuple(shapes, c_size_t, "shapes") + c_shape = _convert_tuple(shape, c_size_t, "shape") c_strides = _convert_tuple(strides, c_ssize_t, "strides") _error_wrapped_call( _DLL.line_sender_buffer_column_f64_arr, self._impl, _column_name(name), c_size_t(rank), - c_shapes, + c_shape, c_strides, ctypes.cast(data, c_uint8_p), c_size_t(length) diff --git a/system_test/test.py b/system_test/test.py index 351a0327..2eeaad86 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -110,19 +110,19 @@ def _mk_linesender(self): QDB_FIXTURE.host, QDB_FIXTURE.http_server_port if QDB_FIXTURE.http else QDB_FIXTURE.line_tcp_port, **kwargs) - + @property def expected_protocol_version(self) -> qls.ProtocolVersion: """The protocol version that we expect to be handling.""" if QDB_FIXTURE.protocol_version is None: if not QDB_FIXTURE.http: return qls.ProtocolVersion.V1 - + if QDB_FIXTURE.version >= FIRST_ARRAYS_RELEASE: return qls.ProtocolVersion.V2 - + return qls.ProtocolVersion.V1 - + return QDB_FIXTURE.protocol_version def _expect_eventual_disconnect(self, sender): @@ -616,6 +616,49 @@ def test_f64_arr_mix_dims(self): except qls.SenderError as e: self.assertIn('cast error from protocol type: DOUBLE[] to column type: DOUBLE[][]', str(e)) + def test_f64_arr_dims_length_overflow(self): + if self.expected_protocol_version < qls.ProtocolVersion.V2: + self.skipTest('communicating over old protocol which does not support arrays') + + table_name = uuid.uuid4().hex + array = np.empty((1 << 29, 0), dtype=np.float64) + try: + with self._mk_linesender() as sender: + (sender.table(table_name) + .column_f64_arr('array', array) + .at_now()) + except qls.SenderError as e: + self.assertIn('dimension length out of range', str(e)) + + def test_f64_arr_total_size_overflow(self): + if self.expected_protocol_version < qls.ProtocolVersion.V2: + self.skipTest('communicating over old protocol which does not support arrays') + + table_name = uuid.uuid4().hex + array = np.empty((1 << 16, 1 << 16), dtype=np.float64) + try: + with self._mk_linesender() as sender: + (sender.table(table_name) + .column_f64_arr('array', array) + .at_now()) + except qls.SenderError as e: + self.assertIn('Array buffer size too big', str(e)) + + def test_f64_arr_max_dims(self): + if self.expected_protocol_version < qls.ProtocolVersion.V2: + self.skipTest('communicating over old protocol which does not support arrays') + + table_name = uuid.uuid4().hex + dims = (1,) * 33 + array = np.empty(dims, dtype=np.float64) + try: + with self._mk_linesender() as sender: + (sender.table(table_name) + .column_f64_arr('array', array) + .at_now()) + except qls.SenderError as e: + self.assertIn('Array dimension mismatch: expected at most 32 dimensions, but got 33', str(e)) + def test_protocol_version_v1(self): if self.expected_protocol_version >= qls.ProtocolVersion.V2: self.skipTest('we are only validating the older protocol here') @@ -625,7 +668,7 @@ def test_protocol_version_v1(self): 0.0, -0.0, 1.0, - -1.0] # Converted to `None`. + -1.0] table_name = uuid.uuid4().hex pending = None @@ -970,10 +1013,7 @@ def test_tcp_transactions(self): self.skipTest('TCP-only test') if QDB_FIXTURE.version <= (7, 3, 7): self.skipTest('No ILP/HTTP support') - version = qls.ProtocolVersion.V2 - if QDB_FIXTURE.version <= (8, 3, 1): - version = qls.ProtocolVersion.V1 - buf = qls.Buffer(version) + buf = qls.Buffer(self.expected_protocol_version) buf.table('t1').column('c1', 'v1').at(time.time_ns()) with self.assertRaisesRegex(qls.SenderError, r'.*Transactional .* not supported.*'): with self._mk_linesender() as sender: From e3a6bece35b44361330da9cc136c23a1b06491eb Mon Sep 17 00:00:00 2001 From: victor Date: Fri, 16 May 2025 12:07:57 +0800 Subject: [PATCH 082/119] remove buffer size overflow test --- system_test/test.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/system_test/test.py b/system_test/test.py index 2eeaad86..54a1e02c 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -630,20 +630,6 @@ def test_f64_arr_dims_length_overflow(self): except qls.SenderError as e: self.assertIn('dimension length out of range', str(e)) - def test_f64_arr_total_size_overflow(self): - if self.expected_protocol_version < qls.ProtocolVersion.V2: - self.skipTest('communicating over old protocol which does not support arrays') - - table_name = uuid.uuid4().hex - array = np.empty((1 << 16, 1 << 16), dtype=np.float64) - try: - with self._mk_linesender() as sender: - (sender.table(table_name) - .column_f64_arr('array', array) - .at_now()) - except qls.SenderError as e: - self.assertIn('Array buffer size too big', str(e)) - def test_f64_arr_max_dims(self): if self.expected_protocol_version < qls.ProtocolVersion.V2: self.skipTest('communicating over old protocol which does not support arrays') From c05753efeee4deb296f823bb3bd0eabff7ecc296 Mon Sep 17 00:00:00 2001 From: victor Date: Fri, 16 May 2025 17:54:38 +0800 Subject: [PATCH 083/119] empty array throw exception. --- questdb-rs/src/ingress/ndarr.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index d2f6345b..cff2c5ce 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -253,6 +253,12 @@ where data: *const u8, data_len: usize, ) -> Result { + if dims == 0 { + return Err(error::fmt!( + ArrayViewError, + "Zero-dimensional arrays are not supported", + )); + } if data_len > MAX_ARRAY_BUFFER_SIZE { return Err(error::fmt!( ArrayViewError, @@ -262,12 +268,11 @@ where )); } let shape = slice::from_raw_parts(shape, dims); - let size = shape - .iter() - .try_fold(std::mem::size_of::(), |acc, &dim| { - acc.checked_mul(dim) - .ok_or_else(|| error::fmt!(ArrayViewError, "Array buffer size too big")) - })?; + let mut size = std::mem::size_of::(); + for &dim in shape { + size *= dim + } + if size != data_len { return Err(error::fmt!( ArrayViewError, From b9e023f3b2fb1d09ab4bf777213f8599be69694b Mon Sep 17 00:00:00 2001 From: victor Date: Fri, 16 May 2025 17:57:33 +0800 Subject: [PATCH 084/119] enrich tests. --- questdb-rs/src/ingress/ndarr.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index cff2c5ce..091258cb 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -268,10 +268,12 @@ where )); } let shape = slice::from_raw_parts(shape, dims); - let mut size = std::mem::size_of::(); - for &dim in shape { - size *= dim - } + let size = shape + .iter() + .try_fold(std::mem::size_of::(), |acc, &dim| { + acc.checked_mul(dim) + .ok_or_else(|| error::fmt!(ArrayViewError, "Array buffer size too big")) + })?; if size != data_len { return Err(error::fmt!( From 3fd415fa12e928d1402ff1293090c9bb2492313a Mon Sep 17 00:00:00 2001 From: amunra Date: Fri, 16 May 2025 12:06:13 +0100 Subject: [PATCH 085/119] test assertion improvements --- questdb-rs/src/tests/http.rs | 30 +++++++++++++----------------- questdb-rs/src/tests/mod.rs | 28 ++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/questdb-rs/src/tests/http.rs b/questdb-rs/src/tests/http.rs index dd8480b3..42b9dcd8 100644 --- a/questdb-rs/src/tests/http.rs +++ b/questdb-rs/src/tests/http.rs @@ -24,7 +24,7 @@ use crate::ingress::{Buffer, Protocol, ProtocolVersion, SenderBuilder, TimestampNanos}; use crate::tests::mock::{certs_dir, HttpResponse, MockServer}; -use crate::tests::TestResult; +use crate::tests::{assert_err_contains, TestResult}; use crate::ErrorCode; use rstest::rstest; use std::io; @@ -446,10 +446,7 @@ fn test_request_timeout( let time_start = std::time::Instant::now(); let res = sender.flush_and_keep(&buffer); let time_elapsed = time_start.elapsed(); - assert!(res.is_err()); - let err = res.unwrap_err(); - assert_eq!(err.code(), ErrorCode::SocketError); - assert!(err.msg().contains("per call")); + assert_err_contains(res, ErrorCode::SocketError, "per call"); assert!(time_elapsed >= request_timeout); Ok(()) } @@ -710,7 +707,7 @@ fn test_transactional( assert_eq!( err.msg(), "Buffer contains lines for multiple tables. \ - Transactional flushes are only supported for buffers containing lines for a single table." + Transactional flushes are only supported for buffers containing lines for a single table.", ); let res = sender.flush_and_keep_with_flags(&buffer2, true); @@ -806,11 +803,11 @@ fn test_sender_auto_protocol_version_unsupported_client() -> TestResult { server.send_settings_response()?; Ok(()) }); - let res1 = sender_builder.build(); - assert!(res1.is_err()); - let e1 = res1.err().unwrap(); - assert_eq!(e1.code(), ErrorCode::ProtocolVersionError); - assert!(e1.msg().contains("Server does not support current client")); + assert_err_contains( + sender_builder.build(), + ErrorCode::ProtocolVersionError, + "Server does not support current client", + ); server_thread.join().unwrap()?; Ok(()) } @@ -822,11 +819,10 @@ fn test_buffer_protocol_version1_not_support_array() -> TestResult { .table("test")? .symbol("sym", "bol")? .column_arr("x", &[1.0f64, 2.0]); - assert!(res.is_err()); - let e1 = res.as_ref().err().unwrap(); - assert_eq!(e1.code(), ErrorCode::ProtocolVersionError); - assert!(e1 - .msg() - .contains("line protocol version v1 does not support array datatype")); + assert_err_contains( + res, + ErrorCode::ProtocolVersionError, + "line protocol version v1 does not support array datatype", + ); Ok(()) } diff --git a/questdb-rs/src/tests/mod.rs b/questdb-rs/src/tests/mod.rs index d68dede9..c63a65d3 100644 --- a/questdb-rs/src/tests/mod.rs +++ b/questdb-rs/src/tests/mod.rs @@ -39,3 +39,31 @@ mod json_tests { pub type TestError = Box; pub type TestResult = std::result::Result<(), TestError>; + +pub fn assert_err_contains( + result: crate::Result, + expected_code: crate::ErrorCode, + expected_msg_contained: &str, +) { + match result { + Ok(_) => panic!( + "Expected error containing '{}', but got Ok({:?})", + expected_msg_contained, result + ), + Err(e) => { + assert_eq!( + e.code(), + expected_code, + "Expected error code {:?}, but got {:?}", + expected_code, + e.code() + ); + assert!( + e.msg().contains(expected_msg_contained), + "Expected error message to contain {:?}, but got {:?}", + expected_msg_contained, + e.msg() + ); + } + } +} From eadbadf9e1517eeebcc421901ff4146731a22393 Mon Sep 17 00:00:00 2001 From: amunra Date: Fri, 16 May 2025 15:21:29 +0100 Subject: [PATCH 086/119] fixed failing test on Windows --- questdb-rs/src/ingress/http.rs | 12 +++++++----- questdb-rs/src/tests/http.rs | 9 ++++++--- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/questdb-rs/src/ingress/http.rs b/questdb-rs/src/ingress/http.rs index a9c281dd..7955933c 100644 --- a/questdb-rs/src/ingress/http.rs +++ b/questdb-rs/src/ingress/http.rs @@ -19,7 +19,6 @@ use ureq::unversioned::transport::{ use crate::ingress::ProtocolVersion; use ureq::unversioned::*; -use ureq::Error::*; use ureq::{http, Body}; #[derive(PartialEq, Debug, Clone)] @@ -256,7 +255,10 @@ fn need_retry(res: Result) -> bool { 599 // Network Connect Timeout Error ) } - Err(err) => matches!(err, Timeout(_) | ConnectionFailed | TooManyRedirects), + Err(err) => matches!( + err, + ureq::Error::Timeout(_) | ureq::Error::ConnectionFailed | ureq::Error::TooManyRedirects + ), } } @@ -429,7 +431,7 @@ pub(super) fn get_supported_protocol_versions( } return Err(fmt!( ProtocolVersionError, - "Failed to detect server's line protocol version, settings url: {}, status code: {}.", + "Could not detect server's line protocol version, settings url: {}, status code: {}.", settings_url, status )); @@ -446,7 +448,7 @@ pub(super) fn get_supported_protocol_versions( } else { fmt!( ProtocolVersionError, - "Failed to detect server's line protocol version, settings url: {}, err: {}.", + "Could not detect server's line protocol version, settings url: {}, err: {}.", settings_url, err ) @@ -455,7 +457,7 @@ pub(super) fn get_supported_protocol_versions( e => { fmt!( ProtocolVersionError, - "Failed to detect server's line protocol version, settings url: {}, err: {}.", + "Could not detect server's line protocol version, settings url: {}, err: {}.", settings_url, e ) diff --git a/questdb-rs/src/tests/http.rs b/questdb-rs/src/tests/http.rs index 42b9dcd8..9773590c 100644 --- a/questdb-rs/src/tests/http.rs +++ b/questdb-rs/src/tests/http.rs @@ -798,17 +798,20 @@ fn test_sender_auto_protocol_version_only_v2() -> TestResult { fn test_sender_auto_protocol_version_unsupported_client() -> TestResult { let mut server = MockServer::new()?.configure_settings_response(&[3, 4]); let sender_builder = server.lsb_http(); - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; server.send_settings_response()?; - Ok(()) + Ok(server) }); assert_err_contains( sender_builder.build(), ErrorCode::ProtocolVersionError, "Server does not support current client", ); - server_thread.join().unwrap()?; + let server = server_thread.join().unwrap()?; + + // We keep the server around til the end of the test to ensure that the response is fully received. + drop(server); Ok(()) } From cba4c33fe9928ea721d8a08160ee95abed0650f2 Mon Sep 17 00:00:00 2001 From: victor Date: Fri, 16 May 2025 23:04:36 +0800 Subject: [PATCH 087/119] fix tests --- cpp_test/test_line_sender.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 8424c133..6933a1c8 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -1031,7 +1031,7 @@ TEST_CASE("Http auto detect line protocol version failed") { std::string msg{se.what()}; CHECK_MESSAGE( - msg.rfind("Failed to detect server's line protocol version", 0) == + msg.rfind("Could not detect server's line protocol version", 0) == 0, msg); } From a25e66cecd311781b4535341c10d48428b97e8bc Mon Sep 17 00:00:00 2001 From: amunra Date: Fri, 16 May 2025 21:22:27 +0100 Subject: [PATCH 088/119] tweaks to speed tests up on Windows --- cpp_test/test_line_sender.cpp | 54 +++++++++++++++--------------- questdb-rs-ffi/src/lib.rs | 2 +- questdb-rs/src/ingress/tests.rs | 8 ++--- system_test/fixture.py | 4 +-- system_test/questdb_line_sender.py | 4 +++ system_test/test.py | 6 ++-- 6 files changed, 42 insertions(+), 36 deletions(-) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 6933a1c8..b1f2769a 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -127,7 +127,7 @@ TEST_CASE("line_sender c api basics") ::line_sender_error_free(err); }}; ::line_sender_utf8 host = {0, nullptr}; - CHECK(::line_sender_utf8_init(&host, 9, "localhost", &err)); + CHECK(::line_sender_utf8_init(&host, 9, "127.0.0.1", &err)); ::line_sender_opts* opts = ::line_sender_opts_new(::line_sender_protocol_tcp, host, server.port()); CHECK_NE(opts, nullptr); @@ -227,7 +227,7 @@ TEST_CASE("line_sender c++ connect disconnect") { questdb::ingress::test::mock_server server; questdb::ingress::line_sender sender{ - questdb::ingress::protocol::tcp, "localhost", server.port()}; + questdb::ingress::protocol::tcp, "127.0.0.1", server.port()}; CHECK_FALSE(sender.must_close()); server.accept(); CHECK(server.recv() == 0); @@ -238,7 +238,7 @@ TEST_CASE("line_sender c++ api basics") questdb::ingress::test::mock_server server; questdb::ingress::line_sender sender{ questdb::ingress::protocol::tcp, - std::string("localhost"), + std::string("127.0.0.1"), std::to_string(server.port()), protocol_version_2}; CHECK_FALSE(sender.must_close()); @@ -285,7 +285,7 @@ TEST_CASE("test multiple lines") { questdb::ingress::test::mock_server server; std::string conf_str = - "tcp::addr=localhost:" + std::to_string(server.port()) + + "tcp::addr=127.0.0.1:" + std::to_string(server.port()) + ";protocol_version=2;"; questdb::ingress::line_sender sender = questdb::ingress::line_sender::from_conf(conf_str); @@ -328,7 +328,7 @@ TEST_CASE("State machine testing -- flush without data.") questdb::ingress::test::mock_server server; questdb::ingress::line_sender sender{ questdb::ingress::protocol::tcp, - std::string_view{"localhost"}, + std::string_view{"127.0.0.1"}, std::to_string(server.port())}; questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); @@ -346,7 +346,7 @@ TEST_CASE("One symbol only - flush before server accept") questdb::ingress::test::mock_server server; questdb::ingress::line_sender sender{ questdb::ingress::protocol::tcp, - std::string{"localhost"}, + std::string{"127.0.0.1"}, server.port()}; // Does not raise - this is unlike InfluxDB spec that disallows this. @@ -368,7 +368,7 @@ TEST_CASE("One column only - server.accept() after flush, before close") { questdb::ingress::test::mock_server server; questdb::ingress::line_sender sender{ - questdb::ingress::protocol::tcp, "localhost", server.port()}; + questdb::ingress::protocol::tcp, "127.0.0.1", server.port()}; // Does not raise - this is unlike the InfluxDB spec that disallows this. questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); @@ -388,7 +388,7 @@ TEST_CASE("Symbol after column") { questdb::ingress::test::mock_server server; questdb::ingress::line_sender sender{ - questdb::ingress::protocol::tcp, "localhost", server.port()}; + questdb::ingress::protocol::tcp, "127.0.0.1", server.port()}; questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table("test").column("t1", "v1"); @@ -552,7 +552,7 @@ TEST_CASE("Sender move testing.") questdb::ingress::test::mock_server server1; questdb::ingress::test::mock_server server2; - questdb::ingress::utf8_view host{"localhost"}; + questdb::ingress::utf8_view host{"127.0.0.1"}; const questdb::ingress::utf8_view& host_ref = host; questdb::ingress::line_sender sender1{ @@ -581,7 +581,7 @@ TEST_CASE("Sender move testing.") CHECK(sender2.must_close()); questdb::ingress::line_sender sender3{ - questdb::ingress::protocol::tcp, "localhost", server2.port()}; + questdb::ingress::protocol::tcp, "127.0.0.1", server2.port()}; CHECK_FALSE(sender3.must_close()); sender3 = std::move(sender2); @@ -614,7 +614,7 @@ TEST_CASE("Bad interface") try { questdb::ingress::opts opts{ - questdb::ingress::protocol::tcp, "localhost", "9009"}; + questdb::ingress::protocol::tcp, "127.0.0.1", "9009"}; opts.bind_interface("dummy_hostname"); questdb::ingress::line_sender sender{opts}; CHECK_MESSAGE(false, "Expected exception"); @@ -637,13 +637,13 @@ TEST_CASE("Bad port") try { questdb::ingress::line_sender sender{ - questdb::ingress::protocol::tcp, "localhost", bad_port}; + questdb::ingress::protocol::tcp, "127.0.0.1", bad_port}; CHECK_MESSAGE(false, "Expected exception"); } catch (const questdb::ingress::line_sender_error& se) { std::string msg{se.what()}; - std::string exp_msg{"\"localhost:" + bad_port + "\": "}; + std::string exp_msg{"\"127.0.0.1:" + bad_port + "\": "}; CHECK_MESSAGE(msg.find(exp_msg) != std::string::npos, msg); } catch (...) @@ -666,7 +666,7 @@ TEST_CASE("Bad connect") // Port 1 is generally the tcpmux service which one would // very much expect to never be running. questdb::ingress::line_sender sender{ - questdb::ingress::protocol::tcp, "localhost", 1}; + questdb::ingress::protocol::tcp, "127.0.0.1", 1}; CHECK_MESSAGE(false, "Expected exception"); } catch (const questdb::ingress::line_sender_error& se) @@ -734,7 +734,7 @@ TEST_CASE("Opts copy ctor, assignment and move testing.") { { questdb::ingress::opts opts1{ - questdb::ingress::protocol::tcp, "localhost", "9009"}; + questdb::ingress::protocol::tcp, "127.0.0.1", "9009"}; questdb::ingress::opts opts2{std::move(opts1)}; } @@ -746,7 +746,7 @@ TEST_CASE("Opts copy ctor, assignment and move testing.") { questdb::ingress::opts opts1{ - questdb::ingress::protocol::tcp, "localhost", "9009"}; + questdb::ingress::protocol::tcp, "127.0.0.1", "9009"}; questdb::ingress::opts opts2{ questdb::ingress::protocol::tcp, "altavista.digital.com", "9009"}; opts1 = std::move(opts2); @@ -765,7 +765,7 @@ TEST_CASE("Test timestamp column.") { questdb::ingress::test::mock_server server; questdb::ingress::line_sender sender{ - questdb::ingress::protocol::tcp, "localhost", server.port()}; + questdb::ingress::protocol::tcp, "127.0.0.1", server.port()}; const auto now = std::chrono::system_clock::now(); const auto now_micros = @@ -900,7 +900,7 @@ TEST_CASE("Empty Buffer") questdb::ingress::test::mock_server server; questdb::ingress::line_sender sender{ - questdb::ingress::protocol::tcp, "localhost", server.port()}; + questdb::ingress::protocol::tcp, "127.0.0.1", server.port()}; CHECK_THROWS_WITH_AS( sender.flush(b1), "State error: Bad call to `flush`, should have called `table` instead.", @@ -914,11 +914,11 @@ TEST_CASE("Empty Buffer") TEST_CASE("Opts from conf") { questdb::ingress::opts opts1 = - questdb::ingress::opts::from_conf("tcp::addr=localhost:9009;"); + questdb::ingress::opts::from_conf("tcp::addr=127.0.0.1:9009;"); questdb::ingress::opts opts2 = questdb::ingress::opts::from_conf("tcps::addr=localhost:9009;"); questdb::ingress::opts opts3 = - questdb::ingress::opts::from_conf("https::addr=localhost:9009;"); + questdb::ingress::opts::from_conf("https::addr=127.0.0.1:9009;"); questdb::ingress::opts opts4 = questdb::ingress::opts::from_conf("https::addr=localhost:9009;"); } @@ -926,9 +926,9 @@ TEST_CASE("Opts from conf") TEST_CASE("HTTP basics") { questdb::ingress::opts opts1{ - questdb::ingress::protocol::http, "localhost", 1, protocol_version_2}; + questdb::ingress::protocol::http, "127.0.0.1", 1, protocol_version_2}; questdb::ingress::opts opts1conf = questdb::ingress::opts::from_conf( - "http::addr=localhost:1;username=user;password=pass;request_timeout=" + "http::addr=127.0.0.1:1;username=user;password=pass;request_timeout=" "5000;retry_timeout=5;protocol_version=2;"); questdb::ingress::opts opts2{ questdb::ingress::protocol::https, @@ -936,7 +936,7 @@ TEST_CASE("HTTP basics") "1", protocol_version_2}; questdb::ingress::opts opts2conf = questdb::ingress::opts::from_conf( - "http::addr=localhost:1;token=token;request_min_throughput=1000;retry_" + "http::addr=127.0.0.1:1;token=token;request_min_throughput=1000;retry_" "timeout=0;protocol_version=2;"); opts1.username("user") .password("pass") @@ -959,7 +959,7 @@ TEST_CASE("HTTP basics") CHECK_THROWS_AS( questdb::ingress::opts::from_conf( - "http::addr=localhost:1;bind_interface=0.0.0.0;"), + "http::addr=127.0.0.1:1;bind_interface=0.0.0.0;"), questdb::ingress::line_sender_error); } @@ -968,7 +968,7 @@ TEST_CASE("line sender protocol version default v1 for tcp") questdb::ingress::test::mock_server server; questdb::ingress::line_sender sender{ questdb::ingress::protocol::tcp, - std::string("localhost"), + std::string("127.0.0.1"), std::to_string(server.port())}; CHECK_FALSE(sender.must_close()); server.accept(); @@ -995,7 +995,7 @@ TEST_CASE("line sender protocol version v2") questdb::ingress::test::mock_server server; questdb::ingress::line_sender sender{ questdb::ingress::protocol::tcp, - std::string("localhost"), + std::string("127.0.0.1"), std::to_string(server.port()), protocol_version_2}; CHECK_FALSE(sender.must_close()); @@ -1023,7 +1023,7 @@ TEST_CASE("Http auto detect line protocol version failed") try { questdb::ingress::opts opts{ - questdb::ingress::protocol::http, "localhost", 1}; + questdb::ingress::protocol::http, "127.0.0.1", 1}; questdb::ingress::line_sender sender1{opts}; CHECK_MESSAGE(false, "Expected exception"); } diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 86bcd275..38bb3bb9 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -84,7 +84,7 @@ macro_rules! upd_opts { // already cleaned up object. // To avoid double-freeing, we need to construct a valid "dummy" // object on top of the memory that is still owned by the caller. - let dummy = SenderBuilder::new(Protocol::Tcp, "localhost", 1); + let dummy = SenderBuilder::new(Protocol::Tcp, "127.0.0.1", 1); ptr::write(builder_ref, dummy); return false; } diff --git a/questdb-rs/src/ingress/tests.rs b/questdb-rs/src/ingress/tests.rs index 599b5ac9..a48021c6 100644 --- a/questdb-rs/src/ingress/tests.rs +++ b/questdb-rs/src/ingress/tests.rs @@ -5,9 +5,9 @@ use tempfile::TempDir; #[cfg(feature = "ilp-over-http")] #[test] fn http_simple() { - let builder = SenderBuilder::from_conf("http::addr=localhost;").unwrap(); + let builder = SenderBuilder::from_conf("http::addr=127.0.0.1;").unwrap(); assert_eq!(builder.protocol, Protocol::Http); - assert_specified_eq(&builder.host, "localhost"); + assert_specified_eq(&builder.host, "127.0.0.1"); assert_specified_eq(&builder.port, Protocol::Http.default_port()); assert!(!builder.protocol.tls_enabled()); } @@ -30,10 +30,10 @@ fn https_simple() { #[test] fn tcp_simple() { - let builder = SenderBuilder::from_conf("tcp::addr=localhost;").unwrap(); + let builder = SenderBuilder::from_conf("tcp::addr=127.0.0.1;").unwrap(); assert_eq!(builder.protocol, Protocol::Tcp); assert_specified_eq(&builder.port, Protocol::Tcp.default_port()); - assert_specified_eq(&builder.host, "localhost"); + assert_specified_eq(&builder.host, "127.0.0.1"); assert!(!builder.protocol.tls_enabled()); } diff --git a/system_test/fixture.py b/system_test/fixture.py index ec145e44..1f70da13 100644 --- a/system_test/fixture.py +++ b/system_test/fixture.py @@ -240,7 +240,7 @@ def __init__(self, root_dir: pathlib.Path, auth=False, wrap_tls=False, http=Fals self._conf_path = self._conf_dir / 'server.conf' self._log = None self._proc = None - self.host = 'localhost' + self.host = '127.0.0.1' self.http_server_port = None self.line_tcp_port = None self.pg_port = None @@ -312,7 +312,7 @@ def check_http_up(): if self._proc.poll() is not None: raise RuntimeError('QuestDB died during startup.') req = urllib.request.Request( - f'http://localhost:{self.http_server_port}/ping', + f'http://127.0.0.1:{self.http_server_port}/ping', method='GET') try: resp = urllib.request.urlopen(req, timeout=1) diff --git a/system_test/questdb_line_sender.py b/system_test/questdb_line_sender.py index 291a3834..c5eb269b 100644 --- a/system_test/questdb_line_sender.py +++ b/system_test/questdb_line_sender.py @@ -780,6 +780,10 @@ def __init__( port: Union[str, int], **kwargs): + if protocol in (Protocol.TCPS, Protocol.HTTPS): + if host == '127.0.0.1': + host = 'localhost' # for TLS connections we need a hostname + self._build_mode = build_mode self._impl = None self._conf = [ diff --git a/system_test/test.py b/system_test/test.py index 54a1e02c..493d081b 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -725,7 +725,9 @@ def _test_example(self, bin_name, table_name, tls=False): ca_path = proj.tls_certs_dir / 'server_rootCA.pem' args.append(str(ca_path)) port = TLS_PROXY_FIXTURE.listen_port - args.extend(['localhost', str(port)]) + args.extend(['localhost', str(port)]) + else: + args.extend(['127.0.0.1', str(port)]) subprocess.check_call(args, cwd=bin_path.parent) # Check inserted data. @@ -800,7 +802,7 @@ def _test_array_example(self, bin_name, table_name): raise RuntimeError(f'Could not find {bin_name}{ext} in {proj.build_dir}') port = QDB_FIXTURE.line_tcp_port args = [str(bin_path)] - args.extend(['localhost', str(port)]) + args.extend(['127.0.0.1', str(port)]) subprocess.check_call(args, cwd=bin_path.parent) resp = retry_check_table(table_name) exp_columns = [ From 3b134d226e2de69db87c19dd4d10956c9e762c33 Mon Sep 17 00:00:00 2001 From: amunra Date: Fri, 16 May 2025 21:36:17 +0100 Subject: [PATCH 089/119] skipping redundant tests --- system_test/test.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/system_test/test.py b/system_test/test.py index 493d081b..57eae909 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -1134,20 +1134,21 @@ def run_with_fixtures(args): global BUILD_MODE last_version = None + latest_protocol = sorted(list(qls.ProtocolVersion))[-1] for questdb_dir, auth, http, protocol_version, build_mode in itertools.product( iter_versions(args), (False, True), # auth (False, True), # http [None] + list(qls.ProtocolVersion), # None is for `auto` list(qls.BuildMode)): - print( - f'Running tests [questdb_dir={questdb_dir}, auth={auth}, http={http}, build_mode={build_mode}, protocol_version={protocol_version}]') - if http and last_version <= (7, 3, 7): - print('Skipping ILP/HTTP tests for versions <= 7.3.7') + if (build_mode in (qls.BuildMode.API, qls.BuildMode.ENV)) and (protocol_version != latest_protocol): continue if http and auth: - print('Skipping auth for ILP/HTTP tests') continue + if auth and (protocol_version != latest_protocol): + continue + print( + f'Running tests [auth={auth}, http={http}, build_mode={build_mode}, protocol_version={protocol_version}]') QDB_FIXTURE = QuestDbFixture( questdb_dir, auth=auth, From 692088e235af32fb9acf3cb20203f0e799cc7905 Mon Sep 17 00:00:00 2001 From: amunra Date: Fri, 16 May 2025 22:05:56 +0100 Subject: [PATCH 090/119] skipping more redundant tests --- system_test/test.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/system_test/test.py b/system_test/test.py index 57eae909..321403cb 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -824,6 +824,8 @@ def test_opposite_auth(self): """ if QDB_FIXTURE.http: self.skipTest('TCP-only test') + if BUILD_MODE != qls.BuildMode.API: + self.skipTest('BuildMode.API-only test') auth = {} if QDB_FIXTURE.auth else AUTH sender = qls.Sender( BUILD_MODE, @@ -856,6 +858,9 @@ def test_unrecognized_auth(self): if not QDB_FIXTURE.auth: self.skipTest('No auth') + if BUILD_MODE != qls.BuildMode.API: + self.skipTest('BuildMode.API-only test') + sender = qls.Sender( BUILD_MODE, qls.Protocol.TCP, @@ -873,6 +878,9 @@ def test_malformed_auth1(self): if not QDB_FIXTURE.auth: self.skipTest('No auth') + if BUILD_MODE != qls.BuildMode.API: + self.skipTest('BuildMode.API-only test') + sender = qls.Sender( BUILD_MODE, qls.Protocol.TCP, @@ -892,6 +900,9 @@ def test_malformed_auth2(self): if not QDB_FIXTURE.auth: self.skipTest('No auth') + if BUILD_MODE != qls.BuildMode.API: + self.skipTest('BuildMode.API-only test') + sender = qls.Sender( BUILD_MODE, qls.Protocol.TCP, @@ -911,6 +922,9 @@ def test_malformed_auth3(self): if not QDB_FIXTURE.auth: self.skipTest('No auth') + if BUILD_MODE != qls.BuildMode.API: + self.skipTest('BuildMode.API-only test') + sender = qls.Sender( BUILD_MODE, qls.Protocol.TCP, @@ -922,6 +936,8 @@ def test_malformed_auth3(self): self._expect_eventual_disconnect(sender) def test_tls_insecure_skip_verify(self): + if BUILD_MODE != qls.BuildMode.API: + self.skipTest('BuildMode.API-only test') protocol = qls.Protocol.HTTPS if QDB_FIXTURE.http else qls.Protocol.TCPS auth = AUTH if QDB_FIXTURE.auth else {} sender = qls.Sender( @@ -934,6 +950,8 @@ def test_tls_insecure_skip_verify(self): self._test_single_symbol_impl(sender) def test_tls_roots(self): + if BUILD_MODE != qls.BuildMode.API: + self.skipTest('BuildMode.API-only test') protocol = qls.Protocol.HTTPS if QDB_FIXTURE.http else qls.Protocol.TCPS auth = auth = AUTH if QDB_FIXTURE.auth else {} sender = qls.Sender( @@ -946,6 +964,8 @@ def test_tls_roots(self): self._test_single_symbol_impl(sender) def _test_tls_ca(self, tls_ca): + if BUILD_MODE != qls.BuildMode.API: + self.skipTest('BuildMode.API-only test') protocol = qls.Protocol.HTTPS if QDB_FIXTURE.http else qls.Protocol.TCPS prev_ssl_cert_file = os.environ.get('SSL_CERT_FILE') try: @@ -1036,6 +1056,10 @@ def parse_args(): '--unittest-help', action='store_true', help='Show unittest --help') + run_p.add_argument( + '--profile', + action='store_true', + help='Run with cProfile') version_g = run_p.add_mutually_exclusive_group() version_g.add_argument( '--last-n', @@ -1194,6 +1218,12 @@ def main(): if args.command == 'list': list_releases(args) else: + profile = args.profile + if profile: + sys.argv.remove("--profile") + import cProfile + cProfile.run('main()', filename='profile.out') + return # Repackage args for unittest's own arg parser. sys.argv[:] = sys.argv[:1] + extra_args show_help = getattr(args, 'unittest_help', False) From 9521d0feff62392ed43e5603c25a077b08567f20 Mon Sep 17 00:00:00 2001 From: amunra Date: Fri, 16 May 2025 22:41:09 +0100 Subject: [PATCH 091/119] reduced some timeouts on expected-timeout tests --- system_test/test.py | 71 ++++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/system_test/test.py b/system_test/test.py index 321403cb..4271e965 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -295,7 +295,7 @@ def test_mismatched_types_across_rows(self): # If HTTP, the error should cause the whole batch to be ignored. # We assert that the table is empty. with self.assertRaises(TimeoutError): - retry_check_table(table_name, timeout_sec=1, log=False) + retry_check_table(table_name, timeout_sec=0.25, log=False) else: # We only ever get the first row back. resp = retry_check_table(table_name, log_ctx=pending) @@ -310,7 +310,7 @@ def test_mismatched_types_across_rows(self): # The second one is dropped and will not appear in results. with self.assertRaises(TimeoutError): - retry_check_table(table_name, min_rows=2, timeout_sec=1, log=False) + retry_check_table(table_name, min_rows=2, timeout_sec=0.25, log=False) def test_at(self): if QDB_FIXTURE.version <= (6, 0, 7, 1): @@ -1014,7 +1014,7 @@ def test_http_transactions(self): self.assertIn('error in line 3', str(e)) with self.assertRaises(TimeoutError): - retry_check_table(table_name, timeout_sec=1, log=False) + retry_check_table(table_name, timeout_sec=0.25, log=False) def test_tcp_transactions(self): if QDB_FIXTURE.http: @@ -1156,47 +1156,46 @@ def run_with_fixtures(args): global QDB_FIXTURE global TLS_PROXY_FIXTURE global BUILD_MODE - last_version = None latest_protocol = sorted(list(qls.ProtocolVersion))[-1] - for questdb_dir, auth, http, protocol_version, build_mode in itertools.product( - iter_versions(args), - (False, True), # auth - (False, True), # http - [None] + list(qls.ProtocolVersion), # None is for `auto` - list(qls.BuildMode)): - if (build_mode in (qls.BuildMode.API, qls.BuildMode.ENV)) and (protocol_version != latest_protocol): - continue - if http and auth: - continue - if auth and (protocol_version != latest_protocol): - continue - print( - f'Running tests [auth={auth}, http={http}, build_mode={build_mode}, protocol_version={protocol_version}]') + for questdb_dir, auth in itertools.product(iter_versions(args), (False, True)): QDB_FIXTURE = QuestDbFixture( questdb_dir, - auth=auth, - http=http, - protocol_version=protocol_version) + auth=auth) TLS_PROXY_FIXTURE = None - BUILD_MODE = build_mode try: + print(f'>>>> STARTING {questdb_dir} [auth={auth}] <<<<') QDB_FIXTURE.start() - # Read the version _after_ a first start so it can rely - # on the live one from the `select build` query. - last_version = QDB_FIXTURE.version - port_to_proxy = QDB_FIXTURE.http_server_port \ - if http else QDB_FIXTURE.line_tcp_port - TLS_PROXY_FIXTURE = TlsProxyFixture(port_to_proxy) - TLS_PROXY_FIXTURE.start() - - QDB_FIXTURE.drop_all_tables() - test_prog = unittest.TestProgram(exit=False) - if not test_prog.result.wasSuccessful(): - sys.exit(1) + for http, protocol_version, build_mode in itertools.product( + (False, True), # http + [None] + list(qls.ProtocolVersion), # None is for `auto` + list(qls.BuildMode)): + if (build_mode in (qls.BuildMode.API, qls.BuildMode.ENV)) and (protocol_version != latest_protocol): + continue + if http and auth: + continue + if auth and (protocol_version != latest_protocol): + continue + print( + f'Running tests [auth={auth}, http={http}, build_mode={build_mode}, protocol_version={protocol_version}]') + # Read the version _after_ a first start so it can rely + # on the live one from the `select build` query. + BUILD_MODE = build_mode + QDB_FIXTURE.http = http + QDB_FIXTURE.protocol_version = protocol_version + port_to_proxy = QDB_FIXTURE.http_server_port \ + if http else QDB_FIXTURE.line_tcp_port + TLS_PROXY_FIXTURE = TlsProxyFixture(port_to_proxy) + TLS_PROXY_FIXTURE.start() + try: + QDB_FIXTURE.drop_all_tables() + test_prog = unittest.TestProgram(exit=False) + if not test_prog.result.wasSuccessful(): + sys.exit(1) + finally: + if TLS_PROXY_FIXTURE: + TLS_PROXY_FIXTURE.stop() finally: - if TLS_PROXY_FIXTURE: - TLS_PROXY_FIXTURE.stop() QDB_FIXTURE.stop() From 54f696940b79b73b561a6ba4e79fbade654ed76d Mon Sep 17 00:00:00 2001 From: amunra Date: Fri, 16 May 2025 22:52:16 +0100 Subject: [PATCH 092/119] skipping even more redundant tests --- system_test/test.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/system_test/test.py b/system_test/test.py index 4271e965..24415b39 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -938,6 +938,8 @@ def test_malformed_auth3(self): def test_tls_insecure_skip_verify(self): if BUILD_MODE != qls.BuildMode.API: self.skipTest('BuildMode.API-only test') + if QDB_FIXTURE.protocol_version != sorted(list(qls.ProtocolVersion))[-1]: + self.skipTest('Skipping tls test for non-latest protocol version') protocol = qls.Protocol.HTTPS if QDB_FIXTURE.http else qls.Protocol.TCPS auth = AUTH if QDB_FIXTURE.auth else {} sender = qls.Sender( @@ -952,6 +954,8 @@ def test_tls_insecure_skip_verify(self): def test_tls_roots(self): if BUILD_MODE != qls.BuildMode.API: self.skipTest('BuildMode.API-only test') + if QDB_FIXTURE.protocol_version != sorted(list(qls.ProtocolVersion))[-1]: + self.skipTest('Skipping tls test for non-latest protocol version') protocol = qls.Protocol.HTTPS if QDB_FIXTURE.http else qls.Protocol.TCPS auth = auth = AUTH if QDB_FIXTURE.auth else {} sender = qls.Sender( @@ -966,6 +970,8 @@ def test_tls_roots(self): def _test_tls_ca(self, tls_ca): if BUILD_MODE != qls.BuildMode.API: self.skipTest('BuildMode.API-only test') + if QDB_FIXTURE.protocol_version != sorted(list(qls.ProtocolVersion))[-1]: + self.skipTest('Skipping tls test for non-latest protocol version') protocol = qls.Protocol.HTTPS if QDB_FIXTURE.http else qls.Protocol.TCPS prev_ssl_cert_file = os.environ.get('SSL_CERT_FILE') try: From bba4192fc7eb34c8196348ff226a47f5731cf1b1 Mon Sep 17 00:00:00 2001 From: amunra Date: Fri, 16 May 2025 22:55:44 +0100 Subject: [PATCH 093/119] ignoring profile result from git-tracked files --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index cd332d3a..9d954b94 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ cmake-build-release .cache questdb-rs/Cargo.lock include/questdb/ingress/line_sender.gen.h -cython/questdb/ingress/line_sender.pxd \ No newline at end of file +cython/questdb/ingress/line_sender.pxd +profile.out \ No newline at end of file From 039ecf611d470ff4e6b8b42ee085853a9e10c3ef Mon Sep 17 00:00:00 2001 From: victor Date: Sat, 17 May 2025 10:14:08 +0800 Subject: [PATCH 094/119] use `assert_err_contains` in all tests. --- questdb-rs/src/ingress/mod.rs | 2 +- questdb-rs/src/tests/http.rs | 161 +++++++++++++++------------------- 2 files changed, 70 insertions(+), 93 deletions(-) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 18f2c546..8f06de4b 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -1123,7 +1123,7 @@ impl Buffer { if self.version == ProtocolVersion::V1 { return Err(error::fmt!( ProtocolVersionError, - "line protocol version v1 does not support array datatype", + "Protocol version v1 does not support array datatype", )); } let ndim = view.ndim(); diff --git a/questdb-rs/src/tests/http.rs b/questdb-rs/src/tests/http.rs index 9773590c..d4c29a02 100644 --- a/questdb-rs/src/tests/http.rs +++ b/questdb-rs/src/tests/http.rs @@ -50,7 +50,7 @@ fn test_two_lines( .at_now()?; let buffer2 = buffer.clone(); - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); @@ -63,12 +63,12 @@ fn test_two_lines( server.send_http_response_q(HttpResponse::empty())?; - Ok(()) + Ok(server) }); let res = sender.flush(&mut buffer); - server_thread.join().unwrap()?; + _ = server_thread.join().unwrap()?; res?; @@ -91,7 +91,7 @@ fn test_text_plain_error( .at_now()?; buffer.table("test")?.column_f64("sym", 2.0)?.at_now()?; let buffer2 = buffer.clone(); - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); @@ -105,19 +105,17 @@ fn test_text_plain_error( .with_body_str("bad wombat"), )?; - Ok(()) + Ok(server) }); - let res = sender.flush(&mut buffer); - - server_thread.join().unwrap()?; - - assert!(res.is_err()); - let err = res.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ServerFlushError); - assert_eq!(err.msg(), "Could not flush buffer: bad wombat"); + assert_err_contains( + sender.flush(&mut buffer), + ErrorCode::ServerFlushError, + "Could not flush buffer: bad wombat", + ); assert!(!buffer.is_empty()); + _ = server_thread.join().unwrap()?; Ok(()) } @@ -137,7 +135,7 @@ fn test_bad_json_error( buffer.table("test")?.column_f64("sym", 2.0)?.at_now()?; let buffer2 = buffer.clone(); - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); @@ -152,12 +150,12 @@ fn test_bad_json_error( })), )?; - Ok(()) + Ok(server) }); let res = sender.flush_and_keep(&buffer); - server_thread.join().unwrap()?; + _ = server_thread.join().unwrap()?; assert!(res.is_err()); let err = res.unwrap_err(); @@ -185,7 +183,7 @@ fn test_json_error( buffer.table("test")?.column_f64("sym", 2.0)?.at_now()?; let buffer2 = buffer.clone(); - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); @@ -203,21 +201,16 @@ fn test_json_error( })), )?; - Ok(()) + Ok(server) }); - let res = sender.flush_and_keep(&buffer); - - server_thread.join().unwrap()?; - - assert!(res.is_err()); - let err = res.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ServerFlushError); - assert_eq!( - err.msg(), - "Could not flush buffer: failed to parse line protocol: invalid field format [id: ABC-2, code: invalid, line: 2]" + assert_err_contains( + sender.flush_and_keep(&buffer), + ErrorCode::ServerFlushError, + "Could not flush buffer: failed to parse line protocol: invalid field format [id: ABC-2, code: invalid, line: 2]", ); + _ = server_thread.join().unwrap()?; Ok(()) } @@ -258,7 +251,7 @@ fn test_old_server_without_ilp_http_support( .at_now()?; let buffer2 = buffer.clone(); - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); @@ -272,21 +265,16 @@ fn test_old_server_without_ilp_http_support( .with_body_str("Not Found"), )?; - Ok(()) + Ok(server) }); - let res = sender.flush_and_keep(&buffer); - - server_thread.join().unwrap()?; - - assert!(res.is_err()); - let err = res.unwrap_err(); - assert_eq!(err.code(), ErrorCode::HttpNotSupported); - assert_eq!( - err.msg(), - "Could not flush buffer: HTTP endpoint does not support ILP." + assert_err_contains( + sender.flush_and_keep(&buffer), + ErrorCode::HttpNotSupported, + "Could not flush buffer: HTTP endpoint does not support ILP.", ); + _ = server_thread.join().unwrap()?; Ok(()) } @@ -309,7 +297,7 @@ fn test_http_basic_auth( .at_now()?; let buffer2 = buffer.clone(); - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); @@ -322,12 +310,12 @@ fn test_http_basic_auth( server.send_http_response_q(HttpResponse::empty())?; - Ok(()) + Ok(server) }); let res = sender.flush(&mut buffer); - server_thread.join().unwrap()?; + _ = server_thread.join().unwrap()?; res?; @@ -350,7 +338,7 @@ fn test_unauthenticated( .at_now()?; let buffer2 = buffer.clone(); - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); @@ -364,23 +352,17 @@ fn test_unauthenticated( .with_header("WWW-Authenticate", "Basic realm=\"Our Site\""), )?; - Ok(()) + Ok(server) }); - let res = sender.flush(&mut buffer); - - server_thread.join().unwrap()?; - - assert!(res.is_err()); - let err = res.unwrap_err(); - assert_eq!(err.code(), ErrorCode::AuthError); - assert_eq!( - err.msg(), - "Could not flush buffer: HTTP endpoint authentication error: Unauthorized [code: 401]" + assert_err_contains( + sender.flush(&mut buffer), + ErrorCode::AuthError, + "Could not flush buffer: HTTP endpoint authentication error: Unauthorized [code: 401]", ); - assert!(!buffer.is_empty()); + _ = server_thread.join().unwrap()?; Ok(()) } @@ -402,7 +384,7 @@ fn test_token_auth( .at_now()?; let buffer2 = buffer.clone(); - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); @@ -412,12 +394,12 @@ fn test_token_auth( server.send_http_response_q(HttpResponse::empty())?; - Ok(()) + Ok(server) }); let res = sender.flush(&mut buffer); - server_thread.join().unwrap()?; + _ = server_thread.join().unwrap()?; res?; @@ -471,7 +453,7 @@ fn test_tls( .column_f64("f1", 0.5)? .at(TimestampNanos::new(10000000))?; let buffer2 = buffer.clone(); - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept_tls_sync()?; let req = server.recv_http_q()?; assert_eq!(req.method(), "POST"); @@ -480,12 +462,12 @@ fn test_tls( server.send_http_response_q(HttpResponse::empty())?; - Ok(()) + Ok(server) }); let res = sender.flush_and_keep(&buffer); - server_thread.join().unwrap()?; + _ = server_thread.join().unwrap()?; // Unpacking the error here allows server errors to bubble first. res?; @@ -510,7 +492,7 @@ fn test_user_agent( .column_f64("f1", 0.5)? .at(TimestampNanos::new(10000000))?; let buffer2 = buffer.clone(); - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; let req = server.recv_http_q()?; assert_eq!(req.header("user-agent"), Some("wallabies/1.2.99")); @@ -518,12 +500,12 @@ fn test_user_agent( server.send_http_response_q(HttpResponse::empty())?; - Ok(()) + Ok(server) }); let res = sender.flush_and_keep(&buffer); - server_thread.join().unwrap()?; + _ = server_thread.join().unwrap()?; // Unpacking the error here allows server errors to bubble first. res?; @@ -549,7 +531,7 @@ fn test_two_retries( .column_f64("f1", 0.5)? .at(TimestampNanos::new(10000000))?; let buffer2 = buffer.clone(); - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; let req = server.recv_http_q()?; assert_eq!(req.body(), buffer2.as_bytes()); @@ -582,12 +564,12 @@ fn test_two_retries( server.send_http_response_q(HttpResponse::empty())?; - Ok(()) + Ok(server) }); let res = sender.flush_and_keep(&buffer); - server_thread.join().unwrap()?; + _ = server_thread.join().unwrap()?; // Unpacking the error here allows server errors to bubble first. res?; @@ -613,7 +595,7 @@ fn test_one_retry( .at(TimestampNanos::new(10000000))?; let buffer2 = buffer.clone(); - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; let req = server.recv_http_q()?; assert_eq!(req.body(), buffer2.as_bytes()); @@ -646,17 +628,16 @@ fn test_one_retry( }; assert_eq!(err.kind(), ErrorKind::TimedOut); - Ok(()) + Ok(server) }); - let res = sender.flush_and_keep(&buffer); - - server_thread.join().unwrap()?; - - let err = res.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ServerFlushError); - assert_eq!(err.msg(), "Could not flush buffer: error 2"); + assert_err_contains( + sender.flush_and_keep(&buffer), + ErrorCode::ServerFlushError, + "Could not flush buffer: error 2", + ); + _ = server_thread.join().unwrap()?; Ok(()) } @@ -690,29 +671,26 @@ fn test_transactional( let buffer3 = buffer2.clone(); assert!(buffer2.transactional()); - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; let req = server.recv_http_q()?; assert_eq!(req.body(), buffer3.as_bytes()); server.send_http_response_q(HttpResponse::empty())?; - Ok(()) + Ok(server) }); - let res = sender.flush_and_keep_with_flags(&buffer1, true); - assert!(res.is_err()); - let err = res.unwrap_err(); - assert_eq!(err.code(), ErrorCode::InvalidApiCall); - assert_eq!( - err.msg(), + assert_err_contains( + sender.flush_and_keep_with_flags(&buffer1, true), + ErrorCode::InvalidApiCall, "Buffer contains lines for multiple tables. \ Transactional flushes are only supported for buffers containing lines for a single table.", ); let res = sender.flush_and_keep_with_flags(&buffer2, true); - server_thread.join().unwrap()?; + _ = server_thread.join().unwrap()?; // Unpacking the error here allows server errors to bubble first. res?; @@ -729,7 +707,7 @@ fn _test_sender_auto_detect_protocol_version( .configure_settings_response(supported_versions.as_deref().unwrap_or(&[])); let sender_builder = server.lsb_http(); - let server_thread = std::thread::spawn(move || -> io::Result<()> { + let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; let req = server.recv_http_q()?; assert_eq!(req.method(), "GET"); @@ -752,7 +730,7 @@ fn _test_sender_auto_detect_protocol_version( let req = server.recv_http_q()?; assert_eq!(req.body(), exp); server.send_http_response_q(HttpResponse::empty())?; - Ok(()) + Ok(server) }); let mut sender = sender_builder.build()?; @@ -764,8 +742,8 @@ fn _test_sender_auto_detect_protocol_version( .column_f64("f1", 0.5)? .at(TimestampNanos::new(10000000))?; let res = sender.flush(&mut buffer); - server_thread.join().unwrap()?; res?; + _ = server_thread.join().unwrap()?; Ok(()) } @@ -808,10 +786,9 @@ fn test_sender_auto_protocol_version_unsupported_client() -> TestResult { ErrorCode::ProtocolVersionError, "Server does not support current client", ); - let server = server_thread.join().unwrap()?; // We keep the server around til the end of the test to ensure that the response is fully received. - drop(server); + _ = server_thread.join().unwrap()?; Ok(()) } @@ -825,7 +802,7 @@ fn test_buffer_protocol_version1_not_support_array() -> TestResult { assert_err_contains( res, ErrorCode::ProtocolVersionError, - "line protocol version v1 does not support array datatype", + "Protocol version v1 does not support array datatype", ); Ok(()) } From b3d982f23a9df47948edf5e7aee2f9a317e33e81 Mon Sep 17 00:00:00 2001 From: victor Date: Sat, 17 May 2025 11:50:26 +0800 Subject: [PATCH 095/119] use `assert_err_contains` in all tests. --- system_test/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system_test/test.py b/system_test/test.py index 24415b39..095e0427 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -705,7 +705,7 @@ def test_protocol_version_v1_array_unsupported(self): sender.column_f64_arr('f64_arr1', array1) sender.at_now() except qls.SenderError as e: - self.assertIn('line protocol version v1 does not support array datatype', str(e)) + self.assertIn('Protocol version v1 does not support array datatype', str(e)) def _test_example(self, bin_name, table_name, tls=False): if BUILD_MODE != qls.BuildMode.API: From 7c75cdb5bf646b4c364820071bb4bb33788b855d Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Mon, 19 May 2025 15:49:44 +0100 Subject: [PATCH 096/119] fixed buffer/sender version checks --- cpp_test/test_line_sender.cpp | 19 ++++++----- questdb-rs/src/ingress/mod.rs | 59 +++++++++------------------------- questdb-rs/src/tests/sender.rs | 21 +++++++++++- 3 files changed, 45 insertions(+), 54 deletions(-) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index b1f2769a..68e1bb06 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -177,16 +177,15 @@ TEST_CASE("line_sender c api basics") 2.7, 48121.5, 4.3}; - CHECK( - ::line_sender_buffer_column_f64_arr( - buffer, - arr_name, - rank, - shape, - strides, - reinterpret_cast(arr_data.data()), - sizeof(arr_data), - &err)); + CHECK(::line_sender_buffer_column_f64_arr( + buffer, + arr_name, + rank, + shape, + strides, + reinterpret_cast(arr_data.data()), + sizeof(arr_data), + &err)); CHECK(::line_sender_buffer_at_nanos(buffer, 10000000, &err)); CHECK(server.recv() == 0); CHECK(::line_sender_buffer_size(buffer) == 150); diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 8f06de4b..51ebef6e 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -1363,12 +1363,6 @@ pub struct Sender { connected: bool, max_buf_size: usize, default_protocol_version: ProtocolVersion, - #[cfg(feature = "ilp-over-http")] - /// List of protocol versions supported by the server. - /// It will be `None` when user explicitly sets `protocol_version` (no first http round trip). - /// Note that when connecting to older servers (responding with 404 or missing version data), - /// it will be init as [`ProtocolVersion::V1`] instead of leaving `None` - supported_protocol_versions: Option>, } impl std::fmt::Debug for Sender { @@ -2571,18 +2565,10 @@ impl SenderBuilder { } }; - let default_protocol_version; - #[cfg(feature = "ilp-over-http")] - let mut supported_protocol_versions: Option> = None; - - match self.protocol_version.deref() { - Some(v) => { - default_protocol_version = *v; - } + let default_protocol_version = match self.protocol_version.deref() { + Some(v) => *v, None => match self.protocol { - Protocol::Tcp | Protocol::Tcps => { - default_protocol_version = ProtocolVersion::V1; - } + Protocol::Tcp | Protocol::Tcps => ProtocolVersion::V1, #[cfg(feature = "ilp-over-http")] Protocol::Http | Protocol::Https => { if let ProtocolHandler::Http(http_state) = &handler { @@ -2594,22 +2580,21 @@ impl SenderBuilder { ); let versions = get_supported_protocol_versions(http_state, settings_url)?; if versions.contains(&ProtocolVersion::V2) { - default_protocol_version = ProtocolVersion::V2; + ProtocolVersion::V2 } else if versions.contains(&ProtocolVersion::V1) { - default_protocol_version = ProtocolVersion::V1; + ProtocolVersion::V1 } else { return Err(error::fmt!( ProtocolVersionError, "Server does not support current client" )); } - supported_protocol_versions = Some(versions); } else { - default_protocol_version = ProtocolVersion::V1; + unreachable!("HTTP handler should be used for HTTP protocol"); } } }, - } + }; if auth.is_some() { descr.push_str("auth=on]"); @@ -2622,8 +2607,6 @@ impl SenderBuilder { connected: true, max_buf_size: *self.max_buf_size, default_protocol_version, - #[cfg(feature = "ilp-over-http")] - supported_protocol_versions, }; Ok(sender) @@ -2858,7 +2841,6 @@ impl Sender { )); } - #[cfg(feature = "ilp-over-http")] self.check_protocol_version(buf.version)?; let bytes = buf.as_bytes(); @@ -2990,27 +2972,18 @@ impl Sender { self.default_protocol_version } - #[cfg(feature = "ilp-over-http")] #[inline(always)] fn check_protocol_version(&self, version: ProtocolVersion) -> Result<()> { - match &self.handler { - ProtocolHandler::Socket(_) => Ok(()), - ProtocolHandler::Http(_) => { - match self.supported_protocol_versions { - Some(ref supported_line_protocols) => { - if supported_line_protocols.contains(&version) { - Ok(()) - } else { - Err(error::fmt!( - ProtocolVersionError, - "Line protocol version {} is not supported by current QuestDB Server", version)) - } - } - // `None` implies user set protocol_version explicitly - None => Ok(()), - } - } + if self.default_protocol_version != version { + return Err(error::fmt!( + ProtocolVersionError, + "Attempting to send with protocol version {} \ + but the sender is configured to use protocol version {}", + version, + self.default_protocol_version + )); } + Ok(()) } } diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index ae2632c2..5d9f3bd9 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -26,6 +26,7 @@ use crate::{ ingress::{ Buffer, CertificateAuthority, Sender, TableName, Timestamp, TimestampMicros, TimestampNanos, }, + tests::assert_err_contains, Error, ErrorCode, }; @@ -606,7 +607,7 @@ fn test_plain_to_tls_server() -> TestResult { let server = MockServer::new()?; let lsb = server.lsb_tcp().auth_timeout(Duration::from_millis(500))?; let server_jh = server.accept_tls(); - let maybe_sender = lsb.build(); + let maybe_sender = lsb.protocol_version(ProtocolVersion::V2)?.build(); let server_err = server_jh.join().unwrap().unwrap_err(); // The server failed to handshake, so disconnected the client. @@ -677,6 +678,24 @@ fn bad_uppercase_addr() { assert!(err.msg() == "Missing \"addr\" parameter in config string"); } +#[test] +fn tcp_mismatched_buffer_and_sender_version() -> TestResult { + let server = MockServer::new()?; + let mut sender = server + .lsb_tcp() + .protocol_version(ProtocolVersion::V2)? + .build()?; + let mut buffer = Buffer::new(ProtocolVersion::V1); + buffer.table("test")?.symbol("t1", "v1")?.at_now()?; + assert_err_contains( + sender.flush(&mut buffer), + ErrorCode::ProtocolVersionError, + "Attempting to send with protocol version v1 \ + but the sender is configured to use protocol version v2", + ); + Ok(()) +} + pub(crate) fn f64_to_bytes(name: &str, value: f64, version: ProtocolVersion) -> Vec { let mut buf = Vec::new(); buf.extend_from_slice(name.as_bytes()); From d9dd58b57eb95ae941a22c33a675177b2ab11e15 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Tue, 20 May 2025 13:49:27 +0100 Subject: [PATCH 097/119] refactoring of the C and C++ public apis and some minor Rust changes --- .bumpversion.cfg | 2 +- CMakeLists.txt | 3 +- cpp_test/test_line_sender.cpp | 55 +++++----- doc/SECURITY.md | 4 +- include/questdb/ingress/line_sender.h | 50 ++++----- include/questdb/ingress/line_sender.hpp | 130 ++++++++---------------- questdb-rs-ffi/Cargo.lock | 4 +- questdb-rs-ffi/Cargo.toml | 2 +- questdb-rs-ffi/src/lib.rs | 43 ++++---- questdb-rs/Cargo.toml | 2 +- questdb-rs/README.md | 2 +- questdb-rs/src/ingress/http.rs | 39 ++++--- questdb-rs/src/ingress/mod.rs | 113 ++++++++++++-------- questdb-rs/src/tests/http.rs | 2 +- questdb-rs/src/tests/sender.rs | 4 +- system_test/questdb_line_sender.py | 16 +-- 16 files changed, 229 insertions(+), 242 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 2a8db0cb..c2feba7f 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 4.0.4 +current_version = 5.0.0-rc1 commit = False tag = False diff --git a/CMakeLists.txt b/CMakeLists.txt index 0fd094fa..b2612af7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,6 @@ cmake_minimum_required(VERSION 3.15.0) -project(c-questdb-client VERSION 4.0.4) +project(c-questdb-client VERSION 5.0.0) +set(PROJECT_PRE_RELEASE "rc1") set(CPACK_PROJECT_NAME ${PROJECT_NAME}) set(CPACK_PROJECT_VERSION ${PROJECT_VERSION}) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 68e1bb06..70a841b5 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -131,7 +131,8 @@ TEST_CASE("line_sender c api basics") ::line_sender_opts* opts = ::line_sender_opts_new(::line_sender_protocol_tcp, host, server.port()); CHECK_NE(opts, nullptr); - line_sender_opts_protocol_version(opts, protocol_version_2, &err); + line_sender_opts_protocol_version( + opts, ::line_sender_protocol_version_2, &err); ::line_sender* sender = ::line_sender_build(opts, &err); line_sender_opts_free(opts); CHECK_NE(sender, nullptr); @@ -235,11 +236,12 @@ TEST_CASE("line_sender c++ connect disconnect") TEST_CASE("line_sender c++ api basics") { questdb::ingress::test::mock_server server; - questdb::ingress::line_sender sender{ + questdb::ingress::opts opts{ questdb::ingress::protocol::tcp, std::string("127.0.0.1"), - std::to_string(server.port()), - protocol_version_2}; + std::to_string(server.port())}; + opts.protocol_version(questdb::ingress::protocol_version::v2); + questdb::ingress::line_sender sender{opts}; CHECK_FALSE(sender.must_close()); server.accept(); CHECK(server.recv() == 0); @@ -508,17 +510,17 @@ TEST_CASE("Buffer move and copy ctor testing") const size_t init_buf_size = 128; questdb::ingress::line_sender_buffer buffer1{ - protocol_version_1, init_buf_size}; + questdb::ingress::protocol_version::v1, init_buf_size}; buffer1.table("buffer1"); CHECK(buffer1.peek() == "buffer1"); questdb::ingress::line_sender_buffer buffer2{ - protocol_version_1, 2 * init_buf_size}; + questdb::ingress::protocol_version::v1, 2 * init_buf_size}; buffer2.table("buffer2"); CHECK(buffer2.peek() == "buffer2"); questdb::ingress::line_sender_buffer buffer3{ - protocol_version_1, 3 * init_buf_size}; + questdb::ingress::protocol_version::v1, 3 * init_buf_size}; buffer3.table("buffer3"); CHECK(buffer3.peek() == "buffer3"); @@ -825,7 +827,8 @@ TEST_CASE("test timestamp_micros and timestamp_nanos::now()") TEST_CASE("Test Marker") { - questdb::ingress::line_sender_buffer buffer{protocol_version_1}; + questdb::ingress::line_sender_buffer buffer{ + questdb::ingress::protocol_version::v1}; buffer.clear_marker(); buffer.clear_marker(); @@ -881,18 +884,22 @@ TEST_CASE("Moved View") TEST_CASE("Empty Buffer") { - questdb::ingress::line_sender_buffer b1{protocol_version_2}; + questdb::ingress::line_sender_buffer b1{ + questdb::ingress::protocol_version::v2}; CHECK(b1.size() == 0); questdb::ingress::line_sender_buffer b2{std::move(b1)}; CHECK(b1.size() == 0); CHECK(b2.size() == 0); - questdb::ingress::line_sender_buffer b3{protocol_version_2}; + questdb::ingress::line_sender_buffer b3{ + questdb::ingress::protocol_version::v2}; b3 = std::move(b2); CHECK(b2.size() == 0); CHECK(b3.size() == 0); - questdb::ingress::line_sender_buffer b4{protocol_version_2}; + questdb::ingress::line_sender_buffer b4{ + questdb::ingress::protocol_version::v2}; b4.table("test").symbol("a", "b").at_now(); - questdb::ingress::line_sender_buffer b5{protocol_version_2}; + questdb::ingress::line_sender_buffer b5{ + questdb::ingress::protocol_version::v2}; b5 = std::move(b4); CHECK(b4.size() == 0); CHECK(b5.size() == 9); @@ -925,24 +932,25 @@ TEST_CASE("Opts from conf") TEST_CASE("HTTP basics") { questdb::ingress::opts opts1{ - questdb::ingress::protocol::http, "127.0.0.1", 1, protocol_version_2}; + questdb::ingress::protocol::http, "127.0.0.1", 1}; questdb::ingress::opts opts1conf = questdb::ingress::opts::from_conf( "http::addr=127.0.0.1:1;username=user;password=pass;request_timeout=" "5000;retry_timeout=5;protocol_version=2;"); questdb::ingress::opts opts2{ - questdb::ingress::protocol::https, - "localhost", - "1", - protocol_version_2}; + questdb::ingress::protocol::https, "localhost", "1"}; questdb::ingress::opts opts2conf = questdb::ingress::opts::from_conf( "http::addr=127.0.0.1:1;token=token;request_min_throughput=1000;retry_" "timeout=0;protocol_version=2;"); - opts1.username("user") + opts1.protocol_version(questdb::ingress::protocol_version::v2) + .username("user") .password("pass") .max_buf_size(1000000) .request_timeout(5000) .retry_timeout(5); - opts2.token("token").request_min_throughput(1000).retry_timeout(0); + opts2.protocol_version(questdb::ingress::protocol_version::v2) + .token("token") + .request_min_throughput(1000) + .retry_timeout(0); questdb::ingress::line_sender sender1{opts1}; questdb::ingress::line_sender sender1conf{opts1conf}; questdb::ingress::line_sender sender2{opts2}; @@ -980,7 +988,7 @@ TEST_CASE("line sender protocol version default v1 for tcp") .column("f1", 0.5) .at(questdb::ingress::timestamp_nanos{10000000}); - CHECK(sender.default_protocol_version() == protocol_version_1); + CHECK(sender.protocol_version() == questdb::ingress::protocol_version::v1); CHECK(server.recv() == 0); CHECK(buffer.size() == 31); sender.flush(buffer); @@ -992,11 +1000,12 @@ TEST_CASE("line sender protocol version default v1 for tcp") TEST_CASE("line sender protocol version v2") { questdb::ingress::test::mock_server server; - questdb::ingress::line_sender sender{ + questdb::ingress::opts opts{ questdb::ingress::protocol::tcp, std::string("127.0.0.1"), - std::to_string(server.port()), - protocol_version_2}; + std::to_string(server.port())}; + opts.protocol_version(questdb::ingress::protocol_version::v2); + questdb::ingress::line_sender sender{opts}; CHECK_FALSE(sender.must_close()); server.accept(); CHECK(server.recv() == 0); diff --git a/doc/SECURITY.md b/doc/SECURITY.md index ccb00f6b..7b87fd0a 100644 --- a/doc/SECURITY.md +++ b/doc/SECURITY.md @@ -35,7 +35,7 @@ A few important technical details on TLS: are managed centrally. For API usage: -* Rust: `SenderBuilder`'s [`auth`](https://docs.rs/questdb-rs/4.0.4/questdb/ingress/struct.SenderBuilder.html#method.auth) - and [`tls`](https://docs.rs/questdb-rs/4.0.4/questdb/ingress/struct.SenderBuilder.html#method.tls) methods. +* Rust: `SenderBuilder`'s [`auth`](https://docs.rs/questdb-rs/5.0.0-rc1/questdb/ingress/struct.SenderBuilder.html#method.auth) + and [`tls`](https://docs.rs/questdb-rs/5.0.0-rc1/questdb/ingress/struct.SenderBuilder.html#method.tls) methods. * C: [examples/line_sender_c_example_auth.c](../examples/line_sender_c_example_auth.c) * C++: [examples/line_sender_cpp_example_auth.cpp](../examples/line_sender_cpp_example_auth.cpp) diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 30a9ce02..991b3f74 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -109,21 +109,21 @@ typedef enum line_sender_protocol } line_sender_protocol; /** The line protocol version used to write data to buffer. */ -typedef enum protocol_version +typedef enum line_sender_protocol_version { /** * Version 1 of Ingestion Line Protocol. * This version is compatible with InfluxDB line protocol. */ - protocol_version_1 = 1, + line_sender_protocol_version_1 = 1, /** * Version 2 of Ingestion Line Protocol. * Uses a binary format serialization for f64, and supports * the array data type. */ - protocol_version_2 = 2, -} protocol_version; + line_sender_protocol_version_2 = 2, +} line_sender_protocol_version; /** Possible sources of the root certificates used to validate the server's * TLS certificate. */ @@ -314,7 +314,8 @@ typedef struct line_sender_buffer line_sender_buffer; * the same as the QuestDB server default. */ LINESENDER_API -line_sender_buffer* line_sender_buffer_new(protocol_version version); +line_sender_buffer* line_sender_buffer_new( + line_sender_protocol_version version); /** * Construct a `line_sender_buffer` with a custom maximum length for table @@ -325,7 +326,7 @@ line_sender_buffer* line_sender_buffer_new(protocol_version version); */ LINESENDER_API line_sender_buffer* line_sender_buffer_with_max_name_len( - size_t max_name_len, protocol_version version); + line_sender_protocol_version version, size_t max_name_len); /** Release the `line_sender_buffer` object. */ LINESENDER_API @@ -756,7 +757,7 @@ bool line_sender_opts_token_y( LINESENDER_API bool line_sender_opts_protocol_version( line_sender_opts* opts, - protocol_version version, + line_sender_protocol_version version, line_sender_error** err_out); /** @@ -918,31 +919,32 @@ line_sender* line_sender_from_conf( LINESENDER_API line_sender* line_sender_from_env(line_sender_error** err_out); -/// Returns sender's default protocol version. -/// 1. User-set value via [`line_sender_opts_protocol_version`] -/// 2. V1 for TCP/TCPS (legacy protocol) -/// 3. Auto-detected version for HTTP/HTTPS +/** + * Return the sender's protocol version. + * This is either the protocol version that was set explicitly, + * or the one that was auto-detected during the connection process. + * If connecting via TCP and not overridden, the value is V1. + */ LINESENDER_API -protocol_version line_sender_default_protocol_version( +line_sender_protocol_version line_sender_get_protocol_version( const line_sender* sender); /** - * Construct a `line_sender_buffer` with a `max_name_len` of `127` and sender's - * default protocol version - * which is the same as the QuestDB server default. + * Returns the configured max_name_len, or the default value of 127. */ -line_sender_buffer* line_sender_buffer_new_for_sender( - const line_sender* sender); +LINESENDER_API +size_t line_sender_get_max_name_len(const line_sender* sender); /** - * Construct a `line_sender_buffer` with sender's default protocol version and - * a custom maximum length for table and column names. This should match the - * `cairo.max.file.name.length` setting of the QuestDB server you're - * connecting to. If the server does not configure it, the default is `127`, - * and you can call `line_sender_buffer_new_for_sender()` instead. + * Construct a `line_sender_buffer` with the sender's + * configured protocol version and other parameters. + * This is equivalent to calling: + * line_sender_buffer_new( + * line_sender_get_protocol_version(sender), + * line_sender_get_max_name_len(sender)) */ -line_sender_buffer* line_sender_buffer_with_max_name_len_for_sender( - const line_sender* sender, size_t max_name_len); +line_sender_buffer* line_sender_buffer_new_for_sender( + const line_sender* sender); /** * Tell whether the sender is no longer usable and must be closed. diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 901e8012..8d5500e2 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -100,6 +100,15 @@ enum class protocol https, }; +enum class protocol_version +{ + /** Ingestion Line Protocol v1. */ + v1 = 1, + + /** Ingestion Line Protocol v2. */ + v2 = 2, +}; + /* Possible sources of the root certificates used to validate the server's TLS * certificate. */ enum class ca @@ -390,18 +399,11 @@ class line_sender_buffer { public: explicit line_sender_buffer( - protocol_version protocol_version, - size_t init_buf_size = 64 * 1024) noexcept - : line_sender_buffer{protocol_version, init_buf_size, 127} - { - } - - line_sender_buffer( protocol_version version, - size_t init_buf_size, - size_t max_name_len) noexcept + size_t init_buf_size = 64 * 1024, + size_t max_name_len = 127) noexcept : _impl{nullptr} - , _protocol_version(version) + , _protocol_version{version} , _init_buf_size{init_buf_size} , _max_name_len{max_name_len} { @@ -814,7 +816,9 @@ class line_sender_buffer if (!_impl) { _impl = ::line_sender_buffer_with_max_name_len( - _max_name_len, _protocol_version); + static_cast<::line_sender_protocol_version>( + static_cast(_protocol_version)), + _max_name_len); ::line_sender_buffer_reserve(_impl, _init_buf_size); } } @@ -833,7 +837,7 @@ class _user_agent static inline ::line_sender_utf8 name() { // Maintained by .bumpversion.cfg - static const char user_agent[] = "questdb/c++/4.0.4"; + static const char user_agent[] = "questdb/c++/5.0.0-rc1"; ::line_sender_utf8 utf8 = ::line_sender_utf8_assert(sizeof(user_agent) - 1, user_agent); return utf8; @@ -892,28 +896,6 @@ class opts ::line_sender_opts_user_agent, _impl, _user_agent::name()); } - /** - * Create a new `opts` instance with the given protocol, hostname and port. - * @param[in] protocol The protocol to use. - * @param[in] host The QuestDB database host. - * @param[in] port The QuestDB tcp or http port. - * @param[in] version The protocol version to use. - * validation. - */ - opts( - protocol protocol, - utf8_view host, - uint16_t port, - protocol_version version) noexcept - : _impl{::line_sender_opts_new( - static_cast<::line_sender_protocol>(protocol), host._impl, port)} - { - line_sender_error::wrapped_call( - ::line_sender_opts_user_agent, _impl, _user_agent::name()); - line_sender_error::wrapped_call( - ::line_sender_opts_protocol_version, _impl, version); - } - /** * Create a new `opts` instance with the given protocol, hostname and * service name. @@ -931,30 +913,6 @@ class opts ::line_sender_opts_user_agent, _impl, _user_agent::name()); } - /** - * Create a new `opts` instance with the given protocol, hostname and - * service name. - * @param[in] protocol The protocol to use. - * @param[in] host The QuestDB database host. - * @param[in] port The QuestDB tcp or http port as service name. - * @param[in] version The protocol version to use. - */ - opts( - protocol protocol, - utf8_view host, - utf8_view port, - protocol_version version) noexcept - : _impl{::line_sender_opts_new_service( - static_cast<::line_sender_protocol>(protocol), - host._impl, - port._impl)} - { - line_sender_error::wrapped_call( - ::line_sender_opts_user_agent, _impl, _user_agent::name()); - line_sender_error::wrapped_call( - ::line_sender_opts_protocol_version, _impl, version); - } - opts(const opts& other) noexcept : _impl{::line_sender_opts_clone(other._impl)} { @@ -1166,6 +1124,16 @@ class opts return *this; } + opts& protocol_version(protocol_version version) noexcept + { + const auto c_protocol_version = + static_cast<::line_sender_protocol_version>( + static_cast(version)); + line_sender_error::wrapped_call( + ::line_sender_opts_protocol_version, _impl, c_protocol_version); + return *this; + } + ~opts() noexcept { reset(); @@ -1255,24 +1223,6 @@ class line_sender { } - line_sender( - protocol protocol, - utf8_view host, - uint16_t port, - protocol_version version) - : line_sender{opts{protocol, host, port, version}} - { - } - - line_sender( - protocol protocol, - utf8_view host, - utf8_view port, - protocol_version version) - : line_sender{opts{protocol, host, port, version}} - { - } - line_sender(const opts& opts) : _impl{ line_sender_error::wrapped_call(::line_sender_build, opts._impl)} @@ -1300,11 +1250,20 @@ class line_sender return *this; } - line_sender_buffer new_buffer( - size_t init_buf_size = 64 * 1024, size_t max_name_len = 127) noexcept + protocol_version protocol_version() const noexcept + { + ensure_impl(); + return static_cast( + static_cast(::line_sender_get_protocol_version(_impl))); + } + + line_sender_buffer new_buffer(size_t init_buf_size = 64 * 1024) noexcept { + ensure_impl(); return line_sender_buffer{ - default_protocol_version(), init_buf_size, max_name_len}; + this->protocol_version(), + init_buf_size, + ::line_sender_get_max_name_len(_impl)}; } /** @@ -1359,22 +1318,13 @@ class line_sender } else { - line_sender_buffer buffer2{default_protocol_version(), 0}; + line_sender_buffer buffer2{this->protocol_version(), 0}; buffer2.may_init(); line_sender_error::wrapped_call( ::line_sender_flush_and_keep, _impl, buffer2._impl); } } - /** - * Returns the QuestDB server's recommended default line protocol version. - */ - protocol_version default_protocol_version() - { - ensure_impl(); - return line_sender_default_protocol_version(_impl); - } - /** * Check if an error occurred previously and the sender must be closed. * This happens when there was an earlier failure. @@ -1405,7 +1355,7 @@ class line_sender } private: - void ensure_impl() + void ensure_impl() const { if (!_impl) throw line_sender_error{ diff --git a/questdb-rs-ffi/Cargo.lock b/questdb-rs-ffi/Cargo.lock index 41476357..7baf1e3d 100644 --- a/questdb-rs-ffi/Cargo.lock +++ b/questdb-rs-ffi/Cargo.lock @@ -266,7 +266,7 @@ dependencies = [ [[package]] name = "questdb-rs" -version = "4.0.4" +version = "5.0.0-rc1" dependencies = [ "base64ct", "dns-lookup", @@ -292,7 +292,7 @@ dependencies = [ [[package]] name = "questdb-rs-ffi" -version = "4.0.4" +version = "5.0.0-rc1" dependencies = [ "cbindgen", "libc", diff --git a/questdb-rs-ffi/Cargo.toml b/questdb-rs-ffi/Cargo.toml index d9d8409c..84298864 100644 --- a/questdb-rs-ffi/Cargo.toml +++ b/questdb-rs-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "questdb-rs-ffi" -version = "4.0.4" +version = "5.0.0-rc1" edition = "2021" publish = false diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 38bb3bb9..402c3305 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -227,16 +227,17 @@ impl From for Protocol { } } -/// The version of Line Protocol used for [`Buffer`]. +/// The version of Ingestion Line Protocol used to communicate with the server. #[repr(C)] #[derive(Debug, Copy, Clone)] pub enum ProtocolVersion { /// Version 1 of Line Protocol. - /// Uses text format serialization for f64. + /// Full-text protocol. + /// When used over HTTP, it is compatible with the InfluxDB line protocol. V1 = 1, /// Version 2 of Ingestion Line Protocol. - /// Uses binary format serialization for f64, and support array data type. + /// Uses binary format serialization for f64, and supports the array data type. V2 = 2, } @@ -613,10 +614,10 @@ pub unsafe extern "C" fn line_sender_buffer_new( /// call `line_sender_buffer_new()` instead. #[no_mangle] pub unsafe extern "C" fn line_sender_buffer_with_max_name_len( - max_name_len: size_t, version: ProtocolVersion, + max_name_len: size_t, ) -> *mut line_sender_buffer { - let buffer = Buffer::with_max_name_len(max_name_len, version.into()); + let buffer = Buffer::with_max_name_len(version.into(), max_name_len); Box::into_raw(Box::new(line_sender_buffer(buffer))) } @@ -1411,15 +1412,20 @@ unsafe fn unwrap_sender_mut<'a>(sender: *mut line_sender) -> &'a mut Sender { &mut (*sender).0 } -/// Returns sender's default protocol version. -/// 1. User-set value via [`line_sender_opts_protocol_version`] -/// 2. V1 for TCP/TCPS (legacy protocol) -/// 3. Auto-detected version for HTTP/HTTPS +/// Return the sender's protocol version. +/// This is either the protocol version that was set explicitly, +/// or the one that was auto-detected during the connection process. +/// If connecting via TCP and not overridden, the value is V1. #[no_mangle] -pub unsafe extern "C" fn line_sender_default_protocol_version( +pub unsafe extern "C" fn line_sender_get_protocol_version( sender: *const line_sender, ) -> ProtocolVersion { - unwrap_sender(sender).default_protocol_version().into() + unwrap_sender(sender).protocol_version().into() +} + +#[no_mangle] +pub unsafe extern "C" fn line_sender_get_max_name_len(sender: *const line_sender) -> size_t { + unwrap_sender(sender).max_name_len() } /// Construct a `line_sender_buffer` with a `max_name_len` of `127` and sender's default protocol version @@ -1433,21 +1439,6 @@ pub unsafe extern "C" fn line_sender_buffer_new_for_sender( Box::into_raw(Box::new(line_sender_buffer(buffer))) } -/// Construct a `line_sender_buffer` with sender's default protocol version and a custom maximum -/// length for table and column names. This should match the `cairo.max.file.name.length` setting of -/// the QuestDB server you're connecting to. -/// If the server does not configure it, the default is `127`, and you can -/// call `line_sender_buffer_new_for_sender()` instead. -#[no_mangle] -pub unsafe extern "C" fn line_sender_buffer_with_max_name_len_for_sender( - sender: *const line_sender, - max_name_len: size_t, -) -> *mut line_sender_buffer { - let sender = unwrap_sender(sender); - let buffer = sender.new_buffer_with_max_name_len(max_name_len); - Box::into_raw(Box::new(line_sender_buffer(buffer))) -} - /// Tell whether the sender is no longer usable and must be closed. /// This happens when there was an earlier failure. /// This fuction is specific to TCP and is not relevant for HTTP. diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index c7abcfd1..a927cfac 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "questdb-rs" -version = "4.0.4" +version = "5.0.0-rc1" edition = "2021" license = "Apache-2.0" description = "QuestDB Client Library for Rust" diff --git a/questdb-rs/README.md b/questdb-rs/README.md index c44c9dcb..19a1cc70 100644 --- a/questdb-rs/README.md +++ b/questdb-rs/README.md @@ -50,7 +50,7 @@ fn main() -> Result<()> { ## Docs Most of the client documentation is on the -[`ingress`](https://docs.rs/questdb-rs/4.0.4/questdb/ingress/) module page. +[`ingress`](https://docs.rs/questdb-rs/5.0.0-rc1/questdb/ingress/) module page. ## Crate features diff --git a/questdb-rs/src/ingress/http.rs b/questdb-rs/src/ingress/http.rs index 7955933c..313b245f 100644 --- a/questdb-rs/src/ingress/http.rs +++ b/questdb-rs/src/ingress/http.rs @@ -1,4 +1,5 @@ use super::conf::ConfigSetting; +use super::MAX_NAME_LEN_DEFAULT; use crate::error::fmt; use crate::{error, Error}; use base64ct::Base64; @@ -406,14 +407,18 @@ pub(super) fn http_send_with_retries( retry_http_send(state, buf, request_timeout, retry_timeout, last_rep) } -/// Return and the server's all supported protocol versions. -/// - For modern servers: Returns explicit version list from QuestDB's server's `/settings` endpoint response -/// - For legacy servers (404 response or missing version field): Automatically falls back to [`ProtocolVersion::V1`] -pub(super) fn get_supported_protocol_versions( +/// Read the server settings from the `/settings` endpoint. +/// This function returns: +/// - A list of supported protocol versions: Default is V1. +/// - The server's max name length: Default is 127. +/// +/// If the server does not support the `/settings` endpoint (404), it returns +/// default values. +pub(super) fn read_server_settings( state: &HttpHandlerState, settings_url: &str, -) -> Result, Error> { - let mut support_versions: Vec = vec![]; +) -> Result<(Vec, usize), Error> { + let default_protocol_version = ProtocolVersion::V1; let response = match http_get_with_retries( state, @@ -426,8 +431,7 @@ pub(super) fn get_supported_protocol_versions( let status = res.status(); _ = res.into_body().read_to_vec(); if status.as_u16() == 404 { - support_versions.push(ProtocolVersion::V1); - return Ok(support_versions); + return Ok((vec![default_protocol_version], MAX_NAME_LEN_DEFAULT)); } return Err(fmt!( ProtocolVersionError, @@ -443,8 +447,7 @@ pub(super) fn get_supported_protocol_versions( let e = match err { ureq::Error::StatusCode(code) => { if code == 404 { - support_versions.push(ProtocolVersion::V1); - return Ok(support_versions); + return Ok((vec![default_protocol_version], MAX_NAME_LEN_DEFAULT)); } else { fmt!( ProtocolVersionError, @@ -468,7 +471,7 @@ pub(super) fn get_supported_protocol_versions( }; let (_, body) = response.into_parts(); - let body_content = body.into_with_config().lossy_utf8(true).read_to_string(); + let body_content = body.into_with_config().read_to_string(); if let Ok(msg) = body_content { let json: serde_json::Value = serde_json::from_str(&msg).map_err(|_| { @@ -479,6 +482,7 @@ pub(super) fn get_supported_protocol_versions( ) })?; + let mut support_versions: Vec = vec![]; if let Some(serde_json::Value::Array(ref values)) = json.get("line.proto.support.versions") { for value in values.iter() { @@ -491,15 +495,20 @@ pub(super) fn get_supported_protocol_versions( } } } else { - support_versions.push(ProtocolVersion::V1); + support_versions.push(default_protocol_version); } + + let max_name_length = json + .get("cairo.max.file.name.length") + .and_then(|v| v.as_u64()) + .unwrap_or(MAX_NAME_LEN_DEFAULT as u64) as usize; + Ok((support_versions, max_name_length)) } else { - return Err(error::fmt!( + Err(error::fmt!( ProtocolVersionError, "Malformed server response, settings url: {}, err: failed to read response body as UTF-8", settings_url - )); + )) } - Ok(support_versions) } #[allow(clippy::result_large_err)] // `ureq::Error` is large enough to cause this warning. diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 51ebef6e..1a3c9750 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -57,13 +57,21 @@ use ring::{ signature::{EcdsaKeyPair, ECDSA_P256_SHA256_FIXED_SIGNING}, }; +pub(crate) const MAX_NAME_LEN_DEFAULT: usize = 127; + /// Defines the maximum allowed dimensions for array data in binary serialization protocols. -pub const MAX_ARRAY_DIMS: usize = 32; +pub(crate) const MAX_ARRAY_DIMS: usize = 32; -/// Line Protocol Version supported by current client. +/// The version of Ingestion Line Protocol used to communicate with the server. #[derive(Debug, Copy, Clone, PartialEq)] pub enum ProtocolVersion { + /// Version 1 of Line Protocol. + /// Full-text protocol. + /// When used over HTTP, it is compatible with the InfluxDB line protocol. V1 = 1, + + /// Version 2 of Ingestion Line Protocol. + /// Uses binary format serialization for f64, and supports the array data type. V2 = 2, } @@ -594,13 +602,7 @@ impl Buffer { /// protocol version. For custom name lengths, use [`with_max_name_len`](Self::with_max_name_len) /// or [`Sender::new_buffer_with_max_name_len`]. pub fn new(protocol_version: ProtocolVersion) -> Self { - Self { - output: Vec::new(), - state: BufferState::new(), - marker: None, - max_name_len: 127, - version: protocol_version, - } + Self::with_max_name_len(protocol_version, MAX_NAME_LEN_DEFAULT) } /// Creates a new [`Buffer`] with a custom maximum name length. @@ -612,10 +614,14 @@ impl Buffer { /// This is equivalent to [`Sender::new_buffer_with_max_name_len`] when using /// the sender's protocol version. For the default name length (127), /// use [`new`](Self::new) or [`Sender::new_buffer`]. - pub fn with_max_name_len(max_name_len: usize, protocol_version: ProtocolVersion) -> Self { - let mut buf = Self::new(protocol_version); - buf.max_name_len = max_name_len; - buf + pub fn with_max_name_len(protocol_version: ProtocolVersion, max_name_len: usize) -> Self { + Self { + output: Vec::new(), + state: BufferState::new(), + marker: None, + max_name_len, + version: protocol_version, + } } /// Pre-allocate to ensure the buffer has enough capacity for at least the @@ -1362,7 +1368,8 @@ pub struct Sender { handler: ProtocolHandler, connected: bool, max_buf_size: usize, - default_protocol_version: ProtocolVersion, + protocol_version: ProtocolVersion, + max_name_len: usize, } impl std::fmt::Debug for Sender { @@ -1813,6 +1820,7 @@ pub struct SenderBuilder { port: ConfigSetting, net_interface: ConfigSetting>, max_buf_size: ConfigSetting, + max_name_len: ConfigSetting, auth_timeout: ConfigSetting, username: ConfigSetting>, password: ConfigSetting>, @@ -1900,6 +1908,9 @@ impl SenderBuilder { )) } }, + "max_name_len" => { + builder.max_name_len(parse_conf_value(key, val)?)? + } "init_buf_size" => { return Err(error::fmt!( @@ -2055,6 +2066,7 @@ impl SenderBuilder { port: ConfigSetting::new_specified(port), net_interface: ConfigSetting::new_default(None), max_buf_size: ConfigSetting::new_default(100 * 1024 * 1024), + max_name_len: ConfigSetting::new_default(MAX_NAME_LEN_DEFAULT), auth_timeout: ConfigSetting::new_default(Duration::from_secs(15)), username: ConfigSetting::new_default(None), password: ConfigSetting::new_default(None), @@ -2216,6 +2228,22 @@ impl SenderBuilder { Ok(self) } + /// The maximum length of a table or column name in bytes. + /// Matches the `cairo.max.file.name.length` setting in the server. + /// The default is 127 bytes. + /// If running over HTTP and protocol version 2 is auto-negotiated, this + /// value is picked up from the server. + pub fn max_name_len(mut self, value: usize) -> Result { + if value < 16 { + return Err(error::fmt!( + ConfigError, + "max_name_len must be at least 16 bytes." + )); + } + self.max_name_len.set_specified("max_name_len", value)?; + Ok(self) + } + #[cfg(feature = "ilp-over-http")] /// Set the cumulative duration spent in retries. /// The value is in milliseconds, and the default is 10 seconds. @@ -2308,7 +2336,7 @@ impl SenderBuilder { .map_err(|io_err| map_io_to_socket_err("Could not set SO_KEEPALIVE: ", io_err))?; sock.set_nodelay(true) .map_err(|io_err| map_io_to_socket_err("Could not set TCP_NODELAY: ", io_err))?; - if let Some(ref host) = self.net_interface.deref() { + if let Some(host) = self.net_interface.deref() { let bind_addr = gai::resolve_host(host.as_str())?; sock.bind(&bind_addr).map_err(|io_err| { map_io_to_socket_err( @@ -2565,7 +2593,9 @@ impl SenderBuilder { } }; - let default_protocol_version = match self.protocol_version.deref() { + let mut max_name_len = *self.max_name_len; + + let protocol_version = match self.protocol_version.deref() { Some(v) => *v, None => match self.protocol { Protocol::Tcp | Protocol::Tcps => ProtocolVersion::V1, @@ -2578,10 +2608,12 @@ impl SenderBuilder { self.host.deref(), self.port.deref() ); - let versions = get_supported_protocol_versions(http_state, settings_url)?; - if versions.contains(&ProtocolVersion::V2) { + let (protocol_versions, server_max_name_len) = + read_server_settings(http_state, settings_url)?; + max_name_len = server_max_name_len; + if protocol_versions.contains(&ProtocolVersion::V2) { ProtocolVersion::V2 - } else if versions.contains(&ProtocolVersion::V1) { + } else if protocol_versions.contains(&ProtocolVersion::V1) { ProtocolVersion::V1 } else { return Err(error::fmt!( @@ -2601,12 +2633,14 @@ impl SenderBuilder { } else { descr.push_str("auth=off]"); } + let sender = Sender { descr, handler, connected: true, max_buf_size: *self.max_buf_size, - default_protocol_version, + protocol_version, + max_name_len, }; Ok(sender) @@ -2808,18 +2842,7 @@ impl Sender { /// the QuestDB server's default maximum name length of 127 characters. /// For custom name lengths, use [`new_buffer_with_max_name_len`](Self::new_buffer_with_max_name_len) pub fn new_buffer(&self) -> Buffer { - Buffer::new(self.default_protocol_version) - } - - /// Creates a new [`Buffer`] with a custom maximum name length. - /// - /// This initializes a buffer using the sender's protocol version and - /// a specified maximum length for table/column names. The value should match - /// your QuestDB server's `cairo.max.file.name.length` configuration. - /// - /// For the default name length (127), use [`new_buffer`](Self::new_buffer) - pub fn new_buffer_with_max_name_len(&self, max_name_len: usize) -> Buffer { - Buffer::with_max_name_len(max_name_len, self.default_protocol_version) + Buffer::with_max_name_len(self.protocol_version, self.max_name_len) } #[allow(unused_variables)] @@ -2964,23 +2987,33 @@ impl Sender { !self.connected } - /// Returns sender's default protocol version. - /// 1. User-set value via [`SenderBuilder::protocol_version`] - /// 2. V1 for TCP/TCPS (legacy protocol) - /// 3. Auto-detected version for HTTP/HTTPS - pub fn default_protocol_version(&self) -> ProtocolVersion { - self.default_protocol_version + /// Return the sender's protocol version. + /// This is either the protocol version that was set explicitly, + /// or the one that was auto-detected during the connection process. + /// If connecting via TCP and not overridden, the value is V1. + pub fn protocol_version(&self) -> ProtocolVersion { + self.protocol_version + } + + /// Return the sender's maxinum name length of any column or table name. + /// This is either set explicitly when constructing the sender, + /// or the default value of 127. + /// When unset and using protocol version 2 over HTTP, the value is read + /// from the server from the `cairo.max.file.name.length` setting in + /// `server.conf` which defaults to 127. + pub fn max_name_len(&self) -> usize { + self.max_name_len } #[inline(always)] fn check_protocol_version(&self, version: ProtocolVersion) -> Result<()> { - if self.default_protocol_version != version { + if self.protocol_version != version { return Err(error::fmt!( ProtocolVersionError, "Attempting to send with protocol version {} \ but the sender is configured to use protocol version {}", version, - self.default_protocol_version + self.protocol_version )); } Ok(()) diff --git a/questdb-rs/src/tests/http.rs b/questdb-rs/src/tests/http.rs index d4c29a02..172267e6 100644 --- a/questdb-rs/src/tests/http.rs +++ b/questdb-rs/src/tests/http.rs @@ -734,7 +734,7 @@ fn _test_sender_auto_detect_protocol_version( }); let mut sender = sender_builder.build()?; - assert_eq!(sender.default_protocol_version(), expect_version); + assert_eq!(sender.protocol_version(), expect_version); let mut buffer = sender.new_buffer(); buffer .table("test")? diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index 5d9f3bd9..9b92bfa5 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -266,7 +266,7 @@ fn test_max_buf_size( #[test] fn test_table_name_too_long() -> TestResult { - let mut buffer = Buffer::with_max_name_len(4, ProtocolVersion::V2); + let mut buffer = Buffer::with_max_name_len(ProtocolVersion::V2, 4); let name = "a name too long"; let err = buffer.table(name).unwrap_err(); assert_eq!(err.code(), ErrorCode::InvalidName); @@ -477,7 +477,7 @@ fn test_chrono_timestamp() -> TestResult { macro_rules! column_name_too_long_test_impl { ($column_fn:ident, $value:expr) => {{ - let mut buffer = Buffer::with_max_name_len(4, ProtocolVersion::V2); + let mut buffer = Buffer::with_max_name_len(ProtocolVersion::V2, 4); let name = "a name too long"; let err = buffer.table("tbl")?.$column_fn(name, $value).unwrap_err(); assert_eq!(err.code(), ErrorCode::InvalidName); diff --git a/system_test/questdb_line_sender.py b/system_test/questdb_line_sender.py index c5eb269b..a617c5e0 100644 --- a/system_test/questdb_line_sender.py +++ b/system_test/questdb_line_sender.py @@ -226,8 +226,8 @@ def set_sig(fn, restype, *argtypes): set_sig( dll.line_sender_buffer_with_max_name_len, c_line_sender_buffer_p, - c_size_t, - c_protocol_version) + c_protocol_version, + c_size_t) set_sig( dll.line_sender_buffer_free, None, @@ -455,15 +455,6 @@ def set_sig(fn, restype, *argtypes): dll.line_sender_from_env, c_line_sender_p, c_line_sender_error_p_p) - set_sig( - dll.line_sender_default_protocol_version, - c_protocol_version, - c_line_sender_p) - set_sig( - dll.line_sender_buffer_with_max_name_len_for_sender, - c_line_sender_buffer_p, - c_line_sender_p, - c_size_t) set_sig( dll.line_sender_must_close, None, @@ -615,7 +606,8 @@ def __init__(self, micros: int): class Buffer: def __init__(self, protocol_version: ProtocolVersion, init_buf_size=65536, max_name_len=127, ): self._impl = _DLL.line_sender_buffer_with_max_name_len( - c_size_t(max_name_len), protocol_version.value[0]) + protocol_version.value[0], + c_size_t(max_name_len)) _DLL.line_sender_buffer_reserve(self._impl, c_size_t(init_buf_size)) def __len__(self): From ac38b3cf4ccd49330d2ee0b13639848a2a722332 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Tue, 20 May 2025 14:38:05 +0100 Subject: [PATCH 098/119] removed convenience constructors, users should prefer conf or env --- cpp_test/test_line_sender.cpp | 52 ++++++++++++------------- include/questdb/ingress/line_sender.h | 13 +++++++ include/questdb/ingress/line_sender.hpp | 23 ++++++----- questdb-rs-ffi/src/lib.rs | 15 +++++++ 4 files changed, 67 insertions(+), 36 deletions(-) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 70a841b5..47f88b05 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -226,8 +226,8 @@ TEST_CASE("Opts service API tests") TEST_CASE("line_sender c++ connect disconnect") { questdb::ingress::test::mock_server server; - questdb::ingress::line_sender sender{ - questdb::ingress::protocol::tcp, "127.0.0.1", server.port()}; + questdb::ingress::line_sender sender{questdb::ingress::opts{ + questdb::ingress::protocol::tcp, "127.0.0.1", server.port()}}; CHECK_FALSE(sender.must_close()); server.accept(); CHECK(server.recv() == 0); @@ -327,10 +327,10 @@ TEST_CASE("test multiple lines") TEST_CASE("State machine testing -- flush without data.") { questdb::ingress::test::mock_server server; - questdb::ingress::line_sender sender{ + questdb::ingress::line_sender sender{questdb::ingress::opts{ questdb::ingress::protocol::tcp, std::string_view{"127.0.0.1"}, - std::to_string(server.port())}; + std::to_string(server.port())}}; questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); CHECK(buffer.size() == 0); @@ -345,10 +345,10 @@ TEST_CASE("State machine testing -- flush without data.") TEST_CASE("One symbol only - flush before server accept") { questdb::ingress::test::mock_server server; - questdb::ingress::line_sender sender{ + questdb::ingress::line_sender sender{questdb::ingress::opts{ questdb::ingress::protocol::tcp, std::string{"127.0.0.1"}, - server.port()}; + server.port()}}; // Does not raise - this is unlike InfluxDB spec that disallows this. questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); @@ -368,8 +368,8 @@ TEST_CASE("One symbol only - flush before server accept") TEST_CASE("One column only - server.accept() after flush, before close") { questdb::ingress::test::mock_server server; - questdb::ingress::line_sender sender{ - questdb::ingress::protocol::tcp, "127.0.0.1", server.port()}; + questdb::ingress::line_sender sender{questdb::ingress::opts{ + questdb::ingress::protocol::tcp, "127.0.0.1", server.port()}}; // Does not raise - this is unlike the InfluxDB spec that disallows this. questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); @@ -388,8 +388,8 @@ TEST_CASE("One column only - server.accept() after flush, before close") TEST_CASE("Symbol after column") { questdb::ingress::test::mock_server server; - questdb::ingress::line_sender sender{ - questdb::ingress::protocol::tcp, "127.0.0.1", server.port()}; + questdb::ingress::line_sender sender{questdb::ingress::opts{ + questdb::ingress::protocol::tcp, "127.0.0.1", server.port()}}; questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table("test").column("t1", "v1"); @@ -556,8 +556,8 @@ TEST_CASE("Sender move testing.") questdb::ingress::utf8_view host{"127.0.0.1"}; const questdb::ingress::utf8_view& host_ref = host; - questdb::ingress::line_sender sender1{ - questdb::ingress::protocol::tcp, host_ref, server1.port()}; + questdb::ingress::line_sender sender1{questdb::ingress::opts{ + questdb::ingress::protocol::tcp, host_ref, server1.port()}}; questdb::ingress::line_sender_buffer buffer = sender1.new_buffer(); buffer.table("test").column("t1", "v1").at_now(); @@ -581,8 +581,8 @@ TEST_CASE("Sender move testing.") CHECK_FALSE(sender1.must_close()); CHECK(sender2.must_close()); - questdb::ingress::line_sender sender3{ - questdb::ingress::protocol::tcp, "127.0.0.1", server2.port()}; + questdb::ingress::line_sender sender3{questdb::ingress::opts{ + questdb::ingress::protocol::tcp, "127.0.0.1", server2.port()}}; CHECK_FALSE(sender3.must_close()); sender3 = std::move(sender2); @@ -593,8 +593,8 @@ TEST_CASE("Bad hostname") { try { - questdb::ingress::line_sender sender{ - questdb::ingress::protocol::tcp, "dummy_hostname", "9009"}; + questdb::ingress::line_sender sender{questdb::ingress::opts{ + questdb::ingress::protocol::tcp, "dummy_hostname", "9009"}}; CHECK_MESSAGE(false, "Expected exception"); } catch (const questdb::ingress::line_sender_error& se) @@ -637,8 +637,8 @@ TEST_CASE("Bad port") const auto test_bad_port = [](std::string bad_port) { try { - questdb::ingress::line_sender sender{ - questdb::ingress::protocol::tcp, "127.0.0.1", bad_port}; + questdb::ingress::line_sender sender{questdb::ingress::opts{ + questdb::ingress::protocol::tcp, "127.0.0.1", bad_port}}; CHECK_MESSAGE(false, "Expected exception"); } catch (const questdb::ingress::line_sender_error& se) @@ -666,8 +666,8 @@ TEST_CASE("Bad connect") { // Port 1 is generally the tcpmux service which one would // very much expect to never be running. - questdb::ingress::line_sender sender{ - questdb::ingress::protocol::tcp, "127.0.0.1", 1}; + questdb::ingress::line_sender sender{questdb::ingress::opts{ + questdb::ingress::protocol::tcp, "127.0.0.1", 1}}; CHECK_MESSAGE(false, "Expected exception"); } catch (const questdb::ingress::line_sender_error& se) @@ -765,8 +765,8 @@ TEST_CASE("Opts copy ctor, assignment and move testing.") TEST_CASE("Test timestamp column.") { questdb::ingress::test::mock_server server; - questdb::ingress::line_sender sender{ - questdb::ingress::protocol::tcp, "127.0.0.1", server.port()}; + questdb::ingress::line_sender sender{questdb::ingress::opts{ + questdb::ingress::protocol::tcp, "127.0.0.1", server.port()}}; const auto now = std::chrono::system_clock::now(); const auto now_micros = @@ -905,8 +905,8 @@ TEST_CASE("Empty Buffer") CHECK(b5.size() == 9); questdb::ingress::test::mock_server server; - questdb::ingress::line_sender sender{ - questdb::ingress::protocol::tcp, "127.0.0.1", server.port()}; + questdb::ingress::line_sender sender{questdb::ingress::opts{ + questdb::ingress::protocol::tcp, "127.0.0.1", server.port()}}; CHECK_THROWS_WITH_AS( sender.flush(b1), "State error: Bad call to `flush`, should have called `table` instead.", @@ -973,10 +973,10 @@ TEST_CASE("HTTP basics") TEST_CASE("line sender protocol version default v1 for tcp") { questdb::ingress::test::mock_server server; - questdb::ingress::line_sender sender{ + questdb::ingress::line_sender sender{questdb::ingress::opts{ questdb::ingress::protocol::tcp, std::string("127.0.0.1"), - std::to_string(server.port())}; + std::to_string(server.port())}}; CHECK_FALSE(sender.must_close()); server.accept(); CHECK(server.recv() == 0); diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 991b3f74..064ea3b1 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -312,6 +312,10 @@ typedef struct line_sender_buffer line_sender_buffer; /** * Construct a `line_sender_buffer` with a `max_name_len` of `127`, which is * the same as the QuestDB server default. + * You should prefer to use `line_sender_for_sender()` instead, which + * automatically creates a buffer of the same protocol version as the sender. + * This is useful as it can rely on the sender's ability to auto-detect the + * protocol version when communicating over HTTP. */ LINESENDER_API line_sender_buffer* line_sender_buffer_new( @@ -621,6 +625,15 @@ LINESENDER_API bool line_sender_buffer_at_now( line_sender_buffer* buffer, line_sender_error** err_out); +/** + * Check whether the buffer is ready to be flushed. + * If this returns false, the buffer is incomplete and cannot be sent, + * and an error message is set to indicate the problem. + */ +LINESENDER_API +bool line_sender_buffer_check_can_flush( + const line_sender_buffer* buffer, line_sender_error** err_out); + /////////// Connecting, sending and disconnecting. /** diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 8d5500e2..9045bacc 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -804,6 +804,19 @@ class line_sender_buffer line_sender_error::wrapped_call(::line_sender_buffer_at_now, _impl); } + void check_can_flush() const + { + if (!_impl) + { + throw line_sender_error{ + line_sender_error_code::invalid_api_call, + "State error: Bad call to `flush`, should have called `table` " + "instead."}; + } + line_sender_error::wrapped_call( + ::line_sender_buffer_check_can_flush, _impl); + } + ~line_sender_buffer() noexcept { if (_impl) @@ -1213,16 +1226,6 @@ class line_sender return {opts::from_env()}; } - line_sender(protocol protocol, utf8_view host, uint16_t port) - : line_sender{opts{protocol, host, port}} - { - } - - line_sender(protocol protocol, utf8_view host, utf8_view port) - : line_sender{opts{protocol, host, port}} - { - } - line_sender(const opts& opts) : _impl{ line_sender_error::wrapped_call(::line_sender_build, opts._impl)} diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 402c3305..49462490 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -1031,6 +1031,21 @@ pub unsafe extern "C" fn line_sender_buffer_at_now( true } +/** + * Check whether the buffer is ready to be flushed. + * If this returns false, the buffer is incomplete and cannot be sent, + * and an error message is set to indicate the problem. + */ +#[no_mangle] +pub unsafe extern "C" fn line_sender_buffer_check_can_flush( + buffer: *const line_sender_buffer, + err_out: *mut *mut line_sender_error, +) -> bool { + let buffer = unwrap_buffer(buffer); + bubble_err_to_c!(err_out, buffer.check_can_flush()); + true +} + /// Accumulates parameters for a new `line_sender` object. pub struct line_sender_opts(SenderBuilder); From e5fc0254ffa41f38d59727e803d0fe40e26d8bf9 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Tue, 20 May 2025 15:26:56 +0100 Subject: [PATCH 099/119] less broken --- system_test/questdb_line_sender.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/system_test/questdb_line_sender.py b/system_test/questdb_line_sender.py index a617c5e0..b58248fd 100644 --- a/system_test/questdb_line_sender.py +++ b/system_test/questdb_line_sender.py @@ -331,6 +331,11 @@ def set_sig(fn, restype, *argtypes): c_bool, c_line_sender_buffer_p, c_line_sender_error_p_p) + set_sig( + dll.line_sender_buffer_check_can_flush, + c_bool, + c_line_sender_buffer_p, + c_line_sender_error_p_p) set_sig( dll.line_sender_opts_new, c_line_sender_opts_p, @@ -482,6 +487,14 @@ def set_sig(fn, restype, *argtypes): c_line_sender_buffer_p, c_bool, c_line_sender_error_p_p) + set_sig( + dll.line_sender_get_protocol_version, + c_protocol_version, + c_line_sender_p) + set_sig( + dll.line_sender_get_max_name_len, + c_size_t, + c_line_sender_p) return dll @@ -824,16 +837,24 @@ def connect(self): def __enter__(self): self.connect() self._buffer = Buffer( - protocol_version=ProtocolVersion.from_int(self.line_sender_default_protocol_version())) + protocol_version=self.protocol_version, + max_name_len=self.max_name_len) return self def _check_connected(self): if not self._impl: raise SenderError('Not connected.') - def line_sender_default_protocol_version(self): + @property + def protocol_version(self): + self._check_connected() + return ProtocolVersion.from_int( + _DLL.line_sender_get_protocol_version(self._impl)) + + @property + def max_name_len(self): self._check_connected() - return _DLL.line_sender_default_protocol_version(self._impl) + return _DLL.line_sender_get_max_name_len(self._impl) def table(self, table: str): self._buffer.table(table) From 81675ade145d69bb738ff66f106b79cd8a7f1232 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Tue, 20 May 2025 15:43:01 +0100 Subject: [PATCH 100/119] gcc fix --- include/questdb/ingress/line_sender.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 9045bacc..cc380957 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -1253,7 +1253,7 @@ class line_sender return *this; } - protocol_version protocol_version() const noexcept + questdb::ingress::protocol_version protocol_version() const noexcept { ensure_impl(); return static_cast( From 5889e05c0f21c93a603371329ae79fb6677ed402 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Tue, 20 May 2025 21:33:17 +0100 Subject: [PATCH 101/119] incomplete: moving C strides view to ffi crate --- doc/DEV_NOTES.md | 6 - questdb-rs-ffi/Cargo.lock | 147 -------- questdb-rs-ffi/Cargo.toml | 7 - questdb-rs-ffi/build.rs | 16 - questdb-rs-ffi/src/lib.rs | 14 +- questdb-rs-ffi/src/ndarr.rs | 497 ++++++++++++++++++++++++++++ questdb-rs/build.rs | 24 ++ questdb-rs/src/error.rs | 24 ++ questdb-rs/src/ingress/http.rs | 24 ++ questdb-rs/src/ingress/mod.rs | 14 +- questdb-rs/src/ingress/ndarr.rs | 254 ++------------ questdb-rs/src/ingress/tests.rs | 24 ++ questdb-rs/src/ingress/timestamp.rs | 24 ++ questdb-rs/src/tests/ndarr.rs | 267 ++------------- 14 files changed, 686 insertions(+), 656 deletions(-) delete mode 100644 questdb-rs-ffi/build.rs create mode 100644 questdb-rs-ffi/src/ndarr.rs diff --git a/doc/DEV_NOTES.md b/doc/DEV_NOTES.md index 66f92c07..7dd5889e 100644 --- a/doc/DEV_NOTES.md +++ b/doc/DEV_NOTES.md @@ -38,15 +38,9 @@ either of the two Rust projects: * [questdb-rs](../questdb-rs/) - Core library * [questdb-rs-ffi](../questdb-rs-ffi/) - C bindings layer. -If you are editing the C functions in the `questdb-rs-ffi` project and what to -see the resulting generated header file, call `cargo build --features gen_h`. - Note that to reduce compile time we don't use cbindgen in the header we ship, which also contains additional formatting and comments. -Similarly, we also support generating Cython bindings via the `gen_cython` -feature. - This generated files should be not be checked in: * `include/questdb/ingress/line_sender.gen.h` * `cython/questdb/ingress/line_sender.pxd` diff --git a/questdb-rs-ffi/Cargo.lock b/questdb-rs-ffi/Cargo.lock index 7baf1e3d..d02045b8 100644 --- a/questdb-rs-ffi/Cargo.lock +++ b/questdb-rs-ffi/Cargo.lock @@ -26,24 +26,6 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" -[[package]] -name = "cbindgen" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadd868a2ce9ca38de7eeafdcec9c7065ef89b42b32f0839278d55f35c54d1ff" -dependencies = [ - "heck", - "indexmap", - "log", - "proc-macro2", - "quote", - "serde", - "serde_json", - "syn", - "tempfile", - "toml", -] - [[package]] name = "cc" version = "1.2.17" @@ -87,28 +69,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "errno" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" -dependencies = [ - "libc", - "windows-sys 0.59.0", -] - -[[package]] -name = "fastrand" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" - [[package]] name = "fnv" version = "1.0.7" @@ -138,18 +98,6 @@ dependencies = [ "wasi 0.14.2+wasi-0.2.4", ] -[[package]] -name = "hashbrown" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" - -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - [[package]] name = "http" version = "1.3.1" @@ -167,16 +115,6 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" -[[package]] -name = "indexmap" -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" -dependencies = [ - "equivalent", - "hashbrown", -] - [[package]] name = "indoc" version = "2.0.6" @@ -195,12 +133,6 @@ version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" -[[package]] -name = "linux-raw-sys" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413" - [[package]] name = "log" version = "0.4.27" @@ -294,7 +226,6 @@ dependencies = [ name = "questdb-rs-ffi" version = "5.0.0-rc1" dependencies = [ - "cbindgen", "libc", "questdb-confstr-ffi", "questdb-rs", @@ -359,19 +290,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rustix" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.59.0", -] - [[package]] name = "rustls" version = "0.23.25" @@ -495,15 +413,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_spanned" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" -dependencies = [ - "serde", -] - [[package]] name = "shlex" version = "1.3.0" @@ -546,53 +455,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "tempfile" -version = "3.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" -dependencies = [ - "fastrand", - "getrandom 0.3.2", - "once_cell", - "rustix", - "windows-sys 0.59.0", -] - -[[package]] -name = "toml" -version = "0.8.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148" -dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", -] - -[[package]] -name = "toml_datetime" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" -dependencies = [ - "serde", -] - -[[package]] -name = "toml_edit" -version = "0.22.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" -dependencies = [ - "indexmap", - "serde", - "serde_spanned", - "toml_datetime", - "winnow", -] - [[package]] name = "unicode-ident" version = "1.0.18" @@ -840,15 +702,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "winnow" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e97b544156e9bebe1a0ffbc03484fc1ffe3100cbce3ffb17eac35f7cdd7ab36" -dependencies = [ - "memchr", -] - [[package]] name = "wit-bindgen-rt" version = "0.39.0" diff --git a/questdb-rs-ffi/Cargo.toml b/questdb-rs-ffi/Cargo.toml index 84298864..bf8bbbc2 100644 --- a/questdb-rs-ffi/Cargo.toml +++ b/questdb-rs-ffi/Cargo.toml @@ -14,16 +14,9 @@ questdb-rs = { path = "../questdb-rs", features = [ libc = "0.2" questdb-confstr-ffi = { version = "0.1.1", optional = true } -[build-dependencies] -cbindgen = { version = "0.28.0", optional = true, default-features = false } - [features] # Expose the config parsing C API. # This used by `py-questdb-client` to parse the config file. # It is exposed here to avoid having multiple copies of the `questdb-confstr` # crate in the final binary. confstr-ffi = ["dep:questdb-confstr-ffi"] - -# Auto-generate the header. This is for dev-debugging-diffing only. -# A hand-crafted header is easier on the eyes. -gen_h = ["cbindgen"] diff --git a/questdb-rs-ffi/build.rs b/questdb-rs-ffi/build.rs deleted file mode 100644 index 51d6ee1d..00000000 --- a/questdb-rs-ffi/build.rs +++ /dev/null @@ -1,16 +0,0 @@ -#[cfg(feature = "gen_h")] -extern crate cbindgen; - -fn main() -> Result<(), Box> { - println!("cargo:rerun-if-changed=build.rs"); - println!("cargo:rerun-if-changed=Cargo.lock"); - - #[cfg(feature = "gen_h")] - { - let crate_dir = std::env::var("CARGO_MANIFEST_DIR")?; - let bindings = cbindgen::generate(crate_dir)?; - bindings.write_to_file("../include/questdb/ingress/line_sender.gen.h"); - } - - Ok(()) -} diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 49462490..cd277803 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -42,6 +42,9 @@ use questdb::{ Error, ErrorCode, }; +mod ndarr; +use ndarr::StrideArrayView; + macro_rules! bubble_err_to_c { ($err_out:expr, $expression:expr) => { bubble_err_to_c!($err_out, $expression, false) @@ -890,13 +893,8 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( ) -> bool { let buffer = unwrap_buffer_mut(buffer); let name = name.as_name(); - let view = match ingress::StrideArrayView::::new( - rank, - shape, - strides, - data_buffer, - data_buffer_len, - ) { + let view = match StrideArrayView::::new(rank, shape, strides, data_buffer, data_buffer_len) + { Ok(value) => value, Err(err) => { let err_ptr = Box::into_raw(Box::new(line_sender_error(err))); @@ -906,7 +904,7 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( }; bubble_err_to_c!( err_out, - buffer.column_arr::, ingress::StrideArrayView<'_, f64>, f64>(name, &view) + buffer.column_arr::, StrideArrayView<'_, f64>, f64>(name, &view) ); true } diff --git a/questdb-rs-ffi/src/ndarr.rs b/questdb-rs-ffi/src/ndarr.rs new file mode 100644 index 00000000..9ec8f2f6 --- /dev/null +++ b/questdb-rs-ffi/src/ndarr.rs @@ -0,0 +1,497 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +use questdb::ingress::ArrayElement; +use questdb::ingress::NdArrayView; +use questdb::ingress::MAX_ARRAY_BUFFER_SIZE; +use questdb::Error; +use std::mem::size_of; +use std::slice; + +/// A view into a multidimensional array with custom memory strides. +// TODO: We are currently evaluating whether to use StrideArrayView or ndarray's view. +// Current benchmarks show that StrideArrayView's iter implementation underperforms(2x) +// compared to ndarray's. +// We should optimise this implementation to be competitive. +// Unfortunately, the `ndarray` crate does not support negative strides +// which we need to support in this FFI crate for efficient iteration of +// numpy arrays coming from Python without copying the data. +#[derive(Debug)] +pub struct StrideArrayView<'a, T> { + dims: usize, + shape: &'a [usize], + strides: &'a [isize], + data: Option<&'a [u8]>, + _marker: std::marker::PhantomData, +} + +impl NdArrayView for StrideArrayView<'_, T> +where + T: ArrayElement, +{ + type Iter<'b> + = RowMajorIter<'b, T> + where + Self: 'b, + T: 'b; + + fn ndim(&self) -> usize { + self.dims + } + + fn dim(&self, index: usize) -> Result { + if index >= self.dims { + return Err(error::fmt!( + ArrayViewError, + "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", + index, + self.dims + )); + } + Ok(self.shape[index]) + } + + fn as_slice(&self) -> Option<&[T]> { + unsafe { + self.is_c_major().then_some(self.data.map(|data| { + slice::from_raw_parts(data.as_ptr() as *const T, data.len() / size_of::()) + })?) + } + } + + fn iter(&self) -> Self::Iter<'_> { + let mut dim_products = Vec::with_capacity(self.dims); + let mut product = 1; + for &dim in self.shape.iter().rev() { + dim_products.push(product); + product *= dim; + } + dim_products.reverse(); + + // consider minus strides + let base_ptr = match self.data { + None => std::ptr::null(), + Some(data) => { + self.strides + .iter() + .enumerate() + .fold(data.as_ptr(), |ptr, (dim, &stride)| { + if stride < 0 { + let dim_size = self.shape[dim] as isize; + unsafe { ptr.offset(stride * (dim_size - 1)) } + } else { + ptr + } + }) + } + }; + + RowMajorIter { + base_ptr, + array: self, + dim_products, + current_linear: 0, + total_elements: self.shape.iter().product(), + } + } +} + +impl StrideArrayView<'_, T> +where + T: ArrayElement, +{ + /// Creates a new strided array view from raw components (unsafe constructor). + /// + /// # Safety + /// Caller must ensure all the following conditions: + /// - `shape` points to a valid array of at least `dims` elements + /// - `strides` points to a valid array of at least `dims` elements + /// - `data` points to a valid memory block of at least `data_len` bytes + /// - Memory layout must satisfy: + /// 1. `data_len ≥ (shape[0]-1)*abs(strides[0]) + ... + (shape[n-1]-1)*abs(strides[n-1]) + size_of::()` + /// 2. All calculated offsets stay within `[0, data_len - size_of::()]` + /// - Lifetime `'a` must outlive the view's usage + /// - Strides are measured in bytes (not elements) + pub unsafe fn new( + dims: usize, + shape: *const usize, + strides: *const isize, + data: *const u8, + data_len: usize, + ) -> Result { + if dims == 0 { + return Err(error::fmt!( + ArrayViewError, + "Zero-dimensional arrays are not supported", + )); + } + if data_len > MAX_ARRAY_BUFFER_SIZE { + return Err(error::fmt!( + ArrayViewError, + "Array buffer size too big: {}, maximum: {}", + data_len, + MAX_ARRAY_BUFFER_SIZE + )); + } + let shape = slice::from_raw_parts(shape, dims); + let size = shape + .iter() + .try_fold(std::mem::size_of::(), |acc, &dim| { + acc.checked_mul(dim) + .ok_or_else(|| error::fmt!(ArrayViewError, "Array buffer size too big")) + })?; + + if size != data_len { + return Err(error::fmt!( + ArrayViewError, + "Array buffer length mismatch (actual: {}, expected: {})", + data_len, + size + )); + } + let strides = slice::from_raw_parts(strides, dims); + let mut slice = None; + if data_len != 0 { + slice = Some(slice::from_raw_parts(data, data_len)); + } + Ok(Self { + dims, + shape, + strides, + data: slice, + _marker: std::marker::PhantomData::, + }) + } + + /// Verifies if the array follows C-style row-major memory layout. + fn is_c_major(&self) -> bool { + match self.data { + None => false, + Some(data) => { + if data.is_empty() { + return false; + } + + let elem_size = size_of::() as isize; + if self.dims == 1 { + return self.strides[0] == elem_size || self.shape[0] == 1; + } + + let mut expected_stride = elem_size; + for (dim, &stride) in self.shape.iter().zip(self.strides).rev() { + if *dim > 1 && stride != expected_stride { + return false; + } + expected_stride *= *dim as isize; + } + true + } + } + } +} + +/// Iterator for traversing a stride array in row-major (C-style) order. +pub struct RowMajorIter<'a, T> { + base_ptr: *const u8, + array: &'a StrideArrayView<'a, T>, + dim_products: Vec, + current_linear: usize, + total_elements: usize, +} + +impl<'a, T> Iterator for RowMajorIter<'a, T> +where + T: ArrayElement, +{ + type Item = &'a T; + fn next(&mut self) -> Option { + if self.current_linear >= self.total_elements { + return None; + } + let mut remaining_index = self.current_linear; + let mut offset = 0; + + for (dim, &dim_factor) in self.dim_products.iter().enumerate() { + let coord = remaining_index / dim_factor; + remaining_index %= dim_factor; + let stride = self.array.strides[dim]; + let actual_coord = if stride >= 0 { + coord + } else { + self.array.shape[dim] - 1 - coord + }; + offset += actual_coord * stride.unsigned_abs(); + } + + self.current_linear += 1; + unsafe { + let ptr = self.base_ptr.add(offset); + Some(&*(ptr as *const T)) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use questdb::ingress::*; + use std::ptr; + type TestResult = Result<(), Box>; + + fn to_bytes(data: &[T]) -> Vec { + data.iter() + .flat_map(|x| { + let bytes = unsafe { + std::slice::from_raw_parts(x as *const T as *const u8, size_of::()) + }; + bytes.to_vec() + }) + .collect() + } + + #[test] + fn test_buffer_basic_write() -> TestResult { + let elem_size = std::mem::size_of::() as isize; + + let test_data = [1.1, 2.2, 3.3, 4.4]; + let array_view: StrideArrayView<'_, f64> = unsafe { + StrideArrayView::new( + 2, + [2, 2].as_ptr(), + [2 * elem_size, elem_size].as_ptr(), + test_data.as_ptr() as *const u8, + test_data.len() * elem_size as usize, + ) + }?; + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("temperature", &array_view)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); + assert_eq!( + &data[19..24], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ArrayColumnTypeTag::Double.into(), + 2u8 + ] + ); + assert_eq!( + &data[24..32], + [2i32.to_le_bytes(), 2i32.to_le_bytes()].concat() + ); + assert_eq!( + &data[32..64], + &[ + 1.1f64.to_ne_bytes(), + 2.2f64.to_le_bytes(), + 3.3f64.to_le_bytes(), + 4.4f64.to_le_bytes(), + ] + .concat() + ); + Ok(()) + } + + #[test] + fn test_stride_array_size_overflow() -> TestResult { + let result = unsafe { + StrideArrayView::::new( + 2, + [u32::MAX as usize, u32::MAX as usize].as_ptr(), + [8, 8].as_ptr(), + ptr::null(), + 0, + ) + }; + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert!(err.msg().contains("Array buffer size too big")); + Ok(()) + } + + #[test] + fn test_stride_view_length_mismatch() -> TestResult { + let elem_size = size_of::() as isize; + let under_data = [1.1]; + let result: Result, Error> = unsafe { + StrideArrayView::new( + 2, + [1, 2].as_ptr(), + [elem_size, elem_size].as_ptr(), + under_data.as_ptr() as *const u8, + under_data.len() * elem_size as usize, + ) + }; + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert!(err + .msg() + .contains("Array buffer length mismatch (actual: 8, expected: 16)")); + + let over_data = [1.1, 2.2, 3.3]; + let result: Result, Error> = unsafe { + StrideArrayView::new( + 2, + [1, 2].as_ptr(), + [elem_size, elem_size].as_ptr(), + over_data.as_ptr() as *const u8, + over_data.len() * elem_size as usize, + ) + }; + + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert!(err + .msg() + .contains("Array buffer length mismatch (actual: 24, expected: 16)")); + Ok(()) + } + + #[test] + fn test_strided_non_contiguous() -> TestResult { + let elem_size = size_of::() as isize; + let col_major_data = [1.0, 3.0, 5.0, 2.0, 4.0, 6.0]; + let shape = [3usize, 2]; + let strides = [elem_size, shape[0] as isize * elem_size]; + + let array_view: StrideArrayView<'_, f64> = unsafe { + StrideArrayView::new( + shape.len(), + shape.as_ptr(), + strides.as_ptr(), + col_major_data.as_ptr() as *const u8, + col_major_data.len() * elem_size as usize, + ) + }?; + + assert_eq!(array_view.ndim(), 2); + assert_eq!(array_view.dim(0), Ok(3)); + assert_eq!(array_view.dim(1), Ok(2)); + assert!(array_view.dim(2).is_err()); + assert!(array_view.as_slice().is_none()); + let mut buffer = vec![0u8; 48]; + write_array_data(&array_view, &mut buffer, 48)?; + + let expected_data = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; + let expected_bytes = unsafe { + std::slice::from_raw_parts( + expected_data.as_ptr() as *const u8, + expected_data.len() * elem_size as usize, + ) + }; + assert_eq!(buffer, expected_bytes); + Ok(()) + } + + #[test] + fn test_negative_strides() -> TestResult { + let elem_size = size_of::(); + let data = [1f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]; + let view = unsafe { + StrideArrayView::::new( + 2, + &[3usize, 3] as *const usize, + &[-24isize, 8] as *const isize, + (data.as_ptr() as *const u8).add(48), + data.len() * elem_size, + ) + }?; + let collected: Vec<_> = view.iter().copied().collect(); + assert!(view.as_slice().is_none()); + let expected_data = vec![7.0, 8.0, 9.0, 4.0, 5.0, 6.0, 1.0, 2.0, 3.0]; + assert_eq!(collected, expected_data); + let mut buffer = vec![0u8; 72]; + write_array_data(&view, &mut buffer, 72)?; + let expected_bytes = unsafe { + std::slice::from_raw_parts( + expected_data.as_ptr() as *const u8, + expected_data.len() * elem_size, + ) + }; + assert_eq!(buffer, expected_bytes); + Ok(()) + } + + #[test] + fn test_basic_edge_cases() -> TestResult { + // empty array + let elem_size = std::mem::size_of::() as isize; + let empty_view: StrideArrayView<'_, f64> = + unsafe { StrideArrayView::new(2, [0, 0].as_ptr(), [0, 0].as_ptr(), ptr::null(), 0)? }; + assert_eq!(empty_view.ndim(), 2); + assert_eq!(empty_view.dim(0), Ok(0)); + assert_eq!(empty_view.dim(1), Ok(0)); + + // single element array + let single_data = [42.0]; + let single_view: StrideArrayView<'_, f64> = unsafe { + StrideArrayView::new( + 1, + [1].as_ptr(), + [elem_size].as_ptr(), + single_data.as_ptr() as *const u8, + elem_size as usize, + ) + }?; + let mut buf = vec![0u8; 8]; + write_array_data(&single_view, &mut buf, 8).unwrap(); + assert_eq!(buf, 42.0f64.to_ne_bytes()); + Ok(()) + } + + #[test] + fn test_stride_array_view() -> TestResult { + // contiguous layout + let test_data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; + let shape = [2usize, 3]; + let strides = [ + (shape[1] * size_of::()) as isize, + size_of::() as isize, + ]; + let array = unsafe { + StrideArrayView::::new( + shape.len(), + shape.as_ptr(), + strides.as_ptr(), + test_data.as_ptr() as *const u8, + test_data.len() * size_of::(), + ) + }?; + + assert_eq!(array.ndim(), 2); + assert_eq!(array.dim(0), Ok(2)); + assert_eq!(array.dim(1), Ok(3)); + assert!(array.dim(2).is_err()); + assert!(array.as_slice().is_some()); + let mut buf = vec![0u8; 48]; + write_array_data(&array, &mut buf, 48).unwrap(); + let expected = to_bytes(&test_data); + assert_eq!(buf, expected); + Ok(()) + } +} diff --git a/questdb-rs/build.rs b/questdb-rs/build.rs index 9a1350f3..98dc34ef 100644 --- a/questdb-rs/build.rs +++ b/questdb-rs/build.rs @@ -1,3 +1,27 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + #[cfg(feature = "json_tests")] pub mod json_tests { use indoc::indoc; diff --git a/questdb-rs/src/error.rs b/questdb-rs/src/error.rs index a6fecf23..0c32f48a 100644 --- a/questdb-rs/src/error.rs +++ b/questdb-rs/src/error.rs @@ -1,3 +1,27 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + use std::fmt::{Display, Formatter}; macro_rules! fmt { diff --git a/questdb-rs/src/ingress/http.rs b/questdb-rs/src/ingress/http.rs index 313b245f..0eb85d77 100644 --- a/questdb-rs/src/ingress/http.rs +++ b/questdb-rs/src/ingress/http.rs @@ -1,3 +1,27 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + use super::conf::ConfigSetting; use super::MAX_NAME_LEN_DEFAULT; use crate::error::fmt; diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 1a3c9750..6b5ec6e4 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -24,7 +24,7 @@ #![doc = include_str!("mod.md")] -pub use self::ndarr::{ArrayElement, NdArrayView, StrideArrayView}; +pub use self::ndarr::{ArrayElement, NdArrayView}; pub use self::timestamp::*; use crate::error::{self, Error, Result}; use crate::gai; @@ -59,8 +59,16 @@ use ring::{ pub(crate) const MAX_NAME_LEN_DEFAULT: usize = 127; -/// Defines the maximum allowed dimensions for array data in binary serialization protocols. -pub(crate) const MAX_ARRAY_DIMS: usize = 32; +/// The maximum allowed dimensions for arrays. +pub const MAX_ARRAY_DIMS: usize = 32; + +// TODO: We should probably agree on a significantly +// _smaller_ limit here, since there's no way +// we've ever tested anything that big. +// My gut feeling is that the maximum array buffer should be +// in the order of 100MB or so. +const MAX_ARRAY_BUFFER_SIZE: usize = i32::MAX as usize; +pub(crate) const MAX_ARRAY_DIM_LEN: usize = 0x0FFF_FFFF; // 1 << 28 - 1 /// The version of Ingestion Line Protocol used to communicate with the server. #[derive(Debug, Copy, Clone, PartialEq)] diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 091258cb..c793094b 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -1,3 +1,27 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + pub trait NdArrayView where T: ArrayElement, @@ -22,14 +46,6 @@ where fn iter(&self) -> Self::Iter<'_>; } -// TODO: We should probably agree on a significantly -// _smaller_ limit here, since there's no way -// we've ever tested anything that big. -// My gut feeling is that the maximum array buffer should be -// in the order of 100MB or so. -const MAX_ARRAY_BUFFER_SIZE: usize = i32::MAX as usize; -pub(crate) const MAX_ARRAY_DIM_LEN: usize = 0x0FFF_FFFF; // 1 << 28 - 1 - pub(crate) fn write_array_data, T>( array: &A, buf: &mut [u8], @@ -145,226 +161,6 @@ impl ArrayElementSealed for f64 { } } -/// A view into a multidimensional array with custom memory strides. -// TODO: We are currently evaluating whether to use StrideArrayView or ndarray's view. -// Current benchmarks show that StrideArrayView's iter implementation underperforms(2x) -// compared to ndarray's. If we proceed with StrideArrayView, we need to -// optimize the iter traversal pattern -#[derive(Debug)] -pub struct StrideArrayView<'a, T> { - dims: usize, - shape: &'a [usize], - strides: &'a [isize], - data: Option<&'a [u8]>, - _marker: std::marker::PhantomData, -} - -impl NdArrayView for StrideArrayView<'_, T> -where - T: ArrayElement, -{ - type Iter<'b> - = RowMajorIter<'b, T> - where - Self: 'b, - T: 'b; - - fn ndim(&self) -> usize { - self.dims - } - - fn dim(&self, index: usize) -> Result { - if index >= self.dims { - return Err(error::fmt!( - ArrayViewError, - "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", - index, - self.dims - )); - } - Ok(self.shape[index]) - } - - fn as_slice(&self) -> Option<&[T]> { - unsafe { - self.is_c_major().then_some(self.data.map(|data| { - slice::from_raw_parts(data.as_ptr() as *const T, data.len() / size_of::()) - })?) - } - } - - fn iter(&self) -> Self::Iter<'_> { - let mut dim_products = Vec::with_capacity(self.dims); - let mut product = 1; - for &dim in self.shape.iter().rev() { - dim_products.push(product); - product *= dim; - } - dim_products.reverse(); - - // consider minus strides - let base_ptr = match self.data { - None => std::ptr::null(), - Some(data) => { - self.strides - .iter() - .enumerate() - .fold(data.as_ptr(), |ptr, (dim, &stride)| { - if stride < 0 { - let dim_size = self.shape[dim] as isize; - unsafe { ptr.offset(stride * (dim_size - 1)) } - } else { - ptr - } - }) - } - }; - - RowMajorIter { - base_ptr, - array: self, - dim_products, - current_linear: 0, - total_elements: self.shape.iter().product(), - } - } -} - -impl StrideArrayView<'_, T> -where - T: ArrayElement, -{ - /// Creates a new strided array view from raw components (unsafe constructor). - /// - /// # Safety - /// Caller must ensure all the following conditions: - /// - `shape` points to a valid array of at least `dims` elements - /// - `strides` points to a valid array of at least `dims` elements - /// - `data` points to a valid memory block of at least `data_len` bytes - /// - Memory layout must satisfy: - /// 1. `data_len ≥ (shape[0]-1)*abs(strides[0]) + ... + (shape[n-1]-1)*abs(strides[n-1]) + size_of::()` - /// 2. All calculated offsets stay within `[0, data_len - size_of::()]` - /// - Lifetime `'a` must outlive the view's usage - /// - Strides are measured in bytes (not elements) - pub unsafe fn new( - dims: usize, - shape: *const usize, - strides: *const isize, - data: *const u8, - data_len: usize, - ) -> Result { - if dims == 0 { - return Err(error::fmt!( - ArrayViewError, - "Zero-dimensional arrays are not supported", - )); - } - if data_len > MAX_ARRAY_BUFFER_SIZE { - return Err(error::fmt!( - ArrayViewError, - "Array buffer size too big: {}, maximum: {}", - data_len, - MAX_ARRAY_BUFFER_SIZE - )); - } - let shape = slice::from_raw_parts(shape, dims); - let size = shape - .iter() - .try_fold(std::mem::size_of::(), |acc, &dim| { - acc.checked_mul(dim) - .ok_or_else(|| error::fmt!(ArrayViewError, "Array buffer size too big")) - })?; - - if size != data_len { - return Err(error::fmt!( - ArrayViewError, - "Array buffer length mismatch (actual: {}, expected: {})", - data_len, - size - )); - } - let strides = slice::from_raw_parts(strides, dims); - let mut slice = None; - if data_len != 0 { - slice = Some(slice::from_raw_parts(data, data_len)); - } - Ok(Self { - dims, - shape, - strides, - data: slice, - _marker: std::marker::PhantomData::, - }) - } - - /// Verifies if the array follows C-style row-major memory layout. - fn is_c_major(&self) -> bool { - match self.data { - None => false, - Some(data) => { - if data.is_empty() { - return false; - } - - let elem_size = size_of::() as isize; - if self.dims == 1 { - return self.strides[0] == elem_size || self.shape[0] == 1; - } - - let mut expected_stride = elem_size; - for (dim, &stride) in self.shape.iter().zip(self.strides).rev() { - if *dim > 1 && stride != expected_stride { - return false; - } - expected_stride *= *dim as isize; - } - true - } - } - } -} - -/// Iterator for traversing a stride array in row-major (C-style) order. -pub struct RowMajorIter<'a, T> { - base_ptr: *const u8, - array: &'a StrideArrayView<'a, T>, - dim_products: Vec, - current_linear: usize, - total_elements: usize, -} - -impl<'a, T> Iterator for RowMajorIter<'a, T> -where - T: ArrayElement, -{ - type Item = &'a T; - fn next(&mut self) -> Option { - if self.current_linear >= self.total_elements { - return None; - } - let mut remaining_index = self.current_linear; - let mut offset = 0; - - for (dim, &dim_factor) in self.dim_products.iter().enumerate() { - let coord = remaining_index / dim_factor; - remaining_index %= dim_factor; - let stride = self.array.strides[dim]; - let actual_coord = if stride >= 0 { - coord - } else { - self.array.shape[dim] - 1 - coord - }; - offset += actual_coord * stride.unsigned_abs(); - } - - self.current_linear += 1; - unsafe { - let ptr = self.base_ptr.add(offset); - Some(&*(ptr as *const T)) - } - } -} - /// impl NdArrayView for one dimension vector impl NdArrayView for Vec { type Iter<'a> @@ -701,6 +497,8 @@ use crate::{error, Error}; use ndarray::{ArrayView, Axis, Dimension}; use std::slice; +use super::{MAX_ARRAY_BUFFER_SIZE, MAX_ARRAY_DIM_LEN}; + #[cfg(feature = "ndarray")] impl NdArrayView for ArrayView<'_, T, D> where diff --git a/questdb-rs/src/ingress/tests.rs b/questdb-rs/src/ingress/tests.rs index a48021c6..69a8bd6d 100644 --- a/questdb-rs/src/ingress/tests.rs +++ b/questdb-rs/src/ingress/tests.rs @@ -1,3 +1,27 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + use super::*; use crate::ErrorCode; use tempfile::TempDir; diff --git a/questdb-rs/src/ingress/timestamp.rs b/questdb-rs/src/ingress/timestamp.rs index 93361484..4176a896 100644 --- a/questdb-rs/src/ingress/timestamp.rs +++ b/questdb-rs/src/ingress/timestamp.rs @@ -1,3 +1,27 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + use crate::error; use std::time::{Duration, SystemTime, UNIX_EPOCH}; diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index 2f27fb3e..30505e5b 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -1,17 +1,37 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + #[cfg(feature = "ndarray")] use crate::ingress::MAX_ARRAY_DIMS; -use crate::ingress::{ - Buffer, NdArrayView, ProtocolVersion, StrideArrayView, ARRAY_BINARY_FORMAT_TYPE, -}; +use crate::ingress::{Buffer, NdArrayView, ProtocolVersion, ARRAY_BINARY_FORMAT_TYPE}; use crate::tests::TestResult; -use crate::{Error, ErrorCode}; +use crate::ErrorCode; -use crate::ingress::ndarr::write_array_data; #[cfg(feature = "ndarray")] use ndarray::{arr1, arr2, arr3, s, ArrayD}; #[cfg(feature = "ndarray")] use std::iter; -use std::ptr; /// QuestDB column type tags that are supported as array element types. #[derive(Clone, Copy)] @@ -45,241 +65,6 @@ impl TryFrom for ArrayColumnTypeTag { } } -fn to_bytes(data: &[T]) -> Vec { - data.iter() - .flat_map(|x| { - let bytes = - unsafe { std::slice::from_raw_parts(x as *const T as *const u8, size_of::()) }; - bytes.to_vec() - }) - .collect() -} - -#[test] -fn test_stride_array_view() -> TestResult { - // contiguous layout - let test_data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; - let shape = [2usize, 3]; - let strides = [ - (shape[1] * size_of::()) as isize, - size_of::() as isize, - ]; - let array = unsafe { - StrideArrayView::::new( - shape.len(), - shape.as_ptr(), - strides.as_ptr(), - test_data.as_ptr() as *const u8, - test_data.len() * size_of::(), - ) - }?; - - assert_eq!(array.ndim(), 2); - assert_eq!(array.dim(0), Ok(2)); - assert_eq!(array.dim(1), Ok(3)); - assert!(array.dim(2).is_err()); - assert!(array.as_slice().is_some()); - let mut buf = vec![0u8; 48]; - write_array_data(&array, &mut buf, 48).unwrap(); - let expected = to_bytes(&test_data); - assert_eq!(buf, expected); - Ok(()) -} - -#[test] -fn test_strided_non_contiguous() -> TestResult { - let elem_size = size_of::() as isize; - let col_major_data = [1.0, 3.0, 5.0, 2.0, 4.0, 6.0]; - let shape = [3usize, 2]; - let strides = [elem_size, shape[0] as isize * elem_size]; - - let array_view: StrideArrayView<'_, f64> = unsafe { - StrideArrayView::new( - shape.len(), - shape.as_ptr(), - strides.as_ptr(), - col_major_data.as_ptr() as *const u8, - col_major_data.len() * elem_size as usize, - ) - }?; - - assert_eq!(array_view.ndim(), 2); - assert_eq!(array_view.dim(0), Ok(3)); - assert_eq!(array_view.dim(1), Ok(2)); - assert!(array_view.dim(2).is_err()); - assert!(array_view.as_slice().is_none()); - let mut buffer = vec![0u8; 48]; - write_array_data(&array_view, &mut buffer, 48)?; - - let expected_data = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; - let expected_bytes = unsafe { - std::slice::from_raw_parts( - expected_data.as_ptr() as *const u8, - expected_data.len() * elem_size as usize, - ) - }; - assert_eq!(buffer, expected_bytes); - Ok(()) -} - -#[test] -fn test_negative_strides() -> TestResult { - let elem_size = size_of::(); - let data = [1f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]; - let view = unsafe { - StrideArrayView::::new( - 2, - &[3usize, 3] as *const usize, - &[-24isize, 8] as *const isize, - (data.as_ptr() as *const u8).add(48), - data.len() * elem_size, - ) - }?; - let collected: Vec<_> = view.iter().copied().collect(); - assert!(view.as_slice().is_none()); - let expected_data = vec![7.0, 8.0, 9.0, 4.0, 5.0, 6.0, 1.0, 2.0, 3.0]; - assert_eq!(collected, expected_data); - let mut buffer = vec![0u8; 72]; - write_array_data(&view, &mut buffer, 72)?; - let expected_bytes = unsafe { - std::slice::from_raw_parts( - expected_data.as_ptr() as *const u8, - expected_data.len() * elem_size, - ) - }; - assert_eq!(buffer, expected_bytes); - Ok(()) -} - -#[test] -fn test_basic_edge_cases() -> TestResult { - // empty array - let elem_size = std::mem::size_of::() as isize; - let empty_view: StrideArrayView<'_, f64> = - unsafe { StrideArrayView::new(2, [0, 0].as_ptr(), [0, 0].as_ptr(), ptr::null(), 0)? }; - assert_eq!(empty_view.ndim(), 2); - assert_eq!(empty_view.dim(0), Ok(0)); - assert_eq!(empty_view.dim(1), Ok(0)); - - // single element array - let single_data = [42.0]; - let single_view: StrideArrayView<'_, f64> = unsafe { - StrideArrayView::new( - 1, - [1].as_ptr(), - [elem_size].as_ptr(), - single_data.as_ptr() as *const u8, - elem_size as usize, - ) - }?; - let mut buf = vec![0u8; 8]; - write_array_data(&single_view, &mut buf, 8).unwrap(); - assert_eq!(buf, 42.0f64.to_ne_bytes()); - Ok(()) -} - -#[test] -fn test_buffer_basic_write() -> TestResult { - let elem_size = std::mem::size_of::() as isize; - - let test_data = [1.1, 2.2, 3.3, 4.4]; - let array_view: StrideArrayView<'_, f64> = unsafe { - StrideArrayView::new( - 2, - [2, 2].as_ptr(), - [2 * elem_size, elem_size].as_ptr(), - test_data.as_ptr() as *const u8, - test_data.len() * elem_size as usize, - ) - }?; - let mut buffer = Buffer::new(ProtocolVersion::V2); - buffer.table("my_test")?; - buffer.column_arr("temperature", &array_view)?; - let data = buffer.as_bytes(); - assert_eq!(&data[0..7], b"my_test"); - assert_eq!(&data[8..19], b"temperature"); - assert_eq!( - &data[19..24], - &[ - b'=', - b'=', - ARRAY_BINARY_FORMAT_TYPE, - ArrayColumnTypeTag::Double.into(), - 2u8 - ] - ); - assert_eq!( - &data[24..32], - [2i32.to_le_bytes(), 2i32.to_le_bytes()].concat() - ); - assert_eq!( - &data[32..64], - &[ - 1.1f64.to_ne_bytes(), - 2.2f64.to_le_bytes(), - 3.3f64.to_le_bytes(), - 4.4f64.to_le_bytes(), - ] - .concat() - ); - Ok(()) -} - -#[test] -fn test_stride_array_size_overflow() -> TestResult { - let result = unsafe { - StrideArrayView::::new( - 2, - [u32::MAX as usize, u32::MAX as usize].as_ptr(), - [8, 8].as_ptr(), - ptr::null(), - 0, - ) - }; - let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ArrayViewError); - assert!(err.msg().contains("Array buffer size too big")); - Ok(()) -} - -#[test] -fn test_stride_view_length_mismatch() -> TestResult { - let elem_size = size_of::() as isize; - let under_data = [1.1]; - let result: Result, Error> = unsafe { - StrideArrayView::new( - 2, - [1, 2].as_ptr(), - [elem_size, elem_size].as_ptr(), - under_data.as_ptr() as *const u8, - under_data.len() * elem_size as usize, - ) - }; - let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ArrayViewError); - assert!(err - .msg() - .contains("Array buffer length mismatch (actual: 8, expected: 16)")); - - let over_data = [1.1, 2.2, 3.3]; - let result: Result, Error> = unsafe { - StrideArrayView::new( - 2, - [1, 2].as_ptr(), - [elem_size, elem_size].as_ptr(), - over_data.as_ptr() as *const u8, - over_data.len() * elem_size as usize, - ) - }; - - let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ArrayViewError); - assert!(err - .msg() - .contains("Array buffer length mismatch (actual: 24, expected: 16)")); - Ok(()) -} - #[test] fn test_build_in_1d_array_normal() -> TestResult { let arr = [1.0f64, 2.0, 3.0, 4.0]; From 0c12e6d9e90e8830c6c6affa61bb462a73d3a628 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 21 May 2025 13:29:58 +0800 Subject: [PATCH 102/119] add max_name_len ffi --- include/questdb/ingress/line_sender.h | 8 ++++++++ include/questdb/ingress/line_sender.hpp | 11 +++++++++++ questdb-rs-ffi/src/lib.rs | 11 +++++++++++ 3 files changed, 30 insertions(+) diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 064ea3b1..650ce2ad 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -821,6 +821,14 @@ LINESENDER_API bool line_sender_opts_max_buf_size( line_sender_opts* opts, size_t max_buf_size, line_sender_error** err_out); +/** + * Set the maximum length of a table or column name in bytes. + * The default is 127 bytes. + */ +LINESENDER_API +bool line_sender_opts_max_name_len( + line_sender_opts* opts, size_t max_name_len, line_sender_error** err_out); + /** * Set the cumulative duration spent in retries. * The value is in milliseconds, and the default is 10 seconds. diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index cc380957..eece0ff8 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -1094,6 +1094,17 @@ class opts return *this; } + /** + * The maximum length of a table or column name in bytes. + * The default is 127 bytes. + */ + opts& max_name_len(size_t max_name_len) + { + line_sender_error::wrapped_call( + ::line_sender_opts_max_name_len, _impl, max_name_len); + return *this; + } + /** * Set the cumulative duration spent in retries. * The value is in milliseconds, and the default is 10 seconds. diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index cd277803..bad81593 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -1261,6 +1261,17 @@ pub unsafe extern "C" fn line_sender_opts_max_buf_size( upd_opts!(opts, err_out, max_buf_size, max_buf_size) } +/// Ser the maximum length of a table or column name in bytes. +/// The default is 127 bytes. +#[no_mangle] +pub unsafe extern "C" fn line_sender_opts_max_name_len( + opts: *mut line_sender_opts, + max_name_len: size_t, + err_out: *mut *mut line_sender_error, +) -> bool { + upd_opts!(opts, err_out, max_name_len, max_name_len) +} + /// Set the cumulative duration spent in retries. /// The value is in milliseconds, and the default is 10 seconds. #[no_mangle] From 8df0379140f006e2bb7e32515d701c14269a97b5 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Wed, 21 May 2025 12:24:52 +0100 Subject: [PATCH 103/119] You shall pass! --- ci/run_all_tests.py | 1 + questdb-rs-ffi/src/ndarr.rs | 91 +++++++++++++++++++++++++++++++---- questdb-rs/src/ingress/mod.rs | 4 +- questdb-rs/src/tests/ndarr.rs | 2 +- 4 files changed, 85 insertions(+), 13 deletions(-) diff --git a/ci/run_all_tests.py b/ci/run_all_tests.py index ee261896..4d7624f5 100644 --- a/ci/run_all_tests.py +++ b/ci/run_all_tests.py @@ -51,6 +51,7 @@ def main(): '--', '--nocapture', cwd='questdb-rs') run_cmd('cargo', 'test', '--features=almost-all-features', '--', '--nocapture', cwd='questdb-rs') + run_cmd('cargo', 'test', cwd='questdb-rs-ffi') run_cmd(str(test_line_sender_path)) run_cmd(str(test_line_sender_path_CXX20)) #run_cmd('python3', str(system_test_path), 'run', '--versions', qdb_v, '-v') diff --git a/questdb-rs-ffi/src/ndarr.rs b/questdb-rs-ffi/src/ndarr.rs index 9ec8f2f6..5ad78181 100644 --- a/questdb-rs-ffi/src/ndarr.rs +++ b/questdb-rs-ffi/src/ndarr.rs @@ -29,6 +29,14 @@ use questdb::Error; use std::mem::size_of; use std::slice; +macro_rules! fmt_error { + ($code:ident, $($arg:tt)*) => { + questdb::Error::new( + questdb::ErrorCode::$code, + format!($($arg)*)) + } +} + /// A view into a multidimensional array with custom memory strides. // TODO: We are currently evaluating whether to use StrideArrayView or ndarray's view. // Current benchmarks show that StrideArrayView's iter implementation underperforms(2x) @@ -62,7 +70,7 @@ where fn dim(&self, index: usize) -> Result { if index >= self.dims { - return Err(error::fmt!( + return Err(fmt_error!( ArrayViewError, "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", index, @@ -141,13 +149,13 @@ where data_len: usize, ) -> Result { if dims == 0 { - return Err(error::fmt!( + return Err(fmt_error!( ArrayViewError, "Zero-dimensional arrays are not supported", )); } if data_len > MAX_ARRAY_BUFFER_SIZE { - return Err(error::fmt!( + return Err(fmt_error!( ArrayViewError, "Array buffer size too big: {}, maximum: {}", data_len, @@ -159,11 +167,11 @@ where .iter() .try_fold(std::mem::size_of::(), |acc, &dim| { acc.checked_mul(dim) - .ok_or_else(|| error::fmt!(ArrayViewError, "Array buffer size too big")) + .ok_or_else(|| fmt_error!(ArrayViewError, "Array buffer size too big")) })?; if size != data_len { - return Err(error::fmt!( + return Err(fmt_error!( ArrayViewError, "Array buffer length mismatch (actual: {}, expected: {})", data_len, @@ -256,8 +264,9 @@ where mod tests { use super::*; use questdb::ingress::*; + use questdb::{Error, ErrorCode}; use std::ptr; - type TestResult = Result<(), Box>; + type TestResult = std::result::Result<(), Box>; fn to_bytes(data: &[T]) -> Vec { data.iter() @@ -270,6 +279,69 @@ mod tests { .collect() } + // Duplicated from `questdb::ingress::ndarr::write_array_data` to avoid leaking it to the public API. + pub(crate) fn write_array_data, T>( + array: &A, + buf: &mut [u8], + expect_size: usize, + ) -> Result<(), Error> + where + T: ArrayElement, + { + // When working with contiguous layout, benchmark shows `copy_from_slice` has better performance than + // `std::ptr::copy_nonoverlapping` on both Arm(Macos) and x86(Linux) platform. + // This may because `copy_from_slice` benefits more from compiler. + if let Some(contiguous) = array.as_slice() { + let bytes = unsafe { + slice::from_raw_parts(contiguous.as_ptr() as *const u8, size_of_val(contiguous)) + }; + + if bytes.len() != expect_size { + return Err(fmt_error!( + ArrayWriteToBufferError, + "Array write buffer length mismatch (actual: {}, expected: {})", + expect_size, + bytes.len() + )); + } + + if buf.len() < bytes.len() { + return Err(fmt_error!( + ArrayWriteToBufferError, + "Buffer capacity {} < required {}", + buf.len(), + bytes.len() + )); + } + + buf[..bytes.len()].copy_from_slice(bytes); + return Ok(()); + } + + // For non-contiguous memory layouts, direct raw pointer operations are preferred. + let elem_size = size_of::(); + let mut total_len = 0; + for (i, &element) in array.iter().enumerate() { + unsafe { + std::ptr::copy_nonoverlapping( + &element as *const T as *const u8, + buf.as_mut_ptr().add(i * elem_size), + elem_size, + ) + } + total_len += elem_size; + } + if total_len != expect_size { + return Err(fmt_error!( + ArrayWriteToBufferError, + "Array write buffer length mismatch (actual: {}, expected: {})", + total_len, + expect_size + )); + } + Ok(()) + } + #[test] fn test_buffer_basic_write() -> TestResult { let elem_size = std::mem::size_of::() as isize; @@ -293,10 +365,8 @@ mod tests { assert_eq!( &data[19..24], &[ - b'=', - b'=', - ARRAY_BINARY_FORMAT_TYPE, - ArrayColumnTypeTag::Double.into(), + b'=', b'=', 14u8, // ARRAY_BINARY_FORMAT_TYPE + 10u8, // ArrayColumnTypeTag::Double.into() 2u8 ] ); @@ -394,6 +464,7 @@ mod tests { assert_eq!(array_view.dim(1), Ok(2)); assert!(array_view.dim(2).is_err()); assert!(array_view.as_slice().is_none()); + let mut buffer = vec![0u8; 48]; write_array_data(&array_view, &mut buffer, 48)?; diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 6b5ec6e4..3fb25dea 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -67,8 +67,8 @@ pub const MAX_ARRAY_DIMS: usize = 32; // we've ever tested anything that big. // My gut feeling is that the maximum array buffer should be // in the order of 100MB or so. -const MAX_ARRAY_BUFFER_SIZE: usize = i32::MAX as usize; -pub(crate) const MAX_ARRAY_DIM_LEN: usize = 0x0FFF_FFFF; // 1 << 28 - 1 +pub const MAX_ARRAY_BUFFER_SIZE: usize = i32::MAX as usize; +pub const MAX_ARRAY_DIM_LEN: usize = 0x0FFF_FFFF; // 1 << 28 - 1 /// The version of Ingestion Line Protocol used to communicate with the server. #[derive(Debug, Copy, Clone, PartialEq)] diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index 30505e5b..48dcbb55 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -22,7 +22,7 @@ * ******************************************************************************/ -#[cfg(feature = "ndarray")] +use crate::ingress::ndarr::write_array_data; use crate::ingress::MAX_ARRAY_DIMS; use crate::ingress::{Buffer, NdArrayView, ProtocolVersion, ARRAY_BINARY_FORMAT_TYPE}; use crate::tests::TestResult; From 48748cb3ae3d4e836ed5d4cfe92e86bba324ff97 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Wed, 21 May 2025 14:09:25 +0100 Subject: [PATCH 104/119] removed no-panic dependency and cleaned up unused code when building without ndarray --- questdb-rs-ffi/Cargo.toml | 13 +++++++++++-- questdb-rs/Cargo.toml | 1 - questdb-rs/src/ingress/mod.rs | 1 - questdb-rs/src/tests/ndarr.rs | 6 ++++-- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/questdb-rs-ffi/Cargo.toml b/questdb-rs-ffi/Cargo.toml index bf8bbbc2..1d461b5d 100644 --- a/questdb-rs-ffi/Cargo.toml +++ b/questdb-rs-ffi/Cargo.toml @@ -9,11 +9,20 @@ name = "questdb_client" crate-type = ["cdylib", "staticlib"] [dependencies] -questdb-rs = { path = "../questdb-rs", features = [ - "insecure-skip-verify", "tls-native-certs", "ilp-over-http"] } libc = "0.2" questdb-confstr-ffi = { version = "0.1.1", optional = true } +[dependencies.questdb-rs] +path = "../questdb-rs" +default-features = false +features = [ + "ring-crypto", + "insecure-skip-verify", + "tls-webpki-certs", + "tls-native-certs", + "ilp-over-http" +] + [features] # Expose the config parsing C API. # This used by `py-questdb-client` to parse the config file. diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index a927cfac..ff7ec416 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -40,7 +40,6 @@ serde_json = { version = "1", optional = true } questdb-confstr = "0.1.1" rand = { version = "0.9.0", optional = true } ndarray = { version = "0.16", optional = true } -no-panic = { version = "0.1", optional = true } [target.'cfg(windows)'.dependencies] winapi = { version = "0.3.9", features = ["ws2def"] } diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 3fb25dea..014c9570 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -2782,7 +2782,6 @@ impl F64Serializer { // This function was taken and customized from the ryu crate. #[cold] - #[cfg_attr(feature = "no-panic", inline)] fn format_nonfinite(&self) -> &'static str { const MANTISSA_MASK: u64 = 0x000fffffffffffff; const SIGN_MASK: u64 = 0x8000000000000000; diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index 48dcbb55..659f8ab4 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -22,12 +22,14 @@ * ******************************************************************************/ -use crate::ingress::ndarr::write_array_data; -use crate::ingress::MAX_ARRAY_DIMS; use crate::ingress::{Buffer, NdArrayView, ProtocolVersion, ARRAY_BINARY_FORMAT_TYPE}; use crate::tests::TestResult; use crate::ErrorCode; +#[cfg(feature = "ndarray")] +use crate::ingress::ndarr::write_array_data; +#[cfg(feature = "ndarray")] +use crate::ingress::MAX_ARRAY_DIMS; #[cfg(feature = "ndarray")] use ndarray::{arr1, arr2, arr3, s, ArrayD}; #[cfg(feature = "ndarray")] From c51fef2bcbf9b53ded2e123ca6d3c5fc61cb5121 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 21 May 2025 22:38:44 +0800 Subject: [PATCH 105/119] export max_array_dims --- questdb-rs-ffi/src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index bad81593..04e0d815 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -305,6 +305,9 @@ impl From for CertificateAuthority { } } +#[no_mangle] +pub static MAX_ARRAY_DIMS: usize = ingress::MAX_ARRAY_DIMS; + /** Error code categorizing the error. */ #[no_mangle] pub unsafe extern "C" fn line_sender_error_get_code( From ea906364406e3a456f3c1751df431095bf03e015 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 21 May 2025 22:42:30 +0800 Subject: [PATCH 106/119] Revert "export max_array_dims" This reverts commit c51fef2bcbf9b53ded2e123ca6d3c5fc61cb5121. --- questdb-rs-ffi/src/lib.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 04e0d815..bad81593 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -305,9 +305,6 @@ impl From for CertificateAuthority { } } -#[no_mangle] -pub static MAX_ARRAY_DIMS: usize = ingress::MAX_ARRAY_DIMS; - /** Error code categorizing the error. */ #[no_mangle] pub unsafe extern "C" fn line_sender_error_get_code( From 5da905e10e3653e5256863221c99e8bb077f7c14 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 22 May 2025 14:17:39 +0800 Subject: [PATCH 107/119] remove comment. --- examples/line_sender_c_example_array.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/line_sender_c_example_array.c b/examples/line_sender_c_example_array.c index f152b2ea..4272bc5b 100644 --- a/examples/line_sender_c_example_array.c +++ b/examples/line_sender_c_example_array.c @@ -29,7 +29,7 @@ static bool example(const char* host, const char* port) conf_str = NULL; buffer = line_sender_buffer_new_for_sender(sender); - line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB 初始缓冲 + line_sender_buffer_reserve(buffer, 64 * 1024); line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("market_orders"); line_sender_column_name symbol_col = QDB_COLUMN_NAME_LITERAL("symbol"); From 1ed8d6c494565d4d92e316badf970d8332fa4c4c Mon Sep 17 00:00:00 2001 From: victor Date: Fri, 23 May 2025 23:10:21 +0800 Subject: [PATCH 108/119] add `line_sender_c_example_array_elem_strides` api --- CMakeLists.txt | 15 +- cpp_test/test_line_sender.cpp | 43 +++- ...ine_sender_c_example_array_byte_strides.c} | 4 +- ...line_sender_c_example_array_elem_strides.c | 99 ++++++++ ...sender_cpp_example_array_byte_strides.cpp} | 4 +- ..._sender_cpp_example_array_elem_strides.cpp | 62 +++++ include/questdb/ingress/line_sender.h | 34 ++- include/questdb/ingress/line_sender.hpp | 11 +- questdb-rs-ffi/src/lib.rs | 70 +++++- questdb-rs-ffi/src/ndarr.rs | 220 ++++++++++++++++-- 10 files changed, 509 insertions(+), 53 deletions(-) rename examples/{line_sender_c_example_array.c => line_sender_c_example_array_byte_strides.c} (96%) create mode 100644 examples/line_sender_c_example_array_elem_strides.c rename examples/{line_sender_cpp_example_array.cpp => line_sender_cpp_example_array_byte_strides.cpp} (92%) create mode 100644 examples/line_sender_cpp_example_array_elem_strides.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b2612af7..2b36780e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,9 +100,13 @@ if (QUESTDB_TESTS_AND_EXAMPLES) examples/concat.c examples/line_sender_c_example.c) compile_example( - line_sender_c_example_array + line_sender_c_example_array_byte_strides examples/concat.c - examples/line_sender_c_example_array.c) + examples/line_sender_c_example_array_byte_strides.c) + compile_example( + line_sender_c_example_array_elem_strides + examples/concat.c + examples/line_sender_c_example_array_elem_strides.c) compile_example( line_sender_c_example_auth examples/concat.c @@ -129,8 +133,11 @@ if (QUESTDB_TESTS_AND_EXAMPLES) line_sender_cpp_example examples/line_sender_cpp_example.cpp) compile_example( - line_sender_cpp_example_array - examples/line_sender_cpp_example_array.cpp) + line_sender_cpp_example_array_byte_strides + examples/line_sender_cpp_example_array_byte_strides.cpp) + compile_example( + line_sender_cpp_example_array_elem_strides + examples/line_sender_cpp_example_array_elem_strides.cpp) compile_example( line_sender_cpp_example_auth examples/line_sender_cpp_example_auth.cpp) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 47f88b05..e2507797 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -178,23 +178,38 @@ TEST_CASE("line_sender c api basics") 2.7, 48121.5, 4.3}; - CHECK(::line_sender_buffer_column_f64_arr( - buffer, - arr_name, - rank, - shape, - strides, - reinterpret_cast(arr_data.data()), - sizeof(arr_data), - &err)); + CHECK( + ::line_sender_buffer_column_f64_arr_byte_strides( + buffer, + arr_name, + rank, + shape, + strides, + reinterpret_cast(arr_data.data()), + sizeof(arr_data), + &err)); + + line_sender_column_name arr_name2 = QDB_COLUMN_NAME_LITERAL("a2"); + intptr_t elem_strides[] = {6, 2, 1}; + CHECK( + ::line_sender_buffer_column_f64_arr_elem_strides( + buffer, + arr_name2, + rank, + shape, + elem_strides, + reinterpret_cast(arr_data.data()), + sizeof(arr_data), + &err)); CHECK(::line_sender_buffer_at_nanos(buffer, 10000000, &err)); CHECK(server.recv() == 0); - CHECK(::line_sender_buffer_size(buffer) == 150); + CHECK(::line_sender_buffer_size(buffer) == 266); CHECK(::line_sender_flush(sender, buffer, &err)); ::line_sender_buffer_free(buffer); CHECK(server.recv() == 1); std::string expect{"test,t1=v1 f1=="}; push_double_to_buffer(expect, 0.5).append(",a1=="); + push_double_arr_to_buffer(expect, arr_data, 3, shape).append(",a2=="); push_double_arr_to_buffer(expect, arr_data, 3, shape).append(" 10000000\n"); CHECK(server.msgs(0) == expect); } @@ -264,19 +279,23 @@ TEST_CASE("line_sender c++ api basics") 2.7, 48121.5, 4.3}; + std::vector elem_strides{6, 2, 1}; buffer.table("test") .symbol("t1", "v1") .symbol("t2", "") .column("f1", 0.5) - .column("a1", rank, shape, strides, arr_data) + .column("a1", rank, shape, strides, arr_data) + .column("a2", rank, shape, elem_strides, arr_data) .at(questdb::ingress::timestamp_nanos{10000000}); CHECK(server.recv() == 0); - CHECK(buffer.size() == 154); + CHECK(buffer.size() == 270); sender.flush(buffer); CHECK(server.recv() == 1); std::string expect{"test,t1=v1,t2= f1=="}; push_double_to_buffer(expect, 0.5).append(",a1=="); + push_double_arr_to_buffer(expect, arr_data, 3, shape.data()) + .append(",a2=="); push_double_arr_to_buffer(expect, arr_data, 3, shape.data()) .append(" 10000000\n"); CHECK(server.msgs(0) == expect); diff --git a/examples/line_sender_c_example_array.c b/examples/line_sender_c_example_array_byte_strides.c similarity index 96% rename from examples/line_sender_c_example_array.c rename to examples/line_sender_c_example_array_byte_strides.c index 4272bc5b..82e076e9 100644 --- a/examples/line_sender_c_example_array.c +++ b/examples/line_sender_c_example_array_byte_strides.c @@ -31,7 +31,7 @@ static bool example(const char* host, const char* port) buffer = line_sender_buffer_new_for_sender(sender); line_sender_buffer_reserve(buffer, 64 * 1024); - line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("market_orders"); + line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("market_orders_byte_strides"); line_sender_column_name symbol_col = QDB_COLUMN_NAME_LITERAL("symbol"); line_sender_column_name book_col = QDB_COLUMN_NAME_LITERAL("order_book"); @@ -60,7 +60,7 @@ static bool example(const char* host, const char* port) 48121.5, 4.3}; - if (!line_sender_buffer_column_f64_arr( + if (!line_sender_buffer_column_f64_arr_byte_strides( buffer, book_col, array_rank, diff --git a/examples/line_sender_c_example_array_elem_strides.c b/examples/line_sender_c_example_array_elem_strides.c new file mode 100644 index 00000000..44c81e05 --- /dev/null +++ b/examples/line_sender_c_example_array_elem_strides.c @@ -0,0 +1,99 @@ +#include +#include +#include +#include +#include "concat.h" + +static bool example(const char* host, const char* port) +{ + line_sender_error* err = NULL; + line_sender* sender = NULL; + line_sender_buffer* buffer = NULL; + char* conf_str = concat("tcp::addr=", host, ":", port, ";protocol_version=2;"); + if (!conf_str) + { + fprintf(stderr, "Could not concatenate configuration string.\n"); + return false; + } + + line_sender_utf8 conf_str_utf8 = {0, NULL}; + if (!line_sender_utf8_init( + &conf_str_utf8, strlen(conf_str), conf_str, &err)) + goto on_error; + + sender = line_sender_from_conf(conf_str_utf8, &err); + if (!sender) + goto on_error; + + free(conf_str); + conf_str = NULL; + + buffer = line_sender_buffer_new_for_sender(sender); + line_sender_buffer_reserve(buffer, 64 * 1024); + + line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("market_orders_elem_strides"); + line_sender_column_name symbol_col = QDB_COLUMN_NAME_LITERAL("symbol"); + line_sender_column_name book_col = QDB_COLUMN_NAME_LITERAL("order_book"); + + if (!line_sender_buffer_table(buffer, table_name, &err)) + goto on_error; + + line_sender_utf8 symbol_val = QDB_UTF8_LITERAL("BTC-USD"); + if (!line_sender_buffer_symbol(buffer, symbol_col, symbol_val, &err)) + goto on_error; + + size_t array_rank = 3; + uintptr_t array_shape[] = {2, 3, 2}; + intptr_t array_strides[] = {6, 2, 1}; + + double array_data[] = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + + if (!line_sender_buffer_column_f64_arr_elem_strides( + buffer, + book_col, + array_rank, + array_shape, + array_strides, + (const uint8_t*)array_data, + sizeof(array_data), + &err)) + goto on_error; + + if (!line_sender_buffer_at_nanos(buffer, line_sender_now_nanos(), &err)) + goto on_error; + + if (!line_sender_flush(sender, buffer, &err)) + goto on_error; + + line_sender_close(sender); + return true; + +on_error:; + size_t err_len = 0; + const char* err_msg = line_sender_error_msg(err, &err_len); + fprintf(stderr, "Error: %.*s\n", (int)err_len, err_msg); + free(conf_str); + line_sender_error_free(err); + line_sender_buffer_free(buffer); + line_sender_close(sender); + return false; +} + +int main(int argc, const char* argv[]) +{ + const char* host = (argc >= 2) ? argv[1] : "localhost"; + const char* port = (argc >= 3) ? argv[2] : "9009"; + return !example(host, port); +} diff --git a/examples/line_sender_cpp_example_array.cpp b/examples/line_sender_cpp_example_array_byte_strides.cpp similarity index 92% rename from examples/line_sender_cpp_example_array.cpp rename to examples/line_sender_cpp_example_array_byte_strides.cpp index 73b56c62..88de8a39 100644 --- a/examples/line_sender_cpp_example_array.cpp +++ b/examples/line_sender_cpp_example_array_byte_strides.cpp @@ -13,7 +13,7 @@ static bool array_example(std::string_view host, std::string_view port) "tcp::addr=" + std::string{host} + ":" + std::string{port} + ";protocol_version=2;"); - const auto table_name = "cpp_market_orders"_tn; + const auto table_name = "cpp_market_orders_byte_strides"_tn; const auto symbol_col = "symbol"_cn; const auto book_col = "order_book"_cn; size_t rank = 3; @@ -36,7 +36,7 @@ static bool array_example(std::string_view host, std::string_view port) questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table(table_name) .symbol(symbol_col, "BTC-USD"_utf8) - .column(book_col, 3, shape, strides, arr_data) + .column(book_col, 3, shape, strides, arr_data) .at(questdb::ingress::timestamp_nanos::now()); sender.flush(buffer); return true; diff --git a/examples/line_sender_cpp_example_array_elem_strides.cpp b/examples/line_sender_cpp_example_array_elem_strides.cpp new file mode 100644 index 00000000..4161f5aa --- /dev/null +++ b/examples/line_sender_cpp_example_array_elem_strides.cpp @@ -0,0 +1,62 @@ +#include +#include +#include + +using namespace std::literals::string_view_literals; +using namespace questdb::ingress::literals; + +static bool array_example(std::string_view host, std::string_view port) +{ + try + { + auto sender = questdb::ingress::line_sender::from_conf( + "tcp::addr=" + std::string{host} + ":" + std::string{port} + + ";protocol_version=2;"); + + const auto table_name = "cpp_market_orders_elem_strides"_tn; + const auto symbol_col = "symbol"_cn; + const auto book_col = "order_book"_cn; + size_t rank = 3; + std::vector shape{2, 3, 2}; + std::vector strides{6, 2, 8}; + std::array arr_data = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); + buffer.table(table_name) + .symbol(symbol_col, "BTC-USD"_utf8) + .column(book_col, 3, shape, strides, arr_data) + .at(questdb::ingress::timestamp_nanos::now()); + sender.flush(buffer); + return true; + } + catch (const questdb::ingress::line_sender_error& err) + { + std::cerr << "[ERROR] " << err.what() << std::endl; + return false; + } +} + +int main(int argc, const char* argv[]) +{ + auto host = "localhost"sv; + if (argc >= 2) + host = std::string_view{argv[1]}; + + auto port = "9009"sv; + if (argc >= 3) + port = std::string_view{argv[2]}; + + return !array_example(host, port); +} diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 650ce2ad..16265832 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -499,7 +499,37 @@ bool line_sender_buffer_column_str( /** * Record a multidimensional array of double for the given column. - * The array data must be stored in row-major order (C-style contiguous layout). + * + * This API uses BYTE-LEVEL STRIDES where the stride values represent the + * number of bytes between consecutive elements along each dimension. + * + * @param[in] buffer Line buffer object. + * @param[in] name Column name. + * @param[in] rank Number of dimensions of the array. + * @param[in] shape Array of dimension sizes (length = `rank`). + * Each element must be a positive integer. + * @param[in] strides Array strides. + * @param[in] data_buffer First array element data. + * @param[in] data_buffer_len Bytes length of the array data. + * @param[out] err_out Set to an error object on failure (if non-NULL). + * @return true on success, false on error. + */ +LINESENDER_API +bool line_sender_buffer_column_f64_arr_byte_strides( + line_sender_buffer* buffer, + line_sender_column_name name, + size_t rank, + const uintptr_t* shape, + const intptr_t* strides, + const uint8_t* data_buffer, + size_t data_buffer_len, + line_sender_error** err_out); + +/** + * Record a multidimensional array of double for the given column. + * + * This function uses ELEMENT-LEVEL STRIDES where the stride values represent + * the number of elements between consecutive elements along each dimension. * * @param[in] buffer Line buffer object. * @param[in] name Column name. @@ -513,7 +543,7 @@ bool line_sender_buffer_column_str( * @return true on success, false on error. */ LINESENDER_API -bool line_sender_buffer_column_f64_arr( +bool line_sender_buffer_column_f64_arr_elem_strides( line_sender_buffer* buffer, line_sender_column_name name, size_t rank, diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index eece0ff8..dc4e0b00 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -643,12 +643,18 @@ class line_sender_buffer /** * Record a multidimensional double-precision array for the given column. * + * @tparam B Strides mode selector: + * - `true` for byte-level strides + * - `false` for element-level strides + * @tparam T Element type (current only `double` is supported). + * @tparam N Number of elements in the flat data array + * * @param name Column name. * @param shape Array dimensions (e.g., [2,3] for a 2x3 matrix). * @param data Array first element data. Size must match product of * dimensions. */ - template + template line_sender_buffer& column( column_name_view name, const size_t rank, @@ -661,7 +667,8 @@ class line_sender_buffer "Only double types are supported for arrays"); may_init(); line_sender_error::wrapped_call( - ::line_sender_buffer_column_f64_arr, + B ? ::line_sender_buffer_column_f64_arr_byte_strides + : ::line_sender_buffer_column_f64_arr_elem_strides, _impl, name._impl, rank, diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index bad81593..033885d3 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -867,12 +867,15 @@ pub unsafe extern "C" fn line_sender_buffer_column_str( true } -/// Record a float multidimensional array value for the given column. +/// Records a float64 multidimensional array with **byte-level strides specification**. +/// +/// The `strides` represent byte offsets between elements along each dimension. +/// /// @param[in] buffer Line buffer object. /// @param[in] name Column name. /// @param[in] rank Array dims. /// @param[in] shape Array shape. -/// @param[in] strides Array strides. +/// @param[in] strides Array strides, represent byte offsets between elements along each dimension. /// @param[in] data_buffer Array **first element** data memory ptr. /// @param[in] data_buffer_len Array data memory length. /// @param[out] err_out Set on error. @@ -881,7 +884,7 @@ pub unsafe extern "C" fn line_sender_buffer_column_str( /// - shape must point to an array of `rank` integers /// - data_buffer must point to a buffer of size `data_buffer_len` bytes #[no_mangle] -pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( +pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_byte_strides( buffer: *mut line_sender_buffer, name: line_sender_column_name, rank: size_t, @@ -893,8 +896,59 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( ) -> bool { let buffer = unwrap_buffer_mut(buffer); let name = name.as_name(); - let view = match StrideArrayView::::new(rank, shape, strides, data_buffer, data_buffer_len) - { + let view = + match StrideArrayView::::new(rank, shape, strides, data_buffer, data_buffer_len) { + Ok(value) => value, + Err(err) => { + let err_ptr = Box::into_raw(Box::new(line_sender_error(err))); + *err_out = err_ptr; + return false; + } + }; + bubble_err_to_c!( + err_out, + buffer.column_arr::, StrideArrayView<'_, f64, 1>, f64>(name, &view) + ); + true +} + +/// Records a float64 multidimensional array with **element count stride specification**. +/// +/// The `strides` represent element counts between elements along each dimension. +/// +/// converted to byte strides using f64 size +/// @param[in] buffer Line buffer object. +/// @param[in] name Column name. +/// @param[in] rank Array dims. +/// @param[in] shape Array shape. +/// @param[in] strides Array strides, represent element counts between elements along each dimension. +/// @param[in] data_buffer Array **first element** data memory ptr. +/// @param[in] data_buffer_len Array data memory length. +/// @param[out] err_out Set on error. +/// # Safety +/// - All pointer parameters must be valid and non-null +/// - shape must point to an array of `rank` integers +/// - data_buffer must point to a buffer of size `data_buffer_len` bytes +#[no_mangle] +pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_elem_strides( + buffer: *mut line_sender_buffer, + name: line_sender_column_name, + rank: size_t, + shape: *const usize, + strides: *const isize, + data_buffer: *const u8, + data_buffer_len: size_t, + err_out: *mut *mut line_sender_error, +) -> bool { + let buffer = unwrap_buffer_mut(buffer); + let name = name.as_name(); + let view = match StrideArrayView::() as isize }>::new( + rank, + shape, + strides, + data_buffer, + data_buffer_len, + ) { Ok(value) => value, Err(err) => { let err_ptr = Box::into_raw(Box::new(line_sender_error(err))); @@ -903,9 +957,9 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( } }; bubble_err_to_c!( - err_out, - buffer.column_arr::, StrideArrayView<'_, f64>, f64>(name, &view) - ); + err_out, + buffer.column_arr::, StrideArrayView<'_, f64, { std::mem::size_of::() as isize }>, f64>(name, &view) + ); true } diff --git a/questdb-rs-ffi/src/ndarr.rs b/questdb-rs-ffi/src/ndarr.rs index 5ad78181..cbc1b452 100644 --- a/questdb-rs-ffi/src/ndarr.rs +++ b/questdb-rs-ffi/src/ndarr.rs @@ -46,7 +46,7 @@ macro_rules! fmt_error { // which we need to support in this FFI crate for efficient iteration of // numpy arrays coming from Python without copying the data. #[derive(Debug)] -pub struct StrideArrayView<'a, T> { +pub struct StrideArrayView<'a, T, const N: isize> { dims: usize, shape: &'a [usize], strides: &'a [isize], @@ -54,12 +54,12 @@ pub struct StrideArrayView<'a, T> { _marker: std::marker::PhantomData, } -impl NdArrayView for StrideArrayView<'_, T> +impl NdArrayView for StrideArrayView<'_, T, N> where T: ArrayElement, { type Iter<'b> - = RowMajorIter<'b, T> + = RowMajorIter<'b, T, N> where Self: 'b, T: 'b; @@ -105,9 +105,10 @@ where .iter() .enumerate() .fold(data.as_ptr(), |ptr, (dim, &stride)| { - if stride < 0 { + let stride_bytes_size = stride * N; + if stride_bytes_size < 0 { let dim_size = self.shape[dim] as isize; - unsafe { ptr.offset(stride * (dim_size - 1)) } + unsafe { ptr.offset(stride_bytes_size * (dim_size - 1)) } } else { ptr } @@ -125,7 +126,7 @@ where } } -impl StrideArrayView<'_, T> +impl StrideArrayView<'_, T, N> where T: ArrayElement, { @@ -203,12 +204,12 @@ where let elem_size = size_of::() as isize; if self.dims == 1 { - return self.strides[0] == elem_size || self.shape[0] == 1; + return self.strides[0] * N == elem_size || self.shape[0] == 1; } let mut expected_stride = elem_size; for (dim, &stride) in self.shape.iter().zip(self.strides).rev() { - if *dim > 1 && stride != expected_stride { + if *dim > 1 && stride * N != expected_stride { return false; } expected_stride *= *dim as isize; @@ -220,15 +221,15 @@ where } /// Iterator for traversing a stride array in row-major (C-style) order. -pub struct RowMajorIter<'a, T> { +pub struct RowMajorIter<'a, T, const N: isize> { base_ptr: *const u8, - array: &'a StrideArrayView<'a, T>, + array: &'a StrideArrayView<'a, T, N>, dim_products: Vec, current_linear: usize, total_elements: usize, } -impl<'a, T> Iterator for RowMajorIter<'a, T> +impl<'a, T, const N: isize> Iterator for RowMajorIter<'a, T, N> where T: ArrayElement, { @@ -243,7 +244,7 @@ where for (dim, &dim_factor) in self.dim_products.iter().enumerate() { let coord = remaining_index / dim_factor; remaining_index %= dim_factor; - let stride = self.array.strides[dim]; + let stride = self.array.strides[dim] * N; let actual_coord = if stride >= 0 { coord } else { @@ -347,7 +348,7 @@ mod tests { let elem_size = std::mem::size_of::() as isize; let test_data = [1.1, 2.2, 3.3, 4.4]; - let array_view: StrideArrayView<'_, f64> = unsafe { + let array_view: StrideArrayView<'_, f64, 1> = unsafe { StrideArrayView::new( 2, [2, 2].as_ptr(), @@ -387,10 +388,55 @@ mod tests { Ok(()) } + #[test] + fn test_buffer_basic_write_with_elem_strides() -> TestResult { + let elem_size = std::mem::size_of::() as isize; + + let test_data = [1.1, 2.2, 3.3, 4.4]; + let array_view: StrideArrayView<'_, f64, 8> = unsafe { + StrideArrayView::new( + 2, + [2, 2].as_ptr(), + [2, 1].as_ptr(), + test_data.as_ptr() as *const u8, + test_data.len() * elem_size as usize, + ) + }?; + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("temperature", &array_view)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); + assert_eq!( + &data[19..24], + &[ + b'=', b'=', 14u8, // ARRAY_BINARY_FORMAT_TYPE + 10u8, // ArrayColumnTypeTag::Double.into() + 2u8 + ] + ); + assert_eq!( + &data[24..32], + [2i32.to_le_bytes(), 2i32.to_le_bytes()].concat() + ); + assert_eq!( + &data[32..64], + &[ + 1.1f64.to_ne_bytes(), + 2.2f64.to_le_bytes(), + 3.3f64.to_le_bytes(), + 4.4f64.to_le_bytes(), + ] + .concat() + ); + Ok(()) + } + #[test] fn test_stride_array_size_overflow() -> TestResult { let result = unsafe { - StrideArrayView::::new( + StrideArrayView::::new( 2, [u32::MAX as usize, u32::MAX as usize].as_ptr(), [8, 8].as_ptr(), @@ -408,11 +454,49 @@ mod tests { fn test_stride_view_length_mismatch() -> TestResult { let elem_size = size_of::() as isize; let under_data = [1.1]; - let result: Result, Error> = unsafe { + let result: Result, Error> = unsafe { + StrideArrayView::new( + 2, + [1, 2].as_ptr(), + [elem_size, elem_size].as_ptr(), + under_data.as_ptr() as *const u8, + under_data.len() * elem_size as usize, + ) + }; + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert!(err + .msg() + .contains("Array buffer length mismatch (actual: 8, expected: 16)")); + + let over_data = [1.1, 2.2, 3.3]; + let result: Result, Error> = unsafe { StrideArrayView::new( 2, [1, 2].as_ptr(), [elem_size, elem_size].as_ptr(), + over_data.as_ptr() as *const u8, + over_data.len() * elem_size as usize, + ) + }; + + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert!(err + .msg() + .contains("Array buffer length mismatch (actual: 24, expected: 16)")); + Ok(()) + } + + #[test] + fn test_stride_view_length_mismatch_with_elem_strides() -> TestResult { + let elem_size = size_of::() as isize; + let under_data = [1.1]; + let result: Result, Error> = unsafe { + StrideArrayView::new( + 2, + [1, 2].as_ptr(), + [1, 1].as_ptr(), under_data.as_ptr() as *const u8, under_data.len() * elem_size as usize, ) @@ -424,7 +508,7 @@ mod tests { .contains("Array buffer length mismatch (actual: 8, expected: 16)")); let over_data = [1.1, 2.2, 3.3]; - let result: Result, Error> = unsafe { + let result: Result, Error> = unsafe { StrideArrayView::new( 2, [1, 2].as_ptr(), @@ -449,7 +533,44 @@ mod tests { let shape = [3usize, 2]; let strides = [elem_size, shape[0] as isize * elem_size]; - let array_view: StrideArrayView<'_, f64> = unsafe { + let array_view: StrideArrayView<'_, f64, 1> = unsafe { + StrideArrayView::new( + shape.len(), + shape.as_ptr(), + strides.as_ptr(), + col_major_data.as_ptr() as *const u8, + col_major_data.len() * elem_size as usize, + ) + }?; + + assert_eq!(array_view.ndim(), 2); + assert_eq!(array_view.dim(0), Ok(3)); + assert_eq!(array_view.dim(1), Ok(2)); + assert!(array_view.dim(2).is_err()); + assert!(array_view.as_slice().is_none()); + + let mut buffer = vec![0u8; 48]; + write_array_data(&array_view, &mut buffer, 48)?; + + let expected_data = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; + let expected_bytes = unsafe { + std::slice::from_raw_parts( + expected_data.as_ptr() as *const u8, + expected_data.len() * elem_size as usize, + ) + }; + assert_eq!(buffer, expected_bytes); + Ok(()) + } + + #[test] + fn test_strided_non_contiguous_with_elem_strides() -> TestResult { + let elem_size = size_of::() as isize; + let col_major_data = [1.0, 3.0, 5.0, 2.0, 4.0, 6.0]; + let shape = [3usize, 2]; + let strides = [1, shape[0] as isize]; + + let array_view: StrideArrayView<'_, f64, 8> = unsafe { StrideArrayView::new( shape.len(), shape.as_ptr(), @@ -484,7 +605,7 @@ mod tests { let elem_size = size_of::(); let data = [1f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]; let view = unsafe { - StrideArrayView::::new( + StrideArrayView::::new( 2, &[3usize, 3] as *const usize, &[-24isize, 8] as *const isize, @@ -508,11 +629,40 @@ mod tests { Ok(()) } + #[test] + fn test_negative_strides_with_elem_strides() -> TestResult { + let elem_size = size_of::(); + let data = [1f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]; + let view = unsafe { + StrideArrayView::::new( + 2, + &[3usize, 3] as *const usize, + &[-3isize, 1] as *const isize, + (data.as_ptr() as *const u8).add(48), + data.len() * elem_size, + ) + }?; + let collected: Vec<_> = view.iter().copied().collect(); + assert!(view.as_slice().is_none()); + let expected_data = vec![7.0, 8.0, 9.0, 4.0, 5.0, 6.0, 1.0, 2.0, 3.0]; + assert_eq!(collected, expected_data); + let mut buffer = vec![0u8; 72]; + write_array_data(&view, &mut buffer, 72)?; + let expected_bytes = unsafe { + std::slice::from_raw_parts( + expected_data.as_ptr() as *const u8, + expected_data.len() * elem_size, + ) + }; + assert_eq!(buffer, expected_bytes); + Ok(()) + } + #[test] fn test_basic_edge_cases() -> TestResult { // empty array let elem_size = std::mem::size_of::() as isize; - let empty_view: StrideArrayView<'_, f64> = + let empty_view: StrideArrayView<'_, f64, 1> = unsafe { StrideArrayView::new(2, [0, 0].as_ptr(), [0, 0].as_ptr(), ptr::null(), 0)? }; assert_eq!(empty_view.ndim(), 2); assert_eq!(empty_view.dim(0), Ok(0)); @@ -520,7 +670,7 @@ mod tests { // single element array let single_data = [42.0]; - let single_view: StrideArrayView<'_, f64> = unsafe { + let single_view: StrideArrayView<'_, f64, 1> = unsafe { StrideArrayView::new( 1, [1].as_ptr(), @@ -545,7 +695,35 @@ mod tests { size_of::() as isize, ]; let array = unsafe { - StrideArrayView::::new( + StrideArrayView::::new( + shape.len(), + shape.as_ptr(), + strides.as_ptr(), + test_data.as_ptr() as *const u8, + test_data.len() * size_of::(), + ) + }?; + + assert_eq!(array.ndim(), 2); + assert_eq!(array.dim(0), Ok(2)); + assert_eq!(array.dim(1), Ok(3)); + assert!(array.dim(2).is_err()); + assert!(array.as_slice().is_some()); + let mut buf = vec![0u8; 48]; + write_array_data(&array, &mut buf, 48).unwrap(); + let expected = to_bytes(&test_data); + assert_eq!(buf, expected); + Ok(()) + } + + #[test] + fn test_stride_array_view_strides_with_elem_size() -> TestResult { + // contiguous layout + let test_data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; + let shape = [2usize, 3]; + let strides = [shape[1] as isize, 1]; + let array = unsafe { + StrideArrayView::() as isize }>::new( shape.len(), shape.as_ptr(), strides.as_ptr(), From 27383c81ec7dc4fc97ba0cb0f37b23ceff712022 Mon Sep 17 00:00:00 2001 From: victor Date: Fri, 23 May 2025 23:12:59 +0800 Subject: [PATCH 109/119] remove buffer size overflow test --- system_test/test.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/system_test/test.py b/system_test/test.py index 095e0427..e7d60dbc 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -776,13 +776,19 @@ def test_cpp_tls_example(self): def test_cpp_array_example(self): self._test_array_example( - 'line_sender_cpp_example_array', - 'cpp_market_orders') + 'line_sender_cpp_example_array_byte_strides', + 'cpp_market_orders_byte_strides', ) + self._test_array_example( + 'line_sender_cpp_example_array_elem_strides', + 'cpp_market_orders_elem_strides', ) def test_c_array_example(self): self._test_array_example( - 'line_sender_c_example_array', - 'market_orders') + 'line_sender_c_example_array_byte_strides', + 'market_orders_byte_strides', ) + self._test_array_example( + 'line_sender_c_example_array_elem_strides', + 'market_orders_elem_strides', ) def _test_array_example(self, bin_name, table_name): if self.expected_protocol_version < qls.ProtocolVersion.V2: @@ -1200,7 +1206,7 @@ def run_with_fixtures(args): sys.exit(1) finally: if TLS_PROXY_FIXTURE: - TLS_PROXY_FIXTURE.stop() + TLS_PROXY_FIXTURE.stop() finally: QDB_FIXTURE.stop() From f8a17712b5fa2a37e9ebf6bc43c433d4a43d3b97 Mon Sep 17 00:00:00 2001 From: victor Date: Fri, 23 May 2025 23:15:15 +0800 Subject: [PATCH 110/119] remove buffer size overflow test --- system_test/questdb_line_sender.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/system_test/questdb_line_sender.py b/system_test/questdb_line_sender.py index b58248fd..0bc73da9 100644 --- a/system_test/questdb_line_sender.py +++ b/system_test/questdb_line_sender.py @@ -290,7 +290,7 @@ def set_sig(fn, restype, *argtypes): c_line_sender_utf8, c_line_sender_error_p_p) set_sig( - dll.line_sender_buffer_column_f64_arr, + dll.line_sender_buffer_column_f64_arr_byte_strides, c_bool, c_line_sender_buffer_p, c_line_sender_column_name, @@ -730,7 +730,7 @@ def _convert_tuple(tpl: tuple[int, ...], c_type: type, name: str) -> ctypes.POIN c_shape = _convert_tuple(shape, c_size_t, "shape") c_strides = _convert_tuple(strides, c_ssize_t, "strides") _error_wrapped_call( - _DLL.line_sender_buffer_column_f64_arr, + _DLL.line_sender_buffer_column_f64_arr_byte_strides, self._impl, _column_name(name), c_size_t(rank), @@ -850,7 +850,7 @@ def protocol_version(self): self._check_connected() return ProtocolVersion.from_int( _DLL.line_sender_get_protocol_version(self._impl)) - + @property def max_name_len(self): self._check_connected() From dc8b43bf0bfdb8a7d4ef237b47acab9131bb240b Mon Sep 17 00:00:00 2001 From: victor Date: Fri, 23 May 2025 23:28:47 +0800 Subject: [PATCH 111/119] fix typo. --- examples/line_sender_cpp_example_array_elem_strides.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/line_sender_cpp_example_array_elem_strides.cpp b/examples/line_sender_cpp_example_array_elem_strides.cpp index 4161f5aa..0065d009 100644 --- a/examples/line_sender_cpp_example_array_elem_strides.cpp +++ b/examples/line_sender_cpp_example_array_elem_strides.cpp @@ -18,7 +18,7 @@ static bool array_example(std::string_view host, std::string_view port) const auto book_col = "order_book"_cn; size_t rank = 3; std::vector shape{2, 3, 2}; - std::vector strides{6, 2, 8}; + std::vector strides{6, 2, 1}; std::array arr_data = { 48123.5, 2.4, From 9ecdad99e8a3fb54080ad77b92d7abf683e8ef7c Mon Sep 17 00:00:00 2001 From: victor Date: Sat, 24 May 2025 00:21:45 +0800 Subject: [PATCH 112/119] adapt server json path format. --- questdb-rs/src/ingress/http.rs | 5 +++-- questdb-rs/src/tests/mock.rs | 13 ++++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/questdb-rs/src/ingress/http.rs b/questdb-rs/src/ingress/http.rs index 0eb85d77..40f92d2c 100644 --- a/questdb-rs/src/ingress/http.rs +++ b/questdb-rs/src/ingress/http.rs @@ -507,7 +507,7 @@ pub(super) fn read_server_settings( })?; let mut support_versions: Vec = vec![]; - if let Some(serde_json::Value::Array(ref values)) = json.get("line.proto.support.versions") + if let Some(serde_json::Value::Array(ref values)) = json.get("config").and_then(|v| v.get("line.proto.support.versions")) { for value in values.iter() { if let Some(v) = value.as_u64() { @@ -523,7 +523,8 @@ pub(super) fn read_server_settings( } let max_name_length = json - .get("cairo.max.file.name.length") + .get("config") + .and_then(|v| v.get("cairo.max.file.name.length")) .and_then(|v| v.as_u64()) .unwrap_or(MAX_NAME_LEN_DEFAULT as u64) as usize; Ok((support_versions, max_name_length)) diff --git a/questdb-rs/src/tests/mock.rs b/questdb-rs/src/tests/mock.rs index 59e98515..2dcd980f 100644 --- a/questdb-rs/src/tests/mock.rs +++ b/questdb-rs/src/tests/mock.rs @@ -314,9 +314,16 @@ impl MockServer { if supported_versions.is_empty() { self.settings_response = serde_json::json!({"version": "8.1.2"}); } else { - self.settings_response = serde_json::json!({ - "line.proto.support.versions": supported_versions - }); + self.settings_response = serde_json::json!( + {"config":{"release.type":"OSS","release.version":"[DEVELOPMENT]", + "line.proto.support.versions":supported_versions, + "ilp.proto.transports":["tcp", "http"], + "posthog.enabled":false, + "posthog.api.key":null, + "cairo.max.file.name.length":127}, + "preferences.version":0, + "preferences":{}} + ); } self } From 3d9ab0e93211ed584c607982ef77d53739df0851 Mon Sep 17 00:00:00 2001 From: victor Date: Sat, 24 May 2025 08:33:41 +0800 Subject: [PATCH 113/119] code format --- questdb-rs-ffi/src/ndarr.rs | 2 +- questdb-rs/src/ingress/http.rs | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/questdb-rs-ffi/src/ndarr.rs b/questdb-rs-ffi/src/ndarr.rs index cbc1b452..c885dece 100644 --- a/questdb-rs-ffi/src/ndarr.rs +++ b/questdb-rs-ffi/src/ndarr.rs @@ -717,7 +717,7 @@ mod tests { } #[test] - fn test_stride_array_view_strides_with_elem_size() -> TestResult { + fn test_stride_array_view_strides_with_elem_strides() -> TestResult { // contiguous layout let test_data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; let shape = [2usize, 3]; diff --git a/questdb-rs/src/ingress/http.rs b/questdb-rs/src/ingress/http.rs index 40f92d2c..68d5e4a6 100644 --- a/questdb-rs/src/ingress/http.rs +++ b/questdb-rs/src/ingress/http.rs @@ -507,7 +507,9 @@ pub(super) fn read_server_settings( })?; let mut support_versions: Vec = vec![]; - if let Some(serde_json::Value::Array(ref values)) = json.get("config").and_then(|v| v.get("line.proto.support.versions")) + if let Some(serde_json::Value::Array(ref values)) = json + .get("config") + .and_then(|v| v.get("line.proto.support.versions")) { for value in values.iter() { if let Some(v) = value.as_u64() { From 0c70977c43aeacc8f25192bdfc185d5c411e6002 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Sun, 25 May 2025 23:18:32 +0300 Subject: [PATCH 114/119] Updated readmes --- README.md | 15 +++++++++++++++ questdb-rs/README.md | 20 +++++++++++++++++--- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 122612d1..22c5f6a1 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,21 @@ For an overview and code examples, see the To understand the protocol in more depth, consult the [protocol reference docs](https://questdb.io/docs/reference/api/ilp/overview/). +## Protocol Versions + +The library supports the following ILP protocol versions. + +These protocol versions are supported over both HTTP and TCP. + +If you use HTTP, the library will automatically detect the server's +latest supported protocol version and use it. If you use TCP, you can specify the +`protocol_version=N` parameter when constructing the `Sender` object. + +| Version | Description | Server Comatibility | +| ------- | ------------------------------------------------------- | --------------------- | +| **1** | Over HTTP it's compatible InfluxDB Line Protocol (ILP) | All QuestDB versions | +| **2** | 64-bit floats sent as binary, adds n-dimentional arrays | 8.4.0+ (2023-10-30) | + ## Getting Started To get started, read the language-specific guides. diff --git a/questdb-rs/README.md b/questdb-rs/README.md index 19a1cc70..565f2c46 100644 --- a/questdb-rs/README.md +++ b/questdb-rs/README.md @@ -9,9 +9,20 @@ Ingestion Line Protocol (ILP) over either HTTP (recommended) or TCP. * [QuestDB Database docs](https://questdb.io/docs/) * [Docs on Ingestion Line Protocol](https://questdb.io/docs/reference/api/ilp/overview/) -When connecting to QuestDB over HTTP, the library will auto-detect the server's -latest supported version and use it. Version 1 is compatible with -the [InfluxDB Line Protocol](https://docs.influxdata.com/influxdb/v2/reference/syntax/line-protocol/). +## Protocol Versions + +The library supports the following ILP protocol versions. + +These protocol versions are supported over both HTTP and TCP. + +If you use HTTP, the library will automatically detect the server's +latest supported protocol version and use it. If you use TCP, you can specify the +`protocol_version=N` parameter when constructing the `Sender` object. + +| Version | Description | Server Comatibility | +| ------- | ------------------------------------------------------- | --------------------- | +| **1** | Over HTTP it's compatible InfluxDB Line Protocol (ILP) | All QuestDB versions | +| **2** | 64-bit floats sent as binary, adds n-dimentional arrays | 8.4.0+ (2023-10-30) | ## Quick Start @@ -78,6 +89,9 @@ These features are opt-in: certificates store. * `insecure-skip-verify`: Allows skipping server certificate validation in TLS (this compromises security). +* `ndarray`: Enables integration with the `ndarray` crate for working with + n-dimensional arrays. Without this feature, you can still send slices, + or integrate custom array types via the `NdArrayView` trait. ## C, C++ and Python APIs From 76296ec883da5e465f180550bbba10ba82ebe1db Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 27 May 2025 14:46:35 +0800 Subject: [PATCH 115/119] add non-contiguous minus strides array. --- questdb-rs-ffi/src/ndarr.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/questdb-rs-ffi/src/ndarr.rs b/questdb-rs-ffi/src/ndarr.rs index c885dece..028d1fea 100644 --- a/questdb-rs-ffi/src/ndarr.rs +++ b/questdb-rs-ffi/src/ndarr.rs @@ -743,4 +743,27 @@ mod tests { assert_eq!(buf, expected); Ok(()) } + + #[test] + fn test_stride_minus_non_contiguous_strides_array() -> TestResult { + let test_data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0]; + let shape = [2usize, 2]; + let strides = [-8, -2]; + let array = unsafe { + StrideArrayView::() as isize }>::new( + shape.len(), + shape.as_ptr(), + strides.as_ptr(), + test_data.as_ptr().add(11) as *const u8, + 4 * size_of::(), + ) + }?; + + let test_data1 = vec![12.0f64, 10.0, 4.0, 2.0]; + let mut buf = vec![0u8; 32]; + write_array_data(&array, &mut buf, 32).unwrap(); + let expected = to_bytes(&test_data1); + assert_eq!(buf, expected); + Ok(()) + } } From 34496b69182c59700c5729afe0f9046187a4e4e4 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Mon, 2 Jun 2025 14:20:22 +0100 Subject: [PATCH 116/119] Reverting back to naming ILP as InfluxDB line protocol, but clarifying compatibility --- README.md | 4 ++-- include/questdb/ingress/line_sender.h | 17 +++++++++-------- include/questdb/ingress/line_sender.hpp | 14 +++++++------- questdb-rs-ffi/src/lib.rs | 17 +++++++++-------- questdb-rs/README.md | 4 ++-- questdb-rs/src/ingress/mod.md | 8 ++++---- questdb-rs/src/ingress/mod.rs | 7 ++++--- 7 files changed, 37 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 22c5f6a1..d649e63d 100644 --- a/README.md +++ b/README.md @@ -3,12 +3,12 @@ This library makes it easy to insert data into [QuestDB](https://questdb.io/). -This client library implements the QuestDB [Ingestion Line Protocol]( +This client library implements the QuestDB's variant of the [InfluxDB Line Protocol]( https://questdb.io/docs/reference/api/ilp/overview/) (ILP) over HTTP and TCP. When connecting to QuestDB over HTTP, the library will auto-detect the server's latest supported version and use it. Version 1 is compatible with -the [InfluxDB Line Protocol](https://docs.influxdata.com/influxdb/v2/reference/syntax/line-protocol/). +the [InfluxDB Database](https://docs.influxdata.com/influxdb/v2/reference/syntax/line-protocol/). * Implementation is in Rust, with no additional [run-time or link-time dependencies](doc/BUILD.md#pre-requisites-and-dependencies) diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 16265832..3efb424b 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -95,16 +95,16 @@ typedef enum line_sender_error_code /** The protocol used to connect with. */ typedef enum line_sender_protocol { - /** Ingestion Line Protocol over TCP. */ + /** InfluxDB Line Protocol over TCP. */ line_sender_protocol_tcp, - /** Ingestion Line Protocol over TCP with TLS. */ + /** InfluxDB Line Protocol over TCP with TLS. */ line_sender_protocol_tcps, - /** Ingestion Line Protocol over HTTP. */ + /** InfluxDB Line Protocol over HTTP. */ line_sender_protocol_http, - /** Ingestion Line Protocol over HTTP with TLS. */ + /** InfluxDB Line Protocol over HTTP with TLS. */ line_sender_protocol_https, } line_sender_protocol; @@ -112,15 +112,16 @@ typedef enum line_sender_protocol typedef enum line_sender_protocol_version { /** - * Version 1 of Ingestion Line Protocol. - * This version is compatible with InfluxDB line protocol. + * Version 1 of InfluxDB Line Protocol. + * This version is compatible with the InfluxDB database. */ line_sender_protocol_version_1 = 1, /** - * Version 2 of Ingestion Line Protocol. + * Version 2 of InfluxDB Line Protocol. * Uses a binary format serialization for f64, and supports * the array data type. + * This version is specific to QuestDB and not compatible with InfluxDB. */ line_sender_protocol_version_2 = 2, } line_sender_protocol_version; @@ -667,7 +668,7 @@ bool line_sender_buffer_check_can_flush( /////////// Connecting, sending and disconnecting. /** - * Inserts data into QuestDB via the Ingestion Line Protocol. + * Inserts data into QuestDB via the InfluxDB Line Protocol. * * Batch up rows in a `line_sender_buffer`, then call `line_sender_flush()` * or one of its variants with this object to send them. diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index dc4e0b00..91fc33d6 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -87,25 +87,25 @@ enum class line_sender_error_code /** The protocol used to connect with. */ enum class protocol { - /** Ingestion Line Protocol over TCP. */ + /** InfluxDB Line Protocol over TCP. */ tcp, - /** Ingestion Line Protocol over TCP with TLS. */ + /** InfluxDB Line Protocol over TCP with TLS. */ tcps, - /** Ingestion Line Protocol over HTTP. */ + /** InfluxDB Line Protocol over HTTP. */ http, - /** Ingestion Line Protocol over HTTP with TLS. */ + /** InfluxDB Line Protocol over HTTP with TLS. */ https, }; enum class protocol_version { - /** Ingestion Line Protocol v1. */ + /** InfluxDB Line Protocol v1. */ v1 = 1, - /** Ingestion Line Protocol v2. */ + /** InfluxDB Line Protocol v2. */ v2 = 2, }; @@ -1191,7 +1191,7 @@ class opts }; /** - * Inserts data into QuestDB via the Ingestion Line Protocol. + * Inserts data into QuestDB via the InfluxDB Line Protocol. * * Batch up rows in a `line_sender_buffer` object, then call * `.flush()` or one of its variants to send. diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 033885d3..150b7412 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -195,16 +195,16 @@ impl From for line_sender_error_code { #[repr(C)] #[derive(Debug, Copy, Clone)] pub enum line_sender_protocol { - /// Ingestion Line Protocol over TCP. + /// InfluxDB Line Protocol over TCP. line_sender_protocol_tcp, - /// Ingestion Line Protocol over TCP with TLS. + /// InfluxDB Line Protocol over TCP with TLS. line_sender_protocol_tcps, - /// Ingestion Line Protocol over HTTP. + /// InfluxDB Line Protocol over HTTP. line_sender_protocol_http, - /// Ingestion Line Protocol over HTTP with TLS. + /// InfluxDB Line Protocol over HTTP with TLS. line_sender_protocol_https, } @@ -230,17 +230,18 @@ impl From for Protocol { } } -/// The version of Ingestion Line Protocol used to communicate with the server. +/// The version of InfluxDB Line Protocol used to communicate with the server. #[repr(C)] #[derive(Debug, Copy, Clone)] pub enum ProtocolVersion { - /// Version 1 of Line Protocol. + /// Version 1 of InfluxDB Line Protocol. /// Full-text protocol. /// When used over HTTP, it is compatible with the InfluxDB line protocol. V1 = 1, - /// Version 2 of Ingestion Line Protocol. + /// Version 2 of InfluxDB Line Protocol. /// Uses binary format serialization for f64, and supports the array data type. + /// This version is specific to QuestDB and is not compatible with InfluxDB. V2 = 2, } @@ -1400,7 +1401,7 @@ pub unsafe extern "C" fn line_sender_opts_free(opts: *mut line_sender_opts) { } } -/// Inserts data into QuestDB via the Ingestion Line Protocol. +/// Inserts data into QuestDB via the InfluxDB Line Protocol. /// /// Batch up rows in a `line_sender_buffer`, then call `line_sender_flush()` or /// one of its variants with this object to send them. diff --git a/questdb-rs/README.md b/questdb-rs/README.md index 565f2c46..e0ef8c93 100644 --- a/questdb-rs/README.md +++ b/questdb-rs/README.md @@ -4,10 +4,10 @@ Official Rust client for [QuestDB](https://questdb.io/), an open-source SQL database designed to process time-series data, faster. The client library is designed for fast ingestion of data into QuestDB via the -Ingestion Line Protocol (ILP) over either HTTP (recommended) or TCP. +InfluxDB Line Protocol (ILP) over either HTTP (recommended) or TCP. * [QuestDB Database docs](https://questdb.io/docs/) -* [Docs on Ingestion Line Protocol](https://questdb.io/docs/reference/api/ilp/overview/) +* [Docs on InfluxDB Line Protocol](https://questdb.io/docs/reference/api/ilp/overview/) ## Protocol Versions diff --git a/questdb-rs/src/ingress/mod.md b/questdb-rs/src/ingress/mod.md index 464713d9..e5f947f0 100644 --- a/questdb-rs/src/ingress/mod.md +++ b/questdb-rs/src/ingress/mod.md @@ -1,7 +1,7 @@ # Fast Ingestion of Data into QuestDB -The `ingress` module implements QuestDB's -[Ingestion Line Protocol](https://questdb.io/docs/reference/api/ilp/overview/) +The `ingress` module implements QuestDB's variant of the +[InfluxDB Line Protocol](https://questdb.io/docs/reference/api/ilp/overview/) (ILP). To get started: @@ -256,7 +256,7 @@ with a high-latency network connection. ### Timestamp Column Name -The Ingestion Line Protocol (ILP) does not give a name to the designated timestamp, +The InfluxDB Line Protocol (ILP) does not give a name to the designated timestamp, so if you let this client auto-create the table, it will have the default `timestamp` name. To use a custom name, say `my_ts`, pre-create the table with the desired timestamp column name: @@ -317,7 +317,7 @@ considerations](https://github.com/questdb/c-questdb-client/blob/main/doc/CONSID document covers these topics: * Threading -* Differences between the Ingestion Line Protocol and QuestDB Data Types +* Differences between the InfluxDB Line Protocol and QuestDB Data Types * Data Quality * Client-side checks and server errors * Flushing diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 014c9570..dfecc7b6 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -70,16 +70,17 @@ pub const MAX_ARRAY_DIMS: usize = 32; pub const MAX_ARRAY_BUFFER_SIZE: usize = i32::MAX as usize; pub const MAX_ARRAY_DIM_LEN: usize = 0x0FFF_FFFF; // 1 << 28 - 1 -/// The version of Ingestion Line Protocol used to communicate with the server. +/// The version of InfluxDB Line Protocol used to communicate with the server. #[derive(Debug, Copy, Clone, PartialEq)] pub enum ProtocolVersion { /// Version 1 of Line Protocol. /// Full-text protocol. - /// When used over HTTP, it is compatible with the InfluxDB line protocol. + /// When used over HTTP, this version is compatible with the InfluxDB database. V1 = 1, - /// Version 2 of Ingestion Line Protocol. + /// Version 2 of InfluxDB Line Protocol. /// Uses binary format serialization for f64, and supports the array data type. + /// This version is specific to QuestDB and is not compatible with InfluxDB. V2 = 2, } From 53db4933e7ac8839b32dc695fc0d703a837176b3 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Mon, 2 Jun 2025 14:43:16 +0100 Subject: [PATCH 117/119] cargo clippy --- questdb-rs/src/tests/mock.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questdb-rs/src/tests/mock.rs b/questdb-rs/src/tests/mock.rs index 2dcd980f..caa407a6 100644 --- a/questdb-rs/src/tests/mock.rs +++ b/questdb-rs/src/tests/mock.rs @@ -581,7 +581,7 @@ impl MockServer { let element_type = match ArrayColumnTypeTag::try_from(accum[index]) { Ok(t) => t, Err(e) => { - return Err(io::Error::new(io::ErrorKind::Other, e)); + return Err(io::Error::other(e)); } }; let mut elems_size = element_type.size(); From fb2735ee96861494859774e2e44f70f385fa0fb6 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Mon, 2 Jun 2025 15:03:35 +0100 Subject: [PATCH 118/119] cargo fmt --- questdb-rs-ffi/src/ndarr.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/questdb-rs-ffi/src/ndarr.rs b/questdb-rs-ffi/src/ndarr.rs index 028d1fea..abf0a571 100644 --- a/questdb-rs-ffi/src/ndarr.rs +++ b/questdb-rs-ffi/src/ndarr.rs @@ -746,7 +746,9 @@ mod tests { #[test] fn test_stride_minus_non_contiguous_strides_array() -> TestResult { - let test_data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0]; + let test_data = vec![ + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, + ]; let shape = [2usize, 2]; let strides = [-8, -2]; let array = unsafe { From fd24e0258f6b86a457037013cc42459e5bb9475b Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Mon, 2 Jun 2025 15:44:23 +0100 Subject: [PATCH 119/119] another cargo clippy fix --- questdb-rs-ffi/src/ndarr.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questdb-rs-ffi/src/ndarr.rs b/questdb-rs-ffi/src/ndarr.rs index abf0a571..ed3de205 100644 --- a/questdb-rs-ffi/src/ndarr.rs +++ b/questdb-rs-ffi/src/ndarr.rs @@ -746,7 +746,7 @@ mod tests { #[test] fn test_stride_minus_non_contiguous_strides_array() -> TestResult { - let test_data = vec![ + let test_data = [ 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, ]; let shape = [2usize, 2];