From 87aa2407a5c1f0de68ab5019ad90234d90d2469e Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 1 Apr 2025 23:02:30 +0800 Subject: [PATCH 01/56] c client nd array begin. --- questdb-rs/src/ingress/mod.rs | 1 + questdb-rs/src/ingress/ndarr.rs | 109 ++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 questdb-rs/src/ingress/ndarr.rs diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index f3d33d8a..b429ef79 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -2691,3 +2691,4 @@ use http::*; #[cfg(test)] mod tests; +mod ndarr; diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs new file mode 100644 index 00000000..41f912f0 --- /dev/null +++ b/questdb-rs/src/ingress/ndarr.rs @@ -0,0 +1,109 @@ +use std::mem::ManuallyDrop; +use std::ptr::NonNull; +use std::slice; + +pub struct Shape(Vec); + +impl From> for Shape { + fn from(vec: Vec) -> Self { + Self(vec) + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct OwnedRepr +{ + ptr: NonNull, + len: usize, + capacity: usize, +} + +impl OwnedRepr { + pub fn from(v: Vec) -> Self + { + let mut v = ManuallyDrop::new(v); + let len = v.len(); + let capacity = v.capacity(); + unsafe { + let ptr = NonNull::new_unchecked(v.as_mut_ptr()); + Self { ptr, len, capacity } + } + } + + pub(crate) fn into_vec(self) -> Vec + { + ManuallyDrop::new(self).take_as_vec() + } + + pub(crate) fn as_slice(&self) -> &[A] + { + unsafe { slice::from_raw_parts(self.ptr.as_ptr(), self.len) } + } + + pub(crate) fn len(&self) -> usize + { + self.len + } + + pub(crate) fn as_ptr(&self) -> *const A + { + self.ptr.as_ptr() + } + + pub(crate) fn as_nonnull_mut(&mut self) -> NonNull + { + self.ptr + } + + fn take_as_vec(&mut self) -> Vec + { + let capacity = self.capacity; + let len = self.len; + self.len = 0; + self.capacity = 0; + unsafe { Vec::from_raw_parts(self.ptr.as_ptr(), len, capacity) } + } +} + +pub struct Array +where + T: ArrayElement, +{ + shape: Shape, + data: OwnedRepr, +} + +impl Array { + fn ndim(&self) -> usize { + self.shape.0.len() + } + + fn dim(&self, index: usize) -> Option { + if index < self.shape.0.len() { + Some(self.shape.0[index]) + } else { + None + } + } + + fn from(v: Vec) -> Self + { + Self::from_shape_vec_unchecked(vec![v.len()], v) + } + + pub fn from_shape_vec_unchecked(shape: Sh, v: Vec) -> Self + where + Sh: Into, + { + Array { + shape: shape.into(), + data: OwnedRepr::from(v), + } + } +} + +pub trait ArrayElement: Copy + 'static {} +impl ArrayElement for f64 {} + + From 6bd3223512e8453300e1f7149fd59991c3f86ec1 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 2 Apr 2025 19:26:35 +0800 Subject: [PATCH 02/56] Ndarray support for rust client. --- questdb-rs/Cargo.toml | 1 + questdb-rs/src/error.rs | 9 ++ questdb-rs/src/ingress/mod.rs | 111 ++++++++++++++++++- questdb-rs/src/ingress/ndarr.rs | 186 ++++++++++++++++++-------------- 4 files changed, 225 insertions(+), 82 deletions(-) diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index 8db1d4da..fee87082 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -35,6 +35,7 @@ ureq = { version = "=2.9", optional = true } serde_json = { version = "1.0.108", optional = true } questdb-confstr = "0.1.0" rand = { version = "0.8.5", optional = true } +ndarray = { version = "0.16.1", optional = true } no-panic = { version = "0.1", optional = true } [target.'cfg(windows)'.dependencies] diff --git a/questdb-rs/src/error.rs b/questdb-rs/src/error.rs index 6848280a..af39b78b 100644 --- a/questdb-rs/src/error.rs +++ b/questdb-rs/src/error.rs @@ -48,6 +48,15 @@ pub enum ErrorCode { /// Bad configuration. ConfigError, + + /// Array has too many dims + ArrayHasTooManyDims, + + /// Array error + ArrayViewError, + + /// Buffer outOfMemory + BufferOutOfMemory, } /// An error that occurred when using QuestDB client library. diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index b429ef79..e0c63ba8 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -39,6 +39,7 @@ use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; +use crate::ingress::ndarr::{ArrayElement, NdArrayView, MAX_DIMS}; use base64ct::{Base64, Base64UrlUnpadded, Encoding}; use ring::rand::SystemRandom; use ring::signature::{EcdsaKeyPair, ECDSA_P256_SHA256_FIXED_SIGNING}; @@ -993,6 +994,112 @@ impl Buffer { Ok(self) } + /// Record a multidimensional array value for the given column. + /// + /// Supports arrays with up to [`MAX_DIMS`] dimensions. The array elements must + /// implement [`ArrayElement`] trait which provides type-to-[`ElemDataType`] mapping. + /// + /// # Examples + /// + /// Basic usage with direct dimension specification: + /// + /// ``` + /// # #[cfg(feature = "ndarray")] + /// # { + /// # use questdb::Result; + /// # use questdb::ingress::Buffer; + /// # use ndarray::array; + /// # fn main() -> Result<()> { + /// # let mut buffer = Buffer::new(); + /// # buffer.table("x")?; + /// // Record a 2D array of f64 values + /// let array_2d = array![[1.1, 2.2], [3.3, 4.4]]; + /// buffer.column_arr("array_col", &array_2d.view())?; + /// # Ok(()) + /// # } + /// # } + /// + /// ``` + /// + /// Using [`ColumnName`] for validated column names: + /// + /// ``` + /// # #[cfg(feature = "ndarray")] + /// # { + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, ColumnName}; + /// # use ndarray::Array3; + /// # fn main() -> Result<()> { + /// # let mut buffer = Buffer::new(); + /// # buffer.table("x1")?; + /// // Record a 3D array of f64 values + /// let array_3d = Array3::from_elem((2, 3, 4), 42f64); + /// let col_name = ColumnName::new("col1")?; + /// buffer.column_arr(col_name, &array_3d.view())?; + /// # Ok(()) + /// # } + /// # } + /// ``` + /// # Errors + /// + /// Returns [`Error`] if: + /// - Array dimensions exceed [`MAX_DIMS`] + /// - Failed to get dimension sizes + /// - Memory allocation fails for array data + /// - Column name validation fails + pub fn column_arr<'a, N, T, D>(&mut self, name: N, view: &T) -> Result<&mut Self> + where + N: TryInto>, + T: NdArrayView, + D: ArrayElement, + Error: From, + { + self.write_column_key(name)?; + + // check dimension less equal than max dims + if MAX_DIMS < view.ndim() { + return Err(error::fmt!( + ArrayHasTooManyDims, + "Array dimension mismatch: expected at most {} dimensions, but got {}", + MAX_DIMS, + view.ndim() + )); + } + + // binary format flag '=' + self.output.push(b'='); + // binary format entity type + self.output.push(ARRAY_BINARY_FORMAT_TYPE); + // ndarr dims + self.output.push(view.ndim() as u8); + // ndarr shapes + self.output.push(D::elem_type().into()); + + let mut reserve_size = size_of::(); + for i in 0..view.ndim() { + let d = view.dim(i).ok_or_else(|| { + error::fmt!(ArrayViewError, "Can not get correct dimensions for dim {}", i) + })?; + + self.output + .extend_from_slice((d as i32).to_le_bytes().as_slice()); + reserve_size = reserve_size + .checked_mul(d) + .ok_or(error::fmt!(ArrayViewError, "Array total elem size overflow"))? + } + + let index = self.output.len(); + let reserve_size = reserve_size + .checked_add(index) + .ok_or(error::fmt!(ArrayViewError, "Array total elem size overflow"))?; + self.output + .try_reserve(reserve_size) + .map_err(|_| error::fmt!(BufferOutOfMemory, "Buffer out of memory"))?; + unsafe { self.output.set_len(reserve_size) } + view.write_row_major_buf(&mut self.output[index..]); + Ok(self) + } + /// Record a timestamp value for the given column. /// /// ``` @@ -2680,6 +2787,8 @@ impl Sender { } } +const ARRAY_BINARY_FORMAT_TYPE: u8 = 14; + mod conf; mod timestamp; @@ -2689,6 +2798,6 @@ mod http; #[cfg(feature = "ilp-over-http")] use http::*; +mod ndarr; #[cfg(test)] mod tests; -mod ndarr; diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 41f912f0..5713ee46 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -1,109 +1,133 @@ -use std::mem::ManuallyDrop; -use std::ptr::NonNull; -use std::slice; +pub(crate) const MAX_DIMS: usize = 32; -pub struct Shape(Vec); +pub trait NdArrayView +where + T: ArrayElement, +{ + /// Returns the number of dimensions (rank) of the array. + fn ndim(&self) -> usize; + + /// Returns the size of the specified dimension. + fn dim(&self, index: usize) -> Option; + + /// Writes array data to buffer in row-major order. + /// + /// # Important Notes + /// - Buffer must be pre-allocated with exact required size + /// - No alignment assumptions should be made about buffer start + /// - Handles both contiguous and non-contiguous memory layouts + fn write_row_major_buf(&self, buff: &mut [u8]); +} -impl From> for Shape { - fn from(vec: Vec) -> Self { - Self(vec) - } +/// Marker trait for valid array element types. +/// +/// Implemented for primitive types that can be stored in arrays. +/// Combines type information with data type classification. +pub trait ArrayElement: Copy + 'static { + /// Returns the corresponding data type classification. + /// + /// This enables runtime type identification while maintaining + /// compile-time type safety. + fn elem_type() -> ElemDataType; } +#[repr(u8)] #[derive(Debug)] -#[repr(C)] -pub struct OwnedRepr -{ - ptr: NonNull, - len: usize, - capacity: usize, +pub enum ElemDataType { + /// Uninitialized/placeholder type + Undefined = 0, + /// Boolean values (true/false) + Boolean, + /// 8-bit signed integer + Byte, + /// 16-bit signed integer + Short, + /// UTF-16 character + Char, + /// 32-bit signed integer + Int, + /// 64-bit signed integer + Long, + /// Date type (days since epoch) + Date, + /// Microsecond-precision timestamp + Timestamp, + /// 32-bit floating point + Float, + /// 64-bit floating point + Double, + /// UTF-8 string data + String, + /// Interned string symbol + Symbol, + /// 256-bit integer value + Long256, + /// Geospatial byte coordinates + GeoByte, + /// Geospatial short coordinates + GeoShort, + /// Geospatial integer coordinates + GeoInt, + /// Geospatial long coordinates + GeoLong, + /// Binary large object + Binary, + /// UUID values + Uuid, } -impl OwnedRepr { - pub fn from(v: Vec) -> Self - { - let mut v = ManuallyDrop::new(v); - let len = v.len(); - let capacity = v.capacity(); - unsafe { - let ptr = NonNull::new_unchecked(v.as_mut_ptr()); - Self { ptr, len, capacity } - } - } - - pub(crate) fn into_vec(self) -> Vec - { - ManuallyDrop::new(self).take_as_vec() - } - - pub(crate) fn as_slice(&self) -> &[A] - { - unsafe { slice::from_raw_parts(self.ptr.as_ptr(), self.len) } - } - - pub(crate) fn len(&self) -> usize - { - self.len - } - - pub(crate) fn as_ptr(&self) -> *const A - { - self.ptr.as_ptr() - } - - pub(crate) fn as_nonnull_mut(&mut self) -> NonNull - { - self.ptr +impl From for u8 { + fn from(val: ElemDataType) -> Self { + val as u8 } +} - fn take_as_vec(&mut self) -> Vec - { - let capacity = self.capacity; - let len = self.len; - self.len = 0; - self.capacity = 0; - unsafe { Vec::from_raw_parts(self.ptr.as_ptr(), len, capacity) } +impl ArrayElement for f64 { + /// Identifies f64 as Double type in QuestDB's type system. + fn elem_type() -> ElemDataType { + ElemDataType::Double } } -pub struct Array +#[cfg(feature = "ndarray")] +use ndarray::{ArrayView, Axis, Dimension}; + +#[cfg(feature = "ndarray")] +impl NdArrayView for ArrayView<'_, T, D> where T: ArrayElement, + D: Dimension, { - shape: Shape, - data: OwnedRepr, -} - -impl Array { fn ndim(&self) -> usize { - self.shape.0.len() + self.ndim() } fn dim(&self, index: usize) -> Option { - if index < self.shape.0.len() { - Some(self.shape.0[index]) + let len = self.len(); + if index < len { + Some(self.len_of(Axis(index))) } else { None } } - fn from(v: Vec) -> Self - { - Self::from_shape_vec_unchecked(vec![v.len()], v) - } + fn write_row_major_buf(&self, buf: &mut [u8]) { + let elem_size = size_of::(); + + if let Some(slice) = self.as_slice() { + let byte_len = size_of_val(slice); + let bytes = + unsafe { std::slice::from_raw_parts(slice.as_ptr() as *const u8, byte_len) }; + buf[..byte_len].copy_from_slice(bytes); + return; + } - pub fn from_shape_vec_unchecked(shape: Sh, v: Vec) -> Self - where - Sh: Into, - { - Array { - shape: shape.into(), - data: OwnedRepr::from(v), + let mut bytes_written = 0; + for &element in self.iter() { + let element_bytes = + unsafe { std::slice::from_raw_parts(&element as *const T as *const _, elem_size) }; + buf[bytes_written..bytes_written + elem_size].copy_from_slice(element_bytes); + bytes_written += elem_size; } } } - -pub trait ArrayElement: Copy + 'static {} -impl ArrayElement for f64 {} - - From b518a9988480766fbe239a8cef5b66bd5bc2e9ad Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 2 Apr 2025 19:30:30 +0800 Subject: [PATCH 03/56] optimize doc. --- questdb-rs/src/error.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/questdb-rs/src/error.rs b/questdb-rs/src/error.rs index af39b78b..1405b61a 100644 --- a/questdb-rs/src/error.rs +++ b/questdb-rs/src/error.rs @@ -49,13 +49,13 @@ pub enum ErrorCode { /// Bad configuration. ConfigError, - /// Array has too many dims + /// Array has too many dims. Currently, only arrays with a maximum [`crate::ingress::ndarr::MAX_DIMS`] dimensions are supported. ArrayHasTooManyDims, - /// Array error + /// Array view internal error. ArrayViewError, - /// Buffer outOfMemory + /// Buffer outOfMemory. BufferOutOfMemory, } From 53a430e0f4fe3c4a5551bd69f04d5272418f0583 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 2 Apr 2025 20:55:32 +0800 Subject: [PATCH 04/56] add tests. --- questdb-rs/src/ingress/mod.rs | 2 +- questdb-rs/src/ingress/ndarr.rs | 4 +- questdb-rs/src/tests/mod.rs | 3 ++ questdb-rs/src/tests/ndarr.rs | 79 +++++++++++++++++++++++++++++++++ questdb-rs/src/tests/sender.rs | 7 +++ 5 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 questdb-rs/src/tests/ndarr.rs diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index e0c63ba8..83d6365b 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -2798,6 +2798,6 @@ mod http; #[cfg(feature = "ilp-over-http")] use http::*; -mod ndarr; +pub(crate) mod ndarr; #[cfg(test)] mod tests; diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 5713ee46..222db184 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -32,7 +32,7 @@ pub trait ArrayElement: Copy + 'static { } #[repr(u8)] -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub enum ElemDataType { /// Uninitialized/placeholder type Undefined = 0, @@ -103,7 +103,7 @@ where } fn dim(&self, index: usize) -> Option { - let len = self.len(); + let len = self.ndim(); if index < len { Some(self.len_of(Axis(index))) } else { diff --git a/questdb-rs/src/tests/mod.rs b/questdb-rs/src/tests/mod.rs index 50266006..60b811a8 100644 --- a/questdb-rs/src/tests/mod.rs +++ b/questdb-rs/src/tests/mod.rs @@ -29,6 +29,9 @@ mod http; mod mock; mod sender; +#[cfg(feature = "ndarray")] +mod ndarr; + #[cfg(feature = "json_tests")] mod json_tests { include!(concat!(env!("OUT_DIR"), "/json_tests.rs")); diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs new file mode 100644 index 00000000..31448131 --- /dev/null +++ b/questdb-rs/src/tests/ndarr.rs @@ -0,0 +1,79 @@ +use crate::ingress::ndarr; +use crate::ingress::ndarr::NdArrayView; +use ndarray::{arr1, arr2, arr3, s}; + +#[test] +fn test_f64_element_type() { + assert_eq!(::elem_type(), ndarr::ElemDataType::Double); + assert_eq!(u8::from(ndarr::ElemDataType::Double), 10); +} + +#[test] +fn test_1d_contiguous_buffer() { + let array = arr1(&[1.0, 2.0, 3.0, 4.0]); + let view = array.view(); + let mut buf = vec![0u8; 4 * size_of::()]; + view.write_row_major_buf(&mut buf); + let expected: Vec = array + .iter() + .flat_map(|&x| x.to_ne_bytes().to_vec()) + .collect(); + assert_eq!(buf, expected); +} + +#[test] +fn test_2d_non_contiguous_buffer() { + let array = arr2(&[[1.0, 2.0], [3.0, 4.0]]); + let transposed = array.view().reversed_axes(); + assert!(!transposed.is_standard_layout()); + let mut buf = vec![0u8; 4 * size_of::()]; + transposed.write_row_major_buf(&mut buf); + let expected = [1.0f64, 3.0, 2.0, 4.0] + .iter() + .flat_map(|&x| x.to_ne_bytes()) + .collect::>(); + assert_eq!(buf, expected); +} + +#[test] +fn test_strided_layout() { + let array = arr2(&[ + [1.0, 2.0, 3.0, 4.0], + [5.0, 6.0, 7.0, 8.0], + [9.0, 10.0, 11.0, 12.0], + [13.0, 14.0, 15.0, 16.0] + ]); + let strided_view = array.slice(s![1..;2, 1..;2]); + assert_eq!(strided_view.dim(), (2, 2)); + let mut buf = vec![0u8; 4 * size_of::()]; + strided_view.write_row_major_buf(&mut buf); + + // expect：6.0, 8.0, 14.0, 16.0 + let expected = [6.0f64, 8.0, 14.0, 16.0] + .iter() + .flat_map(|&x| x.to_ne_bytes()) + .collect::>(); + + assert_eq!(buf, expected); +} + +#[test] +fn test_1d_dimension_info() { + let array = arr1(&[1.0, 2.0, 3.0]); + let view = array.view(); + + assert_eq!(NdArrayView::ndim(&view), 1); + assert_eq!(NdArrayView::dim(&view, 0), Some(3)); + assert_eq!(NdArrayView::dim(&view, 1), None); +} + +#[test] +fn test_complex_dimensions() { + let array = arr3(&[[[1.0], [2.0]], [[3.0], [4.0]]]); + let view = array.view(); + + assert_eq!(NdArrayView::ndim(&view), 3); + assert_eq!(NdArrayView::dim(&view, 0), Some(2)); + assert_eq!(NdArrayView::dim(&view, 1), Some(2)); + assert_eq!(NdArrayView::dim(&view, 2), Some(1)); +} diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index cc0f02fe..1fda0df6 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -35,6 +35,8 @@ use crate::tests::{ }; use core::time::Duration; +#[cfg(feature = "ndarray")] +use ndarray::arr1; use std::{io, time::SystemTime}; #[test] @@ -351,6 +353,11 @@ fn test_str_column_name_too_long() -> TestResult { column_name_too_long_test_impl!(column_str, "value") } +#[cfg(feature = "ndarray")] +fn test_arr_column_name_too_long() -> TestResult { + column_name_too_long_test_impl!(column_arr, &arr1(&[1.0, 2.0, 3.0]).view()) +} + #[test] fn test_tls_with_file_ca() -> TestResult { let mut ca_path = certs_dir(); From b1ebd1dd0b01df742e26b1926fb45267da049d38 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 2 Apr 2025 21:00:34 +0800 Subject: [PATCH 05/56] fix comments. --- questdb-rs/src/ingress/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 83d6365b..223253c7 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -1072,7 +1072,7 @@ impl Buffer { self.output.push(ARRAY_BINARY_FORMAT_TYPE); // ndarr dims self.output.push(view.ndim() as u8); - // ndarr shapes + // ndarr datatype self.output.push(D::elem_type().into()); let mut reserve_size = size_of::(); @@ -1081,6 +1081,7 @@ impl Buffer { error::fmt!(ArrayViewError, "Can not get correct dimensions for dim {}", i) })?; + // ndarr shapes self.output .extend_from_slice((d as i32).to_le_bytes().as_slice()); reserve_size = reserve_size @@ -1096,6 +1097,8 @@ impl Buffer { .try_reserve(reserve_size) .map_err(|_| error::fmt!(BufferOutOfMemory, "Buffer out of memory"))?; unsafe { self.output.set_len(reserve_size) } + + // ndarr data view.write_row_major_buf(&mut self.output[index..]); Ok(self) } From 006af982b8731ab6ab74675910a31d5713890faa Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 3 Apr 2025 09:35:38 +0800 Subject: [PATCH 06/56] fix docs. --- questdb-rs/src/ingress/mod.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 223253c7..03d4b447 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -498,6 +498,7 @@ impl BufferState { /// [`column_i64`](Buffer::column_i64), /// [`column_f64`](Buffer::column_f64), /// [`column_str`](Buffer::column_str), +/// [`column_arr`](Buffer::column_arr), /// [`column_ts`](Buffer::column_ts)). /// * Symbols must appear before columns. /// * A row must be terminated with either [`at`](Buffer::at) or @@ -516,6 +517,7 @@ impl BufferState { /// | [`column_i64`](Buffer::column_i64) | [`INTEGER`](https://questdb.io/docs/reference/api/ilp/columnset-types#integer) | /// | [`column_f64`](Buffer::column_f64) | [`FLOAT`](https://questdb.io/docs/reference/api/ilp/columnset-types#float) | /// | [`column_str`](Buffer::column_str) | [`STRING`](https://questdb.io/docs/reference/api/ilp/columnset-types#string) | +/// | [`column_arr`](Buffer::column_arr) | [`ARRAY`](https://questdb.io/docs/reference/api/ilp/columnset-types#array) | /// | [`column_ts`](Buffer::column_ts) | [`TIMESTAMP`](https://questdb.io/docs/reference/api/ilp/columnset-types#timestamp) | /// /// QuestDB supports both `STRING` and `SYMBOL` column types. @@ -1070,10 +1072,10 @@ impl Buffer { self.output.push(b'='); // binary format entity type self.output.push(ARRAY_BINARY_FORMAT_TYPE); - // ndarr dims - self.output.push(view.ndim() as u8); // ndarr datatype self.output.push(D::elem_type().into()); + // ndarr dims + self.output.push(view.ndim() as u8); let mut reserve_size = size_of::(); for i in 0..view.ndim() { From a31716afaf3dc94267350b4dbb6945a1de149590 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 3 Apr 2025 12:14:02 +0800 Subject: [PATCH 07/56] fix tests and compile error in ffi. --- questdb-rs-ffi/src/lib.rs | 12 +++++ questdb-rs/Cargo.toml | 4 +- questdb-rs/examples/basic.rs | 2 + questdb-rs/examples/http.rs | 2 + questdb-rs/src/ingress/mod.rs | 24 +++++---- questdb-rs/src/tests/mock.rs | 18 ++----- questdb-rs/src/tests/ndarr.rs | 63 ++++++++++++++++++++++-- questdb-rs/src/tests/sender.rs | 90 +++++++++++++++++++++++++++++++--- 8 files changed, 177 insertions(+), 38 deletions(-) diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 63904f66..3a5658ae 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -135,6 +135,15 @@ pub enum line_sender_error_code { /// Bad configuration. line_sender_error_config_error, + + /// Currently, only arrays with a maximum 32 dimensions are supported. + line_sender_error_array_large_dim, + + /// ArrayView internal error, such as failure to get the size of a valid dimension. + line_sender_error_array_view_internal_error, + + /// Buffer Out Of Memory + line_sender_error_buffer_out_of_memory, } impl From for line_sender_error_code { @@ -159,6 +168,9 @@ impl From for line_sender_error_code { line_sender_error_code::line_sender_error_server_flush_error } ErrorCode::ConfigError => line_sender_error_code::line_sender_error_config_error, + ErrorCode::ArrayHasTooManyDims => line_sender_error_code::line_sender_error_array_large_dim, + ErrorCode::ArrayViewError => line_sender_error_code::line_sender_error_array_view_internal_error, + ErrorCode::BufferOutOfMemory => line_sender_error_code::line_sender_error_buffer_out_of_memory, } } } diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index fee87082..53d2e90b 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -75,7 +75,7 @@ chrono_timestamp = ["chrono"] [[example]] name = "basic" -required-features = ["chrono_timestamp"] +required-features = ["chrono_timestamp", "ndarray"] [[example]] name = "auth" @@ -87,4 +87,4 @@ required-features = ["chrono_timestamp"] [[example]] name = "http" -required-features = ["ilp-over-http"] +required-features = ["ilp-over-http", "ndarray"] diff --git a/questdb-rs/examples/basic.rs b/questdb-rs/examples/basic.rs index 6fa665d3..5df32873 100644 --- a/questdb-rs/examples/basic.rs +++ b/questdb-rs/examples/basic.rs @@ -1,4 +1,5 @@ use chrono::{TimeZone, Utc}; +use ndarray::arr1; use questdb::{ ingress::{Buffer, Sender, TimestampNanos}, Result, @@ -17,6 +18,7 @@ fn main() -> Result<()> { .symbol("side", "sell")? .column_f64("price", 2615.54)? .column_f64("amount", 0.00044)? + .column_arr("location", &arr1(&[100.0, 100.1, 100.2]).view())? .at(designated_timestamp)?; //// If you want to pass the current system timestamp, replace with: diff --git a/questdb-rs/examples/http.rs b/questdb-rs/examples/http.rs index 74b2f3e9..3e11dd0f 100644 --- a/questdb-rs/examples/http.rs +++ b/questdb-rs/examples/http.rs @@ -1,3 +1,4 @@ +use ndarray::arr1; use questdb::{ ingress::{Buffer, Sender, TimestampNanos}, Result, @@ -12,6 +13,7 @@ fn main() -> Result<()> { .symbol("side", "sell")? .column_f64("price", 2615.54)? .column_f64("amount", 0.00044)? + .column_arr("location", &arr1(&[100.0, 100.1, 100.2]).view())? .at(TimestampNanos::now())?; sender.flush(&mut buffer)?; Ok(()) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 03d4b447..79df1f6d 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -1080,26 +1080,32 @@ impl Buffer { let mut reserve_size = size_of::(); for i in 0..view.ndim() { let d = view.dim(i).ok_or_else(|| { - error::fmt!(ArrayViewError, "Can not get correct dimensions for dim {}", i) + error::fmt!( + ArrayViewError, + "Can not get correct dimensions for dim {}", + i + ) })?; // ndarr shapes self.output .extend_from_slice((d as i32).to_le_bytes().as_slice()); - reserve_size = reserve_size - .checked_mul(d) - .ok_or(error::fmt!(ArrayViewError, "Array total elem size overflow"))? + reserve_size = reserve_size.checked_mul(d).ok_or(error::fmt!( + ArrayViewError, + "Array total elem size overflow" + ))? } let index = self.output.len(); - let reserve_size = reserve_size - .checked_add(index) - .ok_or(error::fmt!(ArrayViewError, "Array total elem size overflow"))?; + let reserve_size = reserve_size.checked_add(index).ok_or(error::fmt!( + ArrayViewError, + "Array total elem size overflow" + ))?; self.output .try_reserve(reserve_size) .map_err(|_| error::fmt!(BufferOutOfMemory, "Buffer out of memory"))?; unsafe { self.output.set_len(reserve_size) } - + // ndarr data view.write_row_major_buf(&mut self.output[index..]); Ok(self) @@ -2792,7 +2798,7 @@ impl Sender { } } -const ARRAY_BINARY_FORMAT_TYPE: u8 = 14; +pub(crate) const ARRAY_BINARY_FORMAT_TYPE: u8 = 14; mod conf; mod timestamp; diff --git a/questdb-rs/src/tests/mock.rs b/questdb-rs/src/tests/mock.rs index d673a2d1..7f207be5 100644 --- a/questdb-rs/src/tests/mock.rs +++ b/questdb-rs/src/tests/mock.rs @@ -50,7 +50,7 @@ pub struct MockServer { tls_conn: Option, pub host: &'static str, pub port: u16, - pub msgs: Vec, + pub msgs: Vec, } pub fn certs_dir() -> std::path::PathBuf { @@ -519,20 +519,8 @@ impl MockServer { } } - let mut received_count = 0usize; - let mut head = 0usize; - for index in 1..accum.len() { - let last = accum[index]; - let prev = accum[index - 1]; - if (last == b'\n') && (prev != b'\\') { - let tail = index + 1; - let msg = std::str::from_utf8(&accum[head..tail]).unwrap(); - self.msgs.push(msg.to_owned()); - head = tail; - received_count += 1; - } - } - Ok(received_count) + self.msgs = accum; + Ok(self.msgs.len()) } pub fn recv_q(&mut self) -> io::Result { diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index 31448131..f7dab239 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -1,10 +1,16 @@ -use crate::ingress::ndarr; -use crate::ingress::ndarr::NdArrayView; -use ndarray::{arr1, arr2, arr3, s}; +use crate::ingress::ndarr::{ElemDataType, NdArrayView, MAX_DIMS}; +use crate::ingress::{ndarr, Buffer}; +use crate::tests::TestResult; +use crate::{ingress, ErrorCode}; +use ndarray::{arr1, arr2, arr3, s, ArrayD}; +use std::iter; #[test] fn test_f64_element_type() { - assert_eq!(::elem_type(), ndarr::ElemDataType::Double); + assert_eq!( + ::elem_type(), + ndarr::ElemDataType::Double + ); assert_eq!(u8::from(ndarr::ElemDataType::Double), 10); } @@ -41,7 +47,7 @@ fn test_strided_layout() { [1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], [9.0, 10.0, 11.0, 12.0], - [13.0, 14.0, 15.0, 16.0] + [13.0, 14.0, 15.0, 16.0], ]); let strided_view = array.slice(s![1..;2, 1..;2]); assert_eq!(strided_view.dim(), (2, 2)); @@ -77,3 +83,50 @@ fn test_complex_dimensions() { assert_eq!(NdArrayView::dim(&view, 1), Some(2)); assert_eq!(NdArrayView::dim(&view, 2), Some(1)); } + +#[test] +fn test_buffer_array_write() -> TestResult { + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + let array_2d = arr2(&[[1.1, 2.2], [3.3, 4.4]]); + buffer.column_arr("temperature", &array_2d.view())?; + + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); + assert_eq!( + &data[19..24], + &[ + b'=', + b'=', + ingress::ARRAY_BINARY_FORMAT_TYPE, + ElemDataType::Double.into(), + 2u8 + ] + ); + assert_eq!( + &data[24..32], + [2i32.to_le_bytes().as_slice(), 2i32.to_le_bytes().as_slice()].concat() + ); + Ok(()) +} + +#[test] +fn test_buffer_write_max_dimensions() -> TestResult { + let mut buffer = Buffer::new(); + buffer.table("nd_test")?; + let shape: Vec = iter::repeat(1).take(MAX_DIMS).collect(); + let array = ArrayD::::zeros(shape.clone()); + buffer.column_arr("max_dim", &array.view())?; + let data = buffer.as_bytes(); + assert_eq!(data[19], MAX_DIMS as u8); + + // 33 dims error + let shape_invalid: Vec<_> = iter::repeat(1).take(MAX_DIMS + 1).collect(); + let array_invalid = ArrayD::::zeros(shape_invalid); + let result = buffer.column_arr("invalid", &array_invalid.view()); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayHasTooManyDims); + Ok(()) +} diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index 1fda0df6..b69e3a0b 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -23,6 +23,7 @@ ******************************************************************************/ use crate::{ + ingress, ingress::{ Buffer, CertificateAuthority, Sender, TableName, Timestamp, TimestampMicros, TimestampNanos, }, @@ -34,9 +35,12 @@ use crate::tests::{ TestResult, }; +#[cfg(feature = "ndarray")] +use crate::ingress::ndarr::{ElemDataType, NdArrayView}; use core::time::Duration; #[cfg(feature = "ndarray")] -use ndarray::arr1; +use ndarray::{arr1, arr2, ArrayD}; + use std::{io, time::SystemTime}; #[test] @@ -79,8 +83,79 @@ fn test_basics() -> TestResult { sender.flush(&mut buffer)?; assert_eq!(buffer.len(), 0); assert_eq!(buffer.as_bytes(), b""); - assert_eq!(server.recv_q()?, 1); - assert_eq!(server.msgs[0].as_bytes(), exp_byte); + assert_eq!(server.recv_q()?, exp_byte.len()); + assert_eq!(server.msgs, exp_byte); + Ok(()) +} + +#[cfg(feature = "ndarray")] +#[test] +fn test_array_basic() -> TestResult { + let mut server = MockServer::new()?; + let mut sender = server.lsb_tcp().build()?; + server.accept()?; + + let ts = SystemTime::now(); + let ts_nanos_num = ts.duration_since(SystemTime::UNIX_EPOCH)?.as_nanos() as i64; + let ts_nanos = TimestampNanos::from_systemtime(ts)?; + let array_2d = arr2(&[[1.1, 2.2], [3.3, 4.4]]); + let array_3d = ArrayD::::ones(vec![2, 3, 4]); + + let mut buffer = Buffer::new(); + buffer + .table("my_table")? + .symbol("device", "A001")? + .column_f64("f1", 25.5)? + .column_arr("arr2d", &array_2d.view())? + .column_arr("arr3d", &array_3d.view())? + .at(ts_nanos)?; + + assert_eq!(server.recv_q()?, 0); + + let array_header2d = &[ + &[b'='][..], + &[ingress::ARRAY_BINARY_FORMAT_TYPE], + &[ElemDataType::Double as u8], + &[2u8], + &2i32.to_le_bytes(), + &2i32.to_le_bytes(), + ] + .concat(); + let mut array_data2d = vec![0u8; 4 * size_of::()]; + array_2d.view().write_row_major_buf(&mut array_data2d); + + let array_header3d = &[ + &[b'='][..], + &[ingress::ARRAY_BINARY_FORMAT_TYPE], + &[ElemDataType::Double as u8], + &[3u8], + &2i32.to_le_bytes(), + &3i32.to_le_bytes(), + &4i32.to_le_bytes(), + ] + .concat(); + let mut array_data3d = vec![0u8; 24 * size_of::()]; + array_3d.view().write_row_major_buf(&mut array_data3d); + + let exp = &[ + "my_table,device=A001 f1=25.5".as_bytes(), + ",arr2d=".as_bytes(), + array_header2d, + array_data2d.as_slice(), + ",arr3d=".as_bytes(), + array_header3d, + array_data3d.as_slice(), + format!(" {}\n", ts_nanos_num).as_bytes(), + ] + .concat(); + + assert_eq!(buffer.as_bytes(), exp); + assert_eq!(buffer.len(), exp.len()); + sender.flush(&mut buffer)?; + assert_eq!(buffer.len(), 0); + assert_eq!(buffer.as_bytes(), b""); + assert_eq!(server.recv_q()?, exp.len()); + assert_eq!(server.msgs.as_slice(), exp); Ok(()) } @@ -354,6 +429,7 @@ fn test_str_column_name_too_long() -> TestResult { } #[cfg(feature = "ndarray")] +#[test] fn test_arr_column_name_too_long() -> TestResult { column_name_too_long_test_impl!(column_arr, &arr1(&[1.0, 2.0, 3.0]).view()) } @@ -381,8 +457,8 @@ fn test_tls_with_file_ca() -> TestResult { assert_eq!(buffer.as_bytes(), exp); assert_eq!(buffer.len(), exp.len()); sender.flush(&mut buffer)?; - assert_eq!(server.recv_q()?, 1); - assert_eq!(server.msgs[0].as_bytes(), exp); + assert_eq!(server.recv_q()?, exp.len()); + assert_eq!(server.msgs.as_slice(), exp); Ok(()) } @@ -474,8 +550,8 @@ fn test_tls_insecure_skip_verify() -> TestResult { assert_eq!(buffer.as_bytes(), exp); assert_eq!(buffer.len(), exp.len()); sender.flush(&mut buffer)?; - assert_eq!(server.recv_q()?, 1); - assert_eq!(server.msgs[0].as_bytes(), exp); + assert_eq!(server.recv_q()?, exp.len()); + assert_eq!(server.msgs.as_slice(), exp); Ok(()) } From 463a2d3a1752f824d15ed60811e9f7715b9e0975 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 3 Apr 2025 14:21:57 +0800 Subject: [PATCH 08/56] format code. --- questdb-rs-ffi/src/lib.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 3a5658ae..73f1928e 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -168,9 +168,15 @@ impl From for line_sender_error_code { line_sender_error_code::line_sender_error_server_flush_error } ErrorCode::ConfigError => line_sender_error_code::line_sender_error_config_error, - ErrorCode::ArrayHasTooManyDims => line_sender_error_code::line_sender_error_array_large_dim, - ErrorCode::ArrayViewError => line_sender_error_code::line_sender_error_array_view_internal_error, - ErrorCode::BufferOutOfMemory => line_sender_error_code::line_sender_error_buffer_out_of_memory, + ErrorCode::ArrayHasTooManyDims => { + line_sender_error_code::line_sender_error_array_large_dim + } + ErrorCode::ArrayViewError => { + line_sender_error_code::line_sender_error_array_view_internal_error + } + ErrorCode::BufferOutOfMemory => { + line_sender_error_code::line_sender_error_buffer_out_of_memory + } } } } From 44fae08a1ae82d7afdc51c08a840cb567b15d118 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 3 Apr 2025 16:01:12 +0800 Subject: [PATCH 09/56] ffi supported --- questdb-rs-ffi/src/lib.rs | 93 +++++++++++++++++++++++++++++++++-- questdb-rs/src/ingress/mod.rs | 5 +- 2 files changed, 92 insertions(+), 6 deletions(-) diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 73f1928e..20e30a17 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -24,7 +24,7 @@ #![allow(non_camel_case_types, clippy::missing_safety_doc)] -use libc::{c_char, size_t}; +use libc::{c_char, c_int, size_t}; use std::ascii; use std::boxed::Box; use std::convert::{From, Into}; @@ -35,8 +35,8 @@ use std::str; use questdb::{ ingress::{ - Buffer, CertificateAuthority, ColumnName, Protocol, Sender, SenderBuilder, TableName, - TimestampMicros, TimestampNanos, + ArrayElement, Buffer, CertificateAuthority, ColumnName, NdArrayView, Protocol, Sender, + SenderBuilder, TableName, TimestampMicros, TimestampNanos, }, Error, ErrorCode, }; @@ -309,6 +309,56 @@ impl line_sender_utf8 { } } +#[derive(Debug, Copy, Clone)] +struct line_sender_array { + dims: size_t, + shapes: *const i32, + buf_len: size_t, + buf: *const u8, +} + +impl NdArrayView for line_sender_array +where + T: ArrayElement, +{ + fn ndim(&self) -> usize { + self.dims + } + + fn dim(&self, index: usize) -> Option { + if index >= self.dims { + return None; + } + + unsafe { + if self.shapes.is_null() { + return None; + } + + let dim_size = *self.shapes.add(index); + if dim_size < 0 { + None + } else { + Some(dim_size as usize) + } + } + } + + fn write_row_major_buf(&self, buff: &mut [u8]) { + let elem_size = size_of::(); + let total_elements: usize = (0..self.dims) + .filter_map(|i| >::dim(self, i)) + .product(); + assert_eq!( + self.buf_len, + total_elements * elem_size, + "Buffer length mismatch" + ); + let bytes = unsafe { slice::from_raw_parts(self.buf, self.buf_len) }; + buff[..self.buf_len].copy_from_slice(bytes); + } +} + /// An ASCII-safe description of a binary buffer. Trimmed if too long. fn describe_buf(buf: &[u8]) -> String { let max_len = 100usize; @@ -822,6 +872,43 @@ pub unsafe extern "C" fn line_sender_buffer_column_str( true } +/// Record a float multidimensional array value for the given column. +/// @param[in] buffer Line buffer object. +/// @param[in] name Column name. +/// @param[in] rank Array dims. +/// @param[in] shape Array shapes. +/// @param[in] data_buffer Array data memory ptr. +/// @param[in] data_buffer_len Array data memory length. +/// @param[out] err_out Set on error. +/// # Safety +/// - All pointer parameters must be valid and non-null +/// - shape must point to an array of `rank` integers +/// - data_buffer must point to a buffer of size `data_buffer_len` bytes +#[no_mangle] +pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( + buffer: *mut line_sender_buffer, + name: line_sender_column_name, + rank: size_t, + shape: *const c_int, // C array of dimension sizes + data_buffer: *const u8, // Raw array data + data_buffer_len: size_t, // Total bytes length + err_out: *mut *mut line_sender_error, +) -> bool { + let buffer = unwrap_buffer_mut(buffer); + let name = name.as_name(); + let view = line_sender_array { + dims: rank, + shapes: shape, + buf_len: data_buffer_len, + buf: data_buffer, + }; + bubble_err_to_c!( + err_out, + buffer.column_arr::, line_sender_array, f64>(name, &view) + ); + true +} + /// Record a nanosecond timestamp value for the given column. /// @param[in] buffer Line buffer object. /// @param[in] name Column name. diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 79df1f6d..52185ff9 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -24,8 +24,8 @@ #![doc = include_str!("mod.md")] +pub use self::ndarr::*; pub use self::timestamp::*; - use crate::error::{self, Error, Result}; use crate::gai; use crate::ingress::conf::ConfigSetting; @@ -39,7 +39,6 @@ use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; -use crate::ingress::ndarr::{ArrayElement, NdArrayView, MAX_DIMS}; use base64ct::{Base64, Base64UrlUnpadded, Encoding}; use ring::rand::SystemRandom; use ring::signature::{EcdsaKeyPair, ECDSA_P256_SHA256_FIXED_SIGNING}; @@ -2802,6 +2801,7 @@ pub(crate) const ARRAY_BINARY_FORMAT_TYPE: u8 = 14; mod conf; mod timestamp; +pub(crate) mod ndarr; #[cfg(feature = "ilp-over-http")] mod http; @@ -2809,6 +2809,5 @@ mod http; #[cfg(feature = "ilp-over-http")] use http::*; -pub(crate) mod ndarr; #[cfg(test)] mod tests; From 93c6f9decbc3514654c33bcaef1b255433fa5cff Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 3 Apr 2025 16:57:30 +0800 Subject: [PATCH 10/56] ndarray c api --- include/questdb/ingress/line_sender.h | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index cbfa225b..2bd1439b 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -461,6 +461,32 @@ bool line_sender_buffer_column_str( line_sender_utf8 value, line_sender_error** err_out); +/** + * Record a multidimensional array of double for the given column. + * The array data must be stored in row-major order (C-style contiguous layout). + * + * @param[in] buffer Line buffer object. + * @param[in] name Column name. + * @param[in] rank Number of dimensions of the array. + * @param[in] shapes Array of dimension sizes (length = `rank`). + * Each element must be a positive integer. + * @param[in] data_buffer Raw bytes of the array data in little-endian format. + * Size must be `sizeof(double) * (shapes[0] * ... * + * shapes[rank-1])`. + * @param[in] data_buffer_len Byte length of the data buffer. + * @param[out] err_out Set to an error object on failure (if non-NULL). + * @return true on success, false on error. + */ +LINESENDER_API +bool line_sender_buffer_column_f64_arr( + line_sender_buffer* buffer, + line_sender_column_name name, + size_t rank, + const int32_t* shapes, + const uint8_t* data_buffer, + size_t data_buffer_len, + line_sender_error** err_out); + /** * Record a nanosecond timestamp value for the given column. * @param[in] buffer Line buffer object. From e9da55875bbdd3406ad5febe73523a1dc8893934 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 3 Apr 2025 17:05:10 +0800 Subject: [PATCH 11/56] format --- questdb-rs/src/ingress/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 52185ff9..6bdc9fc2 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -2800,8 +2800,8 @@ impl Sender { pub(crate) const ARRAY_BINARY_FORMAT_TYPE: u8 = 14; mod conf; -mod timestamp; pub(crate) mod ndarr; +mod timestamp; #[cfg(feature = "ilp-over-http")] mod http; From 0e8a9e5f7f1ff8ff4157cd8a98a6e902dbce17f1 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 3 Apr 2025 21:36:58 +0800 Subject: [PATCH 12/56] c++ api support. --- include/questdb/ingress/line_sender.h | 2 +- include/questdb/ingress/line_sender.hpp | 34 +++++++++++++++++++++++++ questdb-rs-ffi/src/lib.rs | 12 +++------ 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 2bd1439b..0bd00ec8 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -482,7 +482,7 @@ bool line_sender_buffer_column_f64_arr( line_sender_buffer* buffer, line_sender_column_name name, size_t rank, - const int32_t* shapes, + const uint32_t* shapes, const uint8_t* data_buffer, size_t data_buffer_len, line_sender_error** err_out); diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 19ee2e15..5c2063e6 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -26,6 +26,7 @@ #include "line_sender.h" +#include #include #include #include @@ -624,6 +625,39 @@ class line_sender_buffer return *this; } + /** + * Record a multidimensional double-precision array for the given column. + * + * @param name Column name. + * @param shape Array dimensions (e.g., [2,3] for a 2x3 matrix). + * @param data Array data in row-major order. Size must match product of + * dimensions. + * + * @note Data is stored contiguously in row-major (C-style) order. + * Example: shape [2,3] expects 6 elements ordered as: + * [a11, a12, a13, a21, a22, a23] + */ + template + line_sender_buffer& column( + column_name_view name, + const std::vector& shape, + const std::array& data) + { + static_assert( + std::is_same_v, + "Only double types are supported for arrays"); + may_init(); + line_sender_error::wrapped_call( + ::line_sender_buffer_column_f64_arr, + _impl, + name._impl, + shape.size(), + shape.data(), + reinterpret_cast(data.data()), + sizeof(double) * N); + return *this; + } + /** * Record a string value for the given column. * @param name Column name. diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 20e30a17..1378405d 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -24,7 +24,7 @@ #![allow(non_camel_case_types, clippy::missing_safety_doc)] -use libc::{c_char, c_int, size_t}; +use libc::{c_char, c_uint, size_t}; use std::ascii; use std::boxed::Box; use std::convert::{From, Into}; @@ -312,7 +312,7 @@ impl line_sender_utf8 { #[derive(Debug, Copy, Clone)] struct line_sender_array { dims: size_t, - shapes: *const i32, + shapes: *const u32, buf_len: size_t, buf: *const u8, } @@ -336,11 +336,7 @@ where } let dim_size = *self.shapes.add(index); - if dim_size < 0 { - None - } else { - Some(dim_size as usize) - } + Some(dim_size as usize) } } @@ -889,7 +885,7 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( buffer: *mut line_sender_buffer, name: line_sender_column_name, rank: size_t, - shape: *const c_int, // C array of dimension sizes + shape: *const c_uint, // C array of dimension sizes data_buffer: *const u8, // Raw array data data_buffer_len: size_t, // Total bytes length err_out: *mut *mut line_sender_error, From 261163cb05dd37fbab1561f351ddc5e59dc483cc Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 3 Apr 2025 23:50:34 +0800 Subject: [PATCH 13/56] add include. --- include/questdb/ingress/line_sender.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 5c2063e6..19945589 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -34,6 +34,7 @@ #include #include #include +#include #if __cplusplus >= 202002L # include #endif From a565109a850766f6b5e8f3a041744cf26c80174c Mon Sep 17 00:00:00 2001 From: victor Date: Fri, 4 Apr 2025 00:37:32 +0800 Subject: [PATCH 14/56] code format. --- questdb-rs-ffi/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 1378405d..3b0a2476 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -885,7 +885,7 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( buffer: *mut line_sender_buffer, name: line_sender_column_name, rank: size_t, - shape: *const c_uint, // C array of dimension sizes + shape: *const c_uint, // C array of dimension sizes data_buffer: *const u8, // Raw array data data_buffer_len: size_t, // Total bytes length err_out: *mut *mut line_sender_error, From e161b1daf146062fb4033add6081ff58c7055420 Mon Sep 17 00:00:00 2001 From: victor Date: Sun, 6 Apr 2025 23:01:23 +0800 Subject: [PATCH 15/56] conflict resolve --- questdb-rs/Cargo.toml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index 90ace7f3..6601d9b3 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -39,6 +39,7 @@ ureq = { version = "3.0.10, <3.1.0", default-features = false, features = ["rust serde_json = { version = "1", optional = true } questdb-confstr = "0.1.1" rand = { version = "0.9.0", optional = true } +ndarray = { version = "0.16", optional = true } no-panic = { version = "0.1", optional = true } [target.'cfg(windows)'.dependencies] @@ -93,12 +94,13 @@ almost-all-features = [ "aws-lc-crypto", "insecure-skip-verify", "json_tests", - "chrono_timestamp" + "chrono_timestamp", + "ndarray" ] [[example]] name = "basic" -required-features = ["chrono_timestamp"] +required-features = ["chrono_timestamp", "ndarray"] [[example]] name = "auth" @@ -110,4 +112,4 @@ required-features = ["chrono_timestamp"] [[example]] name = "http" -required-features = ["ilp-over-http"] +required-features = ["ilp-over-http", "ndarray"] From 4a59ec045e7b386b6ffb309f2d46bfcd1ee8160a Mon Sep 17 00:00:00 2001 From: victor Date: Mon, 7 Apr 2025 15:22:46 +0800 Subject: [PATCH 16/56] code review. --- questdb-rs-ffi/Cargo.lock | 81 +++++++++ questdb-rs-ffi/src/lib.rs | 22 +-- questdb-rs/Cargo.toml | 3 +- questdb-rs/src/error.rs | 6 +- questdb-rs/src/ingress/mod.rs | 41 +++-- questdb-rs/src/ingress/ndarr.rs | 80 +++------ questdb-rs/src/tests/mod.rs | 1 - questdb-rs/src/tests/ndarr.rs | 297 ++++++++++++++++++++++++++++++-- questdb-rs/src/tests/sender.rs | 8 +- 9 files changed, 428 insertions(+), 111 deletions(-) diff --git a/questdb-rs-ffi/Cargo.lock b/questdb-rs-ffi/Cargo.lock index 5666057c..248ab0c9 100644 --- a/questdb-rs-ffi/Cargo.lock +++ b/questdb-rs-ffi/Cargo.lock @@ -11,6 +11,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + [[package]] name = "aws-lc-rs" version = "1.13.0" @@ -374,6 +380,16 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +[[package]] +name = "matrixmultiply" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9380b911e3e96d10c1f415da0876389aaf1b56759054eeb0de7df940c456ba1a" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "memchr" version = "2.7.4" @@ -386,6 +402,21 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "ndarray" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + [[package]] name = "nom" version = "7.1.3" @@ -396,6 +427,33 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -414,6 +472,21 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "portable-atomic" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -467,6 +540,8 @@ dependencies = [ "indoc", "itoa", "libc", + "log", + "ndarray", "questdb-confstr", "rand", "rustls", @@ -538,6 +613,12 @@ dependencies = [ "getrandom 0.3.2", ] +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "regex" version = "1.11.1" diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index d78c2e86..3d929e6c 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -142,8 +142,8 @@ pub enum line_sender_error_code { /// ArrayView internal error, such as failure to get the size of a valid dimension. line_sender_error_array_view_internal_error, - /// Buffer Out Of Memory - line_sender_error_buffer_out_of_memory, + /// Write arrayView to sender buffer error. + line_sender_error_array_view_write_to_buffer_error } impl From for line_sender_error_code { @@ -174,8 +174,8 @@ impl From for line_sender_error_code { ErrorCode::ArrayViewError => { line_sender_error_code::line_sender_error_array_view_internal_error } - ErrorCode::BufferOutOfMemory => { - line_sender_error_code::line_sender_error_buffer_out_of_memory + ErrorCode::ArrayWriteToBufferError => { + line_sender_error_code::line_sender_error_array_view_write_to_buffer_error } } } @@ -340,18 +340,10 @@ where } } - fn write_row_major_buf(&self, buff: &mut [u8]) { - let elem_size = size_of::(); - let total_elements: usize = (0..self.dims) - .filter_map(|i| >::dim(self, i)) - .product(); - assert_eq!( - self.buf_len, - total_elements * elem_size, - "Buffer length mismatch" - ); + fn write_row_major(&self, writer: &mut W) -> std::io::Result<()> { let bytes = unsafe { slice::from_raw_parts(self.buf, self.buf_len) }; - buff[..self.buf_len].copy_from_slice(bytes); + writer.write_all(bytes)?; + Ok(()) } } diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index 6601d9b3..50649d37 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -41,6 +41,7 @@ questdb-confstr = "0.1.1" rand = { version = "0.9.0", optional = true } ndarray = { version = "0.16", optional = true } no-panic = { version = "0.1", optional = true } +log = "0.4.27" [target.'cfg(windows)'.dependencies] winapi = { version = "0.3.9", features = ["ws2def"] } @@ -58,7 +59,7 @@ tempfile = "3" webpki-roots = "0.26.8" [features] -default = ["tls-webpki-certs", "ilp-over-http", "aws-lc-crypto"] +default = ["tls-webpki-certs", "ilp-over-http", "aws-lc-crypto", "ndarray"] # Include support for ILP over HTTP. ilp-over-http = ["dep:ureq", "dep:serde_json", "dep:rand"] diff --git a/questdb-rs/src/error.rs b/questdb-rs/src/error.rs index 2958c16d..58f31e74 100644 --- a/questdb-rs/src/error.rs +++ b/questdb-rs/src/error.rs @@ -49,14 +49,14 @@ pub enum ErrorCode { /// Bad configuration. ConfigError, - /// Array has too many dims. Currently, only arrays with a maximum [`crate::ingress::ndarr::MAX_DIMS`] dimensions are supported. + /// Array has too many dims. Currently, only arrays with a maximum [`crate::ingress::MAX_DIMS`] dimensions are supported. ArrayHasTooManyDims, /// Array view internal error. ArrayViewError, - /// Buffer outOfMemory. - BufferOutOfMemory, + /// Array write to buffer error + ArrayWriteToBufferError, } /// An error that occurred when using QuestDB client library. diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 50ed4d23..cc7d98fe 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -24,7 +24,7 @@ #![doc = include_str!("mod.md")] -pub use self::ndarr::*; +pub use self::ndarr::{ArrayElement, ElemDataType, NdArrayView}; pub use self::timestamp::*; use crate::error::{self, Error, Result}; use crate::gai; @@ -37,9 +37,10 @@ use socket2::{Domain, Protocol as SockProtocol, SockAddr, Socket, Type}; use std::collections::HashMap; use std::convert::Infallible; use std::fmt::{Debug, Display, Formatter, Write}; -use std::io::{self, BufRead, BufReader, ErrorKind, Write as IoWrite}; +use std::io::{self, BufRead, BufReader, Cursor, ErrorKind, Write as IoWrite}; use std::ops::Deref; use std::path::PathBuf; +use std::slice::from_raw_parts_mut; use std::str::FromStr; use std::sync::Arc; @@ -55,6 +56,9 @@ use ring::{ signature::{EcdsaKeyPair, ECDSA_P256_SHA256_FIXED_SIGNING}, }; +/// Defines the maximum allowed dimensions for array data in binary serialization protocols. +pub const MAX_DIMS: usize = 32; + #[derive(Debug, Copy, Clone)] enum Op { Table = 1, @@ -1055,7 +1059,6 @@ impl Buffer { /// Returns [`Error`] if: /// - Array dimensions exceed [`MAX_DIMS`] /// - Failed to get dimension sizes - /// - Memory allocation fails for array data /// - Column name validation fails pub fn column_arr<'a, N, T, D>(&mut self, name: N, view: &T) -> Result<&mut Self> where @@ -1104,18 +1107,30 @@ impl Buffer { ))? } + self.output.reserve(reserve_size); let index = self.output.len(); - let reserve_size = reserve_size.checked_add(index).ok_or(error::fmt!( - ArrayViewError, - "Array total elem size overflow" - ))?; - self.output - .try_reserve(reserve_size) - .map_err(|_| error::fmt!(BufferOutOfMemory, "Buffer out of memory"))?; - unsafe { self.output.set_len(reserve_size) } - + let writeable = + unsafe { from_raw_parts_mut(self.output.as_mut_ptr().add(index), reserve_size) }; + let mut cursor = Cursor::new(writeable); // ndarr data - view.write_row_major_buf(&mut self.output[index..]); + if let Err(e) = view.write_row_major(&mut cursor) { + return Err(error::fmt!( + ArrayWriteToBufferError, + "Can not write row major to writer: {}", + e + )); + } + + if cursor.position() != (reserve_size as u64) { + return Err(error::fmt!( + ArrayWriteToBufferError, + "Array write buffer length mismatch (actual: {}, expected: {})", + cursor.position(), + reserve_size + )); + } + + unsafe { self.output.set_len(reserve_size + index) } Ok(self) } diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 222db184..2c0fd0ad 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -1,5 +1,3 @@ -pub(crate) const MAX_DIMS: usize = 32; - pub trait NdArrayView where T: ArrayElement, @@ -10,13 +8,13 @@ where /// Returns the size of the specified dimension. fn dim(&self, index: usize) -> Option; - /// Writes array data to buffer in row-major order. + /// Writes array data to writer in row-major order. /// /// # Important Notes - /// - Buffer must be pre-allocated with exact required size - /// - No alignment assumptions should be made about buffer start + /// - Writer must be pre-allocated with exact required size + /// - No alignment assumptions should be made about writer start /// - Handles both contiguous and non-contiguous memory layouts - fn write_row_major_buf(&self, buff: &mut [u8]); + fn write_row_major(&self, writer: &mut W) -> std::io::Result<()>; } /// Marker trait for valid array element types. @@ -31,49 +29,13 @@ pub trait ArrayElement: Copy + 'static { fn elem_type() -> ElemDataType; } +/// Defines binary format identifiers for array element types compatible with +/// QuestDB's [`ColumnType`]: https://github.com/questdb/questdb/blob/e1853db56ae586d923ca77de01a487cad44093b9/core/src/main/java/io/questdb/cairo/ColumnType.java#L67-L89. #[repr(u8)] -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone, Copy)] pub enum ElemDataType { - /// Uninitialized/placeholder type - Undefined = 0, - /// Boolean values (true/false) - Boolean, - /// 8-bit signed integer - Byte, - /// 16-bit signed integer - Short, - /// UTF-16 character - Char, - /// 32-bit signed integer - Int, - /// 64-bit signed integer - Long, - /// Date type (days since epoch) - Date, - /// Microsecond-precision timestamp - Timestamp, - /// 32-bit floating point - Float, /// 64-bit floating point - Double, - /// UTF-8 string data - String, - /// Interned string symbol - Symbol, - /// 256-bit integer value - Long256, - /// Geospatial byte coordinates - GeoByte, - /// Geospatial short coordinates - GeoShort, - /// Geospatial integer coordinates - GeoInt, - /// Geospatial long coordinates - GeoLong, - /// Binary large object - Binary, - /// UUID values - Uuid, + Double = 0x0A, } impl From for u8 { @@ -111,23 +73,23 @@ where } } - fn write_row_major_buf(&self, buf: &mut [u8]) { - let elem_size = size_of::(); - - if let Some(slice) = self.as_slice() { - let byte_len = size_of_val(slice); - let bytes = - unsafe { std::slice::from_raw_parts(slice.as_ptr() as *const u8, byte_len) }; - buf[..byte_len].copy_from_slice(bytes); - return; + fn write_row_major(&self, writer: &mut W) -> std::io::Result<()> { + if let Some(contiguous) = self.as_slice() { + let bytes = unsafe { + std::slice::from_raw_parts( + contiguous.as_ptr() as *const u8, + size_of_val(contiguous), + ) + }; + return writer.write_all(bytes); } - let mut bytes_written = 0; + let elem_size = size_of::(); for &element in self.iter() { - let element_bytes = + let bytes = unsafe { std::slice::from_raw_parts(&element as *const T as *const _, elem_size) }; - buf[bytes_written..bytes_written + elem_size].copy_from_slice(element_bytes); - bytes_written += elem_size; + writer.write_all(bytes)?; } + Ok(()) } } diff --git a/questdb-rs/src/tests/mod.rs b/questdb-rs/src/tests/mod.rs index f06dba8c..29c1b3f4 100644 --- a/questdb-rs/src/tests/mod.rs +++ b/questdb-rs/src/tests/mod.rs @@ -29,7 +29,6 @@ mod http; mod mock; mod sender; -#[cfg(feature = "ndarray")] mod ndarr; #[cfg(feature = "json_tests")] diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index f7dab239..886bde63 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -1,48 +1,306 @@ -use crate::ingress::ndarr::{ElemDataType, NdArrayView, MAX_DIMS}; -use crate::ingress::{ndarr, Buffer}; +use crate::ingress::{ + ArrayElement, Buffer, ElemDataType, NdArrayView, ARRAY_BINARY_FORMAT_TYPE, MAX_DIMS, +}; use crate::tests::TestResult; -use crate::{ingress, ErrorCode}; +use crate::ErrorCode; + +#[cfg(feature = "ndarray")] use ndarray::{arr1, arr2, arr3, s, ArrayD}; +#[cfg(feature = "ndarray")] use std::iter; +use std::marker::PhantomData; #[test] fn test_f64_element_type() { + assert_eq!(::elem_type(), ElemDataType::Double); + assert_eq!(u8::from(ElemDataType::Double), 10); +} + +struct Array2D { + data: Vec, + rows: usize, + cols: usize, + contiguous: bool, +} + +impl Array2D { + fn new(data: Vec, rows: usize, cols: usize, contiguous: bool) -> Self { + Self { + data, + rows, + cols, + contiguous, + } + } +} + +impl NdArrayView for Array2D { + fn ndim(&self) -> usize { + 2 + } + + fn dim(&self, index: usize) -> Option { + match index { + 0 => Some(self.rows), + 1 => Some(self.cols), + _ => None, + } + } + + fn write_row_major(&self, writer: &mut W) -> std::io::Result<()> { + if self.contiguous { + let bytes = unsafe { + std::slice::from_raw_parts( + self.data.as_ptr() as *const u8, + self.data.len() * std::mem::size_of::(), + ) + }; + return writer.write_all(bytes); + } + + for chunk in self.data.chunks(self.cols) { + let bytes = unsafe { + std::slice::from_raw_parts( + chunk.as_ptr() as *const u8, + chunk.len() * std::mem::size_of::(), + ) + }; + writer.write_all(bytes)?; + } + Ok(()) + } +} + +fn to_bytes(data: &[T]) -> Vec { + data.iter() + .flat_map(|x| { + let bytes = + unsafe { std::slice::from_raw_parts(x as *const T as *const u8, size_of::()) }; + bytes.to_vec() + }) + .collect() +} + +#[test] +fn test_basic_array_view() { + // contiguous layout + let test_data = vec![1.0f64, 2.0, 3.0, 4.0, 5.0, 6.0]; + let array = Array2D::new(test_data.clone(), 2, 3, true); + assert_eq!(array.ndim(), 2); + assert_eq!(array.dim(0), Some(2)); + assert_eq!(array.dim(1), Some(3)); + assert_eq!(array.dim(2), None); + let mut buf = vec![]; + array.write_row_major(&mut buf).unwrap(); + let expected = to_bytes(&test_data); + assert_eq!(buf, expected); + + // non-contiguous layout + let test_data = vec![vec![1.0, 2.0], vec![3.0, 4.0], vec![5.0, 6.0]] + .into_iter() + .flatten() + .collect(); + let array_non_contig = Array2D::new(test_data, 3, 2, false); + let mut buf_non_contig = vec![]; + array_non_contig + .write_row_major(&mut buf_non_contig) + .unwrap(); + let expected_non_contig = to_bytes(&[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]); + assert_eq!(buf_non_contig, expected_non_contig); +} + +#[test] +fn test_basic_edge_cases() { + // empty array + let empty_array = Array2D::::new(vec![], 0, 0, false); + assert_eq!(empty_array.ndim(), 2); + assert_eq!(empty_array.dim(0), Some(0)); + assert_eq!(empty_array.dim(1), Some(0)); + + // single element array + let single = Array2D::new(vec![42.0], 1, 1, true); + let mut buf = vec![]; + single.write_row_major(&mut buf).unwrap(); + assert_eq!(buf, 42.0f64.to_ne_bytes().to_vec()); +} + +#[test] +fn test_buffer_basic_write() -> TestResult { + let test_data = vec![1.1f64, 2.2, 3.3, 4.4]; + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + let array_2d = Array2D::::new(test_data, 2, 2, true); + buffer.column_arr("temperature", &array_2d)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); assert_eq!( - ::elem_type(), - ndarr::ElemDataType::Double + &data[19..24], + &[ + b'=', + b'=', + ARRAY_BINARY_FORMAT_TYPE, + ElemDataType::Double.into(), + 2u8 + ] ); - assert_eq!(u8::from(ndarr::ElemDataType::Double), 10); + assert_eq!( + &data[24..32], + [2i32.to_le_bytes().as_slice(), 2i32.to_le_bytes().as_slice()].concat() + ); + Ok(()) } #[test] -fn test_1d_contiguous_buffer() { +fn test_invalid_dimension() -> TestResult { + struct InvalidDimArray; + impl NdArrayView for InvalidDimArray { + fn ndim(&self) -> usize { + 2 + } + fn dim(&self, _: usize) -> Option { + None + } + fn write_row_major(&self, _: &mut W) -> std::io::Result<()> { + todo!() + } + } + + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + let result = buffer.column_arr("arr1", &InvalidDimArray); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert!(err + .msg() + .contains("Can not get correct dimensions for dim 0")); + Ok(()) +} + +#[test] +fn test_size_overflow() -> TestResult { + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + let data = vec![1.0f64]; + let arr = Array2D:: { + data, + rows: usize::MAX, + cols: usize::MAX, + contiguous: false, + }; + + let result = buffer.column_arr("arr1", &arr); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert!(err.msg().contains("Array total elem size overflow")); + Ok(()) +} + +#[test] +fn test_write_failure() -> TestResult { + struct FaultyArray(PhantomData); + impl NdArrayView for FaultyArray { + fn ndim(&self) -> usize { + 2 + } + fn dim(&self, _: usize) -> Option { + Some(1) + } + fn write_row_major(&self, _: &mut W) -> std::io::Result<()> { + Err(std::io::Error::new( + std::io::ErrorKind::Other, + "mock write error", + )) + } + } + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + let result = buffer.column_arr("arr1", &FaultyArray(PhantomData::)); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayWriteToBufferError); + assert!(err + .msg() + .contains("Can not write row major to writer: mock write error")); + Ok(()) +} + +#[test] +fn test_array_length_mismatch() -> TestResult { + // actual data length is larger than shapes + let test_data = vec![1.1f64, 2.2, 3.3, 4.4]; + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + let array_2d = Array2D::::new(test_data, 1, 2, true); + let result = buffer.column_arr("arr1", &array_2d); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayWriteToBufferError); + assert!(err + .msg() + .contains("Can not write row major to writer: failed to write whole buffer")); + buffer.clear(); + + // actual data length is less than shapes + let test_data = vec![1.1f64]; + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + let array_2d = Array2D::::new(test_data, 1, 2, true); + let result = buffer.column_arr("arr1", &array_2d); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayWriteToBufferError); + assert!(err + .msg() + .contains("Array write buffer length mismatch (actual: 8, expected: 16)")); + buffer.clear(); + + // non-contiguous layout + let test_data = vec![1.1f64]; + let mut buffer = Buffer::new(); + buffer.table("my_test")?; + let array_2d = Array2D::::new(test_data, 1, 2, false); + let result = buffer.column_arr("arr1", &array_2d); + let err = result.unwrap_err(); + assert_eq!(err.code(), ErrorCode::ArrayWriteToBufferError); + assert!(err + .msg() + .contains("Array write buffer length mismatch (actual: 8, expected: 16)")); + Ok(()) +} + +#[cfg(feature = "ndarray")] +#[test] +fn test_1d_contiguous_ndarray_buffer() -> TestResult { let array = arr1(&[1.0, 2.0, 3.0, 4.0]); let view = array.view(); let mut buf = vec![0u8; 4 * size_of::()]; - view.write_row_major_buf(&mut buf); + view.write_row_major(&mut &mut buf[0..])?; let expected: Vec = array .iter() .flat_map(|&x| x.to_ne_bytes().to_vec()) .collect(); assert_eq!(buf, expected); + Ok(()) } +#[cfg(feature = "ndarray")] #[test] -fn test_2d_non_contiguous_buffer() { +fn test_2d_non_contiguous_ndarray_buffer() -> TestResult { let array = arr2(&[[1.0, 2.0], [3.0, 4.0]]); let transposed = array.view().reversed_axes(); assert!(!transposed.is_standard_layout()); let mut buf = vec![0u8; 4 * size_of::()]; - transposed.write_row_major_buf(&mut buf); + transposed.write_row_major(&mut &mut buf[0..])?; let expected = [1.0f64, 3.0, 2.0, 4.0] .iter() .flat_map(|&x| x.to_ne_bytes()) .collect::>(); assert_eq!(buf, expected); + Ok(()) } +#[cfg(feature = "ndarray")] #[test] -fn test_strided_layout() { +fn test_strided_ndarray_layout() -> TestResult { let array = arr2(&[ [1.0, 2.0, 3.0, 4.0], [5.0, 6.0, 7.0, 8.0], @@ -52,7 +310,7 @@ fn test_strided_layout() { let strided_view = array.slice(s![1..;2, 1..;2]); assert_eq!(strided_view.dim(), (2, 2)); let mut buf = vec![0u8; 4 * size_of::()]; - strided_view.write_row_major_buf(&mut buf); + strided_view.write_row_major(&mut &mut buf[0..])?; // expect：6.0, 8.0, 14.0, 16.0 let expected = [6.0f64, 8.0, 14.0, 16.0] @@ -61,10 +319,12 @@ fn test_strided_layout() { .collect::>(); assert_eq!(buf, expected); + Ok(()) } +#[cfg(feature = "ndarray")] #[test] -fn test_1d_dimension_info() { +fn test_1d_dimension_ndarray_info() { let array = arr1(&[1.0, 2.0, 3.0]); let view = array.view(); @@ -73,8 +333,9 @@ fn test_1d_dimension_info() { assert_eq!(NdArrayView::dim(&view, 1), None); } +#[cfg(feature = "ndarray")] #[test] -fn test_complex_dimensions() { +fn test_complex_ndarray_dimensions() { let array = arr3(&[[[1.0], [2.0]], [[3.0], [4.0]]]); let view = array.view(); @@ -84,8 +345,9 @@ fn test_complex_dimensions() { assert_eq!(NdArrayView::dim(&view, 2), Some(1)); } +#[cfg(feature = "ndarray")] #[test] -fn test_buffer_array_write() -> TestResult { +fn test_buffer_ndarray_write() -> TestResult { let mut buffer = Buffer::new(); buffer.table("my_test")?; let array_2d = arr2(&[[1.1, 2.2], [3.3, 4.4]]); @@ -99,7 +361,7 @@ fn test_buffer_array_write() -> TestResult { &[ b'=', b'=', - ingress::ARRAY_BINARY_FORMAT_TYPE, + ARRAY_BINARY_FORMAT_TYPE, ElemDataType::Double.into(), 2u8 ] @@ -111,8 +373,9 @@ fn test_buffer_array_write() -> TestResult { Ok(()) } +#[cfg(feature = "ndarray")] #[test] -fn test_buffer_write_max_dimensions() -> TestResult { +fn test_buffer_write_ndarray_max_dimensions() -> TestResult { let mut buffer = Buffer::new(); buffer.table("nd_test")?; let shape: Vec = iter::repeat(1).take(MAX_DIMS).collect(); diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index 66b8cefd..665bc449 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -122,7 +122,9 @@ fn test_array_basic() -> TestResult { ] .concat(); let mut array_data2d = vec![0u8; 4 * size_of::()]; - array_2d.view().write_row_major_buf(&mut array_data2d); + array_2d + .view() + .write_row_major(&mut &mut array_data2d[0..])?; let array_header3d = &[ &[b'='][..], @@ -135,7 +137,9 @@ fn test_array_basic() -> TestResult { ] .concat(); let mut array_data3d = vec![0u8; 24 * size_of::()]; - array_3d.view().write_row_major_buf(&mut array_data3d); + array_3d + .view() + .write_row_major(&mut &mut array_data3d[0..])?; let exp = &[ "my_table,device=A001 f1=25.5".as_bytes(), From 0c513b1498853370583e848c137e648918c7da30 Mon Sep 17 00:00:00 2001 From: victor Date: Mon, 7 Apr 2025 16:06:33 +0800 Subject: [PATCH 17/56] code format. --- questdb-rs/src/tests/ndarr.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index 886bde63..d6945e69 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -62,7 +62,7 @@ impl NdArrayView for Array2D { let bytes = unsafe { std::slice::from_raw_parts( chunk.as_ptr() as *const u8, - chunk.len() * std::mem::size_of::(), + size_of_val(chunk) ) }; writer.write_all(bytes)?; From 4a5834688eabc79f2b1f84c800ad07101b8e6a70 Mon Sep 17 00:00:00 2001 From: victor Date: Mon, 7 Apr 2025 16:36:23 +0800 Subject: [PATCH 18/56] c example --- cpp_test/mock_server.cpp | 5 ++-- cpp_test/test_line_sender.cpp | 27 ++++++++++++++++++++++ examples/line_sender_c_example.c | 27 ++++++++++++++++++++++ examples/line_sender_c_example_auth.c | 27 ++++++++++++++++++++++ examples/line_sender_c_example_auth_tls.c | 27 ++++++++++++++++++++++ examples/line_sender_c_example_from_conf.c | 27 ++++++++++++++++++++++ examples/line_sender_c_example_from_env.c | 27 ++++++++++++++++++++++ examples/line_sender_c_example_http.c | 27 ++++++++++++++++++++++ examples/line_sender_c_example_tls_ca.c | 27 ++++++++++++++++++++++ 9 files changed, 219 insertions(+), 2 deletions(-) diff --git a/cpp_test/mock_server.cpp b/cpp_test/mock_server.cpp index 0e8a1e85..ad9cf83a 100644 --- a/cpp_test/mock_server.cpp +++ b/cpp_test/mock_server.cpp @@ -24,6 +24,7 @@ #include "mock_server.hpp" +#include #include #if defined(PLATFORM_UNIX) @@ -195,9 +196,9 @@ size_t mock_server::recv(double wait_timeout_sec) if (!wait_for_data(wait_timeout_sec)) return 0; - char chunk[1024]; + std::byte chunk[1024]; size_t chunk_len{sizeof(chunk)}; - std::vector accum; + std::vector accum; for (;;) { wait_for_data(); diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 1510aef2..6f8ca1fc 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -95,6 +95,33 @@ TEST_CASE("line_sender c api basics") CHECK(::line_sender_buffer_table(buffer, table_name, &err)); CHECK(::line_sender_buffer_symbol(buffer, t1_name, v1_utf8, &err)); CHECK(::line_sender_buffer_column_f64(buffer, f1_name, 0.5, &err)); + + line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("order_book"); + // 3D array of doubles + size_t rank = 3; + uint32_t shapes[] = {2, 3, 2}; + double arr_data[] = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + CHECK( + ::line_sender_buffer_column_f64_arr( + buffer, + arr_name, + rank, + shapes, + reinterpret_cast(arr_data), + sizeof(arr_data), + &err)); CHECK(::line_sender_buffer_at_nanos(buffer, 10000000, &err)); CHECK(server.recv() == 0); CHECK(::line_sender_buffer_size(buffer) == 27); diff --git a/examples/line_sender_c_example.c b/examples/line_sender_c_example.c index 0f6805cc..59190e53 100644 --- a/examples/line_sender_c_example.c +++ b/examples/line_sender_c_example.c @@ -55,6 +55,33 @@ static bool example(const char* host, const char* port) if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) goto on_error; + line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("order_book"); + // 3D array of doubles + size_t rank = 3; + uint32_t shapes[] = {2, 3, 2}; + double arr_data[] = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + if (!line_sender_buffer_column_f64_arr( + buffer, + arr_name, + rank, + shapes, + (const uint8_t*)arr_data, + sizeof(arr_data), + &err)) + goto on_error; + // 1997-07-04 04:56:55 UTC int64_t designated_timestamp = 867992215000000000; if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) diff --git a/examples/line_sender_c_example_auth.c b/examples/line_sender_c_example_auth.c index 59a99421..1564da88 100644 --- a/examples/line_sender_c_example_auth.c +++ b/examples/line_sender_c_example_auth.c @@ -60,6 +60,33 @@ static bool example(const char* host, const char* port) if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) goto on_error; + line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("order_book"); + // 3D array of doubles + size_t rank = 3; + uint32_t shapes[] = {2, 3, 2}; + double arr_data[] = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + if (!line_sender_buffer_column_f64_arr( + buffer, + arr_name, + rank, + shapes, + (const uint8_t*)arr_data, + sizeof(arr_data), + &err)) + goto on_error; + // 1997-07-04 04:56:55 UTC int64_t designated_timestamp = 867992215000000000; if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) diff --git a/examples/line_sender_c_example_auth_tls.c b/examples/line_sender_c_example_auth_tls.c index e7454d8d..523e91d1 100644 --- a/examples/line_sender_c_example_auth_tls.c +++ b/examples/line_sender_c_example_auth_tls.c @@ -60,6 +60,33 @@ static bool example(const char* host, const char* port) if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) goto on_error; + line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("order_book"); + // 3D array of doubles + size_t rank = 3; + uint32_t shapes[] = {2, 3, 2}; + double arr_data[] = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + if (!line_sender_buffer_column_f64_arr( + buffer, + arr_name, + rank, + shapes, + (const uint8_t*)arr_data, + sizeof(arr_data), + &err)) + goto on_error; + // 1997-07-04 04:56:55 UTC int64_t designated_timestamp = 867992215000000000; if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) diff --git a/examples/line_sender_c_example_from_conf.c b/examples/line_sender_c_example_from_conf.c index 6f1d4e43..ec3a217f 100644 --- a/examples/line_sender_c_example_from_conf.c +++ b/examples/line_sender_c_example_from_conf.c @@ -44,6 +44,33 @@ int main(int argc, const char* argv[]) if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) goto on_error; + line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("order_book"); + // 3D array of doubles + size_t rank = 3; + uint32_t shapes[] = {2, 3, 2}; + double arr_data[] = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + if (!line_sender_buffer_column_f64_arr( + buffer, + arr_name, + rank, + shapes, + (const uint8_t*)arr_data, + sizeof(arr_data), + &err)) + goto on_error; + // 1997-07-04 04:56:55 UTC int64_t designated_timestamp = 867992215000000000; if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) diff --git a/examples/line_sender_c_example_from_env.c b/examples/line_sender_c_example_from_env.c index e67dbe59..f949b843 100644 --- a/examples/line_sender_c_example_from_env.c +++ b/examples/line_sender_c_example_from_env.c @@ -43,6 +43,33 @@ int main(int argc, const char* argv[]) if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) goto on_error; + line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("order_book"); + // 3D array of doubles + size_t rank = 3; + uint32_t shapes[] = {2, 3, 2}; + double arr_data[] = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + if (!line_sender_buffer_column_f64_arr( + buffer, + arr_name, + rank, + shapes, + (const uint8_t*)arr_data, + sizeof(arr_data), + &err)) + goto on_error; + // 1997-07-04 04:56:55 UTC int64_t designated_timestamp = 867992215000000000; if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) diff --git a/examples/line_sender_c_example_http.c b/examples/line_sender_c_example_http.c index 427ab705..e4dcf662 100644 --- a/examples/line_sender_c_example_http.c +++ b/examples/line_sender_c_example_http.c @@ -54,6 +54,33 @@ static bool example(const char* host, const char* port) if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) goto on_error; + line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("order_book"); + // 3D array of doubles + size_t rank = 3; + uint32_t shapes[] = {2, 3, 2}; + double arr_data[] = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + if (!line_sender_buffer_column_f64_arr( + buffer, + arr_name, + rank, + shapes, + (const uint8_t*)arr_data, + sizeof(arr_data), + &err)) + goto on_error; + // 1997-07-04 04:56:55 UTC int64_t designated_timestamp = 867992215000000000; if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) diff --git a/examples/line_sender_c_example_tls_ca.c b/examples/line_sender_c_example_tls_ca.c index 63151945..d52901fe 100644 --- a/examples/line_sender_c_example_tls_ca.c +++ b/examples/line_sender_c_example_tls_ca.c @@ -61,6 +61,33 @@ static bool example(const char* ca_path, const char* host, const char* port) if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) goto on_error; + line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("order_book"); + // 3D array of doubles + size_t rank = 3; + uint32_t shapes[] = {2, 3, 2}; + double arr_data[] = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + if (!line_sender_buffer_column_f64_arr( + buffer, + arr_name, + rank, + shapes, + (const uint8_t*)arr_data, + sizeof(arr_data), + &err)) + goto on_error; + // 1997-07-04 04:56:55 UTC int64_t designated_timestamp = 867992215000000000; if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) From 1a6cf6f2d2b5d8a3d3222c87e3c10889a6488a6e Mon Sep 17 00:00:00 2001 From: victor Date: Mon, 7 Apr 2025 20:28:09 +0800 Subject: [PATCH 19/56] make mock.rs recv works. --- cpp_test/mock_server.cpp | 5 +-- questdb-rs-ffi/Cargo.lock | 80 --------------------------------- questdb-rs/Cargo.toml | 2 +- questdb-rs/src/ingress/mod.rs | 2 + questdb-rs/src/ingress/ndarr.rs | 11 +++++ questdb-rs/src/tests/mock.rs | 56 +++++++++++++++++++++-- questdb-rs/src/tests/ndarr.rs | 11 +++-- questdb-rs/src/tests/sender.rs | 28 +++++------- 8 files changed, 85 insertions(+), 110 deletions(-) diff --git a/cpp_test/mock_server.cpp b/cpp_test/mock_server.cpp index ad9cf83a..0e8a1e85 100644 --- a/cpp_test/mock_server.cpp +++ b/cpp_test/mock_server.cpp @@ -24,7 +24,6 @@ #include "mock_server.hpp" -#include #include #if defined(PLATFORM_UNIX) @@ -196,9 +195,9 @@ size_t mock_server::recv(double wait_timeout_sec) if (!wait_for_data(wait_timeout_sec)) return 0; - std::byte chunk[1024]; + char chunk[1024]; size_t chunk_len{sizeof(chunk)}; - std::vector accum; + std::vector accum; for (;;) { wait_for_data(); diff --git a/questdb-rs-ffi/Cargo.lock b/questdb-rs-ffi/Cargo.lock index 248ab0c9..a23526e2 100644 --- a/questdb-rs-ffi/Cargo.lock +++ b/questdb-rs-ffi/Cargo.lock @@ -11,12 +11,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "autocfg" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" - [[package]] name = "aws-lc-rs" version = "1.13.0" @@ -380,16 +374,6 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" -[[package]] -name = "matrixmultiply" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9380b911e3e96d10c1f415da0876389aaf1b56759054eeb0de7df940c456ba1a" -dependencies = [ - "autocfg", - "rawpointer", -] - [[package]] name = "memchr" version = "2.7.4" @@ -402,21 +386,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "ndarray" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" -dependencies = [ - "matrixmultiply", - "num-complex", - "num-integer", - "num-traits", - "portable-atomic", - "portable-atomic-util", - "rawpointer", -] - [[package]] name = "nom" version = "7.1.3" @@ -427,33 +396,6 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "num-complex" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - [[package]] name = "once_cell" version = "1.21.3" @@ -472,21 +414,6 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" -[[package]] -name = "portable-atomic" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" - -[[package]] -name = "portable-atomic-util" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" -dependencies = [ - "portable-atomic", -] - [[package]] name = "ppv-lite86" version = "0.2.21" @@ -541,7 +468,6 @@ dependencies = [ "itoa", "libc", "log", - "ndarray", "questdb-confstr", "rand", "rustls", @@ -613,12 +539,6 @@ dependencies = [ "getrandom 0.3.2", ] -[[package]] -name = "rawpointer" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" - [[package]] name = "regex" version = "1.11.1" diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index 50649d37..90461b53 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -59,7 +59,7 @@ tempfile = "3" webpki-roots = "0.26.8" [features] -default = ["tls-webpki-certs", "ilp-over-http", "aws-lc-crypto", "ndarray"] +default = ["tls-webpki-certs", "ilp-over-http", "aws-lc-crypto"] # Include support for ILP over HTTP. ilp-over-http = ["dep:ureq", "dep:serde_json", "dep:rand"] diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index cc7d98fe..8699e572 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -2851,6 +2851,8 @@ impl Sender { } pub(crate) const ARRAY_BINARY_FORMAT_TYPE: u8 = 14; +#[allow(dead_code)] +pub(crate) const DOUBLE_BINARY_FORMAT_TYPE: u8 = 16; mod conf; pub(crate) mod ndarr; diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 2c0fd0ad..a66bd342 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -44,6 +44,17 @@ impl From for u8 { } } +impl TryFrom for ElemDataType { + type Error = String; + + fn try_from(value: u8) -> Result { + match value { + 0x0A => Ok(ElemDataType::Double), + _ => Err(format!("Unknown element type: {}", value)), + } + } +} + impl ArrayElement for f64 { /// Identifies f64 as Double type in QuestDB's type system. fn elem_type() -> ElemDataType { diff --git a/questdb-rs/src/tests/mock.rs b/questdb-rs/src/tests/mock.rs index 5b96d123..49bf150a 100644 --- a/questdb-rs/src/tests/mock.rs +++ b/questdb-rs/src/tests/mock.rs @@ -22,7 +22,7 @@ * ******************************************************************************/ -use crate::ingress::{Protocol, SenderBuilder}; +use crate::ingress::{ElemDataType, Protocol, SenderBuilder}; use core::time::Duration; use mio::event::Event; @@ -36,6 +36,7 @@ use std::net::SocketAddr; use std::sync::Arc; use std::time::Instant; +use crate::ingress; #[cfg(feature = "ilp-over-http")] use std::io::Write; @@ -50,7 +51,7 @@ pub struct MockServer { tls_conn: Option, pub host: &'static str, pub port: u16, - pub msgs: Vec, + pub msgs: Vec>, } pub fn certs_dir() -> std::path::PathBuf { @@ -519,8 +520,55 @@ impl MockServer { } } - self.msgs = accum; - Ok(self.msgs.len()) + let mut received_count = 0usize; + let mut head = 0usize; + let binary_length = 0usize; + let mut index = 1; + + while index < accum.len() { + let last = accum[index]; + let prev = accum[index - 1]; + if last == b'=' && prev == b'=' { + index += 1; + // calc binary length + let binary_type = accum[index]; + if binary_type == ingress::DOUBLE_BINARY_FORMAT_TYPE { + index += size_of::(); + } else if binary_type == ingress::ARRAY_BINARY_FORMAT_TYPE { + index += 1; + let element_type = match ElemDataType::try_from(accum[index]) { + Ok(t) => t, + Err(e) => { + return Err(io::Error::new(io::ErrorKind::Other, e)); + } + }; + let mut elems_size = match element_type { + ElemDataType::Double => size_of::(), + }; + index += 1; + let dims = accum[index] as usize; + index += 1; + for _ in 0..dims { + elems_size = elems_size + * i32::from_le_bytes( + accum[index..index + size_of::()].try_into().unwrap(), + ) as usize; + index += size_of::(); + } + index += elems_size; + } + } else if (last == b'\n') && (prev != b'\\' && binary_length == 0) { + let tail = index + 1; + let msg = &accum[head..tail]; + self.msgs.push(msg.to_vec()); + head = tail; + received_count += 1; + index = tail; + } else { + index += 1; + } + } + Ok(received_count) } pub fn recv_q(&mut self) -> io::Result { diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index d6945e69..ec6d4a08 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -1,5 +1,7 @@ +#[cfg(feature = "ndarray")] +use crate::ingress::MAX_DIMS; use crate::ingress::{ - ArrayElement, Buffer, ElemDataType, NdArrayView, ARRAY_BINARY_FORMAT_TYPE, MAX_DIMS, + ArrayElement, Buffer, ElemDataType, NdArrayView, ARRAY_BINARY_FORMAT_TYPE }; use crate::tests::TestResult; use crate::ErrorCode; @@ -60,10 +62,7 @@ impl NdArrayView for Array2D { for chunk in self.data.chunks(self.cols) { let bytes = unsafe { - std::slice::from_raw_parts( - chunk.as_ptr() as *const u8, - size_of_val(chunk) - ) + std::slice::from_raw_parts(chunk.as_ptr() as *const u8, size_of_val(chunk)) }; writer.write_all(bytes)?; } @@ -162,7 +161,7 @@ fn test_invalid_dimension() -> TestResult { None } fn write_row_major(&self, _: &mut W) -> std::io::Result<()> { - todo!() + Ok(()) } } diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index 665bc449..b3809507 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -22,13 +22,9 @@ * ******************************************************************************/ -use crate::{ - ingress, - ingress::{ - Buffer, CertificateAuthority, Sender, TableName, Timestamp, TimestampMicros, TimestampNanos, - }, - Error, ErrorCode, -}; +use crate::{ingress::{ + Buffer, CertificateAuthority, Sender, TableName, Timestamp, TimestampMicros, TimestampNanos, +}, Error, ErrorCode}; use crate::tests::{ mock::{certs_dir, MockServer}, @@ -36,7 +32,7 @@ use crate::tests::{ }; #[cfg(feature = "ndarray")] -use crate::ingress::ndarr::{ElemDataType, NdArrayView}; +use crate::{ingress, ingress::{ElemDataType, NdArrayView}}; use core::time::Duration; #[cfg(feature = "ndarray")] use ndarray::{arr1, arr2, ArrayD}; @@ -83,8 +79,8 @@ fn test_basics() -> TestResult { sender.flush(&mut buffer)?; assert_eq!(buffer.len(), 0); assert_eq!(buffer.as_bytes(), b""); - assert_eq!(server.recv_q()?, exp_byte.len()); - assert_eq!(server.msgs, exp_byte); + assert_eq!(server.recv_q()?, 1); + assert_eq!(server.msgs[0], exp_byte); Ok(()) } @@ -158,8 +154,8 @@ fn test_array_basic() -> TestResult { sender.flush(&mut buffer)?; assert_eq!(buffer.len(), 0); assert_eq!(buffer.as_bytes(), b""); - assert_eq!(server.recv_q()?, exp.len()); - assert_eq!(server.msgs.as_slice(), exp); + assert_eq!(server.recv_q()?, 1); + assert_eq!(server.msgs[0].as_slice(), exp); Ok(()) } @@ -467,8 +463,8 @@ fn test_tls_with_file_ca() -> TestResult { assert_eq!(buffer.as_bytes(), exp); assert_eq!(buffer.len(), exp.len()); sender.flush(&mut buffer)?; - assert_eq!(server.recv_q()?, exp.len()); - assert_eq!(server.msgs.as_slice(), exp); + assert_eq!(server.recv_q()?, 1); + assert_eq!(server.msgs[0].as_slice(), exp); Ok(()) } @@ -560,8 +556,8 @@ fn test_tls_insecure_skip_verify() -> TestResult { assert_eq!(buffer.as_bytes(), exp); assert_eq!(buffer.len(), exp.len()); sender.flush(&mut buffer)?; - assert_eq!(server.recv_q()?, exp.len()); - assert_eq!(server.msgs.as_slice(), exp); + assert_eq!(server.recv_q()?, 1); + assert_eq!(server.msgs[0].as_slice(), exp); Ok(()) } From 9caed7ad268e95f32907edd77423bab39dae47ef Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Mon, 7 Apr 2025 15:58:26 +0100 Subject: [PATCH 20/56] easier pre-ci 'proj' script to build lint and test changes --- cpp_test/test_line_sender.cpp | 17 ++- proj | 2 + proj.bat | 2 + proj.ps1 | 1 + proj.py | 220 +++++++++++++++++++++++++++++++++ questdb-rs/src/tests/mock.rs | 7 +- questdb-rs/src/tests/ndarr.rs | 4 +- questdb-rs/src/tests/sender.rs | 14 ++- 8 files changed, 247 insertions(+), 20 deletions(-) create mode 100755 proj create mode 100644 proj.bat create mode 100644 proj.ps1 create mode 100755 proj.py diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 6f8ca1fc..80cd72da 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -113,15 +113,14 @@ TEST_CASE("line_sender c api basics") 2.7, 48121.5, 4.3}; - CHECK( - ::line_sender_buffer_column_f64_arr( - buffer, - arr_name, - rank, - shapes, - reinterpret_cast(arr_data), - sizeof(arr_data), - &err)); + CHECK(::line_sender_buffer_column_f64_arr( + buffer, + arr_name, + rank, + shapes, + reinterpret_cast(arr_data), + sizeof(arr_data), + &err)); CHECK(::line_sender_buffer_at_nanos(buffer, 10000000, &err)); CHECK(server.recv() == 0); CHECK(::line_sender_buffer_size(buffer) == 27); diff --git a/proj b/proj new file mode 100755 index 00000000..d4c80095 --- /dev/null +++ b/proj @@ -0,0 +1,2 @@ +#!/bin/bash +python3 proj.py "$@" diff --git a/proj.bat b/proj.bat new file mode 100644 index 00000000..027f5468 --- /dev/null +++ b/proj.bat @@ -0,0 +1,2 @@ +@echo off +python3 proj.py %* diff --git a/proj.ps1 b/proj.ps1 new file mode 100644 index 00000000..7f9e830e --- /dev/null +++ b/proj.ps1 @@ -0,0 +1 @@ +python3 proj.py $args diff --git a/proj.py b/proj.py new file mode 100755 index 00000000..bd36a449 --- /dev/null +++ b/proj.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 + +import sys +sys.dont_write_bytecode = True +import pathlib +import shutil +import shlex +import subprocess +import os + + +PROJ_ROOT = pathlib.Path(__file__).parent + + +def _run(*args, env=None, cwd=None): + """ + Log and run a command within the build dir. + On error, exit with child's return code. + """ + args = [str(arg) for arg in args] + cwd = cwd or PROJ_ROOT + sys.stderr.write('[CMD] ') + if env is not None: + env_str = ' '.join(f'{k}={shlex.quote(v)}' for k, v in env.items()) + sys.stderr.write(f'{env_str} ') + env = {**os.environ, **env} + escaped_cmd = ' '.join(shlex.quote(arg) for arg in args) + sys.stderr.write(f'{escaped_cmd}\n') + ret_code = subprocess.run(args, cwd=str(cwd), env=env).returncode + if ret_code != 0: + sys.exit(ret_code) + + +def _rm(path: pathlib.Path, pattern: str): + paths = path.glob(pattern) + for path in paths: + sys.stderr.write(f'[RM] {path}\n') + path.unlink() + + +def _rmtree(path: pathlib.Path): + if not path.exists(): + return + sys.stderr.write(f'[RMTREE] {path}\n') + shutil.rmtree(path, ignore_errors=True) + + +def _has_command(command: str) -> bool: + """ + Check if a command is available in the system. + """ + return shutil.which(command) is not None + + +COMMANDS = [] + + +def command(fn): + COMMANDS.append(fn.__name__) + return fn + + +@command +def clean(): + _rmtree(PROJ_ROOT / 'build') + _rmtree(PROJ_ROOT / 'build_CXX20') + _rmtree(PROJ_ROOT / 'questdb-rs' / 'target') + _rmtree(PROJ_ROOT / 'questdb-rs-ffi' / 'target') + + +@command +def cmake_cxx17(): + _rmtree(PROJ_ROOT / 'build') + cmd = [ + 'cmake', + '-S', '.', + '-B', 'build', + '-DCMAKE_BUILD_TYPE=Release', + '-DQUESTDB_TESTS_AND_EXAMPLES=ON'] + if _has_command('ninja'): + cmd.insert(1, '-G') + cmd.insert(2, 'Ninja') + _run(*cmd) + + +@command +def cmake_cxx20(): + _rmtree(PROJ_ROOT / 'build_CXX20') + cmd = [ + 'cmake', + '-S', '.', + '-B', 'build_CXX20', + '-DCMAKE_BUILD_TYPE=Release', + '-DQUESTDB_TESTS_AND_EXAMPLES=ON', + '-DCMAKE_CXX_STANDARD=20'] + if _has_command('ninja'): + cmd.insert(1, '-G') + cmd.insert(2, 'Ninja') + _run(*cmd) + + +@command +def build_cxx17(): + if not (PROJ_ROOT / 'build').exists(): + cmake_cxx17() + _run('cmake', '--build', 'build') + + +@command +def build_cxx20(): + if not (PROJ_ROOT / 'build_CXX20').exists(): + cmake_cxx20() + _run('cmake', '--build', 'build_CXX20') + + +@command +def build(): + build_cxx17() + build_cxx20() + + +@command +def lint_rust(): + questdb_rs_path = PROJ_ROOT / 'questdb-rs' + questdb_rs_ffi_path = PROJ_ROOT / 'questdb-rs-ffi' + _run('cargo', 'fmt', '--all', '--', '--check', cwd=questdb_rs_path) + _run('cargo', 'clippy', '--all-targets', '--features', 'almost-all-features', '--', '-D', 'warnings', cwd=questdb_rs_path) + _run('cargo', 'fmt', '--all', '--', '--check', cwd=questdb_rs_ffi_path) + _run('cargo', 'clippy', '--all-targets', '--all-features', '--', '-D', 'warnings', cwd=questdb_rs_ffi_path) + + +@command +def lint_cpp(): + try: + _run( + sys.executable, + PROJ_ROOT / 'ci' / 'format_cpp.py', + '--check') + except subprocess.CalledProcessError: + sys.stderr.write('REMINDER: To fix any C++ formatting issues, run: ./proj format_cpp\n') + raise + + +@command +def lint(): + lint_rust() + lint_cpp() + + +@command +def format_rust(): + questdb_rs_path = PROJ_ROOT / 'questdb-rs' + questdb_rs_ffi_path = PROJ_ROOT / 'questdb-rs-ffi' + _run('cargo', 'fmt', '--all', cwd=questdb_rs_path) + _run('cargo', 'fmt', '--all', cwd=questdb_rs_ffi_path) + + +@command +def format_cpp(): + _run( + sys.executable, + PROJ_ROOT / 'ci' / 'format_cpp.py') + + +@command +def test(): + build() + _run( + sys.executable, + PROJ_ROOT / 'ci' / 'run_all_tests.py') + + +@command +def build_latest_questdb(branch='master'): + questdb_path = PROJ_ROOT / 'questdb' + if not questdb_path.exists(): + _run('git', 'clone', 'https://github.com/questdb/questdb.git') + _run('git', 'fetch', 'origin', branch, cwd=questdb_path) + _run('git', 'switch', branch=questdb_path) + _run('git', 'pull', 'origin', branch=questdb_path) + _run('git', 'submodule', 'update', '--init', '--recursive', cwd=questdb_path) + _run('mvn', 'clean', 'package', '-DskipTests', '-Pbuild-web-console', cwd=questdb_path) + + +@command +def test_vs_latest_questdb(): + questdb_path = PROJ_ROOT / 'questdb' + if not questdb_path.exists(): + build_latest_questdb() + _run( + sys.executable, + PROJ_ROOT / 'system_test' / 'test.py', + '--repo', PROJ_ROOT / 'questdb', + '-v') + + +@command +def all(): + clean() + build() + lint() + test() + test_vs_latest_questdb() + + +def main(): + if len(sys.argv) < 2: + sys.stderr.write('Usage: python3 proj.py \n') + sys.stderr.write('Commands:\n') + for command in COMMANDS: + sys.stderr.write(f' {command}\n') + sys.stderr.write('\n') + sys.exit(0) + fn = sys.argv[1] + args = list(sys.argv)[2:] + globals()[fn](*args) + + +if __name__ == '__main__': + main() diff --git a/questdb-rs/src/tests/mock.rs b/questdb-rs/src/tests/mock.rs index 49bf150a..1d1f438a 100644 --- a/questdb-rs/src/tests/mock.rs +++ b/questdb-rs/src/tests/mock.rs @@ -549,10 +549,9 @@ impl MockServer { let dims = accum[index] as usize; index += 1; for _ in 0..dims { - elems_size = elems_size - * i32::from_le_bytes( - accum[index..index + size_of::()].try_into().unwrap(), - ) as usize; + elems_size *= i32::from_le_bytes( + accum[index..index + size_of::()].try_into().unwrap(), + ) as usize; index += size_of::(); } index += elems_size; diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index ec6d4a08..2d7ee97b 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -1,8 +1,6 @@ #[cfg(feature = "ndarray")] use crate::ingress::MAX_DIMS; -use crate::ingress::{ - ArrayElement, Buffer, ElemDataType, NdArrayView, ARRAY_BINARY_FORMAT_TYPE -}; +use crate::ingress::{ArrayElement, Buffer, ElemDataType, NdArrayView, ARRAY_BINARY_FORMAT_TYPE}; use crate::tests::TestResult; use crate::ErrorCode; diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index b3809507..1f308e25 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -22,9 +22,12 @@ * ******************************************************************************/ -use crate::{ingress::{ - Buffer, CertificateAuthority, Sender, TableName, Timestamp, TimestampMicros, TimestampNanos, -}, Error, ErrorCode}; +use crate::{ + ingress::{ + Buffer, CertificateAuthority, Sender, TableName, Timestamp, TimestampMicros, TimestampNanos, + }, + Error, ErrorCode, +}; use crate::tests::{ mock::{certs_dir, MockServer}, @@ -32,7 +35,10 @@ use crate::tests::{ }; #[cfg(feature = "ndarray")] -use crate::{ingress, ingress::{ElemDataType, NdArrayView}}; +use crate::{ + ingress, + ingress::{ElemDataType, NdArrayView}, +}; use core::time::Duration; #[cfg(feature = "ndarray")] use ndarray::{arr1, arr2, ArrayD}; From c2583f1dbe0aaff7aebf72eefcffc26f19fc28fc Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 8 Apr 2025 01:03:04 +0800 Subject: [PATCH 21/56] c api add strides. --- include/questdb/ingress/line_sender.hpp | 7 +- questdb-rs-ffi/src/lib.rs | 69 ++------- questdb-rs/src/ingress/mod.rs | 3 +- questdb-rs/src/ingress/ndarr.rs | 191 +++++++++++++++++++++++- 4 files changed, 209 insertions(+), 61 deletions(-) diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index aa87a711..1f2ed06e 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -641,7 +641,8 @@ class line_sender_buffer template line_sender_buffer& column( column_name_view name, - const std::vector& shape, + const std::vector& shapes, + const std::vector& strides, const std::array& data) { static_assert( @@ -652,8 +653,8 @@ class line_sender_buffer ::line_sender_buffer_column_f64_arr, _impl, name._impl, - shape.size(), - shape.data(), + shapes.data(), + strides.data(), reinterpret_cast(data.data()), sizeof(double) * N); return *this; diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 3d929e6c..193fe34b 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -24,7 +24,7 @@ #![allow(non_camel_case_types, clippy::missing_safety_doc)] -use libc::{c_char, c_uint, size_t}; +use libc::{c_char, size_t}; use std::ascii; use std::boxed::Box; use std::convert::{From, Into}; @@ -33,13 +33,10 @@ use std::ptr; use std::slice; use std::str; -use questdb::{ - ingress::{ - ArrayElement, Buffer, CertificateAuthority, ColumnName, NdArrayView, Protocol, Sender, - SenderBuilder, TableName, TimestampMicros, TimestampNanos, - }, - Error, ErrorCode, -}; +use questdb::{ingress, ingress::{ + Buffer, CertificateAuthority, ColumnName, Protocol, Sender, + SenderBuilder, TableName, TimestampMicros, TimestampNanos, +}, Error, ErrorCode}; macro_rules! bubble_err_to_c { ($err_out:expr, $expression:expr) => { @@ -309,44 +306,6 @@ impl line_sender_utf8 { } } -#[derive(Debug, Copy, Clone)] -struct line_sender_array { - dims: size_t, - shapes: *const u32, - buf_len: size_t, - buf: *const u8, -} - -impl NdArrayView for line_sender_array -where - T: ArrayElement, -{ - fn ndim(&self) -> usize { - self.dims - } - - fn dim(&self, index: usize) -> Option { - if index >= self.dims { - return None; - } - - unsafe { - if self.shapes.is_null() { - return None; - } - - let dim_size = *self.shapes.add(index); - Some(dim_size as usize) - } - } - - fn write_row_major(&self, writer: &mut W) -> std::io::Result<()> { - let bytes = unsafe { slice::from_raw_parts(self.buf, self.buf_len) }; - writer.write_all(bytes)?; - Ok(()) - } -} - /// An ASCII-safe description of a binary buffer. Trimmed if too long. fn describe_buf(buf: &[u8]) -> String { let max_len = 100usize; @@ -877,22 +836,24 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( buffer: *mut line_sender_buffer, name: line_sender_column_name, rank: size_t, - shape: *const c_uint, // C array of dimension sizes + shapes: *const size_t, // C array of shapes + strides: *const i64, // C array of strides data_buffer: *const u8, // Raw array data data_buffer_len: size_t, // Total bytes length err_out: *mut *mut line_sender_error, ) -> bool { let buffer = unwrap_buffer_mut(buffer); let name = name.as_name(); - let view = line_sender_array { - dims: rank, - shapes: shape, - buf_len: data_buffer_len, - buf: data_buffer, - }; + let view = ingress::ArrayViewWithStrides::::new( + rank, + shapes, + strides as *const isize, + data_buffer, + data_buffer_len, + ); bubble_err_to_c!( err_out, - buffer.column_arr::, line_sender_array, f64>(name, &view) + buffer.column_arr::, ingress::ArrayViewWithStrides<'_, f64>, f64>(name, &view) ); true } diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 8699e572..b9690323 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -24,7 +24,7 @@ #![doc = include_str!("mod.md")] -pub use self::ndarr::{ArrayElement, ElemDataType, NdArrayView}; +pub use self::ndarr::{ArrayElement, ArrayViewWithStrides, ElemDataType, NdArrayView}; pub use self::timestamp::*; use crate::error::{self, Error, Result}; use crate::gai; @@ -1112,6 +1112,7 @@ impl Buffer { let writeable = unsafe { from_raw_parts_mut(self.output.as_mut_ptr().add(index), reserve_size) }; let mut cursor = Cursor::new(writeable); + // ndarr data if let Err(e) = view.write_row_major(&mut cursor) { return Err(error::fmt!( diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index a66bd342..3bb02cb6 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -62,8 +62,180 @@ impl ArrayElement for f64 { } } +#[derive(Debug)] +pub struct ArrayViewWithStrides<'a, T> { + dims: usize, + shapes: &'a [usize], + strides: &'a [isize], + buf_len: usize, + buf: *const u8, + _marker: std::marker::PhantomData, +} + +impl NdArrayView for ArrayViewWithStrides<'_, T> +where + T: ArrayElement, +{ + fn ndim(&self) -> usize { + self.dims + } + + fn dim(&self, index: usize) -> Option { + if index >= self.dims { + return None; + } + + Some(self.shapes[index]) + } + + fn write_row_major(&self, writer: &mut W) -> std::io::Result<()> { + if self.is_c_major() { + let bytes = unsafe { slice::from_raw_parts(self.buf, self.buf_len) }; + writer.write_all(bytes)?; + Ok(()) + } else { + let mut io_slices = Vec::new(); + for element in self.iter() { + io_slices.push(std::io::IoSlice::new(element)); + } + + let mut io_slices: &mut [std::io::IoSlice<'_>] = io_slices.as_mut_slice(); + while !io_slices.is_empty() { + let written = writer.write_vectored(io_slices)?; + if written == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::WriteZero, + "Failed to write all bytes", + )); + } + io_slices = &mut io_slices[written..]; + } + Ok(()) + } + } +} + +impl ArrayViewWithStrides<'_, T> +where + T: ArrayElement, +{ + /// # Safety + /// + /// todo + pub unsafe fn new( + dims: usize, + shapes: *const usize, + strides: *const isize, + data: *const u8, + data_len: usize, + ) -> Self { + let shapes = slice::from_raw_parts(shapes, dims); + let strides = slice::from_raw_parts(strides, dims); + Self { + dims, + shapes, + strides, + buf_len: data_len, + buf: data, + _marker: std::marker::PhantomData::, + } + } + + fn is_c_major(&self) -> bool { + let mut expected_stride = size_of::() as isize; + self.strides + .iter() + .rev() + .skip(1) + .zip(self.shapes.iter().rev()) + .all(|(&actual, &dim)| { + let expected = expected_stride; + expected_stride *= dim as isize; + actual.abs() == expected.abs() + }) + } + + fn iter(&self) -> CMajorIterWithStrides { + let mut dim_products = Vec::with_capacity(self.dims); + let mut product = 1; + for &dim in self.shapes.iter().rev() { + dim_products.push(product); + product *= dim; + } + dim_products.reverse(); + + // consider minus strides + let base_ptr = self + .strides + .iter() + .enumerate() + .fold(self.buf, |ptr, (dim, &stride)| { + if stride < 0 { + let dim_size = self.shapes[dim] as isize; + unsafe { ptr.offset(stride * (dim_size - 1)) } + } else { + ptr + } + }); + CMajorIterWithStrides { + base_ptr, + array: self, + dim_products, + current_linear: 0, + total_elements: self.shapes.iter().product(), + } + } +} + +struct CMajorIterWithStrides<'a, T> { + base_ptr: *const u8, + array: &'a ArrayViewWithStrides<'a, T>, + dim_products: Vec, + current_linear: usize, + total_elements: usize, +} + +impl CMajorIterWithStrides<'_, T> { + fn is_ptr_valid(&self, ptr: *const u8) -> bool { + let start = self.array.buf; + let end = unsafe { start.add(self.array.buf_len) }; + ptr >= start && ptr < end + } +} + +impl<'a, T> Iterator for CMajorIterWithStrides<'a, T> +where + T: ArrayElement, +{ + type Item = &'a [u8]; + fn next(&mut self) -> Option { + if self.current_linear >= self.total_elements { + return None; + } + let mut index = self.current_linear; + let mut offset = 0isize; + + for (dim, &prod) in self.dim_products.iter().enumerate() { + let coord = index / prod; + offset += self.array.strides[dim] * coord as isize; + index %= prod; + } + + self.current_linear += 1; + unsafe { + let ptr = self.base_ptr.offset(offset); + if self.is_ptr_valid(ptr) { + Some(slice::from_raw_parts(ptr, size_of::())) + } else { + None + } + } + } +} + #[cfg(feature = "ndarray")] use ndarray::{ArrayView, Axis, Dimension}; +use std::slice; #[cfg(feature = "ndarray")] impl NdArrayView for ArrayView<'_, T, D> @@ -96,10 +268,23 @@ where } let elem_size = size_of::(); - for &element in self.iter() { + let mut io_slices = Vec::new(); + for element in self.iter() { let bytes = - unsafe { std::slice::from_raw_parts(&element as *const T as *const _, elem_size) }; - writer.write_all(bytes)?; + unsafe { std::slice::from_raw_parts(element as *const T as *const _, elem_size) }; + io_slices.push(std::io::IoSlice::new(bytes)); + } + + let mut io_slices: &mut [std::io::IoSlice<'_>] = io_slices.as_mut_slice(); + while !io_slices.is_empty() { + let written = writer.write_vectored(io_slices)?; + if written == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::WriteZero, + "Failed to write all bytes", + )); + } + io_slices = &mut io_slices[written..]; } Ok(()) } From 0476faf4a594ab3aebd320035670cbd108c7172c Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 8 Apr 2025 16:49:11 +0800 Subject: [PATCH 22/56] add strideArrayView for rust array ingress api --- questdb-rs-ffi/src/lib.rs | 4 +- questdb-rs/src/ingress/mod.rs | 11 +- questdb-rs/src/ingress/ndarr.rs | 296 ++++++++++++++++++----------- questdb-rs/src/tests/ndarr.rs | 318 ++++++++++++++------------------ questdb-rs/src/tests/sender.rs | 11 +- 5 files changed, 340 insertions(+), 300 deletions(-) diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 193fe34b..3ffade93 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -844,7 +844,7 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( ) -> bool { let buffer = unwrap_buffer_mut(buffer); let name = name.as_name(); - let view = ingress::ArrayViewWithStrides::::new( + let view = ingress::StridedArrayView::::new( rank, shapes, strides as *const isize, @@ -853,7 +853,7 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( ); bubble_err_to_c!( err_out, - buffer.column_arr::, ingress::ArrayViewWithStrides<'_, f64>, f64>(name, &view) + buffer.column_arr::, ingress::StridedArrayView<'_, f64>, f64>(name, &view) ); true } diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index b9690323..da093ca6 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -24,7 +24,7 @@ #![doc = include_str!("mod.md")] -pub use self::ndarr::{ArrayElement, ArrayViewWithStrides, ElemDataType, NdArrayView}; +pub use self::ndarr::{ArrayElement, ElemDataType, NdArrayView, StridedArrayView}; pub use self::timestamp::*; use crate::error::{self, Error, Result}; use crate::gai; @@ -1079,6 +1079,7 @@ impl Buffer { )); } + let reserve_size = view.check_data_buf()?; // binary format flag '=' self.output.push(b'='); // binary format entity type @@ -1088,7 +1089,6 @@ impl Buffer { // ndarr dims self.output.push(view.ndim() as u8); - let mut reserve_size = size_of::(); for i in 0..view.ndim() { let d = view.dim(i).ok_or_else(|| { error::fmt!( @@ -1097,14 +1097,9 @@ impl Buffer { i ) })?; - // ndarr shapes self.output .extend_from_slice((d as i32).to_le_bytes().as_slice()); - reserve_size = reserve_size.checked_mul(d).ok_or(error::fmt!( - ArrayViewError, - "Array total elem size overflow" - ))? } self.output.reserve(reserve_size); @@ -1114,7 +1109,7 @@ impl Buffer { let mut cursor = Cursor::new(writeable); // ndarr data - if let Err(e) = view.write_row_major(&mut cursor) { + if let Err(e) = ndarr::write_array_data(view, &mut cursor) { return Err(error::fmt!( ArrayWriteToBufferError, "Can not write row major to writer: {}", diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 3bb02cb6..6be47f69 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -2,19 +2,70 @@ pub trait NdArrayView where T: ArrayElement, { + type Iter<'a>: Iterator + where + Self: 'a, + T: 'a; + /// Returns the number of dimensions (rank) of the array. fn ndim(&self) -> usize; /// Returns the size of the specified dimension. fn dim(&self, index: usize) -> Option; - /// Writes array data to writer in row-major order. + /// Return the array’s data as a slice, if it is c-major-layout. + /// Return `None` otherwise. + fn as_slice(&self) -> Option<&[T]>; + + /// Return an iterator of references to the elements of the array. + /// Iterator element type is `&T`. + fn iter(&self) -> Self::Iter<'_>; + + /// Validates the data buffer size of array is consistency of shapes. /// - /// # Important Notes - /// - Writer must be pre-allocated with exact required size - /// - No alignment assumptions should be made about writer start - /// - Handles both contiguous and non-contiguous memory layouts - fn write_row_major(&self, writer: &mut W) -> std::io::Result<()>; + /// # Returns + /// - `Ok(usize)`: Expected buffer size in bytes if valid + /// - `Err(Error)`: Currently never returned, but reserved for future validation logic + fn check_data_buf(&self) -> Result; +} + +pub(crate) fn write_array_data, T>( + array: &A, + writer: &mut W, +) -> std::io::Result<()> +where + T: ArrayElement, +{ + // First optimization path: write contiguous memory directly + if let Some(contiguous) = array.as_slice() { + let bytes = unsafe { + slice::from_raw_parts(contiguous.as_ptr() as *const u8, size_of_val(contiguous)) + }; + return writer.write_all(bytes); + } + + // Fallback path: non-contiguous memory handling + let elem_size = size_of::(); + let mut io_slices = Vec::new(); + for element in array.iter() { + let bytes = unsafe { slice::from_raw_parts(element as *const T as *const _, elem_size) }; + io_slices.push(std::io::IoSlice::new(bytes)); + } + + let mut io_slices: &mut [IoSlice<'_>] = io_slices.as_mut_slice(); + IoSlice::advance_slices(&mut io_slices, 0); + + while !io_slices.is_empty() { + let written = writer.write_vectored(io_slices)?; + if written == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::WriteZero, + "Failed to write all bytes", + )); + } + IoSlice::advance_slices(&mut io_slices, written); + } + Ok(()) } /// Marker trait for valid array element types. @@ -62,8 +113,9 @@ impl ArrayElement for f64 { } } +/// A view into a multi-dimensional array with custom memory strides. #[derive(Debug)] -pub struct ArrayViewWithStrides<'a, T> { +pub struct StridedArrayView<'a, T> { dims: usize, shapes: &'a [usize], strides: &'a [isize], @@ -72,10 +124,16 @@ pub struct ArrayViewWithStrides<'a, T> { _marker: std::marker::PhantomData, } -impl NdArrayView for ArrayViewWithStrides<'_, T> +impl NdArrayView for StridedArrayView<'_, T> where T: ArrayElement, { + type Iter<'b> + = RowMajorIter<'b, T> + where + Self: 'b, + T: 'b; + fn ndim(&self) -> usize { self.dims } @@ -88,40 +146,82 @@ where Some(self.shapes[index]) } - fn write_row_major(&self, writer: &mut W) -> std::io::Result<()> { + fn as_slice(&self) -> Option<&[T]> { if self.is_c_major() { - let bytes = unsafe { slice::from_raw_parts(self.buf, self.buf_len) }; - writer.write_all(bytes)?; - Ok(()) + Some(unsafe { slice::from_raw_parts(self.buf as *const T, self.buf_len / size_of::()) }) } else { - let mut io_slices = Vec::new(); - for element in self.iter() { - io_slices.push(std::io::IoSlice::new(element)); - } + None + } + } + + fn iter(&self) -> Self::Iter<'_> { + let mut dim_products = Vec::with_capacity(self.dims); + let mut product = 1; + for &dim in self.shapes.iter().rev() { + dim_products.push(product); + product *= dim; + } + dim_products.reverse(); - let mut io_slices: &mut [std::io::IoSlice<'_>] = io_slices.as_mut_slice(); - while !io_slices.is_empty() { - let written = writer.write_vectored(io_slices)?; - if written == 0 { - return Err(std::io::Error::new( - std::io::ErrorKind::WriteZero, - "Failed to write all bytes", - )); + // consider minus strides + let base_ptr = self + .strides + .iter() + .enumerate() + .fold(self.buf, |ptr, (dim, &stride)| { + if stride < 0 { + let dim_size = self.shapes[dim] as isize; + unsafe { ptr.offset(stride * (dim_size - 1)) } + } else { + ptr } - io_slices = &mut io_slices[written..]; - } - Ok(()) + }); + RowMajorIter { + base_ptr, + array: self, + dim_products, + current_linear: 0, + total_elements: self.shapes.iter().product(), } } + + fn check_data_buf(&self) -> Result { + let mut size = size_of::(); + for i in 0..self.dims { + let d = self.shapes[i]; + size = size.checked_mul(d).ok_or(error::fmt!( + ArrayViewError, + "Array total elem size overflow" + ))? + } + if size != self.buf_len { + return Err(error::fmt!( + ArrayWriteToBufferError, + "Array buffer length mismatch (actual: {}, expected: {})", + self.buf_len, + size + )); + } + Ok(size) + } } -impl ArrayViewWithStrides<'_, T> +impl StridedArrayView<'_, T> where T: ArrayElement, { - /// # Safety + /// Creates a new strided array view from raw components (unsafe constructor). /// - /// todo + /// # Safety + /// Caller must ensure all the following conditions: + /// - `shapes` points to a valid array of at least `dims` elements + /// - `strides` points to a valid array of at least `dims` elements + /// - `data` points to a valid memory block of at least `data_len` bytes + /// - Memory layout must satisfy: + /// 1. `data_len ≥ (shape[0]-1)*abs(strides[0]) + ... + (shape[n-1]-1)*abs(strides[n-1]) + size_of::()` + /// 2. All calculated offsets stay within `[0, data_len - size_of::()]` + /// - Lifetime `'a` must outlive the view's usage + /// - Strides are measured in bytes (not elements) pub unsafe fn new( dims: usize, shapes: *const usize, @@ -141,61 +241,50 @@ where } } + /// Verifies if the array follows C-style row-major memory layout. fn is_c_major(&self) -> bool { - let mut expected_stride = size_of::() as isize; - self.strides - .iter() - .rev() - .skip(1) - .zip(self.shapes.iter().rev()) - .all(|(&actual, &dim)| { - let expected = expected_stride; - expected_stride *= dim as isize; - actual.abs() == expected.abs() - }) - } + if self.buf.is_null() || self.buf_len == 0 { + return false; + } - fn iter(&self) -> CMajorIterWithStrides { - let mut dim_products = Vec::with_capacity(self.dims); - let mut product = 1; - for &dim in self.shapes.iter().rev() { - dim_products.push(product); - product *= dim; + if self.dims == 1 { + return self.strides[0] == 1 || self.shapes[0] <= 1; } - dim_products.reverse(); - // consider minus strides - let base_ptr = self - .strides - .iter() - .enumerate() - .fold(self.buf, |ptr, (dim, &stride)| { - if stride < 0 { - let dim_size = self.shapes[dim] as isize; - unsafe { ptr.offset(stride * (dim_size - 1)) } - } else { - ptr + for &d in self.shapes { + if d == 0 { + return true; + } + } + + let mut contig_stride = size_of::(); + for (dim, stride) in self.shapes.iter() + .rev() + .zip(self.strides.iter().rev()) + { + if *dim != 1 { + let s = *stride; + if s.abs() != contig_stride as isize { + return false; } - }); - CMajorIterWithStrides { - base_ptr, - array: self, - dim_products, - current_linear: 0, - total_elements: self.shapes.iter().product(), + contig_stride *= *dim; + } } + + true } } -struct CMajorIterWithStrides<'a, T> { +/// Iterator for traversing a strided array in row-major (C-style) order. +pub struct RowMajorIter<'a, T> { base_ptr: *const u8, - array: &'a ArrayViewWithStrides<'a, T>, + array: &'a StridedArrayView<'a, T>, dim_products: Vec, current_linear: usize, total_elements: usize, } -impl CMajorIterWithStrides<'_, T> { +impl RowMajorIter<'_, T> { fn is_ptr_valid(&self, ptr: *const u8) -> bool { let start = self.array.buf; let end = unsafe { start.add(self.array.buf_len) }; @@ -203,29 +292,35 @@ impl CMajorIterWithStrides<'_, T> { } } -impl<'a, T> Iterator for CMajorIterWithStrides<'a, T> +impl<'a, T> Iterator for RowMajorIter<'a, T> where T: ArrayElement, { - type Item = &'a [u8]; + type Item = &'a T; fn next(&mut self) -> Option { if self.current_linear >= self.total_elements { return None; } - let mut index = self.current_linear; + let mut remaining_index = self.current_linear; let mut offset = 0isize; - for (dim, &prod) in self.dim_products.iter().enumerate() { - let coord = index / prod; - offset += self.array.strides[dim] * coord as isize; - index %= prod; + for (dim, &dim_factor) in self.dim_products.iter().enumerate() { + let coord = remaining_index / dim_factor; + remaining_index %= dim_factor; + let stride = self.array.strides[dim]; + let actual_coord = if stride >= 0 { + coord + } else { + self.array.shapes[dim] - 1 - coord + }; + offset += (actual_coord as isize) * stride; } self.current_linear += 1; unsafe { let ptr = self.base_ptr.offset(offset); if self.is_ptr_valid(ptr) { - Some(slice::from_raw_parts(ptr, size_of::())) + Some(&*(ptr as *const T)) } else { None } @@ -233,8 +328,10 @@ where } } +use crate::{error, Error}; #[cfg(feature = "ndarray")] use ndarray::{ArrayView, Axis, Dimension}; +use std::io::IoSlice; use std::slice; #[cfg(feature = "ndarray")] @@ -243,6 +340,12 @@ where T: ArrayElement, D: Dimension, { + type Iter<'a> + = ndarray::iter::Iter<'a, T, D> + where + Self: 'a, + T: 'a; + fn ndim(&self) -> usize { self.ndim() } @@ -256,36 +359,15 @@ where } } - fn write_row_major(&self, writer: &mut W) -> std::io::Result<()> { - if let Some(contiguous) = self.as_slice() { - let bytes = unsafe { - std::slice::from_raw_parts( - contiguous.as_ptr() as *const u8, - size_of_val(contiguous), - ) - }; - return writer.write_all(bytes); - } + fn iter(&self) -> Self::Iter<'_> { + self.iter() + } - let elem_size = size_of::(); - let mut io_slices = Vec::new(); - for element in self.iter() { - let bytes = - unsafe { std::slice::from_raw_parts(element as *const T as *const _, elem_size) }; - io_slices.push(std::io::IoSlice::new(bytes)); - } + fn as_slice(&self) -> Option<&[T]> { + self.as_slice() + } - let mut io_slices: &mut [std::io::IoSlice<'_>] = io_slices.as_mut_slice(); - while !io_slices.is_empty() { - let written = writer.write_vectored(io_slices)?; - if written == 0 { - return Err(std::io::Error::new( - std::io::ErrorKind::WriteZero, - "Failed to write all bytes", - )); - } - io_slices = &mut io_slices[written..]; - } - Ok(()) + fn check_data_buf(&self) -> Result { + Ok(self.len() * size_of::()) } } diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index 2d7ee97b..d667b9fb 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -1,14 +1,17 @@ #[cfg(feature = "ndarray")] use crate::ingress::MAX_DIMS; -use crate::ingress::{ArrayElement, Buffer, ElemDataType, NdArrayView, ARRAY_BINARY_FORMAT_TYPE}; +use crate::ingress::{ + ArrayElement, Buffer, ElemDataType, NdArrayView, StridedArrayView, ARRAY_BINARY_FORMAT_TYPE, +}; use crate::tests::TestResult; use crate::ErrorCode; +use crate::ingress::ndarr::write_array_data; #[cfg(feature = "ndarray")] use ndarray::{arr1, arr2, arr3, s, ArrayD}; #[cfg(feature = "ndarray")] use std::iter; -use std::marker::PhantomData; +use std::ptr; #[test] fn test_f64_element_type() { @@ -16,58 +19,6 @@ fn test_f64_element_type() { assert_eq!(u8::from(ElemDataType::Double), 10); } -struct Array2D { - data: Vec, - rows: usize, - cols: usize, - contiguous: bool, -} - -impl Array2D { - fn new(data: Vec, rows: usize, cols: usize, contiguous: bool) -> Self { - Self { - data, - rows, - cols, - contiguous, - } - } -} - -impl NdArrayView for Array2D { - fn ndim(&self) -> usize { - 2 - } - - fn dim(&self, index: usize) -> Option { - match index { - 0 => Some(self.rows), - 1 => Some(self.cols), - _ => None, - } - } - - fn write_row_major(&self, writer: &mut W) -> std::io::Result<()> { - if self.contiguous { - let bytes = unsafe { - std::slice::from_raw_parts( - self.data.as_ptr() as *const u8, - self.data.len() * std::mem::size_of::(), - ) - }; - return writer.write_all(bytes); - } - - for chunk in self.data.chunks(self.cols) { - let bytes = unsafe { - std::slice::from_raw_parts(chunk.as_ptr() as *const u8, size_of_val(chunk)) - }; - writer.write_all(bytes)?; - } - Ok(()) - } -} - fn to_bytes(data: &[T]) -> Vec { data.iter() .flat_map(|x| { @@ -79,55 +30,115 @@ fn to_bytes(data: &[T]) -> Vec { } #[test] -fn test_basic_array_view() { +fn test_strided_array_view() -> TestResult { // contiguous layout - let test_data = vec![1.0f64, 2.0, 3.0, 4.0, 5.0, 6.0]; - let array = Array2D::new(test_data.clone(), 2, 3, true); + let test_data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; + let shapes = [2, 3]; + let strides = [ + (shapes[1] * size_of::()) as isize, + size_of::() as isize, + ]; + let array = unsafe { + StridedArrayView::::new( + shapes.len(), + shapes.as_ptr(), + strides.as_ptr(), + test_data.as_ptr() as *const u8, + test_data.len() * size_of::(), + ) + }; + assert_eq!(array.ndim(), 2); assert_eq!(array.dim(0), Some(2)); assert_eq!(array.dim(1), Some(3)); assert_eq!(array.dim(2), None); + assert!(array.as_slice().is_some()); let mut buf = vec![]; - array.write_row_major(&mut buf).unwrap(); + write_array_data(&array, &mut buf).unwrap(); let expected = to_bytes(&test_data); assert_eq!(buf, expected); + Ok(()) +} - // non-contiguous layout - let test_data = vec![vec![1.0, 2.0], vec![3.0, 4.0], vec![5.0, 6.0]] - .into_iter() - .flatten() - .collect(); - let array_non_contig = Array2D::new(test_data, 3, 2, false); - let mut buf_non_contig = vec![]; - array_non_contig - .write_row_major(&mut buf_non_contig) - .unwrap(); - let expected_non_contig = to_bytes(&[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]); - assert_eq!(buf_non_contig, expected_non_contig); +#[test] +fn test_strided_non_contiguous() -> TestResult { + let elem_size = size_of::(); + let col_major_data = vec![1.0, 3.0, 5.0, 2.0, 4.0, 6.0]; + let shapes = [3, 2]; + let strides = [elem_size as isize, (shapes[0] * elem_size) as isize]; + + let array_view: StridedArrayView<'_, f64> = unsafe { + StridedArrayView::new( + shapes.len(), + shapes.as_ptr(), + strides.as_ptr(), + col_major_data.as_ptr() as *const u8, + col_major_data.len() * elem_size, + ) + }; + + assert_eq!(array_view.ndim(), 2); + assert_eq!(array_view.dim(0), Some(3)); + assert_eq!(array_view.dim(1), Some(2)); + assert_eq!(array_view.dim(2), None); + assert!(array_view.as_slice().is_none()); + let mut buffer = Vec::new(); + write_array_data(&array_view, &mut buffer)?; + + let expected_data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; + let expected_bytes = unsafe { + std::slice::from_raw_parts( + expected_data.as_ptr() as *const u8, + expected_data.len() * elem_size, + ) + }; + assert_eq!(buffer, expected_bytes); + Ok(()) } #[test] fn test_basic_edge_cases() { // empty array - let empty_array = Array2D::::new(vec![], 0, 0, false); - assert_eq!(empty_array.ndim(), 2); - assert_eq!(empty_array.dim(0), Some(0)); - assert_eq!(empty_array.dim(1), Some(0)); + let elem_size = std::mem::size_of::(); + let empty_view: StridedArrayView<'_, f64> = + unsafe { StridedArrayView::new(2, [0, 0].as_ptr(), [0, 0].as_ptr(), ptr::null(), 0) }; + assert_eq!(empty_view.ndim(), 2); + assert_eq!(empty_view.dim(0), Some(0)); + assert_eq!(empty_view.dim(1), Some(0)); // single element array - let single = Array2D::new(vec![42.0], 1, 1, true); + let single_data = vec![42.0]; + let single_view: StridedArrayView<'_, f64> = unsafe { + StridedArrayView::new( + 2, + [1, 1].as_ptr(), // 1x1 矩阵 + [elem_size as isize, elem_size as isize].as_ptr(), + single_data.as_ptr() as *const u8, + elem_size, + ) + }; let mut buf = vec![]; - single.write_row_major(&mut buf).unwrap(); - assert_eq!(buf, 42.0f64.to_ne_bytes().to_vec()); + write_array_data(&single_view, &mut buf).unwrap(); + assert_eq!(buf, 42.0f64.to_ne_bytes()); } #[test] fn test_buffer_basic_write() -> TestResult { - let test_data = vec![1.1f64, 2.2, 3.3, 4.4]; + let elem_size = std::mem::size_of::(); + + let test_data = vec![1.1, 2.2, 3.3, 4.4]; + let array_view: StridedArrayView<'_, f64> = unsafe { + StridedArrayView::new( + 2, + [2, 2].as_ptr(), + [2 * elem_size as isize, elem_size as isize].as_ptr(), + test_data.as_ptr() as *const u8, + test_data.len() * elem_size, + ) + }; let mut buffer = Buffer::new(); buffer.table("my_test")?; - let array_2d = Array2D::::new(test_data, 2, 2, true); - buffer.column_arr("temperature", &array_2d)?; + buffer.column_arr("temperature", &array_view)?; let data = buffer.as_bytes(); assert_eq!(&data[0..7], b"my_test"); assert_eq!(&data[8..19], b"temperature"); @@ -143,124 +154,79 @@ fn test_buffer_basic_write() -> TestResult { ); assert_eq!( &data[24..32], - [2i32.to_le_bytes().as_slice(), 2i32.to_le_bytes().as_slice()].concat() + [2i32.to_le_bytes(), 2i32.to_le_bytes()].concat() + ); + assert_eq!( + &data[32..64], + &[ + 1.1f64.to_ne_bytes(), + 2.2f64.to_le_bytes(), + 3.3f64.to_le_bytes(), + 4.4f64.to_le_bytes(), + ].concat() ); - Ok(()) -} - -#[test] -fn test_invalid_dimension() -> TestResult { - struct InvalidDimArray; - impl NdArrayView for InvalidDimArray { - fn ndim(&self) -> usize { - 2 - } - fn dim(&self, _: usize) -> Option { - None - } - fn write_row_major(&self, _: &mut W) -> std::io::Result<()> { - Ok(()) - } - } - - let mut buffer = Buffer::new(); - buffer.table("my_test")?; - let result = buffer.column_arr("arr1", &InvalidDimArray); - assert!(result.is_err()); - let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ArrayViewError); - assert!(err - .msg() - .contains("Can not get correct dimensions for dim 0")); Ok(()) } #[test] fn test_size_overflow() -> TestResult { - let mut buffer = Buffer::new(); - buffer.table("my_test")?; - let data = vec![1.0f64]; - let arr = Array2D:: { - data, - rows: usize::MAX, - cols: usize::MAX, - contiguous: false, + let overflow_view = unsafe { + StridedArrayView::::new( + 2, + [usize::MAX, usize::MAX].as_ptr(), + [8, 8].as_ptr(), + ptr::null(), + 0, + ) }; - let result = buffer.column_arr("arr1", &arr); - let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ArrayViewError); - assert!(err.msg().contains("Array total elem size overflow")); - Ok(()) -} - -#[test] -fn test_write_failure() -> TestResult { - struct FaultyArray(PhantomData); - impl NdArrayView for FaultyArray { - fn ndim(&self) -> usize { - 2 - } - fn dim(&self, _: usize) -> Option { - Some(1) - } - fn write_row_major(&self, _: &mut W) -> std::io::Result<()> { - Err(std::io::Error::new( - std::io::ErrorKind::Other, - "mock write error", - )) - } - } let mut buffer = Buffer::new(); buffer.table("my_test")?; - let result = buffer.column_arr("arr1", &FaultyArray(PhantomData::)); + let result = buffer.column_arr("arr1", &overflow_view); let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ArrayWriteToBufferError); - assert!(err - .msg() - .contains("Can not write row major to writer: mock write error")); + assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert!(err.msg().contains("Array total elem size overflow")); Ok(()) } #[test] fn test_array_length_mismatch() -> TestResult { - // actual data length is larger than shapes - let test_data = vec![1.1f64, 2.2, 3.3, 4.4]; - let mut buffer = Buffer::new(); - buffer.table("my_test")?; - let array_2d = Array2D::::new(test_data, 1, 2, true); - let result = buffer.column_arr("arr1", &array_2d); - let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ArrayWriteToBufferError); - assert!(err - .msg() - .contains("Can not write row major to writer: failed to write whole buffer")); - buffer.clear(); + let elem_size = std::mem::size_of::(); + let under_data = vec![1.1]; + let under_view: StridedArrayView<'_, f64> = unsafe { + StridedArrayView::new( + 2, + [1, 2].as_ptr(), + [elem_size as isize, elem_size as isize].as_ptr(), + under_data.as_ptr() as *const u8, + under_data.len() * elem_size, + ) + }; - // actual data length is less than shapes - let test_data = vec![1.1f64]; let mut buffer = Buffer::new(); buffer.table("my_test")?; - let array_2d = Array2D::::new(test_data, 1, 2, true); - let result = buffer.column_arr("arr1", &array_2d); + let result = buffer.column_arr("arr1", &under_view); let err = result.unwrap_err(); assert_eq!(err.code(), ErrorCode::ArrayWriteToBufferError); - assert!(err - .msg() - .contains("Array write buffer length mismatch (actual: 8, expected: 16)")); - buffer.clear(); + assert!(err.msg().contains("Array buffer length mismatch (actual: 8, expected: 16)")); + + let over_data = vec![1.1, 2.2, 3.3]; + let over_view: StridedArrayView<'_, f64> = unsafe { + StridedArrayView::new( + 2, + [1, 2].as_ptr(), + [elem_size as isize, elem_size as isize].as_ptr(), + over_data.as_ptr() as *const u8, + over_data.len() * elem_size, + ) + }; - // non-contiguous layout - let test_data = vec![1.1f64]; - let mut buffer = Buffer::new(); + buffer.clear(); buffer.table("my_test")?; - let array_2d = Array2D::::new(test_data, 1, 2, false); - let result = buffer.column_arr("arr1", &array_2d); + let result = buffer.column_arr("arr1", &over_view); let err = result.unwrap_err(); assert_eq!(err.code(), ErrorCode::ArrayWriteToBufferError); - assert!(err - .msg() - .contains("Array write buffer length mismatch (actual: 8, expected: 16)")); + assert!(err.msg().contains("Array buffer length mismatch (actual: 24, expected: 16)")); Ok(()) } @@ -270,7 +236,7 @@ fn test_1d_contiguous_ndarray_buffer() -> TestResult { let array = arr1(&[1.0, 2.0, 3.0, 4.0]); let view = array.view(); let mut buf = vec![0u8; 4 * size_of::()]; - view.write_row_major(&mut &mut buf[0..])?; + write_array_data(&view, &mut &mut buf[0..])?; let expected: Vec = array .iter() .flat_map(|&x| x.to_ne_bytes().to_vec()) @@ -286,7 +252,7 @@ fn test_2d_non_contiguous_ndarray_buffer() -> TestResult { let transposed = array.view().reversed_axes(); assert!(!transposed.is_standard_layout()); let mut buf = vec![0u8; 4 * size_of::()]; - transposed.write_row_major(&mut &mut buf[0..])?; + write_array_data(&transposed, &mut &mut buf[0..])?; let expected = [1.0f64, 3.0, 2.0, 4.0] .iter() .flat_map(|&x| x.to_ne_bytes()) @@ -307,7 +273,7 @@ fn test_strided_ndarray_layout() -> TestResult { let strided_view = array.slice(s![1..;2, 1..;2]); assert_eq!(strided_view.dim(), (2, 2)); let mut buf = vec![0u8; 4 * size_of::()]; - strided_view.write_row_major(&mut &mut buf[0..])?; + write_array_data(&strided_view, &mut &mut buf[0..])?; // expect：6.0, 8.0, 14.0, 16.0 let expected = [6.0f64, 8.0, 14.0, 16.0] diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index 1f308e25..9f61e4cd 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -37,12 +37,13 @@ use crate::tests::{ #[cfg(feature = "ndarray")] use crate::{ ingress, - ingress::{ElemDataType, NdArrayView}, + ingress::ElemDataType, }; use core::time::Duration; #[cfg(feature = "ndarray")] use ndarray::{arr1, arr2, ArrayD}; +use crate::ingress::ndarr; use std::{io, time::SystemTime}; #[test] @@ -124,9 +125,7 @@ fn test_array_basic() -> TestResult { ] .concat(); let mut array_data2d = vec![0u8; 4 * size_of::()]; - array_2d - .view() - .write_row_major(&mut &mut array_data2d[0..])?; + ndarr::write_array_data(&array_2d.view(), &mut &mut array_data2d[0..])?; let array_header3d = &[ &[b'='][..], @@ -139,9 +138,7 @@ fn test_array_basic() -> TestResult { ] .concat(); let mut array_data3d = vec![0u8; 24 * size_of::()]; - array_3d - .view() - .write_row_major(&mut &mut array_data3d[0..])?; + ndarr::write_array_data(&array_3d.view(), &mut &mut array_data3d[0..])?; let exp = &[ "my_table,device=A001 f1=25.5".as_bytes(), From 31e57c56d4fc9b6bb9db24a84e92ac13a7ceb93b Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 8 Apr 2025 16:55:35 +0800 Subject: [PATCH 23/56] add strideArrayView for rust array ingress api --- questdb-rs/src/ingress/ndarr.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 6be47f69..d5a71901 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -21,7 +21,7 @@ where /// Iterator element type is `&T`. fn iter(&self) -> Self::Iter<'_>; - /// Validates the data buffer size of array is consistency of shapes. + /// Validates the data buffer size of array is consistency with array [`Shapes`]. /// /// # Returns /// - `Ok(usize)`: Expected buffer size in bytes if valid From 533224a4340ca4de44cae90ba3b75e1835f4adb9 Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 8 Apr 2025 21:07:44 +0800 Subject: [PATCH 24/56] add benchmark test. --- include/questdb/ingress/line_sender.h | 3 +- questdb-rs-ffi/Cargo.lock | 1 - questdb-rs-ffi/src/lib.rs | 18 ++++--- questdb-rs/Cargo.toml | 9 +++- questdb-rs/benches/ndarr.rs | 69 +++++++++++++++++++++++++++ questdb-rs/src/ingress/mod.rs | 50 +++++++++++++++++++ questdb-rs/src/ingress/ndarr.rs | 41 ++++++++++++---- questdb-rs/src/tests/ndarr.rs | 11 +++-- questdb-rs/src/tests/sender.rs | 6 +-- 9 files changed, 181 insertions(+), 27 deletions(-) create mode 100644 questdb-rs/benches/ndarr.rs diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 77877f3d..8e2ad20e 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -482,7 +482,8 @@ bool line_sender_buffer_column_f64_arr( line_sender_buffer* buffer, line_sender_column_name name, size_t rank, - const uint32_t* shapes, + const size_t* shapes, + const int64_t* strides, const uint8_t* data_buffer, size_t data_buffer_len, line_sender_error** err_out); diff --git a/questdb-rs-ffi/Cargo.lock b/questdb-rs-ffi/Cargo.lock index a23526e2..5666057c 100644 --- a/questdb-rs-ffi/Cargo.lock +++ b/questdb-rs-ffi/Cargo.lock @@ -467,7 +467,6 @@ dependencies = [ "indoc", "itoa", "libc", - "log", "questdb-confstr", "rand", "rustls", diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 3ffade93..26e8a760 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -33,10 +33,14 @@ use std::ptr; use std::slice; use std::str; -use questdb::{ingress, ingress::{ - Buffer, CertificateAuthority, ColumnName, Protocol, Sender, - SenderBuilder, TableName, TimestampMicros, TimestampNanos, -}, Error, ErrorCode}; +use questdb::{ + ingress, + ingress::{ + Buffer, CertificateAuthority, ColumnName, Protocol, Sender, SenderBuilder, TableName, + TimestampMicros, TimestampNanos, + }, + Error, ErrorCode, +}; macro_rules! bubble_err_to_c { ($err_out:expr, $expression:expr) => { @@ -140,7 +144,7 @@ pub enum line_sender_error_code { line_sender_error_array_view_internal_error, /// Write arrayView to sender buffer error. - line_sender_error_array_view_write_to_buffer_error + line_sender_error_array_view_write_to_buffer_error, } impl From for line_sender_error_code { @@ -836,8 +840,8 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( buffer: *mut line_sender_buffer, name: line_sender_column_name, rank: size_t, - shapes: *const size_t, // C array of shapes - strides: *const i64, // C array of strides + shapes: *const size_t, // C array of shapes + strides: *const i64, // C array of strides data_buffer: *const u8, // Raw array data data_buffer_len: size_t, // Total bytes length err_out: *mut *mut line_sender_error, diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index 90461b53..ba2a5680 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -41,7 +41,6 @@ questdb-confstr = "0.1.1" rand = { version = "0.9.0", optional = true } ndarray = { version = "0.16", optional = true } no-panic = { version = "0.1", optional = true } -log = "0.4.27" [target.'cfg(windows)'.dependencies] winapi = { version = "0.3.9", features = ["ws2def"] } @@ -57,6 +56,7 @@ mio = { version = "1", features = ["os-poll", "net"] } chrono = "0.4.31" tempfile = "3" webpki-roots = "0.26.8" +criterion = "0.5" [features] default = ["tls-webpki-certs", "ilp-over-http", "aws-lc-crypto"] @@ -85,6 +85,8 @@ json_tests = [] # Enable methods to create timestamp objects from chrono::DateTime objects. chrono_timestamp = ["chrono"] +benchmark = [] + # The `aws-lc-crypto` and `ring-crypto` features are mutually exclusive, # thus compiling with `--all-features` will not work. # Instead compile with `--features almost-all-features`. @@ -99,6 +101,11 @@ almost-all-features = [ "ndarray" ] +[[bench]] +name = "ndarr" +harness = false +required-features = ["benchmark", "ndarray"] + [[example]] name = "basic" required-features = ["chrono_timestamp", "ndarray"] diff --git a/questdb-rs/benches/ndarr.rs b/questdb-rs/benches/ndarr.rs new file mode 100644 index 00000000..babac254 --- /dev/null +++ b/questdb-rs/benches/ndarr.rs @@ -0,0 +1,69 @@ +use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use ndarray::{Array, Array2}; +use questdb::ingress::{Buffer, ColumnName}; + +/// run with +/// ```shell +/// cargo bench --bench ndarr --features="benchmark, ndarray" +/// ``` +fn bench_write_array_data(c: &mut Criterion) { + let mut group = c.benchmark_group("write_array_data"); + let contiguous_array: Array2 = Array::zeros((1000, 1000)); + let non_contiguous_array = contiguous_array.t(); + assert!(contiguous_array.is_standard_layout()); + assert!(!non_contiguous_array.is_standard_layout()); + + let col_name = ColumnName::new("col1").unwrap(); + // Case 1 + group.bench_function("contiguous_writer", |b| { + let mut buffer = Buffer::new(); + buffer.table("x1").unwrap(); + b.iter(|| { + buffer + .column_arr(col_name, black_box(&contiguous_array.view())) + .unwrap(); + }); + buffer.clear(); + }); + + // Case 2 + group.bench_function("contiguous_raw_buffer", |b| { + let mut buffer = Buffer::new(); + buffer.table("x1").unwrap(); + b.iter(|| { + buffer + .column_arr_use_raw_buffer(col_name, black_box(&contiguous_array.view())) + .unwrap(); + }); + buffer.clear(); + }); + + // Case 3 + group.bench_function("non_contiguous_writer", |b| { + let mut buffer = Buffer::new(); + buffer.table("x1").unwrap(); + b.iter(|| { + buffer + .column_arr(col_name, black_box(&non_contiguous_array.view())) + .unwrap(); + }); + buffer.clear(); + }); + + // Case 4 + group.bench_function("non_contiguous_raw_buffer", |b| { + let mut buffer = Buffer::new(); + buffer.table("x1").unwrap(); + b.iter(|| { + buffer + .column_arr_use_raw_buffer(col_name, black_box(&non_contiguous_array.view())) + .unwrap(); + }); + buffer.clear(); + }); + + group.finish(); +} + +criterion_group!(benches, bench_write_array_data); +criterion_main!(benches); diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index da093ca6..2374a16d 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -1130,6 +1130,56 @@ impl Buffer { Ok(self) } + #[cfg(feature = "benchmark")] + pub fn column_arr_use_raw_buffer<'a, N, T, D>(&mut self, name: N, view: &T) -> Result<&mut Self> + where + N: TryInto>, + T: NdArrayView, + D: ArrayElement, + Error: From, + { + self.write_column_key(name)?; + + // check dimension less equal than max dims + if MAX_DIMS < view.ndim() { + return Err(error::fmt!( + ArrayHasTooManyDims, + "Array dimension mismatch: expected at most {} dimensions, but got {}", + MAX_DIMS, + view.ndim() + )); + } + + let reserve_size = view.check_data_buf()?; + // binary format flag '=' + self.output.push(b'='); + // binary format entity type + self.output.push(ARRAY_BINARY_FORMAT_TYPE); + // ndarr datatype + self.output.push(D::elem_type().into()); + // ndarr dims + self.output.push(view.ndim() as u8); + + for i in 0..view.ndim() { + let d = view.dim(i).ok_or_else(|| { + error::fmt!( + ArrayViewError, + "Can not get correct dimensions for dim {}", + i + ) + })?; + // ndarr shapes + self.output + .extend_from_slice((d as i32).to_le_bytes().as_slice()); + } + + self.output.reserve(reserve_size); + let index = self.output.len(); + unsafe { self.output.set_len(reserve_size + index) } + ndarr::write_array_data_use_raw_buffer(&mut self.output[index..], view); + Ok(self) + } + /// Record a timestamp value for the given column. /// /// ``` diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index d5a71901..9d4ca801 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -2,7 +2,7 @@ pub trait NdArrayView where T: ArrayElement, { - type Iter<'a>: Iterator + type Iter<'a>: Iterator where Self: 'a, T: 'a; @@ -29,7 +29,7 @@ where fn check_data_buf(&self) -> Result; } -pub(crate) fn write_array_data, T>( +pub fn write_array_data, T>( array: &A, writer: &mut W, ) -> std::io::Result<()> @@ -68,6 +68,30 @@ where Ok(()) } +#[cfg(feature = "benchmark")] +pub(crate) fn write_array_data_use_raw_buffer, T>(buf: &mut [u8], array: &A) +where + T: ArrayElement, +{ + // First optimization path: write contiguous memory directly + if let Some(contiguous) = array.as_slice() { + let byte_len = size_of_val(contiguous); + let bytes = + unsafe { std::slice::from_raw_parts(contiguous.as_ptr() as *const u8, byte_len) }; + buf[..byte_len].copy_from_slice(bytes); + } + + // Fallback path: non-contiguous memory handling + let mut bytes_written = 0; + let elem_size = size_of::(); + for &element in array.iter() { + let element_bytes = + unsafe { std::slice::from_raw_parts(&element as *const T as *const _, elem_size) }; + buf[bytes_written..bytes_written + elem_size].copy_from_slice(element_bytes); + bytes_written += elem_size; + } +} + /// Marker trait for valid array element types. /// /// Implemented for primitive types that can be stored in arrays. @@ -129,7 +153,7 @@ where T: ArrayElement, { type Iter<'b> - = RowMajorIter<'b, T> + = RowMajorIter<'b, T> where Self: 'b, T: 'b; @@ -148,7 +172,9 @@ where fn as_slice(&self) -> Option<&[T]> { if self.is_c_major() { - Some(unsafe { slice::from_raw_parts(self.buf as *const T, self.buf_len / size_of::()) }) + Some(unsafe { + slice::from_raw_parts(self.buf as *const T, self.buf_len / size_of::()) + }) } else { None } @@ -258,10 +284,7 @@ where } let mut contig_stride = size_of::(); - for (dim, stride) in self.shapes.iter() - .rev() - .zip(self.strides.iter().rev()) - { + for (dim, stride) in self.shapes.iter().rev().zip(self.strides.iter().rev()) { if *dim != 1 { let s = *stride; if s.abs() != contig_stride as isize { @@ -341,7 +364,7 @@ where D: Dimension, { type Iter<'a> - = ndarray::iter::Iter<'a, T, D> + = ndarray::iter::Iter<'a, T, D> where Self: 'a, T: 'a; diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index d667b9fb..e0dd25f4 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -163,7 +163,8 @@ fn test_buffer_basic_write() -> TestResult { 2.2f64.to_le_bytes(), 3.3f64.to_le_bytes(), 4.4f64.to_le_bytes(), - ].concat() + ] + .concat() ); Ok(()) } @@ -208,7 +209,9 @@ fn test_array_length_mismatch() -> TestResult { let result = buffer.column_arr("arr1", &under_view); let err = result.unwrap_err(); assert_eq!(err.code(), ErrorCode::ArrayWriteToBufferError); - assert!(err.msg().contains("Array buffer length mismatch (actual: 8, expected: 16)")); + assert!(err + .msg() + .contains("Array buffer length mismatch (actual: 8, expected: 16)")); let over_data = vec![1.1, 2.2, 3.3]; let over_view: StridedArrayView<'_, f64> = unsafe { @@ -226,7 +229,9 @@ fn test_array_length_mismatch() -> TestResult { let result = buffer.column_arr("arr1", &over_view); let err = result.unwrap_err(); assert_eq!(err.code(), ErrorCode::ArrayWriteToBufferError); - assert!(err.msg().contains("Array buffer length mismatch (actual: 24, expected: 16)")); + assert!(err + .msg() + .contains("Array buffer length mismatch (actual: 24, expected: 16)")); Ok(()) } diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index 9f61e4cd..ede64bc6 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -35,15 +35,11 @@ use crate::tests::{ }; #[cfg(feature = "ndarray")] -use crate::{ - ingress, - ingress::ElemDataType, -}; +use crate::{ingress, ingress::ElemDataType}; use core::time::Duration; #[cfg(feature = "ndarray")] use ndarray::{arr1, arr2, ArrayD}; -use crate::ingress::ndarr; use std::{io, time::SystemTime}; #[test] From 8d30e57e22a0965b1822cb48511171239fa0e41b Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 9 Apr 2025 11:59:35 +0800 Subject: [PATCH 25/56] fix tests --- questdb-rs/src/ingress/ndarr.rs | 42 ++++++++++----------------------- questdb-rs/src/tests/ndarr.rs | 29 +++++++++++++++++++++++ questdb-rs/src/tests/sender.rs | 5 ++-- 3 files changed, 44 insertions(+), 32 deletions(-) diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 9d4ca801..27b873b7 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -273,27 +273,21 @@ where return false; } - if self.dims == 1 { - return self.strides[0] == 1 || self.shapes[0] <= 1; + if self.shapes.iter().any(|&d| d == 0) { + return true; } - - for &d in self.shapes { - if d == 0 { - return true; - } + let elem_size = size_of::() as isize; + if self.dims == 1 { + return self.strides[0] == elem_size || self.shapes[0] == 1; } - let mut contig_stride = size_of::(); - for (dim, stride) in self.shapes.iter().rev().zip(self.strides.iter().rev()) { - if *dim != 1 { - let s = *stride; - if s.abs() != contig_stride as isize { - return false; - } - contig_stride *= *dim; + let mut expected_stride = elem_size; + for (dim, &stride) in self.shapes.iter().zip(self.strides).rev() { + if *dim > 1 && stride != expected_stride { + return false; } + expected_stride *= *dim as isize; } - true } } @@ -307,14 +301,6 @@ pub struct RowMajorIter<'a, T> { total_elements: usize, } -impl RowMajorIter<'_, T> { - fn is_ptr_valid(&self, ptr: *const u8) -> bool { - let start = self.array.buf; - let end = unsafe { start.add(self.array.buf_len) }; - ptr >= start && ptr < end - } -} - impl<'a, T> Iterator for RowMajorIter<'a, T> where T: ArrayElement, @@ -336,17 +322,13 @@ where } else { self.array.shapes[dim] - 1 - coord }; - offset += (actual_coord as isize) * stride; + offset += (actual_coord as isize) * stride.abs(); } self.current_linear += 1; unsafe { let ptr = self.base_ptr.offset(offset); - if self.is_ptr_valid(ptr) { - Some(&*(ptr as *const T)) - } else { - None - } + Some(&*(ptr as *const T)) } } } diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index e0dd25f4..c179e089 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -96,6 +96,35 @@ fn test_strided_non_contiguous() -> TestResult { Ok(()) } +#[test] +fn test_negative_strides() -> TestResult { + let elem_size = size_of::(); + let data = vec![1f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]; + let view = unsafe { + StridedArrayView::::new( + 2, + &[3usize, 3usize] as *const usize, + &[-24isize, 8] as *const isize, + (data.as_ptr() as *const u8).add(48), + data.len() * elem_size, + ) + }; + let collected: Vec<_> = view.iter().copied().collect(); + assert!(view.as_slice().is_none()); + let expected_data = vec![7.0, 8.0, 9.0, 4.0, 5.0, 6.0, 1.0, 2.0, 3.0]; + assert_eq!(collected, expected_data); + let mut buffer = Vec::new(); + write_array_data(&view, &mut buffer)?; + let expected_bytes = unsafe { + std::slice::from_raw_parts( + expected_data.as_ptr() as *const u8, + expected_data.len() * elem_size, + ) + }; + assert_eq!(buffer, expected_bytes); + Ok(()) +} + #[test] fn test_basic_edge_cases() { // empty array diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index ede64bc6..f475b2fd 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -40,6 +40,7 @@ use core::time::Duration; #[cfg(feature = "ndarray")] use ndarray::{arr1, arr2, ArrayD}; +use crate::ingress::ndarr::write_array_data; use std::{io, time::SystemTime}; #[test] @@ -121,7 +122,7 @@ fn test_array_basic() -> TestResult { ] .concat(); let mut array_data2d = vec![0u8; 4 * size_of::()]; - ndarr::write_array_data(&array_2d.view(), &mut &mut array_data2d[0..])?; + write_array_data(&array_2d.view(), &mut &mut array_data2d[0..])?; let array_header3d = &[ &[b'='][..], @@ -134,7 +135,7 @@ fn test_array_basic() -> TestResult { ] .concat(); let mut array_data3d = vec![0u8; 24 * size_of::()]; - ndarr::write_array_data(&array_3d.view(), &mut &mut array_data3d[0..])?; + write_array_data(&array_3d.view(), &mut &mut array_data3d[0..])?; let exp = &[ "my_table,device=A001 f1=25.5".as_bytes(), From 36203a6c5ef1b18f104257ccb32c3ba45ba078df Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 9 Apr 2025 16:18:10 +0800 Subject: [PATCH 26/56] better memory copy. --- questdb-rs/src/ingress/ndarr.rs | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 27b873b7..325435d9 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -76,19 +76,25 @@ where // First optimization path: write contiguous memory directly if let Some(contiguous) = array.as_slice() { let byte_len = size_of_val(contiguous); - let bytes = - unsafe { std::slice::from_raw_parts(contiguous.as_ptr() as *const u8, byte_len) }; - buf[..byte_len].copy_from_slice(bytes); + unsafe { + ptr::copy_nonoverlapping( + contiguous.as_ptr() as *const u8, + buf.as_mut_ptr(), + byte_len, + ) + } } // Fallback path: non-contiguous memory handling - let mut bytes_written = 0; let elem_size = size_of::(); - for &element in array.iter() { - let element_bytes = - unsafe { std::slice::from_raw_parts(&element as *const T as *const _, elem_size) }; - buf[bytes_written..bytes_written + elem_size].copy_from_slice(element_bytes); - bytes_written += elem_size; + for (i, &element) in array.iter().enumerate() { + unsafe { + ptr::copy_nonoverlapping( + &element as *const T as *const u8, + buf.as_mut_ptr().add(i * elem_size), + elem_size, + ) + } } } @@ -337,7 +343,7 @@ use crate::{error, Error}; #[cfg(feature = "ndarray")] use ndarray::{ArrayView, Axis, Dimension}; use std::io::IoSlice; -use std::slice; +use std::{ptr, slice}; #[cfg(feature = "ndarray")] impl NdArrayView for ArrayView<'_, T, D> From feaf5be25830c007386c34a1f1149d28f6dd4d0c Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 9 Apr 2025 16:22:35 +0800 Subject: [PATCH 27/56] code format --- questdb-rs/src/ingress/ndarr.rs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 325435d9..701ecb89 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -77,11 +77,7 @@ where if let Some(contiguous) = array.as_slice() { let byte_len = size_of_val(contiguous); unsafe { - ptr::copy_nonoverlapping( - contiguous.as_ptr() as *const u8, - buf.as_mut_ptr(), - byte_len, - ) + std::ptr::copy_nonoverlapping(contiguous.as_ptr() as *const u8, buf.as_mut_ptr(), byte_len) } } @@ -89,7 +85,7 @@ where let elem_size = size_of::(); for (i, &element) in array.iter().enumerate() { unsafe { - ptr::copy_nonoverlapping( + std::ptr::copy_nonoverlapping( &element as *const T as *const u8, buf.as_mut_ptr().add(i * elem_size), elem_size, @@ -279,9 +275,6 @@ where return false; } - if self.shapes.iter().any(|&d| d == 0) { - return true; - } let elem_size = size_of::() as isize; if self.dims == 1 { return self.strides[0] == elem_size || self.shapes[0] == 1; @@ -343,7 +336,7 @@ use crate::{error, Error}; #[cfg(feature = "ndarray")] use ndarray::{ArrayView, Axis, Dimension}; use std::io::IoSlice; -use std::{ptr, slice}; +use std::slice; #[cfg(feature = "ndarray")] impl NdArrayView for ArrayView<'_, T, D> From c4bed101735f9f98900ff227c45862bc337f628d Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 9 Apr 2025 17:21:46 +0800 Subject: [PATCH 28/56] make c api works. --- cpp_test/test_line_sender.cpp | 19 +++++----- examples/line_sender_c_example.c | 2 ++ include/questdb/ingress/line_sender.h | 10 +++--- include/questdb/ingress/line_sender.hpp | 12 +++---- questdb-rs-ffi/src/lib.rs | 15 ++++---- questdb-rs/src/ingress/ndarr.rs | 48 ++++++++++++------------- questdb-rs/src/tests/ndarr.rs | 46 ++++++++++++------------ questdb-rs/src/tests/sender.rs | 4 +-- 8 files changed, 79 insertions(+), 77 deletions(-) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 80cd72da..198f13cf 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -100,6 +100,7 @@ TEST_CASE("line_sender c api basics") // 3D array of doubles size_t rank = 3; uint32_t shapes[] = {2, 3, 2}; + int32_t strides[] = {48, 16, 8}; double arr_data[] = { 48123.5, 2.4, @@ -113,14 +114,16 @@ TEST_CASE("line_sender c api basics") 2.7, 48121.5, 4.3}; - CHECK(::line_sender_buffer_column_f64_arr( - buffer, - arr_name, - rank, - shapes, - reinterpret_cast(arr_data), - sizeof(arr_data), - &err)); + CHECK( + ::line_sender_buffer_column_f64_arr( + buffer, + arr_name, + rank, + shapes, + strides, + reinterpret_cast(arr_data), + sizeof(arr_data), + &err)); CHECK(::line_sender_buffer_at_nanos(buffer, 10000000, &err)); CHECK(server.recv() == 0); CHECK(::line_sender_buffer_size(buffer) == 27); diff --git a/examples/line_sender_c_example.c b/examples/line_sender_c_example.c index 59190e53..64807fe9 100644 --- a/examples/line_sender_c_example.c +++ b/examples/line_sender_c_example.c @@ -59,6 +59,7 @@ static bool example(const char* host, const char* port) // 3D array of doubles size_t rank = 3; uint32_t shapes[] = {2, 3, 2}; + int32_t strides[] = {48, 16, 8}; double arr_data[] = { 48123.5, 2.4, @@ -77,6 +78,7 @@ static bool example(const char* host, const char* port) arr_name, rank, shapes, + strides, (const uint8_t*)arr_data, sizeof(arr_data), &err)) diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 8e2ad20e..122897d5 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -470,10 +470,8 @@ bool line_sender_buffer_column_str( * @param[in] rank Number of dimensions of the array. * @param[in] shapes Array of dimension sizes (length = `rank`). * Each element must be a positive integer. - * @param[in] data_buffer Raw bytes of the array data in little-endian format. - * Size must be `sizeof(double) * (shapes[0] * ... * - * shapes[rank-1])`. - * @param[in] data_buffer_len Byte length of the data buffer. + * @param[in] data_buffer First array element data. + * @param[in] data_buffer_len Bytes length of the array data. * @param[out] err_out Set to an error object on failure (if non-NULL). * @return true on success, false on error. */ @@ -482,8 +480,8 @@ bool line_sender_buffer_column_f64_arr( line_sender_buffer* buffer, line_sender_column_name name, size_t rank, - const size_t* shapes, - const int64_t* strides, + const uint32_t* shapes, + const int32_t* strides, const uint8_t* data_buffer, size_t data_buffer_len, line_sender_error** err_out); diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 1f2ed06e..1e9adf82 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -631,18 +631,15 @@ class line_sender_buffer * * @param name Column name. * @param shape Array dimensions (e.g., [2,3] for a 2x3 matrix). - * @param data Array data in row-major order. Size must match product of + * @param data Array first element data. Size must match product of * dimensions. - * - * @note Data is stored contiguously in row-major (C-style) order. - * Example: shape [2,3] expects 6 elements ordered as: - * [a11, a12, a13, a21, a22, a23] */ template line_sender_buffer& column( column_name_view name, - const std::vector& shapes, - const std::vector& strides, + const size_t rank, + const std::vector& shapes, + const std::vector& strides, const std::array& data) { static_assert( @@ -653,6 +650,7 @@ class line_sender_buffer ::line_sender_buffer_column_f64_arr, _impl, name._impl, + rank, shapes.data(), strides.data(), reinterpret_cast(data.data()), diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 26e8a760..8f28914b 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -828,7 +828,8 @@ pub unsafe extern "C" fn line_sender_buffer_column_str( /// @param[in] name Column name. /// @param[in] rank Array dims. /// @param[in] shape Array shapes. -/// @param[in] data_buffer Array data memory ptr. +/// @param[in] strides Array strides. +/// @param[in] data_buffer Array **first element** data memory ptr. /// @param[in] data_buffer_len Array data memory length. /// @param[out] err_out Set on error. /// # Safety @@ -840,18 +841,18 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( buffer: *mut line_sender_buffer, name: line_sender_column_name, rank: size_t, - shapes: *const size_t, // C array of shapes - strides: *const i64, // C array of strides - data_buffer: *const u8, // Raw array data - data_buffer_len: size_t, // Total bytes length + shape: *const u32, + strides: *const i32, + data_buffer: *const u8, + data_buffer_len: size_t, err_out: *mut *mut line_sender_error, ) -> bool { let buffer = unwrap_buffer_mut(buffer); let name = name.as_name(); let view = ingress::StridedArrayView::::new( rank, - shapes, - strides as *const isize, + shape, + strides, data_buffer, data_buffer_len, ); diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 701ecb89..4f619358 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -143,8 +143,8 @@ impl ArrayElement for f64 { #[derive(Debug)] pub struct StridedArrayView<'a, T> { dims: usize, - shapes: &'a [usize], - strides: &'a [isize], + shape: &'a [u32], + strides: &'a [i32], buf_len: usize, buf: *const u8, _marker: std::marker::PhantomData, @@ -169,7 +169,7 @@ where return None; } - Some(self.shapes[index]) + Some(self.shape[index] as usize) } fn as_slice(&self) -> Option<&[T]> { @@ -185,7 +185,7 @@ where fn iter(&self) -> Self::Iter<'_> { let mut dim_products = Vec::with_capacity(self.dims); let mut product = 1; - for &dim in self.shapes.iter().rev() { + for &dim in self.shape.iter().rev() { dim_products.push(product); product *= dim; } @@ -198,8 +198,8 @@ where .enumerate() .fold(self.buf, |ptr, (dim, &stride)| { if stride < 0 { - let dim_size = self.shapes[dim] as isize; - unsafe { ptr.offset(stride * (dim_size - 1)) } + let dim_size = self.shape[dim] as isize; + unsafe { ptr.offset(stride as isize * (dim_size - 1)) } } else { ptr } @@ -209,15 +209,15 @@ where array: self, dim_products, current_linear: 0, - total_elements: self.shapes.iter().product(), + total_elements: self.shape.iter().product(), } } fn check_data_buf(&self) -> Result { let mut size = size_of::(); for i in 0..self.dims { - let d = self.shapes[i]; - size = size.checked_mul(d).ok_or(error::fmt!( + let d = self.shape[i]; + size = size.checked_mul(d as usize).ok_or(error::fmt!( ArrayViewError, "Array total elem size overflow" ))? @@ -252,16 +252,16 @@ where /// - Strides are measured in bytes (not elements) pub unsafe fn new( dims: usize, - shapes: *const usize, - strides: *const isize, + shape: *const u32, + strides: *const i32, data: *const u8, data_len: usize, ) -> Self { - let shapes = slice::from_raw_parts(shapes, dims); + let shapes = slice::from_raw_parts(shape, dims); let strides = slice::from_raw_parts(strides, dims); Self { dims, - shapes, + shape: shapes, strides, buf_len: data_len, buf: data, @@ -275,17 +275,17 @@ where return false; } - let elem_size = size_of::() as isize; + let elem_size = size_of::() as i32; if self.dims == 1 { - return self.strides[0] == elem_size || self.shapes[0] == 1; + return self.strides[0] == elem_size || self.shape[0] == 1; } let mut expected_stride = elem_size; - for (dim, &stride) in self.shapes.iter().zip(self.strides).rev() { + for (dim, &stride) in self.shape.iter().zip(self.strides).rev() { if *dim > 1 && stride != expected_stride { return false; } - expected_stride *= *dim as isize; + expected_stride *= *dim as i32; } true } @@ -295,9 +295,9 @@ where pub struct RowMajorIter<'a, T> { base_ptr: *const u8, array: &'a StridedArrayView<'a, T>, - dim_products: Vec, - current_linear: usize, - total_elements: usize, + dim_products: Vec, + current_linear: u32, + total_elements: u32, } impl<'a, T> Iterator for RowMajorIter<'a, T> @@ -310,7 +310,7 @@ where return None; } let mut remaining_index = self.current_linear; - let mut offset = 0isize; + let mut offset = 0; for (dim, &dim_factor) in self.dim_products.iter().enumerate() { let coord = remaining_index / dim_factor; @@ -319,14 +319,14 @@ where let actual_coord = if stride >= 0 { coord } else { - self.array.shapes[dim] - 1 - coord + self.array.shape[dim] - 1 - coord }; - offset += (actual_coord as isize) * stride.abs(); + offset += actual_coord * stride.unsigned_abs(); } self.current_linear += 1; unsafe { - let ptr = self.base_ptr.offset(offset); + let ptr = self.base_ptr.offset(offset as isize); Some(&*(ptr as *const T)) } } diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index c179e089..308c5131 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -33,10 +33,10 @@ fn to_bytes(data: &[T]) -> Vec { fn test_strided_array_view() -> TestResult { // contiguous layout let test_data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; - let shapes = [2, 3]; + let shapes = [2u32, 3]; let strides = [ - (shapes[1] * size_of::()) as isize, - size_of::() as isize, + (shapes[1] * size_of::() as u32) as i32, + size_of::() as i32, ]; let array = unsafe { StridedArrayView::::new( @@ -62,10 +62,10 @@ fn test_strided_array_view() -> TestResult { #[test] fn test_strided_non_contiguous() -> TestResult { - let elem_size = size_of::(); + let elem_size = size_of::() as i32; let col_major_data = vec![1.0, 3.0, 5.0, 2.0, 4.0, 6.0]; - let shapes = [3, 2]; - let strides = [elem_size as isize, (shapes[0] * elem_size) as isize]; + let shapes = [3u32, 2]; + let strides = [elem_size, shapes[0] as i32 * elem_size]; let array_view: StridedArrayView<'_, f64> = unsafe { StridedArrayView::new( @@ -73,7 +73,7 @@ fn test_strided_non_contiguous() -> TestResult { shapes.as_ptr(), strides.as_ptr(), col_major_data.as_ptr() as *const u8, - col_major_data.len() * elem_size, + col_major_data.len() * elem_size as usize, ) }; @@ -89,7 +89,7 @@ fn test_strided_non_contiguous() -> TestResult { let expected_bytes = unsafe { std::slice::from_raw_parts( expected_data.as_ptr() as *const u8, - expected_data.len() * elem_size, + expected_data.len() * elem_size as usize, ) }; assert_eq!(buffer, expected_bytes); @@ -103,8 +103,8 @@ fn test_negative_strides() -> TestResult { let view = unsafe { StridedArrayView::::new( 2, - &[3usize, 3usize] as *const usize, - &[-24isize, 8] as *const isize, + &[3u32, 3] as *const u32, + &[-24i32, 8] as *const i32, (data.as_ptr() as *const u8).add(48), data.len() * elem_size, ) @@ -128,7 +128,7 @@ fn test_negative_strides() -> TestResult { #[test] fn test_basic_edge_cases() { // empty array - let elem_size = std::mem::size_of::(); + let elem_size = std::mem::size_of::() as i32; let empty_view: StridedArrayView<'_, f64> = unsafe { StridedArrayView::new(2, [0, 0].as_ptr(), [0, 0].as_ptr(), ptr::null(), 0) }; assert_eq!(empty_view.ndim(), 2); @@ -140,10 +140,10 @@ fn test_basic_edge_cases() { let single_view: StridedArrayView<'_, f64> = unsafe { StridedArrayView::new( 2, - [1, 1].as_ptr(), // 1x1 矩阵 - [elem_size as isize, elem_size as isize].as_ptr(), + [1, 1].as_ptr(), + [elem_size, elem_size].as_ptr(), single_data.as_ptr() as *const u8, - elem_size, + elem_size as usize, ) }; let mut buf = vec![]; @@ -153,16 +153,16 @@ fn test_basic_edge_cases() { #[test] fn test_buffer_basic_write() -> TestResult { - let elem_size = std::mem::size_of::(); + let elem_size = std::mem::size_of::() as i32; let test_data = vec![1.1, 2.2, 3.3, 4.4]; let array_view: StridedArrayView<'_, f64> = unsafe { StridedArrayView::new( 2, [2, 2].as_ptr(), - [2 * elem_size as isize, elem_size as isize].as_ptr(), + [2 * elem_size, elem_size].as_ptr(), test_data.as_ptr() as *const u8, - test_data.len() * elem_size, + test_data.len() * elem_size as usize, ) }; let mut buffer = Buffer::new(); @@ -203,7 +203,7 @@ fn test_size_overflow() -> TestResult { let overflow_view = unsafe { StridedArrayView::::new( 2, - [usize::MAX, usize::MAX].as_ptr(), + [u32::MAX, u32::MAX].as_ptr(), [8, 8].as_ptr(), ptr::null(), 0, @@ -221,15 +221,15 @@ fn test_size_overflow() -> TestResult { #[test] fn test_array_length_mismatch() -> TestResult { - let elem_size = std::mem::size_of::(); + let elem_size = std::mem::size_of::() as i32; let under_data = vec![1.1]; let under_view: StridedArrayView<'_, f64> = unsafe { StridedArrayView::new( 2, [1, 2].as_ptr(), - [elem_size as isize, elem_size as isize].as_ptr(), + [elem_size, elem_size].as_ptr(), under_data.as_ptr() as *const u8, - under_data.len() * elem_size, + under_data.len() * elem_size as usize, ) }; @@ -247,9 +247,9 @@ fn test_array_length_mismatch() -> TestResult { StridedArrayView::new( 2, [1, 2].as_ptr(), - [elem_size as isize, elem_size as isize].as_ptr(), + [elem_size, elem_size].as_ptr(), over_data.as_ptr() as *const u8, - over_data.len() * elem_size, + over_data.len() * elem_size as usize, ) }; diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index f475b2fd..3edc0eb8 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -34,13 +34,13 @@ use crate::tests::{ TestResult, }; +#[cfg(feature = "ndarray")] +use crate::ingress::ndarr::write_array_data; #[cfg(feature = "ndarray")] use crate::{ingress, ingress::ElemDataType}; use core::time::Duration; #[cfg(feature = "ndarray")] use ndarray::{arr1, arr2, ArrayD}; - -use crate::ingress::ndarr::write_array_data; use std::{io, time::SystemTime}; #[test] From f3011e120b477a10ec6be63a1aeac2d66da1f3f8 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 9 Apr 2025 17:22:21 +0800 Subject: [PATCH 29/56] make c api works. --- examples/line_sender_c_example_auth.c | 2 ++ examples/line_sender_c_example_auth_tls.c | 2 ++ examples/line_sender_c_example_from_conf.c | 2 ++ examples/line_sender_c_example_from_env.c | 2 ++ examples/line_sender_c_example_http.c | 2 ++ examples/line_sender_c_example_tls_ca.c | 2 ++ 6 files changed, 12 insertions(+) diff --git a/examples/line_sender_c_example_auth.c b/examples/line_sender_c_example_auth.c index 1564da88..c6905a9e 100644 --- a/examples/line_sender_c_example_auth.c +++ b/examples/line_sender_c_example_auth.c @@ -64,6 +64,7 @@ static bool example(const char* host, const char* port) // 3D array of doubles size_t rank = 3; uint32_t shapes[] = {2, 3, 2}; + int32_t strides[] = {48, 16, 8}; double arr_data[] = { 48123.5, 2.4, @@ -82,6 +83,7 @@ static bool example(const char* host, const char* port) arr_name, rank, shapes, + strides, (const uint8_t*)arr_data, sizeof(arr_data), &err)) diff --git a/examples/line_sender_c_example_auth_tls.c b/examples/line_sender_c_example_auth_tls.c index 523e91d1..16abb263 100644 --- a/examples/line_sender_c_example_auth_tls.c +++ b/examples/line_sender_c_example_auth_tls.c @@ -64,6 +64,7 @@ static bool example(const char* host, const char* port) // 3D array of doubles size_t rank = 3; uint32_t shapes[] = {2, 3, 2}; + int32_t strides[] = {48, 16, 8}; double arr_data[] = { 48123.5, 2.4, @@ -82,6 +83,7 @@ static bool example(const char* host, const char* port) arr_name, rank, shapes, + strides, (const uint8_t*)arr_data, sizeof(arr_data), &err)) diff --git a/examples/line_sender_c_example_from_conf.c b/examples/line_sender_c_example_from_conf.c index ec3a217f..6b077e58 100644 --- a/examples/line_sender_c_example_from_conf.c +++ b/examples/line_sender_c_example_from_conf.c @@ -48,6 +48,7 @@ int main(int argc, const char* argv[]) // 3D array of doubles size_t rank = 3; uint32_t shapes[] = {2, 3, 2}; + int32_t strides[] = {48, 16, 8}; double arr_data[] = { 48123.5, 2.4, @@ -66,6 +67,7 @@ int main(int argc, const char* argv[]) arr_name, rank, shapes, + strides, (const uint8_t*)arr_data, sizeof(arr_data), &err)) diff --git a/examples/line_sender_c_example_from_env.c b/examples/line_sender_c_example_from_env.c index f949b843..5121bf70 100644 --- a/examples/line_sender_c_example_from_env.c +++ b/examples/line_sender_c_example_from_env.c @@ -47,6 +47,7 @@ int main(int argc, const char* argv[]) // 3D array of doubles size_t rank = 3; uint32_t shapes[] = {2, 3, 2}; + int32_t strides[] = {48, 16, 8}; double arr_data[] = { 48123.5, 2.4, @@ -65,6 +66,7 @@ int main(int argc, const char* argv[]) arr_name, rank, shapes, + strides, (const uint8_t*)arr_data, sizeof(arr_data), &err)) diff --git a/examples/line_sender_c_example_http.c b/examples/line_sender_c_example_http.c index e4dcf662..7d504a7e 100644 --- a/examples/line_sender_c_example_http.c +++ b/examples/line_sender_c_example_http.c @@ -58,6 +58,7 @@ static bool example(const char* host, const char* port) // 3D array of doubles size_t rank = 3; uint32_t shapes[] = {2, 3, 2}; + int32_t strides[] = {48, 16, 8}; double arr_data[] = { 48123.5, 2.4, @@ -76,6 +77,7 @@ static bool example(const char* host, const char* port) arr_name, rank, shapes, + strides, (const uint8_t*)arr_data, sizeof(arr_data), &err)) diff --git a/examples/line_sender_c_example_tls_ca.c b/examples/line_sender_c_example_tls_ca.c index d52901fe..3b8fb48a 100644 --- a/examples/line_sender_c_example_tls_ca.c +++ b/examples/line_sender_c_example_tls_ca.c @@ -65,6 +65,7 @@ static bool example(const char* ca_path, const char* host, const char* port) // 3D array of doubles size_t rank = 3; uint32_t shapes[] = {2, 3, 2}; + int32_t strides[] = {48, 16, 8}; double arr_data[] = { 48123.5, 2.4, @@ -83,6 +84,7 @@ static bool example(const char* ca_path, const char* host, const char* port) arr_name, rank, shapes, + strides, (const uint8_t*)arr_data, sizeof(arr_data), &err)) From 70eaa24a8f68dccfa1aa4b339a2bfaf22412f859 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 9 Apr 2025 17:24:35 +0800 Subject: [PATCH 30/56] code format --- questdb-rs-ffi/src/lib.rs | 9 ++------- questdb-rs/src/ingress/ndarr.rs | 6 +++++- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 8f28914b..b877e31c 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -849,13 +849,8 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr( ) -> bool { let buffer = unwrap_buffer_mut(buffer); let name = name.as_name(); - let view = ingress::StridedArrayView::::new( - rank, - shape, - strides, - data_buffer, - data_buffer_len, - ); + let view = + ingress::StridedArrayView::::new(rank, shape, strides, data_buffer, data_buffer_len); bubble_err_to_c!( err_out, buffer.column_arr::, ingress::StridedArrayView<'_, f64>, f64>(name, &view) diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 4f619358..61626171 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -77,7 +77,11 @@ where if let Some(contiguous) = array.as_slice() { let byte_len = size_of_val(contiguous); unsafe { - std::ptr::copy_nonoverlapping(contiguous.as_ptr() as *const u8, buf.as_mut_ptr(), byte_len) + std::ptr::copy_nonoverlapping( + contiguous.as_ptr() as *const u8, + buf.as_mut_ptr(), + byte_len, + ) } } From 526e8ae7fece238e0e83b406ebb13869d478cb5c Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 9 Apr 2025 21:11:59 +0800 Subject: [PATCH 31/56] make c++ tests work. --- cpp_test/mock_server.cpp | 57 +++++++++++++---- cpp_test/mock_server.hpp | 8 ++- cpp_test/test_line_sender.cpp | 116 ++++++++++++++++++++-------------- questdb-rs/src/tests/mock.rs | 2 +- 4 files changed, 120 insertions(+), 63 deletions(-) diff --git a/cpp_test/mock_server.cpp b/cpp_test/mock_server.cpp index 0e8a1e85..bfc67e0f 100644 --- a/cpp_test/mock_server.cpp +++ b/cpp_test/mock_server.cpp @@ -190,14 +190,21 @@ bool mock_server::wait_for_data(std::optional wait_timeout_sec) return !!count; } +int32_t bytes_to_int32_le(const std::byte* bytes) +{ + return static_cast( + (bytes[0] << 0) | (bytes[1] << 8) | (bytes[2] << 16) | + (bytes[3] << 24)); +} + size_t mock_server::recv(double wait_timeout_sec) { if (!wait_for_data(wait_timeout_sec)) return 0; - char chunk[1024]; + std::byte chunk[1024]; size_t chunk_len{sizeof(chunk)}; - std::vector accum; + std::vector accum; for (;;) { wait_for_data(); @@ -209,24 +216,52 @@ size_t mock_server::recv(double wait_timeout_sec) accum.insert(accum.end(), chunk, chunk + u_count); if (accum.size() < 2) continue; - if ((accum[accum.size() - 1] == '\n') && - (accum[accum.size() - 2] != '\\')) + if ((accum[accum.size() - 1] == std::byte('\n')) && + (accum[accum.size() - 2] != std::byte('\\'))) break; } size_t received_count{0}; - const char* head{&accum[0]}; - for (size_t index = 1; index < accum.size(); ++index) + const std::byte* head{&accum[0]}; + size_t index{1}; + while (index < accum.size()) { - const char& last = accum[index]; - const char& prev = accum[index - 1]; - if ((last == '\n') && (prev != '\\')) + const std::byte& last = accum[index]; + const std::byte& prev = accum[index - 1]; + if (last == std::byte('=') && prev == std::byte('=')) + { + index++; + std::byte& binary_type = accum[index]; + if (binary_type == std::byte(16)) // DOUBLE_BINARY_FORMAT_TYPE + index += sizeof(double) + 1; + else if (binary_type == std::byte(14)) // ARRAY_BINARY_FORMAT_TYPE + { + index++; + const std::byte& array_elem_type = accum[index]; + if (array_elem_type == std::byte(10)) + { + index++; + const size_t dims = size_t(accum[index]); + index++; + size_t data_size{sizeof(double)}; + for (size_t i = 0; i < dims; i++) + { + data_size *= bytes_to_int32_le(&accum[index]); + index += sizeof(int32_t); + } + index += data_size; + } + } + continue; + } + else if ((last == std::byte('\n')) && (prev != std::byte('\\'))) { - const char* tail{&last + 1}; - _msgs.emplace_back(head, tail - head); + const std::byte* tail{&last + 1}; + _msgs.emplace_back(head, tail); head = tail; ++received_count; } + index++; } return received_count; } diff --git a/cpp_test/mock_server.hpp b/cpp_test/mock_server.hpp index ba66efb0..dff8ff4e 100644 --- a/cpp_test/mock_server.hpp +++ b/cpp_test/mock_server.hpp @@ -31,6 +31,7 @@ #include #include "build_env.h" +#include "questdb/ingress/line_sender.hpp" #if defined(PLATFORM_UNIX) typedef int socketfd_t; @@ -60,9 +61,10 @@ class mock_server size_t recv(double wait_timeout_sec = 0.1); - const std::vector& msgs() const + const buffer_view msgs(size_t index) const { - return _msgs; + assert(index < _msgs.size()); + return buffer_view{_msgs[index].data(), _msgs[index].size()}; } void close(); @@ -75,7 +77,7 @@ class mock_server socketfd_t _listen_fd; socketfd_t _conn_fd; uint16_t _port; - std::vector _msgs; + std::vector> _msgs; }; } // namespace questdb::ingress::test diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 198f13cf..7036d268 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -56,6 +56,60 @@ class on_scope_exit F _f; }; +#if __cplusplus >= 202002L +template +bool operator==(std::span lhs, const char (&rhs)[N]) +{ + constexpr size_t bytelen = N - 1; // Exclude null terminator + const std::span rhs_span{ + reinterpret_cast(rhs), bytelen}; + return lhs.size() == bytelen && std::ranges::equal(lhs, rhs_span); +} + +bool operator==(std::span lhs, const std::string& rhs) +{ + const std::span rhs_span{ + reinterpret_cast(rhs.data()), rhs.size()}; + return lhs.size() == rhs.size() && std::ranges::equal(lhs, rhs_span); +} +#else +template +bool operator==( + const questdb::ingress::buffer_view lhs_view, const char (&rhs)[N]) +{ + constexpr size_t bytelen = N - 1; // Exclude null terminator + const questdb::ingress::buffer_view rhs_view{ + reinterpret_cast(rhs), bytelen}; + return lhs_view == rhs_view; +} + +bool operator==( + const questdb::ingress::buffer_view lhs_view, const std::string& rhs) +{ + const questdb::ingress::buffer_view rhs_view{ + reinterpret_cast(rhs.data()), rhs.size()}; + return lhs_view == rhs_view; +} +#endif + +template +void push_double_arr_to_buffer( + std::string& buffer, + std::array data, + size_t rank, + uint32_t* shapes) +{ + buffer.push_back(14); + buffer.push_back(10); + buffer.push_back(static_cast(rank)); + for (size_t i = 0; i < rank; ++i) + buffer.append( + reinterpret_cast(&shapes[i]), sizeof(uint32_t)); + buffer.append( + reinterpret_cast(data.data()), + data.size() * sizeof(double)); +} + TEST_CASE("line_sender c api basics") { questdb::ingress::test::mock_server server; @@ -96,12 +150,12 @@ TEST_CASE("line_sender c api basics") CHECK(::line_sender_buffer_symbol(buffer, t1_name, v1_utf8, &err)); CHECK(::line_sender_buffer_column_f64(buffer, f1_name, 0.5, &err)); - line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("order_book"); + line_sender_column_name arr_name = QDB_COLUMN_NAME_LITERAL("a1"); // 3D array of doubles size_t rank = 3; uint32_t shapes[] = {2, 3, 2}; int32_t strides[] = {48, 16, 8}; - double arr_data[] = { + std::array arr_data = { 48123.5, 2.4, 48124.0, @@ -121,16 +175,19 @@ TEST_CASE("line_sender c api basics") rank, shapes, strides, - reinterpret_cast(arr_data), + reinterpret_cast(arr_data.data()), sizeof(arr_data), &err)); CHECK(::line_sender_buffer_at_nanos(buffer, 10000000, &err)); CHECK(server.recv() == 0); - CHECK(::line_sender_buffer_size(buffer) == 27); + CHECK(::line_sender_buffer_size(buffer) == 143); CHECK(::line_sender_flush(sender, buffer, &err)); ::line_sender_buffer_free(buffer); CHECK(server.recv() == 1); - CHECK(server.msgs().front() == "test,t1=v1 f1=0.5 10000000\n"); + std::string expect{"test,t1=v1 f1=0.5,a1=="}; + push_double_arr_to_buffer(expect, arr_data, 3, shapes); + expect.append(" 10000000\n"); + CHECK(server.msgs(0) == expect); } TEST_CASE("Opts service API tests") @@ -189,7 +246,7 @@ TEST_CASE("line_sender c++ api basics") CHECK(buffer.size() == 31); sender.flush(buffer); CHECK(server.recv() == 1); - CHECK(server.msgs().front() == "test,t1=v1,t2= f1=0.5 10000000\n"); + CHECK(server.msgs(0) == "test,t1=v1,t2= f1=0.5 10000000\n"); } TEST_CASE("test multiple lines") @@ -226,12 +283,11 @@ TEST_CASE("test multiple lines") sender.flush(buffer); CHECK(server.recv() == 2); CHECK( - server.msgs()[0] == + server.msgs(0) == ("metric1,t1=val1,t2=val2 f1=t,f2=12345i," "f3=10.75,f4=\"val3\",f5=\"val4\",f6=\"val5\" 111222233333\n")); CHECK( - server.msgs()[1] == - "metric1,tag3=value\\ 3,tag\\ 4=value:4 field5=f\n"); + server.msgs(1) == "metric1,tag3=value\\ 3,tag\\ 4=value:4 field5=f\n"); } TEST_CASE("State machine testing -- flush without data.") @@ -272,7 +328,7 @@ TEST_CASE("One symbol only - flush before server accept") // but the server hasn't actually accepted the client connection yet. server.accept(); CHECK(server.recv() == 1); - CHECK(server.msgs()[0] == "test,t1=v1\n"); + CHECK(server.msgs(0) == "test,t1=v1\n"); } TEST_CASE("One column only - server.accept() after flush, before close") @@ -292,7 +348,7 @@ TEST_CASE("One column only - server.accept() after flush, before close") sender.close(); CHECK(server.recv() == 1); - CHECK(server.msgs()[0] == "test t1=\"v1\"\n"); + CHECK(server.msgs(0) == "test t1=\"v1\"\n"); } TEST_CASE("Symbol after column") @@ -415,42 +471,6 @@ TEST_CASE("Validation of bad chars in key names.") } } -#if __cplusplus >= 202002L -template -bool operator==(std::span lhs, const char (&rhs)[N]) -{ - constexpr size_t bytelen = N - 1; // Exclude null terminator - const std::span rhs_span{ - reinterpret_cast(rhs), bytelen}; - return lhs.size() == bytelen && std::ranges::equal(lhs, rhs_span); -} - -bool operator==(std::span lhs, const std::string& rhs) -{ - const std::span rhs_span{ - reinterpret_cast(rhs.data()), rhs.size()}; - return lhs.size() == rhs.size() && std::ranges::equal(lhs, rhs_span); -} -#else -template -bool operator==( - const questdb::ingress::buffer_view lhs_view, const char (&rhs)[N]) -{ - constexpr size_t bytelen = N - 1; // Exclude null terminator - const questdb::ingress::buffer_view rhs_view{ - reinterpret_cast(rhs), bytelen}; - return lhs_view == rhs_view; -} - -bool operator==( - const questdb::ingress::buffer_view lhs_view, const std::string& rhs) -{ - const questdb::ingress::buffer_view rhs_view{ - reinterpret_cast(rhs.data()), rhs.size()}; - return lhs_view == rhs_view; -} -#endif - TEST_CASE("Buffer move and copy ctor testing") { const size_t init_buf_size = 128; @@ -744,7 +764,7 @@ TEST_CASE("Test timestamp column.") sender.close(); CHECK(server.recv() == 1); - CHECK(server.msgs()[0] == exp); + CHECK(server.msgs(0) == exp); } TEST_CASE("test timestamp_micros and timestamp_nanos::now()") diff --git a/questdb-rs/src/tests/mock.rs b/questdb-rs/src/tests/mock.rs index 1d1f438a..03fcf527 100644 --- a/questdb-rs/src/tests/mock.rs +++ b/questdb-rs/src/tests/mock.rs @@ -533,7 +533,7 @@ impl MockServer { // calc binary length let binary_type = accum[index]; if binary_type == ingress::DOUBLE_BINARY_FORMAT_TYPE { - index += size_of::(); + index += size_of::() + 1; } else if binary_type == ingress::ARRAY_BINARY_FORMAT_TYPE { index += 1; let element_type = match ElemDataType::try_from(accum[index]) { From 9c16d74fd85d017322dd0a6e5ba5fab4a6abde6d Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 9 Apr 2025 21:28:59 +0800 Subject: [PATCH 32/56] fix compile error. --- cpp_test/mock_server.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp_test/mock_server.hpp b/cpp_test/mock_server.hpp index dff8ff4e..ea6009a2 100644 --- a/cpp_test/mock_server.hpp +++ b/cpp_test/mock_server.hpp @@ -24,8 +24,8 @@ #pragma once +#include #include -#include #include #include #include From 5dae9d0e3ea951f05ab6c3bfc5cc41e1c7c88819 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 9 Apr 2025 21:48:23 +0800 Subject: [PATCH 33/56] make c++20 build happy --- cpp_test/mock_server.hpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/cpp_test/mock_server.hpp b/cpp_test/mock_server.hpp index ea6009a2..8adb6ad7 100644 --- a/cpp_test/mock_server.hpp +++ b/cpp_test/mock_server.hpp @@ -29,9 +29,12 @@ #include #include #include - #include "build_env.h" -#include "questdb/ingress/line_sender.hpp" +#if __cplusplus < 202002L +# include "questdb/ingress/line_sender.hpp" +#else +# include +#endif #if defined(PLATFORM_UNIX) typedef int socketfd_t; @@ -61,10 +64,14 @@ class mock_server size_t recv(double wait_timeout_sec = 0.1); - const buffer_view msgs(size_t index) const +#if __cplusplus >= 202002L + using buffer_view = std::span; +#endif + + buffer_view msgs(size_t index) const { assert(index < _msgs.size()); - return buffer_view{_msgs[index].data(), _msgs[index].size()}; + return {_msgs[index].data(), _msgs[index].size()}; } void close(); From ea522934d56717694af69950647b476fc9738b1b Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 9 Apr 2025 22:12:20 +0800 Subject: [PATCH 34/56] code format --- questdb-rs/src/ingress/ndarr.rs | 4 ++-- questdb-rs/src/tests/ndarr.rs | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 61626171..ff81d034 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -21,7 +21,7 @@ where /// Iterator element type is `&T`. fn iter(&self) -> Self::Iter<'_>; - /// Validates the data buffer size of array is consistency with array [`Shapes`]. + /// Validates the data buffer size of array is consistency with array shapes. /// /// # Returns /// - `Ok(usize)`: Expected buffer size in bytes if valid @@ -111,7 +111,7 @@ pub trait ArrayElement: Copy + 'static { } /// Defines binary format identifiers for array element types compatible with -/// QuestDB's [`ColumnType`]: https://github.com/questdb/questdb/blob/e1853db56ae586d923ca77de01a487cad44093b9/core/src/main/java/io/questdb/cairo/ColumnType.java#L67-L89. +/// QuestDB's ColumnType: . #[repr(u8)] #[derive(Debug, PartialEq, Clone, Copy)] pub enum ElemDataType { diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index 308c5131..99c92222 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -63,7 +63,7 @@ fn test_strided_array_view() -> TestResult { #[test] fn test_strided_non_contiguous() -> TestResult { let elem_size = size_of::() as i32; - let col_major_data = vec![1.0, 3.0, 5.0, 2.0, 4.0, 6.0]; + let col_major_data = [1.0, 3.0, 5.0, 2.0, 4.0, 6.0]; let shapes = [3u32, 2]; let strides = [elem_size, shapes[0] as i32 * elem_size]; @@ -85,7 +85,7 @@ fn test_strided_non_contiguous() -> TestResult { let mut buffer = Vec::new(); write_array_data(&array_view, &mut buffer)?; - let expected_data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; + let expected_data = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]; let expected_bytes = unsafe { std::slice::from_raw_parts( expected_data.as_ptr() as *const u8, @@ -99,7 +99,7 @@ fn test_strided_non_contiguous() -> TestResult { #[test] fn test_negative_strides() -> TestResult { let elem_size = size_of::(); - let data = vec![1f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]; + let data = [1f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]; let view = unsafe { StridedArrayView::::new( 2, @@ -136,7 +136,7 @@ fn test_basic_edge_cases() { assert_eq!(empty_view.dim(1), Some(0)); // single element array - let single_data = vec![42.0]; + let single_data = [42.0]; let single_view: StridedArrayView<'_, f64> = unsafe { StridedArrayView::new( 2, @@ -155,7 +155,7 @@ fn test_basic_edge_cases() { fn test_buffer_basic_write() -> TestResult { let elem_size = std::mem::size_of::() as i32; - let test_data = vec![1.1, 2.2, 3.3, 4.4]; + let test_data = [1.1, 2.2, 3.3, 4.4]; let array_view: StridedArrayView<'_, f64> = unsafe { StridedArrayView::new( 2, @@ -221,8 +221,8 @@ fn test_size_overflow() -> TestResult { #[test] fn test_array_length_mismatch() -> TestResult { - let elem_size = std::mem::size_of::() as i32; - let under_data = vec![1.1]; + let elem_size = size_of::() as i32; + let under_data = [1.1]; let under_view: StridedArrayView<'_, f64> = unsafe { StridedArrayView::new( 2, @@ -242,7 +242,7 @@ fn test_array_length_mismatch() -> TestResult { .msg() .contains("Array buffer length mismatch (actual: 8, expected: 16)")); - let over_data = vec![1.1, 2.2, 3.3]; + let over_data = [1.1, 2.2, 3.3]; let over_view: StridedArrayView<'_, f64> = unsafe { StridedArrayView::new( 2, From 193b887af056a4aab2ea264eee562964a7dc61bf Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 10 Apr 2025 22:02:20 +0800 Subject: [PATCH 35/56] fix cpp problem on windows. --- cpp_test/mock_server.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cpp_test/mock_server.cpp b/cpp_test/mock_server.cpp index bfc67e0f..f2bd8bc5 100644 --- a/cpp_test/mock_server.cpp +++ b/cpp_test/mock_server.cpp @@ -208,8 +208,11 @@ size_t mock_server::recv(double wait_timeout_sec) for (;;) { wait_for_data(); - sock_ssize_t count = - ::recv(_conn_fd, &chunk[0], static_cast(chunk_len), 0); + sock_ssize_t count = ::recv( + _conn_fd, + reinterpret_cast(&chunk[0]), + static_cast(chunk_len), + 0); if (count == -1) throw std::runtime_error{"Bad `recv()`."}; const size_t u_count = static_cast(count); From 523a855e2f6928b3002150cb33e37c35657372c7 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Thu, 10 Apr 2025 17:36:27 +0100 Subject: [PATCH 36/56] lint issues --- questdb-rs/src/ingress/mod.rs | 12 ++++++------ questdb-rs/src/tests/ndarr.rs | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 2374a16d..fc376a6e 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -506,12 +506,12 @@ impl BufferState { /// * A row always starts with [`table`](Buffer::table). /// * A row must contain at least one [`symbol`](Buffer::symbol) or /// column ( -/// [`column_bool`](Buffer::column_bool), -/// [`column_i64`](Buffer::column_i64), -/// [`column_f64`](Buffer::column_f64), -/// [`column_str`](Buffer::column_str), -/// [`column_arr`](Buffer::column_arr), -/// [`column_ts`](Buffer::column_ts)). +/// [`column_bool`](Buffer::column_bool), +/// [`column_i64`](Buffer::column_i64), +/// [`column_f64`](Buffer::column_f64), +/// [`column_str`](Buffer::column_str), +/// [`column_arr`](Buffer::column_arr), +/// [`column_ts`](Buffer::column_ts)). /// * Symbols must appear before columns. /// * A row must be terminated with either [`at`](Buffer::at) or /// [`at_now`](Buffer::at_now). diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index 99c92222..56417d10 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -375,14 +375,14 @@ fn test_buffer_ndarray_write() -> TestResult { fn test_buffer_write_ndarray_max_dimensions() -> TestResult { let mut buffer = Buffer::new(); buffer.table("nd_test")?; - let shape: Vec = iter::repeat(1).take(MAX_DIMS).collect(); + let shape: Vec = iter::repeat_n(1, MAX_DIMS).collect(); let array = ArrayD::::zeros(shape.clone()); buffer.column_arr("max_dim", &array.view())?; let data = buffer.as_bytes(); assert_eq!(data[19], MAX_DIMS as u8); // 33 dims error - let shape_invalid: Vec<_> = iter::repeat(1).take(MAX_DIMS + 1).collect(); + let shape_invalid: Vec<_> = iter::repeat_n(1, MAX_DIMS + 1).collect(); let array_invalid = ArrayD::::zeros(shape_invalid); let result = buffer.column_arr("invalid", &array_invalid.view()); assert!(result.is_err()); From a1fb92d5d2301fa157d568cbde51226b2b2326d7 Mon Sep 17 00:00:00 2001 From: victorgao Date: Mon, 14 Apr 2025 11:11:43 +0800 Subject: [PATCH 37/56] add python tests. --- system_test/questdb_line_sender.py | 55 +++++++++++++++++++++++++++++- system_test/test.py | 47 +++++++++++++++++++++++-- 2 files changed, 99 insertions(+), 3 deletions(-) diff --git a/system_test/questdb_line_sender.py b/system_test/questdb_line_sender.py index 5d4eca85..7ecd4b0c 100644 --- a/system_test/questdb_line_sender.py +++ b/system_test/questdb_line_sender.py @@ -40,6 +40,8 @@ from ast import arg import sys +import numpy + sys.dont_write_bytecode = True import pathlib @@ -53,10 +55,12 @@ c_size_t, c_char_p, c_int, + c_int32, c_int64, c_double, c_uint8, c_uint16, + c_uint32, c_uint64, c_void_p, c_ssize_t) @@ -98,6 +102,10 @@ class c_line_sender_error(ctypes.Structure): c_line_sender_opts_p = ctypes.POINTER(c_line_sender_opts) c_line_sender_error_p = ctypes.POINTER(c_line_sender_error) c_line_sender_error_p_p = ctypes.POINTER(c_line_sender_error_p) +c_int32_p = ctypes.POINTER(c_int32) +c_uint8_p = ctypes.POINTER(c_uint8) +c_uint32_p = ctypes.POINTER(c_uint32) + class c_line_sender_utf8(ctypes.Structure): _fields_ = [("len", c_size_t), ("buf", c_char_p)] @@ -107,7 +115,7 @@ class c_line_sender_table_name(ctypes.Structure): ("buf", c_char_p)] class line_sender_buffer_view(ctypes.Structure): _fields_ = [("len", c_size_t), - ("buf", ctypes.POINTER(c_uint8))] + ("buf", c_uint8_p)] c_line_sender_table_name_p = ctypes.POINTER(c_line_sender_table_name) class c_line_sender_column_name(ctypes.Structure): @@ -237,6 +245,17 @@ def set_sig(fn, restype, *argtypes): c_line_sender_column_name, c_line_sender_utf8, c_line_sender_error_p_p) + set_sig( + dll.line_sender_buffer_column_f64_arr, + c_bool, + c_line_sender_buffer_p, + c_line_sender_column_name, + c_size_t, + c_uint32_p, + c_int32_p, + c_uint8_p, + c_size_t, + c_line_sender_error_p_p) set_sig( dll.line_sender_buffer_column_ts_nanos, c_bool, @@ -627,6 +646,34 @@ def column( '`bool`, `int`, `float` or `str`.') return self + def column_f64_arr(self, name: str, + rank: int, + shapes: tuple[int, ...], + strides: tuple[int, ...], + data: c_void_p, + length: int): + def _convert_tuple(tpl: tuple[int, ...], c_type: type, name: str) -> ctypes.POINTER: + arr_type = c_type * len(tpl) + try: + return arr_type(*[c_type(v) for v in tpl]) + except OverflowError as e: + raise ValueError( + f"{name} value exceeds {c_type.__name__} range" + ) from e + + c_shapes = _convert_tuple(shapes, c_uint32, "shapes") + c_strides = _convert_tuple(strides, c_int32, "strides") + _error_wrapped_call( + _DLL.line_sender_buffer_column_f64_arr, + self._impl, + _column_name(name), + c_size_t(rank), + c_shapes, + c_strides, + ctypes.cast(data, c_uint8_p), + c_size_t(length) + ) + def at_now(self): _error_wrapped_call( _DLL.line_sender_buffer_at_now, @@ -735,6 +782,12 @@ def column( value: Union[bool, int, float, str, TimestampMicros, datetime]): self._buffer.column(name, value) return self + + def column_f64_arr( + self, name: str, + array: numpy.ndarray): + self._buffer.column_f64_arr(name, array.ndim, array.shape, array.strides, array.ctypes.data, array.nbytes) + return self def at_now(self): self._buffer.at_now() diff --git a/system_test/test.py b/system_test/test.py index c28ccd6d..b5498230 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -25,6 +25,9 @@ ################################################################################ import sys + +import numpy as np + sys.dont_write_bytecode = True import os @@ -483,6 +486,45 @@ def test_timestamp_column(self): scrubbed_dataset = [row[:-1] for row in resp['dataset']] self.assertEqual(scrubbed_dataset, exp_dataset) + def test_f64_arr_column(self): + table_name = uuid.uuid4().hex + pending = None + + array1 = np.array( + [ + [[1.1, 2.2], [3.3, 4.4]], + [[5.5, 6.6], [7.7, 8.8]] + ], + dtype=np.float64 + ) + array2 = array1.T + array3 = array1[::-1, ::-1] + + with self._mk_linesender() as sender: + (sender + .table(table_name) + .column_f64_arr('f64_arr1', array1) + .column_f64_arr('f64_arr2', array2) + .column_f64_arr('f64_arr3', array3) + .at_now()) + pending = sender.buffer.peek() + + resp = retry_check_table(table_name, log_ctx=pending) + exp_columns = [ + {'name': 'f64_arr1', 'type': 'DOUBLE[][][]'}, + {'name': 'f64_arr2', 'type': 'DOUBLE[][][]'}, + {'name': 'f64_arr3', 'type': 'DOUBLE[][][]'}, + {'name': 'timestamp', 'type': 'TIMESTAMP'} + ] + self.assertEqual(resp['columns'], exp_columns) + expected_data = [ + ['{{{1.1,2.2},{3.3,4.4}},{{5.5,6.6},{7.7,8.8}}}', + '{{{1.1,5.5},{3.3,7.7}},{{2.2,6.6},{4.4,8.8}}}', + '{{{7.7,8.8},{5.5,6.6}},{{3.3,4.4},{1.1,2.2}}}'] + ] + scrubbed_data = [row[:-1] for row in resp['dataset']] + self.assertEqual(scrubbed_data, expected_data) + def _test_example(self, bin_name, table_name, tls=False): if BUILD_MODE != qls.BuildMode.API: self.skipTest('BuildMode.API-only test') @@ -803,14 +845,15 @@ def run_with_existing(args): global QDB_FIXTURE MockFixture = namedtuple( 'MockFixture', - ('host', 'line_tcp_port', 'http_server_port', 'version', 'http')) + ('host', 'line_tcp_port', 'http_server_port', 'version', 'http', "auth")) host, line_tcp_port, http_server_port = args.existing.split(':') QDB_FIXTURE = MockFixture( host, int(line_tcp_port), int(http_server_port), (999, 999, 999), - True) + True, + False) unittest.main() From 7b75fc64e86e3d6b4d95e73b61b37c3f5a253e99 Mon Sep 17 00:00:00 2001 From: victorgao Date: Mon, 14 Apr 2025 12:11:02 +0800 Subject: [PATCH 38/56] add stridesArrayView performance benchmark. --- questdb-rs/benches/ndarr.rs | 55 +++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/questdb-rs/benches/ndarr.rs b/questdb-rs/benches/ndarr.rs index babac254..bf641cd9 100644 --- a/questdb-rs/benches/ndarr.rs +++ b/questdb-rs/benches/ndarr.rs @@ -1,6 +1,6 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; use ndarray::{Array, Array2}; -use questdb::ingress::{Buffer, ColumnName}; +use questdb::ingress::{Buffer, ColumnName, StridedArrayView}; /// run with /// ```shell @@ -65,5 +65,56 @@ fn bench_write_array_data(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, bench_write_array_data); +// bench NdArrayView and StridedArrayView write performance. +fn bench_array_view(c: &mut Criterion) { + let mut group = c.benchmark_group("write_array_view"); + let col_name = ColumnName::new("col1").unwrap(); + let array: Array2 = Array::ones((1000, 1000)); + let transposed_view = array.t(); + + // Case 1 + group.bench_function("ndarray_view", |b| { + let mut buffer = Buffer::new(); + buffer.table("x1").unwrap(); + b.iter(|| { + buffer + .column_arr(col_name, black_box(&transposed_view)) + .unwrap(); + }); + buffer.clear(); + }); + + let shape: Vec = transposed_view.shape() + .iter() + .map(|&d| d as u32) + .collect(); + let elem_size = size_of::() as i32; + let strides: Vec = transposed_view.strides() + .iter() + .map(|&s| s as i32 * elem_size) // 转换为字节步长 + .collect(); + let view2: StridedArrayView<'_, f64> = unsafe { + StridedArrayView::new( + transposed_view.ndim(), + shape.as_ptr(), + strides.as_ptr(), + transposed_view.as_ptr() as *const u8, + transposed_view.len() * elem_size as usize, + ) + }; + + // Case 2 + group.bench_function("strides_view", |b| { + let mut buffer = Buffer::new(); + buffer.table("x1").unwrap(); + b.iter(|| { + buffer + .column_arr(col_name, black_box(&view2)) + .unwrap(); + }); + buffer.clear(); + }); +} + +criterion_group!(benches, bench_write_array_data, bench_array_view); criterion_main!(benches); From 4b5655aa06917284af04a41bb9dca08b088c8182 Mon Sep 17 00:00:00 2001 From: victorgao Date: Mon, 14 Apr 2025 17:11:01 +0800 Subject: [PATCH 39/56] introduce double binary format protocol. --- questdb-rs/benches/ndarr.rs | 12 ++++-------- questdb-rs/src/ingress/mod.rs | 25 +++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/questdb-rs/benches/ndarr.rs b/questdb-rs/benches/ndarr.rs index bf641cd9..a5b8b704 100644 --- a/questdb-rs/benches/ndarr.rs +++ b/questdb-rs/benches/ndarr.rs @@ -84,12 +84,10 @@ fn bench_array_view(c: &mut Criterion) { buffer.clear(); }); - let shape: Vec = transposed_view.shape() - .iter() - .map(|&d| d as u32) - .collect(); + let shape: Vec = transposed_view.shape().iter().map(|&d| d as u32).collect(); let elem_size = size_of::() as i32; - let strides: Vec = transposed_view.strides() + let strides: Vec = transposed_view + .strides() .iter() .map(|&s| s as i32 * elem_size) // 转换为字节步长 .collect(); @@ -108,9 +106,7 @@ fn bench_array_view(c: &mut Criterion) { let mut buffer = Buffer::new(); buffer.table("x1").unwrap(); b.iter(|| { - buffer - .column_arr(col_name, black_box(&view2)) - .unwrap(); + buffer.column_arr(col_name, black_box(&view2)).unwrap(); }); buffer.clear(); }); diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index fc376a6e..bce94f6f 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -561,6 +561,7 @@ pub struct Buffer { state: BufferState, marker: Option<(usize, BufferState)>, max_name_len: usize, + f64serializer: fn(&mut Vec, f64), } impl Buffer { @@ -572,6 +573,17 @@ impl Buffer { state: BufferState::new(), marker: None, max_name_len: 127, + f64serializer: f64_text_series, + } + } + + pub fn new_with_protocol_version_2() -> Self { + Self { + output: Vec::new(), + state: BufferState::new(), + marker: None, + max_name_len: 127, + f64serializer: f64_binary_series, } } @@ -585,6 +597,7 @@ impl Buffer { pub fn with_max_name_len(max_name_len: usize) -> Self { let mut buf = Self::new(); buf.max_name_len = max_name_len; + buf.f64serializer = f64_text_series; buf } @@ -950,8 +963,7 @@ impl Buffer { Error: From, { self.write_column_key(name)?; - let mut ser = F64Serializer::new(value); - self.output.extend_from_slice(ser.as_str().as_bytes()); + (self.f64serializer)(&mut self.output, value); Ok(self) } @@ -2678,6 +2690,15 @@ fn parse_key_pair(auth: &EcdsaAuthParams) -> Result { }) } +fn f64_text_series(vec: &mut Vec, value: f64) { + let mut ser = F64Serializer::new(value); + vec.extend_from_slice(ser.as_str().as_bytes()) +} + +fn f64_binary_series(vec: &mut Vec, value: f64) { + vec.extend_from_slice(&value.to_be_bytes()) +} + pub(crate) struct F64Serializer { buf: ryu::Buffer, n: f64, From 4076336c888a87b5d281066933f0115fe8d6e946 Mon Sep 17 00:00:00 2001 From: victorgao Date: Mon, 14 Apr 2025 17:22:15 +0800 Subject: [PATCH 40/56] code format --- system_test/questdb_line_sender.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system_test/questdb_line_sender.py b/system_test/questdb_line_sender.py index 7ecd4b0c..38416a4c 100644 --- a/system_test/questdb_line_sender.py +++ b/system_test/questdb_line_sender.py @@ -782,7 +782,7 @@ def column( value: Union[bool, int, float, str, TimestampMicros, datetime]): self._buffer.column(name, value) return self - + def column_f64_arr( self, name: str, array: numpy.ndarray): From 612406f772f5281d82d61b759556a73bacf8e2e3 Mon Sep 17 00:00:00 2001 From: victorgao Date: Tue, 22 Apr 2025 23:12:23 +0800 Subject: [PATCH 41/56] fix little issue --- questdb-rs/src/ingress/mod.rs | 3 ++- questdb-rs/src/ingress/ndarr.rs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index bce94f6f..f80ef031 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -577,6 +577,7 @@ impl Buffer { } } + // todo, make protocol version2 as default option. pub fn new_with_protocol_version_2() -> Self { Self { output: Vec::new(), @@ -2696,7 +2697,7 @@ fn f64_text_series(vec: &mut Vec, value: f64) { } fn f64_binary_series(vec: &mut Vec, value: f64) { - vec.extend_from_slice(&value.to_be_bytes()) + vec.extend_from_slice(&value.to_le_bytes()) } pub(crate) struct F64Serializer { diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index ff81d034..baddd1c1 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -25,7 +25,7 @@ where /// /// # Returns /// - `Ok(usize)`: Expected buffer size in bytes if valid - /// - `Err(Error)`: Currently never returned, but reserved for future validation logic + /// - `Err(Error)`: Otherwise fn check_data_buf(&self) -> Result; } From e1ed4537a53f09dc73bdafc0c589155660683f77 Mon Sep 17 00:00:00 2001 From: victorgao Date: Wed, 23 Apr 2025 13:26:08 +0800 Subject: [PATCH 42/56] f64 binary format test. --- questdb-rs/Cargo.toml | 4 +- questdb-rs/build.rs | 34 +++++++++ questdb-rs/src/ingress/mod.rs | 71 ++++++++++++------- questdb-rs/src/ingress/ndarr.rs | 2 +- .../interop/ilp-client-interop-test.json | 2 + questdb-rs/src/tests/mod.rs | 22 ++++++ questdb-rs/src/tests/sender.rs | 46 ++++++++---- 7 files changed, 143 insertions(+), 38 deletions(-) diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index ba2a5680..3ac9022d 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -23,7 +23,7 @@ socket2 = "0.5.5" dns-lookup = "2.0.4" base64ct = { version = "1.7", features = ["alloc"] } rustls-pemfile = "2.0.0" -ryu = "1.0" +ryu = { version = "1.0", optional = true } itoa = "1.0" aws-lc-rs = { version = "1.13", optional = true } ring = { version = "0.17.14", optional = true } @@ -64,6 +64,8 @@ default = ["tls-webpki-certs", "ilp-over-http", "aws-lc-crypto"] # Include support for ILP over HTTP. ilp-over-http = ["dep:ureq", "dep:serde_json", "dep:rand"] +protocol-version-1 = ["dep:ryu"] + # Allow use OS-provided root TLS certificates tls-native-certs = ["dep:rustls-native-certs"] diff --git a/questdb-rs/build.rs b/questdb-rs/build.rs index 636f55e8..1ce89d26 100644 --- a/questdb-rs/build.rs +++ b/questdb-rs/build.rs @@ -50,6 +50,8 @@ pub mod json_tests { #[derive(Debug, Serialize, Deserialize)] struct Expected { line: Option, + #[serde(rename = "binaryBase64")] + binary_base64: Option, #[serde(rename = "anyLines")] any_lines: Option>, @@ -97,7 +99,10 @@ pub mod json_tests { use crate::{Result, ingress::{Buffer}}; use crate::tests::{TestResult}; + use base64ct::Base64; + use base64ct::Encoding; + #[cfg(feature = "protocol-version-1")] fn matches_any_line(line: &[u8], expected: &[&str]) -> bool { for &exp in expected { if line == exp.as_bytes() { @@ -167,7 +172,36 @@ pub mod json_tests { } } writeln!(output, "{} .at_now()?;", indent)?; + writeln!(output, " println!(\"{{}}\",Base64::encode_string(buffer.as_bytes()));")?; if let Some(expected) = expected { + #[cfg(not(feature = "protocol-version-1"))] + if let Some(ref base64) = expected.binary_base64 { + writeln!(output, " let exp = Base64::decode_vec(\"{}\").unwrap();", base64)?; + writeln!(output, " assert_eq!(buffer.as_bytes(), exp);")?; + } else if let Some(ref line) = expected.line { + let exp_ln = format!("{}\n", line); + writeln!(output, " let exp = {:?};", exp_ln)?; + writeln!(output, " assert_eq!(buffer.as_bytes(), exp.as_bytes());")?; + } else { + let any: Vec = expected + .any_lines + .as_ref() + .unwrap() + .iter() + .map(|line| format!("{}\n", line)) + .collect(); + writeln!(output, " let any = [")?; + for line in any.iter() { + writeln!(output, " {:?},", line)?; + } + writeln!(output, " ];")?; + writeln!( + output, + " assert!(matches_any_line(buffer.as_bytes(), &any));" + )?; + } + + #[cfg(feature = "protocol-version-1")] if let Some(ref line) = expected.line { let exp_ln = format!("{}\n", line); writeln!(output, " let exp = {:?};", exp_ln)?; diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index f80ef031..9dc6690b 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -561,7 +561,6 @@ pub struct Buffer { state: BufferState, marker: Option<(usize, BufferState)>, max_name_len: usize, - f64serializer: fn(&mut Vec, f64), } impl Buffer { @@ -573,18 +572,6 @@ impl Buffer { state: BufferState::new(), marker: None, max_name_len: 127, - f64serializer: f64_text_series, - } - } - - // todo, make protocol version2 as default option. - pub fn new_with_protocol_version_2() -> Self { - Self { - output: Vec::new(), - state: BufferState::new(), - marker: None, - max_name_len: 127, - f64serializer: f64_binary_series, } } @@ -598,7 +585,6 @@ impl Buffer { pub fn with_max_name_len(max_name_len: usize) -> Self { let mut buf = Self::new(); buf.max_name_len = max_name_len; - buf.f64serializer = f64_text_series; buf } @@ -930,6 +916,47 @@ impl Buffer { Ok(self) } + #[cfg(feature = "protocol-version-1")] + /// Record a floating point value for the given column. + /// + /// ``` + /// # use questdb::Result; + /// # use questdb::ingress::Buffer; + /// # fn main() -> Result<()> { + /// # let mut buffer = Buffer::new(); + /// # buffer.table("x")?; + /// buffer.column_f64("col_name", 3.14)?; + /// # Ok(()) + /// # } + /// ``` + /// + /// or + /// + /// ``` + /// # use questdb::Result; + /// # use questdb::ingress::Buffer; + /// use questdb::ingress::ColumnName; + /// + /// # fn main() -> Result<()> { + /// # let mut buffer = Buffer::new(); + /// # buffer.table("x")?; + /// let col_name = ColumnName::new("col_name")?; + /// buffer.column_f64(col_name, 3.14)?; + /// # Ok(()) + /// # } + /// ``` + pub fn column_f64<'a, N>(&mut self, name: N, value: f64) -> Result<&mut Self> + where + N: TryInto>, + Error: From, + { + self.write_column_key(name)?; + let mut ser = F64Serializer::new(value); + self.output.extend_from_slice(ser.as_str().as_bytes()); + Ok(self) + } + + #[cfg(not(feature = "protocol-version-1"))] /// Record a floating point value for the given column. /// /// ``` @@ -964,7 +991,9 @@ impl Buffer { Error: From, { self.write_column_key(name)?; - (self.f64serializer)(&mut self.output, value); + self.output.push(b'='); + self.output.push(DOUBLE_BINARY_FORMAT_TYPE); + self.output.extend_from_slice(&value.to_le_bytes()); Ok(self) } @@ -1021,6 +1050,7 @@ impl Buffer { Ok(self) } + #[cfg(not(feature = "protocol-version-1"))] /// Record a multidimensional array value for the given column. /// /// Supports arrays with up to [`MAX_DIMS`] dimensions. The array elements must @@ -2691,20 +2721,13 @@ fn parse_key_pair(auth: &EcdsaAuthParams) -> Result { }) } -fn f64_text_series(vec: &mut Vec, value: f64) { - let mut ser = F64Serializer::new(value); - vec.extend_from_slice(ser.as_str().as_bytes()) -} - -fn f64_binary_series(vec: &mut Vec, value: f64) { - vec.extend_from_slice(&value.to_le_bytes()) -} - +#[cfg(feature = "protocol-version-1")] pub(crate) struct F64Serializer { buf: ryu::Buffer, n: f64, } +#[cfg(feature = "protocol-version-1")] impl F64Serializer { pub(crate) fn new(n: f64) -> Self { F64Serializer { diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index baddd1c1..129a4934 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -25,7 +25,7 @@ where /// /// # Returns /// - `Ok(usize)`: Expected buffer size in bytes if valid - /// - `Err(Error)`: Otherwise + /// - `Err(Error)`: Otherwise fn check_data_buf(&self) -> Result; } diff --git a/questdb-rs/src/tests/interop/ilp-client-interop-test.json b/questdb-rs/src/tests/interop/ilp-client-interop-test.json index d3e0e259..0acedad7 100644 --- a/questdb-rs/src/tests/interop/ilp-client-interop-test.json +++ b/questdb-rs/src/tests/interop/ilp-client-interop-test.json @@ -32,6 +32,7 @@ ], "result": { "status": "SUCCESS", + "binaryBase64": "dGVzdF90YWJsZSxzeW1fY29sPXN5bV92YWwgc3RyX2NvbD0iZm9vIGJhciBiYXoiLGxvbmdfY29sPTQyaSxkb3VibGVfY29sPT0QAAAAAABARUAsYm9vbF9jb2w9dAo=", "line": "test_table,sym_col=sym_val str_col=\"foo bar baz\",long_col=42i,double_col=42.5,bool_col=t" } }, @@ -73,6 +74,7 @@ ], "result": { "status": "SUCCESS", + "binaryBase64": "ZG91YmxlcyBkMD09EAAAAAAAAAAALGRtMD09EAAAAAAAAACALGQxPT0QAAAAAAAA8D8sZEUxMDA9PRB9w5QlrUmyVCxkMDAwMDAwMT09EI3ttaD3xrA+LGROMDAwMDAwMT09EI3ttaD3xrC+Cg==", "anyLines": [ "doubles d0=0,dm0=-0,d1=1,dE100=1E+100,d0000001=1E-06,dN0000001=-1E-06", "doubles d0=0.0,dm0=-0.0,d1=1.0,dE100=1e100,d0000001=1e-6,dN0000001=-1e-6" diff --git a/questdb-rs/src/tests/mod.rs b/questdb-rs/src/tests/mod.rs index 29c1b3f4..f9f4a8a6 100644 --- a/questdb-rs/src/tests/mod.rs +++ b/questdb-rs/src/tests/mod.rs @@ -21,6 +21,8 @@ * limitations under the License. * ******************************************************************************/ + +#[cfg(feature = "protocol-version-1")] mod f64_serializer; #[cfg(feature = "ilp-over-http")] @@ -38,3 +40,23 @@ mod json_tests { pub type TestError = Box; pub type TestResult = std::result::Result<(), TestError>; + +fn f64_to_bytes(name: &str, value: f64) -> Vec { + let mut buf = Vec::new(); + buf.extend_from_slice(name.as_bytes()); + buf.push(b'='); + + #[cfg(not(feature = "protocol-version-1"))] + { + buf.push(b'='); + buf.push(crate::ingress::DOUBLE_BINARY_FORMAT_TYPE); + buf.extend_from_slice(&value.to_le_bytes()); + buf + } + #[cfg(feature = "protocol-version-1")] + { + let mut ser = crate::ingress::F64Serializer::new(value); + self.output.extend_from_slice(ser.as_str().as_bytes()); + buf + } +} diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index 3edc0eb8..17607a01 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -71,20 +71,25 @@ fn test_basics() -> TestResult { .at(ts_nanos)?; assert_eq!(server.recv_q()?, 0); - let exp = format!( - "test,t1=v1 f1=0.5,ts1=12345t,ts2={}t,ts3={}t {}\n", - ts_micros_num, - ts_nanos_num / 1000i64, - ts_nanos_num - ); - let exp_byte = exp.as_bytes(); - assert_eq!(buffer.as_bytes(), exp_byte); + let exp = &[ + "test,t1=v1 ".as_bytes(), + super::f64_to_bytes("f1", 0.5).as_slice(), + format!( + ",ts1=12345t,ts2={}t,ts3={}t {}\n", + ts_micros_num, + ts_nanos_num / 1000i64, + ts_nanos_num + ) + .as_bytes(), + ] + .concat(); + assert_eq!(buffer.as_bytes(), exp); assert_eq!(buffer.len(), exp.len()); sender.flush(&mut buffer)?; assert_eq!(buffer.len(), 0); assert_eq!(buffer.as_bytes(), b""); assert_eq!(server.recv_q()?, 1); - assert_eq!(server.msgs[0], exp_byte); + assert_eq!(server.msgs[0], *exp); Ok(()) } @@ -138,7 +143,8 @@ fn test_array_basic() -> TestResult { write_array_data(&array_3d.view(), &mut &mut array_data3d[0..])?; let exp = &[ - "my_table,device=A001 f1=25.5".as_bytes(), + "my_table,device=A001 ".as_bytes(), + super::f64_to_bytes("f1", 25.5).as_slice(), ",arr2d=".as_bytes(), array_header2d, array_data2d.as_slice(), @@ -179,6 +185,12 @@ fn test_max_buf_size() -> TestResult { let err = sender.flush(&mut buffer).unwrap_err(); assert_eq!(err.code(), ErrorCode::InvalidApiCall); + #[cfg(not(feature = "protocol-version-1"))] + assert_eq!( + err.msg(), + "Could not flush buffer: Buffer size of 1025 exceeds maximum configured allowed size of 1024 bytes." + ); + #[cfg(feature = "protocol-version-1")] assert_eq!( err.msg(), "Could not flush buffer: Buffer size of 1026 exceeds maximum configured allowed size of 1024 bytes." @@ -459,7 +471,12 @@ fn test_tls_with_file_ca() -> TestResult { .at(TimestampNanos::new(10000000))?; assert_eq!(server.recv_q()?, 0); - let exp = b"test,t1=v1 f1=0.5 10000000\n"; + let exp = &[ + "test,t1=v1 ".as_bytes(), + super::f64_to_bytes("f1", 0.5).as_slice(), + " 10000000\n".as_bytes(), + ] + .concat(); assert_eq!(buffer.as_bytes(), exp); assert_eq!(buffer.len(), exp.len()); sender.flush(&mut buffer)?; @@ -552,7 +569,12 @@ fn test_tls_insecure_skip_verify() -> TestResult { .at(TimestampNanos::new(10000000))?; assert_eq!(server.recv_q()?, 0); - let exp = b"test,t1=v1 f1=0.5 10000000\n"; + let exp = &[ + "test,t1=v1 ".as_bytes(), + super::f64_to_bytes("f1", 0.5).as_slice(), + " 10000000\n".as_bytes(), + ] + .concat(); assert_eq!(buffer.as_bytes(), exp); assert_eq!(buffer.len(), exp.len()); sender.flush(&mut buffer)?; From 888665dfcd0f4865894ba99ea80cf2200df68bf2 Mon Sep 17 00:00:00 2001 From: victorgao Date: Wed, 23 Apr 2025 15:29:57 +0800 Subject: [PATCH 43/56] f64 binary format test. --- questdb-rs-ffi/Cargo.lock | 1 - 1 file changed, 1 deletion(-) diff --git a/questdb-rs-ffi/Cargo.lock b/questdb-rs-ffi/Cargo.lock index 5666057c..709e5c72 100644 --- a/questdb-rs-ffi/Cargo.lock +++ b/questdb-rs-ffi/Cargo.lock @@ -473,7 +473,6 @@ dependencies = [ "rustls-native-certs", "rustls-pemfile", "rustls-pki-types", - "ryu", "serde", "serde_json", "slugify", From 1b3e7556327cf3cb6e944628c652f059893955dd Mon Sep 17 00:00:00 2001 From: victorgao Date: Wed, 23 Apr 2025 15:57:38 +0800 Subject: [PATCH 44/56] fix cpp test for double binary format. --- cpp_test/test_line_sender.cpp | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 7036d268..ecc8d0c1 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -93,7 +93,7 @@ bool operator==( #endif template -void push_double_arr_to_buffer( +std::string& push_double_arr_to_buffer( std::string& buffer, std::array data, size_t rank, @@ -108,6 +108,14 @@ void push_double_arr_to_buffer( buffer.append( reinterpret_cast(data.data()), data.size() * sizeof(double)); + return buffer; +} + +std::string& push_double_to_buffer(std::string& buffer, double data) +{ + buffer.push_back(16); + buffer.append(reinterpret_cast(&data), sizeof(double)); + return buffer; } TEST_CASE("line_sender c api basics") @@ -180,13 +188,14 @@ TEST_CASE("line_sender c api basics") &err)); CHECK(::line_sender_buffer_at_nanos(buffer, 10000000, &err)); CHECK(server.recv() == 0); - CHECK(::line_sender_buffer_size(buffer) == 143); + CHECK(::line_sender_buffer_size(buffer) == 150); CHECK(::line_sender_flush(sender, buffer, &err)); ::line_sender_buffer_free(buffer); CHECK(server.recv() == 1); - std::string expect{"test,t1=v1 f1=0.5,a1=="}; - push_double_arr_to_buffer(expect, arr_data, 3, shapes); - expect.append(" 10000000\n"); + std::string expect{"test,t1=v1 f1=="}; + push_double_to_buffer(expect, 0.5).append(",a1=="); + push_double_arr_to_buffer(expect, arr_data, 3, shapes) + .append(" 10000000\n"); CHECK(server.msgs(0) == expect); } @@ -243,10 +252,12 @@ TEST_CASE("line_sender c++ api basics") .at(questdb::ingress::timestamp_nanos{10000000}); CHECK(server.recv() == 0); - CHECK(buffer.size() == 31); + CHECK(buffer.size() == 38); sender.flush(buffer); CHECK(server.recv() == 1); - CHECK(server.msgs(0) == "test,t1=v1,t2= f1=0.5 10000000\n"); + std::string expect{"test,t1=v1,t2= f1=="}; + push_double_to_buffer(expect, 0.5).append(" 10000000\n"); + CHECK(server.msgs(0) == expect); } TEST_CASE("test multiple lines") @@ -279,13 +290,13 @@ TEST_CASE("test multiple lines") .at_now(); CHECK(server.recv() == 0); - CHECK(buffer.size() == 137); + CHECK(buffer.size() == 142); sender.flush(buffer); CHECK(server.recv() == 2); - CHECK( - server.msgs(0) == - ("metric1,t1=val1,t2=val2 f1=t,f2=12345i," - "f3=10.75,f4=\"val3\",f5=\"val4\",f6=\"val5\" 111222233333\n")); + std::string expect{"metric1,t1=val1,t2=val2 f1=t,f2=12345i,f3=="}; + push_double_to_buffer(expect, 10.75) + .append(",f4=\"val3\",f5=\"val4\",f6=\"val5\" 111222233333\n"); + CHECK(server.msgs(0) == expect); CHECK( server.msgs(1) == "metric1,tag3=value\\ 3,tag\\ 4=value:4 field5=f\n"); } From ceac17d72eb5eed21c411d160aaaa480abaeb8a5 Mon Sep 17 00:00:00 2001 From: victorgao Date: Wed, 23 Apr 2025 16:21:09 +0800 Subject: [PATCH 45/56] fix tests --- ci/run_all_tests.py | 2 ++ questdb-rs/src/ingress/mod.rs | 1 - questdb-rs/src/tests/mod.rs | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/run_all_tests.py b/ci/run_all_tests.py index b4e4dc4a..deffc0e7 100644 --- a/ci/run_all_tests.py +++ b/ci/run_all_tests.py @@ -51,6 +51,8 @@ def main(): '--', '--nocapture', cwd='questdb-rs') run_cmd('cargo', 'test', '--features=almost-all-features', '--', '--nocapture', cwd='questdb-rs') + run_cmd('cargo', 'test', '--features=almost-all-features,protocol-version-1', + '--', '--nocapture', cwd='questdb-rs') run_cmd(str(test_line_sender_path)) run_cmd(str(test_line_sender_path_CXX20)) run_cmd('python3', str(system_test_path), 'run', '--versions', qdb_v, '-v') diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 9dc6690b..b3ce2f75 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -1050,7 +1050,6 @@ impl Buffer { Ok(self) } - #[cfg(not(feature = "protocol-version-1"))] /// Record a multidimensional array value for the given column. /// /// Supports arrays with up to [`MAX_DIMS`] dimensions. The array elements must diff --git a/questdb-rs/src/tests/mod.rs b/questdb-rs/src/tests/mod.rs index f9f4a8a6..28ae717c 100644 --- a/questdb-rs/src/tests/mod.rs +++ b/questdb-rs/src/tests/mod.rs @@ -56,7 +56,7 @@ fn f64_to_bytes(name: &str, value: f64) -> Vec { #[cfg(feature = "protocol-version-1")] { let mut ser = crate::ingress::F64Serializer::new(value); - self.output.extend_from_slice(ser.as_str().as_bytes()); + buf.extend_from_slice(ser.as_str().as_bytes()); buf } } From c38c0839dd20cd187b37ddd58ed4bef35fc9b722 Mon Sep 17 00:00:00 2001 From: victorgao Date: Wed, 23 Apr 2025 16:52:08 +0800 Subject: [PATCH 46/56] use direct buffer case when ndarray is not continus. --- questdb-rs/build.rs | 11 ++++++++-- questdb-rs/src/ingress/mod.rs | 37 ++++++++++++++++++--------------- questdb-rs/src/ingress/ndarr.rs | 1 - questdb-rs/src/tests/sender.rs | 8 +++---- 4 files changed, 33 insertions(+), 24 deletions(-) diff --git a/questdb-rs/build.rs b/questdb-rs/build.rs index 1ce89d26..348605aa 100644 --- a/questdb-rs/build.rs +++ b/questdb-rs/build.rs @@ -172,11 +172,18 @@ pub mod json_tests { } } writeln!(output, "{} .at_now()?;", indent)?; - writeln!(output, " println!(\"{{}}\",Base64::encode_string(buffer.as_bytes()));")?; + writeln!( + output, + " println!(\"{{}}\",Base64::encode_string(buffer.as_bytes()));" + )?; if let Some(expected) = expected { #[cfg(not(feature = "protocol-version-1"))] if let Some(ref base64) = expected.binary_base64 { - writeln!(output, " let exp = Base64::decode_vec(\"{}\").unwrap();", base64)?; + writeln!( + output, + " let exp = Base64::decode_vec(\"{}\").unwrap();", + base64 + )?; writeln!(output, " assert_eq!(buffer.as_bytes(), exp);")?; } else if let Some(ref line) = expected.line { let exp_ln = format!("{}\n", line); diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index b3ce2f75..4474575e 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -1151,24 +1151,27 @@ impl Buffer { let mut cursor = Cursor::new(writeable); // ndarr data - if let Err(e) = ndarr::write_array_data(view, &mut cursor) { - return Err(error::fmt!( - ArrayWriteToBufferError, - "Can not write row major to writer: {}", - e - )); - } - - if cursor.position() != (reserve_size as u64) { - return Err(error::fmt!( - ArrayWriteToBufferError, - "Array write buffer length mismatch (actual: {}, expected: {})", - cursor.position(), - reserve_size - )); + if view.as_slice().is_some() { + if let Err(e) = ndarr::write_array_data(view, &mut cursor) { + return Err(error::fmt!( + ArrayWriteToBufferError, + "Can not write row major to writer: {}", + e + )); + } + if cursor.position() != (reserve_size as u64) { + return Err(error::fmt!( + ArrayWriteToBufferError, + "Array write buffer length mismatch (actual: {}, expected: {})", + cursor.position(), + reserve_size + )); + } + unsafe { self.output.set_len(reserve_size + index) } + } else { + unsafe { self.output.set_len(reserve_size + index) } + ndarr::write_array_data_use_raw_buffer(&mut self.output[index..], view); } - - unsafe { self.output.set_len(reserve_size + index) } Ok(self) } diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index 129a4934..3a324ecd 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -68,7 +68,6 @@ where Ok(()) } -#[cfg(feature = "benchmark")] pub(crate) fn write_array_data_use_raw_buffer, T>(buf: &mut [u8], array: &A) where T: ArrayElement, diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index 17607a01..0ae0765c 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -80,9 +80,9 @@ fn test_basics() -> TestResult { ts_nanos_num / 1000i64, ts_nanos_num ) - .as_bytes(), + .as_bytes(), ] - .concat(); + .concat(); assert_eq!(buffer.as_bytes(), exp); assert_eq!(buffer.len(), exp.len()); sender.flush(&mut buffer)?; @@ -476,7 +476,7 @@ fn test_tls_with_file_ca() -> TestResult { super::f64_to_bytes("f1", 0.5).as_slice(), " 10000000\n".as_bytes(), ] - .concat(); + .concat(); assert_eq!(buffer.as_bytes(), exp); assert_eq!(buffer.len(), exp.len()); sender.flush(&mut buffer)?; @@ -574,7 +574,7 @@ fn test_tls_insecure_skip_verify() -> TestResult { super::f64_to_bytes("f1", 0.5).as_slice(), " 10000000\n".as_bytes(), ] - .concat(); + .concat(); assert_eq!(buffer.as_bytes(), exp); assert_eq!(buffer.len(), exp.len()); sender.flush(&mut buffer)?; From 21cd6f9f304cd327825be0ff8c64c0ad2126b126 Mon Sep 17 00:00:00 2001 From: victorgao Date: Wed, 23 Apr 2025 19:56:43 +0800 Subject: [PATCH 47/56] ci import numpy --- ci/compile.yaml | 4 ++++ system_test/test.py | 4 +--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ci/compile.yaml b/ci/compile.yaml index aa9e1059..4c892ce4 100644 --- a/ci/compile.yaml +++ b/ci/compile.yaml @@ -4,6 +4,10 @@ steps: rustup default $(toolchain) condition: ne(variables['toolchain'], '') displayName: "Update and set Rust toolchain" + - script: | + python -m pip install --upgrade pip + pip install numpy + displayName: 'Install Python Dependencies' - script: cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DQUESTDB_TESTS_AND_EXAMPLES=ON env: JAVA_HOME: $(JAVA_HOME_11_X64) diff --git a/system_test/test.py b/system_test/test.py index b5498230..8ba87399 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -26,16 +26,14 @@ import sys -import numpy as np - sys.dont_write_bytecode = True import os - import pathlib import math import datetime import argparse import unittest +import numpy as np import time import questdb_line_sender as qls import uuid From 1f17a09f9913bbf0ca650520bbc68f8003ce496b Mon Sep 17 00:00:00 2001 From: victorgao Date: Wed, 23 Apr 2025 20:36:09 +0800 Subject: [PATCH 48/56] temp fix ci to specify nd_arr branch --- ci/run_all_tests.py | 5 +++-- ci/run_tests_pipeline.yaml | 31 +++++++++++++++++++++---------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/ci/run_all_tests.py b/ci/run_all_tests.py index deffc0e7..783d52b4 100644 --- a/ci/run_all_tests.py +++ b/ci/run_all_tests.py @@ -41,7 +41,7 @@ def main(): build_cxx20_dir.glob(f'**/test_line_sender{exe_suffix}'))) system_test_path = pathlib.Path('system_test') / 'test.py' - qdb_v = '8.2.3' # The version of QuestDB we'll test against. + #qdb_v = '8.2.3' # The version of QuestDB we'll test against. run_cmd('cargo', 'test', '--', '--nocapture', cwd='questdb-rs') @@ -55,7 +55,8 @@ def main(): '--', '--nocapture', cwd='questdb-rs') run_cmd(str(test_line_sender_path)) run_cmd(str(test_line_sender_path_CXX20)) - run_cmd('python3', str(system_test_path), 'run', '--versions', qdb_v, '-v') + #run_cmd('python3', str(system_test_path), 'run', '--versions', qdb_v, '-v') + run_cmd('python3', str(system_test_path), 'run --repo ./questdb_nd_arr -v') if __name__ == '__main__': diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml index 1fced6be..3e602321 100644 --- a/ci/run_tests_pipeline.yaml +++ b/ci/run_tests_pipeline.yaml @@ -54,6 +54,17 @@ stages: cd questdb-rs cargo build --examples --features almost-all-features displayName: "Build Rust examples" + ############################# temp for test begin ##################### + - script: | + git clone -b nd_arr --depth 1 https://github.com/questdb/questdb.git ./questdb_nd_arr + displayName: git clone questdb + - task: Maven@3 + displayName: "Compile QuestDB" + inputs: + mavenPOMFile: 'questdb/pom.xml' + jdkVersionOption: '1.11' + options: "-DskipTests -Pbuild-web-console" + ############################# temp for test end ##################### - script: python3 ci/run_all_tests.py env: JAVA_HOME: $(JAVA_HOME_11_X64) @@ -114,15 +125,15 @@ stages: lfs: false submodules: false - template: compile.yaml + #- script: | + # git clone --depth 1 https://github.com/questdb/questdb.git + # displayName: git clone questdb + #- task: Maven@3 + # displayName: "Compile QuestDB" + # inputs: + # mavenPOMFile: 'questdb/pom.xml' + # jdkVersionOption: '1.11' + # options: "-DskipTests -Pbuild-web-console" - script: | - git clone --depth 1 https://github.com/questdb/questdb.git - displayName: git clone questdb - - task: Maven@3 - displayName: "Compile QuestDB" - inputs: - mavenPOMFile: 'questdb/pom.xml' - jdkVersionOption: '1.11' - options: "-DskipTests -Pbuild-web-console" - - script: | - python3 system_test/test.py run --repo ./questdb -v + python3 system_test/test.py run --repo ./questdb_nd_arr -v displayName: "integration test" From 73f784a8e7a82491fb0f2b6476ba7a7340283e52 Mon Sep 17 00:00:00 2001 From: victorgao Date: Wed, 23 Apr 2025 20:44:13 +0800 Subject: [PATCH 49/56] temp fix ci to specify nd_arr branch --- ci/run_tests_pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml index 3e602321..4b305507 100644 --- a/ci/run_tests_pipeline.yaml +++ b/ci/run_tests_pipeline.yaml @@ -61,7 +61,7 @@ stages: - task: Maven@3 displayName: "Compile QuestDB" inputs: - mavenPOMFile: 'questdb/pom.xml' + mavenPOMFile: 'questdb_nd_arr/pom.xml' jdkVersionOption: '1.11' options: "-DskipTests -Pbuild-web-console" ############################# temp for test end ##################### From 5de599e9f5ffaa45d6514d9bad8f611beb52de23 Mon Sep 17 00:00:00 2001 From: victorgao Date: Wed, 23 Apr 2025 21:34:09 +0800 Subject: [PATCH 50/56] temp fix ci tests --- ci/run_all_tests.py | 2 +- ci/run_tests_pipeline.yaml | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ci/run_all_tests.py b/ci/run_all_tests.py index 783d52b4..921afe6f 100644 --- a/ci/run_all_tests.py +++ b/ci/run_all_tests.py @@ -56,7 +56,7 @@ def main(): run_cmd(str(test_line_sender_path)) run_cmd(str(test_line_sender_path_CXX20)) #run_cmd('python3', str(system_test_path), 'run', '--versions', qdb_v, '-v') - run_cmd('python3', str(system_test_path), 'run --repo ./questdb_nd_arr -v') + run_cmd('python3', str(system_test_path), 'run' '--repo' './questdb_nd_arr', '-v') if __name__ == '__main__': diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml index 4b305507..25434444 100644 --- a/ci/run_tests_pipeline.yaml +++ b/ci/run_tests_pipeline.yaml @@ -125,15 +125,15 @@ stages: lfs: false submodules: false - template: compile.yaml - #- script: | - # git clone --depth 1 https://github.com/questdb/questdb.git - # displayName: git clone questdb - #- task: Maven@3 - # displayName: "Compile QuestDB" - # inputs: - # mavenPOMFile: 'questdb/pom.xml' - # jdkVersionOption: '1.11' - # options: "-DskipTests -Pbuild-web-console" - script: | - python3 system_test/test.py run --repo ./questdb_nd_arr -v + git clone -b nd_arr --depth 1 https://github.com/questdb/questdb.git + displayName: git clone questdb + - task: Maven@3 + displayName: "Compile QuestDB" + inputs: + mavenPOMFile: 'questdb/pom.xml' + jdkVersionOption: '1.11' + options: "-DskipTests -Pbuild-web-console" + - script: | + python3 system_test/test.py run --repo ./questdb -v displayName: "integration test" From 37f8d2051ac184495190ce703a024924e40a38f7 Mon Sep 17 00:00:00 2001 From: victorgao Date: Wed, 23 Apr 2025 22:41:41 +0800 Subject: [PATCH 51/56] fix ci tests --- ci/run_all_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_all_tests.py b/ci/run_all_tests.py index 921afe6f..4eea548f 100644 --- a/ci/run_all_tests.py +++ b/ci/run_all_tests.py @@ -56,7 +56,7 @@ def main(): run_cmd(str(test_line_sender_path)) run_cmd(str(test_line_sender_path_CXX20)) #run_cmd('python3', str(system_test_path), 'run', '--versions', qdb_v, '-v') - run_cmd('python3', str(system_test_path), 'run' '--repo' './questdb_nd_arr', '-v') + run_cmd('python3', str(system_test_path), 'run', '--repo', './questdb_nd_arr', '-v') if __name__ == '__main__': From 32f2a6383023e21798e570352013527b93f61eac Mon Sep 17 00:00:00 2001 From: victorgao Date: Thu, 24 Apr 2025 00:06:40 +0800 Subject: [PATCH 52/56] fix tests --- system_test/test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/system_test/test.py b/system_test/test.py index 8ba87399..e0541a0e 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -551,6 +551,7 @@ def _test_example(self, bin_name, table_name, tls=False): {'name': 'side', 'type': 'SYMBOL'}, {'name': 'price', 'type': 'DOUBLE'}, {'name': 'amount', 'type': 'DOUBLE'}, + {'dim': 3, 'elemType': 'DOUBLE', 'name': 'order_book', 'type': 'ARRAY'}, {'name': 'timestamp', 'type': 'TIMESTAMP'}] self.assertEqual(resp['columns'], exp_columns) From 1f12f279880fd3b26ec8ad50ecaf02782f7ed180 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 24 Apr 2025 17:08:15 +0800 Subject: [PATCH 53/56] add cpp array example. --- examples/line_sender_cpp_example.cpp | 24 ++- examples/line_sender_cpp_example_auth.cpp | 25 ++- examples/line_sender_cpp_example_auth_tls.cpp | 26 ++- .../line_sender_cpp_example_from_conf.cpp | 18 ++ examples/line_sender_cpp_example_from_env.cpp | 18 ++ examples/line_sender_cpp_example_http.cpp | 24 ++- examples/line_sender_cpp_example_tls_ca.cpp | 28 +++- questdb-rs/build.rs | 4 - system_test/test.py | 156 +++++++++--------- 9 files changed, 224 insertions(+), 99 deletions(-) diff --git a/examples/line_sender_cpp_example.cpp b/examples/line_sender_cpp_example.cpp index 2d90812f..5e3b12d1 100644 --- a/examples/line_sender_cpp_example.cpp +++ b/examples/line_sender_cpp_example.cpp @@ -19,6 +19,23 @@ static bool example(std::string_view host, std::string_view port) const auto side_name = "side"_cn; const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; + const auto order_book_name = "order_book"_cn; + size_t rank = 3; + std::vector shape{2, 3, 2}; + std::vector strides{48, 16, 8}; + std::array arr_data = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; questdb::ingress::line_sender_buffer buffer; buffer @@ -27,6 +44,7 @@ static bool example(std::string_view host, std::string_view port) .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) .column(amount_name, 0.00044) + .column(order_book_name, 3, shape, strides, arr_data) .at(questdb::ingress::timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. @@ -59,9 +77,9 @@ static bool displayed_help(int argc, const char* argv[]) std::cerr << "Usage:\n" << "line_sender_c_example: [HOST [PORT]]\n" - << " HOST: ILP host (defaults to \"localhost\").\n" - << " PORT: ILP port (defaults to \"9009\")." - << std::endl; + << " HOST: ILP host (defaults to \"localhost\").\n" + << " PORT: ILP port (defaults to \"9009\")." + << std::endl; return true; } } diff --git a/examples/line_sender_cpp_example_auth.cpp b/examples/line_sender_cpp_example_auth.cpp index 85e0d6e1..b4cf2b70 100644 --- a/examples/line_sender_cpp_example_auth.cpp +++ b/examples/line_sender_cpp_example_auth.cpp @@ -23,6 +23,24 @@ static bool example(std::string_view host, std::string_view port) const auto side_name = "side"_cn; const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; + const auto order_book_name = "order_book"_cn; + + size_t rank = 3; + std::vector shape{2, 3, 2}; + std::vector strides{48, 16, 8}; + std::array arr_data = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; questdb::ingress::line_sender_buffer buffer; buffer @@ -31,6 +49,7 @@ static bool example(std::string_view host, std::string_view port) .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) .column(amount_name, 0.00044) + .column(order_book_name, 3, shape, strides, arr_data) .at(questdb::ingress::timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. @@ -63,9 +82,9 @@ static bool displayed_help(int argc, const char* argv[]) std::cerr << "Usage:\n" << "line_sender_c_example: [HOST [PORT]]\n" - << " HOST: ILP host (defaults to \"localhost\").\n" - << " PORT: ILP port (defaults to \"9009\")." - << std::endl; + << " HOST: ILP host (defaults to \"localhost\").\n" + << " PORT: ILP port (defaults to \"9009\")." + << std::endl; return true; } } diff --git a/examples/line_sender_cpp_example_auth_tls.cpp b/examples/line_sender_cpp_example_auth_tls.cpp index f100dd04..90e2a46b 100644 --- a/examples/line_sender_cpp_example_auth_tls.cpp +++ b/examples/line_sender_cpp_example_auth_tls.cpp @@ -25,6 +25,23 @@ static bool example( const auto side_name = "side"_cn; const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; + const auto order_book_name = "order_book"_cn; + size_t rank = 3; + std::vector shape{2, 3, 2}; + std::vector strides{48, 16, 8}; + std::array arr_data = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; questdb::ingress::line_sender_buffer buffer; buffer @@ -33,6 +50,7 @@ static bool example( .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) .column(amount_name, 0.00044) + .column(order_book_name, 3, shape, strides, arr_data) .at(questdb::ingress::timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. @@ -64,10 +82,10 @@ static bool displayed_help(int argc, const char* argv[]) { std::cerr << "Usage:\n" - << "line_sender_c_example: CA_PATH [HOST [PORT]]\n" - << " HOST: ILP host (defaults to \"localhost\").\n" - << " PORT: ILP port (defaults to \"9009\")." - << std::endl; + << "line_sender_c_example: CA_PATH [HOST [PORT]]\n" + << " HOST: ILP host (defaults to \"localhost\").\n" + << " PORT: ILP port (defaults to \"9009\")." + << std::endl; return true; } } diff --git a/examples/line_sender_cpp_example_from_conf.cpp b/examples/line_sender_cpp_example_from_conf.cpp index bb71c6e3..d0d1e470 100644 --- a/examples/line_sender_cpp_example_from_conf.cpp +++ b/examples/line_sender_cpp_example_from_conf.cpp @@ -19,6 +19,23 @@ int main(int argc, const char* argv[]) const auto side_name = "side"_cn; const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; + const auto order_book_name = "order_book"_cn; + size_t rank = 3; + std::vector shape{2, 3, 2}; + std::vector strides{48, 16, 8}; + std::array arr_data = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; questdb::ingress::line_sender_buffer buffer; buffer @@ -27,6 +44,7 @@ int main(int argc, const char* argv[]) .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) .column(amount_name, 0.00044) + .column(order_book_name, 3, shape, strides, arr_data) .at(questdb::ingress::timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. diff --git a/examples/line_sender_cpp_example_from_env.cpp b/examples/line_sender_cpp_example_from_env.cpp index 63e99b26..069e4e22 100644 --- a/examples/line_sender_cpp_example_from_env.cpp +++ b/examples/line_sender_cpp_example_from_env.cpp @@ -18,6 +18,23 @@ int main(int argc, const char* argv[]) const auto side_name = "side"_cn; const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; + const auto order_book_name = "order_book"_cn; + size_t rank = 3; + std::vector shape{2, 3, 2}; + std::vector strides{48, 16, 8}; + std::array arr_data = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; questdb::ingress::line_sender_buffer buffer; buffer @@ -26,6 +43,7 @@ int main(int argc, const char* argv[]) .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) .column(amount_name, 0.00044) + .column(order_book_name, 3, shape, strides, arr_data) .at(questdb::ingress::timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. diff --git a/examples/line_sender_cpp_example_http.cpp b/examples/line_sender_cpp_example_http.cpp index 217845a2..43b4bdd5 100644 --- a/examples/line_sender_cpp_example_http.cpp +++ b/examples/line_sender_cpp_example_http.cpp @@ -19,6 +19,23 @@ static bool example(std::string_view host, std::string_view port) const auto side_name = "side"_cn; const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; + const auto order_book_name = "order_book"_cn; + size_t rank = 3; + std::vector shape{2, 3, 2}; + std::vector strides{48, 16, 8}; + std::array arr_data = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; questdb::ingress::line_sender_buffer buffer; buffer @@ -27,6 +44,7 @@ static bool example(std::string_view host, std::string_view port) .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) .column(amount_name, 0.00044) + .column(order_book_name, 3, shape, strides, arr_data) .at(questdb::ingress::timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. @@ -59,9 +77,9 @@ static bool displayed_help(int argc, const char* argv[]) std::cerr << "Usage:\n" << "line_sender_c_example: [HOST [PORT]]\n" - << " HOST: ILP host (defaults to \"localhost\").\n" - << " PORT: ILP port (defaults to \"9009\")." - << std::endl; + << " HOST: ILP host (defaults to \"localhost\").\n" + << " PORT: ILP port (defaults to \"9009\")." + << std::endl; return true; } } diff --git a/examples/line_sender_cpp_example_tls_ca.cpp b/examples/line_sender_cpp_example_tls_ca.cpp index c0327e96..27a91f81 100644 --- a/examples/line_sender_cpp_example_tls_ca.cpp +++ b/examples/line_sender_cpp_example_tls_ca.cpp @@ -27,6 +27,23 @@ static bool example( const auto side_name = "side"_cn; const auto price_name = "price"_cn; const auto amount_name = "amount"_cn; + const auto order_book_name = "order_book"_cn; + size_t rank = 3; + std::vector shape{2, 3, 2}; + std::vector strides{48, 16, 8}; + std::array arr_data = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; questdb::ingress::line_sender_buffer buffer; buffer @@ -35,6 +52,7 @@ static bool example( .symbol(side_name, "sell"_utf8) .column(price_name, 2615.54) .column(amount_name, 0.00044) + .column(order_book_name, 3, shape, strides, arr_data) .at(questdb::ingress::timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. @@ -66,11 +84,11 @@ static bool displayed_help(int argc, const char* argv[]) { std::cerr << "Usage:\n" - << "line_sender_c_example: CA_PATH [HOST [PORT]]\n" - << " CA_PATH: Certificate authority pem file.\n" - << " HOST: ILP host (defaults to \"localhost\").\n" - << " PORT: ILP port (defaults to \"9009\")." - << std::endl; + << "line_sender_c_example: CA_PATH [HOST [PORT]]\n" + << " CA_PATH: Certificate authority pem file.\n" + << " HOST: ILP host (defaults to \"localhost\").\n" + << " PORT: ILP port (defaults to \"9009\")." + << std::endl; return true; } } diff --git a/questdb-rs/build.rs b/questdb-rs/build.rs index 348605aa..bdf72571 100644 --- a/questdb-rs/build.rs +++ b/questdb-rs/build.rs @@ -172,10 +172,6 @@ pub mod json_tests { } } writeln!(output, "{} .at_now()?;", indent)?; - writeln!( - output, - " println!(\"{{}}\",Base64::encode_string(buffer.as_bytes()));" - )?; if let Some(expected) = expected { #[cfg(not(feature = "protocol-version-1"))] if let Some(ref base64) = expected.binary_base64 { diff --git a/system_test/test.py b/system_test/test.py index e0541a0e..5d5e49e9 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -48,7 +48,6 @@ import subprocess from collections import namedtuple - QDB_FIXTURE: QuestDbFixture = None TLS_PROXY_FIXTURE: TlsProxyFixture = None BUILD_MODE = None @@ -73,7 +72,6 @@ def ns_to_qdb_date(at_ts_ns): token_x="-nSHz3evuPl-rGLIlbIZjwOJeWao0rbk53Cll6XEgak", token_y="9iYksF4L5mfmArupv0CMoyVAWjQ4gNIoupdg6N5noG8") - # Bad malformed key AUTH_MALFORMED1 = dict( username="testUser3", @@ -81,7 +79,6 @@ def ns_to_qdb_date(at_ts_ns): token_x="-nSHz3evuPl-rGLIlbIZjwOJeWao0rbk53Cll6XEgak", token_y="9iYksF4L6mfmArupv0CMoyVAWjQ4gNIoupdg6N5noG8") - # Another malformed key where the keys invalid base 64. AUTH_MALFORMED2 = dict( username="testUser4", @@ -89,7 +86,6 @@ def ns_to_qdb_date(at_ts_ns): token_x="-nSHz3evuPl-rGLIlbIZjwOJeWao0rbk5XEgak", token_y="9iYksF4L6mfmArupv0CMoyVAWjQ4gNIou5noG8") - # All the keys are valid, but the username is wrong. AUTH_MALFORMED3 = dict( username="wrongUser", @@ -116,9 +112,9 @@ def _expect_eventual_disconnect(self, sender): for _ in range(1000): time.sleep(0.1) (sender - .table(table_name) - .symbol('s1', 'v1') - .at_now()) + .table(table_name) + .symbol('s1', 'v1') + .at_now()) sender.flush() def test_insert_three_rows(self): @@ -127,13 +123,13 @@ def test_insert_three_rows(self): with self._mk_linesender() as sender: for _ in range(3): (sender - .table(table_name) - .symbol('name_a', 'val_a') - .column('name_b', True) - .column('name_c', 42) - .column('name_d', 2.5) - .column('name_e', 'val_b') - .at_now()) + .table(table_name) + .symbol('name_a', 'val_a') + .column('name_b', True) + .column('name_c', 42) + .column('name_d', 2.5) + .column('name_e', 'val_b') + .at_now()) pending = sender.buffer.peek() sender.flush() @@ -162,12 +158,12 @@ def test_repeated_symbol_and_column_names(self): pending = None with self._mk_linesender() as sender: (sender - .table(table_name) - .symbol('a', 'A') - .symbol('a', 'B') - .column('b', False) - .column('b', 'C') - .at_now()) + .table(table_name) + .symbol('a', 'A') + .symbol('a', 'B') + .column('b', False) + .column('b', 'C') + .at_now()) pending = sender.buffer.peek() resp = retry_check_table(table_name, log_ctx=pending) @@ -189,10 +185,10 @@ def test_same_symbol_and_col_name(self): pending = None with self._mk_linesender() as sender: (sender - .table(table_name) - .symbol('a', 'A') - .column('a', 'B') - .at_now()) + .table(table_name) + .symbol('a', 'A') + .column('a', 'B') + .at_now()) pending = sender.buffer.peek() resp = retry_check_table(table_name, log_ctx=pending) @@ -210,9 +206,9 @@ def _test_single_symbol_impl(self, sender): pending = None with sender: (sender - .table(table_name) - .symbol('a', 'A') - .at_now()) + .table(table_name) + .symbol('a', 'A') + .at_now()) pending = sender.buffer.peek() resp = retry_check_table(table_name, log_ctx=pending) @@ -233,10 +229,10 @@ def test_two_columns(self): pending = None with self._mk_linesender() as sender: (sender - .table(table_name) - .column('a', 'A') - .column('b', 'B') - .at_now()) + .table(table_name) + .column('a', 'A') + .column('b', 'B') + .at_now()) pending = sender.buffer.peek() resp = retry_check_table(table_name, log_ctx=pending) @@ -255,13 +251,13 @@ def test_mismatched_types_across_rows(self): pending = None with self._mk_linesender() as sender: (sender - .table(table_name) - .column('a', 1) # LONG - .at_now()) + .table(table_name) + .column('a', 1) # LONG + .at_now()) (sender - .table(table_name) - .symbol('a', 'B') # SYMBOL - .at_now()) + .table(table_name) + .symbol('a', 'B') # SYMBOL + .at_now()) pending = sender.buffer.peek() @@ -305,9 +301,9 @@ def test_at(self): pending = None with self._mk_linesender() as sender: (sender - .table(table_name) - .symbol('a', 'A') - .at(at_ts_ns)) + .table(table_name) + .symbol('a', 'A') + .at(at_ts_ns)) pending = sender.buffer.peek() resp = retry_check_table(table_name, log_ctx=pending) exp_dataset = [['A', ns_to_qdb_date(at_ts_ns)]] @@ -323,9 +319,9 @@ def test_neg_at(self): with self._mk_linesender() as sender: with self.assertRaisesRegex(qls.SenderError, r'.*Timestamp .* is negative.*'): (sender - .table(table_name) - .symbol('a', 'A') - .at(at_ts_ns)) + .table(table_name) + .symbol('a', 'A') + .at(at_ts_ns)) def test_timestamp_col(self): if QDB_FIXTURE.version <= (6, 0, 7, 1): @@ -335,13 +331,13 @@ def test_timestamp_col(self): pending = None with self._mk_linesender() as sender: (sender - .table(table_name) - .column('a', qls.TimestampMicros(-1000000)) - .at_now()) + .table(table_name) + .column('a', qls.TimestampMicros(-1000000)) + .at_now()) (sender - .table(table_name) - .column('a', qls.TimestampMicros(1000000)) - .at_now()) + .table(table_name) + .column('a', qls.TimestampMicros(1000000)) + .at_now()) pending = sender.buffer.peek() resp = retry_check_table(table_name, log_ctx=pending) @@ -354,16 +350,15 @@ def test_timestamp_col(self): scrubbed_dataset = [row[:-1] for row in resp['dataset']] self.assertEqual(scrubbed_dataset, exp_dataset) - def test_underscores(self): table_name = f'_{uuid.uuid4().hex}_' pending = None with self._mk_linesender() as sender: (sender - .table(table_name) - .symbol('_a_b_c_', 'A') - .column('_d_e_f_', True) - .at_now()) + .table(table_name) + .symbol('_a_b_c_', 'A') + .column('_d_e_f_', True) + .at_now()) pending = sender.buffer.peek() resp = retry_check_table(table_name, log_ctx=pending) @@ -423,16 +418,16 @@ def test_floats(self): 1.23456789012, 1000000000000000000000000.0, -1000000000000000000000000.0, - float("nan"), # Converted to `None`. - float("inf"), # Converted to `None`. + float("nan"), # Converted to `None`. + float("inf"), # Converted to `None`. float("-inf")] # Converted to `None`. - # These values below do not round-trip properly: QuestDB limitation. - # 1.2345678901234567, - # 2.2250738585072014e-308, - # -2.2250738585072014e-308, - # 1.7976931348623157e+308, - # -1.7976931348623157e+308] + # These values below do not round-trip properly: QuestDB limitation. + # 1.2345678901234567, + # 2.2250738585072014e-308, + # -2.2250738585072014e-308, + # 1.7976931348623157e+308, + # -1.7976931348623157e+308] table_name = uuid.uuid4().hex pending = None with self._mk_linesender() as sender: @@ -470,9 +465,9 @@ def test_timestamp_column(self): ts = qls.TimestampMicros(3600000000) # One hour past epoch. with self._mk_linesender() as sender: (sender - .table(table_name) - .column('ts1', ts) - .at_now()) + .table(table_name) + .column('ts1', ts) + .at_now()) pending = sender.buffer.peek() resp = retry_check_table(table_name, log_ctx=pending) @@ -555,7 +550,13 @@ def _test_example(self, bin_name, table_name, tls=False): {'name': 'timestamp', 'type': 'TIMESTAMP'}] self.assertEqual(resp['columns'], exp_columns) - exp_dataset = [['ETH-USD', 'sell', 2615.54, 0.00044]] # Comparison excludes timestamp column. + exp_dataset = [['ETH-USD', + 'sell', + 2615.54, + 0.00044, + [[[48123.5, 2.4], [48124.0, 1.8], [48124.5, 0.9]], + [[48122.5, 3.1], [48122.0, 2.7], [48121.5, 4.3]]]]] + # Comparison excludes timestamp column. scrubbed_dataset = [row[:-1] for row in resp['dataset']] self.assertEqual(scrubbed_dataset, exp_dataset) @@ -611,9 +612,9 @@ def test_opposite_auth(self): # The sending the first line will not fail. (sender - .table(table_name) - .symbol('s1', 'v1') - .at_now()) + .table(table_name) + .symbol('s1', 'v1') + .at_now()) sender.flush() self._expect_eventual_disconnect(sender) @@ -704,7 +705,7 @@ def test_tls_insecure_skip_verify(self): def test_tls_roots(self): protocol = qls.Protocol.HTTPS if QDB_FIXTURE.http else qls.Protocol.TCPS - auth = auth=AUTH if QDB_FIXTURE.auth else {} + auth = auth = AUTH if QDB_FIXTURE.auth else {} sender = qls.Sender( BUILD_MODE, protocol, @@ -720,7 +721,7 @@ def _test_tls_ca(self, tls_ca): try: os.environ['SSL_CERT_FILE'] = str( Project().tls_certs_dir / 'server_rootCA.pem') - auth = auth=AUTH if QDB_FIXTURE.auth else {} + auth = auth = AUTH if QDB_FIXTURE.auth else {} sender = qls.Sender( BUILD_MODE, protocol, @@ -873,11 +874,11 @@ def iter_versions(args): if versions_args: versions = { version: ( - 'https://github.com/questdb/questdb/releases/download/' + - version + - '/questdb-' + - version + - '-no-jre-bin.tar.gz') + 'https://github.com/questdb/questdb/releases/download/' + + version + + '/questdb-' + + version + + '-no-jre-bin.tar.gz') for version in versions_args} else: last_n = getattr(args, 'last_n', None) or 1 @@ -899,7 +900,8 @@ def run_with_fixtures(args): for auth in (False, True): for http in (False, True): for build_mode in list(qls.BuildMode): - print(f'Running tests [questdb_dir={questdb_dir}, auth={auth}, http={http}, build_mode={build_mode}]') + print( + f'Running tests [questdb_dir={questdb_dir}, auth={auth}, http={http}, build_mode={build_mode}]') if http and last_version <= (7, 3, 7): print('Skipping ILP/HTTP tests for versions <= 7.3.7') continue From 89eaa79111b5a0e032a79a4fe264f548c59f5947 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 24 Apr 2025 18:01:04 +0800 Subject: [PATCH 54/56] fix logs --- system_test/test.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/system_test/test.py b/system_test/test.py index 5d5e49e9..8d4ce594 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -481,8 +481,6 @@ def test_timestamp_column(self): def test_f64_arr_column(self): table_name = uuid.uuid4().hex - pending = None - array1 = np.array( [ [[1.1, 2.2], [3.3, 4.4]], @@ -500,9 +498,8 @@ def test_f64_arr_column(self): .column_f64_arr('f64_arr2', array2) .column_f64_arr('f64_arr3', array3) .at_now()) - pending = sender.buffer.peek() - resp = retry_check_table(table_name, log_ctx=pending) + resp = retry_check_table(table_name) exp_columns = [ {'name': 'f64_arr1', 'type': 'DOUBLE[][][]'}, {'name': 'f64_arr2', 'type': 'DOUBLE[][][]'}, From 97c3179613b0ec0eb4f47d7b402062c4b8d0cadc Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 24 Apr 2025 22:35:46 +0800 Subject: [PATCH 55/56] fix array tests --- system_test/test.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/system_test/test.py b/system_test/test.py index 8d4ce594..d2860d9d 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -500,18 +500,14 @@ def test_f64_arr_column(self): .at_now()) resp = retry_check_table(table_name) - exp_columns = [ - {'name': 'f64_arr1', 'type': 'DOUBLE[][][]'}, - {'name': 'f64_arr2', 'type': 'DOUBLE[][][]'}, - {'name': 'f64_arr3', 'type': 'DOUBLE[][][]'}, - {'name': 'timestamp', 'type': 'TIMESTAMP'} - ] + exp_columns = [{'dim': 3, 'elemType': 'DOUBLE', 'name': 'f64_arr1', 'type': 'ARRAY'}, + {'dim': 3, 'elemType': 'DOUBLE', 'name': 'f64_arr2', 'type': 'ARRAY'}, + {'dim': 3, 'elemType': 'DOUBLE', 'name': 'f64_arr3', 'type': 'ARRAY'}, + {'name': 'timestamp', 'type': 'TIMESTAMP'}] self.assertEqual(resp['columns'], exp_columns) - expected_data = [ - ['{{{1.1,2.2},{3.3,4.4}},{{5.5,6.6},{7.7,8.8}}}', - '{{{1.1,5.5},{3.3,7.7}},{{2.2,6.6},{4.4,8.8}}}', - '{{{7.7,8.8},{5.5,6.6}},{{3.3,4.4},{1.1,2.2}}}'] - ] + expected_data = [[[[[1.1, 2.2], [3.3, 4.4]], [[5.5, 6.6], [7.7, 8.8]]], + [[[1.1, 5.5], [3.3, 7.7]], [[2.2, 6.6], [4.4, 8.8]]], + [[[7.7, 8.8], [5.5, 6.6]], [[3.3, 4.4], [1.1, 2.2]]]]] scrubbed_data = [row[:-1] for row in resp['dataset']] self.assertEqual(scrubbed_data, expected_data) From 359a74c3cf85b08537ada4412d0492e1eb9c03cc Mon Sep 17 00:00:00 2001 From: victorgao Date: Fri, 25 Apr 2025 09:27:16 +0800 Subject: [PATCH 56/56] fix binary log. --- system_test/fixture.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/system_test/fixture.py b/system_test/fixture.py index 40dca629..19d6a50e 100644 --- a/system_test/fixture.py +++ b/system_test/fixture.py @@ -414,7 +414,8 @@ def check_table(): except TimeoutError as toe: if log: if log_ctx: - log_ctx = f'\n{textwrap.indent(log_ctx, " ")}\n' + log_ctx_str = log_ctx.decode('utf-8', errors='replace') + log_ctx = f'\n{textwrap.indent(log_ctx_str, " ")}\n' sys.stderr.write( f'Timed out after {timeout_sec} seconds ' + f'waiting for query {sql_query!r}. ' +