From 008c5ed9541b687a363f1e0a920872b888cdc35d Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Thu, 16 Oct 2025 15:23:59 +0200 Subject: [PATCH 01/24] refactor: move the example-aot project to the proper folder --- {example-aot => src/example-aot}/Program.cs | 0 {example-aot => src/example-aot}/example-aot.csproj | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {example-aot => src/example-aot}/Program.cs (100%) rename {example-aot => src/example-aot}/example-aot.csproj (100%) diff --git a/example-aot/Program.cs b/src/example-aot/Program.cs similarity index 100% rename from example-aot/Program.cs rename to src/example-aot/Program.cs diff --git a/example-aot/example-aot.csproj b/src/example-aot/example-aot.csproj similarity index 100% rename from example-aot/example-aot.csproj rename to src/example-aot/example-aot.csproj From 31f29727366cc9531d1c1af0062c775fc2b93c1a Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Thu, 16 Oct 2025 15:24:35 +0200 Subject: [PATCH 02/24] feat: add BufferV3 and support for DECIMAL column values in IBuffer interface --- src/net-questdb-client/Buffers/Buffer.cs | 18 ++- src/net-questdb-client/Buffers/BufferV1.cs | 53 +++++---- src/net-questdb-client/Buffers/BufferV2.cs | 19 ++- src/net-questdb-client/Buffers/BufferV3.cs | 127 +++++++++++++++++++++ src/net-questdb-client/Buffers/IBuffer.cs | 11 +- 5 files changed, 182 insertions(+), 46 deletions(-) create mode 100644 src/net-questdb-client/Buffers/BufferV3.cs diff --git a/src/net-questdb-client/Buffers/Buffer.cs b/src/net-questdb-client/Buffers/Buffer.cs index f801be9..77b92aa 100644 --- a/src/net-questdb-client/Buffers/Buffer.cs +++ b/src/net-questdb-client/Buffers/Buffer.cs @@ -42,15 +42,13 @@ public static class Buffer /// public static IBuffer Create(int bufferSize, int maxNameLen, int maxBufSize, ProtocolVersion version) { - switch (version) + return version switch { - case ProtocolVersion.V1: - return new BufferV1(bufferSize, maxNameLen, maxBufSize); - case ProtocolVersion.V2: - case ProtocolVersion.Auto: - return new BufferV2(bufferSize, maxNameLen, maxBufSize); - } - - throw new NotImplementedException(); + ProtocolVersion.V1 => new BufferV1(bufferSize, maxNameLen, maxBufSize), + ProtocolVersion.V2 => new BufferV2(bufferSize, maxNameLen, maxBufSize), + ProtocolVersion.V3 => new BufferV3(bufferSize, maxNameLen, maxBufSize), + ProtocolVersion.Auto => new BufferV2(bufferSize, maxNameLen, maxBufSize), + _ => throw new NotImplementedException(), + }; } -} \ No newline at end of file +} diff --git a/src/net-questdb-client/Buffers/BufferV1.cs b/src/net-questdb-client/Buffers/BufferV1.cs index 9bd224f..922ecd4 100644 --- a/src/net-questdb-client/Buffers/BufferV1.cs +++ b/src/net-questdb-client/Buffers/BufferV1.cs @@ -136,17 +136,17 @@ public void AtNanos(long timestampNanos) public void Clear() { _currentBufferIndex = 0; - Chunk = _buffers[_currentBufferIndex].Buffer; + Chunk = _buffers[_currentBufferIndex].Buffer; for (var i = 0; i < _buffers.Count; i++) { _buffers[i] = (_buffers[i].Buffer, 0); } - Position = 0; - RowCount = 0; - Length = 0; - WithinTransaction = false; - _currentTableName = ""; + Position = 0; + RowCount = 0; + Length = 0; + WithinTransaction = false; + _currentTableName = ""; _lineStartBufferIndex = 0; _lineStartBufferPosition = 0; } @@ -164,10 +164,10 @@ public void TrimExcessBuffers() /// public void CancelRow() { - _currentBufferIndex = _lineStartBufferIndex; - Length -= Position - _lineStartBufferPosition; - Position = _lineStartBufferPosition; - _hasTable = false; + _currentBufferIndex = _lineStartBufferIndex; + Length -= Position - _lineStartBufferPosition; + Position = _lineStartBufferPosition; + _hasTable = false; } /// @@ -236,10 +236,10 @@ public IBuffer Table(ReadOnlySpan name) GuardTableAlreadySet(); GuardInvalidTableName(name); - _quoted = false; + _quoted = false; _hasTable = true; - _lineStartBufferIndex = _currentBufferIndex; + _lineStartBufferIndex = _currentBufferIndex; _lineStartBufferPosition = Position; EncodeUtf8(name); @@ -394,14 +394,14 @@ public IBuffer Put(long value) new ArgumentOutOfRangeException()); } - Span num = stackalloc byte[20]; - var pos = num.Length; - var remaining = Math.Abs(value); + Span num = stackalloc byte[20]; + var pos = num.Length; + var remaining = Math.Abs(value); do { var digit = remaining % 10; - num[--pos] = (byte)('0' + digit); - remaining /= 10; + num[--pos] = (byte)('0' + digit); + remaining /= 10; } while (remaining != 0); if (value < 0) @@ -414,7 +414,7 @@ public IBuffer Put(long value) num.Slice(pos, len).CopyTo(Chunk.AsSpan(Position)); Position += len; - Length += len; + Length += len; return this; } @@ -457,7 +457,7 @@ public IBuffer Put(byte value) internal void Advance(int by) { Position += by; - Length += by; + Length += by; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -474,8 +474,8 @@ private void FinishLine() { PutAscii('\n'); RowCount++; - _hasTable = false; - _noFields = true; + _hasTable = false; + _noFields = true; _noSymbols = true; GuardExceededMaxBufferSize(); } @@ -540,9 +540,9 @@ private void PutUtf8(char c) NextBuffer(); } - var bytes = Chunk.AsSpan(Position); - Span chars = stackalloc char[1] { c, }; - var byteLength = Encoding.UTF8.GetBytes(chars, bytes); + var bytes = Chunk.AsSpan(Position); + Span chars = stackalloc char[1] { c, }; + var byteLength = Encoding.UTF8.GetBytes(chars, bytes); Advance(byteLength); } @@ -789,4 +789,9 @@ private void GuardFsFileNameLimit(ReadOnlySpan name) $"Name is too long, must be under {_maxNameLen} bytes."); } } + + public virtual IBuffer Column(ReadOnlySpan name, decimal? value) + { + throw new IngressError(ErrorCode.ProtocolVersionError, "Protocol Version does not support DECIMAL types"); + } } \ No newline at end of file diff --git a/src/net-questdb-client/Buffers/BufferV2.cs b/src/net-questdb-client/Buffers/BufferV2.cs index 67a8f18..3a3734b 100644 --- a/src/net-questdb-client/Buffers/BufferV2.cs +++ b/src/net-questdb-client/Buffers/BufferV2.cs @@ -95,10 +95,9 @@ private void PutBinaryLE(T value) where T : struct { var size = Marshal.SizeOf(); EnsureCapacity(size); - var length = Marshal.SizeOf(); - var mem = MemoryMarshal.Cast(Chunk.AsSpan(Position, length)); + var mem = MemoryMarshal.Cast(Chunk.AsSpan(Position, size)); mem[0] = value; - Advance(length); + Advance(size); } // ReSharper disable once InconsistentNaming @@ -114,16 +113,16 @@ private void PutBinaryBE(T value) where T : struct // ReSharper disable once InconsistentNaming private void PutBinaryManyLE(ReadOnlySpan value) where T : struct { - var srcSpan = MemoryMarshal.Cast(value); + var srcSpan = MemoryMarshal.Cast(value); var byteSize = Marshal.SizeOf(); while (srcSpan.Length > 0) { - var dstLength = GetSpareCapacity(); // length + var dstLength = GetSpareCapacity(); // length if (dstLength < byteSize) { NextBuffer(); - dstLength = GetSpareCapacity(); + dstLength = GetSpareCapacity(); } var availLength = dstLength - dstLength % byteSize; // rounded length @@ -133,7 +132,7 @@ private void PutBinaryManyLE(ReadOnlySpan value) where T : struct Advance(srcSpan.Length); return; } - var dstSpan = Chunk.AsSpan(Position, availLength); + var dstSpan = Chunk.AsSpan(Position, availLength); srcSpan.Slice(0, availLength).CopyTo(dstSpan); Advance(availLength); srcSpan = srcSpan.Slice(availLength); @@ -180,7 +179,7 @@ public override IBuffer Column(ReadOnlySpan name, ReadOnlySpan value return PutDoubleArray(name, value); } - private IBuffer PutDoubleArray(ReadOnlySpan name, ReadOnlySpan value) where T : struct + private IBuffer PutDoubleArray(ReadOnlySpan name, ReadOnlySpan value) where T : struct { SetTableIfAppropriate(); PutArrayOfDoubleHeader(name); @@ -199,7 +198,7 @@ public override IBuffer Column(ReadOnlySpan name, Array? value) // The value is null, do not include the column in the message return this; } - + var type = value.GetType().GetElementType(); GuardAgainstNonDoubleTypes(type ?? throw new InvalidOperationException()); if (value.Rank == 1) @@ -207,7 +206,7 @@ public override IBuffer Column(ReadOnlySpan name, Array? value) // Fast path, one dim array return PutDoubleArray(name, (ReadOnlySpan)value!); } - + SetTableIfAppropriate(); PutArrayOfDoubleHeader(name); diff --git a/src/net-questdb-client/Buffers/BufferV3.cs b/src/net-questdb-client/Buffers/BufferV3.cs new file mode 100644 index 0000000..85e5331 --- /dev/null +++ b/src/net-questdb-client/Buffers/BufferV3.cs @@ -0,0 +1,127 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +using System.Buffers.Binary; +using System.Runtime.InteropServices; +using QuestDB.Enums; +using QuestDB.Utils; + +namespace QuestDB.Buffers; + +/// +public class BufferV3 : BufferV2 +{ + /// + public BufferV3(int bufferSize, int maxNameLen, int maxBufSize) : base(bufferSize, maxNameLen, maxBufSize) + { + } + + // Sign mask for the flags field. A value of zero in this bit indicates a + // positive Decimal value, and a value of one in this bit indicates a + // negative Decimal value. + private const int SignMask = unchecked((int)0x80000000); + + // Scale mask for the flags field. This byte in the flags field contains + // the power of 10 to divide the Decimal value by. The scale byte must + // contain a value between 0 and 28 inclusive. + private const int ScaleMask = 0x00FF0000; + + // Number of bits scale is shifted by. + private const int ScaleShift = 16; + + /// + public override IBuffer Column(ReadOnlySpan name, decimal? value) + { + // # Binary Format + // 1. Binary format marker: `'='` (0x3D) + // 2. Type identifier: BinaryFormatType.DECIMAL byte + // 3. Scale: 1 byte (0-76 inclusive) - number of decimal places + // 4. Length: 1 byte - number of bytes in the unscaled value + // 5. Unscaled value: variable-length byte array in two's complement format, big-endian + SetTableIfAppropriate(); + Column(name) + .PutAscii(Constants.BINARY_FORMAT_FLAG) + .Put((byte)BinaryFormatType.DECIMAL); + if (value is null) + { + Put((byte)0); // Scale + Put((byte)0); // Length + return this; + } + + Span parts = stackalloc int[4]; + decimal.GetBits(value.Value, parts); + + int flags = parts[3]; + byte scale = (byte)((flags & ScaleMask) >> ScaleShift); + + // 3. Scale + Put(scale); + + int low = parts[0]; + int mid = parts[1]; + int high = parts[2]; + bool bitSign = false; + bool negative = (flags & SignMask) != 0; + + if (negative) + { + // QuestDB expects negative mantissas in two's complement. + low = ~low + 1; + int c = low == 0 ? 1 : 0; + mid = ~mid + c; + c = mid == 0 && c == 1 ? 1 : 0; + high = ~high + c; + // We may overflow, we need an extra byte to convey the sign. + bitSign = high == 0 && c == 1; + } + else if ((high & 0x80000000) != 0) + { + // If the highest bit is set, we need an extra byte of 0 to convey the sign. + bitSign = true; + } + + var size = bitSign ? 13 : 12; + + // 4. Length + Put((byte)size); + + // 5. Unscaled value + EnsureCapacity(size); + var span = Chunk.AsSpan(Position, size); + var offset = 0; + if (bitSign) + { + span[offset++] = (byte)(negative ? 255 : 0); + } + BinaryPrimitives.WriteInt32BigEndian(span.Slice(offset, 4), high); + offset += 4; + BinaryPrimitives.WriteInt32BigEndian(span.Slice(offset, 4), mid); + offset += 4; + BinaryPrimitives.WriteInt32BigEndian(span.Slice(offset, 4), low); + Advance(size); + + return this; + } +} diff --git a/src/net-questdb-client/Buffers/IBuffer.cs b/src/net-questdb-client/Buffers/IBuffer.cs index 82bfdfe..80fe69b 100644 --- a/src/net-questdb-client/Buffers/IBuffer.cs +++ b/src/net-questdb-client/Buffers/IBuffer.cs @@ -230,7 +230,14 @@ public interface IBuffer /// public IBuffer Column(ReadOnlySpan name, Array? value); - + /// public IBuffer Column(ReadOnlySpan name, IEnumerable value, IEnumerable shape) where T : struct; -} \ No newline at end of file + + /// + /// Records a DECIMAL column value using the ILP binary decimal layout: + /// '=' marker, decimal type id (23), scale byte, mantissa length, and a big-endian + /// two's complement mantissa sourced from . + /// + public IBuffer Column(ReadOnlySpan name, decimal? value); +} From e509af798430fd53334eb6807ebbbb62e5749d03 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 14:43:46 +0200 Subject: [PATCH 03/24] feat: add support for decimal --- README.md | 67 +-- net-questdb-client.sln | 8 +- src/dummy-http-server/DummyHttpServer.cs | 26 +- src/example-aot/example-aot.csproj | 2 +- .../BenchInserts.cs | 2 +- .../net-questdb-client-benchmarks.csproj | 2 +- .../DecimalTestHelpers.cs | 71 +++ .../DummyIlpServer.cs | 28 +- src/net-questdb-client-tests/HttpTests.cs | 544 ++++++++++-------- .../JsonSpecTestRunner.cs | 198 ++++--- .../LineTcpSenderTests.cs | 208 +++---- src/net-questdb-client-tests/TcpTests.cs | 339 ++++++----- src/net-questdb-client/Buffers/BufferV3.cs | 58 +- src/net-questdb-client/Buffers/IBuffer.cs | 6 +- .../Enums/BinaryFormatType.cs | 1 + .../Enums/ProtocolVersion.cs | 7 +- .../Senders/AbstractSender.cs | 13 + src/net-questdb-client/Senders/HttpSender.cs | 52 +- src/net-questdb-client/Senders/ISender.cs | 55 +- src/net-questdb-client/Senders/ISenderV1.cs | 34 ++ src/net-questdb-client/Senders/ISenderV2.cs | 38 ++ src/tcp-client-test/DummyIlpServer.cs | 239 -------- src/tcp-client-test/JsonSpecTestRunner.cs | 202 ------- src/tcp-client-test/certificate.pfx | Bin 2341 -> 0 bytes src/tcp-client-test/tcp-client-test.csproj | 38 -- 25 files changed, 1005 insertions(+), 1233 deletions(-) create mode 100644 src/net-questdb-client-tests/DecimalTestHelpers.cs rename src/{tcp-client-test => net-questdb-client-tests}/LineTcpSenderTests.cs (80%) create mode 100644 src/net-questdb-client/Senders/ISenderV1.cs create mode 100644 src/net-questdb-client/Senders/ISenderV2.cs delete mode 100644 src/tcp-client-test/DummyIlpServer.cs delete mode 100644 src/tcp-client-test/JsonSpecTestRunner.cs delete mode 100644 src/tcp-client-test/certificate.pfx delete mode 100644 src/tcp-client-test/tcp-client-test.csproj diff --git a/README.md b/README.md index 05293d7..be67233 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ using var sender = Sender.New("http::addr=localhost:9000;"); await sender.Table("trades") .Symbol("symbol", "ETH-USD") .Symbol("side", "sell") - .Column("price", 2615.54) + .Column("price", 2615.54m) .Column("amount", 0.00044) .AtAsync(new DateTime(2021, 11, 25, 0, 46, 26)); await sender.SendAsync(); @@ -60,7 +60,7 @@ for(int i = 0; i < 100; i++) sender.Table("trades") .Symbol("symbol", "ETH-USD") .Symbol("side", "sell") - .Column("price", 2615.54) + .Column("price", 2615.54m) .Column("amount", 0.00044) .At(DateTime.UtcNow); } @@ -136,53 +136,54 @@ The config string format is: {http/https/tcp/tcps}::addr={host}:{port};key1=val1;key2=val2;keyN=valN; ``` -| Name | Default | Description | -|--------------------------|----------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `protocol` (schema) | `http` | The transport protocol to use. Options are http(s)/tcp(s). | -| `addr` | `localhost:9000` | The {host}:{port} pair denoting the QuestDB server. By default, port 9000 for HTTP, port 9009 for TCP. | -| `auto_flush` | `on` | Enables or disables auto-flushing functionality. By default, the buffer will be flushed every 75,000 rows, or every 1000ms, whichever comes first. | -| `auto_flush_rows` | `75000 (HTTP)` `600 (TCP)` | The row count after which the buffer will be flushed. Effectively a batch size. | -| `auto_flush_bytes` | `Int.MaxValue` | The byte buffer length which when exceeded, will trigger a flush. | -| `auto_flush_interval` | `1000` | The millisecond interval, which once has elapsed, the next row triggers a flush. | -| `init_buf_size` | `65536` | The starting byte buffer length. Overflowing this buffer will cause the allocation `init_buf_size` bytes (an additional buffer). | -| `max_buf_size` | `104857600` | Maximum size of the byte buffer in bytes. If exceeded, an exception will be thrown. | -| `username` | | The username for authentication. Used for Basic Authentication and TCP JWK Authentication. | -| `password` | | The password for authentication. Used for Basic Authentication. | -| `token` | | The token for authentication. Used for Token Authentication and TCP JWK Authentication. | -| `token_x` | | Un-used. | -| `token_y` | | Un-used. | -| `tls_verify` | `on` | Denotes whether TLS certificates should or should not be verifed. Options are on/unsafe_off. | -| `tls_ca` | | Un-used. | -| `tls_roots` | | Used to specify the filepath for a custom .pem certificate. | -| `tls_roots_password` | | Used to specify the filepath for the private key/password corresponding to the `tls_roots` certificate. | -| `auth_timeout` | `15000` | The time period to wait for authenticating requests, in milliseconds. | -| `request_timeout` | `10000` | Base timeout for HTTP requests before any additional time is added. | -| `request_min_throughput` | `102400` | Expected minimum throughput of requests in bytes per second. Used to add additional time to `request_timeout` to prevent large requests timing out prematurely. | -| `retry_timeout` | `10000` | The time period during which retries will be attempted, in milliseconds. | -| `max_name_len` | `127` | The maximum allowed bytes, in UTF-8 format, for column and table names. | -| `protocol_version` | | Explicitly specifies the version of InfluxDB Line Protocol to use for sender. Valid options are:
• protocol_version=1
• protocol_version=2
• protocol_version=auto (default, if unspecified) | +| Name | Default | Description | +| ------------------------ | -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `protocol` (schema) | `http` | The transport protocol to use. Options are http(s)/tcp(s). | +| `addr` | `localhost:9000` | The {host}:{port} pair denoting the QuestDB server. By default, port 9000 for HTTP, port 9009 for TCP. | +| `auto_flush` | `on` | Enables or disables auto-flushing functionality. By default, the buffer will be flushed every 75,000 rows, or every 1000ms, whichever comes first. | +| `auto_flush_rows` | `75000 (HTTP)` `600 (TCP)` | The row count after which the buffer will be flushed. Effectively a batch size. | +| `auto_flush_bytes` | `Int.MaxValue` | The byte buffer length which when exceeded, will trigger a flush. | +| `auto_flush_interval` | `1000` | The millisecond interval, which once has elapsed, the next row triggers a flush. | +| `init_buf_size` | `65536` | The starting byte buffer length. Overflowing this buffer will cause the allocation `init_buf_size` bytes (an additional buffer). | +| `max_buf_size` | `104857600` | Maximum size of the byte buffer in bytes. If exceeded, an exception will be thrown. | +| `username` | | The username for authentication. Used for Basic Authentication and TCP JWK Authentication. | +| `password` | | The password for authentication. Used for Basic Authentication. | +| `token` | | The token for authentication. Used for Token Authentication and TCP JWK Authentication. | +| `token_x` | | Un-used. | +| `token_y` | | Un-used. | +| `tls_verify` | `on` | Denotes whether TLS certificates should or should not be verifed. Options are on/unsafe_off. | +| `tls_ca` | | Un-used. | +| `tls_roots` | | Used to specify the filepath for a custom .pem certificate. | +| `tls_roots_password` | | Used to specify the filepath for the private key/password corresponding to the `tls_roots` certificate. | +| `auth_timeout` | `15000` | The time period to wait for authenticating requests, in milliseconds. | +| `request_timeout` | `10000` | Base timeout for HTTP requests before any additional time is added. | +| `request_min_throughput` | `102400` | Expected minimum throughput of requests in bytes per second. Used to add additional time to `request_timeout` to prevent large requests timing out prematurely. | +| `retry_timeout` | `10000` | The time period during which retries will be attempted, in milliseconds. | +| `max_name_len` | `127` | The maximum allowed bytes, in UTF-8 format, for column and table names. | +| `protocol_version` | | Explicitly specifies the version of InfluxDB Line Protocol to use for sender. Valid options are:
• protocol_version=1
• protocol_version=2
• protocol_version=3
• protocol_version=auto (default, if unspecified) | ### Protocol Version Behavior details: -| Value | Behavior | -|--------|---------------------------------------------------------------------------------------------------------------------------------------------------------| -| 1 | - Plain text serialization
- Compatible with InfluxDB servers
- No array type support | -| 2 | - Binary encoding for double arrays
- Full support for array | +| Value | Behavior | +| ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | - Plain text serialization
- Compatible with InfluxDB servers
- No array type support | +| 2 | - Binary encoding for double arrays
- Full support for array | +| 3 | - Support for decimal | | `auto` | - **HTTP/HTTPS**: Auto-detects server capability during handshake (supports version negotiation)
- **TCP/TCPS**: Defaults to version 1 for compatibility | ### net-questdb-client specific parameters | Name | Default | Description | -|----------------|----------|---------------------------------------------------------------------------------------| +| -------------- | -------- | ------------------------------------------------------------------------------------- | | `own_socket` | `true` | Specifies whether the internal TCP data stream will own the underlying socket or not. | | `pool_timeout` | `120000` | Sets the idle timeout for HTTP connections in SocketsHttpHandler. | ## Properties and methods | Name | Returns | Description | -|-------------------------------------------------------------------------------------------------------|-----------------|----------------------------------------------------------------------------| +| ----------------------------------------------------------------------------------------------------- | --------------- | -------------------------------------------------------------------------- | | `Length` | `int` | Current length in bytes of the buffer (not capacity!) | | `RowCount` | `int` | Current row count of the buffer | | `LastFlush` | `DateTime` | Returns the UTC DateTime of the last flush sending data to the server. | diff --git a/net-questdb-client.sln b/net-questdb-client.sln index 8f9e3c5..91cec48 100644 --- a/net-questdb-client.sln +++ b/net-questdb-client.sln @@ -5,8 +5,6 @@ VisualStudioVersion = 16.0.30114.105 MinimumVisualStudioVersion = 10.0.40219.1 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "net-questdb-client", "src\net-questdb-client\net-questdb-client.csproj", "{456B1860-0102-48D7-861A-5F9963F3887B}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "tcp-client-test", "src\tcp-client-test\tcp-client-test.csproj", "{22F903D9-4367-46A2-A25A-F4A6BF9105C6}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "example-basic", "src\example-basic\example-basic.csproj", "{121EAA4D-3A73-468C-8CAB-A2A4BEF848CF}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "example-auth-tls", "src\example-auth-tls\example-auth-tls.csproj", "{FBB8181C-6BAB-46C2-A47A-D3566A3997FE}" @@ -21,7 +19,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "example-streaming", "src\ex EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "example-auth-http-tls", "src\example-auth-http-tls\example-auth-http-tls.csproj", "{24D93DBB-3783-423F-81CC-6B9BFD33F6CD}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "example-aot", "example-aot\example-aot.csproj", "{5341FCF0-F71D-4160-8D6E-B5EFDF92E9E8}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "example-aot", "src\example-aot\example-aot.csproj", "{5341FCF0-F71D-4160-8D6E-B5EFDF92E9E8}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -36,10 +34,6 @@ Global {456B1860-0102-48D7-861A-5F9963F3887B}.Debug|Any CPU.Build.0 = Debug|Any CPU {456B1860-0102-48D7-861A-5F9963F3887B}.Release|Any CPU.ActiveCfg = Release|Any CPU {456B1860-0102-48D7-861A-5F9963F3887B}.Release|Any CPU.Build.0 = Release|Any CPU - {22F903D9-4367-46A2-A25A-F4A6BF9105C6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {22F903D9-4367-46A2-A25A-F4A6BF9105C6}.Debug|Any CPU.Build.0 = Debug|Any CPU - {22F903D9-4367-46A2-A25A-F4A6BF9105C6}.Release|Any CPU.ActiveCfg = Release|Any CPU - {22F903D9-4367-46A2-A25A-F4A6BF9105C6}.Release|Any CPU.Build.0 = Release|Any CPU {121EAA4D-3A73-468C-8CAB-A2A4BEF848CF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {121EAA4D-3A73-468C-8CAB-A2A4BEF848CF}.Debug|Any CPU.Build.0 = Debug|Any CPU {121EAA4D-3A73-468C-8CAB-A2A4BEF848CF}.Release|Any CPU.ActiveCfg = Release|Any CPU diff --git a/src/dummy-http-server/DummyHttpServer.cs b/src/dummy-http-server/DummyHttpServer.cs index e11ae42..defac75 100644 --- a/src/dummy-http-server/DummyHttpServer.cs +++ b/src/dummy-http-server/DummyHttpServer.cs @@ -53,10 +53,10 @@ public DummyHttpServer(bool withTokenAuth = false, bool withBasicAuth = false, b .AddConsole(); }); - IlpEndpoint.WithTokenAuth = withTokenAuth; - IlpEndpoint.WithBasicAuth = withBasicAuth; + IlpEndpoint.WithTokenAuth = withTokenAuth; + IlpEndpoint.WithBasicAuth = withBasicAuth; IlpEndpoint.WithRetriableError = withRetriableError; - IlpEndpoint.WithErrorMessage = withErrorMessage; + IlpEndpoint.WithErrorMessage = withErrorMessage; _withStartDelay = withStartDelay; if (withTokenAuth) @@ -103,7 +103,7 @@ public void Clear() IlpEndpoint.ReceiveBuffer.Clear(); IlpEndpoint.ReceiveBytes.Clear(); IlpEndpoint.LastError = null; - IlpEndpoint.Counter = 0; + IlpEndpoint.Counter = 0; } public async Task StartAsync(int port = 29743, int[]? versions = null) @@ -112,10 +112,10 @@ public async Task StartAsync(int port = 29743, int[]? versions = null) { await Task.Delay(_withStartDelay.Value); } - versions ??= new[] { 1, 2, }; - SettingsEndpoint.Versions = versions; - _port = port; - _app.RunAsync($"http://localhost:{port}"); + versions ??= new[] { 1, 2, 3, }; + SettingsEndpoint.Versions = versions; + _port = port; + _ = _app.RunAsync($"http://localhost:{port}"); } public async Task RunAsync() @@ -133,7 +133,7 @@ public StringBuilder GetReceiveBuffer() return IlpEndpoint.ReceiveBuffer; } - public List GetReceiveBytes() + public List GetReceivedBytes() { return IlpEndpoint.ReceiveBytes; } @@ -157,7 +157,7 @@ public async Task Healthcheck() var jwtToken = JwtBearer.CreateToken(o => { o.SigningKey = SigningKey; - o.ExpireAt = DateTime.UtcNow.AddDays(1); + o.ExpireAt = DateTime.UtcNow.AddDays(1); }); return jwtToken; } @@ -172,8 +172,8 @@ public int GetCounter() public string PrintBuffer() { - var bytes = GetReceiveBytes().ToArray(); - var sb = new StringBuilder(); + var bytes = GetReceivedBytes().ToArray(); + var sb = new StringBuilder(); var lastAppend = 0; var i = 0; @@ -242,7 +242,7 @@ public string PrintBuffer() i--; break; default: - throw new NotImplementedException(); + throw new NotImplementedException($"Type {bytes[i]} not implemented"); } lastAppend = i + 1; diff --git a/src/example-aot/example-aot.csproj b/src/example-aot/example-aot.csproj index 28bdf22..b442655 100644 --- a/src/example-aot/example-aot.csproj +++ b/src/example-aot/example-aot.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/net-questdb-client-benchmarks/BenchInserts.cs b/src/net-questdb-client-benchmarks/BenchInserts.cs index 4f49aa5..b68144c 100644 --- a/src/net-questdb-client-benchmarks/BenchInserts.cs +++ b/src/net-questdb-client-benchmarks/BenchInserts.cs @@ -26,7 +26,7 @@ using BenchmarkDotNet.Attributes; using dummy_http_server; using QuestDB; -using tcp_client_test; +using net_questdb_client_tests; #pragma warning disable CS0414 // Field is assigned but its value is never used diff --git a/src/net-questdb-client-benchmarks/net-questdb-client-benchmarks.csproj b/src/net-questdb-client-benchmarks/net-questdb-client-benchmarks.csproj index 58dac62..304cac6 100644 --- a/src/net-questdb-client-benchmarks/net-questdb-client-benchmarks.csproj +++ b/src/net-questdb-client-benchmarks/net-questdb-client-benchmarks.csproj @@ -15,8 +15,8 @@ + - diff --git a/src/net-questdb-client-tests/DecimalTestHelpers.cs b/src/net-questdb-client-tests/DecimalTestHelpers.cs new file mode 100644 index 0000000..2f16fcd --- /dev/null +++ b/src/net-questdb-client-tests/DecimalTestHelpers.cs @@ -0,0 +1,71 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +using System.Text; +using NUnit.Framework; +using QuestDB.Enums; + +namespace net_questdb_client_tests; + +internal static class DecimalTestHelpers +{ + internal static void AssertDecimalField(ReadOnlySpan buffer, + string columnName, + byte expectedScale, + ReadOnlySpan expectedMantissa) + { + var payload = ExtractDecimalPayload(buffer, columnName); + Assert.That(payload.Length, + Is.GreaterThanOrEqualTo(4 + expectedMantissa.Length), + $"Decimal field `{columnName}` payload shorter than expected."); + Assert.That(payload[0], Is.EqualTo((byte)'=')); + Assert.That(payload[1], Is.EqualTo((byte)BinaryFormatType.DECIMAL)); + Assert.That(payload[2], Is.EqualTo(expectedScale), $"Unexpected scale for `{columnName}`."); + Assert.That(payload[3], + Is.EqualTo((byte)expectedMantissa.Length), + $"Unexpected mantissa length for `{columnName}`."); + CollectionAssert.AreEqual(expectedMantissa.ToArray(), payload.Slice(4, expectedMantissa.Length).ToArray(), + $"Mantissa bytes for `{columnName}` did not match expectation."); + } + + internal static void AssertDecimalNullField(ReadOnlySpan buffer, string columnName) + { + var payload = ExtractDecimalPayload(buffer, columnName); + Assert.That(payload.Length, + Is.GreaterThanOrEqualTo(4), + $"Decimal field `{columnName}` payload shorter than expected."); + Assert.That(payload[0], Is.EqualTo((byte)'=')); + Assert.That(payload[1], Is.EqualTo((byte)BinaryFormatType.DECIMAL)); + Assert.That(payload[2], Is.EqualTo(0), $"Unexpected scale for `{columnName}`."); + Assert.That(payload[3], Is.EqualTo(0), $"Unexpected mantissa length for `{columnName}`."); + } + + private static ReadOnlySpan ExtractDecimalPayload(ReadOnlySpan buffer, string columnName) + { + var prefix = Encoding.ASCII.GetBytes($"{columnName}="); + var index = buffer.IndexOf(prefix.AsSpan()); + Assert.That(index, Is.GreaterThanOrEqualTo(0), $"Column `{columnName}` not found in payload."); + return buffer[(index + prefix.Length)..]; + } +} \ No newline at end of file diff --git a/src/net-questdb-client-tests/DummyIlpServer.cs b/src/net-questdb-client-tests/DummyIlpServer.cs index e812a6e..f145836 100644 --- a/src/net-questdb-client-tests/DummyIlpServer.cs +++ b/src/net-questdb-client-tests/DummyIlpServer.cs @@ -51,7 +51,7 @@ public class DummyIlpServer : IDisposable public DummyIlpServer(int port, bool tls) { - _tls = tls; + _tls = tls; _server = new TcpListener(IPAddress.Loopback, port); _server.Start(); } @@ -77,7 +77,7 @@ private async Task AcceptConnections() using var socket = await _server.AcceptSocketAsync(); clientSocket = socket; await using var connection = new NetworkStream(socket, true); - Stream dataStream = connection; + Stream dataStream = connection; if (_tls) { var sslStream = new SslStream(connection); @@ -137,7 +137,7 @@ private async Task RunServerAuth(Stream connection) var pubKey1 = FromBase64String(_publicKeyX); var pubKey2 = FromBase64String(_publicKeyY); - var p = SecNamedCurves.GetByName("secp256r1"); + var p = SecNamedCurves.GetByName("secp256r1"); var parameters = new ECDomainParameters(p.Curve, p.G, p.N, p.H); // Verify the signature @@ -168,8 +168,8 @@ private static string Pad(string text) public static byte[] FromBase64String(string encodedPrivateKey) { var replace = encodedPrivateKey - .Replace('-', '+') - .Replace('_', '/'); + .Replace('-', '+') + .Replace('_', '/'); return Convert.FromBase64String(Pad(replace)); } @@ -178,7 +178,7 @@ private async Task ReceiveUntilEol(Stream connection) var len = 0; while (true) { - var n = await connection.ReadAsync(_buffer.AsMemory(len)); + var n = await connection.ReadAsync(_buffer.AsMemory(len)); var inBuffer = len + n; for (var i = len; i < inBuffer; i++) { @@ -226,13 +226,17 @@ private async Task SaveData(Stream connection, Socket socket) public string GetTextReceived() { return PrintBuffer(); - // return Encoding.UTF8.GetString(_received.GetBuffer(), 0, (int)_received.Length); + } + + public byte[] GetReceivedBytes() + { + return _received.ToArray(); } public string PrintBuffer() { - var bytes = _received.GetBuffer().AsSpan().Slice(0, (int)_received.Length).ToArray(); - var sb = new StringBuilder(); + var bytes = _received.ToArray(); + var sb = new StringBuilder(); var lastAppend = 0; var i = 0; @@ -258,7 +262,7 @@ public string PrintBuffer() for (var j = 0; j < dims; j++) { var lengthBytes = bytes.AsSpan()[i..(i + 4)]; - var _length = MemoryMarshal.Cast(lengthBytes)[0]; + var _length = MemoryMarshal.Cast(lengthBytes)[0]; if (length == 0) { length = _length; @@ -301,7 +305,7 @@ public string PrintBuffer() i--; break; default: - throw new NotImplementedException(); + throw new NotImplementedException("Unknown type: " + bytes[i]); } lastAppend = i + 1; @@ -315,7 +319,7 @@ public string PrintBuffer() public void WithAuth(string keyId, string publicKeyX, string publicKeyY) { - _keyId = keyId; + _keyId = keyId; _publicKeyX = publicKeyX; _publicKeyY = publicKeyY; } diff --git a/src/net-questdb-client-tests/HttpTests.cs b/src/net-questdb-client-tests/HttpTests.cs index 3e76c0c..9d58527 100644 --- a/src/net-questdb-client-tests/HttpTests.cs +++ b/src/net-questdb-client-tests/HttpTests.cs @@ -1,4 +1,3 @@ -// ReSharper disable CommentTypo /******************************************************************************* * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) @@ -47,33 +46,33 @@ public async Task BasicArrayDouble() Sender.New( $"http::addr={Host}:{HttpPort};username=asdasdada;password=asdadad;tls_verify=unsafe_off;auto_flush=off;"); await sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc") - .Column("array", new[] - { - 1.2, 2.6, - 3.1, - }) - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc") + .Column("array", new[] + { + 1.2, 2.6, + 3.1, + }) + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc") - .Column("array", (ReadOnlySpan)new[] - { - 1.5, 2.1, - 3.1, - }) - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 2)); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc") + .Column("array", (ReadOnlySpan)new[] + { + 1.5, 2.1, + 3.1, + }) + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 2)); await sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc") - .Column("array", (ReadOnlySpan)Array.Empty()) - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 3)); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc") + .Column("array", (ReadOnlySpan)Array.Empty()) + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 3)); await sender.SendAsync(); Assert.That( @@ -84,6 +83,58 @@ await sender.Table("metrics") await server.StopAsync(); } + [Test] + public async Task DecimalColumns() + { + using var server = new DummyHttpServer(withBasicAuth: false); + await server.StartAsync(HttpPort, new[] + { + 1, 2, + 3, + }); + using var sender = + Sender.New( + $"http::addr={Host}:{HttpPort};protocol_version=3;tls_verify=unsafe_off;auto_flush=off;"); + + await sender.Table("metrics") + .Symbol("tag", "value") + .Column("dec_pos", 123.45m) + .Column("dec_neg", -123.45m) + .Column("dec_null", (decimal?)null) + .Column("dec_max", decimal.MaxValue) + .Column("dec_min", decimal.MinValue) + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + + await sender.SendAsync(); + + var buffer = server.GetReceivedBytes().ToArray(); + DecimalTestHelpers.AssertDecimalField(buffer, "dec_pos", 2, new byte[] + { + 0x30, 0x39, + }); + DecimalTestHelpers.AssertDecimalField(buffer, "dec_neg", 2, new byte[] + { + 0xCF, 0xC7, + }); + DecimalTestHelpers.AssertDecimalNullField(buffer, "dec_null"); + DecimalTestHelpers.AssertDecimalField(buffer, "dec_max", 0, new byte[] + { + 0x00, + 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, + }); + DecimalTestHelpers.AssertDecimalField(buffer, "dec_min", 0, new byte[] + { + 0xFF, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, + }); + + await server.StopAsync(); + } + [Test] public async Task SendLongArrayAsSpan() { @@ -94,12 +145,12 @@ public async Task SendLongArrayAsSpan() $"http::addr={Host}:{HttpPort};init_buf_size=256;username=asdasdada;password=asdadad;tls_verify=unsafe_off;auto_flush=off;"); sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc"); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc"); - var arrayLen = (1024 - sender.Length) / 8 + 1; - var aray = new double[arrayLen]; + var arrayLen = (1024 - sender.Length) / 8 + 1; + var aray = new double[arrayLen]; var expectedArray = new StringBuilder(); for (var i = 0; i < arrayLen; i++) { @@ -113,12 +164,13 @@ public async Task SendLongArrayAsSpan() } await sender.Column("array", (ReadOnlySpan)aray.AsSpan()) - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.SendAsync(); Assert.That( server.PrintBuffer(), - Is.EqualTo($"metrics,tag=value number=10i,string=\"abc\",array==ARRAY<{arrayLen}>[{expectedArray}] 1000000000\n")); + Is.EqualTo( + $"metrics,tag=value number=10i,string=\"abc\",array==ARRAY<{arrayLen}>[{expectedArray}] 1000000000\n")); await server.StopAsync(); } @@ -136,9 +188,9 @@ await server.StartAsync(HttpPort, new[] $"http::addr={Host}:{HttpPort};username=asdasdada;password=asdadad;tls_verify=unsafe_off;auto_flush=off;"); sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc"); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc"); Assert.That( () => sender.Column("array", new[] @@ -154,17 +206,16 @@ await server.StartAsync(HttpPort, new[] using var server = new DummyHttpServer(withBasicAuth: false); await server.StartAsync(HttpPort, new[] { - 3, 4, - 5, + 4, 5, }); using var sender = Sender.New( $"http::addr={Host}:{HttpPort};username=asdasdada;password=asdadad;tls_verify=unsafe_off;auto_flush=off;"); sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc"); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc"); Assert.That( () => sender.Column("array", new[] @@ -196,15 +247,15 @@ public async Task ArrayNegotiationConnectionIsRetried() await delayedStart; await sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc") - .Column("array", new[] - { - 1.2, 2.6, - 3.1, - }) - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc") + .Column("array", new[] + { + 1.2, 2.6, + 3.1, + }) + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.SendAsync(); @@ -226,9 +277,9 @@ public async Task BasicBinaryDouble() Sender.New( $"http::addr={Host}:{HttpPort};username=asdasdada;password=asdadad;tls_verify=unsafe_off;auto_flush=off;"); await sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 12.2) - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("tag", "value") + .Column("number", 12.2) + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.SendAsync(); Assert.That( @@ -246,18 +297,18 @@ public async Task BasicShapedEnumerableDouble() Sender.New( $"http::addr={Host}:{HttpPort};username=asdasdada;password=asdadad;tls_verify=unsafe_off;auto_flush=off;"); await sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc") - .Column("array", new[] - { - 1.2, 2.6, - 3.1, 4.6, - }.AsEnumerable(), new[] - { - 2, 2, - }.AsEnumerable()) - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc") + .Column("array", new[] + { + 1.2, 2.6, + 3.1, 4.6, + }.AsEnumerable(), new[] + { + 2, 2, + }.AsEnumerable()) + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.SendAsync(); Assert.That( @@ -274,9 +325,9 @@ public void InvalidShapedEnumerableDouble() $"http::addr={Host}:{HttpPort};username=asdasdada;password=asdadad;tls_verify=unsafe_off;auto_flush=off;protocol_version=2;"); sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc"); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc"); Assert.That( () => sender.Column("array", new[] @@ -312,15 +363,15 @@ public async Task BasicFlatArray() Sender.New( $"http::addr={Host}:{HttpPort};username=asdasdada;password=asdadad;tls_verify=unsafe_off;auto_flush=off;"); await sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc") - .Column("array", new[] - { - 1.2, 2.6, - 3.1, 4.6, - }) - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc") + .Column("array", new[] + { + 1.2, 2.6, + 3.1, 4.6, + }) + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.SendAsync(); Assert.That( @@ -344,13 +395,13 @@ public async Task BasicMultidimensionalArrayDouble() await server.StartAsync(HttpPort); using var sender = Sender.New( - $"http::addr={Host}:{HttpPort};username=asdasdada;password=asdadad;tls_verify=unsafe_off;auto_flush=off;protocol_version=V2;"); + $"http::addr={Host}:{HttpPort};username=asdasdada;password=asdadad;tls_verify=unsafe_off;auto_flush=off;protocol_version=2;"); await sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc") - .Column("array", arr) - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc") + .Column("array", arr) + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.SendAsync(); Assert.That( @@ -370,10 +421,10 @@ public async Task AuthBasicFailed() Sender.New( $"https::addr={Host}:{HttpsPort};username=asdasdada;password=asdadad;tls_verify=unsafe_off;auto_flush=off;"); await sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); Assert.That( async () => await sender.SendAsync(), @@ -390,10 +441,10 @@ public async Task AuthBasicSuccess() Sender.New( $"https::addr={Host}:{HttpsPort};username=admin;password=quest;tls_verify=unsafe_off;auto_flush=off;"); await sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.SendAsync(); } @@ -410,10 +461,10 @@ public async Task AuthTokenFailed() for (var i = 0; i < 100; i++) await sender - .Table("test") - .Symbol("foo", "bah") - .Column("num", i) - .AtAsync(DateTime.UtcNow); + .Table("test") + .Symbol("foo", "bah") + .Column("num", i) + .AtAsync(DateTime.UtcNow); Assert.That( async () => await sender.SendAsync(), @@ -435,10 +486,10 @@ public async Task AuthTokenSuccess() for (var i = 0; i < 100; i++) await sender - .Table("test") - .Symbol("foo", "bah") - .Column("num", i) - .AtAsync(DateTime.UtcNow); + .Table("test") + .Symbol("foo", "bah") + .Column("num", i) + .AtAsync(DateTime.UtcNow); await sender.SendAsync(); } @@ -450,10 +501,10 @@ public async Task BasicSend() using var server = new DummyHttpServer(); await server.StartAsync(HttpPort); var sender = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;"); - var ts = DateTime.UtcNow; + var ts = DateTime.UtcNow; await sender.Table("name") - .Column("ts", ts) - .AtAsync(ts); + .Column("ts", ts) + .AtAsync(ts); await sender.SendAsync(); Console.WriteLine(server.GetReceiveBuffer().ToString()); await server.StopAsync(); @@ -467,7 +518,7 @@ public void SendBadSymbol() { var sender = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;protocol_version=1;"); sender.Table("metric name") - .Symbol("t ,a g", "v alu, e"); + .Symbol("t ,a g", "v alu, e"); }, Throws.TypeOf().With.Message.Contains("Column names") ); @@ -481,7 +532,7 @@ public void SendBadColumn() { var sender = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;protocol_version=1;"); sender.Table("metric name") - .Column("t a, g", "v alu e"); + .Column("t a, g", "v alu e"); }, Throws.TypeOf().With.Message.Contains("Column names") ); @@ -494,10 +545,10 @@ public async Task SendLine() await server.StartAsync(HttpPort); var sender = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;"); await sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.SendAsync(); Assert.That( @@ -522,12 +573,12 @@ public async Task SendLineExceedsBuffer() for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -546,17 +597,17 @@ public async Task SendLineExceedsBufferLimit() Sender.New($"http::addr={Host}:{HttpPort};init_buf_size=1024;max_buf_size=2048;auto_flush=off;"); Assert.That(async () => - { - for (var i = 0; i < 500; i++) - await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); - }, - Throws.Exception.With.Message.Contains("maximum buffer size")); + { + for (var i = 0; i < 500; i++) + await sender.Table("table name") + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + }, + Throws.Exception.With.Message.Contains("maximum buffer size")); } [Test] @@ -574,12 +625,12 @@ public async Task SendLineReusesBuffer() for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -588,12 +639,12 @@ await sender.Table("table name") for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -617,12 +668,12 @@ public async Task SendLineTrimsBuffers() for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -632,12 +683,12 @@ await sender.Table("table name") for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -662,18 +713,18 @@ public async Task ServerDisconnects() for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); if (i > 1) { Assert.That(async () => await sender.SendAsync(), - Throws.TypeOf()); + Throws.TypeOf()); break; } @@ -694,11 +745,11 @@ public async Task SendNegativeLongAndDouble() using var ls = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;"); await ls.Table("neg name") - .Column("number1", long.MinValue + 1) - .Column("number2", long.MaxValue) - .Column("number3", double.MinValue) - .Column("number4", double.MaxValue) - .AtAsync(86400000000000); + .Column("number1", long.MinValue + 1) + .Column("number2", long.MaxValue) + .Column("number3", double.MinValue) + .Column("number4", double.MaxValue) + .AtAsync(86400000000000); await ls.SendAsync(); var expected = @@ -716,22 +767,22 @@ public async Task SerialiseDoublesV2() using var ls = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;"); await ls.Table("doubles") - .Column("d0", 0.0) - .Column("dm0", -0.0) - .Column("d1", 1.0) - .Column("dE100", 1E100) - .Column("d0000001", 0.000001) - .Column("dNaN", double.NaN) - .Column("dInf", double.PositiveInfinity) - .Column("dNInf", double.NegativeInfinity) - .AtAsync(86400000000000); + .Column("d0", 0.0) + .Column("dm0", -0.0) + .Column("d1", 1.0) + .Column("dE100", 1E100) + .Column("d0000001", 0.000001) + .Column("dNaN", double.NaN) + .Column("dInf", double.PositiveInfinity) + .Column("dNInf", double.NegativeInfinity) + .AtAsync(86400000000000); await ls.SendAsync(); var expected = "doubles d0=0,dm0=-0,d1=1,dE100=1E+100,d0000001=1E-06,dNaN=NaN,dInf=Infinity,dNInf=-Infinity 86400000000000\n"; Assert.That(srv.PrintBuffer(), Is.EqualTo(expected)); } - + [Test] public async Task SerialiseDoublesV1() { @@ -741,15 +792,15 @@ public async Task SerialiseDoublesV1() using var ls = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;protocol_version=1;"); await ls.Table("doubles") - .Column("d0", 0.0) - .Column("dm0", -0.0) - .Column("d1", 1.0) - .Column("dE100", 1E100) - .Column("d0000001", 0.000001) - .Column("dNaN", double.NaN) - .Column("dInf", double.PositiveInfinity) - .Column("dNInf", double.NegativeInfinity) - .AtAsync(86400000000000); + .Column("d0", 0.0) + .Column("dm0", -0.0) + .Column("d1", 1.0) + .Column("dE100", 1E100) + .Column("d0000001", 0.000001) + .Column("dNaN", double.NaN) + .Column("dInf", double.PositiveInfinity) + .Column("dNInf", double.NegativeInfinity) + .AtAsync(86400000000000); await ls.SendAsync(); var expected = @@ -766,8 +817,8 @@ public async Task SendTimestampColumn() var ts = new DateTime(2022, 2, 24); await sender.Table("name") - .Column("ts", ts) - .AtAsync(ts); + .Column("ts", ts) + .AtAsync(ts); await sender.SendAsync(); @@ -785,8 +836,8 @@ public async Task SendColumnNanos() const long timestampNanos = 1645660800123456789L; await sender.Table("name") - .ColumnNanos("ts", timestampNanos) - .AtAsync(timestampNanos); + .ColumnNanos("ts", timestampNanos) + .AtAsync(timestampNanos); await sender.SendAsync(); @@ -804,8 +855,8 @@ public async Task SendAtNanos() const long timestampNanos = 1645660800987654321L; await sender.Table("name") - .Column("value", 42) - .AtNanosAsync(timestampNanos); + .Column("value", 42) + .AtNanosAsync(timestampNanos); await sender.SendAsync(); @@ -819,8 +870,8 @@ public async Task InvalidState() { using var srv = new DummyHttpServer(); await srv.StartAsync(HttpPort); - using var sender = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;"); - string? nullString = null; + using var sender = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;"); + string? nullString = null; Assert.That( () => sender.Table(nullString), @@ -920,8 +971,8 @@ public async Task InvalidTableName() { using var srv = new DummyHttpServer(); await srv.StartAsync(HttpPort); - using var sender = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;"); - string? nullString = null; + using var sender = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;"); + string? nullString = null; Assert.Throws(() => sender.Table(nullString)); Assert.Throws(() => sender.Column("abc", 123)); @@ -948,19 +999,19 @@ public async Task SendMillionAsyncExplicit() $"http::addr={Host}:{HttpPort};init_buf_size={256 * 1024};auto_flush=off;request_timeout=30000;"); var nowMillisecond = DateTime.Now.Millisecond; - var metric = "metric_name" + nowMillisecond; + var metric = "metric_name" + nowMillisecond; Assert.True(await srv.Healthcheck()); for (var i = 0; i < 1E6; i++) { await sender.Table(metric) - .Symbol("nopoint", "tag" + i % 100) - .Column("counter", i * 1111.1) - .Column("int", i) - .Column("привед", "мед вед") - .AtAsync(new DateTime(2021, 1, 1, i / 360 / 1000 % 60, i / 60 / 1000 % 60, i / 1000 % 60, - i % 1000)); + .Symbol("nopoint", "tag" + i % 100) + .Column("counter", i * 1111.1) + .Column("int", i) + .Column("привед", "мед вед") + .AtAsync(new DateTime(2021, 1, 1, i / 360 / 1000 % 60, i / 60 / 1000 % 60, i / 1000 % 60, + i % 1000)); if (i % 100 == 0) { @@ -978,7 +1029,7 @@ public async Task SendMillionFixedBuffer() await srv.StartAsync(HttpPort); var nowMillisecond = DateTime.Now.Millisecond; - var metric = "metric_name" + nowMillisecond; + var metric = "metric_name" + nowMillisecond; Assert.True(await srv.Healthcheck()); @@ -988,12 +1039,12 @@ public async Task SendMillionFixedBuffer() for (var i = 0; i < 1E6; i++) await sender.Table(metric) - .Symbol("nopoint", "tag" + i % 100) - .Column("counter", i * 1111.1) - .Column("int", i) - .Column("привед", "мед вед") - .AtAsync(new DateTime(2021, 1, 1, i / 360 / 1000 % 60, i / 60 / 1000 % 60, i / 1000 % 60, - i % 1000)); + .Symbol("nopoint", "tag" + i % 100) + .Column("counter", i * 1111.1) + .Column("int", i) + .Column("привед", "мед вед") + .AtAsync(new DateTime(2021, 1, 1, i / 360 / 1000 % 60, i / 60 / 1000 % 60, i / 1000 % 60, + i % 1000)); await sender.SendAsync(); } @@ -1009,8 +1060,8 @@ public async Task SendNegativeLongMin() $"http::addr={Host}:{HttpPort};auto_flush=off;"); Assert.That( () => sender.Table("name") - .Column("number1", long.MinValue) - .AtAsync(DateTime.UtcNow), + .Column("number1", long.MinValue) + .AtAsync(DateTime.UtcNow), Throws.TypeOf().With.Message.Contains("Special case") ); } @@ -1025,8 +1076,8 @@ public async Task SendSpecialStrings() Sender.New( $"http::addr={Host}:{HttpPort};auto_flush=off;"); await sender.Table("neg name") - .Column("привед", " мед\rве\n д") - .AtAsync(86400000000000); + .Column("привед", " мед\rве\n д") + .AtAsync(86400000000000); await sender.SendAsync(); var expected = "neg\\ name привед=\" мед\\\rве\\\n д\" 86400000000000\n"; @@ -1044,9 +1095,9 @@ public async Task SendTagAfterField() $"http::addr={Host}:{HttpPort};auto_flush=off;"); Assert.That( async () => await sender.Table("name") - .Column("number1", 123) - .Symbol("nand", "asdfa") - .AtAsync(DateTime.UtcNow), + .Column("number1", 123) + .Symbol("nand", "asdfa") + .AtAsync(DateTime.UtcNow), Throws.TypeOf() ); } @@ -1064,9 +1115,9 @@ public async Task SendMetricOnce() Assert.That( async () => await sender.Table("name") - .Column("number1", 123) - .Table("nand") - .AtAsync(DateTime.UtcNow), + .Column("number1", 123) + .Table("nand") + .AtAsync(DateTime.UtcNow), Throws.TypeOf() ); } @@ -1219,7 +1270,8 @@ public async Task TransactionMultipleTypes() await sender.Column("foo", 123).AtAsync(86400000000000); await sender.Column("foo", 123d).AtAsync(86400000000000); await sender.Column("foo", new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).AtAsync(86400000000000); - await sender.Column("foo", new DateTimeOffset(new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc))).AtAsync(86400000000000); + await sender.Column("foo", new DateTimeOffset(new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc))) + .AtAsync(86400000000000); await sender.Column("foo", false).AtAsync(86400000000000); await sender.CommitAsync(); @@ -1287,9 +1339,9 @@ public async Task TransactionShouldNotBeAutoFlushed() sender.Transaction("tableName"); for (var i = 0; i < 100; i++) await sender - .Symbol("foo", "bah") - .Column("num", i) - .AtAsync(DateTime.UtcNow); + .Symbol("foo", "bah") + .Column("num", i) + .AtAsync(DateTime.UtcNow); Assert.That(sender.RowCount == 100); Assert.That(sender.WithinTransaction); @@ -1313,9 +1365,9 @@ public async Task TransactionRequiresCommitToComplete() sender.Transaction("tableName"); for (var i = 0; i < 100; i++) await sender - .Symbol("foo", "bah") - .Column("num", i) - .AtAsync(DateTime.UtcNow); + .Symbol("foo", "bah") + .Column("num", i) + .AtAsync(DateTime.UtcNow); Assert.That( async () => await sender.SendAsync(), @@ -1332,9 +1384,9 @@ await sender sender.Transaction("tableName"); for (var i = 0; i < 100; i++) await sender - .Symbol("foo", "bah") - .Column("num", i) - .AtAsync(DateTime.UtcNow); + .Symbol("foo", "bah") + .Column("num", i) + .AtAsync(DateTime.UtcNow); sender.Commit(); } @@ -1500,8 +1552,8 @@ public async Task SendTimestampColumns() using var sender = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;"); sender.Table("foo") - .Symbol("bah", "baz") - .Column("ts1", DateTime.UtcNow).Column("ts2", DateTimeOffset.UtcNow); + .Symbol("bah", "baz") + .Column("ts1", DateTime.UtcNow).Column("ts2", DateTimeOffset.UtcNow); await sender.SendAsync(); } @@ -1514,36 +1566,36 @@ public async Task SendVariousAts() using var sender = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;"); await sender.Table("foo") - .Symbol("bah", "baz") - .AtAsync(DateTime.UtcNow); + .Symbol("bah", "baz") + .AtAsync(DateTime.UtcNow); await sender.Table("foo") - .Symbol("bah", "baz") - .AtAsync(DateTime.UtcNow); + .Symbol("bah", "baz") + .AtAsync(DateTime.UtcNow); await sender.Table("foo") - .Symbol("bah", "baz") - .AtAsync(DateTimeOffset.UtcNow); + .Symbol("bah", "baz") + .AtAsync(DateTimeOffset.UtcNow); await sender.Table("foo") - .Symbol("bah", "baz") - .AtAsync(DateTime.UtcNow.Ticks / 100); + .Symbol("bah", "baz") + .AtAsync(DateTime.UtcNow.Ticks / 100); sender.Table("foo") - .Symbol("bah", "baz") - .At(DateTime.UtcNow); + .Symbol("bah", "baz") + .At(DateTime.UtcNow); sender.Table("foo") - .Symbol("bah", "baz") - .At(DateTime.UtcNow); + .Symbol("bah", "baz") + .At(DateTime.UtcNow); sender.Table("foo") - .Symbol("bah", "baz") - .At(DateTimeOffset.UtcNow); + .Symbol("bah", "baz") + .At(DateTimeOffset.UtcNow); sender.Table("foo") - .Symbol("bah", "baz") - .At(DateTime.UtcNow.Ticks / 100); + .Symbol("bah", "baz") + .At(DateTime.UtcNow.Ticks / 100); await sender.SendAsync(); } @@ -1599,12 +1651,12 @@ public async Task SendManyRequests() for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", i) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(DateTime.UtcNow); + .Symbol("t a g", "v alu, e") + .Column("number", i) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(DateTime.UtcNow); var request = sender.SendAsync(); diff --git a/src/net-questdb-client-tests/JsonSpecTestRunner.cs b/src/net-questdb-client-tests/JsonSpecTestRunner.cs index 514ee47..382a536 100644 --- a/src/net-questdb-client-tests/JsonSpecTestRunner.cs +++ b/src/net-questdb-client-tests/JsonSpecTestRunner.cs @@ -26,9 +26,11 @@ using System.Net; using System.Text; using System.Text.Json; +using System.Text.Json.Serialization; using dummy_http_server; using NUnit.Framework; using QuestDB; +using QuestDB.Senders; // ReSharper disable InconsistentNaming @@ -41,6 +43,50 @@ public class JsonSpecTestRunner private const int HttpPort = 29473; private static readonly TestCase[]? TestCases = ReadTestCases(); + private static async Task ExecuteTestCase(ISender sender, TestCase testCase) + { + sender.Table(testCase.Table); + foreach (var symbol in testCase.Symbols) + { + sender.Symbol(symbol.Name, symbol.Value); + } + + foreach (var column in testCase.Columns) + { + switch (column.Type) + { + case "STRING": + sender.Column(column.Name, ((JsonElement)column.Value).GetString()); + break; + + case "DOUBLE": + sender.Column(column.Name, ((JsonElement)column.Value).GetDouble()); + break; + + case "BOOLEAN": + sender.Column(column.Name, ((JsonElement)column.Value).GetBoolean()); + break; + + case "LONG": + sender.Column(column.Name, (long)((JsonElement)column.Value).GetDouble()); + break; + + case "DECIMAL": + var d = decimal.Parse(((JsonElement)column.Value).GetString()!); + sender.Column(column.Name, d); + break; + + default: + throw new NotSupportedException("Column type not supported: " + column.Type); + } + } + +#pragma warning disable CS0618 // Type or member is obsolete + await sender.AtNowAsync(); +#pragma warning restore CS0618 // Type or member is obsolete + await sender.SendAsync(); + } + [TestCaseSource(nameof(TestCases))] public async Task RunTcp(TestCase testCase) { @@ -48,51 +94,17 @@ public async Task RunTcp(TestCase testCase) srv.AcceptAsync(); using var sender = Sender.New( - $"tcp::addr={IPAddress.Loopback}:{TcpPort};"); + $"tcp::addr={IPAddress.Loopback}:{TcpPort};protocol_version=3"); Exception? exception = null; try { - sender.Table(testCase.table); - foreach (var symbol in testCase.symbols) - { - sender.Symbol(symbol.name, symbol.value); - } - - foreach (var column in testCase.columns) - { - switch (column.type) - { - case "STRING": - sender.Column(column.name, ((JsonElement)column.value).GetString()); - break; - - case "DOUBLE": - sender.Column(column.name, ((JsonElement)column.value).GetDouble()); - break; - - case "BOOLEAN": - sender.Column(column.name, ((JsonElement)column.value).GetBoolean()); - break; - - case "LONG": - sender.Column(column.name, (long)((JsonElement)column.value).GetDouble()); - break; - - default: - throw new NotSupportedException("Column type not supported: " + column.type); - } - } - -#pragma warning disable CS0618 // Type or member is obsolete - await sender.AtNowAsync(); -#pragma warning restore CS0618 // Type or member is obsolete - await sender.SendAsync(); + await ExecuteTestCase(sender, testCase); } catch (Exception? ex) { - if (testCase.result.status == "SUCCESS") + if (testCase.Result.Status == "SUCCESS") { throw; } @@ -100,18 +112,24 @@ public async Task RunTcp(TestCase testCase) exception = ex; } - if (testCase.result.status == "SUCCESS") + if (testCase.Result.Status == "SUCCESS") { - if (testCase.result.anyLines == null || testCase.result.anyLines.Length == 0) + if (testCase.Result.BinaryBase64 != null) + { + var received = srv.GetReceivedBytes(); + var expected = Convert.FromBase64String(testCase.Result.BinaryBase64); + Assert.That(received, Is.EqualTo(expected)); + } + else if (testCase.Result.AnyLines == null || testCase.Result.AnyLines.Length == 0) { - WaitAssert(srv, testCase.result.line + "\n"); + WaitAssert(srv, testCase.Result.Line + "\n"); } else { - WaitAssert(srv, testCase.result.anyLines); + WaitAssert(srv, testCase.Result.AnyLines); } } - else if (testCase.result.status == "ERROR") + else if (testCase.Result.Status == "ERROR") { Assert.NotNull(exception, "Exception should be thrown"); if (exception is NotSupportedException) @@ -121,7 +139,7 @@ public async Task RunTcp(TestCase testCase) } else { - Assert.Fail("Unsupported test case result status: " + testCase.result.status); + Assert.Fail("Unsupported test case result status: " + testCase.Result.Status); } } @@ -134,52 +152,18 @@ public async Task RunHttp(TestCase testCase) Assert.That(await server.Healthcheck()); using var sender = Sender.New( - $"http::addr={IPAddress.Loopback}:{HttpPort};"); + $"http::addr={IPAddress.Loopback}:{HttpPort};protocol_version=3"); Exception? exception = null; try { - sender.Table(testCase.table); - foreach (var symbol in testCase.symbols) - { - sender.Symbol(symbol.name, symbol.value); - } - - foreach (var column in testCase.columns) - { - switch (column.type) - { - case "STRING": - sender.Column(column.name, ((JsonElement)column.value).GetString()); - break; - - case "DOUBLE": - sender.Column(column.name, ((JsonElement)column.value).GetDouble()); - break; - - case "BOOLEAN": - sender.Column(column.name, ((JsonElement)column.value).GetBoolean()); - break; - - case "LONG": - sender.Column(column.name, (long)((JsonElement)column.value).GetDouble()); - break; - - default: - throw new NotSupportedException("Column type not supported: " + column.type); - } - } - -#pragma warning disable CS0618 // Type or member is obsolete - await sender.AtNowAsync(); -#pragma warning restore CS0618 // Type or member is obsolete - await sender.SendAsync(); + await ExecuteTestCase(sender, testCase); } catch (Exception? ex) { TestContext.Write(server.GetLastError()); - if (testCase.result.status == "SUCCESS") + if (testCase.Result.Status == "SUCCESS") { throw; } @@ -187,19 +171,26 @@ public async Task RunHttp(TestCase testCase) exception = ex; } - if (testCase.result.status == "SUCCESS") + if (testCase.Result.Status == "SUCCESS") { - var textReceived = server.PrintBuffer(); - if (testCase.result.anyLines == null || testCase.result.anyLines.Length == 0) + if (testCase.Result.BinaryBase64 != null) + { + var received = server.GetReceivedBytes(); + var expected = Convert.FromBase64String(testCase.Result.BinaryBase64); + Assert.That(received, Is.EqualTo(expected)); + } + else if (testCase.Result.AnyLines == null || testCase.Result.AnyLines.Length == 0) { - Assert.That(textReceived, Is.EqualTo(testCase.result.line + "\n")); + var textReceived = server.PrintBuffer(); + Assert.That(textReceived, Is.EqualTo(testCase.Result.Line + "\n")); } else { - AssertMany(testCase.result.anyLines, textReceived); + var textReceived = server.PrintBuffer(); + AssertMany(testCase.Result.AnyLines, textReceived); } } - else if (testCase.result.status == "ERROR") + else if (testCase.Result.Status == "ERROR") { Assert.NotNull(exception, "Exception should be thrown"); if (exception is NotSupportedException) @@ -209,7 +200,7 @@ public async Task RunHttp(TestCase testCase) } else { - Assert.Fail("Unsupported test case result status: " + testCase.result.status); + Assert.Fail("Unsupported test case result status: " + testCase.Result.Status); } } @@ -258,35 +249,40 @@ private DummyIlpServer CreateTcpListener(int port, bool tls = false) public class TestCase { - public string testName { get; set; } = null!; - public string table { get; set; } = null!; - public TestCaseSymbol[] symbols { get; set; } = null!; - public TestCaseColumn[] columns { get; set; } = null!; - public TestCaseResult result { get; set; } = null!; + [JsonPropertyName("testName")] public string TestName { get; set; } = null!; + [JsonPropertyName("table")] public string Table { get; set; } = null!; + + [JsonPropertyName("minimumProtocolVersion")] + public int? MinimumProtocolVersion { get; set; } + + [JsonPropertyName("symbols")] public TestCaseSymbol[] Symbols { get; set; } = null!; + [JsonPropertyName("columns")] public TestCaseColumn[] Columns { get; set; } = null!; + [JsonPropertyName("result")] public TestCaseResult Result { get; set; } = null!; public override string ToString() { - return testName; + return TestName; } } public class TestCaseSymbol { - public string name { get; set; } = null!; - public string value { get; set; } = null!; + [JsonPropertyName("name")] public string Name { get; set; } = null!; + [JsonPropertyName("value")] public string Value { get; set; } = null!; } public class TestCaseColumn { - public string type { get; set; } = null!; - public string name { get; set; } = null!; - public object value { get; set; } = null!; + [JsonPropertyName("type")] public string Type { get; set; } = null!; + [JsonPropertyName("name")] public string Name { get; set; } = null!; + [JsonPropertyName("value")] public object Value { get; set; } = null!; } public class TestCaseResult { - public string status { get; set; } = null!; - public string line { get; set; } = null!; - public string[]? anyLines { get; set; } = null!; + [JsonPropertyName("status")] public string Status { get; set; } = null!; + [JsonPropertyName("line")] public string Line { get; set; } = null!; + [JsonPropertyName("anyLines")] public string[]? AnyLines { get; set; } = null!; + [JsonPropertyName("binaryBase64")] public string? BinaryBase64 { get; set; } } } \ No newline at end of file diff --git a/src/tcp-client-test/LineTcpSenderTests.cs b/src/net-questdb-client-tests/LineTcpSenderTests.cs similarity index 80% rename from src/tcp-client-test/LineTcpSenderTests.cs rename to src/net-questdb-client-tests/LineTcpSenderTests.cs index c30d61f..efa72d6 100644 --- a/src/tcp-client-test/LineTcpSenderTests.cs +++ b/src/net-questdb-client-tests/LineTcpSenderTests.cs @@ -40,7 +40,7 @@ #pragma warning disable CS0612 // Type or member is obsolete #pragma warning disable CS0618 // Type or member is obsolete -namespace tcp_client_test; +namespace net_questdb_client_tests; [TestFixture] public class LineTcpSenderTests @@ -56,10 +56,10 @@ public async Task SendLine() using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, tlsMode: TlsMode.Disable); ls.Table("metric name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("string", " -=\"") - .At(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("string", " -=\"") + .At(new DateTime(1970, 01, 01, 0, 0, 1)); ls.Send(); @@ -72,17 +72,17 @@ public async Task Authenticate() { using var srv = CreateTcpListener(_port); srv.WithAuth("testUser1", "Vs4e-cOLsVCntsMrZiAGAZtrkPXO00uoRLuA3d7gEcI=", - "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); + "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); srv.AcceptAsync(); using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, tlsMode: TlsMode.Disable); await ls.AuthenticateAsync("testUser1", "NgdiOWDoQNUP18WOnb1xkkEG5TzPYMda5SiUOvT1K0U=", CancellationToken.None); ls.Table("metric name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("string", " -=\"") - .At(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("string", " -=\"") + .At(new DateTime(1970, 01, 01, 0, 0, 1)); ls.Send(); var expected = "metric\\ name,t\\ a\\ g=v\\ alu\\,\\ e number=10i,string=\" -=\\\"\" 1000000000\n"; @@ -94,7 +94,7 @@ public async Task AuthFailWrongKid() { using var srv = CreateTcpListener(_port); srv.WithAuth("testUser1", "Vs4e-cOLsVCntsMrZiAGAZtrkPXO00uoRLuA3d7gEcI=", - "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); + "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); srv.AcceptAsync(); using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, tlsMode: TlsMode.Disable); @@ -113,7 +113,7 @@ public async Task AuthFailWrongKid() public void EcdsaSingnatureLoop() { var privateKey = Convert.FromBase64String("NgdiOWDoQNUP18WOnb1xkkEG5TzPYMda5SiUOvT1K0U="); - var p = SecNamedCurves.GetByName("secp256r1"); + var p = SecNamedCurves.GetByName("secp256r1"); var parameters = new ECDomainParameters(p.Curve, p.G, p.N, p.H); var priKey = new ECPrivateKeyParameters( "ECDSA", @@ -164,12 +164,12 @@ public async Task SendLineExceedsBuffer() for (var i = 0; i < lineCount; i++) { ls.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .At(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .At(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -187,8 +187,8 @@ public async Task SendLineReusesBuffers() srv.AcceptAsync(); using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, 2048, - tlsMode: TlsMode.Disable, - bufferOverflowHandling: BufferOverflowHandling.Extend); + tlsMode: TlsMode.Disable, + bufferOverflowHandling: BufferOverflowHandling.Extend); var lineCount = 500; var expected = "table\\ name,t\\ a\\ g=v\\ alu\\,\\ e number=10i,db\\ l=123.12,string=\" -=\\\"\",при\\ вед=\"медвед\" 1000000000\n"; @@ -196,12 +196,12 @@ public async Task SendLineReusesBuffers() for (var i = 0; i < lineCount; i++) { ls.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .At(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .At(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -210,12 +210,12 @@ public async Task SendLineReusesBuffers() for (var i = 0; i < lineCount; i++) { ls.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .At(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .At(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -240,12 +240,12 @@ public async Task SendLineTrimsBuffers() for (var i = 0; i < lineCount; i++) { ls.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .At(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .At(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -255,12 +255,12 @@ public async Task SendLineTrimsBuffers() for (var i = 0; i < lineCount; i++) { ls.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .At(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .At(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -291,12 +291,12 @@ await LineTcpSender.ConnectAsync( for (var i = 0; i < lineCount; i++) { ls.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .At(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .At(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); try { @@ -323,11 +323,11 @@ public async Task SendNegativeLongAndDouble() using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, tlsMode: TlsMode.Disable); ls.Table("neg name") - .Column("number1", long.MinValue + 1) - .Column("number2", long.MaxValue) - .Column("number3", double.MinValue) - .Column("number4", double.MaxValue) - .AtNow(); + .Column("number1", long.MinValue + 1) + .Column("number2", long.MaxValue) + .Column("number3", double.MinValue) + .Column("number4", double.MaxValue) + .AtNow(); ls.Send(); var expected = @@ -344,15 +344,15 @@ public async Task DoubleSerializationTest() using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, tlsMode: TlsMode.Disable); ls.Table("doubles") - .Column("d0", 0.0) - .Column("dm0", -0.0) - .Column("d1", 1.0) - .Column("dE100", 1E100) - .Column("d0000001", 0.000001) - .Column("dNaN", double.NaN) - .Column("dInf", double.PositiveInfinity) - .Column("dNInf", double.NegativeInfinity) - .AtNow(); + .Column("d0", 0.0) + .Column("dm0", -0.0) + .Column("d1", 1.0) + .Column("dE100", 1E100) + .Column("d0000001", 0.000001) + .Column("dNaN", double.NaN) + .Column("dInf", double.PositiveInfinity) + .Column("dNInf", double.NegativeInfinity) + .AtNow(); ls.Send(); var expected = @@ -370,8 +370,8 @@ public async Task SendTimestampColumn() var ts = new DateTime(2022, 2, 24); ls.Table("name") - .Column("ts", ts) - .At(ts); + .Column("ts", ts) + .At(ts); ls.Send(); @@ -385,18 +385,18 @@ public async Task WithTls() { using var srv = CreateTcpListener(_port, true); srv.WithAuth("testUser1", "Vs4e-cOLsVCntsMrZiAGAZtrkPXO00uoRLuA3d7gEcI=", - "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); + "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); srv.AcceptAsync(); using var ls = await LineTcpSender.ConnectAsync("localhost", _port, tlsMode: TlsMode.AllowAnyServerCertificate); await ls.AuthenticateAsync("testUser1", "NgdiOWDoQNUP18WOnb1xkkEG5TzPYMda5SiUOvT1K0U="); ls.Table("table_name") - .Column("number1", long.MinValue + 1) - .Column("number2", long.MaxValue) - .Column("number3", double.MinValue) - .Column("number4", double.MaxValue) - .AtNow(); + .Column("number1", long.MinValue + 1) + .Column("number2", long.MaxValue) + .Column("number3", double.MinValue) + .Column("number4", double.MaxValue) + .AtNow(); await ls.SendAsync(); @@ -412,7 +412,7 @@ public async Task InvalidState() srv.AcceptAsync(); using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, tlsMode: TlsMode.Disable); - string? nullString = null; + string? nullString = null; Assert.Throws(() => ls.Table(nullString)); Assert.Throws(() => ls.Column("abc", 123)); @@ -488,7 +488,7 @@ public async Task InvalidTableName() srv.AcceptAsync(); using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, tlsMode: TlsMode.Disable); - string? nullString = null; + string? nullString = null; Assert.Throws(() => ls.Table(nullString)); Assert.Throws(() => ls.Column("abc", 123)); @@ -540,18 +540,18 @@ public async Task SendMillionAsyncExplicit() srv.AcceptAsync(); var nowMillisecond = DateTime.Now.Millisecond; - var metric = "metric_name" + nowMillisecond; + var metric = "metric_name" + nowMillisecond; using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, 256 * 1024, - tlsMode: TlsMode.Disable); + tlsMode: TlsMode.Disable); for (var i = 0; i < 1E6; i++) { ls.Table(metric) - .Symbol("nopoint", "tag" + i % 100) - .Column("counter", i * 1111.1) - .Column("int", i) - .Column("привед", "мед вед") - .At(new DateTime(2021, 1, 1, i / 360 / 1000 % 60, i / 60 / 1000 % 60, i / 1000 % 60, i % 1000)); + .Symbol("nopoint", "tag" + i % 100) + .Column("counter", i * 1111.1) + .Column("int", i) + .Column("привед", "мед вед") + .At(new DateTime(2021, 1, 1, i / 360 / 1000 % 60, i / 60 / 1000 % 60, i / 1000 % 60, i % 1000)); if (i % 100 == 0) { @@ -569,18 +569,18 @@ public async Task SendMillionFixedBuffer() srv.AcceptAsync(); var nowMillisecond = DateTime.Now.Millisecond; - var metric = "metric_name" + nowMillisecond; + var metric = "metric_name" + nowMillisecond; using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, 64 * 1024, - BufferOverflowHandling.SendImmediately, TlsMode.Disable); + BufferOverflowHandling.SendImmediately, TlsMode.Disable); for (var i = 0; i < 1E6; i++) { ls.Table(metric) - .Symbol("nopoint", "tag" + i % 100) - .Column("counter", i * 1111.1) - .Column("int", i) - .Column("привед", "мед вед") - .At(new DateTime(2021, 1, 1, i / 360 / 1000 % 60, i / 60 / 1000 % 60, i / 1000 % 60, i % 1000)); + .Symbol("nopoint", "tag" + i % 100) + .Column("counter", i * 1111.1) + .Column("int", i) + .Column("привед", "мед вед") + .At(new DateTime(2021, 1, 1, i / 360 / 1000 % 60, i / 60 / 1000 % 60, i / 1000 % 60, i % 1000)); } await ls.SendAsync(); @@ -605,7 +605,7 @@ public async Task BufferTooSmallForAuth() { using var srv = CreateTcpListener(_port); srv.WithAuth("testUser1", "Vs4e-cOLsVCntsMrZiAGAZtrkPXO00uoRLuA3d7gEcI=", - "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); + "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); srv.AcceptAsync(); using var ls = @@ -614,7 +614,7 @@ public async Task BufferTooSmallForAuth() try { await ls.AuthenticateAsync("testUser1", "NgdiOWDoQNUP18WOnb1xkkEG5TzPYMda5SiUOvT1K0U=", - CancellationToken.None); + CancellationToken.None); Assert.Fail(); } catch (IOException ex) @@ -642,8 +642,8 @@ public async Task SendNegativeLongMin() using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, tlsMode: TlsMode.Disable); Assert.Throws(() => ls.Table("name") - .Column("number1", long.MinValue) - .AtNow() + .Column("number1", long.MinValue) + .AtNow() ); } @@ -655,8 +655,8 @@ public async Task SendSpecialStrings() using var ls = await LineTcpSender.ConnectAsync("127.0.0.1", _port, tlsMode: TlsMode.Disable); ls.Table("neg name") - .Column("привед", " мед\rве\n д") - .AtNow(); + .Column("привед", " мед\rве\n д") + .AtNow(); ls.Send(); var expected = "neg\\ name привед=\" мед\\\rве\\\n д\"\n"; @@ -671,9 +671,9 @@ public async Task SendTagAfterField() using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, tlsMode: TlsMode.Disable); Assert.Throws(() => ls.Table("name") - .Column("number1", 123) - .Symbol("nand", "asdfa") - .AtNow() + .Column("number1", 123) + .Symbol("nand", "asdfa") + .AtNow() ); } @@ -685,9 +685,9 @@ public async Task SendMetricOnce() using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, tlsMode: TlsMode.Disable); Assert.Throws(() => ls.Table("name") - .Column("number1", 123) - .Table("nand") - .AtNow() + .Column("number1", 123) + .Table("nand") + .AtNow() ); } @@ -699,11 +699,11 @@ public async Task StartFromMetric() using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, tlsMode: TlsMode.Disable); Assert.Throws(() => ls.Column("number1", 123) - .AtNow() + .AtNow() ); Assert.Throws(() => ls.Symbol("number1", "1234") - .AtNow() + .AtNow() ); } @@ -740,8 +740,8 @@ public async Task AtWithLongEpochNano() var ts = new DateTime(2022, 2, 24); ls.Table("name") - .Column("ts", ts) - .At((ts.Ticks - DateTime.UnixEpoch.Ticks) * 100); + .Column("ts", ts) + .At((ts.Ticks - DateTime.UnixEpoch.Ticks) * 100); ls.Send(); diff --git a/src/net-questdb-client-tests/TcpTests.cs b/src/net-questdb-client-tests/TcpTests.cs index 32f8548..82e3ecc 100644 --- a/src/net-questdb-client-tests/TcpTests.cs +++ b/src/net-questdb-client-tests/TcpTests.cs @@ -51,10 +51,10 @@ public async Task SendLine() using var sender = Sender.New($"tcp::addr={_host}:{_port};"); await sender.Table("metric name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("string", " -=\"") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("string", " -=\"") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.SendAsync(); @@ -62,6 +62,50 @@ await sender.Table("metric name") WaitAssert(srv, expected); } + [Test] + public async Task SendLineWithDecimalBinaryEncoding() + { + using var srv = CreateTcpListener(_port); + srv.AcceptAsync(); + + using var sender = Sender.New($"tcp::addr={_host}:{_port};protocol_version=3;"); + await sender.Table("metrics") + .Symbol("tag", "value") + .Column("dec_pos", 123.45m) + .Column("dec_neg", -123.45m) + .Column("dec_null", (decimal?)null) + .Column("dec_max", decimal.MaxValue) + .Column("dec_min", decimal.MinValue) + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + + await sender.SendAsync(); + + var buffer = WaitForLineBytes(srv); + DecimalTestHelpers.AssertDecimalField(buffer, "dec_pos", 2, new byte[] + { + 0x30, 0x39, + }); + DecimalTestHelpers.AssertDecimalField(buffer, "dec_neg", 2, new byte[] + { + 0xCF, 0xC7, + }); + DecimalTestHelpers.AssertDecimalNullField(buffer, "dec_null"); + DecimalTestHelpers.AssertDecimalField(buffer, "dec_max", 0, new byte[] + { + 0x00, + 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, + }); + DecimalTestHelpers.AssertDecimalField(buffer, "dec_min", 0, new byte[] + { + 0xFF, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, + }); + } + [Test] public async Task SendLineWithArrayProtocolV2() { @@ -70,18 +114,19 @@ public async Task SendLineWithArrayProtocolV2() using var sender = Sender.New($"tcp::addr={_host}:{_port};protocol_version=2;"); await sender.Table("metric name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("string", " -=\"") - .Column("array", new[] - { - 1.2 - }) - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("string", " -=\"") + .Column("array", new[] + { + 1.2 + }) + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.SendAsync(); - var expected = "metric\\ name,t\\ a\\ g=v\\ alu\\,\\ e number=10i,string=\" -=\\\"\",array==ARRAY<1>[1.2] 1000000000\n"; + var expected = + "metric\\ name,t\\ a\\ g=v\\ alu\\,\\ e number=10i,string=\" -=\\\"\",array==ARRAY<1>[1.2] 1000000000\n"; WaitAssert(srv, expected); } @@ -96,13 +141,13 @@ public void SendLineWithArrayProtocolV1Exception() try { sender.Table("metric name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("string", " -=\"") - .Column("array", new[] - { - 1.2 - }); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("string", " -=\"") + .Column("array", new[] + { + 1.2 + }); } catch (IngressError err) { @@ -125,12 +170,12 @@ public async Task SendLineExceedsBuffer() for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -155,12 +200,12 @@ public async Task SendLineReusesBuffer() for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -169,12 +214,12 @@ await sender.Table("table name") for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -199,12 +244,12 @@ public async Task SendLineTrimsBuffers() for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -214,12 +259,12 @@ await sender.Table("table name") for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -245,12 +290,12 @@ public async Task ServerDisconnects() for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); try { @@ -279,11 +324,11 @@ public async Task SendNegativeLongAndDouble() #pragma warning disable CS0618 // Type or member is obsolete await sender.Table("neg name") - .Column("number1", long.MinValue + 1) - .Column("number2", long.MaxValue) - .Column("number3", double.MinValue) - .Column("number4", double.MaxValue) - .AtNowAsync(); + .Column("number1", long.MinValue + 1) + .Column("number2", long.MaxValue) + .Column("number3", double.MinValue) + .Column("number4", double.MaxValue) + .AtNowAsync(); #pragma warning restore CS0618 // Type or member is obsolete await sender.SendAsync(); @@ -302,15 +347,15 @@ public async Task DoubleSerializationTest() #pragma warning disable CS0618 // Type or member is obsolete await sender.Table("doubles") - .Column("d0", 0.0) - .Column("dm0", -0.0) - .Column("d1", 1.0) - .Column("dE100", 1E100) - .Column("d0000001", 0.000001) - .Column("dNaN", double.NaN) - .Column("dInf", double.PositiveInfinity) - .Column("dNInf", double.NegativeInfinity) - .AtNowAsync(); + .Column("d0", 0.0) + .Column("dm0", -0.0) + .Column("d1", 1.0) + .Column("dE100", 1E100) + .Column("d0000001", 0.000001) + .Column("dNaN", double.NaN) + .Column("dInf", double.PositiveInfinity) + .Column("dNInf", double.NegativeInfinity) + .AtNowAsync(); #pragma warning restore CS0618 // Type or member is obsolete await sender.SendAsync(); @@ -329,8 +374,8 @@ public async Task SendTimestampColumn() var ts = new DateTime(2022, 2, 24); await sender.Table("name") - .Column("ts", ts) - .AtAsync(ts); + .Column("ts", ts) + .AtAsync(ts); await sender.SendAsync(); @@ -349,8 +394,8 @@ public async Task SendColumnNanos() const long timestampNanos = 1645660800123456789L; await sender.Table("name") - .ColumnNanos("ts", timestampNanos) - .AtAsync(timestampNanos); + .ColumnNanos("ts", timestampNanos) + .AtAsync(timestampNanos); await sender.SendAsync(); @@ -369,8 +414,8 @@ public async Task SendAtNanos() const long timestampNanos = 1645660800987654321L; await sender.Table("name") - .Column("value", 42) - .AtNanosAsync(timestampNanos); + .Column("value", 42) + .AtNanosAsync(timestampNanos); await sender.SendAsync(); @@ -385,8 +430,8 @@ public Task InvalidState() using var srv = CreateTcpListener(_port); srv.AcceptAsync(); - using var sender = Sender.New($"tcp::addr={_host}:{_port};"); - string? nullString = null; + using var sender = Sender.New($"tcp::addr={_host}:{_port};"); + string? nullString = null; Assert.That( () => sender.Table(nullString), @@ -488,8 +533,8 @@ public Task InvalidTableName() using var srv = CreateTcpListener(_port); srv.AcceptAsync(); - using var sender = Sender.New($"tcp::addr={_host}:{_port};"); - string? nullString = null; + using var sender = Sender.New($"tcp::addr={_host}:{_port};"); + string? nullString = null; Assert.Throws(() => sender.Table(nullString)); Assert.Throws(() => sender.Column("abc", 123)); @@ -546,19 +591,19 @@ public async Task SendMillionAsyncExplicit() srv.AcceptAsync(); var nowMillisecond = DateTime.Now.Millisecond; - var metric = "metric_name" + nowMillisecond; + var metric = "metric_name" + nowMillisecond; using var sender = Sender.New($"tcp::addr={_host}:{_port};init_buf_size={256 * 1024};"); for (var i = 0; i < 1E6; i++) { await sender.Table(metric) - .Symbol("nopoint", "tag" + i % 100) - .Column("counter", i * 1111.1) - .Column("int", i) - .Column("привед", "мед вед") - .AtAsync(new DateTime(2021, 1, 1, i / 360 / 1000 % 60, i / 60 / 1000 % 60, i / 1000 % 60, - i % 1000)); + .Symbol("nopoint", "tag" + i % 100) + .Column("counter", i * 1111.1) + .Column("int", i) + .Column("привед", "мед вед") + .AtAsync(new DateTime(2021, 1, 1, i / 360 / 1000 % 60, i / 60 / 1000 % 60, i / 1000 % 60, + i % 1000)); if (i % 100 == 0) { @@ -576,7 +621,7 @@ public async Task SendMillionFixedBuffer() srv.AcceptAsync(); var nowMillisecond = DateTime.Now.Millisecond; - var metric = "metric_name" + nowMillisecond; + var metric = "metric_name" + nowMillisecond; using var sender = Sender.New( @@ -585,12 +630,12 @@ public async Task SendMillionFixedBuffer() for (var i = 0; i < 1E6; i++) { await sender.Table(metric) - .Symbol("nopoint", "tag" + i % 100) - .Column("counter", i * 1111.1) - .Column("int", i) - .Column("привед", "мед вед") - .AtAsync(new DateTime(2021, 1, 1, i / 360 / 1000 % 60, i / 60 / 1000 % 60, i / 1000 % 60, - i % 1000)); + .Symbol("nopoint", "tag" + i % 100) + .Column("counter", i * 1111.1) + .Column("int", i) + .Column("привед", "мед вед") + .AtAsync(new DateTime(2021, 1, 1, i / 360 / 1000 % 60, i / 60 / 1000 % 60, i / 1000 % 60, + i % 1000)); } await sender.SendAsync(); @@ -618,8 +663,8 @@ public Task SendNegativeLongMin() Assert.That( #pragma warning disable CS0618 // Type or member is obsolete () => sender.Table("name") - .Column("number1", long.MinValue) - .AtNowAsync(), + .Column("number1", long.MinValue) + .AtNowAsync(), #pragma warning restore CS0618 // Type or member is obsolete Throws.TypeOf().With.Message.Contains("Special case") ); @@ -637,8 +682,8 @@ public async Task SendSpecialStrings() $"tcp::addr={_host}:{_port};"); #pragma warning disable CS0618 // Type or member is obsolete await sender.Table("neg name") - .Column("привед", " мед\rве\n д") - .AtNowAsync(); + .Column("привед", " мед\rве\n д") + .AtNowAsync(); #pragma warning restore CS0618 // Type or member is obsolete await sender.SendAsync(); @@ -659,9 +704,9 @@ public Task SendTagAfterField() Assert.That( #pragma warning disable CS0618 // Type or member is obsolete async () => await sender.Table("name") - .Column("number1", 123) - .Symbol("nand", "asdfa") - .AtNowAsync(), + .Column("number1", 123) + .Symbol("nand", "asdfa") + .AtNowAsync(), #pragma warning restore CS0618 // Type or member is obsolete Throws.TypeOf() ); @@ -681,9 +726,9 @@ public Task SendMetricOnce() Assert.That( #pragma warning disable CS0618 // Type or member is obsolete async () => await sender.Table("name") - .Column("number1", 123) - .Table("nand") - .AtNowAsync(), + .Column("number1", 123) + .Table("nand") + .AtNowAsync(), #pragma warning restore CS0618 // Type or member is obsolete Throws.TypeOf() ); @@ -703,7 +748,7 @@ public Task StartFromMetric() Assert.That( #pragma warning disable CS0618 // Type or member is obsolete async () => await sender.Column("number1", 123) - .AtNowAsync(), + .AtNowAsync(), #pragma warning restore CS0618 // Type or member is obsolete Throws.TypeOf() ); @@ -711,7 +756,7 @@ public Task StartFromMetric() Assert.That( #pragma warning disable CS0618 // Type or member is obsolete async () => await sender.Symbol("number1", "1234") - .AtNowAsync(), + .AtNowAsync(), #pragma warning restore CS0618 // Type or member is obsolete Throws.TypeOf() ); @@ -824,39 +869,39 @@ public async Task SendVariousAts() #pragma warning disable CS0618 // Type or member is obsolete await sender.Table("foo") - .Symbol("bah", "baz") - .AtNowAsync(); + .Symbol("bah", "baz") + .AtNowAsync(); #pragma warning restore CS0618 // Type or member is obsolete await sender.Table("foo") - .Symbol("bah", "baz") - .AtAsync(DateTime.UtcNow); + .Symbol("bah", "baz") + .AtAsync(DateTime.UtcNow); await sender.Table("foo") - .Symbol("bah", "baz") - .AtAsync(DateTimeOffset.UtcNow); + .Symbol("bah", "baz") + .AtAsync(DateTimeOffset.UtcNow); await sender.Table("foo") - .Symbol("bah", "baz") - .AtAsync(DateTime.UtcNow.Ticks / 100); + .Symbol("bah", "baz") + .AtAsync(DateTime.UtcNow.Ticks / 100); #pragma warning disable CS0618 // Type or member is obsolete sender.Table("foo") - .Symbol("bah", "baz") - .AtNow(); + .Symbol("bah", "baz") + .AtNow(); #pragma warning restore CS0618 // Type or member is obsolete sender.Table("foo") - .Symbol("bah", "baz") - .At(DateTime.UtcNow); + .Symbol("bah", "baz") + .At(DateTime.UtcNow); sender.Table("foo") - .Symbol("bah", "baz") - .At(DateTimeOffset.UtcNow); + .Symbol("bah", "baz") + .At(DateTimeOffset.UtcNow); sender.Table("foo") - .Symbol("bah", "baz") - .At(DateTime.UtcNow.Ticks / 100); + .Symbol("bah", "baz") + .At(DateTime.UtcNow.Ticks / 100); await sender.SendAsync(); } @@ -884,7 +929,7 @@ public async Task Authenticate() { using var srv = CreateTcpListener(_port); srv.WithAuth("testUser1", "Vs4e-cOLsVCntsMrZiAGAZtrkPXO00uoRLuA3d7gEcI=", - "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); + "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); srv.AcceptAsync(); using var sender = @@ -892,10 +937,10 @@ public async Task Authenticate() $"tcp::addr={_host}:{_port};username=testUser1;token=NgdiOWDoQNUP18WOnb1xkkEG5TzPYMda5SiUOvT1K0U=;"); await sender.Table("metric name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("string", " -=\"") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("string", " -=\"") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.SendAsync(); var expected = "metric\\ name,t\\ a\\ g=v\\ alu\\,\\ e number=10i,string=\" -=\\\"\" 1000000000\n"; @@ -907,14 +952,14 @@ public Task AuthFailWrongKid() { using var srv = CreateTcpListener(_port); srv.WithAuth("testUser1", "Vs4e-cOLsVCntsMrZiAGAZtrkPXO00uoRLuA3d7gEcI=", - "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); + "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); srv.AcceptAsync(); Assert.That( () => Sender.New($"tcp::addr={_host}:{_port};username=invalid;token=foo=;") - , + , Throws.TypeOf().With.InnerException.TypeOf().With.Message - .Contains("Authentication failed") + .Contains("Authentication failed") ); return Task.CompletedTask; } @@ -924,7 +969,7 @@ public Task AuthFailBadKey() { using var srv = CreateTcpListener(_port); srv.WithAuth("testUser1", "Vs4e-cOLsVCntsMrZiAGAZtrkPXO00uoRLuA3d7gEcI=", - "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); + "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); srv.AcceptAsync(); using var sender = Sender.New( @@ -933,13 +978,13 @@ public Task AuthFailBadKey() Assert.That( async () => { - for (var i = 0; i < 10; i++) + for (var i = 0; i < 100; i++) { await sender.Table("metric name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("string", " -=\"") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("string", " -=\"") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.SendAsync(); Thread.Sleep(10); } @@ -947,7 +992,7 @@ await sender.Table("metric name") Assert.Fail(); }, Throws.TypeOf().With.Message - .Contains("Could not write data to server.") + .Contains("Could not write data to server.") ); return Task.CompletedTask; } @@ -956,7 +1001,7 @@ await sender.Table("metric name") public void EcdsaSignatureLoop() { var privateKey = Convert.FromBase64String("NgdiOWDoQNUP18WOnb1xkkEG5TzPYMda5SiUOvT1K0U="); - var p = SecNamedCurves.GetByName("secp256r1"); + var p = SecNamedCurves.GetByName("secp256r1"); var parameters = new ECDomainParameters(p.Curve, p.G, p.N, p.H); var m = new byte[512]; @@ -997,9 +1042,25 @@ private DummyIlpServer CreateTcpListener(int port, bool tls = false) return new DummyIlpServer(port, tls); } + private static byte[] WaitForLineBytes(DummyIlpServer server) + { + for (var i = 0; i < 500; i++) + { + var bytes = server.GetReceivedBytes(); + if (bytes.Length > 0 && bytes[^1] == (byte)'\n') + { + return bytes; + } + + Thread.Sleep(10); + } + + Assert.Fail("Timed out waiting for decimal ILP payload."); + return Array.Empty(); + } + private byte[] FromBase64String(string text) { return DummyIlpServer.FromBase64String(text); } - } \ No newline at end of file diff --git a/src/net-questdb-client/Buffers/BufferV3.cs b/src/net-questdb-client/Buffers/BufferV3.cs index 85e5331..3609637 100644 --- a/src/net-questdb-client/Buffers/BufferV3.cs +++ b/src/net-questdb-client/Buffers/BufferV3.cs @@ -25,7 +25,6 @@ using System.Buffers.Binary; using System.Runtime.InteropServices; using QuestDB.Enums; -using QuestDB.Utils; namespace QuestDB.Buffers; @@ -65,61 +64,58 @@ public override IBuffer Column(ReadOnlySpan name, decimal? value) .Put((byte)BinaryFormatType.DECIMAL); if (value is null) { - Put((byte)0); // Scale - Put((byte)0); // Length + Put(0); // Scale + Put(0); // Length return this; } Span parts = stackalloc int[4]; decimal.GetBits(value.Value, parts); - int flags = parts[3]; - byte scale = (byte)((flags & ScaleMask) >> ScaleShift); + var flags = parts[3]; + var scale = (byte)((flags & ScaleMask) >> ScaleShift); // 3. Scale Put(scale); - int low = parts[0]; - int mid = parts[1]; - int high = parts[2]; - bool bitSign = false; - bool negative = (flags & SignMask) != 0; + var low = parts[0]; + var mid = parts[1]; + var high = parts[2]; + var negative = (flags & SignMask) != 0 && value != 0; if (negative) { // QuestDB expects negative mantissas in two's complement. low = ~low + 1; - int c = low == 0 ? 1 : 0; + var c = low == 0 ? 1 : 0; mid = ~mid + c; c = mid == 0 && c == 1 ? 1 : 0; high = ~high + c; - // We may overflow, we need an extra byte to convey the sign. - bitSign = high == 0 && c == 1; } - else if ((high & 0x80000000) != 0) - { - // If the highest bit is set, we need an extra byte of 0 to convey the sign. - bitSign = true; - } - - var size = bitSign ? 13 : 12; + // We write the byte array on the stack first so that we can compress (remove unnecessary bytes) it later. + Span span = stackalloc byte[13]; + var signByte = (byte)(negative ? 255 : 0); + span[0] = signByte; + BinaryPrimitives.WriteInt32BigEndian(span.Slice(1, 4), high); + BinaryPrimitives.WriteInt32BigEndian(span.Slice(5, 4), mid); + BinaryPrimitives.WriteInt32BigEndian(span.Slice(9, 4), low); + + // Compress + var start = 0; + for (; + // We can strip prefix bits that are 0 (if positive) or 1 (if negative) as long as we keep at least + // one of it in front to convey the sign. + start < span.Length - 1 && span[start] == signByte && ((span[start + 1] ^ signByte) & 0x80) == 0; + start++) ; + // 4. Length + var size = span.Length - start; Put((byte)size); // 5. Unscaled value EnsureCapacity(size); - var span = Chunk.AsSpan(Position, size); - var offset = 0; - if (bitSign) - { - span[offset++] = (byte)(negative ? 255 : 0); - } - BinaryPrimitives.WriteInt32BigEndian(span.Slice(offset, 4), high); - offset += 4; - BinaryPrimitives.WriteInt32BigEndian(span.Slice(offset, 4), mid); - offset += 4; - BinaryPrimitives.WriteInt32BigEndian(span.Slice(offset, 4), low); + span.Slice(start, size).CopyTo(Chunk.AsSpan(Position, size)); Advance(size); return this; diff --git a/src/net-questdb-client/Buffers/IBuffer.cs b/src/net-questdb-client/Buffers/IBuffer.cs index 80fe69b..6074853 100644 --- a/src/net-questdb-client/Buffers/IBuffer.cs +++ b/src/net-questdb-client/Buffers/IBuffer.cs @@ -235,9 +235,7 @@ public interface IBuffer public IBuffer Column(ReadOnlySpan name, IEnumerable value, IEnumerable shape) where T : struct; /// - /// Records a DECIMAL column value using the ILP binary decimal layout: - /// '=' marker, decimal type id (23), scale byte, mantissa length, and a big-endian - /// two's complement mantissa sourced from . + /// Records a DECIMAL column value using the ILP binary decimal layout. /// public IBuffer Column(ReadOnlySpan name, decimal? value); -} +} \ No newline at end of file diff --git a/src/net-questdb-client/Enums/BinaryFormatType.cs b/src/net-questdb-client/Enums/BinaryFormatType.cs index 495901f..b0ee5b6 100644 --- a/src/net-questdb-client/Enums/BinaryFormatType.cs +++ b/src/net-questdb-client/Enums/BinaryFormatType.cs @@ -31,4 +31,5 @@ public enum BinaryFormatType : byte { DOUBLE = 16, ARRAY = 14, + DECIMAL = 23, } \ No newline at end of file diff --git a/src/net-questdb-client/Enums/ProtocolVersion.cs b/src/net-questdb-client/Enums/ProtocolVersion.cs index d6bff49..3904cc2 100644 --- a/src/net-questdb-client/Enums/ProtocolVersion.cs +++ b/src/net-questdb-client/Enums/ProtocolVersion.cs @@ -40,7 +40,12 @@ public enum ProtocolVersion V1, /// - /// Text ILP with binary extensions for DOUBLE and ARRAY[DOUBLE] + /// Text ILP with binary for DOUBLE and support for the ARRAY[DOUBLE] datatype /// V2, + + /// + /// Support for the DECIMAL datatype + /// + V3, } \ No newline at end of file diff --git a/src/net-questdb-client/Senders/AbstractSender.cs b/src/net-questdb-client/Senders/AbstractSender.cs index 6fcd1b0..54a3e26 100644 --- a/src/net-questdb-client/Senders/AbstractSender.cs +++ b/src/net-questdb-client/Senders/AbstractSender.cs @@ -94,6 +94,13 @@ public ISender Column(ReadOnlySpan name, long value) return this; } + /// + public ISender Column(ReadOnlySpan name, int value) + { + Buffer.Column(name, value); + return this; + } + /// public ISender Column(ReadOnlySpan name, bool value) { @@ -312,4 +319,10 @@ private void GuardLastFlushNotSet() LastFlush = DateTime.UtcNow; } } + + public ISender Column(ReadOnlySpan name, decimal? value) + { + Buffer.Column(name, value); + return this; + } } \ No newline at end of file diff --git a/src/net-questdb-client/Senders/HttpSender.cs b/src/net-questdb-client/Senders/HttpSender.cs index 2ed207f..cae2d10 100644 --- a/src/net-questdb-client/Senders/HttpSender.cs +++ b/src/net-questdb-client/Senders/HttpSender.cs @@ -59,7 +59,7 @@ internal class HttpSender : AbstractSender public HttpSender(SenderOptions options) { - _sendRequestFactory = GenerateRequest; + _sendRequestFactory = GenerateRequest; _settingRequestFactory = GenerateSettingsRequest; Options = options; @@ -75,12 +75,12 @@ private void Build() _handler = new SocketsHttpHandler { PooledConnectionIdleTimeout = Options.pool_timeout, - MaxConnectionsPerServer = 1, + MaxConnectionsPerServer = 1, }; if (Options.protocol == ProtocolType.https) { - _handler.SslOptions.TargetHost = Options.Host; + _handler.SslOptions.TargetHost = Options.Host; _handler.SslOptions.EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13; if (Options.tls_verify == TlsVerifyType.unsafe_off) @@ -116,13 +116,13 @@ private void Build() } } - _handler.ConnectTimeout = Options.auth_timeout; + _handler.ConnectTimeout = Options.auth_timeout; _handler.PreAuthenticate = true; _client = new HttpClient(_handler); var uri = new UriBuilder(Options.protocol.ToString(), Options.Host, Options.Port); _client.BaseAddress = uri.Uri; - _client.Timeout = Timeout.InfiniteTimeSpan; + _client.Timeout = Timeout.InfiniteTimeSpan; if (!string.IsNullOrEmpty(Options.username) && !string.IsNullOrEmpty(Options.password)) { @@ -141,7 +141,7 @@ private void Build() if (protocolVersion == ProtocolVersion.Auto) { - // We need to see if this will be a V1 or a V2 + // We need to select the last version that both client and server support. // Other clients use 1 second timeout for "/settings", follow same practice here. using var response = SendWithRetries(default, _settingRequestFactory, TimeSpan.FromSeconds(1)); if (!response.IsSuccessStatusCode) @@ -162,17 +162,9 @@ private void Build() { try { - var json = response.Content.ReadFromJsonAsync().Result!; + var json = response.Content.ReadFromJsonAsync().Result!; var versions = json.Config?.LineProtoSupportVersions!; - foreach (var element in versions) - { - if (element == (int)ProtocolVersion.V2) - { - // V2 is supported, use it. - protocolVersion = ProtocolVersion.V2; - break; - } - } + protocolVersion = (ProtocolVersion)versions.Where(v => v <= (int)ProtocolVersion.V3).Max(); } catch { @@ -217,8 +209,8 @@ private CancellationTokenSource GenerateRequestCts(CancellationToken ct = defaul private HttpRequestMessage GenerateRequest() { var request = new HttpRequestMessage(HttpMethod.Post, "/write") - { Content = new BufferStreamContent(Buffer), }; - request.Content.Headers.ContentType = new MediaTypeHeaderValue("text/plain") { CharSet = "utf-8", }; + { Content = new BufferStreamContent(Buffer), }; + request.Content.Headers.ContentType = new MediaTypeHeaderValue("text/plain") { CharSet = "utf-8", }; request.Content.Headers.ContentLength = Buffer.Length; return request; } @@ -331,7 +323,7 @@ public override void Send(CancellationToken ct = default) if (response.IsSuccessStatusCode) { LastFlush = (response.Headers.Date ?? DateTime.UtcNow).UtcDateTime; - success = true; + success = true; return; } @@ -365,9 +357,9 @@ public override void Send(CancellationToken ct = default) private HttpResponseMessage SendWithRetries(CancellationToken ct, Func requestFactory, TimeSpan retryTimeout) { - HttpResponseMessage? response = null; - CancellationTokenSource cts = GenerateRequestCts(ct); - HttpRequestMessage request = requestFactory(); + HttpResponseMessage? response = null; + CancellationTokenSource cts = GenerateRequestCts(ct); + HttpRequestMessage request = requestFactory(); try { @@ -381,7 +373,7 @@ private HttpResponseMessage SendWithRetries(CancellationToken ct, Func TimeSpan.Zero) - // retry if appropriate - error that's retriable, and retries are enabled + // retry if appropriate - error that's retriable, and retries are enabled { if (response == null // if it was a cannot correct error || (!response.IsSuccessStatusCode // or some other http error @@ -406,7 +398,7 @@ private HttpResponseMessage SendWithRetries(CancellationToken ct, Func /// Interface representing implementations. ///
-public interface ISender : ISenderV2 -{ -} - -/// -/// Version 1 of the Sender API. -/// -public interface ISenderV1 : IDisposable +public interface ISender : IDisposable { /// /// Represents the current length of the buffer in UTF-8 bytes. @@ -134,19 +127,22 @@ public interface ISenderV1 : IDisposable /// Itself public ISender Column(ReadOnlySpan name, ReadOnlySpan value); - /// + /// public ISender Column(ReadOnlySpan name, long value); - /// + /// + public ISender Column(ReadOnlySpan name, int value); + + /// public ISender Column(ReadOnlySpan name, bool value); - /// + /// public ISender Column(ReadOnlySpan name, double value); - /// + /// public ISender Column(ReadOnlySpan name, DateTime value); - /// + /// public ISender Column(ReadOnlySpan name, DateTimeOffset value); /// @@ -217,15 +213,9 @@ public interface ISenderV1 : IDisposable /// Clears the sender's buffer. /// public void Clear(); -} -/// -/// Version 2 of the Sender API, adding ARRAY and binary DOUBLE support. -/// -public interface ISenderV2 : ISenderV1 -{ /// + /// cref="Column{T}(ReadOnlySpan{char},IEnumerable{T},IEnumerable{int})" /> public ISender Column(ReadOnlySpan name, IEnumerable value, IEnumerable shape) where T : struct; /// @@ -238,7 +228,7 @@ public interface ISenderV2 : ISenderV1 public ISender Column(ReadOnlySpan name, Array value); /// + /// cref="Column{T}(ReadOnlySpan{char},IEnumerable{T},IEnumerable{int})" /> public ISender Column(ReadOnlySpan name, ReadOnlySpan value) where T : struct; /// @@ -249,7 +239,7 @@ public interface ISenderV2 : ISenderV1 /// Itself public ISender Column(ReadOnlySpan name, string? value) { - return ((ISenderV1)this).Column(name, value); + return Column(name, value.AsSpan()); } /// @@ -261,7 +251,7 @@ public ISender NullableColumn(ReadOnlySpan name, IEnumerable? value, Column(name, value, shape); } - return (ISender)this; + return this; } /// @@ -272,7 +262,7 @@ public ISender NullableColumn(ReadOnlySpan name, Array? value) Column(name, value); } - return (ISender)this; + return this; } /// @@ -283,7 +273,7 @@ public ISender NullableColumn(ReadOnlySpan name, string? value) Column(name, value); } - return (ISender)this; + return this; } /// @@ -294,7 +284,7 @@ public ISender NullableColumn(ReadOnlySpan name, long? value) return Column(name, value ?? throw new InvalidOperationException()); } - return (ISender)this; + return this; } /// @@ -305,7 +295,7 @@ public ISender NullableColumn(ReadOnlySpan name, bool? value) return Column(name, value ?? throw new InvalidOperationException()); } - return (ISender)this; + return this; } /// @@ -316,7 +306,7 @@ public ISender NullableColumn(ReadOnlySpan name, double? value) return Column(name, value ?? throw new InvalidOperationException()); } - return (ISender)this; + return this; } /// @@ -327,7 +317,7 @@ public ISender NullableColumn(ReadOnlySpan name, DateTime? value) return Column(name, value ?? throw new InvalidOperationException()); } - return (ISender)this; + return this; } /// @@ -338,6 +328,11 @@ public ISender NullableColumn(ReadOnlySpan name, DateTimeOffset? value) return Column(name, value ?? throw new InvalidOperationException()); } - return (ISender)this; + return this; } + + /// + /// Adds a DECIMAL column in the binary format. + /// + public ISender Column(ReadOnlySpan name, decimal? value); } \ No newline at end of file diff --git a/src/net-questdb-client/Senders/ISenderV1.cs b/src/net-questdb-client/Senders/ISenderV1.cs new file mode 100644 index 0000000..c52e0e5 --- /dev/null +++ b/src/net-questdb-client/Senders/ISenderV1.cs @@ -0,0 +1,34 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +// ReSharper disable InconsistentNaming + +namespace QuestDB.Senders; + +/// +/// Version 1 of the Sender API. +/// +public interface ISenderV1 : ISender +{ +} \ No newline at end of file diff --git a/src/net-questdb-client/Senders/ISenderV2.cs b/src/net-questdb-client/Senders/ISenderV2.cs new file mode 100644 index 0000000..7f7bfbe --- /dev/null +++ b/src/net-questdb-client/Senders/ISenderV2.cs @@ -0,0 +1,38 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +// ReSharper disable CommentTypo + +using QuestDB.Utils; + +// ReSharper disable InconsistentNaming + +namespace QuestDB.Senders; + +/// +/// Version 2 of the Sender API, adding ARRAY and binary DOUBLE support. +/// +public interface ISenderV2 : ISender +{ +} \ No newline at end of file diff --git a/src/tcp-client-test/DummyIlpServer.cs b/src/tcp-client-test/DummyIlpServer.cs deleted file mode 100644 index 55c39d0..0000000 --- a/src/tcp-client-test/DummyIlpServer.cs +++ /dev/null @@ -1,239 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - - -using System; -using System.IO; -using System.Net; -using System.Net.Security; -using System.Net.Sockets; -using System.Security.Cryptography.X509Certificates; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Org.BouncyCastle.Asn1.Sec; -using Org.BouncyCastle.Crypto.Parameters; -using Org.BouncyCastle.Math; -using Org.BouncyCastle.Security; - -// ReSharper disable NonAtomicCompoundOperator - -namespace tcp_client_test; - -public class DummyIlpServer : IDisposable -{ - private readonly byte[] _buffer = new byte[2048]; - private readonly CancellationTokenSource _cancellationTokenSource = new(); - private readonly MemoryStream _received = new(); - private readonly TcpListener _server; - private readonly bool _tls; - private string? _keyId; - private string? _publicKeyX; - private string? _publicKeyY; - private volatile int _totalReceived; - - public DummyIlpServer(int port, bool tls) - { - _tls = tls; - _server = new TcpListener(IPAddress.Loopback, port); - _server.Start(); - } - - public int TotalReceived => _totalReceived; - - public void Dispose() - { - _cancellationTokenSource.Cancel(); - _server.Stop(); - } - - public void AcceptAsync() - { - Task.Run(AcceptConnections); - } - - private async Task AcceptConnections() - { - Socket? clientSocket = null; - try - { - using var socket = await _server.AcceptSocketAsync(); - clientSocket = socket; - await using var connection = new NetworkStream(socket, true); - Stream dataStream = connection; - if (_tls) - { - var sslStream = new SslStream(connection); - dataStream = sslStream; - await sslStream.AuthenticateAsServerAsync(GetCertificate()); - } - - if (_keyId != null) - { - await RunServerAuth(dataStream); - } - - await SaveData(dataStream, socket); - } - catch (SocketException ex) - { - Console.WriteLine($"Error {ex.ErrorCode}: Server socket error."); - } - finally - { - clientSocket?.Dispose(); - } - } - - private X509Certificate GetCertificate() - { - return X509Certificate.CreateFromCertFile("certificate.pfx"); - } - - private async Task RunServerAuth(Stream connection) - { - var receivedLen = await ReceiveUntilEol(connection); - - var requestedKeyId = Encoding.UTF8.GetString(_buffer, 0, receivedLen); - if (requestedKeyId != _keyId) - { - connection.Close(); - return; - } - - var challenge = new byte[512]; - GenerateRandomBytes(challenge, 512); - await connection.WriteAsync(challenge); - _buffer[0] = (byte)'\n'; - await connection.WriteAsync(_buffer.AsMemory(0, 1)); - - receivedLen = await ReceiveUntilEol(connection); - var signatureRaw = Encoding.UTF8.GetString(_buffer.AsSpan(0, receivedLen)); - Console.WriteLine(signatureRaw); - var signature = Convert.FromBase64String(Pad(signatureRaw)); - - if (_publicKeyX == null || _publicKeyY == null) - { - throw new InvalidOperationException("public key not set"); - } - - var pubKey1 = FromBase64String(_publicKeyX); - var pubKey2 = FromBase64String(_publicKeyY); - - var p = SecNamedCurves.GetByName("secp256r1"); - var parameters = new ECDomainParameters(p.Curve, p.G, p.N, p.H); - - // Verify the signature - var pubKey = new ECPublicKeyParameters( - parameters.Curve.CreatePoint(new BigInteger(1, pubKey1), new BigInteger(1, pubKey2)), - parameters); - - var ecdsa = SignerUtilities.GetSigner("SHA-256withECDSA"); - ecdsa.Init(false, pubKey); - ecdsa.BlockUpdate(challenge, 0, challenge.Length); - if (!ecdsa.VerifySignature(signature)) - { - connection.Close(); - } - } - - private static string Pad(string text) - { - var padding = 3 - (text.Length + 3) % 4; - if (padding == 0) - { - return text; - } - - return text + new string('=', padding); - } - - public static byte[] FromBase64String(string encodedPrivateKey) - { - var replace = encodedPrivateKey - .Replace('-', '+') - .Replace('_', '/'); - return Convert.FromBase64String(Pad(replace)); - } - - private async Task ReceiveUntilEol(Stream connection) - { - var len = 0; - while (true) - { - var n = await connection.ReadAsync(_buffer.AsMemory(len)); - var inBuffer = len + n; - for (var i = len; i < inBuffer; i++) - { - if (_buffer[i] == '\n') - { - if (i + 1 < inBuffer) - { - _received.Write(_buffer, i + 1, inBuffer - i - 1); - _totalReceived += inBuffer - i; - } - - return i; - } - } - - len += n; - } - } - - private void GenerateRandomBytes(byte[] buffer, int length) - { - var rnd = new Random(DateTime.Now.Millisecond); - rnd.NextBytes(buffer.AsSpan(length)); - } - - private async Task SaveData(Stream connection, Socket socket) - { - while (!_cancellationTokenSource.IsCancellationRequested && socket.Connected) - { - var received = await connection.ReadAsync(_buffer); - if (received > 0) - { - _received.Write(_buffer, 0, received); - _totalReceived += received; - } - else - { - return; - } - } - } - - public string GetTextReceived() - { - return Encoding.UTF8.GetString(_received.GetBuffer(), 0, (int)_received.Length); - } - - public void WithAuth(string keyId, string publicKeyX, string publicKeyY) - { - _keyId = keyId; - _publicKeyX = publicKeyX; - _publicKeyY = publicKeyY; - } -} \ No newline at end of file diff --git a/src/tcp-client-test/JsonSpecTestRunner.cs b/src/tcp-client-test/JsonSpecTestRunner.cs deleted file mode 100644 index 8dcf605..0000000 --- a/src/tcp-client-test/JsonSpecTestRunner.cs +++ /dev/null @@ -1,202 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - - -using System; -using System.IO; -using System.Linq; -using System.Net; -using System.Text; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using NUnit.Framework; -using QuestDB; - -// ReSharper disable InconsistentNaming - -#pragma warning disable CS0618 // Type or member is obsolete - -namespace tcp_client_test; - -[TestFixture] -public class JsonSpecTestRunner -{ - private const int Port = 29472; - private static readonly TestCase[]? TestCases = ReadTestCases(); - - [TestCaseSource(nameof(TestCases))] - public async Task Run(TestCase testCase) - { - using var srv = CreateTcpListener(Port); - srv.AcceptAsync(); - -#pragma warning disable CS0612 // Type or member is obsolete - using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), Port, tlsMode: TlsMode.Disable); -#pragma warning restore CS0612 // Type or member is obsolete - Exception? exception = null; - - try - { - ls.Table(testCase.table); - foreach (var symbol in testCase.symbols) - { - ls.Symbol(symbol.name, symbol.value); - } - - foreach (var column in testCase.columns) - { - switch (column.type) - { - case "STRING": - ls.Column(column.name, ((JsonElement)column.value).GetString()); - break; - - case "DOUBLE": - ls.Column(column.name, ((JsonElement)column.value).GetDouble()); - break; - - case "BOOLEAN": - ls.Column(column.name, ((JsonElement)column.value).GetBoolean()); - break; - - case "LONG": - ls.Column(column.name, (long)((JsonElement)column.value).GetDouble()); - break; - - default: - throw new NotSupportedException("Column type not supported: " + column.type); - } - } - - ls.AtNow(); - ls.Send(); - } - catch (Exception? ex) - { - if (testCase.result.status == "SUCCESS") - { - throw; - } - - exception = ex; - } - - if (testCase.result.status == "SUCCESS") - { - if (testCase.result.anyLines == null || testCase.result.anyLines.Length == 0) - { - WaitAssert(srv, testCase.result.line + "\n"); - } - else - { - WaitAssert(srv, testCase.result.anyLines); - } - } - else if (testCase.result.status == "ERROR") - { - Assert.NotNull(exception, "Exception should be thrown"); - if (exception is NotSupportedException) - { - throw exception; - } - } - else - { - Assert.Fail("Unsupported test case result status: " + testCase.result.status); - } - } - - private void WaitAssert(DummyIlpServer srv, string[] resultAnyLines) - { - var minExpectedLen = resultAnyLines.Select(l => Encoding.UTF8.GetBytes(l).Length).Min(); - for (var i = 0; i < 500 && srv.TotalReceived < minExpectedLen; i++) - { - Thread.Sleep(10); - } - - var textReceived = srv.GetTextReceived(); - var anyMatch = resultAnyLines.Any(l => (l + "\n").Equals(textReceived)); - if (!anyMatch) - { - Assert.Fail(textReceived + ": did not match any expected results"); - } - } - - private static void WaitAssert(DummyIlpServer srv, string expected) - { - var expectedLen = Encoding.UTF8.GetBytes(expected).Length; - for (var i = 0; i < 500 && srv.TotalReceived < expectedLen; i++) - { - Thread.Sleep(10); - } - - Assert.AreEqual(expected, srv.GetTextReceived()); - } - - private DummyIlpServer CreateTcpListener(int port, bool tls = false) - { - return new DummyIlpServer(port, tls); - } - - private static TestCase[]? ReadTestCases() - { - using var jsonFile = File.OpenRead("ilp-client-interop-test.json"); - return JsonSerializer.Deserialize(jsonFile); - } - - public class TestCase - { - public string testName { get; set; } = null!; - public string table { get; set; } = null!; - public TestCaseSymbol[] symbols { get; set; } = null!; - public TestCaseColumn[] columns { get; set; } = null!; - public TestCaseResult result { get; set; } = null!; - - public override string ToString() - { - return testName; - } - } - - public class TestCaseSymbol - { - public string name { get; set; } = null!; - public string value { get; set; } = null!; - } - - public class TestCaseColumn - { - public string type { get; set; } = null!; - public string name { get; set; } = null!; - public object value { get; set; } = null!; - } - - public class TestCaseResult - { - public string status { get; set; } = null!; - public string line { get; set; } = null!; - public string[]? anyLines { get; set; } = null!; - } -} \ No newline at end of file diff --git a/src/tcp-client-test/certificate.pfx b/src/tcp-client-test/certificate.pfx deleted file mode 100644 index 1c4c174468c43b75febe6e27421ff2be390ec892..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2341 zcmY+EcQhM{9>YD2cV_-`Mmx~XyJSyrhhlE(1PF)Ah=msZfHjD*aZLp z(?sCt)Tq;eqm`op-h8H34pGtWjHbb5X9`>e@>sIj?rl(!gSJxA+f>4O#+y5G zGJE{Fzzt~jD@$l#n52j%?;-jqzaQ$5R#9VIcTi8&tume8ZXOKRdYPOul$(sZ;Y-zw z_a@va`D52vKG#S>$}`Jyrtp>Z2c^9k14UnyOl?k{iCK;ZrOdODNUC1ywO5kUJ8GjP zJC5{zz&HofF|22?>1d4!bTo>Je*!Zi&%2U%&PD%f_PSEx6Q-jcbj^S6eQRW=Pf;Y* zkEbA~Yq>W{Bi^N6#Z<1<5fq4-KbW> zmk$=+-Oat(fS6sfncZmp#I8ogTr8qb^d{zLR%Cs5S2I}qGk?440gAQTa>;?c5-z8) z>$43LFlUh??zT@|*qhE9k-kOV_|i~n^1TF8fZAHKwDma8&ggUwB=T-pW0j>o{%O-} zLCnaZd5ySL%h+I2_a=4|x0ydu^9YKC7&0)sRn+urH;n1soe0F-e;ytlis$O$>w73f@PAStGrrfbY^bbL)_8Pfk?qu%C^?^DS; z8 zv7I*6x_VH$4YEIRsPXB0)`Yc6a2$63`0Lk@bubQqFQB{KmzF_rc9w@v?;Ko*UX?w< z2qW%?X0FCcnjU#665OSpgqzCDaKt5gQ09W`r|7pQ7pz@nne4j~m#e?EBkIIRfQjvN zolw>{(LlTb!FT+iw9wM9z?oXN-k$B^12GYE;0u8y%VIzS9np=%X_ItXpX!OJ+C#24 zThd^^Cpi*X&*1v4B4dBSB?81;9KP~1_%qm`SLeJZn@b+)t#h^ll$1@a0gOqJx5(=% z+&OR@z54$MSBRsR0pjQ-e`2wpCk$f#kA6(FfI=K_2aW@7{-2d}zpO;;Y-(#V%vk)g z5{Lu#ca_pW-W8_5v&R=hD`AXfLF;0Fz}R|5)b!TRLj9~5N6eQqU--gSyetsvmb+2s zf)*X5h@szej|B7YZ-sC3f0vNWA4XDm(mr#cqYW80lwa5-tXV#2#2a=nqYDhyah}n9 z;sXw(L;7hlJdLM!H{!JaR3H_$orCho9A>q-6+p-A;qS zGl6LnA4z`l@n<*ni?S~&()%o`CgP~~YIoEX+_*^Qu>#Rr^?8Z1pFsW-W$g;+F83=XutNPm>a^Q$mZZ9a`Pi+bO ztyLFF?Mrr)M`n29t?KsNdaon1_-*S6XNpvtoNABKPQrMxo;R~UjLd}%Ykc_S_?A(h zIpLYY_uiL|k73N-X?K81nr!*luu;_O0^@j|xcMd!Yque(BydxH@|gP(5Y^c#DDR;l zuz|03)ionMkJve1t9hetSGds4@Tah0>?~2m?Y2+C@D%!@8H7oPnUbLg;qYix~_b>keSyhTY{V? zoDaoXXcJH8#79Y5{Z&G(TG_~_&r1y<=lcAD@vjjJOlWo<%&Y-29Z|G+L?CA$c^*gW zq&N;xOye&vc7Z}cmS#c4gNBDQsME&ME+3ni5RtX4WHBZ;Q>D>TvcJQwH#( zRNQUn(qkD~9%0ptGv{v?aJK-O4}0}9Blif7@!p!s?#?tk2iz{RTYS#!kj(h7vcQ2C z@4!HLDIw-Ba(9Ii@S?F&pEvES8W+`$=QQ?(BUW^^ZhDa&rNgLiP1pMp;@k{aV{aZt z8iibi?@7*6EVX8MMS*M+GTKQg9IlWGZ@43T!-_p80AK(4Y@?qeED7l;+>w`Nl@@KS zj(eMcxzaf{=sjlzxnri0>kj5ywu`9SZ4-%3khpZPB1d7V)I7rr&; zXTGAASbzRZX&+bDx&(4T__D{HOrMNUBFk+*OS#H(@{=z|Q9!4L8(uwVYG<4SLNZQa z_cfk^x@p}|J38$0&!wotkUC~bg9b2 z|8KOH6WI5=(mqO5kkpb8@yO$H9^|CJ=P`Y&ehXShp?w#19U14Zjb=eq=D*Vs;^Hd9 za$48VovbI9Xg<)ruOinh9vGD|=HXx4HT;+-%N<~Y{!p&kw)~8>yBmv_oLUQ(3HF7m zsI6er$j$pR#HHW~_>9JIFXwYNn+@2P)Z6DEy0MyX@=5%`?0Q{R0Zjh8u8AuPMX5q_ z*W^Qn;>rjP>9B-c3Gx>TGD(T-C*xCHWiEXS4*l;sGG3MD1ab&{Rk&pbUU2_%;B|4C zqFw%UjIs)shTniQg6PE|bO0WD8X(7TU{6-mDl6_@i?uask(;ebHOLhRE^rDuvkkhP I{A(ru1&V`LzyJUM diff --git a/src/tcp-client-test/tcp-client-test.csproj b/src/tcp-client-test/tcp-client-test.csproj deleted file mode 100644 index 009274b..0000000 --- a/src/tcp-client-test/tcp-client-test.csproj +++ /dev/null @@ -1,38 +0,0 @@ - - - - tcp_client_test - enable - false - net6.0;net7.0;net8.0;net9.0 - Library - - - - - - - - - - - - - - - - - Always - - - - - - ilp-client-interop-test.json - Always - - - - - - From 16635e954d63acd23a64712851fc910691fb2683 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 14:48:24 +0200 Subject: [PATCH 04/24] fix: update Buffer creation for ProtocolVersion.Auto to use BufferV3 --- src/net-questdb-client/Buffers/Buffer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net-questdb-client/Buffers/Buffer.cs b/src/net-questdb-client/Buffers/Buffer.cs index 77b92aa..b4d8644 100644 --- a/src/net-questdb-client/Buffers/Buffer.cs +++ b/src/net-questdb-client/Buffers/Buffer.cs @@ -47,7 +47,7 @@ public static IBuffer Create(int bufferSize, int maxNameLen, int maxBufSize, Pro ProtocolVersion.V1 => new BufferV1(bufferSize, maxNameLen, maxBufSize), ProtocolVersion.V2 => new BufferV2(bufferSize, maxNameLen, maxBufSize), ProtocolVersion.V3 => new BufferV3(bufferSize, maxNameLen, maxBufSize), - ProtocolVersion.Auto => new BufferV2(bufferSize, maxNameLen, maxBufSize), + ProtocolVersion.Auto => new BufferV3(bufferSize, maxNameLen, maxBufSize), _ => throw new NotImplementedException(), }; } From ea890c883034e65185de5989a439d8185148f7ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Dalmon?= <38668811+RaphDal@users.noreply.github.com> Date: Fri, 17 Oct 2025 16:32:06 +0200 Subject: [PATCH 05/24] typo: fix typo in readme Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index be67233..5c38533 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ The config string format is: | `token` | | The token for authentication. Used for Token Authentication and TCP JWK Authentication. | | `token_x` | | Un-used. | | `token_y` | | Un-used. | -| `tls_verify` | `on` | Denotes whether TLS certificates should or should not be verifed. Options are on/unsafe_off. | +| `tls_verify` | `on` | Denotes whether TLS certificates should or should not be verified. Options are on/unsafe_off. | | `tls_ca` | | Un-used. | | `tls_roots` | | Used to specify the filepath for a custom .pem certificate. | | `tls_roots_password` | | Used to specify the filepath for the private key/password corresponding to the `tls_roots` certificate. | From cfdb79e7e7a8496ba2c39b5c2309eb1c4aba31d2 Mon Sep 17 00:00:00 2001 From: "coderabbitai[bot]" <136622811+coderabbitai[bot]@users.noreply.github.com> Date: Fri, 17 Oct 2025 14:32:45 +0000 Subject: [PATCH 06/24] =?UTF-8?q?=F0=9F=93=9D=20Add=20docstrings=20to=20`r?= =?UTF-8?q?d=5Fdecimal`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Docstrings generation was requested by @RaphDal. * https://github.com/questdb/net-questdb-client/pull/52#issuecomment-3415850079 The following files were modified: * `src/dummy-http-server/DummyHttpServer.cs` * `src/net-questdb-client-tests/DecimalTestHelpers.cs` * `src/net-questdb-client-tests/DummyIlpServer.cs` * `src/net-questdb-client-tests/JsonSpecTestRunner.cs` * `src/net-questdb-client/Buffers/Buffer.cs` * `src/net-questdb-client/Buffers/BufferV1.cs` * `src/net-questdb-client/Buffers/BufferV2.cs` * `src/net-questdb-client/Buffers/BufferV3.cs` * `src/net-questdb-client/Buffers/IBuffer.cs` * `src/net-questdb-client/Senders/AbstractSender.cs` * `src/net-questdb-client/Senders/HttpSender.cs` * `src/net-questdb-client/Senders/ISender.cs` --- src/dummy-http-server/DummyHttpServer.cs | 44 +++++- .../DecimalTestHelpers.cs | 21 +++ .../DummyIlpServer.cs | 51 +++++++ .../JsonSpecTestRunner.cs | 18 +++ src/net-questdb-client/Buffers/Buffer.cs | 12 +- src/net-questdb-client/Buffers/BufferV1.cs | 64 +++++++- src/net-questdb-client/Buffers/BufferV2.cs | 29 +++- src/net-questdb-client/Buffers/BufferV3.cs | 16 +- src/net-questdb-client/Buffers/IBuffer.cs | 22 ++- .../Senders/AbstractSender.cs | 16 +- src/net-questdb-client/Senders/HttpSender.cs | 46 +++++- src/net-questdb-client/Senders/ISender.cs | 140 +++++++++++++++--- 12 files changed, 437 insertions(+), 42 deletions(-) diff --git a/src/dummy-http-server/DummyHttpServer.cs b/src/dummy-http-server/DummyHttpServer.cs index d2d5080..fa2109f 100644 --- a/src/dummy-http-server/DummyHttpServer.cs +++ b/src/dummy-http-server/DummyHttpServer.cs @@ -42,6 +42,15 @@ public class DummyHttpServer : IDisposable private int _port = 29743; private readonly TimeSpan? _withStartDelay; + /// + /// Initializes a configurable in-process dummy HTTP server used for testing endpoints. + /// + /// If true, enable JWT bearer authentication and authorization. + /// If true, enable basic authentication behavior in the test endpoint. + /// If true, configure the test endpoint to produce retriable error responses. + /// If true, include error messages in test error responses. + /// Optional delay applied when starting the server. + /// If true, require client TLS certificates for HTTPS connections. public DummyHttpServer(bool withTokenAuth = false, bool withBasicAuth = false, bool withRetriableError = false, bool withErrorMessage = false, TimeSpan? withStartDelay = null, bool requireClientCert = false) { @@ -108,6 +117,13 @@ public void Dispose() _app.StopAsync().Wait(); } + /// + /// Clears the in-memory receive buffers and resets the endpoint error state and counter. + /// + /// + /// Empties IlpEndpoint.ReceiveBuffer and IlpEndpoint.ReceiveBytes, sets IlpEndpoint.LastError to null, + /// and sets IlpEndpoint.Counter to zero. + /// public void Clear() { IlpEndpoint.ReceiveBuffer.Clear(); @@ -116,6 +132,12 @@ public void Clear() IlpEndpoint.Counter = 0; } + /// + /// Starts the HTTP server on the specified port and configures the supported protocol versions. + /// + /// Port to listen on (defaults to 29743). + /// Array of supported protocol versions; defaults to {1, 2, 3} when null. + /// A task that completes after any configured startup delay has elapsed and the server's background run task has been initiated. public async Task StartAsync(int port = 29743, int[]? versions = null) { if (_withStartDelay.HasValue) @@ -128,6 +150,9 @@ public async Task StartAsync(int port = 29743, int[]? versions = null) _ = _app.RunAsync($"http://localhost:{port}"); } + /// + /// Starts the web application and listens for HTTP requests on http://localhost:{_port}. + /// public async Task RunAsync() { await _app.RunAsync($"http://localhost:{_port}"); @@ -138,11 +163,19 @@ public async Task StopAsync() await _app.StopAsync(); } + /// + /// Gets the server's in-memory text buffer of received data. + /// + /// The mutable containing the accumulated received text; modifying it updates the server's buffer. public StringBuilder GetReceiveBuffer() { return IlpEndpoint.ReceiveBuffer; } + /// + /// Gets the in-memory list of bytes received by the ILP endpoint. + /// + /// The mutable list of bytes received by the endpoint. public List GetReceivedBytes() { return IlpEndpoint.ReceiveBytes; @@ -160,6 +193,10 @@ public async Task Healthcheck() } + /// + /// Generates a JWT for the test server when the provided credentials match the server's static username and password. + /// + /// The JWT string when credentials are valid; null otherwise. The issued token is valid for one day. public string? GetJwtToken(string username, string password) { if (username == Username && password == Password) @@ -180,6 +217,11 @@ public int GetCounter() return IlpEndpoint.Counter; } + /// + /// Produces a human-readable string representation of the server's received-bytes buffer, interpreting embedded markers and formatting arrays and numeric values. + /// + /// The formatted textual representation of the received bytes buffer. + /// Thrown when the buffer contains an unsupported type code. public string PrintBuffer() { var bytes = GetReceivedBytes().ToArray(); @@ -263,4 +305,4 @@ public string PrintBuffer() sb.Append(Encoding.UTF8.GetString(bytes, lastAppend, i - lastAppend)); return sb.ToString(); } -} +} \ No newline at end of file diff --git a/src/net-questdb-client-tests/DecimalTestHelpers.cs b/src/net-questdb-client-tests/DecimalTestHelpers.cs index 2f16fcd..77142e4 100644 --- a/src/net-questdb-client-tests/DecimalTestHelpers.cs +++ b/src/net-questdb-client-tests/DecimalTestHelpers.cs @@ -30,6 +30,13 @@ namespace net_questdb_client_tests; internal static class DecimalTestHelpers { + /// + /// Asserts that the buffer contains a decimal field for the specified column with the given scale and mantissa bytes. + /// + /// The encoded row payload to search for the column's decimal field. + /// The name of the column whose decimal payload is expected in the buffer. + /// The expected scale byte of the decimal field. + /// The expected mantissa bytes of the decimal field. internal static void AssertDecimalField(ReadOnlySpan buffer, string columnName, byte expectedScale, @@ -49,6 +56,14 @@ internal static void AssertDecimalField(ReadOnlySpan buffer, $"Mantissa bytes for `{columnName}` did not match expectation."); } + /// + /// Asserts that the buffer contains a null decimal field payload for the specified column. + /// + /// Buffer containing the encoded record(s) to inspect. + /// Name of the column whose decimal payload should be null. + /// + /// Verifies the payload starts with '=' then the DECIMAL type marker, and that both scale and mantissa length are zero. + /// internal static void AssertDecimalNullField(ReadOnlySpan buffer, string columnName) { var payload = ExtractDecimalPayload(buffer, columnName); @@ -61,6 +76,12 @@ internal static void AssertDecimalNullField(ReadOnlySpan buffer, string co Assert.That(payload[3], Is.EqualTo(0), $"Unexpected mantissa length for `{columnName}`."); } + /// + /// Locate and return the payload bytes for a decimal column identified by name. + /// + /// The byte span containing the encoded record payload to search. + /// The column name whose payload prefix ("columnName=") will be located. + /// The slice of immediately after the found "columnName=" prefix. private static ReadOnlySpan ExtractDecimalPayload(ReadOnlySpan buffer, string columnName) { var prefix = Encoding.ASCII.GetBytes($"{columnName}="); diff --git a/src/net-questdb-client-tests/DummyIlpServer.cs b/src/net-questdb-client-tests/DummyIlpServer.cs index f145836..a108f74 100644 --- a/src/net-questdb-client-tests/DummyIlpServer.cs +++ b/src/net-questdb-client-tests/DummyIlpServer.cs @@ -49,6 +49,11 @@ public class DummyIlpServer : IDisposable private string? _publicKeyY; private volatile int _totalReceived; + /// + /// Initializes the dummy ILP server and starts a TCP listener bound to the loopback interface. + /// + /// TCP port to listen on. + /// If true, enables TLS for incoming connections. public DummyIlpServer(int port, bool tls) { _tls = tls; @@ -69,6 +74,12 @@ public void AcceptAsync() Task.Run(AcceptConnections); } + /// + /// Accepts a single incoming connection, optionally negotiates TLS and performs server authentication, then reads and saves data from the client. + /// + /// + /// Handles one client socket from the listener, wraps the connection with TLS if configured, invokes server-auth when credentials are set, and delegates continuous data receipt to the save routine. Socket errors are caught and the client socket is disposed on exit. + /// private async Task AcceptConnections() { Socket? clientSocket = null; @@ -107,6 +118,11 @@ private X509Certificate GetCertificate() return X509Certificate.CreateFromCertFile("certificate.pfx"); } + /// + /// Performs the server-side authentication handshake over the given stream using a challenge-response ECDSA verification. + /// + /// Stream used for the authentication handshake; the method may write to it and will close it if the requested key id mismatches or the signature verification fails. + /// Thrown when the configured public key coordinates are not set. private async Task RunServerAuth(Stream connection) { var receivedLen = await ReceiveUntilEol(connection); @@ -165,6 +181,11 @@ private static string Pad(string text) return text + new string('=', padding); } + /// + /// Decode a Base64 string that may use URL-safe characters and missing padding into its raw byte representation. + /// + /// A Base64-encoded string which may use '-' and '_' instead of '+' and '/' and may omit padding. + /// The decoded bytes represented by the normalized Base64 input. public static byte[] FromBase64String(string encodedPrivateKey) { var replace = encodedPrivateKey @@ -173,6 +194,11 @@ public static byte[] FromBase64String(string encodedPrivateKey) return Convert.FromBase64String(Pad(replace)); } + /// + /// Reads bytes from the provided stream until a newline ('\n') byte is encountered, storing any bytes that follow the newline from the final read into the server's receive buffer. + /// + /// The stream to read incoming bytes from. + /// The index position of the newline byte within the internal read buffer. private async Task ReceiveUntilEol(Stream connection) { var len = 0; @@ -223,16 +249,35 @@ private async Task SaveData(Stream connection, Socket socket) } } + /// + /// Produces a human-readable representation of the data received from the connected client. + /// + /// A formatted string containing the contents of the server's received buffer. public string GetTextReceived() { return PrintBuffer(); } + /// + /// Gets a copy of all bytes received so far. + /// + /// A byte array containing the raw bytes received up to this point. public byte[] GetReceivedBytes() { return _received.ToArray(); } + /// + /// Converts the server's accumulated receive buffer into a human-readable string by decoding UTF-8 text and expanding embedded binary markers into readable representations. + /// + /// + /// The method scans the internal receive buffer for the marker sequence "==". After the marker a type byte determines how the following bytes are interpreted: + /// - type 14: formats a multi-dimensional array of doubles as "ARRAY<dim1,dim2,...>[v1,v2,...]". + /// - type 16: formats a single double value. + /// All bytes outside these marked sections are decoded as UTF-8 text and included verbatim. + /// + /// A formatted string containing the decoded UTF-8 text and expanded representations of any detected binary markers. + /// Thrown when an unknown type marker is encountered after the marker sequence. public string PrintBuffer() { var bytes = _received.ToArray(); @@ -317,6 +362,12 @@ public string PrintBuffer() return sb.ToString(); } + /// + /// Enables server-side authentication by configuring the expected key identifier and the ECDSA public key coordinates. + /// + /// The key identifier expected from the client during authentication. + /// Base64-encoded X coordinate of the ECDSA public key (secp256r1). + /// Base64-encoded Y coordinate of the ECDSA public key (secp256r1). public void WithAuth(string keyId, string publicKeyX, string publicKeyY) { _keyId = keyId; diff --git a/src/net-questdb-client-tests/JsonSpecTestRunner.cs b/src/net-questdb-client-tests/JsonSpecTestRunner.cs index 382a536..0fd46e4 100644 --- a/src/net-questdb-client-tests/JsonSpecTestRunner.cs +++ b/src/net-questdb-client-tests/JsonSpecTestRunner.cs @@ -43,6 +43,12 @@ public class JsonSpecTestRunner private const int HttpPort = 29473; private static readonly TestCase[]? TestCases = ReadTestCases(); + /// + /// Populate the provided sender with the test case's table, symbols, and columns, then send the prepared row. + /// + /// The ISender to configure and use for sending the test case row. + /// The test case containing table name, symbols, and typed columns to write. + /// A task that completes when the prepared row has been sent. private static async Task ExecuteTestCase(ISender sender, TestCase testCase) { sender.Table(testCase.Table); @@ -87,6 +93,10 @@ private static async Task ExecuteTestCase(ISender sender, TestCase testCase) await sender.SendAsync(); } + /// + /// Executes the provided test case by sending its configured table, symbols, and columns to a local TCP listener and asserting the listener's received output against the test case's expected result. + /// + /// The test case to run; provides table, symbols, columns to send and a Result describing the expected validation (Status, Line, AnyLines, or BinaryBase64). [TestCaseSource(nameof(TestCases))] public async Task RunTcp(TestCase testCase) { @@ -143,6 +153,10 @@ public async Task RunTcp(TestCase testCase) } } + /// + /// Executes the provided test case by sending data over HTTP to a dummy server using a QuestDB sender and validates the server's response according to the test case result. + /// + /// The test case describing table, symbols, columns, and expected result (status, line(s), or base64 binary) to execute and validate. [TestCaseSource(nameof(TestCases))] public async Task RunHttp(TestCase testCase) { @@ -259,6 +273,10 @@ public class TestCase [JsonPropertyName("columns")] public TestCaseColumn[] Columns { get; set; } = null!; [JsonPropertyName("result")] public TestCaseResult Result { get; set; } = null!; + /// + /// Provides the test case name for display and logging. + /// + /// The TestName of the test case. public override string ToString() { return TestName; diff --git a/src/net-questdb-client/Buffers/Buffer.cs b/src/net-questdb-client/Buffers/Buffer.cs index b4d8644..5ea5e2e 100644 --- a/src/net-questdb-client/Buffers/Buffer.cs +++ b/src/net-questdb-client/Buffers/Buffer.cs @@ -39,7 +39,15 @@ public static class Buffer /// /// /// - /// + /// + /// Creates a concrete IBuffer implementation configured for the specified protocol version. + /// + /// Size in bytes of each buffer segment. + /// Maximum allowed length for names stored in the buffer. + /// Maximum total buffer capacity. + /// Protocol version that determines which concrete buffer implementation to create. + /// An instance corresponding to the specified protocol version. + /// Thrown when an unsupported protocol version is provided. public static IBuffer Create(int bufferSize, int maxNameLen, int maxBufSize, ProtocolVersion version) { return version switch @@ -51,4 +59,4 @@ public static IBuffer Create(int bufferSize, int maxNameLen, int maxBufSize, Pro _ => throw new NotImplementedException(), }; } -} +} \ No newline at end of file diff --git a/src/net-questdb-client/Buffers/BufferV1.cs b/src/net-questdb-client/Buffers/BufferV1.cs index 922ecd4..7cde1cf 100644 --- a/src/net-questdb-client/Buffers/BufferV1.cs +++ b/src/net-questdb-client/Buffers/BufferV1.cs @@ -132,7 +132,13 @@ public void AtNanos(long timestampNanos) FinishLine(); } - /// + /// + /// Resets the buffer to its initial empty state and clears all written data. + /// + /// + /// Clears lengths of all allocated chunks, resets the active chunk and write position, + /// resets row and total-length counters, exits any transaction state, and clears the current table and line start markers. + /// public void Clear() { _currentBufferIndex = 0; @@ -161,7 +167,13 @@ public void TrimExcessBuffers() } } - /// + /// + /// Reverts the current (in-progress) row to its start position, removing any bytes written for that row. + /// + /// + /// Restores the active buffer index, adjusts the total Length and current Position to the saved line start, + /// and clears the table-set flag for the cancelled row. + /// public void CancelRow() { _currentBufferIndex = _lineStartBufferIndex; @@ -223,7 +235,15 @@ public void WriteToStream(Stream stream, CancellationToken ct = default) stream.Flush(); } - /// + /// + /// Sets the table name for the current row and encodes it into the buffer, beginning a new line context. + /// + /// The table name to write; must meet filesystem length limits and protocol naming rules. + /// This buffer instance to support fluent calls. + /// + /// Thrown with ErrorCode.InvalidApiCall if a transaction is active for a different table or if a table has already been set for the current line. + /// Thrown with ErrorCode.InvalidName if the provided name violates length or character restrictions. + /// public IBuffer Table(ReadOnlySpan name) { GuardFsFileNameLimit(name); @@ -385,7 +405,11 @@ public void Put(ReadOnlySpan chars) EncodeUtf8(chars); } - /// + /// + /// Appends the decimal ASCII representation of the specified 64-bit integer to the buffer. + /// + /// The current buffer instance. + /// Thrown when the value is , which cannot be represented by this method; the error contains an inner . public IBuffer Put(long value) { if (value == long.MinValue) @@ -453,6 +477,10 @@ public IBuffer Put(byte value) } + /// + /// Advance the current buffer write position and the overall length by a given number of bytes. + /// + /// The number of bytes to add to both and . [MethodImpl(MethodImplOptions.AggressiveInlining)] internal void Advance(int by) { @@ -460,6 +488,12 @@ internal void Advance(int by) Length += by; } + /// + /// Sets the buffer's current table to the stored table name when a transaction is active and no table has been set for the current row. + /// + /// + /// Has no effect if not within a transaction or if a table has already been set for the current row. + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] internal void SetTableIfAppropriate() { @@ -469,6 +503,9 @@ internal void SetTableIfAppropriate() } } + /// + /// Finalizes the current row: terminates it with a newline, increments the completed row counter, resets per-row flags, and enforces the buffer size limit. + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] private void FinishLine() { @@ -533,6 +570,12 @@ internal void EnsureCapacity(int additional) } } + /// + /// Encodes the specified character as UTF-8 into the current chunk and advances the write position by the number of bytes written. + /// + /// + /// If the current chunk has fewer than four bytes free, switches to the next buffer before writing. + /// private void PutUtf8(char c) { if (Position + 4 >= Chunk.Length) @@ -780,7 +823,11 @@ private static void GuardInvalidColumnName(ReadOnlySpan columnName) /// Check that the file name is not too long. /// /// - /// + /// + /// Validates that the UTF-8 encoded byte length of the given name is within the configured maximum. + /// + /// The name to validate (measured in UTF-8 bytes). + /// Thrown with if the name exceeds the maximum allowed byte length. private void GuardFsFileNameLimit(ReadOnlySpan name) { if (Encoding.UTF8.GetBytes(name.ToString()).Length > _maxNameLen) @@ -790,6 +837,13 @@ private void GuardFsFileNameLimit(ReadOnlySpan name) } } + /// + /// Attempts to add a DECIMAL column to the current row; DECIMAL types are not supported by Protocol Version V1. + /// + /// The column name to write. + /// The decimal value to write, or null to indicate absence. + /// The buffer instance for fluent chaining. + /// Always thrown with to indicate DECIMAL is unsupported. public virtual IBuffer Column(ReadOnlySpan name, decimal? value) { throw new IngressError(ErrorCode.ProtocolVersionError, "Protocol Version does not support DECIMAL types"); diff --git a/src/net-questdb-client/Buffers/BufferV2.cs b/src/net-questdb-client/Buffers/BufferV2.cs index 3a3734b..c0c5db2 100644 --- a/src/net-questdb-client/Buffers/BufferV2.cs +++ b/src/net-questdb-client/Buffers/BufferV2.cs @@ -90,7 +90,10 @@ private void GuardAgainstNonDoubleTypes(Type t) } } - // ReSharper disable once InconsistentNaming + /// + /// Writes the provided value into the buffer as little-endian raw bytes and advances the buffer position by the value's size. + /// + /// A value whose raw bytes will be written into the buffer in little-endian order. private void PutBinaryLE(T value) where T : struct { var size = Marshal.SizeOf(); @@ -110,7 +113,10 @@ private void PutBinaryBE(T value) where T : struct MemoryMarshal.Cast(slot).Reverse(); } - // ReSharper disable once InconsistentNaming + /// + /// Writes a sequence of values into the buffer in little-endian binary form, handling chunk boundaries and advancing the buffer position. + /// + /// A span of values whose raw bytes will be written as little-endian binary (elements are written whole; partial element writes are not performed). private void PutBinaryManyLE(ReadOnlySpan value) where T : struct { var srcSpan = MemoryMarshal.Cast(value); @@ -172,13 +178,22 @@ private void PutBinary(T value) where T : struct } } - /// + /// + /// Writes a column whose value is the provided span of doubles encoded as a binary double array. + /// + /// The current buffer instance. public override IBuffer Column(ReadOnlySpan name, ReadOnlySpan value) where T : struct { GuardAgainstNonDoubleTypes(typeof(T)); return PutDoubleArray(name, value); } + /// + /// Writes a one-dimensional double array column encoded in the buffer's binary double-array format. + /// + /// The column name. + /// A span of elements representing the array; elements must be of type `double`. + /// The current buffer instance. private IBuffer PutDoubleArray(ReadOnlySpan name, ReadOnlySpan value) where T : struct { SetTableIfAppropriate(); @@ -190,7 +205,13 @@ private IBuffer PutDoubleArray(ReadOnlySpan name, ReadOnlySpan value return this; } - /// + /// + /// Add a column with the given name whose value is provided by the specified double array (1D or multi-dimensional). + /// + /// The column name to write. + /// An array of doubles to write. If null the column is omitted. For a 1D array the values are written as a single-dimension double array; for multi-dimensional arrays the rank and each dimension length are written followed by the elements in row-major order. + /// This buffer instance. + /// Thrown when the array's element type cannot be determined. public override IBuffer Column(ReadOnlySpan name, Array? value) { if (value == null) diff --git a/src/net-questdb-client/Buffers/BufferV3.cs b/src/net-questdb-client/Buffers/BufferV3.cs index 3609637..729e66e 100644 --- a/src/net-questdb-client/Buffers/BufferV3.cs +++ b/src/net-questdb-client/Buffers/BufferV3.cs @@ -31,7 +31,12 @@ namespace QuestDB.Buffers; /// public class BufferV3 : BufferV2 { - /// + /// + /// Initializes a new instance of BufferV3 with the specified buffer and name length limits. + /// + /// Initial size of the internal write buffer, in bytes. + /// Maximum allowed length for column names, in characters. + /// Maximum allowed internal buffer size, in bytes. public BufferV3(int bufferSize, int maxNameLen, int maxBufSize) : base(bufferSize, maxNameLen, maxBufSize) { } @@ -49,7 +54,12 @@ public BufferV3(int bufferSize, int maxNameLen, int maxBufSize) : base(bufferSiz // Number of bits scale is shifted by. private const int ScaleShift = 16; - /// + /// + /// Writes a decimal column in QuestDB's binary column format (scale, length, and two's-complement big-endian unscaled value). + /// + /// Column name to write. + /// Nullable decimal value to encode; when null writes zero scale and zero length. + /// The buffer instance for call chaining. public override IBuffer Column(ReadOnlySpan name, decimal? value) { // # Binary Format @@ -120,4 +130,4 @@ public override IBuffer Column(ReadOnlySpan name, decimal? value) return this; } -} +} \ No newline at end of file diff --git a/src/net-questdb-client/Buffers/IBuffer.cs b/src/net-questdb-client/Buffers/IBuffer.cs index 6074853..6598b74 100644 --- a/src/net-questdb-client/Buffers/IBuffer.cs +++ b/src/net-questdb-client/Buffers/IBuffer.cs @@ -228,14 +228,30 @@ public interface IBuffer /// public IBuffer Column(ReadOnlySpan name, ReadOnlySpan value) where T : struct; - /// + /// +/// Writes an array column value for the current row. +/// +/// The column name. +/// The array to write as the column value, or null to record a NULL value. +/// The same buffer instance for fluent chaining. public IBuffer Column(ReadOnlySpan name, Array? value); - /// + /// +/// Writes a column with the specified name using the provided enumerable of values and shape information. +/// +/// The column name. +/// An enumerable of values for the column; elements are of the value type `T`. +/// An enumerable of integers describing the multidimensional shape/length(s) for the values. +/// The same instance for call chaining. public IBuffer Column(ReadOnlySpan name, IEnumerable value, IEnumerable shape) where T : struct; /// /// Records a DECIMAL column value using the ILP binary decimal layout. - /// + /// +/// Writes a DECIMAL column with the specified name using the ILP binary decimal layout. +/// +/// The column name. +/// The decimal value to write, or `null` to write a NULL column. +/// The buffer instance for method chaining. public IBuffer Column(ReadOnlySpan name, decimal? value); } \ No newline at end of file diff --git a/src/net-questdb-client/Senders/AbstractSender.cs b/src/net-questdb-client/Senders/AbstractSender.cs index 54a3e26..290c96e 100644 --- a/src/net-questdb-client/Senders/AbstractSender.cs +++ b/src/net-questdb-client/Senders/AbstractSender.cs @@ -94,7 +94,12 @@ public ISender Column(ReadOnlySpan name, long value) return this; } - /// + /// + /// Appends an integer-valued column with the specified name to the current buffered row. + /// + /// The column name. + /// The integer value to append for the column. + /// The same instance to allow fluent chaining. public ISender Column(ReadOnlySpan name, int value) { Buffer.Column(name, value); @@ -311,6 +316,9 @@ private void FlushIfNecessary(CancellationToken ct = default) } } + /// + /// Sets to the current UTC time if it has not been initialized. + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] private void GuardLastFlushNotSet() { @@ -320,6 +328,12 @@ private void GuardLastFlushNotSet() } } + /// + /// Adds a nullable decimal column value to the current row in the buffer. + /// + /// The column name. + /// The decimal value to write, or null to emit a null for the column. + /// The same instance for fluent chaining. public ISender Column(ReadOnlySpan name, decimal? value) { Buffer.Column(name, value); diff --git a/src/net-questdb-client/Senders/HttpSender.cs b/src/net-questdb-client/Senders/HttpSender.cs index d748653..9f50f86 100644 --- a/src/net-questdb-client/Senders/HttpSender.cs +++ b/src/net-questdb-client/Senders/HttpSender.cs @@ -57,6 +57,10 @@ internal class HttpSender : AbstractSender private readonly Func _sendRequestFactory; private readonly Func _settingRequestFactory; + /// + /// Initializes a new HttpSender configured according to the provided options. + /// + /// Configuration for the sender, including connection endpoint, TLS and certificate settings, buffering and protocol parameters, authentication, and timeouts. public HttpSender(SenderOptions options) { _sendRequestFactory = GenerateRequest; @@ -70,6 +74,17 @@ public HttpSender(string confStr) : this(new SenderOptions(confStr)) { } + /// + /// Configure and initialize the SocketsHttpHandler and HttpClient, set TLS and authentication options, determine the Line Protocol version (probing /settings when set to Auto), and create the internal send buffer. + /// + /// + /// - Applies pool and connection settings from Options. + /// - When using HTTPS, configures TLS protocols, optional remote-certificate validation override (when tls_verify is unsafe_off), optional custom root CA installation, and optional client certificates. + /// - Sets connection timeout, PreAuthenticate, BaseAddress, and disables HttpClient timeout. + /// - Adds Basic or Bearer Authorization header when credentials or token are provided. + /// - If protocol_version is Auto, probes the server's /settings with a 1-second retry window to select the highest mutually supported protocol up to V3, falling back to V1 on errors or unexpected responses. + /// - Initializes the Buffer with init_buf_size, max_name_len, max_buf_size, and the chosen protocol version. + /// private void Build() { _handler = new SocketsHttpHandler @@ -211,7 +226,10 @@ private CancellationTokenSource GenerateRequestCts(CancellationToken ct = defaul /// /// Creates a new HTTP request with appropriate encoding. /// - /// + /// + /// Create an HTTP POST request targeting "/write" with the sender's buffer as the request body. + /// + /// An configured with the buffer as the request body, Content-Type set to "text/plain" with charset "utf-8", and Content-Length set to the buffer length. private HttpRequestMessage GenerateRequest() { var request = new HttpRequestMessage(HttpMethod.Post, "/write") @@ -307,7 +325,15 @@ public override void Rollback() Buffer.Clear(); } - /// + /// + /// Sends the current buffer synchronously to the server, applying configured retries and handling server-side errors. + /// + /// + /// Validates that a pending transaction is being committed before sending. If the buffer is empty this method returns immediately. + /// On success updates from the server response date; on failure sets to now. The buffer is always cleared after the operation. + /// + /// Cancellation token to cancel the send operation. + /// Thrown with if a transaction is open but not committing, or with for server/transport errors. public override void Send(CancellationToken ct = default) { if (WithinTransaction && !CommittingTransaction) @@ -361,6 +387,15 @@ public override void Send(CancellationToken ct = default) } } + /// + /// Sends an HTTP request produced by and retries on transient connection or server errors until a successful response is received or elapses. + /// + /// Cancellation token used to cancel the overall operation and linked to per-request timeouts. + /// Factory that produces a fresh for each attempt. + /// Maximum duration to keep retrying transient failures; retries are skipped if this is zero. + /// The final returned by the server for a successful request. + /// Thrown with when a connection could not be established within the allowed retries. + /// The caller is responsible for disposing the returned ./// private HttpResponseMessage SendWithRetries(CancellationToken ct, Func requestFactory, TimeSpan retryTimeout) { HttpResponseMessage? response = null; @@ -452,6 +487,11 @@ private void HandleErrorJson(HttpResponseMessage response) } } + /// + /// Read an error payload from the HTTP response (JSON if possible, otherwise raw text) and throw an IngressError containing the server reason and the parsed error details. + /// + /// The HTTP response containing a JSON or plain-text error body. + /// Always thrown with ; the message contains response.ReasonPhrase followed by the deserialized JSON error or the raw response body. private async Task HandleErrorJsonAsync(HttpResponseMessage response) { await using var respStream = await response.Content.ReadAsStreamAsync(); @@ -618,4 +658,4 @@ public override void Dispose() Buffer.Clear(); Buffer.TrimExcessBuffers(); } -} +} \ No newline at end of file diff --git a/src/net-questdb-client/Senders/ISender.cs b/src/net-questdb-client/Senders/ISender.cs index c7227a5..6535519 100644 --- a/src/net-questdb-client/Senders/ISender.cs +++ b/src/net-questdb-client/Senders/ISender.cs @@ -124,25 +124,55 @@ public interface ISender : IDisposable /// /// The name of the column /// The value for the column - /// Itself + /// +/// Adds a column (field) with the specified string value to the current row. +/// +/// The column name. +/// The column value as a character span. +/// The sender instance for fluent chaining. public ISender Column(ReadOnlySpan name, ReadOnlySpan value); - /// + /// +/// Adds a column with the specified name and 64-bit integer value to the current row. +/// +/// The column (field) name. +/// The 64-bit integer value for the column. +/// The current sender instance for method chaining. public ISender Column(ReadOnlySpan name, long value); /// public ISender Column(ReadOnlySpan name, int value); - /// + /// +/// Adds a boolean field column with the specified name and value to the current row. +/// +/// The column (field) name. +/// The boolean value to store in the column. +/// The same instance to allow fluent chaining. public ISender Column(ReadOnlySpan name, bool value); - /// + /// +/// Adds a double-precision field column to the current row. +/// +/// The column (field) name. +/// The column's double-precision value. +/// The same instance for fluent chaining. public ISender Column(ReadOnlySpan name, double value); - /// + /// +/// Adds a column (field) with the specified DateTime value to the current row. +/// +/// The column name. +/// The DateTime value to add. +/// The same instance for fluent chaining. public ISender Column(ReadOnlySpan name, DateTime value); - /// + /// +/// Adds a column with the specified name and DateTimeOffset value to the current row. +/// +/// The column name. +/// The DateTimeOffset value to store for the column (used as a timestamp value). +/// The sender instance for fluent chaining. public ISender Column(ReadOnlySpan name, DateTimeOffset value); /// @@ -211,11 +241,20 @@ public interface ISender : IDisposable /// /// Clears the sender's buffer. - /// + /// +/// Clears the sender's internal buffer and resets buffer-related state, removing all pending rows. +/// public void Clear(); /// + /// +/// Adds a column to the current row using a sequence of value-type elements and an explicit multidimensional shape. +/// +/// The element value type stored in the column. +/// The column name. +/// A sequence of elements that form the column's data. +/// A sequence of integers describing the dimensions of the array representation; dimension lengths must match the number of elements in when multiplied together. +/// The same instance for fluent chaining. public ISender Column(ReadOnlySpan name, IEnumerable value, IEnumerable shape) where T : struct; /// @@ -224,11 +263,21 @@ public interface ISender : IDisposable /// /// /// - /// + /// +/// Adds a column whose value is provided as a native array; multidimensional (non-jagged) arrays are supported. +/// +/// The column name. +/// A native array containing the column data. Multidimensional arrays are treated as shaped data (do not pass jagged arrays). +/// The sender instance for fluent chaining. public ISender Column(ReadOnlySpan name, Array value); /// + /// +/// Adds a column with the specified name and a sequence of value-type elements from a span to the current row. +/// +/// The column (field) name. +/// A contiguous sequence of value-type elements representing the column data. +/// The same instance to allow fluent chaining. public ISender Column(ReadOnlySpan name, ReadOnlySpan value) where T : struct; /// @@ -236,13 +285,24 @@ public interface ISender : IDisposable /// /// The name of the column /// The value for the column - /// Itself + /// + /// Adds a column with the specified string value to the current row. + /// + /// The column name. + /// The column's string value; may be null. + /// The same sender instance for fluent chaining. public ISender Column(ReadOnlySpan name, string? value) { return Column(name, value.AsSpan()); } - /// + /// + /// Adds a column whose value is a sequence of value-type elements with the given multidimensional shape when both and are provided; no action is taken if either is null. + /// + /// The column name. + /// The sequence of elements for the column, or null to skip adding the column. + /// The dimensions describing the array shape, or null to skip adding the column. + /// This sender instance for fluent chaining. public ISender NullableColumn(ReadOnlySpan name, IEnumerable? value, IEnumerable? shape) where T : struct { @@ -254,7 +314,12 @@ public ISender NullableColumn(ReadOnlySpan name, IEnumerable? value, return this; } - /// + /// + /// Adds a column using a native array value when the provided array is non-null. + /// + /// The column name. + /// The array to use as the column value; if null, no column is added. Multidimensional arrays are supported (non-jagged). + /// The same instance for fluent chaining. public ISender NullableColumn(ReadOnlySpan name, Array? value) { if (value != null) @@ -265,7 +330,12 @@ public ISender NullableColumn(ReadOnlySpan name, Array? value) return this; } - /// + /// + /// Adds a string column with the given name when the provided value is not null. + /// + /// The column name. + /// The string value to add; if null, no column is added. + /// The current sender instance for fluent chaining. public ISender NullableColumn(ReadOnlySpan name, string? value) { if (value != null) @@ -276,7 +346,12 @@ public ISender NullableColumn(ReadOnlySpan name, string? value) return this; } - /// + /// + /// Adds a long column with the specified name when the provided nullable value has a value; does nothing when the value is null. + /// + /// The column name. + /// The nullable long value to add as a column; if null the sender is unchanged. + /// The current sender instance for fluent chaining. public ISender NullableColumn(ReadOnlySpan name, long? value) { if (value != null) @@ -287,7 +362,12 @@ public ISender NullableColumn(ReadOnlySpan name, long? value) return this; } - /// + /// + /// Adds a boolean column with the given name when a value is provided; does nothing if the value is null. + /// + /// The column name. + /// The nullable boolean value to add as a column. + /// The current sender instance to allow fluent chaining. public ISender NullableColumn(ReadOnlySpan name, bool? value) { if (value != null) @@ -298,7 +378,12 @@ public ISender NullableColumn(ReadOnlySpan name, bool? value) return this; } - /// + /// + /// Adds a column with the given double value when the value is non-null; otherwise no column is added and the sender is unchanged. + /// + /// The column name. + /// The column value; if non-null, the value is written as a double field. + /// The sender instance after the operation. public ISender NullableColumn(ReadOnlySpan name, double? value) { if (value != null) @@ -309,7 +394,12 @@ public ISender NullableColumn(ReadOnlySpan name, double? value) return this; } - /// + /// + /// Adds a DateTime column with the specified name when a value is provided; no action is taken if the value is null. + /// + /// The column name. + /// The nullable DateTime value to add as a column. + /// The current instance for fluent chaining; unchanged if is null. public ISender NullableColumn(ReadOnlySpan name, DateTime? value) { if (value != null) @@ -320,7 +410,12 @@ public ISender NullableColumn(ReadOnlySpan name, DateTime? value) return this; } - /// + /// + /// Adds a column with the given name and DateTimeOffset value when a value is provided; does nothing if the value is null. + /// + /// The column name. + /// The DateTimeOffset value to add; if null the column is not added. + /// The same instance to allow fluent chaining. public ISender NullableColumn(ReadOnlySpan name, DateTimeOffset? value) { if (value != null) @@ -333,6 +428,11 @@ public ISender NullableColumn(ReadOnlySpan name, DateTimeOffset? value) /// /// Adds a DECIMAL column in the binary format. - /// + /// +/// Adds a decimal column in binary format to the current row. +/// +/// The column name. +/// The decimal value to add; may be null to represent a NULL field. +/// The sender instance for fluent call chaining. public ISender Column(ReadOnlySpan name, decimal? value); } \ No newline at end of file From c9492d78ce756287e79fc32ea5d9915fa4f2a1ca Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 16:37:59 +0200 Subject: [PATCH 07/24] refactor: replace direct byte comparison with WaitAssert for improved reliability in test case validation --- src/net-questdb-client-tests/JsonSpecTestRunner.cs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/net-questdb-client-tests/JsonSpecTestRunner.cs b/src/net-questdb-client-tests/JsonSpecTestRunner.cs index 382a536..5e78717 100644 --- a/src/net-questdb-client-tests/JsonSpecTestRunner.cs +++ b/src/net-questdb-client-tests/JsonSpecTestRunner.cs @@ -116,9 +116,8 @@ public async Task RunTcp(TestCase testCase) { if (testCase.Result.BinaryBase64 != null) { - var received = srv.GetReceivedBytes(); var expected = Convert.FromBase64String(testCase.Result.BinaryBase64); - Assert.That(received, Is.EqualTo(expected)); + WaitAssert(srv, expected); } else if (testCase.Result.AnyLines == null || testCase.Result.AnyLines.Length == 0) { @@ -236,6 +235,17 @@ private static void WaitAssert(DummyIlpServer srv, string expected) Assert.That(srv.GetTextReceived(), Is.EqualTo(expected)); } + private static void WaitAssert(DummyIlpServer srv, byte[] expected) + { + var expectedLen = expected.Length; + for (var i = 0; i < 500 && srv.TotalReceived < expectedLen; i++) + { + Thread.Sleep(10); + } + + Assert.That(srv.GetReceivedBytes(), Is.EqualTo(expected)); + } + private DummyIlpServer CreateTcpListener(int port, bool tls = false) { return new DummyIlpServer(port, tls); From 4850e7fb7d1a7939d10baefacbb981878b826803 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 16:41:43 +0200 Subject: [PATCH 08/24] fix: handle null value in Column method of ISender interface --- src/net-questdb-client/Senders/ISender.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/net-questdb-client/Senders/ISender.cs b/src/net-questdb-client/Senders/ISender.cs index c7227a5..468860c 100644 --- a/src/net-questdb-client/Senders/ISender.cs +++ b/src/net-questdb-client/Senders/ISender.cs @@ -239,6 +239,7 @@ public interface ISender : IDisposable /// Itself public ISender Column(ReadOnlySpan name, string? value) { + if (value is null) return this; return Column(name, value.AsSpan()); } From 5319a316cf01572aed11081b013695eb97080b11 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 16:48:31 +0200 Subject: [PATCH 09/24] fix: handle null values in DECIMAL case of JsonSpecTestRunner --- .../JsonSpecTestRunner.cs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/net-questdb-client-tests/JsonSpecTestRunner.cs b/src/net-questdb-client-tests/JsonSpecTestRunner.cs index 5e78717..fc194af 100644 --- a/src/net-questdb-client-tests/JsonSpecTestRunner.cs +++ b/src/net-questdb-client-tests/JsonSpecTestRunner.cs @@ -23,6 +23,7 @@ ******************************************************************************/ +using System.Globalization; using System.Net; using System.Text; using System.Text.Json; @@ -68,12 +69,20 @@ private static async Task ExecuteTestCase(ISender sender, TestCase testCase) break; case "LONG": - sender.Column(column.Name, (long)((JsonElement)column.Value).GetDouble()); + sender.Column(column.Name, ((JsonElement)column.Value).GetInt64()); break; case "DECIMAL": - var d = decimal.Parse(((JsonElement)column.Value).GetString()!); - sender.Column(column.Name, d); + var value = ((JsonElement)column.Value).GetString(); + if (value is null) + { + sender.Column(column.Name, (decimal?)null); + } + else + { + var d = decimal.Parse(value, NumberStyles.Number, CultureInfo.InvariantCulture); + sender.Column(column.Name, d); + } break; default: From e85fd36d80ef9807e3cd3cf4bb9d08a9cf24ca4f Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 16:56:42 +0200 Subject: [PATCH 10/24] refactor: remove unused using directive for System.Runtime.InteropServices in BufferV3 --- src/net-questdb-client/Buffers/BufferV3.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/net-questdb-client/Buffers/BufferV3.cs b/src/net-questdb-client/Buffers/BufferV3.cs index 3609637..ae1dd90 100644 --- a/src/net-questdb-client/Buffers/BufferV3.cs +++ b/src/net-questdb-client/Buffers/BufferV3.cs @@ -23,7 +23,6 @@ ******************************************************************************/ using System.Buffers.Binary; -using System.Runtime.InteropServices; using QuestDB.Enums; namespace QuestDB.Buffers; From 079195efc886d7cba3dbd369e468af0e88d3db15 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 17:01:48 +0200 Subject: [PATCH 11/24] refactor: improve XML documentation for Buffer and IBuffer interfaces --- src/net-questdb-client/Buffers/Buffer.cs | 8 -- src/net-questdb-client/Buffers/IBuffer.cs | 32 ++--- src/net-questdb-client/Senders/HttpSender.cs | 43 +++---- src/net-questdb-client/Senders/ISender.cs | 127 ++++++++----------- 4 files changed, 90 insertions(+), 120 deletions(-) diff --git a/src/net-questdb-client/Buffers/Buffer.cs b/src/net-questdb-client/Buffers/Buffer.cs index 5ea5e2e..9c17b04 100644 --- a/src/net-questdb-client/Buffers/Buffer.cs +++ b/src/net-questdb-client/Buffers/Buffer.cs @@ -31,14 +31,6 @@ namespace QuestDB.Buffers; /// public static class Buffer { - /// - /// Creates an IBuffer instance, based on the provided protocol version. - /// - /// - /// - /// - /// - /// /// /// Creates a concrete IBuffer implementation configured for the specified protocol version. /// diff --git a/src/net-questdb-client/Buffers/IBuffer.cs b/src/net-questdb-client/Buffers/IBuffer.cs index 6598b74..628128a 100644 --- a/src/net-questdb-client/Buffers/IBuffer.cs +++ b/src/net-questdb-client/Buffers/IBuffer.cs @@ -229,29 +229,29 @@ public interface IBuffer public IBuffer Column(ReadOnlySpan name, ReadOnlySpan value) where T : struct; /// -/// Writes an array column value for the current row. -/// -/// The column name. -/// The array to write as the column value, or null to record a NULL value. -/// The same buffer instance for fluent chaining. + /// Writes an array column value for the current row. + /// + /// The column name. + /// The array to write as the column value, or null to record a NULL value. + /// The same buffer instance for fluent chaining. public IBuffer Column(ReadOnlySpan name, Array? value); /// -/// Writes a column with the specified name using the provided enumerable of values and shape information. -/// -/// The column name. -/// An enumerable of values for the column; elements are of the value type `T`. -/// An enumerable of integers describing the multidimensional shape/length(s) for the values. -/// The same instance for call chaining. + /// Writes a column with the specified name using the provided enumerable of values and shape information. + /// + /// The column name. + /// An enumerable of values for the column; elements are of the value type `T`. + /// An enumerable of integers describing the multidimensional shape/length(s) for the values. + /// The same instance for call chaining. public IBuffer Column(ReadOnlySpan name, IEnumerable value, IEnumerable shape) where T : struct; /// /// Records a DECIMAL column value using the ILP binary decimal layout. /// -/// Writes a DECIMAL column with the specified name using the ILP binary decimal layout. -/// -/// The column name. -/// The decimal value to write, or `null` to write a NULL column. -/// The buffer instance for method chaining. + /// Writes a DECIMAL column with the specified name using the ILP binary decimal layout. + /// + /// The column name. + /// The decimal value to write, or `null` to write a NULL column. + /// The buffer instance for method chaining. public IBuffer Column(ReadOnlySpan name, decimal? value); } \ No newline at end of file diff --git a/src/net-questdb-client/Senders/HttpSender.cs b/src/net-questdb-client/Senders/HttpSender.cs index 9f50f86..3cb644a 100644 --- a/src/net-questdb-client/Senders/HttpSender.cs +++ b/src/net-questdb-client/Senders/HttpSender.cs @@ -149,9 +149,9 @@ private void Build() { _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", - Convert.ToBase64String( - Encoding.ASCII.GetBytes( - $"{Options.username}:{Options.password}"))); + Convert.ToBase64String( + Encoding.ASCII.GetBytes( + $"{Options.username}:{Options.password}"))); } else if (!string.IsNullOrEmpty(Options.token)) { @@ -223,9 +223,6 @@ private CancellationTokenSource GenerateRequestCts(CancellationToken ct = defaul return cts; } - /// - /// Creates a new HTTP request with appropriate encoding. - /// /// /// Create an HTTP POST request targeting "/write" with the sender's buffer as the request body. /// @@ -233,7 +230,7 @@ private CancellationTokenSource GenerateRequestCts(CancellationToken ct = defaul private HttpRequestMessage GenerateRequest() { var request = new HttpRequestMessage(HttpMethod.Post, "/write") - { Content = new BufferStreamContent(Buffer), }; + { Content = new BufferStreamContent(Buffer), }; request.Content.Headers.ContentType = new MediaTypeHeaderValue("text/plain") { CharSet = "utf-8", }; request.Content.Headers.ContentLength = Buffer.Length; return request; @@ -260,13 +257,13 @@ public override ISender Transaction(ReadOnlySpan tableName) if (WithinTransaction) { throw new IngressError(ErrorCode.InvalidApiCall, - "Cannot start another transaction - only one allowed at a time."); + "Cannot start another transaction - only one allowed at a time."); } if (Length > 0) { throw new IngressError(ErrorCode.InvalidApiCall, - "Buffer must be clear before you can start a transaction."); + "Buffer must be clear before you can start a transaction."); } Buffer.Transaction(tableName); @@ -274,7 +271,6 @@ public override ISender Transaction(ReadOnlySpan tableName) } /// - /// /> public override void Commit(CancellationToken ct = default) { try @@ -396,7 +392,8 @@ public override void Send(CancellationToken ct = default) /// The final returned by the server for a successful request. /// Thrown with when a connection could not be established within the allowed retries. /// The caller is responsible for disposing the returned ./// - private HttpResponseMessage SendWithRetries(CancellationToken ct, Func requestFactory, TimeSpan retryTimeout) + private HttpResponseMessage SendWithRetries(CancellationToken ct, Func requestFactory, + TimeSpan retryTimeout) { HttpResponseMessage? response = null; CancellationTokenSource cts = GenerateRequestCts(ct); @@ -414,9 +411,9 @@ private HttpResponseMessage SendWithRetries(CancellationToken ct, Func TimeSpan.Zero) - // retry if appropriate - error that's retriable, and retries are enabled + // retry if appropriate - error that's retriable, and retries are enabled { - if (response == null // if it was a cannot correct error + if (response == null // if it was a cannot correct error || (!response.IsSuccessStatusCode // or some other http error && IsRetriableError(response.StatusCode))) { @@ -426,9 +423,9 @@ private HttpResponseMessage SendWithRetries(CancellationToken ct, Func TimeSpan.Zero) { - if (response == null // if it was a cannot correct error + if (response == null // if it was a cannot correct error || (!response.IsSuccessStatusCode // or some other http error && IsRetriableError(response.StatusCode))) { @@ -552,9 +549,9 @@ public override async Task SendAsync(CancellationToken ct = default) while (retryTimer.Elapsed < Options.retry_timeout // whilst we can still retry && ( - response == null || // either we can't connect - (!response.IsSuccessStatusCode && // or we have another http error - IsRetriableError(response.StatusCode))) + response == null || // either we can't connect + (!response.IsSuccessStatusCode && // or we have another http error + IsRetriableError(response.StatusCode))) ) { retryInterval = TimeSpan.FromMilliseconds(Math.Min(retryInterval.TotalMilliseconds * 2, 1000)); @@ -574,7 +571,7 @@ public override async Task SendAsync(CancellationToken ct = default) try { response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, - cts.Token); + cts.Token); } catch (HttpRequestException) { @@ -588,7 +585,7 @@ public override async Task SendAsync(CancellationToken ct = default) if (response == null) { throw new IngressError(ErrorCode.ServerFlushError, - $"Cannot connect to `{Options.Host}:{Options.Port}`"); + $"Cannot connect to `{Options.Host}:{Options.Port}`"); } // return if ok diff --git a/src/net-questdb-client/Senders/ISender.cs b/src/net-questdb-client/Senders/ISender.cs index 6535519..5735f13 100644 --- a/src/net-questdb-client/Senders/ISender.cs +++ b/src/net-questdb-client/Senders/ISender.cs @@ -22,12 +22,8 @@ * ******************************************************************************/ -// ReSharper disable CommentTypo - using QuestDB.Utils; -// ReSharper disable InconsistentNaming - namespace QuestDB.Senders; /// @@ -46,7 +42,7 @@ public interface ISender : IDisposable public int RowCount { get; } /// - /// Represents whether or not the Sender is in a transactional state. + /// Represents whether the Sender is in a transactional state. /// public bool WithinTransaction { get; } @@ -125,54 +121,54 @@ public interface ISender : IDisposable /// The name of the column /// The value for the column /// -/// Adds a column (field) with the specified string value to the current row. -/// -/// The column name. -/// The column value as a character span. -/// The sender instance for fluent chaining. + /// Adds a column (field) with the specified string value to the current row. + /// + /// The column name. + /// The column value as a character span. + /// The sender instance for fluent chaining. public ISender Column(ReadOnlySpan name, ReadOnlySpan value); /// -/// Adds a column with the specified name and 64-bit integer value to the current row. -/// -/// The column (field) name. -/// The 64-bit integer value for the column. -/// The current sender instance for method chaining. + /// Adds a column with the specified name and 64-bit integer value to the current row. + /// + /// The column (field) name. + /// The 64-bit integer value for the column. + /// The current sender instance for method chaining. public ISender Column(ReadOnlySpan name, long value); /// public ISender Column(ReadOnlySpan name, int value); /// -/// Adds a boolean field column with the specified name and value to the current row. -/// -/// The column (field) name. -/// The boolean value to store in the column. -/// The same instance to allow fluent chaining. + /// Adds a boolean field column with the specified name and value to the current row. + /// + /// The column (field) name. + /// The boolean value to store in the column. + /// The same instance to allow fluent chaining. public ISender Column(ReadOnlySpan name, bool value); /// -/// Adds a double-precision field column to the current row. -/// -/// The column (field) name. -/// The column's double-precision value. -/// The same instance for fluent chaining. + /// Adds a double-precision field column to the current row. + /// + /// The column (field) name. + /// The column's double-precision value. + /// The same instance for fluent chaining. public ISender Column(ReadOnlySpan name, double value); /// -/// Adds a column (field) with the specified DateTime value to the current row. -/// -/// The column name. -/// The DateTime value to add. -/// The same instance for fluent chaining. + /// Adds a column (field) with the specified DateTime value to the current row. + /// + /// The column name. + /// The DateTime value to add. + /// The same instance for fluent chaining. public ISender Column(ReadOnlySpan name, DateTime value); /// -/// Adds a column with the specified name and DateTimeOffset value to the current row. -/// -/// The column name. -/// The DateTimeOffset value to store for the column (used as a timestamp value). -/// The sender instance for fluent chaining. + /// Adds a column with the specified name and DateTimeOffset value to the current row. + /// + /// The column name. + /// The DateTimeOffset value to store for the column (used as a timestamp value). + /// The sender instance for fluent chaining. public ISender Column(ReadOnlySpan name, DateTimeOffset value); /// @@ -242,49 +238,36 @@ public interface ISender : IDisposable /// /// Clears the sender's buffer. /// -/// Clears the sender's internal buffer and resets buffer-related state, removing all pending rows. -/// + /// Clears the sender's internal buffer and resets buffer-related state, removing all pending rows. + /// public void Clear(); - /// -/// Adds a column to the current row using a sequence of value-type elements and an explicit multidimensional shape. -/// -/// The element value type stored in the column. -/// The column name. -/// A sequence of elements that form the column's data. -/// A sequence of integers describing the dimensions of the array representation; dimension lengths must match the number of elements in when multiplied together. -/// The same instance for fluent chaining. + /// Adds a column to the current row using a sequence of value-type elements and an explicit multidimensional shape. + /// + /// The element value type stored in the column. + /// The column name. + /// A sequence of elements that form the column's data. + /// A sequence of integers describing the dimensions of the array representation; dimension lengths must match the number of elements in when multiplied together. + /// The same instance for fluent chaining. public ISender Column(ReadOnlySpan name, IEnumerable value, IEnumerable shape) where T : struct; /// - /// Adds an ARRAY to the current row. - /// Arrays are n-dimensional non-jagged arrays. + /// Adds a column whose value is provided as a native array; multidimensional (non-jagged) arrays are supported. /// - /// - /// - /// -/// Adds a column whose value is provided as a native array; multidimensional (non-jagged) arrays are supported. -/// -/// The column name. -/// A native array containing the column data. Multidimensional arrays are treated as shaped data (do not pass jagged arrays). -/// The sender instance for fluent chaining. + /// The column name. + /// A native array containing the column data. Multidimensional arrays are treated as shaped data (do not pass jagged arrays). + /// The sender instance for fluent chaining. public ISender Column(ReadOnlySpan name, Array value); - /// -/// Adds a column with the specified name and a sequence of value-type elements from a span to the current row. -/// -/// The column (field) name. -/// A contiguous sequence of value-type elements representing the column data. -/// The same instance to allow fluent chaining. + /// Adds a column with the specified name and a sequence of value-type elements from a span to the current row. + /// + /// The column (field) name. + /// A contiguous sequence of value-type elements representing the column data. + /// The same instance to allow fluent chaining. public ISender Column(ReadOnlySpan name, ReadOnlySpan value) where T : struct; - /// - /// Adds a column (field) to the current row. - /// - /// The name of the column - /// The value for the column /// /// Adds a column with the specified string value to the current row. /// @@ -427,12 +410,10 @@ public ISender NullableColumn(ReadOnlySpan name, DateTimeOffset? value) } /// - /// Adds a DECIMAL column in the binary format. - /// -/// Adds a decimal column in binary format to the current row. -/// -/// The column name. -/// The decimal value to add; may be null to represent a NULL field. -/// The sender instance for fluent call chaining. + /// Adds a decimal column in binary format to the current row. + /// + /// The column name. + /// The decimal value to add; may be null to represent a NULL field. + /// The sender instance for fluent call chaining. public ISender Column(ReadOnlySpan name, decimal? value); } \ No newline at end of file From 92eadc4b9463d0efcf51d0ba4b9a5cdf4b97fa89 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 17:07:17 +0200 Subject: [PATCH 12/24] fix: handle nullable decimal values in BufferV3 --- src/net-questdb-client/Buffers/BufferV3.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/net-questdb-client/Buffers/BufferV3.cs b/src/net-questdb-client/Buffers/BufferV3.cs index e82d9c8..64bb41c 100644 --- a/src/net-questdb-client/Buffers/BufferV3.cs +++ b/src/net-questdb-client/Buffers/BufferV3.cs @@ -90,7 +90,7 @@ public override IBuffer Column(ReadOnlySpan name, decimal? value) var low = parts[0]; var mid = parts[1]; var high = parts[2]; - var negative = (flags & SignMask) != 0 && value != 0; + var negative = (flags & SignMask) != 0 && value.Value != 0m; if (negative) { @@ -109,7 +109,7 @@ public override IBuffer Column(ReadOnlySpan name, decimal? value) BinaryPrimitives.WriteInt32BigEndian(span.Slice(1, 4), high); BinaryPrimitives.WriteInt32BigEndian(span.Slice(5, 4), mid); BinaryPrimitives.WriteInt32BigEndian(span.Slice(9, 4), low); - + // Compress var start = 0; for (; @@ -117,7 +117,7 @@ public override IBuffer Column(ReadOnlySpan name, decimal? value) // one of it in front to convey the sign. start < span.Length - 1 && span[start] == signByte && ((span[start + 1] ^ signByte) & 0x80) == 0; start++) ; - + // 4. Length var size = span.Length - start; Put((byte)size); From 65640912b1dc8b42ca659ebfdf83251d9168b693 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 18:12:01 +0200 Subject: [PATCH 13/24] fix: fix culture in dummy servers. --- src/dummy-http-server/DummyHttpServer.cs | 122 +++++++++--------- .../DummyIlpServer.cs | 112 ++++++++-------- 2 files changed, 115 insertions(+), 119 deletions(-) diff --git a/src/dummy-http-server/DummyHttpServer.cs b/src/dummy-http-server/DummyHttpServer.cs index fa2109f..91a21bc 100644 --- a/src/dummy-http-server/DummyHttpServer.cs +++ b/src/dummy-http-server/DummyHttpServer.cs @@ -52,15 +52,15 @@ public class DummyHttpServer : IDisposable /// Optional delay applied when starting the server. /// If true, require client TLS certificates for HTTPS connections. public DummyHttpServer(bool withTokenAuth = false, bool withBasicAuth = false, bool withRetriableError = false, - bool withErrorMessage = false, TimeSpan? withStartDelay = null, bool requireClientCert = false) + bool withErrorMessage = false, TimeSpan? withStartDelay = null, bool requireClientCert = false) { var bld = WebApplication.CreateBuilder(); bld.Services.AddLogging(builder => { builder.AddFilter("Microsoft", LogLevel.Warning) - .AddFilter("System", LogLevel.Warning) - .AddConsole(); + .AddFilter("System", LogLevel.Warning) + .AddConsole(); }); IlpEndpoint.WithTokenAuth = withTokenAuth; @@ -72,7 +72,7 @@ public DummyHttpServer(bool withTokenAuth = false, bool withBasicAuth = false, b if (withTokenAuth) { bld.Services.AddAuthenticationJwtBearer(s => s.SigningKey = SigningKey) - .AddAuthorization(); + .AddAuthorization(); } @@ -92,7 +92,7 @@ public DummyHttpServer(bool withTokenAuth = false, bool withBasicAuth = false, b o.Limits.MaxRequestBodySize = 1073741824; o.ListenLocalhost(29474, - options => { options.UseHttps(); }); + options => { options.UseHttps(); }); o.ListenLocalhost(29473); }); @@ -144,6 +144,7 @@ public async Task StartAsync(int port = 29743, int[]? versions = null) { await Task.Delay(_withStartDelay.Value); } + versions ??= new[] { 1, 2, 3, }; SettingsEndpoint.Versions = versions; _port = port; @@ -231,74 +232,71 @@ public string PrintBuffer() var i = 0; for (; i < bytes.Length; i++) { - if (bytes[i] == (byte)'=') + if (bytes[i] == (byte)'=' && i > 0 && bytes[i - 1] == (byte)'=') { - if (bytes[i - 1] == (byte)'=') + sb.Append(Encoding.UTF8.GetString(bytes, lastAppend, i + 1 - lastAppend)); + switch (bytes[++i]) { - sb.Append(Encoding.UTF8.GetString(bytes, lastAppend, i + 1 - lastAppend)); - switch (bytes[++i]) - { - case 14: - sb.Append("ARRAY<"); - var type = bytes[++i]; + case 14: + sb.Append("ARRAY<"); + var type = bytes[++i]; - Debug.Assert(type == 10); - var dims = bytes[++i]; + Debug.Assert(type == 10); + var dims = bytes[++i]; - ++i; + ++i; - long length = 0; - for (var j = 0; j < dims; j++) + long length = 0; + for (var j = 0; j < dims; j++) + { + var lengthBytes = bytes.AsSpan()[i..(i + 4)]; + var lengthValue = MemoryMarshal.Cast(lengthBytes)[0]; + if (length == 0) { - var lengthBytes = bytes.AsSpan()[i..(i + 4)]; - var lengthValue = MemoryMarshal.Cast(lengthBytes)[0]; - if (length == 0) - { - length = lengthValue; - } - else - { - length *= lengthValue; - } - - sb.Append(lengthValue); - sb.Append(','); - i += 4; + length = lengthValue; } - - sb.Remove(sb.Length - 1, 1); - sb.Append('>'); - - var doubleBytes = - MemoryMarshal.Cast(bytes.AsSpan().Slice(i, (int)(length * 8))); - - - sb.Append('['); - for (var j = 0; j < length; j++) + else { - sb.Append(doubleBytes[j]); - sb.Append(','); + length *= lengthValue; } - sb.Remove(sb.Length - 1, 1); - sb.Append(']'); - - i += (int)(length * 8); - i--; - break; - case 16: - sb.Remove(sb.Length - 1, 1); - var doubleValue = MemoryMarshal.Cast(bytes.AsSpan().Slice(++i, 8)); - sb.Append(doubleValue[0].ToString(CultureInfo.InvariantCulture)); - i += 8; - i--; - break; - default: - throw new NotImplementedException($"Type {bytes[i]} not implemented"); - } - - lastAppend = i + 1; + sb.Append(lengthValue); + sb.Append(','); + i += 4; + } + + sb.Remove(sb.Length - 1, 1); + sb.Append('>'); + + var doubleBytes = + MemoryMarshal.Cast(bytes.AsSpan().Slice(i, (int)(length * 8))); + + + sb.Append('['); + for (var j = 0; j < length; j++) + { + sb.Append(doubleBytes[j].ToString(CultureInfo.InvariantCulture)); + sb.Append(','); + } + + sb.Remove(sb.Length - 1, 1); + sb.Append(']'); + + i += (int)(length * 8); + i--; + break; + case 16: + sb.Remove(sb.Length - 1, 1); + var doubleValue = MemoryMarshal.Cast(bytes.AsSpan().Slice(++i, 8)); + sb.Append(doubleValue[0].ToString(CultureInfo.InvariantCulture)); + i += 8; + i--; + break; + default: + throw new NotImplementedException($"Type {bytes[i]} not implemented"); } + + lastAppend = i + 1; } } diff --git a/src/net-questdb-client-tests/DummyIlpServer.cs b/src/net-questdb-client-tests/DummyIlpServer.cs index a108f74..feff38e 100644 --- a/src/net-questdb-client-tests/DummyIlpServer.cs +++ b/src/net-questdb-client-tests/DummyIlpServer.cs @@ -24,6 +24,7 @@ using System.Diagnostics; +using System.Globalization; using System.Net; using System.Net.Security; using System.Net.Sockets; @@ -287,74 +288,71 @@ public string PrintBuffer() var i = 0; for (; i < bytes.Length; i++) { - if (bytes[i] == (byte)'=') + if (bytes[i] == (byte)'=' && i > 0 && bytes[i - 1] == (byte)'=') { - if (bytes[i - 1] == (byte)'=') + sb.Append(Encoding.UTF8.GetString(bytes, lastAppend, i + 1 - lastAppend)); + switch (bytes[++i]) { - sb.Append(Encoding.UTF8.GetString(bytes, lastAppend, i + 1 - lastAppend)); - switch (bytes[++i]) - { - case 14: - sb.Append("ARRAY<"); - var type = bytes[++i]; + case 14: + sb.Append("ARRAY<"); + var type = bytes[++i]; - Debug.Assert(type == 10); - var dims = bytes[++i]; + Debug.Assert(type == 10); + var dims = bytes[++i]; - ++i; + ++i; - long length = 0; - for (var j = 0; j < dims; j++) + long length = 0; + for (var j = 0; j < dims; j++) + { + var lengthBytes = bytes.AsSpan()[i..(i + 4)]; + var _length = MemoryMarshal.Cast(lengthBytes)[0]; + if (length == 0) { - var lengthBytes = bytes.AsSpan()[i..(i + 4)]; - var _length = MemoryMarshal.Cast(lengthBytes)[0]; - if (length == 0) - { - length = _length; - } - else - { - length *= _length; - } - - sb.Append(_length); - sb.Append(','); - i += 4; + length = _length; } - - sb.Remove(sb.Length - 1, 1); - sb.Append('>'); - - var doubleBytes = - MemoryMarshal.Cast(bytes.AsSpan().Slice(i, (int)(length * 8))); - - - sb.Append('['); - for (var j = 0; j < length; j++) + else { - sb.Append(doubleBytes[j]); - sb.Append(','); + length *= _length; } - sb.Remove(sb.Length - 1, 1); - sb.Append(']'); - - i += (int)(length * 8); - i--; - break; - case 16: - sb.Remove(sb.Length - 1, 1); - var doubleValue = MemoryMarshal.Cast(bytes.AsSpan().Slice(++i, 8)); - sb.Append(doubleValue[0]); - i += 8; - i--; - break; - default: - throw new NotImplementedException("Unknown type: " + bytes[i]); - } - - lastAppend = i + 1; + sb.Append(_length); + sb.Append(','); + i += 4; + } + + sb.Remove(sb.Length - 1, 1); + sb.Append('>'); + + var doubleBytes = + MemoryMarshal.Cast(bytes.AsSpan().Slice(i, (int)(length * 8))); + + + sb.Append('['); + for (var j = 0; j < length; j++) + { + sb.Append(doubleBytes[j].ToString(CultureInfo.InvariantCulture)); + sb.Append(','); + } + + sb.Remove(sb.Length - 1, 1); + sb.Append(']'); + + i += (int)(length * 8); + i--; + break; + case 16: + sb.Remove(sb.Length - 1, 1); + var doubleValue = MemoryMarshal.Cast(bytes.AsSpan().Slice(++i, 8)); + sb.Append(doubleValue[0].ToString(CultureInfo.InvariantCulture)); + i += 8; + i--; + break; + default: + throw new NotImplementedException($"Type {bytes[i]} not implemented"); } + + lastAppend = i + 1; } } From d2b5163832b210724ce3fe9db3d64a16e9f60dc8 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 18:13:31 +0200 Subject: [PATCH 14/24] refactor: enhance XML documentation across Buffer classes and senders --- .../Buffers/BufferStreamContent.cs | 4 + src/net-questdb-client/Buffers/BufferV1.cs | 78 ++++++++++++------- src/net-questdb-client/Buffers/BufferV2.cs | 63 ++++++++++++++- src/net-questdb-client/Buffers/IBuffer.cs | 56 ++++++++++--- .../Senders/AbstractSender.cs | 14 +++- src/net-questdb-client/Senders/HttpSender.cs | 24 ++++-- src/net-questdb-client/Senders/TcpSender.cs | 42 +++++++--- 7 files changed, 226 insertions(+), 55 deletions(-) diff --git a/src/net-questdb-client/Buffers/BufferStreamContent.cs b/src/net-questdb-client/Buffers/BufferStreamContent.cs index 6edbc00..7dc2bad 100644 --- a/src/net-questdb-client/Buffers/BufferStreamContent.cs +++ b/src/net-questdb-client/Buffers/BufferStreamContent.cs @@ -32,6 +32,10 @@ namespace QuestDB.Buffers; /// internal class BufferStreamContent : HttpContent { + /// + /// Initializes a new instance of the class. + /// + /// The buffer to wrap for HTTP streaming. public BufferStreamContent(IBuffer buffer) { Buffer = buffer; diff --git a/src/net-questdb-client/Buffers/BufferV1.cs b/src/net-questdb-client/Buffers/BufferV1.cs index 7cde1cf..365fa3a 100644 --- a/src/net-questdb-client/Buffers/BufferV1.cs +++ b/src/net-questdb-client/Buffers/BufferV1.cs @@ -46,7 +46,12 @@ public class BufferV1 : IBuffer private bool _noSymbols = true; private bool _quoted; - /// + /// + /// Initializes a new instance of BufferV1 for writing ILP (InfluxDB Line Protocol) messages. + /// + /// Initial size of each buffer chunk, in bytes. + /// Maximum allowed UTF-8 byte length for table and column names. + /// Maximum total buffer size across all chunks, in bytes. public BufferV1(int bufferSize, int maxNameLen, int maxBufSize) { Chunk = new byte[bufferSize]; @@ -76,13 +81,13 @@ public IBuffer Transaction(ReadOnlySpan tableName) if (WithinTransaction) { throw new IngressError(ErrorCode.InvalidApiCall, - "Cannot start another transaction - only one allowed at a time."); + "Cannot start another transaction - only one allowed at a time."); } if (Length > 0) { throw new IngressError(ErrorCode.InvalidApiCall, - "Buffer must be clear before you can start a transaction."); + "Buffer must be clear before you can start a transaction."); } GuardInvalidTableName(tableName); @@ -250,7 +255,7 @@ public IBuffer Table(ReadOnlySpan name) if (WithinTransaction && name != _currentTableName) { throw new IngressError(ErrorCode.InvalidApiCall, - "Transactions can only be for one table."); + "Transactions can only be for one table."); } GuardTableAlreadySet(); @@ -372,7 +377,7 @@ public IBuffer ColumnNanos(ReadOnlySpan name, long timestampNanos) return this; } - /// + /// public IBuffer EncodeUtf8(ReadOnlySpan name) { foreach (var c in name) @@ -390,7 +395,7 @@ public IBuffer EncodeUtf8(ReadOnlySpan name) return this; } - /// + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public IBuffer PutAscii(char c) { @@ -398,7 +403,7 @@ public IBuffer PutAscii(char c) return this; } - /// + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Put(ReadOnlySpan chars) { @@ -415,7 +420,7 @@ public IBuffer Put(long value) if (value == long.MinValue) { throw new IngressError(ErrorCode.InvalidApiCall, "Special case, long.MinValue cannot be handled by QuestDB", - new ArgumentOutOfRangeException()); + new ArgumentOutOfRangeException()); } Span num = stackalloc byte[20]; @@ -443,26 +448,26 @@ public IBuffer Put(long value) return this; } - /// + /// public virtual IBuffer Column(ReadOnlySpan name, ReadOnlySpan value) where T : struct { throw new IngressError(ErrorCode.ProtocolVersionError, "Protocol Version V1 does not support ARRAY types"); } - /// + /// public virtual IBuffer Column(ReadOnlySpan name, Array? value) { throw new IngressError(ErrorCode.ProtocolVersionError, "Protocol Version V1 does not support ARRAY types"); } - /// + /// public virtual IBuffer Column(ReadOnlySpan name, IEnumerable value, IEnumerable shape) where T : struct { throw new IngressError(ErrorCode.ProtocolVersionError, "Protocol Version V1 does not support ARRAY types"); } - /// + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public IBuffer Put(byte value) { @@ -527,10 +532,16 @@ private void GuardExceededMaxBufferSize() if (Length > _maxBufSize) { throw new IngressError(ErrorCode.InvalidApiCall, - $"Exceeded maximum buffer size. Current: {Length} Maximum: {_maxBufSize}"); + $"Exceeded maximum buffer size. Current: {Length} Maximum: {_maxBufSize}"); } } + /// + /// Writes the column name to the buffer and prepares for writing the column value by appending the appropriate separator and equals sign. + /// + /// The column name to write. + /// The buffer instance for fluent chaining. + /// Thrown if the table is not set, the column name is invalid, or the name exceeds the maximum length. internal IBuffer Column(ReadOnlySpan columnName) { GuardFsFileNameLimit(columnName); @@ -550,16 +561,26 @@ internal IBuffer Column(ReadOnlySpan columnName) return EncodeUtf8(columnName).PutAscii('='); } + /// + /// Validates that the requested additional byte count does not exceed the chunk size. + /// + /// The number of additional bytes requested. + /// Thrown with if the requested size exceeds the chunk length. [MethodImpl(MethodImplOptions.AggressiveInlining)] internal void GuardAgainstOversizedChunk(int additional) { if (additional > Chunk.Length) { throw new IngressError(ErrorCode.InvalidApiCall, - "tried to allocate oversized chunk: " + additional + " bytes"); + "tried to allocate oversized chunk: " + additional + " bytes"); } } + /// + /// Ensures that the current chunk has enough space to write the specified number of additional bytes; switches to the next buffer chunk if needed. + /// + /// The number of additional bytes required. + /// Thrown if the requested size exceeds the chunk size. [MethodImpl(MethodImplOptions.AggressiveInlining)] internal void EnsureCapacity(int additional) { @@ -571,11 +592,9 @@ internal void EnsureCapacity(int additional) } /// - /// Encodes the specified character as UTF-8 into the current chunk and advances the write position by the number of bytes written. + /// Writes a non-ASCII character as UTF-8 to the buffer, switching to the next buffer chunk if insufficient space remains. /// - /// - /// If the current chunk has fewer than four bytes free, switches to the next buffer before writing. - /// + /// The character to encode and write. private void PutUtf8(char c) { if (Position + 4 >= Chunk.Length) @@ -589,6 +608,13 @@ private void PutUtf8(char c) Advance(byteLength); } + /// + /// Writes an ASCII character to the buffer, applying ILP escaping rules based on context (quoted or unquoted). + /// + /// The ASCII character to write. + /// + /// Escapes space, comma, equals, newline, carriage return, quote, and backslash characters according to ILP protocol requirements. + /// private void PutSpecial(char c) { switch (c) @@ -702,7 +728,7 @@ private static void GuardInvalidTableName(ReadOnlySpan tableName) if (tableName.IsEmpty) { throw new IngressError(ErrorCode.InvalidName, - "Table names must have a non-zero length."); + "Table names must have a non-zero length."); } var prev = '\0'; @@ -715,7 +741,7 @@ private static void GuardInvalidTableName(ReadOnlySpan tableName) if (i == 0 || i == tableName.Length - 1 || prev == '.') { throw new IngressError(ErrorCode.InvalidName, - $"Bad string {tableName}. Found invalid dot `.` at position {i}."); + $"Bad string {tableName}. Found invalid dot `.` at position {i}."); } break; @@ -750,10 +776,10 @@ private static void GuardInvalidTableName(ReadOnlySpan tableName) case '\x000f': case '\x007f': throw new IngressError(ErrorCode.InvalidName, - $"Bad string {tableName}. Table names can't contain a {c} character, which was found at byte position {i}"); + $"Bad string {tableName}. Table names can't contain a {c} character, which was found at byte position {i}"); case '\xfeff': throw new IngressError(ErrorCode.InvalidName, - $"Bad string {tableName}. Table names can't contain a UTF-8 BOM character, was was found at byte position {i}."); + $"Bad string {tableName}. Table names can't contain a UTF-8 BOM character, was was found at byte position {i}."); } prev = c; @@ -770,7 +796,7 @@ private static void GuardInvalidColumnName(ReadOnlySpan columnName) if (columnName.IsEmpty) { throw new IngressError(ErrorCode.InvalidName, - "Column names must have a non-zero length."); + "Column names must have a non-zero length."); } for (var i = 0; i < columnName.Length; i++) @@ -811,10 +837,10 @@ private static void GuardInvalidColumnName(ReadOnlySpan columnName) case '\x000f': case '\x007f': throw new IngressError(ErrorCode.InvalidName, - $"Bad string {columnName}. Column names can't contain a {c} character, which was found at byte position {i}"); + $"Bad string {columnName}. Column names can't contain a {c} character, which was found at byte position {i}"); case '\xfeff': throw new IngressError(ErrorCode.InvalidName, - $"Bad string {columnName}. Column names can't contain a UTF-8 BOM character, was was found at byte position {i}."); + $"Bad string {columnName}. Column names can't contain a UTF-8 BOM character, was was found at byte position {i}."); } } } @@ -833,7 +859,7 @@ private void GuardFsFileNameLimit(ReadOnlySpan name) if (Encoding.UTF8.GetBytes(name.ToString()).Length > _maxNameLen) { throw new IngressError(ErrorCode.InvalidApiCall, - $"Name is too long, must be under {_maxNameLen} bytes."); + $"Name is too long, must be under {_maxNameLen} bytes."); } } diff --git a/src/net-questdb-client/Buffers/BufferV2.cs b/src/net-questdb-client/Buffers/BufferV2.cs index c0c5db2..ac3a23f 100644 --- a/src/net-questdb-client/Buffers/BufferV2.cs +++ b/src/net-questdb-client/Buffers/BufferV2.cs @@ -32,12 +32,25 @@ namespace QuestDB.Buffers; /// public class BufferV2 : BufferV1 { - /// + /// + /// Initializes a new instance of BufferV2 supporting ARRAY and binary DOUBLE types. + /// + /// Initial size of the internal write buffer, in bytes. + /// Maximum allowed length for column names, in characters. + /// Maximum allowed internal buffer size, in bytes. public BufferV2(int bufferSize, int maxNameLen, int maxBufSize) : base(bufferSize, maxNameLen, maxBufSize) { } - /// + /// + /// Writes a multidimensional double array column with the specified name, elements, and shape to the buffer. + /// + /// The element type (must be double). + /// The column name. + /// An enumerable of double values forming the array elements. + /// An enumerable of integers describing the dimensions; product must equal the element count. + /// The buffer instance for fluent chaining. + /// Thrown if T is not double, shape is invalid, or shape does not match element count. public override IBuffer Column(ReadOnlySpan name, IEnumerable value, IEnumerable shape) where T : struct { @@ -81,6 +94,11 @@ public override IBuffer Column(ReadOnlySpan name, IEnumerable value, return this; } + /// + /// Validates that the provided type is double; throws if not. + /// + /// The type to validate. + /// Thrown with if the type is not double. [MethodImpl(MethodImplOptions.AggressiveInlining)] private void GuardAgainstNonDoubleTypes(Type t) { @@ -103,6 +121,11 @@ private void PutBinaryLE(T value) where T : struct Advance(size); } + /// + /// Writes the provided value into the buffer as big-endian raw bytes and advances the buffer position by the value's size. + /// + /// A value type. + /// The value to write in big-endian byte order. // ReSharper disable once InconsistentNaming private void PutBinaryBE(T value) where T : struct { @@ -124,12 +147,13 @@ private void PutBinaryManyLE(ReadOnlySpan value) where T : struct while (srcSpan.Length > 0) { - var dstLength = GetSpareCapacity(); // length + var dstLength = GetSpareCapacity(); // length if (dstLength < byteSize) { NextBuffer(); dstLength = GetSpareCapacity(); } + var availLength = dstLength - dstLength % byteSize; // rounded length if (srcSpan.Length < availLength) @@ -138,6 +162,7 @@ private void PutBinaryManyLE(ReadOnlySpan value) where T : struct Advance(srcSpan.Length); return; } + var dstSpan = Chunk.AsSpan(Position, availLength); srcSpan.Slice(0, availLength).CopyTo(dstSpan); Advance(availLength); @@ -145,6 +170,11 @@ private void PutBinaryManyLE(ReadOnlySpan value) where T : struct } } + /// + /// Writes a sequence of values into the buffer in big-endian binary form by writing each element individually. + /// + /// A value type. + /// A span of values to write in big-endian byte order. // ReSharper disable once InconsistentNaming private void PutBinaryManyBE(ReadOnlySpan value) where T : struct { @@ -154,6 +184,11 @@ private void PutBinaryManyBE(ReadOnlySpan value) where T : struct } } + /// + /// Writes a sequence of values into the buffer in the platform's native byte order (little-endian or big-endian). + /// + /// A value type. + /// A span of values to write. private void PutBinaryMany(ReadOnlySpan value) where T : struct { if (BitConverter.IsLittleEndian) @@ -166,6 +201,11 @@ private void PutBinaryMany(ReadOnlySpan value) where T : struct } } + /// + /// Writes a single value into the buffer in the platform's native byte order (little-endian or big-endian). + /// + /// A value type. + /// The value to write. private void PutBinary(T value) where T : struct { if (BitConverter.IsLittleEndian) @@ -246,6 +286,11 @@ public override IBuffer Column(ReadOnlySpan name, Array? value) return this; } + /// + /// Reserves space in the buffer for a value of type T and returns a span to write the value later, advancing the buffer position. + /// + /// A value type. + /// An out parameter that receives a span covering the reserved space for writing. private void PutBinaryDeferred(out Span span) where T : struct { var length = Marshal.SizeOf(); @@ -254,6 +299,10 @@ private void PutBinaryDeferred(out Span span) where T : struct Advance(length); } + /// + /// Writes the ILP binary format header for a double array column. + /// + /// The column name. private void PutArrayOfDoubleHeader(ReadOnlySpan name) { Column(name) @@ -263,6 +312,10 @@ private void PutArrayOfDoubleHeader(ReadOnlySpan name) } + /// + /// Writes the ILP binary format header for a double column. + /// + /// The column name. private void PutDoubleHeader(ReadOnlySpan name) { Column(name) @@ -270,6 +323,10 @@ private void PutDoubleHeader(ReadOnlySpan name) .Put((byte)BinaryFormatType.DOUBLE); } + /// + /// Calculates the remaining available space in the current buffer chunk. + /// + /// The number of bytes remaining in the current chunk. private int GetSpareCapacity() { return Chunk.Length - Position; diff --git a/src/net-questdb-client/Buffers/IBuffer.cs b/src/net-questdb-client/Buffers/IBuffer.cs index 628128a..b16b60f 100644 --- a/src/net-questdb-client/Buffers/IBuffer.cs +++ b/src/net-questdb-client/Buffers/IBuffer.cs @@ -58,7 +58,11 @@ public interface IBuffer // ReSharper disable once InconsistentNaming public int RowCount { get; protected set; } - /// + /// + /// Encodes the specified character span to UTF-8 and appends it to the buffer. + /// + /// The character span to encode and append. + /// The buffer instance for fluent chaining. public IBuffer EncodeUtf8(ReadOnlySpan name); /// @@ -194,7 +198,10 @@ public interface IBuffer /// public void CancelRow(); - /// + /// + /// Gets the current chunk of the buffer as a read-only byte span for sending. + /// + /// A read-only span of bytes representing the current chunk. public ReadOnlySpan GetSendBuffer(); /// @@ -213,19 +220,43 @@ public interface IBuffer /// When writing to stream fails. public void WriteToStream(Stream stream, CancellationToken ct = default); - /// + /// + /// Appends a single ASCII character to the buffer. + /// + /// The ASCII character to append. + /// The buffer instance for fluent chaining. public IBuffer PutAscii(char c); - /// + /// + /// Appends a 64-bit integer value in ASCII decimal representation to the buffer. + /// + /// The long value to append. + /// The buffer instance for fluent chaining. public IBuffer Put(long value); - /// + /// + /// Appends a character span to the buffer by encoding it as UTF-8. + /// + /// The character span to encode and append. public void Put(ReadOnlySpan chars); - /// + /// + /// Appends a single byte to the buffer. + /// + /// The byte value to append. + /// The buffer instance for fluent chaining. public IBuffer Put(byte value); - /// + /// + /// Adds a column with the specified name and a span of value-type elements. + /// + /// The element type; must be a value type. + /// The column name. + /// A span of value-type elements representing the column data. + /// The buffer instance for fluent chaining. + /// + /// This method requires protocol version 2 or later. It will throw an with if used with protocol version 1. + /// public IBuffer Column(ReadOnlySpan name, ReadOnlySpan value) where T : struct; /// @@ -234,6 +265,9 @@ public interface IBuffer /// The column name. /// The array to write as the column value, or null to record a NULL value. /// The same buffer instance for fluent chaining. + /// + /// This method requires protocol version 2 or later. It will throw an with if used with protocol version 1. + /// public IBuffer Column(ReadOnlySpan name, Array? value); /// @@ -243,15 +277,19 @@ public interface IBuffer /// An enumerable of values for the column; elements are of the value type `T`. /// An enumerable of integers describing the multidimensional shape/length(s) for the values. /// The same instance for call chaining. + /// + /// This method requires protocol version 2 or later. It will throw an with if used with protocol version 1. + /// public IBuffer Column(ReadOnlySpan name, IEnumerable value, IEnumerable shape) where T : struct; - /// - /// Records a DECIMAL column value using the ILP binary decimal layout. /// /// Writes a DECIMAL column with the specified name using the ILP binary decimal layout. /// /// The column name. /// The decimal value to write, or `null` to write a NULL column. /// The buffer instance for method chaining. + /// + /// This method requires protocol version 3 or later. It will throw an with if used with protocol version 1 or 2. + /// public IBuffer Column(ReadOnlySpan name, decimal? value); } \ No newline at end of file diff --git a/src/net-questdb-client/Senders/AbstractSender.cs b/src/net-questdb-client/Senders/AbstractSender.cs index 290c96e..27f74b6 100644 --- a/src/net-questdb-client/Senders/AbstractSender.cs +++ b/src/net-questdb-client/Senders/AbstractSender.cs @@ -303,7 +303,19 @@ private ValueTask FlushIfNecessaryAsync(CancellationToken ct = default) return ValueTask.CompletedTask; } - /// + /// + /// Synchronously checks auto-flush conditions and sends the buffer if thresholds are met. + /// + /// A user-provided cancellation token. + /// + /// Auto-flushing is triggered based on: + /// + /// - the number of buffered ILP rows. + /// - the current length of the buffer in UTF-8 bytes. + /// - the elapsed time interval since the last flush. + /// + /// Has no effect within a transaction or if is set to . + /// private void FlushIfNecessary(CancellationToken ct = default) { if (Options.auto_flush == AutoFlushType.on && !WithinTransaction && diff --git a/src/net-questdb-client/Senders/HttpSender.cs b/src/net-questdb-client/Senders/HttpSender.cs index 3cb644a..841b006 100644 --- a/src/net-questdb-client/Senders/HttpSender.cs +++ b/src/net-questdb-client/Senders/HttpSender.cs @@ -70,6 +70,10 @@ public HttpSender(SenderOptions options) Build(); } + /// + /// Initializes a new instance of by parsing a configuration string. + /// + /// Configuration string in QuestDB connection string format. public HttpSender(string confStr) : this(new SenderOptions(confStr)) { } @@ -207,15 +211,20 @@ private void Build() ); } + /// + /// Creates an HTTP GET request to the /settings endpoint for querying server capabilities. + /// + /// A new configured for the /settings endpoint. private static HttpRequestMessage GenerateSettingsRequest() { return new HttpRequestMessage(HttpMethod.Get, "/settings"); } /// - /// Creates a new HTTP request with appropriate encoding and timeout. + /// Creates a new cancellation token source linked to the provided token and configured with the calculated request timeout. /// - /// + /// Optional cancellation token to link. + /// A configured with the request timeout. private CancellationTokenSource GenerateRequestCts(CancellationToken ct = default) { var cts = CancellationTokenSource.CreateLinkedTokenSource(ct); @@ -469,6 +478,11 @@ private HttpResponseMessage SendWithRetries(CancellationToken ct, Func + /// Reads and deserializes a JSON error response from the HTTP response, then throws an with the error details. + /// + /// The HTTP response containing a JSON error body. + /// Always thrown with ; the message combines the response reason phrase with the deserialized JSON error or raw response text. private void HandleErrorJson(HttpResponseMessage response) { using var respStream = response.Content.ReadAsStream(); @@ -623,10 +637,10 @@ public override async Task SendAsync(CancellationToken ct = default) } /// - /// Specifies whether a negative will lead to a retry or to an exception. + /// Determines whether the specified HTTP status code represents a transient error that should be retried. /// - /// The - /// + /// The HTTP status code to check. + /// true if the error is transient and retriable (e.g., 500, 503, 504, 509, 523, 524, 529, 599); otherwise, false. // ReSharper disable once IdentifierTypo private static bool IsRetriableError(HttpStatusCode code) { diff --git a/src/net-questdb-client/Senders/TcpSender.cs b/src/net-questdb-client/Senders/TcpSender.cs index dad37d5..eb8435c 100644 --- a/src/net-questdb-client/Senders/TcpSender.cs +++ b/src/net-questdb-client/Senders/TcpSender.cs @@ -46,16 +46,31 @@ internal class TcpSender : AbstractSender private Secp256r1SignatureGenerator? _signatureGenerator; private Socket _underlyingSocket = null!; + /// + /// Initializes a new instance of configured with the provided options. + /// + /// Configuration options for the TCP sender including host, port, TLS settings, and authentication. public TcpSender(SenderOptions options) { Options = options; Build(); } + /// + /// Initializes a new instance of by parsing a configuration string. + /// + /// Configuration string in QuestDB connection string format. public TcpSender(string confStr) : this(new SenderOptions(confStr)) { } + /// + /// Initializes the TCP connection, configures TLS if required, creates the buffer, and performs authentication if credentials are provided. + /// + /// + /// Establishes the TCP socket connection, wraps it in SSL/TLS if protocol is tcps, validates certificates according to options, performs ECDSA authentication if a token is provided, and initializes the ILP buffer. + /// + /// Thrown if TLS handshake fails, authentication fails, or connection cannot be established. private void Build() { Buffer = Buffers.Buffer.Create( @@ -65,9 +80,9 @@ private void Build() Options.protocol_version == ProtocolVersion.Auto ? ProtocolVersion.V1 : Options.protocol_version ); - var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, System.Net.Sockets.ProtocolType.Tcp); + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, System.Net.Sockets.ProtocolType.Tcp); NetworkStream? networkStream = null; - SslStream? sslStream = null; + SslStream? sslStream = null; try { socket.ConnectAsync(Options.Host, Options.Port).Wait(); @@ -100,7 +115,7 @@ private void Build() } _underlyingSocket = socket; - _dataStream = dataStream; + _dataStream = dataStream; if (!string.IsNullOrEmpty(Options.token)) { var authTimeout = new CancellationTokenSource(); @@ -154,19 +169,19 @@ private async ValueTask AuthenticateAsync(CancellationToken ct = default) } /// - /// Receives a chunk of data from the TCP stream. + /// Asynchronously reads bytes from the TCP stream until the specified terminator character is received or the buffer is full. /// - /// - /// - /// - /// + /// The terminator character indicating the end of the message. + /// Cancellation token to cancel the read operation. + /// The number of bytes read, excluding the terminator character. + /// Thrown with if the connection is closed before the terminator is received or if the buffer is too small. private async ValueTask ReceiveUntil(char endChar, CancellationToken cancellationToken) { var totalReceived = 0; while (totalReceived < Buffer.Chunk.Length) { var received = await _dataStream.ReadAsync(Buffer.Chunk, totalReceived, - Buffer.Chunk.Length - totalReceived, cancellationToken); + Buffer.Chunk.Length - totalReceived, cancellationToken); if (received > 0) { totalReceived += received; @@ -185,10 +200,15 @@ private async ValueTask ReceiveUntil(char endChar, CancellationToken cancel throw new IngressError(ErrorCode.SocketError, "Buffer is too small to receive the message."); } + /// + /// Decodes a URL-safe Base64-encoded string (using - and _ instead of + and /) and returns the decoded byte array. + /// + /// The URL-safe Base64-encoded private key string. + /// The decoded byte array. private static byte[] FromBase64String(string encodedPrivateKey) { var urlUnsafe = encodedPrivateKey.Replace('-', '+').Replace('_', '/'); - var padding = 3 - (urlUnsafe.Length + 3) % 4; + var padding = 3 - (urlUnsafe.Length + 3) % 4; if (padding != 0) { urlUnsafe += new string('=', padding); @@ -260,4 +280,4 @@ public override void Dispose() Buffer.Clear(); Buffer.TrimExcessBuffers(); } -} +} \ No newline at end of file From 3bf58c8db23ad30848a9e20197ba355d9ae8c309 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Dalmon?= <38668811+RaphDal@users.noreply.github.com> Date: Fri, 17 Oct 2025 18:33:53 +0200 Subject: [PATCH 15/24] Update src/net-questdb-client/Senders/ISender.cs Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- src/net-questdb-client/Senders/ISender.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/net-questdb-client/Senders/ISender.cs b/src/net-questdb-client/Senders/ISender.cs index a5310b2..3b117e7 100644 --- a/src/net-questdb-client/Senders/ISender.cs +++ b/src/net-questdb-client/Senders/ISender.cs @@ -235,8 +235,6 @@ public interface ISender : IDisposable /// public void CancelRow(); - /// - /// Clears the sender's buffer. /// /// Clears the sender's internal buffer and resets buffer-related state, removing all pending rows. /// From 89d923c7b90c54482a718b1d968703e31bbc93f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Dalmon?= <38668811+RaphDal@users.noreply.github.com> Date: Fri, 17 Oct 2025 18:36:48 +0200 Subject: [PATCH 16/24] Update src/net-questdb-client/Buffers/BufferV3.cs Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- src/net-questdb-client/Buffers/BufferV3.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net-questdb-client/Buffers/BufferV3.cs b/src/net-questdb-client/Buffers/BufferV3.cs index 64bb41c..185a6b8 100644 --- a/src/net-questdb-client/Buffers/BufferV3.cs +++ b/src/net-questdb-client/Buffers/BufferV3.cs @@ -64,7 +64,7 @@ public override IBuffer Column(ReadOnlySpan name, decimal? value) // # Binary Format // 1. Binary format marker: `'='` (0x3D) // 2. Type identifier: BinaryFormatType.DECIMAL byte - // 3. Scale: 1 byte (0-76 inclusive) - number of decimal places + // 3. Scale: 1 byte (0-28 for .NET decimal) - number of decimal places // 4. Length: 1 byte - number of bytes in the unscaled value // 5. Unscaled value: variable-length byte array in two's complement format, big-endian SetTableIfAppropriate(); From 82a07fcad87d6026e25ed8814204208542f3482a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Dalmon?= <38668811+RaphDal@users.noreply.github.com> Date: Fri, 17 Oct 2025 18:37:26 +0200 Subject: [PATCH 17/24] Update src/net-questdb-client/Senders/HttpSender.cs Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- src/net-questdb-client/Senders/HttpSender.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net-questdb-client/Senders/HttpSender.cs b/src/net-questdb-client/Senders/HttpSender.cs index 841b006..12ac3ea 100644 --- a/src/net-questdb-client/Senders/HttpSender.cs +++ b/src/net-questdb-client/Senders/HttpSender.cs @@ -445,7 +445,7 @@ private HttpResponseMessage SendWithRetries(CancellationToken ct, Func Date: Fri, 17 Oct 2025 18:39:33 +0200 Subject: [PATCH 18/24] fix: correct error message wording for invalid table and column names --- src/net-questdb-client/Buffers/BufferV1.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/net-questdb-client/Buffers/BufferV1.cs b/src/net-questdb-client/Buffers/BufferV1.cs index 365fa3a..b76d4fb 100644 --- a/src/net-questdb-client/Buffers/BufferV1.cs +++ b/src/net-questdb-client/Buffers/BufferV1.cs @@ -779,7 +779,7 @@ private static void GuardInvalidTableName(ReadOnlySpan tableName) $"Bad string {tableName}. Table names can't contain a {c} character, which was found at byte position {i}"); case '\xfeff': throw new IngressError(ErrorCode.InvalidName, - $"Bad string {tableName}. Table names can't contain a UTF-8 BOM character, was was found at byte position {i}."); + $"Bad string {tableName}. Table names can't contain a UTF-8 BOM character, which was found at byte position {i}."); } prev = c; @@ -840,7 +840,7 @@ private static void GuardInvalidColumnName(ReadOnlySpan columnName) $"Bad string {columnName}. Column names can't contain a {c} character, which was found at byte position {i}"); case '\xfeff': throw new IngressError(ErrorCode.InvalidName, - $"Bad string {columnName}. Column names can't contain a UTF-8 BOM character, was was found at byte position {i}."); + $"Bad string {columnName}. Column names can't contain a UTF-8 BOM character, which was found at byte position {i}."); } } } From c20de6857402aea073f57d7cc0614d8b12bfea30 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 18:54:34 +0200 Subject: [PATCH 19/24] fix: ensure two's complement conversion is unchecked for negative mantissas in BufferV3 --- src/net-questdb-client/Buffers/BufferV3.cs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/net-questdb-client/Buffers/BufferV3.cs b/src/net-questdb-client/Buffers/BufferV3.cs index 185a6b8..0a90f89 100644 --- a/src/net-questdb-client/Buffers/BufferV3.cs +++ b/src/net-questdb-client/Buffers/BufferV3.cs @@ -95,11 +95,14 @@ public override IBuffer Column(ReadOnlySpan name, decimal? value) if (negative) { // QuestDB expects negative mantissas in two's complement. - low = ~low + 1; - var c = low == 0 ? 1 : 0; - mid = ~mid + c; - c = mid == 0 && c == 1 ? 1 : 0; - high = ~high + c; + unchecked + { + low = ~low + 1; + var c = low == 0 ? 1 : 0; + mid = ~mid + c; + c = mid == 0 && c == 1 ? 1 : 0; + high = ~high + c; + } } // We write the byte array on the stack first so that we can compress (remove unnecessary bytes) it later. From b59e1ce5ec77989cbc02e315d70eec5210d78203 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 19:13:40 +0200 Subject: [PATCH 20/24] fix: improve buffer management in BufferV1 by resetting line start length --- src/net-questdb-client-tests/TcpTests.cs | 28 +++++++++++----------- src/net-questdb-client/Buffers/BufferV1.cs | 7 +++++- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/src/net-questdb-client-tests/TcpTests.cs b/src/net-questdb-client-tests/TcpTests.cs index 82e3ecc..ada2154 100644 --- a/src/net-questdb-client-tests/TcpTests.cs +++ b/src/net-questdb-client-tests/TcpTests.cs @@ -560,23 +560,23 @@ public async Task CancelLine() using var sender = Sender.New($"tcp::addr={_host}:{_port};"); - sender.Table("good"); - sender.Symbol("asdf", "sdfad"); - sender.Column("ddd", 123); -#pragma warning disable CS0618 // Type or member is obsolete - await sender.AtNowAsync(); -#pragma warning restore CS0618 // Type or member is obsolete + await sender + .Table("good") + .Symbol("asdf", "sdfad") + .Column("ddd", 123) + .AtAsync(DateTime.UtcNow); + + await sender + .Table("bad") + .Symbol("asdf", "sdfad") + .Column("asdf", 123) + .AtAsync(DateTime.UtcNow); - sender.Table("bad"); - sender.Symbol("asdf", "sdfad"); - sender.Column("asdf", 123); -#pragma warning disable CS0618 // Type or member is obsolete - await sender.AtNowAsync(); -#pragma warning restore CS0618 // Type or member is obsolete sender.CancelRow(); - sender.Table("good"); - await sender.AtAsync(new DateTime(1970, 1, 2)); + await sender + .Table("good") + .AtAsync(new DateTime(1970, 1, 2)); await sender.SendAsync(); var expected = "good,asdf=sdfad ddd=123i\n" + diff --git a/src/net-questdb-client/Buffers/BufferV1.cs b/src/net-questdb-client/Buffers/BufferV1.cs index b76d4fb..e4fc0fe 100644 --- a/src/net-questdb-client/Buffers/BufferV1.cs +++ b/src/net-questdb-client/Buffers/BufferV1.cs @@ -40,6 +40,7 @@ public class BufferV1 : IBuffer private int _currentBufferIndex; private string _currentTableName = null!; private bool _hasTable; + private int _lineStartLength; private int _lineStartBufferIndex; private int _lineStartBufferPosition; private bool _noFields = true; @@ -158,6 +159,7 @@ public void Clear() Length = 0; WithinTransaction = false; _currentTableName = ""; + _lineStartLength = 0; _lineStartBufferIndex = 0; _lineStartBufferPosition = 0; } @@ -182,9 +184,11 @@ public void TrimExcessBuffers() public void CancelRow() { _currentBufferIndex = _lineStartBufferIndex; - Length -= Position - _lineStartBufferPosition; + Chunk = _buffers[_currentBufferIndex].Buffer; + Length = _lineStartLength; Position = _lineStartBufferPosition; _hasTable = false; + } /// @@ -264,6 +268,7 @@ public IBuffer Table(ReadOnlySpan name) _quoted = false; _hasTable = true; + _lineStartLength = Length; _lineStartBufferIndex = _currentBufferIndex; _lineStartBufferPosition = Position; From 1be64bc35774aeb3d281bfe0f7d0900a0268dbab Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 17 Oct 2025 19:39:30 +0200 Subject: [PATCH 21/24] fix: update timestamp in TcpTests to use a specific date for consistency --- src/net-questdb-client-tests/TcpTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net-questdb-client-tests/TcpTests.cs b/src/net-questdb-client-tests/TcpTests.cs index ada2154..49d7895 100644 --- a/src/net-questdb-client-tests/TcpTests.cs +++ b/src/net-questdb-client-tests/TcpTests.cs @@ -564,7 +564,7 @@ await sender .Table("good") .Symbol("asdf", "sdfad") .Column("ddd", 123) - .AtAsync(DateTime.UtcNow); + .AtAsync(new DateTime(1970, 1, 2)); await sender .Table("bad") From 1495cb17393355891c45c272e852930cb889479e Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Mon, 20 Oct 2025 13:52:58 +0200 Subject: [PATCH 22/24] fix: update TcpTests to use AtNowAsync for current timestamp in test cases --- src/net-questdb-client-tests/TcpTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net-questdb-client-tests/TcpTests.cs b/src/net-questdb-client-tests/TcpTests.cs index 49d7895..5a43135 100644 --- a/src/net-questdb-client-tests/TcpTests.cs +++ b/src/net-questdb-client-tests/TcpTests.cs @@ -564,7 +564,7 @@ await sender .Table("good") .Symbol("asdf", "sdfad") .Column("ddd", 123) - .AtAsync(new DateTime(1970, 1, 2)); + .AtNowAsync(); await sender .Table("bad") From bc20a0dc200cad9777d263ea6c3d283fe7b4f5f5 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 24 Oct 2025 11:31:39 +0200 Subject: [PATCH 23/24] fix: don't write column when the value is null --- .../DecimalTestHelpers.cs | 20 ------------------- src/net-questdb-client-tests/HttpTests.cs | 5 +++-- src/net-questdb-client-tests/TcpTests.cs | 3 ++- src/net-questdb-client/Buffers/BufferV3.cs | 11 +++++----- 4 files changed, 10 insertions(+), 29 deletions(-) diff --git a/src/net-questdb-client-tests/DecimalTestHelpers.cs b/src/net-questdb-client-tests/DecimalTestHelpers.cs index 77142e4..e28d9f2 100644 --- a/src/net-questdb-client-tests/DecimalTestHelpers.cs +++ b/src/net-questdb-client-tests/DecimalTestHelpers.cs @@ -56,26 +56,6 @@ internal static void AssertDecimalField(ReadOnlySpan buffer, $"Mantissa bytes for `{columnName}` did not match expectation."); } - /// - /// Asserts that the buffer contains a null decimal field payload for the specified column. - /// - /// Buffer containing the encoded record(s) to inspect. - /// Name of the column whose decimal payload should be null. - /// - /// Verifies the payload starts with '=' then the DECIMAL type marker, and that both scale and mantissa length are zero. - /// - internal static void AssertDecimalNullField(ReadOnlySpan buffer, string columnName) - { - var payload = ExtractDecimalPayload(buffer, columnName); - Assert.That(payload.Length, - Is.GreaterThanOrEqualTo(4), - $"Decimal field `{columnName}` payload shorter than expected."); - Assert.That(payload[0], Is.EqualTo((byte)'=')); - Assert.That(payload[1], Is.EqualTo((byte)BinaryFormatType.DECIMAL)); - Assert.That(payload[2], Is.EqualTo(0), $"Unexpected scale for `{columnName}`."); - Assert.That(payload[3], Is.EqualTo(0), $"Unexpected mantissa length for `{columnName}`."); - } - /// /// Locate and return the payload bytes for a decimal column identified by name. /// diff --git a/src/net-questdb-client-tests/HttpTests.cs b/src/net-questdb-client-tests/HttpTests.cs index 5376baa..b2e14c2 100644 --- a/src/net-questdb-client-tests/HttpTests.cs +++ b/src/net-questdb-client-tests/HttpTests.cs @@ -117,7 +117,8 @@ await sender.Table("metrics") { 0xCF, 0xC7, }); - DecimalTestHelpers.AssertDecimalNullField(buffer, "dec_null"); + var prefix = Encoding.UTF8.GetBytes("dec_null="); + Assert.That(buffer.AsSpan().IndexOf(prefix), Is.EqualTo(-1)); DecimalTestHelpers.AssertDecimalField(buffer, "dec_max", 0, new byte[] { 0x00, @@ -1720,4 +1721,4 @@ public async Task FailsWhenExpectingCert() await server.StopAsync(); } -} +} \ No newline at end of file diff --git a/src/net-questdb-client-tests/TcpTests.cs b/src/net-questdb-client-tests/TcpTests.cs index 5a43135..7bbe2c6 100644 --- a/src/net-questdb-client-tests/TcpTests.cs +++ b/src/net-questdb-client-tests/TcpTests.cs @@ -89,7 +89,8 @@ await sender.Table("metrics") { 0xCF, 0xC7, }); - DecimalTestHelpers.AssertDecimalNullField(buffer, "dec_null"); + var prefix = Encoding.UTF8.GetBytes("dec_null="); + Assert.That(buffer.AsSpan().IndexOf(prefix), Is.EqualTo(-1)); DecimalTestHelpers.AssertDecimalField(buffer, "dec_max", 0, new byte[] { 0x00, diff --git a/src/net-questdb-client/Buffers/BufferV3.cs b/src/net-questdb-client/Buffers/BufferV3.cs index 0a90f89..d0e1a74 100644 --- a/src/net-questdb-client/Buffers/BufferV3.cs +++ b/src/net-questdb-client/Buffers/BufferV3.cs @@ -61,6 +61,11 @@ public BufferV3(int bufferSize, int maxNameLen, int maxBufSize) : base(bufferSiz /// The buffer instance for call chaining. public override IBuffer Column(ReadOnlySpan name, decimal? value) { + if (value is null) + { + return this; + } + // # Binary Format // 1. Binary format marker: `'='` (0x3D) // 2. Type identifier: BinaryFormatType.DECIMAL byte @@ -71,12 +76,6 @@ public override IBuffer Column(ReadOnlySpan name, decimal? value) Column(name) .PutAscii(Constants.BINARY_FORMAT_FLAG) .Put((byte)BinaryFormatType.DECIMAL); - if (value is null) - { - Put(0); // Scale - Put(0); // Length - return this; - } Span parts = stackalloc int[4]; decimal.GetBits(value.Value, parts); From 331083e38dad1652a57dbde4b85946868390e332 Mon Sep 17 00:00:00 2001 From: Raphael DALMON Date: Fri, 24 Oct 2025 11:31:48 +0200 Subject: [PATCH 24/24] test: add BufferTests for decimal negation scenarios --- src/net-questdb-client-tests/BufferTests.cs | 174 ++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 src/net-questdb-client-tests/BufferTests.cs diff --git a/src/net-questdb-client-tests/BufferTests.cs b/src/net-questdb-client-tests/BufferTests.cs new file mode 100644 index 0000000..2dbeaa6 --- /dev/null +++ b/src/net-questdb-client-tests/BufferTests.cs @@ -0,0 +1,174 @@ +using NUnit.Framework; +using QuestDB.Buffers; + +namespace net_questdb_client_tests; + +public class BufferTests +{ + [Test] + public void DecimalNegationSimple() + { + var buffer = new BufferV3(128, 128, 128); + // Test simple negative decimal without carry propagation + // -1.0m has unscaled value of -10 (with scale 1) + buffer.Table("negation_test") + .Symbol("tag", "simple") + .Column("dec_neg_one", -1.0m) + .At(new DateTime(1970, 01, 01, 0, 0, 1)); + + // -10 in two's complement: 0xF6 (since 10 = 0x0A, ~0x0A + 1 = 0xF5 + 1 = 0xF6) + DecimalTestHelpers.AssertDecimalField(buffer.GetSendBuffer(), "dec_neg_one", 1, new byte[] + { + 0xF6, + }); + } + + [Test] + public void DecimalNegationCarryLowToMid() + { + var buffer = new BufferV3(128, 128, 128); + + // Test carry propagation from low to mid part + // Decimal with low=0x00000001, mid=0x00000000, high=0x00000000 + // After negation: low becomes 0xFFFFFFFF (overflow with carry), mid gets carry + // This decimal is: -(2^96 / 10^28 + 1) which causes carry propagation + // Use -4294967296 (which is -2^32) to force carry: low=0x00000000, mid=0x00000001 + // After negation: low = ~0 + 1 = 0, mid = ~1 + 1 = 0xFFFFFFFE + 1 = 0xFFFFFFFF + var decimalValue = -4294967296m; // -2^32 + + buffer.Table("negation_test") + .Symbol("tag", "carry_low_mid") + .Column("dec_carry", decimalValue) + .At(new DateTime(1970, 01, 01, 0, 0, 1)); + + // -4294967296 has bits: low=0, mid=1, high=0 + // Two's complement: low = ~0 + 1 = 0 (with carry), mid = ~1 + carry = 0xFFFFFFFE + 1 = 0xFFFFFFFF + // Result should be: 0xFF, 0x00, 0x00, 0x00, 0x00 (compressed) + DecimalTestHelpers.AssertDecimalField(buffer.GetSendBuffer(), "dec_carry", 0, new byte[] + { + 0xFF, + 0x00, 0x00, 0x00, 0x00, + }); + } + + [Test] + public void DecimalNegationCarryFullPropagation() + { + var buffer = new BufferV3(128, 128, 128); + + // Test carry propagation through all parts (low -> mid -> high) + // Create a decimal where low=0, mid=0, high=1 + // This is 2^64 = 18446744073709551616 + // After negation: low = ~0 + 1 = 0 (carry), mid = ~0 + 1 = 0 (carry), high = ~1 + 1 = 0xFFFFFFFE + 1 = 0xFFFFFFFF + var decimalValue = -18446744073709551616m; // -2^64 + + buffer.Table("negation_test") + .Symbol("tag", "carry_full") + .Column("dec_full_carry", decimalValue) + .At(new DateTime(1970, 01, 01, 0, 0, 1)); + + // -18446744073709551616 has bits: low=0, mid=0, high=1 + // Two's complement propagates carry through all parts + // Result: high=0xFFFFFFFF, mid=0, low=0 + // In big-endian with sign byte: 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + DecimalTestHelpers.AssertDecimalField(buffer.GetSendBuffer(), "dec_full_carry", 0, new byte[] + { + 0xFF, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + }); + } + + [Test] + public void DecimalNegationZeroEdgeCase() + { + var buffer = new BufferV3(128, 128, 128); + + // Test that -0.0m is treated as positive zero (line 92 in BufferV3.cs) + // The code checks: var negative = (flags & SignMask) != 0 && value.Value != 0m; + // This ensures that -0.0m doesn't get negated + buffer.Table("negation_test") + .Symbol("tag", "zero") + .Column("dec_zero", 0.0m) + .Column("dec_neg_zero", -0.0m) + .At(new DateTime(1970, 01, 01, 0, 0, 1)); + + // Both 0.0m and -0.0m should be encoded as positive zero: 0x00 + DecimalTestHelpers.AssertDecimalField(buffer.GetSendBuffer(), "dec_zero", 1, new byte[] + { + 0x00, + }); + DecimalTestHelpers.AssertDecimalField(buffer.GetSendBuffer(), "dec_neg_zero", 1, new byte[] + { + 0x00, + }); + } + + [Test] + public void DecimalNegationSmallestValue() + { + var buffer = new BufferV3(128, 128, 128); + + // Test negation of the smallest representable positive value: 0.0000000000000000000000000001m + // This has low=1, mid=0, high=0, scale=28 + // After negation: low = ~1 + 1 = 0xFFFFFFFE + 1 = 0xFFFFFFFF + var decimalValue = -0.0000000000000000000000000001m; + + buffer.Table("negation_test") + .Symbol("tag", "smallest") + .Column("dec_smallest", decimalValue) + .At(new DateTime(1970, 01, 01, 0, 0, 1)); + + // Result should be: 0xFF (single byte, compressed) + DecimalTestHelpers.AssertDecimalField(buffer.GetSendBuffer(), "dec_smallest", 28, new byte[] + { + 0xFF, + }); + } + + [Test] + public void DecimalNegationWithHighScale() + { + var buffer = new BufferV3(128, 128, 128); + + // Test negation with high scale value + // -0.00000001m has scale=8 + var decimalValue = -0.00000001m; // -10^-8 + + buffer.Table("negation_test") + .Symbol("tag", "high_scale") + .Column("dec_high_scale", decimalValue) + .At(new DateTime(1970, 01, 01, 0, 0, 1)); + + // -1 with scale 8 = 0xFF in two's complement + DecimalTestHelpers.AssertDecimalField(buffer.GetSendBuffer(), "dec_high_scale", 8, new byte[] + { + 0xFF, + }); + } + + [Test] + public void DecimalNegationBoundaryCarry() + { + var buffer = new BufferV3(128, 128, 128); + + // Test a value where low=0xFFFFFFFF (all ones), which after negation becomes 1 + // This is the value 4294967295 (2^32 - 1) + // After negation: low = ~0xFFFFFFFF + 1 = 0x00000000 + 1 = 0x00000001 + // So -4294967295 should give us: 0xFF, 0xFF, 0xFF, 0xFF, 0x01 + var decimalValue = -4294967295m; + + buffer.Table("negation_test") + .Symbol("tag", "boundary") + .Column("dec_boundary", decimalValue) + .At(new DateTime(1970, 01, 01, 0, 0, 1)); + + // Two's complement of 4294967295: negates to 0xFFFFFFFF00000001 (represented in big-endian) + DecimalTestHelpers.AssertDecimalField(buffer.GetSendBuffer(), "dec_boundary", 0, new byte[] + { + 0xFF, + 0xFF, 0xFF, 0xFF, 0x01, + }); + } + +} \ No newline at end of file