diff --git a/README.md b/README.md index 05293d7..5c38533 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ using var sender = Sender.New("http::addr=localhost:9000;"); await sender.Table("trades") .Symbol("symbol", "ETH-USD") .Symbol("side", "sell") - .Column("price", 2615.54) + .Column("price", 2615.54m) .Column("amount", 0.00044) .AtAsync(new DateTime(2021, 11, 25, 0, 46, 26)); await sender.SendAsync(); @@ -60,7 +60,7 @@ for(int i = 0; i < 100; i++) sender.Table("trades") .Symbol("symbol", "ETH-USD") .Symbol("side", "sell") - .Column("price", 2615.54) + .Column("price", 2615.54m) .Column("amount", 0.00044) .At(DateTime.UtcNow); } @@ -136,53 +136,54 @@ The config string format is: {http/https/tcp/tcps}::addr={host}:{port};key1=val1;key2=val2;keyN=valN; ``` -| Name | Default | Description | -|--------------------------|----------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `protocol` (schema) | `http` | The transport protocol to use. Options are http(s)/tcp(s). | -| `addr` | `localhost:9000` | The {host}:{port} pair denoting the QuestDB server. By default, port 9000 for HTTP, port 9009 for TCP. | -| `auto_flush` | `on` | Enables or disables auto-flushing functionality. By default, the buffer will be flushed every 75,000 rows, or every 1000ms, whichever comes first. | -| `auto_flush_rows` | `75000 (HTTP)` `600 (TCP)` | The row count after which the buffer will be flushed. Effectively a batch size. | -| `auto_flush_bytes` | `Int.MaxValue` | The byte buffer length which when exceeded, will trigger a flush. | -| `auto_flush_interval` | `1000` | The millisecond interval, which once has elapsed, the next row triggers a flush. | -| `init_buf_size` | `65536` | The starting byte buffer length. Overflowing this buffer will cause the allocation `init_buf_size` bytes (an additional buffer). | -| `max_buf_size` | `104857600` | Maximum size of the byte buffer in bytes. If exceeded, an exception will be thrown. | -| `username` | | The username for authentication. Used for Basic Authentication and TCP JWK Authentication. | -| `password` | | The password for authentication. Used for Basic Authentication. | -| `token` | | The token for authentication. Used for Token Authentication and TCP JWK Authentication. | -| `token_x` | | Un-used. | -| `token_y` | | Un-used. | -| `tls_verify` | `on` | Denotes whether TLS certificates should or should not be verifed. Options are on/unsafe_off. | -| `tls_ca` | | Un-used. | -| `tls_roots` | | Used to specify the filepath for a custom .pem certificate. | -| `tls_roots_password` | | Used to specify the filepath for the private key/password corresponding to the `tls_roots` certificate. | -| `auth_timeout` | `15000` | The time period to wait for authenticating requests, in milliseconds. | -| `request_timeout` | `10000` | Base timeout for HTTP requests before any additional time is added. | -| `request_min_throughput` | `102400` | Expected minimum throughput of requests in bytes per second. Used to add additional time to `request_timeout` to prevent large requests timing out prematurely. | -| `retry_timeout` | `10000` | The time period during which retries will be attempted, in milliseconds. | -| `max_name_len` | `127` | The maximum allowed bytes, in UTF-8 format, for column and table names. | -| `protocol_version` | | Explicitly specifies the version of InfluxDB Line Protocol to use for sender. Valid options are:
• protocol_version=1
• protocol_version=2
• protocol_version=auto (default, if unspecified) | +| Name | Default | Description | +| ------------------------ | -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `protocol` (schema) | `http` | The transport protocol to use. Options are http(s)/tcp(s). | +| `addr` | `localhost:9000` | The {host}:{port} pair denoting the QuestDB server. By default, port 9000 for HTTP, port 9009 for TCP. | +| `auto_flush` | `on` | Enables or disables auto-flushing functionality. By default, the buffer will be flushed every 75,000 rows, or every 1000ms, whichever comes first. | +| `auto_flush_rows` | `75000 (HTTP)` `600 (TCP)` | The row count after which the buffer will be flushed. Effectively a batch size. | +| `auto_flush_bytes` | `Int.MaxValue` | The byte buffer length which when exceeded, will trigger a flush. | +| `auto_flush_interval` | `1000` | The millisecond interval, which once has elapsed, the next row triggers a flush. | +| `init_buf_size` | `65536` | The starting byte buffer length. Overflowing this buffer will cause the allocation `init_buf_size` bytes (an additional buffer). | +| `max_buf_size` | `104857600` | Maximum size of the byte buffer in bytes. If exceeded, an exception will be thrown. | +| `username` | | The username for authentication. Used for Basic Authentication and TCP JWK Authentication. | +| `password` | | The password for authentication. Used for Basic Authentication. | +| `token` | | The token for authentication. Used for Token Authentication and TCP JWK Authentication. | +| `token_x` | | Un-used. | +| `token_y` | | Un-used. | +| `tls_verify` | `on` | Denotes whether TLS certificates should or should not be verified. Options are on/unsafe_off. | +| `tls_ca` | | Un-used. | +| `tls_roots` | | Used to specify the filepath for a custom .pem certificate. | +| `tls_roots_password` | | Used to specify the filepath for the private key/password corresponding to the `tls_roots` certificate. | +| `auth_timeout` | `15000` | The time period to wait for authenticating requests, in milliseconds. | +| `request_timeout` | `10000` | Base timeout for HTTP requests before any additional time is added. | +| `request_min_throughput` | `102400` | Expected minimum throughput of requests in bytes per second. Used to add additional time to `request_timeout` to prevent large requests timing out prematurely. | +| `retry_timeout` | `10000` | The time period during which retries will be attempted, in milliseconds. | +| `max_name_len` | `127` | The maximum allowed bytes, in UTF-8 format, for column and table names. | +| `protocol_version` | | Explicitly specifies the version of InfluxDB Line Protocol to use for sender. Valid options are:
• protocol_version=1
• protocol_version=2
• protocol_version=3
• protocol_version=auto (default, if unspecified) | ### Protocol Version Behavior details: -| Value | Behavior | -|--------|---------------------------------------------------------------------------------------------------------------------------------------------------------| -| 1 | - Plain text serialization
- Compatible with InfluxDB servers
- No array type support | -| 2 | - Binary encoding for double arrays
- Full support for array | +| Value | Behavior | +| ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | - Plain text serialization
- Compatible with InfluxDB servers
- No array type support | +| 2 | - Binary encoding for double arrays
- Full support for array | +| 3 | - Support for decimal | | `auto` | - **HTTP/HTTPS**: Auto-detects server capability during handshake (supports version negotiation)
- **TCP/TCPS**: Defaults to version 1 for compatibility | ### net-questdb-client specific parameters | Name | Default | Description | -|----------------|----------|---------------------------------------------------------------------------------------| +| -------------- | -------- | ------------------------------------------------------------------------------------- | | `own_socket` | `true` | Specifies whether the internal TCP data stream will own the underlying socket or not. | | `pool_timeout` | `120000` | Sets the idle timeout for HTTP connections in SocketsHttpHandler. | ## Properties and methods | Name | Returns | Description | -|-------------------------------------------------------------------------------------------------------|-----------------|----------------------------------------------------------------------------| +| ----------------------------------------------------------------------------------------------------- | --------------- | -------------------------------------------------------------------------- | | `Length` | `int` | Current length in bytes of the buffer (not capacity!) | | `RowCount` | `int` | Current row count of the buffer | | `LastFlush` | `DateTime` | Returns the UTC DateTime of the last flush sending data to the server. | diff --git a/net-questdb-client.sln b/net-questdb-client.sln index 8f9e3c5..91cec48 100644 --- a/net-questdb-client.sln +++ b/net-questdb-client.sln @@ -5,8 +5,6 @@ VisualStudioVersion = 16.0.30114.105 MinimumVisualStudioVersion = 10.0.40219.1 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "net-questdb-client", "src\net-questdb-client\net-questdb-client.csproj", "{456B1860-0102-48D7-861A-5F9963F3887B}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "tcp-client-test", "src\tcp-client-test\tcp-client-test.csproj", "{22F903D9-4367-46A2-A25A-F4A6BF9105C6}" -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "example-basic", "src\example-basic\example-basic.csproj", "{121EAA4D-3A73-468C-8CAB-A2A4BEF848CF}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "example-auth-tls", "src\example-auth-tls\example-auth-tls.csproj", "{FBB8181C-6BAB-46C2-A47A-D3566A3997FE}" @@ -21,7 +19,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "example-streaming", "src\ex EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "example-auth-http-tls", "src\example-auth-http-tls\example-auth-http-tls.csproj", "{24D93DBB-3783-423F-81CC-6B9BFD33F6CD}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "example-aot", "example-aot\example-aot.csproj", "{5341FCF0-F71D-4160-8D6E-B5EFDF92E9E8}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "example-aot", "src\example-aot\example-aot.csproj", "{5341FCF0-F71D-4160-8D6E-B5EFDF92E9E8}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -36,10 +34,6 @@ Global {456B1860-0102-48D7-861A-5F9963F3887B}.Debug|Any CPU.Build.0 = Debug|Any CPU {456B1860-0102-48D7-861A-5F9963F3887B}.Release|Any CPU.ActiveCfg = Release|Any CPU {456B1860-0102-48D7-861A-5F9963F3887B}.Release|Any CPU.Build.0 = Release|Any CPU - {22F903D9-4367-46A2-A25A-F4A6BF9105C6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {22F903D9-4367-46A2-A25A-F4A6BF9105C6}.Debug|Any CPU.Build.0 = Debug|Any CPU - {22F903D9-4367-46A2-A25A-F4A6BF9105C6}.Release|Any CPU.ActiveCfg = Release|Any CPU - {22F903D9-4367-46A2-A25A-F4A6BF9105C6}.Release|Any CPU.Build.0 = Release|Any CPU {121EAA4D-3A73-468C-8CAB-A2A4BEF848CF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {121EAA4D-3A73-468C-8CAB-A2A4BEF848CF}.Debug|Any CPU.Build.0 = Debug|Any CPU {121EAA4D-3A73-468C-8CAB-A2A4BEF848CF}.Release|Any CPU.ActiveCfg = Release|Any CPU diff --git a/src/dummy-http-server/DummyHttpServer.cs b/src/dummy-http-server/DummyHttpServer.cs index 2dd36f6..91a21bc 100644 --- a/src/dummy-http-server/DummyHttpServer.cs +++ b/src/dummy-http-server/DummyHttpServer.cs @@ -42,28 +42,37 @@ public class DummyHttpServer : IDisposable private int _port = 29743; private readonly TimeSpan? _withStartDelay; + /// + /// Initializes a configurable in-process dummy HTTP server used for testing endpoints. + /// + /// If true, enable JWT bearer authentication and authorization. + /// If true, enable basic authentication behavior in the test endpoint. + /// If true, configure the test endpoint to produce retriable error responses. + /// If true, include error messages in test error responses. + /// Optional delay applied when starting the server. + /// If true, require client TLS certificates for HTTPS connections. public DummyHttpServer(bool withTokenAuth = false, bool withBasicAuth = false, bool withRetriableError = false, - bool withErrorMessage = false, TimeSpan? withStartDelay = null, bool requireClientCert = false) + bool withErrorMessage = false, TimeSpan? withStartDelay = null, bool requireClientCert = false) { var bld = WebApplication.CreateBuilder(); bld.Services.AddLogging(builder => { builder.AddFilter("Microsoft", LogLevel.Warning) - .AddFilter("System", LogLevel.Warning) - .AddConsole(); + .AddFilter("System", LogLevel.Warning) + .AddConsole(); }); - IlpEndpoint.WithTokenAuth = withTokenAuth; - IlpEndpoint.WithBasicAuth = withBasicAuth; + IlpEndpoint.WithTokenAuth = withTokenAuth; + IlpEndpoint.WithBasicAuth = withBasicAuth; IlpEndpoint.WithRetriableError = withRetriableError; - IlpEndpoint.WithErrorMessage = withErrorMessage; + IlpEndpoint.WithErrorMessage = withErrorMessage; _withStartDelay = withStartDelay; if (withTokenAuth) { bld.Services.AddAuthenticationJwtBearer(s => s.SigningKey = SigningKey) - .AddAuthorization(); + .AddAuthorization(); } @@ -83,7 +92,7 @@ public DummyHttpServer(bool withTokenAuth = false, bool withBasicAuth = false, b o.Limits.MaxRequestBodySize = 1073741824; o.ListenLocalhost(29474, - options => { options.UseHttps(); }); + options => { options.UseHttps(); }); o.ListenLocalhost(29473); }); @@ -108,26 +117,43 @@ public void Dispose() _app.StopAsync().Wait(); } + /// + /// Clears the in-memory receive buffers and resets the endpoint error state and counter. + /// + /// + /// Empties IlpEndpoint.ReceiveBuffer and IlpEndpoint.ReceiveBytes, sets IlpEndpoint.LastError to null, + /// and sets IlpEndpoint.Counter to zero. + /// public void Clear() { IlpEndpoint.ReceiveBuffer.Clear(); IlpEndpoint.ReceiveBytes.Clear(); IlpEndpoint.LastError = null; - IlpEndpoint.Counter = 0; + IlpEndpoint.Counter = 0; } + /// + /// Starts the HTTP server on the specified port and configures the supported protocol versions. + /// + /// Port to listen on (defaults to 29743). + /// Array of supported protocol versions; defaults to {1, 2, 3} when null. + /// A task that completes after any configured startup delay has elapsed and the server's background run task has been initiated. public async Task StartAsync(int port = 29743, int[]? versions = null) { if (_withStartDelay.HasValue) { await Task.Delay(_withStartDelay.Value); } - versions ??= new[] { 1, 2, }; - SettingsEndpoint.Versions = versions; - _port = port; - _app.RunAsync($"http://localhost:{port}"); + + versions ??= new[] { 1, 2, 3, }; + SettingsEndpoint.Versions = versions; + _port = port; + _ = _app.RunAsync($"http://localhost:{port}"); } + /// + /// Starts the web application and listens for HTTP requests on http://localhost:{_port}. + /// public async Task RunAsync() { await _app.RunAsync($"http://localhost:{_port}"); @@ -138,12 +164,20 @@ public async Task StopAsync() await _app.StopAsync(); } + /// + /// Gets the server's in-memory text buffer of received data. + /// + /// The mutable containing the accumulated received text; modifying it updates the server's buffer. public StringBuilder GetReceiveBuffer() { return IlpEndpoint.ReceiveBuffer; } - public List GetReceiveBytes() + /// + /// Gets the in-memory list of bytes received by the ILP endpoint. + /// + /// The mutable list of bytes received by the endpoint. + public List GetReceivedBytes() { return IlpEndpoint.ReceiveBytes; } @@ -160,6 +194,10 @@ public async Task Healthcheck() } + /// + /// Generates a JWT for the test server when the provided credentials match the server's static username and password. + /// + /// The JWT string when credentials are valid; null otherwise. The issued token is valid for one day. public string? GetJwtToken(string username, string password) { if (username == Username && password == Password) @@ -167,7 +205,7 @@ public async Task Healthcheck() var jwtToken = JwtBearer.CreateToken(o => { o.SigningKey = SigningKey; - o.ExpireAt = DateTime.UtcNow.AddDays(1); + o.ExpireAt = DateTime.UtcNow.AddDays(1); }); return jwtToken; } @@ -180,87 +218,89 @@ public int GetCounter() return IlpEndpoint.Counter; } + /// + /// Produces a human-readable string representation of the server's received-bytes buffer, interpreting embedded markers and formatting arrays and numeric values. + /// + /// The formatted textual representation of the received bytes buffer. + /// Thrown when the buffer contains an unsupported type code. public string PrintBuffer() { - var bytes = GetReceiveBytes().ToArray(); - var sb = new StringBuilder(); + var bytes = GetReceivedBytes().ToArray(); + var sb = new StringBuilder(); var lastAppend = 0; var i = 0; for (; i < bytes.Length; i++) { - if (bytes[i] == (byte)'=') + if (bytes[i] == (byte)'=' && i > 0 && bytes[i - 1] == (byte)'=') { - if (bytes[i - 1] == (byte)'=') + sb.Append(Encoding.UTF8.GetString(bytes, lastAppend, i + 1 - lastAppend)); + switch (bytes[++i]) { - sb.Append(Encoding.UTF8.GetString(bytes, lastAppend, i + 1 - lastAppend)); - switch (bytes[++i]) - { - case 14: - sb.Append("ARRAY<"); - var type = bytes[++i]; + case 14: + sb.Append("ARRAY<"); + var type = bytes[++i]; - Debug.Assert(type == 10); - var dims = bytes[++i]; + Debug.Assert(type == 10); + var dims = bytes[++i]; - ++i; + ++i; - long length = 0; - for (var j = 0; j < dims; j++) + long length = 0; + for (var j = 0; j < dims; j++) + { + var lengthBytes = bytes.AsSpan()[i..(i + 4)]; + var lengthValue = MemoryMarshal.Cast(lengthBytes)[0]; + if (length == 0) { - var lengthBytes = bytes.AsSpan()[i..(i + 4)]; - var lengthValue = MemoryMarshal.Cast(lengthBytes)[0]; - if (length == 0) - { - length = lengthValue; - } - else - { - length *= lengthValue; - } - - sb.Append(lengthValue); - sb.Append(','); - i += 4; + length = lengthValue; } - - sb.Remove(sb.Length - 1, 1); - sb.Append('>'); - - var doubleBytes = - MemoryMarshal.Cast(bytes.AsSpan().Slice(i, (int)(length * 8))); - - - sb.Append('['); - for (var j = 0; j < length; j++) + else { - sb.Append(doubleBytes[j]); - sb.Append(','); + length *= lengthValue; } - sb.Remove(sb.Length - 1, 1); - sb.Append(']'); - - i += (int)(length * 8); - i--; - break; - case 16: - sb.Remove(sb.Length - 1, 1); - var doubleValue = MemoryMarshal.Cast(bytes.AsSpan().Slice(++i, 8)); - sb.Append(doubleValue[0].ToString(CultureInfo.InvariantCulture)); - i += 8; - i--; - break; - default: - throw new NotImplementedException(); - } - - lastAppend = i + 1; + sb.Append(lengthValue); + sb.Append(','); + i += 4; + } + + sb.Remove(sb.Length - 1, 1); + sb.Append('>'); + + var doubleBytes = + MemoryMarshal.Cast(bytes.AsSpan().Slice(i, (int)(length * 8))); + + + sb.Append('['); + for (var j = 0; j < length; j++) + { + sb.Append(doubleBytes[j].ToString(CultureInfo.InvariantCulture)); + sb.Append(','); + } + + sb.Remove(sb.Length - 1, 1); + sb.Append(']'); + + i += (int)(length * 8); + i--; + break; + case 16: + sb.Remove(sb.Length - 1, 1); + var doubleValue = MemoryMarshal.Cast(bytes.AsSpan().Slice(++i, 8)); + sb.Append(doubleValue[0].ToString(CultureInfo.InvariantCulture)); + i += 8; + i--; + break; + default: + throw new NotImplementedException($"Type {bytes[i]} not implemented"); } + + lastAppend = i + 1; } } sb.Append(Encoding.UTF8.GetString(bytes, lastAppend, i - lastAppend)); return sb.ToString(); } -} +} \ No newline at end of file diff --git a/example-aot/Program.cs b/src/example-aot/Program.cs similarity index 100% rename from example-aot/Program.cs rename to src/example-aot/Program.cs diff --git a/example-aot/example-aot.csproj b/src/example-aot/example-aot.csproj similarity index 83% rename from example-aot/example-aot.csproj rename to src/example-aot/example-aot.csproj index 28bdf22..b442655 100644 --- a/example-aot/example-aot.csproj +++ b/src/example-aot/example-aot.csproj @@ -11,7 +11,7 @@ - + diff --git a/src/net-questdb-client-benchmarks/BenchInserts.cs b/src/net-questdb-client-benchmarks/BenchInserts.cs index 4f49aa5..b68144c 100644 --- a/src/net-questdb-client-benchmarks/BenchInserts.cs +++ b/src/net-questdb-client-benchmarks/BenchInserts.cs @@ -26,7 +26,7 @@ using BenchmarkDotNet.Attributes; using dummy_http_server; using QuestDB; -using tcp_client_test; +using net_questdb_client_tests; #pragma warning disable CS0414 // Field is assigned but its value is never used diff --git a/src/net-questdb-client-benchmarks/net-questdb-client-benchmarks.csproj b/src/net-questdb-client-benchmarks/net-questdb-client-benchmarks.csproj index 58dac62..304cac6 100644 --- a/src/net-questdb-client-benchmarks/net-questdb-client-benchmarks.csproj +++ b/src/net-questdb-client-benchmarks/net-questdb-client-benchmarks.csproj @@ -15,8 +15,8 @@ + - diff --git a/src/net-questdb-client-tests/BufferTests.cs b/src/net-questdb-client-tests/BufferTests.cs new file mode 100644 index 0000000..2dbeaa6 --- /dev/null +++ b/src/net-questdb-client-tests/BufferTests.cs @@ -0,0 +1,174 @@ +using NUnit.Framework; +using QuestDB.Buffers; + +namespace net_questdb_client_tests; + +public class BufferTests +{ + [Test] + public void DecimalNegationSimple() + { + var buffer = new BufferV3(128, 128, 128); + // Test simple negative decimal without carry propagation + // -1.0m has unscaled value of -10 (with scale 1) + buffer.Table("negation_test") + .Symbol("tag", "simple") + .Column("dec_neg_one", -1.0m) + .At(new DateTime(1970, 01, 01, 0, 0, 1)); + + // -10 in two's complement: 0xF6 (since 10 = 0x0A, ~0x0A + 1 = 0xF5 + 1 = 0xF6) + DecimalTestHelpers.AssertDecimalField(buffer.GetSendBuffer(), "dec_neg_one", 1, new byte[] + { + 0xF6, + }); + } + + [Test] + public void DecimalNegationCarryLowToMid() + { + var buffer = new BufferV3(128, 128, 128); + + // Test carry propagation from low to mid part + // Decimal with low=0x00000001, mid=0x00000000, high=0x00000000 + // After negation: low becomes 0xFFFFFFFF (overflow with carry), mid gets carry + // This decimal is: -(2^96 / 10^28 + 1) which causes carry propagation + // Use -4294967296 (which is -2^32) to force carry: low=0x00000000, mid=0x00000001 + // After negation: low = ~0 + 1 = 0, mid = ~1 + 1 = 0xFFFFFFFE + 1 = 0xFFFFFFFF + var decimalValue = -4294967296m; // -2^32 + + buffer.Table("negation_test") + .Symbol("tag", "carry_low_mid") + .Column("dec_carry", decimalValue) + .At(new DateTime(1970, 01, 01, 0, 0, 1)); + + // -4294967296 has bits: low=0, mid=1, high=0 + // Two's complement: low = ~0 + 1 = 0 (with carry), mid = ~1 + carry = 0xFFFFFFFE + 1 = 0xFFFFFFFF + // Result should be: 0xFF, 0x00, 0x00, 0x00, 0x00 (compressed) + DecimalTestHelpers.AssertDecimalField(buffer.GetSendBuffer(), "dec_carry", 0, new byte[] + { + 0xFF, + 0x00, 0x00, 0x00, 0x00, + }); + } + + [Test] + public void DecimalNegationCarryFullPropagation() + { + var buffer = new BufferV3(128, 128, 128); + + // Test carry propagation through all parts (low -> mid -> high) + // Create a decimal where low=0, mid=0, high=1 + // This is 2^64 = 18446744073709551616 + // After negation: low = ~0 + 1 = 0 (carry), mid = ~0 + 1 = 0 (carry), high = ~1 + 1 = 0xFFFFFFFE + 1 = 0xFFFFFFFF + var decimalValue = -18446744073709551616m; // -2^64 + + buffer.Table("negation_test") + .Symbol("tag", "carry_full") + .Column("dec_full_carry", decimalValue) + .At(new DateTime(1970, 01, 01, 0, 0, 1)); + + // -18446744073709551616 has bits: low=0, mid=0, high=1 + // Two's complement propagates carry through all parts + // Result: high=0xFFFFFFFF, mid=0, low=0 + // In big-endian with sign byte: 0xFF, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + DecimalTestHelpers.AssertDecimalField(buffer.GetSendBuffer(), "dec_full_carry", 0, new byte[] + { + 0xFF, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + }); + } + + [Test] + public void DecimalNegationZeroEdgeCase() + { + var buffer = new BufferV3(128, 128, 128); + + // Test that -0.0m is treated as positive zero (line 92 in BufferV3.cs) + // The code checks: var negative = (flags & SignMask) != 0 && value.Value != 0m; + // This ensures that -0.0m doesn't get negated + buffer.Table("negation_test") + .Symbol("tag", "zero") + .Column("dec_zero", 0.0m) + .Column("dec_neg_zero", -0.0m) + .At(new DateTime(1970, 01, 01, 0, 0, 1)); + + // Both 0.0m and -0.0m should be encoded as positive zero: 0x00 + DecimalTestHelpers.AssertDecimalField(buffer.GetSendBuffer(), "dec_zero", 1, new byte[] + { + 0x00, + }); + DecimalTestHelpers.AssertDecimalField(buffer.GetSendBuffer(), "dec_neg_zero", 1, new byte[] + { + 0x00, + }); + } + + [Test] + public void DecimalNegationSmallestValue() + { + var buffer = new BufferV3(128, 128, 128); + + // Test negation of the smallest representable positive value: 0.0000000000000000000000000001m + // This has low=1, mid=0, high=0, scale=28 + // After negation: low = ~1 + 1 = 0xFFFFFFFE + 1 = 0xFFFFFFFF + var decimalValue = -0.0000000000000000000000000001m; + + buffer.Table("negation_test") + .Symbol("tag", "smallest") + .Column("dec_smallest", decimalValue) + .At(new DateTime(1970, 01, 01, 0, 0, 1)); + + // Result should be: 0xFF (single byte, compressed) + DecimalTestHelpers.AssertDecimalField(buffer.GetSendBuffer(), "dec_smallest", 28, new byte[] + { + 0xFF, + }); + } + + [Test] + public void DecimalNegationWithHighScale() + { + var buffer = new BufferV3(128, 128, 128); + + // Test negation with high scale value + // -0.00000001m has scale=8 + var decimalValue = -0.00000001m; // -10^-8 + + buffer.Table("negation_test") + .Symbol("tag", "high_scale") + .Column("dec_high_scale", decimalValue) + .At(new DateTime(1970, 01, 01, 0, 0, 1)); + + // -1 with scale 8 = 0xFF in two's complement + DecimalTestHelpers.AssertDecimalField(buffer.GetSendBuffer(), "dec_high_scale", 8, new byte[] + { + 0xFF, + }); + } + + [Test] + public void DecimalNegationBoundaryCarry() + { + var buffer = new BufferV3(128, 128, 128); + + // Test a value where low=0xFFFFFFFF (all ones), which after negation becomes 1 + // This is the value 4294967295 (2^32 - 1) + // After negation: low = ~0xFFFFFFFF + 1 = 0x00000000 + 1 = 0x00000001 + // So -4294967295 should give us: 0xFF, 0xFF, 0xFF, 0xFF, 0x01 + var decimalValue = -4294967295m; + + buffer.Table("negation_test") + .Symbol("tag", "boundary") + .Column("dec_boundary", decimalValue) + .At(new DateTime(1970, 01, 01, 0, 0, 1)); + + // Two's complement of 4294967295: negates to 0xFFFFFFFF00000001 (represented in big-endian) + DecimalTestHelpers.AssertDecimalField(buffer.GetSendBuffer(), "dec_boundary", 0, new byte[] + { + 0xFF, + 0xFF, 0xFF, 0xFF, 0x01, + }); + } + +} \ No newline at end of file diff --git a/src/net-questdb-client-tests/DecimalTestHelpers.cs b/src/net-questdb-client-tests/DecimalTestHelpers.cs new file mode 100644 index 0000000..e28d9f2 --- /dev/null +++ b/src/net-questdb-client-tests/DecimalTestHelpers.cs @@ -0,0 +1,72 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +using System.Text; +using NUnit.Framework; +using QuestDB.Enums; + +namespace net_questdb_client_tests; + +internal static class DecimalTestHelpers +{ + /// + /// Asserts that the buffer contains a decimal field for the specified column with the given scale and mantissa bytes. + /// + /// The encoded row payload to search for the column's decimal field. + /// The name of the column whose decimal payload is expected in the buffer. + /// The expected scale byte of the decimal field. + /// The expected mantissa bytes of the decimal field. + internal static void AssertDecimalField(ReadOnlySpan buffer, + string columnName, + byte expectedScale, + ReadOnlySpan expectedMantissa) + { + var payload = ExtractDecimalPayload(buffer, columnName); + Assert.That(payload.Length, + Is.GreaterThanOrEqualTo(4 + expectedMantissa.Length), + $"Decimal field `{columnName}` payload shorter than expected."); + Assert.That(payload[0], Is.EqualTo((byte)'=')); + Assert.That(payload[1], Is.EqualTo((byte)BinaryFormatType.DECIMAL)); + Assert.That(payload[2], Is.EqualTo(expectedScale), $"Unexpected scale for `{columnName}`."); + Assert.That(payload[3], + Is.EqualTo((byte)expectedMantissa.Length), + $"Unexpected mantissa length for `{columnName}`."); + CollectionAssert.AreEqual(expectedMantissa.ToArray(), payload.Slice(4, expectedMantissa.Length).ToArray(), + $"Mantissa bytes for `{columnName}` did not match expectation."); + } + + /// + /// Locate and return the payload bytes for a decimal column identified by name. + /// + /// The byte span containing the encoded record payload to search. + /// The column name whose payload prefix ("columnName=") will be located. + /// The slice of immediately after the found "columnName=" prefix. + private static ReadOnlySpan ExtractDecimalPayload(ReadOnlySpan buffer, string columnName) + { + var prefix = Encoding.ASCII.GetBytes($"{columnName}="); + var index = buffer.IndexOf(prefix.AsSpan()); + Assert.That(index, Is.GreaterThanOrEqualTo(0), $"Column `{columnName}` not found in payload."); + return buffer[(index + prefix.Length)..]; + } +} \ No newline at end of file diff --git a/src/net-questdb-client-tests/DummyIlpServer.cs b/src/net-questdb-client-tests/DummyIlpServer.cs index e812a6e..feff38e 100644 --- a/src/net-questdb-client-tests/DummyIlpServer.cs +++ b/src/net-questdb-client-tests/DummyIlpServer.cs @@ -24,6 +24,7 @@ using System.Diagnostics; +using System.Globalization; using System.Net; using System.Net.Security; using System.Net.Sockets; @@ -49,9 +50,14 @@ public class DummyIlpServer : IDisposable private string? _publicKeyY; private volatile int _totalReceived; + /// + /// Initializes the dummy ILP server and starts a TCP listener bound to the loopback interface. + /// + /// TCP port to listen on. + /// If true, enables TLS for incoming connections. public DummyIlpServer(int port, bool tls) { - _tls = tls; + _tls = tls; _server = new TcpListener(IPAddress.Loopback, port); _server.Start(); } @@ -69,6 +75,12 @@ public void AcceptAsync() Task.Run(AcceptConnections); } + /// + /// Accepts a single incoming connection, optionally negotiates TLS and performs server authentication, then reads and saves data from the client. + /// + /// + /// Handles one client socket from the listener, wraps the connection with TLS if configured, invokes server-auth when credentials are set, and delegates continuous data receipt to the save routine. Socket errors are caught and the client socket is disposed on exit. + /// private async Task AcceptConnections() { Socket? clientSocket = null; @@ -77,7 +89,7 @@ private async Task AcceptConnections() using var socket = await _server.AcceptSocketAsync(); clientSocket = socket; await using var connection = new NetworkStream(socket, true); - Stream dataStream = connection; + Stream dataStream = connection; if (_tls) { var sslStream = new SslStream(connection); @@ -107,6 +119,11 @@ private X509Certificate GetCertificate() return X509Certificate.CreateFromCertFile("certificate.pfx"); } + /// + /// Performs the server-side authentication handshake over the given stream using a challenge-response ECDSA verification. + /// + /// Stream used for the authentication handshake; the method may write to it and will close it if the requested key id mismatches or the signature verification fails. + /// Thrown when the configured public key coordinates are not set. private async Task RunServerAuth(Stream connection) { var receivedLen = await ReceiveUntilEol(connection); @@ -137,7 +154,7 @@ private async Task RunServerAuth(Stream connection) var pubKey1 = FromBase64String(_publicKeyX); var pubKey2 = FromBase64String(_publicKeyY); - var p = SecNamedCurves.GetByName("secp256r1"); + var p = SecNamedCurves.GetByName("secp256r1"); var parameters = new ECDomainParameters(p.Curve, p.G, p.N, p.H); // Verify the signature @@ -165,20 +182,30 @@ private static string Pad(string text) return text + new string('=', padding); } + /// + /// Decode a Base64 string that may use URL-safe characters and missing padding into its raw byte representation. + /// + /// A Base64-encoded string which may use '-' and '_' instead of '+' and '/' and may omit padding. + /// The decoded bytes represented by the normalized Base64 input. public static byte[] FromBase64String(string encodedPrivateKey) { var replace = encodedPrivateKey - .Replace('-', '+') - .Replace('_', '/'); + .Replace('-', '+') + .Replace('_', '/'); return Convert.FromBase64String(Pad(replace)); } + /// + /// Reads bytes from the provided stream until a newline ('\n') byte is encountered, storing any bytes that follow the newline from the final read into the server's receive buffer. + /// + /// The stream to read incoming bytes from. + /// The index position of the newline byte within the internal read buffer. private async Task ReceiveUntilEol(Stream connection) { var len = 0; while (true) { - var n = await connection.ReadAsync(_buffer.AsMemory(len)); + var n = await connection.ReadAsync(_buffer.AsMemory(len)); var inBuffer = len + n; for (var i = len; i < inBuffer; i++) { @@ -223,89 +250,109 @@ private async Task SaveData(Stream connection, Socket socket) } } + /// + /// Produces a human-readable representation of the data received from the connected client. + /// + /// A formatted string containing the contents of the server's received buffer. public string GetTextReceived() { return PrintBuffer(); - // return Encoding.UTF8.GetString(_received.GetBuffer(), 0, (int)_received.Length); } + /// + /// Gets a copy of all bytes received so far. + /// + /// A byte array containing the raw bytes received up to this point. + public byte[] GetReceivedBytes() + { + return _received.ToArray(); + } + + /// + /// Converts the server's accumulated receive buffer into a human-readable string by decoding UTF-8 text and expanding embedded binary markers into readable representations. + /// + /// + /// The method scans the internal receive buffer for the marker sequence "==". After the marker a type byte determines how the following bytes are interpreted: + /// - type 14: formats a multi-dimensional array of doubles as "ARRAY<dim1,dim2,...>[v1,v2,...]". + /// - type 16: formats a single double value. + /// All bytes outside these marked sections are decoded as UTF-8 text and included verbatim. + /// + /// A formatted string containing the decoded UTF-8 text and expanded representations of any detected binary markers. + /// Thrown when an unknown type marker is encountered after the marker sequence. public string PrintBuffer() { - var bytes = _received.GetBuffer().AsSpan().Slice(0, (int)_received.Length).ToArray(); - var sb = new StringBuilder(); + var bytes = _received.ToArray(); + var sb = new StringBuilder(); var lastAppend = 0; var i = 0; for (; i < bytes.Length; i++) { - if (bytes[i] == (byte)'=') + if (bytes[i] == (byte)'=' && i > 0 && bytes[i - 1] == (byte)'=') { - if (bytes[i - 1] == (byte)'=') + sb.Append(Encoding.UTF8.GetString(bytes, lastAppend, i + 1 - lastAppend)); + switch (bytes[++i]) { - sb.Append(Encoding.UTF8.GetString(bytes, lastAppend, i + 1 - lastAppend)); - switch (bytes[++i]) - { - case 14: - sb.Append("ARRAY<"); - var type = bytes[++i]; + case 14: + sb.Append("ARRAY<"); + var type = bytes[++i]; - Debug.Assert(type == 10); - var dims = bytes[++i]; + Debug.Assert(type == 10); + var dims = bytes[++i]; - ++i; + ++i; - long length = 0; - for (var j = 0; j < dims; j++) + long length = 0; + for (var j = 0; j < dims; j++) + { + var lengthBytes = bytes.AsSpan()[i..(i + 4)]; + var _length = MemoryMarshal.Cast(lengthBytes)[0]; + if (length == 0) { - var lengthBytes = bytes.AsSpan()[i..(i + 4)]; - var _length = MemoryMarshal.Cast(lengthBytes)[0]; - if (length == 0) - { - length = _length; - } - else - { - length *= _length; - } - - sb.Append(_length); - sb.Append(','); - i += 4; + length = _length; } - - sb.Remove(sb.Length - 1, 1); - sb.Append('>'); - - var doubleBytes = - MemoryMarshal.Cast(bytes.AsSpan().Slice(i, (int)(length * 8))); - - - sb.Append('['); - for (var j = 0; j < length; j++) + else { - sb.Append(doubleBytes[j]); - sb.Append(','); + length *= _length; } - sb.Remove(sb.Length - 1, 1); - sb.Append(']'); - - i += (int)(length * 8); - i--; - break; - case 16: - sb.Remove(sb.Length - 1, 1); - var doubleValue = MemoryMarshal.Cast(bytes.AsSpan().Slice(++i, 8)); - sb.Append(doubleValue[0]); - i += 8; - i--; - break; - default: - throw new NotImplementedException(); - } - - lastAppend = i + 1; + sb.Append(_length); + sb.Append(','); + i += 4; + } + + sb.Remove(sb.Length - 1, 1); + sb.Append('>'); + + var doubleBytes = + MemoryMarshal.Cast(bytes.AsSpan().Slice(i, (int)(length * 8))); + + + sb.Append('['); + for (var j = 0; j < length; j++) + { + sb.Append(doubleBytes[j].ToString(CultureInfo.InvariantCulture)); + sb.Append(','); + } + + sb.Remove(sb.Length - 1, 1); + sb.Append(']'); + + i += (int)(length * 8); + i--; + break; + case 16: + sb.Remove(sb.Length - 1, 1); + var doubleValue = MemoryMarshal.Cast(bytes.AsSpan().Slice(++i, 8)); + sb.Append(doubleValue[0].ToString(CultureInfo.InvariantCulture)); + i += 8; + i--; + break; + default: + throw new NotImplementedException($"Type {bytes[i]} not implemented"); } + + lastAppend = i + 1; } } @@ -313,9 +360,15 @@ public string PrintBuffer() return sb.ToString(); } + /// + /// Enables server-side authentication by configuring the expected key identifier and the ECDSA public key coordinates. + /// + /// The key identifier expected from the client during authentication. + /// Base64-encoded X coordinate of the ECDSA public key (secp256r1). + /// Base64-encoded Y coordinate of the ECDSA public key (secp256r1). public void WithAuth(string keyId, string publicKeyX, string publicKeyY) { - _keyId = keyId; + _keyId = keyId; _publicKeyX = publicKeyX; _publicKeyY = publicKeyY; } diff --git a/src/net-questdb-client-tests/HttpTests.cs b/src/net-questdb-client-tests/HttpTests.cs index 580438b..b2e14c2 100644 --- a/src/net-questdb-client-tests/HttpTests.cs +++ b/src/net-questdb-client-tests/HttpTests.cs @@ -1,4 +1,3 @@ -// ReSharper disable CommentTypo /******************************************************************************* * ___ _ ____ ____ * / _ \ _ _ ___ ___| |_| _ \| __ ) @@ -48,33 +47,33 @@ public async Task BasicArrayDouble() Sender.New( $"http::addr={Host}:{HttpPort};username=asdasdada;password=asdadad;tls_verify=unsafe_off;auto_flush=off;"); await sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc") - .Column("array", new[] - { - 1.2, 2.6, - 3.1, - }) - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc") + .Column("array", new[] + { + 1.2, 2.6, + 3.1, + }) + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc") - .Column("array", (ReadOnlySpan)new[] - { - 1.5, 2.1, - 3.1, - }) - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 2)); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc") + .Column("array", (ReadOnlySpan)new[] + { + 1.5, 2.1, + 3.1, + }) + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 2)); await sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc") - .Column("array", (ReadOnlySpan)Array.Empty()) - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 3)); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc") + .Column("array", (ReadOnlySpan)Array.Empty()) + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 3)); await sender.SendAsync(); Assert.That( @@ -85,6 +84,59 @@ await sender.Table("metrics") await server.StopAsync(); } + [Test] + public async Task DecimalColumns() + { + using var server = new DummyHttpServer(withBasicAuth: false); + await server.StartAsync(HttpPort, new[] + { + 1, 2, + 3, + }); + using var sender = + Sender.New( + $"http::addr={Host}:{HttpPort};protocol_version=3;tls_verify=unsafe_off;auto_flush=off;"); + + await sender.Table("metrics") + .Symbol("tag", "value") + .Column("dec_pos", 123.45m) + .Column("dec_neg", -123.45m) + .Column("dec_null", (decimal?)null) + .Column("dec_max", decimal.MaxValue) + .Column("dec_min", decimal.MinValue) + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + + await sender.SendAsync(); + + var buffer = server.GetReceivedBytes().ToArray(); + DecimalTestHelpers.AssertDecimalField(buffer, "dec_pos", 2, new byte[] + { + 0x30, 0x39, + }); + DecimalTestHelpers.AssertDecimalField(buffer, "dec_neg", 2, new byte[] + { + 0xCF, 0xC7, + }); + var prefix = Encoding.UTF8.GetBytes("dec_null="); + Assert.That(buffer.AsSpan().IndexOf(prefix), Is.EqualTo(-1)); + DecimalTestHelpers.AssertDecimalField(buffer, "dec_max", 0, new byte[] + { + 0x00, + 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, + }); + DecimalTestHelpers.AssertDecimalField(buffer, "dec_min", 0, new byte[] + { + 0xFF, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, + }); + + await server.StopAsync(); + } + [Test] public async Task SendLongArrayAsSpan() { @@ -95,12 +147,12 @@ public async Task SendLongArrayAsSpan() $"http::addr={Host}:{HttpPort};init_buf_size=256;username=asdasdada;password=asdadad;tls_verify=unsafe_off;auto_flush=off;"); sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc"); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc"); - var arrayLen = (1024 - sender.Length) / 8 + 1; - var aray = new double[arrayLen]; + var arrayLen = (1024 - sender.Length) / 8 + 1; + var aray = new double[arrayLen]; var expectedArray = new StringBuilder(); for (var i = 0; i < arrayLen; i++) { @@ -114,12 +166,13 @@ public async Task SendLongArrayAsSpan() } await sender.Column("array", (ReadOnlySpan)aray.AsSpan()) - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.SendAsync(); Assert.That( server.PrintBuffer(), - Is.EqualTo($"metrics,tag=value number=10i,string=\"abc\",array==ARRAY<{arrayLen}>[{expectedArray}] 1000000000\n")); + Is.EqualTo( + $"metrics,tag=value number=10i,string=\"abc\",array==ARRAY<{arrayLen}>[{expectedArray}] 1000000000\n")); await server.StopAsync(); } @@ -137,9 +190,9 @@ await server.StartAsync(HttpPort, new[] $"http::addr={Host}:{HttpPort};username=asdasdada;password=asdadad;tls_verify=unsafe_off;auto_flush=off;"); sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc"); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc"); Assert.That( () => sender.Column("array", new[] @@ -155,17 +208,16 @@ await server.StartAsync(HttpPort, new[] using var server = new DummyHttpServer(withBasicAuth: false); await server.StartAsync(HttpPort, new[] { - 3, 4, - 5, + 4, 5, }); using var sender = Sender.New( $"http::addr={Host}:{HttpPort};username=asdasdada;password=asdadad;tls_verify=unsafe_off;auto_flush=off;"); sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc"); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc"); Assert.That( () => sender.Column("array", new[] @@ -197,15 +249,15 @@ public async Task ArrayNegotiationConnectionIsRetried() await delayedStart; await sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc") - .Column("array", new[] - { - 1.2, 2.6, - 3.1, - }) - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc") + .Column("array", new[] + { + 1.2, 2.6, + 3.1, + }) + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.SendAsync(); @@ -227,9 +279,9 @@ public async Task BasicBinaryDouble() Sender.New( $"http::addr={Host}:{HttpPort};username=asdasdada;password=asdadad;tls_verify=unsafe_off;auto_flush=off;"); await sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 12.2) - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("tag", "value") + .Column("number", 12.2) + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.SendAsync(); Assert.That( @@ -247,18 +299,18 @@ public async Task BasicShapedEnumerableDouble() Sender.New( $"http::addr={Host}:{HttpPort};username=asdasdada;password=asdadad;tls_verify=unsafe_off;auto_flush=off;"); await sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc") - .Column("array", new[] - { - 1.2, 2.6, - 3.1, 4.6, - }.AsEnumerable(), new[] - { - 2, 2, - }.AsEnumerable()) - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc") + .Column("array", new[] + { + 1.2, 2.6, + 3.1, 4.6, + }.AsEnumerable(), new[] + { + 2, 2, + }.AsEnumerable()) + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.SendAsync(); Assert.That( @@ -275,9 +327,9 @@ public void InvalidShapedEnumerableDouble() $"http::addr={Host}:{HttpPort};username=asdasdada;password=asdadad;tls_verify=unsafe_off;auto_flush=off;protocol_version=2;"); sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc"); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc"); Assert.That( () => sender.Column("array", new[] @@ -313,15 +365,15 @@ public async Task BasicFlatArray() Sender.New( $"http::addr={Host}:{HttpPort};username=asdasdada;password=asdadad;tls_verify=unsafe_off;auto_flush=off;"); await sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc") - .Column("array", new[] - { - 1.2, 2.6, - 3.1, 4.6, - }) - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc") + .Column("array", new[] + { + 1.2, 2.6, + 3.1, 4.6, + }) + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.SendAsync(); Assert.That( @@ -345,13 +397,13 @@ public async Task BasicMultidimensionalArrayDouble() await server.StartAsync(HttpPort); using var sender = Sender.New( - $"http::addr={Host}:{HttpPort};username=asdasdada;password=asdadad;tls_verify=unsafe_off;auto_flush=off;protocol_version=V2;"); + $"http::addr={Host}:{HttpPort};username=asdasdada;password=asdadad;tls_verify=unsafe_off;auto_flush=off;protocol_version=2;"); await sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc") - .Column("array", arr) - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc") + .Column("array", arr) + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.SendAsync(); Assert.That( @@ -371,10 +423,10 @@ public async Task AuthBasicFailed() Sender.New( $"https::addr={Host}:{HttpsPort};username=asdasdada;password=asdadad;tls_verify=unsafe_off;auto_flush=off;"); await sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); Assert.That( async () => await sender.SendAsync(), @@ -391,10 +443,10 @@ public async Task AuthBasicSuccess() Sender.New( $"https::addr={Host}:{HttpsPort};username=admin;password=quest;tls_verify=unsafe_off;auto_flush=off;"); await sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.SendAsync(); } @@ -411,10 +463,10 @@ public async Task AuthTokenFailed() for (var i = 0; i < 100; i++) await sender - .Table("test") - .Symbol("foo", "bah") - .Column("num", i) - .AtAsync(DateTime.UtcNow); + .Table("test") + .Symbol("foo", "bah") + .Column("num", i) + .AtAsync(DateTime.UtcNow); Assert.That( async () => await sender.SendAsync(), @@ -436,10 +488,10 @@ public async Task AuthTokenSuccess() for (var i = 0; i < 100; i++) await sender - .Table("test") - .Symbol("foo", "bah") - .Column("num", i) - .AtAsync(DateTime.UtcNow); + .Table("test") + .Symbol("foo", "bah") + .Column("num", i) + .AtAsync(DateTime.UtcNow); await sender.SendAsync(); } @@ -451,10 +503,10 @@ public async Task BasicSend() using var server = new DummyHttpServer(); await server.StartAsync(HttpPort); var sender = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;"); - var ts = DateTime.UtcNow; + var ts = DateTime.UtcNow; await sender.Table("name") - .Column("ts", ts) - .AtAsync(ts); + .Column("ts", ts) + .AtAsync(ts); await sender.SendAsync(); Console.WriteLine(server.GetReceiveBuffer().ToString()); await server.StopAsync(); @@ -468,7 +520,7 @@ public void SendBadSymbol() { var sender = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;protocol_version=1;"); sender.Table("metric name") - .Symbol("t ,a g", "v alu, e"); + .Symbol("t ,a g", "v alu, e"); }, Throws.TypeOf().With.Message.Contains("Column names") ); @@ -482,7 +534,7 @@ public void SendBadColumn() { var sender = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;protocol_version=1;"); sender.Table("metric name") - .Column("t a, g", "v alu e"); + .Column("t a, g", "v alu e"); }, Throws.TypeOf().With.Message.Contains("Column names") ); @@ -495,10 +547,10 @@ public async Task SendLine() await server.StartAsync(HttpPort); var sender = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;"); await sender.Table("metrics") - .Symbol("tag", "value") - .Column("number", 10) - .Column("string", "abc") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("tag", "value") + .Column("number", 10) + .Column("string", "abc") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.SendAsync(); Assert.That( @@ -523,12 +575,12 @@ public async Task SendLineExceedsBuffer() for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -547,17 +599,17 @@ public async Task SendLineExceedsBufferLimit() Sender.New($"http::addr={Host}:{HttpPort};init_buf_size=1024;max_buf_size=2048;auto_flush=off;"); Assert.That(async () => - { - for (var i = 0; i < 500; i++) - await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); - }, - Throws.Exception.With.Message.Contains("maximum buffer size")); + { + for (var i = 0; i < 500; i++) + await sender.Table("table name") + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + }, + Throws.Exception.With.Message.Contains("maximum buffer size")); } [Test] @@ -575,12 +627,12 @@ public async Task SendLineReusesBuffer() for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -589,12 +641,12 @@ await sender.Table("table name") for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -618,12 +670,12 @@ public async Task SendLineTrimsBuffers() for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -633,12 +685,12 @@ await sender.Table("table name") for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -663,18 +715,18 @@ public async Task ServerDisconnects() for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); if (i > 1) { Assert.That(async () => await sender.SendAsync(), - Throws.TypeOf()); + Throws.TypeOf()); break; } @@ -695,11 +747,11 @@ public async Task SendNegativeLongAndDouble() using var ls = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;"); await ls.Table("neg name") - .Column("number1", long.MinValue + 1) - .Column("number2", long.MaxValue) - .Column("number3", double.MinValue) - .Column("number4", double.MaxValue) - .AtAsync(86400000000000); + .Column("number1", long.MinValue + 1) + .Column("number2", long.MaxValue) + .Column("number3", double.MinValue) + .Column("number4", double.MaxValue) + .AtAsync(86400000000000); await ls.SendAsync(); var expected = @@ -717,22 +769,22 @@ public async Task SerialiseDoublesV2() using var ls = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;"); await ls.Table("doubles") - .Column("d0", 0.0) - .Column("dm0", -0.0) - .Column("d1", 1.0) - .Column("dE100", 1E100) - .Column("d0000001", 0.000001) - .Column("dNaN", double.NaN) - .Column("dInf", double.PositiveInfinity) - .Column("dNInf", double.NegativeInfinity) - .AtAsync(86400000000000); + .Column("d0", 0.0) + .Column("dm0", -0.0) + .Column("d1", 1.0) + .Column("dE100", 1E100) + .Column("d0000001", 0.000001) + .Column("dNaN", double.NaN) + .Column("dInf", double.PositiveInfinity) + .Column("dNInf", double.NegativeInfinity) + .AtAsync(86400000000000); await ls.SendAsync(); var expected = "doubles d0=0,dm0=-0,d1=1,dE100=1E+100,d0000001=1E-06,dNaN=NaN,dInf=Infinity,dNInf=-Infinity 86400000000000\n"; Assert.That(srv.PrintBuffer(), Is.EqualTo(expected)); } - + [Test] public async Task SerialiseDoublesV1() { @@ -742,15 +794,15 @@ public async Task SerialiseDoublesV1() using var ls = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;protocol_version=1;"); await ls.Table("doubles") - .Column("d0", 0.0) - .Column("dm0", -0.0) - .Column("d1", 1.0) - .Column("dE100", 1E100) - .Column("d0000001", 0.000001) - .Column("dNaN", double.NaN) - .Column("dInf", double.PositiveInfinity) - .Column("dNInf", double.NegativeInfinity) - .AtAsync(86400000000000); + .Column("d0", 0.0) + .Column("dm0", -0.0) + .Column("d1", 1.0) + .Column("dE100", 1E100) + .Column("d0000001", 0.000001) + .Column("dNaN", double.NaN) + .Column("dInf", double.PositiveInfinity) + .Column("dNInf", double.NegativeInfinity) + .AtAsync(86400000000000); await ls.SendAsync(); var expected = @@ -767,8 +819,8 @@ public async Task SendTimestampColumn() var ts = new DateTime(2022, 2, 24); await sender.Table("name") - .Column("ts", ts) - .AtAsync(ts); + .Column("ts", ts) + .AtAsync(ts); await sender.SendAsync(); @@ -786,8 +838,8 @@ public async Task SendColumnNanos() const long timestampNanos = 1645660800123456789L; await sender.Table("name") - .ColumnNanos("ts", timestampNanos) - .AtAsync(timestampNanos); + .ColumnNanos("ts", timestampNanos) + .AtAsync(timestampNanos); await sender.SendAsync(); @@ -805,8 +857,8 @@ public async Task SendAtNanos() const long timestampNanos = 1645660800987654321L; await sender.Table("name") - .Column("value", 42) - .AtNanosAsync(timestampNanos); + .Column("value", 42) + .AtNanosAsync(timestampNanos); await sender.SendAsync(); @@ -820,8 +872,8 @@ public async Task InvalidState() { using var srv = new DummyHttpServer(); await srv.StartAsync(HttpPort); - using var sender = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;"); - string? nullString = null; + using var sender = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;"); + string? nullString = null; Assert.That( () => sender.Table(nullString), @@ -921,8 +973,8 @@ public async Task InvalidTableName() { using var srv = new DummyHttpServer(); await srv.StartAsync(HttpPort); - using var sender = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;"); - string? nullString = null; + using var sender = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;"); + string? nullString = null; Assert.Throws(() => sender.Table(nullString)); Assert.Throws(() => sender.Column("abc", 123)); @@ -949,19 +1001,19 @@ public async Task SendMillionAsyncExplicit() $"http::addr={Host}:{HttpPort};init_buf_size={256 * 1024};auto_flush=off;request_timeout=30000;"); var nowMillisecond = DateTime.Now.Millisecond; - var metric = "metric_name" + nowMillisecond; + var metric = "metric_name" + nowMillisecond; Assert.True(await srv.Healthcheck()); for (var i = 0; i < 1E6; i++) { await sender.Table(metric) - .Symbol("nopoint", "tag" + i % 100) - .Column("counter", i * 1111.1) - .Column("int", i) - .Column("привед", "мед вед") - .AtAsync(new DateTime(2021, 1, 1, i / 360 / 1000 % 60, i / 60 / 1000 % 60, i / 1000 % 60, - i % 1000)); + .Symbol("nopoint", "tag" + i % 100) + .Column("counter", i * 1111.1) + .Column("int", i) + .Column("привед", "мед вед") + .AtAsync(new DateTime(2021, 1, 1, i / 360 / 1000 % 60, i / 60 / 1000 % 60, i / 1000 % 60, + i % 1000)); if (i % 100 == 0) { @@ -979,7 +1031,7 @@ public async Task SendMillionFixedBuffer() await srv.StartAsync(HttpPort); var nowMillisecond = DateTime.Now.Millisecond; - var metric = "metric_name" + nowMillisecond; + var metric = "metric_name" + nowMillisecond; Assert.True(await srv.Healthcheck()); @@ -989,12 +1041,12 @@ public async Task SendMillionFixedBuffer() for (var i = 0; i < 1E6; i++) await sender.Table(metric) - .Symbol("nopoint", "tag" + i % 100) - .Column("counter", i * 1111.1) - .Column("int", i) - .Column("привед", "мед вед") - .AtAsync(new DateTime(2021, 1, 1, i / 360 / 1000 % 60, i / 60 / 1000 % 60, i / 1000 % 60, - i % 1000)); + .Symbol("nopoint", "tag" + i % 100) + .Column("counter", i * 1111.1) + .Column("int", i) + .Column("привед", "мед вед") + .AtAsync(new DateTime(2021, 1, 1, i / 360 / 1000 % 60, i / 60 / 1000 % 60, i / 1000 % 60, + i % 1000)); await sender.SendAsync(); } @@ -1010,8 +1062,8 @@ public async Task SendNegativeLongMin() $"http::addr={Host}:{HttpPort};auto_flush=off;"); Assert.That( () => sender.Table("name") - .Column("number1", long.MinValue) - .AtAsync(DateTime.UtcNow), + .Column("number1", long.MinValue) + .AtAsync(DateTime.UtcNow), Throws.TypeOf().With.Message.Contains("Special case") ); } @@ -1026,8 +1078,8 @@ public async Task SendSpecialStrings() Sender.New( $"http::addr={Host}:{HttpPort};auto_flush=off;"); await sender.Table("neg name") - .Column("привед", " мед\rве\n д") - .AtAsync(86400000000000); + .Column("привед", " мед\rве\n д") + .AtAsync(86400000000000); await sender.SendAsync(); var expected = "neg\\ name привед=\" мед\\\rве\\\n д\" 86400000000000\n"; @@ -1045,9 +1097,9 @@ public async Task SendTagAfterField() $"http::addr={Host}:{HttpPort};auto_flush=off;"); Assert.That( async () => await sender.Table("name") - .Column("number1", 123) - .Symbol("nand", "asdfa") - .AtAsync(DateTime.UtcNow), + .Column("number1", 123) + .Symbol("nand", "asdfa") + .AtAsync(DateTime.UtcNow), Throws.TypeOf() ); } @@ -1065,9 +1117,9 @@ public async Task SendMetricOnce() Assert.That( async () => await sender.Table("name") - .Column("number1", 123) - .Table("nand") - .AtAsync(DateTime.UtcNow), + .Column("number1", 123) + .Table("nand") + .AtAsync(DateTime.UtcNow), Throws.TypeOf() ); } @@ -1220,7 +1272,8 @@ public async Task TransactionMultipleTypes() await sender.Column("foo", 123).AtAsync(86400000000000); await sender.Column("foo", 123d).AtAsync(86400000000000); await sender.Column("foo", new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).AtAsync(86400000000000); - await sender.Column("foo", new DateTimeOffset(new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc))).AtAsync(86400000000000); + await sender.Column("foo", new DateTimeOffset(new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc))) + .AtAsync(86400000000000); await sender.Column("foo", false).AtAsync(86400000000000); await sender.CommitAsync(); @@ -1288,9 +1341,9 @@ public async Task TransactionShouldNotBeAutoFlushed() sender.Transaction("tableName"); for (var i = 0; i < 100; i++) await sender - .Symbol("foo", "bah") - .Column("num", i) - .AtAsync(DateTime.UtcNow); + .Symbol("foo", "bah") + .Column("num", i) + .AtAsync(DateTime.UtcNow); Assert.That(sender.RowCount == 100); Assert.That(sender.WithinTransaction); @@ -1314,9 +1367,9 @@ public async Task TransactionRequiresCommitToComplete() sender.Transaction("tableName"); for (var i = 0; i < 100; i++) await sender - .Symbol("foo", "bah") - .Column("num", i) - .AtAsync(DateTime.UtcNow); + .Symbol("foo", "bah") + .Column("num", i) + .AtAsync(DateTime.UtcNow); Assert.That( async () => await sender.SendAsync(), @@ -1333,9 +1386,9 @@ await sender sender.Transaction("tableName"); for (var i = 0; i < 100; i++) await sender - .Symbol("foo", "bah") - .Column("num", i) - .AtAsync(DateTime.UtcNow); + .Symbol("foo", "bah") + .Column("num", i) + .AtAsync(DateTime.UtcNow); sender.Commit(); } @@ -1501,8 +1554,8 @@ public async Task SendTimestampColumns() using var sender = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;"); sender.Table("foo") - .Symbol("bah", "baz") - .Column("ts1", DateTime.UtcNow).Column("ts2", DateTimeOffset.UtcNow); + .Symbol("bah", "baz") + .Column("ts1", DateTime.UtcNow).Column("ts2", DateTimeOffset.UtcNow); await sender.SendAsync(); } @@ -1515,36 +1568,36 @@ public async Task SendVariousAts() using var sender = Sender.New($"http::addr={Host}:{HttpPort};auto_flush=off;"); await sender.Table("foo") - .Symbol("bah", "baz") - .AtAsync(DateTime.UtcNow); + .Symbol("bah", "baz") + .AtAsync(DateTime.UtcNow); await sender.Table("foo") - .Symbol("bah", "baz") - .AtAsync(DateTime.UtcNow); + .Symbol("bah", "baz") + .AtAsync(DateTime.UtcNow); await sender.Table("foo") - .Symbol("bah", "baz") - .AtAsync(DateTimeOffset.UtcNow); + .Symbol("bah", "baz") + .AtAsync(DateTimeOffset.UtcNow); await sender.Table("foo") - .Symbol("bah", "baz") - .AtAsync(DateTime.UtcNow.Ticks / 100); + .Symbol("bah", "baz") + .AtAsync(DateTime.UtcNow.Ticks / 100); sender.Table("foo") - .Symbol("bah", "baz") - .At(DateTime.UtcNow); + .Symbol("bah", "baz") + .At(DateTime.UtcNow); sender.Table("foo") - .Symbol("bah", "baz") - .At(DateTime.UtcNow); + .Symbol("bah", "baz") + .At(DateTime.UtcNow); sender.Table("foo") - .Symbol("bah", "baz") - .At(DateTimeOffset.UtcNow); + .Symbol("bah", "baz") + .At(DateTimeOffset.UtcNow); sender.Table("foo") - .Symbol("bah", "baz") - .At(DateTime.UtcNow.Ticks / 100); + .Symbol("bah", "baz") + .At(DateTime.UtcNow.Ticks / 100); await sender.SendAsync(); } @@ -1600,12 +1653,12 @@ public async Task SendManyRequests() for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", i) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(DateTime.UtcNow); + .Symbol("t a g", "v alu, e") + .Column("number", i) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(DateTime.UtcNow); var request = sender.SendAsync(); @@ -1668,4 +1721,4 @@ public async Task FailsWhenExpectingCert() await server.StopAsync(); } -} +} \ No newline at end of file diff --git a/src/net-questdb-client-tests/JsonSpecTestRunner.cs b/src/net-questdb-client-tests/JsonSpecTestRunner.cs index 514ee47..f8093e9 100644 --- a/src/net-questdb-client-tests/JsonSpecTestRunner.cs +++ b/src/net-questdb-client-tests/JsonSpecTestRunner.cs @@ -23,12 +23,15 @@ ******************************************************************************/ +using System.Globalization; using System.Net; using System.Text; using System.Text.Json; +using System.Text.Json.Serialization; using dummy_http_server; using NUnit.Framework; using QuestDB; +using QuestDB.Senders; // ReSharper disable InconsistentNaming @@ -41,6 +44,68 @@ public class JsonSpecTestRunner private const int HttpPort = 29473; private static readonly TestCase[]? TestCases = ReadTestCases(); + /// + /// Populate the provided sender with the test case's table, symbols, and columns, then send the prepared row. + /// + /// The ISender to configure and use for sending the test case row. + /// The test case containing table name, symbols, and typed columns to write. + /// A task that completes when the prepared row has been sent. + private static async Task ExecuteTestCase(ISender sender, TestCase testCase) + { + sender.Table(testCase.Table); + foreach (var symbol in testCase.Symbols) + { + sender.Symbol(symbol.Name, symbol.Value); + } + + foreach (var column in testCase.Columns) + { + switch (column.Type) + { + case "STRING": + sender.Column(column.Name, ((JsonElement)column.Value).GetString()); + break; + + case "DOUBLE": + sender.Column(column.Name, ((JsonElement)column.Value).GetDouble()); + break; + + case "BOOLEAN": + sender.Column(column.Name, ((JsonElement)column.Value).GetBoolean()); + break; + + case "LONG": + sender.Column(column.Name, ((JsonElement)column.Value).GetInt64()); + break; + + case "DECIMAL": + var value = ((JsonElement)column.Value).GetString(); + if (value is null) + { + sender.Column(column.Name, (decimal?)null); + } + else + { + var d = decimal.Parse(value, NumberStyles.Number, CultureInfo.InvariantCulture); + sender.Column(column.Name, d); + } + break; + + default: + throw new NotSupportedException("Column type not supported: " + column.Type); + } + } + +#pragma warning disable CS0618 // Type or member is obsolete + await sender.AtNowAsync(); +#pragma warning restore CS0618 // Type or member is obsolete + await sender.SendAsync(); + } + + /// + /// Executes the provided test case by sending its configured table, symbols, and columns to a local TCP listener and asserting the listener's received output against the test case's expected result. + /// + /// The test case to run; provides table, symbols, columns to send and a Result describing the expected validation (Status, Line, AnyLines, or BinaryBase64). [TestCaseSource(nameof(TestCases))] public async Task RunTcp(TestCase testCase) { @@ -48,51 +113,17 @@ public async Task RunTcp(TestCase testCase) srv.AcceptAsync(); using var sender = Sender.New( - $"tcp::addr={IPAddress.Loopback}:{TcpPort};"); + $"tcp::addr={IPAddress.Loopback}:{TcpPort};protocol_version=3"); Exception? exception = null; try { - sender.Table(testCase.table); - foreach (var symbol in testCase.symbols) - { - sender.Symbol(symbol.name, symbol.value); - } - - foreach (var column in testCase.columns) - { - switch (column.type) - { - case "STRING": - sender.Column(column.name, ((JsonElement)column.value).GetString()); - break; - - case "DOUBLE": - sender.Column(column.name, ((JsonElement)column.value).GetDouble()); - break; - - case "BOOLEAN": - sender.Column(column.name, ((JsonElement)column.value).GetBoolean()); - break; - - case "LONG": - sender.Column(column.name, (long)((JsonElement)column.value).GetDouble()); - break; - - default: - throw new NotSupportedException("Column type not supported: " + column.type); - } - } - -#pragma warning disable CS0618 // Type or member is obsolete - await sender.AtNowAsync(); -#pragma warning restore CS0618 // Type or member is obsolete - await sender.SendAsync(); + await ExecuteTestCase(sender, testCase); } catch (Exception? ex) { - if (testCase.result.status == "SUCCESS") + if (testCase.Result.Status == "SUCCESS") { throw; } @@ -100,18 +131,23 @@ public async Task RunTcp(TestCase testCase) exception = ex; } - if (testCase.result.status == "SUCCESS") + if (testCase.Result.Status == "SUCCESS") { - if (testCase.result.anyLines == null || testCase.result.anyLines.Length == 0) + if (testCase.Result.BinaryBase64 != null) { - WaitAssert(srv, testCase.result.line + "\n"); + var expected = Convert.FromBase64String(testCase.Result.BinaryBase64); + WaitAssert(srv, expected); + } + else if (testCase.Result.AnyLines == null || testCase.Result.AnyLines.Length == 0) + { + WaitAssert(srv, testCase.Result.Line + "\n"); } else { - WaitAssert(srv, testCase.result.anyLines); + WaitAssert(srv, testCase.Result.AnyLines); } } - else if (testCase.result.status == "ERROR") + else if (testCase.Result.Status == "ERROR") { Assert.NotNull(exception, "Exception should be thrown"); if (exception is NotSupportedException) @@ -121,10 +157,14 @@ public async Task RunTcp(TestCase testCase) } else { - Assert.Fail("Unsupported test case result status: " + testCase.result.status); + Assert.Fail("Unsupported test case result status: " + testCase.Result.Status); } } + /// + /// Executes the provided test case by sending data over HTTP to a dummy server using a QuestDB sender and validates the server's response according to the test case result. + /// + /// The test case describing table, symbols, columns, and expected result (status, line(s), or base64 binary) to execute and validate. [TestCaseSource(nameof(TestCases))] public async Task RunHttp(TestCase testCase) { @@ -134,52 +174,18 @@ public async Task RunHttp(TestCase testCase) Assert.That(await server.Healthcheck()); using var sender = Sender.New( - $"http::addr={IPAddress.Loopback}:{HttpPort};"); + $"http::addr={IPAddress.Loopback}:{HttpPort};protocol_version=3"); Exception? exception = null; try { - sender.Table(testCase.table); - foreach (var symbol in testCase.symbols) - { - sender.Symbol(symbol.name, symbol.value); - } - - foreach (var column in testCase.columns) - { - switch (column.type) - { - case "STRING": - sender.Column(column.name, ((JsonElement)column.value).GetString()); - break; - - case "DOUBLE": - sender.Column(column.name, ((JsonElement)column.value).GetDouble()); - break; - - case "BOOLEAN": - sender.Column(column.name, ((JsonElement)column.value).GetBoolean()); - break; - - case "LONG": - sender.Column(column.name, (long)((JsonElement)column.value).GetDouble()); - break; - - default: - throw new NotSupportedException("Column type not supported: " + column.type); - } - } - -#pragma warning disable CS0618 // Type or member is obsolete - await sender.AtNowAsync(); -#pragma warning restore CS0618 // Type or member is obsolete - await sender.SendAsync(); + await ExecuteTestCase(sender, testCase); } catch (Exception? ex) { TestContext.Write(server.GetLastError()); - if (testCase.result.status == "SUCCESS") + if (testCase.Result.Status == "SUCCESS") { throw; } @@ -187,19 +193,26 @@ public async Task RunHttp(TestCase testCase) exception = ex; } - if (testCase.result.status == "SUCCESS") + if (testCase.Result.Status == "SUCCESS") { - var textReceived = server.PrintBuffer(); - if (testCase.result.anyLines == null || testCase.result.anyLines.Length == 0) + if (testCase.Result.BinaryBase64 != null) + { + var received = server.GetReceivedBytes(); + var expected = Convert.FromBase64String(testCase.Result.BinaryBase64); + Assert.That(received, Is.EqualTo(expected)); + } + else if (testCase.Result.AnyLines == null || testCase.Result.AnyLines.Length == 0) { - Assert.That(textReceived, Is.EqualTo(testCase.result.line + "\n")); + var textReceived = server.PrintBuffer(); + Assert.That(textReceived, Is.EqualTo(testCase.Result.Line + "\n")); } else { - AssertMany(testCase.result.anyLines, textReceived); + var textReceived = server.PrintBuffer(); + AssertMany(testCase.Result.AnyLines, textReceived); } } - else if (testCase.result.status == "ERROR") + else if (testCase.Result.Status == "ERROR") { Assert.NotNull(exception, "Exception should be thrown"); if (exception is NotSupportedException) @@ -209,7 +222,7 @@ public async Task RunHttp(TestCase testCase) } else { - Assert.Fail("Unsupported test case result status: " + testCase.result.status); + Assert.Fail("Unsupported test case result status: " + testCase.Result.Status); } } @@ -245,6 +258,17 @@ private static void WaitAssert(DummyIlpServer srv, string expected) Assert.That(srv.GetTextReceived(), Is.EqualTo(expected)); } + private static void WaitAssert(DummyIlpServer srv, byte[] expected) + { + var expectedLen = expected.Length; + for (var i = 0; i < 500 && srv.TotalReceived < expectedLen; i++) + { + Thread.Sleep(10); + } + + Assert.That(srv.GetReceivedBytes(), Is.EqualTo(expected)); + } + private DummyIlpServer CreateTcpListener(int port, bool tls = false) { return new DummyIlpServer(port, tls); @@ -258,35 +282,44 @@ private DummyIlpServer CreateTcpListener(int port, bool tls = false) public class TestCase { - public string testName { get; set; } = null!; - public string table { get; set; } = null!; - public TestCaseSymbol[] symbols { get; set; } = null!; - public TestCaseColumn[] columns { get; set; } = null!; - public TestCaseResult result { get; set; } = null!; + [JsonPropertyName("testName")] public string TestName { get; set; } = null!; + [JsonPropertyName("table")] public string Table { get; set; } = null!; + + [JsonPropertyName("minimumProtocolVersion")] + public int? MinimumProtocolVersion { get; set; } + + [JsonPropertyName("symbols")] public TestCaseSymbol[] Symbols { get; set; } = null!; + [JsonPropertyName("columns")] public TestCaseColumn[] Columns { get; set; } = null!; + [JsonPropertyName("result")] public TestCaseResult Result { get; set; } = null!; + /// + /// Provides the test case name for display and logging. + /// + /// The TestName of the test case. public override string ToString() { - return testName; + return TestName; } } public class TestCaseSymbol { - public string name { get; set; } = null!; - public string value { get; set; } = null!; + [JsonPropertyName("name")] public string Name { get; set; } = null!; + [JsonPropertyName("value")] public string Value { get; set; } = null!; } public class TestCaseColumn { - public string type { get; set; } = null!; - public string name { get; set; } = null!; - public object value { get; set; } = null!; + [JsonPropertyName("type")] public string Type { get; set; } = null!; + [JsonPropertyName("name")] public string Name { get; set; } = null!; + [JsonPropertyName("value")] public object Value { get; set; } = null!; } public class TestCaseResult { - public string status { get; set; } = null!; - public string line { get; set; } = null!; - public string[]? anyLines { get; set; } = null!; + [JsonPropertyName("status")] public string Status { get; set; } = null!; + [JsonPropertyName("line")] public string Line { get; set; } = null!; + [JsonPropertyName("anyLines")] public string[]? AnyLines { get; set; } = null!; + [JsonPropertyName("binaryBase64")] public string? BinaryBase64 { get; set; } } } \ No newline at end of file diff --git a/src/tcp-client-test/LineTcpSenderTests.cs b/src/net-questdb-client-tests/LineTcpSenderTests.cs similarity index 80% rename from src/tcp-client-test/LineTcpSenderTests.cs rename to src/net-questdb-client-tests/LineTcpSenderTests.cs index c30d61f..efa72d6 100644 --- a/src/tcp-client-test/LineTcpSenderTests.cs +++ b/src/net-questdb-client-tests/LineTcpSenderTests.cs @@ -40,7 +40,7 @@ #pragma warning disable CS0612 // Type or member is obsolete #pragma warning disable CS0618 // Type or member is obsolete -namespace tcp_client_test; +namespace net_questdb_client_tests; [TestFixture] public class LineTcpSenderTests @@ -56,10 +56,10 @@ public async Task SendLine() using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, tlsMode: TlsMode.Disable); ls.Table("metric name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("string", " -=\"") - .At(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("string", " -=\"") + .At(new DateTime(1970, 01, 01, 0, 0, 1)); ls.Send(); @@ -72,17 +72,17 @@ public async Task Authenticate() { using var srv = CreateTcpListener(_port); srv.WithAuth("testUser1", "Vs4e-cOLsVCntsMrZiAGAZtrkPXO00uoRLuA3d7gEcI=", - "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); + "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); srv.AcceptAsync(); using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, tlsMode: TlsMode.Disable); await ls.AuthenticateAsync("testUser1", "NgdiOWDoQNUP18WOnb1xkkEG5TzPYMda5SiUOvT1K0U=", CancellationToken.None); ls.Table("metric name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("string", " -=\"") - .At(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("string", " -=\"") + .At(new DateTime(1970, 01, 01, 0, 0, 1)); ls.Send(); var expected = "metric\\ name,t\\ a\\ g=v\\ alu\\,\\ e number=10i,string=\" -=\\\"\" 1000000000\n"; @@ -94,7 +94,7 @@ public async Task AuthFailWrongKid() { using var srv = CreateTcpListener(_port); srv.WithAuth("testUser1", "Vs4e-cOLsVCntsMrZiAGAZtrkPXO00uoRLuA3d7gEcI=", - "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); + "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); srv.AcceptAsync(); using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, tlsMode: TlsMode.Disable); @@ -113,7 +113,7 @@ public async Task AuthFailWrongKid() public void EcdsaSingnatureLoop() { var privateKey = Convert.FromBase64String("NgdiOWDoQNUP18WOnb1xkkEG5TzPYMda5SiUOvT1K0U="); - var p = SecNamedCurves.GetByName("secp256r1"); + var p = SecNamedCurves.GetByName("secp256r1"); var parameters = new ECDomainParameters(p.Curve, p.G, p.N, p.H); var priKey = new ECPrivateKeyParameters( "ECDSA", @@ -164,12 +164,12 @@ public async Task SendLineExceedsBuffer() for (var i = 0; i < lineCount; i++) { ls.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .At(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .At(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -187,8 +187,8 @@ public async Task SendLineReusesBuffers() srv.AcceptAsync(); using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, 2048, - tlsMode: TlsMode.Disable, - bufferOverflowHandling: BufferOverflowHandling.Extend); + tlsMode: TlsMode.Disable, + bufferOverflowHandling: BufferOverflowHandling.Extend); var lineCount = 500; var expected = "table\\ name,t\\ a\\ g=v\\ alu\\,\\ e number=10i,db\\ l=123.12,string=\" -=\\\"\",при\\ вед=\"медвед\" 1000000000\n"; @@ -196,12 +196,12 @@ public async Task SendLineReusesBuffers() for (var i = 0; i < lineCount; i++) { ls.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .At(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .At(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -210,12 +210,12 @@ public async Task SendLineReusesBuffers() for (var i = 0; i < lineCount; i++) { ls.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .At(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .At(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -240,12 +240,12 @@ public async Task SendLineTrimsBuffers() for (var i = 0; i < lineCount; i++) { ls.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .At(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .At(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -255,12 +255,12 @@ public async Task SendLineTrimsBuffers() for (var i = 0; i < lineCount; i++) { ls.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .At(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .At(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -291,12 +291,12 @@ await LineTcpSender.ConnectAsync( for (var i = 0; i < lineCount; i++) { ls.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .At(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .At(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); try { @@ -323,11 +323,11 @@ public async Task SendNegativeLongAndDouble() using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, tlsMode: TlsMode.Disable); ls.Table("neg name") - .Column("number1", long.MinValue + 1) - .Column("number2", long.MaxValue) - .Column("number3", double.MinValue) - .Column("number4", double.MaxValue) - .AtNow(); + .Column("number1", long.MinValue + 1) + .Column("number2", long.MaxValue) + .Column("number3", double.MinValue) + .Column("number4", double.MaxValue) + .AtNow(); ls.Send(); var expected = @@ -344,15 +344,15 @@ public async Task DoubleSerializationTest() using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, tlsMode: TlsMode.Disable); ls.Table("doubles") - .Column("d0", 0.0) - .Column("dm0", -0.0) - .Column("d1", 1.0) - .Column("dE100", 1E100) - .Column("d0000001", 0.000001) - .Column("dNaN", double.NaN) - .Column("dInf", double.PositiveInfinity) - .Column("dNInf", double.NegativeInfinity) - .AtNow(); + .Column("d0", 0.0) + .Column("dm0", -0.0) + .Column("d1", 1.0) + .Column("dE100", 1E100) + .Column("d0000001", 0.000001) + .Column("dNaN", double.NaN) + .Column("dInf", double.PositiveInfinity) + .Column("dNInf", double.NegativeInfinity) + .AtNow(); ls.Send(); var expected = @@ -370,8 +370,8 @@ public async Task SendTimestampColumn() var ts = new DateTime(2022, 2, 24); ls.Table("name") - .Column("ts", ts) - .At(ts); + .Column("ts", ts) + .At(ts); ls.Send(); @@ -385,18 +385,18 @@ public async Task WithTls() { using var srv = CreateTcpListener(_port, true); srv.WithAuth("testUser1", "Vs4e-cOLsVCntsMrZiAGAZtrkPXO00uoRLuA3d7gEcI=", - "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); + "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); srv.AcceptAsync(); using var ls = await LineTcpSender.ConnectAsync("localhost", _port, tlsMode: TlsMode.AllowAnyServerCertificate); await ls.AuthenticateAsync("testUser1", "NgdiOWDoQNUP18WOnb1xkkEG5TzPYMda5SiUOvT1K0U="); ls.Table("table_name") - .Column("number1", long.MinValue + 1) - .Column("number2", long.MaxValue) - .Column("number3", double.MinValue) - .Column("number4", double.MaxValue) - .AtNow(); + .Column("number1", long.MinValue + 1) + .Column("number2", long.MaxValue) + .Column("number3", double.MinValue) + .Column("number4", double.MaxValue) + .AtNow(); await ls.SendAsync(); @@ -412,7 +412,7 @@ public async Task InvalidState() srv.AcceptAsync(); using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, tlsMode: TlsMode.Disable); - string? nullString = null; + string? nullString = null; Assert.Throws(() => ls.Table(nullString)); Assert.Throws(() => ls.Column("abc", 123)); @@ -488,7 +488,7 @@ public async Task InvalidTableName() srv.AcceptAsync(); using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, tlsMode: TlsMode.Disable); - string? nullString = null; + string? nullString = null; Assert.Throws(() => ls.Table(nullString)); Assert.Throws(() => ls.Column("abc", 123)); @@ -540,18 +540,18 @@ public async Task SendMillionAsyncExplicit() srv.AcceptAsync(); var nowMillisecond = DateTime.Now.Millisecond; - var metric = "metric_name" + nowMillisecond; + var metric = "metric_name" + nowMillisecond; using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, 256 * 1024, - tlsMode: TlsMode.Disable); + tlsMode: TlsMode.Disable); for (var i = 0; i < 1E6; i++) { ls.Table(metric) - .Symbol("nopoint", "tag" + i % 100) - .Column("counter", i * 1111.1) - .Column("int", i) - .Column("привед", "мед вед") - .At(new DateTime(2021, 1, 1, i / 360 / 1000 % 60, i / 60 / 1000 % 60, i / 1000 % 60, i % 1000)); + .Symbol("nopoint", "tag" + i % 100) + .Column("counter", i * 1111.1) + .Column("int", i) + .Column("привед", "мед вед") + .At(new DateTime(2021, 1, 1, i / 360 / 1000 % 60, i / 60 / 1000 % 60, i / 1000 % 60, i % 1000)); if (i % 100 == 0) { @@ -569,18 +569,18 @@ public async Task SendMillionFixedBuffer() srv.AcceptAsync(); var nowMillisecond = DateTime.Now.Millisecond; - var metric = "metric_name" + nowMillisecond; + var metric = "metric_name" + nowMillisecond; using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, 64 * 1024, - BufferOverflowHandling.SendImmediately, TlsMode.Disable); + BufferOverflowHandling.SendImmediately, TlsMode.Disable); for (var i = 0; i < 1E6; i++) { ls.Table(metric) - .Symbol("nopoint", "tag" + i % 100) - .Column("counter", i * 1111.1) - .Column("int", i) - .Column("привед", "мед вед") - .At(new DateTime(2021, 1, 1, i / 360 / 1000 % 60, i / 60 / 1000 % 60, i / 1000 % 60, i % 1000)); + .Symbol("nopoint", "tag" + i % 100) + .Column("counter", i * 1111.1) + .Column("int", i) + .Column("привед", "мед вед") + .At(new DateTime(2021, 1, 1, i / 360 / 1000 % 60, i / 60 / 1000 % 60, i / 1000 % 60, i % 1000)); } await ls.SendAsync(); @@ -605,7 +605,7 @@ public async Task BufferTooSmallForAuth() { using var srv = CreateTcpListener(_port); srv.WithAuth("testUser1", "Vs4e-cOLsVCntsMrZiAGAZtrkPXO00uoRLuA3d7gEcI=", - "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); + "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); srv.AcceptAsync(); using var ls = @@ -614,7 +614,7 @@ public async Task BufferTooSmallForAuth() try { await ls.AuthenticateAsync("testUser1", "NgdiOWDoQNUP18WOnb1xkkEG5TzPYMda5SiUOvT1K0U=", - CancellationToken.None); + CancellationToken.None); Assert.Fail(); } catch (IOException ex) @@ -642,8 +642,8 @@ public async Task SendNegativeLongMin() using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, tlsMode: TlsMode.Disable); Assert.Throws(() => ls.Table("name") - .Column("number1", long.MinValue) - .AtNow() + .Column("number1", long.MinValue) + .AtNow() ); } @@ -655,8 +655,8 @@ public async Task SendSpecialStrings() using var ls = await LineTcpSender.ConnectAsync("127.0.0.1", _port, tlsMode: TlsMode.Disable); ls.Table("neg name") - .Column("привед", " мед\rве\n д") - .AtNow(); + .Column("привед", " мед\rве\n д") + .AtNow(); ls.Send(); var expected = "neg\\ name привед=\" мед\\\rве\\\n д\"\n"; @@ -671,9 +671,9 @@ public async Task SendTagAfterField() using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, tlsMode: TlsMode.Disable); Assert.Throws(() => ls.Table("name") - .Column("number1", 123) - .Symbol("nand", "asdfa") - .AtNow() + .Column("number1", 123) + .Symbol("nand", "asdfa") + .AtNow() ); } @@ -685,9 +685,9 @@ public async Task SendMetricOnce() using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, tlsMode: TlsMode.Disable); Assert.Throws(() => ls.Table("name") - .Column("number1", 123) - .Table("nand") - .AtNow() + .Column("number1", 123) + .Table("nand") + .AtNow() ); } @@ -699,11 +699,11 @@ public async Task StartFromMetric() using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), _port, tlsMode: TlsMode.Disable); Assert.Throws(() => ls.Column("number1", 123) - .AtNow() + .AtNow() ); Assert.Throws(() => ls.Symbol("number1", "1234") - .AtNow() + .AtNow() ); } @@ -740,8 +740,8 @@ public async Task AtWithLongEpochNano() var ts = new DateTime(2022, 2, 24); ls.Table("name") - .Column("ts", ts) - .At((ts.Ticks - DateTime.UnixEpoch.Ticks) * 100); + .Column("ts", ts) + .At((ts.Ticks - DateTime.UnixEpoch.Ticks) * 100); ls.Send(); diff --git a/src/net-questdb-client-tests/TcpTests.cs b/src/net-questdb-client-tests/TcpTests.cs index 32f8548..7bbe2c6 100644 --- a/src/net-questdb-client-tests/TcpTests.cs +++ b/src/net-questdb-client-tests/TcpTests.cs @@ -51,10 +51,10 @@ public async Task SendLine() using var sender = Sender.New($"tcp::addr={_host}:{_port};"); await sender.Table("metric name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("string", " -=\"") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("string", " -=\"") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.SendAsync(); @@ -62,6 +62,51 @@ await sender.Table("metric name") WaitAssert(srv, expected); } + [Test] + public async Task SendLineWithDecimalBinaryEncoding() + { + using var srv = CreateTcpListener(_port); + srv.AcceptAsync(); + + using var sender = Sender.New($"tcp::addr={_host}:{_port};protocol_version=3;"); + await sender.Table("metrics") + .Symbol("tag", "value") + .Column("dec_pos", 123.45m) + .Column("dec_neg", -123.45m) + .Column("dec_null", (decimal?)null) + .Column("dec_max", decimal.MaxValue) + .Column("dec_min", decimal.MinValue) + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + + await sender.SendAsync(); + + var buffer = WaitForLineBytes(srv); + DecimalTestHelpers.AssertDecimalField(buffer, "dec_pos", 2, new byte[] + { + 0x30, 0x39, + }); + DecimalTestHelpers.AssertDecimalField(buffer, "dec_neg", 2, new byte[] + { + 0xCF, 0xC7, + }); + var prefix = Encoding.UTF8.GetBytes("dec_null="); + Assert.That(buffer.AsSpan().IndexOf(prefix), Is.EqualTo(-1)); + DecimalTestHelpers.AssertDecimalField(buffer, "dec_max", 0, new byte[] + { + 0x00, + 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, + }); + DecimalTestHelpers.AssertDecimalField(buffer, "dec_min", 0, new byte[] + { + 0xFF, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, + }); + } + [Test] public async Task SendLineWithArrayProtocolV2() { @@ -70,18 +115,19 @@ public async Task SendLineWithArrayProtocolV2() using var sender = Sender.New($"tcp::addr={_host}:{_port};protocol_version=2;"); await sender.Table("metric name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("string", " -=\"") - .Column("array", new[] - { - 1.2 - }) - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("string", " -=\"") + .Column("array", new[] + { + 1.2 + }) + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.SendAsync(); - var expected = "metric\\ name,t\\ a\\ g=v\\ alu\\,\\ e number=10i,string=\" -=\\\"\",array==ARRAY<1>[1.2] 1000000000\n"; + var expected = + "metric\\ name,t\\ a\\ g=v\\ alu\\,\\ e number=10i,string=\" -=\\\"\",array==ARRAY<1>[1.2] 1000000000\n"; WaitAssert(srv, expected); } @@ -96,13 +142,13 @@ public void SendLineWithArrayProtocolV1Exception() try { sender.Table("metric name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("string", " -=\"") - .Column("array", new[] - { - 1.2 - }); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("string", " -=\"") + .Column("array", new[] + { + 1.2 + }); } catch (IngressError err) { @@ -125,12 +171,12 @@ public async Task SendLineExceedsBuffer() for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -155,12 +201,12 @@ public async Task SendLineReusesBuffer() for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -169,12 +215,12 @@ await sender.Table("table name") for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -199,12 +245,12 @@ public async Task SendLineTrimsBuffers() for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -214,12 +260,12 @@ await sender.Table("table name") for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); } @@ -245,12 +291,12 @@ public async Task ServerDisconnects() for (var i = 0; i < lineCount; i++) { await sender.Table("table name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("db l", 123.12) - .Column("string", " -=\"") - .Column("при вед", "медвед") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("db l", 123.12) + .Column("string", " -=\"") + .Column("при вед", "медвед") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); totalExpectedSb.Append(expected); try { @@ -279,11 +325,11 @@ public async Task SendNegativeLongAndDouble() #pragma warning disable CS0618 // Type or member is obsolete await sender.Table("neg name") - .Column("number1", long.MinValue + 1) - .Column("number2", long.MaxValue) - .Column("number3", double.MinValue) - .Column("number4", double.MaxValue) - .AtNowAsync(); + .Column("number1", long.MinValue + 1) + .Column("number2", long.MaxValue) + .Column("number3", double.MinValue) + .Column("number4", double.MaxValue) + .AtNowAsync(); #pragma warning restore CS0618 // Type or member is obsolete await sender.SendAsync(); @@ -302,15 +348,15 @@ public async Task DoubleSerializationTest() #pragma warning disable CS0618 // Type or member is obsolete await sender.Table("doubles") - .Column("d0", 0.0) - .Column("dm0", -0.0) - .Column("d1", 1.0) - .Column("dE100", 1E100) - .Column("d0000001", 0.000001) - .Column("dNaN", double.NaN) - .Column("dInf", double.PositiveInfinity) - .Column("dNInf", double.NegativeInfinity) - .AtNowAsync(); + .Column("d0", 0.0) + .Column("dm0", -0.0) + .Column("d1", 1.0) + .Column("dE100", 1E100) + .Column("d0000001", 0.000001) + .Column("dNaN", double.NaN) + .Column("dInf", double.PositiveInfinity) + .Column("dNInf", double.NegativeInfinity) + .AtNowAsync(); #pragma warning restore CS0618 // Type or member is obsolete await sender.SendAsync(); @@ -329,8 +375,8 @@ public async Task SendTimestampColumn() var ts = new DateTime(2022, 2, 24); await sender.Table("name") - .Column("ts", ts) - .AtAsync(ts); + .Column("ts", ts) + .AtAsync(ts); await sender.SendAsync(); @@ -349,8 +395,8 @@ public async Task SendColumnNanos() const long timestampNanos = 1645660800123456789L; await sender.Table("name") - .ColumnNanos("ts", timestampNanos) - .AtAsync(timestampNanos); + .ColumnNanos("ts", timestampNanos) + .AtAsync(timestampNanos); await sender.SendAsync(); @@ -369,8 +415,8 @@ public async Task SendAtNanos() const long timestampNanos = 1645660800987654321L; await sender.Table("name") - .Column("value", 42) - .AtNanosAsync(timestampNanos); + .Column("value", 42) + .AtNanosAsync(timestampNanos); await sender.SendAsync(); @@ -385,8 +431,8 @@ public Task InvalidState() using var srv = CreateTcpListener(_port); srv.AcceptAsync(); - using var sender = Sender.New($"tcp::addr={_host}:{_port};"); - string? nullString = null; + using var sender = Sender.New($"tcp::addr={_host}:{_port};"); + string? nullString = null; Assert.That( () => sender.Table(nullString), @@ -488,8 +534,8 @@ public Task InvalidTableName() using var srv = CreateTcpListener(_port); srv.AcceptAsync(); - using var sender = Sender.New($"tcp::addr={_host}:{_port};"); - string? nullString = null; + using var sender = Sender.New($"tcp::addr={_host}:{_port};"); + string? nullString = null; Assert.Throws(() => sender.Table(nullString)); Assert.Throws(() => sender.Column("abc", 123)); @@ -515,23 +561,23 @@ public async Task CancelLine() using var sender = Sender.New($"tcp::addr={_host}:{_port};"); - sender.Table("good"); - sender.Symbol("asdf", "sdfad"); - sender.Column("ddd", 123); -#pragma warning disable CS0618 // Type or member is obsolete - await sender.AtNowAsync(); -#pragma warning restore CS0618 // Type or member is obsolete + await sender + .Table("good") + .Symbol("asdf", "sdfad") + .Column("ddd", 123) + .AtNowAsync(); + + await sender + .Table("bad") + .Symbol("asdf", "sdfad") + .Column("asdf", 123) + .AtAsync(DateTime.UtcNow); - sender.Table("bad"); - sender.Symbol("asdf", "sdfad"); - sender.Column("asdf", 123); -#pragma warning disable CS0618 // Type or member is obsolete - await sender.AtNowAsync(); -#pragma warning restore CS0618 // Type or member is obsolete sender.CancelRow(); - sender.Table("good"); - await sender.AtAsync(new DateTime(1970, 1, 2)); + await sender + .Table("good") + .AtAsync(new DateTime(1970, 1, 2)); await sender.SendAsync(); var expected = "good,asdf=sdfad ddd=123i\n" + @@ -546,19 +592,19 @@ public async Task SendMillionAsyncExplicit() srv.AcceptAsync(); var nowMillisecond = DateTime.Now.Millisecond; - var metric = "metric_name" + nowMillisecond; + var metric = "metric_name" + nowMillisecond; using var sender = Sender.New($"tcp::addr={_host}:{_port};init_buf_size={256 * 1024};"); for (var i = 0; i < 1E6; i++) { await sender.Table(metric) - .Symbol("nopoint", "tag" + i % 100) - .Column("counter", i * 1111.1) - .Column("int", i) - .Column("привед", "мед вед") - .AtAsync(new DateTime(2021, 1, 1, i / 360 / 1000 % 60, i / 60 / 1000 % 60, i / 1000 % 60, - i % 1000)); + .Symbol("nopoint", "tag" + i % 100) + .Column("counter", i * 1111.1) + .Column("int", i) + .Column("привед", "мед вед") + .AtAsync(new DateTime(2021, 1, 1, i / 360 / 1000 % 60, i / 60 / 1000 % 60, i / 1000 % 60, + i % 1000)); if (i % 100 == 0) { @@ -576,7 +622,7 @@ public async Task SendMillionFixedBuffer() srv.AcceptAsync(); var nowMillisecond = DateTime.Now.Millisecond; - var metric = "metric_name" + nowMillisecond; + var metric = "metric_name" + nowMillisecond; using var sender = Sender.New( @@ -585,12 +631,12 @@ public async Task SendMillionFixedBuffer() for (var i = 0; i < 1E6; i++) { await sender.Table(metric) - .Symbol("nopoint", "tag" + i % 100) - .Column("counter", i * 1111.1) - .Column("int", i) - .Column("привед", "мед вед") - .AtAsync(new DateTime(2021, 1, 1, i / 360 / 1000 % 60, i / 60 / 1000 % 60, i / 1000 % 60, - i % 1000)); + .Symbol("nopoint", "tag" + i % 100) + .Column("counter", i * 1111.1) + .Column("int", i) + .Column("привед", "мед вед") + .AtAsync(new DateTime(2021, 1, 1, i / 360 / 1000 % 60, i / 60 / 1000 % 60, i / 1000 % 60, + i % 1000)); } await sender.SendAsync(); @@ -618,8 +664,8 @@ public Task SendNegativeLongMin() Assert.That( #pragma warning disable CS0618 // Type or member is obsolete () => sender.Table("name") - .Column("number1", long.MinValue) - .AtNowAsync(), + .Column("number1", long.MinValue) + .AtNowAsync(), #pragma warning restore CS0618 // Type or member is obsolete Throws.TypeOf().With.Message.Contains("Special case") ); @@ -637,8 +683,8 @@ public async Task SendSpecialStrings() $"tcp::addr={_host}:{_port};"); #pragma warning disable CS0618 // Type or member is obsolete await sender.Table("neg name") - .Column("привед", " мед\rве\n д") - .AtNowAsync(); + .Column("привед", " мед\rве\n д") + .AtNowAsync(); #pragma warning restore CS0618 // Type or member is obsolete await sender.SendAsync(); @@ -659,9 +705,9 @@ public Task SendTagAfterField() Assert.That( #pragma warning disable CS0618 // Type or member is obsolete async () => await sender.Table("name") - .Column("number1", 123) - .Symbol("nand", "asdfa") - .AtNowAsync(), + .Column("number1", 123) + .Symbol("nand", "asdfa") + .AtNowAsync(), #pragma warning restore CS0618 // Type or member is obsolete Throws.TypeOf() ); @@ -681,9 +727,9 @@ public Task SendMetricOnce() Assert.That( #pragma warning disable CS0618 // Type or member is obsolete async () => await sender.Table("name") - .Column("number1", 123) - .Table("nand") - .AtNowAsync(), + .Column("number1", 123) + .Table("nand") + .AtNowAsync(), #pragma warning restore CS0618 // Type or member is obsolete Throws.TypeOf() ); @@ -703,7 +749,7 @@ public Task StartFromMetric() Assert.That( #pragma warning disable CS0618 // Type or member is obsolete async () => await sender.Column("number1", 123) - .AtNowAsync(), + .AtNowAsync(), #pragma warning restore CS0618 // Type or member is obsolete Throws.TypeOf() ); @@ -711,7 +757,7 @@ public Task StartFromMetric() Assert.That( #pragma warning disable CS0618 // Type or member is obsolete async () => await sender.Symbol("number1", "1234") - .AtNowAsync(), + .AtNowAsync(), #pragma warning restore CS0618 // Type or member is obsolete Throws.TypeOf() ); @@ -824,39 +870,39 @@ public async Task SendVariousAts() #pragma warning disable CS0618 // Type or member is obsolete await sender.Table("foo") - .Symbol("bah", "baz") - .AtNowAsync(); + .Symbol("bah", "baz") + .AtNowAsync(); #pragma warning restore CS0618 // Type or member is obsolete await sender.Table("foo") - .Symbol("bah", "baz") - .AtAsync(DateTime.UtcNow); + .Symbol("bah", "baz") + .AtAsync(DateTime.UtcNow); await sender.Table("foo") - .Symbol("bah", "baz") - .AtAsync(DateTimeOffset.UtcNow); + .Symbol("bah", "baz") + .AtAsync(DateTimeOffset.UtcNow); await sender.Table("foo") - .Symbol("bah", "baz") - .AtAsync(DateTime.UtcNow.Ticks / 100); + .Symbol("bah", "baz") + .AtAsync(DateTime.UtcNow.Ticks / 100); #pragma warning disable CS0618 // Type or member is obsolete sender.Table("foo") - .Symbol("bah", "baz") - .AtNow(); + .Symbol("bah", "baz") + .AtNow(); #pragma warning restore CS0618 // Type or member is obsolete sender.Table("foo") - .Symbol("bah", "baz") - .At(DateTime.UtcNow); + .Symbol("bah", "baz") + .At(DateTime.UtcNow); sender.Table("foo") - .Symbol("bah", "baz") - .At(DateTimeOffset.UtcNow); + .Symbol("bah", "baz") + .At(DateTimeOffset.UtcNow); sender.Table("foo") - .Symbol("bah", "baz") - .At(DateTime.UtcNow.Ticks / 100); + .Symbol("bah", "baz") + .At(DateTime.UtcNow.Ticks / 100); await sender.SendAsync(); } @@ -884,7 +930,7 @@ public async Task Authenticate() { using var srv = CreateTcpListener(_port); srv.WithAuth("testUser1", "Vs4e-cOLsVCntsMrZiAGAZtrkPXO00uoRLuA3d7gEcI=", - "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); + "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); srv.AcceptAsync(); using var sender = @@ -892,10 +938,10 @@ public async Task Authenticate() $"tcp::addr={_host}:{_port};username=testUser1;token=NgdiOWDoQNUP18WOnb1xkkEG5TzPYMda5SiUOvT1K0U=;"); await sender.Table("metric name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("string", " -=\"") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("string", " -=\"") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.SendAsync(); var expected = "metric\\ name,t\\ a\\ g=v\\ alu\\,\\ e number=10i,string=\" -=\\\"\" 1000000000\n"; @@ -907,14 +953,14 @@ public Task AuthFailWrongKid() { using var srv = CreateTcpListener(_port); srv.WithAuth("testUser1", "Vs4e-cOLsVCntsMrZiAGAZtrkPXO00uoRLuA3d7gEcI=", - "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); + "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); srv.AcceptAsync(); Assert.That( () => Sender.New($"tcp::addr={_host}:{_port};username=invalid;token=foo=;") - , + , Throws.TypeOf().With.InnerException.TypeOf().With.Message - .Contains("Authentication failed") + .Contains("Authentication failed") ); return Task.CompletedTask; } @@ -924,7 +970,7 @@ public Task AuthFailBadKey() { using var srv = CreateTcpListener(_port); srv.WithAuth("testUser1", "Vs4e-cOLsVCntsMrZiAGAZtrkPXO00uoRLuA3d7gEcI=", - "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); + "ANhR2AZSs4ar9urE5AZrJqu469X0r7gZ1BBEdcrAuL_6"); srv.AcceptAsync(); using var sender = Sender.New( @@ -933,13 +979,13 @@ public Task AuthFailBadKey() Assert.That( async () => { - for (var i = 0; i < 10; i++) + for (var i = 0; i < 100; i++) { await sender.Table("metric name") - .Symbol("t a g", "v alu, e") - .Column("number", 10) - .Column("string", " -=\"") - .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); + .Symbol("t a g", "v alu, e") + .Column("number", 10) + .Column("string", " -=\"") + .AtAsync(new DateTime(1970, 01, 01, 0, 0, 1)); await sender.SendAsync(); Thread.Sleep(10); } @@ -947,7 +993,7 @@ await sender.Table("metric name") Assert.Fail(); }, Throws.TypeOf().With.Message - .Contains("Could not write data to server.") + .Contains("Could not write data to server.") ); return Task.CompletedTask; } @@ -956,7 +1002,7 @@ await sender.Table("metric name") public void EcdsaSignatureLoop() { var privateKey = Convert.FromBase64String("NgdiOWDoQNUP18WOnb1xkkEG5TzPYMda5SiUOvT1K0U="); - var p = SecNamedCurves.GetByName("secp256r1"); + var p = SecNamedCurves.GetByName("secp256r1"); var parameters = new ECDomainParameters(p.Curve, p.G, p.N, p.H); var m = new byte[512]; @@ -997,9 +1043,25 @@ private DummyIlpServer CreateTcpListener(int port, bool tls = false) return new DummyIlpServer(port, tls); } + private static byte[] WaitForLineBytes(DummyIlpServer server) + { + for (var i = 0; i < 500; i++) + { + var bytes = server.GetReceivedBytes(); + if (bytes.Length > 0 && bytes[^1] == (byte)'\n') + { + return bytes; + } + + Thread.Sleep(10); + } + + Assert.Fail("Timed out waiting for decimal ILP payload."); + return Array.Empty(); + } + private byte[] FromBase64String(string text) { return DummyIlpServer.FromBase64String(text); } - } \ No newline at end of file diff --git a/src/net-questdb-client/Buffers/Buffer.cs b/src/net-questdb-client/Buffers/Buffer.cs index f801be9..9c17b04 100644 --- a/src/net-questdb-client/Buffers/Buffer.cs +++ b/src/net-questdb-client/Buffers/Buffer.cs @@ -32,25 +32,23 @@ namespace QuestDB.Buffers; public static class Buffer { /// - /// Creates an IBuffer instance, based on the provided protocol version. + /// Creates a concrete IBuffer implementation configured for the specified protocol version. /// - /// - /// - /// - /// - /// - /// + /// Size in bytes of each buffer segment. + /// Maximum allowed length for names stored in the buffer. + /// Maximum total buffer capacity. + /// Protocol version that determines which concrete buffer implementation to create. + /// An instance corresponding to the specified protocol version. + /// Thrown when an unsupported protocol version is provided. public static IBuffer Create(int bufferSize, int maxNameLen, int maxBufSize, ProtocolVersion version) { - switch (version) + return version switch { - case ProtocolVersion.V1: - return new BufferV1(bufferSize, maxNameLen, maxBufSize); - case ProtocolVersion.V2: - case ProtocolVersion.Auto: - return new BufferV2(bufferSize, maxNameLen, maxBufSize); - } - - throw new NotImplementedException(); + ProtocolVersion.V1 => new BufferV1(bufferSize, maxNameLen, maxBufSize), + ProtocolVersion.V2 => new BufferV2(bufferSize, maxNameLen, maxBufSize), + ProtocolVersion.V3 => new BufferV3(bufferSize, maxNameLen, maxBufSize), + ProtocolVersion.Auto => new BufferV3(bufferSize, maxNameLen, maxBufSize), + _ => throw new NotImplementedException(), + }; } } \ No newline at end of file diff --git a/src/net-questdb-client/Buffers/BufferStreamContent.cs b/src/net-questdb-client/Buffers/BufferStreamContent.cs index 6edbc00..7dc2bad 100644 --- a/src/net-questdb-client/Buffers/BufferStreamContent.cs +++ b/src/net-questdb-client/Buffers/BufferStreamContent.cs @@ -32,6 +32,10 @@ namespace QuestDB.Buffers; /// internal class BufferStreamContent : HttpContent { + /// + /// Initializes a new instance of the class. + /// + /// The buffer to wrap for HTTP streaming. public BufferStreamContent(IBuffer buffer) { Buffer = buffer; diff --git a/src/net-questdb-client/Buffers/BufferV1.cs b/src/net-questdb-client/Buffers/BufferV1.cs index 9bd224f..e4fc0fe 100644 --- a/src/net-questdb-client/Buffers/BufferV1.cs +++ b/src/net-questdb-client/Buffers/BufferV1.cs @@ -40,13 +40,19 @@ public class BufferV1 : IBuffer private int _currentBufferIndex; private string _currentTableName = null!; private bool _hasTable; + private int _lineStartLength; private int _lineStartBufferIndex; private int _lineStartBufferPosition; private bool _noFields = true; private bool _noSymbols = true; private bool _quoted; - /// + /// + /// Initializes a new instance of BufferV1 for writing ILP (InfluxDB Line Protocol) messages. + /// + /// Initial size of each buffer chunk, in bytes. + /// Maximum allowed UTF-8 byte length for table and column names. + /// Maximum total buffer size across all chunks, in bytes. public BufferV1(int bufferSize, int maxNameLen, int maxBufSize) { Chunk = new byte[bufferSize]; @@ -76,13 +82,13 @@ public IBuffer Transaction(ReadOnlySpan tableName) if (WithinTransaction) { throw new IngressError(ErrorCode.InvalidApiCall, - "Cannot start another transaction - only one allowed at a time."); + "Cannot start another transaction - only one allowed at a time."); } if (Length > 0) { throw new IngressError(ErrorCode.InvalidApiCall, - "Buffer must be clear before you can start a transaction."); + "Buffer must be clear before you can start a transaction."); } GuardInvalidTableName(tableName); @@ -132,21 +138,28 @@ public void AtNanos(long timestampNanos) FinishLine(); } - /// + /// + /// Resets the buffer to its initial empty state and clears all written data. + /// + /// + /// Clears lengths of all allocated chunks, resets the active chunk and write position, + /// resets row and total-length counters, exits any transaction state, and clears the current table and line start markers. + /// public void Clear() { _currentBufferIndex = 0; - Chunk = _buffers[_currentBufferIndex].Buffer; + Chunk = _buffers[_currentBufferIndex].Buffer; for (var i = 0; i < _buffers.Count; i++) { _buffers[i] = (_buffers[i].Buffer, 0); } - Position = 0; - RowCount = 0; - Length = 0; - WithinTransaction = false; - _currentTableName = ""; + Position = 0; + RowCount = 0; + Length = 0; + WithinTransaction = false; + _currentTableName = ""; + _lineStartLength = 0; _lineStartBufferIndex = 0; _lineStartBufferPosition = 0; } @@ -161,13 +174,21 @@ public void TrimExcessBuffers() } } - /// + /// + /// Reverts the current (in-progress) row to its start position, removing any bytes written for that row. + /// + /// + /// Restores the active buffer index, adjusts the total Length and current Position to the saved line start, + /// and clears the table-set flag for the cancelled row. + /// public void CancelRow() { - _currentBufferIndex = _lineStartBufferIndex; - Length -= Position - _lineStartBufferPosition; - Position = _lineStartBufferPosition; - _hasTable = false; + _currentBufferIndex = _lineStartBufferIndex; + Chunk = _buffers[_currentBufferIndex].Buffer; + Length = _lineStartLength; + Position = _lineStartBufferPosition; + _hasTable = false; + } /// @@ -223,23 +244,32 @@ public void WriteToStream(Stream stream, CancellationToken ct = default) stream.Flush(); } - /// + /// + /// Sets the table name for the current row and encodes it into the buffer, beginning a new line context. + /// + /// The table name to write; must meet filesystem length limits and protocol naming rules. + /// This buffer instance to support fluent calls. + /// + /// Thrown with ErrorCode.InvalidApiCall if a transaction is active for a different table or if a table has already been set for the current line. + /// Thrown with ErrorCode.InvalidName if the provided name violates length or character restrictions. + /// public IBuffer Table(ReadOnlySpan name) { GuardFsFileNameLimit(name); if (WithinTransaction && name != _currentTableName) { throw new IngressError(ErrorCode.InvalidApiCall, - "Transactions can only be for one table."); + "Transactions can only be for one table."); } GuardTableAlreadySet(); GuardInvalidTableName(name); - _quoted = false; + _quoted = false; _hasTable = true; - _lineStartBufferIndex = _currentBufferIndex; + _lineStartLength = Length; + _lineStartBufferIndex = _currentBufferIndex; _lineStartBufferPosition = Position; EncodeUtf8(name); @@ -352,7 +382,7 @@ public IBuffer ColumnNanos(ReadOnlySpan name, long timestampNanos) return this; } - /// + /// public IBuffer EncodeUtf8(ReadOnlySpan name) { foreach (var c in name) @@ -370,7 +400,7 @@ public IBuffer EncodeUtf8(ReadOnlySpan name) return this; } - /// + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public IBuffer PutAscii(char c) { @@ -378,30 +408,34 @@ public IBuffer PutAscii(char c) return this; } - /// + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Put(ReadOnlySpan chars) { EncodeUtf8(chars); } - /// + /// + /// Appends the decimal ASCII representation of the specified 64-bit integer to the buffer. + /// + /// The current buffer instance. + /// Thrown when the value is , which cannot be represented by this method; the error contains an inner . public IBuffer Put(long value) { if (value == long.MinValue) { throw new IngressError(ErrorCode.InvalidApiCall, "Special case, long.MinValue cannot be handled by QuestDB", - new ArgumentOutOfRangeException()); + new ArgumentOutOfRangeException()); } - Span num = stackalloc byte[20]; - var pos = num.Length; - var remaining = Math.Abs(value); + Span num = stackalloc byte[20]; + var pos = num.Length; + var remaining = Math.Abs(value); do { var digit = remaining % 10; - num[--pos] = (byte)('0' + digit); - remaining /= 10; + num[--pos] = (byte)('0' + digit); + remaining /= 10; } while (remaining != 0); if (value < 0) @@ -414,31 +448,31 @@ public IBuffer Put(long value) num.Slice(pos, len).CopyTo(Chunk.AsSpan(Position)); Position += len; - Length += len; + Length += len; return this; } - /// + /// public virtual IBuffer Column(ReadOnlySpan name, ReadOnlySpan value) where T : struct { throw new IngressError(ErrorCode.ProtocolVersionError, "Protocol Version V1 does not support ARRAY types"); } - /// + /// public virtual IBuffer Column(ReadOnlySpan name, Array? value) { throw new IngressError(ErrorCode.ProtocolVersionError, "Protocol Version V1 does not support ARRAY types"); } - /// + /// public virtual IBuffer Column(ReadOnlySpan name, IEnumerable value, IEnumerable shape) where T : struct { throw new IngressError(ErrorCode.ProtocolVersionError, "Protocol Version V1 does not support ARRAY types"); } - /// + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public IBuffer Put(byte value) { @@ -453,13 +487,23 @@ public IBuffer Put(byte value) } + /// + /// Advance the current buffer write position and the overall length by a given number of bytes. + /// + /// The number of bytes to add to both and . [MethodImpl(MethodImplOptions.AggressiveInlining)] internal void Advance(int by) { Position += by; - Length += by; + Length += by; } + /// + /// Sets the buffer's current table to the stored table name when a transaction is active and no table has been set for the current row. + /// + /// + /// Has no effect if not within a transaction or if a table has already been set for the current row. + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] internal void SetTableIfAppropriate() { @@ -469,13 +513,16 @@ internal void SetTableIfAppropriate() } } + /// + /// Finalizes the current row: terminates it with a newline, increments the completed row counter, resets per-row flags, and enforces the buffer size limit. + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] private void FinishLine() { PutAscii('\n'); RowCount++; - _hasTable = false; - _noFields = true; + _hasTable = false; + _noFields = true; _noSymbols = true; GuardExceededMaxBufferSize(); } @@ -490,10 +537,16 @@ private void GuardExceededMaxBufferSize() if (Length > _maxBufSize) { throw new IngressError(ErrorCode.InvalidApiCall, - $"Exceeded maximum buffer size. Current: {Length} Maximum: {_maxBufSize}"); + $"Exceeded maximum buffer size. Current: {Length} Maximum: {_maxBufSize}"); } } + /// + /// Writes the column name to the buffer and prepares for writing the column value by appending the appropriate separator and equals sign. + /// + /// The column name to write. + /// The buffer instance for fluent chaining. + /// Thrown if the table is not set, the column name is invalid, or the name exceeds the maximum length. internal IBuffer Column(ReadOnlySpan columnName) { GuardFsFileNameLimit(columnName); @@ -513,16 +566,26 @@ internal IBuffer Column(ReadOnlySpan columnName) return EncodeUtf8(columnName).PutAscii('='); } + /// + /// Validates that the requested additional byte count does not exceed the chunk size. + /// + /// The number of additional bytes requested. + /// Thrown with if the requested size exceeds the chunk length. [MethodImpl(MethodImplOptions.AggressiveInlining)] internal void GuardAgainstOversizedChunk(int additional) { if (additional > Chunk.Length) { throw new IngressError(ErrorCode.InvalidApiCall, - "tried to allocate oversized chunk: " + additional + " bytes"); + "tried to allocate oversized chunk: " + additional + " bytes"); } } + /// + /// Ensures that the current chunk has enough space to write the specified number of additional bytes; switches to the next buffer chunk if needed. + /// + /// The number of additional bytes required. + /// Thrown if the requested size exceeds the chunk size. [MethodImpl(MethodImplOptions.AggressiveInlining)] internal void EnsureCapacity(int additional) { @@ -533,6 +596,10 @@ internal void EnsureCapacity(int additional) } } + /// + /// Writes a non-ASCII character as UTF-8 to the buffer, switching to the next buffer chunk if insufficient space remains. + /// + /// The character to encode and write. private void PutUtf8(char c) { if (Position + 4 >= Chunk.Length) @@ -540,12 +607,19 @@ private void PutUtf8(char c) NextBuffer(); } - var bytes = Chunk.AsSpan(Position); - Span chars = stackalloc char[1] { c, }; - var byteLength = Encoding.UTF8.GetBytes(chars, bytes); + var bytes = Chunk.AsSpan(Position); + Span chars = stackalloc char[1] { c, }; + var byteLength = Encoding.UTF8.GetBytes(chars, bytes); Advance(byteLength); } + /// + /// Writes an ASCII character to the buffer, applying ILP escaping rules based on context (quoted or unquoted). + /// + /// The ASCII character to write. + /// + /// Escapes space, comma, equals, newline, carriage return, quote, and backslash characters according to ILP protocol requirements. + /// private void PutSpecial(char c) { switch (c) @@ -659,7 +733,7 @@ private static void GuardInvalidTableName(ReadOnlySpan tableName) if (tableName.IsEmpty) { throw new IngressError(ErrorCode.InvalidName, - "Table names must have a non-zero length."); + "Table names must have a non-zero length."); } var prev = '\0'; @@ -672,7 +746,7 @@ private static void GuardInvalidTableName(ReadOnlySpan tableName) if (i == 0 || i == tableName.Length - 1 || prev == '.') { throw new IngressError(ErrorCode.InvalidName, - $"Bad string {tableName}. Found invalid dot `.` at position {i}."); + $"Bad string {tableName}. Found invalid dot `.` at position {i}."); } break; @@ -707,10 +781,10 @@ private static void GuardInvalidTableName(ReadOnlySpan tableName) case '\x000f': case '\x007f': throw new IngressError(ErrorCode.InvalidName, - $"Bad string {tableName}. Table names can't contain a {c} character, which was found at byte position {i}"); + $"Bad string {tableName}. Table names can't contain a {c} character, which was found at byte position {i}"); case '\xfeff': throw new IngressError(ErrorCode.InvalidName, - $"Bad string {tableName}. Table names can't contain a UTF-8 BOM character, was was found at byte position {i}."); + $"Bad string {tableName}. Table names can't contain a UTF-8 BOM character, which was found at byte position {i}."); } prev = c; @@ -727,7 +801,7 @@ private static void GuardInvalidColumnName(ReadOnlySpan columnName) if (columnName.IsEmpty) { throw new IngressError(ErrorCode.InvalidName, - "Column names must have a non-zero length."); + "Column names must have a non-zero length."); } for (var i = 0; i < columnName.Length; i++) @@ -768,10 +842,10 @@ private static void GuardInvalidColumnName(ReadOnlySpan columnName) case '\x000f': case '\x007f': throw new IngressError(ErrorCode.InvalidName, - $"Bad string {columnName}. Column names can't contain a {c} character, which was found at byte position {i}"); + $"Bad string {columnName}. Column names can't contain a {c} character, which was found at byte position {i}"); case '\xfeff': throw new IngressError(ErrorCode.InvalidName, - $"Bad string {columnName}. Column names can't contain a UTF-8 BOM character, was was found at byte position {i}."); + $"Bad string {columnName}. Column names can't contain a UTF-8 BOM character, which was found at byte position {i}."); } } } @@ -780,13 +854,29 @@ private static void GuardInvalidColumnName(ReadOnlySpan columnName) /// Check that the file name is not too long. /// /// - /// + /// + /// Validates that the UTF-8 encoded byte length of the given name is within the configured maximum. + /// + /// The name to validate (measured in UTF-8 bytes). + /// Thrown with if the name exceeds the maximum allowed byte length. private void GuardFsFileNameLimit(ReadOnlySpan name) { if (Encoding.UTF8.GetBytes(name.ToString()).Length > _maxNameLen) { throw new IngressError(ErrorCode.InvalidApiCall, - $"Name is too long, must be under {_maxNameLen} bytes."); + $"Name is too long, must be under {_maxNameLen} bytes."); } } + + /// + /// Attempts to add a DECIMAL column to the current row; DECIMAL types are not supported by Protocol Version V1. + /// + /// The column name to write. + /// The decimal value to write, or null to indicate absence. + /// The buffer instance for fluent chaining. + /// Always thrown with to indicate DECIMAL is unsupported. + public virtual IBuffer Column(ReadOnlySpan name, decimal? value) + { + throw new IngressError(ErrorCode.ProtocolVersionError, "Protocol Version does not support DECIMAL types"); + } } \ No newline at end of file diff --git a/src/net-questdb-client/Buffers/BufferV2.cs b/src/net-questdb-client/Buffers/BufferV2.cs index 67a8f18..ac3a23f 100644 --- a/src/net-questdb-client/Buffers/BufferV2.cs +++ b/src/net-questdb-client/Buffers/BufferV2.cs @@ -32,12 +32,25 @@ namespace QuestDB.Buffers; /// public class BufferV2 : BufferV1 { - /// + /// + /// Initializes a new instance of BufferV2 supporting ARRAY and binary DOUBLE types. + /// + /// Initial size of the internal write buffer, in bytes. + /// Maximum allowed length for column names, in characters. + /// Maximum allowed internal buffer size, in bytes. public BufferV2(int bufferSize, int maxNameLen, int maxBufSize) : base(bufferSize, maxNameLen, maxBufSize) { } - /// + /// + /// Writes a multidimensional double array column with the specified name, elements, and shape to the buffer. + /// + /// The element type (must be double). + /// The column name. + /// An enumerable of double values forming the array elements. + /// An enumerable of integers describing the dimensions; product must equal the element count. + /// The buffer instance for fluent chaining. + /// Thrown if T is not double, shape is invalid, or shape does not match element count. public override IBuffer Column(ReadOnlySpan name, IEnumerable value, IEnumerable shape) where T : struct { @@ -81,6 +94,11 @@ public override IBuffer Column(ReadOnlySpan name, IEnumerable value, return this; } + /// + /// Validates that the provided type is double; throws if not. + /// + /// The type to validate. + /// Thrown with if the type is not double. [MethodImpl(MethodImplOptions.AggressiveInlining)] private void GuardAgainstNonDoubleTypes(Type t) { @@ -90,17 +108,24 @@ private void GuardAgainstNonDoubleTypes(Type t) } } - // ReSharper disable once InconsistentNaming + /// + /// Writes the provided value into the buffer as little-endian raw bytes and advances the buffer position by the value's size. + /// + /// A value whose raw bytes will be written into the buffer in little-endian order. private void PutBinaryLE(T value) where T : struct { var size = Marshal.SizeOf(); EnsureCapacity(size); - var length = Marshal.SizeOf(); - var mem = MemoryMarshal.Cast(Chunk.AsSpan(Position, length)); + var mem = MemoryMarshal.Cast(Chunk.AsSpan(Position, size)); mem[0] = value; - Advance(length); + Advance(size); } + /// + /// Writes the provided value into the buffer as big-endian raw bytes and advances the buffer position by the value's size. + /// + /// A value type. + /// The value to write in big-endian byte order. // ReSharper disable once InconsistentNaming private void PutBinaryBE(T value) where T : struct { @@ -111,20 +136,24 @@ private void PutBinaryBE(T value) where T : struct MemoryMarshal.Cast(slot).Reverse(); } - // ReSharper disable once InconsistentNaming + /// + /// Writes a sequence of values into the buffer in little-endian binary form, handling chunk boundaries and advancing the buffer position. + /// + /// A span of values whose raw bytes will be written as little-endian binary (elements are written whole; partial element writes are not performed). private void PutBinaryManyLE(ReadOnlySpan value) where T : struct { - var srcSpan = MemoryMarshal.Cast(value); + var srcSpan = MemoryMarshal.Cast(value); var byteSize = Marshal.SizeOf(); while (srcSpan.Length > 0) { - var dstLength = GetSpareCapacity(); // length + var dstLength = GetSpareCapacity(); // length if (dstLength < byteSize) { NextBuffer(); - dstLength = GetSpareCapacity(); + dstLength = GetSpareCapacity(); } + var availLength = dstLength - dstLength % byteSize; // rounded length if (srcSpan.Length < availLength) @@ -133,13 +162,19 @@ private void PutBinaryManyLE(ReadOnlySpan value) where T : struct Advance(srcSpan.Length); return; } - var dstSpan = Chunk.AsSpan(Position, availLength); + + var dstSpan = Chunk.AsSpan(Position, availLength); srcSpan.Slice(0, availLength).CopyTo(dstSpan); Advance(availLength); srcSpan = srcSpan.Slice(availLength); } } + /// + /// Writes a sequence of values into the buffer in big-endian binary form by writing each element individually. + /// + /// A value type. + /// A span of values to write in big-endian byte order. // ReSharper disable once InconsistentNaming private void PutBinaryManyBE(ReadOnlySpan value) where T : struct { @@ -149,6 +184,11 @@ private void PutBinaryManyBE(ReadOnlySpan value) where T : struct } } + /// + /// Writes a sequence of values into the buffer in the platform's native byte order (little-endian or big-endian). + /// + /// A value type. + /// A span of values to write. private void PutBinaryMany(ReadOnlySpan value) where T : struct { if (BitConverter.IsLittleEndian) @@ -161,6 +201,11 @@ private void PutBinaryMany(ReadOnlySpan value) where T : struct } } + /// + /// Writes a single value into the buffer in the platform's native byte order (little-endian or big-endian). + /// + /// A value type. + /// The value to write. private void PutBinary(T value) where T : struct { if (BitConverter.IsLittleEndian) @@ -173,14 +218,23 @@ private void PutBinary(T value) where T : struct } } - /// + /// + /// Writes a column whose value is the provided span of doubles encoded as a binary double array. + /// + /// The current buffer instance. public override IBuffer Column(ReadOnlySpan name, ReadOnlySpan value) where T : struct { GuardAgainstNonDoubleTypes(typeof(T)); return PutDoubleArray(name, value); } - private IBuffer PutDoubleArray(ReadOnlySpan name, ReadOnlySpan value) where T : struct + /// + /// Writes a one-dimensional double array column encoded in the buffer's binary double-array format. + /// + /// The column name. + /// A span of elements representing the array; elements must be of type `double`. + /// The current buffer instance. + private IBuffer PutDoubleArray(ReadOnlySpan name, ReadOnlySpan value) where T : struct { SetTableIfAppropriate(); PutArrayOfDoubleHeader(name); @@ -191,7 +245,13 @@ private IBuffer PutDoubleArray(ReadOnlySpan name, ReadOnlySpan value return this; } - /// + /// + /// Add a column with the given name whose value is provided by the specified double array (1D or multi-dimensional). + /// + /// The column name to write. + /// An array of doubles to write. If null the column is omitted. For a 1D array the values are written as a single-dimension double array; for multi-dimensional arrays the rank and each dimension length are written followed by the elements in row-major order. + /// This buffer instance. + /// Thrown when the array's element type cannot be determined. public override IBuffer Column(ReadOnlySpan name, Array? value) { if (value == null) @@ -199,7 +259,7 @@ public override IBuffer Column(ReadOnlySpan name, Array? value) // The value is null, do not include the column in the message return this; } - + var type = value.GetType().GetElementType(); GuardAgainstNonDoubleTypes(type ?? throw new InvalidOperationException()); if (value.Rank == 1) @@ -207,7 +267,7 @@ public override IBuffer Column(ReadOnlySpan name, Array? value) // Fast path, one dim array return PutDoubleArray(name, (ReadOnlySpan)value!); } - + SetTableIfAppropriate(); PutArrayOfDoubleHeader(name); @@ -226,6 +286,11 @@ public override IBuffer Column(ReadOnlySpan name, Array? value) return this; } + /// + /// Reserves space in the buffer for a value of type T and returns a span to write the value later, advancing the buffer position. + /// + /// A value type. + /// An out parameter that receives a span covering the reserved space for writing. private void PutBinaryDeferred(out Span span) where T : struct { var length = Marshal.SizeOf(); @@ -234,6 +299,10 @@ private void PutBinaryDeferred(out Span span) where T : struct Advance(length); } + /// + /// Writes the ILP binary format header for a double array column. + /// + /// The column name. private void PutArrayOfDoubleHeader(ReadOnlySpan name) { Column(name) @@ -243,6 +312,10 @@ private void PutArrayOfDoubleHeader(ReadOnlySpan name) } + /// + /// Writes the ILP binary format header for a double column. + /// + /// The column name. private void PutDoubleHeader(ReadOnlySpan name) { Column(name) @@ -250,6 +323,10 @@ private void PutDoubleHeader(ReadOnlySpan name) .Put((byte)BinaryFormatType.DOUBLE); } + /// + /// Calculates the remaining available space in the current buffer chunk. + /// + /// The number of bytes remaining in the current chunk. private int GetSpareCapacity() { return Chunk.Length - Position; diff --git a/src/net-questdb-client/Buffers/BufferV3.cs b/src/net-questdb-client/Buffers/BufferV3.cs new file mode 100644 index 0000000..d0e1a74 --- /dev/null +++ b/src/net-questdb-client/Buffers/BufferV3.cs @@ -0,0 +1,134 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +using System.Buffers.Binary; +using QuestDB.Enums; + +namespace QuestDB.Buffers; + +/// +public class BufferV3 : BufferV2 +{ + /// + /// Initializes a new instance of BufferV3 with the specified buffer and name length limits. + /// + /// Initial size of the internal write buffer, in bytes. + /// Maximum allowed length for column names, in characters. + /// Maximum allowed internal buffer size, in bytes. + public BufferV3(int bufferSize, int maxNameLen, int maxBufSize) : base(bufferSize, maxNameLen, maxBufSize) + { + } + + // Sign mask for the flags field. A value of zero in this bit indicates a + // positive Decimal value, and a value of one in this bit indicates a + // negative Decimal value. + private const int SignMask = unchecked((int)0x80000000); + + // Scale mask for the flags field. This byte in the flags field contains + // the power of 10 to divide the Decimal value by. The scale byte must + // contain a value between 0 and 28 inclusive. + private const int ScaleMask = 0x00FF0000; + + // Number of bits scale is shifted by. + private const int ScaleShift = 16; + + /// + /// Writes a decimal column in QuestDB's binary column format (scale, length, and two's-complement big-endian unscaled value). + /// + /// Column name to write. + /// Nullable decimal value to encode; when null writes zero scale and zero length. + /// The buffer instance for call chaining. + public override IBuffer Column(ReadOnlySpan name, decimal? value) + { + if (value is null) + { + return this; + } + + // # Binary Format + // 1. Binary format marker: `'='` (0x3D) + // 2. Type identifier: BinaryFormatType.DECIMAL byte + // 3. Scale: 1 byte (0-28 for .NET decimal) - number of decimal places + // 4. Length: 1 byte - number of bytes in the unscaled value + // 5. Unscaled value: variable-length byte array in two's complement format, big-endian + SetTableIfAppropriate(); + Column(name) + .PutAscii(Constants.BINARY_FORMAT_FLAG) + .Put((byte)BinaryFormatType.DECIMAL); + + Span parts = stackalloc int[4]; + decimal.GetBits(value.Value, parts); + + var flags = parts[3]; + var scale = (byte)((flags & ScaleMask) >> ScaleShift); + + // 3. Scale + Put(scale); + + var low = parts[0]; + var mid = parts[1]; + var high = parts[2]; + var negative = (flags & SignMask) != 0 && value.Value != 0m; + + if (negative) + { + // QuestDB expects negative mantissas in two's complement. + unchecked + { + low = ~low + 1; + var c = low == 0 ? 1 : 0; + mid = ~mid + c; + c = mid == 0 && c == 1 ? 1 : 0; + high = ~high + c; + } + } + + // We write the byte array on the stack first so that we can compress (remove unnecessary bytes) it later. + Span span = stackalloc byte[13]; + var signByte = (byte)(negative ? 255 : 0); + span[0] = signByte; + BinaryPrimitives.WriteInt32BigEndian(span.Slice(1, 4), high); + BinaryPrimitives.WriteInt32BigEndian(span.Slice(5, 4), mid); + BinaryPrimitives.WriteInt32BigEndian(span.Slice(9, 4), low); + + // Compress + var start = 0; + for (; + // We can strip prefix bits that are 0 (if positive) or 1 (if negative) as long as we keep at least + // one of it in front to convey the sign. + start < span.Length - 1 && span[start] == signByte && ((span[start + 1] ^ signByte) & 0x80) == 0; + start++) ; + + // 4. Length + var size = span.Length - start; + Put((byte)size); + + // 5. Unscaled value + EnsureCapacity(size); + span.Slice(start, size).CopyTo(Chunk.AsSpan(Position, size)); + Advance(size); + + return this; + } +} \ No newline at end of file diff --git a/src/net-questdb-client/Buffers/IBuffer.cs b/src/net-questdb-client/Buffers/IBuffer.cs index 82bfdfe..b16b60f 100644 --- a/src/net-questdb-client/Buffers/IBuffer.cs +++ b/src/net-questdb-client/Buffers/IBuffer.cs @@ -58,7 +58,11 @@ public interface IBuffer // ReSharper disable once InconsistentNaming public int RowCount { get; protected set; } - /// + /// + /// Encodes the specified character span to UTF-8 and appends it to the buffer. + /// + /// The character span to encode and append. + /// The buffer instance for fluent chaining. public IBuffer EncodeUtf8(ReadOnlySpan name); /// @@ -194,7 +198,10 @@ public interface IBuffer /// public void CancelRow(); - /// + /// + /// Gets the current chunk of the buffer as a read-only byte span for sending. + /// + /// A read-only span of bytes representing the current chunk. public ReadOnlySpan GetSendBuffer(); /// @@ -213,24 +220,76 @@ public interface IBuffer /// When writing to stream fails. public void WriteToStream(Stream stream, CancellationToken ct = default); - /// + /// + /// Appends a single ASCII character to the buffer. + /// + /// The ASCII character to append. + /// The buffer instance for fluent chaining. public IBuffer PutAscii(char c); - /// + /// + /// Appends a 64-bit integer value in ASCII decimal representation to the buffer. + /// + /// The long value to append. + /// The buffer instance for fluent chaining. public IBuffer Put(long value); - /// + /// + /// Appends a character span to the buffer by encoding it as UTF-8. + /// + /// The character span to encode and append. public void Put(ReadOnlySpan chars); - /// + /// + /// Appends a single byte to the buffer. + /// + /// The byte value to append. + /// The buffer instance for fluent chaining. public IBuffer Put(byte value); - /// + /// + /// Adds a column with the specified name and a span of value-type elements. + /// + /// The element type; must be a value type. + /// The column name. + /// A span of value-type elements representing the column data. + /// The buffer instance for fluent chaining. + /// + /// This method requires protocol version 2 or later. It will throw an with if used with protocol version 1. + /// public IBuffer Column(ReadOnlySpan name, ReadOnlySpan value) where T : struct; - /// + /// + /// Writes an array column value for the current row. + /// + /// The column name. + /// The array to write as the column value, or null to record a NULL value. + /// The same buffer instance for fluent chaining. + /// + /// This method requires protocol version 2 or later. It will throw an with if used with protocol version 1. + /// public IBuffer Column(ReadOnlySpan name, Array? value); - - /// + + /// + /// Writes a column with the specified name using the provided enumerable of values and shape information. + /// + /// The column name. + /// An enumerable of values for the column; elements are of the value type `T`. + /// An enumerable of integers describing the multidimensional shape/length(s) for the values. + /// The same instance for call chaining. + /// + /// This method requires protocol version 2 or later. It will throw an with if used with protocol version 1. + /// public IBuffer Column(ReadOnlySpan name, IEnumerable value, IEnumerable shape) where T : struct; + + /// + /// Writes a DECIMAL column with the specified name using the ILP binary decimal layout. + /// + /// The column name. + /// The decimal value to write, or `null` to write a NULL column. + /// The buffer instance for method chaining. + /// + /// This method requires protocol version 3 or later. It will throw an with if used with protocol version 1 or 2. + /// + public IBuffer Column(ReadOnlySpan name, decimal? value); } \ No newline at end of file diff --git a/src/net-questdb-client/Enums/BinaryFormatType.cs b/src/net-questdb-client/Enums/BinaryFormatType.cs index 495901f..b0ee5b6 100644 --- a/src/net-questdb-client/Enums/BinaryFormatType.cs +++ b/src/net-questdb-client/Enums/BinaryFormatType.cs @@ -31,4 +31,5 @@ public enum BinaryFormatType : byte { DOUBLE = 16, ARRAY = 14, + DECIMAL = 23, } \ No newline at end of file diff --git a/src/net-questdb-client/Enums/ProtocolVersion.cs b/src/net-questdb-client/Enums/ProtocolVersion.cs index d6bff49..3904cc2 100644 --- a/src/net-questdb-client/Enums/ProtocolVersion.cs +++ b/src/net-questdb-client/Enums/ProtocolVersion.cs @@ -40,7 +40,12 @@ public enum ProtocolVersion V1, /// - /// Text ILP with binary extensions for DOUBLE and ARRAY[DOUBLE] + /// Text ILP with binary for DOUBLE and support for the ARRAY[DOUBLE] datatype /// V2, + + /// + /// Support for the DECIMAL datatype + /// + V3, } \ No newline at end of file diff --git a/src/net-questdb-client/Senders/AbstractSender.cs b/src/net-questdb-client/Senders/AbstractSender.cs index 6fcd1b0..27f74b6 100644 --- a/src/net-questdb-client/Senders/AbstractSender.cs +++ b/src/net-questdb-client/Senders/AbstractSender.cs @@ -94,6 +94,18 @@ public ISender Column(ReadOnlySpan name, long value) return this; } + /// + /// Appends an integer-valued column with the specified name to the current buffered row. + /// + /// The column name. + /// The integer value to append for the column. + /// The same instance to allow fluent chaining. + public ISender Column(ReadOnlySpan name, int value) + { + Buffer.Column(name, value); + return this; + } + /// public ISender Column(ReadOnlySpan name, bool value) { @@ -291,7 +303,19 @@ private ValueTask FlushIfNecessaryAsync(CancellationToken ct = default) return ValueTask.CompletedTask; } - /// + /// + /// Synchronously checks auto-flush conditions and sends the buffer if thresholds are met. + /// + /// A user-provided cancellation token. + /// + /// Auto-flushing is triggered based on: + /// + /// - the number of buffered ILP rows. + /// - the current length of the buffer in UTF-8 bytes. + /// - the elapsed time interval since the last flush. + /// + /// Has no effect within a transaction or if is set to . + /// private void FlushIfNecessary(CancellationToken ct = default) { if (Options.auto_flush == AutoFlushType.on && !WithinTransaction && @@ -304,6 +328,9 @@ private void FlushIfNecessary(CancellationToken ct = default) } } + /// + /// Sets to the current UTC time if it has not been initialized. + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] private void GuardLastFlushNotSet() { @@ -312,4 +339,16 @@ private void GuardLastFlushNotSet() LastFlush = DateTime.UtcNow; } } + + /// + /// Adds a nullable decimal column value to the current row in the buffer. + /// + /// The column name. + /// The decimal value to write, or null to emit a null for the column. + /// The same instance for fluent chaining. + public ISender Column(ReadOnlySpan name, decimal? value) + { + Buffer.Column(name, value); + return this; + } } \ No newline at end of file diff --git a/src/net-questdb-client/Senders/HttpSender.cs b/src/net-questdb-client/Senders/HttpSender.cs index da4e641..12ac3ea 100644 --- a/src/net-questdb-client/Senders/HttpSender.cs +++ b/src/net-questdb-client/Senders/HttpSender.cs @@ -57,30 +57,49 @@ internal class HttpSender : AbstractSender private readonly Func _sendRequestFactory; private readonly Func _settingRequestFactory; + /// + /// Initializes a new HttpSender configured according to the provided options. + /// + /// Configuration for the sender, including connection endpoint, TLS and certificate settings, buffering and protocol parameters, authentication, and timeouts. public HttpSender(SenderOptions options) { - _sendRequestFactory = GenerateRequest; + _sendRequestFactory = GenerateRequest; _settingRequestFactory = GenerateSettingsRequest; Options = options; Build(); } + /// + /// Initializes a new instance of by parsing a configuration string. + /// + /// Configuration string in QuestDB connection string format. public HttpSender(string confStr) : this(new SenderOptions(confStr)) { } + /// + /// Configure and initialize the SocketsHttpHandler and HttpClient, set TLS and authentication options, determine the Line Protocol version (probing /settings when set to Auto), and create the internal send buffer. + /// + /// + /// - Applies pool and connection settings from Options. + /// - When using HTTPS, configures TLS protocols, optional remote-certificate validation override (when tls_verify is unsafe_off), optional custom root CA installation, and optional client certificates. + /// - Sets connection timeout, PreAuthenticate, BaseAddress, and disables HttpClient timeout. + /// - Adds Basic or Bearer Authorization header when credentials or token are provided. + /// - If protocol_version is Auto, probes the server's /settings with a 1-second retry window to select the highest mutually supported protocol up to V3, falling back to V1 on errors or unexpected responses. + /// - Initializes the Buffer with init_buf_size, max_name_len, max_buf_size, and the chosen protocol version. + /// private void Build() { _handler = new SocketsHttpHandler { PooledConnectionIdleTimeout = Options.pool_timeout, - MaxConnectionsPerServer = 1, + MaxConnectionsPerServer = 1, }; if (Options.protocol == ProtocolType.https) { - _handler.SslOptions.TargetHost = Options.Host; + _handler.SslOptions.TargetHost = Options.Host; _handler.SslOptions.EnabledSslProtocols = SslProtocols.Tls12 | SslProtocols.Tls13; if (Options.tls_verify == TlsVerifyType.unsafe_off) @@ -122,21 +141,21 @@ private void Build() } } - _handler.ConnectTimeout = Options.auth_timeout; + _handler.ConnectTimeout = Options.auth_timeout; _handler.PreAuthenticate = true; _client = new HttpClient(_handler); var uri = new UriBuilder(Options.protocol.ToString(), Options.Host, Options.Port); _client.BaseAddress = uri.Uri; - _client.Timeout = Timeout.InfiniteTimeSpan; + _client.Timeout = Timeout.InfiniteTimeSpan; if (!string.IsNullOrEmpty(Options.username) && !string.IsNullOrEmpty(Options.password)) { _client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", - Convert.ToBase64String( - Encoding.ASCII.GetBytes( - $"{Options.username}:{Options.password}"))); + Convert.ToBase64String( + Encoding.ASCII.GetBytes( + $"{Options.username}:{Options.password}"))); } else if (!string.IsNullOrEmpty(Options.token)) { @@ -147,7 +166,7 @@ private void Build() if (protocolVersion == ProtocolVersion.Auto) { - // We need to see if this will be a V1 or a V2 + // We need to select the last version that both client and server support. // Other clients use 1 second timeout for "/settings", follow same practice here. using var response = SendWithRetries(default, _settingRequestFactory, TimeSpan.FromSeconds(1)); if (!response.IsSuccessStatusCode) @@ -168,17 +187,9 @@ private void Build() { try { - var json = response.Content.ReadFromJsonAsync().Result!; + var json = response.Content.ReadFromJsonAsync().Result!; var versions = json.Config?.LineProtoSupportVersions!; - foreach (var element in versions) - { - if (element == (int)ProtocolVersion.V2) - { - // V2 is supported, use it. - protocolVersion = ProtocolVersion.V2; - break; - } - } + protocolVersion = (ProtocolVersion)versions.Where(v => v <= (int)ProtocolVersion.V3).Max(); } catch { @@ -200,15 +211,20 @@ private void Build() ); } + /// + /// Creates an HTTP GET request to the /settings endpoint for querying server capabilities. + /// + /// A new configured for the /settings endpoint. private static HttpRequestMessage GenerateSettingsRequest() { return new HttpRequestMessage(HttpMethod.Get, "/settings"); } /// - /// Creates a new HTTP request with appropriate encoding and timeout. + /// Creates a new cancellation token source linked to the provided token and configured with the calculated request timeout. /// - /// + /// Optional cancellation token to link. + /// A configured with the request timeout. private CancellationTokenSource GenerateRequestCts(CancellationToken ct = default) { var cts = CancellationTokenSource.CreateLinkedTokenSource(ct); @@ -217,14 +233,14 @@ private CancellationTokenSource GenerateRequestCts(CancellationToken ct = defaul } /// - /// Creates a new HTTP request with appropriate encoding. + /// Create an HTTP POST request targeting "/write" with the sender's buffer as the request body. /// - /// + /// An configured with the buffer as the request body, Content-Type set to "text/plain" with charset "utf-8", and Content-Length set to the buffer length. private HttpRequestMessage GenerateRequest() { var request = new HttpRequestMessage(HttpMethod.Post, "/write") { Content = new BufferStreamContent(Buffer), }; - request.Content.Headers.ContentType = new MediaTypeHeaderValue("text/plain") { CharSet = "utf-8", }; + request.Content.Headers.ContentType = new MediaTypeHeaderValue("text/plain") { CharSet = "utf-8", }; request.Content.Headers.ContentLength = Buffer.Length; return request; } @@ -250,13 +266,13 @@ public override ISender Transaction(ReadOnlySpan tableName) if (WithinTransaction) { throw new IngressError(ErrorCode.InvalidApiCall, - "Cannot start another transaction - only one allowed at a time."); + "Cannot start another transaction - only one allowed at a time."); } if (Length > 0) { throw new IngressError(ErrorCode.InvalidApiCall, - "Buffer must be clear before you can start a transaction."); + "Buffer must be clear before you can start a transaction."); } Buffer.Transaction(tableName); @@ -264,7 +280,6 @@ public override ISender Transaction(ReadOnlySpan tableName) } /// - /// /> public override void Commit(CancellationToken ct = default) { try @@ -315,7 +330,15 @@ public override void Rollback() Buffer.Clear(); } - /// + /// + /// Sends the current buffer synchronously to the server, applying configured retries and handling server-side errors. + /// + /// + /// Validates that a pending transaction is being committed before sending. If the buffer is empty this method returns immediately. + /// On success updates from the server response date; on failure sets to now. The buffer is always cleared after the operation. + /// + /// Cancellation token to cancel the send operation. + /// Thrown with if a transaction is open but not committing, or with for server/transport errors. public override void Send(CancellationToken ct = default) { if (WithinTransaction && !CommittingTransaction) @@ -337,7 +360,7 @@ public override void Send(CancellationToken ct = default) if (response.IsSuccessStatusCode) { LastFlush = (response.Headers.Date ?? DateTime.UtcNow).UtcDateTime; - success = true; + success = true; return; } @@ -369,11 +392,21 @@ public override void Send(CancellationToken ct = default) } } - private HttpResponseMessage SendWithRetries(CancellationToken ct, Func requestFactory, TimeSpan retryTimeout) + /// + /// Sends an HTTP request produced by and retries on transient connection or server errors until a successful response is received or elapses. + /// + /// Cancellation token used to cancel the overall operation and linked to per-request timeouts. + /// Factory that produces a fresh for each attempt. + /// Maximum duration to keep retrying transient failures; retries are skipped if this is zero. + /// The final returned by the server for a successful request. + /// Thrown with when a connection could not be established within the allowed retries. + /// The caller is responsible for disposing the returned ./// + private HttpResponseMessage SendWithRetries(CancellationToken ct, Func requestFactory, + TimeSpan retryTimeout) { - HttpResponseMessage? response = null; - CancellationTokenSource cts = GenerateRequestCts(ct); - HttpRequestMessage request = requestFactory(); + HttpResponseMessage? response = null; + CancellationTokenSource cts = GenerateRequestCts(ct); + HttpRequestMessage request = requestFactory(); try { @@ -389,7 +422,7 @@ private HttpResponseMessage SendWithRetries(CancellationToken ct, Func TimeSpan.Zero) // retry if appropriate - error that's retriable, and retries are enabled { - if (response == null // if it was a cannot correct error + if (response == null // if it was a cannot correct error || (!response.IsSuccessStatusCode // or some other http error && IsRetriableError(response.StatusCode))) { @@ -399,9 +432,9 @@ private HttpResponseMessage SendWithRetries(CancellationToken ct, Func + /// Reads and deserializes a JSON error response from the HTTP response, then throws an with the error details. + /// + /// The HTTP response containing a JSON error body. + /// Always thrown with ; the message combines the response reason phrase with the deserialized JSON error or raw response text. private void HandleErrorJson(HttpResponseMessage response) { using var respStream = response.Content.ReadAsStream(); @@ -460,6 +498,11 @@ private void HandleErrorJson(HttpResponseMessage response) } } + /// + /// Read an error payload from the HTTP response (JSON if possible, otherwise raw text) and throw an IngressError containing the server reason and the parsed error details. + /// + /// The HTTP response containing a JSON or plain-text error body. + /// Always thrown with ; the message contains response.ReasonPhrase followed by the deserialized JSON error or the raw response body. private async Task HandleErrorJsonAsync(HttpResponseMessage response) { await using var respStream = await response.Content.ReadAsStreamAsync(); @@ -471,7 +514,7 @@ private async Task HandleErrorJsonAsync(HttpResponseMessage response) catch (JsonException) { using var strReader = new StreamReader(respStream); - var errorStr = await strReader.ReadToEndAsync(); + var errorStr = await strReader.ReadToEndAsync(); throw new IngressError(ErrorCode.ServerFlushError, $"{response.ReasonPhrase}. {errorStr}"); } } @@ -489,14 +532,14 @@ public override async Task SendAsync(CancellationToken ct = default) return; } - HttpRequestMessage? request = null; - CancellationTokenSource? cts = null; - HttpResponseMessage? response = null; + HttpRequestMessage? request = null; + CancellationTokenSource? cts = null; + HttpResponseMessage? response = null; try { request = GenerateRequest(); - cts = GenerateRequestCts(ct); + cts = GenerateRequestCts(ct); try { @@ -510,7 +553,7 @@ public override async Task SendAsync(CancellationToken ct = default) // retry if appropriate - error that's retriable, and retries are enabled if (Options.retry_timeout > TimeSpan.Zero) { - if (response == null // if it was a cannot correct error + if (response == null // if it was a cannot correct error || (!response.IsSuccessStatusCode // or some other http error && IsRetriableError(response.StatusCode))) { @@ -520,9 +563,9 @@ public override async Task SendAsync(CancellationToken ct = default) while (retryTimer.Elapsed < Options.retry_timeout // whilst we can still retry && ( - response == null || // either we can't connect - (!response.IsSuccessStatusCode && // or we have another http error - IsRetriableError(response.StatusCode))) + response == null || // either we can't connect + (!response.IsSuccessStatusCode && // or we have another http error + IsRetriableError(response.StatusCode))) ) { retryInterval = TimeSpan.FromMilliseconds(Math.Min(retryInterval.TotalMilliseconds * 2, 1000)); @@ -534,7 +577,7 @@ public override async Task SendAsync(CancellationToken ct = default) cts = null; request = GenerateRequest(); - cts = GenerateRequestCts(ct); + cts = GenerateRequestCts(ct); var jitter = TimeSpan.FromMilliseconds(Random.Shared.Next(0, 10) - 10 / 2.0); await Task.Delay(retryInterval + jitter, cts.Token); @@ -542,7 +585,7 @@ public override async Task SendAsync(CancellationToken ct = default) try { response = await _client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, - cts.Token); + cts.Token); } catch (HttpRequestException) { @@ -556,7 +599,7 @@ public override async Task SendAsync(CancellationToken ct = default) if (response == null) { throw new IngressError(ErrorCode.ServerFlushError, - $"Cannot connect to `{Options.Host}:{Options.Port}`"); + $"Cannot connect to `{Options.Host}:{Options.Port}`"); } // return if ok @@ -594,10 +637,10 @@ public override async Task SendAsync(CancellationToken ct = default) } /// - /// Specifies whether a negative will lead to a retry or to an exception. + /// Determines whether the specified HTTP status code represents a transient error that should be retried. /// - /// The - /// + /// The HTTP status code to check. + /// true if the error is transient and retriable (e.g., 500, 503, 504, 509, 523, 524, 529, 599); otherwise, false. // ReSharper disable once IdentifierTypo private static bool IsRetriableError(HttpStatusCode code) { @@ -626,4 +669,4 @@ public override void Dispose() Buffer.Clear(); Buffer.TrimExcessBuffers(); } -} +} \ No newline at end of file diff --git a/src/net-questdb-client/Senders/ISender.cs b/src/net-questdb-client/Senders/ISender.cs index 26101ee..3b117e7 100644 --- a/src/net-questdb-client/Senders/ISender.cs +++ b/src/net-questdb-client/Senders/ISender.cs @@ -22,25 +22,14 @@ * ******************************************************************************/ -// ReSharper disable CommentTypo - using QuestDB.Utils; -// ReSharper disable InconsistentNaming - namespace QuestDB.Senders; /// /// Interface representing implementations. /// -public interface ISender : ISenderV2 -{ -} - -/// -/// Version 1 of the Sender API. -/// -public interface ISenderV1 : IDisposable +public interface ISender : IDisposable { /// /// Represents the current length of the buffer in UTF-8 bytes. @@ -53,7 +42,7 @@ public interface ISenderV1 : IDisposable public int RowCount { get; } /// - /// Represents whether or not the Sender is in a transactional state. + /// Represents whether the Sender is in a transactional state. /// public bool WithinTransaction { get; } @@ -131,22 +120,55 @@ public interface ISenderV1 : IDisposable /// /// The name of the column /// The value for the column - /// Itself + /// + /// Adds a column (field) with the specified string value to the current row. + /// + /// The column name. + /// The column value as a character span. + /// The sender instance for fluent chaining. public ISender Column(ReadOnlySpan name, ReadOnlySpan value); - /// + /// + /// Adds a column with the specified name and 64-bit integer value to the current row. + /// + /// The column (field) name. + /// The 64-bit integer value for the column. + /// The current sender instance for method chaining. public ISender Column(ReadOnlySpan name, long value); - /// + /// + public ISender Column(ReadOnlySpan name, int value); + + /// + /// Adds a boolean field column with the specified name and value to the current row. + /// + /// The column (field) name. + /// The boolean value to store in the column. + /// The same instance to allow fluent chaining. public ISender Column(ReadOnlySpan name, bool value); - /// + /// + /// Adds a double-precision field column to the current row. + /// + /// The column (field) name. + /// The column's double-precision value. + /// The same instance for fluent chaining. public ISender Column(ReadOnlySpan name, double value); - /// + /// + /// Adds a column (field) with the specified DateTime value to the current row. + /// + /// The column name. + /// The DateTime value to add. + /// The same instance for fluent chaining. public ISender Column(ReadOnlySpan name, DateTime value); - /// + /// + /// Adds a column with the specified name and DateTimeOffset value to the current row. + /// + /// The column name. + /// The DateTimeOffset value to store for the column (used as a timestamp value). + /// The sender instance for fluent chaining. public ISender Column(ReadOnlySpan name, DateTimeOffset value); /// @@ -214,45 +236,55 @@ public interface ISenderV1 : IDisposable public void CancelRow(); /// - /// Clears the sender's buffer. + /// Clears the sender's internal buffer and resets buffer-related state, removing all pending rows. /// public void Clear(); -} -/// -/// Version 2 of the Sender API, adding ARRAY and binary DOUBLE support. -/// -public interface ISenderV2 : ISenderV1 -{ - /// + /// + /// Adds a column to the current row using a sequence of value-type elements and an explicit multidimensional shape. + /// + /// The element value type stored in the column. + /// The column name. + /// A sequence of elements that form the column's data. + /// A sequence of integers describing the dimensions of the array representation; dimension lengths must match the number of elements in when multiplied together. + /// The same instance for fluent chaining. public ISender Column(ReadOnlySpan name, IEnumerable value, IEnumerable shape) where T : struct; /// - /// Adds an ARRAY to the current row. - /// Arrays are n-dimensional non-jagged arrays. + /// Adds a column whose value is provided as a native array; multidimensional (non-jagged) arrays are supported. /// - /// - /// - /// + /// The column name. + /// A native array containing the column data. Multidimensional arrays are treated as shaped data (do not pass jagged arrays). + /// The sender instance for fluent chaining. public ISender Column(ReadOnlySpan name, Array value); - /// + /// + /// Adds a column with the specified name and a sequence of value-type elements from a span to the current row. + /// + /// The column (field) name. + /// A contiguous sequence of value-type elements representing the column data. + /// The same instance to allow fluent chaining. public ISender Column(ReadOnlySpan name, ReadOnlySpan value) where T : struct; /// - /// Adds a column (field) to the current row. + /// Adds a column with the specified string value to the current row. /// - /// The name of the column - /// The value for the column - /// Itself + /// The column name. + /// The column's string value; may be null. + /// The same sender instance for fluent chaining. public ISender Column(ReadOnlySpan name, string? value) { - return ((ISenderV1)this).Column(name, value); + if (value is null) return this; + return Column(name, value.AsSpan()); } - /// + /// + /// Adds a column whose value is a sequence of value-type elements with the given multidimensional shape when both and are provided; no action is taken if either is null. + /// + /// The column name. + /// The sequence of elements for the column, or null to skip adding the column. + /// The dimensions describing the array shape, or null to skip adding the column. + /// This sender instance for fluent chaining. public ISender NullableColumn(ReadOnlySpan name, IEnumerable? value, IEnumerable? shape) where T : struct { @@ -261,10 +293,15 @@ public ISender NullableColumn(ReadOnlySpan name, IEnumerable? value, Column(name, value, shape); } - return (ISender)this; + return this; } - /// + /// + /// Adds a column using a native array value when the provided array is non-null. + /// + /// The column name. + /// The array to use as the column value; if null, no column is added. Multidimensional arrays are supported (non-jagged). + /// The same instance for fluent chaining. public ISender NullableColumn(ReadOnlySpan name, Array? value) { if (value != null) @@ -272,10 +309,15 @@ public ISender NullableColumn(ReadOnlySpan name, Array? value) Column(name, value); } - return (ISender)this; + return this; } - /// + /// + /// Adds a string column with the given name when the provided value is not null. + /// + /// The column name. + /// The string value to add; if null, no column is added. + /// The current sender instance for fluent chaining. public ISender NullableColumn(ReadOnlySpan name, string? value) { if (value != null) @@ -283,10 +325,15 @@ public ISender NullableColumn(ReadOnlySpan name, string? value) Column(name, value); } - return (ISender)this; + return this; } - /// + /// + /// Adds a long column with the specified name when the provided nullable value has a value; does nothing when the value is null. + /// + /// The column name. + /// The nullable long value to add as a column; if null the sender is unchanged. + /// The current sender instance for fluent chaining. public ISender NullableColumn(ReadOnlySpan name, long? value) { if (value != null) @@ -294,10 +341,15 @@ public ISender NullableColumn(ReadOnlySpan name, long? value) return Column(name, value ?? throw new InvalidOperationException()); } - return (ISender)this; + return this; } - /// + /// + /// Adds a boolean column with the given name when a value is provided; does nothing if the value is null. + /// + /// The column name. + /// The nullable boolean value to add as a column. + /// The current sender instance to allow fluent chaining. public ISender NullableColumn(ReadOnlySpan name, bool? value) { if (value != null) @@ -305,10 +357,15 @@ public ISender NullableColumn(ReadOnlySpan name, bool? value) return Column(name, value ?? throw new InvalidOperationException()); } - return (ISender)this; + return this; } - /// + /// + /// Adds a column with the given double value when the value is non-null; otherwise no column is added and the sender is unchanged. + /// + /// The column name. + /// The column value; if non-null, the value is written as a double field. + /// The sender instance after the operation. public ISender NullableColumn(ReadOnlySpan name, double? value) { if (value != null) @@ -316,10 +373,15 @@ public ISender NullableColumn(ReadOnlySpan name, double? value) return Column(name, value ?? throw new InvalidOperationException()); } - return (ISender)this; + return this; } - /// + /// + /// Adds a DateTime column with the specified name when a value is provided; no action is taken if the value is null. + /// + /// The column name. + /// The nullable DateTime value to add as a column. + /// The current instance for fluent chaining; unchanged if is null. public ISender NullableColumn(ReadOnlySpan name, DateTime? value) { if (value != null) @@ -327,10 +389,15 @@ public ISender NullableColumn(ReadOnlySpan name, DateTime? value) return Column(name, value ?? throw new InvalidOperationException()); } - return (ISender)this; + return this; } - /// + /// + /// Adds a column with the given name and DateTimeOffset value when a value is provided; does nothing if the value is null. + /// + /// The column name. + /// The DateTimeOffset value to add; if null the column is not added. + /// The same instance to allow fluent chaining. public ISender NullableColumn(ReadOnlySpan name, DateTimeOffset? value) { if (value != null) @@ -338,6 +405,14 @@ public ISender NullableColumn(ReadOnlySpan name, DateTimeOffset? value) return Column(name, value ?? throw new InvalidOperationException()); } - return (ISender)this; + return this; } + + /// + /// Adds a decimal column in binary format to the current row. + /// + /// The column name. + /// The decimal value to add; may be null to represent a NULL field. + /// The sender instance for fluent call chaining. + public ISender Column(ReadOnlySpan name, decimal? value); } \ No newline at end of file diff --git a/src/net-questdb-client/Senders/ISenderV1.cs b/src/net-questdb-client/Senders/ISenderV1.cs new file mode 100644 index 0000000..c52e0e5 --- /dev/null +++ b/src/net-questdb-client/Senders/ISenderV1.cs @@ -0,0 +1,34 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +// ReSharper disable InconsistentNaming + +namespace QuestDB.Senders; + +/// +/// Version 1 of the Sender API. +/// +public interface ISenderV1 : ISender +{ +} \ No newline at end of file diff --git a/src/net-questdb-client/Senders/ISenderV2.cs b/src/net-questdb-client/Senders/ISenderV2.cs new file mode 100644 index 0000000..7f7bfbe --- /dev/null +++ b/src/net-questdb-client/Senders/ISenderV2.cs @@ -0,0 +1,38 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2024 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +// ReSharper disable CommentTypo + +using QuestDB.Utils; + +// ReSharper disable InconsistentNaming + +namespace QuestDB.Senders; + +/// +/// Version 2 of the Sender API, adding ARRAY and binary DOUBLE support. +/// +public interface ISenderV2 : ISender +{ +} \ No newline at end of file diff --git a/src/net-questdb-client/Senders/TcpSender.cs b/src/net-questdb-client/Senders/TcpSender.cs index dad37d5..eb8435c 100644 --- a/src/net-questdb-client/Senders/TcpSender.cs +++ b/src/net-questdb-client/Senders/TcpSender.cs @@ -46,16 +46,31 @@ internal class TcpSender : AbstractSender private Secp256r1SignatureGenerator? _signatureGenerator; private Socket _underlyingSocket = null!; + /// + /// Initializes a new instance of configured with the provided options. + /// + /// Configuration options for the TCP sender including host, port, TLS settings, and authentication. public TcpSender(SenderOptions options) { Options = options; Build(); } + /// + /// Initializes a new instance of by parsing a configuration string. + /// + /// Configuration string in QuestDB connection string format. public TcpSender(string confStr) : this(new SenderOptions(confStr)) { } + /// + /// Initializes the TCP connection, configures TLS if required, creates the buffer, and performs authentication if credentials are provided. + /// + /// + /// Establishes the TCP socket connection, wraps it in SSL/TLS if protocol is tcps, validates certificates according to options, performs ECDSA authentication if a token is provided, and initializes the ILP buffer. + /// + /// Thrown if TLS handshake fails, authentication fails, or connection cannot be established. private void Build() { Buffer = Buffers.Buffer.Create( @@ -65,9 +80,9 @@ private void Build() Options.protocol_version == ProtocolVersion.Auto ? ProtocolVersion.V1 : Options.protocol_version ); - var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, System.Net.Sockets.ProtocolType.Tcp); + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, System.Net.Sockets.ProtocolType.Tcp); NetworkStream? networkStream = null; - SslStream? sslStream = null; + SslStream? sslStream = null; try { socket.ConnectAsync(Options.Host, Options.Port).Wait(); @@ -100,7 +115,7 @@ private void Build() } _underlyingSocket = socket; - _dataStream = dataStream; + _dataStream = dataStream; if (!string.IsNullOrEmpty(Options.token)) { var authTimeout = new CancellationTokenSource(); @@ -154,19 +169,19 @@ private async ValueTask AuthenticateAsync(CancellationToken ct = default) } /// - /// Receives a chunk of data from the TCP stream. + /// Asynchronously reads bytes from the TCP stream until the specified terminator character is received or the buffer is full. /// - /// - /// - /// - /// + /// The terminator character indicating the end of the message. + /// Cancellation token to cancel the read operation. + /// The number of bytes read, excluding the terminator character. + /// Thrown with if the connection is closed before the terminator is received or if the buffer is too small. private async ValueTask ReceiveUntil(char endChar, CancellationToken cancellationToken) { var totalReceived = 0; while (totalReceived < Buffer.Chunk.Length) { var received = await _dataStream.ReadAsync(Buffer.Chunk, totalReceived, - Buffer.Chunk.Length - totalReceived, cancellationToken); + Buffer.Chunk.Length - totalReceived, cancellationToken); if (received > 0) { totalReceived += received; @@ -185,10 +200,15 @@ private async ValueTask ReceiveUntil(char endChar, CancellationToken cancel throw new IngressError(ErrorCode.SocketError, "Buffer is too small to receive the message."); } + /// + /// Decodes a URL-safe Base64-encoded string (using - and _ instead of + and /) and returns the decoded byte array. + /// + /// The URL-safe Base64-encoded private key string. + /// The decoded byte array. private static byte[] FromBase64String(string encodedPrivateKey) { var urlUnsafe = encodedPrivateKey.Replace('-', '+').Replace('_', '/'); - var padding = 3 - (urlUnsafe.Length + 3) % 4; + var padding = 3 - (urlUnsafe.Length + 3) % 4; if (padding != 0) { urlUnsafe += new string('=', padding); @@ -260,4 +280,4 @@ public override void Dispose() Buffer.Clear(); Buffer.TrimExcessBuffers(); } -} +} \ No newline at end of file diff --git a/src/tcp-client-test/DummyIlpServer.cs b/src/tcp-client-test/DummyIlpServer.cs deleted file mode 100644 index 55c39d0..0000000 --- a/src/tcp-client-test/DummyIlpServer.cs +++ /dev/null @@ -1,239 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - - -using System; -using System.IO; -using System.Net; -using System.Net.Security; -using System.Net.Sockets; -using System.Security.Cryptography.X509Certificates; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Org.BouncyCastle.Asn1.Sec; -using Org.BouncyCastle.Crypto.Parameters; -using Org.BouncyCastle.Math; -using Org.BouncyCastle.Security; - -// ReSharper disable NonAtomicCompoundOperator - -namespace tcp_client_test; - -public class DummyIlpServer : IDisposable -{ - private readonly byte[] _buffer = new byte[2048]; - private readonly CancellationTokenSource _cancellationTokenSource = new(); - private readonly MemoryStream _received = new(); - private readonly TcpListener _server; - private readonly bool _tls; - private string? _keyId; - private string? _publicKeyX; - private string? _publicKeyY; - private volatile int _totalReceived; - - public DummyIlpServer(int port, bool tls) - { - _tls = tls; - _server = new TcpListener(IPAddress.Loopback, port); - _server.Start(); - } - - public int TotalReceived => _totalReceived; - - public void Dispose() - { - _cancellationTokenSource.Cancel(); - _server.Stop(); - } - - public void AcceptAsync() - { - Task.Run(AcceptConnections); - } - - private async Task AcceptConnections() - { - Socket? clientSocket = null; - try - { - using var socket = await _server.AcceptSocketAsync(); - clientSocket = socket; - await using var connection = new NetworkStream(socket, true); - Stream dataStream = connection; - if (_tls) - { - var sslStream = new SslStream(connection); - dataStream = sslStream; - await sslStream.AuthenticateAsServerAsync(GetCertificate()); - } - - if (_keyId != null) - { - await RunServerAuth(dataStream); - } - - await SaveData(dataStream, socket); - } - catch (SocketException ex) - { - Console.WriteLine($"Error {ex.ErrorCode}: Server socket error."); - } - finally - { - clientSocket?.Dispose(); - } - } - - private X509Certificate GetCertificate() - { - return X509Certificate.CreateFromCertFile("certificate.pfx"); - } - - private async Task RunServerAuth(Stream connection) - { - var receivedLen = await ReceiveUntilEol(connection); - - var requestedKeyId = Encoding.UTF8.GetString(_buffer, 0, receivedLen); - if (requestedKeyId != _keyId) - { - connection.Close(); - return; - } - - var challenge = new byte[512]; - GenerateRandomBytes(challenge, 512); - await connection.WriteAsync(challenge); - _buffer[0] = (byte)'\n'; - await connection.WriteAsync(_buffer.AsMemory(0, 1)); - - receivedLen = await ReceiveUntilEol(connection); - var signatureRaw = Encoding.UTF8.GetString(_buffer.AsSpan(0, receivedLen)); - Console.WriteLine(signatureRaw); - var signature = Convert.FromBase64String(Pad(signatureRaw)); - - if (_publicKeyX == null || _publicKeyY == null) - { - throw new InvalidOperationException("public key not set"); - } - - var pubKey1 = FromBase64String(_publicKeyX); - var pubKey2 = FromBase64String(_publicKeyY); - - var p = SecNamedCurves.GetByName("secp256r1"); - var parameters = new ECDomainParameters(p.Curve, p.G, p.N, p.H); - - // Verify the signature - var pubKey = new ECPublicKeyParameters( - parameters.Curve.CreatePoint(new BigInteger(1, pubKey1), new BigInteger(1, pubKey2)), - parameters); - - var ecdsa = SignerUtilities.GetSigner("SHA-256withECDSA"); - ecdsa.Init(false, pubKey); - ecdsa.BlockUpdate(challenge, 0, challenge.Length); - if (!ecdsa.VerifySignature(signature)) - { - connection.Close(); - } - } - - private static string Pad(string text) - { - var padding = 3 - (text.Length + 3) % 4; - if (padding == 0) - { - return text; - } - - return text + new string('=', padding); - } - - public static byte[] FromBase64String(string encodedPrivateKey) - { - var replace = encodedPrivateKey - .Replace('-', '+') - .Replace('_', '/'); - return Convert.FromBase64String(Pad(replace)); - } - - private async Task ReceiveUntilEol(Stream connection) - { - var len = 0; - while (true) - { - var n = await connection.ReadAsync(_buffer.AsMemory(len)); - var inBuffer = len + n; - for (var i = len; i < inBuffer; i++) - { - if (_buffer[i] == '\n') - { - if (i + 1 < inBuffer) - { - _received.Write(_buffer, i + 1, inBuffer - i - 1); - _totalReceived += inBuffer - i; - } - - return i; - } - } - - len += n; - } - } - - private void GenerateRandomBytes(byte[] buffer, int length) - { - var rnd = new Random(DateTime.Now.Millisecond); - rnd.NextBytes(buffer.AsSpan(length)); - } - - private async Task SaveData(Stream connection, Socket socket) - { - while (!_cancellationTokenSource.IsCancellationRequested && socket.Connected) - { - var received = await connection.ReadAsync(_buffer); - if (received > 0) - { - _received.Write(_buffer, 0, received); - _totalReceived += received; - } - else - { - return; - } - } - } - - public string GetTextReceived() - { - return Encoding.UTF8.GetString(_received.GetBuffer(), 0, (int)_received.Length); - } - - public void WithAuth(string keyId, string publicKeyX, string publicKeyY) - { - _keyId = keyId; - _publicKeyX = publicKeyX; - _publicKeyY = publicKeyY; - } -} \ No newline at end of file diff --git a/src/tcp-client-test/JsonSpecTestRunner.cs b/src/tcp-client-test/JsonSpecTestRunner.cs deleted file mode 100644 index 8dcf605..0000000 --- a/src/tcp-client-test/JsonSpecTestRunner.cs +++ /dev/null @@ -1,202 +0,0 @@ -/******************************************************************************* - * ___ _ ____ ____ - * / _ \ _ _ ___ ___| |_| _ \| __ ) - * | | | | | | |/ _ \/ __| __| | | | _ \ - * | |_| | |_| | __/\__ \ |_| |_| | |_) | - * \__\_\\__,_|\___||___/\__|____/|____/ - * - * Copyright (c) 2014-2019 Appsicle - * Copyright (c) 2019-2024 QuestDB - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - ******************************************************************************/ - - -using System; -using System.IO; -using System.Linq; -using System.Net; -using System.Text; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using NUnit.Framework; -using QuestDB; - -// ReSharper disable InconsistentNaming - -#pragma warning disable CS0618 // Type or member is obsolete - -namespace tcp_client_test; - -[TestFixture] -public class JsonSpecTestRunner -{ - private const int Port = 29472; - private static readonly TestCase[]? TestCases = ReadTestCases(); - - [TestCaseSource(nameof(TestCases))] - public async Task Run(TestCase testCase) - { - using var srv = CreateTcpListener(Port); - srv.AcceptAsync(); - -#pragma warning disable CS0612 // Type or member is obsolete - using var ls = await LineTcpSender.ConnectAsync(IPAddress.Loopback.ToString(), Port, tlsMode: TlsMode.Disable); -#pragma warning restore CS0612 // Type or member is obsolete - Exception? exception = null; - - try - { - ls.Table(testCase.table); - foreach (var symbol in testCase.symbols) - { - ls.Symbol(symbol.name, symbol.value); - } - - foreach (var column in testCase.columns) - { - switch (column.type) - { - case "STRING": - ls.Column(column.name, ((JsonElement)column.value).GetString()); - break; - - case "DOUBLE": - ls.Column(column.name, ((JsonElement)column.value).GetDouble()); - break; - - case "BOOLEAN": - ls.Column(column.name, ((JsonElement)column.value).GetBoolean()); - break; - - case "LONG": - ls.Column(column.name, (long)((JsonElement)column.value).GetDouble()); - break; - - default: - throw new NotSupportedException("Column type not supported: " + column.type); - } - } - - ls.AtNow(); - ls.Send(); - } - catch (Exception? ex) - { - if (testCase.result.status == "SUCCESS") - { - throw; - } - - exception = ex; - } - - if (testCase.result.status == "SUCCESS") - { - if (testCase.result.anyLines == null || testCase.result.anyLines.Length == 0) - { - WaitAssert(srv, testCase.result.line + "\n"); - } - else - { - WaitAssert(srv, testCase.result.anyLines); - } - } - else if (testCase.result.status == "ERROR") - { - Assert.NotNull(exception, "Exception should be thrown"); - if (exception is NotSupportedException) - { - throw exception; - } - } - else - { - Assert.Fail("Unsupported test case result status: " + testCase.result.status); - } - } - - private void WaitAssert(DummyIlpServer srv, string[] resultAnyLines) - { - var minExpectedLen = resultAnyLines.Select(l => Encoding.UTF8.GetBytes(l).Length).Min(); - for (var i = 0; i < 500 && srv.TotalReceived < minExpectedLen; i++) - { - Thread.Sleep(10); - } - - var textReceived = srv.GetTextReceived(); - var anyMatch = resultAnyLines.Any(l => (l + "\n").Equals(textReceived)); - if (!anyMatch) - { - Assert.Fail(textReceived + ": did not match any expected results"); - } - } - - private static void WaitAssert(DummyIlpServer srv, string expected) - { - var expectedLen = Encoding.UTF8.GetBytes(expected).Length; - for (var i = 0; i < 500 && srv.TotalReceived < expectedLen; i++) - { - Thread.Sleep(10); - } - - Assert.AreEqual(expected, srv.GetTextReceived()); - } - - private DummyIlpServer CreateTcpListener(int port, bool tls = false) - { - return new DummyIlpServer(port, tls); - } - - private static TestCase[]? ReadTestCases() - { - using var jsonFile = File.OpenRead("ilp-client-interop-test.json"); - return JsonSerializer.Deserialize(jsonFile); - } - - public class TestCase - { - public string testName { get; set; } = null!; - public string table { get; set; } = null!; - public TestCaseSymbol[] symbols { get; set; } = null!; - public TestCaseColumn[] columns { get; set; } = null!; - public TestCaseResult result { get; set; } = null!; - - public override string ToString() - { - return testName; - } - } - - public class TestCaseSymbol - { - public string name { get; set; } = null!; - public string value { get; set; } = null!; - } - - public class TestCaseColumn - { - public string type { get; set; } = null!; - public string name { get; set; } = null!; - public object value { get; set; } = null!; - } - - public class TestCaseResult - { - public string status { get; set; } = null!; - public string line { get; set; } = null!; - public string[]? anyLines { get; set; } = null!; - } -} \ No newline at end of file diff --git a/src/tcp-client-test/certificate.pfx b/src/tcp-client-test/certificate.pfx deleted file mode 100644 index 1c4c174..0000000 Binary files a/src/tcp-client-test/certificate.pfx and /dev/null differ diff --git a/src/tcp-client-test/tcp-client-test.csproj b/src/tcp-client-test/tcp-client-test.csproj deleted file mode 100644 index 009274b..0000000 --- a/src/tcp-client-test/tcp-client-test.csproj +++ /dev/null @@ -1,38 +0,0 @@ - - - - tcp_client_test - enable - false - net6.0;net7.0;net8.0;net9.0 - Library - - - - - - - - - - - - - - - - - Always - - - - - - ilp-client-interop-test.json - Always - - - - - -