diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index e4419f3eaa..d1f43fdeb4 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -323,3 +323,36 @@ jobs: run: | ./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build" env BUILD_DRIVER_MANAGER=0 ./ci/scripts/python_test.sh "$(pwd)" "$(pwd)/build" + + flightsql_interop: + name: "FlightSQL C# Interop" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + - name: Get required Go version + run: | + (. .env && echo "GO_VERSION=${GO}") >> $GITHUB_ENV + - uses: actions/setup-go@v5 + with: + go-version: "${{ env.GO_VERSION }}" + check-latest: true + cache: true + cache-dependency-path: go/adbc/go.sum + - name: Build ADBC Driver + working-directory: go/adbc/pkg + run: | + make libadbc_driver_flightsql.so + - name: Start Test Servers + run: | + docker compose up --wait --detach spiceai-test + - name: Test Driver against Spice.ai OSS + env: + FLIGHTSQL_INTEROP_TEST_CONFIG_FILE: "../../../../../csharp/configs/flightsql-spiceai.json" + run: | + dotnet test ./csharp/test/Drivers/Interop/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.Interop.FlightSql.csproj + - name: Stop Test Servers + run: | + docker compose down diff --git a/ci/docker/spiceai-init.dockerfile b/ci/docker/spiceai-init.dockerfile new file mode 100644 index 0000000000..50f4875df9 --- /dev/null +++ b/ci/docker/spiceai-init.dockerfile @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM spiceai/spiceai:1.4.0-models AS spiceai + +FROM debian:bookworm-slim + +RUN apt update \ + && apt install --yes ca-certificates libssl3 curl --no-install-recommends + +COPY --from=spiceai /usr/local/bin/spiced /usr/local/bin/spiced + +# Create the Spicepod configuration file +COPY < array.Data.DataType.TypeId == ArrowTypeId.Decimal256 ? ((Decimal256Array)array).GetString(index) : ((StringArray)array).GetString(index); + case ArrowTypeId.LargeString: + return (array, index) =>((LargeStringArray)array).GetString(index); #if NET6_0_OR_GREATER case ArrowTypeId.Time32: return (array, index) => ((Time32Array)array).GetTime(index); diff --git a/csharp/test/Drivers/Interop/FlightSql/ClientTests.cs b/csharp/test/Drivers/Interop/FlightSql/ClientTests.cs index 47b76af4e6..9b681e60f6 100644 --- a/csharp/test/Drivers/Interop/FlightSql/ClientTests.cs +++ b/csharp/test/Drivers/Interop/FlightSql/ClientTests.cs @@ -159,12 +159,12 @@ public void CanClientExecuteQueryWithNoResults() { foreach (FlightSqlTestEnvironment environment in _environments) { - environment.Query = "SELECT * WHERE 0=1"; + environment.Query = "SELECT 1 WHERE 0=1"; // The query simulates no results environment.ExpectedResultsCount = 0; using (Adbc.Client.AdbcConnection adbcConnection = GetFlightSqlAdbcConnectionUsingConnectionString(environment, _testConfiguration)) { - Tests.ClientTests.CanClientExecuteQuery(adbcConnection, environment, additionalCommandOptionsSetter: null, environment.Name); + Tests.ClientTests.CanClientExecuteQuery(adbcConnection, environment, additionalCommandOptionsSetter: null, environmentName: environment.Name); } } } diff --git a/csharp/test/Drivers/Interop/FlightSql/FlightSqlData.cs b/csharp/test/Drivers/Interop/FlightSql/FlightSqlData.cs index 07b0e24f34..42c5a2323a 100644 --- a/csharp/test/Drivers/Interop/FlightSql/FlightSqlData.cs +++ b/csharp/test/Drivers/Interop/FlightSql/FlightSqlData.cs @@ -50,6 +50,8 @@ FlightSqlTestEnvironmentType environmentType return GetDuckDbSampleData(); case FlightSqlTestEnvironmentType.SQLite: return GetSQLiteSampleData(); + case FlightSqlTestEnvironmentType.SpiceAI: + return GetSpiceAISampleData(); default: throw new InvalidOperationException("Unknown environment type."); } @@ -204,5 +206,73 @@ private static SampleDataBuilder GetSQLiteSampleData() return sampleDataBuilder; } + + private static SampleDataBuilder GetSpiceAISampleData() + { + SampleDataBuilder sampleDataBuilder = new SampleDataBuilder(); + + sampleDataBuilder.Samples.Add( + + // Primitive types + new SampleData() + { + Query = "SELECT " + + "TRUE AS \"Boolean\", " + + "32767::SMALLINT AS \"SmallInt\", " + + "2147483647::INTEGER AS \"Integer\", " + + "9223372036854775807::BIGINT AS \"BigInt\", " + + "3.14::REAL AS \"Real\", " + + "3.141592653589793::DOUBLE PRECISION AS \"Double\", " + + "'12345.67'::NUMERIC(10,2) AS \"Decimal\", " + + "'Hello, Arrow!'::TEXT AS \"Varchar\", " + + "'2024-09-10'::DATE AS \"Date\", " + + "'12:34:56'::TIME AS \"Time\", " + + "'2024-09-10 12:34:56'::TIMESTAMP AS \"Timestamp\", " + + "INTERVAL '1 year' AS \"Interval\"", + ExpectedValues = new List() + { + new ColumnNetTypeArrowTypeValue("Boolean", typeof(bool), typeof(BooleanType), true), + new ColumnNetTypeArrowTypeValue("SmallInt", typeof(short), typeof(Int16Type), (short)32767), + new ColumnNetTypeArrowTypeValue("Integer", typeof(int), typeof(Int32Type), 2147483647), + new ColumnNetTypeArrowTypeValue("BigInt", typeof(long), typeof(Int64Type), 9223372036854775807L), + new ColumnNetTypeArrowTypeValue("Real", typeof(float), typeof(FloatType), 3.14f), + new ColumnNetTypeArrowTypeValue("Double", typeof(double), typeof(DoubleType), 3.141592653589793d), + new ColumnNetTypeArrowTypeValue("Decimal", typeof(SqlDecimal), typeof(Decimal128Type), new SqlDecimal(12345.67m)), + new ColumnNetTypeArrowTypeValue("Varchar", typeof(string), typeof(StringType), "Hello, Arrow!"), + new ColumnNetTypeArrowTypeValue("Date", typeof(DateTime), typeof(Date32Type), new DateTime(2024, 9, 10)), +#if NET6_0_OR_GREATER + new ColumnNetTypeArrowTypeValue("Time", typeof(TimeOnly), typeof(Time64Type), new TimeOnly(12, 34, 56)), +#else + new ColumnNetTypeArrowTypeValue("Time", typeof(TimeSpan), typeof(Time64Type), new TimeSpan(12, 34, 56)), +#endif + new ColumnNetTypeArrowTypeValue("Timestamp", typeof(DateTimeOffset), typeof(TimestampType), new DateTimeOffset(new DateTime(2024, 9, 10, 12, 34, 56), TimeSpan.Zero)), + new ColumnNetTypeArrowTypeValue("Interval", typeof(MonthDayNanosecondInterval), typeof(IntervalType), new MonthDayNanosecondInterval(12, 0, 0)), + } + } + ); + + Dictionary struct_record = new Dictionary(); + struct_record["c0"] = 0; + struct_record["c1"] = "Test Value"; + + sampleDataBuilder.Samples.Add( + // Lists and Structs + new SampleData() + { + StructBehavior = "Strict", + Query = "SELECT " + + "ARRAY[n_regionkey, n_nationkey] AS \"List\", " + + "struct(n_regionkey, 'Test Value') AS Struct "+ + "FROM nation WHERE n_regionkey = 0 AND n_nationkey = 5", + ExpectedValues = new List() + { + new ColumnNetTypeArrowTypeValue("List", typeof(Int32Array), typeof(ListType), new Int32Array.Builder().AppendRange(new[] { 0, 5}).Build()), + new ColumnNetTypeArrowTypeValue("Struct", typeof(Dictionary), typeof(StructType), struct_record), + } + } + ); + + return sampleDataBuilder; + } } } diff --git a/csharp/test/Drivers/Interop/FlightSql/FlightSqlTestConfiguration.cs b/csharp/test/Drivers/Interop/FlightSql/FlightSqlTestConfiguration.cs index 971fa9163d..f9af1795ef 100644 --- a/csharp/test/Drivers/Interop/FlightSql/FlightSqlTestConfiguration.cs +++ b/csharp/test/Drivers/Interop/FlightSql/FlightSqlTestConfiguration.cs @@ -40,7 +40,8 @@ internal enum FlightSqlTestEnvironmentType Denodo, Dremio, DuckDB, - SQLite + SQLite, + SpiceAI, } internal class FlightSqlTestEnvironment : TestConfiguration diff --git a/csharp/test/Drivers/Interop/FlightSql/readme.md b/csharp/test/Drivers/Interop/FlightSql/readme.md index d3fc3162c9..323b9b7b0f 100644 --- a/csharp/test/Drivers/Interop/FlightSql/readme.md +++ b/csharp/test/Drivers/Interop/FlightSql/readme.md @@ -32,6 +32,7 @@ A growing number of data sources support Arrow Flight SQL. This library has test - [Dremio](https://docs.dremio.com/current/sonar/developing-client-apps/arrow-flight-sql/) - [DuckDB](https://github.com/voltrondata/SQLFlite) - [SQLite](https://github.com/voltrondata/SQLFlite) +- [Spice.ai OSS](https://github.com/spiceai/spiceai) It is recommended you test your data source with the Flight SQL Go driver to ensure compatibility, since each data source can implement the Flight protocol slightly differently. @@ -49,6 +50,7 @@ A sample configuration file is provided in the Resources directory. The configur - Denodo - DuckDB - SQLite + - SpiceAI - **tableTypes**: The table types to include in the GetObjects call - **sqlFile**: A path to a SQL file to run queries to test CRUD operations - **metadata**: Used for the GetObjects calls @@ -83,3 +85,31 @@ simultaneously. To use multiple data sources, you can configure them like: ... } ``` + +### Spice.ai OSS Configuration + +Use the following command to run a local test instance of Spice.ai OSS. An example test configuration is available at [flightsql-spiceai.json](/csharp/configs/flightsql-spiceai.json). + +```bash +docker compose up spiceai-test +``` + +Output: + +```console +[+] Running 1/0 + ✔ Container adbc-spiceai Created 0.0s +Attaching to adbc-spiceai +adbc-spiceai | 2025-06-12T22:44:38.684347Z INFO spiced: Starting runtime v1.4.0+models +adbc-spiceai | 2025-06-12T22:44:38.684737Z INFO runtime::init::caching: Initialized results cache; max size: 128.00 MiB, item ttl: 1s +adbc-spiceai | 2025-06-12T22:44:38.684772Z INFO runtime::init::caching: Initialized search results cache; +adbc-spiceai | 2025-06-12T22:44:39.043113Z INFO runtime::init::dataset: Initializing dataset nation +adbc-spiceai | 2025-06-12T22:44:39.044803Z INFO runtime::metrics_server: Spice Runtime Metrics listening on 0.0.0.0:9090 +adbc-spiceai | 2025-06-12T22:44:39.044866Z INFO runtime::opentelemetry: Spice Runtime OpenTelemetry listening on 127.0.0.1:50052 +adbc-spiceai | 2025-06-12T22:44:39.044926Z INFO runtime::http: Spice Runtime HTTP listening on 0.0.0.0:8090 +adbc-spiceai | 2025-06-12T22:44:39.044351Z INFO runtime::flight: Spice Runtime Flight listening on 0.0.0.0:50051 +adbc-spiceai | 2025-06-12T22:44:39.848102Z INFO runtime::init::dataset: Dataset nation registered (s3://spiceai-demo-datasets/tpch/nation/), acceleration (arrow), results cache enabled. +adbc-spiceai | 2025-06-12T22:44:39.849943Z INFO runtime::accelerated_table::refresh_task: Loading data for dataset nation +adbc-spiceai | 2025-06-12T22:44:40.537458Z INFO runtime::accelerated_table::refresh_task: Loaded 25 rows (3.35 kiB) for dataset nation in 687ms. +adbc-spiceai | 2025-06-12T22:44:40.568947Z INFO runtime: All components are loaded. Spice runtime is ready! +```