test(csharp/src/Drivers): add Spice.ai test target

.github/workflows/integration.yml

@@ -323,3 +323,36 @@ jobs:
         run: |
           ./ci/scripts/python_build.sh "$(pwd)" "$(pwd)/build"
           env BUILD_DRIVER_MANAGER=0 ./ci/scripts/python_test.sh "$(pwd)" "$(pwd)/build"
+
+  flightsql_interop:
+    name: "FlightSQL C# Interop"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Get required Go version
+        run: |
+          (. .env && echo "GO_VERSION=${GO}") >> $GITHUB_ENV
+      - uses: actions/setup-go@v5
+        with:
+          go-version: "${{ env.GO_VERSION }}"
+          check-latest: true
+          cache: true
+          cache-dependency-path: go/adbc/go.sum
+      - name: Build ADBC Driver
+        working-directory: go/adbc/pkg
+        run: |
+          make libadbc_driver_flightsql.so
+      - name: Start Test Servers
+        run: |
+          docker compose up --wait --detach spiceai-test
+      - name: Test Driver against Spice.ai OSS
+        env:
+          FLIGHTSQL_INTEROP_TEST_CONFIG_FILE: "../../../../../csharp/configs/flightsql-spiceai.json"
+        run: |
+          dotnet test ./csharp/test/Drivers/Interop/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.Interop.FlightSql.csproj
+      - name: Stop Test Servers
+        run: |
+          docker compose down

ci/docker/spiceai-init.dockerfile

@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+FROM spiceai/spiceai:1.4.0-models AS spiceai
+
+FROM debian:bookworm-slim
+
+RUN apt update \
+  && apt install --yes ca-certificates libssl3 curl --no-install-recommends
+
+COPY --from=spiceai /usr/local/bin/spiced /usr/local/bin/spiced
+
+# Create the Spicepod configuration file
+COPY <<EOF /app/spicepod.yaml
+version: v1
+kind: Spicepod
+name: app
+
+datasets:
+  - from: s3://spiceai-demo-datasets/tpch/nation/
+    name: nation
+    params:
+      file_format: parquet
+      client_timeout: 60s
+    acceleration:
+      enabled: true
+EOF
+
+EXPOSE 8090
+EXPOSE 9090
+EXPOSE 50051
+
+WORKDIR /app
+
+# Start the Spice runtime with specified arguments
+CMD ["/usr/local/bin/spiced", "--http", "0.0.0.0:8090", "--metrics", "0.0.0.0:9090", "--flight", "0.0.0.0:50051"]

compose.yaml

@@ -293,6 +293,20 @@ services:
     ports:
       - "5432:5432"
 
+  spiceai-test:
+    container_name: adbc-spiceai
+    build:
+      context: .
+      dockerfile: ci/docker/spiceai-init.dockerfile
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "http://localhost:8090/v1/ready"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    ports:
+      - "50051:50051"
+      - "8090:8090"
+
   ################################ Verification ################################
 
   verify-all-ubuntu:

csharp/configs/flightsql-spiceai.json

@@ -0,0 +1,28 @@
+{
+    "driverPath": "../../../../../go/adbc/pkg/libadbc_driver_flightsql.so",
+    "driverEntryPoint": "FlightSqlDriverInit",
+    "testEnvironments": [
+        "SpiceAI_Local"
+    ],
+    "environments": {
+        "SpiceAI_Local": {
+            "uri": "grpc://127.0.0.1:50051",
+            "type": "SpiceAI",
+            "sslSkipVerify": true,
+            "supportsWriteUpdate": false,
+            "supportsCatalogs": true,
+            "caseSensitive": true,
+            "tableTypes": [
+                "BASE TABLE"
+            ],
+            "metadata": {
+                "catalog": "spice",
+                "schema": "public",
+                "table": "nation",
+                "expectedColumnCount": 4
+            },
+            "query": "select * from nation limit 5",
+            "expectedResults": 5
+        }
+    }
+}

csharp/src/Apache.Arrow.Adbc/Extensions/IArrowArrayExtensions.cs

@@ -103,6 +103,8 @@ public static class IArrowArrayExtensions
                     return ((Int64Array)arrowArray).GetValue(index);
                 case ArrowTypeId.String:
                     return ((StringArray)arrowArray).GetString(index);
+                case ArrowTypeId.LargeString:
+                    return ((LargeStringArray)arrowArray).GetString(index);
 #if NET6_0_OR_GREATER
                 case ArrowTypeId.Time32:
                     return ((Time32Array)arrowArray).GetTime(index);
@@ -235,6 +237,8 @@ public static class IArrowArrayExtensions
                     return (array, index) => array.Data.DataType.TypeId == ArrowTypeId.Decimal256 ?
                         ((Decimal256Array)array).GetString(index) :
                         ((StringArray)array).GetString(index);
+                case ArrowTypeId.LargeString:
+                    return (array, index) =>((LargeStringArray)array).GetString(index);
 #if NET6_0_OR_GREATER
                 case ArrowTypeId.Time32:
                     return (array, index) => ((Time32Array)array).GetTime(index);

csharp/test/Drivers/Interop/FlightSql/ClientTests.cs

@@ -159,12 +159,12 @@ public void CanClientExecuteQueryWithNoResults()
         {
             foreach (FlightSqlTestEnvironment environment in _environments)
             {
-                environment.Query = "SELECT * WHERE 0=1";
+                environment.Query = "SELECT 1 WHERE 0=1"; // The query simulates no results
                 environment.ExpectedResultsCount = 0;
 
                 using (Adbc.Client.AdbcConnection adbcConnection = GetFlightSqlAdbcConnectionUsingConnectionString(environment, _testConfiguration))
                 {
-                    Tests.ClientTests.CanClientExecuteQuery(adbcConnection, environment, additionalCommandOptionsSetter: null, environment.Name);
+                    Tests.ClientTests.CanClientExecuteQuery(adbcConnection, environment, additionalCommandOptionsSetter: null, environmentName: environment.Name);
                 }
             }
         }

csharp/test/Drivers/Interop/FlightSql/FlightSqlData.cs

@@ -50,6 +50,8 @@ FlightSqlTestEnvironmentType environmentType
                     return GetDuckDbSampleData();
                 case FlightSqlTestEnvironmentType.SQLite:
                     return GetSQLiteSampleData();
+                case FlightSqlTestEnvironmentType.SpiceAI:
+                    return GetSpiceAISampleData();
                 default:
                     throw new InvalidOperationException("Unknown environment type.");
             }
@@ -204,5 +206,73 @@ private static SampleDataBuilder GetSQLiteSampleData()
 
             return sampleDataBuilder;
         }
+
+        private static SampleDataBuilder GetSpiceAISampleData()
+        {
+            SampleDataBuilder sampleDataBuilder = new SampleDataBuilder();
+
+            sampleDataBuilder.Samples.Add(
+
+                // Primitive types
+                new SampleData()
+                {
+                    Query = "SELECT " +
+                            "TRUE AS \"Boolean\", " +
+                            "32767::SMALLINT AS \"SmallInt\", " +
+                            "2147483647::INTEGER AS \"Integer\", " +
+                            "9223372036854775807::BIGINT AS \"BigInt\", " +
+                            "3.14::REAL AS \"Real\", " +
+                            "3.141592653589793::DOUBLE PRECISION AS \"Double\", " +
+                            "'12345.67'::NUMERIC(10,2) AS \"Decimal\", " +
+                            "'Hello, Arrow!'::TEXT AS \"Varchar\", " +
+                            "'2024-09-10'::DATE AS \"Date\", " +
+                            "'12:34:56'::TIME AS \"Time\", " +
+                            "'2024-09-10 12:34:56'::TIMESTAMP AS \"Timestamp\", " +
+                            "INTERVAL '1 year' AS \"Interval\"",
+                    ExpectedValues = new List<ColumnNetTypeArrowTypeValue>()
+                    {
+                        new ColumnNetTypeArrowTypeValue("Boolean", typeof(bool), typeof(BooleanType), true),
+                        new ColumnNetTypeArrowTypeValue("SmallInt", typeof(short), typeof(Int16Type), (short)32767),
+                        new ColumnNetTypeArrowTypeValue("Integer", typeof(int), typeof(Int32Type), 2147483647),
+                        new ColumnNetTypeArrowTypeValue("BigInt", typeof(long), typeof(Int64Type), 9223372036854775807L),
+                        new ColumnNetTypeArrowTypeValue("Real", typeof(float), typeof(FloatType), 3.14f),
+                        new ColumnNetTypeArrowTypeValue("Double", typeof(double), typeof(DoubleType), 3.141592653589793d),
+                        new ColumnNetTypeArrowTypeValue("Decimal", typeof(SqlDecimal), typeof(Decimal128Type), new SqlDecimal(12345.67m)),
+                        new ColumnNetTypeArrowTypeValue("Varchar", typeof(string), typeof(StringType), "Hello, Arrow!"),
+                        new ColumnNetTypeArrowTypeValue("Date", typeof(DateTime), typeof(Date32Type), new DateTime(2024, 9, 10)),
+#if NET6_0_OR_GREATER
+                        new ColumnNetTypeArrowTypeValue("Time", typeof(TimeOnly), typeof(Time64Type), new TimeOnly(12, 34, 56)),
+#else
+                        new ColumnNetTypeArrowTypeValue("Time", typeof(TimeSpan), typeof(Time64Type), new TimeSpan(12, 34, 56)),
+#endif
+                        new ColumnNetTypeArrowTypeValue("Timestamp", typeof(DateTimeOffset), typeof(TimestampType), new DateTimeOffset(new DateTime(2024, 9, 10, 12, 34, 56), TimeSpan.Zero)),
+                        new ColumnNetTypeArrowTypeValue("Interval", typeof(MonthDayNanosecondInterval), typeof(IntervalType), new MonthDayNanosecondInterval(12, 0, 0)),
+                    }
+                }
+            );
+
+            Dictionary<string, object?> struct_record = new Dictionary<string, object?>();
+            struct_record["c0"] = 0;
+            struct_record["c1"] = "Test Value";
+
+            sampleDataBuilder.Samples.Add(
+                // Lists and Structs
+                new SampleData()
+                {
+                    StructBehavior = "Strict",
+                    Query = "SELECT " +
+                            "ARRAY[n_regionkey, n_nationkey] AS \"List\", " +
+                            "struct(n_regionkey, 'Test Value') AS Struct "+
+                            "FROM nation WHERE n_regionkey = 0 AND n_nationkey = 5",
+                    ExpectedValues = new List<ColumnNetTypeArrowTypeValue>()
+                    {
+                        new ColumnNetTypeArrowTypeValue("List", typeof(Int32Array), typeof(ListType), new Int32Array.Builder().AppendRange(new[] { 0, 5}).Build()),
+                        new ColumnNetTypeArrowTypeValue("Struct", typeof(Dictionary<string, object?>), typeof(StructType), struct_record),
+                    }
+                }
+            );
+
+            return sampleDataBuilder;
+        }
     }
 }

csharp/test/Drivers/Interop/FlightSql/FlightSqlTestConfiguration.cs

@@ -40,7 +40,8 @@ internal enum FlightSqlTestEnvironmentType
         Denodo,
         Dremio,
         DuckDB,
-        SQLite
+        SQLite,
+        SpiceAI,
     }
 
     internal class FlightSqlTestEnvironment : TestConfiguration

csharp/test/Drivers/Interop/FlightSql/readme.md

@@ -32,6 +32,7 @@ A growing number of data sources support Arrow Flight SQL. This library has test
 - [Dremio](https://docs.dremio.com/current/sonar/developing-client-apps/arrow-flight-sql/)
 - [DuckDB](https://github.com/voltrondata/SQLFlite)
 - [SQLite](https://github.com/voltrondata/SQLFlite)
+- [Spice.ai OSS](https://github.com/spiceai/spiceai)
 
 It is recommended you test your data source with the Flight SQL Go driver to ensure compatibility, since each data source can implement the Flight protocol slightly differently.
 
@@ -49,6 +50,7 @@ A sample configuration file is provided in the Resources directory. The configur
     - Denodo
     - DuckDB
     - SQLite
+    - SpiceAI
 - **tableTypes**: The table types to include in the GetObjects call
 - **sqlFile**: A path to a SQL file to run queries to test CRUD operations
 - **metadata**: Used for the GetObjects calls
@@ -83,3 +85,31 @@ simultaneously. To use multiple data sources, you can configure them like:
            ...
         }
 ```
+
+### Spice.ai OSS Configuration
+
+Use the following command to run a local test instance of Spice.ai OSS. An example test configuration is available at [flightsql-spiceai.json](/csharp/configs/flightsql-spiceai.json).
+
+```bash
+docker compose up spiceai-test
+```
+
+Output:
+
+```console
+[+] Running 1/0
+ ✔ Container adbc-spiceai  Created                                                    0.0s
+Attaching to adbc-spiceai
+adbc-spiceai  | 2025-06-12T22:44:38.684347Z  INFO spiced: Starting runtime v1.4.0+models
+adbc-spiceai  | 2025-06-12T22:44:38.684737Z  INFO runtime::init::caching: Initialized results cache; max size: 128.00 MiB, item ttl: 1s
+adbc-spiceai  | 2025-06-12T22:44:38.684772Z  INFO runtime::init::caching: Initialized search results cache;
+adbc-spiceai  | 2025-06-12T22:44:39.043113Z  INFO runtime::init::dataset: Initializing dataset nation
+adbc-spiceai  | 2025-06-12T22:44:39.044803Z  INFO runtime::metrics_server: Spice Runtime Metrics listening on 0.0.0.0:9090
+adbc-spiceai  | 2025-06-12T22:44:39.044866Z  INFO runtime::opentelemetry: Spice Runtime OpenTelemetry listening on 127.0.0.1:50052
+adbc-spiceai  | 2025-06-12T22:44:39.044926Z  INFO runtime::http: Spice Runtime HTTP listening on 0.0.0.0:8090
+adbc-spiceai  | 2025-06-12T22:44:39.044351Z  INFO runtime::flight: Spice Runtime Flight listening on 0.0.0.0:50051
+adbc-spiceai  | 2025-06-12T22:44:39.848102Z  INFO runtime::init::dataset: Dataset nation registered (s3://spiceai-demo-datasets/tpch/nation/), acceleration (arrow), results cache enabled.
+adbc-spiceai  | 2025-06-12T22:44:39.849943Z  INFO runtime::accelerated_table::refresh_task: Loading data for dataset nation
+adbc-spiceai  | 2025-06-12T22:44:40.537458Z  INFO runtime::accelerated_table::refresh_task: Loaded 25 rows (3.35 kiB) for dataset nation in 687ms.
+adbc-spiceai  | 2025-06-12T22:44:40.568947Z  INFO runtime: All components are loaded. Spice runtime is ready!
+```