diff --git a/csharp/hiveserver2 b/csharp/hiveserver2 index edab9f8b8..ae2dedf28 160000 --- a/csharp/hiveserver2 +++ b/csharp/hiveserver2 @@ -1 +1 @@ -Subproject commit edab9f8b8a209f9f3c16796a109441836e1ea470 +Subproject commit ae2dedf289f4548ac25c7b9841bbf492c8610f5d diff --git a/csharp/src/ColumnMetadataHelper.cs b/csharp/src/ColumnMetadataHelper.cs new file mode 100644 index 000000000..257ad345d --- /dev/null +++ b/csharp/src/ColumnMetadataHelper.cs @@ -0,0 +1,271 @@ +/* + * Copyright (c) 2025 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using AdbcDrivers.HiveServer2.Hive2; +using static AdbcDrivers.HiveServer2.Hive2.HiveServer2Connection; + +namespace AdbcDrivers.Databricks +{ + /// + /// Computes column metadata (data type codes, column sizes, decimal digits, + /// buffer lengths, etc.) from SQL type name strings. Provides defaults for + /// metadata fields that are not returned directly by the server. + /// + internal static class ColumnMetadataHelper + { + private static readonly Dictionary s_baseTypeToCodeMap = new(StringComparer.OrdinalIgnoreCase) + { + { "BOOLEAN", (short)ColumnTypeId.BOOLEAN }, + { "TINYINT", (short)ColumnTypeId.TINYINT }, + { "SMALLINT", (short)ColumnTypeId.SMALLINT }, + { "INTEGER", (short)ColumnTypeId.INTEGER }, + { "BIGINT", (short)ColumnTypeId.BIGINT }, + { "FLOAT", (short)ColumnTypeId.FLOAT }, + { "REAL", (short)ColumnTypeId.REAL }, + { "DOUBLE", (short)ColumnTypeId.DOUBLE }, + { "DECIMAL", (short)ColumnTypeId.DECIMAL }, + { "NUMERIC", (short)ColumnTypeId.NUMERIC }, + { "CHAR", (short)ColumnTypeId.CHAR }, + { "NCHAR", (short)ColumnTypeId.NCHAR }, + { "STRING", (short)ColumnTypeId.VARCHAR }, + { "VARCHAR", (short)ColumnTypeId.VARCHAR }, + { "NVARCHAR", (short)ColumnTypeId.NVARCHAR }, + { "LONGVARCHAR", (short)ColumnTypeId.LONGVARCHAR }, + { "LONGNVARCHAR", (short)ColumnTypeId.LONGNVARCHAR }, + { "BINARY", (short)ColumnTypeId.BINARY }, + { "VARBINARY", (short)ColumnTypeId.VARBINARY }, + { "LONGVARBINARY", (short)ColumnTypeId.LONGVARBINARY }, + { "DATE", (short)ColumnTypeId.DATE }, + { "TIMESTAMP", (short)ColumnTypeId.TIMESTAMP }, + { "ARRAY", (short)ColumnTypeId.ARRAY }, + { "MAP", (short)ColumnTypeId.JAVA_OBJECT }, + { "STRUCT", (short)ColumnTypeId.STRUCT }, + { "NULL", (short)ColumnTypeId.NULL }, + { "VOID", (short)ColumnTypeId.NULL }, + { "INTERVAL", (short)ColumnTypeId.OTHER }, + { "VARIANT", (short)ColumnTypeId.OTHER }, + { "OTHER", (short)ColumnTypeId.OTHER }, + }; + + private static readonly Dictionary s_aliasToBaseType = new(StringComparer.OrdinalIgnoreCase) + { + { "BYTE", "TINYINT" }, + { "SHORT", "SMALLINT" }, + { "LONG", "BIGINT" }, + }; + + private static readonly HashSet s_numericBaseTypes = new(StringComparer.OrdinalIgnoreCase) + { + "TINYINT", "SMALLINT", "INTEGER", "BIGINT", + "FLOAT", "REAL", "DOUBLE", "DECIMAL", "NUMERIC" + }; + + private static readonly HashSet s_charBaseTypes = new(StringComparer.OrdinalIgnoreCase) + { + "STRING", "VARCHAR", "CHAR", "NCHAR", "NVARCHAR", + "LONGVARCHAR", "LONGNVARCHAR" + }; + + internal static short GetDataTypeCode(string typeName) + { + string baseName = GetBaseTypeName(typeName); + if (s_baseTypeToCodeMap.TryGetValue(baseName, out short code)) + return code; + return (short)ColumnTypeId.OTHER; + } + + internal static string GetBaseTypeName(string typeName) + { + if (SqlTypeNameParser.TryParse(typeName, out SqlTypeNameParserResult? result)) + { + return result!.BaseTypeName; + } + string upper = typeName.Trim().ToUpperInvariant(); + if (s_aliasToBaseType.TryGetValue(upper, out string? canonical)) + return canonical; + return upper; + } + + internal static int? GetColumnSizeDefault(string typeName) + { + string baseName = GetBaseTypeName(typeName); + switch (baseName) + { + case "BOOLEAN": + case "TINYINT": + return 1; + case "SMALLINT": + return 2; + case "INTEGER": + case "FLOAT": + case "REAL": + case "DATE": + return 4; + case "BIGINT": + case "DOUBLE": + case "TIMESTAMP": + return 8; + case "DECIMAL": + case "NUMERIC": + return GetParsedPrecision(typeName) ?? SqlDecimalTypeParser.DecimalPrecisionDefault; + case "VARCHAR": + case "LONGVARCHAR": + case "LONGNVARCHAR": + case "NVARCHAR": + return GetParsedColumnSize(typeName) ?? SqlVarcharTypeParser.VarcharColumnSizeDefault; + case "STRING": + return int.MaxValue; + case "CHAR": + case "NCHAR": + return GetParsedColumnSize(typeName) ?? 255; + case "BINARY": + case "VARBINARY": + case "LONGVARBINARY": + return 0; + case "NULL": + case "VOID": + return 1; + case "INTERVAL": + return GetIntervalSize(typeName); + default: + return 0; + } + } + + internal static int? GetDecimalDigitsDefault(string typeName) + { + string baseName = GetBaseTypeName(typeName); + return baseName switch + { + "DECIMAL" or "NUMERIC" => GetParsedScale(typeName) ?? SqlDecimalTypeParser.DecimalScaleDefault, + "FLOAT" or "REAL" => 7, + "DOUBLE" => 15, + "TIMESTAMP" => 6, + _ => 0 + }; + } + + internal static int? GetBufferLength(string typeName) + { + string baseName = GetBaseTypeName(typeName); + switch (baseName) + { + case "BOOLEAN": + case "TINYINT": + return 1; + case "SMALLINT": + return 2; + case "INTEGER": + case "FLOAT": + case "REAL": + return 4; + case "BIGINT": + case "DOUBLE": + case "TIMESTAMP": + case "DATE": + return 8; + case "DECIMAL": + case "NUMERIC": + int precision = GetParsedPrecision(typeName) ?? SqlDecimalTypeParser.DecimalPrecisionDefault; + return ((precision + 8) / 9) * 5 + 1; + default: + return null; + } + } + + internal static short? GetNumPrecRadix(string typeName) + { + string baseName = GetBaseTypeName(typeName); + return s_numericBaseTypes.Contains(baseName) ? (short)10 : null; + } + + internal static int? GetCharOctetLength(string typeName) + { + string baseName = GetBaseTypeName(typeName); + return s_charBaseTypes.Contains(baseName) ? GetColumnSizeDefault(typeName) : null; + } + + internal static short? GetSqlDatetimeSub(string typeName) + { + string baseName = GetBaseTypeName(typeName); + return baseName switch + { + "DATE" => 1, + "TIMESTAMP" => 3, + _ => null + }; + } + + internal static void PopulateTableInfoFromTypeName( + TableInfo tableInfo, + string columnName, + string typeName, + int ordinalPosition, + bool isNullable = true, + string? comment = null, + string? columnDefault = null) + { + tableInfo.ColumnName.Add(columnName); + tableInfo.TypeName.Add(typeName); + tableInfo.ColType.Add(GetDataTypeCode(typeName)); + tableInfo.BaseTypeName.Add(GetBaseTypeName(typeName)); + tableInfo.Precision.Add(GetColumnSizeDefault(typeName)); + int? scale = GetDecimalDigitsDefault(typeName); + tableInfo.Scale.Add(scale.HasValue ? (short)scale.Value : null); + tableInfo.OrdinalPosition.Add(ordinalPosition); + tableInfo.Nullable.Add(isNullable ? (short)1 : (short)0); + tableInfo.IsNullable.Add(isNullable ? "YES" : "NO"); + tableInfo.IsAutoIncrement.Add(false); + tableInfo.ColumnDefault.Add(columnDefault ?? ""); + } + + private static int? GetParsedPrecision(string typeName) + { + if (SqlTypeNameParser.TryParse(typeName, out SqlTypeNameParserResult? result) + && result is SqlDecimalParserResult decimalResult) + return decimalResult.Precision; + return null; + } + + private static int? GetParsedScale(string typeName) + { + if (SqlTypeNameParser.TryParse(typeName, out SqlTypeNameParserResult? result) + && result is SqlDecimalParserResult decimalResult) + return decimalResult.Scale; + return null; + } + + private static int? GetParsedColumnSize(string typeName) + { + if (SqlTypeNameParser.TryParse(typeName, out SqlTypeNameParserResult? result) + && result is SqlCharVarcharParserResult charResult) + return charResult.ColumnSize; + return null; + } + + private static int GetIntervalSize(string typeName) + { + string upper = typeName.Trim().ToUpperInvariant(); + if (upper.Contains("YEAR") || upper.Contains("MONTH")) + return 4; + if (upper.Contains("DAY") || upper.Contains("HOUR") || upper.Contains("MINUTE") || upper.Contains("SECOND")) + return 8; + return 4; + } + } +} diff --git a/csharp/src/DatabricksStatement.cs b/csharp/src/DatabricksStatement.cs index cb8c80d49..bf93435a5 100644 --- a/csharp/src/DatabricksStatement.cs +++ b/csharp/src/DatabricksStatement.cs @@ -380,20 +380,14 @@ protected override async Task GetCatalogsAsync(CancellationToken ca new("reason", "Multiple catalog support disabled") ]); - // Create a schema with a single column TABLE_CAT - var field = new Field("TABLE_CAT", StringType.Default, true); - var schema = new Schema(new[] { field }, null); - - // Create a single row with value "SPARK" + var schema = MetadataSchemaFactory.CreateCatalogsSchema(); var builder = new StringArray.Builder(); builder.Append("SPARK"); - var array = builder.Build(); activity?.SetTag(SemanticConventions.Db.Response.ReturnedRows, 1); activity?.AddEvent("statement.get_catalogs.complete"); - // Return the result without making an RPC call - return new QueryResult(1, new HiveServer2Connection.HiveInfoArrowStream(schema, new[] { array })); + return new QueryResult(1, new HiveInfoArrowStream(schema, new IArrowArray[] { builder.Build() })); } // If EnableMultipleCatalogSupport is true, delegate to base class implementation @@ -433,23 +427,10 @@ protected override async Task GetSchemasAsync(CancellationToken can new("reason", "Multiple catalog support disabled and catalog is not null") ]); - // Create a schema with TABLE_SCHEM and TABLE_CATALOG columns - var fields = new[] - { - new Field("TABLE_SCHEM", StringType.Default, true), - new Field("TABLE_CATALOG", StringType.Default, true) - }; - var schema = new Schema(fields, null); - - // Create empty arrays for both columns - var catalogArray = new StringArray.Builder().Build(); - var schemaArray = new StringArray.Builder().Build(); - activity?.SetTag(SemanticConventions.Db.Response.ReturnedRows, 0); activity?.AddEvent("statement.get_schemas.complete"); - // Return empty result - return new QueryResult(0, new HiveServer2Connection.HiveInfoArrowStream(schema, new[] { catalogArray, schemaArray })); + return MetadataSchemaFactory.CreateEmptySchemasResult(); } // Call the base implementation with the potentially modified catalog name @@ -492,42 +473,10 @@ protected override async Task GetTablesAsync(CancellationToken canc new("reason", "Multiple catalog support disabled and catalog is not null") ]); - // Correct schema for GetTables - var fields = new[] - { - new Field("TABLE_CAT", StringType.Default, true), - new Field("TABLE_SCHEM", StringType.Default, true), - new Field("TABLE_NAME", StringType.Default, true), - new Field("TABLE_TYPE", StringType.Default, true), - new Field("REMARKS", StringType.Default, true), - new Field("TYPE_CAT", StringType.Default, true), - new Field("TYPE_SCHEM", StringType.Default, true), - new Field("TYPE_NAME", StringType.Default, true), - new Field("SELF_REFERENCING_COL_NAME", StringType.Default, true), - new Field("REF_GENERATION", StringType.Default, true) - }; - var schema = new Schema(fields, null); - - // Create empty arrays for all columns - var arrays = new IArrowArray[] - { - new StringArray.Builder().Build(), // TABLE_CAT - new StringArray.Builder().Build(), // TABLE_SCHEM - new StringArray.Builder().Build(), // TABLE_NAME - new StringArray.Builder().Build(), // TABLE_TYPE - new StringArray.Builder().Build(), // REMARKS - new StringArray.Builder().Build(), // TYPE_CAT - new StringArray.Builder().Build(), // TYPE_SCHEM - new StringArray.Builder().Build(), // TYPE_NAME - new StringArray.Builder().Build(), // SELF_REFERENCING_COL_NAME - new StringArray.Builder().Build() // REF_GENERATION - }; - activity?.SetTag(SemanticConventions.Db.Response.ReturnedRows, 0); activity?.AddEvent("statement.get_tables.complete"); - // Return empty result - return new QueryResult(0, new HiveServer2Connection.HiveInfoArrowStream(schema, arrays)); + return MetadataSchemaFactory.CreateEmptyTablesResult(); } // Call the base implementation with the potentially modified catalog name @@ -571,7 +520,7 @@ protected override async Task GetColumnsAsync(CancellationToken can ]); // Correct schema for GetColumns - var schema = CreateColumnMetadataSchema(); + var schema = MetadataSchemaFactory.CreateColumnMetadataSchema(); // Create empty arrays for all columns var arrays = CreateColumnMetadataEmptyArray(); @@ -580,7 +529,7 @@ protected override async Task GetColumnsAsync(CancellationToken can activity?.AddEvent("statement.get_columns.complete"); // Return empty result - return new QueryResult(0, new HiveServer2Connection.HiveInfoArrowStream(schema, arrays)); + return new QueryResult(0, new HiveInfoArrowStream(schema, arrays)); } // Call the base implementation with the potentially modified catalog name @@ -608,25 +557,7 @@ protected override async Task GetColumnsAsync(CancellationToken can /// internal bool ShouldReturnEmptyPkFkResult() { - if (!enablePKFK) - return true; - - var catalogInvalid = string.IsNullOrEmpty(CatalogName) || - string.Equals(CatalogName, "SPARK", StringComparison.OrdinalIgnoreCase) || - string.Equals(CatalogName, "hive_metastore", StringComparison.OrdinalIgnoreCase); - - var foreignCatalogInvalid = string.IsNullOrEmpty(ForeignCatalogName) || - string.Equals(ForeignCatalogName, "SPARK", StringComparison.OrdinalIgnoreCase) || - string.Equals(ForeignCatalogName, "hive_metastore", StringComparison.OrdinalIgnoreCase); - - // Handle special catalog cases - // Only when both catalog and foreignCatalog is Invalid, we return empty results - if (catalogInvalid && foreignCatalogInvalid) - { - return true; - } - - return false; + return MetadataUtilities.ShouldReturnEmptyPKFKResult(CatalogName, ForeignCatalogName, enablePKFK); } protected override async Task GetPrimaryKeysAsync(CancellationToken cancellationToken = default) @@ -660,28 +591,7 @@ protected override async Task GetPrimaryKeysAsync(CancellationToken private QueryResult EmptyPrimaryKeysResult() { - var fields = new[] - { - new Field("TABLE_CAT", StringType.Default, true), - new Field("TABLE_SCHEM", StringType.Default, true), - new Field("TABLE_NAME", StringType.Default, true), - new Field("COLUMN_NAME", StringType.Default, true), - new Field("KEQ_SEQ", Int32Type.Default, true), - new Field("PK_NAME", StringType.Default, true) - }; - var schema = new Schema(fields, null); - - var arrays = new IArrowArray[] - { - new StringArray.Builder().Build(), // TABLE_CAT - new StringArray.Builder().Build(), // TABLE_SCHEM - new StringArray.Builder().Build(), // TABLE_NAME - new StringArray.Builder().Build(), // COLUMN_NAME - new Int32Array.Builder().Build(), // KEQ_SEQ - new StringArray.Builder().Build() // PK_NAME - }; - - return new QueryResult(0, new HiveServer2Connection.HiveInfoArrowStream(schema, arrays)); + return MetadataSchemaFactory.CreateEmptyPrimaryKeysResult(); } protected override async Task GetCrossReferenceAsync(CancellationToken cancellationToken = default) @@ -744,44 +654,7 @@ protected override async Task GetCrossReferenceAsForeignTableAsync( private QueryResult EmptyCrossReferenceResult() { - var fields = new[] - { - new Field("PKTABLE_CAT", StringType.Default, true), - new Field("PKTABLE_SCHEM", StringType.Default, true), - new Field("PKTABLE_NAME", StringType.Default, true), - new Field("PKCOLUMN_NAME", StringType.Default, true), - new Field("FKTABLE_CAT", StringType.Default, true), - new Field("FKTABLE_SCHEM", StringType.Default, true), - new Field("FKTABLE_NAME", StringType.Default, true), - new Field("FKCOLUMN_NAME", StringType.Default, true), - new Field("KEQ_SEQ", Int32Type.Default, true), - new Field("UPDATE_RULE", Int32Type.Default, true), - new Field("DELETE_RULE", Int32Type.Default, true), - new Field("FK_NAME", StringType.Default, true), - new Field("PK_NAME", StringType.Default, true), - new Field("DEFERRABILITY", Int32Type.Default, true) - }; - var schema = new Schema(fields, null); - - var arrays = new IArrowArray[] - { - new StringArray.Builder().Build(), // PKTABLE_CAT - new StringArray.Builder().Build(), // PKTABLE_SCHEM - new StringArray.Builder().Build(), // PKTABLE_NAME - new StringArray.Builder().Build(), // PKCOLUMN_NAME - new StringArray.Builder().Build(), // FKTABLE_CAT - new StringArray.Builder().Build(), // FKTABLE_SCHEM - new StringArray.Builder().Build(), // FKTABLE_NAME - new StringArray.Builder().Build(), // FKCOLUMN_NAME - new Int32Array.Builder().Build(), // KEQ_SEQ - new Int32Array.Builder().Build(), // UPDATE_RULE - new Int32Array.Builder().Build(), // DELETE_RULE - new StringArray.Builder().Build(), // FK_NAME - new StringArray.Builder().Build(), // PK_NAME - new Int32Array.Builder().Build() // DEFERRABILITY - }; - - return new QueryResult(0, new HiveServer2Connection.HiveInfoArrowStream(schema, arrays)); + return MetadataSchemaFactory.CreateEmptyCrossReferenceResult(); } protected override async Task GetColumnsExtendedAsync(CancellationToken cancellationToken = default) @@ -840,7 +713,7 @@ protected override async Task GetColumnsExtendedAsync(CancellationT return baseResult; } - var columnMetadataSchema = CreateColumnMetadataSchema(); + var columnMetadataSchema = MetadataSchemaFactory.CreateColumnMetadataSchema(); if (descResult.Stream == null) { @@ -894,41 +767,6 @@ protected override async Task GetColumnsExtendedAsync(CancellationT public override string AssemblyVersion => DatabricksConnection.s_assemblyVersion; - /// - /// Creates the schema for the column metadata result set. - /// This schema is used for the GetColumns metadata query. - /// - private static Schema CreateColumnMetadataSchema() - { - var fields = new[] - { - new Field("TABLE_CAT", StringType.Default, true), - new Field("TABLE_SCHEM", StringType.Default, true), - new Field("TABLE_NAME", StringType.Default, true), - new Field("COLUMN_NAME", StringType.Default, true), - new Field("DATA_TYPE", Int32Type.Default, true), - new Field("TYPE_NAME", StringType.Default, true), - new Field("COLUMN_SIZE", Int32Type.Default, true), - new Field("BUFFER_LENGTH", Int8Type.Default, true), - new Field("DECIMAL_DIGITS", Int32Type.Default, true), - new Field("NUM_PREC_RADIX", Int32Type.Default, true), - new Field("NULLABLE", Int32Type.Default, true), - new Field("REMARKS", StringType.Default, true), - new Field("COLUMN_DEF", StringType.Default, true), - new Field("SQL_DATA_TYPE", Int32Type.Default, true), - new Field("SQL_DATETIME_SUB", Int32Type.Default, true), - new Field("CHAR_OCTET_LENGTH", Int32Type.Default, true), - new Field("ORDINAL_POSITION", Int32Type.Default, true), - new Field("IS_NULLABLE", StringType.Default, true), - new Field("SCOPE_CATALOG", StringType.Default, true), - new Field("SCOPE_SCHEMA", StringType.Default, true), - new Field("SCOPE_TABLE", StringType.Default, true), - new Field("SOURCE_DATA_TYPE", Int16Type.Default, true), - new Field("IS_AUTO_INCREMENT", StringType.Default, true), - new Field("BASE_TYPE_NAME", StringType.Default, true) - }; - return new Schema(fields, null); - } /// /// Creates an empty array for each column in the column metadata schema. @@ -944,7 +782,7 @@ private static IArrowArray[] CreateColumnMetadataEmptyArray() new Int32Array.Builder().Build(), // DATA_TYPE new StringArray.Builder().Build(), // TYPE_NAME new Int32Array.Builder().Build(), // COLUMN_SIZE - new Int8Array.Builder().Build(), // BUFFER_LENGTH + new Int32Array.Builder().Build(), // BUFFER_LENGTH new Int32Array.Builder().Build(), // DECIMAL_DIGITS new Int32Array.Builder().Build(), // NUM_PREC_RADIX new Int32Array.Builder().Build(), // NULLABLE @@ -964,7 +802,7 @@ private static IArrowArray[] CreateColumnMetadataEmptyArray() ]; } - private QueryResult CreateExtendedColumnsResult(Schema columnMetadataSchema, DescTableExtendedResult descResult) + internal static QueryResult CreateExtendedColumnsResult(Schema columnMetadataSchema, DescTableExtendedResult descResult) { var allFields = new List(columnMetadataSchema.FieldsList); foreach (var field in PrimaryKeyFields) @@ -1140,7 +978,7 @@ private QueryResult CreateExtendedColumnsResult(Schema columnMetadataSchema, Des fkColumnKeySeqBuilder.Build() }; - return new QueryResult(descResult.Columns.Count, new HiveServer2Connection.HiveInfoArrowStream(combinedSchema, combinedData)); + return new QueryResult(descResult.Columns.Count, new HiveInfoArrowStream(combinedSchema, combinedData)); } } } diff --git a/csharp/src/FlatColumnsResultBuilder.cs b/csharp/src/FlatColumnsResultBuilder.cs new file mode 100644 index 000000000..9ce79649a --- /dev/null +++ b/csharp/src/FlatColumnsResultBuilder.cs @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2025 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Generic; +using AdbcDrivers.HiveServer2; +using AdbcDrivers.HiveServer2.Hive2; +using Apache.Arrow; +using Apache.Arrow.Adbc; + +namespace AdbcDrivers.Databricks +{ + /// + /// Builds a flat columns result set with all 24 standard metadata columns, + /// including computed fields like BUFFER_LENGTH, NUM_PREC_RADIX, + /// SQL_DATETIME_SUB, and CHAR_OCTET_LENGTH. + /// + internal static class FlatColumnsResultBuilder + { + internal static QueryResult BuildFlatColumnsResult( + IEnumerable<(string catalog, string schema, string table, TableInfo info)> tables) + { + var tableCatBuilder = new StringArray.Builder(); + var tableSchemaBuilder = new StringArray.Builder(); + var tableNameBuilder = new StringArray.Builder(); + var columnNameBuilder = new StringArray.Builder(); + var dataTypeBuilder = new Int32Array.Builder(); + var typeNameBuilder = new StringArray.Builder(); + var columnSizeBuilder = new Int32Array.Builder(); + var bufferLengthBuilder = new Int32Array.Builder(); + var decimalDigitsBuilder = new Int32Array.Builder(); + var numPrecRadixBuilder = new Int32Array.Builder(); + var nullableBuilder = new Int32Array.Builder(); + var remarksBuilder = new StringArray.Builder(); + var columnDefBuilder = new StringArray.Builder(); + var sqlDataTypeBuilder = new Int32Array.Builder(); + var sqlDatetimeSubBuilder = new Int32Array.Builder(); + var charOctetLengthBuilder = new Int32Array.Builder(); + var ordinalPositionBuilder = new Int32Array.Builder(); + var isNullableBuilder = new StringArray.Builder(); + var scopeCatalogBuilder = new StringArray.Builder(); + var scopeSchemaBuilder = new StringArray.Builder(); + var scopeTableBuilder = new StringArray.Builder(); + var sourceDataTypeBuilder = new Int16Array.Builder(); + var isAutoIncrementBuilder = new StringArray.Builder(); + var baseTypeNameBuilder = new StringArray.Builder(); + int totalRows = 0; + + foreach (var (catalog, schema, table, info) in tables) + { + for (int i = 0; i < info.ColumnName.Count; i++) + { + tableCatBuilder.Append(catalog); + tableSchemaBuilder.Append(schema); + tableNameBuilder.Append(table); + columnNameBuilder.Append(info.ColumnName[i]); + dataTypeBuilder.Append(info.ColType[i]); + typeNameBuilder.Append(info.TypeName[i]); + + if (info.Precision[i].HasValue) columnSizeBuilder.Append(info.Precision[i]!.Value); else columnSizeBuilder.AppendNull(); + + int? bufLen = ColumnMetadataHelper.GetBufferLength(info.TypeName[i]); + if (bufLen.HasValue) bufferLengthBuilder.Append(bufLen.Value); else bufferLengthBuilder.AppendNull(); + + if (info.Scale[i].HasValue) decimalDigitsBuilder.Append(info.Scale[i]!.Value); else decimalDigitsBuilder.AppendNull(); + + short? radix = ColumnMetadataHelper.GetNumPrecRadix(info.TypeName[i]); + if (radix.HasValue) numPrecRadixBuilder.Append(radix.Value); else numPrecRadixBuilder.AppendNull(); + + nullableBuilder.Append(info.Nullable[i]); + remarksBuilder.Append(""); + columnDefBuilder.AppendNull(); + + sqlDataTypeBuilder.Append(info.ColType[i]); + + short? dtSub = ColumnMetadataHelper.GetSqlDatetimeSub(info.TypeName[i]); + if (dtSub.HasValue) sqlDatetimeSubBuilder.Append(dtSub.Value); else sqlDatetimeSubBuilder.AppendNull(); + + int? charOctet = ColumnMetadataHelper.GetCharOctetLength(info.TypeName[i]); + if (charOctet.HasValue) charOctetLengthBuilder.Append(charOctet.Value); else charOctetLengthBuilder.AppendNull(); + + ordinalPositionBuilder.Append(info.OrdinalPosition[i]); + isNullableBuilder.Append(info.IsNullable[i]); + scopeCatalogBuilder.AppendNull(); + scopeSchemaBuilder.AppendNull(); + scopeTableBuilder.AppendNull(); + sourceDataTypeBuilder.AppendNull(); + isAutoIncrementBuilder.Append(info.IsAutoIncrement[i] ? "YES" : "NO"); + baseTypeNameBuilder.Append(info.BaseTypeName[i]); + totalRows++; + } + } + + var resultSchema = MetadataSchemaFactory.CreateColumnMetadataSchema(); + + var dataArrays = new IArrowArray[] + { + tableCatBuilder.Build(), + tableSchemaBuilder.Build(), + tableNameBuilder.Build(), + columnNameBuilder.Build(), + dataTypeBuilder.Build(), + typeNameBuilder.Build(), + columnSizeBuilder.Build(), + bufferLengthBuilder.Build(), + decimalDigitsBuilder.Build(), + numPrecRadixBuilder.Build(), + nullableBuilder.Build(), + remarksBuilder.Build(), + columnDefBuilder.Build(), + sqlDataTypeBuilder.Build(), + sqlDatetimeSubBuilder.Build(), + charOctetLengthBuilder.Build(), + ordinalPositionBuilder.Build(), + isNullableBuilder.Build(), + scopeCatalogBuilder.Build(), + scopeSchemaBuilder.Build(), + scopeTableBuilder.Build(), + sourceDataTypeBuilder.Build(), + isAutoIncrementBuilder.Build(), + baseTypeNameBuilder.Build() + }; + + return new QueryResult(totalRows, new HiveInfoArrowStream(resultSchema, dataArrays)); + } + } +} diff --git a/csharp/src/MetadataUtilities.cs b/csharp/src/MetadataUtilities.cs new file mode 100644 index 000000000..9391abf3d --- /dev/null +++ b/csharp/src/MetadataUtilities.cs @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2025 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; + +namespace AdbcDrivers.Databricks +{ + internal static class MetadataUtilities + { + internal static string? NormalizeSparkCatalog(string? catalogName) + { + if (string.IsNullOrEmpty(catalogName)) + return catalogName; + + if (string.Equals(catalogName, "SPARK", StringComparison.OrdinalIgnoreCase)) + return null; + + return catalogName; + } + + internal static bool IsInvalidPKFKCatalog(string? catalogName) + { + return string.IsNullOrEmpty(catalogName) || + string.Equals(catalogName, "SPARK", StringComparison.OrdinalIgnoreCase) || + string.Equals(catalogName, "hive_metastore", StringComparison.OrdinalIgnoreCase); + } + + internal static bool ShouldReturnEmptyPKFKResult(string? catalogName, string? foreignCatalogName, bool enablePKFK) + { + if (!enablePKFK) + return true; + + return IsInvalidPKFKCatalog(catalogName) && IsInvalidPKFKCatalog(foreignCatalogName); + } + + internal static string? BuildQualifiedTableName(string? catalogName, string? schemaName, string? tableName) + { + if (string.IsNullOrEmpty(tableName)) + return tableName; + + var parts = new System.Collections.Generic.List(); + + if (!string.IsNullOrEmpty(schemaName)) + { + if (!string.IsNullOrEmpty(catalogName) && !catalogName!.Equals("SPARK", StringComparison.OrdinalIgnoreCase)) + { + parts.Add($"`{catalogName.Replace("`", "``")}`"); + } + parts.Add($"`{schemaName!.Replace("`", "``")}`"); + } + + parts.Add($"`{tableName!.Replace("`", "``")}`"); + + return string.Join(".", parts); + } + } +} diff --git a/csharp/src/Result/DescTableExtendedResult.cs b/csharp/src/Result/DescTableExtendedResult.cs index 30c321da4..a8ab98f3b 100644 --- a/csharp/src/Result/DescTableExtendedResult.cs +++ b/csharp/src/Result/DescTableExtendedResult.cs @@ -25,6 +25,7 @@ using System.Collections.Generic; using System.Text.Json.Serialization; using System.Text.RegularExpressions; +using AdbcDrivers.HiveServer2.Hive2; using static AdbcDrivers.HiveServer2.Hive2.HiveServer2Connection; namespace AdbcDrivers.Databricks.Result @@ -88,112 +89,38 @@ public ColumnTypeId DataType { get { - string normalizedTypeName = Type.Name.Trim().ToUpper(); - - return normalizedTypeName switch - { - "BOOLEAN" => ColumnTypeId.BOOLEAN, - "TINYINT" or "BYTE" => ColumnTypeId.TINYINT, - "SMALLINT" or "SHORT" => ColumnTypeId.SMALLINT, - "INT" or "INTEGER" => ColumnTypeId.INTEGER, - "BIGINT" or "LONG" => ColumnTypeId.BIGINT, - "FLOAT" or "REAL" => ColumnTypeId.FLOAT, - "DOUBLE" => ColumnTypeId.DOUBLE, - "DECIMAL" or "NUMERIC" => ColumnTypeId.DECIMAL, - - "CHAR" => ColumnTypeId.CHAR, - "STRING" or "VARCHAR" => ColumnTypeId.VARCHAR, - "BINARY" => ColumnTypeId.BINARY, - - "TIMESTAMP" => ColumnTypeId.TIMESTAMP, - "TIMESTAMP_LTZ" => ColumnTypeId.TIMESTAMP, - "TIMESTAMP_NTZ" => ColumnTypeId.TIMESTAMP, - "DATE" => ColumnTypeId.DATE, - - "ARRAY" => ColumnTypeId.ARRAY, - "MAP" => ColumnTypeId.JAVA_OBJECT, - "STRUCT" => ColumnTypeId.STRUCT, - "INTERVAL" => ColumnTypeId.OTHER, // Intervals don't have a direct JDBC mapping - "VOID" => ColumnTypeId.NULL, - "VARIANT" => ColumnTypeId.OTHER, - _ => ColumnTypeId.OTHER // Default fallback for unknown types - }; + var code = (ColumnTypeId)ColumnMetadataHelper.GetDataTypeCode(Type.Name); + // REAL maps to FLOAT for backward compatibility + return code == ColumnTypeId.REAL ? ColumnTypeId.FLOAT : code; } } [JsonIgnore] - public bool IsNumber - { - get - { - return DataType switch - { - ColumnTypeId.TINYINT or ColumnTypeId.SMALLINT or ColumnTypeId.INTEGER or - ColumnTypeId.BIGINT or ColumnTypeId.FLOAT or ColumnTypeId.DOUBLE or - ColumnTypeId.DECIMAL or ColumnTypeId.NUMERIC => true, - _ => false - }; - } - } + public bool IsNumber => ColumnMetadataHelper.GetNumPrecRadix(Type.Name) != null; [JsonIgnore] - public int DecimalDigits - { - get - { - return DataType switch - { - ColumnTypeId.DECIMAL or ColumnTypeId.NUMERIC => Type.Scale ?? 0, - ColumnTypeId.DOUBLE => 15, - ColumnTypeId.FLOAT or ColumnTypeId.REAL => 7, - ColumnTypeId.TIMESTAMP => 6, - _ => 0 - }; - } - } + public int DecimalDigits => ColumnMetadataHelper.GetDecimalDigitsDefault(Type.FullTypeName) ?? 0; - /// - /// Get column size - /// - /// Currently the query `DESC TABLE EXTNEDED AS JSON` does not return the column size, - /// we can calculate it based on the data type and some type specific properties - /// [JsonIgnore] public int? ColumnSize { get { - return DataType switch + // For INTERVAL types, FullTypeName may not include the qualifier + // when only StartUnit is set. Use StartUnit directly in that case. + if (Type.Name.Trim().Equals("INTERVAL", StringComparison.OrdinalIgnoreCase) + && !string.IsNullOrEmpty(Type.StartUnit) + && Type.EndUnit == null) { - ColumnTypeId.TINYINT or ColumnTypeId.BOOLEAN => 1, - ColumnTypeId.SMALLINT => 2, - ColumnTypeId.INTEGER or ColumnTypeId.FLOAT or ColumnTypeId.DATE => 4, - ColumnTypeId.BIGINT or ColumnTypeId.DOUBLE or ColumnTypeId.TIMESTAMP or ColumnTypeId.TIMESTAMP_WITH_TIMEZONE => 8, - ColumnTypeId.CHAR => Type.Length, - ColumnTypeId.VARCHAR => Type.Name.Trim().ToUpper() == "STRING" ? int.MaxValue : Type.Length, - ColumnTypeId.DECIMAL => Type.Precision ?? 0, - ColumnTypeId.NULL => 1, - _ => Type.Name.Trim().ToUpper() == "INTERVAL" ? GetIntervalSize() : 0 - }; - } - } - - private int GetIntervalSize() - { - if (String.IsNullOrEmpty(Type.StartUnit)) - { - return 0; + return Type.StartUnit!.ToUpper() switch + { + "YEAR" or "MONTH" => 4, + "DAY" or "HOUR" or "MINUTE" or "SECOND" => 8, + _ => 4 + }; + } + return ColumnMetadataHelper.GetColumnSizeDefault(Type.FullTypeName); } - - // Check whether interval is yearMonthIntervalQualifier or dayTimeIntervalQualifier - // yearMonthIntervalQualifier size is 4, dayTimeIntervalQualifier size is 8 - // see https://docs.databricks.com/aws/en/sql/language-manual/data-types/interval-type - return Type.StartUnit!.ToUpper() switch - { - "YEAR" or "MONTH" => 4, - "DAY" or "HOUR" or "MINUTE" or "SECOND" => 8, - _ => 4 - }; } } diff --git a/csharp/src/StatementExecution/MetadataCommandBase.cs b/csharp/src/StatementExecution/MetadataCommandBase.cs new file mode 100644 index 000000000..8992bbf35 --- /dev/null +++ b/csharp/src/StatementExecution/MetadataCommandBase.cs @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2025 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text; + +namespace AdbcDrivers.Databricks.StatementExecution +{ + internal abstract class MetadataCommandBase + { + protected const string InAllCatalogs = " IN ALL CATALOGS"; + protected const string LikeFormat = " LIKE '{0}'"; + protected const string SchemaLikeFormat = " SCHEMA LIKE '{0}'"; + protected const string TableLikeFormat = " TABLE LIKE '{0}'"; + protected const string InCatalogFormat = " IN CATALOG {0}"; + protected const string InSchemaFormat = " IN SCHEMA {0}"; + protected const string InTableFormat = " IN TABLE {0}"; + + public abstract string Build(); + + protected static string QuoteIdentifier(string identifier) + { + return $"`{identifier.Replace("`", "``")}`"; + } + + protected static string ConvertPattern(string? pattern) + { + if (string.IsNullOrEmpty(pattern)) + return "*"; + + var result = new StringBuilder(pattern!.Length); + bool escapeNext = false; + + for (int i = 0; i < pattern.Length; i++) + { + char c = pattern[i]; + + if (c == '\\') + { + if (i + 1 < pattern.Length && pattern[i + 1] == '\\') + { + result.Append("\\\\"); + i++; + } + else + { + escapeNext = !escapeNext; + if (!escapeNext) + result.Append('\\'); + } + } + else if (escapeNext) + { + result.Append(c); + escapeNext = false; + } + else if (c == '%') + { + result.Append('*'); + } + else if (c == '_') + { + result.Append('.'); + } + else if (c == '\'') + { + result.Append("''"); + } + else + { + result.Append(c); + } + } + + return result.ToString(); + } + + protected static void AppendCatalogScope(StringBuilder sql, string? catalog) + { + if (catalog == null) + sql.Append(InAllCatalogs); + else + sql.Append(string.Format(InCatalogFormat, QuoteIdentifier(catalog))); + } + } +} diff --git a/csharp/src/StatementExecution/ShowCatalogsCommand.cs b/csharp/src/StatementExecution/ShowCatalogsCommand.cs new file mode 100644 index 000000000..ea534e203 --- /dev/null +++ b/csharp/src/StatementExecution/ShowCatalogsCommand.cs @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2025 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text; + +namespace AdbcDrivers.Databricks.StatementExecution +{ + internal class ShowCatalogsCommand : MetadataCommandBase + { + private readonly string? _catalogPattern; + + public ShowCatalogsCommand(string? catalogPattern = null) + { + _catalogPattern = catalogPattern; + } + + public override string Build() + { + var sql = new StringBuilder("SHOW CATALOGS"); + if (_catalogPattern != null) + sql.Append(string.Format(LikeFormat, ConvertPattern(_catalogPattern))); + return sql.ToString(); + } + } +} diff --git a/csharp/src/StatementExecution/ShowColumnsCommand.cs b/csharp/src/StatementExecution/ShowColumnsCommand.cs new file mode 100644 index 000000000..a24269df5 --- /dev/null +++ b/csharp/src/StatementExecution/ShowColumnsCommand.cs @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2025 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text; + +namespace AdbcDrivers.Databricks.StatementExecution +{ + internal class ShowColumnsCommand : MetadataCommandBase + { + private readonly string? _catalog; + private readonly string? _schemaPattern; + private readonly string? _tablePattern; + private readonly string? _columnPattern; + + public ShowColumnsCommand(string? catalog, string? schemaPattern = null, + string? tablePattern = null, string? columnPattern = null) + { + _catalog = catalog; + _schemaPattern = schemaPattern; + _tablePattern = tablePattern; + _columnPattern = columnPattern; + } + + public override string Build() + { + var sql = new StringBuilder("SHOW COLUMNS"); + AppendCatalogScope(sql, _catalog); + if (_schemaPattern != null) + sql.Append(string.Format(SchemaLikeFormat, ConvertPattern(_schemaPattern))); + if (_tablePattern != null) + sql.Append(string.Format(TableLikeFormat, ConvertPattern(_tablePattern))); + if (_columnPattern != null && _columnPattern != "%") + sql.Append(string.Format(LikeFormat, ConvertPattern(_columnPattern))); + return sql.ToString(); + } + } +} diff --git a/csharp/src/StatementExecution/ShowKeysCommand.cs b/csharp/src/StatementExecution/ShowKeysCommand.cs new file mode 100644 index 000000000..96d252f46 --- /dev/null +++ b/csharp/src/StatementExecution/ShowKeysCommand.cs @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2025 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Text; + +namespace AdbcDrivers.Databricks.StatementExecution +{ + internal class ShowKeysCommand : MetadataCommandBase + { + private readonly string _catalog; + private readonly string _schema; + private readonly string _table; + + public ShowKeysCommand(string catalog, string schema, string table) + { + _catalog = catalog ?? throw new ArgumentNullException(nameof(catalog)); + _schema = schema ?? throw new ArgumentNullException(nameof(schema)); + _table = table ?? throw new ArgumentNullException(nameof(table)); + } + + public override string Build() + { + var sql = new StringBuilder("SHOW KEYS"); + sql.Append(string.Format(InCatalogFormat, QuoteIdentifier(_catalog))); + sql.Append(string.Format(InSchemaFormat, QuoteIdentifier(_schema))); + sql.Append(string.Format(InTableFormat, QuoteIdentifier(_table))); + return sql.ToString(); + } + } + + internal class ShowForeignKeysCommand : MetadataCommandBase + { + private readonly string _catalog; + private readonly string _schema; + private readonly string _table; + + public ShowForeignKeysCommand(string catalog, string schema, string table) + { + _catalog = catalog ?? throw new ArgumentNullException(nameof(catalog)); + _schema = schema ?? throw new ArgumentNullException(nameof(schema)); + _table = table ?? throw new ArgumentNullException(nameof(table)); + } + + public override string Build() + { + var sql = new StringBuilder("SHOW FOREIGN KEYS"); + sql.Append(string.Format(InCatalogFormat, QuoteIdentifier(_catalog))); + sql.Append(string.Format(InSchemaFormat, QuoteIdentifier(_schema))); + sql.Append(string.Format(InTableFormat, QuoteIdentifier(_table))); + return sql.ToString(); + } + } +} diff --git a/csharp/src/StatementExecution/ShowSchemasCommand.cs b/csharp/src/StatementExecution/ShowSchemasCommand.cs new file mode 100644 index 000000000..8801af528 --- /dev/null +++ b/csharp/src/StatementExecution/ShowSchemasCommand.cs @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2025 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text; + +namespace AdbcDrivers.Databricks.StatementExecution +{ + internal class ShowSchemasCommand : MetadataCommandBase + { + private readonly string? _catalog; + private readonly string? _schemaPattern; + + public ShowSchemasCommand(string? catalog, string? schemaPattern = null) + { + _catalog = catalog; + _schemaPattern = schemaPattern; + } + + public override string Build() + { + var sql = new StringBuilder("SHOW SCHEMAS"); + if (_catalog == null) + sql.Append(InAllCatalogs); + else + sql.Append($" IN {QuoteIdentifier(_catalog)}"); + if (_schemaPattern != null) + sql.Append(string.Format(LikeFormat, ConvertPattern(_schemaPattern))); + return sql.ToString(); + } + } +} diff --git a/csharp/src/StatementExecution/ShowTablesCommand.cs b/csharp/src/StatementExecution/ShowTablesCommand.cs new file mode 100644 index 000000000..c39f983ba --- /dev/null +++ b/csharp/src/StatementExecution/ShowTablesCommand.cs @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2025 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Text; + +namespace AdbcDrivers.Databricks.StatementExecution +{ + internal class ShowTablesCommand : MetadataCommandBase + { + private readonly string? _catalog; + private readonly string? _schemaPattern; + private readonly string? _tablePattern; + + public ShowTablesCommand(string? catalog, string? schemaPattern = null, string? tablePattern = null) + { + _catalog = catalog; + _schemaPattern = schemaPattern; + _tablePattern = tablePattern; + } + + public override string Build() + { + var sql = new StringBuilder("SHOW TABLES"); + AppendCatalogScope(sql, _catalog); + if (_schemaPattern != null) + sql.Append(string.Format(SchemaLikeFormat, ConvertPattern(_schemaPattern))); + if (_tablePattern != null) + sql.Append(string.Format(LikeFormat, ConvertPattern(_tablePattern))); + return sql.ToString(); + } + } +} diff --git a/csharp/src/StatementExecution/StatementExecutionConnection.cs b/csharp/src/StatementExecution/StatementExecutionConnection.cs index a913a99c3..d4336fc01 100644 --- a/csharp/src/StatementExecution/StatementExecutionConnection.cs +++ b/csharp/src/StatementExecution/StatementExecutionConnection.cs @@ -16,15 +16,19 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Net.Http; using System.Threading; using System.Threading.Tasks; using AdbcDrivers.Databricks.Http; +using AdbcDrivers.HiveServer2.Hive2; +using AdbcDrivers.HiveServer2.Spark; using Apache.Arrow; using Apache.Arrow.Adbc; -using AdbcDrivers.HiveServer2.Spark; using Apache.Arrow.Adbc.Tracing; using Apache.Arrow.Ipc; +using Apache.Arrow.Types; +using static Apache.Arrow.Adbc.AdbcConnection; namespace AdbcDrivers.Databricks.StatementExecution { @@ -33,7 +37,7 @@ namespace AdbcDrivers.Databricks.StatementExecution /// Manages session lifecycle and creates statements for query execution. /// Extends TracingConnection for consistent tracing support with Thrift protocol. /// - internal class StatementExecutionConnection : TracingConnection + internal class StatementExecutionConnection : TracingConnection, IGetObjectsDataProvider { private readonly IStatementExecutionClient _client; private readonly string _warehouseId; @@ -378,31 +382,284 @@ public override AdbcStatement CreateStatement() this); // Pass connection as TracingConnection for tracing support } - /// - /// Gets objects (metadata) from the database. - /// public override IArrowArrayStream GetObjects(GetObjectsDepth depth, string? catalogPattern, string? schemaPattern, string? tableNamePattern, IReadOnlyList? tableTypes, string? columnNamePattern) { - // TODO (PECO-2792): Implement metadata operations via SQL queries - throw new NotImplementedException("Metadata operations are not yet implemented for Statement Execution API (PECO-2792)"); + return this.TraceActivity(activity => + { + activity?.SetTag("depth", depth.ToString()); + activity?.SetTag("catalog_pattern", catalogPattern ?? "(none)"); + activity?.SetTag("schema_pattern", schemaPattern ?? "(none)"); + activity?.SetTag("table_pattern", tableNamePattern ?? "(none)"); + activity?.SetTag("column_pattern", columnNamePattern ?? "(none)"); + + return GetObjectsResultBuilder.BuildGetObjectsResult( + this, depth, catalogPattern, schemaPattern, + tableNamePattern, tableTypes, columnNamePattern); + }, nameof(GetObjects)); + } + + public override IArrowArrayStream GetInfo(IReadOnlyList codes) + { + return this.TraceActivity(activity => + { + var supportedCodes = new AdbcInfoCode[] + { + AdbcInfoCode.DriverName, + AdbcInfoCode.DriverVersion, + AdbcInfoCode.DriverArrowVersion, + AdbcInfoCode.VendorName, + AdbcInfoCode.VendorVersion, + AdbcInfoCode.VendorSql, + }; + + if (codes == null || codes.Count == 0) + codes = supportedCodes; + + activity?.SetTag("requested_codes", string.Join(",", codes)); + + var values = new Dictionary + { + { AdbcInfoCode.DriverName, "ADBC Databricks Driver" }, + { AdbcInfoCode.DriverVersion, AssemblyVersion }, + { AdbcInfoCode.DriverArrowVersion, "1.0.0" }, + { AdbcInfoCode.VendorName, "Databricks" }, + { AdbcInfoCode.VendorVersion, AssemblyVersion }, + { AdbcInfoCode.VendorSql, true }, + }; + + return MetadataSchemaFactory.BuildGetInfoResult(codes, values); + }, nameof(GetInfo)); } - /// - /// Gets table types from the database. - /// public override IArrowArrayStream GetTableTypes() { - // TODO (PECO-2792): Implement metadata operations via SQL queries - throw new NotImplementedException("Metadata operations are not yet implemented for Statement Execution API (PECO-2792)"); + return this.TraceActivity(activity => + { + var builder = new StringArray.Builder(); + builder.Append("TABLE"); + builder.Append("VIEW"); + var schema = new Schema(new[] { new Field("table_type", StringType.Default, false) }, null); + return new HiveInfoArrowStream(schema, new IArrowArray[] { builder.Build() }); + }, nameof(GetTableTypes)); } - /// - /// Gets the schema for a specific table. - /// public override Schema GetTableSchema(string? catalog, string? dbSchema, string tableName) { - // TODO (PECO-2792): Implement metadata operations via SQL queries - throw new NotImplementedException("Metadata operations are not yet implemented for Statement Execution API (PECO-2792)"); + return this.TraceActivity(activity => + { + activity?.SetTag("catalog", catalog ?? "(none)"); + activity?.SetTag("db_schema", dbSchema ?? "(none)"); + activity?.SetTag("table_name", tableName); + + var cancellationToken = CreateMetadataTimeoutToken(); + string sql = new ShowColumnsCommand( + catalog ?? _catalog, dbSchema, tableName).Build(); + activity?.SetTag("sql_query", sql); + var batches = ExecuteMetadataSql(sql, cancellationToken); + + var fields = new List(); + foreach (var batch in batches) + { + var colNameArray = TryGetColumn(batch, "col_name"); + var columnTypeArray = TryGetColumn(batch, "columnType"); + var isNullableArray = TryGetColumn(batch, "isNullable"); + + if (colNameArray == null || columnTypeArray == null) continue; + + for (int i = 0; i < batch.Length; i++) + { + if (colNameArray.IsNull(i) || columnTypeArray.IsNull(i)) continue; + + string colName = colNameArray.GetString(i); + string colType = columnTypeArray.GetString(i); + bool nullable = isNullableArray == null || isNullableArray.IsNull(i) || + !isNullableArray.GetString(i).Equals("false", StringComparison.OrdinalIgnoreCase); + + short typeCode = ColumnMetadataHelper.GetDataTypeCode(colType); + IArrowType arrowType = HiveServer2Connection.GetArrowType(typeCode, colType, false, null, null); + fields.Add(new Field(colName, arrowType, nullable)); + } + } + + activity?.SetTag("result_fields", fields.Count); + return new Schema(fields, null); + }, nameof(GetTableSchema)); + } + + // IGetObjectsDataProvider implementation + + IReadOnlyList IGetObjectsDataProvider.GetCatalogs(string? catalogPattern) + { + var cancellationToken = CreateMetadataTimeoutToken(); + string sql = new ShowCatalogsCommand(catalogPattern).Build(); + var batches = ExecuteMetadataSql(sql, cancellationToken); + var result = new List(); + foreach (var batch in batches) + { + var catalogArray = TryGetColumn(batch, "catalog"); + if (catalogArray == null) continue; + for (int i = 0; i < batch.Length; i++) + { + if (!catalogArray.IsNull(i)) + result.Add(catalogArray.GetString(i)); + } + } + return result; + } + + IReadOnlyList<(string catalog, string schema)> IGetObjectsDataProvider.GetSchemas(string? catalogPattern, string? schemaPattern) + { + var cancellationToken = CreateMetadataTimeoutToken(); + string sql = new ShowSchemasCommand(catalogPattern, schemaPattern).Build(); + var batches = ExecuteMetadataSql(sql, cancellationToken); + var result = new List<(string, string)>(); + foreach (var batch in batches) + { + var schemaArray = TryGetColumn(batch, "databaseName"); + if (schemaArray == null) continue; + + // SHOW SCHEMAS IN ALL CATALOGS has catalog_name column; + // SHOW SCHEMAS IN `catalog` does not — use the catalogPattern as the catalog. + var catalogArray = TryGetColumn(batch, "catalog_name"); + + for (int i = 0; i < batch.Length; i++) + { + if (schemaArray.IsNull(i)) continue; + string catalog = catalogArray != null && !catalogArray.IsNull(i) + ? catalogArray.GetString(i) + : catalogPattern ?? ""; + result.Add((catalog, schemaArray.GetString(i))); + } + } + return result; + } + + IReadOnlyList<(string catalog, string schema, string table, string tableType)> IGetObjectsDataProvider.GetTables( + string? catalogPattern, string? schemaPattern, string? tableNamePattern, IReadOnlyList? tableTypes) + { + var cancellationToken = CreateMetadataTimeoutToken(); + string sql = new ShowTablesCommand(catalogPattern, schemaPattern, tableNamePattern).Build(); + var batches = ExecuteMetadataSql(sql, cancellationToken); + var result = new List<(string, string, string, string)>(); + foreach (var batch in batches) + { + var catalogArray = TryGetColumn(batch, "catalogName"); + var schemaArray = TryGetColumn(batch, "namespace"); + var tableArray = TryGetColumn(batch, "tableName"); + var tableTypeArray = TryGetColumn(batch, "tableType"); + + if (catalogArray == null || schemaArray == null || tableArray == null) continue; + + for (int i = 0; i < batch.Length; i++) + { + if (catalogArray.IsNull(i) || schemaArray.IsNull(i) || tableArray.IsNull(i)) continue; + + string tableType = "TABLE"; + if (tableTypeArray != null && !tableTypeArray.IsNull(i)) + { + string serverType = tableTypeArray.GetString(i); + if (!string.IsNullOrEmpty(serverType)) + tableType = serverType; + } + + if (tableTypes != null && tableTypes.Count > 0 && !tableTypes.Contains(tableType)) + continue; + + result.Add((catalogArray.GetString(i), schemaArray.GetString(i), + tableArray.GetString(i), tableType)); + } + } + return result; + } + + void IGetObjectsDataProvider.PopulateColumnInfo(string? catalogPattern, string? schemaPattern, + string? tablePattern, string? columnPattern, + Dictionary>> catalogMap) + { + var cancellationToken = CreateMetadataTimeoutToken(); + string sql = new ShowColumnsCommand(catalogPattern, schemaPattern, tablePattern, columnPattern).Build(); + var batches = ExecuteMetadataSql(sql, cancellationToken); + + var tablePositions = new Dictionary(); + + foreach (var batch in batches) + { + var catalogArray = TryGetColumn(batch, "catalogName"); + var schemaArray = TryGetColumn(batch, "namespace"); + var tableNameArray = TryGetColumn(batch, "tableName"); + var colNameArray = TryGetColumn(batch, "col_name"); + var columnTypeArray = TryGetColumn(batch, "columnType"); + var isNullableArray = TryGetColumn(batch, "isNullable"); + + if (catalogArray == null || schemaArray == null || tableNameArray == null || + colNameArray == null || columnTypeArray == null) continue; + + for (int i = 0; i < batch.Length; i++) + { + if (catalogArray.IsNull(i) || schemaArray.IsNull(i) || tableNameArray.IsNull(i) || + colNameArray.IsNull(i) || columnTypeArray.IsNull(i)) continue; + + string cat = catalogArray.GetString(i); + string sch = schemaArray.GetString(i); + string tbl = tableNameArray.GetString(i); + string colName = colNameArray.GetString(i); + string colType = columnTypeArray.GetString(i); + + if (string.IsNullOrEmpty(colName)) continue; + + string tableKey = $"{cat}.{sch}.{tbl}"; + if (!tablePositions.ContainsKey(tableKey)) + tablePositions[tableKey] = 1; + int position = tablePositions[tableKey]++; + + bool nullable = isNullableArray == null || isNullableArray.IsNull(i) || + !isNullableArray.GetString(i).Equals("false", StringComparison.OrdinalIgnoreCase); + + if (catalogMap.TryGetValue(cat, out var schemaMap) + && schemaMap.TryGetValue(sch, out var tableMap) + && tableMap.TryGetValue(tbl, out var tableInfo)) + { + ColumnMetadataHelper.PopulateTableInfoFromTypeName( + tableInfo, colName, colType, position, nullable); + } + } + } + } + + private static T? TryGetColumn(RecordBatch batch, string name) where T : class, IArrowArray + { + try + { + return batch.Column(name) as T; + } + catch (ArgumentOutOfRangeException) + { + return null; + } + } + + internal List ExecuteMetadataSql(string sql, CancellationToken cancellationToken = default) + { + var batches = new List(); + using var stmt = CreateStatement(); + stmt.SqlQuery = sql; + var result = stmt.ExecuteQuery(); + using var stream = result.Stream; + if (stream == null) return batches; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + var batch = stream.ReadNextRecordBatchAsync(cancellationToken).AsTask().GetAwaiter().GetResult(); + if (batch == null) break; + batches.Add(batch); + } + return batches; + } + + internal CancellationToken CreateMetadataTimeoutToken() + { + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(_waitTimeoutSeconds)); + return cts.Token; } /// diff --git a/csharp/src/StatementExecution/StatementExecutionStatement.cs b/csharp/src/StatementExecution/StatementExecutionStatement.cs index d2f894819..8ab0c5638 100644 --- a/csharp/src/StatementExecution/StatementExecutionStatement.cs +++ b/csharp/src/StatementExecution/StatementExecutionStatement.cs @@ -22,6 +22,9 @@ using System.Threading; using System.Threading.Tasks; using AdbcDrivers.Databricks.Reader.CloudFetch; +using AdbcDrivers.Databricks.Result; +using AdbcDrivers.HiveServer2; +using AdbcDrivers.HiveServer2.Hive2; using Apache.Arrow; using Apache.Arrow.Adbc; using Apache.Arrow.Adbc.Tracing; @@ -60,10 +63,23 @@ internal class StatementExecutionStatement : TracingStatement // HTTP client for CloudFetch downloads private readonly HttpClient _httpClient; + // Connection reference for metadata queries + private readonly StatementExecutionConnection _connection; + // Statement state private string? _currentStatementId; private string? _sqlQuery; + // Metadata command support + private bool _isMetadataCommand; + private string? _metadataCatalogName; + private string? _metadataSchemaName; + private string? _metadataTableName; + private string? _metadataColumnName; + private string? _metadataForeignCatalogName; + private string? _metadataForeignSchemaName; + private string? _metadataForeignTableName; + public StatementExecutionStatement( IStatementExecutionClient client, string? sessionId, @@ -80,8 +96,9 @@ public StatementExecutionStatement( System.Buffers.ArrayPool lz4BufferPool, HttpClient httpClient, StatementExecutionConnection connection) - : base(connection) // Initialize TracingStatement base class with TracingConnection + : base(connection) { + _connection = connection ?? throw new ArgumentNullException(nameof(connection)); _client = client ?? throw new ArgumentNullException(nameof(client)); _sessionId = sessionId; _warehouseId = warehouseId ?? throw new ArgumentNullException(nameof(warehouseId)); @@ -107,19 +124,52 @@ public override string? SqlQuery set => _sqlQuery = value; } - /// - /// Executes the query and returns a result set. - /// + public override void SetOption(string key, string value) + { + switch (key) + { + case ApacheParameters.IsMetadataCommand: + _isMetadataCommand = bool.TryParse(value, out bool b) && b; + break; + case ApacheParameters.CatalogName: + _metadataCatalogName = value; + break; + case ApacheParameters.SchemaName: + _metadataSchemaName = value; + break; + case ApacheParameters.TableName: + _metadataTableName = value; + break; + case ApacheParameters.ColumnName: + _metadataColumnName = value; + break; + case ApacheParameters.ForeignCatalogName: + _metadataForeignCatalogName = value; + break; + case ApacheParameters.ForeignSchemaName: + _metadataForeignSchemaName = value; + break; + case ApacheParameters.ForeignTableName: + _metadataForeignTableName = value; + break; + default: + base.SetOption(key, value); + break; + } + } + public override QueryResult ExecuteQuery() { return ExecuteQueryAsync(CancellationToken.None).GetAwaiter().GetResult(); } - /// - /// Executes the query asynchronously and returns a result set. - /// public async Task ExecuteQueryAsync(CancellationToken cancellationToken = default) { + if (_isMetadataCommand) + { + return ExecuteMetadataCommand(); + } + if (string.IsNullOrEmpty(_sqlQuery)) { throw new InvalidOperationException("SQL query is required"); @@ -613,6 +663,396 @@ private static async Task FetchAllChunksAsync( } } + // Metadata command routing + + private string? EffectiveCatalog => MetadataUtilities.NormalizeSparkCatalog(_metadataCatalogName) ?? _catalog; + + private QueryResult ExecuteMetadataCommand() + { + return _sqlQuery?.ToLowerInvariant() switch + { + "getcatalogs" => GetCatalogs(), + "getschemas" => GetSchemas(), + "gettables" => GetTables(), + "getcolumns" => GetColumns(), + "getcolumnsextended" => GetColumnsExtended(), + "getprimarykeys" => GetPrimaryKeys(), + "getcrossreference" => GetCrossReference(), + _ => throw new NotSupportedException($"Metadata command '{_sqlQuery}' is not supported"), + }; + } + + private QueryResult GetCatalogs() + { + return this.TraceActivity(activity => + { + activity?.SetTag("catalog_pattern", _metadataCatalogName ?? "(none)"); + + string sql = new ShowCatalogsCommand(_metadataCatalogName).Build(); + activity?.SetTag("sql_query", sql); + var batches = _connection.ExecuteMetadataSql(sql, _connection.CreateMetadataTimeoutToken()); + + var tableCatBuilder = new StringArray.Builder(); + int count = 0; + foreach (var batch in batches) + { + var catalogArray = TryGetColumn(batch, "catalog"); + if (catalogArray == null) continue; + for (int i = 0; i < batch.Length; i++) + { + if (!catalogArray.IsNull(i)) + { + tableCatBuilder.Append(catalogArray.GetString(i)); + count++; + } + } + } + + activity?.SetTag("result_count", count); + var schema = MetadataSchemaFactory.CreateCatalogsSchema(); + return new QueryResult(count, new HiveInfoArrowStream(schema, new IArrowArray[] { tableCatBuilder.Build() })); + }, "GetCatalogs"); + } + + private QueryResult GetSchemas() + { + return this.TraceActivity(activity => + { + activity?.SetTag("catalog", EffectiveCatalog ?? "(none)"); + activity?.SetTag("schema_pattern", _metadataSchemaName ?? "(none)"); + + string sql = new ShowSchemasCommand(EffectiveCatalog, _metadataSchemaName).Build(); + activity?.SetTag("sql_query", sql); + var batches = _connection.ExecuteMetadataSql(sql, _connection.CreateMetadataTimeoutToken()); + + var tableSchemaBuilder = new StringArray.Builder(); + var tableCatalogBuilder = new StringArray.Builder(); + int count = 0; + foreach (var batch in batches) + { + var schemaArray = TryGetColumn(batch, "databaseName"); + var catalogArray = TryGetColumn(batch, "catalog_name"); + if (schemaArray == null) continue; + for (int i = 0; i < batch.Length; i++) + { + if (schemaArray.IsNull(i)) continue; + tableSchemaBuilder.Append(schemaArray.GetString(i)); + string catalog = catalogArray != null && !catalogArray.IsNull(i) + ? catalogArray.GetString(i) + : EffectiveCatalog ?? ""; + tableCatalogBuilder.Append(catalog); + count++; + } + } + + activity?.SetTag("result_count", count); + var schema = MetadataSchemaFactory.CreateSchemasSchema(); + return new QueryResult(count, new HiveInfoArrowStream(schema, new IArrowArray[] + { + tableSchemaBuilder.Build(), tableCatalogBuilder.Build() + })); + }, "GetSchemas"); + } + + private QueryResult GetTables() + { + return this.TraceActivity(activity => + { + activity?.SetTag("catalog", EffectiveCatalog ?? "(none)"); + activity?.SetTag("schema_pattern", _metadataSchemaName ?? "(none)"); + activity?.SetTag("table_pattern", _metadataTableName ?? "(none)"); + + string sql = new ShowTablesCommand(EffectiveCatalog, _metadataSchemaName, _metadataTableName).Build(); + activity?.SetTag("sql_query", sql); + var batches = _connection.ExecuteMetadataSql(sql, _connection.CreateMetadataTimeoutToken()); + + var tableCatBuilder = new StringArray.Builder(); + var tableSchemaBuilder = new StringArray.Builder(); + var tableNameBuilder = new StringArray.Builder(); + var tableTypeBuilder = new StringArray.Builder(); + var remarksBuilder = new StringArray.Builder(); + var typeCatBuilder = new StringArray.Builder(); + var typeSchemaBuilder = new StringArray.Builder(); + var typeNameBuilder = new StringArray.Builder(); + var selfRefColBuilder = new StringArray.Builder(); + var refGenBuilder = new StringArray.Builder(); + int count = 0; + foreach (var batch in batches) + { + var catalogArray = TryGetColumn(batch, "catalogName"); + var schemaArray = TryGetColumn(batch, "namespace"); + var tableArray = TryGetColumn(batch, "tableName"); + var tableTypeArray = TryGetColumn(batch, "tableType"); + var remarksArray = TryGetColumn(batch, "remarks"); + if (catalogArray == null || schemaArray == null || tableArray == null) continue; + for (int i = 0; i < batch.Length; i++) + { + if (catalogArray.IsNull(i) || schemaArray.IsNull(i) || tableArray.IsNull(i)) continue; + tableCatBuilder.Append(catalogArray.GetString(i)); + tableSchemaBuilder.Append(schemaArray.GetString(i)); + tableNameBuilder.Append(tableArray.GetString(i)); + string tableType = tableTypeArray != null && !tableTypeArray.IsNull(i) ? tableTypeArray.GetString(i) : "TABLE"; + tableTypeBuilder.Append(tableType); + remarksBuilder.Append(remarksArray != null && !remarksArray.IsNull(i) ? remarksArray.GetString(i) : ""); + typeCatBuilder.AppendNull(); + typeSchemaBuilder.AppendNull(); + typeNameBuilder.AppendNull(); + selfRefColBuilder.AppendNull(); + refGenBuilder.AppendNull(); + count++; + } + } + + activity?.SetTag("result_count", count); + var schema = MetadataSchemaFactory.CreateTablesSchema(); + return new QueryResult(count, new HiveInfoArrowStream(schema, new IArrowArray[] + { + tableCatBuilder.Build(), tableSchemaBuilder.Build(), tableNameBuilder.Build(), + tableTypeBuilder.Build(), remarksBuilder.Build(), typeCatBuilder.Build(), + typeSchemaBuilder.Build(), typeNameBuilder.Build(), selfRefColBuilder.Build(), + refGenBuilder.Build() + })); + }, "GetTables"); + } + + private QueryResult GetColumns() + { + return this.TraceActivity(activity => + { + activity?.SetTag("catalog", EffectiveCatalog ?? "(none)"); + activity?.SetTag("schema_pattern", _metadataSchemaName ?? "(none)"); + activity?.SetTag("table_pattern", _metadataTableName ?? "(none)"); + activity?.SetTag("column_pattern", _metadataColumnName ?? "(none)"); + + string sql = new ShowColumnsCommand( + EffectiveCatalog, _metadataSchemaName, + _metadataTableName, _metadataColumnName).Build(); + activity?.SetTag("sql_query", sql); + var batches = _connection.ExecuteMetadataSql(sql, _connection.CreateMetadataTimeoutToken()); + + var tableInfos = new Dictionary(); + + foreach (var batch in batches) + { + var catalogArray = TryGetColumn(batch, "catalogName"); + var schemaArray = TryGetColumn(batch, "namespace"); + var tableArray = TryGetColumn(batch, "tableName"); + var colNameArray = TryGetColumn(batch, "col_name"); + var columnTypeArray = TryGetColumn(batch, "columnType"); + var isNullableArray = TryGetColumn(batch, "isNullable"); + + if (catalogArray == null || schemaArray == null || tableArray == null || + colNameArray == null || columnTypeArray == null) continue; + + for (int i = 0; i < batch.Length; i++) + { + if (catalogArray.IsNull(i) || schemaArray.IsNull(i) || tableArray.IsNull(i) || + colNameArray.IsNull(i) || columnTypeArray.IsNull(i)) continue; + + string cat = catalogArray.GetString(i); + string sch = schemaArray.GetString(i); + string tbl = tableArray.GetString(i); + string key = $"{cat}.{sch}.{tbl}"; + + if (!tableInfos.ContainsKey(key)) + tableInfos[key] = (cat, sch, tbl, new TableInfo("TABLE")); + + var entry = tableInfos[key]; + bool nullable = isNullableArray == null || isNullableArray.IsNull(i) || + !isNullableArray.GetString(i).Equals("false", StringComparison.OrdinalIgnoreCase); + + ColumnMetadataHelper.PopulateTableInfoFromTypeName( + entry.info, + colNameArray.GetString(i), + columnTypeArray.GetString(i), + entry.info.ColumnName.Count, + nullable); + } + } + + activity?.SetTag("result_tables", tableInfos.Count); + return FlatColumnsResultBuilder.BuildFlatColumnsResult(tableInfos.Values); + }, "GetColumns"); + } + + private QueryResult GetColumnsExtended() + { + return this.TraceActivity(activity => + { + activity?.SetTag("catalog", EffectiveCatalog ?? "(none)"); + activity?.SetTag("schema", _metadataSchemaName ?? "(none)"); + activity?.SetTag("table", _metadataTableName ?? "(none)"); + + string? fullTableName = MetadataUtilities.BuildQualifiedTableName( + EffectiveCatalog, _metadataSchemaName, _metadataTableName); + + if (string.IsNullOrEmpty(fullTableName)) + throw new ArgumentException("Catalog, schema, and table name are required for GetColumnsExtended"); + + string query = $"DESC TABLE EXTENDED {fullTableName} AS JSON"; + activity?.SetTag("sql_query", query); + var batches = _connection.ExecuteMetadataSql(query, _connection.CreateMetadataTimeoutToken()); + + string? resultJson = null; + foreach (var batch in batches) + { + if (batch.Length > 0) + { + resultJson = ((StringArray)batch.Column(0)).GetString(0); + break; + } + } + + if (string.IsNullOrEmpty(resultJson)) + throw new FormatException($"Empty result from {query}"); + + var descResult = System.Text.Json.JsonSerializer.Deserialize(resultJson!); + if (descResult == null) + throw new FormatException($"Failed to parse JSON result from {query}"); + + activity?.SetTag("result_columns", descResult.Columns?.Count ?? 0); + activity?.SetTag("result_pk_count", descResult.PrimaryKeys?.Count ?? 0); + activity?.SetTag("result_fk_count", descResult.ForeignKeys?.Count ?? 0); + + return DatabricksStatement.CreateExtendedColumnsResult( + MetadataSchemaFactory.CreateColumnMetadataSchema(), descResult); + }, "GetColumnsExtended"); + } + + private QueryResult GetPrimaryKeys() + { + return this.TraceActivity(activity => + { + activity?.SetTag("catalog", _metadataCatalogName ?? "(none)"); + activity?.SetTag("schema", _metadataSchemaName ?? "(none)"); + activity?.SetTag("table", _metadataTableName ?? "(none)"); + + if (MetadataUtilities.IsInvalidPKFKCatalog(_metadataCatalogName)) + return MetadataSchemaFactory.CreateEmptyPrimaryKeysResult(); + + if (string.IsNullOrEmpty(_metadataCatalogName) || string.IsNullOrEmpty(_metadataSchemaName) || + string.IsNullOrEmpty(_metadataTableName)) + return MetadataSchemaFactory.CreateEmptyPrimaryKeysResult(); + + List batches; + try + { + string sql = new ShowKeysCommand(_metadataCatalogName!, _metadataSchemaName!, _metadataTableName!).Build(); + activity?.SetTag("sql_query", sql); + batches = _connection.ExecuteMetadataSql(sql, _connection.CreateMetadataTimeoutToken()); + } + catch + { + return MetadataSchemaFactory.CreateEmptyPrimaryKeysResult(); + } + + var keys = new List<(string, string, string, string, int, string)>(); + int seq = 0; + foreach (var batch in batches) + { + var colNameArray = TryGetColumn(batch, "col_name"); + var keyNameArray = TryGetColumn(batch, "constraintName"); + var keySeqArray = TryGetColumn(batch, "keySeq"); + if (colNameArray == null) continue; + for (int i = 0; i < batch.Length; i++) + { + if (colNameArray.IsNull(i)) continue; + int keySeq = keySeqArray != null && !keySeqArray.IsNull(i) ? keySeqArray.GetValue(i)!.Value : ++seq; + string pkName = keyNameArray != null && !keyNameArray.IsNull(i) ? keyNameArray.GetString(i) : ""; + keys.Add((_metadataCatalogName!, _metadataSchemaName!, _metadataTableName!, + colNameArray.GetString(i), keySeq, pkName)); + } + } + + activity?.SetTag("result_count", keys.Count); + return MetadataSchemaFactory.BuildPrimaryKeysResult(keys); + }, "GetPrimaryKeys"); + } + + private QueryResult GetCrossReference() + { + return this.TraceActivity(activity => + { + activity?.SetTag("pk_catalog", _metadataCatalogName ?? "(none)"); + activity?.SetTag("pk_schema", _metadataSchemaName ?? "(none)"); + activity?.SetTag("pk_table", _metadataTableName ?? "(none)"); + activity?.SetTag("fk_catalog", _metadataForeignCatalogName ?? "(none)"); + activity?.SetTag("fk_schema", _metadataForeignSchemaName ?? "(none)"); + activity?.SetTag("fk_table", _metadataForeignTableName ?? "(none)"); + + if (MetadataUtilities.IsInvalidPKFKCatalog(_metadataForeignCatalogName)) + return MetadataSchemaFactory.CreateEmptyCrossReferenceResult(); + + if (string.IsNullOrEmpty(_metadataForeignCatalogName) || string.IsNullOrEmpty(_metadataForeignSchemaName) || + string.IsNullOrEmpty(_metadataForeignTableName)) + return MetadataSchemaFactory.CreateEmptyCrossReferenceResult(); + + List batches; + try + { + string sql = new ShowForeignKeysCommand( + _metadataForeignCatalogName!, _metadataForeignSchemaName!, _metadataForeignTableName!).Build(); + activity?.SetTag("sql_query", sql); + batches = _connection.ExecuteMetadataSql(sql, _connection.CreateMetadataTimeoutToken()); + } + catch + { + return MetadataSchemaFactory.CreateEmptyCrossReferenceResult(); + } + + var refs = new List<(string, string, string, string, string, string, string, string, int, int, int, string, string?, int)>(); + int seq = 0; + foreach (var batch in batches) + { + var pkCatalogArray = TryGetColumn(batch, "parentCatalogName"); + var pkSchemaArray = TryGetColumn(batch, "parentNamespace"); + var pkTableArray = TryGetColumn(batch, "parentTableName"); + var pkColArray = TryGetColumn(batch, "parentColName"); + var fkCatalogArray = TryGetColumn(batch, "catalogName"); + var fkSchemaArray = TryGetColumn(batch, "namespace"); + var fkTableArray = TryGetColumn(batch, "tableName"); + var fkColArray = TryGetColumn(batch, "col_name"); + var fkNameArray = TryGetColumn(batch, "constraintName"); + var fkKeySeqArray = TryGetColumn(batch, "keySeq"); + var fkUpdateRuleArray = TryGetColumn(batch, "updateRule"); + var fkDeleteRuleArray = TryGetColumn(batch, "deleteRule"); + var fkDeferrabilityArray = TryGetColumn(batch, "deferrability"); + + if (fkColArray == null) continue; + + for (int i = 0; i < batch.Length; i++) + { + if (fkColArray.IsNull(i)) continue; + refs.Add(( + pkCatalogArray != null && !pkCatalogArray.IsNull(i) ? pkCatalogArray.GetString(i) : _metadataCatalogName ?? "", + pkSchemaArray != null && !pkSchemaArray.IsNull(i) ? pkSchemaArray.GetString(i) : _metadataSchemaName ?? "", + pkTableArray != null && !pkTableArray.IsNull(i) ? pkTableArray.GetString(i) : _metadataTableName ?? "", + pkColArray != null && !pkColArray.IsNull(i) ? pkColArray.GetString(i) : "", + fkCatalogArray != null && !fkCatalogArray.IsNull(i) ? fkCatalogArray.GetString(i) : _metadataForeignCatalogName!, + fkSchemaArray != null && !fkSchemaArray.IsNull(i) ? fkSchemaArray.GetString(i) : _metadataForeignSchemaName!, + fkTableArray != null && !fkTableArray.IsNull(i) ? fkTableArray.GetString(i) : _metadataForeignTableName!, + fkColArray.GetString(i), + fkKeySeqArray != null && !fkKeySeqArray.IsNull(i) ? fkKeySeqArray.GetValue(i)!.Value : ++seq, + fkUpdateRuleArray != null && !fkUpdateRuleArray.IsNull(i) ? fkUpdateRuleArray.GetValue(i)!.Value : 0, + fkDeleteRuleArray != null && !fkDeleteRuleArray.IsNull(i) ? fkDeleteRuleArray.GetValue(i)!.Value : 0, + fkNameArray != null && !fkNameArray.IsNull(i) ? fkNameArray.GetString(i) : "", + (string?)null, + fkDeferrabilityArray != null && !fkDeferrabilityArray.IsNull(i) ? fkDeferrabilityArray.GetValue(i)!.Value : 5 + )); + } + } + + activity?.SetTag("result_count", refs.Count); + return MetadataSchemaFactory.BuildCrossReferenceResult(refs); + }, "GetCrossReference"); + } + + private static T? TryGetColumn(RecordBatch batch, string name) where T : class, IArrowArray + { + try { return batch.Column(name) as T; } + catch (ArgumentOutOfRangeException) { return null; } + } + // TracingStatement implementation public override string AssemblyVersion => GetType().Assembly.GetName().Version?.ToString() ?? "1.0.0"; public override string AssemblyName => "AdbcDrivers.Databricks"; diff --git a/csharp/test/ColumnMetadataHelperTests.cs b/csharp/test/ColumnMetadataHelperTests.cs new file mode 100644 index 000000000..d23214da6 --- /dev/null +++ b/csharp/test/ColumnMetadataHelperTests.cs @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2025 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using AdbcDrivers.Databricks; +using Xunit; +using static AdbcDrivers.HiveServer2.Hive2.HiveServer2Connection; + +namespace AdbcDrivers.Databricks.Tests +{ + public class ColumnMetadataHelperTests + { + [Theory] + [InlineData("BOOLEAN", (short)ColumnTypeId.BOOLEAN)] + [InlineData("TINYINT", (short)ColumnTypeId.TINYINT)] + [InlineData("BYTE", (short)ColumnTypeId.TINYINT)] + [InlineData("SMALLINT", (short)ColumnTypeId.SMALLINT)] + [InlineData("SHORT", (short)ColumnTypeId.SMALLINT)] + [InlineData("INT", (short)ColumnTypeId.INTEGER)] + [InlineData("INTEGER", (short)ColumnTypeId.INTEGER)] + [InlineData("BIGINT", (short)ColumnTypeId.BIGINT)] + [InlineData("LONG", (short)ColumnTypeId.BIGINT)] + [InlineData("FLOAT", (short)ColumnTypeId.FLOAT)] + [InlineData("DOUBLE", (short)ColumnTypeId.DOUBLE)] + [InlineData("DECIMAL", (short)ColumnTypeId.DECIMAL)] + [InlineData("DECIMAL(10,2)", (short)ColumnTypeId.DECIMAL)] + [InlineData("DEC", (short)ColumnTypeId.DECIMAL)] + [InlineData("NUMERIC", (short)ColumnTypeId.DECIMAL)] // SqlTypeNameParser normalizes NUMERIC → DECIMAL + [InlineData("STRING", (short)ColumnTypeId.VARCHAR)] + [InlineData("VARCHAR", (short)ColumnTypeId.VARCHAR)] + [InlineData("VARCHAR(255)", (short)ColumnTypeId.VARCHAR)] + [InlineData("CHAR", (short)ColumnTypeId.CHAR)] + [InlineData("BINARY", (short)ColumnTypeId.BINARY)] + [InlineData("DATE", (short)ColumnTypeId.DATE)] + [InlineData("TIMESTAMP", (short)ColumnTypeId.TIMESTAMP)] + [InlineData("TIMESTAMP_NTZ", (short)ColumnTypeId.TIMESTAMP)] + [InlineData("TIMESTAMP_LTZ", (short)ColumnTypeId.TIMESTAMP)] + [InlineData("ARRAY", (short)ColumnTypeId.ARRAY)] + [InlineData("MAP", (short)ColumnTypeId.JAVA_OBJECT)] + [InlineData("STRUCT", (short)ColumnTypeId.STRUCT)] + [InlineData("VOID", (short)ColumnTypeId.NULL)] + [InlineData("INTERVAL YEAR TO MONTH", (short)ColumnTypeId.OTHER)] + [InlineData("UNKNOWN_TYPE", (short)ColumnTypeId.OTHER)] + public void GetDataTypeCode_ReturnsCorrectCode(string typeName, short expectedCode) + { + Assert.Equal(expectedCode, ColumnMetadataHelper.GetDataTypeCode(typeName)); + } + + [Theory] + [InlineData("DECIMAL(10,2)", "DECIMAL")] + [InlineData("VARCHAR(255)", "VARCHAR")] + [InlineData("ARRAY", "ARRAY")] + [InlineData("MAP", "MAP")] + [InlineData("STRUCT", "STRUCT")] + [InlineData("INT", "INTEGER")] + [InlineData("INTEGER", "INTEGER")] + [InlineData("DEC", "DECIMAL")] + [InlineData("TIMESTAMP_NTZ", "TIMESTAMP")] + [InlineData("TIMESTAMP_LTZ", "TIMESTAMP")] + [InlineData("BYTE", "TINYINT")] + [InlineData("SHORT", "SMALLINT")] + [InlineData("LONG", "BIGINT")] + [InlineData("STRING", "STRING")] + [InlineData("BOOLEAN", "BOOLEAN")] + [InlineData("DOUBLE", "DOUBLE")] + public void GetBaseTypeName_ReturnsCorrectName(string typeName, string expectedBase) + { + Assert.Equal(expectedBase, ColumnMetadataHelper.GetBaseTypeName(typeName)); + } + + [Theory] + [InlineData("BOOLEAN", 1)] + [InlineData("TINYINT", 1)] + [InlineData("SMALLINT", 2)] + [InlineData("INT", 4)] + [InlineData("INTEGER", 4)] + [InlineData("BIGINT", 8)] + [InlineData("FLOAT", 4)] + [InlineData("DOUBLE", 8)] + [InlineData("TIMESTAMP", 8)] + [InlineData("DATE", 4)] + [InlineData("DECIMAL(10,2)", 10)] + [InlineData("DECIMAL", 10)] + [InlineData("VARCHAR(255)", 255)] + [InlineData("STRING", int.MaxValue)] + [InlineData("CHAR(50)", 50)] + public void GetColumnSizeDefault_ReturnsCorrectSize(string typeName, int expectedSize) + { + Assert.Equal(expectedSize, ColumnMetadataHelper.GetColumnSizeDefault(typeName)); + } + + [Theory] + [InlineData("DECIMAL(10,2)", 2)] + [InlineData("DECIMAL", 0)] + [InlineData("FLOAT", 7)] + [InlineData("DOUBLE", 15)] + [InlineData("TIMESTAMP", 6)] + [InlineData("INT", 0)] + [InlineData("STRING", 0)] + [InlineData("BOOLEAN", 0)] + public void GetDecimalDigitsDefault_ReturnsCorrectDigits(string typeName, int expectedDigits) + { + Assert.Equal(expectedDigits, ColumnMetadataHelper.GetDecimalDigitsDefault(typeName)); + } + + [Theory] + [InlineData("BOOLEAN", 1)] + [InlineData("TINYINT", 1)] + [InlineData("SMALLINT", 2)] + [InlineData("INT", 4)] + [InlineData("BIGINT", 8)] + [InlineData("FLOAT", 4)] + [InlineData("DOUBLE", 8)] + [InlineData("DECIMAL(10,2)", 11)] + public void GetBufferLength_ReturnsCorrectLength(string typeName, int expectedLength) + { + Assert.Equal(expectedLength, ColumnMetadataHelper.GetBufferLength(typeName)); + } + + [Fact] + public void GetBufferLength_ReturnsNullForNonNumeric() + { + Assert.Null(ColumnMetadataHelper.GetBufferLength("STRING")); + Assert.Null(ColumnMetadataHelper.GetBufferLength("ARRAY")); + } + + [Theory] + [InlineData("INT", (short)10)] + [InlineData("DECIMAL", (short)10)] + [InlineData("FLOAT", (short)10)] + [InlineData("DOUBLE", (short)10)] + [InlineData("BIGINT", (short)10)] + public void GetNumPrecRadix_Returns10ForNumeric(string typeName, short expected) + { + Assert.Equal(expected, ColumnMetadataHelper.GetNumPrecRadix(typeName)); + } + + [Theory] + [InlineData("STRING")] + [InlineData("BOOLEAN")] + [InlineData("DATE")] + [InlineData("TIMESTAMP")] + [InlineData("ARRAY")] + public void GetNumPrecRadix_ReturnsNullForNonNumeric(string typeName) + { + Assert.Null(ColumnMetadataHelper.GetNumPrecRadix(typeName)); + } + + [Theory] + [InlineData("STRING", int.MaxValue)] + [InlineData("VARCHAR(255)", 255)] + [InlineData("CHAR(50)", 50)] + public void GetCharOctetLength_ReturnsForCharTypes(string typeName, int expectedLength) + { + Assert.Equal(expectedLength, ColumnMetadataHelper.GetCharOctetLength(typeName)); + } + + [Theory] + [InlineData("INT")] + [InlineData("DECIMAL(10,2)")] + [InlineData("BOOLEAN")] + public void GetCharOctetLength_ReturnsNullForNonCharTypes(string typeName) + { + Assert.Null(ColumnMetadataHelper.GetCharOctetLength(typeName)); + } + + [Theory] + [InlineData("INTERVAL YEAR TO MONTH", 4)] + [InlineData("INTERVAL DAY TO SECOND", 8)] + [InlineData("INTERVAL HOUR TO SECOND", 8)] + public void GetColumnSizeDefault_HandlesIntervalTypes(string typeName, int expectedSize) + { + Assert.Equal(expectedSize, ColumnMetadataHelper.GetColumnSizeDefault(typeName)); + } + + [Theory] + [InlineData(" DECIMAL(10,2) ", (short)ColumnTypeId.DECIMAL)] + [InlineData("decimal", (short)ColumnTypeId.DECIMAL)] + [InlineData(" int ", (short)ColumnTypeId.INTEGER)] + public void GetDataTypeCode_HandlesTrimAndCase(string typeName, short expectedCode) + { + Assert.Equal(expectedCode, ColumnMetadataHelper.GetDataTypeCode(typeName)); + } + } +} diff --git a/csharp/test/Unit/Result/DescTableExtendedResultTest.cs b/csharp/test/Unit/Result/DescTableExtendedResultTest.cs index 91bd87e1b..4bf4f1da3 100644 --- a/csharp/test/Unit/Result/DescTableExtendedResultTest.cs +++ b/csharp/test/Unit/Result/DescTableExtendedResultTest.cs @@ -351,7 +351,7 @@ public void TestTableWithConstraints(string constraints, string[] primaryKeys, s [InlineData("CHAR", ColumnTypeId.CHAR, false, 20)] [InlineData("VARCHAR", ColumnTypeId.VARCHAR, false, 20)] [InlineData("STRING", ColumnTypeId.VARCHAR, false, int.MaxValue)] - [InlineData("BINARY", ColumnTypeId.BINARY, false, 0)] + [InlineData("BINARY", ColumnTypeId.BINARY, false, int.MaxValue)] [InlineData("DATE", ColumnTypeId.DATE, false, 4)] [InlineData("TIMESTAMP", ColumnTypeId.TIMESTAMP, false, 8)] [InlineData("TIMESTAMP_LTZ", ColumnTypeId.TIMESTAMP, false, 8)]