[JDBC-V2] Fix some metadata resultsets return inconsistent data on getObject()

clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java

@@ -38,6 +38,7 @@
 import java.io.Serializable;
 import java.lang.reflect.Array;
 import java.math.BigInteger;
+import java.sql.SQLException;
 import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -90,6 +91,7 @@ public final class ClickHouseColumn implements Serializable {
     private List<ClickHouseColumn> nested;
     private List<String> parameters;
     private ClickHouseEnum enumConstants;
+    private ValueFunction valueFunction;
 
     private int arrayLevel;
     private ClickHouseColumn arrayBaseColumn;
@@ -787,6 +789,18 @@ public boolean isNestedType() {
         return dataType.isNested();
     }
 
+    public boolean hasValueFunction() {
+        return valueFunction != null;
+    }
+
+    public void setValueFunction(ValueFunction valueFunction) {
+        this.valueFunction = valueFunction;
+    }
+
+    public ValueFunction getValueFunction() {
+        return valueFunction;
+    }
+
     public int getArrayNestedLevel() {
         return arrayLevel;
     }
@@ -1126,4 +1140,9 @@ public String toString() {
         }
         return builder.append(' ').append(originalTypeName).toString();
     }
+
+    public interface ValueFunction {
+
+        Object produceValue(Object[] row);
+    }
 }

client-v2/src/main/java/com/clickhouse/client/api/data_formats/ClickHouseBinaryFormatReader.java

@@ -1,6 +1,7 @@
 package com.clickhouse.client.api.data_formats;
 
 import com.clickhouse.client.api.metadata.TableSchema;
+import com.clickhouse.data.ClickHouseColumn;
 import com.clickhouse.data.value.ClickHouseBitmap;
 import com.clickhouse.data.value.ClickHouseGeoMultiPolygonValue;
 import com.clickhouse.data.value.ClickHouseGeoPointValue;
@@ -545,4 +546,13 @@ public interface ClickHouseBinaryFormatReader extends AutoCloseable {
     TemporalAmount getTemporalAmount(int index);
 
     TemporalAmount getTemporalAmount(String colName);
+
+    /**
+     * ! Experimental ! Might change in the future.
+     * Sets a value function of a column. If column has a value function then reader will pass current row
+     * as Object[] to a function. The least is responsible for returning correct value or null.
+     * @param index - column index starting with 1
+     * @param function - function that will be used to calculate column value from current row.
+     */
+    default void setValueFunction(int index, ClickHouseColumn.ValueFunction function) {}
 }

client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/AbstractBinaryFormatReader.java

@@ -131,6 +131,11 @@ public boolean readToPOJO(Map<String, POJOFieldDeserializer> deserializers, Obje
         return true;
     }
 
+    @Override
+    public void setValueFunction(int index, ClickHouseColumn.ValueFunction function) {
+        columns[index - 1].setValueFunction(function);
+    }
+
     /**
      * It is still internal method and should be used with care.
      * Usually this method is called to read next record into internal object and affects hasNext() method.
@@ -148,7 +153,11 @@ public boolean readRecord(Map<String, Object> record) throws IOException {
         }
 
         boolean firstColumn = true;
+        boolean hasValueFunctionColumn = false;
         for (ClickHouseColumn column : columns) {
+            if (column.hasValueFunction()) {
+                hasValueFunctionColumn = true;
+            }
             try {
                 Object val = binaryStreamReader.readValue(column);
                 if (val != null) {
@@ -165,6 +174,16 @@ public boolean readRecord(Map<String, Object> record) throws IOException {
                 throw e;
             }
         }
+
+        if (hasValueFunctionColumn) {
+            // This variant of readRecord is called only for POJO serialization and this logic should be avoided.
+            Object[] row = record.values().toArray();
+            for (ClickHouseColumn column : columns) {
+                if (column.hasValueFunction()) {
+                    record.put(column.getColumnName(), column.getValueFunction().produceValue(row));
+                }
+            }
+        }
         return true;
     }
 
@@ -174,9 +193,14 @@ protected boolean readRecord(Object[] record) throws IOException {
         }
 
         boolean firstColumn = true;
+        boolean hasValueFunctionColumn = false;
         for (int i = 0; i < columns.length; i++) {
             try {
-                Object val = binaryStreamReader.readValue(columns[i]);
+                ClickHouseColumn column = columns[i];
+                if (column.hasValueFunction()) {
+                    hasValueFunctionColumn = true;
+                }
+                Object val = binaryStreamReader.readValue(column);
                 if (val != null) {
                     record[i] = val;
                 } else {
@@ -191,9 +215,19 @@ protected boolean readRecord(Object[] record) throws IOException {
                 throw e;
             }
         }
+
+        if (hasValueFunctionColumn) {
+            for  (int i = 0; i < columns.length; i++) {
+                ClickHouseColumn column = columns[i];
+                if (column.hasValueFunction()) {
+                    record[i] = column.getValueFunction().produceValue(record);
+                }
+            }
+        }
         return true;
     }
 
+
     @Override
     public <T> T readValue(int colIndex) {
         if (colIndex < 1 || colIndex > getSchema().getColumns().size()) {

jdbc-v2/src/main/java/com/clickhouse/jdbc/ResultSetImpl.java

@@ -3,6 +3,7 @@
 import com.clickhouse.client.api.data_formats.ClickHouseBinaryFormatReader;
 import com.clickhouse.client.api.metadata.TableSchema;
 import com.clickhouse.client.api.query.QueryResponse;
+import com.clickhouse.data.ClickHouseColumn;
 import com.clickhouse.data.ClickHouseDataType;
 import com.clickhouse.jdbc.internal.ExceptionUtils;
 import com.clickhouse.jdbc.internal.JdbcUtils;
@@ -40,7 +41,7 @@
 
 public class ResultSetImpl implements ResultSet, JdbcV2Wrapper {
     private static final Logger log = LoggerFactory.getLogger(ResultSetImpl.class);
-    private ResultSetMetaData metaData;
+    private ResultSetMetaDataImpl metaData;
     protected ClickHouseBinaryFormatReader reader;
     private QueryResponse response;
     private boolean closed;
@@ -138,6 +139,14 @@ public void close() throws SQLException {
         }
     }
 
+    public void setValueFunction(int colIndex, ClickHouseColumn.ValueFunction valueFunction) {
+        reader.setValueFunction(colIndex, valueFunction);
+    }
+
+    public void hideLastNColumns(int n) {
+        metaData.setColumnCount(metaData.getOriginalColumnCount() - n);
+    }
+
     @Override
     public boolean wasNull() throws SQLException {
         checkClosed();

jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImpl.java

@@ -11,7 +11,6 @@
 import com.clickhouse.jdbc.ResultSetImpl;
 import com.clickhouse.jdbc.internal.ExceptionUtils;
 import com.clickhouse.jdbc.internal.JdbcUtils;
-import com.clickhouse.jdbc.internal.MetadataResultSet;
 import com.clickhouse.logging.Logger;
 import com.clickhouse.logging.LoggerFactory;
 
@@ -22,6 +21,7 @@
 import java.sql.SQLException;
 import java.sql.SQLFeatureNotSupportedException;
 import java.sql.SQLType;
+import java.sql.Types;
 import java.util.Arrays;
 
 public class DatabaseMetaDataImpl implements java.sql.DatabaseMetaData, JdbcV2Wrapper {
@@ -222,7 +222,7 @@ public String getStringFunctions() throws SQLException {
     public String getSystemFunctions() throws SQLException {
         // took from below URL(not from system.functions):
         // https://clickhouse.com/docs/en/sql-reference/functions/other-functions/
-        return "bar,basename,blockNumber,blockSerializedSize,blockSize,buildId,byteSize,countDigits,currentDatabase,currentProfiles,currentRoles,currentUser,defaultProfiles,defaultRoles,defaultValueOfArgumentType,defaultValueOfTypeName,dumpColumnStructure,enabledProfiles,enabledRoles,errorCodeToName,filesystemAvailable,filesystemCapacity,filesystemFree,finalizeAggregation,formatReadableQuantity,formatReadableSize,formatReadableTimeDelta,FQDN,getMacro,getServerPort,getSetting,getSizeOfEnumType,greatest,hasColumnInTable,hostName,identity,ifNotFinite,ignore,indexHint,initializeAggregation,initialQueryID,isConstant,isDecimalOverflow,isFinite,isInfinite,isNaN,joinGet,least,MACNumToString,MACStringToNum,MACStringToOUI,materialize,modelEvaluate,neighbor,queryID,randomFixedString,randomPrintableASCII,randomString,randomStringUTF8,replicate,rowNumberInAllBlocks,rowNumberInBlock,runningAccumulate,runningConcurrency,runningDifference,runningDifferenceStartingWithFirstValue,shardCount ,shardNum,sleep,sleepEachRow,tcpPort,throwIf,toColumnTypeName,toTypeName,transform,uptime,version,visibleWidth";
+        return "bar,basename,blockNumber,blockSerializedSize,blockSize,buildId,byteSize,countDigits,currentDatabase,currentProfiles,currentRoles,currentUser,defaultProfiles,defaultRoles,defaultValueOfArgumentType,defaultValueOfTypeName,dumpColumnStructure,enabledProfiles,enabledRoles,errorCodeToName,filesystemAvailable,filesystemCapacity,filesystemFree,finalizeAggregation,formatReadableQuantity,formatReadableSize,formatReadableTimeDelta,FQDN,getMacro,getServerPort,getSetting,getSizeOfEnumType,greatest,hasColumnInTable,hostName,identity,ifNotFinite,ignore,indexHint,initializeAggregation,initialQueryID,isConstant,isDecimalOverflow,isFinite,isInfinite,isNaN,joinGet,least,MACNumToString,MACStringToNum,MACStringToOUI,materialize,modelEvaluate,neighbor,queryID,randomFixedString,randomPrintableASCII,randomString,randomStringUTF8,replicate,rowNumberInAllBlocks,rowNumberInBlock,runningAccumulate,runningConcurrency,runningDifference,runningDifferenceStartingWithFirstValue,shardCount ,shardNum,sleep,sleepEachRow,tcpPort,throwIf,toColumnTypeName,toTypeName,e,uptime,version,visibleWidth";
     }
 
     @Override
@@ -830,18 +830,19 @@ public ResultSet getTableTypes() throws SQLException {
         }
     }
 
-    private static final ClickHouseColumn DATA_TYPE_COL = ClickHouseColumn.of("DATA_TYPE", ClickHouseDataType.Int32.name()) ;
+    private static final int GET_COLUMNS_TYPE_NAME_COL = 6;
+
+    private static final int GET_COLUMNS_DATA_TYPE_COL = 5;
     @Override
     @SuppressWarnings({"squid:S2095", "squid:S2077"})
     public ResultSet getColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) throws SQLException {
-        //TODO: Best way to convert type to JDBC data type
         // TODO: handle useCatalogs == true and return schema catalog name
-        String sql = "SELECT " +
+        final String sql = "SELECT " +
                 catalogPlaceholder + " AS TABLE_CAT, " +
                 "database AS TABLE_SCHEM, " +
                 "table AS TABLE_NAME, " +
                 "name AS COLUMN_NAME, " +
-                "system.columns.type AS DATA_TYPE, " +
+                "toInt32(" + Types.OTHER + ") AS DATA_TYPE, " +
                 "type AS TYPE_NAME, " +
                 "toInt32(" + generateSqlTypeSizes("system.columns.type") + ") AS COLUMN_SIZE, " +
                 "toInt32(0) AS BUFFER_LENGTH, " +
@@ -867,8 +868,9 @@ public ResultSet getColumns(String catalog, String schemaPattern, String tableNa
                 " AND name LIKE " + SQLUtils.enquoteLiteral(columnNamePattern == null ? "%" : columnNamePattern) +
                 " ORDER BY TABLE_SCHEM, TABLE_NAME, ORDINAL_POSITION";
         try {
-            return new MetadataResultSet((ResultSetImpl) connection.createStatement().executeQuery(sql))
-                    .transform(DATA_TYPE_COL.getColumnName(), DATA_TYPE_COL, DatabaseMetaDataImpl::columnDataTypeToSqlType);
+            ResultSetImpl rs = (ResultSetImpl) connection.createStatement().executeQuery(sql);
+            rs.setValueFunction(GET_COLUMNS_DATA_TYPE_COL, GET_COLUMNS_DATA_TYPE_FUNC);
+            return rs;
         } catch (Exception e) {
             throw ExceptionUtils.toSqlState(e);
         }
@@ -887,17 +889,22 @@ private static String generateSqlTypeSizes(String columnName) {
         return sql.toString();
     }
 
-    private static String columnDataTypeToSqlType(String value) {
-        SQLType type = JdbcUtils.CLICKHOUSE_TYPE_NAME_TO_SQL_TYPE_MAP.get(value);
-        if (type == null) {
-            try {
-                type = JdbcUtils.convertToSqlType(ClickHouseColumn.of("v1", value).getDataType());
-            } catch (Exception e) {
-                log.error("Failed to convert column data type to SQL type: {}", value, e);
-                type = JDBCType.OTHER; // In case of error, return SQL type 0
+    private static final ClickHouseColumn.ValueFunction GET_COLUMNS_DATA_TYPE_FUNC = dataTypeValueFunction(GET_COLUMNS_TYPE_NAME_COL);
+
+    private static ClickHouseColumn.ValueFunction dataTypeValueFunction(int srcColIndex) {
+        return row -> {
+            String typeName = (String) row[srcColIndex - 1];
+            SQLType type = JdbcUtils.CLICKHOUSE_TYPE_NAME_TO_SQL_TYPE_MAP.get(typeName);
+            if (type == null) {
+                try {
+                    type = JdbcUtils.convertToSqlType(ClickHouseColumn.of("v1", typeName).getDataType());
+                } catch (Exception e) {
+                    log.error("Failed to convert column data type to SQL type: {}", typeName, e);
+                    type = JDBCType.OTHER; // In case of error, return SQL type 0
+                }
             }
-        }
-        return String.valueOf(type.getVendorTypeNumber());
+            return type.getVendorTypeNumber();
+        };
     }
 
     @Override
@@ -1067,26 +1074,23 @@ public ResultSet getCrossReference(String parentCatalog, String parentSchema, St
         }
     }
 
-    private static final ClickHouseColumn NULLABLE_COL = ClickHouseColumn.of("NULLABLE", ClickHouseDataType.Int16.name());
+    private static final int TYPE_INFO_DATA_TYPE_COL = 2;
+    private static final int TYPE_INFO_NULLABILITY_COL = 7;
     @Override
     @SuppressWarnings({"squid:S2095"})
     public ResultSet getTypeInfo() throws SQLException {
         try {
-            return new MetadataResultSet((ResultSetImpl) connection.createStatement().executeQuery(DATA_TYPE_INFO_SQL))
-                    .transform(DATA_TYPE_COL.getColumnName(), DATA_TYPE_COL, DatabaseMetaDataImpl::dataTypeToSqlTypeInt)
-                    .transform(NULLABLE_COL.getColumnName(), NULLABLE_COL, DatabaseMetaDataImpl::dataTypeNullability);
+            ResultSetImpl rs = (ResultSetImpl) connection.createStatement().executeQuery(DATA_TYPE_INFO_SQL);
+            rs.setValueFunction(TYPE_INFO_DATA_TYPE_COL, TYPE_INFO_DATA_TYPE_VALUE_FUNC);
+            rs.setValueFunction(TYPE_INFO_NULLABILITY_COL, DatabaseMetaDataImpl::dataTypeNullability);
+            return rs;
         } catch (Exception e) {
             throw ExceptionUtils.toSqlState(e);
         }
     }
 
-    private static String dataTypeToSqlTypeInt(String type) {
-        SQLType sqlType = JdbcUtils.CLICKHOUSE_TYPE_NAME_TO_SQL_TYPE_MAP.get(type);
-        return sqlType == null ? String.valueOf(JDBCType.OTHER.getVendorTypeNumber()) :
-                String.valueOf(sqlType.getVendorTypeNumber());
-    }
-
-    private static String dataTypeNullability(String type) {
+    private static String dataTypeNullability(Object[] row) {
+        String type = (String) row[DATA_TYPE_INFO_SQL_TYPE_NAME_COL - 1];
         if (type.equals(ClickHouseDataType.Nullable.name()) || type.equals(ClickHouseDataType.Dynamic.name())) {
             return String.valueOf(java.sql.DatabaseMetaData.typeNullable);
         }
@@ -1095,21 +1099,23 @@ private static String dataTypeNullability(String type) {
 
     private static final String DATA_TYPE_INFO_SQL = getDataTypeInfoSql();
 
+    private static final int DATA_TYPE_INFO_SQL_TYPE_NAME_COL = 13;
+
     private static String getDataTypeInfoSql() {
         StringBuilder sql = new StringBuilder("SELECT " +
                 "name AS TYPE_NAME, " +
-                "if(empty(alias_to), name, alias_to) AS DATA_TYPE, " + // passing type name or alias if exists to map then
+                "0::Int32 AS DATA_TYPE, " + // passing type name or alias if exists to map then
                 "attrs.c2::Nullable(Int32) AS PRECISION, " +
                 "NULL::Nullable(String) AS LITERAL_PREFIX, " +
                 "NULL::Nullable(String) AS LITERAL_SUFFIX, " +
                 "NULL::Nullable(String) AS CREATE_PARAMS, " +
-                "name AS NULLABLE, " + // passing type name to map for nullable
+                "0::Int16 AS NULLABLE, " + // passing type name to map for nullable
                 "not(dt.case_insensitive)::Boolean AS CASE_SENSITIVE, " +
                 java.sql.DatabaseMetaData.typeSearchable + "::Int16 AS SEARCHABLE, " +
                 "not(attrs.c3)::Boolean AS UNSIGNED_ATTRIBUTE, " +
                 "false AS FIXED_PREC_SCALE, " +
                 "false AS AUTO_INCREMENT, " +
-                "name AS LOCAL_TYPE_NAME, " +
+                "if(empty(alias_to), name, alias_to) AS LOCAL_TYPE_NAME, " +
                 "attrs.c4::Nullable(Int16) AS MINIMUM_SCALE, " +
                 "attrs.c5::Nullable(Int16) AS MAXIMUM_SCALE, " +
                 "0::Nullable(Int32) AS SQL_DATA_TYPE, " +
@@ -1134,6 +1140,8 @@ private static String getDataTypeInfoSql() {
         return sql.toString();
     }
 
+    private static final ClickHouseColumn.ValueFunction TYPE_INFO_DATA_TYPE_VALUE_FUNC = dataTypeValueFunction(DATA_TYPE_INFO_SQL_TYPE_NAME_COL);
+
     @Override
     public ResultSet getIndexInfo(String catalog, String schema, String table, boolean unique, boolean approximate) throws SQLException {
         try {

jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/ResultSetMetaDataImpl.java

@@ -5,6 +5,8 @@
 import com.clickhouse.jdbc.JdbcV2Wrapper;
 import com.clickhouse.jdbc.internal.ExceptionUtils;
 import com.clickhouse.jdbc.internal.JdbcUtils;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.UnmodifiableListIterator;
 
 import java.sql.SQLException;
 import java.util.List;
@@ -22,13 +24,16 @@ public class ResultSetMetaDataImpl implements java.sql.ResultSetMetaData, JdbcV2
 
     private final Map<ClickHouseDataType, Class<?>> typeClassMap;
 
+    private int columnCount;
+
     public ResultSetMetaDataImpl(List<ClickHouseColumn> columns, String schema, String catalog, String tableName,
                                  Map<ClickHouseDataType, Class<?>> typeClassMap) {
-        this.columns = columns;
+        this.columns = ImmutableList.copyOf(columns);
         this.schema = schema;
         this.catalog = catalog;
         this.tableName = tableName;
         this.typeClassMap = typeClassMap;
+        this.columnCount = columns.size();
     }
 
     private ClickHouseColumn getColumn(int column) throws SQLException {
@@ -41,9 +46,24 @@ private ClickHouseColumn getColumn(int column) throws SQLException {
 
     @Override
     public int getColumnCount() throws SQLException {
+        return columnCount;
+    }
+
+    public int getOriginalColumnCount() {
         return columns.size();
     }
 
+    /**
+     * This method used to truncate list of column so it is possible to
+     * "hide" columns from the end of the list.
+     * Note: we use this to implement column replacement. it is needed when DB calculation is too hard compare to a
+     * programmatic approach.
+     * @param columnCount
+     */
+    public void setColumnCount(int columnCount) {
+        this.columnCount = columnCount;
+    }
+
     @Override
     public boolean isAutoIncrement(int column) throws SQLException {
         return false; // no auto-incremental types