Skip to content

Commit d0c17ad

Browse files
chernserchernser
authored
Merge pull request #2531 from ClickHouse/v2_issue_2462
[client-v2] Adding JSON support with predefined paths
2 parents c9b62cb + 1bc5b01 commit d0c17ad

File tree

6 files changed

+173
-7
lines changed

6 files changed

+173
-7
lines changed

‎clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java

Lines changed: 72 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,9 @@
3737

3838
import java.io.Serializable;
3939
import java.lang.reflect.Array;
40-
import java.math.BigInteger;
4140
import java.time.OffsetDateTime;
4241
import java.util.ArrayList;
4342
import java.util.Arrays;
44-
import java.util.Collection;
4543
import java.util.Collections;
4644
import java.util.Comparator;
4745
import java.util.HashMap;
@@ -51,6 +49,7 @@
5149
import java.util.Objects;
5250
import java.util.Set;
5351
import java.util.TimeZone;
52+
import java.util.stream.Collectors;
5453

5554
/**
5655
* This class represents a column defined in database.
@@ -72,6 +71,7 @@ public final class ClickHouseColumn implements Serializable {
7271
private static final String KEYWORD_MAP = ClickHouseDataType.Map.name();
7372
private static final String KEYWORD_NESTED = ClickHouseDataType.Nested.name();
7473
private static final String KEYWORD_VARIANT = ClickHouseDataType.Variant.name();
74+
private static final String KEYWORD_JSON = ClickHouseDataType.JSON.name();
7575

7676
private int columnCount;
7777
private int columnIndex;
@@ -90,6 +90,7 @@ public final class ClickHouseColumn implements Serializable {
9090
private List<ClickHouseColumn> nested;
9191
private List<String> parameters;
9292
private ClickHouseEnum enumConstants;
93+
private Map<String, ClickHouseColumn> jsonPredefinedPaths;
9394

9495
private int arrayLevel;
9596
private ClickHouseColumn arrayBaseColumn;
@@ -504,6 +505,23 @@ protected static int readColumn(String args, int startIndex, int len, String nam
504505
}
505506
}
506507
}
508+
} else if (args.startsWith(KEYWORD_JSON, i)) {
509+
int index = args.indexOf('(', i + KEYWORD_JSON.length());
510+
if (index > i) {
511+
i = ClickHouseUtils.skipBrackets(args, index, len, '(');
512+
String originalTypeName = args.substring(startIndex, i);
513+
List<ClickHouseColumn> nestedColumns = new ArrayList<>();
514+
515+
List<String> parameters = new ArrayList<>();
516+
parseJSONColumn(args.substring(index + 1, i - 1), nestedColumns, parameters);
517+
nestedColumns.sort(Comparator.comparing(o -> o.getDataType().name()));
518+
column = new ClickHouseColumn(ClickHouseDataType.JSON, name, originalTypeName, nullable, lowCardinality,
519+
parameters, nestedColumns);
520+
column.jsonPredefinedPaths = nestedColumns.stream().collect(Collectors.toMap(ClickHouseColumn::getColumnName,
521+
c -> c));
522+
fixedLength = false;
523+
estimatedLength++;
524+
}
507525
}
508526

509527
if (column == null) {
@@ -658,6 +676,54 @@ public static List<ClickHouseColumn> parse(String args) {
658676
return Collections.unmodifiableList(c);
659677
}
660678

679+
public static final String JSON_MAX_PATHS_PARAM = "max_dynamic_paths";
680+
public static final String JSON_MAX_DYN_TYPES_PARAM = "max_dynamic_types";
681+
public static final String JSON_SKIP_MARKER = "SKIP";
682+
683+
public static void parseJSONColumn(String args, List<ClickHouseColumn> nestedColumns, List<String> parameters) {
684+
if (args == null || args.isEmpty()) {
685+
return;
686+
}
687+
688+
String name = null;
689+
ClickHouseColumn column = null;
690+
StringBuilder builder = new StringBuilder();
691+
int i =0;
692+
int len = args.length();
693+
while (i < len) {
694+
char ch = args.charAt(i);
695+
if (Character.isWhitespace(ch)) {
696+
i++;
697+
continue;
698+
}
699+
700+
if (name == null) { // column name
701+
i = ClickHouseUtils.readNameOrQuotedString(args, i, len, builder) - 1;
702+
name = builder.toString();
703+
if (name.startsWith(JSON_SKIP_MARKER)) {
704+
name = null; // skip parameters
705+
i = ClickHouseUtils.skipContentsUntil(args, i, len, ',') - 1;
706+
} else if ( name.startsWith(JSON_MAX_PATHS_PARAM) || name.startsWith(JSON_MAX_DYN_TYPES_PARAM)) {
707+
parameters.add(name);
708+
name = null;
709+
i = ClickHouseUtils.skipContentsUntil(args, i, len, ',') - 1;
710+
}
711+
builder.setLength(0);
712+
} else if (column == null) { // now type
713+
LinkedList<ClickHouseColumn> colList = new LinkedList<>();
714+
i = readColumn(args, i, len, name, colList) - 1;
715+
column = colList.getFirst();
716+
nestedColumns.add(column);
717+
} else { // prepare for next column
718+
i = ClickHouseUtils.skipContentsUntil(args, i, len, ',') - 1;
719+
name = null;
720+
column = null;
721+
}
722+
723+
i++;
724+
}
725+
}
726+
661727
public ClickHouseColumn(ClickHouseDataType dataType, String columnName, String originalTypeName, boolean nullable,
662728
boolean lowCardinality, List<String> parameters, List<ClickHouseColumn> nestedColumns) {
663729
this(dataType, columnName, originalTypeName, nullable, lowCardinality, parameters, nestedColumns, ClickHouseEnum.EMPTY);
@@ -954,6 +1020,10 @@ public ClickHouseAggregateFunction getAggregateFunction() {
9541020
return aggFuncType;
9551021
}
9561022

1023+
public Map<String, ClickHouseColumn> getJsonPredefinedPaths() {
1024+
return jsonPredefinedPaths;
1025+
}
1026+
9571027
public ClickHouseArraySequence newArrayValue(ClickHouseDataConfig config) {
9581028
int level = arrayLevel;
9591029
ClickHouseArraySequence value;

‎clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseUtils.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,6 @@
5151
import java.util.function.Supplier;
5252
import java.util.function.UnaryOperator;
5353

54-
@Deprecated
5554
public final class ClickHouseUtils {
5655
private static final boolean IS_UNIX;
5756
private static final boolean IS_WINDOWS;

‎clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
package com.clickhouse.data;
22

33
import java.math.BigInteger;
4+
import java.util.Arrays;
45
import java.util.Collections;
56
import java.util.LinkedList;
67
import java.util.List;
8+
import java.util.Map;
79

810
import org.testng.Assert;
911
import org.testng.annotations.DataProvider;
@@ -441,4 +443,26 @@ public boolean isWidenUnsignedTypes() {
441443
}
442444
}
443445
}
446+
447+
@Test(groups = {"unit"}, dataProvider = "testJSONBinaryFormat_dp")
448+
public void testJSONBinaryFormat(String jsonDef, int params, List<String> predefinedPaths) throws Exception {
449+
ClickHouseColumn column = ClickHouseColumn.of("v", jsonDef);
450+
Assert.assertEquals(column.getNestedColumns().size(), predefinedPaths.size(), "predefined paths count mismatch");
451+
Assert.assertEquals(column.getParameters().size(), params, "parameters count mismatch");
452+
}
453+
454+
@DataProvider
455+
public Object[][] testJSONBinaryFormat_dp() {
456+
457+
return new Object[][] {
458+
{"JSON", 0, Collections.emptyList()},
459+
{"JSON()", 0, Collections.emptyList()},
460+
{"JSON(stat.name String, count Int32)", 0, Arrays.asList("stat.name", "count")},
461+
{"JSON(stat.name String, `comments` String)", 0, Arrays.asList("stat.name", "comments")},
462+
{"JSON(max_dynamic_paths=3, stat.name String, count Int8, SKIP alt_count)", 1, Arrays.asList("stat.name", "count")},
463+
{"JSON(max_dynamic_paths=3, stat.name String, SKIP REGEXP '^-.*')", 1, Arrays.asList("stat.name")},
464+
{"JSON(max_dynamic_paths=3,SKIP REGEXP '^-.*',SKIP ff, flags Array(Array(Array(Int8))), SKIP alt_count)", 1, Arrays.asList("flags")},
465+
{"JSON(max_dynamic_types=3,max_dynamic_paths=3, SKIP REGEXP '^-.*',SKIP ff, flags Array(Array(Array(Int8))), SKIP alt_count)", 2, Arrays.asList("flags")},
466+
};
467+
}
444468
}

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,7 @@ public <T> T readValue(ClickHouseColumn column, Class<?> typeHint) throws IOExce
226226
if (jsonAsString) {
227227
return (T) readString(input);
228228
} else {
229-
return (T) readJsonData(input);
229+
return (T) readJsonData(input, actualColumn);
230230
}
231231
// case Object: // deprecated https://clickhouse.com/docs/en/sql-reference/data-types/object-data-type
232232
case Array:
@@ -1192,16 +1192,20 @@ private ClickHouseColumn readDynamicData() throws IOException {
11921192

11931193
private static final ClickHouseColumn JSON_PLACEHOLDER_COL = ClickHouseColumn.parse("v Dynamic").get(0);
11941194

1195-
private Map<String, Object> readJsonData(InputStream input) throws IOException {
1195+
private Map<String, Object> readJsonData(InputStream input, ClickHouseColumn column) throws IOException {
11961196
int numOfPaths = readVarInt(input);
11971197
if (numOfPaths == 0) {
11981198
return Collections.emptyMap();
11991199
}
12001200

12011201
Map<String, Object> obj = new HashMap<>();
1202+
1203+
final Map<String, ClickHouseColumn> predefinedColumns = column.getJsonPredefinedPaths();
12021204
for (int i = 0; i < numOfPaths; i++) {
12031205
String path = readString(input);
1204-
Object value = readValue(JSON_PLACEHOLDER_COL);
1206+
ClickHouseColumn dataColumn = predefinedColumns == null? JSON_PLACEHOLDER_COL :
1207+
predefinedColumns.getOrDefault(path, JSON_PLACEHOLDER_COL);
1208+
Object value = readValue(dataColumn);
12051209
obj.put(path, value);
12061210
}
12071211
return obj;

‎client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,14 @@
77
import com.clickhouse.client.api.Client;
88
import com.clickhouse.client.api.DataTypeUtils;
99
import com.clickhouse.client.api.command.CommandSettings;
10+
import com.clickhouse.client.api.data_formats.ClickHouseBinaryFormatReader;
11+
import com.clickhouse.client.api.data_formats.internal.SerializerUtils;
1012
import com.clickhouse.client.api.enums.Protocol;
1113
import com.clickhouse.client.api.insert.InsertSettings;
1214
import com.clickhouse.client.api.metadata.TableSchema;
1315
import com.clickhouse.client.api.query.GenericRecord;
1416
import com.clickhouse.client.api.query.QueryResponse;
17+
import com.clickhouse.client.api.sql.SQLUtils;
1518
import com.clickhouse.data.ClickHouseDataType;
1619
import com.clickhouse.data.ClickHouseVersion;
1720
import lombok.AllArgsConstructor;
@@ -874,6 +877,45 @@ private void testVariantWith(String withWhat, String[] fields, Object[] values,
874877
}
875878
}
876879

880+
@Test(groups = {"integration"}, dataProvider = "testJSONBinaryFormat_dp")
881+
public void testJSONBinaryFormat(String jsonDef) throws Exception {
882+
if (isVersionMatch("(,24.8]")) {
883+
return;
884+
}
885+
886+
final String table = "test_json_binary_format";
887+
final String jsonCol = "value " + jsonDef;
888+
final String jsonValue = "{\"count\": 1000, \"stat\": {\"float\": 0.999, \"name\": \"temp\" }}";
889+
890+
client.execute("DROP TABLE IF EXISTS " + table).get().close();
891+
client.execute(tableDefinition(table, jsonCol),
892+
(CommandSettings) new CommandSettings()
893+
.serverSetting("enable_json_type", "1")
894+
.serverSetting("allow_experimental_json_type", "1")).get().close();
895+
client.execute("INSERT INTO " + table + " VALUES (" + SQLUtils.enquoteLiteral(jsonValue) + ")").get().close();
896+
897+
try (QueryResponse queryResponse = client.query("SELECT * FROM " + table + " LIMIT 1").get()) {
898+
ClickHouseBinaryFormatReader reader = client.newBinaryFormatReader(queryResponse);
899+
Map<String, Object> row = reader.next();
900+
Object value = row.get("value");
901+
Assert.assertNotNull(value);
902+
}
903+
}
904+
905+
@DataProvider
906+
public Object[][] testJSONBinaryFormat_dp() {
907+
908+
return new Object[][] {
909+
{"JSON"},
910+
{"JSON()"},
911+
{"JSON(stat.name String, count Int32)"},
912+
{"JSON(stat.name String, `comments` String)"},
913+
{"JSON(max_dynamic_paths=3, stat.name String, SKIP alt_count)"},
914+
{"JSON(max_dynamic_paths=3, stat.name String, SKIP REGEXP '^-.*')"},
915+
{"JSON(max_dynamic_paths=3,SKIP REGEXP '^-.*',SKIP ff, flags Array(Array(Array(Int8))), SKIP alt_count)"},
916+
};
917+
}
918+
877919
public static String tableDefinition(String table, String... columns) {
878920
StringBuilder sb = new StringBuilder();
879921
sb.append("CREATE TABLE " + table + " ( ");

‎jdbc-v2/src/test/java/com/clickhouse/jdbc/DataTypeTests.java

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import com.clickhouse.client.api.ClientConfigProperties;
44
import com.clickhouse.client.api.DataTypeUtils;
55
import com.clickhouse.client.api.internal.ServerSettings;
6+
import com.clickhouse.client.api.sql.SQLUtils;
67
import com.clickhouse.data.ClickHouseVersion;
78
import com.clickhouse.data.Tuple;
89
import org.slf4j.Logger;
@@ -47,6 +48,7 @@
4748

4849
import static org.testng.Assert.assertEquals;
4950
import static org.testng.Assert.assertFalse;
51+
import static org.testng.Assert.assertNotEquals;
5052
import static org.testng.Assert.assertNull;
5153
import static org.testng.Assert.assertThrows;
5254
import static org.testng.Assert.assertTrue;
@@ -1372,7 +1374,32 @@ public void testJSONWritingAsString() throws SQLException {
13721374
}
13731375
}
13741376

1375-
@Test(groups = { "integration" }, enabled = false)
1377+
@Test(groups = { "integration" })
1378+
public void testReadingJSONBinary() throws SQLException {
1379+
if (ClickHouseVersion.of(getServerVersion()).check("(,24.8]")) {
1380+
return; // JSON was introduced in 24.10
1381+
}
1382+
1383+
Properties properties = new Properties();
1384+
properties.put(ClientConfigProperties.serverSetting("allow_experimental_json_type"), "1");
1385+
try (Connection conn = getJdbcConnection(properties);
1386+
Statement stmt = conn.createStatement()) {
1387+
1388+
final String json = "{\"count\": 1000, \"event\": { \"name\": \"start\", \"value\": 0.10} }";
1389+
String sql = String.format("SELECT %1$s::JSON(), %1$s::JSON(count Int16)", SQLUtils.enquoteLiteral(json));
1390+
try (ResultSet rs = stmt.executeQuery(sql)) {
1391+
rs.next();
1392+
1393+
Map<String, Object> val1 = (Map<String, Object>) rs.getObject(1);
1394+
assertEquals(val1.get("count"), 1000L);
1395+
Map<String, Object> val2 = (Map<String, Object>) rs.getObject(2);
1396+
assertEquals(val2.get("count"), (short)1000);
1397+
}
1398+
}
1399+
}
1400+
1401+
1402+
@Test(groups = { "integration" }, enabled = false)
13761403
public void testGeometricTypesSimpleStatement() throws SQLException {
13771404
// TODO: add LineString and MultiLineString support
13781405
runQuery("CREATE TABLE test_geometric (order Int8, "

0 commit comments

Comments
 (0)