Skip to content

Commit 865b915

Browse files
miomiocatmiomiocat
authored
[Enhancement] improve error message handling of JNI connector and support char type of paimon connector (StarRocks#28044)
Fixes StarRocks#27992 - fail fast for unsupported types of paimon table - output error massage of JNI to client terminal - support char type of paimon connector - some minor refactor Signed-off-by: miomiocat <[email protected]>
1 parent 282bb77 commit 865b915

File tree

8 files changed

+141
-59
lines changed

8 files changed

+141
-59
lines changed

‎be/src/connector/hive_connector.cpp

+4
Original file line numberDiff line numberDiff line change
@@ -523,6 +523,10 @@ Status HiveDataSource::_init_scanner(RuntimeState* state) {
523523
RETURN_IF_ERROR(scanner->init(state, scanner_params));
524524
Status st = scanner->open(state);
525525
if (!st.ok()) {
526+
if (scanner->is_jni_scanner()) {
527+
return st;
528+
}
529+
526530
auto msg = fmt::format("file = {}", native_file_path);
527531

528532
// After catching the AWS 404 file not found error and returning it to the FE,

‎be/src/exec/hdfs_scanner.h

+1
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,7 @@ class HdfsScanner {
310310
virtual Status do_get_next(RuntimeState* runtime_state, ChunkPtr* chunk) = 0;
311311
virtual Status do_init(RuntimeState* runtime_state, const HdfsScannerParams& scanner_params) = 0;
312312
virtual void do_update_counter(HdfsScanProfile* profile);
313+
virtual bool is_jni_scanner() { return false; }
313314

314315
void enter_pending_queue();
315316
// how long it stays inside pending queue.

‎be/src/exec/jni_scanner.cpp

+23-15
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,12 @@
2525
namespace starrocks {
2626

2727
Status JniScanner::_check_jni_exception(JNIEnv* _jni_env, const std::string& message) {
28-
if (_jni_env->ExceptionCheck()) {
28+
if (jthrowable thr = _jni_env->ExceptionOccurred(); thr) {
29+
std::string jni_error_message = JVMFunctionHelper::getInstance().dumpExceptionString(thr);
2930
_jni_env->ExceptionDescribe();
3031
_jni_env->ExceptionClear();
31-
return Status::InternalError(message);
32+
_jni_env->DeleteLocalRef(thr);
33+
return Status::InternalError(message + " java exception details: " + jni_error_message);
3234
}
3335
return Status::OK();
3436
}
@@ -139,10 +141,11 @@ Status JniScanner::_get_next_chunk(JNIEnv* _jni_env, long* chunk_meta) {
139141
return Status::OK();
140142
}
141143

142-
template <LogicalType type, typename CppType>
144+
template <LogicalType type>
143145
Status JniScanner::_append_primitive_data(const FillColumnArgs& args) {
144146
char* column_ptr = static_cast<char*>(next_chunk_meta_as_ptr());
145147
using ColumnType = typename starrocks::RunTimeColumnType<type>;
148+
using CppType = typename starrocks::RunTimeCppType<type>;
146149
auto* runtime_column = down_cast<ColumnType*>(args.column);
147150
runtime_column->resize_uninitialized(args.num_rows);
148151
memcpy(runtime_column->get_data().data(), column_ptr, args.num_rows * sizeof(CppType));
@@ -169,12 +172,13 @@ Status JniScanner::_append_string_data(const FillColumnArgs& args) {
169172
return Status::OK();
170173
}
171174

172-
template <LogicalType type, typename CppType>
175+
template <LogicalType type>
173176
Status JniScanner::_append_decimal_data(const FillColumnArgs& args) {
174177
int* offset_ptr = static_cast<int*>(next_chunk_meta_as_ptr());
175178
char* column_ptr = static_cast<char*>(next_chunk_meta_as_ptr());
176179

177180
using ColumnType = typename starrocks::RunTimeColumnType<type>;
181+
using CppType = typename starrocks::RunTimeCppType<type>;
178182
auto* runtime_column = down_cast<ColumnType*>(args.column);
179183
runtime_column->resize_uninitialized(args.num_rows);
180184
CppType* runtime_data = runtime_column->get_data().data();
@@ -365,37 +369,41 @@ Status JniScanner::_fill_column(FillColumnArgs* pargs) {
365369
pargs->column = data_column;
366370
pargs->nulls = null_data.data();
367371
} else {
368-
// otherwise we skil this chunk meta, because in Java side
369-
// we assume every column starswith `null_column`.
372+
// otherwise we skip this chunk meta, because in Java side
373+
// we assume every column starts with `null_column`.
370374
}
371375

372376
LogicalType column_type = args.slot_type.type;
373377
if (column_type == LogicalType::TYPE_BOOLEAN) {
374-
RETURN_IF_ERROR((_append_primitive_data<TYPE_BOOLEAN, uint8_t>(args)));
378+
RETURN_IF_ERROR((_append_primitive_data<TYPE_BOOLEAN>(args)));
379+
} else if (column_type == LogicalType::TYPE_TINYINT) {
380+
RETURN_IF_ERROR((_append_primitive_data<TYPE_TINYINT>(args)));
375381
} else if (column_type == LogicalType::TYPE_SMALLINT) {
376-
RETURN_IF_ERROR((_append_primitive_data<TYPE_SMALLINT, int16_t>(args)));
382+
RETURN_IF_ERROR((_append_primitive_data<TYPE_SMALLINT>(args)));
377383
} else if (column_type == LogicalType::TYPE_INT) {
378-
RETURN_IF_ERROR((_append_primitive_data<TYPE_INT, int32_t>(args)));
384+
RETURN_IF_ERROR((_append_primitive_data<TYPE_INT>(args)));
379385
} else if (column_type == LogicalType::TYPE_FLOAT) {
380-
RETURN_IF_ERROR((_append_primitive_data<TYPE_FLOAT, float>(args)));
386+
RETURN_IF_ERROR((_append_primitive_data<TYPE_FLOAT>(args)));
381387
} else if (column_type == LogicalType::TYPE_BIGINT) {
382-
RETURN_IF_ERROR((_append_primitive_data<TYPE_BIGINT, int64_t>(args)));
388+
RETURN_IF_ERROR((_append_primitive_data<TYPE_BIGINT>(args)));
383389
} else if (column_type == LogicalType::TYPE_DOUBLE) {
384-
RETURN_IF_ERROR((_append_primitive_data<TYPE_DOUBLE, double>(args)));
390+
RETURN_IF_ERROR((_append_primitive_data<TYPE_DOUBLE>(args)));
385391
} else if (column_type == LogicalType::TYPE_VARCHAR) {
386392
RETURN_IF_ERROR((_append_string_data<TYPE_VARCHAR>(args)));
387393
} else if (column_type == LogicalType::TYPE_CHAR) {
388394
RETURN_IF_ERROR((_append_string_data<TYPE_CHAR>(args)));
395+
} else if (column_type == LogicalType::TYPE_VARBINARY) {
396+
RETURN_IF_ERROR((_append_string_data<TYPE_VARBINARY>(args)));
389397
} else if (column_type == LogicalType::TYPE_DATE) {
390398
RETURN_IF_ERROR((_append_date_data(args)));
391399
} else if (column_type == LogicalType::TYPE_DATETIME) {
392400
RETURN_IF_ERROR((_append_datetime_data(args)));
393401
} else if (column_type == LogicalType::TYPE_DECIMAL32) {
394-
RETURN_IF_ERROR((_append_decimal_data<TYPE_DECIMAL32, int32_t>(args)));
402+
RETURN_IF_ERROR((_append_decimal_data<TYPE_DECIMAL32>(args)));
395403
} else if (column_type == LogicalType::TYPE_DECIMAL64) {
396-
RETURN_IF_ERROR((_append_decimal_data<TYPE_DECIMAL64, int64_t>(args)));
404+
RETURN_IF_ERROR((_append_decimal_data<TYPE_DECIMAL64>(args)));
397405
} else if (column_type == LogicalType::TYPE_DECIMAL128) {
398-
RETURN_IF_ERROR((_append_decimal_data<TYPE_DECIMAL128, int128_t>(args)));
406+
RETURN_IF_ERROR((_append_decimal_data<TYPE_DECIMAL128>(args)));
399407
} else if (column_type == LogicalType::TYPE_ARRAY) {
400408
RETURN_IF_ERROR((_append_array_data(args)));
401409
} else if (column_type == LogicalType::TYPE_MAP) {

‎be/src/exec/jni_scanner.h

+3-2
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ class JniScanner : public HdfsScanner {
3535
void do_close(RuntimeState* runtime_state) noexcept override;
3636
Status do_get_next(RuntimeState* runtime_state, ChunkPtr* chunk) override;
3737
Status do_init(RuntimeState* runtime_state, const HdfsScannerParams& scanner_params) override;
38+
bool is_jni_scanner() override { return true; }
3839

3940
private:
4041
struct FillColumnArgs {
@@ -57,10 +58,10 @@ class JniScanner : public HdfsScanner {
5758

5859
Status _get_next_chunk(JNIEnv* _jni_env, long* chunk_meta);
5960

60-
template <LogicalType type, typename CppType>
61+
template <LogicalType type>
6162
Status _append_primitive_data(const FillColumnArgs& args);
6263

63-
template <LogicalType type, typename CppType>
64+
template <LogicalType type>
6465
Status _append_decimal_data(const FillColumnArgs& args);
6566

6667
template <LogicalType type>

‎fe/fe-core/src/main/java/com/starrocks/connector/ColumnTypeConverter.java

+35-24
Original file line numberDiff line numberDiff line change
@@ -32,18 +32,17 @@
3232
import org.apache.avro.Schema;
3333
import org.apache.iceberg.types.Types;
3434
import org.apache.paimon.types.BigIntType;
35+
import org.apache.paimon.types.BinaryType;
3536
import org.apache.paimon.types.BooleanType;
36-
import org.apache.paimon.types.DataField;
37+
import org.apache.paimon.types.CharType;
3738
import org.apache.paimon.types.DataTypeDefaultVisitor;
3839
import org.apache.paimon.types.DateType;
3940
import org.apache.paimon.types.DecimalType;
4041
import org.apache.paimon.types.DoubleType;
4142
import org.apache.paimon.types.FloatType;
4243
import org.apache.paimon.types.IntType;
43-
import org.apache.paimon.types.RowType;
4444
import org.apache.paimon.types.SmallIntType;
4545
import org.apache.paimon.types.TimestampType;
46-
import org.apache.paimon.types.TinyIntType;
4746
import org.apache.paimon.types.VarCharType;
4847

4948
import java.util.ArrayList;
@@ -394,6 +393,14 @@ private static class PaimonToHiveTypeVisitor extends DataTypeDefaultVisitor<Type
394393

395394
private static final PaimonToHiveTypeVisitor INSTANCE = new PaimonToHiveTypeVisitor();
396395

396+
public Type visit(BinaryType binaryType) {
397+
return ScalarType.createType(PrimitiveType.VARBINARY);
398+
}
399+
400+
public Type visit(CharType charType) {
401+
return ScalarType.createCharType(charType.getLength());
402+
}
403+
397404
public Type visit(VarCharType varCharType) {
398405
return ScalarType.createDefaultExternalTableString();
399406
}
@@ -406,9 +413,10 @@ public Type visit(DecimalType decimalType) {
406413
return ScalarType.createUnifiedDecimalType(decimalType.getPrecision(), decimalType.getScale());
407414
}
408415

409-
public Type visit(TinyIntType tinyIntType) {
410-
return ScalarType.createType(PrimitiveType.TINYINT);
411-
}
416+
// TODO: uncomment this and unit test case when this type is supported in paimon connector
417+
//public Type visit(TinyIntType tinyIntType) {
418+
// return ScalarType.createType(PrimitiveType.TINYINT);
419+
//}
412420

413421
public Type visit(SmallIntType smallIntType) {
414422
return ScalarType.createType(PrimitiveType.SMALLINT);
@@ -438,24 +446,27 @@ public Type visit(TimestampType timestampType) {
438446
return ScalarType.createType(PrimitiveType.DATETIME);
439447
}
440448

441-
public Type visit(org.apache.paimon.types.ArrayType arrayType) {
442-
return new ArrayType(fromPaimonType(arrayType.getElementType()));
443-
}
444-
445-
public Type visit(org.apache.paimon.types.MapType mapType) {
446-
return new MapType(fromPaimonType(mapType.getKeyType()), fromPaimonType(mapType.getValueType()));
447-
}
448-
449-
public Type visit(RowType rowType) {
450-
List<DataField> fields = rowType.getFields();
451-
ArrayList<StructField> structFields = new ArrayList<>(fields.size());
452-
for (DataField field : fields) {
453-
String fieldName = field.name();
454-
Type fieldType = fromPaimonType(field.type());
455-
structFields.add(new StructField(fieldName, fieldType));
456-
}
457-
return new StructType(structFields);
458-
}
449+
// TODO: uncomment this and unit test case when this type is supported in paimon connector
450+
//public Type visit(org.apache.paimon.types.ArrayType arrayType) {
451+
// return new ArrayType(fromPaimonType(arrayType.getElementType()));
452+
//}
453+
454+
// TODO: uncomment this and unit test case when this type is supported in paimon connector
455+
//public Type visit(org.apache.paimon.types.MapType mapType) {
456+
// return new MapType(fromPaimonType(mapType.getKeyType()), fromPaimonType(mapType.getValueType()));
457+
//}
458+
459+
// TODO: uncomment this and unit test case when this type is supported in paimon connector
460+
//public Type visit(RowType rowType) {
461+
// List<DataField> fields = rowType.getFields();
462+
// ArrayList<StructField> structFields = new ArrayList<>(fields.size());
463+
// for (DataField field : fields) {
464+
// String fieldName = field.name();
465+
// Type fieldType = fromPaimonType(field.type());
466+
// structFields.add(new StructField(fieldName, fieldType));
467+
// }
468+
// return new StructType(structFields);
469+
//}
459470

460471
@Override
461472
protected Type defaultMethod(org.apache.paimon.types.DataType dataType) {

‎fe/fe-core/src/test/java/com/starrocks/connector/paimon/PaimonColumnConverterTest.java

+63-16
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,9 @@
2020
import com.starrocks.connector.ColumnTypeConverter;
2121
import org.apache.paimon.types.ArrayType;
2222
import org.apache.paimon.types.BigIntType;
23+
import org.apache.paimon.types.BinaryType;
2324
import org.apache.paimon.types.BooleanType;
25+
import org.apache.paimon.types.CharType;
2426
import org.apache.paimon.types.DataField;
2527
import org.apache.paimon.types.DateType;
2628
import org.apache.paimon.types.DecimalType;
@@ -34,6 +36,7 @@
3436
import org.apache.paimon.types.TinyIntType;
3537
import org.apache.paimon.types.VarCharType;
3638
import org.junit.Assert;
39+
import org.junit.Ignore;
3740
import org.junit.Test;
3841

3942
import java.util.Arrays;
@@ -42,7 +45,22 @@
4245
public class PaimonColumnConverterTest {
4346

4447
@Test
45-
public void testConvertString() {
48+
public void testConvertBinary() {
49+
BinaryType paimonType = new BinaryType();
50+
Type result = ColumnTypeConverter.fromPaimonType(paimonType);
51+
Assert.assertEquals(result, Type.VARBINARY);
52+
}
53+
54+
@Test
55+
public void testConvertChar() {
56+
CharType paimonType = new CharType(10);
57+
Type result = ColumnTypeConverter.fromPaimonType(paimonType);
58+
Type srType = ScalarType.createCharType(10);
59+
Assert.assertEquals(result, srType);
60+
}
61+
62+
@Test
63+
public void testConvertVarchar() {
4664
VarCharType paimonType = new VarCharType();
4765
Type result = ColumnTypeConverter.fromPaimonType(paimonType);
4866
Type srType = ScalarType.createDefaultExternalTableString();
@@ -56,13 +74,52 @@ public void testConvertBool() {
5674
Assert.assertEquals(result, Type.BOOLEAN);
5775
}
5876

77+
@Test
78+
public void testConvertDecimal() {
79+
int precision = 9;
80+
int scale = 5;
81+
DecimalType paimonType = new DecimalType(precision, scale);
82+
Type result = ColumnTypeConverter.fromPaimonType(paimonType);
83+
Type srType = ScalarType.createUnifiedDecimalType(precision, scale);
84+
Assert.assertEquals(result, srType);
85+
}
86+
87+
88+
@Ignore
89+
public void testConvertTinyInt() {
90+
TinyIntType paimonType = new TinyIntType();
91+
Type result = ColumnTypeConverter.fromPaimonType(paimonType);
92+
Assert.assertEquals(result, Type.TINYINT);
93+
}
94+
95+
@Test
96+
public void testConvertSmallint() {
97+
SmallIntType paimonType = new SmallIntType();
98+
Type result = ColumnTypeConverter.fromPaimonType(paimonType);
99+
Assert.assertEquals(result, Type.SMALLINT);
100+
}
101+
59102
@Test
60103
public void testConvertInt() {
61104
IntType paimonType = new IntType();
62105
Type result = ColumnTypeConverter.fromPaimonType(paimonType);
63106
Assert.assertEquals(result, Type.INT);
64107
}
65108

109+
@Test
110+
public void testConvertBigint() {
111+
BigIntType paimonType = new BigIntType();
112+
Type result = ColumnTypeConverter.fromPaimonType(paimonType);
113+
Assert.assertEquals(result, Type.BIGINT);
114+
}
115+
116+
@Test
117+
public void testConvertFlout() {
118+
FloatType paimonType = new FloatType();
119+
Type result = ColumnTypeConverter.fromPaimonType(paimonType);
120+
Assert.assertEquals(result, Type.FLOAT);
121+
}
122+
66123
@Test
67124
public void testConvertDouble() {
68125
DoubleType paimonType = new DoubleType();
@@ -84,17 +141,7 @@ public void testConvertDatetime() {
84141
Assert.assertEquals(result, Type.DATETIME);
85142
}
86143

87-
@Test
88-
public void testConvertDecimal() {
89-
int precision = 9;
90-
int scale = 5;
91-
DecimalType paimonType = new DecimalType(precision, scale);
92-
Type result = ColumnTypeConverter.fromPaimonType(paimonType);
93-
Type srType = ScalarType.createUnifiedDecimalType(precision, scale);
94-
Assert.assertEquals(result, srType);
95-
}
96-
97-
@Test
144+
@Ignore
98145
public void testConvertArray() {
99146
ArrayType paimonType = new ArrayType(new SmallIntType());
100147
Type result = ColumnTypeConverter.fromPaimonType(paimonType);
@@ -103,7 +150,7 @@ public void testConvertArray() {
103150
Assert.assertEquals(Type.SMALLINT, srType.getItemType());
104151
}
105152

106-
@Test
153+
@Ignore
107154
public void testConvertMap() {
108155
MapType paimonType = new MapType(new VarCharType(20), new TimestampType());
109156
Type result = ColumnTypeConverter.fromPaimonType(paimonType);
@@ -113,19 +160,19 @@ public void testConvertMap() {
113160
Assert.assertEquals(Type.DATETIME, srType.getValueType());
114161
}
115162

116-
@Test
163+
@Ignore
117164
public void testConvertStruct() {
118165
List<DataField> fields =
119166
Arrays.asList(
120-
new DataField(0, "f0", new TinyIntType()),
167+
new DataField(0, "f0", new BinaryType()),
121168
new DataField(1, "f1", new BigIntType()),
122169
new DataField(2, "f2", new FloatType()));
123170
RowType paimonType = new RowType(fields);
124171
Type result = ColumnTypeConverter.fromPaimonType(paimonType);
125172
Assert.assertTrue(result instanceof StructType);
126173
StructType srType = (StructType) result;
127174
Assert.assertEquals(3, srType.getFields().size());
128-
Assert.assertEquals(Type.TINYINT, srType.getField("f0").getType());
175+
Assert.assertEquals(Type.VARBINARY, srType.getField("f0").getType());
129176
Assert.assertEquals(Type.BIGINT, srType.getField("f1").getType());
130177
Assert.assertEquals(Type.FLOAT, srType.getField("f2").getType());
131178
}

‎java-extensions/jni-connector/src/main/java/com/starrocks/jni/connector/ColumnType.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,7 @@ private void parse(StringScanner scanner) {
198198
}
199199

200200
if (typeValue == null) {
201-
throw new RuntimeException("Unknown type: " + t);
201+
throw new RuntimeException("Unsupported type: " + t);
202202
}
203203
}
204204

0 commit comments

Comments
 (0)