Skip to content

Commit 8ed71ae

Browse files
kokila-19kokila-19
committed
HIVE-29570: Fix MERGE rewrite parse failure by correctly quoting qualified column names when they are function names(like date)
1 parent 4619ff8 commit 8ed71ae

4 files changed

Lines changed: 307 additions & 4 deletions

File tree

‎ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -354,8 +354,20 @@ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx, Obje
354354
throws SemanticException {
355355
UnparseTranslator unparseTranslator = ((QuotedIdExpressionContext)procCtx).getUnparseTranslator();
356356
ASTNode identifier = (ASTNode) nd;
357-
String id = identifier.getText();
358-
if (FunctionRegistry.getFunctionInfo(id) != null){
357+
/*
358+
* Quote identifiers during unparse.
359+
*
360+
* Only skip quoting for function names.
361+
* Always quote column names, even if they match function names.
362+
* For example, use `alias`.`date` instead of `alias`.date.
363+
*/
364+
ASTNode parent = (ASTNode) identifier.getParent();
365+
if (parent != null
366+
&& (parent.getType() == HiveParser.TOK_FUNCTION
367+
|| parent.getType() == HiveParser.TOK_FUNCTIONDI
368+
|| parent.getType() == HiveParser.TOK_FUNCTIONSTAR)
369+
&& parent.getChildCount() > 0
370+
&& parent.getChild(0) == identifier) {
359371
return null;
360372
}
361373

‎ql/src/test/queries/clientpositive/sqlmerge.q‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,15 @@ explain merge into acidTbl_n0 as t using nonAcidOrcTbl_n0 s ON t.a = s.a
2222
WHEN MATCHED AND s.a > 8 THEN DELETE
2323
WHEN MATCHED THEN UPDATE SET b = 7
2424
WHEN NOT MATCHED THEN INSERT VALUES(s.a, s.b);
25+
26+
-- MERGE rewrite must preserve quoting for qualified identifiers like s.`date` when column name is function keyword
27+
drop table if exists src_table;
28+
drop table if exists tgt_table;
29+
create table src_table(a int, `date` int) clustered by (a) into 2 buckets stored as orc
30+
TBLPROPERTIES ('transactional'='true');
31+
create table tgt_table(a int, `date` int) clustered by (a) into 2 buckets stored as orc
32+
TBLPROPERTIES ('transactional'='true');
33+
34+
explain merge into tgt_table t using src_table s ON t.a = s.a
35+
WHEN MATCHED THEN UPDATE SET `date` = s.`date`
36+
WHEN NOT MATCHED THEN INSERT VALUES(s.`a`, s.`date`);

‎ql/src/test/results/clientpositive/llap/sqlmerge.q.out‎

Lines changed: 279 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -819,3 +819,282 @@ STAGE PLANS:
819819
Stats Work
820820
Basic Stats Work:
821821

822+
PREHOOK: query: drop table if exists src_table
823+
PREHOOK: type: DROPTABLE
824+
PREHOOK: Output: database:default
825+
POSTHOOK: query: drop table if exists src_table
826+
POSTHOOK: type: DROPTABLE
827+
POSTHOOK: Output: database:default
828+
PREHOOK: query: drop table if exists tgt_table
829+
PREHOOK: type: DROPTABLE
830+
PREHOOK: Output: database:default
831+
POSTHOOK: query: drop table if exists tgt_table
832+
POSTHOOK: type: DROPTABLE
833+
POSTHOOK: Output: database:default
834+
PREHOOK: query: create table src_table(a int, `date` int) clustered by (a) into 2 buckets stored as orc
835+
TBLPROPERTIES ('transactional'='true')
836+
PREHOOK: type: CREATETABLE
837+
PREHOOK: Output: database:default
838+
PREHOOK: Output: default@src_table
839+
POSTHOOK: query: create table src_table(a int, `date` int) clustered by (a) into 2 buckets stored as orc
840+
TBLPROPERTIES ('transactional'='true')
841+
POSTHOOK: type: CREATETABLE
842+
POSTHOOK: Output: database:default
843+
POSTHOOK: Output: default@src_table
844+
PREHOOK: query: create table tgt_table(a int, `date` int) clustered by (a) into 2 buckets stored as orc
845+
TBLPROPERTIES ('transactional'='true')
846+
PREHOOK: type: CREATETABLE
847+
PREHOOK: Output: database:default
848+
PREHOOK: Output: default@tgt_table
849+
POSTHOOK: query: create table tgt_table(a int, `date` int) clustered by (a) into 2 buckets stored as orc
850+
TBLPROPERTIES ('transactional'='true')
851+
POSTHOOK: type: CREATETABLE
852+
POSTHOOK: Output: database:default
853+
POSTHOOK: Output: default@tgt_table
854+
PREHOOK: query: explain merge into tgt_table t using src_table s ON t.a = s.a
855+
WHEN MATCHED THEN UPDATE SET `date` = s.`date`
856+
WHEN NOT MATCHED THEN INSERT VALUES(s.`a`, s.`date`)
857+
PREHOOK: type: QUERY
858+
PREHOOK: Input: default@src_table
859+
PREHOOK: Input: default@tgt_table
860+
PREHOOK: Output: default@merge_tmp_table
861+
PREHOOK: Output: default@tgt_table
862+
PREHOOK: Output: default@tgt_table
863+
POSTHOOK: query: explain merge into tgt_table t using src_table s ON t.a = s.a
864+
WHEN MATCHED THEN UPDATE SET `date` = s.`date`
865+
WHEN NOT MATCHED THEN INSERT VALUES(s.`a`, s.`date`)
866+
POSTHOOK: type: QUERY
867+
POSTHOOK: Input: default@src_table
868+
POSTHOOK: Input: default@tgt_table
869+
POSTHOOK: Output: default@merge_tmp_table
870+
POSTHOOK: Output: default@tgt_table
871+
POSTHOOK: Output: default@tgt_table
872+
STAGE DEPENDENCIES:
873+
Stage-3 is a root stage
874+
Stage-4 depends on stages: Stage-3
875+
Stage-0 depends on stages: Stage-4
876+
Stage-5 depends on stages: Stage-0
877+
Stage-1 depends on stages: Stage-4
878+
Stage-6 depends on stages: Stage-1
879+
Stage-2 depends on stages: Stage-4
880+
Stage-7 depends on stages: Stage-2
881+
882+
STAGE PLANS:
883+
Stage: Stage-3
884+
Tez
885+
#### A masked pattern was here ####
886+
Edges:
887+
Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 6 (SIMPLE_EDGE)
888+
Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
889+
Reducer 4 <- Reducer 2 (SIMPLE_EDGE)
890+
Reducer 5 <- Reducer 2 (SIMPLE_EDGE)
891+
#### A masked pattern was here ####
892+
Vertices:
893+
Map 1
894+
Map Operator Tree:
895+
TableScan
896+
alias: s
897+
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
898+
Select Operator
899+
expressions: a (type: int), date (type: int)
900+
outputColumnNames: _col0, _col1
901+
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
902+
Reduce Output Operator
903+
key expressions: _col0 (type: int)
904+
null sort order: z
905+
sort order: +
906+
Map-reduce partition columns: _col0 (type: int)
907+
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
908+
value expressions: _col1 (type: int)
909+
Execution mode: vectorized, llap
910+
LLAP IO: may be used (ACID table)
911+
Map 6
912+
Map Operator Tree:
913+
TableScan
914+
alias: tgt_table
915+
filterExpr: a is not null (type: boolean)
916+
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
917+
Filter Operator
918+
predicate: a is not null (type: boolean)
919+
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
920+
Select Operator
921+
expressions: ROW__ID (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), a (type: int)
922+
outputColumnNames: _col0, _col1
923+
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
924+
Reduce Output Operator
925+
key expressions: _col1 (type: int)
926+
null sort order: z
927+
sort order: +
928+
Map-reduce partition columns: _col1 (type: int)
929+
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: NONE
930+
value expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
931+
Execution mode: vectorized, llap
932+
LLAP IO: may be used (ACID table)
933+
Reducer 2
934+
Execution mode: llap
935+
Reduce Operator Tree:
936+
Merge Join Operator
937+
condition map:
938+
Left Outer Join 0 to 1
939+
keys:
940+
0 _col0 (type: int)
941+
1 _col1 (type: int)
942+
outputColumnNames: _col0, _col1, _col2, _col3
943+
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
944+
Select Operator
945+
expressions: _col2 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), _col1 (type: int), _col3 (type: int), _col0 (type: int)
946+
outputColumnNames: _col0, _col1, _col2, _col3
947+
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
948+
Filter Operator
949+
predicate: _col2 is null (type: boolean)
950+
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
951+
Select Operator
952+
expressions: _col3 (type: int), _col1 (type: int)
953+
outputColumnNames: _col0, _col1
954+
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
955+
Reduce Output Operator
956+
key expressions: _col0 (type: int)
957+
null sort order: a
958+
sort order: +
959+
Map-reduce partition columns: _col0 (type: int)
960+
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
961+
value expressions: _col1 (type: int)
962+
Filter Operator
963+
predicate: (_col2 = _col3) (type: boolean)
964+
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
965+
Select Operator
966+
expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), _col2 (type: int), _col1 (type: int)
967+
outputColumnNames: _col0, _col1, _col2
968+
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
969+
Reduce Output Operator
970+
key expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
971+
null sort order: z
972+
sort order: +
973+
Map-reduce partition columns: UDFToInteger(_col0) (type: int)
974+
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
975+
value expressions: _col1 (type: int), _col2 (type: int)
976+
Filter Operator
977+
predicate: (_col2 = _col3) (type: boolean)
978+
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
979+
Select Operator
980+
expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
981+
outputColumnNames: _col0
982+
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
983+
Group By Operator
984+
aggregations: count()
985+
keys: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
986+
minReductionHashAggr: 0.99
987+
mode: hash
988+
outputColumnNames: _col0, _col1
989+
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
990+
Reduce Output Operator
991+
key expressions: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
992+
null sort order: z
993+
sort order: +
994+
Map-reduce partition columns: _col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
995+
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
996+
value expressions: _col1 (type: bigint)
997+
Reducer 3
998+
Execution mode: vectorized, llap
999+
Reduce Operator Tree:
1000+
Select Operator
1001+
expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: int)
1002+
outputColumnNames: _col0, _col1
1003+
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
1004+
File Output Operator
1005+
compressed: false
1006+
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
1007+
table:
1008+
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
1009+
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
1010+
serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
1011+
name: default.tgt_table
1012+
Write Type: INSERT
1013+
Reducer 4
1014+
Execution mode: vectorized, llap
1015+
Reduce Operator Tree:
1016+
Select Operator
1017+
expressions: KEY.reducesinkkey0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>), VALUE._col0 (type: int), VALUE._col1 (type: int)
1018+
outputColumnNames: _col0, _col1, _col2
1019+
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
1020+
File Output Operator
1021+
compressed: false
1022+
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
1023+
table:
1024+
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
1025+
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
1026+
serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
1027+
name: default.tgt_table
1028+
Write Type: UPDATE
1029+
Reducer 5
1030+
Execution mode: llap
1031+
Reduce Operator Tree:
1032+
Group By Operator
1033+
aggregations: count(VALUE._col0)
1034+
keys: KEY._col0 (type: struct<writeid:bigint,bucketid:int,rowid:bigint>)
1035+
mode: mergepartial
1036+
outputColumnNames: _col0, _col1
1037+
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
1038+
Filter Operator
1039+
predicate: (_col1 > 1L) (type: boolean)
1040+
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
1041+
Select Operator
1042+
expressions: cardinality_violation(_col0) (type: int)
1043+
outputColumnNames: _col0
1044+
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
1045+
File Output Operator
1046+
compressed: false
1047+
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
1048+
table:
1049+
input format: org.apache.hadoop.mapred.TextInputFormat
1050+
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
1051+
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
1052+
name: default.merge_tmp_table
1053+
1054+
Stage: Stage-4
1055+
Dependency Collection
1056+
1057+
Stage: Stage-0
1058+
Move Operator
1059+
tables:
1060+
replace: false
1061+
table:
1062+
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
1063+
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
1064+
serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
1065+
name: default.tgt_table
1066+
Write Type: INSERT
1067+
1068+
Stage: Stage-5
1069+
Stats Work
1070+
Basic Stats Work:
1071+
1072+
Stage: Stage-1
1073+
Move Operator
1074+
tables:
1075+
replace: false
1076+
table:
1077+
input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
1078+
output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
1079+
serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
1080+
name: default.tgt_table
1081+
Write Type: UPDATE
1082+
1083+
Stage: Stage-6
1084+
Stats Work
1085+
Basic Stats Work:
1086+
1087+
Stage: Stage-2
1088+
Move Operator
1089+
tables:
1090+
replace: false
1091+
table:
1092+
input format: org.apache.hadoop.mapred.TextInputFormat
1093+
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
1094+
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
1095+
name: default.merge_tmp_table
1096+
1097+
Stage: Stage-7
1098+
Stats Work
1099+
Basic Stats Work:
1100+

‎ql/src/test/results/clientpositive/tez/explainanalyze_3.q.out‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -375,10 +375,10 @@ Stage-0
375375
Number of rows:1
376376
TableScan [TS_0]
377377

378-
PREHOOK: query: DROP TEMPORARY MACRO SIGMOID
378+
PREHOOK: query: DROP TEMPORARY MACRO `SIGMOID`
379379
PREHOOK: type: DROPMACRO
380380
PREHOOK: Output: database:default
381-
POSTHOOK: query: DROP TEMPORARY MACRO SIGMOID
381+
POSTHOOK: query: DROP TEMPORARY MACRO `SIGMOID`
382382
POSTHOOK: type: DROPMACRO
383383
POSTHOOK: Output: database:default
384384
PREHOOK: query: explain analyze DROP TEMPORARY MACRO SIGMOID

0 commit comments

Comments
 (0)