Merge branch 'master' into HIVE-29059-show_create_table_truncates_after_tab

Author: cyanzheng2926

itests/src/test/resources/testconfiguration.properties

@@ -181,6 +181,7 @@ minillap.query.files=\
   skip_header_footer_aggr.q,\
   skip_header_footer_proj.q,\
   str_to_map.q,\
+  str_to_map_utf8.q,\
   table_nonprintable.q,\
   temp_table_add_part_with_loc.q,\
   temp_table_add_partition_with_location.q,\

ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java

@@ -1530,6 +1530,17 @@ public static boolean isRuntimeConstant(GenericUDF genericUDF) {
     return false;
   }
 
+  /**
+   * Returns whether the return type of a GenericUDF is nullable.
+   */
+  public static boolean isNullable(GenericUDF genericUDF) {
+    if (genericUDF == null) {
+      return true;
+    }
+    UDFType udfType = AnnotationUtils.getAnnotation(genericUDF.getClass(), UDFType.class);
+    return udfType == null || udfType.nullable();
+  }
+
   /**
    * Returns whether the expression, for a single query, returns the same result given
    * the same arguments/children. This includes deterministic functions as well as runtime

ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java

@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.hive.ql.exec.vector;
 
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -465,7 +466,7 @@ private void assignRowColumn(
           {
             if (object instanceof String) {
               String string = (String) object;
-              byte[] bytes = string.getBytes();
+              byte[] bytes = string.getBytes(StandardCharsets.UTF_8);
               ((BytesColumnVector) columnVector).setVal(
                   batchIndex, bytes, 0, bytes.length);
             } else {

ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java

@@ -520,17 +520,17 @@ private static void setVector(Object row,
       }
         break;
     case DECIMAL:
-      DecimalColumnVector dcv = (DecimalColumnVector) batch.cols[offset + colIndex];
+      ColumnVector dcv = batch.cols[offset + colIndex];
       if (writableCol != null) {
         dcv.isNull[rowIndex] = false;
         HiveDecimalWritable wobj = (HiveDecimalWritable) writableCol;
-        dcv.set(rowIndex, wobj);
+        ((IDecimalColumnVector) dcv).set(rowIndex, wobj);
       } else {
         setNullColIsNullValue(dcv, rowIndex);
       }
       break;
     default:
-      throw new HiveException("Vectorizaton is not supported for datatype:" +
+      throw new HiveException("Vectorization is not supported for datatype:" +
           poi.getPrimitiveCategory());
     }
   }

ql/src/java/org/apache/hadoop/hive/ql/parse/type/HiveFunctionHelper.java

@@ -135,12 +135,16 @@ public RelDataType getReturnType(FunctionInfo fi, List<RexNode> inputs)
       inputsOIs[i] = createObjectInspector(inputs.get(i));
     }
     // 2) Initialize and obtain return type
-    ObjectInspector oi = fi.getGenericUDF() != null ?
-        fi.getGenericUDF().initializeAndFoldConstants(inputsOIs) :
+    GenericUDF genericUDF = fi.getGenericUDF();
+    ObjectInspector oi = genericUDF != null ?
+        genericUDF.initializeAndFoldConstants(inputsOIs) :
         fi.getGenericUDTF().initialize(inputsOIs);
     // 3) Convert to RelDataType
     return TypeConverter.convert(
-        TypeInfoUtils.getTypeInfoFromObjectInspector(oi), rexBuilder.getTypeFactory());
+        TypeInfoUtils.getTypeInfoFromObjectInspector(oi),
+        FunctionRegistry.isNullable(genericUDF),
+        rexBuilder.getTypeFactory()
+    );
   }
 
   /**

ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotNull.java

@@ -25,6 +25,7 @@
 import org.apache.hadoop.hive.ql.exec.vector.expressions.IsNotNull;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.SelectColumnIsNotNull;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.UDFType;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
 import org.apache.hadoop.io.BooleanWritable;
@@ -37,6 +38,7 @@
     value = "_FUNC_ a - Returns true if a is not NULL and false otherwise")
 @VectorizedExpressions({IsNotNull.class, SelectColumnIsNotNull.class})
 @NDV(maxNdv = 2)
+@UDFType(nullable = false)
 public class GenericUDFOPNotNull extends GenericUDF {
   private final BooleanWritable result = new BooleanWritable();
 

ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNull.java

@@ -25,6 +25,7 @@
 import org.apache.hadoop.hive.ql.exec.vector.expressions.IsNull;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.SelectColumnIsNull;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.UDFType;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
 import org.apache.hadoop.io.BooleanWritable;
@@ -36,6 +37,7 @@
 @Description(name = "isnull", value = "_FUNC_ a - Returns true if a is NULL and false otherwise")
 @VectorizedExpressions({IsNull.class, SelectColumnIsNull.class})
 @NDV(maxNdv = 2)
+@UDFType(nullable = false)
 public class GenericUDFOPNull extends GenericUDF {
   private final BooleanWritable result = new BooleanWritable();
 

ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizedBatchUtil.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec.vector;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestVectorizedBatchUtil {
+
+  @Test
+  public void testSetDecimal64ColumnVector() throws Exception {
+    Decimal64ColumnVector dec64ColVector = new Decimal64ColumnVector(10, 2);
+    VectorizedRowBatch batch = new VectorizedRowBatch(1);
+    batch.cols[0] = dec64ColVector;
+
+    Object[] row = new Object[] {new HiveDecimalWritable("123.45")};
+
+    StructObjectInspector oi =
+        ObjectInspectorFactory.getStandardStructObjectInspector(
+            List.of("col1"),
+            List.of(PrimitiveObjectInspectorFactory.writableHiveDecimalObjectInspector));
+
+    DataOutputBuffer buffer = new DataOutputBuffer();
+
+    try {
+      VectorizedBatchUtil.addProjectedRowToBatchFrom(row, oi, 0, batch, buffer);
+
+      Assert.assertEquals(12345L, dec64ColVector.vector[0]);
+    } catch (ClassCastException e) {
+      Assert.fail(
+          "ClassCastException thrown when adding projected row with Decimal64ColumnVector: "
+              + e.getMessage());
+    }
+  }
+}

ql/src/test/queries/clientpositive/is_null_is_not_null_nested.q

@@ -0,0 +1,19 @@
+CREATE TABLE test (id INT, name STRING);
+
+-- always FALSE
+EXPLAIN CBO SELECT id, (name IS NULL) IS NULL FROM test;
+
+-- always FALSE
+EXPLAIN CBO SELECT id, (name IS NOT NULL) IS NULL FROM test;
+
+-- always TRUE
+EXPLAIN CBO SELECT id, (name IS NULL) IS NOT NULL FROM test;
+
+-- always TRUE
+EXPLAIN CBO SELECT id, (name IS NOT NULL) IS NOT NULL FROM test;
+
+-- always-true condition: no rows filtered
+EXPLAIN CBO SELECT id FROM test WHERE (name IS NULL) IS NOT NULL;
+
+-- always-false condition: all rows filtered
+EXPLAIN CBO SELECT id FROM test WHERE (name IS NULL) IS NULL;

ql/src/test/queries/clientpositive/str_to_map_utf8.q

@@ -0,0 +1,55 @@
+-- HIVE-28728: STR_TO_MAP() must preserve UTF-8 in vectorized execution when JVM default charset is not UTF-8.
+-- Tez container opts below force US-ASCII in Tez tasks
+-- Use driver-level mimic for testing with llap: -Dmaven.test.jvm.args="-Dfile.encoding=US-ASCII"
+
+SET tez.am.launch.cmd-opts=-Dfile.encoding=US-ASCII;
+SET hive.tez.java.opts=-Dfile.encoding=US-ASCII;
+SET hive.vectorized.execution.enabled=true;
+SET hive.fetch.task.conversion=none;
+
+DROP TABLE IF EXISTS hive28728_src;
+DROP TABLE IF EXISTS hive28728_result;
+DROP TABLE IF EXISTS hive28728_multi;
+DROP TABLE IF EXISTS hive28728_result_novec;
+
+CREATE TABLE hive28728_src (id string, name string, multi string) STORED AS ORC;
+INSERT INTO hive28728_src VALUES
+  ('100','hive', 'en:1'),
+  ('200','spark', null),
+  ('300','oozie', 'a:1,b:2'),
+  ('400','airflow', 'ascii:值'),
+  ('500','优惠活动', '上海:北京,优惠活动:折扣'),
+  ('600','日本語', 'val:1,val:2');
+
+SELECT STR_TO_MAP(CONCAT(id, ':', name), ',', ':') FROM hive28728_src ORDER BY id;
+SELECT STR_TO_MAP(multi, ',', ':') FROM hive28728_src WHERE multi IS NOT NULL ORDER BY id;
+SELECT STR_TO_MAP(multi, ',', ':')['优惠活动'] FROM hive28728_src WHERE id = '500';
+SELECT STR_TO_MAP('优惠活动:折扣,北京:海淀', ',', ':');
+
+SELECT STR_TO_MAP(multi, ',', ':') FROM hive28728_src WHERE id = '200';
+SELECT STR_TO_MAP('700', ',', ':');
+
+-- Vectorized INSERT OVERWRITE
+CREATE TABLE hive28728_result (cd MAP<STRING, STRING>) STORED AS ORC;
+INSERT OVERWRITE TABLE hive28728_result
+  SELECT STR_TO_MAP(CONCAT(id, ':', name), ',', ':') FROM hive28728_src;
+SELECT * FROM hive28728_result ORDER BY cd;
+
+CREATE TABLE hive28728_multi (cd MAP<STRING, STRING>) STORED AS ORC;
+INSERT OVERWRITE TABLE hive28728_multi
+  SELECT STR_TO_MAP(multi, ',', ':') FROM hive28728_src WHERE multi IS NOT NULL ORDER BY id;
+SELECT * FROM hive28728_multi ORDER BY cd;
+
+-- Non-vectorized baseline
+SET hive.vectorized.execution.enabled=false;
+CREATE TABLE hive28728_result_novec (cd MAP<STRING, STRING>) STORED AS ORC;
+INSERT OVERWRITE TABLE hive28728_result_novec
+  SELECT STR_TO_MAP(CONCAT(id, ':', name), ',', ':') FROM hive28728_src;
+SELECT * FROM hive28728_result_novec ORDER BY cd;
+
+SELECT STR_TO_MAP(CONCAT(id, ':', name), ',', ':') FROM hive28728_src ORDER BY id;
+
+DROP TABLE IF EXISTS hive28728_src;
+DROP TABLE IF EXISTS hive28728_result;
+DROP TABLE IF EXISTS hive28728_multi;
+DROP TABLE IF EXISTS hive28728_result_novec;