Kusto-phase3: rebase v24.2.1.2248-stable

Kusto-phase3: rebase v24.2.1.2248-stable fixing tests.

Kusto-phase3: rebase v24.2.1.2248-stable fix has any ip functions.

Kusto-phase3: rebase v24.2.1.2248-stable provide additional protection on ip functions.

Author: kothiga

src/Common/IntervalKind.h

@@ -42,7 +42,7 @@ struct IntervalKind
     Float64 toSeconds() const;
 
     /// Chooses an interval kind based on number of seconds.
-    /// For example, `IntervalKind::fromAvgSeconds(3600)` returns `IntervalKind::Hour`.
+    /// For example, `IntervalKind::fromAvgSeconds(3600)` returns `IntervalKind::Kind::Hour`.
     static IntervalKind fromAvgSeconds(Int64 num_seconds);
 
     /// Returns whether IntervalKind has a fixed number of seconds (e.g. Day) or non-fixed(e.g. Month)
@@ -54,16 +54,16 @@ struct IntervalKind
     const char * toLowercasedKeyword() const;
 
     /// Returns the string which can be passed to the `unit` parameter of the dateDiff() function.
-    /// For example, `IntervalKind{IntervalKind::Day}.getDateDiffParameter()` returns "day".
+    /// For example, `IntervalKind{IntervalKind::Kind::Day}.getDateDiffParameter()` returns "day".
     const char * toDateDiffUnit() const;
 
     /// Returns the name of the function converting a number to the interval data type.
-    /// For example, `IntervalKind{IntervalKind::Day}.getToIntervalDataTypeFunctionName()`
+    /// For example, `IntervalKind{IntervalKind::Kind::Day}.getToIntervalDataTypeFunctionName()`
     /// returns "toIntervalDay".
     const char * toNameOfFunctionToIntervalDataType() const;
 
     /// Returns the name of the function extracting time part from a date or a time.
-    /// For example, `IntervalKind{IntervalKind::Day}.getExtractTimePartFunctionName()`
+    /// For example, `IntervalKind{IntervalKind::Kind::Day}.getExtractTimePartFunctionName()`
     /// returns "toDayOfMonth".
     const char * toNameOfFunctionExtractTimePart() const;
 

src/Core/SettingsChangesHistory.h

@@ -141,7 +141,9 @@ static std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges> sett
     {"23.12", {{"allow_suspicious_ttl_expressions", true, false, "It is a new setting, and in previous versions the behavior was equivalent to allowing."},
               {"input_format_parquet_allow_missing_columns", false, true, "Allow missing columns in Parquet files by default"},
               {"input_format_orc_allow_missing_columns", false, true, "Allow missing columns in ORC files by default"},
-              {"input_format_arrow_allow_missing_columns", false, true, "Allow missing columns in Arrow files by default"}}},
+              {"input_format_arrow_allow_missing_columns", false, true, "Allow missing columns in Arrow files by default"},
+              {"split_parts_ranges_into_intersecting_and_non_intersecting_final", false, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"},
+              {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"}}},
     {"23.9", {{"optimize_group_by_constant_keys", false, true, "Optimize group by constant keys by default"},
               {"input_format_json_try_infer_named_tuples_from_objects", false, true, "Try to infer named Tuples from JSON objects by default"},
               {"input_format_json_read_numbers_as_strings", false, true, "Allow to read numbers as strings in JSON formats by default"},

src/Functions/FunctionBinaryArithmetic.h

@@ -1723,29 +1723,7 @@ class FunctionBinaryArithmetic : public IFunction
 
                 if constexpr (!std::is_same_v<ResultDataType, InvalidType>)
                 {
-                    if constexpr (is_div_int || is_div_int_or_zero)
-                        type_res = std::make_shared<ResultDataType>();
-                    else if constexpr (IsDataTypeDecimal<LeftDataType> && IsDataTypeDecimal<RightDataType>)
-                    {
-                        if constexpr (is_division)
-                        {
-                            if (context->getSettingsRef().decimal_check_overflow)
-                            {
-                                /// Check overflow by using operands scale (based on big decimal division implementation details):
-                                /// big decimal arithmetic is based on big integers, decimal operands are converted to big integers
-                                /// i.e. int_operand = decimal_operand*10^scale
-                                /// For division, left operand will be scaled by right operand scale also to do big integer division,
-                                /// BigInt result = left*10^(left_scale + right_scale) / right * 10^right_scale
-                                /// So, we can check upfront possible overflow just by checking max scale used for left operand
-                                /// Note: it doesn't detect all possible overflow during big decimal division
-                                if (left.getScale() + right.getScale() > ResultDataType::maxPrecision())
-                                    throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "Overflow during decimal division");
-                            }
-                        }
-                        ResultDataType result_type = decimalResultType<is_multiply, is_division>(left, right);
-                        type_res = std::make_shared<ResultDataType>(result_type.getPrecision(), result_type.getScale());
-                    }
-                    else if constexpr (((IsDataTypeDecimal<LeftDataType> && IsFloatingPoint<RightDataType>) ||
+                    if constexpr (((IsDataTypeDecimal<LeftDataType> && IsFloatingPoint<RightDataType>) ||
                         (IsDataTypeDecimal<RightDataType> && IsFloatingPoint<LeftDataType>)))
                     {
                         type_res = std::make_shared<DataTypeFloat64>();
@@ -1807,6 +1785,28 @@ class FunctionBinaryArithmetic : public IFunction
 
                         return static_cast<bool>(nested_type);
                     }
+                    else if constexpr (IsDataTypeDecimal<LeftDataType> && IsDataTypeDecimal<RightDataType>)
+                    {
+                        if constexpr (is_division)
+                        {
+                            if (context->getSettingsRef().decimal_check_overflow)
+                            {
+                                /// Check overflow by using operands scale (based on big decimal division implementation details):
+                                /// big decimal arithmetic is based on big integers, decimal operands are converted to big integers
+                                /// i.e. int_operand = decimal_operand*10^scale
+                                /// For division, left operand will be scaled by right operand scale also to do big integer division,
+                                /// BigInt result = left*10^(left_scale + right_scale) / right * 10^right_scale
+                                /// So, we can check upfront possible overflow just by checking max scale used for left operand
+                                /// Note: it doesn't detect all possible overflow during big decimal division
+                                if (left.getScale() + right.getScale() > ResultDataType::maxPrecision())
+                                    throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "Overflow during decimal division");
+                            }
+                        }
+                        ResultDataType result_type = decimalResultType<is_multiply, is_division>(left, right);
+                        type_res = std::make_shared<ResultDataType>(result_type.getPrecision(), result_type.getScale());
+                    }
+                    else if constexpr (is_div_int || is_div_int_or_zero)
+                        type_res = std::make_shared<ResultDataType>();
                     else
                         type_res = std::make_shared<ResultDataType>();
                     return true;

src/Functions/FunctionsConversion.h

@@ -3316,7 +3316,7 @@ class FunctionCast final : public FunctionCastBase
                         switch (interval_kind)
                         {
 #define DECLARE_CASE(NAME) \
-    case IntervalKind::NAME: \
+    case IntervalKind::Kind::NAME: \
         return FunctionToInterval##NAME::create();
                             FOR_EACH_INTERVAL_KIND(DECLARE_CASE)
 #undef DECLARE_CASE

src/Functions/Kusto/KqlRange.cpp

@@ -809,7 +809,7 @@ class FunctionKqlRange : public IFunction
                 total_values = total_elements;
         }
 
-        auto out = ColumnArray::create(std::make_shared<DataTypeInterval>(IntervalKind::Nanosecond)->createColumn());
+        auto out = ColumnArray::create(std::make_shared<DataTypeInterval>(IntervalKind::Kind::Nanosecond)->createColumn());
         IColumn & out_data = out->getData();
         IColumn::Offsets & out_offsets = out->getOffsets();
 

src/Functions/Kusto/kqlDateTime.cpp

@@ -95,11 +95,11 @@ ColumnPtr FunctionKqlDateTime<input_policy>::executeImpl(
 
     const ColumnsWithTypeAndName addition_args{
         asArgument(converted, "converted"),
-        createConstColumnWithTypeAndName<DataTypeInterval>(50, "interval_50", IntervalKind::Nanosecond)};
+        createConstColumnWithTypeAndName<DataTypeInterval>(50, "interval_50", IntervalKind::Kind::Nanosecond)};
     const auto sum = executeFunctionCall(context, "plus", addition_args, input_rows_count);
 
     const ColumnsWithTypeAndName to_start_of_interval_args{
-        asArgument(sum, "sum"), createConstColumnWithTypeAndName<DataTypeInterval>(100, "interval_100", IntervalKind::Nanosecond)};
+        asArgument(sum, "sum"), createConstColumnWithTypeAndName<DataTypeInterval>(100, "interval_100", IntervalKind::Kind::Nanosecond)};
     const auto [rounded_column, _] = executeFunctionCall(context, "toStartOfInterval", to_start_of_interval_args, input_rows_count);
 
     return wrapInNullable(rounded_column, conversion_args, result_type, input_rows_count);

src/Functions/Kusto/kqlHasAnyIp.cpp

@@ -1,9 +1,9 @@
 #include <optional>
 #include <ranges>
-#include <regex>
 #include <Columns/ColumnArray.h>
 #include <Columns/ColumnDecimal.h>
 #include <Columns/ColumnString.h>
+#include <Common/OptimizedRegularExpression.h>
 #include <DataTypes/DataTypeString.h>
 #include <DataTypes/DataTypesNumber.h>
 #include <Functions/FunctionFactory.h>
@@ -397,17 +397,26 @@ class FunctionKqlHasIpGeneric : public IFunction
             const auto ips = extractIpsFromArguments(arguments, result_type, context, i);
 
             std::string source = arguments[0].column->getDataAt(i).toString();
-            const std::regex ip_finder(Func::regex);
-            std::smatch matches;
 
-            while (!res && std::regex_search(source, matches, ip_finder))
+            OptimizedRegularExpression ip_finder(Func::regex);
+            OptimizedRegularExpression::MatchVec matches;
+            
+            unsigned num_matches;
+            while (!res && (num_matches = ip_finder.match(source, matches)))
             {
-                res = Func::checkRegexMatch(matches[2].str(), result_type, context, ips);
+                if (num_matches <= 3) 
+                    continue;
 
-                source = matches.suffix().str();
+                res = Func::checkRegexMatch(source.substr(matches[2].offset, matches[2].length), result_type, context, ips);
+
+                if (matches[3].length == 0)
+                    source = "";
+                else
+                    source = source.substr(matches[3].offset, source.length());
             }
             result_column.push_back(static_cast<UInt8>(res));
         }
+
         return result;
     }
 

src/Functions/Kusto/kqlIndexOfRegex.cpp

@@ -1,21 +1,13 @@
 #include <Columns/ColumnNullable.h>
 #include <Columns/ColumnString.h>
 #include <Columns/ColumnsNumber.h>
+#include <Common/re2.h>
 #include <DataTypes/DataTypeString.h>
 #include <DataTypes/DataTypesNumber.h>
 #include <Functions/FunctionFactory.h>
 #include <Functions/FunctionHelpers.h>
 #include <Functions/IFunction.h>
 
-#ifdef __clang__
-#  pragma clang diagnostic push
-#  pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
-#endif
-#include <re2/re2.h>
-#ifdef __clang__
-#  pragma clang diagnostic pop
-#endif
-
 namespace DB::ErrorCodes
 {
 extern const int CANNOT_COMPILE_REGEXP;

src/Functions/Kusto/kqlToTimespan.cpp

@@ -79,7 +79,7 @@ ColumnPtr FunctionKqlToTimespan::executeImpl(
 
 DataTypePtr FunctionKqlToTimespan::getReturnTypeImpl(const DataTypes &) const
 {
-    return makeNullable(std::make_shared<DataTypeInterval>(IntervalKind::Nanosecond));
+    return makeNullable(std::make_shared<DataTypeInterval>(IntervalKind::Kind::Nanosecond));
 }
 
 REGISTER_FUNCTION(KqlToTimespan)

src/Functions/dateTime64Diff.cpp

@@ -92,7 +92,7 @@ DataTypePtr FunctionDateTime64Diff::getReturnTypeImpl(const DataTypes & argument
             rhs->getName(),
             getName());
 
-    return std::make_shared<DataTypeInterval>(IntervalKind::Nanosecond);
+    return std::make_shared<DataTypeInterval>(IntervalKind::Kind::Nanosecond);
 }
 
 REGISTER_FUNCTION(DateTime64Diff)