feat: Improve schema-based search accuracy using log_surgeon::Query.

components/core/CMakeLists.txt

@@ -774,4 +774,7 @@ if(CLP_BUILD_TESTING)
     target_compile_features(unitTest
             PRIVATE cxx_std_20
             )
+    target_compile_definitions(unitTest
+            PRIVATE CLP_BUILD_TESTING
+            )
 endif()

components/core/src/clp/GrepCore.cpp

@@ -1,35 +1,36 @@
 #include "GrepCore.hpp"
 
+#include <algorithm>
 #include <cstddef>
 #include <set>
 #include <string>
 #include <string_view>
+#include <unordered_map>
+#include <variant>
+#include <vector>
 
 #include <log_surgeon/Constants.hpp>
 #include <string_utils/string_utils.hpp>
 
+#include "EncodedVariableInterpreter.hpp"
 #include "ir/parsing.hpp"
-#include "LogSurgeonReader.hpp"
-#include "QueryToken.hpp"
 #include "StringReader.hpp"
 
 using clp::ir::is_delim;
 using clp::string_utils::is_alphabet;
 using clp::string_utils::is_wildcard;
+using log_surgeon::SymbolId::TokenFloat;
+using log_surgeon::SymbolId::TokenInt;
+using log_surgeon::wildcard_query_parser::QueryInterpretation;
+using log_surgeon::wildcard_query_parser::StaticQueryToken;
+using log_surgeon::wildcard_query_parser::VariableQueryToken;
+using std::holds_alternative;
+using std::set;
 using std::string;
+using std::unordered_map;
+using std::vector;
 
 namespace clp {
-namespace {
-/**
- * Wraps the tokens returned from the log_surgeon lexer, and stores the variable ids of the tokens
- * in a search query in a set. This allows for optimized search performance.
- */
-class SearchToken : public log_surgeon::Token {
-public:
-    std::set<int> m_type_ids_set;
-};
-}  // namespace
-
 bool GrepCore::get_bounds_of_next_potential_var(
         string const& value,
         size_t& begin_pos,
@@ -150,124 +151,134 @@ bool GrepCore::get_bounds_of_next_potential_var(
     return (value_length != begin_pos);
 }
 
-bool GrepCore::get_bounds_of_next_potential_var(
-        string const& value,
-        size_t& begin_pos,
-        size_t& end_pos,
-        bool& is_var,
-        log_surgeon::lexers::ByteLexer& lexer
-) {
-    size_t const value_length = value.length();
-    if (end_pos >= value_length) {
-        return false;
+auto GrepCore::normalize_interpretations(set<QueryInterpretation> const& interpretations)
+        -> set<QueryInterpretation> {
+    set<QueryInterpretation> normalized_interpretations;
+    for (auto const& interpretation : interpretations) {
+        QueryInterpretation normalized_interpretation;
+        for (auto const& token : interpretation.get_logtype()) {
+            auto const& src_string{
+                    holds_alternative<VariableQueryToken>(token)
+                            ? std::get<VariableQueryToken>(token).get_query_substring()
+                            : std::get<StaticQueryToken>(token).get_query_substring()
+            };
+            string normalized_string;
+            normalized_string.reserve(src_string.size());
+            for (auto const c : src_string) {
+                if (c != '*' || normalized_string.empty() || normalized_string.back() != '*') {
+                    normalized_string += c;
+                }
+            }
+
+            if (holds_alternative<VariableQueryToken>(token)) {
+                auto const& variable_token{std::get<VariableQueryToken>(token)};
+                normalized_interpretation.append_variable_token(
+                        variable_token.get_variable_type(),
+                        normalized_string,
+                        variable_token.get_contains_wildcard()
+                );
+            } else {
+                normalized_interpretation.append_static_token(normalized_string);
+            }
+        }
+        normalized_interpretations.insert(normalized_interpretation);
     }
+    return normalized_interpretations;
+}
 
-    is_var = false;
-    bool contains_wildcard = false;
-    while (false == is_var && false == contains_wildcard && begin_pos < value_length) {
-        // Start search at end of last token
-        begin_pos = end_pos;
+auto GrepCore::get_wildcard_encodable_positions(QueryInterpretation const& interpretation)
+        -> vector<size_t> {
+    auto const logtype{interpretation.get_logtype()};
+    vector<size_t> wildcard_encodable_positions;
+    wildcard_encodable_positions.reserve(logtype.size());
 
-        // Find variable begin or wildcard
-        bool is_escaped = false;
-        for (; begin_pos < value_length; ++begin_pos) {
-            char c = value[begin_pos];
+    for (size_t i{0}; i < logtype.size(); ++i) {
+        auto const& token{logtype[i]};
+        if (holds_alternative<VariableQueryToken>(token)) {
+            auto const& var_token{std::get<VariableQueryToken>(token)};
+            auto const var_type{static_cast<log_surgeon::SymbolId>(var_token.get_variable_type())};
+            bool const is_int{TokenInt == var_type};
+            bool const is_float{TokenFloat == var_type};
+            if (var_token.get_contains_wildcard() && (is_int || is_float)) {
+                wildcard_encodable_positions.push_back(i);
+            }
+        }
+    }
+    return wildcard_encodable_positions;
+}
 
-            if (is_escaped) {
-                is_escaped = false;
+auto GrepCore::generate_logtype_string(
+        QueryInterpretation const& interpretation,
+        vector<size_t> const& wildcard_encodable_positions,
+        vector<bool> const& mask_encoded_flags
+) -> string {
+    string logtype_string;
 
-                if (false == lexer.is_delimiter(c)) {
-                    // Found escaped non-delimiter, so reverse the index to retain the escape
-                    // character
-                    --begin_pos;
-                    break;
-                }
-            } else if ('\\' == c) {
-                // Escape character
-                is_escaped = true;
-            } else {
-                if (is_wildcard(c)) {
-                    contains_wildcard = true;
-                    break;
-                }
-                if (false == lexer.is_delimiter(c)) {
-                    break;
-                }
-            }
+    // Reserve size for `logtype_string`.
+    size_t logtype_string_size{0};
+    auto const logtype{interpretation.get_logtype()};
+    for (auto const& token : logtype) {
+        if (holds_alternative<StaticQueryToken>(token)) {
+            auto const& static_token{std::get<StaticQueryToken>(token)};
+            logtype_string_size += static_token.get_query_substring().size();
+        } else {
+            logtype_string_size++;
         }
+    }
+    logtype_string.reserve(logtype_string_size);
 
-        // Find next delimiter
-        is_escaped = false;
-        end_pos = begin_pos;
-        for (; end_pos < value_length; ++end_pos) {
-            char c = value[end_pos];
+    // Generate `logtype_string`.
+    for (size_t i{0}; i < logtype.size(); ++i) {
+        auto const& token{logtype[i]};
+        if (holds_alternative<StaticQueryToken>(token)) {
+            logtype_string += std::get<StaticQueryToken>(token).get_query_substring();
+            continue;
+        }
 
-            if (is_escaped) {
-                is_escaped = false;
+        auto const& var_token{std::get<VariableQueryToken>(token)};
+        auto const& raw_string{var_token.get_query_substring()};
+        auto const var_type{static_cast<log_surgeon::SymbolId>(var_token.get_variable_type())};
+        bool const is_int{TokenInt == var_type};
+        bool const is_float{TokenFloat == var_type};
 
-                if (lexer.is_delimiter(c)) {
-                    // Found escaped delimiter, so reverse the index to retain the escape character
-                    --end_pos;
-                    break;
+        if (wildcard_encodable_positions.end()
+            != std::ranges::find(
+                    wildcard_encodable_positions.begin(),
+                    wildcard_encodable_positions.end(),
+                    i
+            ))
+        {
+            if (mask_encoded_flags[i]) {
+                if (is_int) {
+                    EncodedVariableInterpreter::add_int_var(logtype_string);
+                } else {
+                    EncodedVariableInterpreter::add_float_var(logtype_string);
                 }
-            } else if ('\\' == c) {
-                // Escape character
-                is_escaped = true;
             } else {
-                if (is_wildcard(c)) {
-                    contains_wildcard = true;
-                } else if (lexer.is_delimiter(c)) {
-                    // Found delimiter that's not also a wildcard
-                    break;
-                }
+                EncodedVariableInterpreter::add_dict_var(logtype_string);
             }
+            continue;
         }
 
-        if (end_pos > begin_pos) {
-            bool has_prefix_wildcard = ('*' == value[begin_pos]) || ('?' == value[begin_pos]);
-            bool has_suffix_wildcard = ('*' == value[end_pos - 1]) || ('?' == value[end_pos - 1]);
-            bool has_wildcard_in_middle = false;
-            for (size_t i = begin_pos + 1; i < end_pos - 1; ++i) {
-                if (('*' == value[i] || '?' == value[i]) && value[i - 1] != '\\') {
-                    has_wildcard_in_middle = true;
-                    break;
-                }
-            }
-            SearchToken search_token;
-            if (has_wildcard_in_middle || has_prefix_wildcard) {
-                // DO NOTHING
-            } else {
-                StringReader string_reader;
-                LogSurgeonReader reader_wrapper(string_reader);
-                log_surgeon::ParserInputBuffer parser_input_buffer;
-                if (has_suffix_wildcard) {  // text*
-                    // TODO: creating a string reader, setting it equal to a string, to read it into
-                    // the ParserInputBuffer, seems like a convoluted way to set a string equal to a
-                    // string, should be improved when adding a SearchParser to log_surgeon
-                    string_reader.open(value.substr(begin_pos, end_pos - begin_pos - 1));
-                    parser_input_buffer.read_if_safe(reader_wrapper);
-                    lexer.reset();
-                    lexer.scan_with_wildcard(parser_input_buffer, value[end_pos - 1], search_token);
-                } else {  // no wildcards
-                    string_reader.open(value.substr(begin_pos, end_pos - begin_pos));
-                    parser_input_buffer.read_if_safe(reader_wrapper);
-                    lexer.reset();
-                    auto [err, token] = lexer.scan(parser_input_buffer);
-                    if (log_surgeon::ErrorCode::Success != err) {
-                        return false;
-                    }
-                    search_token = SearchToken{token.value()};
-                    search_token.m_type_ids_set.insert(search_token.m_type_ids_ptr->at(0));
-                }
-                auto const& type = search_token.m_type_ids_ptr->at(0);
-                if (type != static_cast<int>(log_surgeon::SymbolId::TokenUncaughtString)
-                    && type != static_cast<int>(log_surgeon::SymbolId::TokenEnd))
-                {
-                    is_var = true;
-                }
-            }
+        encoded_variable_t encoded_var{0};
+        if (is_int
+            && EncodedVariableInterpreter::convert_string_to_representable_integer_var(
+                    raw_string,
+                    encoded_var
+            ))
+        {
+            EncodedVariableInterpreter::add_int_var(logtype_string);
+        } else if (is_float
+                   && EncodedVariableInterpreter::convert_string_to_representable_float_var(
+                           raw_string,
+                           encoded_var
+                   ))
+        {
+            EncodedVariableInterpreter::add_float_var(logtype_string);
+        } else {
+            EncodedVariableInterpreter::add_dict_var(logtype_string);
         }
     }
-    return (value_length != begin_pos);
+    return logtype_string;
 }
 }  // namespace clp