feat: Generate all possible wildcard subqueries for non-capture schema-based search, preventing unnecessary archive decompression. (#1313)

Co-authored-by: SharafMohamed <SharafMohamed@users.noreply.github.com>
Co-authored-by: davidlion <david.lion@yscope.com>

Author: SharafMohamed

components/core/CMakeLists.txt

@@ -575,6 +575,8 @@ set(SOURCE_FILES_unitTest
         src/clp/GrepCore.hpp
         src/clp/hash_utils.cpp
         src/clp/hash_utils.hpp
+        src/clp/SchemaSearcher.cpp
+        src/clp/SchemaSearcher.hpp
         src/clp/ir/constants.hpp
         src/clp/ir/EncodedTextAst.cpp
         src/clp/ir/EncodedTextAst.hpp
@@ -704,6 +706,11 @@ set(SOURCE_FILES_unitTest
         tests/clp_s_test_utils.cpp
         tests/clp_s_test_utils.hpp
         tests/LogSuppressor.hpp
+        tests/MockLogTypeDictionary.hpp
+        tests/MockVariableDictionary.hpp
+        tests/SchemaSearcherTest.hpp
+        tests/search_test_utils.cpp
+        tests/search_test_utils.hpp
         tests/TestOutputCleaner.hpp
         tests/test-BoundedReader.cpp
         tests/test-BufferedReader.cpp
@@ -731,6 +738,7 @@ set(SOURCE_FILES_unitTest
         tests/test-ParserWithUserSchema.cpp
         tests/test-query_methods.cpp
         tests/test-regex_utils.cpp
+        tests/test-SchemaSearcher.cpp
         tests/test-Segment.cpp
         tests/test-SQLiteDB.cpp
         tests/test-Stopwatch.cpp
@@ -790,4 +798,7 @@ if(CLP_ENABLE_TESTS)
     target_compile_features(unitTest
             PRIVATE cxx_std_20
             )
+    target_compile_definitions(unitTest
+            PRIVATE CLP_ENABLE_TESTS
+            )
 endif()

components/core/src/clp/GrepCore.cpp

@@ -1,35 +1,19 @@
 #include "GrepCore.hpp"
 
 #include <cstddef>
-#include <set>
 #include <string>
 #include <string_view>
 
-#include <log_surgeon/Constants.hpp>
 #include <string_utils/string_utils.hpp>
 
-#include "ir/parsing.hpp"
-#include "LogSurgeonReader.hpp"
-#include "QueryToken.hpp"
-#include "StringReader.hpp"
+#include <clp/ir/parsing.hpp>
 
 using clp::ir::is_delim;
 using clp::string_utils::is_alphabet;
 using clp::string_utils::is_wildcard;
 using std::string;
 
 namespace clp {
-namespace {
-/**
- * Wraps the tokens returned from the log_surgeon lexer, and stores the variable ids of the tokens
- * in a search query in a set. This allows for optimized search performance.
- */
-class SearchToken : public log_surgeon::Token {
-public:
-    std::set<int> m_type_ids_set;
-};
-}  // namespace
-
 bool GrepCore::get_bounds_of_next_potential_var(
         string const& value,
         size_t& begin_pos,
@@ -149,125 +133,4 @@ bool GrepCore::get_bounds_of_next_potential_var(
 
     return (value_length != begin_pos);
 }
-
-bool GrepCore::get_bounds_of_next_potential_var(
-        string const& value,
-        size_t& begin_pos,
-        size_t& end_pos,
-        bool& is_var,
-        log_surgeon::lexers::ByteLexer& lexer
-) {
-    size_t const value_length = value.length();
-    if (end_pos >= value_length) {
-        return false;
-    }
-
-    is_var = false;
-    bool contains_wildcard = false;
-    while (false == is_var && false == contains_wildcard && begin_pos < value_length) {
-        // Start search at end of last token
-        begin_pos = end_pos;
-
-        // Find variable begin or wildcard
-        bool is_escaped = false;
-        for (; begin_pos < value_length; ++begin_pos) {
-            char c = value[begin_pos];
-
-            if (is_escaped) {
-                is_escaped = false;
-
-                if (false == lexer.is_delimiter(c)) {
-                    // Found escaped non-delimiter, so reverse the index to retain the escape
-                    // character
-                    --begin_pos;
-                    break;
-                }
-            } else if ('\\' == c) {
-                // Escape character
-                is_escaped = true;
-            } else {
-                if (is_wildcard(c)) {
-                    contains_wildcard = true;
-                    break;
-                }
-                if (false == lexer.is_delimiter(c)) {
-                    break;
-                }
-            }
-        }
-
-        // Find next delimiter
-        is_escaped = false;
-        end_pos = begin_pos;
-        for (; end_pos < value_length; ++end_pos) {
-            char c = value[end_pos];
-
-            if (is_escaped) {
-                is_escaped = false;
-
-                if (lexer.is_delimiter(c)) {
-                    // Found escaped delimiter, so reverse the index to retain the escape character
-                    --end_pos;
-                    break;
-                }
-            } else if ('\\' == c) {
-                // Escape character
-                is_escaped = true;
-            } else {
-                if (is_wildcard(c)) {
-                    contains_wildcard = true;
-                } else if (lexer.is_delimiter(c)) {
-                    // Found delimiter that's not also a wildcard
-                    break;
-                }
-            }
-        }
-
-        if (end_pos > begin_pos) {
-            bool has_prefix_wildcard = ('*' == value[begin_pos]) || ('?' == value[begin_pos]);
-            bool has_suffix_wildcard = ('*' == value[end_pos - 1]) || ('?' == value[end_pos - 1]);
-            bool has_wildcard_in_middle = false;
-            for (size_t i = begin_pos + 1; i < end_pos - 1; ++i) {
-                if (('*' == value[i] || '?' == value[i]) && value[i - 1] != '\\') {
-                    has_wildcard_in_middle = true;
-                    break;
-                }
-            }
-            SearchToken search_token;
-            if (has_wildcard_in_middle || has_prefix_wildcard) {
-                // DO NOTHING
-            } else {
-                StringReader string_reader;
-                LogSurgeonReader reader_wrapper(string_reader);
-                log_surgeon::ParserInputBuffer parser_input_buffer;
-                if (has_suffix_wildcard) {  // text*
-                    // TODO: creating a string reader, setting it equal to a string, to read it into
-                    // the ParserInputBuffer, seems like a convoluted way to set a string equal to a
-                    // string, should be improved when adding a SearchParser to log_surgeon
-                    string_reader.open(value.substr(begin_pos, end_pos - begin_pos - 1));
-                    parser_input_buffer.read_if_safe(reader_wrapper);
-                    lexer.reset();
-                    lexer.scan_with_wildcard(parser_input_buffer, value[end_pos - 1], search_token);
-                } else {  // no wildcards
-                    string_reader.open(value.substr(begin_pos, end_pos - begin_pos));
-                    parser_input_buffer.read_if_safe(reader_wrapper);
-                    lexer.reset();
-                    auto [err, token] = lexer.scan(parser_input_buffer);
-                    if (log_surgeon::ErrorCode::Success != err) {
-                        return false;
-                    }
-                    search_token = SearchToken{token.value()};
-                    search_token.m_type_ids_set.insert(search_token.get_type_ids()->at(0));
-                }
-                auto const& type = search_token.get_type_ids()->at(0);
-                if (type != static_cast<int>(log_surgeon::SymbolId::TokenUncaughtString)
-                    && type != static_cast<int>(log_surgeon::SymbolId::TokenEnd))
-                {
-                    is_var = true;
-                }
-            }
-        }
-    }
-    return (value_length != begin_pos);
-}
 }  // namespace clp

components/core/src/clp/GrepCore.hpp

@@ -13,14 +13,15 @@
 #include <string_utils/constants.hpp>
 #include <string_utils/string_utils.hpp>
 
-#include "Defs.h"
-#include "EncodedVariableInterpreter.hpp"
-#include "ir/parsing.hpp"
-#include "ir/types.hpp"
-#include "LogTypeDictionaryReaderReq.hpp"
-#include "Query.hpp"
-#include "QueryToken.hpp"
-#include "VariableDictionaryReaderReq.hpp"
+#include <clp/Defs.h>
+#include <clp/EncodedVariableInterpreter.hpp>
+#include <clp/ir/parsing.hpp>
+#include <clp/ir/types.hpp>
+#include <clp/LogTypeDictionaryReaderReq.hpp>
+#include <clp/Query.hpp>
+#include <clp/QueryToken.hpp>
+#include <clp/SchemaSearcher.hpp>
+#include <clp/VariableDictionaryReaderReq.hpp>
 
 namespace clp {
 class GrepCore {
@@ -75,24 +76,6 @@ class GrepCore {
             bool& is_var
     );
 
-    /**
-     * Returns bounds of next potential variable (either a definite variable or a token with
-     * wildcards)
-     * @param value String containing token
-     * @param begin_pos Begin position of last token, changes to begin position of next token
-     * @param end_pos End position of last token, changes to end position of next token
-     * @param is_var Whether the token is definitely a variable
-     * @param lexer DFA for determining if input is in the schema
-     * @return true if another potential variable was found, false otherwise
-     */
-    static bool get_bounds_of_next_potential_var(
-            std::string const& value,
-            size_t& begin_pos,
-            size_t& end_pos,
-            bool& is_var,
-            log_surgeon::lexers::ByteLexer& lexer
-    );
-
 private:
     // Types
     enum class SubQueryMatchabilityResult : uint8_t {
@@ -163,13 +146,18 @@ std::optional<Query> GrepCore::process_raw_query(
         log_surgeon::lexers::ByteLexer& lexer,
         bool use_heuristic
 ) {
-    // Split search_string into tokens with wildcards
-    std::vector<QueryToken> query_tokens;
-    size_t begin_pos = 0;
-    size_t end_pos = 0;
-    bool is_var;
-    std::string search_string_for_sub_queries{search_string};
-    if (use_heuristic) {
+    std::vector<SubQuery> sub_queries;
+    if (false == use_heuristic) {
+        sub_queries
+                = SchemaSearcher::search(search_string, lexer, logtype_dict, var_dict, ignore_case);
+    } else {
+        // Split search_string into tokens with wildcards
+        std::vector<QueryToken> query_tokens;
+        size_t begin_pos{0};
+        size_t end_pos{0};
+        bool is_var{false};
+        std::string search_string_for_sub_queries{search_string};
+
         // Replace unescaped '?' wildcards with '*' wildcards since we currently have no support for
         // generating sub-queries with '?' wildcards. The final wildcard match on the decompressed
         // message uses the original wildcards, so correctness will be maintained.
@@ -192,70 +180,55 @@ std::optional<Query> GrepCore::process_raw_query(
         {
             query_tokens.emplace_back(search_string_for_sub_queries, begin_pos, end_pos, is_var);
         }
-    } else {
-        while (get_bounds_of_next_potential_var(
-                search_string_for_sub_queries,
-                begin_pos,
-                end_pos,
-                is_var,
-                lexer
-        ))
-        {
-            query_tokens.emplace_back(search_string_for_sub_queries, begin_pos, end_pos, is_var);
-        }
-    }
-
-    // Get pointers to all ambiguous tokens. Exclude tokens with wildcards in the middle since we
-    // fall-back to decompression + wildcard matching for those.
-    std::vector<QueryToken*> ambiguous_tokens;
-    for (auto& query_token : query_tokens) {
-        if (!query_token.has_greedy_wildcard_in_middle() && query_token.is_ambiguous_token()) {
-            ambiguous_tokens.push_back(&query_token);
+        // Get pointers to all ambiguous tokens. Exclude tokens with wildcards in the middle since
+        // we fall-back to decompression + wildcard matching for those.
+        std::vector<QueryToken*> ambiguous_tokens;
+        for (auto& query_token : query_tokens) {
+            if (false == query_token.has_greedy_wildcard_in_middle()
+                && query_token.is_ambiguous_token())
+            {
+                ambiguous_tokens.push_back(&query_token);
+            }
         }
-    }
 
-    // Generate a sub-query for each combination of ambiguous tokens
-    // E.g., if there are two ambiguous tokens each of which could be a logtype or variable, we need
-    // to create:
-    // - (token1 as logtype) (token2 as logtype)
-    // - (token1 as logtype) (token2 as var)
-    // - (token1 as var) (token2 as logtype)
-    // - (token1 as var) (token2 as var)
-    std::vector<SubQuery> sub_queries;
-    std::string logtype;
-    bool type_of_one_token_changed = true;
-    while (type_of_one_token_changed) {
-        SubQuery sub_query;
-
-        // Compute logtypes and variables for query
-        auto matchability = generate_logtypes_and_vars_for_subquery(
-                logtype_dict,
-                var_dict,
-                search_string_for_sub_queries,
-                query_tokens,
-                ignore_case,
-                sub_query
-        );
-        switch (matchability) {
-            case SubQueryMatchabilityResult::SupercedesAllSubQueries:
-                // Since other sub-queries will be superceded by this one, we can stop processing
-                // now
-                return Query{search_begin_ts, search_end_ts, ignore_case, search_string, {}};
-            case SubQueryMatchabilityResult::MayMatch:
-                sub_queries.push_back(std::move(sub_query));
-                break;
-            case SubQueryMatchabilityResult::WontMatch:
-            default:
-                // Do nothing
-                break;
-        }
+        // Generate a sub-query for each combination of ambiguous tokens
+        // E.g., if there are two ambiguous tokens each of which could be a logtype or variable, we
+        // need to create:
+        // - (token1 as logtype) (token2 as logtype)
+        // - (token1 as logtype) (token2 as var)
+        // - (token1 as var) (token2 as logtype)
+        // - (token1 as var) (token2 as var)
+        bool type_of_one_token_changed{true};
+        while (type_of_one_token_changed) {
+            SubQuery sub_query;
+            auto matchability{generate_logtypes_and_vars_for_subquery(
+                    logtype_dict,
+                    var_dict,
+                    search_string_for_sub_queries,
+                    query_tokens,
+                    ignore_case,
+                    sub_query
+            )};
+            switch (matchability) {
+                case SubQueryMatchabilityResult::SupercedesAllSubQueries:
+                    // Since other sub-queries will be superceded by this one, we can stop
+                    // processing now.
+                    return Query{search_begin_ts, search_end_ts, ignore_case, search_string, {}};
+                case SubQueryMatchabilityResult::MayMatch:
+                    sub_queries.push_back(std::move(sub_query));
+                    break;
+                case SubQueryMatchabilityResult::WontMatch:
+                default:
+                    break;
+            }
 
-        // Update combination of ambiguous tokens
-        type_of_one_token_changed = false;
-        for (auto* ambiguous_token : ambiguous_tokens) {
-            if (ambiguous_token->change_to_next_possible_type()) {
-                type_of_one_token_changed = true;
-                break;
+            // Update combination of ambiguous tokens
+            type_of_one_token_changed = false;
+            for (auto* ambiguous_token : ambiguous_tokens) {
+                if (ambiguous_token->change_to_next_possible_type()) {
+                    type_of_one_token_changed = true;
+                    break;
+                }
             }
         }
     }