[filters] fix hit count

Related to #1639

Author: tstack

src/filter_observer.cc

@@ -62,8 +62,9 @@ line_filter_observer::logline_new_lines(const logfile& lf,
 
     for (; ll_begin != ll_end; ++ll_begin) {
         auto sbr_copy = sbr.clone();
-        if (lf.get_format() != nullptr) {
-            lf.get_format()->get_subline(
+        auto* format = lf.get_format_ptr();
+        if (format != nullptr) {
+            format->get_subline(
                 lf.get_format_file_state(), *ll_begin, sbr_copy);
         }
         sbr_copy.erase_ansi();

src/logfile.cc

@@ -310,6 +310,26 @@ logfile::file_options_have_changed()
     return tz_changed;
 }
 
+void
+logfile::reset_internal_state_for_reindex()
+{
+    this->lf_index.clear();
+    this->lf_index_size = 0;
+    this->lf_level_stats = {};
+    this->lf_partial_line = false;
+    this->lf_longest_line = 0;
+    this->lf_sort_needed = true;
+    this->lf_out_of_time_order_count = 0;
+    this->lf_pattern_locks.pl_lines.clear();
+    this->lf_value_stats.clear();
+    this->lf_opids.writeAccess()->clear();
+    this->lf_thread_ids.writeAccess()->clear();
+    this->lf_allocator.reset();
+    if (this->lf_logline_observer) {
+        this->lf_logline_observer->logline_clear(*this);
+    }
+}
+
 logfile::map_entry_result
 logfile::find_content_map_entry(file_off_t offset, map_read_requirement req)
 {
@@ -1155,15 +1175,7 @@ logfile::process_prefix(shared_buffer_ref& sbr,
             }
 
             if (format_changed) {
-                this->lf_index_size = 0;
-                this->lf_level_stats = log_level_stats{};
-                this->lf_longest_line = 0;
-                this->lf_out_of_time_order_count = 0;
-                this->lf_opids.writeAccess()->clear();
-                this->lf_thread_ids.writeAccess()->clear();
-                this->lf_value_stats.clear();
-                this->lf_pattern_locks.pl_lines.clear();
-                this->lf_index.clear();
+                this->reset_internal_state_for_reindex();
                 retval = true;
             } else {
                 /*
@@ -1361,29 +1373,7 @@ logfile::rebuild_index(std::optional<ui_clock::time_point> deadline)
     {
         log_info("%s: format has changed, rebuilding",
                  this->lf_filename_as_string.c_str());
-        this->lf_index.clear();
-        this->lf_index_size = 0;
-        this->lf_partial_line = false;
-        this->lf_longest_line = 0;
-        this->lf_sort_needed = true;
-        this->lf_pattern_locks.pl_lines.clear();
-        this->lf_value_stats.clear();
-        {
-            safe::WriteAccess<safe_opid_state> writable_opid_map(
-                this->lf_opids);
-
-            writable_opid_map->los_opid_ranges.clear();
-            writable_opid_map->los_sub_in_use.clear();
-        }
-        {
-            auto tids = this->lf_thread_ids.writeAccess();
-
-            tids->ltis_tid_ranges.clear();
-        }
-        this->lf_allocator.reset();
-        if (this->lf_logline_observer) {
-            this->lf_logline_observer->logline_clear(*this);
-        }
+        this->reset_internal_state_for_reindex();
     }
     this->lf_zoned_to_local_state = dts_cfg.c_zoned_to_local;
 

src/logfile.hh

@@ -547,6 +547,8 @@ private:
 
     bool file_options_have_changed();
 
+    void reset_internal_state_for_reindex();
+
     std::filesystem::path lf_filename;
     std::string lf_filename_as_string;
     logfile_open_options lf_options;

src/textview_curses.cc

@@ -58,6 +58,8 @@ text_filter::revert_to_last(logfile_filter_state& lfs, size_t rollback_size)
         = lfs.tfs_last_message_matched[this->lf_index];
     lfs.tfs_lines_for_message[this->lf_index]
         = lfs.tfs_last_lines_for_message[this->lf_index];
+    lfs.tfs_hits_for_message[this->lf_index]
+        = lfs.tfs_last_hits_for_message[this->lf_index];
 
     for (size_t lpc = 0; lpc < lfs.tfs_lines_for_message[this->lf_index]; lpc++)
     {
@@ -92,6 +94,9 @@ text_filter::add_line(logfile_filter_state& lfs,
     lfs.tfs_message_matched[this->lf_index]
         = lfs.tfs_message_matched[this->lf_index] || retval;
     lfs.tfs_lines_for_message[this->lf_index] += 1;
+    if (retval) {
+        lfs.tfs_hits_for_message[this->lf_index] += 1;
+    }
 
     return retval;
 }
@@ -117,16 +122,18 @@ text_filter::end_of_message(logfile_filter_state& lfs)
             lfs.tfs_mask[line_number] &= ~mask;
         }
         lfs.tfs_filter_count[this->lf_index] += 1;
-        if (lfs.tfs_message_matched[this->lf_index]) {
-            lfs.tfs_filter_hits[this->lf_index] += 1;
-        }
     }
+    lfs.tfs_filter_hits[this->lf_index]
+        += lfs.tfs_hits_for_message[this->lf_index];
     lfs.tfs_last_message_matched[this->lf_index]
         = lfs.tfs_message_matched[this->lf_index];
     lfs.tfs_last_lines_for_message[this->lf_index]
         = lfs.tfs_lines_for_message[this->lf_index];
+    lfs.tfs_last_hits_for_message[this->lf_index]
+        = lfs.tfs_hits_for_message[this->lf_index];
     lfs.tfs_message_matched[this->lf_index] = false;
     lfs.tfs_lines_for_message[this->lf_index] = 0;
+    lfs.tfs_hits_for_message[this->lf_index] = 0;
 }
 
 void
@@ -1604,6 +1611,7 @@ logfile_filter_state::logfile_filter_state(std::shared_ptr<logfile> lf)
     memset(this->tfs_filter_hits, 0, sizeof(this->tfs_filter_hits));
     memset(this->tfs_message_matched, 0, sizeof(this->tfs_message_matched));
     memset(this->tfs_lines_for_message, 0, sizeof(this->tfs_lines_for_message));
+    memset(this->tfs_hits_for_message, 0, sizeof(this->tfs_hits_for_message));
     memset(this->tfs_last_message_matched,
            0,
            sizeof(this->tfs_last_message_matched));
@@ -1623,10 +1631,12 @@ logfile_filter_state::clear()
 void
 logfile_filter_state::clear_for_rebuild()
 {
+    log_debug("clearing filter state");
     memset(this->tfs_filter_count, 0, sizeof(this->tfs_filter_count));
     memset(this->tfs_filter_hits, 0, sizeof(this->tfs_filter_hits));
     memset(this->tfs_message_matched, 0, sizeof(this->tfs_message_matched));
     memset(this->tfs_lines_for_message, 0, sizeof(this->tfs_lines_for_message));
+    memset(this->tfs_hits_for_message, 0, sizeof(this->tfs_hits_for_message));
     memset(this->tfs_last_message_matched,
            0,
            sizeof(this->tfs_last_message_matched));
@@ -1644,6 +1654,7 @@ logfile_filter_state::clear_filter_state(size_t index)
     this->tfs_filter_hits[index] = 0;
     this->tfs_message_matched[index] = false;
     this->tfs_lines_for_message[index] = 0;
+    this->tfs_hits_for_message[index] = 0;
     this->tfs_last_message_matched[index] = false;
     this->tfs_last_lines_for_message[index] = 0;
 }

src/textview_curses.hh

@@ -84,8 +84,10 @@ public:
     int tfs_filter_hits[MAX_FILTERS];
     bool tfs_message_matched[MAX_FILTERS];
     size_t tfs_lines_for_message[MAX_FILTERS];
+    size_t tfs_hits_for_message[MAX_FILTERS];
     bool tfs_last_message_matched[MAX_FILTERS];
     size_t tfs_last_lines_for_message[MAX_FILTERS];
+    size_t tfs_last_hits_for_message[MAX_FILTERS];
     std::vector<uint32_t> tfs_mask;
     std::vector<uint32_t> tfs_index;
 };

test/Makefile.am

@@ -359,6 +359,7 @@ dist_noinst_DATA = \
 	logfile_epoch.0 \
 	logfile_epoch.1 \
 	logfile_filter.0 \
+	logfile_filter.1 \
 	logfile_for_join.0 \
 	logfile_generic.0 \
 	logfile_generic.1 \

test/logfile_filter.1

@@ -0,0 +1,5 @@
+hello
+hello
+bye
+Mar 22 13:41:18 Timothys-MacBook-Pro syslogd[557]: ASL Sender Statistics
+hello

test/test_sql_views_vtab.sh

@@ -128,6 +128,12 @@ view_name,filter_id,hits
 log,1,2
 EOF
 
+run_cap_test ${lnav_test} -n \
+    -c ":filter-in hello" \
+    -c ";SELECT * FROM lnav_view_filter_stats" \
+    -c ":write-csv-to -" \
+    ${test_dir}/logfile_filter.1
+
 run_test ${lnav_test} -n \
     -c ";INSERT INTO lnav_view_filters (view_name, language, pattern) VALUES ('log', 'sql', ':sc_bytes = 134')" \
     ${test_dir}/logfile_access_log.0