[stats] compute cardinality for identifiers using hyperloglog

Author: tstack

NEWS.md

@@ -57,6 +57,16 @@ Features:
   the raw value to the base unit implied by `suffix` —
   e.g. a field storing milliseconds with `"suffix": "s"`
   declares `"divisor": 1000`.
+* The details overlay now shows per-column statistics for
+  the focused message.  Numeric columns get a `min..max of
+  N` range summary on the value line, plus a `p50/p90/p99`
+  percentile sub-line for columns with enough samples to
+  characterize the distribution shape.  Identifier and
+  metrics-text columns get an estimated distinct-value
+  count (`~K distinct of N`).  Distinct counts are computed
+  from a HyperLogLog sketch (~4 KB per text column,
+  ~1.6% standard error).  Stats render in the column's
+  declared unit when one is set.
 
 Interface Changes:
 * Moving horizontally now defaults to moving to the

src/CMakeLists.txt

@@ -838,6 +838,8 @@ add_library(
         third-party/date/include/date/tz.h
         third-party/date/include/date/islamic.h
 
+        third-party/cpp-HyperLogLog/include/hyperloglog.hpp
+
         third-party/digestible/include/digestible/digestible.h
 
         third-party/intervaltree/IntervalTree.h
@@ -872,6 +874,7 @@ set(lnav_SRCS lnav.cc file_vtab.cc all_ids_vtabs.cc breakpoint_vtab.cc metrics_v
 target_include_directories(diag PUBLIC . fmtlib ${CMAKE_CURRENT_BINARY_DIR}
         third-party
         third-party/base64/include
+        third-party/cpp-HyperLogLog/include
         third-party/date/include
         third-party/digestible/include
         third-party/lnav-rs-ext

src/Makefile.am

@@ -181,6 +181,7 @@ AM_CPPFLAGS = \
 	-I$(srcdir)/fmtlib \
 	-I$(srcdir)/third-party \
 	-I$(srcdir)/third-party/base64/include \
+	-I$(srcdir)/third-party/cpp-HyperLogLog/include \
 	-I$(srcdir)/third-party/date/include \
 	-I$(srcdir)/third-party/digestible/include \
 	-I$(srcdir)/third-party/notcurses/include \
@@ -459,6 +460,7 @@ THIRD_PARTY_SRCS = \
     third-party/CLI/Split.hpp \
     third-party/CLI/TypeTools.hpp \
     third-party/CLI/ConfigFwd.hpp \
+    third-party/cpp-HyperLogLog/include/hyperloglog.hpp \
     third-party/digestible/include/digestible/digestible.h \
     third-party/doctest-root/doctest/doctest.h \
     third-party/intervaltree/IntervalTree.h \

src/field_overlay_source.cc

@@ -540,7 +540,7 @@ field_overlay_source::build_field_lines(const listview_curses& lv,
         auto* curr_elf = dynamic_cast<external_log_format*>(curr_format);
         const auto format_name = curr_format->get_name().to_string();
         attr_line_t al;
-        auto value_str = lv.to_string();
+        auto value_str = lv.to_humanized_string();
 
         if (curr_format != last_format) {
             this->fos_lines.emplace_back(" Known message fields for table "
@@ -554,6 +554,7 @@ field_overlay_source::build_field_lines(const listview_curses& lv,
 
         std::string field_name, orig_field_name;
         line_range hl_range;
+        size_t prefix_len = 0;
         al.append(" ").append("|", VC_GRAPHIC.value(NCACS_LTEE)).append(" ");
         if (meta.lvm_struct_name.empty()) {
             if (curr_elf && curr_elf->elf_body_field == meta.lvm_name) {
@@ -585,11 +586,11 @@ field_overlay_source::build_field_lines(const listview_curses& lv,
                     al.append(":bar_chart:"_emoji).append(" ");
                     break;
             }
-            auto prefix_len = al.column_width();
+            prefix_len = al.column_width() + this->fos_known_key_size;
             hl_range.lr_start = al.get_string().length();
             al.append(field_name);
             hl_range.lr_end = al.get_string().length();
-            al.pad_to(prefix_len + this->fos_known_key_size);
+            al.pad_to(prefix_len);
 
             this->fos_row_to_field_meta.emplace(this->fos_lines.size(),
                                                 row_info{meta, value_str});
@@ -599,6 +600,7 @@ field_overlay_source::build_field_lines(const listview_curses& lv,
                                                meta.lvm_name.get());
             hl_range.lr_start = al.get_string().length();
             al.append(jget_str.in());
+            prefix_len = al.column_width();
             hl_range.lr_end = al.get_string().length();
 
             this->fos_row_to_field_meta.emplace(
@@ -607,29 +609,6 @@ field_overlay_source::build_field_lines(const listview_curses& lv,
         readline_sql_highlighter_int(
             al, lnav::sql::dialect::sqlite, std::nullopt, hl_range);
 
-        if (!meta.lvm_unit_suffix.empty()) {
-            std::optional<double> numeric;
-            switch (meta.lvm_kind) {
-                case value_kind_t::VALUE_INTEGER:
-                    numeric = (double) lv.lv_value.i;
-                    break;
-                case value_kind_t::VALUE_FLOAT:
-                    numeric = lv.lv_value.d;
-                    break;
-                default:
-                    break;
-            }
-            if (numeric) {
-                if (meta.lvm_unit_divisor != 0.0
-                    && meta.lvm_unit_divisor != 1.0)
-                {
-                    *numeric /= meta.lvm_unit_divisor;
-                }
-                value_str = humanize::format(
-                    *numeric, meta.lvm_unit_suffix.to_string_fragment());
-            }
-        }
-
         if (meta.lvm_kind == value_kind_t::VALUE_TIMESTAMP) {
             auto dts = curr_format->build_time_scanner();
             exttm tm;
@@ -653,8 +632,66 @@ field_overlay_source::build_field_lines(const listview_curses& lv,
 
         al.append(" = ").append(scrub_ws(value_str.c_str()));
 
+        // Per-column stats summary: numeric columns get a min..max
+        // range and total count; text columns get an HLL-estimated
+        // distinct count.  Both render in the column's unit if one is
+        // declared, mirroring the value's own formatting above.
+        const logline_value_stats* stats = nullptr;
+        const auto* curr_lf = this->fos_log_helper.ldh_file.get();
+        if (curr_lf != nullptr) {
+            stats = curr_lf->stats_for_value(meta.lvm_name);
+        }
+
+        if (stats != nullptr) {
+            std::string summary;
+            if (stats->lvs_count > 0) {
+                summary
+                    = fmt::format(FMT_STRING("  {}..{} of {}"),
+                                  meta.to_humanized_value(stats->lvs_min_value),
+                                  meta.to_humanized_value(stats->lvs_max_value),
+                                  stats->lvs_count);
+            } else if (auto est = stats->distinct_estimate(); est) {
+                summary = fmt::format(FMT_STRING("  ~{:.0f} distinct of {}"),
+                                      est.value(),
+                                      stats->lvs_text_count);
+            }
+            if (!summary.empty()) {
+                al.append(attr_line_t(summary).with_attr_for_all(
+                    VC_ROLE.value(role_t::VCR_COMMENT)));
+            }
+        }
+
         this->fos_lines.emplace_back(al);
 
+        // Numeric percentile sub-line: typical / tail / extreme.
+        // Suppressed when the sample is too small to be statistically
+        // meaningful, when the distribution is degenerate (single
+        // value), or when the upper percentiles all collapse to the
+        // max — in those cases the inline `min..max of N` already
+        // tells the whole story.
+        if (stats != nullptr && stats->lvs_count >= 20
+            && stats->lvs_min_value < stats->lvs_max_value)
+        {
+            const auto p50 = stats->lvs_tdigest.quantile(50);
+            const auto p90 = stats->lvs_tdigest.quantile(90);
+            const auto p99 = stats->lvs_tdigest.quantile(99);
+            if (!(p50 == p99 && p99 == stats->lvs_max_value)) {
+                attr_line_t pct_line;
+                pct_line.append("    ")
+                    .with_attr(string_attr(line_range{1, 2},
+                                           VC_GRAPHIC.value(NCACS_VLINE)))
+                    .with_attr(string_attr(line_range{1, 2},
+                                           VC_ROLE.value(role_t::VCR_COMMENT)))
+                    .pad_to(prefix_len + 5)
+                    .append(fmt::format(FMT_STRING("p50={}  p90={}  p99={}"),
+                                        meta.to_humanized_value(p50),
+                                        meta.to_humanized_value(p90),
+                                        meta.to_humanized_value(p99)));
+                pct_line.with_attr_for_all(VC_ROLE.value(role_t::VCR_COMMENT));
+                this->fos_lines.emplace_back(pct_line);
+            }
+        }
+
         if (meta.lvm_kind == value_kind_t::VALUE_STRUCT) {
             json_string js = extract(value_str.c_str());
 

src/log_format.cc

@@ -390,6 +390,30 @@ logline_value_meta::to_chart_type() const
     return retval;
 }
 
+std::string
+logline_value_meta::to_humanized_value(int64_t i) const
+{
+    if (!this->lvm_unit_suffix.empty()
+        || (this->lvm_unit_divisor != 0.0 && this->lvm_unit_divisor != 1.0))
+    {
+        double d = i;
+        if (this->lvm_unit_divisor != 0.0 && this->lvm_unit_divisor != 1.0) {
+            d /= this->lvm_unit_divisor;
+        }
+        return humanize::format(d, this->lvm_unit_suffix.to_string_fragment());
+    }
+    return fmt::to_string(i);
+}
+
+std::string
+logline_value_meta::to_humanized_value(double d) const
+{
+    if (this->lvm_unit_divisor != 0.0 && this->lvm_unit_divisor != 1.0) {
+        d /= this->lvm_unit_divisor;
+    }
+    return humanize::format(d, this->lvm_unit_suffix.to_string_fragment());
+}
+
 struct line_range
 logline_value::origin_in_full_msg(const char* msg, ssize_t len) const
 {
@@ -592,6 +616,19 @@ logline_value::to_string() const
     return {buffer};
 }
 
+std::string
+logline_value::to_humanized_string() const
+{
+    switch (this->lv_meta.lvm_kind) {
+        case value_kind_t::VALUE_INTEGER:
+            return this->lv_meta.to_humanized_value(this->lv_value.i);
+        case value_kind_t::VALUE_FLOAT:
+            return this->lv_meta.to_humanized_value(this->lv_value.d);
+        default:
+            return this->to_string();
+    }
+}
+
 string_fragment
 logline_value::to_string_fragment(ArenaAlloc::Alloc<char>& alloc) const
 {
@@ -2291,6 +2328,9 @@ external_log_format::scan_tabular(logfile& lf,
                 if (field_sf.length() > lvs.lvs_width) {
                     lvs.lvs_width = field_sf.length();
                 }
+                if (vd->vd_meta.lvm_identifier && !field_sf.empty()) {
+                    lvs.add_text(field_sf);
+                }
             }
             // CSV cells may carry a `""`-escaped double-quote literal;
             // collapse those before any downstream comparison or
@@ -2603,6 +2643,15 @@ external_log_format::scan(logfile& lf,
             if (cap_size > lvs.lvs_width) {
                 lvs.lvs_width = cap_size;
             }
+            // Identifier fields are explicitly excluded from numeric
+            // ingest in `ingest_numeric_value`; route them to the
+            // distinct-count estimator instead so columns like opid,
+            // hostname, request_id surface a useful cardinality.
+            if (ivd.ivd_value_def->vd_meta.lvm_identifier) {
+                if (auto cap = md[ivd.ivd_index]) {
+                    lvs.add_text(*cap);
+                }
+            }
         }
 
         for (auto value_index : fpat->p_numeric_value_indexes) {
@@ -5967,6 +6016,11 @@ external_log_format::value_line_count(scan_batch_context& sbc,
         }
         if (val) {
             lvs.add_value(val.value());
+        } else if (vd->vd_meta.lvm_identifier && str != nullptr && len > 0) {
+            // Identifier fields parsed as strings (no numeric `val`)
+            // contribute to the column's distinct-count estimate
+            // instead, mirroring the regex/tabular paths.
+            lvs.add_text(string_fragment::from_bytes(str, len));
         }
     }
 
@@ -6206,6 +6260,18 @@ logline_value_stats::merge(const logline_value_stats& other)
         this->lvs_width = other.lvs_width;
     }
 
+    // Distinct-count merge runs before the lvs_count == 0 short-circuit
+    // because a text-only column has count == 0 but may still carry an
+    // HLL whose registers we need to fold in.
+    this->lvs_text_count += other.lvs_text_count;
+    if (other.lvs_distinct) {
+        if (this->lvs_distinct) {
+            this->lvs_distinct->merge(other.lvs_distinct.value());
+        } else {
+            this->lvs_distinct = other.lvs_distinct;
+        }
+    }
+
     if (other.lvs_count == 0) {
         return;
     }
@@ -6239,6 +6305,31 @@ logline_value_stats::add_value(double value)
     this->lvs_tdigest.insert(value);
 }
 
+void
+logline_value_stats::add_text(string_fragment sf)
+{
+    if (!this->lvs_distinct) {
+        this->lvs_distinct.emplace(12);
+    }
+    this->lvs_distinct->add(sf.data(), static_cast<uint32_t>(sf.length()));
+    this->lvs_text_count += 1;
+}
+
+std::optional<double>
+logline_value_stats::distinct_estimate() const
+{
+    if (!this->lvs_distinct) {
+        return std::nullopt;
+    }
+    return this->lvs_distinct->estimate();
+}
+
+void
+logline_value_stats::finalize()
+{
+    this->lvs_tdigest.merge();
+}
+
 std::vector<logline_value_meta>
 external_log_format::get_value_metadata() const
 {

src/log_format_fwd.hh

@@ -52,6 +52,7 @@
 #include "base/time_util.hh"
 #include "byte_array.hh"
 #include "digestible/digestible.h"
+#include "hyperloglog.hpp"
 #include "log_level.hh"
 #include "pcrepp/pcre2pp.hh"
 #include "robin_hood/robin_hood.h"
@@ -181,12 +182,32 @@ struct logline_value_stats {
 
     void add_value(double value);
 
+    // Feed a non-numeric cell into the per-column distinct-count
+    // estimator.  Lazy-initializes `lvs_distinct` on first call so
+    // numeric-only columns don't pay the ~4KB register-vector cost.
+    void add_text(string_fragment sf);
+
+    // HLL cardinality estimate for the column, or std::nullopt if no
+    // text cells were ever fed in.
+    std::optional<double> distinct_estimate() const;
+
+    // Compact the t-digest's pending buffer into ordered centroids so
+    // `lvs_tdigest.quantile(p)` returns accurate percentiles.  Call
+    // once after the per-file scan completes; no-op for repeated
+    // calls.
+    void finalize();
+
     int64_t lvs_width{0};
     int64_t lvs_count{0};
+    int64_t lvs_text_count{0};
     double lvs_total{0};
     double lvs_min_value{std::numeric_limits<double>::max()};
     double lvs_max_value{-std::numeric_limits<double>::max()};
     digestible::tdigest<double> lvs_tdigest{200};
+    // p=12 → ~4KB registers, ~1.6% standard error.  Right size for
+    // typical metrics text columns (HTTP methods, status codes,
+    // hostnames, op IDs all ≪ 2^52 — the post-XXH3 ceiling).
+    std::optional<hll::HyperLogLog> lvs_distinct;
 };
 
 struct pattern_for_lines {
@@ -624,6 +645,10 @@ struct logline_value_meta {
 
     chart_type_t to_chart_type() const;
 
+    std::string to_humanized_value(int64_t i) const;
+
+    std::string to_humanized_value(double d) const;
+
     intern_string_t lvm_name;
     value_kind_t lvm_kind;
     column_t lvm_column{external_column{}};
@@ -695,6 +720,8 @@ public:
 
     std::string to_string() const;
 
+    std::string to_humanized_string() const;
+
     string_fragment to_string_fragment(ArenaAlloc::Alloc<char>& alloc) const;
 
     const char* text_value() const;