[ui] report background tasks

Author: tstack

NEWS.md

@@ -2,11 +2,38 @@
 ## lnav v0.14.0
 
 Features:
-* Log formats can now specify source file/line and thread
-  ID with the `src-file-field`, `src-line-field`, and
-  `thread-id-field` properties. These fields can then be
-  accessed in the SQL vtables as `log_src_file`,
-  `log_src_line`, and `log_thread_id`.
+* Introducing "Log-Oriented Debugging", a collection of
+  features to streamline mapping log messages back to
+  the source code that generated them.  For example,
+  given the log message "Hello, Steve!" and the source
+  directory containing the log statement.  lnav can
+  find the line of code that generated the message,
+  such as `logging.info("Hello, %s!", name)`, and
+  determine the value of the substituted variables
+  (`name` => `Steve`). This functionality is
+  implemented using the
+  [log2src](https://github.com/ttiimm/log2src) project.
+  The following features have been added in support of
+  this functionality:
+  - The `:add-source-path` command was added to specify
+    the source directories to be scanned for log
+    statements.
+  - Log formats can now specify source file/line and
+    thread ID with the `src-file-field`, `src-line-field`,
+    and `thread-id-field` properties. These fields can
+    then be accessed in the SQL vtables as `log_src_file`,
+    `log_src_line`, and `log_thread_id`.
+  - The `:breakpoint`, `:toggle-breakpoint`, and
+    `:clear-breakpoints` commands have been added to
+    support setting/clearing breakpoints for log messages.
+    The `CTRL-B` shortcut toggles a breakpoint on the
+    focused line in the LOG view.  Also, if the log
+    format specifies source file/line fields, the first
+    character of the source file will be underlined and
+    can be clicked to toggle a breakpoint.  Once
+    breakpoints have been added, you can press `F7`/`F8`
+    to move to the previous/next log message that
+    matches a breakpoint.
 * The `all_opids` and `all_thread_ids` virtual tables
   have been added to make it simple to discover all of
   the operations and threads across all log files.
@@ -28,21 +55,17 @@ Features:
   - The `prefers` property is a regular expression that
     will be tested against the full path to be opened.
     If matched, that editor will be chosen.
-* The `:breakpoint`, `:toggle-breakpoint`, and
-  `:clear-breakpoints` commands have been added to support
-  setting/clearing breakpoints for log messages.  The
-  `CTRL-B` shortcut toggles a breakpoint on the focused
-  line in the LOG view.  Also, if the log format specifies
-  source file/line fields, the first character of the
-  source file will be underlined and can be clicked to
-  toggle a breakpoint.  Once breakpoints have been added,
-  you can press `F7`/`F8` to move to the previous/next log
-  message that matches a breakpoint.
 * The `:external-access` command has been added to open a
   localhost HTTP port that can be used to remotely control
   lnav. Requests can be sent to execute commands and poll
   for changes in the view state.
 
+Interface changes:
+* If there are background tasks, like the processing done
+  by `:add-source-path`, a panel with progress bars for
+  each operation will be shown just above the bottom
+  status bar.
+
 ## lnav v0.13.2
 
 Bug Fixes:

src/CMakeLists.txt

@@ -564,6 +564,7 @@ add_library(
         plain_text_source.cc
         pretty_printer.cc
         preview_status_source.cc
+        progress_source.cc
         pugixml/pugixml.cpp
         readline_callbacks.cc
         readline_highlighters.cc
@@ -701,6 +702,7 @@ add_library(
         plain_text_source.hh
         pretty_printer.hh
         preview_status_source.hh
+        progress_source.hh
         pugixml/pugiconfig.hpp
         pugixml/pugixml.hpp
         readline_callbacks.hh

src/Makefile.am

@@ -346,6 +346,7 @@ noinst_HEADERS = \
 	pollable.hh \
 	pretty_printer.hh \
 	preview_status_source.hh \
+	progress_source.hh \
 	ptimec.hh \
 	ptimec_spec.hh \
 	readline_callbacks.hh \
@@ -544,6 +545,7 @@ libdiag_a_SOURCES = \
 	pollable.cc \
 	pretty_printer.cc \
 	preview_status_source.cc \
+	progress_source.cc \
 	ptimec_rt.cc \
 	readline_callbacks.cc \
 	readline_highlighters.cc \

src/base/progress.hh

@@ -30,13 +30,62 @@
 #ifndef lnav_progress_hh
 #define lnav_progress_hh
 
+#include <cstdint>
+#include <string>
+#include <vector>
+
+#include "distributed_slice.hh"
+#include "lnav.console.hh"
+#include "safe/safe.h"
+
 namespace lnav {
 
-enum class progress_result_t {
+enum class progress_result_t : uint8_t {
     ok,
     interrupt,
 };
 
+enum class progress_status_t : uint8_t {
+    idle,
+    working,
+};
+
+struct task_progress {
+    std::string tp_id;
+    progress_status_t tp_status{progress_status_t::idle};
+    size_t tp_version{0};
+    std::string tp_step;
+    size_t tp_completed{0};
+    size_t tp_total{0};
+    std::vector<console::user_message> tp_messages;
+};
+
+using progress_reporter_t = task_progress(*)();
+
+struct progress_tracker {
+    using task_container = dist_slice_container<progress_reporter_t>;
+
+    using safe_task_container = safe::Safe<task_container*>;
+
+    static safe_task_container& get_tasks()
+    {
+        static auto pt = progress_tracker();
+
+        return pt.pt_tasks;
+    }
+
+private:
+    safe_task_container pt_tasks;
+
+    progress_tracker()
+    {
+        static auto inner
+            = DIST_SLICE_CONTAINER(progress_reporter_t, prog_reps);
+
+        *this->pt_tasks.writeAccess() = &inner;
+    }
+};
+
 }  // namespace lnav
 
 #endif

src/cmds.scripting.cc

@@ -477,8 +477,70 @@ com_sh(exec_context& ec, std::string cmdline, std::vector<std::string>& args)
 }
 
 #ifdef HAVE_RUST_DEPS
+
+static lnav::task_progress
+ext_prog_rep()
+{
+    auto ext = lnav_rs_ext::get_status();
+    auto status = ext.status == lnav_rs_ext::Status::idle
+        ? lnav::progress_status_t::idle
+        : lnav::progress_status_t::working;
+    std::vector<lnav::console::user_message> msgs_out;
+    for (const auto& err : ext.messages) {
+        auto um = lnav::console::user_message::error((std::string) err.error)
+                      .with_reason((std::string) err.source)
+                      .with_help((std::string) err.help);
+        msgs_out.emplace_back(um);
+    }
+    auto retval = lnav::task_progress{
+        (std::string) ext.id,
+        status,
+        ext.version,
+        (std::string) ext.current_step,
+        ext.completed,
+        ext.total,
+        std::move(msgs_out),
+    };
+
+    return retval;
+}
+
+DIST_SLICE(prog_reps) lnav::progress_reporter_t EXT_PROG_REP = ext_prog_rep;
+
 namespace lnav_rs_ext {
 
+LnavLogLevel
+get_lnav_log_level()
+{
+    switch (lnav_log_level) {
+        case lnav_log_level_t::TRACE:
+            return LnavLogLevel::trace;
+        case lnav_log_level_t::DEBUG:
+            return LnavLogLevel::debug;
+        case lnav_log_level_t::INFO:
+            return LnavLogLevel::info;
+        case lnav_log_level_t::WARNING:
+            return LnavLogLevel::warning;
+        case lnav_log_level_t::ERROR:
+            return LnavLogLevel::error;
+    }
+
+    return LnavLogLevel::info;
+}
+
+void
+log_msg(LnavLogLevel level, ::rust::Str file, uint32_t line, ::rust::Str msg)
+{
+    auto ln_level = static_cast<lnav_log_level_t>(level);
+
+    ::log_msg(ln_level,
+              ((std::string) file).c_str(),
+              line,
+              "%.*s",
+              msg.size(),
+              msg.data());
+}
+
 ::rust::String
 version_info()
 {

src/command_executor.cc

@@ -285,6 +285,7 @@ execute_sql(exec_context& ec, const std::string& sql, std::string& alt_msg)
         log_info("compiling PRQL: %s", stmt_str.c_str());
 
 #if HAVE_RUST_DEPS
+        extern rust::Vec<lnav_rs_ext::SourceTreeElement> sqlite_extension_prql;
         auto opts = lnav_rs_ext::Options{true, "sql.sqlite", true};
 
         auto tree = sqlite_extension_prql;

src/ext.longpoll.cc

@@ -32,6 +32,9 @@
 
 #include "ext.longpoll.hh"
 
+#include "base/injector.hh"
+#include "base/itertools.enumerate.hh"
+#include "base/progress.hh"
 #include "config.h"
 #include "lnav_rs_ext.cxx.hh"
 #include "safe/safe.h"
@@ -51,30 +54,119 @@ using safe_pollers_t = safe::Safe<pollers>;
 
 safe_pollers_t POLLERS;
 
-PollInput
+PollResult
 longpoll(const PollInput& pi)
 {
-    auto p = POLLERS.writeAccess<std::unique_lock>();
+    auto pi_retval = PollInput{
+        0,
+    };
+    auto timeout = 10000ms;
 
-    if (pi.view_states.log == p->p_latest_state.vs_log
-        && pi.view_states.text == p->p_latest_state.vs_text)
     {
-        p->p_pollers.emplace_front(pi);
-        auto iter = p->p_pollers.begin();
-
-        p->p_condvar.wait_for(p.lock, 10s);
-        p->p_pollers.erase(iter);
+        auto& bts = lnav::progress_tracker::get_tasks();
+
+        for (const auto& bt : **bts.readAccess()) {
+            auto tp = bt();
+            if (tp.tp_status == lnav::progress_status_t::working) {
+                timeout = 333ms;
+                break;
+            }
+        }
     }
 
-    return PollInput{
-        0,
-        ViewStates{
+    {
+        auto p = POLLERS.writeAccess<std::unique_lock>();
+        auto views_are_same = pi.view_states.log == p->p_latest_state.vs_log
+            && pi.view_states.text == p->p_latest_state.vs_text;
+        auto tasks_are_same = true;
+
+        {
+            auto& bts = lnav::progress_tracker::get_tasks();
+            auto* task_cont = *bts.readAccess();
+            tasks_are_same = pi.task_states.size() == task_cont->size();
+            if (tasks_are_same) {
+                for (size_t lpc = 0; lpc < pi.task_states.size(); lpc++) {
+                    if (lpc >= task_cont->size()) {
+                        continue;
+                    }
+
+                    auto tp = (*std::next(task_cont->begin(), lpc))();
+                    if (tp.tp_version != pi.task_states[lpc]) {
+                        tasks_are_same = false;
+                        break;
+                    }
+                }
+            }
+        }
+
+        if (views_are_same && tasks_are_same) {
+            p->p_pollers.emplace_front(pi);
+            auto iter = p->p_pollers.begin();
+
+            p->p_condvar.wait_for(p.lock, timeout);
+            p->p_pollers.erase(iter);
+        }
+        pi_retval.view_states = ViewStates{
             ::rust::String::lossy(p->p_latest_state.vs_log),
             ::rust::String::lossy(p->p_latest_state.vs_text),
-        },
+        };
+    }
+
+    ::rust::Vec<ExtProgress> bt_out;
+    {
+        auto& bts = lnav::progress_tracker::get_tasks();
+        for (const auto& [index, bt] :
+             lnav::itertools::enumerate(**bts.readAccess()))
+        {
+            auto tp = bt();
+            pi_retval.task_states.emplace_back(tp.tp_version);
+            if (tp.tp_version == 0) {
+                continue;
+            }
+            if (tp.tp_status == lnav::progress_status_t::idle
+                && index < pi.task_states.size()
+                && pi.task_states[index] == tp.tp_version)
+            {
+                continue;
+            }
+
+            ::rust::Vec<ExtError> errors_out;
+            for (const auto& msg : tp.tp_messages) {
+                errors_out.emplace_back(ExtError{
+                    msg.um_message.al_string,
+                    msg.um_reason.al_string,
+                    msg.um_help.al_string,
+                });
+            }
+
+            auto ep = ExtProgress{
+                tp.tp_id,
+                tp.tp_status == lnav::progress_status_t::idle ? Status::idle
+                                                              : Status::working,
+                tp.tp_version,
+                tp.tp_step,
+                tp.tp_completed,
+                tp.tp_total,
+            };
+            bt_out.emplace_back(ep);
+        }
+    }
+
+    return PollResult{
+        pi_retval,
+        std::move(bt_out),
     };
 }
 
+void
+notify_pollers()
+{
+#    ifdef HAVE_RUST_DEPS
+    auto p = POLLERS.writeAccess<std::unique_lock>();
+    p->p_condvar.notify_all();
+#    endif
+}
+
 }  // namespace lnav_rs_ext
 #endif
 

src/ext.longpoll.hh

@@ -39,6 +39,8 @@ struct view_states {
     std::string vs_text;
 };
 
+void notify_pollers();
+
 void notify_pollers(const view_states& vs);
 
 }  // namespace lnav::ext