[textfile] render invalid utf

Author: tstack

src/log_format_fwd.hh

@@ -487,7 +487,8 @@ struct subline_options {
                            const subline_options& rhs)
     {
         return lhs.full_message == rhs.full_message
-            && lhs.hash_hack == rhs.hash_hack;
+            && lhs.hash_hack == rhs.hash_hack
+            && lhs.scrub_invalid_utf8 == rhs.scrub_invalid_utf8;
     }
 
     friend bool operator!=(const subline_options& lhs,
@@ -498,6 +499,7 @@ struct subline_options {
 
     bool full_message{false};
     bool hash_hack{false};
+    bool scrub_invalid_utf8{true};
 };
 
 enum class value_kind_t : int {

src/logfile.cc

@@ -1983,7 +1983,9 @@ logfile::read_line(iterator ll, subline_options opts)
         return this->lf_line_buffer.read_range(get_range_res)
             .map([&ll, &get_range_res, &opts, this](auto sbr) {
                 sbr.rtrim(is_line_ending);
-                if (!get_range_res.fr_metadata.m_valid_utf) {
+                if (opts.scrub_invalid_utf8
+                    && !get_range_res.fr_metadata.m_valid_utf)
+                {
                     scrub_to_utf8(sbr.get_writable_data(), sbr.length());
                     sbr.get_metadata().m_valid_utf = true;
                 }

src/logfile.hh

@@ -286,7 +286,7 @@ public:
 
     bool is_closed() const { return this->lf_is_closed; }
 
-    struct timeval original_line_time(iterator ll);
+    timeval original_line_time(iterator ll);
 
     Result<shared_buffer_ref, std::string> read_line(iterator ll,
                                                      subline_options opts = {});

src/textfile_sub_source.cc

@@ -173,13 +173,16 @@ textfile_sub_source::text_value_for_line(textview_curses& tc,
     }
 
     const auto ll = lf->begin() + lfo->lfo_filter_state.tfs_index[line];
-    auto read_result = lf->read_line(ll);
+    auto read_opts = subline_options{};
+    read_opts.scrub_invalid_utf8 = false;
+    auto read_result = lf->read_line(ll, read_opts);
     this->tss_line_indent_size = 0;
     this->tss_plain_line_attrs.clear();
     if (read_result.isOk()) {
         auto sbr = read_result.unwrap();
         value_out = to_string(sbr);
-        if (sbr.get_metadata().m_has_ansi) {
+        const auto& meta = sbr.get_metadata();
+        if (meta.m_valid_utf && meta.m_has_ansi) {
             scrub_ansi_string(value_out, &this->tss_plain_line_attrs);
         }
         for (const auto& ch : value_out) {
@@ -852,7 +855,9 @@ textfile_sub_source::rescan_files(textfile_sub_source::scan_callback& callback,
                                 "discovery",
                                 lf->get_path_for_key().c_str());
                             iter->fvs_mtime = st.st_mtime;
-                            iter->fvs_file_size = lf->get_index_size();
+                            iter->fvs_file_size = st.st_size;
+                            iter->fvs_file_indexed_size = lf->get_index_size();
+                            iter->fvs_error = "skipping meta discovery";
                         } else {
                             auto content
                                 = attr_line_t(read_file_res.rfr_content);

src/view_curses.cc

@@ -50,6 +50,7 @@
 #include "shlex.hh"
 #include "terminfo-files.h"
 #include "terminfo/terminfo.h"
+#include "unistr.h"
 #include "uniwidth.h"
 #include "xterm_mouse.hh"
 
@@ -365,25 +366,29 @@ view_curses::mvwattrline(ncplane* window,
                     break;
                 }
 
-                auto exp_read_start = expanded_line.size();
+                if (ch <= 0x7f) {
+                    expanded_line.push_back(ch);
+                    curr_ch_col_count = 1;
+                    char_index += 1;
+                    lpc += 1;
+                    break;
+                }
+
                 auto lpc_start = lpc;
-                auto read_res
-                    = ww898::utf::utf8::read([&line, &expanded_line, &lpc] {
-                          auto ch = line[lpc++];
-                          expanded_line.push_back(ch);
-                          return ch;
-                      });
-
-                if (read_res.isErr()) {
-                    log_trace(
-                        "error:%d:%zu:%s", y, x + lpc, read_res.unwrapErr());
-                    expanded_line.resize(exp_read_start);
-                    expanded_line.push_back('?');
+                ucs4_t wch;
+                auto read_res = u8_mbtoucr(
+                    &wch, (uint8_t*) line.data() + lpc, line.size() - lpc);
+                if (read_res <= 0) {
+                    expanded_line.append("\ufffd");
+                    sa.emplace_back(line_range{(int) lpc, (int) lpc + 1},
+                                    VC_ROLE.value(role_t::VCR_NON_ASCII));
                     curr_ch_col_count = 1;
                     char_index += 1;
-                    lpc = lpc_start + 1;
+                    lpc += 1;
                 } else {
-                    auto wch = read_res.unwrap();
+                    for (size_t i = 0; i < read_res; i++) {
+                        expanded_line.push_back(line[lpc++]);
+                    }
                     if (wch == L'\u200d') {
                         join_start_index = char_index - last_ch_col_count;
                         continue;