Skip to content

Commit 01beb45

Browse files
tstacktstack
committed
[headless] make the warning for non-UTF files nicer
1 parent ee991c6 commit 01beb45

39 files changed

Lines changed: 1692 additions & 1651 deletions

File tree

‎src/base/attr_line.builder.cc‎

Lines changed: 59 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -30,54 +30,70 @@
3030
#include "attr_line.builder.hh"
3131

3232
attr_line_builder&
33-
attr_line_builder::append_as_hexdump(const string_fragment& sf)
33+
attr_line_builder::append_as_hexdump(const string_fragment& in)
3434
{
35-
auto byte_off = size_t{0};
36-
for (auto ch : sf) {
37-
if (byte_off == 8) {
38-
this->append(" ");
35+
auto first = true;
36+
auto remaining = in;
37+
while (!remaining.empty()) {
38+
if (first) {
39+
first = false;
40+
} else {
41+
this->append("\n");
3942
}
40-
std::optional<role_t> ro;
41-
if (ch == '\0') {
42-
ro = role_t::VCR_NULL;
43-
} else if (isspace(ch) || iscntrl(ch)) {
44-
ro = role_t::VCR_ASCII_CTRL;
45-
} else if (!isprint(ch)) {
46-
ro = role_t::VCR_NON_ASCII;
43+
44+
auto [sf, rest] = remaining.split_n(16);
45+
auto byte_off = size_t{0};
46+
for (auto ch : sf) {
47+
if (byte_off == 8) {
48+
this->append(" ");
49+
}
50+
std::optional<role_t> ro;
51+
if (ch == '\0') {
52+
ro = role_t::VCR_NULL;
53+
} else if (isspace(ch) || iscntrl(ch)) {
54+
ro = role_t::VCR_ASCII_CTRL;
55+
} else if (!isprint(ch)) {
56+
ro = role_t::VCR_NON_ASCII;
57+
}
58+
auto ag = ro.has_value()
59+
? this->with_attr(VC_ROLE.value(ro.value()))
60+
: this->with_default();
61+
this->appendf(FMT_STRING(" {:0>2x}"), ch);
62+
byte_off += 1;
4763
}
48-
auto ag = ro.has_value() ? this->with_attr(VC_ROLE.value(ro.value()))
49-
: this->with_default();
50-
this->appendf(FMT_STRING(" {:0>2x}"), ch);
51-
byte_off += 1;
52-
}
53-
for (; byte_off < 16; byte_off++) {
54-
if (byte_off == 8) {
55-
this->append(" ");
64+
for (; byte_off < 16; byte_off++) {
65+
if (byte_off == 8) {
66+
this->append(" ");
67+
}
68+
this->append(" ");
5669
}
57-
this->append(" ");
58-
}
59-
this->append(" ");
60-
byte_off = 0;
61-
for (auto ch : sf) {
62-
if (byte_off == 8) {
63-
this->append(" ");
70+
this->append(" ");
71+
byte_off = 0;
72+
for (auto ch : sf) {
73+
if (byte_off == 8) {
74+
this->append(" ");
75+
}
76+
if (ch == '\0') {
77+
auto ag = this->with_attr(VC_ROLE.value(role_t::VCR_NULL));
78+
this->append("\u22c4");
79+
} else if (isspace(ch)) {
80+
auto ag
81+
= this->with_attr(VC_ROLE.value(role_t::VCR_ASCII_CTRL));
82+
this->append("_");
83+
} else if (iscntrl(ch)) {
84+
auto ag
85+
= this->with_attr(VC_ROLE.value(role_t::VCR_ASCII_CTRL));
86+
this->append("\u2022");
87+
} else if (isprint(ch)) {
88+
this->alb_line.get_string().push_back(ch);
89+
} else {
90+
auto ag = this->with_attr(VC_ROLE.value(role_t::VCR_NON_ASCII));
91+
this->append("\u00d7");
92+
}
93+
byte_off += 1;
6494
}
65-
if (ch == '\0') {
66-
auto ag = this->with_attr(VC_ROLE.value(role_t::VCR_NULL));
67-
this->append("\u22c4");
68-
} else if (isspace(ch)) {
69-
auto ag = this->with_attr(VC_ROLE.value(role_t::VCR_ASCII_CTRL));
70-
this->append("_");
71-
} else if (iscntrl(ch)) {
72-
auto ag = this->with_attr(VC_ROLE.value(role_t::VCR_ASCII_CTRL));
73-
this->append("\u2022");
74-
} else if (isprint(ch)) {
75-
this->alb_line.get_string().push_back(ch);
76-
} else {
77-
auto ag = this->with_attr(VC_ROLE.value(role_t::VCR_NON_ASCII));
78-
this->append("\u00d7");
79-
}
80-
byte_off += 1;
95+
96+
remaining = rest;
8197
}
8298

8399
return *this;

‎src/base/attr_line.cc‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -324,29 +324,29 @@ consume(const string_fragment text)
324324
auto word_find_res
325325
= WORD_RE.find_in(text, PCRE2_NO_UTF_CHECK).ignore_error();
326326
if (word_find_res) {
327-
auto split_res = text.split_n(word_find_res->f_all.length()).value();
327+
auto split_res = text.split_n(word_find_res->f_all.length());
328328

329329
return word{split_res.first, split_res.second};
330330
}
331331

332332
if (isspace(text.front())) {
333-
auto split_res = text.split_n(1).value();
333+
auto split_res = text.split_n(1);
334334

335335
return space{split_res.first, split_res.second};
336336
}
337337

338338
auto space_find_res
339339
= SPACE_RE.find_in(text, PCRE2_NO_UTF_CHECK).ignore_error();
340340
if (space_find_res) {
341-
auto split_res = text.split_n(space_find_res->f_all.length()).value();
341+
auto split_res = text.split_n(space_find_res->f_all.length());
342342

343343
return space{split_res.first, split_res.second};
344344
}
345345

346346
auto next_char_byte_index = text.column_to_byte_index(1);
347347
auto split_res = text.split_n(next_char_byte_index);
348348

349-
return word{split_res->first, split_res->second};
349+
return word{split_res.first, split_res.second};
350350
}
351351

352352
} // namespace text_stream
@@ -463,8 +463,8 @@ attr_line_t::insert(size_t index,
463463
require_ge(text_to_wrap.length(), pre_len);
464464
auto pre_pair = text_to_wrap.split_n(pre_len);
465465
next_chunk = text_stream::word{
466-
pre_pair->first,
467-
pre_pair->second,
466+
pre_pair.first,
467+
pre_pair.second,
468468
};
469469
}
470470
if (!next_chunk.valid()) {

‎src/base/intern_string.cc‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -237,14 +237,12 @@ string_fragment::consume_n(int amount) const
237237
};
238238
}
239239

240-
string_fragment::split_result
240+
string_fragment::split_n_result
241241
string_fragment::split_n(int amount) const
242242
{
243-
if (amount > this->length()) {
244-
return std::nullopt;
245-
}
243+
amount = std::min(amount, this->length());
246244

247-
return std::make_pair(
245+
return {
248246
string_fragment{
249247
this->sf_string,
250248
this->sf_begin,
@@ -254,7 +252,8 @@ string_fragment::split_n(int amount) const
254252
this->sf_string,
255253
this->sf_begin + amount,
256254
this->sf_end,
257-
});
255+
},
256+
};
258257
}
259258

260259
std::vector<string_fragment>

‎src/base/intern_string.hh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -601,7 +601,9 @@ struct string_fragment {
601601
return std::nullopt;
602602
}
603603

604-
split_result split_n(int amount) const;
604+
using split_n_result = std::pair<string_fragment, string_fragment>;
605+
606+
split_n_result split_n(int amount) const;
605607

606608
std::vector<string_fragment> split_lines() const;
607609

‎src/base/lnav.console.cc‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -396,8 +396,10 @@ role_to_style(const role_t role,
396396
case role_t::VCR_HIDDEN:
397397
case role_t::VCR_WARNING:
398398
case role_t::VCR_RE_REPEAT:
399+
case role_t::VCR_NON_ASCII:
399400
line_style |= fmt::fg(fmt::terminal_color::yellow);
400401
break;
402+
case role_t::VCR_ASCII_CTRL:
401403
case role_t::VCR_COMMENT:
402404
case role_t::VCR_DIFF_ADD:
403405
line_style |= fmt::fg(fmt::terminal_color::green);
@@ -417,6 +419,7 @@ role_to_style(const role_t role,
417419
line_style
418420
|= fmt::emphasis::bold | fmt::fg(fmt::terminal_color::magenta);
419421
break;
422+
case role_t::VCR_NULL:
420423
case role_t::VCR_KEYWORD:
421424
case role_t::VCR_RE_SPECIAL:
422425
line_style

‎src/lnav.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4299,7 +4299,7 @@ SELECT tbl_name FROM sqlite_master WHERE sql LIKE 'CREATE VIRTUAL TABLE%'
42994299
}
43004300
}
43014301

4302-
for (auto& pair : cmd_results) {
4302+
for (const auto& pair : cmd_results) {
43034303
if (pair.first.isErr()) {
43044304
lnav::console::print(stderr, pair.first.unwrapErr());
43054305
output_view = false;

‎src/lnav.indexing.cc‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -479,9 +479,20 @@ rebuild_indexes(std::optional<ui_clock::time_point> deadline)
479479
void
480480
rebuild_indexes_repeatedly()
481481
{
482-
for (size_t attempt = 0; attempt < 10 && rebuild_indexes().rir_changes > 0;
483-
attempt++)
484-
{
482+
for (size_t attempt = 0; attempt < 50; attempt++) {
483+
auto rebuild_res = rebuild_indexes();
484+
if (!rebuild_res.rir_completed) {
485+
log_info("rebuilding indexes did not finish, retrying...");
486+
continue;
487+
}
488+
if (rebuild_res.rir_rescan_needed) {
489+
log_info("rebuilding indexes needs a rescan...");
490+
rescan_files(false);
491+
continue;
492+
}
493+
if (rebuild_res.rir_changes == 0) {
494+
break;
495+
}
485496
log_info("continuing to rebuild indexes...");
486497
}
487498
}

‎src/log_format.cc‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2083,8 +2083,12 @@ external_log_format::scan(logfile& lf,
20832083
const value_def& vd = *ivd.ivd_value_def;
20842084
auto num_cap = md[ivd.ivd_index];
20852085

2086+
if (vd.vd_meta.lvm_identifier) {
2087+
continue;
2088+
}
2089+
20862090
if (num_cap && num_cap->is_valid()) {
2087-
const struct scaling_factor* scaling = nullptr;
2091+
const scaling_factor* scaling = nullptr;
20882092

20892093
if (ivd.ivd_unit_field_index >= 0) {
20902094
auto unit_cap = md[ivd.ivd_unit_field_index];

‎src/logfile.cc‎

Lines changed: 28 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1544,6 +1544,18 @@ logfile::rebuild_index(std::optional<ui_clock::time_point> deadline)
15441544
}
15451545
prev_range = li.li_file_range;
15461546

1547+
auto read_result
1548+
= this->lf_line_buffer.read_range(li.li_file_range);
1549+
if (read_result.isErr()) {
1550+
log_error("%s:read failure -- %s",
1551+
this->lf_filename_as_string.c_str(),
1552+
read_result.unwrapErr().c_str());
1553+
this->close();
1554+
return rebuild_result_t::INVALID;
1555+
}
1556+
1557+
auto sbr = read_result.unwrap();
1558+
15471559
if (this->lf_format == nullptr
15481560
&& !this->lf_options.loo_non_utf_is_visible
15491561
&& !li.li_utf8_scan_result.is_valid())
@@ -1552,9 +1564,21 @@ logfile::rebuild_index(std::optional<ui_clock::time_point> deadline)
15521564
this->lf_filename_as_string.c_str());
15531565
this->lf_indexing = false;
15541566
this->lf_options.loo_is_visible = false;
1555-
auto utf8_error_um
1556-
= lnav::console::user_message::error("invalid UTF-8")
1557-
.with_reason(
1567+
attr_line_t hex;
1568+
attr_line_builder alb(hex);
1569+
alb.append_as_hexdump(sbr.to_string_fragment());
1570+
auto snip = lnav::console::snippet::from(
1571+
source_location{
1572+
intern_string::lookup(this->lf_filename),
1573+
(int) this->lf_index.size() + 1,
1574+
},
1575+
hex);
1576+
auto note_um
1577+
= lnav::console::user_message::warning(
1578+
attr_line_t("skipping indexing for ")
1579+
.append_quoted(this->lf_filename))
1580+
.with_reason("File contains invalid UTF-8")
1581+
.with_note(
15581582
attr_line_t(li.li_utf8_scan_result.usr_message)
15591583
.append(" at line ")
15601584
.append(lnav::roles::number(fmt::to_string(
@@ -1563,11 +1587,8 @@ logfile::rebuild_index(std::optional<ui_clock::time_point> deadline)
15631587
.append(lnav::roles::number(fmt::to_string(
15641588
li.li_utf8_scan_result.usr_valid_frag
15651589
.sf_end))))
1590+
.with_snippet(snip)
15661591
.move();
1567-
auto note_um = lnav::console::user_message::warning(
1568-
"skipping indexing for file")
1569-
.with_reason(utf8_error_um)
1570-
.move();
15711592
this->lf_notes.writeAccess()->insert(note_type::not_utf,
15721593
note_um);
15731594
if (this->lf_logfile_observer != nullptr) {
@@ -1633,18 +1654,6 @@ logfile::rebuild_index(std::optional<ui_clock::time_point> deadline)
16331654
}
16341655
}
16351656

1636-
auto read_result
1637-
= this->lf_line_buffer.read_range(li.li_file_range);
1638-
if (read_result.isErr()) {
1639-
log_error("%s:read failure -- %s",
1640-
this->lf_filename_as_string.c_str(),
1641-
read_result.unwrapErr().c_str());
1642-
this->close();
1643-
return rebuild_result_t::INVALID;
1644-
}
1645-
1646-
auto sbr = read_result.unwrap();
1647-
16481657
if (!li.li_utf8_scan_result.is_valid()) {
16491658
log_warning(
16501659
"%s: invalid UTF-8 detected at L%zu:C%d/%lld (O:%lld) -- "

‎src/textinput_curses.cc‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1358,11 +1358,8 @@ textinput_curses::handle_key(const ncinput& ch)
13581358
} else if (!this->tc_selection) {
13591359
const auto& al = this->tc_lines[this->tc_cursor.y];
13601360
auto line_sf = al.to_string_fragment();
1361-
const auto [before, after]
1362-
= line_sf
1363-
.split_n(
1364-
line_sf.column_to_byte_index(this->tc_cursor.x))
1365-
.value();
1361+
const auto [before, after] = line_sf.split_n(
1362+
line_sf.column_to_byte_index(this->tc_cursor.x));
13661363
auto match_opt = PREFIX_RE.capture_from(before)
13671364
.into(md)
13681365
.matches()
@@ -2118,11 +2115,11 @@ textinput_curses::do_update()
21182115
mvwattrline(this->tc_window, y, this->vc_x, al, lr);
21192116

21202117
if (!abort_msg_shown && this->tc_abort_requested) {
2121-
static auto REQ_MSG = attr_line_t(" Press ")
2122-
.append("Esc"_hotkey)
2123-
.append(" to abort ")
2124-
.with_attr_for_all(VC_ROLE.value(
2125-
role_t::VCR_STATUS));
2118+
static auto REQ_MSG
2119+
= attr_line_t(" Press ")
2120+
.append("Esc"_hotkey)
2121+
.append(" to abort ")
2122+
.with_attr_for_all(VC_ROLE.value(role_t::VCR_STATUS));
21262123
auto msg_lr = line_range{0, 0 + dim.dr_width};
21272124
mvwattrline(
21282125
this->tc_window,

0 commit comments

Comments
 (0)