Skip to content

Commit 577b946

Browse files
tstacktstack
committed
[ansi_scrubber] inform the user of unsupported escape sequences
1 parent e826ca0 commit 577b946

20 files changed

Lines changed: 226 additions & 37 deletions

‎NEWS.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,11 @@ Bug Fixes:
4040
read and crash in some situations.
4141
* Fix a lockup when viewing a file that contained log
4242
messages and lots of binary data.
43+
* More hardening for some diabolical inputs:
44+
- Unsupported escape-sequences were ignored before,
45+
but they are displayed now.
46+
- Checks for archives with file paths that could
47+
escape containment.
4348

4449

4550
## lnav v0.14.0

‎src/base/ansi_scrubber.cc‎

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,11 +40,20 @@
4040
#include "pcrepp/pcre2pp.hh"
4141
#include "scn/scan.h"
4242

43+
using std::string_literals::operator""s;
44+
4345
static const lnav::pcre2pp::code&
4446
ansi_regex()
4547
{
4648
static const auto retval = lnav::pcre2pp::code::from_const(
47-
R"(\x00|\x1b\[([\d=;:\?]*)([a-zA-Z])|\x1b\](\d+);(.*?)(?:\x07|\x1b\\)|(?:\X\x08\X)+|(\x16+))");
49+
R"(
50+
\x00 # NUL
51+
| \x1b \[ ([\d=;:\?]*) ([a-zA-Z]) # CSI: ESC [ <params> <final>
52+
| \x1b \] (\d+) ; (.*?) (?:\x07 | \x1b\\) # OSC: ESC ] <code> ; <payload> (BEL|ST)
53+
| (?: \X \x08 \X )+ # Overstrike: glyph BS glyph
54+
| (\x16+) # SYN run
55+
)",
56+
PCRE2_EXTENDED);
4857

4958
return retval;
5059
}
@@ -290,7 +299,7 @@ scrub_ansi_string(std::string& str, string_attrs_t* sa)
290299

291300
if (osc_id) {
292301
switch (osc_id->value()) {
293-
case 8:
302+
case 8: {
294303
auto split_res = md[4]->split_pair(semi_pred);
295304
if (split_res) {
296305
// auto params = split_res->first;
@@ -313,6 +322,20 @@ scrub_ansi_string(std::string& str, string_attrs_t* sa)
313322
}
314323
}
315324
break;
325+
}
326+
default: {
327+
if (sa != nullptr) {
328+
lr.lr_start = cp_dst;
329+
lr.lr_end = cp_dst;
330+
tmp_sa.emplace_back(
331+
lr,
332+
SA_UNSUPPORTED.value(fmt::format(
333+
FMT_STRING("ANSI sequence: OSC {} {}"),
334+
md[3],
335+
md[4])));
336+
}
337+
break;
338+
}
316339
}
317340
}
318341
} else if (md[1]) {
@@ -468,6 +491,19 @@ scrub_ansi_string(std::string& str, string_attrs_t* sa)
468491
}
469492
break;
470493
}
494+
default: {
495+
if (sa != nullptr) {
496+
lr.lr_start = cp_dst;
497+
lr.lr_end = cp_dst;
498+
tmp_sa.emplace_back(
499+
lr,
500+
SA_UNSUPPORTED.value(fmt::format(
501+
FMT_STRING("ANSI sequence: ESC [ {} {}"),
502+
seq,
503+
terminator)));
504+
}
505+
break;
506+
}
471507
}
472508
}
473509
if (md[1] || md[3] || md[5]) {
@@ -485,6 +521,7 @@ scrub_ansi_string(std::string& str, string_attrs_t* sa)
485521
sa.sa_range.lr_end = cp_dst;
486522
if (sa.sa_range.empty()) {
487523
sa.sa_type = &SA_INVALID;
524+
sa.sa_value = "zero-length attribute"s;
488525
}
489526
}
490527
if (sf.sf_end < str.size()) {

‎src/base/intern_string.hh‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1023,6 +1023,20 @@ struct formatter<std::error_code> : formatter<string_view> {
10231023
return formatter<string_view>::format(ec.message(), ctx);
10241024
}
10251025
};
1026+
1027+
template<typename I>
1028+
struct formatter<std::optional<I>> : formatter<I> {
1029+
template<typename FormatContext>
1030+
auto format(const std::optional<I>& opt, FormatContext& ctx) const
1031+
{
1032+
if (!opt) {
1033+
return formatter<string_view>::format("\u2205", ctx);
1034+
}
1035+
1036+
return formatter<I>::format(opt.value(), ctx);
1037+
}
1038+
};
1039+
10261040
} // namespace fmt
10271041

10281042
namespace std {

‎src/base/lnav.console.cc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -633,6 +633,8 @@ println(FILE* file, const attr_line_t& al)
633633
if (color_opt) {
634634
fg_style = fmt::fg(color_opt.value());
635635
}
636+
} else if (attr.sa_type == &SA_UNSUPPORTED) {
637+
line_style |= fmt::fg(fmt::terminal_color::yellow);
636638
} else if (attr.sa_type == &VC_STYLE) {
637639
auto saw = string_attr_wrapper<text_attrs>(&attr);
638640
auto style = saw.get();

‎src/base/string_attr_type.cc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ constexpr string_attr_type<void> SA_REMOVED("removed");
4646
constexpr string_attr_type<void> SA_PREFORMATTED("preformatted");
4747
constexpr string_attr_type<std::string> SA_INVALID("invalid");
4848
constexpr string_attr_type<std::string> SA_ERROR("error");
49+
constexpr string_attr_type<std::string> SA_UNSUPPORTED("unsupported");
4950
constexpr string_attr_type<int64_t> SA_LEVEL("level");
5051
constexpr string_attr_type<int64_t> SA_ORIGIN_OFFSET("origin-offset");
5152
constexpr string_attr_type<text_format_t> SA_QUOTED_TEXT("quoted-text");

‎src/base/string_attr_type.hh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -438,6 +438,7 @@ extern const string_attr_type<void> SA_REMOVED;
438438
extern const string_attr_type<void> SA_PREFORMATTED;
439439
extern const string_attr_type<std::string> SA_INVALID;
440440
extern const string_attr_type<std::string> SA_ERROR;
441+
extern const string_attr_type<std::string> SA_UNSUPPORTED;
441442
extern const string_attr_type<int64_t> SA_LEVEL;
442443
extern const string_attr_type<int64_t> SA_ORIGIN_OFFSET;
443444
extern const string_attr_type<text_format_t> SA_QUOTED_TEXT;

‎src/field_overlay_source.cc‎

Lines changed: 49 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ field_overlay_source::build_field_lines(const listview_curses& lv,
8080
bool display = false;
8181

8282
if (ll->is_time_skewed()
83-
|| ll->get_msg_level() == log_level_t::LEVEL_INVALID)
83+
|| ll->get_msg_level() == log_level_t::LEVEL_INVALID || ll->has_ansi())
8484
{
8585
display = true;
8686
}
@@ -124,7 +124,7 @@ field_overlay_source::build_field_lines(const listview_curses& lv,
124124
}
125125

126126
if (ll->get_msg_level() == LEVEL_INVALID) {
127-
for (const auto& sattr : this->fos_log_helper.ldh_line_attrs) {
127+
for (const auto& sattr : this->fos_log_helper.ldh_attr_line.al_attrs) {
128128
if (sattr.sa_type != &SA_INVALID) {
129129
continue;
130130
}
@@ -165,6 +165,36 @@ field_overlay_source::build_field_lines(const listview_curses& lv,
165165
curr_timestamp[ts_len] = '\0';
166166
}
167167

168+
auto first_nl = this->fos_log_helper.ldh_attr_line.al_string.find('\n');
169+
for (const auto& attr : this->fos_log_helper.ldh_attr_line.al_attrs) {
170+
if (attr.sa_type != &SA_UNSUPPORTED) {
171+
continue;
172+
}
173+
if (first_nl != std::string::npos && attr.sa_range.lr_start >= first_nl)
174+
{
175+
continue;
176+
}
177+
178+
const auto& msg
179+
= attr.sa_value.get<decltype(SA_UNSUPPORTED)::value_type>();
180+
auto msg_al
181+
= attr_line_t()
182+
.pad_to(this->fos_lss.get_filename_offset()
183+
+ attr.sa_range.lr_start)
184+
.append(ui_icon_t::warning)
185+
.append(" ")
186+
.append("Unsupported:"_h2)
187+
.append(" ")
188+
.append(msg)
189+
.with_attr_for_all(VC_ROLE.value(role_t::VCR_WARNING));
190+
this->fos_lines.emplace_back(msg_al);
191+
}
192+
193+
if (ll->is_continued()) {
194+
// only need to display unsupported escapes for a continued line
195+
return;
196+
}
197+
168198
if (ll->is_time_skewed()) {
169199
time_lr.lr_start = 1;
170200
time_lr.lr_end = 2;
@@ -275,7 +305,9 @@ field_overlay_source::build_field_lines(const listview_curses& lv,
275305
this->fos_lines.emplace_back(time_line);
276306
}
277307

278-
if (this->fos_contexts.empty() || !this->fos_contexts.top().c_show) {
308+
if (this->fos_contexts.empty() || !this->fos_contexts.top().c_show
309+
|| row != lv.get_selection())
310+
{
279311
return;
280312
}
281313

@@ -352,7 +384,9 @@ field_overlay_source::build_field_lines(const listview_curses& lv,
352384
pattern_al,
353385
pattern_al.length(),
354386
line_range{skip, (int) pattern_al.length()});
355-
this->fos_lines.emplace_back(pattern_al);
387+
for (const auto& pattern_line_al : pattern_al.split_lines()) {
388+
this->fos_lines.emplace_back(pattern_line_al);
389+
}
356390
}
357391

358392
if (this->fos_log_helper.ldh_line_values.lvv_opid_value) {
@@ -939,8 +973,10 @@ field_overlay_source::list_value_for_overlay(
939973
vis_line_t row,
940974
std::vector<attr_line_t>& value_out)
941975
{
942-
// log_debug("value for overlay %d", row);
943-
if (row == lv.get_selection()) {
976+
auto line_pair_opt = this->fos_lss.find_line_with_file(row);
977+
if (row == lv.get_selection()
978+
|| (line_pair_opt && line_pair_opt->second->has_ansi()))
979+
{
944980
this->build_field_lines(lv, row);
945981
value_out = this->fos_lines;
946982
}
@@ -1211,22 +1247,22 @@ field_overlay_source::list_header_for_overlay(const listview_curses& lv,
12111247
attr_line_t retval;
12121248

12131249
retval.append(this->fos_lss.get_filename_offset(), ' ');
1214-
if (this->fos_contexts.top().c_show) {
1250+
if (lv.get_selection() == vl && this->fos_contexts.top().c_show) {
12151251
retval.appendf(FMT_STRING("\u258C Line {:L} parser details."),
12161252
(int) vl);
12171253
if (media == media_t::display) {
12181254
retval.append(" Press ")
12191255
.append("p"_hotkey)
12201256
.append(" to hide this panel.");
12211257
}
1222-
} else {
1258+
} else if (this->fos_lss.find_bookmark_metadata(vl)) {
12231259
retval.append("\u258C Line ")
12241260
.append(
12251261
lnav::roles::number(fmt::format(FMT_STRING("{:L}"), (int) vl)))
12261262
.append(" metadata");
12271263
}
12281264

1229-
if (media == media_t::display) {
1265+
if (!retval.empty() && media == media_t::display) {
12301266
if (lv.get_overlay_selection()) {
12311267
retval.append(" ")
12321268
.append("SPC"_hotkey)
@@ -1242,5 +1278,9 @@ field_overlay_source::list_header_for_overlay(const listview_curses& lv,
12421278
}
12431279
}
12441280

1281+
if (retval.empty()) {
1282+
return std::nullopt;
1283+
}
1284+
12451285
return retval;
12461286
}

‎src/formats/block_log.json‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,13 @@
55
"description": "A generic format for logs, like cron, that have a date at the start of a block.",
66
"regex": {
77
"std": {
8-
"pattern": "^(?<timestamp>\\S{3,8} \\w{3}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2} \\w+ \\d{4})\\s*(?<body>.*)$"
8+
"pattern": "^(?<timestamp>\\S{3,8} \\w{3}\\s+\\d{1,2} \\d{2}:\\d{2}:\\d{2} \\w+ \\d{4})\\s*(?:\\n(?<body>.*))?$"
99
},
1010
"sq-brackets": {
11-
"pattern": "^\\[(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3,6})?(?:Z|[-+]\\d{2}:?\\d{2})?)\\]\\s*(?<body>.*)$"
11+
"pattern": "^\\[(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3,6})?(?:Z|[-+]\\d{2}:?\\d{2})?)\\]\\s*(?:\\n(?<body>.*))?$"
12+
},
13+
"line-separator": {
14+
"pattern": "^(?<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{3,6})?(?:Z|[-+]\\d{2}:?\\d{2})?)\\s*(?:-+|=+)?(?:\\n(?<body>.*))?$"
1215
}
1316
},
1417
"sample": [
@@ -17,6 +20,9 @@
1720
},
1821
{
1922
"line": "[2021-05-21T21:58:57.022497Z]"
23+
},
24+
{
25+
"line": "2026-04-27T13:51:22+00:00 ============================================================="
2026
}
2127
]
2228
}

‎src/log_data_helper.cc‎

Lines changed: 42 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@
3333

3434
#include "log_data_helper.hh"
3535

36+
#include "base/ansi_scrubber.hh"
37+
#include "base/attr_line.builder.hh"
3638
#include "config.h"
3739
#include "lnav_util.hh"
3840
#include "logfile.hh"
@@ -56,7 +58,8 @@ log_data_helper::clear()
5658
this->ldh_extra_json.clear();
5759
this->ldh_json_pairs.clear();
5860
this->ldh_xml_pairs.clear();
59-
this->ldh_line_attrs.clear();
61+
this->ldh_share_manager.invalidate_refs();
62+
this->ldh_attr_line.clear();
6063
}
6164

6265
bool
@@ -75,36 +78,56 @@ log_data_helper::load_line(content_line_t line, bool allow_middle)
7578
}
7679
this->ldh_line = ll;
7780
if (!ll->is_message()) {
81+
this->ldh_share_manager.invalidate_refs();
7882
this->ldh_parser.reset();
7983
this->ldh_scanner.reset();
8084
this->ldh_namer.reset();
8185
this->ldh_extra_json.clear();
8286
this->ldh_json_pairs.clear();
8387
this->ldh_xml_pairs.clear();
84-
this->ldh_line_attrs.clear();
88+
this->ldh_attr_line.clear();
8589
this->ldh_msg_format.clear();
8690
#ifdef HAVE_RUST_DEPS
8791
this->ldh_src_ref = std::nullopt;
8892
this->ldh_src_vars.clear();
8993
#endif
94+
95+
if (ll->is_continued() && ll->has_ansi() && ll->is_valid_utf()) {
96+
auto read_res = this->ldh_file->read_line(ll);
97+
if (read_res.isOk()) {
98+
this->ldh_attr_line.al_string = to_string(read_res.unwrap());
99+
scrub_ansi_string(this->ldh_attr_line.al_string,
100+
&this->ldh_attr_line.al_attrs);
101+
retval = true;
102+
}
103+
}
90104
} else {
91105
auto format = this->ldh_file->get_format();
92-
auto& sa = this->ldh_line_attrs;
93106

94107
#ifdef HAVE_RUST_DEPS
95108
this->ldh_src_ref = std::nullopt;
96109
this->ldh_src_vars.clear();
97110
#endif
111+
this->ldh_share_manager.invalidate_refs();
98112
this->ldh_parser.reset();
99113
this->ldh_scanner.reset();
100114
this->ldh_namer.reset();
101-
this->ldh_line_attrs.clear();
115+
this->ldh_attr_line.clear();
102116
this->ldh_line_values.clear();
103-
this->ldh_file->read_full_message(ll, this->ldh_line_values.lvv_sbr);
104-
this->ldh_line_values.lvv_sbr.erase_ansi();
117+
auto& sbr = this->ldh_line_values.lvv_sbr;
118+
this->ldh_file->read_full_message(ll, sbr);
119+
auto& sbr_meta = sbr.get_metadata();
120+
this->ldh_attr_line.al_string = to_string(sbr);
121+
if (sbr_meta.m_valid_utf && sbr_meta.m_has_ansi) {
122+
scrub_ansi_string(this->ldh_attr_line.al_string,
123+
&this->ldh_attr_line.al_attrs);
124+
sbr.share(this->ldh_share_manager,
125+
this->ldh_attr_line.al_string.data(),
126+
this->ldh_attr_line.al_string.size());
127+
}
105128
format->annotate(this->ldh_file.get(),
106129
this->ldh_line_index,
107-
sa,
130+
this->ldh_attr_line.al_attrs,
108131
this->ldh_line_values);
109132

110133
this->ldh_extra_json.clear();
@@ -190,6 +213,17 @@ log_data_helper::load_line(content_line_t line, bool allow_middle)
190213
retval = true;
191214
}
192215

216+
if (retval) {
217+
for (auto& sa : this->ldh_attr_line.al_attrs) {
218+
if (sa.sa_type != &VC_HYPERLINK) {
219+
continue;
220+
}
221+
sa.sa_type = &SA_UNSUPPORTED;
222+
sa.sa_value = fmt::format(FMT_STRING("hyperlink <{}>"),
223+
sa.sa_value.get<std::string>());
224+
}
225+
}
226+
193227
return retval;
194228
}
195229

@@ -205,7 +239,7 @@ log_data_helper::parse_body()
205239
}
206240

207241
auto& sbr = this->ldh_line_values.lvv_sbr;
208-
auto& sa = this->ldh_line_attrs;
242+
auto& sa = this->ldh_attr_line.al_attrs;
209243
auto body = find_string_attr_range(sa, &SA_BODY);
210244
if (body.lr_start == -1) {
211245
body.lr_start = this->ldh_line_values.lvv_sbr.length();

0 commit comments

Comments
 (0)