Skip to content

Commit e6ee476

Browse files
tstacktstack
committed
[logfile] cache conversion of path to string
1 parent ad8997a commit e6ee476

2 files changed

Lines changed: 57 additions & 43 deletions

File tree

‎src/logfile.cc‎

Lines changed: 45 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ logfile::open(std::filesystem::path filename,
135135
(int) lf_fd,
136136
(long long) lf->lf_stat.st_size,
137137
(long long) lf->lf_stat.st_mtime,
138-
lf->lf_filename.c_str());
138+
lf->lf_filename_as_string.c_str());
139139
if (lf->lf_actual_path) {
140140
log_info(" actual_path=%s", lf->lf_actual_path->c_str());
141141
}
@@ -155,7 +155,9 @@ logfile::open(std::filesystem::path filename,
155155

156156
const auto& hdr = lf->lf_line_buffer.get_header_data();
157157
if (hdr.valid()) {
158-
log_info("%s: has header %d", lf->lf_filename.c_str(), hdr.valid());
158+
log_info("%s: has header %d",
159+
lf->lf_filename_as_string.c_str(),
160+
hdr.valid());
159161
hdr.match(
160162
[&lf](const lnav::gzip::header& gzhdr) {
161163
if (!gzhdr.empty()) {
@@ -212,7 +214,7 @@ logfile::open(std::filesystem::path filename,
212214
}
213215

214216
lf->file_options_have_changed();
215-
lf->lf_content_id = hasher().update(lf->lf_filename).to_string();
217+
lf->lf_content_id = hasher().update(lf->lf_filename_as_string).to_string();
216218

217219
ensure(lf->invariant());
218220

@@ -221,14 +223,15 @@ logfile::open(std::filesystem::path filename,
221223

222224
logfile::logfile(std::filesystem::path filename,
223225
const logfile_open_options& loo)
224-
: lf_filename(std::move(filename)), lf_options(loo)
226+
: lf_filename(std::move(filename)),
227+
lf_filename_as_string(lf_filename.string()), lf_options(loo)
225228
{
226229
this->lf_opids.writeAccess()->los_opid_ranges.reserve(64);
227230
}
228231

229232
logfile::~logfile()
230233
{
231-
log_info("destructing logfile: %s", this->lf_filename.c_str());
234+
log_info("destructing logfile: %s", this->lf_filename_as_string.c_str());
232235
}
233236

234237
bool
@@ -256,7 +259,8 @@ logfile::file_options_have_changed()
256259
}
257260

258261
this->lf_file_options = new_options;
259-
log_info("%s: file options have changed", this->lf_filename.c_str());
262+
log_info("%s: file options have changed",
263+
this->lf_filename_as_string.c_str());
260264
if (this->lf_file_options) {
261265
log_info(
262266
" tz=%s",
@@ -373,7 +377,7 @@ logfile::process_prefix(shared_buffer_ref& sbr,
373377
* tend to be sufficiently different that there are few ambiguities...
374378
*/
375379
log_trace("logfile[%s]: scanning line %d (offset: %d; size: %d)",
376-
this->lf_filename.c_str(),
380+
this->lf_filename_as_string.c_str(),
377381
this->lf_index.size(),
378382
li.li_file_range.fr_offset,
379383
li.li_file_range.fr_size);
@@ -391,13 +395,13 @@ logfile::process_prefix(shared_buffer_ref& sbr,
391395
continue;
392396
}
393397

394-
auto match_res = curr->match_name(this->lf_filename);
398+
auto match_res = curr->match_name(this->lf_filename_as_string);
395399
if (match_res.is<log_format::name_mismatched>()) {
396400
auto nm = match_res.get<log_format::name_mismatched>();
397401
if (li.li_file_range.fr_offset == 0) {
398402
log_debug("(%s) does not match file name: %s",
399403
curr->get_name().get(),
400-
this->lf_filename.c_str());
404+
this->lf_filename_as_string.c_str());
401405
}
402406
auto regex_al = attr_line_t(nm.nm_pattern);
403407
lnav::snippets::regex_highlighter(
@@ -539,7 +543,7 @@ logfile::process_prefix(shared_buffer_ref& sbr,
539543

540544
if (!scan_count) {
541545
log_info("%s: no formats available to scan, no longer detecting",
542-
this->lf_filename.c_str());
546+
this->lf_filename_as_string.c_str());
543547
this->lf_options.loo_detect_format = false;
544548
}
545549

@@ -552,7 +556,7 @@ logfile::process_prefix(shared_buffer_ref& sbr,
552556
auto winner = best_match.value();
553557
auto* curr = winner.first;
554558
log_info("%s:%d:log format found -- %s",
555-
this->lf_filename.c_str(),
559+
this->lf_filename_as_string.c_str(),
556560
this->lf_index.size(),
557561
curr->get_name().get());
558562

@@ -577,7 +581,7 @@ logfile::process_prefix(shared_buffer_ref& sbr,
577581
for (auto& td_pair : this->lf_format->lf_tag_defs) {
578582
bool matches = td_pair.second->ftd_paths.empty();
579583
for (const auto& pr : td_pair.second->ftd_paths) {
580-
if (pr.matches(this->lf_filename.c_str())) {
584+
if (pr.matches(this->lf_filename_as_string.c_str())) {
581585
matches = true;
582586
break;
583587
}
@@ -587,7 +591,7 @@ logfile::process_prefix(shared_buffer_ref& sbr,
587591
}
588592

589593
log_info("%s: found applicable tag definition /%s/tags/%s",
590-
this->lf_filename.c_str(),
594+
this->lf_filename_as_string.c_str(),
591595
this->lf_format->get_name().get(),
592596
td_pair.second->ftd_name.c_str());
593597
this->lf_applicable_taggers.emplace_back(td_pair.second);
@@ -597,7 +601,7 @@ logfile::process_prefix(shared_buffer_ref& sbr,
597601
for (auto& pd_pair : this->lf_format->lf_partition_defs) {
598602
bool matches = pd_pair.second->fpd_paths.empty();
599603
for (const auto& pr : pd_pair.second->fpd_paths) {
600-
if (pr.matches(this->lf_filename.c_str())) {
604+
if (pr.matches(this->lf_filename_as_string.c_str())) {
601605
matches = true;
602606
break;
603607
}
@@ -609,7 +613,7 @@ logfile::process_prefix(shared_buffer_ref& sbr,
609613
log_info(
610614
"%s: found applicable partition definition "
611615
"/%s/partitions/%s",
612-
this->lf_filename.c_str(),
616+
this->lf_filename_as_string.c_str(),
613617
this->lf_format->get_name().get(),
614618
pd_pair.second->fpd_name.c_str());
615619
this->lf_applicable_partitioners.emplace_back(pd_pair.second);
@@ -786,7 +790,7 @@ logfile::rebuild_index(std::optional<ui_clock::time_point> deadline)
786790
|| this->lf_format->format_changed())))
787791
{
788792
log_info("%s: format has changed, rebuilding",
789-
this->lf_filename.c_str());
793+
this->lf_filename_as_string.c_str());
790794
this->lf_index.clear();
791795
this->lf_index_size = 0;
792796
this->lf_partial_line = false;
@@ -841,7 +845,7 @@ logfile::rebuild_index(std::optional<ui_clock::time_point> deadline)
841845

842846
log_info(
843847
"%s: overwrite content_id double check: old:%s; now:%s",
844-
this->lf_filename.c_str(),
848+
this->lf_filename_as_string.c_str(),
845849
this->lf_content_id.c_str(),
846850
curr_content_id.c_str());
847851
if (this->lf_content_id == curr_content_id) {
@@ -857,7 +861,7 @@ logfile::rebuild_index(std::optional<ui_clock::time_point> deadline)
857861
if (is_truncated || is_overwritten) {
858862
log_info("overwritten file detected, closing -- %s new: %" PRId64
859863
"/%" PRId64 " old: %" PRId64 "/%" PRId64,
860-
this->lf_filename.c_str(),
864+
this->lf_filename_as_string.c_str(),
861865
st.st_size,
862866
st.st_mtime,
863867
this->lf_stat.st_size,
@@ -890,7 +894,8 @@ logfile::rebuild_index(std::optional<ui_clock::time_point> deadline)
890894
}
891895

892896
if (begin_size == 0 && !has_format) {
893-
log_debug("scanning file... %s", this->lf_filename.c_str());
897+
log_debug("scanning file... %s",
898+
this->lf_filename_as_string.c_str());
894899
}
895900

896901
if (!this->lf_index.empty()) {
@@ -923,7 +928,7 @@ logfile::rebuild_index(std::optional<ui_clock::time_point> deadline)
923928
if (read_result.isErr()) {
924929
log_info(
925930
"overwritten file detected, closing -- %s (%s)",
926-
this->lf_filename.c_str(),
931+
this->lf_filename_as_string.c_str(),
927932
read_result.unwrapErr().c_str());
928933
this->close();
929934
return rebuild_result_t::INVALID;
@@ -932,7 +937,7 @@ logfile::rebuild_index(std::optional<ui_clock::time_point> deadline)
932937
auto sbr = read_result.unwrap();
933938
if (!sbr.to_string_fragment().endswith("\n")) {
934939
log_info("overwritten file detected, closing -- %s",
935-
this->lf_filename.c_str());
940+
this->lf_filename_as_string.c_str());
936941
this->close();
937942
return rebuild_result_t::INVALID;
938943
}
@@ -949,7 +954,7 @@ logfile::rebuild_index(std::optional<ui_clock::time_point> deadline)
949954

950955
if (read_result.isErr()) {
951956
log_info("overwritten file detected, closing -- %s (%s)",
952-
this->lf_filename.c_str(),
957+
this->lf_filename_as_string.c_str(),
953958
read_result.unwrapErr().c_str());
954959
this->close();
955960
return rebuild_result_t::INVALID;
@@ -972,7 +977,7 @@ logfile::rebuild_index(std::optional<ui_clock::time_point> deadline)
972977
if (ui_clock::now() > deadline.value()) {
973978
if (has_format) {
974979
log_warning("with format ran past deadline! -- %s",
975-
this->lf_filename.c_str());
980+
this->lf_filename_as_string.c_str());
976981
limit = 1000;
977982
} else {
978983
limit = 100;
@@ -984,8 +989,9 @@ logfile::rebuild_index(std::optional<ui_clock::time_point> deadline)
984989
}
985990
}
986991
if (!has_format) {
987-
log_debug(
988-
"loading file... %s:%d", this->lf_filename.c_str(), begin_size);
992+
log_debug("loading file... %s:%d",
993+
this->lf_filename_as_string.c_str(),
994+
begin_size);
989995
}
990996
scan_batch_context sbc{this->lf_allocator};
991997
sbc.sbc_opids.los_opid_ranges.reserve(32);
@@ -995,7 +1001,7 @@ logfile::rebuild_index(std::optional<ui_clock::time_point> deadline)
9951001

9961002
if (load_result.isErr()) {
9971003
log_error("%s: load next line failure -- %s",
998-
this->lf_filename.c_str(),
1004+
this->lf_filename_as_string.c_str(),
9991005
load_result.unwrapErr().c_str());
10001006
this->close();
10011007
return rebuild_result_t::INVALID;
@@ -1013,7 +1019,7 @@ logfile::rebuild_index(std::optional<ui_clock::time_point> deadline)
10131019
&& !li.li_utf8_scan_result.is_valid())
10141020
{
10151021
log_info("file is not utf, hiding: %s",
1016-
this->lf_filename.c_str());
1022+
this->lf_filename_as_string.c_str());
10171023
this->lf_indexing = false;
10181024
this->lf_options.loo_is_visible = false;
10191025
auto utf8_error_um
@@ -1107,7 +1113,7 @@ logfile::rebuild_index(std::optional<ui_clock::time_point> deadline)
11071113
= this->lf_line_buffer.read_range(li.li_file_range);
11081114
if (read_result.isErr()) {
11091115
log_error("%s:read failure -- %s",
1110-
this->lf_filename.c_str(),
1116+
this->lf_filename_as_string.c_str(),
11111117
read_result.unwrapErr().c_str());
11121118
this->close();
11131119
return rebuild_result_t::INVALID;
@@ -1120,7 +1126,7 @@ logfile::rebuild_index(std::optional<ui_clock::time_point> deadline)
11201126
attr_line_builder alb(al);
11211127
log_warning(
11221128
"%s: invalid UTF-8 detected at L%d:C%d/%d (O:%lld) -- %s",
1123-
this->lf_filename.c_str(),
1129+
this->lf_filename_as_string.c_str(),
11241130
this->lf_index.size() + 1,
11251131
li.li_utf8_scan_result.usr_valid_frag.sf_end,
11261132
li.li_file_range.fr_size,
@@ -1160,7 +1166,7 @@ logfile::rebuild_index(std::optional<ui_clock::time_point> deadline)
11601166
log_debug(
11611167
"%s: rollbacked line %zu matched filter, forcing "
11621168
"full sort",
1163-
this->lf_filename.c_str(),
1169+
this->lf_filename_as_string.c_str(),
11641170
rollback_index_start);
11651171
sort_needed = true;
11661172
}
@@ -1273,7 +1279,7 @@ logfile::rebuild_index(std::optional<ui_clock::time_point> deadline)
12731279
&& st.st_size >= this->lf_options.loo_visible_size_limit)
12741280
{
12751281
log_info("file has unknown format and is too large: %s",
1276-
this->lf_filename.c_str());
1282+
this->lf_filename_as_string.c_str());
12771283
this->lf_indexing = false;
12781284
auto note_um
12791285
= lnav::console::user_message::warning(
@@ -1302,7 +1308,7 @@ logfile::rebuild_index(std::optional<ui_clock::time_point> deadline)
13021308
begin_rusage,
13031309
this->lf_activity.la_initial_index_rusage);
13041310
log_info("Resource usage for initial indexing of file: %s:%d-%d",
1305-
this->lf_filename.c_str(),
1311+
this->lf_filename_as_string.c_str(),
13061312
begin_size,
13071313
this->lf_index.size());
13081314
log_rusage(lnav_log_level_t::INFO,
@@ -1333,15 +1339,15 @@ logfile::rebuild_index(std::optional<ui_clock::time_point> deadline)
13331339
}
13341340
log_debug(
13351341
"%s: opid_map size: count=%zu; sizeof(otr)=%zu; alloc=%zu",
1336-
this->lf_filename.c_str(),
1342+
this->lf_filename_as_string.c_str(),
13371343
writable_opid_map->los_opid_ranges.size(),
13381344
sizeof(opid_time_range),
13391345
this->lf_allocator.getNumBytesAllocated());
13401346
}
13411347

13421348
if (begin_size > this->lf_index.size()) {
13431349
log_info("overwritten file detected, closing -- %s",
1344-
this->lf_filename.c_str());
1350+
this->lf_filename_as_string.c_str());
13451351
this->close();
13461352
return rebuild_result_t::INVALID;
13471353
}
@@ -1375,7 +1381,7 @@ logfile::rebuild_index(std::optional<ui_clock::time_point> deadline)
13751381
if (this->lf_out_of_time_order_count) {
13761382
log_info("Detected %d out-of-time-order lines in file: %s",
13771383
this->lf_out_of_time_order_count,
1378-
this->lf_filename.c_str());
1384+
this->lf_filename_as_string.c_str());
13791385
this->lf_out_of_time_order_count = 0;
13801386
}
13811387

@@ -1456,7 +1462,7 @@ logfile::read_full_message(const_iterator ll,
14561462

14571463
#if 0
14581464
log_debug(
1459-
"%s: reading msg at %d", this->lf_filename.c_str(), ll->get_offset());
1465+
"%s: reading msg at %d", this->lf_filename_as_string.c_str(), ll->get_offset());
14601466
#endif
14611467

14621468
msg_out.disown();
@@ -1706,6 +1712,7 @@ logfile::set_filename(const std::string& filename)
17061712
{
17071713
if (this->lf_filename != filename) {
17081714
this->lf_filename = filename;
1715+
this->lf_filename_as_string = this->lf_filename.string();
17091716
std::filesystem::path p(filename);
17101717
this->lf_basename = p.filename();
17111718
}
@@ -1770,7 +1777,8 @@ logfile::dump_stats()
17701777
if (buf_stats.empty()) {
17711778
return;
17721779
}
1773-
log_info("line buffer stats for file: %s", this->lf_filename.c_str());
1780+
log_info("line buffer stats for file: %s",
1781+
this->lf_filename_as_string.c_str());
17741782
log_info(" file_size=%lld", this->lf_line_buffer.get_file_size());
17751783
log_info(" buffer_size=%ld", this->lf_line_buffer.get_buffer_size());
17761784
log_info(" read_hist=[%4lu %4lu %4lu %4lu %4lu %4lu %4lu %4lu %4lu %4lu]",

‎src/logfile.hh‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,11 @@ public:
132132
return this->lf_filename;
133133
}
134134

135+
const std::string get_filename_as_string() const
136+
{
137+
return this->lf_filename_as_string;
138+
}
139+
135140
std::filesystem::path get_path_for_key() const
136141
{
137142
return this->lf_actual_path.value_or(this->lf_filename);
@@ -198,10 +203,7 @@ public:
198203

199204
int get_time_offset_line() const { return this->lf_time_offset_line; }
200205

201-
const timeval& get_time_offset() const
202-
{
203-
return this->lf_time_offset;
204-
}
206+
const timeval& get_time_offset() const { return this->lf_time_offset; }
205207

206208
void adjust_content_time(int line,
207209
const struct timeval& tv,
@@ -452,12 +454,14 @@ public:
452454
return this->lf_file_options;
453455
}
454456

455-
const std::set<intern_string_t>& get_mismatched_formats()
457+
const robin_hood::unordered_set<intern_string_t, intern_hasher>&
458+
get_mismatched_formats() const
456459
{
457460
return this->lf_mismatched_formats;
458461
}
459462

460463
const std::vector<lnav::console::user_message>& get_format_match_messages()
464+
const
461465
{
462466
return this->lf_format_match_messages;
463467
}
@@ -501,6 +505,7 @@ private:
501505
bool file_options_have_changed();
502506

503507
std::filesystem::path lf_filename;
508+
std::string lf_filename_as_string;
504509
logfile_open_options lf_options;
505510
logfile_activity lf_activity;
506511
bool lf_named_file{true};
@@ -540,7 +545,8 @@ private:
540545
std::optional<tm> lf_cached_base_tm;
541546

542547
std::optional<std::pair<file_off_t, size_t>> lf_next_line_cache;
543-
std::set<intern_string_t> lf_mismatched_formats;
548+
robin_hood::unordered_set<intern_string_t, intern_hasher>
549+
lf_mismatched_formats;
544550
robin_hood::unordered_map<uint32_t, bookmark_metadata> lf_bookmark_metadata;
545551

546552
std::vector<std::shared_ptr<format_tag_def>> lf_applicable_taggers;

0 commit comments

Comments
 (0)