Skip to content

Commit 31f3049

Browse files
tstacktstack
committed
[ptime] fix UTC/GMT check and where time localization is done
Some format matching refinements as well. Related to #1429
1 parent 96611b6 commit 31f3049

10 files changed

Lines changed: 201 additions & 139 deletions

‎src/grep_proc.cc‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -268,13 +268,12 @@ grep_proc<LineType>::child_loop()
268268
}
269269
}
270270

271-
if (line != -1 && stop_line.ru_type == until_type_t::eof) {
271+
if (line > 0 && stop_line.ru_type == until_type_t::eof) {
272272
// When scanning to the end of the source, we need to return the
273273
// highest line that was seen so that the next request that
274274
// continues from the end works properly.
275275
fmt::println(stdout, FMT_STRING("h{}"), line - 1);
276276
}
277-
this->gp_highest_line = line - 1_vl;
278277
this->child_term();
279278
}
280279
}
@@ -315,16 +314,18 @@ grep_proc<LineType>::dispatch_line(const string_fragment& line)
315314
auto sv = line.to_string_view();
316315
auto h_scan_res = scn::scan<int>(sv, "h{}");
317316
if (h_scan_res) {
318-
this->gp_highest_line = LineType{h_scan_res->value()};
317+
auto highest_line = LineType{h_scan_res->value()};
318+
if (highest_line > this->gp_highest_line) {
319+
this->gp_highest_line = highest_line;
320+
}
319321
} else {
320322
auto ll_scan_res = scn::scan<int>(sv, "{}");
321323
if (ll_scan_res) {
322-
this->gp_last_line = LineType{ll_scan_res->value()};
324+
auto last_line = LineType{ll_scan_res->value()};
323325
/* Starting a new line with matches. */
324-
ensure(this->gp_last_line >= 0);
325326
/* Pass the match offsets to the sink delegate. */
326327
if (this->gp_sink != nullptr) {
327-
this->gp_sink->grep_match(*this, this->gp_last_line);
328+
this->gp_sink->grep_match(*this, last_line);
328329
}
329330
} else {
330331
log_error(
@@ -452,6 +453,7 @@ grep_proc<LineType>::invalidate()
452453
}
453454
}
454455
this->gp_queue.clear();
456+
this->gp_highest_line = LineType{0};
455457
this->cleanup();
456458
return *this;
457459
}

‎src/grep_proc.hh‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -276,16 +276,11 @@ protected:
276276

277277
/** The queue of search requests. */
278278
std::deque<std::pair<LineType, request_until_t>> gp_queue;
279-
LineType gp_last_line{0}; /*<
280-
* The last line number received from
281-
* the child. For multiple matches,
282-
* the line number is only sent once.
283-
*/
284-
LineType gp_highest_line; /*< The highest numbered line processed
285-
* by the grep child process. This
286-
* value is used when the start line
287-
* for a queued request is -1.
288-
*/
279+
LineType gp_highest_line{0}; /*< The highest numbered line processed
280+
* by the grep child process. This
281+
* value is used when the start line
282+
* for a queued request is -1.
283+
*/
289284
grep_proc_sink<LineType>* gp_sink{nullptr}; /*< The sink delegate. */
290285
grep_proc_control* gp_control{nullptr}; /*< The control delegate. */
291286
};

‎src/log_format.cc‎

Lines changed: 70 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1105,9 +1105,13 @@ struct json_log_userdata {
11051105
.first
11061106
== field_frag)
11071107
{
1108-
return format
1108+
auto retval = format
11091109
->elf_value_def_read_order[this->jlu_read_order_index++]
11101110
.second;
1111+
if (retval != nullptr) {
1112+
this->jlu_precision += 1;
1113+
}
1114+
return retval;
11111115
}
11121116

11131117
format->elf_value_def_read_order.resize(this->jlu_read_order_index);
@@ -1116,6 +1120,9 @@ struct json_log_userdata {
11161120
format->elf_value_def_read_order.emplace_back(vd_iter->first,
11171121
vd_iter->second);
11181122
this->jlu_read_order_index += 1;
1123+
if (vd_iter->second != nullptr) {
1124+
this->jlu_precision += 1;
1125+
}
11191126
return vd_iter->second;
11201127
}
11211128

@@ -1158,6 +1165,7 @@ struct json_log_userdata {
11581165
std::stack<size_t> jlu_sub_start;
11591166
uint32_t jlu_quality{0};
11601167
uint32_t jlu_strikes{0};
1168+
uint32_t jlu_precision{0};
11611169
std::vector<bool> jlu_format_hits;
11621170
shared_buffer_ref& jlu_shared_buffer;
11631171
scan_batch_context* jlu_batch_context;
@@ -1884,7 +1892,7 @@ external_log_format::scan_json(std::vector<logline>& dst,
18841892
if (jlu.jlu_quality > 0) {
18851893
jlu.jlu_quality += 3000;
18861894
}
1887-
return scan_match{jlu.jlu_quality, jlu.jlu_strikes};
1895+
return scan_match{jlu.jlu_quality, jlu.jlu_strikes, jlu.jlu_precision};
18881896
}
18891897

18901898
log_format::scan_result_t
@@ -2411,9 +2419,37 @@ external_log_format::annotate(logfile* lf,
24112419
}
24122420

24132421
if (cap) {
2414-
values.lvv_values.emplace_back(
2415-
vd.vd_meta, line, to_line_range(cap.value()));
2416-
values.lvv_values.back().apply_scaling(scaling);
2422+
if (vd.vd_meta.lvm_kind == value_kind_t::VALUE_TIMESTAMP) {
2423+
auto dts = this->build_time_scanner();
2424+
exttm tm;
2425+
timeval tv;
2426+
auto val_sf = cap.value();
2427+
2428+
if (dts.scan(val_sf.data(),
2429+
val_sf.length(),
2430+
this->get_timestamp_formats(),
2431+
&tm,
2432+
tv,
2433+
true))
2434+
{
2435+
char ts[64];
2436+
tm.et_gmtoff = tm.et_orig_gmtoff;
2437+
auto len = dts.ftime(
2438+
ts, sizeof(ts), this->get_timestamp_formats(), tm);
2439+
ts[len] = '\0';
2440+
values.lvv_values.emplace_back(vd.vd_meta,
2441+
std::string{ts, len});
2442+
values.lvv_values.back().lv_origin
2443+
= to_line_range(cap.value());
2444+
} else {
2445+
values.lvv_values.emplace_back(
2446+
vd.vd_meta, line, to_line_range(cap.value()));
2447+
}
2448+
} else {
2449+
values.lvv_values.emplace_back(
2450+
vd.vd_meta, line, to_line_range(cap.value()));
2451+
values.lvv_values.back().apply_scaling(scaling);
2452+
}
24172453
} else {
24182454
values.lvv_values.emplace_back(vd.vd_meta);
24192455
}
@@ -2734,6 +2770,35 @@ rewrite_json_field(yajlpp_parse_context* ypc,
27342770
jlu->jlu_format->get_value_meta(field_name,
27352771
value_kind_t::VALUE_TEXT),
27362772
std::string{time_buf});
2773+
} else if (vd != nullptr
2774+
&& vd->vd_meta.lvm_kind == value_kind_t::VALUE_TIMESTAMP)
2775+
{
2776+
auto dts = jlu->jlu_format->build_time_scanner();
2777+
exttm tm;
2778+
timeval tv;
2779+
2780+
if (dts.scan((const char*) str,
2781+
len,
2782+
jlu->jlu_format->get_timestamp_formats(),
2783+
&tm,
2784+
tv,
2785+
true))
2786+
{
2787+
char ts[64];
2788+
tm.et_gmtoff = tm.et_orig_gmtoff;
2789+
auto tslen = dts.ftime(
2790+
ts, sizeof(ts), jlu->jlu_format->get_timestamp_formats(), tm);
2791+
ts[tslen] = '\0';
2792+
jlu->jlu_format->jlf_line_values.lvv_values.emplace_back(
2793+
jlu->jlu_format->get_value_meta(
2794+
ypc, vd, value_kind_t::VALUE_TIMESTAMP),
2795+
std::string{(const char*) ts, tslen});
2796+
} else {
2797+
jlu->jlu_format->jlf_line_values.lvv_values.emplace_back(
2798+
jlu->jlu_format->get_value_meta(
2799+
ypc, vd, value_kind_t::VALUE_TEXT),
2800+
std::string{(const char*) str, len});
2801+
}
27372802
} else if (jlu->jlu_shared_buffer.contains((const char*) str)) {
27382803
auto str_offset = (int) ((const char*) str - jlu->jlu_line_value);
27392804
if (field_name == jlu->jlu_format->elf_body_field) {
@@ -2959,31 +3024,6 @@ external_log_format::get_subline(const log_format_file_state& lffs,
29593024
.value()]
29603025
.get();
29613026
}
2962-
if (lv_iter->lv_meta.lvm_kind
2963-
== value_kind_t::VALUE_TIMESTAMP)
2964-
{
2965-
auto dts = this->build_time_scanner();
2966-
exttm tm;
2967-
timeval tv;
2968-
2969-
if (dts.scan(str.c_str(),
2970-
str.size(),
2971-
this->get_timestamp_formats(),
2972-
&tm,
2973-
tv,
2974-
true))
2975-
{
2976-
char ts[64];
2977-
tm.et_gmtoff = tm.et_orig_gmtoff;
2978-
auto len = dts.ftime(
2979-
ts,
2980-
sizeof(ts),
2981-
this->get_timestamp_formats(),
2982-
tm);
2983-
ts[len] = '\0';
2984-
str = ts;
2985-
}
2986-
}
29873027
while (endswith(str, "\n")) {
29883028
str.pop_back();
29893029
}
@@ -3247,31 +3287,6 @@ external_log_format::get_subline(const log_format_file_state& lffs,
32473287
}
32483288

32493289
auto str = lv.to_string();
3250-
if (lv.lv_meta.lvm_kind
3251-
== value_kind_t::VALUE_TIMESTAMP)
3252-
{
3253-
auto dts = this->build_time_scanner();
3254-
exttm tm;
3255-
timeval tv;
3256-
3257-
if (dts.scan(str.c_str(),
3258-
str.size(),
3259-
this->get_timestamp_formats(),
3260-
&tm,
3261-
tv,
3262-
true))
3263-
{
3264-
char ts[64];
3265-
tm.et_gmtoff = tm.et_orig_gmtoff;
3266-
auto len = dts.ftime(
3267-
ts,
3268-
sizeof(ts),
3269-
this->get_timestamp_formats(),
3270-
tm);
3271-
ts[len] = '\0';
3272-
str = ts;
3273-
}
3274-
}
32753290
while (endswith(str, "\n")) {
32763291
str.pop_back();
32773292
}

‎src/log_format.hh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@ public:
161161
struct scan_match {
162162
uint32_t sm_quality{0};
163163
uint32_t sm_strikes{0};
164+
uint32_t sm_precision{0};
164165
};
165166

166167
struct scan_error {

‎src/log_format_fwd.hh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,12 @@ struct log_opid_state {
138138
const string_fragment& subid,
139139
const std::chrono::microseconds& us,
140140
log_level_t level);
141+
142+
void clear()
143+
{
144+
this->los_opid_ranges.clear();
145+
this->los_sub_in_use.clear();
146+
}
141147
};
142148

143149
struct thread_id_time_range {
@@ -166,6 +172,8 @@ struct log_thread_id_state {
166172
log_thread_id_map::iterator insert_tid(ArenaAlloc::Alloc<char>& alloc,
167173
const string_fragment& tid,
168174
const std::chrono::microseconds& us);
175+
176+
void clear() { this->ltis_tid_ranges.clear(); }
169177
};
170178

171179
struct logline_value_stats {

0 commit comments

Comments
 (0)