Skip to content

Commit aa5e8cc

Browse files
tstacktstack
committed
[cmds] add "--all" flag to ":write-jsonlines-to"
1 parent 05f4bd5 commit aa5e8cc

9 files changed

Lines changed: 188 additions & 120 deletions

‎NEWS.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ Features:
4646
the standard log fields (`log_path`, `log_time`,
4747
`log_level`, `log_opid`, `log_line_link`) along with any
4848
format-specific parsed values, comments, and tags.
49+
The `--all` flag can be used to write all visible log
50+
lines instead of only marked lines.
4951
* Introducing "Log-Oriented Debugging", a collection of
5052
features to streamline mapping log messages back to
5153
the source code that generated them. For example,

‎src/cmds.io.cc‎

Lines changed: 164 additions & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,105 @@ json_write_row(exec_context& ec,
233233
}
234234
}
235235

236+
static void
237+
json_write_logmsg(yajl_gen handle,
238+
const logline_window::logmsg_info& li,
239+
lnav::text_anonymizer& ta,
240+
bool anonymize)
241+
{
242+
const auto& values = li.get_values();
243+
const auto& ll = li.get_logline();
244+
245+
{
246+
yajlpp_map obj_map(handle);
247+
248+
obj_map.gen("log_path");
249+
obj_map.gen(li.get_file_ptr()->get_filename());
250+
251+
obj_map.gen("log_time");
252+
{
253+
char ts[64];
254+
auto ts_len = sql_strftime(ts, sizeof(ts), ll.get_timeval(), 'T');
255+
obj_map.gen(ts, ts_len);
256+
}
257+
258+
obj_map.gen("log_level");
259+
obj_map.gen(ll.get_level_name());
260+
261+
if (values.lvv_opid_value) {
262+
obj_map.gen("log_opid");
263+
obj_map.gen(*values.lvv_opid_value);
264+
}
265+
266+
auto hash_res = li.get_line_hash();
267+
if (hash_res.isOk()) {
268+
auto hash = hash_res.unwrap();
269+
auto link
270+
= fmt::format(FMT_STRING("#msg{:016x}-{}"),
271+
ll.get_time<std::chrono::microseconds>().count(),
272+
hash);
273+
obj_map.gen("log_line_link");
274+
obj_map.gen(link);
275+
}
276+
277+
auto meta_opt = li.get_metadata();
278+
if (meta_opt) {
279+
auto* meta = *meta_opt;
280+
if (!meta->bm_comment.empty()) {
281+
obj_map.gen("log_comment");
282+
obj_map.gen(meta->bm_comment);
283+
}
284+
if (!meta->bm_tags.empty()) {
285+
obj_map.gen("log_tags");
286+
{
287+
yajlpp_array arr(handle);
288+
for (const auto& tag : meta->bm_tags) {
289+
arr.gen(tag);
290+
}
291+
}
292+
}
293+
}
294+
295+
for (const auto& lv : values.lvv_values) {
296+
if (lv.lv_meta.lvm_hidden) {
297+
continue;
298+
}
299+
300+
obj_map.gen(lv.lv_meta.lvm_name.get());
301+
switch (lv.lv_meta.lvm_kind) {
302+
case value_kind_t::VALUE_NULL:
303+
obj_map.gen();
304+
break;
305+
case value_kind_t::VALUE_BOOLEAN:
306+
obj_map.gen(lv.lv_value.i ? true : false);
307+
break;
308+
case value_kind_t::VALUE_INTEGER:
309+
obj_map.gen(lv.lv_value.i);
310+
break;
311+
case value_kind_t::VALUE_FLOAT:
312+
obj_map.gen(lv.lv_value.d);
313+
break;
314+
case value_kind_t::VALUE_JSON: {
315+
auto tv = lv.text_value();
316+
yajl_gen_number(handle, tv, lv.text_length());
317+
break;
318+
}
319+
default: {
320+
if (anonymize) {
321+
auto anon = ta.next(
322+
std::string(lv.text_value(), lv.text_length()));
323+
obj_map.gen(anon);
324+
} else {
325+
obj_map.gen(lv.text_value(), lv.text_length());
326+
}
327+
break;
328+
}
329+
}
330+
}
331+
}
332+
yajl_gen_reset(handle, "\n");
333+
}
334+
236335
static Result<std::string, lnav::console::user_message>
237336
com_save_to(exec_context& ec,
238337
std::string cmdline,
@@ -272,6 +371,12 @@ com_save_to(exec_context& ec,
272371
split_args.erase(anon_iter);
273372
anonymize = true;
274373
}
374+
bool write_all = false;
375+
auto all_iter = std::find(split_args.begin(), split_args.end(), "--all");
376+
if (all_iter != split_args.end()) {
377+
split_args.erase(all_iter);
378+
write_all = true;
379+
}
275380

276381
auto* tc = *lnav_data.ld_view_stack.top();
277382
auto opt_view_name = find_arg(split_args, "--view");
@@ -319,10 +424,12 @@ com_save_to(exec_context& ec,
319424
"no query result to write, use ';' to execute a query");
320425
}
321426
} else if (tc == &lnav_data.ld_views[LNV_LOG]) {
322-
all_user_marks = combined_user_marks(tc->get_bookmarks());
323-
if (all_user_marks.empty()) {
324-
return ec.make_error(
325-
"no lines marked to write, use 'm' to mark lines");
427+
if (!write_all) {
428+
all_user_marks = combined_user_marks(tc->get_bookmarks());
429+
if (all_user_marks.empty()) {
430+
return ec.make_error(
431+
"no lines marked to write, use 'm' to mark lines");
432+
}
326433
}
327434
} else {
328435
return ec.make_error(
@@ -719,126 +826,66 @@ com_save_to(exec_context& ec,
719826
yajlpp_gen gen;
720827

721828
yajl_gen_config(gen, yajl_gen_beautify, 0);
722-
yajl_gen_config(gen, yajl_gen_print_callback, yajl_writer, outfile);
829+
830+
auto flush_gen = [&gen, outfile]() {
831+
const unsigned char* buf;
832+
size_t len;
833+
834+
yajl_gen_get_buf(gen, &buf, &len);
835+
if (len > 0) {
836+
fwrite(buf, 1, len, outfile);
837+
yajl_gen_clear(gen);
838+
}
839+
};
723840

724841
if (tc == &lnav_data.ld_views[LNV_LOG]) {
725842
auto& lss = lnav_data.ld_log_source;
726843

727-
for (auto iter = all_user_marks.bv_tree.begin();
728-
iter != all_user_marks.bv_tree.end();
729-
++iter)
730-
{
731-
if (ec.ec_dry_run && line_count > 10) {
732-
break;
733-
}
734-
735-
auto lw = lss.window_at(*iter);
844+
if (write_all) {
845+
auto inner_height = tc->get_inner_height();
846+
auto lw = lss.window_to_end(0_vl);
736847

737848
for (const auto& li : *lw) {
738-
const auto& values = li.get_values();
739-
const auto& ll = li.get_logline();
740-
741-
{
742-
yajlpp_map obj_map(gen);
743-
744-
obj_map.gen("log_path");
745-
obj_map.gen(li.get_file_ptr()->get_filename());
746-
747-
obj_map.gen("log_time");
849+
if (ec.ec_dry_run && line_count > 10) {
850+
break;
851+
}
852+
json_write_logmsg(gen, li, ta, anonymize);
853+
line_count += 1;
854+
if (line_count % 1000 == 0) {
855+
flush_gen();
856+
if (write_progress(line_count,
857+
static_cast<size_t>(inner_height))
858+
== lnav::progress_result_t::interrupt)
748859
{
749-
char ts[64];
750-
auto ts_len = sql_strftime(
751-
ts, sizeof(ts), ll.get_timeval(), 'T');
752-
obj_map.gen(ts, ts_len);
753-
}
754-
755-
obj_map.gen("log_level");
756-
obj_map.gen(ll.get_level_name());
757-
758-
if (values.lvv_opid_value) {
759-
obj_map.gen("log_opid");
760-
obj_map.gen(*values.lvv_opid_value);
761-
}
762-
763-
auto hash_res = li.get_line_hash();
764-
if (hash_res.isOk()) {
765-
auto hash = hash_res.unwrap();
766-
auto link = fmt::format(
767-
FMT_STRING("#msg{:016x}-{}"),
768-
ll.get_time<std::chrono::microseconds>()
769-
.count(),
770-
hash);
771-
obj_map.gen("log_line_link");
772-
obj_map.gen(link);
773-
}
774-
775-
auto meta_opt = li.get_metadata();
776-
if (meta_opt) {
777-
auto* meta = *meta_opt;
778-
if (!meta->bm_comment.empty()) {
779-
obj_map.gen("log_comment");
780-
obj_map.gen(meta->bm_comment);
781-
}
782-
if (!meta->bm_tags.empty()) {
783-
obj_map.gen("log_tags");
784-
{
785-
yajlpp_array arr(gen);
786-
for (const auto& tag : meta->bm_tags) {
787-
arr.gen(tag);
788-
}
789-
}
790-
}
860+
break;
791861
}
862+
}
863+
}
864+
} else {
865+
for (auto iter = all_user_marks.bv_tree.begin();
866+
iter != all_user_marks.bv_tree.end();
867+
++iter)
868+
{
869+
if (ec.ec_dry_run && line_count > 10) {
870+
break;
871+
}
792872

793-
for (const auto& lv : values.lvv_values) {
794-
if (lv.lv_meta.lvm_hidden) {
795-
continue;
796-
}
873+
auto lw = lss.window_at(*iter);
797874

798-
obj_map.gen(lv.lv_meta.lvm_name.get());
799-
switch (lv.lv_meta.lvm_kind) {
800-
case value_kind_t::VALUE_NULL:
801-
obj_map.gen();
802-
break;
803-
case value_kind_t::VALUE_BOOLEAN:
804-
obj_map.gen(lv.lv_value.i ? true : false);
805-
break;
806-
case value_kind_t::VALUE_INTEGER:
807-
obj_map.gen(lv.lv_value.i);
808-
break;
809-
case value_kind_t::VALUE_FLOAT:
810-
obj_map.gen(lv.lv_value.d);
811-
break;
812-
case value_kind_t::VALUE_JSON: {
813-
auto tv = lv.text_value();
814-
yajl_gen_number(gen, tv, lv.text_length());
815-
break;
816-
}
817-
default: {
818-
if (anonymize) {
819-
auto anon = ta.next(std::string(
820-
lv.text_value(), lv.text_length()));
821-
obj_map.gen(anon);
822-
} else {
823-
obj_map.gen(lv.text_value(),
824-
lv.text_length());
825-
}
826-
break;
827-
}
828-
}
829-
}
875+
for (const auto& li : *lw) {
876+
json_write_logmsg(gen, li, ta, anonymize);
877+
line_count += 1;
830878
}
831-
yajl_gen_reset(gen, "\n");
832-
}
833879

834-
if (line_count > 0 && line_count % 1000 == 0) {
835-
if (write_progress(line_count, all_user_marks.size())
836-
== lnav::progress_result_t::interrupt)
837-
{
838-
break;
880+
if (line_count > 0 && line_count % 1000 == 0) {
881+
flush_gen();
882+
if (write_progress(line_count, all_user_marks.size())
883+
== lnav::progress_result_t::interrupt)
884+
{
885+
break;
886+
}
839887
}
840888
}
841-
line_count += 1;
842889
}
843890
} else {
844891
for (size_t row = 0; row < dls.dls_row_cursors.size(); row++) {
@@ -849,6 +896,7 @@ com_save_to(exec_context& ec,
849896
json_write_row(ec, gen, row, ta, anonymize);
850897
yajl_gen_reset(gen, "\n");
851898
if (row > 0 && row % 1000 == 0) {
899+
flush_gen();
852900
if (write_progress(row, dls.dls_row_cursors.size())
853901
== lnav::progress_result_t::interrupt)
854902
{
@@ -858,6 +906,7 @@ com_save_to(exec_context& ec,
858906
line_count += 1;
859907
}
860908
}
909+
flush_gen();
861910
} else if (args[0] == "write-screen-to") {
862911
bool wrapped = tc->get_word_wrap();
863912
auto orig_top = tc->get_top();
@@ -2224,8 +2273,12 @@ static readline_context::command_t IO_COMMANDS[] = {
22242273
com_save_to,
22252274

22262275
help_text(":write-jsonlines-to")
2227-
.with_summary("Write SQL results or marked log lines to "
2276+
.with_summary("Write SQL results or log lines to "
22282277
"the given file in JSON Lines format")
2278+
.with_parameter(help_text("--all",
2279+
"Write all visible log lines instead of "
2280+
"only marked lines")
2281+
.flag())
22292282
.with_parameter(
22302283
help_text("--anonymize", "Anonymize the JSON values").flag())
22312284
.with_parameter(

‎src/internals/cmd-ref.rst‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1883,12 +1883,13 @@
18831883

18841884
.. _write_jsonlines_to:
18851885

1886-
:write-jsonlines-to *\[--anonymize\]* *path*
1887-
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
1886+
:write-jsonlines-to *\[--all\]* *\[--anonymize\]* *path*
1887+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
18881888

1889-
Write SQL results or marked log lines to the given file in JSON Lines format
1889+
Write SQL results or log lines to the given file in JSON Lines format
18901890

18911891
**Parameters**
1892+
* **--all** --- Write all visible log lines instead of only marked lines
18921893
* **--anonymize** --- Anonymize the JSON values
18931894
* **path\*** --- The path to the file to write
18941895

‎test/expected/Makefile.am‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,8 @@ dist_noinst_DATA = \
234234
test_cmds.sh_a190bfc279fa046a823864f1484f899d27d22953.out \
235235
test_cmds.sh_a5742238bad948b1372d32f7a491f03fa4e8b711.err \
236236
test_cmds.sh_a5742238bad948b1372d32f7a491f03fa4e8b711.out \
237+
test_cmds.sh_a5a1ddb5c9da963d52ccffc7b341e54325ecbe49.err \
238+
test_cmds.sh_a5a1ddb5c9da963d52ccffc7b341e54325ecbe49.out \
237239
test_cmds.sh_a6c431f2871ea96cfdf4e11465b3bca543c7b678.err \
238240
test_cmds.sh_a6c431f2871ea96cfdf4e11465b3bca543c7b678.out \
239241
test_cmds.sh_a813f4cb5e937f218eb10859e5c8132d06eefc1b.err \

‎test/expected/test_cmds.sh_a5a1ddb5c9da963d52ccffc7b341e54325ecbe49.err‎

Whitespace-only changes.

‎test/expected/test_cmds.sh_a5a1ddb5c9da963d52ccffc7b341e54325ecbe49.out‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{"log_path":"{test_dir}/logfile_access_log.0","log_time":"2009-07-20T22:59:26.000000","log_level":"info","log_opid":"87ecc75c2bfe711642086a9b8546ab05","log_line_link":"#msg00046f2b16f0cf80-v1:3f7e0f10f2473f83b2b4eacccfc9b4e2","c_ip":"192.168.202.254","cs_username":null,"timestamp":"20/Jul/2009:22:59:26 +0000","cs_method":"GET","cs_uri_stem":"/vmw/cgi/tramp","cs_uri_query":null,"cs_version":"HTTP/1.0","sc_status":200,"sc_bytes":134,"cs_referer":null,"cs_user_agent":"gPXE/0.9.7","body":""}
2+
{"log_path":"{test_dir}/logfile_access_log.0","log_time":"2009-07-20T22:59:29.000000","log_level":"error","log_opid":"87ecc75c2bfe711642086a9b8546ab05","log_line_link":"#msg00046f2b171e9640-v1:7ad4831dd06e0d6b0ffd965c7d65285f","c_ip":"192.168.202.254","cs_username":null,"timestamp":"20/Jul/2009:22:59:29 +0000","cs_method":"GET","cs_uri_stem":"/vmw/vSphere/default/vmkboot.gz","cs_uri_query":null,"cs_version":"HTTP/1.0","sc_status":404,"sc_bytes":46210,"cs_referer":null,"cs_user_agent":"gPXE/0.9.7","body":""}
3+
{"log_path":"{test_dir}/logfile_access_log.0","log_time":"2009-07-20T22:59:29.000000","log_level":"info","log_opid":"87ecc75c2bfe711642086a9b8546ab05","log_line_link":"#msg00046f2b171e9640-v1:b05c1bdfe75cde41e151c89087e31951","c_ip":"192.168.202.254","cs_username":null,"timestamp":"20/Jul/2009:22:59:29 +0000","cs_method":"GET","cs_uri_stem":"/vmw/vSphere/default/vmkernel.gz","cs_uri_query":null,"cs_version":"HTTP/1.0","sc_status":200,"sc_bytes":78929,"cs_referer":null,"cs_user_agent":"gPXE/0.9.7","body":""}

‎test/expected/test_cmds.sh_b6a3bb78e9d60e5e1f5ce5b18e40d2f1662707ab.out‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1941,11 +1941,13 @@ For support questions, email:
19411941

19421942

19431943

1944-
:write-jsonlines-to [--anonymize] path
1944+
:write-jsonlines-to [--all] [--anonymize] path
19451945
══════════════════════════════════════════════════════════════════════
1946-
Write SQL results or marked log lines to the given file in JSON
1947-
Lines format
1946+
Write SQL results or log lines to the given file in JSON Lines
1947+
format
19481948
Parameters
1949+
--all Write all visible log lines instead of
1950+
only marked lines
19491951
--anonymize Anonymize the JSON values
19501952
path The path to the file to write
19511953
See Also

0 commit comments

Comments
 (0)