Skip to content

Commit 2b55c16

Browse files
stacknilstacknil
committed
test(parser): add mixed coverage artifact
1 parent ec10016 commit 2b55c16

7 files changed

Lines changed: 204 additions & 2 deletions

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ All notable user-visible changes should be recorded here.
1111
- Added `verdict_boundary` to JSON findings and advanced the report artifact contract to `loglens.report.v2`.
1212
- Expanded parser coverage for `Accepted publickey` and selected `pam_faillock` / `pam_sss` variants.
1313
- Added a 150-line sanitized mixed auth corpus fixture covering Ubuntu / Debian-style `auth.log`, RHEL-family `secure`-style syslog, unknown lines, malformed source IPs, and blank-line handling.
14+
- Added a reviewer-facing parser coverage JSON artifact for the mixed auth corpus.
1415
- Added compact host-level summaries for multi-host reports.
1516
- Added optional CSV export for findings and warnings when explicitly requested.
1617

‎assets/mixed_auth_parser_coverage.json‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
{
2+
"artifact": "loglens.parser_coverage_sample",
3+
"schema_version": 1,
4+
"fixture": "assets/mixed_auth_corpus.log",
5+
"input_mode": "syslog_legacy",
6+
"assume_year": 2026,
7+
"parser_quality": {
8+
"total_input_lines": 150,
9+
"total_lines": 140,
10+
"skipped_blank_lines": 10,
11+
"parsed_lines": 90,
12+
"unparsed_lines": 50,
13+
"parse_success_rate": 0.6428571429,
14+
"top_unknown_patterns": [
15+
{"pattern": "invalid_month_token", "count": 10},
16+
{"pattern": "malformed_source_ip", "count": 10},
17+
{"pattern": "pam_unix_session_closed", "count": 10},
18+
{"pattern": "program_cron", "count": 10},
19+
{"pattern": "sshd_connection_closed_preauth", "count": 10}
20+
],
21+
"failure_categories": [
22+
{"category": "known_program_unknown_message", "count": 10},
23+
{"category": "malformed_source_ip", "count": 10},
24+
{"category": "unknown_program", "count": 10},
25+
{"category": "unknown_timestamp", "count": 10},
26+
{"category": "unsupported_pam_variant", "count": 10}
27+
]
28+
},
29+
"parsed_event_count": 90,
30+
"warning_count": 50,
31+
"event_type_counts": [
32+
{"event_type": "ssh_accepted_publickey", "count": 10},
33+
{"event_type": "ssh_invalid_user", "count": 10},
34+
{"event_type": "ssh_failed_publickey", "count": 10},
35+
{"event_type": "pam_auth_failure", "count": 30},
36+
{"event_type": "sudo_command", "count": 10},
37+
{"event_type": "sudo_auth_failure", "count": 10},
38+
{"event_type": "su_auth_failure", "count": 10}
39+
],
40+
"warnings": [
41+
{"line_number": 10, "category": "known_program_unknown_message", "reason": "unrecognized auth pattern: sshd_connection_closed_preauth"},
42+
{"line_number": 11, "category": "unsupported_pam_variant", "reason": "unrecognized auth pattern: pam_unix_session_closed"},
43+
{"line_number": 12, "category": "unknown_program", "reason": "unrecognized auth pattern: program_cron"},
44+
{"line_number": 13, "category": "malformed_source_ip", "reason": "malformed source IP"},
45+
{"line_number": 14, "category": "unknown_timestamp", "reason": "invalid month token"},
46+
{"line_number": 25, "category": "known_program_unknown_message", "reason": "unrecognized auth pattern: sshd_connection_closed_preauth"},
47+
{"line_number": 26, "category": "unsupported_pam_variant", "reason": "unrecognized auth pattern: pam_unix_session_closed"},
48+
{"line_number": 27, "category": "unknown_program", "reason": "unrecognized auth pattern: program_cron"},
49+
{"line_number": 28, "category": "malformed_source_ip", "reason": "malformed source IP"},
50+
{"line_number": 29, "category": "unknown_timestamp", "reason": "invalid month token"},
51+
{"line_number": 40, "category": "known_program_unknown_message", "reason": "unrecognized auth pattern: sshd_connection_closed_preauth"},
52+
{"line_number": 41, "category": "unsupported_pam_variant", "reason": "unrecognized auth pattern: pam_unix_session_closed"},
53+
{"line_number": 42, "category": "unknown_program", "reason": "unrecognized auth pattern: program_cron"},
54+
{"line_number": 43, "category": "malformed_source_ip", "reason": "malformed source IP"},
55+
{"line_number": 44, "category": "unknown_timestamp", "reason": "invalid month token"},
56+
{"line_number": 55, "category": "known_program_unknown_message", "reason": "unrecognized auth pattern: sshd_connection_closed_preauth"},
57+
{"line_number": 56, "category": "unsupported_pam_variant", "reason": "unrecognized auth pattern: pam_unix_session_closed"},
58+
{"line_number": 57, "category": "unknown_program", "reason": "unrecognized auth pattern: program_cron"},
59+
{"line_number": 58, "category": "malformed_source_ip", "reason": "malformed source IP"},
60+
{"line_number": 59, "category": "unknown_timestamp", "reason": "invalid month token"},
61+
{"line_number": 70, "category": "known_program_unknown_message", "reason": "unrecognized auth pattern: sshd_connection_closed_preauth"},
62+
{"line_number": 71, "category": "unsupported_pam_variant", "reason": "unrecognized auth pattern: pam_unix_session_closed"},
63+
{"line_number": 72, "category": "unknown_program", "reason": "unrecognized auth pattern: program_cron"},
64+
{"line_number": 73, "category": "malformed_source_ip", "reason": "malformed source IP"},
65+
{"line_number": 74, "category": "unknown_timestamp", "reason": "invalid month token"},
66+
{"line_number": 85, "category": "known_program_unknown_message", "reason": "unrecognized auth pattern: sshd_connection_closed_preauth"},
67+
{"line_number": 86, "category": "unsupported_pam_variant", "reason": "unrecognized auth pattern: pam_unix_session_closed"},
68+
{"line_number": 87, "category": "unknown_program", "reason": "unrecognized auth pattern: program_cron"},
69+
{"line_number": 88, "category": "malformed_source_ip", "reason": "malformed source IP"},
70+
{"line_number": 89, "category": "unknown_timestamp", "reason": "invalid month token"},
71+
{"line_number": 100, "category": "known_program_unknown_message", "reason": "unrecognized auth pattern: sshd_connection_closed_preauth"},
72+
{"line_number": 101, "category": "unsupported_pam_variant", "reason": "unrecognized auth pattern: pam_unix_session_closed"},
73+
{"line_number": 102, "category": "unknown_program", "reason": "unrecognized auth pattern: program_cron"},
74+
{"line_number": 103, "category": "malformed_source_ip", "reason": "malformed source IP"},
75+
{"line_number": 104, "category": "unknown_timestamp", "reason": "invalid month token"},
76+
{"line_number": 115, "category": "known_program_unknown_message", "reason": "unrecognized auth pattern: sshd_connection_closed_preauth"},
77+
{"line_number": 116, "category": "unsupported_pam_variant", "reason": "unrecognized auth pattern: pam_unix_session_closed"},
78+
{"line_number": 117, "category": "unknown_program", "reason": "unrecognized auth pattern: program_cron"},
79+
{"line_number": 118, "category": "malformed_source_ip", "reason": "malformed source IP"},
80+
{"line_number": 119, "category": "unknown_timestamp", "reason": "invalid month token"},
81+
{"line_number": 130, "category": "known_program_unknown_message", "reason": "unrecognized auth pattern: sshd_connection_closed_preauth"},
82+
{"line_number": 131, "category": "unsupported_pam_variant", "reason": "unrecognized auth pattern: pam_unix_session_closed"},
83+
{"line_number": 132, "category": "unknown_program", "reason": "unrecognized auth pattern: program_cron"},
84+
{"line_number": 133, "category": "malformed_source_ip", "reason": "malformed source IP"},
85+
{"line_number": 134, "category": "unknown_timestamp", "reason": "invalid month token"},
86+
{"line_number": 145, "category": "known_program_unknown_message", "reason": "unrecognized auth pattern: sshd_connection_closed_preauth"},
87+
{"line_number": 146, "category": "unsupported_pam_variant", "reason": "unrecognized auth pattern: pam_unix_session_closed"},
88+
{"line_number": 147, "category": "unknown_program", "reason": "unrecognized auth pattern: program_cron"},
89+
{"line_number": 148, "category": "malformed_source_ip", "reason": "malformed source IP"},
90+
{"line_number": 150, "category": "unknown_timestamp", "reason": "invalid month token"}
91+
]
92+
}

‎docs/parser-conformance-matrix.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ coverage telemetry path.
132132
| [`assets/parser_auth_families_syslog.log`](../assets/parser_auth_families_syslog.log) | Selected `sshd`, `pam_unix`, `pam_faillock`, `pam_sss`, and session-opened auth-family support, plus five unsupported PAM-family telemetry buckets |
133133
| [`assets/parser_auth_families_journalctl_short_full.log`](../assets/parser_auth_families_journalctl_short_full.log) | Same auth-family event and warning shape as the syslog auth-family fixture, with journalctl timestamp parsing |
134134
| [`assets/noisy_auth_sample.log`](../assets/noisy_auth_sample.log) and [`tests/fixtures/parser_matrix/noisy_auth_expected.json`](../tests/fixtures/parser_matrix/noisy_auth_expected.json) | Noisy syslog coverage fixture with malformed lines, blank lines, unsupported auth-family evidence, irrelevant service lines, and locked parser quality counts |
135-
| [`assets/mixed_auth_corpus.log`](../assets/mixed_auth_corpus.log) | 150-line sanitized mixed syslog corpus with Ubuntu / Debian-style `auth.log` and RHEL-family `secure` host labels, 90 parsed events, 50 parser warnings, 10 blank lines, and locked unknown-pattern and failure-category coverage |
135+
| [`assets/mixed_auth_corpus.log`](../assets/mixed_auth_corpus.log) and [`assets/mixed_auth_parser_coverage.json`](../assets/mixed_auth_parser_coverage.json) | 150-line sanitized mixed syslog corpus with Ubuntu / Debian-style `auth.log` and RHEL-family `secure` host labels, 90 parsed events, 50 parser warnings, 10 blank lines, and locked unknown-pattern and failure-category coverage |
136136

137137
## Review Rule
138138

‎docs/parser-contract.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ Parsed successes and audit-only events remain reportable but do not count as bru
8989
| [`assets/parser_auth_families_syslog.log`](../assets/parser_auth_families_syslog.log) | Syslog PAM/auth-family parser coverage |
9090
| [`assets/parser_auth_families_journalctl_short_full.log`](../assets/parser_auth_families_journalctl_short_full.log) | Journalctl PAM/auth-family parser coverage |
9191
| [`assets/noisy_auth_sample.log`](../assets/noisy_auth_sample.log) and [`tests/fixtures/parser_matrix/noisy_auth_expected.json`](../tests/fixtures/parser_matrix/noisy_auth_expected.json) | Noisy syslog parser-coverage matrix for malformed, unsupported, blank, irrelevant, multi-host, and unusual-username input |
92+
| [`assets/mixed_auth_corpus.log`](../assets/mixed_auth_corpus.log) and [`assets/mixed_auth_parser_coverage.json`](../assets/mixed_auth_parser_coverage.json) | 150-line mixed auth corpus plus reviewer-facing parser coverage artifact for dirty syslog input |
9293
| [`tests/test_report_contracts.cpp`](../tests/test_report_contracts.cpp) | Stable report-shape expectations for generated artifacts |
9394

9495
## Non-goals

‎docs/parser-coverage-notes.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ The locked expected coverage summary lives in [`tests/fixtures/parser_matrix/noi
2929

3030
The corpus repeats ten small evidence batches. Each batch includes recognized `sshd`, `sudo`, `su`, `pam_unix`, `pam_faillock`, and `pam_sss` evidence; unsupported `sshd` preauth and `pam_unix` session-close telemetry; an unsupported service program; a malformed source IP; an invalid timestamp; and one blank line.
3131

32+
For reviewer inspection without running the test suite, [`assets/mixed_auth_parser_coverage.json`](../assets/mixed_auth_parser_coverage.json) captures the deterministic parser coverage view for this corpus: parser-quality counters, normalized event-type counts, unknown-pattern buckets, failure categories, and warning line references.
33+
3234
Locked parser expectations:
3335

3436
- `total_input_lines`: 150

‎docs/reviewer-path.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ This path is for reviewers who want to understand LogLens quickly without readin
1010
| What log formats are supported? | [`docs/parser-contract.md`](./parser-contract.md) | Can name `syslog_legacy` and `journalctl_short_full` behavior |
1111
| What artifacts does it produce? | [`docs/report-artifacts.md`](./report-artifacts.md) and report-contract fixtures | Can inspect Markdown, JSON, and optional CSV outputs |
1212
| How do rules use evidence? | [`docs/rule-catalog.md`](./rule-catalog.md) | Can explain grouping keys, windows, thresholds, and unsupported-evidence boundaries |
13-
| Can the parser behavior be trusted? | Parser contract, fixture matrix, and parser coverage fields | Can see known, unknown, and malformed line handling |
13+
| Can the parser behavior be trusted? | Parser contract, fixture matrix, and [`assets/mixed_auth_parser_coverage.json`](../assets/mixed_auth_parser_coverage.json) | Can see known, unknown, and malformed line handling |
1414
| What proves the main claims? | [`docs/quality-gates.md`](./quality-gates.md) | Can map claims to tests, fixtures, docs, and repeatable commands |
1515
| How should a finding be interpreted? | [`docs/case-study-linux-auth-bruteforce.md`](./case-study-linux-auth-bruteforce.md) | Can trace raw evidence to normalized events, findings, warnings, and non-goals |
1616
| How does it behave on larger local inputs? | [`docs/performance-envelope.md`](./performance-envelope.md) | Can state the local 1k/10k/100k-line envelope and its caveats |
@@ -43,6 +43,7 @@ Inspect:
4343
- [`tests/fixtures/report_contracts/syslog_legacy/report.json`](../tests/fixtures/report_contracts/syslog_legacy/report.json)
4444
- [`docs/report-artifacts.md`](./report-artifacts.md)
4545
- [`docs/parser-contract.md`](./parser-contract.md)
46+
- [`assets/mixed_auth_parser_coverage.json`](../assets/mixed_auth_parser_coverage.json)
4647
- [`docs/quality-gates.md`](./quality-gates.md)
4748
- [`docs/rule-catalog.md`](./rule-catalog.md)
4849
- [`docs/case-study-linux-auth-bruteforce.md`](./case-study-linux-auth-bruteforce.md)

‎tests/test_parser.cpp‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#include <stdexcept>
1010
#include <string>
1111
#include <string_view>
12+
#include <utility>
1213
#include <vector>
1314

1415
namespace {
@@ -107,6 +108,42 @@ std::size_t failure_category_count(const loglens::ParserQualityMetrics& quality,
107108
return it == quality.failure_categories.end() ? 0 : it->count;
108109
}
109110

111+
std::vector<std::pair<loglens::EventType, std::size_t>> parser_event_type_counts(
112+
const std::vector<loglens::Event>& events) {
113+
std::vector<std::pair<loglens::EventType, std::size_t>> counts{
114+
{loglens::EventType::SshFailedPassword, 0},
115+
{loglens::EventType::SshAcceptedPassword, 0},
116+
{loglens::EventType::SshAcceptedPublicKey, 0},
117+
{loglens::EventType::SshAcceptedKeyboardInteractive, 0},
118+
{loglens::EventType::SshInvalidUser, 0},
119+
{loglens::EventType::SshFailedPublicKey, 0},
120+
{loglens::EventType::SshFailedKeyboardInteractive, 0},
121+
{loglens::EventType::SshMaxAuthTries, 0},
122+
{loglens::EventType::PamAuthFailure, 0},
123+
{loglens::EventType::SessionOpened, 0},
124+
{loglens::EventType::SudoCommand, 0},
125+
{loglens::EventType::SudoAuthFailure, 0},
126+
{loglens::EventType::SudoPolicyDenied, 0},
127+
{loglens::EventType::SuAuthFailure, 0}};
128+
129+
for (const auto& event : events) {
130+
for (auto& [type, count] : counts) {
131+
if (type == event.event_type) {
132+
++count;
133+
break;
134+
}
135+
}
136+
}
137+
138+
counts.erase(
139+
std::remove_if(counts.begin(), counts.end(), [](const auto& entry) {
140+
return entry.second == 0;
141+
}),
142+
counts.end());
143+
144+
return counts;
145+
}
146+
110147
std::string noisy_auth_coverage_json(const loglens::ParseReport& result) {
111148
std::ostringstream output;
112149
output << "{\n"
@@ -156,6 +193,70 @@ std::string noisy_auth_coverage_json(const loglens::ParseReport& result) {
156193
return output.str();
157194
}
158195

196+
std::string mixed_auth_coverage_json(const loglens::ParseReport& result) {
197+
std::ostringstream output;
198+
const auto event_counts = parser_event_type_counts(result.events);
199+
200+
output << "{\n"
201+
<< " \"artifact\": \"loglens.parser_coverage_sample\",\n"
202+
<< " \"schema_version\": 1,\n"
203+
<< " \"fixture\": \"assets/mixed_auth_corpus.log\",\n"
204+
<< " \"input_mode\": \"" << loglens::to_string(result.metadata.input_mode) << "\",\n"
205+
<< " \"assume_year\": " << *result.metadata.assume_year << ",\n"
206+
<< " \"parser_quality\": {\n"
207+
<< " \"total_input_lines\": " << total_input_lines(result) << ",\n"
208+
<< " \"total_lines\": " << result.quality.total_lines << ",\n"
209+
<< " \"skipped_blank_lines\": " << result.quality.skipped_blank_lines << ",\n"
210+
<< " \"parsed_lines\": " << result.quality.parsed_lines << ",\n"
211+
<< " \"unparsed_lines\": " << result.quality.unparsed_lines << ",\n"
212+
<< " \"parse_success_rate\": " << std::fixed << std::setprecision(10)
213+
<< result.quality.parse_success_rate << ",\n"
214+
<< " \"top_unknown_patterns\": [\n";
215+
216+
for (std::size_t index = 0; index < result.quality.top_unknown_patterns.size(); ++index) {
217+
const auto& entry = result.quality.top_unknown_patterns[index];
218+
output << " {\"pattern\": \"" << entry.pattern << "\", \"count\": " << entry.count << "}";
219+
output << (index + 1 == result.quality.top_unknown_patterns.size() ? "\n" : ",\n");
220+
}
221+
222+
output << " ],\n"
223+
<< " \"failure_categories\": [\n";
224+
225+
for (std::size_t index = 0; index < result.quality.failure_categories.size(); ++index) {
226+
const auto& entry = result.quality.failure_categories[index];
227+
output << " {\"category\": \"" << loglens::to_string(entry.category)
228+
<< "\", \"count\": " << entry.count << "}";
229+
output << (index + 1 == result.quality.failure_categories.size() ? "\n" : ",\n");
230+
}
231+
232+
output << " ]\n"
233+
<< " },\n"
234+
<< " \"parsed_event_count\": " << result.events.size() << ",\n"
235+
<< " \"warning_count\": " << result.warnings.size() << ",\n"
236+
<< " \"event_type_counts\": [\n";
237+
238+
for (std::size_t index = 0; index < event_counts.size(); ++index) {
239+
const auto& [type, count] = event_counts[index];
240+
output << " {\"event_type\": \"" << loglens::to_string(type) << "\", \"count\": " << count << "}";
241+
output << (index + 1 == event_counts.size() ? "\n" : ",\n");
242+
}
243+
244+
output << " ],\n"
245+
<< " \"warnings\": [\n";
246+
247+
for (std::size_t index = 0; index < result.warnings.size(); ++index) {
248+
const auto& warning = result.warnings[index];
249+
output << " {\"line_number\": " << warning.line_number
250+
<< ", \"category\": \"" << loglens::to_string(warning.category) << "\""
251+
<< ", \"reason\": \"" << warning.reason << "\"}";
252+
output << (index + 1 == result.warnings.size() ? "\n" : ",\n");
253+
}
254+
255+
output << " ]\n"
256+
<< "}\n";
257+
return output.str();
258+
}
259+
159260
void test_invalid_user_failure() {
160261
const auto parser = make_syslog_parser();
161262
std::string error;
@@ -1186,6 +1287,10 @@ void test_mixed_auth_corpus_fixture_file() {
11861287
"expected ten unknown-timestamp failures");
11871288
expect(failure_category_count(result.quality, loglens::ParserFailureCategory::UnsupportedPamVariant) == 10,
11881289
"expected ten unsupported-PAM-variant failures");
1290+
1291+
const auto actual = mixed_auth_coverage_json(result);
1292+
const auto expected = read_text_file(asset_path("mixed_auth_parser_coverage.json"));
1293+
expect(actual == expected, "expected mixed auth parser coverage artifact to match fixture");
11891294
}
11901295

11911296
} // namespace

0 commit comments

Comments
 (0)