Skip to content

Commit ccbfc51

Browse files
MDA2AVclaude
andcommitted
Fix STRICT evaluator to match C# scoring: exclude Warn from scored count
The C# probe dynamically excludes Warn-verdict tests from the score. The STRICT dict was using static scored:False flags that disagreed. Now the Python evaluator excludes Warn verdicts from the scored count (matching C# behavior), and removes unnecessary scored:False flags. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent d349f4d commit ccbfc51

1 file changed

Lines changed: 14 additions & 16 deletions

File tree

.github/workflows/probe.yml

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -314,31 +314,31 @@ jobs:
314314
},
315315
'SMUG-CL-TRAILING-SPACE': {
316316
'accept': [400], 'close_ok': True, 'timeout_ok': False,
317-
'warn_on_2xx': True, 'scored': False,
317+
'warn_on_2xx': True,
318318
'expected': '400 or 2xx',
319319
'reason': 'Trailing space in CL — OWS trimming is valid per RFC 9110 §5.5'
320320
},
321321
'SMUG-HEADER-INJECTION': {
322322
'accept': [400], 'close_ok': True, 'timeout_ok': False,
323-
'warn_on_2xx': True, 'scored': False,
323+
'warn_on_2xx': True,
324324
'expected': '400 or 2xx',
325325
'reason': 'Payload is two valid headers on the wire — 2xx is RFC-compliant'
326326
},
327327
'SMUG-TE-DOUBLE-CHUNKED': {
328328
'accept': [400], 'close_ok': True, 'timeout_ok': False,
329-
'warn_on_2xx': True, 'scored': False,
329+
'warn_on_2xx': True,
330330
'expected': '400 or 2xx',
331331
'reason': 'Duplicate chunked TE with CL — 4xx is strict, 2xx is tolerable'
332332
},
333333
'SMUG-CL-EXTRA-LEADING-SP': {
334334
'accept': [400], 'close_ok': True, 'timeout_ok': False,
335-
'warn_on_2xx': True, 'scored': False,
335+
'warn_on_2xx': True,
336336
'expected': '400 or 2xx',
337337
'reason': 'Extra OWS after colon is valid per RFC 9110 §5.5'
338338
},
339339
'SMUG-TE-CASE-MISMATCH': {
340340
'accept': [400], 'close_ok': True, 'timeout_ok': False,
341-
'warn_on_2xx': True, 'scored': False,
341+
'warn_on_2xx': True,
342342
'expected': '400 or 2xx',
343343
'reason': 'Case-insensitive TE matching is valid per RFC — 2xx is compliant'
344344
},
@@ -411,19 +411,19 @@ jobs:
411411
},
412412
'COMP-LEADING-CRLF': {
413413
'accept': [400], 'close_ok': True, 'timeout_ok': False,
414-
'warn_on_2xx': True, 'scored': False,
414+
'warn_on_2xx': True,
415415
'expected': '400 or 2xx',
416416
'reason': 'Leading CRLF — server MAY ignore per RFC 9112 §2.2'
417417
},
418418
'COMP-ABSOLUTE-FORM': {
419419
'accept': [400], 'close_ok': True, 'timeout_ok': False,
420-
'warn_on_2xx': True, 'scored': False,
420+
'warn_on_2xx': True,
421421
'expected': '400 or 2xx',
422422
'reason': 'Absolute-form is valid per RFC 9112 §3.2.2'
423423
},
424424
'COMP-METHOD-CASE': {
425425
'accept': [400, 405, 501], 'close_ok': True, 'timeout_ok': False,
426-
'warn_on_2xx': True, 'scored': False,
426+
'warn_on_2xx': True,
427427
'expected': '400/405/501 or 2xx',
428428
'reason': 'Methods are case-sensitive per RFC 9110 §9.1 — 2xx means case-insensitive'
429429
},
@@ -523,7 +523,7 @@ jobs:
523523
},
524524
'SMUG-CHUNKED-WITH-PARAMS': {
525525
'accept': [400], 'close_ok': True, 'timeout_ok': False,
526-
'warn_on_2xx': True, 'scored': False,
526+
'warn_on_2xx': True,
527527
'expected': '400 or 2xx',
528528
'reason': 'Parameters on chunked encoding — some servers ignore'
529529
},
@@ -561,7 +561,7 @@ jobs:
561561
},
562562
'MAL-CL-TAB-BEFORE-VALUE': {
563563
'accept': [400], 'close_ok': True, 'timeout_ok': False,
564-
'warn_on_2xx': True, 'scored': False,
564+
'warn_on_2xx': True,
565565
'expected': '400 or 2xx',
566566
'reason': 'Tab as OWS is valid per RFC 9110 §5.5 — 2xx is compliant'
567567
},
@@ -601,10 +601,9 @@ jobs:
601601
'reason': 'Unknown Expect value — 417 is correct, 2xx means ignored (RFC 9110 §10.1.1)'
602602
},
603603
'COMP-UPGRADE-INVALID-VER': {
604-
'accept': [c for c in range(100, 600) if c != 101],
605-
'close_ok': True, 'timeout_ok': False,
606-
'scored': False,
607-
'expected': '!101',
604+
'accept': [426], 'close_ok': True, 'timeout_ok': False,
605+
'warn_on_2xx': True,
606+
'expected': '426 or 2xx',
608607
'reason': 'Unsupported WebSocket version — 426 is correct, 101 is failure (RFC 6455 §4.4)'
609608
},
610609
'COMP-METHOD-TRACE': {
@@ -736,7 +735,6 @@ jobs:
736735
'COMP-CHUNKED-EXTENSION': {
737736
'accept': list(range(200, 300)) + [400],
738737
'close_ok': True, 'timeout_ok': False,
739-
'scored': False,
740738
'expected': '2xx or 400',
741739
'reason': 'Chunk extensions are valid per RFC 9112 §7.1.1 — 400 means unsupported'
742740
},
@@ -784,7 +782,7 @@ jobs:
784782
'durationMs': r.get('durationMs', 0),
785783
})
786784
787-
scored_results = [r for r in results if r['scored']]
785+
scored_results = [r for r in results if r['scored'] and r['verdict'] != 'Warn']
788786
total = len(scored_results)
789787
passed = sum(1 for r in scored_results if r['verdict'] == 'Pass')
790788
warned = sum(1 for r in results if r['verdict'] == 'Warn')

0 commit comments

Comments
 (0)