GSoC 2026 Module B — Week 4: evaluation harness + recall-first gold corrections + prompt tuning (OWASP#976)

* feat(module-b): add Stage 2 evaluation harness

* chore(module-b): align gold labels to recall-first (worked examples + heading-only)

* feat(module-b): sharpen Stage 2 prompt for organizational meta-content

* fix(module-b): guard eval report against empty dataset

Author: manshusainishab

application/tests/noise_filter/fixtures/labeled_data.json

@@ -27,8 +27,8 @@
       "id": "document/4-Web_Application_Security_Testing/02-Configuration_and_Deployment_Management_Testing/10-Test_for_Subdomain_Takeover.md",
       "path": "document/4-Web_Application_Security_Testing/02-Configuration_and_Deployment_Management_Testing/10-Test_for_Subdomain_Takeover.md"
     },
-    "label": "KNOWLEDGE",
-    "label_rationale": "",
+    "label": "NOISE",
+    "label_rationale": "Heading/title-only chunk with no methodology body [heading-only tiny-chunk -> NOISE, consistent rule 2026-07-03; was KNOWLEDGE]",
     "labeled_by": "manshusainishab",
     "labeled_at": "2026-05-29"
   },
@@ -96,8 +96,8 @@
       "id": "document/4-Web_Application_Security_Testing/02-Configuration_and_Deployment_Management_Testing/10-Test_for_Subdomain_Takeover.md",
       "path": "document/4-Web_Application_Security_Testing/02-Configuration_and_Deployment_Management_Testing/10-Test_for_Subdomain_Takeover.md"
     },
-    "label": "UNCERTAIN",
-    "label_rationale": "borderline: structured as an example but is the primary methodology unit for testing GitHub-specific subdomain takeover. Prompt iteration must clarify whether canonical WSTG worked examples count as KNOWLEDGE or as \"additional example\" NOISE.",
+    "label": "KNOWLEDGE",
+    "label_rationale": "Worked attack-scenario walkthrough (GitHub subdomain takeover) -- security content [recall-first correction 2026-07-03; was UNCERTAIN]",
     "labeled_by": "manshusainishab",
     "labeled_at": "2026-05-29"
   },
@@ -131,8 +131,8 @@
       "id": "document/4-Web_Application_Security_Testing/02-Configuration_and_Deployment_Management_Testing/10-Test_for_Subdomain_Takeover.md",
       "path": "document/4-Web_Application_Security_Testing/02-Configuration_and_Deployment_Management_Testing/10-Test_for_Subdomain_Takeover.md"
     },
-    "label": "UNCERTAIN",
-    "label_rationale": "same case as chunk 3 — scenario walkthrough under Summary. Linked for prompt-iteration consistency.",
+    "label": "KNOWLEDGE",
+    "label_rationale": "Worked attack-scenario walkthrough (expired-domain subdomain takeover) [recall-first correction 2026-07-03; was UNCERTAIN]",
     "labeled_by": "manshusainishab",
     "labeled_at": "2026-05-29"
   },
@@ -548,8 +548,8 @@
       "id": "document/4-Web_Application_Security_Testing/04-Authentication_Testing/11-Testing_Multi-Factor_Authentication.md",
       "path": "document/4-Web_Application_Security_Testing/04-Authentication_Testing/11-Testing_Multi-Factor_Authentication.md"
     },
-    "label": "KNOWLEDGE",
-    "label_rationale": "",
+    "label": "NOISE",
+    "label_rationale": "Heading/title-only chunk with no methodology body [heading-only tiny-chunk -> NOISE, consistent rule 2026-07-03; was KNOWLEDGE]",
     "labeled_by": "manshusainishab",
     "labeled_at": "2026-05-29"
   },
@@ -1748,8 +1748,8 @@
       "id": "cheatsheets/DOM_based_XSS_Prevention_Cheat_Sheet.md",
       "path": "cheatsheets/DOM_based_XSS_Prevention_Cheat_Sheet.md"
     },
-    "label": "KNOWLEDGE",
-    "label_rationale": "",
+    "label": "NOISE",
+    "label_rationale": "Heading/title-only chunk with no methodology body [heading-only tiny-chunk -> NOISE, consistent rule 2026-07-03; was KNOWLEDGE]",
     "labeled_by": "manshusainishab",
     "labeled_at": "2026-05-29"
   },
@@ -1851,8 +1851,8 @@
       "id": "cheatsheets/DOM_based_XSS_Prevention_Cheat_Sheet.md",
       "path": "cheatsheets/DOM_based_XSS_Prevention_Cheat_Sheet.md"
     },
-    "label": "KNOWLEDGE",
-    "label_rationale": "",
+    "label": "NOISE",
+    "label_rationale": "Heading/title-only chunk with no methodology body [heading-only tiny-chunk -> NOISE, consistent rule 2026-07-03; was KNOWLEDGE]",
     "labeled_by": "manshusainishab",
     "labeled_at": "2026-05-29"
   },
@@ -1887,8 +1887,8 @@
       "id": "cheatsheets/DOM_based_XSS_Prevention_Cheat_Sheet.md",
       "path": "cheatsheets/DOM_based_XSS_Prevention_Cheat_Sheet.md"
     },
-    "label": "UNCERTAIN",
-    "label_rationale": "DOM sinks (innerHTML/outerHTML) framed as examples but functionally an attack-vector catalog. Same pattern as chunks 3-4; needs prompt clarification on canonical-primitive vs illustrative-example.",
+    "label": "KNOWLEDGE",
+    "label_rationale": "Catalog of dangerous DOM sinks (innerHTML/outerHTML) = XSS attack vectors [recall-first correction 2026-07-03; was UNCERTAIN]",
     "labeled_by": "manshusainishab",
     "labeled_at": "2026-05-29"
   },
@@ -1923,8 +1923,8 @@
       "id": "cheatsheets/DOM_based_XSS_Prevention_Cheat_Sheet.md",
       "path": "cheatsheets/DOM_based_XSS_Prevention_Cheat_Sheet.md"
     },
-    "label": "UNCERTAIN",
-    "label_rationale": "same case as chunk 55 — DOM sinks framed as examples. Linked for prompt-iteration consistency.",
+    "label": "KNOWLEDGE",
+    "label_rationale": "Catalog of dangerous DOM methods (document.write/writeln) = XSS attack vectors [recall-first correction 2026-07-03; was UNCERTAIN]",
     "labeled_by": "manshusainishab",
     "labeled_at": "2026-05-29"
   },
@@ -3068,8 +3068,8 @@
       "id": "Website/content/en/user-day/owasp-samm-to-the-rescue.md",
       "path": "Website/content/en/user-day/owasp-samm-to-the-rescue.md"
     },
-    "label": "UNCERTAIN",
-    "label_rationale": "talk abstract with substantive attack content (AWS CodeBuild StartBuild pipeline poisoning) wrapped in event-page framing. Prompt iteration must clarify whether talk teasers with named attack techniques count as KNOWLEDGE.",
+    "label": "KNOWLEDGE",
+    "label_rationale": "Talk abstract naming a concrete technique (CodeBuild pipeline poisoning) [recall-first correction 2026-07-03; was UNCERTAIN]",
     "labeled_by": "manshusainishab",
     "labeled_at": "2026-05-29"
   },

application/tests/noise_filter/llm_classifier_test.py

@@ -83,7 +83,7 @@ def test_all_few_shot_examples_embedded(self) -> None:
     def test_few_shot_label_distribution(self) -> None:
         labels = [e["label"] for e in FEW_SHOT_EXAMPLES]
         self.assertEqual(labels.count("KNOWLEDGE"), 5)
-        self.assertEqual(labels.count("NOISE"), 3)
+        self.assertEqual(labels.count("NOISE"), 5)
         self.assertEqual(labels.count("UNCERTAIN"), 2)
 
 

application/utils/noise_filter/prompts.py

@@ -27,9 +27,15 @@
 configurations, EVEN clarifications, typo fixes, expanded examples, \
 restatements of well-known material, added reference links, or restructured \
 security sections. When in doubt, choose KNOWLEDGE.
-- NOISE: ONLY chunks with no security signal at all -- sponsorship pages, \
-meeting notes, CI/build configuration, release tags, website layout, \
-contributor onboarding, project governance.
+- NOISE: ONLY chunks with no security-knowledge content at all -- sponsorship \
+pages, meeting notes, CI/build configuration, release tags, website layout, \
+contributor onboarding, project governance. This includes content *about* a \
+security project rather than security knowledge itself: version/release \
+announcements, licenses, translation notes, citation/reference-format \
+instructions, tables of contents, project titles/banners, and event/talk \
+teasers -- even when they name a security standard or project (e.g. "ASVS", \
+"SAMM"). Naming a security project is not the same as containing security \
+knowledge.
 - UNCERTAIN: only when genuinely 50/50 even after applying the bias above. \
 This label is rare.
 
@@ -133,6 +139,33 @@
         "confidence": 0.95,
         "reasoning": "CI release-tagging workflow; build infrastructure, not security knowledge.",
     },
+    {
+        "heading_path": [
+            "OWASP Application Security Verification Standard",
+            "Latest Stable Version",
+        ],
+        "text": (
+            "The latest stable version is 5.0.0 (dated May 2025), available as a "
+            "PDF and in the GitHub repository. See the release notes for changes "
+            "since 4.0."
+        ),
+        "label": "NOISE",
+        "confidence": 0.9,
+        "reasoning": "Version/release announcement about a security standard -- content about the project, not security knowledge.",
+    },
+    {
+        "heading_path": [
+            "OWASP Application Security Verification Standard",
+            "License",
+        ],
+        "text": (
+            "The entire project content is under the Creative Commons "
+            "Attribution-Share Alike v4.0 license."
+        ),
+        "label": "NOISE",
+        "confidence": 0.95,
+        "reasoning": "Licensing information about a security project -- organizational, no security methodology.",
+    },
     {
         "heading_path": ["About"],
         "text": (