docs: add checkpoint B5 documentation and refine test comments

Signed-off-by: Abhijeet Saharan <abhijeetsaharan2236@gmail.com>

Author: Abhijeet2409

application/tests/cheatsheet_extractor_test.py

@@ -30,15 +30,16 @@
 
 EMPTY_MD = ""
 
-# No ## headings — _extract_summary raises, _fallback_summary matches # via _ANY_HEADING_RE
-# and extracts body until len(markdown) since there is no next heading.
+# No ## Introduction heading exists, so summary extraction 
+# falls back to the first available body content.
 BODY_UNDER_H1_MD = """\
 # Single Heading Cheat Sheet
 
 Body text directly under H1, no subheadings at all.
 """
 
-# Leading spaces before # and ##malformed (no space) — both handled by \s* and (?!#) regex
+# Leading whitespace before H1 and malformed ## headings 
+# should still be normalized and extracted correctly.
 MALFORMED_MD = """\
    # Malformed Title
 
@@ -55,8 +56,8 @@ class TestNormal(unittest.TestCase):
     def setUp(self):
         self.record = extract_cheatsheet_record(NORMAL_MD, SOURCE_PATH)
 
-    # source, source_id, hyperlink, raw_markdown_path are derived from SOURCE_PATH
-    # and are independent of markdown content — verified once here for all cases
+    # source-derived fields should remain deterministic and
+    # independent of markdown content across all extraction paths.
     def test_source(self):
         self.assertEqual(self.record.source, "owasp_cheatsheets")
 
@@ -76,8 +77,8 @@ def test_summary(self):
         self.assertEqual(self.record.summary, "Storage guidance.")
 
     def test_summary_bounded(self):
-        # SUMMARY_MAX_LENGTH truncation happens in CheatsheetRecord.__post_init__
-        # for every record — testing once here covers all cases
+        # Summary truncation is enforced centrally via
+        # CheatsheetRecord.__post_init__.
         self.assertLessEqual(len(self.record.summary), SUMMARY_MAX_LENGTH)
 
     def test_headings(self):
@@ -114,7 +115,7 @@ def test_title_is_fallback(self):
         self.assertEqual(self.record.title, "No title found.")
 
     def test_summary_no_summary_found(self):
-        # No headings at all — _fallback_summary returns this literal string
+        # Empty markdown should trigger terminal summary fallback.
         self.assertEqual(self.record.summary, "No summary found.")
 
     def test_headings_empty(self):
@@ -132,11 +133,11 @@ def test_title(self):
         self.assertEqual(self.record.title, "Single Heading Cheat Sheet")
 
     def test_summary_from_fallback_via_h1(self):
-        # _fallback_summary matches # heading, extracts body until len(markdown)
+        # Summary fallback should extract body content beneath the H1 section.
         self.assertIn("body text", self.record.summary.lower())
 
     def test_headings_empty(self):
-        # _HEADING_RE only matches ## — no ## present here
+        # No valid ## headings should produce an empty headings list.
         self.assertEqual(self.record.headings, [])
 
     def test_fallback_used(self):

docs/rfc-structured-extraction.md

@@ -0,0 +1,258 @@
+# RFC Workstream B — Structured Extraction
+
+This document explains the implementation and behavior of RFC Workstream B
+(Structured Extraction) from Cheatsheet to CRE Mapping RFC.
+
+The goal of this module is to convert OWASP Cheat Sheet markdown into a
+deterministic structured object that downstream RFC workstreams can consume
+for categorization, retrieval, reranking, and mapping generation.
+
+The implementation is primarily located in:
+
+* `cheatsheet_defs.py`
+* `cheatsheet_extractor.py`
+
+---
+
+## Sources for more context
+
+* RFC:
+  `docs/rfc/cheatsheets-llm-autonomous-mapping-rfc.md`
+
+* Checkpoints B1 & B2 implementation PR:
+  `https://github.com/OWASP/OpenCRE/pull/912`
+
+* Checkpoints B3 & B4 implementation PR:
+  `https://github.com/OWASP/OpenCRE/pull/921`
+
+---
+
+## What Workstream B implements
+
+**The implementation strictly follows the RFC extraction contract and prioritizes deterministic extraction behavior.**
+
+It defines a typed dataclass named `CheatsheetRecord`.
+
+This object represents the structured extraction result returned from:
+
+```python
+extract_cheatsheet_record(markdown, source_path)
+```
+
+The extractor parses OWASP Cheat Sheet markdown and returns normalized
+structured information about a cheatsheet.
+
+`CheatsheetRecord` contains:
+
+* `source`
+* `source_id`
+* `title`
+* `hyperlink`
+* `summary`
+* `headings`
+* `raw_markdown_path`
+* `category_hints`
+* `metadata`
+
+---
+
+## Fallback behavior
+
+The extractor contains fallback functions capable of handling incomplete or
+malformed markdown containing:
+
+* missing titles,
+* missing summary sources,
+* malformed headings.
+
+These fallback paths ensure that extraction still returns a valid
+`CheatsheetRecord` object instead of failing entirely.
+
+Fallback behavior is explicitly surfaced through:
+
+```json
+"metadata": {
+  "fallback_used": "true"
+}
+```
+
+This allows downstream workstreams to identify records that required fallback
+logic during extraction and downstream normalization.
+
+---
+
+## Fallback decision tree
+
+```text
+extract_cheatsheet_record(markdown, source_path)
+
+│
+├── _extract_title(markdown)
+│     ├── H1 title exists
+│     │      → extract and normalize title
+│     │
+│     └── H1 title missing
+│            → _fallback_title()
+│            → "No title found."
+│            → metadata["fallback_used"] = "true"
+│
+└── _extract_summary(markdown)
+      ├── "Introduction" heading exists with body content
+      │      → body beneath "Introduction" extracted as summary
+      │      → summary normalized and truncated upto specifc length.
+      │
+      └── Introduction section missing or invalid
+             → _fallback_summary(markdown)
+             │
+             ├── first heading with body content exists
+             │      → its body returned as summary
+             │
+             └── no usable heading/body content exists
+                    → "No summary found."
+             → metadata["fallback_used"] = "true"
+```
+
+---
+
+## Extraction examples
+
+The following examples demonstrate deterministic extractor behavior across
+different markdown shapes.
+
+Notes:
+
+* Currently, `category_hints` is intentionally returned as an initial empty
+  list during v1.
+
+* `raw_markdown_path`, `hyperlink`, and `source_id` are derived from
+  `source_path` (Module A) and are independent of markdown content.
+
+---
+
+## 1. Normal cheat sheet
+
+### Example Input
+
+```markdown
+# Secrets Management Cheat Sheet
+
+## Introduction
+Storage guidance.
+
+## Architectural Patterns
+Use vaults and environment isolation.
+```
+
+### Output
+
+```json
+{
+  "source": "owasp_cheatsheets",
+  "source_id": "Secrets_Management_Cheat_Sheet",
+  "title": "Secrets Management Cheat Sheet",
+  "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Secrets_Management_Cheat_Sheet.html",
+  "summary": "Storage guidance.",
+  "headings": ["Introduction", "Architectural Patterns"],
+  "raw_markdown_path": "cheatsheets/Secrets_Management_Cheat_Sheet.md",
+  "category_hints": [],
+  "metadata": {
+    "parser_version": "v1",
+    "fallback_used": "false"
+  }
+}
+```
+
+### Notes
+
+* No fallback logic was required.
+
+---
+
+## 2. Missing H1 (fallback title)
+
+### Input
+
+```markdown
+## Introduction
+No H1 present.
+
+## Details
+More content.
+```
+
+### Output
+
+```json
+{
+  "source": "owasp_cheatsheets",
+  "source_id": "Example_Cheat_Sheet",
+  "title": "No title found.",
+  "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Example_Cheat_Sheet.html",
+  "summary": "No H1 present.",
+  "headings": ["Introduction", "Details"],
+  "raw_markdown_path": "cheatsheets/Example_Cheat_Sheet.md",
+  "category_hints": [],
+  "metadata": {
+    "parser_version": "v1",
+    "fallback_used": "true"
+  }
+}
+```
+
+### Notes
+
+* No H1 title exists, so the title defaults to `"No title found."`
+
+---
+
+## 3. Missing Introduction section (summary fallback)
+
+### Input
+
+```markdown
+# Single Heading Cheat Sheet
+
+## Authentication
+
+### Storage
+Secrets should be encrypted.
+```
+
+### Output
+
+```json
+{
+  "source": "owasp_cheatsheets",
+  "source_id": "Single_Heading_Cheat_Sheet",
+  "title": "Single Heading Cheat Sheet",
+  "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Single_Heading_Cheat_Sheet.html",
+  "summary": "Secrets should be encrypted.",
+  "headings": ["Authentication"],
+  "raw_markdown_path": "cheatsheets/Single_Heading_Cheat_Sheet.md",
+  "category_hints": [],
+  "metadata": {
+    "parser_version": "v1",
+    "fallback_used": "true"
+  }
+}
+```
+
+### Notes
+
+* No `Introduction` heading exists, so summary fallback logic is used.
+* The fallback scans all headings and returns the first non-empty body it
+  finds — in this case the content beneath `### Storage`.
+* Only `##`-level headings appear in `headings` — `### Storage` is excluded.
+
+---
+
+## Additional behavior notes
+
+It is ensured that markdown files with malformed title/headings such as:
+
+* With leading whitespace (e.g. `   # My Title`, `   ## My Heading`)
+* No space after the marker (e.g. `##Authentication`)
+
+are extracted correctly.
+
+---