feat: implement structured extraction checkpoints B1 and B2 (#912)

* feat: implement structured extraction checkpoints B1 and B2

Signed-off-by: Abhijeet Saharan <abhijeetsaharan2236@gmail.com>

* docs: improve formatting

Signed-off-by: Abhijeet <abhijeetsaharan2236@gmail.com>

* fix: improve normalization of required string fields

Signed-off-by: Abhijeet <abhijeetsaharan2236@gmail.com>

* docs: add docstrings

Signed-off-by: Abhijeet Saharan <abhijeetsaharan2236@gmail.com>

* fix: validate normalized string field values correctly

Signed-off-by: Abhijeet Saharan <abhijeetsaharan2236@gmail.com>

---------

Signed-off-by: Abhijeet Saharan <abhijeetsaharan2236@gmail.com>
Signed-off-by: Abhijeet <abhijeetsaharan2236@gmail.com>

Author: Abhijeet2409

application/defs/cheatsheet_defs.py

@@ -0,0 +1,81 @@
+from dataclasses import dataclass, field
+from typing import Dict, List
+
+SUMMARY_MAX_LENGTH = 500
+
+
+@dataclass
+class CheatsheetRecord:
+    """Structured representation of an OWASP cheatsheet record."""
+
+    source: str = field(default="owasp_cheatsheets", init=False)
+    source_id: str
+    title: str
+    hyperlink: str
+    summary: str
+    headings: List[str]
+    raw_markdown_path: str
+    category_hints: List[str] = field(default_factory=list)
+    metadata: Dict[str, str] = field(default_factory=dict)
+
+    def __post_init__(self):
+        """Normalize and validate CheatsheetRecord fields."""
+
+        required_str_fields = {
+            "source_id": self.source_id,
+            "title": self.title,
+            "hyperlink": self.hyperlink,
+            "summary": self.summary,
+            "raw_markdown_path": self.raw_markdown_path,
+        }
+
+        # Summary-specific normalization
+        self.summary = self.summary.strip()[:SUMMARY_MAX_LENGTH]
+
+        # Normalize fields which require string values.
+        for field_name, value in required_str_fields.items():
+            if isinstance(value, str):
+                setattr(self, field_name, value.strip())
+
+        list_str_fields = {
+            "headings": self.headings,
+            "category_hints": self.category_hints,
+        }
+
+        # Validate fields which require string values.
+        for field_name in required_str_fields:
+            value = getattr(self, field_name)
+
+            if not isinstance(value, str) or not value:
+                raise ValueError(
+                    f"CheatsheetRecord: field '{field_name}' "
+                    f"must be a non-empty string, got {value!r}"
+                )
+
+        # Validate fields which require list[str] values.
+        for field_name, value in list_str_fields.items():
+            if not isinstance(value, list):
+                raise ValueError(
+                    f"CheatsheetRecord: field '{field_name}' "
+                    f"must be a list[str], got {type(value)!r}"
+                )
+
+            for item in value:
+                if not isinstance(item, str):
+                    raise ValueError(
+                        f"CheatsheetRecord: value of '{field_name}' "
+                        f"must be a string, got {item!r}"
+                    )
+
+        # Validate input for metadata.
+        if not isinstance(self.metadata, dict):
+            raise ValueError(
+                "CheatsheetRecord: field 'metadata' must be a dict[str, str]"
+            )
+
+        for key, value in self.metadata.items():
+            if not isinstance(key, str) or not isinstance(value, str):
+                raise ValueError(
+                    "CheatsheetRecord: metadata keys and values must be strings, "
+                    f"got {key!r}: {value!r}"
+                )

application/utils/external_project_parsers/parsers/cheatsheet_extractor.py

@@ -0,0 +1,96 @@
+import os
+import re
+
+from application.defs.cheatsheet_defs import CheatsheetRecord
+
+PARSER_VERSION = "v1"
+FALLBACK_USED = "false"
+
+CANONICAL_BASE_URL = "https://cheatsheetseries.owasp.org/cheatsheets/"
+
+_TITLE_RE = re.compile(r"^#\s+(?P<title>.+)$", re.MULTILINE)
+_HEADING_RE = re.compile(r"^##\s+(?P<heading>.+)$", re.MULTILINE)
+_ANY_HEADING_RE = re.compile(r"^#{1,6}\s+.+$", re.MULTILINE)
+
+
+def _derive_source_id(source_path: str) -> str:
+    """Derive the cheatsheet source ID from the markdown file path."""
+
+    basename = os.path.basename(source_path)
+    source_id, _ = os.path.splitext(basename)
+
+    return source_id
+
+
+def _derive_hyperlink(source_path: str) -> str:
+    """Generate the canonical OWASP cheatsheet hyperlink."""
+
+    source_id = _derive_source_id(source_path)
+
+    return f"{CANONICAL_BASE_URL}{source_id}.html"
+
+
+def _extract_body_after_heading(markdown: str, heading_match: re.Match) -> str:
+    """Extract body content until the next markdown heading."""
+
+    start = heading_match.end()
+    next_heading = _ANY_HEADING_RE.search(markdown, start)
+    end = next_heading.start() if next_heading else len(markdown)
+
+    return markdown[start:end].strip()
+
+
+def _extract_summary(markdown: str) -> str:
+    """Extract a summary section from cheatsheet markdown."""
+
+    all_heading_matches = list(_ANY_HEADING_RE.finditer(markdown))
+
+    for match in all_heading_matches:
+        heading_text = match.group().lstrip("#").strip()
+
+        if heading_text.lower() == "introduction":
+            body = _extract_body_after_heading(markdown, match)
+
+            if body:
+                return body
+
+            break
+
+    for match in all_heading_matches:
+        body = _extract_body_after_heading(markdown, match)
+
+        if body:
+            return body
+
+    raise ValueError("_extract_summary: no summary could be extracted from markdown.")
+
+
+def extract_cheatsheet_record(
+    markdown: str,
+    source_path: str,
+) -> CheatsheetRecord:
+    """Extract a structured CheatsheetRecord from markdown content."""
+
+    title_match = _TITLE_RE.search(markdown)
+    title = title_match.group("title").strip()
+
+    headings = [m.group("heading").strip() for m in _HEADING_RE.finditer(markdown)]
+
+    summary = _extract_summary(markdown)
+
+    source_id = _derive_source_id(source_path)
+    hyperlink = _derive_hyperlink(source_path)
+
+    return CheatsheetRecord(
+        source_id=source_id,
+        title=title,
+        hyperlink=hyperlink,
+        summary=summary,
+        headings=headings,
+        raw_markdown_path=source_path,
+        category_hints=[],
+        metadata={
+            "parser_version": PARSER_VERSION,
+            "fallback_used": FALLBACK_USED,
+        },
+    )