Skip to content

Commit c067497

Browse files
skypank-coderskypank-coder
authored
Merge branch 'main' into feat/frontend-useuser-header-auth
2 parents 1f8c74a + 0e16c2e commit c067497

3 files changed

Lines changed: 484 additions & 20 deletions

File tree

‎application/tests/cheatsheet_extractor_test.py‎

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
import unittest
2+
from application.utils.external_project_parsers.parsers.cheatsheet_extractor import (
3+
extract_cheatsheet_record,
4+
)
5+
from application.defs.cheatsheet_defs import SUMMARY_MAX_LENGTH
6+
7+
SOURCE_PATH = "cheatsheets/Secrets_Management_Cheat_Sheet.md"
8+
EXPECTED_SOURCE_ID = "Secrets_Management_Cheat_Sheet"
9+
EXPECTED_HYPERLINK = (
10+
"https://cheatsheetseries.owasp.org/cheatsheets/Secrets_Management_Cheat_Sheet.html"
11+
)
12+
13+
NORMAL_MD = """\
14+
# Secrets Management Cheat Sheet
15+
16+
## Introduction
17+
Storage guidance.
18+
19+
## Architectural Patterns
20+
Use vaults and environment isolation.
21+
"""
22+
23+
MISSING_H1_MD = """\
24+
## Introduction
25+
No H1 present.
26+
27+
## Details
28+
More content.
29+
"""
30+
31+
EMPTY_MD = ""
32+
33+
# No ## Introduction heading exists, so summary extraction
34+
# falls back to the first available body content.
35+
BODY_UNDER_H1_MD = """\
36+
# Single Heading Cheat Sheet
37+
38+
Body text directly under H1, no subheadings at all.
39+
"""
40+
41+
# Leading whitespace before H1 and malformed ## headings
42+
# should still be normalized and extracted correctly.
43+
MALFORMED_MD = """\
44+
# Malformed Title
45+
46+
##malformed
47+
48+
## Introduction
49+
Some intro text.
50+
51+
## Valid Heading
52+
"""
53+
54+
55+
class TestNormal(unittest.TestCase):
56+
def setUp(self):
57+
self.record = extract_cheatsheet_record(NORMAL_MD, SOURCE_PATH)
58+
59+
# source-derived fields should remain deterministic and
60+
# independent of markdown content across all extraction paths.
61+
def test_source(self):
62+
self.assertEqual(self.record.source, "owasp_cheatsheets")
63+
64+
def test_source_id(self):
65+
self.assertEqual(self.record.source_id, EXPECTED_SOURCE_ID)
66+
67+
def test_hyperlink(self):
68+
self.assertEqual(self.record.hyperlink, EXPECTED_HYPERLINK)
69+
70+
def test_raw_markdown_path(self):
71+
self.assertEqual(self.record.raw_markdown_path, SOURCE_PATH)
72+
73+
def test_title(self):
74+
self.assertEqual(self.record.title, "Secrets Management Cheat Sheet")
75+
76+
def test_summary(self):
77+
self.assertEqual(self.record.summary, "Storage guidance.")
78+
79+
def test_summary_bounded(self):
80+
# Summary truncation is enforced centrally via
81+
# CheatsheetRecord.__post_init__.
82+
self.assertLessEqual(len(self.record.summary), SUMMARY_MAX_LENGTH)
83+
84+
def test_headings(self):
85+
self.assertIn("Introduction", self.record.headings)
86+
self.assertIn("Architectural Patterns", self.record.headings)
87+
88+
def test_fallback_not_used(self):
89+
self.assertEqual(self.record.metadata["fallback_used"], "false")
90+
91+
92+
class TestMissingH1(unittest.TestCase):
93+
def setUp(self):
94+
self.record = extract_cheatsheet_record(MISSING_H1_MD, SOURCE_PATH)
95+
96+
def test_title_is_fallback(self):
97+
self.assertEqual(self.record.title, "No title found.")
98+
99+
def test_summary_from_introduction(self):
100+
self.assertIn("no h1", self.record.summary.lower())
101+
102+
def test_headings_extracted(self):
103+
self.assertIn("Introduction", self.record.headings)
104+
self.assertIn("Details", self.record.headings)
105+
106+
def test_fallback_used(self):
107+
self.assertEqual(self.record.metadata["fallback_used"], "true")
108+
109+
110+
class TestEmptyMarkdown(unittest.TestCase):
111+
def setUp(self):
112+
self.record = extract_cheatsheet_record(EMPTY_MD, SOURCE_PATH)
113+
114+
def test_title_is_fallback(self):
115+
self.assertEqual(self.record.title, "No title found.")
116+
117+
def test_summary_no_summary_found(self):
118+
# Empty markdown should trigger terminal summary fallback.
119+
self.assertEqual(self.record.summary, "No summary found.")
120+
121+
def test_headings_empty(self):
122+
self.assertEqual(self.record.headings, [])
123+
124+
def test_fallback_used(self):
125+
self.assertEqual(self.record.metadata["fallback_used"], "true")
126+
127+
128+
class TestBodyUnderH1(unittest.TestCase):
129+
def setUp(self):
130+
self.record = extract_cheatsheet_record(BODY_UNDER_H1_MD, SOURCE_PATH)
131+
132+
def test_title(self):
133+
self.assertEqual(self.record.title, "Single Heading Cheat Sheet")
134+
135+
def test_summary_from_fallback_via_h1(self):
136+
# Summary fallback should extract body content beneath the H1 section.
137+
self.assertIn("body text", self.record.summary.lower())
138+
139+
def test_headings_empty(self):
140+
# No valid ## headings should produce an empty headings list.
141+
self.assertEqual(self.record.headings, [])
142+
143+
def test_fallback_used(self):
144+
self.assertEqual(self.record.metadata["fallback_used"], "true")
145+
146+
147+
class TestMalformedHeadings(unittest.TestCase):
148+
def setUp(self):
149+
self.record = extract_cheatsheet_record(MALFORMED_MD, SOURCE_PATH)
150+
151+
def test_malformed_h1_extracted(self):
152+
self.assertEqual(self.record.title, "Malformed Title")
153+
154+
def test_malformed_h2_in_headings(self):
155+
self.assertIn("malformed", self.record.headings)
156+
157+
def test_valid_headings_also_extracted(self):
158+
self.assertIn("Introduction", self.record.headings)
159+
self.assertIn("Valid Heading", self.record.headings)
160+
161+
def test_summary_from_introduction(self):
162+
self.assertIn("intro", self.record.summary.lower())
163+
164+
def test_fallback_not_used(self):
165+
self.assertEqual(self.record.metadata["fallback_used"], "false")
166+
167+
168+
if __name__ == "__main__":
169+
unittest.main()

‎application/utils/external_project_parsers/parsers/cheatsheet_extractor.py‎

Lines changed: 59 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,27 @@
1+
import logging
12
import os
23
import re
34

45
from application.defs.cheatsheet_defs import CheatsheetRecord
56

67
PARSER_VERSION = "v1"
7-
FALLBACK_USED = "false"
88

99
CANONICAL_BASE_URL = "https://cheatsheetseries.owasp.org/cheatsheets/"
1010

11-
_TITLE_RE = re.compile(r"^#\s+(?P<title>.+)$", re.MULTILINE)
12-
_HEADING_RE = re.compile(r"^##\s+(?P<heading>.+)$", re.MULTILINE)
13-
_ANY_HEADING_RE = re.compile(r"^#{1,6}\s+.+$", re.MULTILINE)
11+
_TITLE_RE = re.compile(
12+
r"^\s*#(?!#)\s*(?P<title>.+?)$",
13+
re.MULTILINE,
14+
)
15+
16+
_HEADING_RE = re.compile(
17+
r"^\s*##(?!#)\s*(?P<heading>.+?)$",
18+
re.MULTILINE,
19+
)
20+
21+
_ANY_HEADING_RE = re.compile(
22+
r"^\s*#{1,6}(?!#)\s*.+?$",
23+
re.MULTILINE,
24+
)
1425

1526

1627
def _derive_source_id(source_path: str) -> str:
@@ -41,28 +52,44 @@ def _extract_body_after_heading(markdown: str, heading_match: re.Match) -> str:
4152

4253

4354
def _extract_summary(markdown: str) -> str:
44-
"""Extract a summary section from cheatsheet markdown."""
45-
46-
all_heading_matches = list(_ANY_HEADING_RE.finditer(markdown))
55+
"""Extract summary from Introduction section in cheatsheet markdown."""
4756

48-
for match in all_heading_matches:
49-
heading_text = match.group().lstrip("#").strip()
50-
51-
if heading_text.lower() == "introduction":
57+
for match in _ANY_HEADING_RE.finditer(markdown):
58+
if match.group().strip().lstrip("#").strip().lower() == "introduction":
5259
body = _extract_body_after_heading(markdown, match)
53-
5460
if body:
5561
return body
5662

57-
break
63+
raise ValueError(
64+
"_extract_summary: no suitable summary section could be extracted from markdown."
65+
)
5866

59-
for match in all_heading_matches:
60-
body = _extract_body_after_heading(markdown, match)
6167

68+
def _extract_title(markdown: str) -> str:
69+
"""Extract H1 title from cheatsheet markdown."""
70+
71+
match = _TITLE_RE.search(markdown)
72+
if not match:
73+
raise ValueError("_extract_title: no title found in markdown.")
74+
75+
return match.group("title").strip()
76+
77+
78+
def _fallback_title() -> str:
79+
"""Return fallback title for malformed markdown."""
80+
81+
return "No title found."
82+
83+
84+
def _fallback_summary(markdown: str) -> str:
85+
"""Return first non-empty paragraph after any heading, or 'No summary found.'"""
86+
87+
for match in _ANY_HEADING_RE.finditer(markdown):
88+
body = _extract_body_after_heading(markdown, match)
6289
if body:
6390
return body
6491

65-
raise ValueError("_extract_summary: no summary could be extracted from markdown.")
92+
return "No summary found."
6693

6794

6895
def extract_cheatsheet_record(
@@ -71,12 +98,24 @@ def extract_cheatsheet_record(
7198
) -> CheatsheetRecord:
7299
"""Extract a structured CheatsheetRecord from markdown content."""
73100

74-
title_match = _TITLE_RE.search(markdown)
75-
title = title_match.group("title").strip()
101+
fallback_used = "false"
102+
103+
try:
104+
title = _extract_title(markdown)
105+
except ValueError as e:
106+
logging.warning(str(e))
107+
title = _fallback_title()
108+
fallback_used = "true"
76109

110+
# Headings can be empty.
77111
headings = [m.group("heading").strip() for m in _HEADING_RE.finditer(markdown)]
78112

79-
summary = _extract_summary(markdown)
113+
try:
114+
summary = _extract_summary(markdown)
115+
except ValueError as e:
116+
logging.warning(str(e))
117+
summary = _fallback_summary(markdown)
118+
fallback_used = "true"
80119

81120
source_id = _derive_source_id(source_path)
82121
hyperlink = _derive_hyperlink(source_path)
@@ -91,6 +130,6 @@ def extract_cheatsheet_record(
91130
category_hints=[],
92131
metadata={
93132
"parser_version": PARSER_VERSION,
94-
"fallback_used": FALLBACK_USED,
133+
"fallback_used": fallback_used,
95134
},
96135
)

0 commit comments

Comments
 (0)