Normalize OWASP cheat sheet references

Author: Bornunique911

application/tests/cheatsheets_parser_test.py

@@ -69,8 +69,37 @@ class Repo:
         self.maxDiff = None
         for name, nodes in entries.results.items():
             self.assertEqual(name, parser.name)
-            self.assertEqual(len(nodes), 1)
-            self.assertEqual(expected.todict(), nodes[0].todict())
+            sections = {node.section for node in nodes}
+            self.assertIn("Secrets Management Cheat Sheet", sections)
+            secret_entry = [
+                node
+                for node in nodes
+                if node.section == "Secrets Management Cheat Sheet"
+            ][0]
+            self.assertEqual(expected.todict(), secret_entry.todict())
+
+    def test_register_supplemental_cheatsheets(self) -> None:
+        for cre_id, name in [
+            ("118-110", "API/web services"),
+            ("724-770", "Technical application access control"),
+            ("623-550", "Denial Of Service protection"),
+        ]:
+            self.collection.add_cre(defs.CRE(name=name, id=cre_id))
+
+        entries = cheatsheets_parser.Cheatsheets().register_supplemental_cheatsheets(
+            cache=self.collection
+        )
+        rest = [
+            entry for entry in entries if entry.section == "REST Security Cheat Sheet"
+        ][0]
+        self.assertEqual(
+            "https://cheatsheetseries.owasp.org/cheatsheets/REST_Security_Cheat_Sheet.html",
+            rest.hyperlink,
+        )
+        self.assertEqual(
+            ["118-110", "724-770", "623-550"],
+            [link.document.id for link in rest.links],
+        )
 
     cheatsheets_md = """ # Secrets Management Cheat Sheet
 

application/utils/external_project_parsers/data/owasp_cheatsheets_supplement.json

@@ -0,0 +1,47 @@
+[
+  {
+    "section": "Authorization Cheat Sheet",
+    "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Authorization_Cheat_Sheet.html",
+    "cre_ids": ["128-128", "117-371"]
+  },
+  {
+    "section": "REST Security Cheat Sheet",
+    "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/REST_Security_Cheat_Sheet.html",
+    "cre_ids": ["118-110", "724-770", "623-550"]
+  },
+  {
+    "section": "Server Side Request Forgery Prevention Cheat Sheet",
+    "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Server_Side_Request_Forgery_Prevention_Cheat_Sheet.html",
+    "cre_ids": ["028-728", "657-084"]
+  },
+  {
+    "section": "Docker Security Cheat Sheet",
+    "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Docker_Security_Cheat_Sheet.html",
+    "cre_ids": ["233-748", "486-813"]
+  },
+  {
+    "section": "Kubernetes Security Cheat Sheet",
+    "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Kubernetes_Security_Cheat_Sheet.html",
+    "cre_ids": ["467-784", "233-748", "486-813"]
+  },
+  {
+    "section": "Secure Cloud Architecture Cheat Sheet",
+    "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Secure_Cloud_Architecture_Cheat_Sheet.html",
+    "cre_ids": ["155-155", "467-784"]
+  },
+  {
+    "section": "LLM Prompt Injection Prevention Cheat Sheet",
+    "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/LLM_Prompt_Injection_Prevention_Cheat_Sheet.html",
+    "cre_ids": ["161-451", "760-764"]
+  },
+  {
+    "section": "AI Agent Security Cheat Sheet",
+    "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/AI_Agent_Security_Cheat_Sheet.html",
+    "cre_ids": ["117-371", "650-560", "126-668"]
+  },
+  {
+    "section": "Secure AI Model Ops Cheat Sheet",
+    "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Secure_AI_Model_Ops_Cheat_Sheet.html",
+    "cre_ids": ["148-853", "613-285", "613-287"]
+  }
+]

application/utils/external_project_parsers/parsers/cheatsheets_parser.py

@@ -6,6 +6,9 @@
 import os
 import re
 from application.utils.external_project_parsers import base_parser_defs
+import json
+from pathlib import Path
+import logging
 from application.utils.external_project_parsers.base_parser_defs import (
     ParserInterface,
     ParseResult,
@@ -16,6 +19,12 @@
 class Cheatsheets(ParserInterface):
     name = "OWASP Cheat Sheets"
     cheatsheetseries_base_url = "https://cheatsheetseries.owasp.org/cheatsheets"
+    supplement_data_file = (
+        Path(__file__).resolve().parent.parent
+        / "data"
+        / "owasp_cheatsheets_supplement.json"
+    )
+    logger = logging.getLogger(__name__)
 
     def cheatsheet(
         self, section: str, hyperlink: str, tags: List[str]
@@ -41,10 +50,22 @@ def official_cheatsheet_url(self, markdown_filename: str) -> str:
     def parse(self, cache: db.Node_collection, ph: prompt_client.PromptHandler):
         c_repo = "https://github.com/OWASP/CheatSheetSeries.git"
         cheatsheets_path = "cheatsheets/"
-        repo = git.clone(c_repo, sparse_paths=["cheatsheets"], sparse_cone=True)
-        cheatsheets = self.register_cheatsheets(
-            repo=repo, cache=cache, cheatsheets_path=cheatsheets_path, repo_path=c_repo
-        )
+        cheatsheets = []
+        try:
+            repo = git.clone(c_repo, sparse_paths=["cheatsheets"], sparse_cone=True)
+            cheatsheets = self.register_cheatsheets(
+                repo=repo,
+                cache=cache,
+                cheatsheets_path=cheatsheets_path,
+                repo_path=c_repo,
+            )
+        except Exception as exc:
+            self.logger.warning(
+                "Unable to clone OWASP CheatSheetSeries, continuing with supplemental cheat sheets only: %s",
+                exc,
+            )
+        cheatsheets.extend(self.register_supplemental_cheatsheets(cache=cache))
+        cheatsheets = self.deduplicate_entries(cheatsheets)
         results = {self.name: cheatsheets}
         base_parser_defs.validate_classification_tags(results)
         return ParseResult(results=results)
@@ -80,3 +101,36 @@ def register_cheatsheets(
                         )
                     standard_entries.append(cs)
         return standard_entries
+
+    def register_supplemental_cheatsheets(self, cache: db.Node_collection):
+        with self.supplement_data_file.open("r", encoding="utf-8") as handle:
+            supplement_entries = json.load(handle)
+
+        standard_entries = []
+        for entry in supplement_entries:
+            cs = self.cheatsheet(
+                section=entry["section"],
+                hyperlink=entry["hyperlink"],
+                tags=[],
+            )
+            for cre_id in entry.get("cre_ids", []):
+                cres = cache.get_CREs(external_id=cre_id)
+                for cre in cres:
+                    try:
+                        cs.add_link(
+                            defs.Link(
+                                document=cre.shallow_copy(),
+                                ltype=defs.LinkTypes.AutomaticallyLinkedTo,
+                            )
+                        )
+                    except Exception:
+                        continue
+            if cs.links:
+                standard_entries.append(cs)
+        return standard_entries
+
+    def deduplicate_entries(self, entries: List[defs.Standard]) -> List[defs.Standard]:
+        deduped = {}
+        for entry in entries:
+            deduped[(entry.section, entry.hyperlink)] = entry
+        return list(deduped.values())