Normalize OWASP cheat sheet references

Author: Bornunique911

application/tests/cheatsheets_parser_test.py

@@ -42,15 +42,44 @@ class Repo:
         )
         expected = defs.Standard(
             name="OWASP Cheat Sheets",
-            hyperlink="https://github.com/foo/bar/tree/master/cs.md",
+            hyperlink="https://cheatsheetseries.owasp.org/cheatsheets/cs.html",
             section="Secrets Management Cheat Sheet",
             links=[defs.Link(document=cre, ltype=defs.LinkTypes.LinkedTo)],
         )
         self.maxDiff = None
         for name, nodes in entries.results.items():
             self.assertEqual(name, cheatsheets_parser.Cheatsheets().name)
-            self.assertEqual(len(nodes), 1)
-            self.assertCountEqual(expected.todict(), nodes[0].todict())
+            sections = {node.section for node in nodes}
+            self.assertIn("Secrets Management Cheat Sheet", sections)
+            secret_entry = [
+                node
+                for node in nodes
+                if node.section == "Secrets Management Cheat Sheet"
+            ][0]
+            self.assertCountEqual(expected.todict(), secret_entry.todict())
+
+    def test_register_supplemental_cheatsheets(self) -> None:
+        for cre_id, name in [
+            ("118-110", "API/web services"),
+            ("724-770", "Technical application access control"),
+            ("623-550", "Denial Of Service protection"),
+        ]:
+            self.collection.add_cre(defs.CRE(name=name, id=cre_id))
+
+        entries = cheatsheets_parser.Cheatsheets().register_supplemental_cheatsheets(
+            cache=self.collection
+        )
+        rest = [
+            entry for entry in entries if entry.section == "REST Security Cheat Sheet"
+        ][0]
+        self.assertEqual(
+            "https://cheatsheetseries.owasp.org/cheatsheets/REST_Security_Cheat_Sheet.html",
+            rest.hyperlink,
+        )
+        self.assertEqual(
+            ["118-110", "724-770", "623-550"],
+            [link.document.id for link in rest.links],
+        )
 
     cheatsheets_md = """ # Secrets Management Cheat Sheet
 

application/utils/external_project_parsers/data/owasp_cheatsheets_supplement.json

@@ -0,0 +1,47 @@
+[
+  {
+    "section": "Authorization Cheat Sheet",
+    "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Authorization_Cheat_Sheet.html",
+    "cre_ids": ["128-128", "117-371"]
+  },
+  {
+    "section": "REST Security Cheat Sheet",
+    "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/REST_Security_Cheat_Sheet.html",
+    "cre_ids": ["118-110", "724-770", "623-550"]
+  },
+  {
+    "section": "Server Side Request Forgery Prevention Cheat Sheet",
+    "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Server_Side_Request_Forgery_Prevention_Cheat_Sheet.html",
+    "cre_ids": ["028-728", "657-084"]
+  },
+  {
+    "section": "Docker Security Cheat Sheet",
+    "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Docker_Security_Cheat_Sheet.html",
+    "cre_ids": ["233-748", "486-813"]
+  },
+  {
+    "section": "Kubernetes Security Cheat Sheet",
+    "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Kubernetes_Security_Cheat_Sheet.html",
+    "cre_ids": ["467-784", "233-748", "486-813"]
+  },
+  {
+    "section": "Secure Cloud Architecture Cheat Sheet",
+    "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Secure_Cloud_Architecture_Cheat_Sheet.html",
+    "cre_ids": ["155-155", "467-784"]
+  },
+  {
+    "section": "LLM Prompt Injection Prevention Cheat Sheet",
+    "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/LLM_Prompt_Injection_Prevention_Cheat_Sheet.html",
+    "cre_ids": ["161-451", "760-764"]
+  },
+  {
+    "section": "AI Agent Security Cheat Sheet",
+    "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/AI_Agent_Security_Cheat_Sheet.html",
+    "cre_ids": ["117-371", "650-560", "126-668"]
+  },
+  {
+    "section": "Secure AI Model Ops Cheat Sheet",
+    "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Secure_AI_Model_Ops_Cheat_Sheet.html",
+    "cre_ids": ["148-853", "613-285", "613-287"]
+  }
+]

application/utils/external_project_parsers/parsers/cheatsheets_parser.py

@@ -5,6 +5,9 @@
 from application.defs import cre_defs as defs
 import os
 import re
+import json
+from pathlib import Path
+import logging
 from application.utils.external_project_parsers.base_parser_defs import (
     ParserInterface,
     ParseResult,
@@ -14,6 +17,13 @@
 
 class Cheatsheets(ParserInterface):
     name = "OWASP Cheat Sheets"
+    cheatsheetseries_base_url = "https://cheatsheetseries.owasp.org/cheatsheets"
+    supplement_data_file = (
+        Path(__file__).resolve().parent.parent
+        / "data"
+        / "owasp_cheatsheets_supplement.json"
+    )
+    logger = logging.getLogger(__name__)
 
     def cheatsheet(
         self, section: str, hyperlink: str, tags: List[str]
@@ -25,13 +35,29 @@ def cheatsheet(
             hyperlink=hyperlink,
         )
 
+    def official_cheatsheet_url(self, markdown_filename: str) -> str:
+        html_name = os.path.splitext(markdown_filename)[0] + ".html"
+        return f"{self.cheatsheetseries_base_url}/{html_name}"
+
     def parse(self, cache: db.Node_collection, ph: prompt_client.PromptHandler):
         c_repo = "https://github.com/OWASP/CheatSheetSeries.git"
         cheatsheets_path = "cheatsheets/"
-        repo = git.clone(c_repo)
-        cheatsheets = self.register_cheatsheets(
-            repo=repo, cache=cache, cheatsheets_path=cheatsheets_path, repo_path=c_repo
-        )
+        cheatsheets = []
+        try:
+            repo = git.clone(c_repo)
+            cheatsheets = self.register_cheatsheets(
+                repo=repo,
+                cache=cache,
+                cheatsheets_path=cheatsheets_path,
+                repo_path=c_repo,
+            )
+        except Exception as exc:
+            self.logger.warning(
+                "Unable to clone OWASP CheatSheetSeries, continuing with supplemental cheat sheets only: %s",
+                exc,
+            )
+        cheatsheets.extend(self.register_supplemental_cheatsheets(cache=cache))
+        cheatsheets = self.deduplicate_entries(cheatsheets)
         return ParseResult(results={self.name: cheatsheets})
 
     def register_cheatsheets(
@@ -55,7 +81,7 @@ def register_cheatsheets(
                     name = title.group("title")
                     cre_id = cre.group("cre")
                     cres = cache.get_CREs(external_id=cre_id)
-                    hyperlink = f"{repo_path.replace('.git','')}/tree/master/{cheatsheets_path}{mdfile}"
+                    hyperlink = self.official_cheatsheet_url(mdfile)
                     cs = self.cheatsheet(section=name, hyperlink=hyperlink, tags=[])
                     for cre in cres:
                         cs.add_link(
@@ -65,3 +91,36 @@ def register_cheatsheets(
                         )
                     standard_entries.append(cs)
         return standard_entries
+
+    def register_supplemental_cheatsheets(self, cache: db.Node_collection):
+        with self.supplement_data_file.open("r", encoding="utf-8") as handle:
+            supplement_entries = json.load(handle)
+
+        standard_entries = []
+        for entry in supplement_entries:
+            cs = self.cheatsheet(
+                section=entry["section"],
+                hyperlink=entry["hyperlink"],
+                tags=[],
+            )
+            for cre_id in entry.get("cre_ids", []):
+                cres = cache.get_CREs(external_id=cre_id)
+                for cre in cres:
+                    try:
+                        cs.add_link(
+                            defs.Link(
+                                document=cre.shallow_copy(),
+                                ltype=defs.LinkTypes.AutomaticallyLinkedTo,
+                            )
+                        )
+                    except Exception:
+                        continue
+            if cs.links:
+                standard_entries.append(cs)
+        return standard_entries
+
+    def deduplicate_entries(self, entries: List[defs.Standard]) -> List[defs.Standard]:
+        deduped = {}
+        for entry in entries:
+            deduped[(entry.section, entry.hyperlink)] = entry
+        return list(deduped.values())