Merge branch 'dev' into security/remove-pickle

Author: Maffooch

dojo/tools/api_sonarqube/importer.py

@@ -2,7 +2,9 @@
 import re
 import textwrap
 
+import bleach
 import html2text
+import markdown
 from django.conf import settings
 from django.core.exceptions import ValidationError
 from lxml import etree
@@ -16,6 +18,44 @@
 
 
 class SonarQubeApiImporter:
+    ALLOWED_RULE_DESCRIPTION_TAGS = [
+        "a",
+        "b",
+        "blockquote",
+        "br",
+        "code",
+        "em",
+        "h1",
+        "h2",
+        "h3",
+        "h4",
+        "h5",
+        "h6",
+        "i",
+        "li",
+        "ol",
+        "p",
+        "pre",
+        "strong",
+        "table",
+        "tbody",
+        "td",
+        "th",
+        "thead",
+        "tr",
+        "ul",
+    ]
+    ALLOWED_RULE_DESCRIPTION_ATTRIBUTES = {
+        "a": [
+            "href",
+            "title",
+        ],
+    }
+    ALLOWED_RULE_DESCRIPTION_PROTOCOLS = [
+        "http",
+        "https",
+        "mailto",
+    ]
 
     """
     This class imports from SonarQube (SQ) all open/confirmed SQ issues related to the project related to the test as
@@ -153,13 +193,13 @@ def import_issues(self, test):
                 except KeyError:
                     sonarqube_permalink = "No permalink \n"
 
-                # custom (user defined) SQ rules may not have 'htmlDesc'
-                if "htmlDesc" in rule:
+                rule_details = self.get_rule_details(rule)
+                if rule_details:
                     description = self.clean_rule_description_html(
-                        rule["htmlDesc"],
+                        rule_details,
                     )
-                    cwe = self.clean_cwe(rule["htmlDesc"])
-                    references = sonarqube_permalink + self.get_references(rule["htmlDesc"])
+                    cwe = self.clean_cwe(rule_details)
+                    references = sonarqube_permalink + self.get_references(rule_details)
                 else:
                     description = ""
                     cwe = None
@@ -338,8 +378,10 @@ def import_hotspots(self, test):
 
     @staticmethod
     def clean_rule_description_html(raw_html):
+        if not raw_html:
+            return ""
         search = re.search(
-            r"^(.*?)(?:(<h2>See</h2>)|(<b>References</b>))",
+            r"^(.*?)(?:(<h2>See</h2>)|(<h2>References</h2>)|(<b>References</b>))",
             raw_html,
             re.DOTALL,
         )
@@ -356,6 +398,36 @@ def clean_cwe(raw_html):
             return int(search.group(1))
         return None
 
+    @staticmethod
+    def get_rule_details(rule):
+        if html_desc := rule.get("htmlDesc"):
+            return SonarQubeApiImporter.sanitize_rule_details(html_desc)
+        if not (md_desc := rule.get("mdDesc")):
+            return ""
+        # SonarQube 2025.x can return markdown-only rule descriptions, including
+        # inline HTML that should still be treated as markdown content.
+        return SonarQubeApiImporter.sanitize_rule_details(
+            markdown.markdown(md_desc, extensions=["extra"]),
+        )
+
+    @staticmethod
+    def sanitize_rule_details(description):
+        if not description:
+            return ""
+        sanitized_description = re.sub(
+            r"<(script|style)\b[^>]*>.*?</\1>",
+            "",
+            description,
+            flags=re.DOTALL | re.IGNORECASE,
+        )
+        return bleach.clean(
+            sanitized_description,
+            tags=SonarQubeApiImporter.ALLOWED_RULE_DESCRIPTION_TAGS,
+            attributes=SonarQubeApiImporter.ALLOWED_RULE_DESCRIPTION_ATTRIBUTES,
+            protocols=SonarQubeApiImporter.ALLOWED_RULE_DESCRIPTION_PROTOCOLS,
+            strip=True,
+        )
+
     @staticmethod
     def convert_sonar_severity(sonar_severity):
         sev = sonar_severity.lower()
@@ -382,6 +454,8 @@ def convert_scanner_confidence(sonar_scanner_confidence):
 
     @staticmethod
     def get_references(vuln_details):
+        if not vuln_details:
+            return ""
         parser = etree.HTMLParser()
         details = etree.fromstring(vuln_details, parser)
 

dojo/utils.py

@@ -1794,35 +1794,18 @@ def async_delete_task(model_label, pk, **kwargs):
 
         # Step 2: Prepare duplicate clusters (must happen before any deletion)
         # When CASCADE_DELETE=True, reconfigure_duplicate_cluster skips reconfiguration —
-        # we handle that below by expanding scope to include outside duplicates.
+        # and deletes any outside scope duplicates to avoid FK violations during chunked deletion.
         prepare_duplicates_for_delete(obj)
 
-        # Step 3: Delete outside-scope duplicates first — these point to findings
-        # in the main scope via duplicate_finding FK, so they must be removed before
-        # the originals to avoid FK violations during chunked deletion.
-        scope_ids = finding_qs.values_list("id", flat=True)
-        outside_dupes_qs = (
-            Finding.objects.filter(duplicate_finding_id__in=scope_ids)
-            .exclude(id__in=scope_ids)
-        )
+        # Step 3: Delete the main scope findings
         chunk_size = get_setting("ASYNC_OBEJECT_DELETE_CHUNK_SIZE")
-        outside_count = outside_dupes_qs.count()
-        if outside_count:
-            logger.info("ASYNC_DELETE: Deleting %d outside-scope duplicates first", outside_count)
-            bulk_delete_findings(
-                outside_dupes_qs,
-                chunk_size=chunk_size,
-                cascade_root=cascade_root,
-            )
-
-        # Step 4: Delete the main scope findings
         bulk_delete_findings(
             finding_qs,
             chunk_size=chunk_size,
             cascade_root=cascade_root,
         )
 
-    # Step 5: Delete all remaining related objects (Tests, Engagements,
+    # Step 4: Delete all remaining related objects (Tests, Engagements,
     # Endpoints, etc.) via SQL cascade. Findings are already gone, so
     # skip_relations={Finding} avoids walking empty relations.
     # Single transaction is fine here — the heavy relations (Findings,
@@ -1831,12 +1814,12 @@ def async_delete_task(model_label, pk, **kwargs):
     with transaction.atomic():
         cascade_delete_related_objects(type(obj), pk_query, skip_relations={Finding})
 
-    # Step 6: Delete the top-level object via ORM to fire Django signals
+    # Step 5: Delete the top-level object via ORM to fire Django signals
     # (post_delete notifications, pghistory audit, Pro signals).
     # All children are already gone so this is a single-row DELETE.
     obj.delete()
 
-    # Step 7: Recalculate product grade once (Engagement/Test deletes only). Skip when the
+    # Step 6: Recalculate product grade once (Engagement/Test deletes only). Skip when the
     # deleted object is the Product itself — it is removed in step 6 and grading is pointless.
     # For Product TYpe deletiongs we don't have a product instance, so this never fires.
     if product and not isinstance(obj, Product):

unittests/scans/api_sonarqube/rule_md_desc_only.json

@@ -0,0 +1,32 @@
+{
+    "key": "typescript:S1854",
+    "repo": "typescript",
+    "name": "Dead stores should be removed",
+    "createdAt": "2018-01-17T10:11:21-0500",
+    "mdDesc": "A dead store happens when a local variable is assigned a value that is not read by any subsequent instruction. Calculating or retrieving a value only to then overwrite it or throw it away, could indicate a serious error in the code. Even if it's not an error, it is at best a waste of resources. Therefore all calculated values should be used.\n\n## Noncompliant Code Example\n\ni = a + b; // Noncompliant; calculation result not used before value is overwritten\ni = compute();\n\n## Compliant Solution\n\ni = a + b;\ni += compute();\n\n## Exceptions\n\nThis rule ignores initializations to -1, 0, 1, `null`, `true`, `false`, `\"\"`, `[]` and `{}`.\n\n## See\n\n- [MITRE, CWE-563](http://cwe.mitre.org/data/definitions/563.html) - Assignment to Variable without Use ('Unused Variable')\n- [CERT, MSC13-C.](https://www.securecoding.cert.org/confluence/x/QYA5) - Detect and remove unused values\n- [CERT, MSC56-J.](https://www.securecoding.cert.org/confluence/x/uQCSBg) - Detect and remove superfluous code and values\n",
+    "severity": "MAJOR",
+    "status": "READY",
+    "isTemplate": false,
+    "tags": [],
+    "sysTags": [
+        "cert",
+        "cwe",
+        "unused"
+    ],
+    "lang": "ts",
+    "langName": "TypeScript",
+    "params": [],
+    "defaultDebtRemFnType": "CONSTANT_ISSUE",
+    "defaultDebtRemFnOffset": "15min",
+    "debtOverloaded": false,
+    "debtRemFnType": "CONSTANT_ISSUE",
+    "debtRemFnOffset": "15min",
+    "defaultRemFnType": "CONSTANT_ISSUE",
+    "defaultRemFnBaseEffort": "15min",
+    "remFnType": "CONSTANT_ISSUE",
+    "remFnBaseEffort": "15min",
+    "remFnOverloaded": false,
+    "scope": "MAIN",
+    "isExternal": false,
+    "type": "CODE_SMELL"
+}