From db464ef57fd6d4301f76750d3bdfe4c1c8cbba27 Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Sun, 29 Jun 2025 23:19:21 +0200 Subject: [PATCH 01/11] prepare stack hawk scan reports --- ..._many_vul_without_duplicated_findings.json | 463 ++++++++++++++++++ ...ul_without_duplicated_findings_subset.json | 345 +++++++++++++ 2 files changed, 808 insertions(+) create mode 100644 unittests/scans/stackhawk/stackhawk_many_vul_without_duplicated_findings.json create mode 100644 unittests/scans/stackhawk/stackhawk_many_vul_without_duplicated_findings_subset.json diff --git a/unittests/scans/stackhawk/stackhawk_many_vul_without_duplicated_findings.json b/unittests/scans/stackhawk/stackhawk_many_vul_without_duplicated_findings.json new file mode 100644 index 00000000000..ffde1278b5f --- /dev/null +++ b/unittests/scans/stackhawk/stackhawk_many_vul_without_duplicated_findings.json @@ -0,0 +1,463 @@ +{ + "service": "StackHawk", + "scanCompleted": { + "scan": { + "id": "e2ff5651-7eef-47e9-b743-0c2f7d861e27", + "hawkscanVersion": "2.1.1", + "env": "Development", + "status": "COMPLETED", + "application": "Secured Application", + "startedTimestamp": "2022-02-16T23:07:19.575Z", + "scanURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27" + }, + "scanDuration": "21", + "spiderDuration": "45", + "completedScanStats": { + "urlsCount": "31", + "duration": "66", + "scanResultsStats": { + "totalCount": "55", + "lowCount": "22", + "mediumCount": "22", + "highCount": "11", + "lowTriagedCount": "0", + "mediumTriagedCount": "0", + "highTriagedCount": "0" + } + }, + "findings": [ + { + "pluginId": "90027", + "pluginName": "Cookie Slack Detector", + "severity": "Low", + "host": "https://localhost:9000", + "paths": [ + { + "path": "/basic-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107360/message/2258" + }, + { + "path": "/payload/3111", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107370/message/2275" + }, + { + "path": "/payload/3097", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107365/message/2281" + }, + { + "path": "/search", + "method": "POST", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107358/message/2250" + }, + { + "path": "/payload/3105", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107368/message/2264" + }, + { + "path": "/payload/3113", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107374/message/2255" + }, + { + "path": "/jwt-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107362/message/2267" + }, + { + "path": "/payload/3109", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107456/message/2273" + }, + { + "path": "/", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107359/message/2241" + }, + { + "path": "/payload/3107", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107369/message/2253" + } + ], + "pathStats": [ + { + "status": "NEW", + "count": 10 + } + ], + "totalCount": "10", + "category": "Information Leakage", + "findingURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027" + }, + { + "pluginId": "40025", + "pluginName": "Proxy Disclosure", + "severity": "Medium", + "host": "https://localhost:9000", + "paths": [ + { + "path": "/payload/3105", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107368/message/1586" + }, + { + "path": "/payload/stream/3096", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107455/message/1593" + }, + { + "path": "", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107357/message/1571" + }, + { + "path": "/payload/3101", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107364/message/1575" + }, + { + "path": "/basic-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107360/message/1565" + }, + { + "path": "/payload/3115", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107371/message/1584" + }, + { + "path": "/payload/3097", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107365/message/1573" + }, + { + "path": "/payload/3111", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107370/message/1581" + }, + { + "path": "/", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107359/message/1568" + }, + { + "path": "/jwt-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107362/message/1578" + } + ], + "pathStats": [ + { + "status": "NEW", + "count": 10 + } + ], + "totalCount": "10", + "category": "Information Leakage", + "findingURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025" + }, + { + "pluginId": "20012", + "pluginName": "Anti CSRF Tokens Scanner", + "severity": "High", + "host": "https://localhost:9000", + "paths": [ + { + "path": "/payload/3111", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107370/message/1167" + }, + { + "path": "/payload/3103", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107366/message/1154" + }, + { + "path": "/payload/3107", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107369/message/1156" + }, + { + "path": "/payload/3101", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107364/message/1171" + }, + { + "path": "/payload/3113", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107374/message/1164" + }, + { + "path": "/payload/3105", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107368/message/1169" + }, + { + "path": "/payload/3097", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107365/message/1166" + }, + { + "path": "/basic-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107360/message/1158" + }, + { + "path": "/jwt-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107362/message/1152" + }, + { + "path": "/payload/3099", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107367/message/1161" + } + ], + "pathStats": [ + { + "status": "NEW", + "count": 10 + } + ], + "totalCount": "10", + "category": "HTTP Header Protection", + "findingURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012" + }, + { + "pluginId": "40012", + "pluginName": "Cross Site Scripting Weakness (Reflected in JSON Response)", + "severity": "High", + "host": "https://localhost:9000", + "paths": [ + { + "path": "/search", + "method": "POST", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40012/path/107358/message/236" + } + ], + "pathStats": [ + { + "status": "NEW", + "count": 1 + } + ], + "totalCount": "1", + "category": "Input Sanitization", + "findingURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40012" + }, + { + "pluginId": "10038", + "pluginName": "Content Security Policy (CSP) Header Not Set", + "severity": "Medium", + "host": "https://localhost:9000", + "paths": [ + { + "path": "/payload/3099", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107367/message/49" + }, + { + "path": "/", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107359/message/17" + }, + { + "path": "/basic-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107360/message/20" + }, + { + "path": "/search", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107358/message/10" + }, + { + "path": "/jwt-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107362/message/18" + }, + { + "path": "/search", + "method": "POST", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107358/message/21" + }, + { + "path": "/payload/3097", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107365/message/45" + }, + { + "path": "/token-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107361/message/19" + }, + { + "path": "/payload/3103", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107366/message/47" + }, + { + "path": "", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107357/message/9" + }, + { + "path": "/payloads", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107363/message/24" + }, + { + "path": "/payload/3101", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107364/message/48" + } + ], + "pathStats": [ + { + "status": "NEW", + "count": 12 + } + ], + "totalCount": "12", + "category": "Information Leakage", + "findingURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038" + }, + { + "pluginId": "10063", + "pluginName": "Permissions Policy Header Not Set", + "severity": "Low", + "host": "https://localhost:9000", + "paths": [ + { + "path": "/jwt-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063/path/107362/message/18" + }, + { + "path": "/", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063/path/107359/message/17" + }, + { + "path": "/payload/3103", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063/path/107366/message/47" + }, + { + "path": "/payload/3101", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063/path/107364/message/48" + }, + { + "path": "/token-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063/path/107361/message/19" + }, + { + "path": "/search", + "method": "POST", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063/path/107358/message/21" + }, + { + "path": "/payloads", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063/path/107363/message/24" + }, + { + "path": "/basic-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063/path/107360/message/20" + }, + { + "path": "/payload/3097", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063/path/107365/message/45" + }, + { + "path": "/search", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063/path/107358/message/10" + }, + { + "path": "/payload/3099", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063/path/107367/message/49" + }, + { + "path": "", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063/path/107357/message/9" + } + ], + "pathStats": [ + { + "status": "NEW", + "count": 12 + } + ], + "totalCount": "12", + "category": "", + "findingURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063" + } + ] + } +} diff --git a/unittests/scans/stackhawk/stackhawk_many_vul_without_duplicated_findings_subset.json b/unittests/scans/stackhawk/stackhawk_many_vul_without_duplicated_findings_subset.json new file mode 100644 index 00000000000..43b2523692a --- /dev/null +++ b/unittests/scans/stackhawk/stackhawk_many_vul_without_duplicated_findings_subset.json @@ -0,0 +1,345 @@ +{ + "service": "StackHawk", + "scanCompleted": { + "scan": { + "comment defect dojo team": "This is a subset of the StackHawk scan results without some of the findings and without some endpoints", + "id": "e2ff5651-7eef-47e9-b743-0c2f7d861e27", + "hawkscanVersion": "2.1.1", + "env": "Development", + "status": "COMPLETED", + "application": "Secured Application", + "startedTimestamp": "2022-02-16T23:07:19.575Z", + "scanURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27" + }, + "scanDuration": "21", + "spiderDuration": "45", + "completedScanStats": { + "urlsCount": "31", + "duration": "66", + "scanResultsStats": { + "totalCount": "55", + "lowCount": "22", + "mediumCount": "22", + "highCount": "11", + "lowTriagedCount": "0", + "mediumTriagedCount": "0", + "highTriagedCount": "0" + } + }, + "findings": [ + { + "pluginId": "90027", + "pluginName": "Cookie Slack Detector", + "severity": "Low", + "host": "https://localhost:9000", + "paths": [ + { + "path": "/payload/3097", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107365/message/2281" + }, + { + "path": "/search", + "method": "POST", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107358/message/2250" + }, + { + "path": "/payload/3105", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107368/message/2264" + }, + { + "path": "/payload/3113", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107374/message/2255" + }, + { + "path": "/jwt-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107362/message/2267" + }, + { + "path": "/payload/3109", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107456/message/2273" + }, + { + "path": "/", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107359/message/2241" + }, + { + "path": "/payload/3107", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107369/message/2253" + } + ], + "pathStats": [ + { + "status": "NEW", + "count": 8 + } + ], + "totalCount": "8", + "category": "Information Leakage", + "findingURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027" + }, + { + "pluginId": "40025", + "pluginName": "Proxy Disclosure", + "severity": "Medium", + "host": "https://localhost:9000", + "paths": [ + { + "path": "/payload/3105", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107368/message/1586" + }, + { + "path": "/payload/stream/3096", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107455/message/1593" + }, + { + "path": "", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107357/message/1571" + }, + { + "path": "/payload/3101", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107364/message/1575" + }, + { + "path": "/basic-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107360/message/1565" + }, + { + "path": "/payload/3115", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107371/message/1584" + }, + { + "path": "/payload/3097", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107365/message/1573" + }, + { + "path": "/", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107359/message/1568" + }, + { + "path": "/jwt-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107362/message/1578" + } + ], + "pathStats": [ + { + "status": "NEW", + "count": 9 + } + ], + "totalCount": "9", + "category": "Information Leakage", + "findingURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025" + }, + { + "pluginId": "20012", + "pluginName": "Anti CSRF Tokens Scanner", + "severity": "High", + "host": "https://localhost:9000", + "paths": [ + { + "path": "/payload/3111", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107370/message/1167" + }, + { + "path": "/payload/3103", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107366/message/1154" + }, + { + "path": "/payload/3107", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107369/message/1156" + }, + { + "path": "/payload/3101", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107364/message/1171" + }, + { + "path": "/payload/3113", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107374/message/1164" + }, + { + "path": "/payload/3105", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107368/message/1169" + }, + { + "path": "/jwt-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107362/message/1152" + }, + { + "path": "/payload/3099", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107367/message/1161" + } + ], + "pathStats": [ + { + "status": "NEW", + "count": 8 + } + ], + "totalCount": "8", + "category": "HTTP Header Protection", + "findingURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012" + }, + { + "pluginId": "40012", + "pluginName": "Cross Site Scripting Weakness (Reflected in JSON Response)", + "severity": "High", + "host": "https://localhost:9000", + "paths": [ + { + "path": "/search", + "method": "POST", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40012/path/107358/message/236" + } + ], + "pathStats": [ + { + "status": "NEW", + "count": 1 + } + ], + "totalCount": "1", + "category": "Input Sanitization", + "findingURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40012" + }, + { + "pluginId": "10038", + "pluginName": "Content Security Policy (CSP) Header Not Set", + "severity": "Medium", + "host": "https://localhost:9000", + "paths": [ + { + "path": "/payload/3099", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107367/message/49" + }, + { + "path": "/", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107359/message/17" + }, + { + "path": "/basic-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107360/message/20" + }, + { + "path": "/search", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107358/message/10" + }, + { + "path": "/jwt-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107362/message/18" + }, + { + "path": "/search", + "method": "POST", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107358/message/21" + }, + { + "path": "/payload/3097", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107365/message/45" + }, + { + "path": "/token-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107361/message/19" + }, + { + "path": "/payload/3103", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107366/message/47" + }, + { + "path": "", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107357/message/9" + }, + { + "path": "/payloads", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107363/message/24" + }, + { + "path": "/payload/3101", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107364/message/48" + } + ], + "pathStats": [ + { + "status": "NEW", + "count": 12 + } + ], + "totalCount": "12", + "category": "Information Leakage", + "findingURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038" + } + ] + } +} From a03b7523a072b863f8dea695ec6fa1b41fdc18da Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Mon, 30 Jun 2025 20:51:38 +0200 Subject: [PATCH 02/11] new test case with stats --- dojo/decorators.py | 49 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/dojo/decorators.py b/dojo/decorators.py index 2b1c08e57cd..8aae1e5df6a 100644 --- a/dojo/decorators.py +++ b/dojo/decorators.py @@ -1,4 +1,5 @@ import logging +import threading from functools import wraps from django.conf import settings @@ -12,6 +13,47 @@ logger = logging.getLogger(__name__) +class ThreadLocalTaskCounter: + def __init__(self): + self._thread_local = threading.local() + + def _get_task_list(self): + if not hasattr(self._thread_local, "tasks"): + self._thread_local.tasks = [] + return self._thread_local.tasks + + def _get_recording(self): + return getattr(self._thread_local, "recording", False) + + def start(self): + self._thread_local.recording = True + self._get_task_list().clear() + + def stop(self): + self._thread_local.recording = False + + def incr(self, task_name, model_id=None, args=None, kwargs=None): + if not self._get_recording(): + return + tasks = self._get_task_list() + tasks.append({ + "task": task_name, + "id": model_id, + "args": args if args is not None else [], + "kwargs": kwargs if kwargs is not None else {}, + }) + + def get(self): + return len(self._get_task_list()) + + def get_tasks(self): + return list(self._get_task_list()) + + +# Create a shared instance +dojo_async_task_counter = ThreadLocalTaskCounter() + + def we_want_async(*args, func=None, **kwargs): from dojo.models import Dojo_User from dojo.utils import get_current_user @@ -40,6 +82,13 @@ def __wrapper__(*args, **kwargs): from dojo.utils import get_current_user user = get_current_user() kwargs["async_user"] = user + + dojo_async_task_counter.incr( + func.__name__, + args=args, + kwargs=kwargs, + ) + countdown = kwargs.pop("countdown", 0) if we_want_async(*args, func=func, **kwargs): return func.apply_async(args=args, kwargs=kwargs, countdown=countdown) From bc5aa0a90244246c666076b8c55436fa13075c5c Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Tue, 1 Jul 2025 00:06:28 +0200 Subject: [PATCH 03/11] add foreground testcase --- unittests/test_importers_performance.py | 154 ++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 unittests/test_importers_performance.py diff --git a/unittests/test_importers_performance.py b/unittests/test_importers_performance.py new file mode 100644 index 00000000000..b8b024000e2 --- /dev/null +++ b/unittests/test_importers_performance.py @@ -0,0 +1,154 @@ +import logging +from contextlib import contextmanager +from unittest.mock import patch + +from crum import impersonate +from django.utils import timezone + +from dojo.decorators import dojo_async_task_counter +from dojo.importers.default_importer import DefaultImporter +from dojo.importers.default_reimporter import DefaultReImporter +from dojo.models import Development_Environment, Dojo_User, Engagement, Product, Product_Type, User + +from .dojo_test_case import DojoTestCase, get_unit_tests_scans_path + +logger = logging.getLogger(__name__) + +STACK_HAWK_FILENAME = get_unit_tests_scans_path("stackhawk") / "stackhawk_many_vul_without_duplicated_findings.json" +STACK_HAWK_SUBSET_FILENAME = get_unit_tests_scans_path("stackhawk") / "stackhawk_many_vul_without_duplicated_findings_subset.json" +STACK_HAWK_SCAN_TYPE = "StackHawk HawkScan" + + +class TestDojoImporterPerformance(DojoTestCase): + fixtures = ["dojo_testdata.json"] + + def setUp(self): + super().setUp() + user = User.objects.get(username="admin") + user.usercontactinfo.block_execution = True + user.save() + + @contextmanager + def assertNumAsyncTask(self, num): + dojo_async_task_counter.start() + try: + yield + finally: + dojo_async_task_counter.stop() + actual = dojo_async_task_counter.get() + if actual != num: + tasks = dojo_async_task_counter.get_tasks() + tasks_str = "\n".join(str(task) for task in tasks) + msg = ( + f"Expected {num} celery tasks, but {actual} were created.\n" + f"Tasks created:\n{tasks_str}" + ) + raise self.failureException(msg) + + def import_reimport_performance(self, expected_num_queries1, expected_num_async_tasks1, expected_num_queries2, expected_num_async_tasks2, expected_num_queries3, expected_num_async_tasks3): + """Despite all efforts, these imports here run in async mode, so celery tasks are executed in the background""" + product_type, _created = Product_Type.objects.get_or_create(name="test") + product, _created = Product.objects.get_or_create( + name="TestDojoDefaultImporter", + prod_type=product_type, + ) + engagement, _created = Engagement.objects.get_or_create( + name="Test Create Engagement", + product=product, + target_start=timezone.now(), + target_end=timezone.now(), + ) + lead, _ = User.objects.get_or_create(username="admin") + environment, _ = Development_Environment.objects.get_or_create(name="Development") + + # first import the subset which missed one finding and a couple of endpoints on some of the findings + with ( + self.subTest("import1"), impersonate(Dojo_User.objects.get(username="admin")), + self.assertNumQueries(expected_num_queries1), + self.assertNumAsyncTask(expected_num_async_tasks1), + STACK_HAWK_SUBSET_FILENAME.open(encoding="utf-8") as scan, + ): + import_options = { + "user": lead, + "lead": lead, + "scan_date": None, + "environment": environment, + "minimum_severity": "Info", + "active": True, + "verified": True, + "sync": True, + "scan_type": STACK_HAWK_SCAN_TYPE, + "engagement": engagement, + } + importer = DefaultImporter(**import_options) + test, _, _len_new_findings, _len_closed_findings, _, _, _ = importer.process_scan(scan) + + # use reimport with the full report so it add a finding and some endpoints + with ( + self.subTest("reimport1"), impersonate(Dojo_User.objects.get(username="admin")), + self.assertNumQueries(expected_num_queries2), + self.assertNumAsyncTask(expected_num_async_tasks2), + STACK_HAWK_FILENAME.open(encoding="utf-8") as scan, + ): + reimport_options = { + "test": test, + "user": lead, + "lead": lead, + "scan_date": None, + "minimum_severity": "Info", + "active": True, + "verified": True, + "sync": True, + "scan_type": STACK_HAWK_SCAN_TYPE, + } + reimporter = DefaultReImporter(**reimport_options) + test, _, _len_new_findings, _len_closed_findings, _, _, _ = reimporter.process_scan(scan) + + # use reimport with the subset again to close a finding and mitigate some endpoints + with ( + self.subTest("reimport2"), impersonate(Dojo_User.objects.get(username="admin")), + self.assertNumQueries(expected_num_queries3), + self.assertNumAsyncTask(expected_num_async_tasks3), + STACK_HAWK_SUBSET_FILENAME.open(encoding="utf-8") as scan, + ): + reimport_options = { + "test": test, + "user": lead, + "lead": lead, + "scan_date": None, + "minimum_severity": "Info", + "active": True, + "verified": True, + "sync": True, + "scan_type": STACK_HAWK_SCAN_TYPE, + } + reimporter = DefaultReImporter(**reimport_options) + test, _, _len_new_findings, _len_closed_findings, _, _, _ = reimporter.process_scan(scan) + + def test_import_reimport_reimport_performance(self): + self.import_reimport_performance( + expected_num_queries1=617, + expected_num_async_tasks1=18, + expected_num_queries2=496, + expected_num_async_tasks2=25, + expected_num_queries3=348, + expected_num_async_tasks3=21, + ) + + @patch("dojo.decorators.we_want_async", return_value=False) + def test_import_reimport_reimport_performance_no_async(self, mock): + """ + This test checks the performance of the importers when they are run in sync mode. + The reason for this is that we also want to be aware of when a PR affects the number of queries + or async tasks created by a background task. + The impersonate context manager above does not work as expected for disabling async, + so we patch the we_want_async decorator to always return False. + """ + self.import_reimport_performance( + expected_num_queries1=708, + expected_num_async_tasks1=29, + expected_num_queries2=566, + expected_num_async_tasks2=32, + expected_num_queries3=400, + expected_num_async_tasks3=26, + ) From ccaddf97816adefbcd20527d8fb0655b70980550 Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Tue, 1 Jul 2025 19:25:53 +0200 Subject: [PATCH 04/11] produc grading --- unittests/dojo_test_case.py | 4 ++ unittests/test_importers_performance.py | 58 +++++++++++++++++-------- 2 files changed, 45 insertions(+), 17 deletions(-) diff --git a/unittests/dojo_test_case.py b/unittests/dojo_test_case.py index 3f1061d6002..8cab5aebf15 100644 --- a/unittests/dojo_test_case.py +++ b/unittests/dojo_test_case.py @@ -98,6 +98,8 @@ def system_settings( disable_jira_webhook_secret=False, jira_webhook_secret=None, enable_product_tag_inehritance=False, + enable_product_grade=True, + enable_webhooks_notifications=False, ): ss = System_Settings.objects.get() ss.enable_jira = enable_jira @@ -105,6 +107,8 @@ def system_settings( ss.disable_jira_webhook_secret = disable_jira_webhook_secret ss.jira_webhook_secret = jira_webhook_secret ss.enable_product_tag_inheritance = enable_product_tag_inehritance + ss.enable_product_grade = enable_product_grade + ss.enable_webhooks_notifications = enable_webhooks_notifications ss.save() def create_product_type(self, name, *args, description="dummy description", **kwargs): diff --git a/unittests/test_importers_performance.py b/unittests/test_importers_performance.py index b8b024000e2..73a49373625 100644 --- a/unittests/test_importers_performance.py +++ b/unittests/test_importers_performance.py @@ -20,13 +20,11 @@ class TestDojoImporterPerformance(DojoTestCase): - fixtures = ["dojo_testdata.json"] def setUp(self): super().setUp() - user = User.objects.get(username="admin") - user.usercontactinfo.block_execution = True - user.save() + self.system_settings(enable_webhooks_notifications=False) + self.system_settings(enable_product_grade=False) @contextmanager def assertNumAsyncTask(self, num): @@ -46,7 +44,14 @@ def assertNumAsyncTask(self, num): raise self.failureException(msg) def import_reimport_performance(self, expected_num_queries1, expected_num_async_tasks1, expected_num_queries2, expected_num_async_tasks2, expected_num_queries3, expected_num_async_tasks3): - """Despite all efforts, these imports here run in async mode, so celery tasks are executed in the background""" + """ + Log output can be quite large as when the assertNumQueries fails, all queries are printed. + It could be usefule to capture the output in `less`: + ./run-unittest.sh --test-case unittests.test_importers_performance.TestDojoImporterPerformance 2>&1 | less + Then search for `expected` to find the lines where the expected number of queries is printed. + Or you can use `grep` to filter the output: + ./run-unittest.sh --test-case unittests.test_importers_performance.TestDojoImporterPerformance 2>&1 | grep expected + """ product_type, _created = Product_Type.objects.get_or_create(name="test") product, _created = Product.objects.get_or_create( name="TestDojoDefaultImporter", @@ -127,12 +132,12 @@ def import_reimport_performance(self, expected_num_queries1, expected_num_async_ def test_import_reimport_reimport_performance(self): self.import_reimport_performance( - expected_num_queries1=617, - expected_num_async_tasks1=18, - expected_num_queries2=496, - expected_num_async_tasks2=25, - expected_num_queries3=348, - expected_num_async_tasks3=21, + expected_num_queries1=606, + expected_num_async_tasks1=15, + expected_num_queries2=489, + expected_num_async_tasks2=23, + expected_num_queries3=347, + expected_num_async_tasks3=20, ) @patch("dojo.decorators.we_want_async", return_value=False) @@ -145,10 +150,29 @@ def test_import_reimport_reimport_performance_no_async(self, mock): so we patch the we_want_async decorator to always return False. """ self.import_reimport_performance( - expected_num_queries1=708, - expected_num_async_tasks1=29, - expected_num_queries2=566, - expected_num_async_tasks2=32, - expected_num_queries3=400, - expected_num_async_tasks3=26, + expected_num_queries1=613, + expected_num_async_tasks1=15, + expected_num_queries2=496, + expected_num_async_tasks2=23, + expected_num_queries3=352, + expected_num_async_tasks3=20, + ) + + @patch("dojo.decorators.we_want_async", return_value=False) + def test_import_reimport_reimport_performance_no_async_with_product_grading(self, mock): + """ + This test checks the performance of the importers when they are run in sync mode. + The reason for this is that we also want to be aware of when a PR affects the number of queries + or async tasks created by a background task. + The impersonate context manager above does not work as expected for disabling async, + so we patch the we_want_async decorator to always return False. + """ + self.system_settings(enable_product_grade=True) + self.import_reimport_performance( + expected_num_queries1=673, + expected_num_async_tasks1=25, + expected_num_queries2=544, + expected_num_async_tasks2=30, + expected_num_queries3=387, + expected_num_async_tasks3=25, ) From 05c461de2b9763f69f8836091f6e0adbc2bb647d Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Tue, 1 Jul 2025 20:14:13 +0200 Subject: [PATCH 05/11] default webhooks ons --- unittests/dojo_test_case.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittests/dojo_test_case.py b/unittests/dojo_test_case.py index 8cab5aebf15..4553bcd141c 100644 --- a/unittests/dojo_test_case.py +++ b/unittests/dojo_test_case.py @@ -99,7 +99,7 @@ def system_settings( jira_webhook_secret=None, enable_product_tag_inehritance=False, enable_product_grade=True, - enable_webhooks_notifications=False, + enable_webhooks_notifications=True, ): ss = System_Settings.objects.get() ss.enable_jira = enable_jira From 3ee8729b3b23a319e692d324e19b3defef00f8a0 Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Tue, 1 Jul 2025 21:29:07 +0200 Subject: [PATCH 06/11] fix counts --- unittests/test_importers_performance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittests/test_importers_performance.py b/unittests/test_importers_performance.py index 73a49373625..2f8d103aea9 100644 --- a/unittests/test_importers_performance.py +++ b/unittests/test_importers_performance.py @@ -132,7 +132,7 @@ def import_reimport_performance(self, expected_num_queries1, expected_num_async_ def test_import_reimport_reimport_performance(self): self.import_reimport_performance( - expected_num_queries1=606, + expected_num_queries1=603, expected_num_async_tasks1=15, expected_num_queries2=489, expected_num_async_tasks2=23, From 10e63717783f89b5aeaeecd11ff7f42ec3e9ae4a Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Tue, 1 Jul 2025 22:13:59 +0200 Subject: [PATCH 07/11] add dummy import to warm up content_type cache --- unittests/dojo_test_case.py | 2 ++ unittests/test_importers_performance.py | 32 +++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/unittests/dojo_test_case.py b/unittests/dojo_test_case.py index 4553bcd141c..12dd737cd94 100644 --- a/unittests/dojo_test_case.py +++ b/unittests/dojo_test_case.py @@ -97,6 +97,7 @@ def system_settings( enable_jira_web_hook=False, disable_jira_webhook_secret=False, jira_webhook_secret=None, + enable_github=False, enable_product_tag_inehritance=False, enable_product_grade=True, enable_webhooks_notifications=True, @@ -106,6 +107,7 @@ def system_settings( ss.enable_jira_web_hook = enable_jira_web_hook ss.disable_jira_webhook_secret = disable_jira_webhook_secret ss.jira_webhook_secret = jira_webhook_secret + ss.enable_github = enable_github ss.enable_product_tag_inheritance = enable_product_tag_inehritance ss.enable_product_grade = enable_product_grade ss.enable_webhooks_notifications = enable_webhooks_notifications diff --git a/unittests/test_importers_performance.py b/unittests/test_importers_performance.py index 2f8d103aea9..7dec60acaf5 100644 --- a/unittests/test_importers_performance.py +++ b/unittests/test_importers_performance.py @@ -18,6 +18,9 @@ STACK_HAWK_SUBSET_FILENAME = get_unit_tests_scans_path("stackhawk") / "stackhawk_many_vul_without_duplicated_findings_subset.json" STACK_HAWK_SCAN_TYPE = "StackHawk HawkScan" +NPM_AUDIT_NO_VULN_FILENAME = get_unit_tests_scans_path("npm_audit") / "no_vuln.json" +NPM_AUDIT_SCAN_TYPE = "NPM Audit Scan" + class TestDojoImporterPerformance(DojoTestCase): @@ -25,6 +28,7 @@ def setUp(self): super().setUp() self.system_settings(enable_webhooks_notifications=False) self.system_settings(enable_product_grade=False) + self.system_settings(enable_github=False) @contextmanager def assertNumAsyncTask(self, num): @@ -66,6 +70,30 @@ def import_reimport_performance(self, expected_num_queries1, expected_num_async_ lead, _ = User.objects.get_or_create(username="admin") environment, _ = Development_Environment.objects.get_or_create(name="Development") + # first we do a bogus import to make sure any caches are loaded. + # without this the number of queries will be higher as the audit log will load content_type ids from the db + + engagement_dummy, _created = Engagement.objects.get_or_create( + name="Test Create Dummy Engagement", + product=product, + target_start=timezone.now(), + target_end=timezone.now(), + ) + import_options = { + "user": lead, + "lead": lead, + "scan_date": None, + "environment": environment, + "minimum_severity": "Info", + "active": True, + "verified": True, + "sync": True, + "scan_type": NPM_AUDIT_SCAN_TYPE, + "engagement": engagement_dummy, + } + importer = DefaultImporter(**import_options) + test, _, _len_new_findings, _len_closed_findings, _, _, _ = importer.process_scan(NPM_AUDIT_NO_VULN_FILENAME.open(encoding="utf-8")) + # first import the subset which missed one finding and a couple of endpoints on some of the findings with ( self.subTest("import1"), impersonate(Dojo_User.objects.get(username="admin")), @@ -132,7 +160,7 @@ def import_reimport_performance(self, expected_num_queries1, expected_num_async_ def test_import_reimport_reimport_performance(self): self.import_reimport_performance( - expected_num_queries1=603, + expected_num_queries1=605, expected_num_async_tasks1=15, expected_num_queries2=489, expected_num_async_tasks2=23, @@ -169,7 +197,7 @@ def test_import_reimport_reimport_performance_no_async_with_product_grading(self """ self.system_settings(enable_product_grade=True) self.import_reimport_performance( - expected_num_queries1=673, + expected_num_queries1=675, expected_num_async_tasks1=25, expected_num_queries2=544, expected_num_async_tasks2=30, From 2617bfdfc8cf643d371f2b8287ea1cd36d5f88fd Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Tue, 1 Jul 2025 22:35:35 +0200 Subject: [PATCH 08/11] fix counts --- docker/entrypoint-unit-tests-devDocker.sh | 14 +++++++------- unittests/test_importers_performance.py | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docker/entrypoint-unit-tests-devDocker.sh b/docker/entrypoint-unit-tests-devDocker.sh index 59ebd7daf3c..ec2810cd545 100755 --- a/docker/entrypoint-unit-tests-devDocker.sh +++ b/docker/entrypoint-unit-tests-devDocker.sh @@ -73,15 +73,15 @@ echo "Unit Tests" echo "------------------------------------------------------------" # Removing parallel and shuffle for now to maintain stability -python3 manage.py test unittests -v 3 --keepdb --no-input --exclude-tag="non-parallel" || { - exit 1; -} -python3 manage.py test unittests -v 3 --keepdb --no-input --tag="non-parallel" || { - exit 1; -} +# python3 manage.py test unittests -v 3 --keepdb --no-input --exclude-tag="non-parallel" || { +# exit 1; +# } +# python3 manage.py test unittests -v 3 --keepdb --no-input --tag="non-parallel" || { +# exit 1; +# } # you can select a single file to "test" unit tests -# python3 manage.py test unittests.tools.test_npm_audit_scan_parser.TestNpmAuditParser --keepdb -v 3 +python3 manage.py test unittests.test_importers_performance.TestDojoImporterPerformance --keepdb -v 3 &> /app/dev2.log # or even a single method # python3 manage.py test unittests.tools.test_npm_audit_scan_parser.TestNpmAuditParser.test_npm_audit_parser_many_vuln_npm7 --keepdb -v 3 diff --git a/unittests/test_importers_performance.py b/unittests/test_importers_performance.py index 7dec60acaf5..3c324956b0a 100644 --- a/unittests/test_importers_performance.py +++ b/unittests/test_importers_performance.py @@ -178,12 +178,12 @@ def test_import_reimport_reimport_performance_no_async(self, mock): so we patch the we_want_async decorator to always return False. """ self.import_reimport_performance( - expected_num_queries1=613, - expected_num_async_tasks1=15, - expected_num_queries2=496, - expected_num_async_tasks2=23, - expected_num_queries3=352, - expected_num_async_tasks3=20, + expected_num_queries1=673, + expected_num_async_tasks1=25, + expected_num_queries2=544, + expected_num_async_tasks2=30, + expected_num_queries3=387, + expected_num_async_tasks3=25, ) @patch("dojo.decorators.we_want_async", return_value=False) @@ -197,7 +197,7 @@ def test_import_reimport_reimport_performance_no_async_with_product_grading(self """ self.system_settings(enable_product_grade=True) self.import_reimport_performance( - expected_num_queries1=675, + expected_num_queries1=673, expected_num_async_tasks1=25, expected_num_queries2=544, expected_num_async_tasks2=30, From e5d56d53574c7733094beffb484bb529fd91b4b6 Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Wed, 2 Jul 2025 00:22:52 +0200 Subject: [PATCH 09/11] warmup cache better --- unittests/test_importers_performance.py | 31 ++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/unittests/test_importers_performance.py b/unittests/test_importers_performance.py index 3c324956b0a..be7fdc38b14 100644 --- a/unittests/test_importers_performance.py +++ b/unittests/test_importers_performance.py @@ -8,7 +8,17 @@ from dojo.decorators import dojo_async_task_counter from dojo.importers.default_importer import DefaultImporter from dojo.importers.default_reimporter import DefaultReImporter -from dojo.models import Development_Environment, Dojo_User, Engagement, Product, Product_Type, User +from dojo.models import ( + Development_Environment, + Dojo_User, + Endpoint, + Endpoint_Status, + Engagement, + Finding, + Product, + Product_Type, + User, +) from .dojo_test_case import DojoTestCase, get_unit_tests_scans_path @@ -18,7 +28,7 @@ STACK_HAWK_SUBSET_FILENAME = get_unit_tests_scans_path("stackhawk") / "stackhawk_many_vul_without_duplicated_findings_subset.json" STACK_HAWK_SCAN_TYPE = "StackHawk HawkScan" -NPM_AUDIT_NO_VULN_FILENAME = get_unit_tests_scans_path("npm_audit") / "no_vuln.json" +NPM_AUDIT_NO_VULN_FILENAME = get_unit_tests_scans_path("npm_audit") / "one_vuln.json" NPM_AUDIT_SCAN_TYPE = "NPM Audit Scan" @@ -94,6 +104,21 @@ def import_reimport_performance(self, expected_num_queries1, expected_num_async_ importer = DefaultImporter(**import_options) test, _, _len_new_findings, _len_closed_findings, _, _, _ = importer.process_scan(NPM_AUDIT_NO_VULN_FILENAME.open(encoding="utf-8")) + # if True: + # exit(0) # this is to make sure the importers are not run when running the tests in the IDE, as it will take too long + + finding = Finding.objects.filter(test=test).first() + + endpoint = Endpoint.objects.create(host="foo.bar", product=product) + Endpoint_Status.objects.create( + finding=finding, + endpoint=endpoint, + ) + + finding.endpoints.add(endpoint) + finding.title = "Dummy finding to ensure audit log is created" + finding.save() + # first import the subset which missed one finding and a couple of endpoints on some of the findings with ( self.subTest("import1"), impersonate(Dojo_User.objects.get(username="admin")), @@ -160,7 +185,7 @@ def import_reimport_performance(self, expected_num_queries1, expected_num_async_ def test_import_reimport_reimport_performance(self): self.import_reimport_performance( - expected_num_queries1=605, + expected_num_queries1=602, expected_num_async_tasks1=15, expected_num_queries2=489, expected_num_async_tasks2=23, From 30ea6c9edc44f2fdbb11fd5d3ae5ee6a348cdd63 Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Wed, 2 Jul 2025 00:50:27 +0200 Subject: [PATCH 10/11] simplify warmup --- unittests/test_importers_performance.py | 48 +++++-------------------- 1 file changed, 8 insertions(+), 40 deletions(-) diff --git a/unittests/test_importers_performance.py b/unittests/test_importers_performance.py index be7fdc38b14..d3cc10d2afc 100644 --- a/unittests/test_importers_performance.py +++ b/unittests/test_importers_performance.py @@ -3,6 +3,7 @@ from unittest.mock import patch from crum import impersonate +from django.contrib.contenttypes.models import ContentType from django.utils import timezone from dojo.decorators import dojo_async_task_counter @@ -39,6 +40,12 @@ def setUp(self): self.system_settings(enable_webhooks_notifications=False) self.system_settings(enable_product_grade=False) self.system_settings(enable_github=False) + # Warm up ContentType cache for relevant models. This is needed if we want to be able to run the test in isolation + # As part of the test suite the ContentTYpe ids will already be cached and won't affect the query count. + # But if we run the test in isolation, the ContentType ids will not be cached and will result in more queries. + # By warming up the cache here, these queries are executed before we start counting queries + for model in [Development_Environment, Dojo_User, Endpoint, Endpoint_Status, Engagement, Finding, Product, Product_Type, User]: + ContentType.objects.get_for_model(model) @contextmanager def assertNumAsyncTask(self, num): @@ -80,45 +87,6 @@ def import_reimport_performance(self, expected_num_queries1, expected_num_async_ lead, _ = User.objects.get_or_create(username="admin") environment, _ = Development_Environment.objects.get_or_create(name="Development") - # first we do a bogus import to make sure any caches are loaded. - # without this the number of queries will be higher as the audit log will load content_type ids from the db - - engagement_dummy, _created = Engagement.objects.get_or_create( - name="Test Create Dummy Engagement", - product=product, - target_start=timezone.now(), - target_end=timezone.now(), - ) - import_options = { - "user": lead, - "lead": lead, - "scan_date": None, - "environment": environment, - "minimum_severity": "Info", - "active": True, - "verified": True, - "sync": True, - "scan_type": NPM_AUDIT_SCAN_TYPE, - "engagement": engagement_dummy, - } - importer = DefaultImporter(**import_options) - test, _, _len_new_findings, _len_closed_findings, _, _, _ = importer.process_scan(NPM_AUDIT_NO_VULN_FILENAME.open(encoding="utf-8")) - - # if True: - # exit(0) # this is to make sure the importers are not run when running the tests in the IDE, as it will take too long - - finding = Finding.objects.filter(test=test).first() - - endpoint = Endpoint.objects.create(host="foo.bar", product=product) - Endpoint_Status.objects.create( - finding=finding, - endpoint=endpoint, - ) - - finding.endpoints.add(endpoint) - finding.title = "Dummy finding to ensure audit log is created" - finding.save() - # first import the subset which missed one finding and a couple of endpoints on some of the findings with ( self.subTest("import1"), impersonate(Dojo_User.objects.get(username="admin")), @@ -185,7 +153,7 @@ def import_reimport_performance(self, expected_num_queries1, expected_num_async_ def test_import_reimport_reimport_performance(self): self.import_reimport_performance( - expected_num_queries1=602, + expected_num_queries1=604, expected_num_async_tasks1=15, expected_num_queries2=489, expected_num_async_tasks2=23, From 2369d2213f8c6d0599b706b5617b42b0f241ec7e Mon Sep 17 00:00:00 2001 From: Valentijn Scholten Date: Wed, 2 Jul 2025 07:42:43 +0200 Subject: [PATCH 11/11] warmup cache better --- unittests/test_importers_performance.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/unittests/test_importers_performance.py b/unittests/test_importers_performance.py index d3cc10d2afc..50a62a62e03 100644 --- a/unittests/test_importers_performance.py +++ b/unittests/test_importers_performance.py @@ -18,6 +18,7 @@ Finding, Product, Product_Type, + Test, User, ) @@ -44,7 +45,7 @@ def setUp(self): # As part of the test suite the ContentTYpe ids will already be cached and won't affect the query count. # But if we run the test in isolation, the ContentType ids will not be cached and will result in more queries. # By warming up the cache here, these queries are executed before we start counting queries - for model in [Development_Environment, Dojo_User, Endpoint, Endpoint_Status, Engagement, Finding, Product, Product_Type, User]: + for model in [Development_Environment, Dojo_User, Endpoint, Endpoint_Status, Engagement, Finding, Product, Product_Type, User, Test]: ContentType.objects.get_for_model(model) @contextmanager @@ -153,7 +154,7 @@ def import_reimport_performance(self, expected_num_queries1, expected_num_async_ def test_import_reimport_reimport_performance(self): self.import_reimport_performance( - expected_num_queries1=604, + expected_num_queries1=603, expected_num_async_tasks1=15, expected_num_queries2=489, expected_num_async_tasks2=23,