diff --git a/docker/entrypoint-unit-tests-devDocker.sh b/docker/entrypoint-unit-tests-devDocker.sh index 59ebd7daf3c..ec2810cd545 100755 --- a/docker/entrypoint-unit-tests-devDocker.sh +++ b/docker/entrypoint-unit-tests-devDocker.sh @@ -73,15 +73,15 @@ echo "Unit Tests" echo "------------------------------------------------------------" # Removing parallel and shuffle for now to maintain stability -python3 manage.py test unittests -v 3 --keepdb --no-input --exclude-tag="non-parallel" || { - exit 1; -} -python3 manage.py test unittests -v 3 --keepdb --no-input --tag="non-parallel" || { - exit 1; -} +# python3 manage.py test unittests -v 3 --keepdb --no-input --exclude-tag="non-parallel" || { +# exit 1; +# } +# python3 manage.py test unittests -v 3 --keepdb --no-input --tag="non-parallel" || { +# exit 1; +# } # you can select a single file to "test" unit tests -# python3 manage.py test unittests.tools.test_npm_audit_scan_parser.TestNpmAuditParser --keepdb -v 3 +python3 manage.py test unittests.test_importers_performance.TestDojoImporterPerformance --keepdb -v 3 &> /app/dev2.log # or even a single method # python3 manage.py test unittests.tools.test_npm_audit_scan_parser.TestNpmAuditParser.test_npm_audit_parser_many_vuln_npm7 --keepdb -v 3 diff --git a/dojo/decorators.py b/dojo/decorators.py index 2b1c08e57cd..8aae1e5df6a 100644 --- a/dojo/decorators.py +++ b/dojo/decorators.py @@ -1,4 +1,5 @@ import logging +import threading from functools import wraps from django.conf import settings @@ -12,6 +13,47 @@ logger = logging.getLogger(__name__) +class ThreadLocalTaskCounter: + def __init__(self): + self._thread_local = threading.local() + + def _get_task_list(self): + if not hasattr(self._thread_local, "tasks"): + self._thread_local.tasks = [] + return self._thread_local.tasks + + def _get_recording(self): + return getattr(self._thread_local, "recording", False) + + def start(self): + self._thread_local.recording = True + self._get_task_list().clear() + + def stop(self): + self._thread_local.recording = False + + def incr(self, task_name, model_id=None, args=None, kwargs=None): + if not self._get_recording(): + return + tasks = self._get_task_list() + tasks.append({ + "task": task_name, + "id": model_id, + "args": args if args is not None else [], + "kwargs": kwargs if kwargs is not None else {}, + }) + + def get(self): + return len(self._get_task_list()) + + def get_tasks(self): + return list(self._get_task_list()) + + +# Create a shared instance +dojo_async_task_counter = ThreadLocalTaskCounter() + + def we_want_async(*args, func=None, **kwargs): from dojo.models import Dojo_User from dojo.utils import get_current_user @@ -40,6 +82,13 @@ def __wrapper__(*args, **kwargs): from dojo.utils import get_current_user user = get_current_user() kwargs["async_user"] = user + + dojo_async_task_counter.incr( + func.__name__, + args=args, + kwargs=kwargs, + ) + countdown = kwargs.pop("countdown", 0) if we_want_async(*args, func=func, **kwargs): return func.apply_async(args=args, kwargs=kwargs, countdown=countdown) diff --git a/unittests/dojo_test_case.py b/unittests/dojo_test_case.py index 3f1061d6002..12dd737cd94 100644 --- a/unittests/dojo_test_case.py +++ b/unittests/dojo_test_case.py @@ -97,14 +97,20 @@ def system_settings( enable_jira_web_hook=False, disable_jira_webhook_secret=False, jira_webhook_secret=None, + enable_github=False, enable_product_tag_inehritance=False, + enable_product_grade=True, + enable_webhooks_notifications=True, ): ss = System_Settings.objects.get() ss.enable_jira = enable_jira ss.enable_jira_web_hook = enable_jira_web_hook ss.disable_jira_webhook_secret = disable_jira_webhook_secret ss.jira_webhook_secret = jira_webhook_secret + ss.enable_github = enable_github ss.enable_product_tag_inheritance = enable_product_tag_inehritance + ss.enable_product_grade = enable_product_grade + ss.enable_webhooks_notifications = enable_webhooks_notifications ss.save() def create_product_type(self, name, *args, description="dummy description", **kwargs): diff --git a/unittests/scans/stackhawk/stackhawk_many_vul_without_duplicated_findings.json b/unittests/scans/stackhawk/stackhawk_many_vul_without_duplicated_findings.json new file mode 100644 index 00000000000..ffde1278b5f --- /dev/null +++ b/unittests/scans/stackhawk/stackhawk_many_vul_without_duplicated_findings.json @@ -0,0 +1,463 @@ +{ + "service": "StackHawk", + "scanCompleted": { + "scan": { + "id": "e2ff5651-7eef-47e9-b743-0c2f7d861e27", + "hawkscanVersion": "2.1.1", + "env": "Development", + "status": "COMPLETED", + "application": "Secured Application", + "startedTimestamp": "2022-02-16T23:07:19.575Z", + "scanURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27" + }, + "scanDuration": "21", + "spiderDuration": "45", + "completedScanStats": { + "urlsCount": "31", + "duration": "66", + "scanResultsStats": { + "totalCount": "55", + "lowCount": "22", + "mediumCount": "22", + "highCount": "11", + "lowTriagedCount": "0", + "mediumTriagedCount": "0", + "highTriagedCount": "0" + } + }, + "findings": [ + { + "pluginId": "90027", + "pluginName": "Cookie Slack Detector", + "severity": "Low", + "host": "https://localhost:9000", + "paths": [ + { + "path": "/basic-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107360/message/2258" + }, + { + "path": "/payload/3111", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107370/message/2275" + }, + { + "path": "/payload/3097", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107365/message/2281" + }, + { + "path": "/search", + "method": "POST", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107358/message/2250" + }, + { + "path": "/payload/3105", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107368/message/2264" + }, + { + "path": "/payload/3113", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107374/message/2255" + }, + { + "path": "/jwt-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107362/message/2267" + }, + { + "path": "/payload/3109", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107456/message/2273" + }, + { + "path": "/", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107359/message/2241" + }, + { + "path": "/payload/3107", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107369/message/2253" + } + ], + "pathStats": [ + { + "status": "NEW", + "count": 10 + } + ], + "totalCount": "10", + "category": "Information Leakage", + "findingURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027" + }, + { + "pluginId": "40025", + "pluginName": "Proxy Disclosure", + "severity": "Medium", + "host": "https://localhost:9000", + "paths": [ + { + "path": "/payload/3105", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107368/message/1586" + }, + { + "path": "/payload/stream/3096", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107455/message/1593" + }, + { + "path": "", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107357/message/1571" + }, + { + "path": "/payload/3101", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107364/message/1575" + }, + { + "path": "/basic-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107360/message/1565" + }, + { + "path": "/payload/3115", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107371/message/1584" + }, + { + "path": "/payload/3097", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107365/message/1573" + }, + { + "path": "/payload/3111", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107370/message/1581" + }, + { + "path": "/", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107359/message/1568" + }, + { + "path": "/jwt-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107362/message/1578" + } + ], + "pathStats": [ + { + "status": "NEW", + "count": 10 + } + ], + "totalCount": "10", + "category": "Information Leakage", + "findingURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025" + }, + { + "pluginId": "20012", + "pluginName": "Anti CSRF Tokens Scanner", + "severity": "High", + "host": "https://localhost:9000", + "paths": [ + { + "path": "/payload/3111", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107370/message/1167" + }, + { + "path": "/payload/3103", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107366/message/1154" + }, + { + "path": "/payload/3107", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107369/message/1156" + }, + { + "path": "/payload/3101", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107364/message/1171" + }, + { + "path": "/payload/3113", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107374/message/1164" + }, + { + "path": "/payload/3105", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107368/message/1169" + }, + { + "path": "/payload/3097", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107365/message/1166" + }, + { + "path": "/basic-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107360/message/1158" + }, + { + "path": "/jwt-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107362/message/1152" + }, + { + "path": "/payload/3099", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107367/message/1161" + } + ], + "pathStats": [ + { + "status": "NEW", + "count": 10 + } + ], + "totalCount": "10", + "category": "HTTP Header Protection", + "findingURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012" + }, + { + "pluginId": "40012", + "pluginName": "Cross Site Scripting Weakness (Reflected in JSON Response)", + "severity": "High", + "host": "https://localhost:9000", + "paths": [ + { + "path": "/search", + "method": "POST", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40012/path/107358/message/236" + } + ], + "pathStats": [ + { + "status": "NEW", + "count": 1 + } + ], + "totalCount": "1", + "category": "Input Sanitization", + "findingURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40012" + }, + { + "pluginId": "10038", + "pluginName": "Content Security Policy (CSP) Header Not Set", + "severity": "Medium", + "host": "https://localhost:9000", + "paths": [ + { + "path": "/payload/3099", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107367/message/49" + }, + { + "path": "/", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107359/message/17" + }, + { + "path": "/basic-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107360/message/20" + }, + { + "path": "/search", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107358/message/10" + }, + { + "path": "/jwt-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107362/message/18" + }, + { + "path": "/search", + "method": "POST", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107358/message/21" + }, + { + "path": "/payload/3097", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107365/message/45" + }, + { + "path": "/token-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107361/message/19" + }, + { + "path": "/payload/3103", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107366/message/47" + }, + { + "path": "", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107357/message/9" + }, + { + "path": "/payloads", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107363/message/24" + }, + { + "path": "/payload/3101", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107364/message/48" + } + ], + "pathStats": [ + { + "status": "NEW", + "count": 12 + } + ], + "totalCount": "12", + "category": "Information Leakage", + "findingURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038" + }, + { + "pluginId": "10063", + "pluginName": "Permissions Policy Header Not Set", + "severity": "Low", + "host": "https://localhost:9000", + "paths": [ + { + "path": "/jwt-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063/path/107362/message/18" + }, + { + "path": "/", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063/path/107359/message/17" + }, + { + "path": "/payload/3103", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063/path/107366/message/47" + }, + { + "path": "/payload/3101", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063/path/107364/message/48" + }, + { + "path": "/token-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063/path/107361/message/19" + }, + { + "path": "/search", + "method": "POST", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063/path/107358/message/21" + }, + { + "path": "/payloads", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063/path/107363/message/24" + }, + { + "path": "/basic-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063/path/107360/message/20" + }, + { + "path": "/payload/3097", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063/path/107365/message/45" + }, + { + "path": "/search", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063/path/107358/message/10" + }, + { + "path": "/payload/3099", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063/path/107367/message/49" + }, + { + "path": "", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063/path/107357/message/9" + } + ], + "pathStats": [ + { + "status": "NEW", + "count": 12 + } + ], + "totalCount": "12", + "category": "", + "findingURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10063" + } + ] + } +} diff --git a/unittests/scans/stackhawk/stackhawk_many_vul_without_duplicated_findings_subset.json b/unittests/scans/stackhawk/stackhawk_many_vul_without_duplicated_findings_subset.json new file mode 100644 index 00000000000..43b2523692a --- /dev/null +++ b/unittests/scans/stackhawk/stackhawk_many_vul_without_duplicated_findings_subset.json @@ -0,0 +1,345 @@ +{ + "service": "StackHawk", + "scanCompleted": { + "scan": { + "comment defect dojo team": "This is a subset of the StackHawk scan results without some of the findings and without some endpoints", + "id": "e2ff5651-7eef-47e9-b743-0c2f7d861e27", + "hawkscanVersion": "2.1.1", + "env": "Development", + "status": "COMPLETED", + "application": "Secured Application", + "startedTimestamp": "2022-02-16T23:07:19.575Z", + "scanURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27" + }, + "scanDuration": "21", + "spiderDuration": "45", + "completedScanStats": { + "urlsCount": "31", + "duration": "66", + "scanResultsStats": { + "totalCount": "55", + "lowCount": "22", + "mediumCount": "22", + "highCount": "11", + "lowTriagedCount": "0", + "mediumTriagedCount": "0", + "highTriagedCount": "0" + } + }, + "findings": [ + { + "pluginId": "90027", + "pluginName": "Cookie Slack Detector", + "severity": "Low", + "host": "https://localhost:9000", + "paths": [ + { + "path": "/payload/3097", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107365/message/2281" + }, + { + "path": "/search", + "method": "POST", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107358/message/2250" + }, + { + "path": "/payload/3105", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107368/message/2264" + }, + { + "path": "/payload/3113", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107374/message/2255" + }, + { + "path": "/jwt-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107362/message/2267" + }, + { + "path": "/payload/3109", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107456/message/2273" + }, + { + "path": "/", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107359/message/2241" + }, + { + "path": "/payload/3107", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027/path/107369/message/2253" + } + ], + "pathStats": [ + { + "status": "NEW", + "count": 8 + } + ], + "totalCount": "8", + "category": "Information Leakage", + "findingURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/90027" + }, + { + "pluginId": "40025", + "pluginName": "Proxy Disclosure", + "severity": "Medium", + "host": "https://localhost:9000", + "paths": [ + { + "path": "/payload/3105", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107368/message/1586" + }, + { + "path": "/payload/stream/3096", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107455/message/1593" + }, + { + "path": "", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107357/message/1571" + }, + { + "path": "/payload/3101", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107364/message/1575" + }, + { + "path": "/basic-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107360/message/1565" + }, + { + "path": "/payload/3115", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107371/message/1584" + }, + { + "path": "/payload/3097", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107365/message/1573" + }, + { + "path": "/", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107359/message/1568" + }, + { + "path": "/jwt-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025/path/107362/message/1578" + } + ], + "pathStats": [ + { + "status": "NEW", + "count": 9 + } + ], + "totalCount": "9", + "category": "Information Leakage", + "findingURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40025" + }, + { + "pluginId": "20012", + "pluginName": "Anti CSRF Tokens Scanner", + "severity": "High", + "host": "https://localhost:9000", + "paths": [ + { + "path": "/payload/3111", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107370/message/1167" + }, + { + "path": "/payload/3103", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107366/message/1154" + }, + { + "path": "/payload/3107", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107369/message/1156" + }, + { + "path": "/payload/3101", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107364/message/1171" + }, + { + "path": "/payload/3113", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107374/message/1164" + }, + { + "path": "/payload/3105", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107368/message/1169" + }, + { + "path": "/jwt-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107362/message/1152" + }, + { + "path": "/payload/3099", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012/path/107367/message/1161" + } + ], + "pathStats": [ + { + "status": "NEW", + "count": 8 + } + ], + "totalCount": "8", + "category": "HTTP Header Protection", + "findingURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/20012" + }, + { + "pluginId": "40012", + "pluginName": "Cross Site Scripting Weakness (Reflected in JSON Response)", + "severity": "High", + "host": "https://localhost:9000", + "paths": [ + { + "path": "/search", + "method": "POST", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40012/path/107358/message/236" + } + ], + "pathStats": [ + { + "status": "NEW", + "count": 1 + } + ], + "totalCount": "1", + "category": "Input Sanitization", + "findingURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/40012" + }, + { + "pluginId": "10038", + "pluginName": "Content Security Policy (CSP) Header Not Set", + "severity": "Medium", + "host": "https://localhost:9000", + "paths": [ + { + "path": "/payload/3099", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107367/message/49" + }, + { + "path": "/", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107359/message/17" + }, + { + "path": "/basic-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107360/message/20" + }, + { + "path": "/search", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107358/message/10" + }, + { + "path": "/jwt-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107362/message/18" + }, + { + "path": "/search", + "method": "POST", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107358/message/21" + }, + { + "path": "/payload/3097", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107365/message/45" + }, + { + "path": "/token-auth", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107361/message/19" + }, + { + "path": "/payload/3103", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107366/message/47" + }, + { + "path": "", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107357/message/9" + }, + { + "path": "/payloads", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107363/message/24" + }, + { + "path": "/payload/3101", + "method": "GET", + "status": "NEW", + "pathURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038/path/107364/message/48" + } + ], + "pathStats": [ + { + "status": "NEW", + "count": 12 + } + ], + "totalCount": "12", + "category": "Information Leakage", + "findingURL": "https://app.stackhawk.com/scans/e2ff5651-7eef-47e9-b743-0c2f7d861e27/finding/10038" + } + ] + } +} diff --git a/unittests/test_importers_performance.py b/unittests/test_importers_performance.py new file mode 100644 index 00000000000..50a62a62e03 --- /dev/null +++ b/unittests/test_importers_performance.py @@ -0,0 +1,200 @@ +import logging +from contextlib import contextmanager +from unittest.mock import patch + +from crum import impersonate +from django.contrib.contenttypes.models import ContentType +from django.utils import timezone + +from dojo.decorators import dojo_async_task_counter +from dojo.importers.default_importer import DefaultImporter +from dojo.importers.default_reimporter import DefaultReImporter +from dojo.models import ( + Development_Environment, + Dojo_User, + Endpoint, + Endpoint_Status, + Engagement, + Finding, + Product, + Product_Type, + Test, + User, +) + +from .dojo_test_case import DojoTestCase, get_unit_tests_scans_path + +logger = logging.getLogger(__name__) + +STACK_HAWK_FILENAME = get_unit_tests_scans_path("stackhawk") / "stackhawk_many_vul_without_duplicated_findings.json" +STACK_HAWK_SUBSET_FILENAME = get_unit_tests_scans_path("stackhawk") / "stackhawk_many_vul_without_duplicated_findings_subset.json" +STACK_HAWK_SCAN_TYPE = "StackHawk HawkScan" + +NPM_AUDIT_NO_VULN_FILENAME = get_unit_tests_scans_path("npm_audit") / "one_vuln.json" +NPM_AUDIT_SCAN_TYPE = "NPM Audit Scan" + + +class TestDojoImporterPerformance(DojoTestCase): + + def setUp(self): + super().setUp() + self.system_settings(enable_webhooks_notifications=False) + self.system_settings(enable_product_grade=False) + self.system_settings(enable_github=False) + # Warm up ContentType cache for relevant models. This is needed if we want to be able to run the test in isolation + # As part of the test suite the ContentTYpe ids will already be cached and won't affect the query count. + # But if we run the test in isolation, the ContentType ids will not be cached and will result in more queries. + # By warming up the cache here, these queries are executed before we start counting queries + for model in [Development_Environment, Dojo_User, Endpoint, Endpoint_Status, Engagement, Finding, Product, Product_Type, User, Test]: + ContentType.objects.get_for_model(model) + + @contextmanager + def assertNumAsyncTask(self, num): + dojo_async_task_counter.start() + try: + yield + finally: + dojo_async_task_counter.stop() + actual = dojo_async_task_counter.get() + if actual != num: + tasks = dojo_async_task_counter.get_tasks() + tasks_str = "\n".join(str(task) for task in tasks) + msg = ( + f"Expected {num} celery tasks, but {actual} were created.\n" + f"Tasks created:\n{tasks_str}" + ) + raise self.failureException(msg) + + def import_reimport_performance(self, expected_num_queries1, expected_num_async_tasks1, expected_num_queries2, expected_num_async_tasks2, expected_num_queries3, expected_num_async_tasks3): + """ + Log output can be quite large as when the assertNumQueries fails, all queries are printed. + It could be usefule to capture the output in `less`: + ./run-unittest.sh --test-case unittests.test_importers_performance.TestDojoImporterPerformance 2>&1 | less + Then search for `expected` to find the lines where the expected number of queries is printed. + Or you can use `grep` to filter the output: + ./run-unittest.sh --test-case unittests.test_importers_performance.TestDojoImporterPerformance 2>&1 | grep expected + """ + product_type, _created = Product_Type.objects.get_or_create(name="test") + product, _created = Product.objects.get_or_create( + name="TestDojoDefaultImporter", + prod_type=product_type, + ) + engagement, _created = Engagement.objects.get_or_create( + name="Test Create Engagement", + product=product, + target_start=timezone.now(), + target_end=timezone.now(), + ) + lead, _ = User.objects.get_or_create(username="admin") + environment, _ = Development_Environment.objects.get_or_create(name="Development") + + # first import the subset which missed one finding and a couple of endpoints on some of the findings + with ( + self.subTest("import1"), impersonate(Dojo_User.objects.get(username="admin")), + self.assertNumQueries(expected_num_queries1), + self.assertNumAsyncTask(expected_num_async_tasks1), + STACK_HAWK_SUBSET_FILENAME.open(encoding="utf-8") as scan, + ): + import_options = { + "user": lead, + "lead": lead, + "scan_date": None, + "environment": environment, + "minimum_severity": "Info", + "active": True, + "verified": True, + "sync": True, + "scan_type": STACK_HAWK_SCAN_TYPE, + "engagement": engagement, + } + importer = DefaultImporter(**import_options) + test, _, _len_new_findings, _len_closed_findings, _, _, _ = importer.process_scan(scan) + + # use reimport with the full report so it add a finding and some endpoints + with ( + self.subTest("reimport1"), impersonate(Dojo_User.objects.get(username="admin")), + self.assertNumQueries(expected_num_queries2), + self.assertNumAsyncTask(expected_num_async_tasks2), + STACK_HAWK_FILENAME.open(encoding="utf-8") as scan, + ): + reimport_options = { + "test": test, + "user": lead, + "lead": lead, + "scan_date": None, + "minimum_severity": "Info", + "active": True, + "verified": True, + "sync": True, + "scan_type": STACK_HAWK_SCAN_TYPE, + } + reimporter = DefaultReImporter(**reimport_options) + test, _, _len_new_findings, _len_closed_findings, _, _, _ = reimporter.process_scan(scan) + + # use reimport with the subset again to close a finding and mitigate some endpoints + with ( + self.subTest("reimport2"), impersonate(Dojo_User.objects.get(username="admin")), + self.assertNumQueries(expected_num_queries3), + self.assertNumAsyncTask(expected_num_async_tasks3), + STACK_HAWK_SUBSET_FILENAME.open(encoding="utf-8") as scan, + ): + reimport_options = { + "test": test, + "user": lead, + "lead": lead, + "scan_date": None, + "minimum_severity": "Info", + "active": True, + "verified": True, + "sync": True, + "scan_type": STACK_HAWK_SCAN_TYPE, + } + reimporter = DefaultReImporter(**reimport_options) + test, _, _len_new_findings, _len_closed_findings, _, _, _ = reimporter.process_scan(scan) + + def test_import_reimport_reimport_performance(self): + self.import_reimport_performance( + expected_num_queries1=603, + expected_num_async_tasks1=15, + expected_num_queries2=489, + expected_num_async_tasks2=23, + expected_num_queries3=347, + expected_num_async_tasks3=20, + ) + + @patch("dojo.decorators.we_want_async", return_value=False) + def test_import_reimport_reimport_performance_no_async(self, mock): + """ + This test checks the performance of the importers when they are run in sync mode. + The reason for this is that we also want to be aware of when a PR affects the number of queries + or async tasks created by a background task. + The impersonate context manager above does not work as expected for disabling async, + so we patch the we_want_async decorator to always return False. + """ + self.import_reimport_performance( + expected_num_queries1=673, + expected_num_async_tasks1=25, + expected_num_queries2=544, + expected_num_async_tasks2=30, + expected_num_queries3=387, + expected_num_async_tasks3=25, + ) + + @patch("dojo.decorators.we_want_async", return_value=False) + def test_import_reimport_reimport_performance_no_async_with_product_grading(self, mock): + """ + This test checks the performance of the importers when they are run in sync mode. + The reason for this is that we also want to be aware of when a PR affects the number of queries + or async tasks created by a background task. + The impersonate context manager above does not work as expected for disabling async, + so we patch the we_want_async decorator to always return False. + """ + self.system_settings(enable_product_grade=True) + self.import_reimport_performance( + expected_num_queries1=673, + expected_num_async_tasks1=25, + expected_num_queries2=544, + expected_num_async_tasks2=30, + expected_num_queries3=387, + expected_num_async_tasks3=25, + )