Skip to content

Commit d6cf51c

Browse files
valentijnscholtenvalentijnscholten
committed
initial command
1 parent b769410 commit d6cf51c

1 file changed

Lines changed: 179 additions & 0 deletions

File tree

‎dojo/management/commands/import_all_unittest_scans.py‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
import json
2+
import logging
3+
import os
4+
from importlib import import_module
5+
from importlib.util import find_spec
6+
from inspect import isclass
7+
from pathlib import Path
8+
9+
from django.core.management.base import BaseCommand
10+
from django.urls import reverse
11+
from django.utils import timezone
12+
from rest_framework.authtoken.models import Token
13+
from rest_framework.test import APIClient
14+
15+
import dojo.tools.factory
16+
from dojo.models import Engagement, Product, Product_Type
17+
from unittests.test_dashboard import User
18+
19+
logger = logging.getLogger(__name__)
20+
21+
22+
class Command(BaseCommand):
23+
24+
help = (
25+
"Command to import all scans available in unittests folder"
26+
)
27+
28+
def get_test_admin(self, *args, **kwargs):
29+
return User.objects.get(username="admin")
30+
31+
def import_scan(self, payload, expected_http_status_code):
32+
testuser = self.get_test_admin()
33+
token = Token.objects.get(user=testuser)
34+
self.client = APIClient()
35+
self.client.credentials(HTTP_AUTHORIZATION="Token " + token.key)
36+
37+
logger.debug("import_scan payload %s", payload)
38+
response = self.client.post(reverse("importscan-list"), payload)
39+
if expected_http_status_code != response.status_code:
40+
msg = f"Expected HTTP status code {expected_http_status_code}, got {response.status_code}: {response.content[:1000]}"
41+
raise AssertionError(
42+
msg,
43+
)
44+
return json.loads(response.content)
45+
46+
def import_scan_with_params(self, filename, scan_type="ZAP Scan", engagement=1, minimum_severity="Low", *, active=True, verified=False,
47+
push_to_jira=None, endpoint_to_add=None, tags=None, close_old_findings=False, group_by=None, engagement_name=None,
48+
product_name=None, product_type_name=None, auto_create_context=None, expected_http_status_code=201, test_title=None,
49+
scan_date=None, service=None, force_active=True, force_verified=True):
50+
51+
with (Path("unittests/scans") / filename).open(encoding="utf-8") as testfile:
52+
payload = {
53+
"minimum_severity": minimum_severity,
54+
"scan_type": scan_type,
55+
"file": testfile,
56+
"version": "1.0.1",
57+
"close_old_findings": close_old_findings,
58+
}
59+
60+
if active is not None:
61+
payload["active"] = active
62+
63+
if verified is not None:
64+
payload["verified"] = verified
65+
66+
if engagement:
67+
payload["engagement"] = engagement
68+
69+
if engagement_name:
70+
payload["engagement_name"] = engagement_name
71+
72+
if product_name:
73+
payload["product_name"] = product_name
74+
75+
if product_type_name:
76+
payload["product_type_name"] = product_type_name
77+
78+
if auto_create_context:
79+
payload["auto_create_context"] = auto_create_context
80+
81+
if push_to_jira is not None:
82+
payload["push_to_jira"] = push_to_jira
83+
84+
if endpoint_to_add is not None:
85+
payload["endpoint_to_add"] = endpoint_to_add
86+
87+
if tags is not None:
88+
payload["tags"] = tags
89+
90+
if group_by is not None:
91+
payload["group_by"] = group_by
92+
93+
if test_title is not None:
94+
payload["test_title"] = test_title
95+
96+
if scan_date is not None:
97+
payload["scan_date"] = scan_date
98+
99+
if service is not None:
100+
payload["service"] = service
101+
102+
return self.import_scan(payload, expected_http_status_code)
103+
104+
def handle(self, *args, **options):
105+
prod_type = Product_Type.objects.first()
106+
prod, _ = Product.objects.get_or_create(prod_type=prod_type, name="prod name")
107+
eng, _ = Engagement.objects.get_or_create(product=prod, name="valentijn engagement", target_start=timezone.now(), target_end=timezone.now())
108+
109+
error_count = 0
110+
error_messages = {}
111+
# iterate through the modules in the current package
112+
package_dir = str(Path(dojo.tools.factory.__file__).resolve().parent)
113+
for module_name in os.listdir(package_dir): # noqa: PTH208
114+
# check if it's dir
115+
if (Path(package_dir) / module_name).is_dir():
116+
try:
117+
# check if it's a Python module
118+
if find_spec(f"dojo.tools.{module_name}.parser"):
119+
# import the module and iterate through its attributes
120+
module = import_module(f"dojo.tools.{module_name}.parser")
121+
for attribute_name in dir(module):
122+
attribute = getattr(module, attribute_name)
123+
if isclass(attribute) and attribute_name.lower() == module_name.replace("_", "") + "parser":
124+
logger.info(f"Loading {module_name} parser")
125+
scan_dir = Path("unittests") / "scans" / module_name
126+
for scan_file in scan_dir.glob("*.json"):
127+
if scan_file.name != "report_invalid.json": # meterian
128+
if scan_file.name != "single_finding_no_libraryId.json": # checkmarx_osa
129+
if scan_file.name not in ["issue_7897.json", "empty_with_error.json", "many_vuln_npm7.json"]: # npm_audit # noqa: PLR6201
130+
if scan_file.name != "threat_composer_no_threats_with_error.json": # threat_composer
131+
if scan_file.name != "very_many_vulns.json": # jfrog_xray
132+
try:
133+
logger.debug(f"Importing scan {scan_file.name} using {module_name} parser")
134+
parser = attribute()
135+
# with scan_file.open(encoding="utf-8") as f:
136+
# findings = parser.get_findings(f, Test())
137+
result = self.import_scan_with_params(
138+
filename=module_name + "/" + scan_file.name,
139+
scan_type=parser.get_scan_types()[0],
140+
engagement=eng.id,
141+
)
142+
# logger.debug(f"Result of import: {result}")
143+
# raise Exception(f"Scan {scan_file.name} is not expected to be imported, but it was.")
144+
logger.info(f"Imported findings from {module_name + scan_file.name}")
145+
except Exception as e:
146+
logger.error(f"Error importing scan {module_name + scan_file.name}: {e}")
147+
error_count += 1
148+
error_messages[module_name + "/" + scan_file.name] = result.get("message", str(e))
149+
150+
except:
151+
logger.exception(f"failed to load {module_name}")
152+
raise
153+
154+
logger.error(f"Error count: {error_count}")
155+
for scan, message in error_messages.items():
156+
logger.error(f"Error importing scan {scan}: {message}")
157+
158+
159+
# errors:
160+
161+
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:154] Error count: 18
162+
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan generic/generic_empty.json: Expected HTTP status code 201, got 400: b'{"message":"[\\"Required fields are missing: [\'description\', \'severity\', \'title\']\\"]","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
163+
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan generic/test_with_image_no_ext.json: Expected HTTP status code 201, got 400: b'{"message":"[\'Unsupported extension. Supported extensions are as follows: .txt, .pdf, .json, .xml, .csv, .yml, .png, .jpeg, .sarif, .xlsx, .doc, .html, .js, .nessus, .zip, .fpr\']","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
164+
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan generic/generic_invalid.json: Expected HTTP status code 201, got 400: b'{"message":"[\\"Not allowed fields are present: [\'invalid_field\', \'last_status_update\']\\"]","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
165+
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan whitehat_sentinel/empty_file.json: Expected HTTP status code 201, got 400: b'{"message":"[\'collection key not present or there were not findings present.\']","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
166+
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan gitlab_api_fuzzing/gitlab_api_fuzzing_invalid.json: Expected HTTP status code 201, got 500: b'{"message":"Internal server error, check logs for details","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
167+
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan burp_graphql/null_title.json: Expected HTTP status code 201, got 400: b'{"message":"[\'Issue does not have a name\']","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
168+
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan stackhawk/oddly_familiar_json_that_isnt_us.json: Expected HTTP status code 201, got 400: b'{"message":"[\' Unexpected JSON format provided. Need help? Check out the StackHawk Docs at https://docs.stackhawk.com/workflow-integrations/defect-dojo.html\']","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
169+
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan stackhawk/invalid.json: Expected HTTP status code 201, got 400: b'{"message":"[\' Unexpected JSON format provided. Need help? Check out the StackHawk Docs at https://docs.stackhawk.com/workflow-integrations/defect-dojo.html\']","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
170+
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan kubehunter/empty.json: Expected HTTP status code 201, got 400: b'{"message":"[\'Expecting value: line 1 column 1 (char 0)\']","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
171+
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan threagile/bad_formatted_risks_file.json: Expected HTTP status code 201, got 500: b'{"message":"Internal server error, check logs for details","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
172+
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan hydra/oddly_familiar_json_that_isnt_us.json: Expected HTTP status code 201, got 400: b'{"message":"[\\"Unexpected JSON format provided. That doesn\'t look like a Hydra scan!\\"]","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
173+
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan hydra/invalid.json: Expected HTTP status code 201, got 400: b'{"message":"[\\"Unexpected JSON format provided. That doesn\'t look like a Hydra scan!\\"]","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
174+
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan anchore_enterprise/invalid_checks_format.json: Expected HTTP status code 201, got 400: b'{"message":"[\\"Invalid format: \'result\' key not found\\"]","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
175+
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan risk_recon/bad_key.json: Expected HTTP status code 201, got 500: b'{"message":"Internal server error, check logs for details","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
176+
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan risk_recon/bad_url.json: Expected HTTP status code 201, got 500: b'{"message":"Internal server error, check logs for details","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
177+
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan osv_scanner/some_findings.json: Expected HTTP status code 201, got 500: b'{"message":"Internal server error, check logs for details","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
178+
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan coverity_api/wrong.json: Expected HTTP status code 201, got 400: b'{"message":"[\\"(\'Report file is not a well-formed Coverity REST view report\', \'wrong.json\')\\"]","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
179+
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan govulncheck/empty.json: Expected HTTP status code 201, got 400: b'{"message":"[\'Invalid JSON format\']","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'

0 commit comments

Comments
 (0)