Skip to content

Commit 11e7873

Browse files
valentijnscholtenvalentijnscholten
committed
working version
1 parent 2a9182f commit 11e7873

1 file changed

Lines changed: 81 additions & 54 deletions

File tree

‎dojo/management/commands/import_all_unittest_scans.py‎

Lines changed: 81 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import json
22
import logging
33
import os
4+
from datetime import datetime
45
from importlib import import_module
56
from importlib.util import find_spec
67
from inspect import isclass
@@ -22,9 +23,27 @@
2223
class Command(BaseCommand):
2324

2425
help = (
26+
"EXPERIMENTAL: May be changed/deprecated/removed without prior notice. "
2527
"Command to import all scans available in unittests folder"
2628
)
2729

30+
def add_arguments(self, parser):
31+
parser.add_argument(
32+
"--product-name-prefix",
33+
type=lambda s: s if len(s) <= 250 else parser.error("product-name-prefix must be at most 250 characters"),
34+
help="Prefix to use for product names, defaults to 'All scans <today>'. Max length 250 characters.",
35+
)
36+
parser.add_argument(
37+
"--include-very-big-scans",
38+
action="store_true",
39+
default=False,
40+
help="Include very big scans like jfrog_xray very_many_vulns.json (default: False)",
41+
)
42+
parser.add_argument("--tests-per-engagement", type=int, default=10, help="Number of tests per engagement before a new engagement is created, defaults to 10")
43+
parser.add_argument("--engagements-per-product", type=int, default=50, help="Number of engagements per product before a new product is created, defaults to 50")
44+
parser.add_argument("--products-per-product-type", type=int, default=15, help="Number of products per product type before a new product type is created, defaults to 15")
45+
parser.add_argument("--number-of-runs", type=int, default=1, help="Number of times to run the import of all sample scans, defaults to 1")
46+
2847
def get_test_admin(self, *args, **kwargs):
2948
return User.objects.get(username="admin")
3049

@@ -34,7 +53,6 @@ def import_scan(self, payload, expected_http_status_code):
3453
self.client = APIClient()
3554
self.client.credentials(HTTP_AUTHORIZATION="Token " + token.key)
3655

37-
logger.debug("import_scan payload %s", payload)
3856
response = self.client.post(reverse("importscan-list"), payload)
3957
if expected_http_status_code != response.status_code:
4058
msg = f"Expected HTTP status code {expected_http_status_code}, got {response.status_code}: {response.content[:1000]}"
@@ -101,16 +119,36 @@ def import_scan_with_params(self, filename, scan_type="ZAP Scan", engagement=1,
101119

102120
return self.import_scan(payload, expected_http_status_code)
103121

104-
def handle(self, *args, **options):
105-
prod_type = Product_Type.objects.first()
106-
prod, _ = Product.objects.get_or_create(prod_type=prod_type, name="prod name")
107-
eng, _ = Engagement.objects.get_or_create(product=prod, name="valentijn engagement", target_start=timezone.now(), target_end=timezone.now())
122+
def import_all_unittest_scans(self, product_name_prefix=None, tests_per_engagement=10, engagements_per_product=50, products_per_product_type=15, *, include_very_big_scans=False, **kwargs):
123+
logger.info(f"product_name_prefix: {product_name_prefix}, tests_per_engagement: {tests_per_engagement}, engagements_per_product: {engagements_per_product}, products_per_product_type: {products_per_product_type}")
124+
product_type_prefix = "Sample scans " + datetime.now().strftime("%Y-%m-%d %H:%M:%S")
125+
product_type_index = 1
126+
127+
product_index = 1
128+
engagement_index = 1
129+
tests_index = 1
108130

109131
error_count = 0
110132
error_messages = {}
111133
# iterate through the modules in the current package
112134
package_dir = str(Path(dojo.tools.factory.__file__).resolve().parent)
113135
for module_name in os.listdir(package_dir): # noqa: PTH208
136+
if tests_index > tests_per_engagement:
137+
tests_index = 1
138+
engagement_index += 1
139+
140+
if engagement_index > engagements_per_product:
141+
engagement_index = 1
142+
product_index += 1
143+
144+
if product_index > products_per_product_type:
145+
product_index = 1
146+
product_type_index += 1
147+
148+
prod_type, _ = Product_Type.objects.get_or_create(name=product_type_prefix + f" {product_type_index}")
149+
prod, _ = Product.objects.get_or_create(prod_type=prod_type, name=product_name_prefix + f" {product_type_index}:{product_index}", description="Sample scans for unittesting")
150+
eng, _ = Engagement.objects.get_or_create(product=prod, name="Sample scan engagement" + f" {engagement_index}", target_start=timezone.now(), target_end=timezone.now())
151+
114152
# check if it's dir
115153
if (Path(package_dir) / module_name).is_dir():
116154
try:
@@ -121,59 +159,48 @@ def handle(self, *args, **options):
121159
for attribute_name in dir(module):
122160
attribute = getattr(module, attribute_name)
123161
if isclass(attribute) and attribute_name.lower() == module_name.replace("_", "") + "parser":
124-
logger.info(f"Loading {module_name} parser")
162+
logger.debug(f"Loading {module_name} parser")
125163
scan_dir = Path("unittests") / "scans" / module_name
126164
for scan_file in scan_dir.glob("*.json"):
127-
if scan_file.name != "report_invalid.json": # meterian
128-
if scan_file.name != "single_finding_no_libraryId.json": # checkmarx_osa
129-
if scan_file.name not in ["issue_7897.json", "empty_with_error.json", "many_vuln_npm7.json"]: # npm_audit # noqa: PLR6201
130-
if scan_file.name != "threat_composer_no_threats_with_error.json": # threat_composer
131-
if scan_file.name != "very_many_vulns.json": # jfrog_xray
132-
try:
133-
logger.debug(f"Importing scan {scan_file.name} using {module_name} parser")
134-
parser = attribute()
135-
# with scan_file.open(encoding="utf-8") as f:
136-
# findings = parser.get_findings(f, Test())
137-
result = self.import_scan_with_params(
138-
filename=module_name + "/" + scan_file.name,
139-
scan_type=parser.get_scan_types()[0],
140-
engagement=eng.id,
141-
)
142-
# logger.debug(f"Result of import: {result}")
143-
# raise Exception(f"Scan {scan_file.name} is not expected to be imported, but it was.")
144-
logger.info(f"Imported findings from {module_name + scan_file.name}")
145-
except Exception as e:
146-
logger.error(f"Error importing scan {module_name + scan_file.name}: {e}")
147-
error_count += 1
148-
error_messages[module_name + "/" + scan_file.name] = result.get("message", str(e))
165+
if include_very_big_scans or scan_file.name != "very_many_vulns.json": # jfrog_xray file is huge and takes too long to import
166+
try:
167+
logger.info(f"Importing scan {scan_file.name} using {module_name} parser into {prod.name}:{eng.name}")
168+
parser = attribute()
169+
# with scan_file.open(encoding="utf-8") as f:
170+
# findings = parser.get_findings(f, Test())
171+
result = self.import_scan_with_params(
172+
filename=module_name + "/" + scan_file.name,
173+
scan_type=parser.get_scan_types()[0],
174+
engagement=eng.id,
175+
)
176+
# logger.debug(f"Result of import: {result}")
177+
# raise Exception(f"Scan {scan_file.name} is not expected to be imported, but it was.")
178+
logger.debug(f"Imported findings from {module_name + scan_file.name}")
179+
tests_index += 1
180+
except Exception as e:
181+
logger.error(f"Error importing scan {module_name + scan_file.name}: {e}")
182+
error_count += 1
183+
error_messages[module_name + "/" + scan_file.name] = result.get("message", str(e))
149184

150185
except:
151186
logger.exception(f"failed to load {module_name}")
152187
raise
153188

154-
logger.error(f"Error count: {error_count}")
155-
for scan, message in error_messages.items():
156-
logger.error(f"Error importing scan {scan}: {message}")
157-
158-
159-
# errors:
160-
161-
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:154] Error count: 18
162-
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan generic/generic_empty.json: Expected HTTP status code 201, got 400: b'{"message":"[\\"Required fields are missing: [\'description\', \'severity\', \'title\']\\"]","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
163-
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan generic/test_with_image_no_ext.json: Expected HTTP status code 201, got 400: b'{"message":"[\'Unsupported extension. Supported extensions are as follows: .txt, .pdf, .json, .xml, .csv, .yml, .png, .jpeg, .sarif, .xlsx, .doc, .html, .js, .nessus, .zip, .fpr\']","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
164-
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan generic/generic_invalid.json: Expected HTTP status code 201, got 400: b'{"message":"[\\"Not allowed fields are present: [\'invalid_field\', \'last_status_update\']\\"]","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
165-
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan whitehat_sentinel/empty_file.json: Expected HTTP status code 201, got 400: b'{"message":"[\'collection key not present or there were not findings present.\']","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
166-
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan gitlab_api_fuzzing/gitlab_api_fuzzing_invalid.json: Expected HTTP status code 201, got 500: b'{"message":"Internal server error, check logs for details","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
167-
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan burp_graphql/null_title.json: Expected HTTP status code 201, got 400: b'{"message":"[\'Issue does not have a name\']","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
168-
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan stackhawk/oddly_familiar_json_that_isnt_us.json: Expected HTTP status code 201, got 400: b'{"message":"[\' Unexpected JSON format provided. Need help? Check out the StackHawk Docs at https://docs.stackhawk.com/workflow-integrations/defect-dojo.html\']","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
169-
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan stackhawk/invalid.json: Expected HTTP status code 201, got 400: b'{"message":"[\' Unexpected JSON format provided. Need help? Check out the StackHawk Docs at https://docs.stackhawk.com/workflow-integrations/defect-dojo.html\']","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
170-
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan kubehunter/empty.json: Expected HTTP status code 201, got 400: b'{"message":"[\'Expecting value: line 1 column 1 (char 0)\']","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
171-
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan threagile/bad_formatted_risks_file.json: Expected HTTP status code 201, got 500: b'{"message":"Internal server error, check logs for details","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
172-
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan hydra/oddly_familiar_json_that_isnt_us.json: Expected HTTP status code 201, got 400: b'{"message":"[\\"Unexpected JSON format provided. That doesn\'t look like a Hydra scan!\\"]","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
173-
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan hydra/invalid.json: Expected HTTP status code 201, got 400: b'{"message":"[\\"Unexpected JSON format provided. That doesn\'t look like a Hydra scan!\\"]","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
174-
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan anchore_enterprise/invalid_checks_format.json: Expected HTTP status code 201, got 400: b'{"message":"[\\"Invalid format: \'result\' key not found\\"]","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
175-
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan risk_recon/bad_key.json: Expected HTTP status code 201, got 500: b'{"message":"Internal server error, check logs for details","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
176-
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan risk_recon/bad_url.json: Expected HTTP status code 201, got 500: b'{"message":"Internal server error, check logs for details","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
177-
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan osv_scanner/some_findings.json: Expected HTTP status code 201, got 500: b'{"message":"Internal server error, check logs for details","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
178-
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan coverity_api/wrong.json: Expected HTTP status code 201, got 400: b'{"message":"[\\"(\'Report file is not a well-formed Coverity REST view report\', \'wrong.json\')\\"]","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
179-
# [25/Jun/2025 18:06:15] ERROR [dojo.management.commands.import_all_unittest_scans:156] Error importing scan govulncheck/empty.json: Expected HTTP status code 201, got 400: b'{"message":"[\'Invalid JSON format\']","pro":["Pro comes with support. Try today for free or email us at hello@defectdojo.com"]}'
189+
logger.error(f"Error count: {error_count}")
190+
for scan, message in error_messages.items():
191+
logger.error(f"Error importing scan {scan}: {message}")
192+
193+
def handle(self, *args, **options):
194+
logger.info("EXPERIMENTAL: This command may be changed/deprecated/removed without prior notice.")
195+
for i in range(options.get("number_of_runs", 1)):
196+
product_name_prefix = options.get("product_name_prefix")
197+
if not product_name_prefix:
198+
today = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
199+
product_name_prefix = f"Sample scan product {i + 1} {today}"
200+
self.import_all_unittest_scans(
201+
product_name_prefix=product_name_prefix,
202+
tests_per_engagement=options.get("tests_per_engagement"),
203+
engagements_per_product=options.get("engagements_per_product"),
204+
products_per_product_type=options.get("products_per_product_type"),
205+
include_very_big_scans=options.get("include_very_big_scans"),
206+
)

0 commit comments

Comments
 (0)