Merge pull request #1 from OPPIDA/spotbugs

Author: nolliv22

.pre-commit-config.yaml

@@ -1,4 +1,9 @@
+default_install_hook_types:
+  - pre-commit
+  - commit-msg
+  
 fail_fast: true
+
 repos:
   - repo: https://github.com/astral-sh/uv-pre-commit
     rev: 0.8.9
@@ -15,4 +20,11 @@ repos:
     hooks:
       - id: ruff-check
         args: [--fix]
-      - id: ruff-format
+      - id: ruff-format
+
+  - repo: https://github.com/compilerla/conventional-pre-commit
+    rev: v4.3.0
+    hooks:
+      - id: conventional-pre-commit
+        stages: [commit-msg]
+        args: []

codesectools/cli.py

@@ -68,24 +68,28 @@ def status(
     if sasts or (not sasts and not datasets):
         table = Table(show_lines=True)
         table.add_column("SAST", justify="center", no_wrap=True)
+        table.add_column("Type", justify="center", no_wrap=True)
         table.add_column("Status", justify="center", no_wrap=True)
         table.add_column("Note", justify="center")
         for sast_name, sast_data in SASTS_ALL.items():
             if sast_data["status"] == "full":
                 table.add_row(
                     sast_name,
+                    sast_data["sast"].__bases__[0].__name__,
                     "Full ✅",
                     "[b]Analysis[/b] and [b]result parsing[/b] are available",
                 )
             elif sast_data["status"] == "partial":
                 table.add_row(
                     sast_name,
+                    sast_data["sast"].__bases__[0].__name__,
                     "Partial ⚠️",
                     f"Only [b]result parsing[/b] is available\nMissing: [red]{sast_data['missing']}[/red]",
                 )
             else:
                 table.add_row(
                     sast_name,
+                    sast_data["sast"].__bases__[0].__name__,
                     "None ❌",
                     f"[b]Nothing[/b] is available\nMissing: [red]{sast_data['missing']}[/red]",
                 )
@@ -143,18 +147,24 @@ def get_downloadable() -> dict[str, DownloadableRequirement | Dataset]:
 
 
 if DOWNLOADABLE := get_downloadable():
-
-    @cli.command()
-    def download(
-        name: Annotated[
-            str,
-            typer.Argument(
-                click_type=Choice(["all"] + list(DOWNLOADABLE)),
-                metavar="NAME",
-            ),
-        ],
-    ) -> None:
-        """Download missing resources."""
+    download_hidden = False
+    download_arg_type = str
+    download_arg_value = typer.Argument(
+        click_type=Choice(["all"] + list(DOWNLOADABLE)),
+        metavar="NAME",
+    )
+else:
+    download_hidden = True
+    download_arg_type = Optional[str]
+    download_arg_value = None
+
+
+@cli.command(hidden=download_hidden)
+def download(name: download_arg_type = download_arg_value) -> None:
+    """Download any missing resources that are available for download."""
+    if name is None:
+        print("All downloadable resources have been retrieved.")
+    else:
         if name == "all":
             targets = DOWNLOADABLE.values()
         else:

codesectools/datasets/BenchmarkJava/dataset.py

@@ -6,11 +6,12 @@
 """
 
 import csv
+from pathlib import Path
 from typing import Self
 
 import git
 
-from codesectools.datasets.core.dataset import File, FileDataset
+from codesectools.datasets.core.dataset import File, PrebuiltFileDataset
 from codesectools.shared.cwe import CWE, CWEs
 
 
@@ -50,7 +51,7 @@ def __init__(
         self.vuln_type = vuln_type
 
 
-class BenchmarkJava(FileDataset):
+class BenchmarkJava(PrebuiltFileDataset):
     """Represents the BenchmarkJava dataset.
 
     This class handles the loading of the dataset, which includes Java test files
@@ -67,6 +68,9 @@ class BenchmarkJava(FileDataset):
     license = "GPL-2.0"
     license_url = "https://github.com/OWASP-Benchmark/BenchmarkJava/blob/master/LICENSE"
 
+    build_command = "mvn clean compile"
+    prebuilt_expected = (Path("target/classes/org/owasp/benchmark/testcode"), "*.class")
+
     def __init__(self, lang: None | str = None) -> None:
         """Initialize the BenchmarkJava dataset.
 
@@ -97,21 +101,8 @@ def __eq__(self, other: str | Self) -> bool:
 
     def download_files(self: Self) -> None:
         """Download the dataset files from the official Git repository."""
-        repo = git.Repo.clone_from(
-            "https://github.com/OWASP-Benchmark/BenchmarkJava.git",
-            self.directory,
-            depth=1,
-            sparse=True,
-            filter=["tree:0"],
-        )
-        repo.git.sparse_checkout(
-            "set",
-            "--no-cone",
-            *[
-                "src/main/java/org/owasp/benchmark/testcode/",
-                "expectedresults-1.2.csv",
-                "LICENSE",
-            ],
+        git.Repo.clone_from(
+            "https://github.com/OWASP-Benchmark/BenchmarkJava.git", self.directory
         )
 
     def load_dataset(self) -> list[TestCode]:

codesectools/datasets/core/dataset.py

@@ -9,6 +9,7 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
+from pathlib import Path
 from typing import TYPE_CHECKING
 
 import git
@@ -21,7 +22,6 @@
 from codesectools.utils import USER_CACHE_DIR
 
 if TYPE_CHECKING:
-    from pathlib import Path
     from typing import Self
 
     from codesectools.sasts.core.parser import AnalysisResult, Defect
@@ -132,6 +132,31 @@ def list_dataset_full_names(cls) -> list[str]:
         return sorted([f"{cls.name}_{lang}" for lang in cls.supported_languages])
 
 
+class PrebuiltDatasetMixin:
+    """Provide functionality for datasets that require a build step."""
+
+    build_command: str
+    prebuilt_expected: tuple[Path, str]
+
+    def is_built(self) -> bool:
+        """Check if the dataset has been built."""
+        if not self.build_command:
+            return False
+        prebuilt_dir, prebuilt_glob = self.prebuilt_expected
+        prebuilt_path = USER_CACHE_DIR / self.name / prebuilt_dir
+        if not prebuilt_path.is_dir():
+            return False
+        if not self.list_prebuilt_files():
+            return False
+        return True
+
+    def list_prebuilt_files(self) -> list[Path]:
+        """List the pre-built artefact files."""
+        prebuilt_dir, prebuilt_glob = self.prebuilt_expected
+        prebuilt_path = USER_CACHE_DIR / self.name / prebuilt_dir
+        return list(prebuilt_path.glob(prebuilt_glob))
+
+
 class DatasetUnit:
     """Base class for a single unit within a dataset.
 
@@ -271,7 +296,7 @@ def validate(self, analysis_result: AnalysisResult) -> FileDatasetData:
             if not defect.cwe or defect.cwe.id == -1:
                 continue
 
-            file_cwe_pair = (defect.file_path, defect.cwe)
+            file_cwe_pair = (Path(defect.file).name, defect.cwe)  # TODO: USE FULL PATH
             if file_cwe_pair not in unique_reported_defects:
                 unique_reported_defects[file_cwe_pair] = defect
 
@@ -329,6 +354,12 @@ def validate(self, analysis_result: AnalysisResult) -> FileDatasetData:
         )
 
 
+class PrebuiltFileDataset(PrebuiltDatasetMixin, FileDataset):
+    """Represent a file-based dataset that requires a build step."""
+
+    pass
+
+
 class FileDatasetData(BenchmarkData):
     """Store the results of validating an analysis against a FileDataset.
 

codesectools/sasts/__init__.py

@@ -22,31 +22,32 @@
 from codesectools.utils import SASTS_DIR
 
 SASTS_ALL = {}
-for child in SASTS_DIR.iterdir():
+for child in (SASTS_DIR / "tools").iterdir():
     if child.is_dir():
-        if list(child.glob("sast.py")) and child.name not in ["all", "core"]:
-            sast_name = child.name
-
-            sast_module = importlib.import_module(
-                f"codesectools.sasts.{sast_name}.sast"
-            )
-
-            sast: SAST = getattr(sast_module, f"{sast_name}SAST")
-            sast_instance = sast()
-            analysis_result: AnalysisResult = getattr(
-                sast_module, f"{sast_name}AnalysisResult"
-            )
-
-            cli_module = importlib.import_module(f"codesectools.sasts.{sast_name}.cli")
-            cli_factory: typer.Typer = getattr(cli_module, f"{sast_name}CLIFactory")
-
-            SASTS_ALL[sast_name] = {
-                "status": sast_instance.status,
-                "missing": sast_instance.missing,
-                "properties": sast_instance.properties,
-                "sast": sast,
-                "analysis_result": analysis_result,
-                "cli_factory": cli_factory,
-            }
+        sast_name = child.name
+
+        sast_module = importlib.import_module(
+            f"codesectools.sasts.tools.{sast_name}.sast"
+        )
+
+        sast: SAST = getattr(sast_module, f"{sast_name}SAST")
+        sast_instance = sast()
+        analysis_result: AnalysisResult = getattr(
+            sast_module, f"{sast_name}AnalysisResult"
+        )
+
+        cli_module = importlib.import_module(
+            f"codesectools.sasts.tools.{sast_name}.cli"
+        )
+        cli_factory: typer.Typer = getattr(cli_module, f"{sast_name}CLIFactory")
+
+        SASTS_ALL[sast_name] = {
+            "status": sast_instance.status,
+            "missing": sast_instance.missing,
+            "properties": sast_instance.properties,
+            "sast": sast,
+            "analysis_result": analysis_result,
+            "cli_factory": cli_factory,
+        }
 
 SASTS_ALL = dict(sorted(SASTS_ALL.items()))

codesectools/sasts/all/cli.py

@@ -20,6 +20,7 @@
 from codesectools.sasts import SASTS_ALL
 from codesectools.sasts.all.graphics import ProjectGraphics
 from codesectools.sasts.all.sast import AllSAST
+from codesectools.sasts.core.sast import PrebuiltSAST
 
 
 def build_cli() -> typer.Typer:
@@ -70,6 +71,15 @@ def analyze(
                 metavar="LANG",
             ),
         ],
+        # Additional options
+        artifact_dir: Annotated[
+            Path | None,
+            typer.Option(
+                help="Pre-built artifacts directory (for PrebuiltSAST only)",
+                metavar="ARTIFACT_DIR",
+            ),
+        ] = None,
+        # Common NOT REQUIRED option
         overwrite: Annotated[
             bool,
             typer.Option(
@@ -80,16 +90,27 @@ def analyze(
     ) -> None:
         """Run analysis on the current project with all available SASTs."""
         for sast in all_sast.sasts:
+            if isinstance(sast, PrebuiltSAST) and artifact_dir is None:
+                print(f"{sast.name} required pre-built artifacts for analysis")
+                print(
+                    "Please provide the directory with artifacts (with --artifact-dir) to include this tool"
+                )
+                continue
+
             output_dir = sast.output_dir / Path.cwd().name
             if output_dir.is_dir():
                 if overwrite:
                     shutil.rmtree(output_dir)
-                    sast.run_analysis(lang, Path.cwd(), output_dir)
+                    sast.run_analysis(
+                        lang, Path.cwd(), output_dir, artifact_dir=artifact_dir
+                    )
                 else:
                     print(f"Found existing analysis result at {output_dir}")
                     print("Use --overwrite to overwrite it")
             else:
-                sast.run_analysis(lang, Path.cwd(), output_dir)
+                sast.run_analysis(
+                    lang, Path.cwd(), output_dir, artifact_dir=artifact_dir
+                )
 
     @cli.command(help="Benchmark a dataset using all SASTs.")
     def benchmark(

codesectools/sasts/all/parser.py

@@ -149,8 +149,8 @@ def stats_by_scores(self) -> dict:
 
             defect_locations = {}
             for defect in defects:
-                if location := defect.location:
-                    start, end = location
+                if any(defect.location):
+                    start, end = defect.location
                     for line in range(start, end + 1):
                         if not defect_locations.get(line):
                             defect_locations[line] = []
@@ -202,11 +202,12 @@ def prepare_report_data(self) -> dict:
 
             locations = []
             for defect in defects:
-                start, end = defect.location
-                if start and end:
-                    locations.append(
-                        (defect.sast, defect.cwe, defect.message, (start, end))
-                    )
+                if any(defect.location):
+                    start, end = defect.location
+                    if start and end:
+                        locations.append(
+                            (defect.sast, defect.cwe, defect.message, (start, end))
+                        )
 
             report["defects"][defect_file] = {
                 "score": scores[defect_file]["score"],