Add a new test for environments without SAST tools

.github/workflows/run-tests.yaml → .github/workflows/ci.yaml

@@ -1,4 +1,4 @@
-name: Run tests on Pull Request
+name: ci
 
 on:
   pull_request:
@@ -15,5 +15,14 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v3
 
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+
+      - name: Setup project
+        run: uv sync --locked --all-extras
+
+      - name: Run check
+        run: uv run ruff check && uv run ty check
+
       - name: Run tests
         run: make test

.github/workflows/deploy-docs.yaml → .github/workflows/docs.yaml

@@ -1,12 +1,12 @@
-name: Publish docs to GitHub Pages
+name: docs
 on:
   push:
     branches:
       - main
 permissions:
   contents: write
 jobs:
-  deploy:
+  docs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

Makefile

@@ -14,16 +14,18 @@ check:		## Lint, format, and type-check the code
 
 test:		## Run tests in a Docker container
 	@docker compose build 1>/dev/null
-	@docker compose run --rm test
+	@docker compose run --rm no-sast
+	@docker compose run --rm with-sast
 
 test-force:	## Run tests in a Docker container while ignoring any stored state
 	@docker volume rm codesectools_pytest-cache 2>&1 1>/dev/null || true
 	@docker compose build 1>/dev/null
-	@docker compose run --rm test
+	@docker compose run --rm no-sast
+	@docker compose run --rm with-sast
 
 test-debug:	## Spawn an interactive shell in the test container to debug
 	@docker compose build
 	@docker compose run --rm test /bin/bash
 
-doc-serve:	## Serve the documentation locally
+docs-serve:	## Serve the documentation locally
 	@mkdocs serve

codesectools/cli.py

@@ -135,7 +135,7 @@ def get_downloadable() -> dict[str, DownloadableRequirement | Dataset]:
         sast = sast_data["sast"]
         for req in sast.requirements.all:
             if isinstance(req, DownloadableRequirement):
-                if not req.is_fulfilled():
+                if not req.is_fulfilled() and req.dependencies_fulfilled():
                     downloadable[req.name] = req
 
     for dataset_name, dataset in DATASETS_ALL.items():

codesectools/sasts/core/cli.py

@@ -114,7 +114,9 @@ def install() -> None:
                 install_help += (
                     f"{'❌' if req in missing_reqs else '✅'} [b]{req}[/b]\n"
                 )
-                if req.instruction:
+                if req.depends_on:
+                    install_help += f"- Depends on: [red]{', '.join([str(r) for r in req.depends_on])}[/red]\n"
+                elif req.instruction:
                     install_help += f"- Instruction: [red]{req.instruction}[/red]\n"
                 if req.url:
                     install_help += f"- URL: [u]{req.url}[/u]\n"

codesectools/sasts/core/sast/requirements.py

@@ -3,7 +3,7 @@
 import shutil
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import Any, Literal
+from typing import Any, Literal, Self
 
 import requests
 import typer
@@ -21,6 +21,7 @@ class SASTRequirement(ABC):
     def __init__(
         self,
         name: str,
+        depends_on: list[Self] | None = None,
         instruction: str | None = None,
         url: str | None = None,
         doc: bool = False,
@@ -29,12 +30,14 @@ def __init__(
 
         Args:
             name: The name of the requirement.
+            depends_on: A list of other requirements that must be fulfilled first.
             instruction: A short instruction on how to download the requirement.
             url: A URL for more detailed instructions.
-            doc: A flag indicating if the instruction is available in the documentaton.
+            doc: A flag indicating if the instruction is available in the documentation.
 
         """
         self.name = name
+        self.depends_on = depends_on
         self.instruction = instruction
         self.url = url
         self.doc = doc
@@ -44,6 +47,12 @@ def is_fulfilled(self, **kwargs: Any) -> bool:
         """Check if the requirement is met."""
         pass
 
+    def dependencies_fulfilled(self) -> bool:
+        """Check if all dependencies for this requirement are fulfilled."""
+        if not self.depends_on:
+            return True
+        return all(dependency.is_fulfilled() for dependency in self.depends_on)
+
     def __repr__(self) -> str:
         """Return a developer-friendly string representation of the requirement."""
         return f"{self.__class__.__name__}({self.name})"
@@ -55,6 +64,7 @@ class DownloadableRequirement(SASTRequirement):
     def __init__(
         self,
         name: str,
+        depends_on: list[SASTRequirement] | None = None,
         instruction: str | None = None,
         url: str | None = None,
         doc: bool = False,
@@ -65,13 +75,16 @@ def __init__(
 
         Args:
             name: The name of the requirement.
+            depends_on: A list of other requirements that must be fulfilled first.
             instruction: A short instruction on how to download the requirement.
             url: A URL for more detailed instructions.
-            doc: A flag indicating if the instruction is available in the documentaton.
+            doc: A flag indicating if the instruction is available in the documentation.
 
         """
         instruction = f"cstools download {name}"
-        super().__init__(name, instruction, url, doc)
+        super().__init__(
+            name=name, depends_on=depends_on, instruction=instruction, url=url, doc=doc
+        )
 
     @abstractmethod
     def download(self, **kwargs: Any) -> None:
@@ -85,6 +98,7 @@ class Config(SASTRequirement):
     def __init__(
         self,
         name: str,
+        depends_on: list[SASTRequirement] | None = None,
         instruction: str | None = None,
         url: str | None = None,
         doc: bool = False,
@@ -93,12 +107,15 @@ def __init__(
 
         Args:
             name: The name of the requirement.
+            depends_on: A list of other requirements that must be fulfilled first.
             instruction: A short instruction on how to download the requirement.
             url: A URL for more detailed instructions.
-            doc: A flag indicating if the instruction is available in the documentaton.
+            doc: A flag indicating if the instruction is available in the documentation.
 
         """
-        super().__init__(name, instruction, url, doc)
+        super().__init__(
+            name=name, depends_on=depends_on, instruction=instruction, url=url, doc=doc
+        )
 
     def is_fulfilled(self, sast_name: str, **kwargs: Any) -> bool:
         """Check if the configuration file exists for the given SAST tool."""
@@ -111,6 +128,7 @@ class Binary(SASTRequirement):
     def __init__(
         self,
         name: str,
+        depends_on: list[SASTRequirement] | None = None,
         instruction: str | None = None,
         url: str | None = None,
         doc: bool = False,
@@ -119,12 +137,15 @@ def __init__(
 
         Args:
             name: The name of the requirement.
+            depends_on: A list of other requirements that must be fulfilled first.
             instruction: A short instruction on how to download the requirement.
             url: A URL for more detailed instructions.
-            doc: A flag indicating if the instruction is available in the documentaton.
+            doc: A flag indicating if the instruction is available in the documentation.
 
         """
-        super().__init__(name, instruction, url, doc)
+        super().__init__(
+            name=name, depends_on=depends_on, instruction=instruction, url=url, doc=doc
+        )
 
     def is_fulfilled(self, **kwargs: Any) -> bool:
         """Check if the binary is available in the system's PATH."""
@@ -140,6 +161,7 @@ def __init__(
         repo_url: str,
         license: str,
         license_url: str,
+        depends_on: list[SASTRequirement] | None = None,
         instruction: str | None = None,
         url: str | None = None,
         doc: bool = False,
@@ -151,12 +173,15 @@ def __init__(
             repo_url: The URL of the Git repository to clone.
             license: The license of the repository.
             license_url: A URL for the repository's license.
+            depends_on: A list of other requirements that must be fulfilled first.
             instruction: A short instruction on how to download the requirement.
             url: A URL for more detailed instructions.
-            doc: A flag indicating if the instruction is available in the documentaton.
+            doc: A flag indicating if the instruction is available in the documentation.
 
         """
-        super().__init__(name, instruction, url, doc)
+        super().__init__(
+            name=name, depends_on=depends_on, instruction=instruction, url=url, doc=doc
+        )
         self.repo_url = repo_url
         self.license = license
         self.license_url = license_url
@@ -206,6 +231,7 @@ def __init__(
         file_url: str,
         license: str,
         license_url: str,
+        depends_on: list[SASTRequirement] | None = None,
         instruction: str | None = None,
         url: str | None = None,
         doc: bool = False,
@@ -218,12 +244,15 @@ def __init__(
             file_url: The URL to download the file from.
             license: The license of the file.
             license_url: A URL for the file's license.
+            depends_on: A list of other requirements that must be fulfilled first.
             instruction: A short instruction on how to download the requirement.
             url: A URL for more detailed instructions.
-            doc: A flag indicating if the instruction is available in the documentaton.
+            doc: A flag indicating if the instruction is available in the documentation.
 
         """
-        super().__init__(name, instruction, url, doc)
+        super().__init__(
+            name=name, depends_on=depends_on, instruction=instruction, url=url, doc=doc
+        )
         self.parent_dir = parent_dir
         self.file_url = file_url
         self.license = license

codesectools/sasts/tools/SpotBugs/sast.py

@@ -40,10 +40,13 @@ class SpotBugsSAST(PrebuiltSAST):
     properties = SASTProperties(free=True, offline=True)
     requirements = SASTRequirements(
         full_reqs=[
-            Binary("spotbugs", url="https://github.com/spotbugs/spotbugs"),
+            binary := Binary("spotbugs", url="https://github.com/spotbugs/spotbugs"),
             File(
                 name="findsecbugs-plugin-1.14.0.jar",
-                parent_dir=Path(shutil.which("spotbugs")).parent.parent / "plugin",
+                depends_on=[binary],
+                parent_dir=Path(shutil.which("spotbugs")).parent.parent / "plugin"
+                if shutil.which("spotbugs")
+                else Path("/tmp"),
                 file_url="https://search.maven.org/remotecontent?filepath=com/h3xstream/findsecbugs/findsecbugs-plugin/1.14.0/findsecbugs-plugin-1.14.0.jar",
                 license="LGPL-3.0",
                 license_url="https://find-sec-bugs.github.io/license.htm",

docker-compose.yml

@@ -1,9 +1,26 @@
 services:
-  test:
-    image: cstools_test
+  no-sast:
+    image: cstools_no-sast
     build:
       context: .
       dockerfile: tests/Dockerfile
+      target: no-sast
+
+    tty: true
+    stdin_open: true
+
+    volumes:      
+      - pytest-cache:/app/.pytest_cache
+
+    environment:
+      _TYPER_STANDARD_TRACEBACK: 1
+
+  with-sast:
+    image: cstools_with-sast
+    build:
+      context: .
+      dockerfile: tests/Dockerfile
+      target: with-sast
 
     tty: true
     stdin_open: true

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "CodeSecTools"
-version = "0.10.0"
+version = "0.10.1"
 description = "A framework for code security that provides abstractions for static analysis tools and datasets to support their integration, testing, and evaluation."
 readme = "README.md"
 license = "AGPL-3.0-only"

requirements.txt

@@ -497,9 +497,9 @@ tqdm==4.67.1 \
     --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 \
     --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2
     # via codesectools
-typer==0.19.2 \
-    --hash=sha256:755e7e19670ffad8283db353267cb81ef252f595aa6834a0d1ca9312d9326cb9 \
-    --hash=sha256:9ad824308ded0ad06cc716434705f691d4ee0bfd0fb081839d2e426860e7fdca
+typer==0.20.0 \
+    --hash=sha256:1aaf6494031793e4876fb0bacfa6a912b551cf43c1e63c800df8b1a866720c37 \
+    --hash=sha256:5b463df6793ec1dca6213a3cf4c0f03bc6e322ac5e16e13ddd622a889489784a
     # via codesectools
 typing-extensions==4.15.0 \
     --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \

tests/Dockerfile

@@ -12,8 +12,8 @@ COPY codesectools /app/codesectools
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv sync --locked --no-dev --extra test
 
-# =========================== Test stage ===========================
-FROM python:3.12-slim-bookworm AS test
+# =========================== Base ===========================
+FROM python:3.12-slim-bookworm AS test-base
 
 RUN apt update -qq && \
     DEBIAN_FRONTEND=noninteractive \
@@ -24,6 +24,23 @@ RUN apt update -qq && \
     -y -qq --no-install-recommends && \
     rm -rf /var/lib/apt/lists/*
 
+# =========================== No SAST Tools ===========================
+FROM test-base AS no-sast
+ENV TEST_TYPE=no-sast
+
+# === Run tests ===
+COPY --from=builder --chown=app:app /app /app
+ENV PATH="/app/.venv/bin:$PATH"
+
+WORKDIR /app
+COPY tests /app/tests
+
+CMD ["pytest"]
+
+# =========================== With SAST Tools ===========================
+FROM test-base AS with-sast
+ENV TEST_TYPE=with-sast
+
 # === Free SAST tools only ===
 # Semgrep Community Edition
 RUN pip install --no-cache semgrep

tests/conftest.py

@@ -8,6 +8,9 @@
 
 import pytest
 
+test_type = os.environ.get("TEST_TYPE")
+state_file = Path(f".pytest_cache/state_{test_type}.json")
+
 
 def gen_state() -> dict[str, str]:
     """Generate a state dictionary of source file paths and their SHA256 hashes.
@@ -29,7 +32,6 @@ def source_code_changed() -> bool:
 
     Compares the current state with a saved state in '.pytest_cache/state.json'.
     """
-    state_file = Path(".pytest_cache/state.json")
     if not state_file.is_file():
         return True
 
@@ -67,7 +69,6 @@ def pytest_sessionfinish(session: pytest.Session) -> None:
     """
     if session.testscollected > 0 and session.testsfailed == 0:
         new_state = gen_state()
-        state_file = Path(".pytest_cache/state.json")
         state_file.parent.mkdir(exist_ok=True, parents=True)
         with state_file.open("w") as f:
             json.dump(new_state, f, indent=2)