tests: add tests for cli output using subprocess.run

Author: Saharsh1123

.github/workflows/tests.yaml

@@ -11,17 +11,6 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Install test dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install pytest ruff
-
-      - name: Run Ruff
-        run: ruff check .
-
-      - name: Run tests
-        run: pytest
-
       - name: Check out repository
         uses: actions/checkout@v4
 
@@ -33,7 +22,10 @@ jobs:
       - name: Install test dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install pytest
+          pip install pytest ruff
+
+      - name: Run Ruff
+        run: ruff check .
 
       - name: Run tests
         run: pytest

tests/test_cli.py

@@ -0,0 +1,250 @@
+import json
+import subprocess
+import sys
+
+
+def run_cli(*args):
+    """
+    Run the SentinelScan CLI with the provided arguments.
+
+    Uses the current Python interpreter so tests work locally and in CI.
+    """
+    return subprocess.run(
+        [sys.executable, "main.py", *args],
+        capture_output=True,
+        text=True,
+    )
+
+
+def parse_json_output(result):
+    """
+    Parse CLI stdout as JSON.
+    """
+    return json.loads(result.stdout)
+
+
+# Ensure JSON mode runs successfully and returns valid JSON
+def test_cli_json_output_is_valid_json():
+    result = run_cli("test_dirs", "--json")
+
+    assert result.returncode == 0
+
+    data = parse_json_output(result)
+
+    assert isinstance(data, list)
+    assert len(data) > 0
+
+
+# Ensure JSON findings use the expected schema
+def test_cli_json_schema():
+    result = run_cli("test_dirs", "--json")
+
+    assert result.returncode == 0
+
+    data = parse_json_output(result)
+
+    assert isinstance(data, list)
+    assert len(data) > 0
+
+    required_keys = {"line", "file", "rule", "severity", "value"}
+
+    for finding in data:
+        assert required_keys.issubset(finding.keys())
+
+
+# Ensure JSON finding fields have the expected data types
+def test_cli_json_field_types():
+    result = run_cli("test_dirs", "--json")
+
+    assert result.returncode == 0
+
+    data = parse_json_output(result)
+
+    assert len(data) > 0
+
+    for finding in data:
+        assert isinstance(finding["line"], int)
+        assert isinstance(finding["file"], str)
+        assert isinstance(finding["rule"], str)
+        assert isinstance(finding["severity"], str)
+        assert isinstance(finding["value"], str)
+
+
+# Ensure JSON mode does not include human-readable CLI text
+def test_cli_json_output_is_pure_json():
+    result = run_cli("test_dirs", "--json")
+
+    assert result.returncode == 0
+
+    assert "Scanning" not in result.stdout
+    assert "--- Findings ---" not in result.stdout
+    assert "Total findings" not in result.stdout
+
+    data = parse_json_output(result)
+    assert isinstance(data, list)
+
+
+# Ensure normal CLI mode prints human-readable scan output
+def test_cli_text_output_contains_expected_sections():
+    result = run_cli("test_dirs")
+
+    assert result.returncode == 0
+
+    assert "Scanning" in result.stdout
+    assert "--- Findings ---" in result.stdout
+    assert "Total findings" in result.stdout
+
+
+# Ensure normal CLI output contains finding details
+def test_cli_text_output_contains_finding_details():
+    result = run_cli("test_dirs")
+
+    assert result.returncode == 0
+
+    assert "[HIGH]" in result.stdout
+    assert "→" in result.stdout
+
+
+# Ensure HIGH severity filtering works in JSON mode
+def test_cli_json_severity_high_filter():
+    result = run_cli("test_dirs", "--json", "--severity", "HIGH")
+
+    assert result.returncode == 0
+
+    data = parse_json_output(result)
+
+    assert len(data) > 0
+    assert all(finding["severity"] == "HIGH" for finding in data)
+
+
+# Ensure MEDIUM severity filtering works in JSON mode
+def test_cli_json_severity_medium_filter():
+    result = run_cli("test_dirs", "--json", "--severity", "MEDIUM")
+
+    assert result.returncode == 0
+
+    data = parse_json_output(result)
+
+    assert len(data) > 0
+    assert all(finding["severity"] == "MEDIUM" for finding in data)
+
+
+# Ensure HIGH severity filtering works in text mode
+def test_cli_text_severity_high_filter():
+    result = run_cli("test_dirs", "--severity", "HIGH")
+
+    assert result.returncode == 0
+
+    assert "[HIGH]" in result.stdout
+    assert "[MEDIUM]" not in result.stdout
+
+
+# Ensure MEDIUM severity filtering works in text mode
+def test_cli_text_severity_medium_filter():
+    result = run_cli("test_dirs", "--severity", "MEDIUM")
+
+    assert result.returncode == 0
+
+    assert "[MEDIUM]" in result.stdout
+    assert "[HIGH]" not in result.stdout
+
+
+# Ensure JSON output and severity filtering work together
+def test_cli_json_and_severity_combined():
+    result = run_cli("test_dirs", "--json", "--severity", "HIGH")
+
+    assert result.returncode == 0
+
+    data = parse_json_output(result)
+
+    assert len(data) > 0
+    assert all(finding["severity"] == "HIGH" for finding in data)
+
+
+# Ensure invalid severity values are rejected by argparse
+def test_cli_invalid_severity_fails():
+    result = run_cli("test_dirs", "--severity", "CRITICAL")
+
+    assert result.returncode != 0
+    assert "invalid choice" in result.stderr
+
+
+# Ensure invalid paths are handled gracefully
+def test_cli_invalid_path_prints_error():
+    result = run_cli("does_not_exist")
+
+    assert "[ERROR]" in result.stdout or "[ERROR]" in result.stderr
+
+
+# Ensure benign Python files produce no findings in text mode
+def test_cli_no_findings_text_output(tmp_path):
+    benign_file = tmp_path / "safe.py"
+    benign_file.write_text('username = "notsecret"\n', encoding="utf-8")
+
+    result = run_cli(str(tmp_path))
+
+    assert result.returncode == 0
+
+    assert "No vulnerabilities found." in result.stdout
+
+
+# Ensure benign Python files produce an empty JSON list
+def test_cli_no_findings_json_output(tmp_path):
+    benign_file = tmp_path / "safe.py"
+    benign_file.write_text('username = "notsecret"\n', encoding="utf-8")
+
+    result = run_cli(str(tmp_path), "--json")
+
+    assert result.returncode == 0
+
+    data = parse_json_output(result)
+
+    assert data == []
+
+
+# Ensure CLI can scan temporary directories, not just checked-in fixtures
+def test_cli_detects_secret_in_temp_directory(tmp_path):
+    vulnerable_file = tmp_path / "vulnerable.py"
+    vulnerable_file.write_text('password = "abcdef"\n', encoding="utf-8")
+
+    result = run_cli(str(tmp_path), "--json")
+
+    assert result.returncode == 0
+
+    data = parse_json_output(result)
+
+    assert data == [
+        {
+            "line": 1,
+            "file": str(vulnerable_file),
+            "rule": "Password",
+            "severity": "HIGH",
+            "value": "abcdef",
+        }
+    ]
+
+
+# Ensure JSON output can be parsed even after severity filtering removes all findings
+def test_cli_json_filter_with_no_matching_findings(tmp_path):
+    medium_file = tmp_path / "medium.py"
+    medium_file.write_text('token = "abcdef"\n', encoding="utf-8")
+
+    result = run_cli(str(tmp_path), "--json", "--severity", "HIGH")
+
+    assert result.returncode == 0
+
+    data = parse_json_output(result)
+
+    assert data == []
+
+
+# Ensure text output handles severity filters with no matching findings
+def test_cli_text_filter_with_no_matching_findings(tmp_path):
+    medium_file = tmp_path / "medium.py"
+    medium_file.write_text('token = "abcdef"\n', encoding="utf-8")
+
+    result = run_cli(str(tmp_path), "--severity", "HIGH")
+
+    assert result.returncode == 0
+
+    assert "No vulnerabilities found." in result.stdout