RSMetaCheck action fails on detection #96

Author: Anas-Elhounsri

src/rsmetacheck/cli.py

@@ -1,8 +1,10 @@
 import argparse
+import json
 import os
+import sys
 from pathlib import Path
 
-from rsmetacheck.config import load_analysis_config
+from rsmetacheck.config import AnalysisConfig, load_analysis_config
 from rsmetacheck.run_analyzer import run_analysis
 from rsmetacheck.run_somef import (
     ensure_somef_configured,
@@ -11,6 +13,30 @@
 )
 
 
+def _exit_on_findings(analysis_output: str, analysis_config: AnalysisConfig) -> None:
+    try:
+        with open(analysis_output) as f:
+            data = json.load(f)
+    except Exception as e:
+        print(f"Warning: Could not read analysis output to evaluate exit code: {e}")
+        return
+
+    summary = data.get("summary", {})
+    pitfalls = summary.get("total_pitfalls_detected", 0)
+    warnings = summary.get("total_warnings_detected", 0)
+
+    should_fail = False
+    if analysis_config.fail_on_pitfalls and pitfalls > 0:
+        print(f"CI gate: {pitfalls} pitfall(s) detected (fail_on_pitfalls=true).")
+        should_fail = True
+    if analysis_config.fail_on_warnings and warnings > 0:
+        print(f"CI gate: {warnings} warning(s) detected (fail_on_warnings=true).")
+        should_fail = True
+
+    if should_fail:
+        sys.exit(1)
+
+
 def cli():
     parser = argparse.ArgumentParser(
         description="Detect metadata pitfalls in software repositories using SoMEF."
@@ -118,6 +144,8 @@ def cli():
             analysis_config=analysis_config,
         )
 
+        _exit_on_findings(args.analysis_output, analysis_config)
+
     else:
         ensure_somef_configured()
 
@@ -178,6 +206,8 @@ def cli():
             analysis_config=analysis_config,
         )
 
+        _exit_on_findings(args.analysis_output, analysis_config)
+
 
 if __name__ == "__main__":
     cli()

src/rsmetacheck/config.py

@@ -15,6 +15,8 @@ class AnalysisConfig:
     ignored_checks: Set[str] = field(default_factory=set)
     exclude_files: list[str] = field(default_factory=list)
     check_parameters: Dict[str, Dict[str, Any]] = field(default_factory=dict)
+    fail_on_pitfalls: bool = True
+    fail_on_warnings: bool = True
     profile: Optional[str] = None
     source_path: Optional[Path] = None
 
@@ -129,6 +131,9 @@ def load_analysis_config(
     base_exclude_files = _normalize_exclude_files(raw.get("exclude_files", []))
     base_parameters = _normalize_parameters(raw.get("parameters", {}))
 
+    fail_on_pitfalls = raw.get("fail_on_pitfalls", True)
+    fail_on_warnings = raw.get("fail_on_warnings", True)
+
     profile_ignore: Set[str] = set()
     profile_exclude_files: list[str] = []
     profile_parameters: Dict[str, Dict[str, Any]] = {}
@@ -146,12 +151,19 @@ def load_analysis_config(
         profile_exclude_files = _normalize_exclude_files(selected.get("exclude_files", []))
         profile_parameters = _normalize_parameters(selected.get("parameters", {}))
 
+        if "fail_on_pitfalls" in selected:
+            fail_on_pitfalls = selected["fail_on_pitfalls"]
+        if "fail_on_warnings" in selected:
+            fail_on_warnings = selected["fail_on_warnings"]
+
     merged_parameters = _merge_parameters(base_parameters, profile_parameters)
 
     return AnalysisConfig(
         ignored_checks=base_ignore | profile_ignore,
         exclude_files=base_exclude_files + profile_exclude_files,
         check_parameters=merged_parameters,
+        fail_on_pitfalls=fail_on_pitfalls,
+        fail_on_warnings=fail_on_warnings,
         profile=selected_profile,
         source_path=resolved_path,
     )

tests/test_cli.py

@@ -1,8 +1,11 @@
 """Unit tests verifying CLI argument handling and dispatch."""
 
 import importlib
+import json
 from unittest.mock import MagicMock
 
+import pytest
+
 from rsmetacheck.config import AnalysisConfig
 
 cli_module = importlib.import_module("rsmetacheck.cli")
@@ -539,3 +542,125 @@ def test_cli_config_load_error_stops_execution(monkeypatch, tmp_path, capsys):
     captured = capsys.readouterr()
     assert "Error loading config" in captured.out
     run_analysis_mock.assert_not_called()
+
+
+def test_exit_on_findings_pitfalls_cause_exit(tmp_path, capsys):
+    """Exit code 1 when pitfalls detected and fail_on_pitfalls=True."""
+    analysis_file = tmp_path / "analysis.json"
+    analysis_file.write_text(json.dumps({
+        "summary": {
+            "total_pitfalls_detected": 3,
+            "total_warnings_detected": 0,
+        }
+    }))
+    config = AnalysisConfig(fail_on_pitfalls=True, fail_on_warnings=False)
+
+    with pytest.raises(SystemExit) as exc_info:
+        cli_module._exit_on_findings(str(analysis_file), config)
+
+    assert exc_info.value.code == 1
+    captured = capsys.readouterr()
+    assert "CI gate: 3 pitfall(s) detected" in captured.out
+
+
+def test_exit_on_findings_warnings_cause_exit(tmp_path, capsys):
+    """Exit code 1 when warnings detected and fail_on_warnings=True."""
+    analysis_file = tmp_path / "analysis.json"
+    analysis_file.write_text(json.dumps({
+        "summary": {
+            "total_pitfalls_detected": 0,
+            "total_warnings_detected": 2,
+        }
+    }))
+    config = AnalysisConfig(fail_on_pitfalls=False, fail_on_warnings=True)
+
+    with pytest.raises(SystemExit) as exc_info:
+        cli_module._exit_on_findings(str(analysis_file), config)
+
+    assert exc_info.value.code == 1
+    captured = capsys.readouterr()
+    assert "CI gate: 2 warning(s) detected" in captured.out
+
+
+def test_exit_on_findings_both_cause_exit(tmp_path, capsys):
+    """Exit code 1 when both pitfalls and warnings detected with both flags True."""
+    analysis_file = tmp_path / "analysis.json"
+    analysis_file.write_text(json.dumps({
+        "summary": {
+            "total_pitfalls_detected": 1,
+            "total_warnings_detected": 1,
+        }
+    }))
+    config = AnalysisConfig(fail_on_pitfalls=True, fail_on_warnings=True)
+
+    with pytest.raises(SystemExit) as exc_info:
+        cli_module._exit_on_findings(str(analysis_file), config)
+
+    assert exc_info.value.code == 1
+    captured = capsys.readouterr()
+    assert "CI gate: 1 pitfall(s) detected" in captured.out
+    assert "CI gate: 1 warning(s) detected" in captured.out
+
+
+def test_exit_on_findings_no_exit_when_flags_false(tmp_path):
+    """No exit when fail flags are False even with findings."""
+    analysis_file = tmp_path / "analysis.json"
+    analysis_file.write_text(json.dumps({
+        "summary": {
+            "total_pitfalls_detected": 5,
+            "total_warnings_detected": 5,
+        }
+    }))
+    config = AnalysisConfig(fail_on_pitfalls=False, fail_on_warnings=False)
+
+    # Should not raise SystemExit
+    cli_module._exit_on_findings(str(analysis_file), config)
+
+
+def test_exit_on_findings_no_exit_with_zero_findings(tmp_path):
+    """No exit when zero pitfalls and warnings even with flags True."""
+    analysis_file = tmp_path / "analysis.json"
+    analysis_file.write_text(json.dumps({
+        "summary": {
+            "total_pitfalls_detected": 0,
+            "total_warnings_detected": 0,
+        }
+    }))
+    config = AnalysisConfig(fail_on_pitfalls=True, fail_on_warnings=True)
+
+    # Should not raise SystemExit
+    cli_module._exit_on_findings(str(analysis_file), config)
+
+
+def test_exit_on_findings_missing_file_prints_warning(tmp_path, capsys):
+    """Missing analysis file should print warning and not exit."""
+    config = AnalysisConfig(fail_on_pitfalls=True, fail_on_warnings=True)
+
+    cli_module._exit_on_findings(str(tmp_path / "nonexistent.json"), config)
+
+    captured = capsys.readouterr()
+    assert "Warning: Could not read analysis output" in captured.out
+
+
+def test_exit_on_findings_malformed_json_prints_warning(tmp_path, capsys):
+    """Malformed analysis file should print warning and not exit."""
+    analysis_file = tmp_path / "analysis.json"
+    analysis_file.write_text("not valid json")
+
+    config = AnalysisConfig(fail_on_pitfalls=True, fail_on_warnings=True)
+
+    cli_module._exit_on_findings(str(analysis_file), config)
+
+    captured = capsys.readouterr()
+    assert "Warning: Could not read analysis output" in captured.out
+
+
+def test_exit_on_findings_missing_summary_fields_treated_as_zero(tmp_path):
+    """Missing summary fields should be treated as 0 (no exit)."""
+    analysis_file = tmp_path / "analysis.json"
+    analysis_file.write_text(json.dumps({"summary": {}}))
+
+    config = AnalysisConfig(fail_on_pitfalls=True, fail_on_warnings=True)
+
+    # Should not raise SystemExit
+    cli_module._exit_on_findings(str(analysis_file), config)

tests/test_config.py

@@ -1,4 +1,4 @@
-from rsmetacheck.config import load_analysis_config
+from rsmetacheck.config import AnalysisConfig, load_analysis_config
 
 
 def test_load_analysis_config_auto_detects_default_file(tmp_path):
@@ -59,3 +59,57 @@ def test_load_analysis_config_missing_profile_raises(tmp_path):
         assert False, "Expected ValueError"
     except ValueError:
         pass
+
+
+def test_analysis_config_fail_flags_default_to_true():
+    """fail_on_pitfalls and fail_on_warnings should default to True."""
+    config = AnalysisConfig()
+    assert config.fail_on_pitfalls is True
+    assert config.fail_on_warnings is True
+
+
+def test_load_analysis_config_reads_fail_flags_from_toml(tmp_path):
+    """fail_on_pitfalls and fail_on_warnings should be read from TOML root."""
+    config_file = tmp_path / ".rsmetacheck.toml"
+    config_file.write_text(
+        """
+fail_on_pitfalls = false
+fail_on_warnings = false
+""".strip()
+    )
+
+    config = load_analysis_config(cwd=tmp_path)
+
+    assert config.fail_on_pitfalls is False
+    assert config.fail_on_warnings is False
+
+
+def test_load_analysis_config_fail_flags_default_when_absent(tmp_path):
+    """When TOML has no fail flags, they should default to True."""
+    config_file = tmp_path / ".rsmetacheck.toml"
+    config_file.write_text('ignore = ["P001"]\n')
+
+    config = load_analysis_config(cwd=tmp_path)
+
+    assert config.fail_on_pitfalls is True
+    assert config.fail_on_warnings is True
+
+
+def test_load_analysis_config_profile_overrides_fail_flags(tmp_path):
+    """Profile-level fail flags should override base values."""
+    config_file = tmp_path / ".rsmetacheck.toml"
+    config_file.write_text(
+        """
+fail_on_pitfalls = true
+fail_on_warnings = true
+
+[profiles.permissive]
+fail_on_pitfalls = false
+fail_on_warnings = false
+""".strip()
+    )
+
+    config = load_analysis_config(cwd=tmp_path, profile="permissive")
+
+    assert config.fail_on_pitfalls is False
+    assert config.fail_on_warnings is False