feat: return status depending on the checker's severity

.github/workflows/build-and-test.yaml

@@ -52,7 +52,7 @@ jobs:
           uv run t4viz scene ./tests/sample/t4dataset -o ./output
           uv run t4viz instance ./tests/sample/t4dataset 90f0c98d1a040d5360847f576c5528f8 -o ./output
           uv run t4viz pointcloud ./tests/sample/t4dataset -o ./output
-          uv run t4sanity ./tests/sample -iw
+          uv run t4sanity ./tests/sample/t4dataset
 
       - name: Get test coverage
         if: ${{ steps.is-changed.outputs.changes == 'true'}} && ${{ matrix.python-version == '3.10' }}

docs/cli/t4sanity.md

@@ -10,10 +10,11 @@ $ t4sanity -h
 ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
 ╭─ Options ─────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
 │ --version             -v             Show the application version and exit.                                           │
-│ --output              -o       TEXT  Path to output JSON file.                                                        │
-│ --revision            -rv      TEXT  Specify if you want to check the specific version.                               │
-│ --exclude             -e       TEXT  Exclude specific rules or rule groups.                                           │
+│ --output              -o       TEXT  Path to output JSON file. [default: None]                                        │
+│ --revision            -rv      TEXT  Specify if you want to check the specific version. [default: None]               │
+│ --exclude             -e       TEXT  Exclude specific rules or rule groups. [default: None]                           │
 │ --include-warning     -iw            Indicates whether to report any warnings.                                        │
+│ --strict              -s             Indicates whether warnings are treated as failures.                              │
 │ --install-completion                 Install completion for the current shell.                                        │
 │ --show-completion                    Show completion for the current shell, to copy it or customize the installation. │
 │ --help                -h             Show this message and exit.                                                      │
@@ -58,11 +59,11 @@ $ t4sanity <DATA_ROOT>
   STR008: ✅
   ...
 
-+-----------+---------+---------+-------+---------+----------+-------+
-| DatasetID | Version | Status  | Rules | Success | Failures | Skips |
-+-----------+---------+---------+-------+---------+----------+-------+
-| dataset1  |    0    | SUCCESS |  44   |   44    |    0     |   0   |
-+-----------+---------+---------+-------+---------+----------+-------+
++-----------+---------+---------+-------+---------+----------+-------+----------+
+| DatasetID | Version | Status  | Rules | Success | Failures | Skips | Warnings |
++-----------+---------+---------+-------+---------+----------+-------+----------+
+| dataset1  |         | SUCCESS |  49   |   43    |    0     |   2   |    4     |
++-----------+---------+---------+-------+---------+----------+-------+----------+
 ```
 
 ### Dump Results as JSON
@@ -83,6 +84,7 @@ Then a JSON file named `result.json` will be generated as follows:
     {
         "id": "<RuleID: str>",
         "name": "<RuleName: str>",
+        "severity": "<WARNING/ERROR: str>",
         "description": "<Description: str>",
         "status": "<SUCCESS/FAILURE/SKIPPED: str>",
         "reasons": "<[<Reason1>, <Reason2>, ...]: [str; N] | null>" // Failure or skipped reasons, null if success
@@ -99,3 +101,14 @@ With `-e; --excludes` option enables us to exclude specific checks by specifying
 # Exclude STR001 and all FMT-relevant rules
 t4sanity <DATA_ROOT> -e STR001 -e FMT
 ```
+
+### Strict Mode
+
+Basically, rules whose **severity is WARNING** will be treated as success.
+
+With `-s; --strict` option enables us to treat warnings as failures:
+
+```shell
+# Run strict mode
+t4sanity <DATA_ROOT> -s
+```

docs/schema/requirement.md

@@ -7,9 +7,9 @@
 | `STR001` | `version-dir-presence`        | `Warn`   | `version/` directory exists under the dataset root directory.        |
 | `STR002` | `annotation-dir-presence`     | `Error`  | `annotation/` directory exists under the dataset root directory.     |
 | `STR003` | `data-dir-presence`           | `Error`  | `data/` directory exists under the dataset root directory.           |
-| `STR004` | `map-dir-presence`            | `Error`  | `map/` directory exists under the dataset root directory.            |
-| `STR005` | `bag-dir-presence`            | `Error`  | `input_bag/` directory exists under the dataset root directory.      |
-| `STR006` | `status-file-presence`        | `Error`  | `status.json` file exists under the dataset root directory.          |
+| `STR004` | `map-dir-presence`            | `Warn`   | `map/` directory exists under the dataset root directory.            |
+| `STR005` | `bag-dir-presence`            | `Warn`   | `input_bag/` directory exists under the dataset root directory.      |
+| `STR006` | `status-file-presence`        | `Warn`   | `status.json` file exists under the dataset root directory.          |
 | `STR007` | `schema-files-presence`       | `Error`  | Mandatory schema JSON files exist under the `annotation/` directory. |
 | `STR008` | `lanelet-file-presence`       | `Warn`   | `lanelet2_map.osm` file exists under the `map/` directory.           |
 | `STR009` | `pointcloud-map-dir-presence` | `Warn`   | `pointcloud_map.pcd` directory exists under the `map/` directory.    |

t4_devkit/cli/sanity.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import sys
+
 import typer
 
 from t4_devkit.common.io import save_json
@@ -37,6 +39,9 @@ def main(
     include_warning: bool = typer.Option(
         False, "-iw", "--include-warning", help="Indicates whether to report any warnings."
     ),
+    strict: bool = typer.Option(
+        False, "-s", "--strict", help="Indicates whether warnings are treated as failures."
+    ),
 ) -> None:
     result = sanity_check(
         data_root=data_root,
@@ -45,8 +50,13 @@ def main(
         include_warning=include_warning,
     )
 
-    print_sanity_result(result)
+    print_sanity_result(result, strict=strict)
 
     if output:
         serialized = serialize_dataclass(result)
         save_json(serialized, output)
+
+    if result.is_success(strict=strict):
+        sys.exit(0)
+    else:
+        sys.exit(1)

t4_devkit/sanity/checker.py

@@ -1,11 +1,12 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
+from enum import Enum
 from typing import TYPE_CHECKING, NewType
 
 from returns.maybe import Maybe, Nothing, Some
 
-from .result import make_failure, make_success, make_skipped
+from .result import make_report, make_skipped
 
 if TYPE_CHECKING:
     from .context import SanityContext
@@ -16,30 +17,51 @@
 RuleName = NewType("RuleName", str)
 
 
+class Severity(str, Enum):
+    """Severity levels for sanity checkers."""
+
+    WARNING = "WARNING"
+    ERROR = "ERROR"
+
+    def is_warning(self) -> bool:
+        """Return `True` if the severity is WARNING."""
+        return self == Severity.WARNING
+
+    def is_error(self) -> bool:
+        """Return `True` if the severity is ERROR."""
+        return self == Severity.ERROR
+
+
 class Checker(ABC):
     """Base class for sanity checkers."""
 
     name: RuleName
     description: str
+    severity: Severity
 
     def __init__(self, id: RuleID) -> None:
         self.id = id
 
     def __call__(self, context: SanityContext) -> Report:
         match self.can_skip(context):
             case Some(skip):
-                return make_skipped(self.id, self.name, self.description, skip)
+                return make_skipped(self.id, self.name, self.severity, self.description, skip)
 
         reasons = self.check(context)
-        if reasons:
-            return make_failure(self.id, self.name, self.description, reasons)
-        else:
-            return make_success(self.id, self.name, self.description)
+        return make_report(self.id, self.name, self.severity, self.description, reasons)
 
     def can_skip(self, _: SanityContext) -> Maybe[Reason]:
         """Return a skip reason if the checker should be skipped."""
         return Nothing
 
     @abstractmethod
-    def check(self, context: SanityContext) -> list[Reason]:
+    def check(self, context: SanityContext) -> list[Reason] | None:
+        """Return a list of reasons if the checker fails, or None if it passes.
+
+        Args:
+            context (SanityContext): The sanity context.
+
+        Returns:
+            A list of reasons if the checker fails, or None if it passes.
+        """
         pass

t4_devkit/sanity/format/base.py

@@ -21,6 +21,7 @@ class FieldTypeChecker(Checker):
 
     Attributes:
         name (RuleName): The name of the rule.
+        severity (Severity): The severity of the rule.
         description (str): The description of the rule.
         schema (SchemaName): The schema name to check.
     """
@@ -36,24 +37,24 @@ def can_skip(self, context: SanityContext) -> Maybe[Reason]:
             case _:
                 return Nothing
 
-    def check(self, context: SanityContext) -> list[Reason]:
+    def check(self, context: SanityContext) -> list[Reason] | None:
         filepath = context.to_schema_file(self.schema).unwrap()
 
         if self.schema.is_optional() and not filepath.exists():
-            return []
+            return None
 
         records = load_json_safe(filepath)
         return _build_records(self.schema, records.unwrap())
 
 
-def _build_records(schema: SchemaName, records: list[dict]) -> list[Reason]:
+def _build_records(schema: SchemaName, records: list[dict]) -> list[Reason] | None:
     module = SCHEMAS.get(schema)
     failures = []
     for record in records:
         conversion = _safe_from_dict(module, record)
         if not is_successful(conversion):
             failures.append(Reason(f"[{schema.name}] {record['token']}: {conversion.failure()}"))
-    return failures
+    return failures if failures else None
 
 
 @safe

t4_devkit/sanity/format/fmt001.py

@@ -2,11 +2,10 @@
 
 from t4_devkit.schema import SchemaName
 
-from ..checker import RuleID, RuleName
+from ..checker import RuleID, RuleName, Severity
 from ..registry import CHECKERS
 from .base import FieldTypeChecker
 
-
 __all__ = ["FMT001"]
 
 
@@ -15,5 +14,6 @@ class FMT001(FieldTypeChecker):
     """A checker of FMT001."""
 
     name = RuleName("attribute-field")
+    severity = Severity.ERROR
     description = "All types of 'Attribute' fields are valid."
     schema = SchemaName.ATTRIBUTE

t4_devkit/sanity/format/fmt002.py

@@ -2,11 +2,10 @@
 
 from t4_devkit.schema import SchemaName
 
-from ..checker import RuleID, RuleName
+from ..checker import RuleID, RuleName, Severity
 from ..registry import CHECKERS
 from .base import FieldTypeChecker
 
-
 __all__ = ["FMT002"]
 
 
@@ -15,5 +14,6 @@ class FMT002(FieldTypeChecker):
     """A checker of FMT002."""
 
     name = RuleName("calibrated-sensor-field")
+    severity = Severity.ERROR
     description = "All types of 'CalibratedSensor' fields are valid."
     schema = SchemaName.CALIBRATED_SENSOR

t4_devkit/sanity/format/fmt003.py

@@ -2,11 +2,10 @@
 
 from t4_devkit.schema import SchemaName
 
-from ..checker import RuleID, RuleName
+from ..checker import RuleID, RuleName, Severity
 from ..registry import CHECKERS
 from .base import FieldTypeChecker
 
-
 __all__ = ["FMT003"]
 
 
@@ -15,5 +14,6 @@ class FMT003(FieldTypeChecker):
     """A checker of FMT003."""
 
     name = RuleName("category-field")
+    severity = Severity.ERROR
     description = "All types of 'Category' fields are valid."
     schema = SchemaName.CATEGORY

t4_devkit/sanity/format/fmt004.py

@@ -2,11 +2,10 @@
 
 from t4_devkit.schema import SchemaName
 
-from ..checker import RuleID, RuleName
+from ..checker import RuleID, RuleName, Severity
 from ..registry import CHECKERS
 from .base import FieldTypeChecker
 
-
 __all__ = ["FMT004"]
 
 
@@ -15,5 +14,6 @@ class FMT004(FieldTypeChecker):
     """A checker of FMT004."""
 
     name = RuleName("ego-pose-field")
+    severity = Severity.ERROR
     description = "All types of 'EgoPose' fields are valid."
     schema = SchemaName.EGO_POSE

t4_devkit/sanity/format/fmt005.py

@@ -2,11 +2,10 @@
 
 from t4_devkit.schema import SchemaName
 
-from ..checker import RuleID, RuleName
+from ..checker import RuleID, RuleName, Severity
 from ..registry import CHECKERS
 from .base import FieldTypeChecker
 
-
 __all__ = ["FMT005"]
 
 
@@ -15,5 +14,6 @@ class FMT005(FieldTypeChecker):
     """A checker of FMT005."""
 
     name = RuleName("instance-field")
+    severity = Severity.ERROR
     description = "All types of 'Instance' fields are valid."
     schema = SchemaName.INSTANCE

t4_devkit/sanity/format/fmt006.py

@@ -2,7 +2,7 @@
 
 from t4_devkit.schema import SchemaName
 
-from ..checker import RuleID, RuleName
+from ..checker import RuleID, RuleName, Severity
 from ..registry import CHECKERS
 from .base import FieldTypeChecker
 
@@ -14,5 +14,6 @@ class FMT006(FieldTypeChecker):
     """A checker of FMT006."""
 
     name = RuleName("log-field")
+    severity = Severity.ERROR
     description = "All types of 'Log' fields are valid."
     schema = SchemaName.LOG

t4_devkit/sanity/format/fmt007.py

@@ -2,7 +2,7 @@
 
 from t4_devkit.schema import SchemaName
 
-from ..checker import RuleID, RuleName
+from ..checker import RuleID, RuleName, Severity
 from ..registry import CHECKERS
 from .base import FieldTypeChecker
 
@@ -14,5 +14,6 @@ class FMT007(FieldTypeChecker):
     """A checker of FMT007."""
 
     name = RuleName("map-field")
+    severity = Severity.ERROR
     description = "All types of 'Map' fields are valid."
     schema = SchemaName.MAP

t4_devkit/sanity/format/fmt008.py

@@ -2,11 +2,10 @@
 
 from t4_devkit.schema import SchemaName
 
-from ..checker import RuleID, RuleName
+from ..checker import RuleID, RuleName, Severity
 from ..registry import CHECKERS
 from .base import FieldTypeChecker
 
-
 __all__ = ["FMT008"]
 
 
@@ -15,5 +14,6 @@ class FMT008(FieldTypeChecker):
     """A checker of FMT008."""
 
     name = RuleName("sample-field")
+    severity = Severity.ERROR
     description = "All types of 'Sample' fields are valid."
     schema = SchemaName.SAMPLE

t4_devkit/sanity/format/fmt009.py

@@ -2,11 +2,10 @@
 
 from t4_devkit.schema import SchemaName
 
-from ..checker import RuleID, RuleName
+from ..checker import RuleID, RuleName, Severity
 from ..registry import CHECKERS
 from .base import FieldTypeChecker
 
-
 __all__ = ["FMT009"]
 
 
@@ -15,5 +14,6 @@ class FMT009(FieldTypeChecker):
     """A checker of FMT009."""
 
     name = RuleName("sample-annotation-field")
+    severity = Severity.ERROR
     description = "All types of 'SampleAnnotation' fields are valid."
     schema = SchemaName.SAMPLE_ANNOTATION