diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index f640895..d33e49b 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -52,7 +52,7 @@ jobs: uv run t4viz scene ./tests/sample/t4dataset -o ./output uv run t4viz instance ./tests/sample/t4dataset 90f0c98d1a040d5360847f576c5528f8 -o ./output uv run t4viz pointcloud ./tests/sample/t4dataset -o ./output - uv run t4sanity ./tests/sample -iw + uv run t4sanity ./tests/sample/t4dataset - name: Get test coverage if: ${{ steps.is-changed.outputs.changes == 'true'}} && ${{ matrix.python-version == '3.10' }} diff --git a/docs/cli/t4sanity.md b/docs/cli/t4sanity.md index 3a6a583..79c2613 100644 --- a/docs/cli/t4sanity.md +++ b/docs/cli/t4sanity.md @@ -10,10 +10,11 @@ $ t4sanity -h ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Options ─────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ │ --version -v Show the application version and exit. │ -│ --output -o TEXT Path to output JSON file. │ -│ --revision -rv TEXT Specify if you want to check the specific version. │ -│ --exclude -e TEXT Exclude specific rules or rule groups. │ +│ --output -o TEXT Path to output JSON file. [default: None] │ +│ --revision -rv TEXT Specify if you want to check the specific version. [default: None] │ +│ --exclude -e TEXT Exclude specific rules or rule groups. [default: None] │ │ --include-warning -iw Indicates whether to report any warnings. │ +│ --strict -s Indicates whether warnings are treated as failures. │ │ --install-completion Install completion for the current shell. │ │ --show-completion Show completion for the current shell, to copy it or customize the installation. │ │ --help -h Show this message and exit. │ @@ -58,11 +59,11 @@ $ t4sanity STR008: ✅ ... -+-----------+---------+---------+-------+---------+----------+-------+ -| DatasetID | Version | Status | Rules | Success | Failures | Skips | -+-----------+---------+---------+-------+---------+----------+-------+ -| dataset1 | 0 | SUCCESS | 44 | 44 | 0 | 0 | -+-----------+---------+---------+-------+---------+----------+-------+ ++-----------+---------+---------+-------+---------+----------+-------+----------+ +| DatasetID | Version | Status | Rules | Success | Failures | Skips | Warnings | ++-----------+---------+---------+-------+---------+----------+-------+----------+ +| dataset1 | | SUCCESS | 49 | 43 | 0 | 2 | 4 | ++-----------+---------+---------+-------+---------+----------+-------+----------+ ``` ### Dump Results as JSON @@ -83,6 +84,7 @@ Then a JSON file named `result.json` will be generated as follows: { "id": "", "name": "", + "severity": "", "description": "", "status": "", "reasons": "<[, , ...]: [str; N] | null>" // Failure or skipped reasons, null if success @@ -99,3 +101,14 @@ With `-e; --excludes` option enables us to exclude specific checks by specifying # Exclude STR001 and all FMT-relevant rules t4sanity -e STR001 -e FMT ``` + +### Strict Mode + +Basically, rules whose **severity is WARNING** will be treated as success. + +With `-s; --strict` option enables us to treat warnings as failures: + +```shell +# Run strict mode +t4sanity -s +``` diff --git a/docs/schema/requirement.md b/docs/schema/requirement.md index 9988179..9e8d773 100644 --- a/docs/schema/requirement.md +++ b/docs/schema/requirement.md @@ -7,9 +7,9 @@ | `STR001` | `version-dir-presence` | `Warn` | `version/` directory exists under the dataset root directory. | | `STR002` | `annotation-dir-presence` | `Error` | `annotation/` directory exists under the dataset root directory. | | `STR003` | `data-dir-presence` | `Error` | `data/` directory exists under the dataset root directory. | -| `STR004` | `map-dir-presence` | `Error` | `map/` directory exists under the dataset root directory. | -| `STR005` | `bag-dir-presence` | `Error` | `input_bag/` directory exists under the dataset root directory. | -| `STR006` | `status-file-presence` | `Error` | `status.json` file exists under the dataset root directory. | +| `STR004` | `map-dir-presence` | `Warn` | `map/` directory exists under the dataset root directory. | +| `STR005` | `bag-dir-presence` | `Warn` | `input_bag/` directory exists under the dataset root directory. | +| `STR006` | `status-file-presence` | `Warn` | `status.json` file exists under the dataset root directory. | | `STR007` | `schema-files-presence` | `Error` | Mandatory schema JSON files exist under the `annotation/` directory. | | `STR008` | `lanelet-file-presence` | `Warn` | `lanelet2_map.osm` file exists under the `map/` directory. | | `STR009` | `pointcloud-map-dir-presence` | `Warn` | `pointcloud_map.pcd` directory exists under the `map/` directory. | diff --git a/t4_devkit/cli/sanity.py b/t4_devkit/cli/sanity.py index 63847c5..8682d93 100644 --- a/t4_devkit/cli/sanity.py +++ b/t4_devkit/cli/sanity.py @@ -1,5 +1,7 @@ from __future__ import annotations +import sys + import typer from t4_devkit.common.io import save_json @@ -37,6 +39,9 @@ def main( include_warning: bool = typer.Option( False, "-iw", "--include-warning", help="Indicates whether to report any warnings." ), + strict: bool = typer.Option( + False, "-s", "--strict", help="Indicates whether warnings are treated as failures." + ), ) -> None: result = sanity_check( data_root=data_root, @@ -45,8 +50,13 @@ def main( include_warning=include_warning, ) - print_sanity_result(result) + print_sanity_result(result, strict=strict) if output: serialized = serialize_dataclass(result) save_json(serialized, output) + + if result.is_success(strict=strict): + sys.exit(0) + else: + sys.exit(1) diff --git a/t4_devkit/sanity/checker.py b/t4_devkit/sanity/checker.py index f434b0c..a43fe2a 100644 --- a/t4_devkit/sanity/checker.py +++ b/t4_devkit/sanity/checker.py @@ -1,11 +1,12 @@ from __future__ import annotations from abc import ABC, abstractmethod +from enum import Enum from typing import TYPE_CHECKING, NewType from returns.maybe import Maybe, Nothing, Some -from .result import make_failure, make_success, make_skipped +from .result import make_report, make_skipped if TYPE_CHECKING: from .context import SanityContext @@ -16,11 +17,27 @@ RuleName = NewType("RuleName", str) +class Severity(str, Enum): + """Severity levels for sanity checkers.""" + + WARNING = "WARNING" + ERROR = "ERROR" + + def is_warning(self) -> bool: + """Return `True` if the severity is WARNING.""" + return self == Severity.WARNING + + def is_error(self) -> bool: + """Return `True` if the severity is ERROR.""" + return self == Severity.ERROR + + class Checker(ABC): """Base class for sanity checkers.""" name: RuleName description: str + severity: Severity def __init__(self, id: RuleID) -> None: self.id = id @@ -28,18 +45,23 @@ def __init__(self, id: RuleID) -> None: def __call__(self, context: SanityContext) -> Report: match self.can_skip(context): case Some(skip): - return make_skipped(self.id, self.name, self.description, skip) + return make_skipped(self.id, self.name, self.severity, self.description, skip) reasons = self.check(context) - if reasons: - return make_failure(self.id, self.name, self.description, reasons) - else: - return make_success(self.id, self.name, self.description) + return make_report(self.id, self.name, self.severity, self.description, reasons) def can_skip(self, _: SanityContext) -> Maybe[Reason]: """Return a skip reason if the checker should be skipped.""" return Nothing @abstractmethod - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: + """Return a list of reasons if the checker fails, or None if it passes. + + Args: + context (SanityContext): The sanity context. + + Returns: + A list of reasons if the checker fails, or None if it passes. + """ pass diff --git a/t4_devkit/sanity/format/base.py b/t4_devkit/sanity/format/base.py index 3670eb1..39f2b57 100644 --- a/t4_devkit/sanity/format/base.py +++ b/t4_devkit/sanity/format/base.py @@ -21,6 +21,7 @@ class FieldTypeChecker(Checker): Attributes: name (RuleName): The name of the rule. + severity (Severity): The severity of the rule. description (str): The description of the rule. schema (SchemaName): The schema name to check. """ @@ -36,24 +37,24 @@ def can_skip(self, context: SanityContext) -> Maybe[Reason]: case _: return Nothing - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: filepath = context.to_schema_file(self.schema).unwrap() if self.schema.is_optional() and not filepath.exists(): - return [] + return None records = load_json_safe(filepath) return _build_records(self.schema, records.unwrap()) -def _build_records(schema: SchemaName, records: list[dict]) -> list[Reason]: +def _build_records(schema: SchemaName, records: list[dict]) -> list[Reason] | None: module = SCHEMAS.get(schema) failures = [] for record in records: conversion = _safe_from_dict(module, record) if not is_successful(conversion): failures.append(Reason(f"[{schema.name}] {record['token']}: {conversion.failure()}")) - return failures + return failures if failures else None @safe diff --git a/t4_devkit/sanity/format/fmt001.py b/t4_devkit/sanity/format/fmt001.py index b156ee0..b70ad07 100644 --- a/t4_devkit/sanity/format/fmt001.py +++ b/t4_devkit/sanity/format/fmt001.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT001"] @@ -15,5 +14,6 @@ class FMT001(FieldTypeChecker): """A checker of FMT001.""" name = RuleName("attribute-field") + severity = Severity.ERROR description = "All types of 'Attribute' fields are valid." schema = SchemaName.ATTRIBUTE diff --git a/t4_devkit/sanity/format/fmt002.py b/t4_devkit/sanity/format/fmt002.py index 240498b..ccd6d85 100644 --- a/t4_devkit/sanity/format/fmt002.py +++ b/t4_devkit/sanity/format/fmt002.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT002"] @@ -15,5 +14,6 @@ class FMT002(FieldTypeChecker): """A checker of FMT002.""" name = RuleName("calibrated-sensor-field") + severity = Severity.ERROR description = "All types of 'CalibratedSensor' fields are valid." schema = SchemaName.CALIBRATED_SENSOR diff --git a/t4_devkit/sanity/format/fmt003.py b/t4_devkit/sanity/format/fmt003.py index 8baf068..33352e8 100644 --- a/t4_devkit/sanity/format/fmt003.py +++ b/t4_devkit/sanity/format/fmt003.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT003"] @@ -15,5 +14,6 @@ class FMT003(FieldTypeChecker): """A checker of FMT003.""" name = RuleName("category-field") + severity = Severity.ERROR description = "All types of 'Category' fields are valid." schema = SchemaName.CATEGORY diff --git a/t4_devkit/sanity/format/fmt004.py b/t4_devkit/sanity/format/fmt004.py index 685f650..ccfbc8b 100644 --- a/t4_devkit/sanity/format/fmt004.py +++ b/t4_devkit/sanity/format/fmt004.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT004"] @@ -15,5 +14,6 @@ class FMT004(FieldTypeChecker): """A checker of FMT004.""" name = RuleName("ego-pose-field") + severity = Severity.ERROR description = "All types of 'EgoPose' fields are valid." schema = SchemaName.EGO_POSE diff --git a/t4_devkit/sanity/format/fmt005.py b/t4_devkit/sanity/format/fmt005.py index 0610280..b8d5d57 100644 --- a/t4_devkit/sanity/format/fmt005.py +++ b/t4_devkit/sanity/format/fmt005.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT005"] @@ -15,5 +14,6 @@ class FMT005(FieldTypeChecker): """A checker of FMT005.""" name = RuleName("instance-field") + severity = Severity.ERROR description = "All types of 'Instance' fields are valid." schema = SchemaName.INSTANCE diff --git a/t4_devkit/sanity/format/fmt006.py b/t4_devkit/sanity/format/fmt006.py index b35ed1e..23a4132 100644 --- a/t4_devkit/sanity/format/fmt006.py +++ b/t4_devkit/sanity/format/fmt006.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker @@ -14,5 +14,6 @@ class FMT006(FieldTypeChecker): """A checker of FMT006.""" name = RuleName("log-field") + severity = Severity.ERROR description = "All types of 'Log' fields are valid." schema = SchemaName.LOG diff --git a/t4_devkit/sanity/format/fmt007.py b/t4_devkit/sanity/format/fmt007.py index a62ef8d..f2e9eab 100644 --- a/t4_devkit/sanity/format/fmt007.py +++ b/t4_devkit/sanity/format/fmt007.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker @@ -14,5 +14,6 @@ class FMT007(FieldTypeChecker): """A checker of FMT007.""" name = RuleName("map-field") + severity = Severity.ERROR description = "All types of 'Map' fields are valid." schema = SchemaName.MAP diff --git a/t4_devkit/sanity/format/fmt008.py b/t4_devkit/sanity/format/fmt008.py index 7bc978e..9d7a743 100644 --- a/t4_devkit/sanity/format/fmt008.py +++ b/t4_devkit/sanity/format/fmt008.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT008"] @@ -15,5 +14,6 @@ class FMT008(FieldTypeChecker): """A checker of FMT008.""" name = RuleName("sample-field") + severity = Severity.ERROR description = "All types of 'Sample' fields are valid." schema = SchemaName.SAMPLE diff --git a/t4_devkit/sanity/format/fmt009.py b/t4_devkit/sanity/format/fmt009.py index 9211e44..12e38fb 100644 --- a/t4_devkit/sanity/format/fmt009.py +++ b/t4_devkit/sanity/format/fmt009.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT009"] @@ -15,5 +14,6 @@ class FMT009(FieldTypeChecker): """A checker of FMT009.""" name = RuleName("sample-annotation-field") + severity = Severity.ERROR description = "All types of 'SampleAnnotation' fields are valid." schema = SchemaName.SAMPLE_ANNOTATION diff --git a/t4_devkit/sanity/format/fmt010.py b/t4_devkit/sanity/format/fmt010.py index 0235bc4..447193c 100644 --- a/t4_devkit/sanity/format/fmt010.py +++ b/t4_devkit/sanity/format/fmt010.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT010"] @@ -15,5 +14,6 @@ class FMT010(FieldTypeChecker): """A checker of FMT010.""" name = RuleName("sample-data-field") + severity = Severity.ERROR description = "All types of 'SampleData' fields are valid." schema = SchemaName.SAMPLE_DATA diff --git a/t4_devkit/sanity/format/fmt011.py b/t4_devkit/sanity/format/fmt011.py index 612b3de..368efba 100644 --- a/t4_devkit/sanity/format/fmt011.py +++ b/t4_devkit/sanity/format/fmt011.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT011"] @@ -15,5 +14,6 @@ class FMT011(FieldTypeChecker): """A checker of FMT011.""" name = RuleName("scene-field") + severity = Severity.ERROR description = "All types of 'Scene' fields are valid." schema = SchemaName.SCENE diff --git a/t4_devkit/sanity/format/fmt012.py b/t4_devkit/sanity/format/fmt012.py index d98169f..a70bcbb 100644 --- a/t4_devkit/sanity/format/fmt012.py +++ b/t4_devkit/sanity/format/fmt012.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT012"] @@ -15,5 +14,6 @@ class FMT012(FieldTypeChecker): """A checker of FMT012.""" name = RuleName("sensor-field") + severity = Severity.ERROR description = "All types of 'Sensor' fields are valid." schema = SchemaName.SENSOR diff --git a/t4_devkit/sanity/format/fmt013.py b/t4_devkit/sanity/format/fmt013.py index 3318f72..028e6b9 100644 --- a/t4_devkit/sanity/format/fmt013.py +++ b/t4_devkit/sanity/format/fmt013.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT013"] @@ -15,5 +14,6 @@ class FMT013(FieldTypeChecker): """A checker of FMT013.""" name = RuleName("visibility-field") + severity = Severity.ERROR description = "All types of 'Visibility' fields are valid." schema = SchemaName.VISIBILITY diff --git a/t4_devkit/sanity/format/fmt014.py b/t4_devkit/sanity/format/fmt014.py index 85fa09c..7b4ed4b 100644 --- a/t4_devkit/sanity/format/fmt014.py +++ b/t4_devkit/sanity/format/fmt014.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT014"] @@ -15,5 +14,6 @@ class FMT014(FieldTypeChecker): """A checker of FMT014.""" name = RuleName("lidarseg-field") + severity = Severity.ERROR description = "All types of 'LidarSeg' fields are valid." schema = SchemaName.LIDARSEG diff --git a/t4_devkit/sanity/format/fmt015.py b/t4_devkit/sanity/format/fmt015.py index 02b61fb..e56dc68 100644 --- a/t4_devkit/sanity/format/fmt015.py +++ b/t4_devkit/sanity/format/fmt015.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT015"] @@ -15,5 +14,6 @@ class FMT015(FieldTypeChecker): """A checker of FMT015.""" name = RuleName("object-ann-field") + severity = Severity.ERROR description = "All types of 'ObjectAnn' fields are valid." schema = SchemaName.OBJECT_ANN diff --git a/t4_devkit/sanity/format/fmt016.py b/t4_devkit/sanity/format/fmt016.py index 59bc770..84df925 100644 --- a/t4_devkit/sanity/format/fmt016.py +++ b/t4_devkit/sanity/format/fmt016.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT016"] @@ -15,5 +14,6 @@ class FMT016(FieldTypeChecker): """A checker of FMT016.""" name = RuleName("surface-ann-field") + severity = Severity.ERROR description = "All types of 'SurfaceAnn' fields are valid." schema = SchemaName.SURFACE_ANN diff --git a/t4_devkit/sanity/format/fmt017.py b/t4_devkit/sanity/format/fmt017.py index ea5727b..ca52efe 100644 --- a/t4_devkit/sanity/format/fmt017.py +++ b/t4_devkit/sanity/format/fmt017.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT017"] @@ -15,5 +14,6 @@ class FMT017(FieldTypeChecker): """A checker of FMT017.""" name = RuleName("keypoint-field") + severity = Severity.ERROR description = "All types of 'Keypoint' fields are valid." schema = SchemaName.KEYPOINT diff --git a/t4_devkit/sanity/format/fmt018.py b/t4_devkit/sanity/format/fmt018.py index 6e4df61..73b57ca 100644 --- a/t4_devkit/sanity/format/fmt018.py +++ b/t4_devkit/sanity/format/fmt018.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT018"] @@ -15,5 +14,6 @@ class FMT018(FieldTypeChecker): """A checker of FMT018.""" name = RuleName("vehicle-state-field") + severity = Severity.ERROR description = "All types of 'VehicleState' fields are valid." schema = SchemaName.VEHICLE_STATE diff --git a/t4_devkit/sanity/record/base.py b/t4_devkit/sanity/record/base.py index e172355..162f7f5 100644 --- a/t4_devkit/sanity/record/base.py +++ b/t4_devkit/sanity/record/base.py @@ -20,6 +20,7 @@ class RecordCountChecker(Checker): Attributes: name (RuleName): The name of the rule. + severity (Severity): The severity of the rule. description (str): The description of the rule. schema (SchemaName): The schema name to check. """ @@ -36,19 +37,19 @@ def can_skip(self, context: SanityContext) -> Maybe[Reason]: case _: return Maybe.from_value(Reason("Missing 'annotation' directory path")) - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: filepath = context.to_schema_file(self.schema).unwrap() records = load_json_safe(filepath).unwrap() return self.check_count(records) @abstractmethod - def check_count(self, records: list[dict]) -> list[Reason]: + def check_count(self, records: list[dict]) -> list[Reason] | None: """Check the count of records. Args: records (list[dict]): The list of records to check. Returns: - A list of reasons for any issues found, otherwise an empty list. + A list of reasons for any issues found, otherwise None. """ pass diff --git a/t4_devkit/sanity/record/rec001.py b/t4_devkit/sanity/record/rec001.py index 2923595..01c7927 100644 --- a/t4_devkit/sanity/record/rec001.py +++ b/t4_devkit/sanity/record/rec001.py @@ -2,12 +2,11 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason from .base import RecordCountChecker - __all__ = ["REC001"] @@ -16,13 +15,14 @@ class REC001(RecordCountChecker): """A checker of REC001.""" name = RuleName("scene-single") + severity = Severity.ERROR description = "'Scene' record is a single." schema = SchemaName.SCENE - def check_count(self, records: list[dict]) -> list[Reason]: + def check_count(self, records: list[dict]) -> list[Reason] | None: num_scene = len(records) return ( - [] + None if num_scene == 1 else [Reason(f"'Scene' must contain exactly one element: {num_scene}")] ) diff --git a/t4_devkit/sanity/record/rec002.py b/t4_devkit/sanity/record/rec002.py index 0d802c9..86d2e9f 100644 --- a/t4_devkit/sanity/record/rec002.py +++ b/t4_devkit/sanity/record/rec002.py @@ -2,12 +2,11 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason from .base import RecordCountChecker - __all__ = ["REC002"] @@ -16,9 +15,10 @@ class REC002(RecordCountChecker): """A checker of REC002.""" name = RuleName("sample-not-empty") + severity = Severity.ERROR description = "'Sample' record is not empty." schema = SchemaName.SAMPLE - def check_count(self, records: list[dict]) -> list[Reason]: + def check_count(self, records: list[dict]) -> list[Reason] | None: num_sample = len(records) - return [Reason("'Sample' record must not be empty")] if num_sample == 0 else [] + return [Reason("'Sample' record must not be empty")] if num_sample == 0 else None diff --git a/t4_devkit/sanity/record/rec003.py b/t4_devkit/sanity/record/rec003.py index e11301b..d4f2d98 100644 --- a/t4_devkit/sanity/record/rec003.py +++ b/t4_devkit/sanity/record/rec003.py @@ -2,12 +2,11 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason from .base import RecordCountChecker - __all__ = ["REC003"] @@ -16,9 +15,10 @@ class REC003(RecordCountChecker): """A checker of REC003.""" name = RuleName("sample-data-not-empty") + severity = Severity.ERROR description = "'SampleData' record is not empty." schema = SchemaName.SAMPLE_DATA - def check_count(self, records: list[dict]) -> list[Reason]: + def check_count(self, records: list[dict]) -> list[Reason] | None: num_sample_data = len(records) - return [Reason("'SampleData' record must not be empty")] if num_sample_data == 0 else [] + return [Reason("'SampleData' record must not be empty")] if num_sample_data == 0 else None diff --git a/t4_devkit/sanity/record/rec004.py b/t4_devkit/sanity/record/rec004.py index 048fdfa..a9a12ea 100644 --- a/t4_devkit/sanity/record/rec004.py +++ b/t4_devkit/sanity/record/rec004.py @@ -2,12 +2,11 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason from .base import RecordCountChecker - __all__ = ["REC004"] @@ -16,9 +15,10 @@ class REC004(RecordCountChecker): """A checker of REC004.""" name = RuleName("ego-pose-not-empty") + severity = Severity.ERROR description = "'EgoPose' record is not empty." schema = SchemaName.EGO_POSE - def check_count(self, records: list[dict]) -> list[Reason]: + def check_count(self, records: list[dict]) -> list[Reason] | None: num_ego_pose = len(records) - return [Reason("'EgoPose' record must not be empty")] if num_ego_pose == 0 else [] + return [Reason("'EgoPose' record must not be empty")] if num_ego_pose == 0 else None diff --git a/t4_devkit/sanity/record/rec005.py b/t4_devkit/sanity/record/rec005.py index 9464e44..b4cd15a 100644 --- a/t4_devkit/sanity/record/rec005.py +++ b/t4_devkit/sanity/record/rec005.py @@ -2,12 +2,11 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason from .base import RecordCountChecker - __all__ = ["REC005"] @@ -16,13 +15,14 @@ class REC005(RecordCountChecker): """A checker of REC005.""" name = RuleName("calibrated-sensor-not-empty") + severity = Severity.ERROR description = "'CalibratedSensor' record is not empty." schema = SchemaName.CALIBRATED_SENSOR - def check_count(self, records: list[dict]) -> list[Reason]: + def check_count(self, records: list[dict]) -> list[Reason] | None: num_calibrated_sensor = len(records) return ( [Reason("'CalibratedSensor' record must not be empty")] if num_calibrated_sensor == 0 - else [] + else None ) diff --git a/t4_devkit/sanity/record/rec006.py b/t4_devkit/sanity/record/rec006.py index 802d659..8a251af 100644 --- a/t4_devkit/sanity/record/rec006.py +++ b/t4_devkit/sanity/record/rec006.py @@ -2,12 +2,11 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason from .base import RecordCountChecker - __all__ = ["REC006"] @@ -16,9 +15,10 @@ class REC006(RecordCountChecker): """A checker of REC006.""" name = RuleName("instance-not-empty") + severity = Severity.ERROR description = "'Instance' record is not empty." schema = SchemaName.INSTANCE - def check_count(self, records: list[dict]) -> list[Reason]: + def check_count(self, records: list[dict]) -> list[Reason] | None: num_instance = len(records) - return [Reason("'Instance' record must not be empty")] if num_instance == 0 else [] + return [Reason("'Instance' record must not be empty")] if num_instance == 0 else None diff --git a/t4_devkit/sanity/reference/base.py b/t4_devkit/sanity/reference/base.py index f7a8ea4..04ba8a3 100644 --- a/t4_devkit/sanity/reference/base.py +++ b/t4_devkit/sanity/reference/base.py @@ -19,6 +19,7 @@ class RecordReferenceChecker(Checker): Attributes: name (RuleName): The name of the rule. + severity (Severity): The severity of the rule. description (str): The description of the rule. source (SchemaName): The source schema name. target (SchemaName): The target schema name. @@ -43,7 +44,7 @@ def can_skip(self, context: SanityContext) -> Maybe[Reason]: case _: return Maybe.from_value(Reason("Missing 'annotation' directory path")) - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: source_file = context.to_schema_file(self.source).unwrap() target_file = context.to_schema_file(self.target).unwrap() source_records = load_json_safe(source_file).unwrap() @@ -55,7 +56,7 @@ def check(self, context: SanityContext) -> list[Reason]: for record in source_records if record[self.reference] not in target_tokens and self.is_additional_condition_ok(record) - ] + ] or None def is_additional_condition_ok(self, record: dict[str, Any]) -> bool: """Return True if the additional condition is met. @@ -74,6 +75,7 @@ class FileReferenceChecker(Checker): Attributes: name (RuleName): The name of the rule. + severity (Severity): The severity of the rule. description (str): The description of the rule. schema (SchemaName): The schema name to check. """ diff --git a/t4_devkit/sanity/reference/ref001.py b/t4_devkit/sanity/reference/ref001.py index dcce036..007debe 100644 --- a/t4_devkit/sanity/reference/ref001.py +++ b/t4_devkit/sanity/reference/ref001.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker - __all__ = ["REF001"] @@ -15,6 +14,7 @@ class REF001(RecordReferenceChecker): """A checker of REF001.""" name = RuleName("scene-to-log") + severity = Severity.ERROR description = "'Scene.log_token' refers to 'Log' record." source = SchemaName.SCENE target = SchemaName.LOG diff --git a/t4_devkit/sanity/reference/ref002.py b/t4_devkit/sanity/reference/ref002.py index e4fc05f..ea2a25f 100644 --- a/t4_devkit/sanity/reference/ref002.py +++ b/t4_devkit/sanity/reference/ref002.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker - __all__ = ["REF002"] @@ -15,6 +14,7 @@ class REF002(RecordReferenceChecker): """A checker of REF002.""" name = RuleName("scene-to-first-sample") + severity = Severity.ERROR description = "'Scene.first_sample_token' refers to 'Sample' record." source = SchemaName.SCENE target = SchemaName.SAMPLE diff --git a/t4_devkit/sanity/reference/ref003.py b/t4_devkit/sanity/reference/ref003.py index 1a16684..a13677f 100644 --- a/t4_devkit/sanity/reference/ref003.py +++ b/t4_devkit/sanity/reference/ref003.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker - __all__ = ["REF003"] @@ -15,6 +14,7 @@ class REF003(RecordReferenceChecker): """A checker of REF003.""" name = RuleName("scene-to-last-sample") + severity = Severity.ERROR description = "'Scene.last_sample_token' refers to 'Sample' record." source = SchemaName.SCENE target = SchemaName.SAMPLE diff --git a/t4_devkit/sanity/reference/ref004.py b/t4_devkit/sanity/reference/ref004.py index 1176553..8ae5498 100644 --- a/t4_devkit/sanity/reference/ref004.py +++ b/t4_devkit/sanity/reference/ref004.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker - __all__ = ["REF004"] @@ -15,6 +14,7 @@ class REF004(RecordReferenceChecker): """A checker of REF004.""" name = RuleName("sample-to-scene") + severity = Severity.ERROR description = "'Sample.scene_token' refers to 'Scene' record." source = SchemaName.SAMPLE target = SchemaName.SCENE diff --git a/t4_devkit/sanity/reference/ref005.py b/t4_devkit/sanity/reference/ref005.py index 6bc4c40..58ef57c 100644 --- a/t4_devkit/sanity/reference/ref005.py +++ b/t4_devkit/sanity/reference/ref005.py @@ -4,11 +4,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker - __all__ = ["REF005"] @@ -17,6 +16,7 @@ class REF005(RecordReferenceChecker): """A checker of REF005.""" name = RuleName("sample-data-to-sample") + severity = Severity.ERROR description = "'SampleData.sample_token' refers to 'Sample' record." source = SchemaName.SAMPLE_DATA target = SchemaName.SAMPLE diff --git a/t4_devkit/sanity/reference/ref006.py b/t4_devkit/sanity/reference/ref006.py index f16b5f3..638bf30 100644 --- a/t4_devkit/sanity/reference/ref006.py +++ b/t4_devkit/sanity/reference/ref006.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -14,6 +14,7 @@ class REF006(RecordReferenceChecker): """A checker of REF006.""" name = RuleName("sample-data-to-ego-pose") + severity = Severity.ERROR description = "'SampleData.ego_pose_token' refers to 'EgoPose' record." source = SchemaName.SAMPLE_DATA target = SchemaName.EGO_POSE diff --git a/t4_devkit/sanity/reference/ref007.py b/t4_devkit/sanity/reference/ref007.py index 35b2e08..5cc7f1f 100644 --- a/t4_devkit/sanity/reference/ref007.py +++ b/t4_devkit/sanity/reference/ref007.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -14,6 +14,7 @@ class REF007(RecordReferenceChecker): """A checker of REF007.""" name = RuleName("sample-data-to-calibrated-sensor") + severity = Severity.ERROR description = "'SampleData.calibrated_sensor_token' refers to 'CalibratedSensor' record." source = SchemaName.SAMPLE_DATA target = SchemaName.CALIBRATED_SENSOR diff --git a/t4_devkit/sanity/reference/ref008.py b/t4_devkit/sanity/reference/ref008.py index da3e969..dccec5c 100644 --- a/t4_devkit/sanity/reference/ref008.py +++ b/t4_devkit/sanity/reference/ref008.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker - __all__ = ["REF008"] @@ -15,6 +14,7 @@ class REF008(RecordReferenceChecker): """A checker of REF008.""" name = RuleName("calibrated-sensor-to-sensor") + severity = Severity.ERROR description = "'CalibratedSensor.sensor_token' refers to 'Sensor' record." source = SchemaName.CALIBRATED_SENSOR target = SchemaName.SENSOR diff --git a/t4_devkit/sanity/reference/ref009.py b/t4_devkit/sanity/reference/ref009.py index cb1218f..193ac78 100644 --- a/t4_devkit/sanity/reference/ref009.py +++ b/t4_devkit/sanity/reference/ref009.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker - __all__ = ["REF009"] @@ -15,6 +14,7 @@ class REF009(RecordReferenceChecker): """A checker of REF009.""" name = RuleName("instance-to-category") + severity = Severity.ERROR description = "'Instance.category_token' refers to 'Category' record." source = SchemaName.INSTANCE target = SchemaName.CATEGORY diff --git a/t4_devkit/sanity/reference/ref010.py b/t4_devkit/sanity/reference/ref010.py index 7ce58cb..613ee7a 100644 --- a/t4_devkit/sanity/reference/ref010.py +++ b/t4_devkit/sanity/reference/ref010.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker - __all__ = ["REF010"] @@ -15,6 +14,7 @@ class REF010(RecordReferenceChecker): """A checker of REF010.""" name = RuleName("instance-to-first-sample-annotation") + severity = Severity.ERROR description = "'Instance.first_annotation_token' refers to 'SampleAnnotation' record." source = SchemaName.INSTANCE target = SchemaName.SAMPLE_ANNOTATION diff --git a/t4_devkit/sanity/reference/ref011.py b/t4_devkit/sanity/reference/ref011.py index a374a35..caac4c7 100644 --- a/t4_devkit/sanity/reference/ref011.py +++ b/t4_devkit/sanity/reference/ref011.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker - __all__ = ["REF011"] @@ -15,6 +14,7 @@ class REF011(RecordReferenceChecker): """A checker of REF011.""" name = RuleName("instance-to-last-sample-annotation") + severity = Severity.ERROR description = "'Instance.last_annotation_token' refers to 'SampleAnnotation' record." source = SchemaName.INSTANCE target = SchemaName.SAMPLE_ANNOTATION diff --git a/t4_devkit/sanity/reference/ref012.py b/t4_devkit/sanity/reference/ref012.py index f786993..5e317e2 100644 --- a/t4_devkit/sanity/reference/ref012.py +++ b/t4_devkit/sanity/reference/ref012.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker - __all__ = ["REF012"] @@ -15,6 +14,7 @@ class REF012(RecordReferenceChecker): """A checker of REF012.""" name = RuleName("lidarset-to-sample-data") + severity = Severity.ERROR description = "'LidarSeg.sample_data_token' refers to 'SampleData' record." source = SchemaName.LIDARSEG target = SchemaName.SAMPLE_DATA diff --git a/t4_devkit/sanity/reference/ref013.py b/t4_devkit/sanity/reference/ref013.py index dd5b2e1..6adf0d0 100644 --- a/t4_devkit/sanity/reference/ref013.py +++ b/t4_devkit/sanity/reference/ref013.py @@ -4,7 +4,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason from ..safety import load_json_safe @@ -22,10 +22,11 @@ class REF013(FileReferenceChecker): """A checker of REF013.""" name = RuleName("sample-data-filename-presence") + severity = Severity.ERROR description = "'SampleData.filename' exists." schema = SchemaName.SAMPLE_DATA - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: filepath = context.to_schema_file(self.schema).unwrap() records = load_json_safe(filepath).unwrap() data_root = context.data_root.unwrap() @@ -33,4 +34,4 @@ def check(self, context: SanityContext) -> list[Reason]: Reason(f"File not found: {record['filename']}") for record in records if not data_root.joinpath(record["filename"]).exists() - ] + ] or None diff --git a/t4_devkit/sanity/reference/ref014.py b/t4_devkit/sanity/reference/ref014.py index 227e7f1..09d07ca 100644 --- a/t4_devkit/sanity/reference/ref014.py +++ b/t4_devkit/sanity/reference/ref014.py @@ -4,7 +4,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason from ..safety import load_json_safe @@ -22,10 +22,11 @@ class REF014(FileReferenceChecker): """A checker of REF014.""" name = RuleName("sample-data-filename-presence") + severity = Severity.ERROR description = "'SampleData.filename' exists." schema = SchemaName.SAMPLE_DATA - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: filepath = context.to_schema_file(self.schema).unwrap() records = load_json_safe(filepath).unwrap() data_root = context.data_root.unwrap() @@ -34,4 +35,4 @@ def check(self, context: SanityContext) -> list[Reason]: for record in records if record.get("info_filename") is not None and not data_root.joinpath(record["info_filename"]).exists() - ] + ] or None diff --git a/t4_devkit/sanity/reference/ref015.py b/t4_devkit/sanity/reference/ref015.py index 2bfaf9b..c12313b 100644 --- a/t4_devkit/sanity/reference/ref015.py +++ b/t4_devkit/sanity/reference/ref015.py @@ -4,7 +4,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason from ..safety import load_json_safe @@ -22,10 +22,11 @@ class REF015(FileReferenceChecker): """A checker of REF015.""" name = RuleName("lidarseg-filename-presence") + severity = Severity.ERROR description = "'LidarSeg.filename' exists." schema = SchemaName.LIDARSEG - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: filepath = context.to_schema_file(self.schema).unwrap() records = load_json_safe(filepath).unwrap() data_root = context.data_root.unwrap() @@ -33,4 +34,4 @@ def check(self, context: SanityContext) -> list[Reason]: Reason(f"File not found: {record['filename']}") for record in records if not data_root.joinpath(record["filename"]).exists() - ] + ] or None diff --git a/t4_devkit/sanity/result.py b/t4_devkit/sanity/result.py index e29d0b1..7fbe42c 100644 --- a/t4_devkit/sanity/result.py +++ b/t4_devkit/sanity/result.py @@ -8,31 +8,19 @@ from typing_extensions import Self if TYPE_CHECKING: - from .checker import RuleID, RuleName + from .checker import RuleID, RuleName, Severity from .context import SanityContext __all__ = ["Status", "Report", "SanityResult", "print_sanity_result"] class Status(str, Enum): - """Status of a report.""" + """Runtime outcome per checker.""" SUCCESS = "SUCCESS" FAILURE = "FAILURE" SKIPPED = "SKIPPED" - def is_success(self) -> bool: - """Check if the status is success.""" - return self == Status.SUCCESS - - def is_failure(self) -> bool: - """Check if the status is failure.""" - return self == Status.FAILURE - - def is_skipped(self) -> bool: - """Check if the status is skipped.""" - return self == Status.SKIPPED - Reason = NewType("Reason", str) @@ -44,6 +32,7 @@ class Report: Attributes: id (RuleID): The ID of the rule. name (RuleName): The name of the rule. + severity (Severity): The severity of the rule. description (str): The description of the rule. status (Status): The status of the report. reasons (list[Reason] | None): The list of reasons for the report if the report is a failure or skipped. @@ -51,39 +40,55 @@ class Report: id: RuleID name: RuleName + severity: Severity description: str status: Status reasons: list[Reason] | None = field(default=None) def __attrs_post_init__(self) -> None: - if self.is_success(): + if self.status == Status.SUCCESS: assert self.reasons is None, "Success report cannot have reasons" else: - assert self.reasons is not None, "Failure report must have reasons" + assert self.reasons is not None, "Non-success report must have reasons" - def is_success(self) -> bool: - return self.status == Status.SUCCESS + def is_success(self, *, strict: bool = False) -> bool: + """Check if the status is success.""" + return ( + self.status == Status.SUCCESS + or self.is_skipped() + or (not strict and self.severity.is_warning()) + ) - def is_failure(self) -> bool: - return self.status == Status.FAILURE + def is_failure(self, *, strict: bool = False) -> bool: + """Check if the status is failure.""" + return (self.status == Status.FAILURE and self.severity.is_error()) or not ( + self.is_success(strict=strict) or self.is_skipped() + ) def is_skipped(self) -> bool: + """Check if the status is skipped.""" return self.status == Status.SKIPPED -def make_success(id: RuleID, name: RuleName, description: str) -> Report: - """Make a success report for the given rule.""" - return Report(id, name, description, Status.SUCCESS) +def make_report( + id: RuleID, + name: RuleName, + severity: Severity, + description: str, + reasons: list[Reason] | None = None, +) -> Report: + """Make a report for the given rule.""" + if reasons: + return Report(id, name, severity, description, Status.FAILURE, reasons) + else: + return Report(id, name, severity, description, Status.SUCCESS) -def make_skipped(id: RuleID, name: RuleName, description: str, reason: Reason) -> Report: +def make_skipped( + id: RuleID, name: RuleName, severity: Severity, description: str, reason: Reason +) -> Report: """Make a skipped report for the given rule.""" - return Report(id, name, description, Status.SKIPPED, [reason]) - - -def make_failure(id: RuleID, name: RuleName, description: str, reasons: list[Reason]) -> Report: - """Make a failure report for the given rule.""" - return Report(id, name, description, Status.FAILURE, reasons) + return Report(id, name, severity, description, Status.SKIPPED, [reason]) @define @@ -117,14 +122,37 @@ def from_context(cls, context: SanityContext, reports: list[Report]) -> Self: reports=reports, ) - def __repr__(self) -> str: + def is_success(self, *, strict: bool = False) -> bool: + """Return True if all reports are successful, False otherwise. + + Args: + strict (bool): Whether to consider warnings as failures. + + Returns: + True if all reports are successful, False otherwise. + """ + return all(report.is_success(strict=strict) for report in self.reports) + + def to_str(self, *, strict: bool = False) -> str: + """Return a string representation of the result. + + Args: + strict (bool): Whether to consider warnings as failures. + + Returns: + A string representation of the result. + """ string = f"=== DatasetID: {self.dataset_id} ===\n" for report in self.reports: - if report.is_failure(): + if not report.is_success(strict=strict): string += f"\033[31m {report.id}:\033[0m\n" for reason in report.reasons or []: string += f"\033[31m - {reason}\033[0m\n" elif report.is_skipped(): + string += f"\033[36m {report.id}: [SKIPPED]\033[0m\n" + for reason in report.reasons or []: + string += f"\033[36m - {reason}\033[0m\n" + elif report.severity.is_warning() and report.reasons: string += f"\033[33m {report.id}:\033[0m\n" for reason in report.reasons or []: string += f"\033[33m - {reason}\033[0m\n" @@ -133,35 +161,51 @@ def __repr__(self) -> str: return string -def print_sanity_result(result: SanityResult) -> None: +def print_sanity_result(result: SanityResult, *, strict: bool = False) -> None: """Print detailed and summary results of a sanity check. Args: result (SanityResult): The result of a sanity check. """ # print detailed result - print(result) + print(result.to_str(strict=strict)) # print summary result - success = sum(1 for rp in result.reports if rp.is_success()) - failures = sum(1 for rp in result.reports if rp.is_failure()) + success = sum(1 for rp in result.reports if rp.is_success(strict=strict)) + failures = sum(1 for rp in result.reports if not rp.is_success(strict=strict)) skips = sum(1 for rp in result.reports if rp.is_skipped()) + + # just count the number of warnings + warnings = sum(1 for rp in result.reports if rp.severity.is_warning() and rp.reasons) + summary_rows = [ [ result.dataset_id, result.version, - "\033[31mFAILURE\033[0m" if failures > 0 else "\033[32mSUCCESS\033[0m", + "\033[32mSUCCESS\033[0m" + if result.is_success(strict=strict) + else "\033[31mFAILURE\033[0m", len(result.reports), success, failures, skips, + warnings, ] ] print( tabulate( summary_rows, - headers=["DatasetID", "Version", "Status", "Rules", "Success", "Failures", "Skips"], + headers=[ + "DatasetID", + "Version", + "Status", + "Rules", + "Success", + "Failures", + "Skips", + "Warnings", + ], tablefmt="pretty", ), ) diff --git a/t4_devkit/sanity/structure/str001.py b/t4_devkit/sanity/structure/str001.py index 7a0dbf3..c3d0a0b 100644 --- a/t4_devkit/sanity/structure/str001.py +++ b/t4_devkit/sanity/structure/str001.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName +from ..checker import Checker, RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason @@ -20,11 +20,12 @@ class STR001(Checker): """A checker of STR001.""" name = RuleName("version-dir-presence") + severity = Severity.WARNING description = "'version/' directory exists under the dataset root directory." - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: match context.version: case Some(_): - return [] + return None case _: return [Reason("'version' directory doesn't exist")] diff --git a/t4_devkit/sanity/structure/str002.py b/t4_devkit/sanity/structure/str002.py index 356f13a..baee3cc 100644 --- a/t4_devkit/sanity/structure/str002.py +++ b/t4_devkit/sanity/structure/str002.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName +from ..checker import Checker, RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason @@ -20,13 +20,14 @@ class STR002(Checker): """A checker of STR002.""" name = RuleName("annotation-dir-presence") + severity = Severity.ERROR description = "'annotation/' directory exists under the dataset root directory." - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: match context.annotation_dir: case Some(x): return ( - [] + None if x.exists() else [Reason(f"Path to 'annotation' not found: {x.as_posix()}")] ) diff --git a/t4_devkit/sanity/structure/str003.py b/t4_devkit/sanity/structure/str003.py index b3a012f..bc95188 100644 --- a/t4_devkit/sanity/structure/str003.py +++ b/t4_devkit/sanity/structure/str003.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName +from ..checker import Checker, RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason @@ -20,13 +20,14 @@ class STR003(Checker): """A checker of STR003.""" name = RuleName("data-dir-presence") + severity = Severity.ERROR description = "'data/' directory exists under the dataset root directory." - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: match context.sensor_data_dir: case Some(x): return ( - [] + None if x.exists() else [Reason(f"Path to 'data' directory not found: {x.as_posix()}")] ) diff --git a/t4_devkit/sanity/structure/str004.py b/t4_devkit/sanity/structure/str004.py index b13c9f9..5c4944a 100644 --- a/t4_devkit/sanity/structure/str004.py +++ b/t4_devkit/sanity/structure/str004.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName +from ..checker import Checker, RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason @@ -20,11 +20,12 @@ class STR004(Checker): """A checker of STR004.""" name = RuleName("map-dir-presence") + severity = Severity.WARNING description = "'map/' directory exists under the dataset root directory." - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: match context.map_dir: case Some(x): - return [] if x.exists() else [Reason(f"Path to 'map' not found: {x.as_posix()}")] + return None if x.exists() else [Reason(f"Path to 'map' not found: {x.as_posix()}")] case _: return [Reason("dataset directory doesn't contain 'map' directory")] diff --git a/t4_devkit/sanity/structure/str005.py b/t4_devkit/sanity/structure/str005.py index 6bd43e8..7860a9d 100644 --- a/t4_devkit/sanity/structure/str005.py +++ b/t4_devkit/sanity/structure/str005.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName +from ..checker import Checker, RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason @@ -20,13 +20,16 @@ class STR005(Checker): """A checker of STR005.""" name = RuleName("bag-dir-presence") + severity = Severity.WARNING description = "'input_bag/' directory exists under the dataset root directory." - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: match context.bag_dir: case Some(x): return ( - [] if x.exists() else [Reason(f"Path to 'input_bag' not found: {x.as_posix()}")] + None + if x.exists() + else [Reason(f"Path to 'input_bag' not found: {x.as_posix()}")] ) case _: return [Reason("dataset directory doesn't contain 'input_bag' directory")] diff --git a/t4_devkit/sanity/structure/str006.py b/t4_devkit/sanity/structure/str006.py index 992e94a..09da99b 100644 --- a/t4_devkit/sanity/structure/str006.py +++ b/t4_devkit/sanity/structure/str006.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName +from ..checker import Checker, RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason @@ -20,13 +20,14 @@ class STR006(Checker): """A checker of STR006.""" name = RuleName("status-json-presence") + severity = Severity.WARNING description = "'status.json' file exists under the dataset root directory." - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: match context.status_json: case Some(x): return ( - [] + None if x.exists() else [Reason(f"Path to 'status.json' not found: {x.as_posix()}")] ) diff --git a/t4_devkit/sanity/structure/str007.py b/t4_devkit/sanity/structure/str007.py index fac9bbf..0819559 100644 --- a/t4_devkit/sanity/structure/str007.py +++ b/t4_devkit/sanity/structure/str007.py @@ -6,7 +6,7 @@ from t4_devkit.schema import SchemaName -from ..checker import Checker, RuleID, RuleName +from ..checker import Checker, RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason @@ -22,9 +22,10 @@ class STR007(Checker): """A checker of STR007.""" name = RuleName("schema-file-presence") + severity = Severity.ERROR description = "Mandatory schema JSON files exist under the `annotation/` directory." - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: failures = [] for schema in SchemaName: match context.to_schema_file(schema): @@ -33,4 +34,4 @@ def check(self, context: SanityContext) -> list[Reason]: failures.append(Reason(f"schema file '{schema.filename}' not found")) case _: failures.append(Reason(f"schema file '{schema.filename}' not found")) - return failures + return failures if failures else None diff --git a/t4_devkit/sanity/structure/str008.py b/t4_devkit/sanity/structure/str008.py index 196fa78..d7f25d8 100644 --- a/t4_devkit/sanity/structure/str008.py +++ b/t4_devkit/sanity/structure/str008.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName +from ..checker import Checker, RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason @@ -19,9 +19,10 @@ class STR008(Checker): """A checker of STR008.""" name = RuleName("lanelet-file-presence") + severity = Severity.WARNING description = "'lanelet2_map.osm' file exists under the 'map/' directory." - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: match context.map_dir: case Some(x): if not x.exists(): @@ -30,7 +31,7 @@ def check(self, context: SanityContext) -> list[Reason]: return ( [Reason(f"Lanelet2 map file not found: {lanelet_file.as_posix()}")] if not lanelet_file.exists() - else [] + else None ) case _: return [Reason("dataset directory doesn't contain 'map' directory")] diff --git a/t4_devkit/sanity/structure/str009.py b/t4_devkit/sanity/structure/str009.py index df23c9c..6f97220 100644 --- a/t4_devkit/sanity/structure/str009.py +++ b/t4_devkit/sanity/structure/str009.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName +from ..checker import Checker, RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason @@ -19,9 +19,10 @@ class STR009(Checker): """A checker of STR009.""" name = RuleName("pointcloud-map-dir-presence") + severity = Severity.WARNING description = "'pointcloud_map.pcd' directory exists under the 'map/' directory." - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: match context.map_dir: case Some(x): if not x.exists(): @@ -30,7 +31,7 @@ def check(self, context: SanityContext) -> list[Reason]: return ( [Reason(f"PCD map directory not found: {pointcloud_map_dir.as_posix()}")] if not pointcloud_map_dir.exists() - else [] + else None ) case _: return [Reason("dataset directory doesn't contain 'map' directory")] diff --git a/t4_devkit/sanity/tier4/tiv001.py b/t4_devkit/sanity/tier4/tiv001.py index 0ee376b..e9174dd 100644 --- a/t4_devkit/sanity/tier4/tiv001.py +++ b/t4_devkit/sanity/tier4/tiv001.py @@ -2,15 +2,16 @@ from typing import TYPE_CHECKING +from returns.maybe import Maybe, Nothing, Some from returns.pipeline import is_successful from returns.result import Result, safe -from returns.maybe import Maybe, Nothing, Some -from ..checker import Checker, RuleID, RuleName -from ..result import Reason -from ..registry import CHECKERS from t4_devkit import Tier4 +from ..checker import Checker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + if TYPE_CHECKING: from ..context import SanityContext @@ -22,6 +23,7 @@ class TIV001(Checker): """A checker for TIV001.""" name = RuleName("load-tier4") + severity = Severity.ERROR description = "Ensure 'Tier4' instance is loaded successfully." def can_skip(self, context: SanityContext) -> Maybe[Reason]: @@ -33,10 +35,10 @@ def can_skip(self, context: SanityContext) -> Maybe[Reason]: case _: return Nothing - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: result = _load_tier4_safe(context) return ( - [] if is_successful(result) else [Reason(f"Failed to load Tier4: {result.failure()}")] + None if is_successful(result) else [Reason(f"Failed to load Tier4: {result.failure()}")] )