From 3f5d52b9cb2fda4528f73af2ee5074a3cb469445 Mon Sep 17 00:00:00 2001 From: ktro2828 Date: Tue, 11 Nov 2025 18:44:21 +0900 Subject: [PATCH 1/6] feat: return status depending on the checker's severity Signed-off-by: ktro2828 --- docs/cli/t4sanity.md | 13 +++-- docs/schema/requirement.md | 6 +- t4_devkit/sanity/checker.py | 21 +++++-- t4_devkit/sanity/format/base.py | 1 + t4_devkit/sanity/format/fmt001.py | 4 +- t4_devkit/sanity/format/fmt002.py | 4 +- t4_devkit/sanity/format/fmt003.py | 4 +- t4_devkit/sanity/format/fmt004.py | 4 +- t4_devkit/sanity/format/fmt005.py | 4 +- t4_devkit/sanity/format/fmt006.py | 3 +- t4_devkit/sanity/format/fmt007.py | 3 +- t4_devkit/sanity/format/fmt008.py | 4 +- t4_devkit/sanity/format/fmt009.py | 4 +- t4_devkit/sanity/format/fmt010.py | 4 +- t4_devkit/sanity/format/fmt011.py | 4 +- t4_devkit/sanity/format/fmt012.py | 4 +- t4_devkit/sanity/format/fmt013.py | 4 +- t4_devkit/sanity/format/fmt014.py | 4 +- t4_devkit/sanity/format/fmt015.py | 4 +- t4_devkit/sanity/format/fmt016.py | 4 +- t4_devkit/sanity/format/fmt017.py | 4 +- t4_devkit/sanity/format/fmt018.py | 4 +- t4_devkit/sanity/record/base.py | 1 + t4_devkit/sanity/record/rec001.py | 4 +- t4_devkit/sanity/record/rec002.py | 4 +- t4_devkit/sanity/record/rec003.py | 4 +- t4_devkit/sanity/record/rec004.py | 4 +- t4_devkit/sanity/record/rec005.py | 4 +- t4_devkit/sanity/record/rec006.py | 4 +- t4_devkit/sanity/reference/base.py | 2 + t4_devkit/sanity/reference/ref001.py | 4 +- t4_devkit/sanity/reference/ref002.py | 4 +- t4_devkit/sanity/reference/ref003.py | 4 +- t4_devkit/sanity/reference/ref004.py | 4 +- t4_devkit/sanity/reference/ref005.py | 4 +- t4_devkit/sanity/reference/ref006.py | 3 +- t4_devkit/sanity/reference/ref007.py | 3 +- t4_devkit/sanity/reference/ref008.py | 4 +- t4_devkit/sanity/reference/ref009.py | 4 +- t4_devkit/sanity/reference/ref010.py | 4 +- t4_devkit/sanity/reference/ref011.py | 4 +- t4_devkit/sanity/reference/ref012.py | 4 +- t4_devkit/sanity/reference/ref013.py | 3 +- t4_devkit/sanity/reference/ref014.py | 3 +- t4_devkit/sanity/reference/ref015.py | 3 +- t4_devkit/sanity/result.py | 84 ++++++++++++++++++++-------- t4_devkit/sanity/structure/str001.py | 3 +- t4_devkit/sanity/structure/str002.py | 3 +- t4_devkit/sanity/structure/str003.py | 3 +- t4_devkit/sanity/structure/str004.py | 3 +- t4_devkit/sanity/structure/str005.py | 3 +- t4_devkit/sanity/structure/str006.py | 3 +- t4_devkit/sanity/structure/str007.py | 3 +- t4_devkit/sanity/structure/str008.py | 3 +- t4_devkit/sanity/structure/str009.py | 3 +- t4_devkit/sanity/tier4/tiv001.py | 10 ++-- 56 files changed, 193 insertions(+), 121 deletions(-) diff --git a/docs/cli/t4sanity.md b/docs/cli/t4sanity.md index 3a6a583..87a7640 100644 --- a/docs/cli/t4sanity.md +++ b/docs/cli/t4sanity.md @@ -58,11 +58,11 @@ $ t4sanity STR008: ✅ ... -+-----------+---------+---------+-------+---------+----------+-------+ -| DatasetID | Version | Status | Rules | Success | Failures | Skips | -+-----------+---------+---------+-------+---------+----------+-------+ -| dataset1 | 0 | SUCCESS | 44 | 44 | 0 | 0 | -+-----------+---------+---------+-------+---------+----------+-------+ ++-----------+---------+---------+-------+---------+----------+----------+-------+ +| DatasetID | Version | Status | Rules | Success | Warnings | Failures | Skips | ++-----------+---------+---------+-------+---------+----------+----------+-------+ +| dataset1 | | SUCCESS | 49 | 43 | 4 | 0 | 2 | ++-----------+---------+---------+-------+---------+----------+----------+-------+ ``` ### Dump Results as JSON @@ -83,8 +83,9 @@ Then a JSON file named `result.json` will be generated as follows: { "id": "", "name": "", + "severity": "", "description": "", - "status": "", + "status": "", "reasons": "<[, , ...]: [str; N] | null>" // Failure or skipped reasons, null if success }, ] diff --git a/docs/schema/requirement.md b/docs/schema/requirement.md index 9988179..9e8d773 100644 --- a/docs/schema/requirement.md +++ b/docs/schema/requirement.md @@ -7,9 +7,9 @@ | `STR001` | `version-dir-presence` | `Warn` | `version/` directory exists under the dataset root directory. | | `STR002` | `annotation-dir-presence` | `Error` | `annotation/` directory exists under the dataset root directory. | | `STR003` | `data-dir-presence` | `Error` | `data/` directory exists under the dataset root directory. | -| `STR004` | `map-dir-presence` | `Error` | `map/` directory exists under the dataset root directory. | -| `STR005` | `bag-dir-presence` | `Error` | `input_bag/` directory exists under the dataset root directory. | -| `STR006` | `status-file-presence` | `Error` | `status.json` file exists under the dataset root directory. | +| `STR004` | `map-dir-presence` | `Warn` | `map/` directory exists under the dataset root directory. | +| `STR005` | `bag-dir-presence` | `Warn` | `input_bag/` directory exists under the dataset root directory. | +| `STR006` | `status-file-presence` | `Warn` | `status.json` file exists under the dataset root directory. | | `STR007` | `schema-files-presence` | `Error` | Mandatory schema JSON files exist under the `annotation/` directory. | | `STR008` | `lanelet-file-presence` | `Warn` | `lanelet2_map.osm` file exists under the `map/` directory. | | `STR009` | `pointcloud-map-dir-presence` | `Warn` | `pointcloud_map.pcd` directory exists under the `map/` directory. | diff --git a/t4_devkit/sanity/checker.py b/t4_devkit/sanity/checker.py index f434b0c..7208311 100644 --- a/t4_devkit/sanity/checker.py +++ b/t4_devkit/sanity/checker.py @@ -1,11 +1,12 @@ from __future__ import annotations from abc import ABC, abstractmethod +from enum import Enum from typing import TYPE_CHECKING, NewType from returns.maybe import Maybe, Nothing, Some -from .result import make_failure, make_success, make_skipped +from .result import make_failure, make_skipped, make_success, make_warning if TYPE_CHECKING: from .context import SanityContext @@ -16,11 +17,19 @@ RuleName = NewType("RuleName", str) +class Severity(str, Enum): + """Severity levels for sanity checkers.""" + + ERROR = "ERROR" + WARNING = "WARNING" + + class Checker(ABC): """Base class for sanity checkers.""" name: RuleName description: str + severity: Severity def __init__(self, id: RuleID) -> None: self.id = id @@ -28,13 +37,17 @@ def __init__(self, id: RuleID) -> None: def __call__(self, context: SanityContext) -> Report: match self.can_skip(context): case Some(skip): - return make_skipped(self.id, self.name, self.description, skip) + return make_skipped(self.id, self.name, self.severity, self.description, skip) reasons = self.check(context) if reasons: - return make_failure(self.id, self.name, self.description, reasons) + return ( + make_failure(self.id, self.name, self.severity, self.description, reasons) + if self.severity == Severity.ERROR + else make_warning(self.id, self.name, self.severity, self.description, reasons) + ) else: - return make_success(self.id, self.name, self.description) + return make_success(self.id, self.name, self.severity, self.description) def can_skip(self, _: SanityContext) -> Maybe[Reason]: """Return a skip reason if the checker should be skipped.""" diff --git a/t4_devkit/sanity/format/base.py b/t4_devkit/sanity/format/base.py index 3670eb1..ae31276 100644 --- a/t4_devkit/sanity/format/base.py +++ b/t4_devkit/sanity/format/base.py @@ -21,6 +21,7 @@ class FieldTypeChecker(Checker): Attributes: name (RuleName): The name of the rule. + severity (Severity): The severity of the rule. description (str): The description of the rule. schema (SchemaName): The schema name to check. """ diff --git a/t4_devkit/sanity/format/fmt001.py b/t4_devkit/sanity/format/fmt001.py index b156ee0..b70ad07 100644 --- a/t4_devkit/sanity/format/fmt001.py +++ b/t4_devkit/sanity/format/fmt001.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT001"] @@ -15,5 +14,6 @@ class FMT001(FieldTypeChecker): """A checker of FMT001.""" name = RuleName("attribute-field") + severity = Severity.ERROR description = "All types of 'Attribute' fields are valid." schema = SchemaName.ATTRIBUTE diff --git a/t4_devkit/sanity/format/fmt002.py b/t4_devkit/sanity/format/fmt002.py index 240498b..ccd6d85 100644 --- a/t4_devkit/sanity/format/fmt002.py +++ b/t4_devkit/sanity/format/fmt002.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT002"] @@ -15,5 +14,6 @@ class FMT002(FieldTypeChecker): """A checker of FMT002.""" name = RuleName("calibrated-sensor-field") + severity = Severity.ERROR description = "All types of 'CalibratedSensor' fields are valid." schema = SchemaName.CALIBRATED_SENSOR diff --git a/t4_devkit/sanity/format/fmt003.py b/t4_devkit/sanity/format/fmt003.py index 8baf068..33352e8 100644 --- a/t4_devkit/sanity/format/fmt003.py +++ b/t4_devkit/sanity/format/fmt003.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT003"] @@ -15,5 +14,6 @@ class FMT003(FieldTypeChecker): """A checker of FMT003.""" name = RuleName("category-field") + severity = Severity.ERROR description = "All types of 'Category' fields are valid." schema = SchemaName.CATEGORY diff --git a/t4_devkit/sanity/format/fmt004.py b/t4_devkit/sanity/format/fmt004.py index 685f650..ccfbc8b 100644 --- a/t4_devkit/sanity/format/fmt004.py +++ b/t4_devkit/sanity/format/fmt004.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT004"] @@ -15,5 +14,6 @@ class FMT004(FieldTypeChecker): """A checker of FMT004.""" name = RuleName("ego-pose-field") + severity = Severity.ERROR description = "All types of 'EgoPose' fields are valid." schema = SchemaName.EGO_POSE diff --git a/t4_devkit/sanity/format/fmt005.py b/t4_devkit/sanity/format/fmt005.py index 0610280..b8d5d57 100644 --- a/t4_devkit/sanity/format/fmt005.py +++ b/t4_devkit/sanity/format/fmt005.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT005"] @@ -15,5 +14,6 @@ class FMT005(FieldTypeChecker): """A checker of FMT005.""" name = RuleName("instance-field") + severity = Severity.ERROR description = "All types of 'Instance' fields are valid." schema = SchemaName.INSTANCE diff --git a/t4_devkit/sanity/format/fmt006.py b/t4_devkit/sanity/format/fmt006.py index b35ed1e..23a4132 100644 --- a/t4_devkit/sanity/format/fmt006.py +++ b/t4_devkit/sanity/format/fmt006.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker @@ -14,5 +14,6 @@ class FMT006(FieldTypeChecker): """A checker of FMT006.""" name = RuleName("log-field") + severity = Severity.ERROR description = "All types of 'Log' fields are valid." schema = SchemaName.LOG diff --git a/t4_devkit/sanity/format/fmt007.py b/t4_devkit/sanity/format/fmt007.py index a62ef8d..f2e9eab 100644 --- a/t4_devkit/sanity/format/fmt007.py +++ b/t4_devkit/sanity/format/fmt007.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker @@ -14,5 +14,6 @@ class FMT007(FieldTypeChecker): """A checker of FMT007.""" name = RuleName("map-field") + severity = Severity.ERROR description = "All types of 'Map' fields are valid." schema = SchemaName.MAP diff --git a/t4_devkit/sanity/format/fmt008.py b/t4_devkit/sanity/format/fmt008.py index 7bc978e..9d7a743 100644 --- a/t4_devkit/sanity/format/fmt008.py +++ b/t4_devkit/sanity/format/fmt008.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT008"] @@ -15,5 +14,6 @@ class FMT008(FieldTypeChecker): """A checker of FMT008.""" name = RuleName("sample-field") + severity = Severity.ERROR description = "All types of 'Sample' fields are valid." schema = SchemaName.SAMPLE diff --git a/t4_devkit/sanity/format/fmt009.py b/t4_devkit/sanity/format/fmt009.py index 9211e44..12e38fb 100644 --- a/t4_devkit/sanity/format/fmt009.py +++ b/t4_devkit/sanity/format/fmt009.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT009"] @@ -15,5 +14,6 @@ class FMT009(FieldTypeChecker): """A checker of FMT009.""" name = RuleName("sample-annotation-field") + severity = Severity.ERROR description = "All types of 'SampleAnnotation' fields are valid." schema = SchemaName.SAMPLE_ANNOTATION diff --git a/t4_devkit/sanity/format/fmt010.py b/t4_devkit/sanity/format/fmt010.py index 0235bc4..447193c 100644 --- a/t4_devkit/sanity/format/fmt010.py +++ b/t4_devkit/sanity/format/fmt010.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT010"] @@ -15,5 +14,6 @@ class FMT010(FieldTypeChecker): """A checker of FMT010.""" name = RuleName("sample-data-field") + severity = Severity.ERROR description = "All types of 'SampleData' fields are valid." schema = SchemaName.SAMPLE_DATA diff --git a/t4_devkit/sanity/format/fmt011.py b/t4_devkit/sanity/format/fmt011.py index 612b3de..368efba 100644 --- a/t4_devkit/sanity/format/fmt011.py +++ b/t4_devkit/sanity/format/fmt011.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT011"] @@ -15,5 +14,6 @@ class FMT011(FieldTypeChecker): """A checker of FMT011.""" name = RuleName("scene-field") + severity = Severity.ERROR description = "All types of 'Scene' fields are valid." schema = SchemaName.SCENE diff --git a/t4_devkit/sanity/format/fmt012.py b/t4_devkit/sanity/format/fmt012.py index d98169f..a70bcbb 100644 --- a/t4_devkit/sanity/format/fmt012.py +++ b/t4_devkit/sanity/format/fmt012.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT012"] @@ -15,5 +14,6 @@ class FMT012(FieldTypeChecker): """A checker of FMT012.""" name = RuleName("sensor-field") + severity = Severity.ERROR description = "All types of 'Sensor' fields are valid." schema = SchemaName.SENSOR diff --git a/t4_devkit/sanity/format/fmt013.py b/t4_devkit/sanity/format/fmt013.py index 3318f72..028e6b9 100644 --- a/t4_devkit/sanity/format/fmt013.py +++ b/t4_devkit/sanity/format/fmt013.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT013"] @@ -15,5 +14,6 @@ class FMT013(FieldTypeChecker): """A checker of FMT013.""" name = RuleName("visibility-field") + severity = Severity.ERROR description = "All types of 'Visibility' fields are valid." schema = SchemaName.VISIBILITY diff --git a/t4_devkit/sanity/format/fmt014.py b/t4_devkit/sanity/format/fmt014.py index 85fa09c..7b4ed4b 100644 --- a/t4_devkit/sanity/format/fmt014.py +++ b/t4_devkit/sanity/format/fmt014.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT014"] @@ -15,5 +14,6 @@ class FMT014(FieldTypeChecker): """A checker of FMT014.""" name = RuleName("lidarseg-field") + severity = Severity.ERROR description = "All types of 'LidarSeg' fields are valid." schema = SchemaName.LIDARSEG diff --git a/t4_devkit/sanity/format/fmt015.py b/t4_devkit/sanity/format/fmt015.py index 02b61fb..e56dc68 100644 --- a/t4_devkit/sanity/format/fmt015.py +++ b/t4_devkit/sanity/format/fmt015.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT015"] @@ -15,5 +14,6 @@ class FMT015(FieldTypeChecker): """A checker of FMT015.""" name = RuleName("object-ann-field") + severity = Severity.ERROR description = "All types of 'ObjectAnn' fields are valid." schema = SchemaName.OBJECT_ANN diff --git a/t4_devkit/sanity/format/fmt016.py b/t4_devkit/sanity/format/fmt016.py index 59bc770..84df925 100644 --- a/t4_devkit/sanity/format/fmt016.py +++ b/t4_devkit/sanity/format/fmt016.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT016"] @@ -15,5 +14,6 @@ class FMT016(FieldTypeChecker): """A checker of FMT016.""" name = RuleName("surface-ann-field") + severity = Severity.ERROR description = "All types of 'SurfaceAnn' fields are valid." schema = SchemaName.SURFACE_ANN diff --git a/t4_devkit/sanity/format/fmt017.py b/t4_devkit/sanity/format/fmt017.py index ea5727b..ca52efe 100644 --- a/t4_devkit/sanity/format/fmt017.py +++ b/t4_devkit/sanity/format/fmt017.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT017"] @@ -15,5 +14,6 @@ class FMT017(FieldTypeChecker): """A checker of FMT017.""" name = RuleName("keypoint-field") + severity = Severity.ERROR description = "All types of 'Keypoint' fields are valid." schema = SchemaName.KEYPOINT diff --git a/t4_devkit/sanity/format/fmt018.py b/t4_devkit/sanity/format/fmt018.py index 6e4df61..73b57ca 100644 --- a/t4_devkit/sanity/format/fmt018.py +++ b/t4_devkit/sanity/format/fmt018.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import FieldTypeChecker - __all__ = ["FMT018"] @@ -15,5 +14,6 @@ class FMT018(FieldTypeChecker): """A checker of FMT018.""" name = RuleName("vehicle-state-field") + severity = Severity.ERROR description = "All types of 'VehicleState' fields are valid." schema = SchemaName.VEHICLE_STATE diff --git a/t4_devkit/sanity/record/base.py b/t4_devkit/sanity/record/base.py index e172355..7b9b505 100644 --- a/t4_devkit/sanity/record/base.py +++ b/t4_devkit/sanity/record/base.py @@ -20,6 +20,7 @@ class RecordCountChecker(Checker): Attributes: name (RuleName): The name of the rule. + severity (Severity): The severity of the rule. description (str): The description of the rule. schema (SchemaName): The schema name to check. """ diff --git a/t4_devkit/sanity/record/rec001.py b/t4_devkit/sanity/record/rec001.py index 2923595..e01e485 100644 --- a/t4_devkit/sanity/record/rec001.py +++ b/t4_devkit/sanity/record/rec001.py @@ -2,12 +2,11 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason from .base import RecordCountChecker - __all__ = ["REC001"] @@ -16,6 +15,7 @@ class REC001(RecordCountChecker): """A checker of REC001.""" name = RuleName("scene-single") + severity = Severity.ERROR description = "'Scene' record is a single." schema = SchemaName.SCENE diff --git a/t4_devkit/sanity/record/rec002.py b/t4_devkit/sanity/record/rec002.py index 0d802c9..cea4656 100644 --- a/t4_devkit/sanity/record/rec002.py +++ b/t4_devkit/sanity/record/rec002.py @@ -2,12 +2,11 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason from .base import RecordCountChecker - __all__ = ["REC002"] @@ -16,6 +15,7 @@ class REC002(RecordCountChecker): """A checker of REC002.""" name = RuleName("sample-not-empty") + severity = Severity.ERROR description = "'Sample' record is not empty." schema = SchemaName.SAMPLE diff --git a/t4_devkit/sanity/record/rec003.py b/t4_devkit/sanity/record/rec003.py index e11301b..0d110dd 100644 --- a/t4_devkit/sanity/record/rec003.py +++ b/t4_devkit/sanity/record/rec003.py @@ -2,12 +2,11 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason from .base import RecordCountChecker - __all__ = ["REC003"] @@ -16,6 +15,7 @@ class REC003(RecordCountChecker): """A checker of REC003.""" name = RuleName("sample-data-not-empty") + severity = Severity.ERROR description = "'SampleData' record is not empty." schema = SchemaName.SAMPLE_DATA diff --git a/t4_devkit/sanity/record/rec004.py b/t4_devkit/sanity/record/rec004.py index 048fdfa..840e813 100644 --- a/t4_devkit/sanity/record/rec004.py +++ b/t4_devkit/sanity/record/rec004.py @@ -2,12 +2,11 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason from .base import RecordCountChecker - __all__ = ["REC004"] @@ -16,6 +15,7 @@ class REC004(RecordCountChecker): """A checker of REC004.""" name = RuleName("ego-pose-not-empty") + severity = Severity.ERROR description = "'EgoPose' record is not empty." schema = SchemaName.EGO_POSE diff --git a/t4_devkit/sanity/record/rec005.py b/t4_devkit/sanity/record/rec005.py index 9464e44..c5bf39c 100644 --- a/t4_devkit/sanity/record/rec005.py +++ b/t4_devkit/sanity/record/rec005.py @@ -2,12 +2,11 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason from .base import RecordCountChecker - __all__ = ["REC005"] @@ -16,6 +15,7 @@ class REC005(RecordCountChecker): """A checker of REC005.""" name = RuleName("calibrated-sensor-not-empty") + severity = Severity.ERROR description = "'CalibratedSensor' record is not empty." schema = SchemaName.CALIBRATED_SENSOR diff --git a/t4_devkit/sanity/record/rec006.py b/t4_devkit/sanity/record/rec006.py index 802d659..e6f1f11 100644 --- a/t4_devkit/sanity/record/rec006.py +++ b/t4_devkit/sanity/record/rec006.py @@ -2,12 +2,11 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason from .base import RecordCountChecker - __all__ = ["REC006"] @@ -16,6 +15,7 @@ class REC006(RecordCountChecker): """A checker of REC006.""" name = RuleName("instance-not-empty") + severity = Severity.ERROR description = "'Instance' record is not empty." schema = SchemaName.INSTANCE diff --git a/t4_devkit/sanity/reference/base.py b/t4_devkit/sanity/reference/base.py index f7a8ea4..eb8b420 100644 --- a/t4_devkit/sanity/reference/base.py +++ b/t4_devkit/sanity/reference/base.py @@ -19,6 +19,7 @@ class RecordReferenceChecker(Checker): Attributes: name (RuleName): The name of the rule. + severity (Severity): The severity of the rule. description (str): The description of the rule. source (SchemaName): The source schema name. target (SchemaName): The target schema name. @@ -74,6 +75,7 @@ class FileReferenceChecker(Checker): Attributes: name (RuleName): The name of the rule. + severity (Severity): The severity of the rule. description (str): The description of the rule. schema (SchemaName): The schema name to check. """ diff --git a/t4_devkit/sanity/reference/ref001.py b/t4_devkit/sanity/reference/ref001.py index dcce036..007debe 100644 --- a/t4_devkit/sanity/reference/ref001.py +++ b/t4_devkit/sanity/reference/ref001.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker - __all__ = ["REF001"] @@ -15,6 +14,7 @@ class REF001(RecordReferenceChecker): """A checker of REF001.""" name = RuleName("scene-to-log") + severity = Severity.ERROR description = "'Scene.log_token' refers to 'Log' record." source = SchemaName.SCENE target = SchemaName.LOG diff --git a/t4_devkit/sanity/reference/ref002.py b/t4_devkit/sanity/reference/ref002.py index e4fc05f..ea2a25f 100644 --- a/t4_devkit/sanity/reference/ref002.py +++ b/t4_devkit/sanity/reference/ref002.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker - __all__ = ["REF002"] @@ -15,6 +14,7 @@ class REF002(RecordReferenceChecker): """A checker of REF002.""" name = RuleName("scene-to-first-sample") + severity = Severity.ERROR description = "'Scene.first_sample_token' refers to 'Sample' record." source = SchemaName.SCENE target = SchemaName.SAMPLE diff --git a/t4_devkit/sanity/reference/ref003.py b/t4_devkit/sanity/reference/ref003.py index 1a16684..a13677f 100644 --- a/t4_devkit/sanity/reference/ref003.py +++ b/t4_devkit/sanity/reference/ref003.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker - __all__ = ["REF003"] @@ -15,6 +14,7 @@ class REF003(RecordReferenceChecker): """A checker of REF003.""" name = RuleName("scene-to-last-sample") + severity = Severity.ERROR description = "'Scene.last_sample_token' refers to 'Sample' record." source = SchemaName.SCENE target = SchemaName.SAMPLE diff --git a/t4_devkit/sanity/reference/ref004.py b/t4_devkit/sanity/reference/ref004.py index 1176553..8ae5498 100644 --- a/t4_devkit/sanity/reference/ref004.py +++ b/t4_devkit/sanity/reference/ref004.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker - __all__ = ["REF004"] @@ -15,6 +14,7 @@ class REF004(RecordReferenceChecker): """A checker of REF004.""" name = RuleName("sample-to-scene") + severity = Severity.ERROR description = "'Sample.scene_token' refers to 'Scene' record." source = SchemaName.SAMPLE target = SchemaName.SCENE diff --git a/t4_devkit/sanity/reference/ref005.py b/t4_devkit/sanity/reference/ref005.py index 6bc4c40..58ef57c 100644 --- a/t4_devkit/sanity/reference/ref005.py +++ b/t4_devkit/sanity/reference/ref005.py @@ -4,11 +4,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker - __all__ = ["REF005"] @@ -17,6 +16,7 @@ class REF005(RecordReferenceChecker): """A checker of REF005.""" name = RuleName("sample-data-to-sample") + severity = Severity.ERROR description = "'SampleData.sample_token' refers to 'Sample' record." source = SchemaName.SAMPLE_DATA target = SchemaName.SAMPLE diff --git a/t4_devkit/sanity/reference/ref006.py b/t4_devkit/sanity/reference/ref006.py index f16b5f3..638bf30 100644 --- a/t4_devkit/sanity/reference/ref006.py +++ b/t4_devkit/sanity/reference/ref006.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -14,6 +14,7 @@ class REF006(RecordReferenceChecker): """A checker of REF006.""" name = RuleName("sample-data-to-ego-pose") + severity = Severity.ERROR description = "'SampleData.ego_pose_token' refers to 'EgoPose' record." source = SchemaName.SAMPLE_DATA target = SchemaName.EGO_POSE diff --git a/t4_devkit/sanity/reference/ref007.py b/t4_devkit/sanity/reference/ref007.py index 35b2e08..5cc7f1f 100644 --- a/t4_devkit/sanity/reference/ref007.py +++ b/t4_devkit/sanity/reference/ref007.py @@ -2,7 +2,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker @@ -14,6 +14,7 @@ class REF007(RecordReferenceChecker): """A checker of REF007.""" name = RuleName("sample-data-to-calibrated-sensor") + severity = Severity.ERROR description = "'SampleData.calibrated_sensor_token' refers to 'CalibratedSensor' record." source = SchemaName.SAMPLE_DATA target = SchemaName.CALIBRATED_SENSOR diff --git a/t4_devkit/sanity/reference/ref008.py b/t4_devkit/sanity/reference/ref008.py index da3e969..dccec5c 100644 --- a/t4_devkit/sanity/reference/ref008.py +++ b/t4_devkit/sanity/reference/ref008.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker - __all__ = ["REF008"] @@ -15,6 +14,7 @@ class REF008(RecordReferenceChecker): """A checker of REF008.""" name = RuleName("calibrated-sensor-to-sensor") + severity = Severity.ERROR description = "'CalibratedSensor.sensor_token' refers to 'Sensor' record." source = SchemaName.CALIBRATED_SENSOR target = SchemaName.SENSOR diff --git a/t4_devkit/sanity/reference/ref009.py b/t4_devkit/sanity/reference/ref009.py index cb1218f..193ac78 100644 --- a/t4_devkit/sanity/reference/ref009.py +++ b/t4_devkit/sanity/reference/ref009.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker - __all__ = ["REF009"] @@ -15,6 +14,7 @@ class REF009(RecordReferenceChecker): """A checker of REF009.""" name = RuleName("instance-to-category") + severity = Severity.ERROR description = "'Instance.category_token' refers to 'Category' record." source = SchemaName.INSTANCE target = SchemaName.CATEGORY diff --git a/t4_devkit/sanity/reference/ref010.py b/t4_devkit/sanity/reference/ref010.py index 7ce58cb..613ee7a 100644 --- a/t4_devkit/sanity/reference/ref010.py +++ b/t4_devkit/sanity/reference/ref010.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker - __all__ = ["REF010"] @@ -15,6 +14,7 @@ class REF010(RecordReferenceChecker): """A checker of REF010.""" name = RuleName("instance-to-first-sample-annotation") + severity = Severity.ERROR description = "'Instance.first_annotation_token' refers to 'SampleAnnotation' record." source = SchemaName.INSTANCE target = SchemaName.SAMPLE_ANNOTATION diff --git a/t4_devkit/sanity/reference/ref011.py b/t4_devkit/sanity/reference/ref011.py index a374a35..caac4c7 100644 --- a/t4_devkit/sanity/reference/ref011.py +++ b/t4_devkit/sanity/reference/ref011.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker - __all__ = ["REF011"] @@ -15,6 +14,7 @@ class REF011(RecordReferenceChecker): """A checker of REF011.""" name = RuleName("instance-to-last-sample-annotation") + severity = Severity.ERROR description = "'Instance.last_annotation_token' refers to 'SampleAnnotation' record." source = SchemaName.INSTANCE target = SchemaName.SAMPLE_ANNOTATION diff --git a/t4_devkit/sanity/reference/ref012.py b/t4_devkit/sanity/reference/ref012.py index f786993..5e317e2 100644 --- a/t4_devkit/sanity/reference/ref012.py +++ b/t4_devkit/sanity/reference/ref012.py @@ -2,11 +2,10 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from .base import RecordReferenceChecker - __all__ = ["REF012"] @@ -15,6 +14,7 @@ class REF012(RecordReferenceChecker): """A checker of REF012.""" name = RuleName("lidarset-to-sample-data") + severity = Severity.ERROR description = "'LidarSeg.sample_data_token' refers to 'SampleData' record." source = SchemaName.LIDARSEG target = SchemaName.SAMPLE_DATA diff --git a/t4_devkit/sanity/reference/ref013.py b/t4_devkit/sanity/reference/ref013.py index dd5b2e1..b5e8dcb 100644 --- a/t4_devkit/sanity/reference/ref013.py +++ b/t4_devkit/sanity/reference/ref013.py @@ -4,7 +4,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason from ..safety import load_json_safe @@ -22,6 +22,7 @@ class REF013(FileReferenceChecker): """A checker of REF013.""" name = RuleName("sample-data-filename-presence") + severity = Severity.ERROR description = "'SampleData.filename' exists." schema = SchemaName.SAMPLE_DATA diff --git a/t4_devkit/sanity/reference/ref014.py b/t4_devkit/sanity/reference/ref014.py index 227e7f1..e4e6a66 100644 --- a/t4_devkit/sanity/reference/ref014.py +++ b/t4_devkit/sanity/reference/ref014.py @@ -4,7 +4,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason from ..safety import load_json_safe @@ -22,6 +22,7 @@ class REF014(FileReferenceChecker): """A checker of REF014.""" name = RuleName("sample-data-filename-presence") + severity = Severity.ERROR description = "'SampleData.filename' exists." schema = SchemaName.SAMPLE_DATA diff --git a/t4_devkit/sanity/reference/ref015.py b/t4_devkit/sanity/reference/ref015.py index 2bfaf9b..4bf3591 100644 --- a/t4_devkit/sanity/reference/ref015.py +++ b/t4_devkit/sanity/reference/ref015.py @@ -4,7 +4,7 @@ from t4_devkit.schema import SchemaName -from ..checker import RuleID, RuleName +from ..checker import RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason from ..safety import load_json_safe @@ -22,6 +22,7 @@ class REF015(FileReferenceChecker): """A checker of REF015.""" name = RuleName("lidarseg-filename-presence") + severity = Severity.ERROR description = "'LidarSeg.filename' exists." schema = SchemaName.LIDARSEG diff --git a/t4_devkit/sanity/result.py b/t4_devkit/sanity/result.py index e29d0b1..4f4e09f 100644 --- a/t4_devkit/sanity/result.py +++ b/t4_devkit/sanity/result.py @@ -8,31 +8,20 @@ from typing_extensions import Self if TYPE_CHECKING: - from .checker import RuleID, RuleName + from .checker import RuleID, RuleName, Severity from .context import SanityContext __all__ = ["Status", "Report", "SanityResult", "print_sanity_result"] class Status(str, Enum): - """Status of a report.""" + """Runtime outcome per checker.""" SUCCESS = "SUCCESS" + WARNING = "WARNING" FAILURE = "FAILURE" SKIPPED = "SKIPPED" - def is_success(self) -> bool: - """Check if the status is success.""" - return self == Status.SUCCESS - - def is_failure(self) -> bool: - """Check if the status is failure.""" - return self == Status.FAILURE - - def is_skipped(self) -> bool: - """Check if the status is skipped.""" - return self == Status.SKIPPED - Reason = NewType("Reason", str) @@ -44,6 +33,7 @@ class Report: Attributes: id (RuleID): The ID of the rule. name (RuleName): The name of the rule. + severity (Severity): The severity of the rule. description (str): The description of the rule. status (Status): The status of the report. reasons (list[Reason] | None): The list of reasons for the report if the report is a failure or skipped. @@ -51,6 +41,7 @@ class Report: id: RuleID name: RuleName + severity: Severity description: str status: Status reasons: list[Reason] | None = field(default=None) @@ -62,28 +53,58 @@ def __attrs_post_init__(self) -> None: assert self.reasons is not None, "Failure report must have reasons" def is_success(self) -> bool: + """Check if the status is success.""" return self.status == Status.SUCCESS + def is_warning(self) -> bool: + """Check if the status is warning.""" + return self.status == Status.WARNING + def is_failure(self) -> bool: + """Check if the status is failure.""" return self.status == Status.FAILURE def is_skipped(self) -> bool: + """Check if the status is skipped.""" return self.status == Status.SKIPPED -def make_success(id: RuleID, name: RuleName, description: str) -> Report: +def make_success(id: RuleID, name: RuleName, severity: Severity, description: str) -> Report: """Make a success report for the given rule.""" - return Report(id, name, description, Status.SUCCESS) - - -def make_skipped(id: RuleID, name: RuleName, description: str, reason: Reason) -> Report: + return Report(id, name, severity, description, Status.SUCCESS) + + +def make_warning( + id: RuleID, + name: RuleName, + severity: Severity, + description: str, + reasons: list[Reason], +) -> Report: + """Make a warning report for the given rule.""" + return Report(id, name, severity, description, Status.WARNING, reasons) + + +def make_skipped( + id: RuleID, + name: RuleName, + severity: Severity, + description: str, + reason: Reason, +) -> Report: """Make a skipped report for the given rule.""" - return Report(id, name, description, Status.SKIPPED, [reason]) + return Report(id, name, severity, description, Status.SKIPPED, [reason]) -def make_failure(id: RuleID, name: RuleName, description: str, reasons: list[Reason]) -> Report: +def make_failure( + id: RuleID, + name: RuleName, + severity: Severity, + description: str, + reasons: list[Reason], +) -> Report: """Make a failure report for the given rule.""" - return Report(id, name, description, Status.FAILURE, reasons) + return Report(id, name, severity, description, Status.FAILURE, reasons) @define @@ -124,10 +145,14 @@ def __repr__(self) -> str: string += f"\033[31m {report.id}:\033[0m\n" for reason in report.reasons or []: string += f"\033[31m - {reason}\033[0m\n" - elif report.is_skipped(): + elif report.is_warning(): string += f"\033[33m {report.id}:\033[0m\n" for reason in report.reasons or []: string += f"\033[33m - {reason}\033[0m\n" + elif report.is_skipped(): + string += f"\033[36m {report.id}: [SKIPPED]\033[0m\n" + for reason in report.reasons or []: + string += f"\033[36m - {reason}\033[0m\n" else: string += f"\033[32m {report.id}: ✅\033[0m\n" return string @@ -144,6 +169,7 @@ def print_sanity_result(result: SanityResult) -> None: # print summary result success = sum(1 for rp in result.reports if rp.is_success()) + warnings = sum(1 for rp in result.reports if rp.is_warning()) failures = sum(1 for rp in result.reports if rp.is_failure()) skips = sum(1 for rp in result.reports if rp.is_skipped()) summary_rows = [ @@ -153,6 +179,7 @@ def print_sanity_result(result: SanityResult) -> None: "\033[31mFAILURE\033[0m" if failures > 0 else "\033[32mSUCCESS\033[0m", len(result.reports), success, + warnings, failures, skips, ] @@ -161,7 +188,16 @@ def print_sanity_result(result: SanityResult) -> None: print( tabulate( summary_rows, - headers=["DatasetID", "Version", "Status", "Rules", "Success", "Failures", "Skips"], + headers=[ + "DatasetID", + "Version", + "Status", + "Rules", + "Success", + "Warnings", + "Failures", + "Skips", + ], tablefmt="pretty", ), ) diff --git a/t4_devkit/sanity/structure/str001.py b/t4_devkit/sanity/structure/str001.py index 7a0dbf3..f8d3d2a 100644 --- a/t4_devkit/sanity/structure/str001.py +++ b/t4_devkit/sanity/structure/str001.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName +from ..checker import Checker, RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason @@ -20,6 +20,7 @@ class STR001(Checker): """A checker of STR001.""" name = RuleName("version-dir-presence") + severity = Severity.WARNING description = "'version/' directory exists under the dataset root directory." def check(self, context: SanityContext) -> list[Reason]: diff --git a/t4_devkit/sanity/structure/str002.py b/t4_devkit/sanity/structure/str002.py index 356f13a..b3b8548 100644 --- a/t4_devkit/sanity/structure/str002.py +++ b/t4_devkit/sanity/structure/str002.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName +from ..checker import Checker, RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason @@ -20,6 +20,7 @@ class STR002(Checker): """A checker of STR002.""" name = RuleName("annotation-dir-presence") + severity = Severity.ERROR description = "'annotation/' directory exists under the dataset root directory." def check(self, context: SanityContext) -> list[Reason]: diff --git a/t4_devkit/sanity/structure/str003.py b/t4_devkit/sanity/structure/str003.py index b3a012f..83a5e0f 100644 --- a/t4_devkit/sanity/structure/str003.py +++ b/t4_devkit/sanity/structure/str003.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName +from ..checker import Checker, RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason @@ -20,6 +20,7 @@ class STR003(Checker): """A checker of STR003.""" name = RuleName("data-dir-presence") + severity = Severity.ERROR description = "'data/' directory exists under the dataset root directory." def check(self, context: SanityContext) -> list[Reason]: diff --git a/t4_devkit/sanity/structure/str004.py b/t4_devkit/sanity/structure/str004.py index b13c9f9..efd2c2c 100644 --- a/t4_devkit/sanity/structure/str004.py +++ b/t4_devkit/sanity/structure/str004.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName +from ..checker import Checker, RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason @@ -20,6 +20,7 @@ class STR004(Checker): """A checker of STR004.""" name = RuleName("map-dir-presence") + severity = Severity.WARNING description = "'map/' directory exists under the dataset root directory." def check(self, context: SanityContext) -> list[Reason]: diff --git a/t4_devkit/sanity/structure/str005.py b/t4_devkit/sanity/structure/str005.py index 6bd43e8..63d561e 100644 --- a/t4_devkit/sanity/structure/str005.py +++ b/t4_devkit/sanity/structure/str005.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName +from ..checker import Checker, RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason @@ -20,6 +20,7 @@ class STR005(Checker): """A checker of STR005.""" name = RuleName("bag-dir-presence") + severity = Severity.WARNING description = "'input_bag/' directory exists under the dataset root directory." def check(self, context: SanityContext) -> list[Reason]: diff --git a/t4_devkit/sanity/structure/str006.py b/t4_devkit/sanity/structure/str006.py index 992e94a..5a9a928 100644 --- a/t4_devkit/sanity/structure/str006.py +++ b/t4_devkit/sanity/structure/str006.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName +from ..checker import Checker, RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason @@ -20,6 +20,7 @@ class STR006(Checker): """A checker of STR006.""" name = RuleName("status-json-presence") + severity = Severity.WARNING description = "'status.json' file exists under the dataset root directory." def check(self, context: SanityContext) -> list[Reason]: diff --git a/t4_devkit/sanity/structure/str007.py b/t4_devkit/sanity/structure/str007.py index fac9bbf..b7727b7 100644 --- a/t4_devkit/sanity/structure/str007.py +++ b/t4_devkit/sanity/structure/str007.py @@ -6,7 +6,7 @@ from t4_devkit.schema import SchemaName -from ..checker import Checker, RuleID, RuleName +from ..checker import Checker, RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason @@ -22,6 +22,7 @@ class STR007(Checker): """A checker of STR007.""" name = RuleName("schema-file-presence") + severity = Severity.ERROR description = "Mandatory schema JSON files exist under the `annotation/` directory." def check(self, context: SanityContext) -> list[Reason]: diff --git a/t4_devkit/sanity/structure/str008.py b/t4_devkit/sanity/structure/str008.py index 196fa78..3980fb1 100644 --- a/t4_devkit/sanity/structure/str008.py +++ b/t4_devkit/sanity/structure/str008.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName +from ..checker import Checker, RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason @@ -19,6 +19,7 @@ class STR008(Checker): """A checker of STR008.""" name = RuleName("lanelet-file-presence") + severity = Severity.WARNING description = "'lanelet2_map.osm' file exists under the 'map/' directory." def check(self, context: SanityContext) -> list[Reason]: diff --git a/t4_devkit/sanity/structure/str009.py b/t4_devkit/sanity/structure/str009.py index df23c9c..bbde87f 100644 --- a/t4_devkit/sanity/structure/str009.py +++ b/t4_devkit/sanity/structure/str009.py @@ -4,7 +4,7 @@ from returns.maybe import Some -from ..checker import Checker, RuleID, RuleName +from ..checker import Checker, RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason @@ -19,6 +19,7 @@ class STR009(Checker): """A checker of STR009.""" name = RuleName("pointcloud-map-dir-presence") + severity = Severity.WARNING description = "'pointcloud_map.pcd' directory exists under the 'map/' directory." def check(self, context: SanityContext) -> list[Reason]: diff --git a/t4_devkit/sanity/tier4/tiv001.py b/t4_devkit/sanity/tier4/tiv001.py index 0ee376b..daddf45 100644 --- a/t4_devkit/sanity/tier4/tiv001.py +++ b/t4_devkit/sanity/tier4/tiv001.py @@ -2,15 +2,16 @@ from typing import TYPE_CHECKING +from returns.maybe import Maybe, Nothing, Some from returns.pipeline import is_successful from returns.result import Result, safe -from returns.maybe import Maybe, Nothing, Some -from ..checker import Checker, RuleID, RuleName -from ..result import Reason -from ..registry import CHECKERS from t4_devkit import Tier4 +from ..checker import Checker, RuleID, RuleName, Severity +from ..registry import CHECKERS +from ..result import Reason + if TYPE_CHECKING: from ..context import SanityContext @@ -22,6 +23,7 @@ class TIV001(Checker): """A checker for TIV001.""" name = RuleName("load-tier4") + severity = Severity.ERROR description = "Ensure 'Tier4' instance is loaded successfully." def can_skip(self, context: SanityContext) -> Maybe[Reason]: From c99fbf7c408ca7de38e111081b5fa8b2b3436fe4 Mon Sep 17 00:00:00 2001 From: Kotaro Uetake <60615504+ktro2828@users.noreply.github.com> Date: Tue, 11 Nov 2025 22:24:40 +0900 Subject: [PATCH 2/6] Update t4_devkit/sanity/result.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- t4_devkit/sanity/result.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t4_devkit/sanity/result.py b/t4_devkit/sanity/result.py index 4f4e09f..783c3f0 100644 --- a/t4_devkit/sanity/result.py +++ b/t4_devkit/sanity/result.py @@ -50,7 +50,7 @@ def __attrs_post_init__(self) -> None: if self.is_success(): assert self.reasons is None, "Success report cannot have reasons" else: - assert self.reasons is not None, "Failure report must have reasons" + assert self.reasons is not None, "Non-success report must have reasons" def is_success(self) -> bool: """Check if the status is success.""" From cebfaedf03c6afeb7b0689a12603802545027a9c Mon Sep 17 00:00:00 2001 From: ktro2828 Date: Wed, 12 Nov 2025 13:25:53 +0900 Subject: [PATCH 3/6] feat: remove warning from status Signed-off-by: ktro2828 --- docs/cli/t4sanity.md | 14 ++++++------- t4_devkit/sanity/checker.py | 18 +++++++++------- t4_devkit/sanity/result.py | 41 +++++++------------------------------ 3 files changed, 25 insertions(+), 48 deletions(-) diff --git a/docs/cli/t4sanity.md b/docs/cli/t4sanity.md index 87a7640..4ee44b1 100644 --- a/docs/cli/t4sanity.md +++ b/docs/cli/t4sanity.md @@ -58,11 +58,11 @@ $ t4sanity STR008: ✅ ... -+-----------+---------+---------+-------+---------+----------+----------+-------+ -| DatasetID | Version | Status | Rules | Success | Warnings | Failures | Skips | -+-----------+---------+---------+-------+---------+----------+----------+-------+ -| dataset1 | | SUCCESS | 49 | 43 | 4 | 0 | 2 | -+-----------+---------+---------+-------+---------+----------+----------+-------+ ++-----------+---------+---------+-------+---------+----------+-------+----------+ +| DatasetID | Version | Status | Rules | Success | Failures | Skips | Warnings | ++-----------+---------+---------+-------+---------+----------+-------+----------+ +| dataset1 | | SUCCESS | 49 | 43 | 0 | 2 | 4 | ++-----------+---------+---------+-------+---------+----------+-------+----------+ ``` ### Dump Results as JSON @@ -83,9 +83,9 @@ Then a JSON file named `result.json` will be generated as follows: { "id": "", "name": "", - "severity": "", + "severity": "", "description": "", - "status": "", + "status": "", "reasons": "<[, , ...]: [str; N] | null>" // Failure or skipped reasons, null if success }, ] diff --git a/t4_devkit/sanity/checker.py b/t4_devkit/sanity/checker.py index 7208311..7236273 100644 --- a/t4_devkit/sanity/checker.py +++ b/t4_devkit/sanity/checker.py @@ -6,7 +6,7 @@ from returns.maybe import Maybe, Nothing, Some -from .result import make_failure, make_skipped, make_success, make_warning +from .result import make_failure, make_skipped, make_success if TYPE_CHECKING: from .context import SanityContext @@ -20,8 +20,16 @@ class Severity(str, Enum): """Severity levels for sanity checkers.""" - ERROR = "ERROR" WARNING = "WARNING" + ERROR = "ERROR" + + def is_warning(self) -> bool: + """Return `True` if the severity is WARNING.""" + return self == Severity.WARNING + + def is_error(self) -> bool: + """Return `True` if the severity is ERROR.""" + return self == Severity.ERROR class Checker(ABC): @@ -41,11 +49,7 @@ def __call__(self, context: SanityContext) -> Report: reasons = self.check(context) if reasons: - return ( - make_failure(self.id, self.name, self.severity, self.description, reasons) - if self.severity == Severity.ERROR - else make_warning(self.id, self.name, self.severity, self.description, reasons) - ) + return make_failure(self.id, self.name, self.severity, self.description, reasons) else: return make_success(self.id, self.name, self.severity, self.description) diff --git a/t4_devkit/sanity/result.py b/t4_devkit/sanity/result.py index 783c3f0..8fe3c0e 100644 --- a/t4_devkit/sanity/result.py +++ b/t4_devkit/sanity/result.py @@ -18,7 +18,6 @@ class Status(str, Enum): """Runtime outcome per checker.""" SUCCESS = "SUCCESS" - WARNING = "WARNING" FAILURE = "FAILURE" SKIPPED = "SKIPPED" @@ -56,10 +55,6 @@ def is_success(self) -> bool: """Check if the status is success.""" return self.status == Status.SUCCESS - def is_warning(self) -> bool: - """Check if the status is warning.""" - return self.status == Status.WARNING - def is_failure(self) -> bool: """Check if the status is failure.""" return self.status == Status.FAILURE @@ -74,34 +69,15 @@ def make_success(id: RuleID, name: RuleName, severity: Severity, description: st return Report(id, name, severity, description, Status.SUCCESS) -def make_warning( - id: RuleID, - name: RuleName, - severity: Severity, - description: str, - reasons: list[Reason], -) -> Report: - """Make a warning report for the given rule.""" - return Report(id, name, severity, description, Status.WARNING, reasons) - - def make_skipped( - id: RuleID, - name: RuleName, - severity: Severity, - description: str, - reason: Reason, + id: RuleID, name: RuleName, severity: Severity, description: str, reason: Reason ) -> Report: """Make a skipped report for the given rule.""" return Report(id, name, severity, description, Status.SKIPPED, [reason]) def make_failure( - id: RuleID, - name: RuleName, - severity: Severity, - description: str, - reasons: list[Reason], + id: RuleID, name: RuleName, severity: Severity, description: str, reasons: list[Reason] ) -> Report: """Make a failure report for the given rule.""" return Report(id, name, severity, description, Status.FAILURE, reasons) @@ -145,10 +121,6 @@ def __repr__(self) -> str: string += f"\033[31m {report.id}:\033[0m\n" for reason in report.reasons or []: string += f"\033[31m - {reason}\033[0m\n" - elif report.is_warning(): - string += f"\033[33m {report.id}:\033[0m\n" - for reason in report.reasons or []: - string += f"\033[33m - {reason}\033[0m\n" elif report.is_skipped(): string += f"\033[36m {report.id}: [SKIPPED]\033[0m\n" for reason in report.reasons or []: @@ -169,9 +141,10 @@ def print_sanity_result(result: SanityResult) -> None: # print summary result success = sum(1 for rp in result.reports if rp.is_success()) - warnings = sum(1 for rp in result.reports if rp.is_warning()) - failures = sum(1 for rp in result.reports if rp.is_failure()) + failures = sum(1 for rp in result.reports if rp.severity.is_error() and rp.is_failure()) skips = sum(1 for rp in result.reports if rp.is_skipped()) + warnings = sum(1 for rp in result.reports if rp.severity.is_warning() and rp.is_failure()) + summary_rows = [ [ result.dataset_id, @@ -179,9 +152,9 @@ def print_sanity_result(result: SanityResult) -> None: "\033[31mFAILURE\033[0m" if failures > 0 else "\033[32mSUCCESS\033[0m", len(result.reports), success, - warnings, failures, skips, + warnings, ] ] @@ -194,9 +167,9 @@ def print_sanity_result(result: SanityResult) -> None: "Status", "Rules", "Success", - "Warnings", "Failures", "Skips", + "Warnings", ], tablefmt="pretty", ), From 29f3e11aa7967ee3d0b17acf47bb17e51519dbaa Mon Sep 17 00:00:00 2001 From: ktro2828 Date: Wed, 12 Nov 2025 15:07:17 +0900 Subject: [PATCH 4/6] feat: add strict option Signed-off-by: ktro2828 --- docs/cli/t4sanity.md | 18 ++++++++++-- t4_devkit/cli/sanity.py | 4 +++ t4_devkit/sanity/checker.py | 19 ++++++++---- t4_devkit/sanity/context.py | 5 ++-- t4_devkit/sanity/format/base.py | 8 +++--- t4_devkit/sanity/record/base.py | 6 ++-- t4_devkit/sanity/record/rec001.py | 4 +-- t4_devkit/sanity/record/rec002.py | 4 +-- t4_devkit/sanity/record/rec003.py | 4 +-- t4_devkit/sanity/record/rec004.py | 4 +-- t4_devkit/sanity/record/rec005.py | 4 +-- t4_devkit/sanity/record/rec006.py | 4 +-- t4_devkit/sanity/reference/base.py | 4 +-- t4_devkit/sanity/reference/ref013.py | 4 +-- t4_devkit/sanity/reference/ref014.py | 4 +-- t4_devkit/sanity/reference/ref015.py | 4 +-- t4_devkit/sanity/result.py | 43 ++++++++++++++++++---------- t4_devkit/sanity/run.py | 4 ++- t4_devkit/sanity/structure/str001.py | 4 +-- t4_devkit/sanity/structure/str002.py | 4 +-- t4_devkit/sanity/structure/str003.py | 4 +-- t4_devkit/sanity/structure/str004.py | 4 +-- t4_devkit/sanity/structure/str005.py | 6 ++-- t4_devkit/sanity/structure/str006.py | 4 +-- t4_devkit/sanity/structure/str007.py | 4 +-- t4_devkit/sanity/structure/str008.py | 4 +-- t4_devkit/sanity/structure/str009.py | 4 +-- t4_devkit/sanity/tier4/tiv001.py | 4 +-- 28 files changed, 115 insertions(+), 74 deletions(-) diff --git a/docs/cli/t4sanity.md b/docs/cli/t4sanity.md index 4ee44b1..79c2613 100644 --- a/docs/cli/t4sanity.md +++ b/docs/cli/t4sanity.md @@ -10,10 +10,11 @@ $ t4sanity -h ╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Options ─────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ │ --version -v Show the application version and exit. │ -│ --output -o TEXT Path to output JSON file. │ -│ --revision -rv TEXT Specify if you want to check the specific version. │ -│ --exclude -e TEXT Exclude specific rules or rule groups. │ +│ --output -o TEXT Path to output JSON file. [default: None] │ +│ --revision -rv TEXT Specify if you want to check the specific version. [default: None] │ +│ --exclude -e TEXT Exclude specific rules or rule groups. [default: None] │ │ --include-warning -iw Indicates whether to report any warnings. │ +│ --strict -s Indicates whether warnings are treated as failures. │ │ --install-completion Install completion for the current shell. │ │ --show-completion Show completion for the current shell, to copy it or customize the installation. │ │ --help -h Show this message and exit. │ @@ -100,3 +101,14 @@ With `-e; --excludes` option enables us to exclude specific checks by specifying # Exclude STR001 and all FMT-relevant rules t4sanity -e STR001 -e FMT ``` + +### Strict Mode + +Basically, rules whose **severity is WARNING** will be treated as success. + +With `-s; --strict` option enables us to treat warnings as failures: + +```shell +# Run strict mode +t4sanity -s +``` diff --git a/t4_devkit/cli/sanity.py b/t4_devkit/cli/sanity.py index 63847c5..5b558d4 100644 --- a/t4_devkit/cli/sanity.py +++ b/t4_devkit/cli/sanity.py @@ -37,12 +37,16 @@ def main( include_warning: bool = typer.Option( False, "-iw", "--include-warning", help="Indicates whether to report any warnings." ), + strict: bool = typer.Option( + False, "-s", "--strict", help="Indicates whether warnings are treated as failures." + ), ) -> None: result = sanity_check( data_root=data_root, revision=revision, excludes=excludes, include_warning=include_warning, + strict=strict, ) print_sanity_result(result) diff --git a/t4_devkit/sanity/checker.py b/t4_devkit/sanity/checker.py index 7236273..8d06b2d 100644 --- a/t4_devkit/sanity/checker.py +++ b/t4_devkit/sanity/checker.py @@ -6,7 +6,7 @@ from returns.maybe import Maybe, Nothing, Some -from .result import make_failure, make_skipped, make_success +from .result import make_report, make_skipped if TYPE_CHECKING: from .context import SanityContext @@ -48,15 +48,22 @@ def __call__(self, context: SanityContext) -> Report: return make_skipped(self.id, self.name, self.severity, self.description, skip) reasons = self.check(context) - if reasons: - return make_failure(self.id, self.name, self.severity, self.description, reasons) - else: - return make_success(self.id, self.name, self.severity, self.description) + return make_report( + self.id, self.name, self.severity, self.description, reasons, strict=context.strict + ) def can_skip(self, _: SanityContext) -> Maybe[Reason]: """Return a skip reason if the checker should be skipped.""" return Nothing @abstractmethod - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: + """Return a list of reasons if the checker fails, or None if it passes. + + Args: + context (SanityContext): The sanity context. + + Returns: + A list of reasons if the checker fails, or None if it passes. + """ pass diff --git a/t4_devkit/sanity/context.py b/t4_devkit/sanity/context.py index 2ba355f..64a843d 100644 --- a/t4_devkit/sanity/context.py +++ b/t4_devkit/sanity/context.py @@ -15,12 +15,13 @@ @define class SanityContext: metadata: Maybe[DBMetadata] + strict: bool @classmethod - def from_path(cls, data_root: str, revision: str | None = None) -> Self: + def from_path(cls, data_root: str, revision: str | None = None, strict: bool = False) -> Self: metadata_result = _load_metadata_safe(data_root, revision=revision) metadata = metadata_result.unwrap() if is_successful(metadata_result) else None - return cls(Maybe.from_optional(metadata)) + return cls(Maybe.from_optional(metadata), strict) @property def data_root(self) -> Maybe[Path]: diff --git a/t4_devkit/sanity/format/base.py b/t4_devkit/sanity/format/base.py index ae31276..39f2b57 100644 --- a/t4_devkit/sanity/format/base.py +++ b/t4_devkit/sanity/format/base.py @@ -37,24 +37,24 @@ def can_skip(self, context: SanityContext) -> Maybe[Reason]: case _: return Nothing - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: filepath = context.to_schema_file(self.schema).unwrap() if self.schema.is_optional() and not filepath.exists(): - return [] + return None records = load_json_safe(filepath) return _build_records(self.schema, records.unwrap()) -def _build_records(schema: SchemaName, records: list[dict]) -> list[Reason]: +def _build_records(schema: SchemaName, records: list[dict]) -> list[Reason] | None: module = SCHEMAS.get(schema) failures = [] for record in records: conversion = _safe_from_dict(module, record) if not is_successful(conversion): failures.append(Reason(f"[{schema.name}] {record['token']}: {conversion.failure()}")) - return failures + return failures if failures else None @safe diff --git a/t4_devkit/sanity/record/base.py b/t4_devkit/sanity/record/base.py index 7b9b505..162f7f5 100644 --- a/t4_devkit/sanity/record/base.py +++ b/t4_devkit/sanity/record/base.py @@ -37,19 +37,19 @@ def can_skip(self, context: SanityContext) -> Maybe[Reason]: case _: return Maybe.from_value(Reason("Missing 'annotation' directory path")) - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: filepath = context.to_schema_file(self.schema).unwrap() records = load_json_safe(filepath).unwrap() return self.check_count(records) @abstractmethod - def check_count(self, records: list[dict]) -> list[Reason]: + def check_count(self, records: list[dict]) -> list[Reason] | None: """Check the count of records. Args: records (list[dict]): The list of records to check. Returns: - A list of reasons for any issues found, otherwise an empty list. + A list of reasons for any issues found, otherwise None. """ pass diff --git a/t4_devkit/sanity/record/rec001.py b/t4_devkit/sanity/record/rec001.py index e01e485..01c7927 100644 --- a/t4_devkit/sanity/record/rec001.py +++ b/t4_devkit/sanity/record/rec001.py @@ -19,10 +19,10 @@ class REC001(RecordCountChecker): description = "'Scene' record is a single." schema = SchemaName.SCENE - def check_count(self, records: list[dict]) -> list[Reason]: + def check_count(self, records: list[dict]) -> list[Reason] | None: num_scene = len(records) return ( - [] + None if num_scene == 1 else [Reason(f"'Scene' must contain exactly one element: {num_scene}")] ) diff --git a/t4_devkit/sanity/record/rec002.py b/t4_devkit/sanity/record/rec002.py index cea4656..86d2e9f 100644 --- a/t4_devkit/sanity/record/rec002.py +++ b/t4_devkit/sanity/record/rec002.py @@ -19,6 +19,6 @@ class REC002(RecordCountChecker): description = "'Sample' record is not empty." schema = SchemaName.SAMPLE - def check_count(self, records: list[dict]) -> list[Reason]: + def check_count(self, records: list[dict]) -> list[Reason] | None: num_sample = len(records) - return [Reason("'Sample' record must not be empty")] if num_sample == 0 else [] + return [Reason("'Sample' record must not be empty")] if num_sample == 0 else None diff --git a/t4_devkit/sanity/record/rec003.py b/t4_devkit/sanity/record/rec003.py index 0d110dd..d4f2d98 100644 --- a/t4_devkit/sanity/record/rec003.py +++ b/t4_devkit/sanity/record/rec003.py @@ -19,6 +19,6 @@ class REC003(RecordCountChecker): description = "'SampleData' record is not empty." schema = SchemaName.SAMPLE_DATA - def check_count(self, records: list[dict]) -> list[Reason]: + def check_count(self, records: list[dict]) -> list[Reason] | None: num_sample_data = len(records) - return [Reason("'SampleData' record must not be empty")] if num_sample_data == 0 else [] + return [Reason("'SampleData' record must not be empty")] if num_sample_data == 0 else None diff --git a/t4_devkit/sanity/record/rec004.py b/t4_devkit/sanity/record/rec004.py index 840e813..a9a12ea 100644 --- a/t4_devkit/sanity/record/rec004.py +++ b/t4_devkit/sanity/record/rec004.py @@ -19,6 +19,6 @@ class REC004(RecordCountChecker): description = "'EgoPose' record is not empty." schema = SchemaName.EGO_POSE - def check_count(self, records: list[dict]) -> list[Reason]: + def check_count(self, records: list[dict]) -> list[Reason] | None: num_ego_pose = len(records) - return [Reason("'EgoPose' record must not be empty")] if num_ego_pose == 0 else [] + return [Reason("'EgoPose' record must not be empty")] if num_ego_pose == 0 else None diff --git a/t4_devkit/sanity/record/rec005.py b/t4_devkit/sanity/record/rec005.py index c5bf39c..b4cd15a 100644 --- a/t4_devkit/sanity/record/rec005.py +++ b/t4_devkit/sanity/record/rec005.py @@ -19,10 +19,10 @@ class REC005(RecordCountChecker): description = "'CalibratedSensor' record is not empty." schema = SchemaName.CALIBRATED_SENSOR - def check_count(self, records: list[dict]) -> list[Reason]: + def check_count(self, records: list[dict]) -> list[Reason] | None: num_calibrated_sensor = len(records) return ( [Reason("'CalibratedSensor' record must not be empty")] if num_calibrated_sensor == 0 - else [] + else None ) diff --git a/t4_devkit/sanity/record/rec006.py b/t4_devkit/sanity/record/rec006.py index e6f1f11..8a251af 100644 --- a/t4_devkit/sanity/record/rec006.py +++ b/t4_devkit/sanity/record/rec006.py @@ -19,6 +19,6 @@ class REC006(RecordCountChecker): description = "'Instance' record is not empty." schema = SchemaName.INSTANCE - def check_count(self, records: list[dict]) -> list[Reason]: + def check_count(self, records: list[dict]) -> list[Reason] | None: num_instance = len(records) - return [Reason("'Instance' record must not be empty")] if num_instance == 0 else [] + return [Reason("'Instance' record must not be empty")] if num_instance == 0 else None diff --git a/t4_devkit/sanity/reference/base.py b/t4_devkit/sanity/reference/base.py index eb8b420..04ba8a3 100644 --- a/t4_devkit/sanity/reference/base.py +++ b/t4_devkit/sanity/reference/base.py @@ -44,7 +44,7 @@ def can_skip(self, context: SanityContext) -> Maybe[Reason]: case _: return Maybe.from_value(Reason("Missing 'annotation' directory path")) - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: source_file = context.to_schema_file(self.source).unwrap() target_file = context.to_schema_file(self.target).unwrap() source_records = load_json_safe(source_file).unwrap() @@ -56,7 +56,7 @@ def check(self, context: SanityContext) -> list[Reason]: for record in source_records if record[self.reference] not in target_tokens and self.is_additional_condition_ok(record) - ] + ] or None def is_additional_condition_ok(self, record: dict[str, Any]) -> bool: """Return True if the additional condition is met. diff --git a/t4_devkit/sanity/reference/ref013.py b/t4_devkit/sanity/reference/ref013.py index b5e8dcb..6adf0d0 100644 --- a/t4_devkit/sanity/reference/ref013.py +++ b/t4_devkit/sanity/reference/ref013.py @@ -26,7 +26,7 @@ class REF013(FileReferenceChecker): description = "'SampleData.filename' exists." schema = SchemaName.SAMPLE_DATA - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: filepath = context.to_schema_file(self.schema).unwrap() records = load_json_safe(filepath).unwrap() data_root = context.data_root.unwrap() @@ -34,4 +34,4 @@ def check(self, context: SanityContext) -> list[Reason]: Reason(f"File not found: {record['filename']}") for record in records if not data_root.joinpath(record["filename"]).exists() - ] + ] or None diff --git a/t4_devkit/sanity/reference/ref014.py b/t4_devkit/sanity/reference/ref014.py index e4e6a66..09d07ca 100644 --- a/t4_devkit/sanity/reference/ref014.py +++ b/t4_devkit/sanity/reference/ref014.py @@ -26,7 +26,7 @@ class REF014(FileReferenceChecker): description = "'SampleData.filename' exists." schema = SchemaName.SAMPLE_DATA - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: filepath = context.to_schema_file(self.schema).unwrap() records = load_json_safe(filepath).unwrap() data_root = context.data_root.unwrap() @@ -35,4 +35,4 @@ def check(self, context: SanityContext) -> list[Reason]: for record in records if record.get("info_filename") is not None and not data_root.joinpath(record["info_filename"]).exists() - ] + ] or None diff --git a/t4_devkit/sanity/reference/ref015.py b/t4_devkit/sanity/reference/ref015.py index 4bf3591..c12313b 100644 --- a/t4_devkit/sanity/reference/ref015.py +++ b/t4_devkit/sanity/reference/ref015.py @@ -26,7 +26,7 @@ class REF015(FileReferenceChecker): description = "'LidarSeg.filename' exists." schema = SchemaName.LIDARSEG - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: filepath = context.to_schema_file(self.schema).unwrap() records = load_json_safe(filepath).unwrap() data_root = context.data_root.unwrap() @@ -34,4 +34,4 @@ def check(self, context: SanityContext) -> list[Reason]: Reason(f"File not found: {record['filename']}") for record in records if not data_root.joinpath(record["filename"]).exists() - ] + ] or None diff --git a/t4_devkit/sanity/result.py b/t4_devkit/sanity/result.py index 8fe3c0e..d01b43c 100644 --- a/t4_devkit/sanity/result.py +++ b/t4_devkit/sanity/result.py @@ -46,9 +46,9 @@ class Report: reasons: list[Reason] | None = field(default=None) def __attrs_post_init__(self) -> None: - if self.is_success(): - assert self.reasons is None, "Success report cannot have reasons" - else: + if self.is_success() and self.severity.is_error(): + assert self.reasons is None, "Success report for error rule cannot have reasons" + elif self.is_failure(): assert self.reasons is not None, "Non-success report must have reasons" def is_success(self) -> bool: @@ -64,9 +64,27 @@ def is_skipped(self) -> bool: return self.status == Status.SKIPPED -def make_success(id: RuleID, name: RuleName, severity: Severity, description: str) -> Report: - """Make a success report for the given rule.""" - return Report(id, name, severity, description, Status.SUCCESS) +def make_report( + id: RuleID, + name: RuleName, + severity: Severity, + description: str, + reasons: list[Reason] | None = None, + *, + strict: bool = False, +) -> Report: + """Make a report for the given rule.""" + if reasons: + if severity.is_warning(): + return ( + Report(id, name, severity, description, Status.FAILURE, reasons) + if strict + else Report(id, name, severity, description, Status.SUCCESS, reasons) + ) + else: + return Report(id, name, severity, description, Status.FAILURE, reasons) + else: + return Report(id, name, severity, description, Status.SUCCESS) def make_skipped( @@ -76,13 +94,6 @@ def make_skipped( return Report(id, name, severity, description, Status.SKIPPED, [reason]) -def make_failure( - id: RuleID, name: RuleName, severity: Severity, description: str, reasons: list[Reason] -) -> Report: - """Make a failure report for the given rule.""" - return Report(id, name, severity, description, Status.FAILURE, reasons) - - @define class SanityResult: """The result of a Sanity check. @@ -141,9 +152,11 @@ def print_sanity_result(result: SanityResult) -> None: # print summary result success = sum(1 for rp in result.reports if rp.is_success()) - failures = sum(1 for rp in result.reports if rp.severity.is_error() and rp.is_failure()) + failures = sum(1 for rp in result.reports if rp.is_failure()) skips = sum(1 for rp in result.reports if rp.is_skipped()) - warnings = sum(1 for rp in result.reports if rp.severity.is_warning() and rp.is_failure()) + + # just count the number of warnings + warnings = sum(1 for rp in result.reports if rp.severity.is_warning() and rp.reasons) summary_rows = [ [ diff --git a/t4_devkit/sanity/run.py b/t4_devkit/sanity/run.py index cd62ab9..3d75f8f 100644 --- a/t4_devkit/sanity/run.py +++ b/t4_devkit/sanity/run.py @@ -16,6 +16,7 @@ def sanity_check( *, include_warning: bool = False, excludes: Sequence[str] | None = None, + strict: bool = False, ) -> SanityResult: """Run sanity checks on the given data root. @@ -24,6 +25,7 @@ def sanity_check( revision (str | None, optional): The revision to check. If None, the latest revision is used. include_warning (bool, optional): Whether to include warning checks. excludes (Sequence[str] | None, optional): A list of rule names or groups to exclude. + strict (bool, optional): Indicate whether checkers whose severity is set to "WARNING" should fail. Returns: A SanityResult object. @@ -34,7 +36,7 @@ def sanity_check( else: warnings.simplefilter("ignore") - context = SanityContext.from_path(data_root, revision=revision) + context = SanityContext.from_path(data_root, revision=revision, strict=strict) checkers = CHECKERS.build(excludes=excludes) reports = [checker(context) for checker in checkers] diff --git a/t4_devkit/sanity/structure/str001.py b/t4_devkit/sanity/structure/str001.py index f8d3d2a..c3d0a0b 100644 --- a/t4_devkit/sanity/structure/str001.py +++ b/t4_devkit/sanity/structure/str001.py @@ -23,9 +23,9 @@ class STR001(Checker): severity = Severity.WARNING description = "'version/' directory exists under the dataset root directory." - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: match context.version: case Some(_): - return [] + return None case _: return [Reason("'version' directory doesn't exist")] diff --git a/t4_devkit/sanity/structure/str002.py b/t4_devkit/sanity/structure/str002.py index b3b8548..baee3cc 100644 --- a/t4_devkit/sanity/structure/str002.py +++ b/t4_devkit/sanity/structure/str002.py @@ -23,11 +23,11 @@ class STR002(Checker): severity = Severity.ERROR description = "'annotation/' directory exists under the dataset root directory." - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: match context.annotation_dir: case Some(x): return ( - [] + None if x.exists() else [Reason(f"Path to 'annotation' not found: {x.as_posix()}")] ) diff --git a/t4_devkit/sanity/structure/str003.py b/t4_devkit/sanity/structure/str003.py index 83a5e0f..bc95188 100644 --- a/t4_devkit/sanity/structure/str003.py +++ b/t4_devkit/sanity/structure/str003.py @@ -23,11 +23,11 @@ class STR003(Checker): severity = Severity.ERROR description = "'data/' directory exists under the dataset root directory." - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: match context.sensor_data_dir: case Some(x): return ( - [] + None if x.exists() else [Reason(f"Path to 'data' directory not found: {x.as_posix()}")] ) diff --git a/t4_devkit/sanity/structure/str004.py b/t4_devkit/sanity/structure/str004.py index efd2c2c..5c4944a 100644 --- a/t4_devkit/sanity/structure/str004.py +++ b/t4_devkit/sanity/structure/str004.py @@ -23,9 +23,9 @@ class STR004(Checker): severity = Severity.WARNING description = "'map/' directory exists under the dataset root directory." - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: match context.map_dir: case Some(x): - return [] if x.exists() else [Reason(f"Path to 'map' not found: {x.as_posix()}")] + return None if x.exists() else [Reason(f"Path to 'map' not found: {x.as_posix()}")] case _: return [Reason("dataset directory doesn't contain 'map' directory")] diff --git a/t4_devkit/sanity/structure/str005.py b/t4_devkit/sanity/structure/str005.py index 63d561e..7860a9d 100644 --- a/t4_devkit/sanity/structure/str005.py +++ b/t4_devkit/sanity/structure/str005.py @@ -23,11 +23,13 @@ class STR005(Checker): severity = Severity.WARNING description = "'input_bag/' directory exists under the dataset root directory." - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: match context.bag_dir: case Some(x): return ( - [] if x.exists() else [Reason(f"Path to 'input_bag' not found: {x.as_posix()}")] + None + if x.exists() + else [Reason(f"Path to 'input_bag' not found: {x.as_posix()}")] ) case _: return [Reason("dataset directory doesn't contain 'input_bag' directory")] diff --git a/t4_devkit/sanity/structure/str006.py b/t4_devkit/sanity/structure/str006.py index 5a9a928..09da99b 100644 --- a/t4_devkit/sanity/structure/str006.py +++ b/t4_devkit/sanity/structure/str006.py @@ -23,11 +23,11 @@ class STR006(Checker): severity = Severity.WARNING description = "'status.json' file exists under the dataset root directory." - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: match context.status_json: case Some(x): return ( - [] + None if x.exists() else [Reason(f"Path to 'status.json' not found: {x.as_posix()}")] ) diff --git a/t4_devkit/sanity/structure/str007.py b/t4_devkit/sanity/structure/str007.py index b7727b7..0819559 100644 --- a/t4_devkit/sanity/structure/str007.py +++ b/t4_devkit/sanity/structure/str007.py @@ -25,7 +25,7 @@ class STR007(Checker): severity = Severity.ERROR description = "Mandatory schema JSON files exist under the `annotation/` directory." - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: failures = [] for schema in SchemaName: match context.to_schema_file(schema): @@ -34,4 +34,4 @@ def check(self, context: SanityContext) -> list[Reason]: failures.append(Reason(f"schema file '{schema.filename}' not found")) case _: failures.append(Reason(f"schema file '{schema.filename}' not found")) - return failures + return failures if failures else None diff --git a/t4_devkit/sanity/structure/str008.py b/t4_devkit/sanity/structure/str008.py index 3980fb1..d7f25d8 100644 --- a/t4_devkit/sanity/structure/str008.py +++ b/t4_devkit/sanity/structure/str008.py @@ -22,7 +22,7 @@ class STR008(Checker): severity = Severity.WARNING description = "'lanelet2_map.osm' file exists under the 'map/' directory." - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: match context.map_dir: case Some(x): if not x.exists(): @@ -31,7 +31,7 @@ def check(self, context: SanityContext) -> list[Reason]: return ( [Reason(f"Lanelet2 map file not found: {lanelet_file.as_posix()}")] if not lanelet_file.exists() - else [] + else None ) case _: return [Reason("dataset directory doesn't contain 'map' directory")] diff --git a/t4_devkit/sanity/structure/str009.py b/t4_devkit/sanity/structure/str009.py index bbde87f..6f97220 100644 --- a/t4_devkit/sanity/structure/str009.py +++ b/t4_devkit/sanity/structure/str009.py @@ -22,7 +22,7 @@ class STR009(Checker): severity = Severity.WARNING description = "'pointcloud_map.pcd' directory exists under the 'map/' directory." - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: match context.map_dir: case Some(x): if not x.exists(): @@ -31,7 +31,7 @@ def check(self, context: SanityContext) -> list[Reason]: return ( [Reason(f"PCD map directory not found: {pointcloud_map_dir.as_posix()}")] if not pointcloud_map_dir.exists() - else [] + else None ) case _: return [Reason("dataset directory doesn't contain 'map' directory")] diff --git a/t4_devkit/sanity/tier4/tiv001.py b/t4_devkit/sanity/tier4/tiv001.py index daddf45..e9174dd 100644 --- a/t4_devkit/sanity/tier4/tiv001.py +++ b/t4_devkit/sanity/tier4/tiv001.py @@ -35,10 +35,10 @@ def can_skip(self, context: SanityContext) -> Maybe[Reason]: case _: return Nothing - def check(self, context: SanityContext) -> list[Reason]: + def check(self, context: SanityContext) -> list[Reason] | None: result = _load_tier4_safe(context) return ( - [] if is_successful(result) else [Reason(f"Failed to load Tier4: {result.failure()}")] + None if is_successful(result) else [Reason(f"Failed to load Tier4: {result.failure()}")] ) From 3f9a1d387dc211f4aa2269b0ee3d0a800920444a Mon Sep 17 00:00:00 2001 From: ktro2828 Date: Wed, 12 Nov 2025 19:03:23 +0900 Subject: [PATCH 5/6] feat: always set Status.FAILURE if the checker with Severity.WARNING failed, but their reports are treated as sucuess unless strict=True Signed-off-by: ktro2828 --- t4_devkit/cli/sanity.py | 10 ++++-- t4_devkit/sanity/checker.py | 4 +-- t4_devkit/sanity/context.py | 5 ++- t4_devkit/sanity/result.py | 70 ++++++++++++++++++++++++------------- t4_devkit/sanity/run.py | 4 +-- 5 files changed, 58 insertions(+), 35 deletions(-) diff --git a/t4_devkit/cli/sanity.py b/t4_devkit/cli/sanity.py index 5b558d4..8682d93 100644 --- a/t4_devkit/cli/sanity.py +++ b/t4_devkit/cli/sanity.py @@ -1,5 +1,7 @@ from __future__ import annotations +import sys + import typer from t4_devkit.common.io import save_json @@ -46,11 +48,15 @@ def main( revision=revision, excludes=excludes, include_warning=include_warning, - strict=strict, ) - print_sanity_result(result) + print_sanity_result(result, strict=strict) if output: serialized = serialize_dataclass(result) save_json(serialized, output) + + if result.is_success(strict=strict): + sys.exit(0) + else: + sys.exit(1) diff --git a/t4_devkit/sanity/checker.py b/t4_devkit/sanity/checker.py index 8d06b2d..a43fe2a 100644 --- a/t4_devkit/sanity/checker.py +++ b/t4_devkit/sanity/checker.py @@ -48,9 +48,7 @@ def __call__(self, context: SanityContext) -> Report: return make_skipped(self.id, self.name, self.severity, self.description, skip) reasons = self.check(context) - return make_report( - self.id, self.name, self.severity, self.description, reasons, strict=context.strict - ) + return make_report(self.id, self.name, self.severity, self.description, reasons) def can_skip(self, _: SanityContext) -> Maybe[Reason]: """Return a skip reason if the checker should be skipped.""" diff --git a/t4_devkit/sanity/context.py b/t4_devkit/sanity/context.py index 64a843d..2ba355f 100644 --- a/t4_devkit/sanity/context.py +++ b/t4_devkit/sanity/context.py @@ -15,13 +15,12 @@ @define class SanityContext: metadata: Maybe[DBMetadata] - strict: bool @classmethod - def from_path(cls, data_root: str, revision: str | None = None, strict: bool = False) -> Self: + def from_path(cls, data_root: str, revision: str | None = None) -> Self: metadata_result = _load_metadata_safe(data_root, revision=revision) metadata = metadata_result.unwrap() if is_successful(metadata_result) else None - return cls(Maybe.from_optional(metadata), strict) + return cls(Maybe.from_optional(metadata)) @property def data_root(self) -> Maybe[Path]: diff --git a/t4_devkit/sanity/result.py b/t4_devkit/sanity/result.py index d01b43c..7fbe42c 100644 --- a/t4_devkit/sanity/result.py +++ b/t4_devkit/sanity/result.py @@ -46,18 +46,24 @@ class Report: reasons: list[Reason] | None = field(default=None) def __attrs_post_init__(self) -> None: - if self.is_success() and self.severity.is_error(): - assert self.reasons is None, "Success report for error rule cannot have reasons" - elif self.is_failure(): + if self.status == Status.SUCCESS: + assert self.reasons is None, "Success report cannot have reasons" + else: assert self.reasons is not None, "Non-success report must have reasons" - def is_success(self) -> bool: + def is_success(self, *, strict: bool = False) -> bool: """Check if the status is success.""" - return self.status == Status.SUCCESS + return ( + self.status == Status.SUCCESS + or self.is_skipped() + or (not strict and self.severity.is_warning()) + ) - def is_failure(self) -> bool: + def is_failure(self, *, strict: bool = False) -> bool: """Check if the status is failure.""" - return self.status == Status.FAILURE + return (self.status == Status.FAILURE and self.severity.is_error()) or not ( + self.is_success(strict=strict) or self.is_skipped() + ) def is_skipped(self) -> bool: """Check if the status is skipped.""" @@ -70,19 +76,10 @@ def make_report( severity: Severity, description: str, reasons: list[Reason] | None = None, - *, - strict: bool = False, ) -> Report: """Make a report for the given rule.""" if reasons: - if severity.is_warning(): - return ( - Report(id, name, severity, description, Status.FAILURE, reasons) - if strict - else Report(id, name, severity, description, Status.SUCCESS, reasons) - ) - else: - return Report(id, name, severity, description, Status.FAILURE, reasons) + return Report(id, name, severity, description, Status.FAILURE, reasons) else: return Report(id, name, severity, description, Status.SUCCESS) @@ -125,10 +122,29 @@ def from_context(cls, context: SanityContext, reports: list[Report]) -> Self: reports=reports, ) - def __repr__(self) -> str: + def is_success(self, *, strict: bool = False) -> bool: + """Return True if all reports are successful, False otherwise. + + Args: + strict (bool): Whether to consider warnings as failures. + + Returns: + True if all reports are successful, False otherwise. + """ + return all(report.is_success(strict=strict) for report in self.reports) + + def to_str(self, *, strict: bool = False) -> str: + """Return a string representation of the result. + + Args: + strict (bool): Whether to consider warnings as failures. + + Returns: + A string representation of the result. + """ string = f"=== DatasetID: {self.dataset_id} ===\n" for report in self.reports: - if report.is_failure(): + if not report.is_success(strict=strict): string += f"\033[31m {report.id}:\033[0m\n" for reason in report.reasons or []: string += f"\033[31m - {reason}\033[0m\n" @@ -136,23 +152,27 @@ def __repr__(self) -> str: string += f"\033[36m {report.id}: [SKIPPED]\033[0m\n" for reason in report.reasons or []: string += f"\033[36m - {reason}\033[0m\n" + elif report.severity.is_warning() and report.reasons: + string += f"\033[33m {report.id}:\033[0m\n" + for reason in report.reasons or []: + string += f"\033[33m - {reason}\033[0m\n" else: string += f"\033[32m {report.id}: ✅\033[0m\n" return string -def print_sanity_result(result: SanityResult) -> None: +def print_sanity_result(result: SanityResult, *, strict: bool = False) -> None: """Print detailed and summary results of a sanity check. Args: result (SanityResult): The result of a sanity check. """ # print detailed result - print(result) + print(result.to_str(strict=strict)) # print summary result - success = sum(1 for rp in result.reports if rp.is_success()) - failures = sum(1 for rp in result.reports if rp.is_failure()) + success = sum(1 for rp in result.reports if rp.is_success(strict=strict)) + failures = sum(1 for rp in result.reports if not rp.is_success(strict=strict)) skips = sum(1 for rp in result.reports if rp.is_skipped()) # just count the number of warnings @@ -162,7 +182,9 @@ def print_sanity_result(result: SanityResult) -> None: [ result.dataset_id, result.version, - "\033[31mFAILURE\033[0m" if failures > 0 else "\033[32mSUCCESS\033[0m", + "\033[32mSUCCESS\033[0m" + if result.is_success(strict=strict) + else "\033[31mFAILURE\033[0m", len(result.reports), success, failures, diff --git a/t4_devkit/sanity/run.py b/t4_devkit/sanity/run.py index 3d75f8f..cd62ab9 100644 --- a/t4_devkit/sanity/run.py +++ b/t4_devkit/sanity/run.py @@ -16,7 +16,6 @@ def sanity_check( *, include_warning: bool = False, excludes: Sequence[str] | None = None, - strict: bool = False, ) -> SanityResult: """Run sanity checks on the given data root. @@ -25,7 +24,6 @@ def sanity_check( revision (str | None, optional): The revision to check. If None, the latest revision is used. include_warning (bool, optional): Whether to include warning checks. excludes (Sequence[str] | None, optional): A list of rule names or groups to exclude. - strict (bool, optional): Indicate whether checkers whose severity is set to "WARNING" should fail. Returns: A SanityResult object. @@ -36,7 +34,7 @@ def sanity_check( else: warnings.simplefilter("ignore") - context = SanityContext.from_path(data_root, revision=revision, strict=strict) + context = SanityContext.from_path(data_root, revision=revision) checkers = CHECKERS.build(excludes=excludes) reports = [checker(context) for checker in checkers] From 88e45ce6ecbf76bac86b18f2d323f090ee7b48a1 Mon Sep 17 00:00:00 2001 From: ktro2828 Date: Fri, 14 Nov 2025 12:09:29 +0900 Subject: [PATCH 6/6] ci: update CLI test Signed-off-by: ktro2828 --- .github/workflows/build-and-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index f640895..d33e49b 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -52,7 +52,7 @@ jobs: uv run t4viz scene ./tests/sample/t4dataset -o ./output uv run t4viz instance ./tests/sample/t4dataset 90f0c98d1a040d5360847f576c5528f8 -o ./output uv run t4viz pointcloud ./tests/sample/t4dataset -o ./output - uv run t4sanity ./tests/sample -iw + uv run t4sanity ./tests/sample/t4dataset - name: Get test coverage if: ${{ steps.is-changed.outputs.changes == 'true'}} && ${{ matrix.python-version == '3.10' }}