Skip to content

Commit 92c8d5b

Browse files
ktro2828ktro2828
authored
feat: add a validator to check impossible empty tokens (#241)
Signed-off-by: ktro2828 <kotaro.uetake@tier4.jp>
1 parent c8ab6fb commit 92c8d5b

13 files changed

Lines changed: 52 additions & 42 deletions

File tree

‎docs/schema/table.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -158,8 +158,8 @@ instance {
158158
"category_token": <str> -- Foreign key to the `Category` table.
159159
"instance_name": <str> -- Consists of the dataset name and the instance ID separated by colons `::`, such as `<DATASET_ID>::<INSTANCE_ID>`.
160160
"nbr_annotations": <int> -- Number of annotations associated with this instance.
161-
"first_annotation_token": <str> -- Foreign key to the first `SampleAnnotation` table associated with this instance.
162-
"last_annotation_token": <str> -- Foreign key to the last `SampleAnnotation` table associated with this instance.
161+
"first_annotation_token": <str> -- Foreign key to the first `SampleAnnotation` table associated with this instance. Empty string `""` if the dataset only contains 2D annotations.
162+
"last_annotation_token": <str> -- Foreign key to the last `SampleAnnotation` table associated with this instance. Empty string `""` if the dataset only contains 2D annotations.
163163
}
164164
```
165165

@@ -245,7 +245,7 @@ If the `is_key_frame=True`, the timestamp should be very close to the associated
245245
```json
246246
sample_data {
247247
"token": <str> -- Unique record identifier.
248-
"sample_token": <str> -- Foreign key to the `Sample` table.
248+
"sample_token": <str> -- Foreign key to the `Sample` table. Empty string `""` if this is not a key frame.
249249
"ego_pose_token": <str> -- Foreign key to the `EgoPose` table.
250250
"calibrated_sensor_token": <str> -- Foreign key to the `CalibratedSensor` table.
251251
"filename": <str> -- Relative path from a dataset root directory to the sensor data file.

‎t4_devkit/schema/tables/base.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
from abc import ABC
44
from secrets import token_hex
5-
from typing import Any, TypeVar
5+
from typing import Any, Sized, TypeVar
66

77
from attrs import define, field, validators
88

@@ -11,11 +11,17 @@
1111
__all__ = ["SchemaBase", "SchemaTable"]
1212

1313

14+
def impossible_empty(instance, attribute, value: Sized) -> None:
15+
"""A validator that raises ValueError if value is empty."""
16+
if len(value) == 0:
17+
raise ValueError(f"{attribute.name} cannot be empty")
18+
19+
1420
@define
1521
class SchemaBase(ABC):
1622
"""Abstract base dataclass of schema tables."""
1723

18-
token: str = field(validator=validators.instance_of(str))
24+
token: str = field(validator=(validators.instance_of(str), impossible_empty))
1925

2026
@classmethod
2127
def from_json(cls, filepath: str) -> list[SchemaTable]:

‎t4_devkit/schema/tables/calibrated_sensor.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
from t4_devkit.typing import CameraDistortion, CameraIntrinsic, Quaternion, Vector3
77

88
from ..name import SchemaName
9-
from .base import SchemaBase
9+
from .base import SchemaBase, impossible_empty
1010
from .registry import SCHEMAS
1111

1212
__all__ = ["CalibratedSensor"]
@@ -26,7 +26,7 @@ class CalibratedSensor(SchemaBase):
2626
camera_distortion (CameraDistortion): Camera distortion array. Empty for sensors that are not cameras.
2727
"""
2828

29-
sensor_token: str = field(validator=validators.instance_of(str))
29+
sensor_token: str = field(validator=(validators.instance_of(str), impossible_empty))
3030
translation: Vector3 = field(converter=Vector3)
3131
rotation: Quaternion = field(converter=to_quaternion)
3232
camera_intrinsic: CameraIntrinsic = field(converter=CameraIntrinsic)

‎t4_devkit/schema/tables/instance.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
from attrs import define, field, validators
44

55
from ..name import SchemaName
6-
from .base import SchemaBase
6+
from .base import SchemaBase, impossible_empty
77
from .registry import SCHEMAS
88

99
__all__ = ["Instance"]
@@ -20,10 +20,12 @@ class Instance(SchemaBase):
2020
instance_name (str): Dataset name and instance ID defined in annotation tool.
2121
nbr_annotations (int): Number of annotations of this instance.
2222
first_annotation_token (str): Foreign key pointing to the first annotation of this instance.
23+
Empty if the dataset only contains 2D annotations.
2324
last_annotation_token (str): Foreign key pointing to the last annotation of this instance.
25+
Empty if the dataset only contains 2D annotations.
2426
"""
2527

26-
category_token: str = field(validator=validators.instance_of(str))
28+
category_token: str = field(validator=(validators.instance_of(str), impossible_empty))
2729
instance_name: str = field(validator=validators.instance_of(str))
2830
nbr_annotations: int = field(validator=validators.instance_of(int))
2931
first_annotation_token: str = field(validator=validators.instance_of(str))

‎t4_devkit/schema/tables/keypoint.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
from attrs import define, field, validators
77

88
from ..name import SchemaName
9-
from .base import SchemaBase
9+
from .base import SchemaBase, impossible_empty
1010
from .registry import SCHEMAS
1111

1212
if TYPE_CHECKING:
@@ -29,10 +29,10 @@ class Keypoint(SchemaBase):
2929
num_keypoints (int): The number of keypoints to be annotated.
3030
"""
3131

32-
sample_data_token: str = field(validator=validators.instance_of(str))
33-
instance_token: str = field(validator=validators.instance_of(str))
32+
sample_data_token: str = field(validator=(validators.instance_of(str), impossible_empty))
33+
instance_token: str = field(validator=(validators.instance_of(str), impossible_empty))
3434
category_tokens: list[str] = field(
35-
validator=validators.deep_iterable(validators.instance_of(str))
35+
validator=validators.deep_iterable((validators.instance_of(str), impossible_empty))
3636
)
3737
keypoints: KeypointLike = field(converter=np.array)
3838
num_keypoints: int = field(validator=validators.instance_of(int))

‎t4_devkit/schema/tables/lidarseg.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
from attrs import define, field, validators
44

55
from ..name import SchemaName
6-
from .base import SchemaBase
6+
from .base import SchemaBase, impossible_empty
77
from .registry import SCHEMAS
88

99
__all__ = ["LidarSeg"]
@@ -21,4 +21,4 @@ class LidarSeg(SchemaBase):
2121
"""
2222

2323
filename: str = field(validator=validators.instance_of(str))
24-
sample_data_token: str = field(validator=validators.instance_of(str))
24+
sample_data_token: str = field(validator=(validators.instance_of(str), impossible_empty))

‎t4_devkit/schema/tables/map.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
from attrs import define, field, validators
44

55
from ..name import SchemaName
6-
from .base import SchemaBase
6+
from .base import SchemaBase, impossible_empty
77
from .registry import SCHEMAS
88

99
__all__ = ["Map"]
@@ -21,6 +21,8 @@ class Map(SchemaBase):
2121
filename (str): Relative path to the file with the map mask.
2222
"""
2323

24-
log_tokens: list[str] = field(validator=validators.deep_iterable(validators.instance_of(str)))
24+
log_tokens: list[str] = field(
25+
validator=validators.deep_iterable((validators.instance_of(str), impossible_empty))
26+
)
2527
category: str = field(validator=validators.instance_of(str))
2628
filename: str = field(validator=validators.instance_of(str))

‎t4_devkit/schema/tables/object_ann.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
from ..name import SchemaName
1212
from .autolabel_metadata import AutolabelMixin
13-
from .base import SchemaBase
13+
from .base import SchemaBase, impossible_empty
1414
from .registry import SCHEMAS
1515

1616
if TYPE_CHECKING:
@@ -76,11 +76,11 @@ class ObjectAnn(SchemaBase, AutolabelMixin):
7676
category_name (str): Category name. This should be set after instantiated.
7777
"""
7878

79-
sample_data_token: str = field(validator=validators.instance_of(str))
80-
instance_token: str = field(validator=validators.instance_of(str))
81-
category_token: str = field(validator=validators.instance_of(str))
79+
sample_data_token: str = field(validator=(validators.instance_of(str), impossible_empty))
80+
instance_token: str = field(validator=(validators.instance_of(str), impossible_empty))
81+
category_token: str = field(validator=(validators.instance_of(str), impossible_empty))
8282
attribute_tokens: list[str] = field(
83-
validator=validators.deep_iterable(validators.instance_of(str))
83+
validator=validators.deep_iterable((validators.instance_of(str), impossible_empty))
8484
)
8585
bbox: Roi = field(converter=Roi)
8686
mask: RLEMask | None = field(

‎t4_devkit/schema/tables/sample.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
from attrs import define, field, validators
44

55
from ..name import SchemaName
6-
from .base import SchemaBase
6+
from .base import SchemaBase, impossible_empty
77
from .registry import SCHEMAS
88

99
__all__ = ["Sample"]
@@ -34,8 +34,8 @@ class Sample(SchemaBase):
3434
"""
3535

3636
timestamp: int = field(validator=validators.instance_of(int))
37-
scene_token: str = field(validator=validators.instance_of(str))
38-
next: str = field(validator=validators.instance_of(str)) # noqa: A003
37+
scene_token: str = field(validator=(validators.instance_of(str), impossible_empty))
38+
next: str = field(validator=validators.instance_of(str))
3939
prev: str = field(validator=validators.instance_of(str))
4040

4141
# shortcuts

‎t4_devkit/schema/tables/sample_annotation.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77

88
from ..name import SchemaName
99
from .autolabel_metadata import AutolabelMixin
10-
from .base import SchemaBase
10+
from .base import SchemaBase, impossible_empty
1111
from .registry import SCHEMAS
1212

1313
__all__ = ["SampleAnnotation"]
@@ -48,18 +48,18 @@ class SampleAnnotation(SchemaBase, AutolabelMixin):
4848
category_name (str): Category name. This should be set after instantiated.
4949
"""
5050

51-
sample_token: str = field(validator=validators.instance_of(str))
52-
instance_token: str = field(validator=validators.instance_of(str))
51+
sample_token: str = field(validator=(validators.instance_of(str), impossible_empty))
52+
instance_token: str = field(validator=(validators.instance_of(str), impossible_empty))
5353
attribute_tokens: list[str] = field(
54-
validator=validators.deep_iterable(validators.instance_of(str))
54+
validator=validators.deep_iterable((validators.instance_of(str), impossible_empty))
5555
)
56-
visibility_token: str = field(validator=validators.instance_of(str))
56+
visibility_token: str = field(validator=(validators.instance_of(str), impossible_empty))
5757
translation: Vector3 = field(converter=Vector3)
5858
size: Vector3 = field(converter=Vector3)
5959
rotation: Quaternion = field(converter=to_quaternion)
6060
num_lidar_pts: int = field(validator=validators.instance_of(int))
6161
num_radar_pts: int = field(validator=validators.instance_of(int))
62-
next: str = field(validator=validators.instance_of(str)) # noqa: A003
62+
next: str = field(validator=validators.instance_of(str))
6363
prev: str = field(validator=validators.instance_of(str))
6464
velocity: Vector3 | None = field(default=None, converter=converters.optional(Vector3))
6565
acceleration: Vector3 | None = field(default=None, converter=converters.optional(Vector3))

0 commit comments

Comments
 (0)