diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 330251fe..d8efdb4d 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -81,6 +81,9 @@ jobs: source .venv/bin/activate ruff check --config ruff.toml hololinked/client ruff check --config ruff.toml hololinked/serializers + ruff check --config ruff.toml hololinked/schema_validators + ruff check --config ruff.toml hololinked/serialization.py + ruff check --config ruff.toml hololinked/schemas.py - name: run ty type checker if: matrix.tool == 'ty' @@ -88,6 +91,9 @@ jobs: source .venv/bin/activate ty check hololinked/client ty check hololinked/serializers + ty check hololinked/schema_validators + ty check hololinked/serialization.py + ty check hololinked/schemas.py scan: name: security scan (${{ matrix.tool }}) diff --git a/hololinked/__init__.py b/hololinked/__init__.py index a9711135..d67847d4 100644 --- a/hololinked/__init__.py +++ b/hololinked/__init__.py @@ -3,5 +3,9 @@ __version__ = "0.4.0" from .config import global_config # noqa -import hololinked.core # noqa: F401 +from .serialization import Serializers as Serializers +from .schemas import JSONSchema as JSONSchema, SchemaValidatorClasses as SchemaValidatorClasses + +import hololinked.core # noqa: F401 # this one is lazy for most part import hololinked.serializers # noqa: F401 +import hololinked.schema_validators # noqa: F401 diff --git a/hololinked/client/factory.py b/hololinked/client/factory.py index d15698bc..bdb03e38 100644 --- a/hololinked/client/factory.py +++ b/hololinked/client/factory.py @@ -13,6 +13,7 @@ from paho.mqtt.client import CallbackAPIVersion, MQTTMessage, MQTTProtocolVersion from paho.mqtt.client import Client as PahoMQTTClient +from hololinked import Serializers from hololinked.client.abstractions import ( ConsumedThingAction, ConsumedThingEvent, @@ -25,7 +26,7 @@ OAuthDirectAccessGrant, ) from hololinked.constants import ZMQ_TRANSPORTS -from hololinked.core import Serializers, Thing +from hololinked.core import Thing from hololinked.td.interaction_affordance import ( ActionAffordance, EventAffordance, diff --git a/hololinked/client/http/consumed_interactions.py b/hololinked/client/http/consumed_interactions.py index bf55ccdf..1bda4de0 100644 --- a/hololinked/client/http/consumed_interactions.py +++ b/hololinked/client/http/consumed_interactions.py @@ -12,6 +12,7 @@ import httpx import structlog +from hololinked import Serializers from hololinked.client.abstractions import ( SSE, ConsumedThingAction, @@ -20,7 +21,6 @@ ) from hololinked.client.exceptions import raise_local_exception from hololinked.constants import Operations -from hololinked.core import Serializers from hololinked.td.forms import Form from hololinked.td.interaction_affordance import ( ActionAffordance, diff --git a/hololinked/client/mqtt/consumed_interactions.py b/hololinked/client/mqtt/consumed_interactions.py index 1b75070c..1c90d181 100644 --- a/hololinked/client/mqtt/consumed_interactions.py +++ b/hololinked/client/mqtt/consumed_interactions.py @@ -9,8 +9,8 @@ from paho.mqtt.client import Client as PahoMQTTClient from paho.mqtt.client import MQTTMessage +from hololinked import Serializers from hololinked.client.abstractions import SSE, ConsumedThingEvent -from hololinked.core import Serializers from hololinked.core.interfaces import BaseSerializer # noqa: F401 from hololinked.td.forms import Form from hololinked.td.interaction_affordance import EventAffordance, PropertyAffordance diff --git a/hololinked/constants.py b/hololinked/constants.py index 09b1d75a..6507c7d2 100644 --- a/hololinked/constants.py +++ b/hololinked/constants.py @@ -8,6 +8,7 @@ # types JSONSerializable = typing.Union[str, int, float, bool, None, typing.Dict[str, typing.Any], typing.List] JSON = typing.Dict[str, JSONSerializable] +JSONSchema = typing.Dict[str, str | typing.Dict[str, typing.Any] | typing.List[typing.Dict[str, typing.Any]]] byte_types = (bytes, bytearray, memoryview) diff --git a/hololinked/core/__init__.py b/hololinked/core/__init__.py index 29e7fbcb..d74902bf 100644 --- a/hololinked/core/__init__.py +++ b/hololinked/core/__init__.py @@ -4,12 +4,57 @@ State machines, meta classes, descriptor registries and the concrete implementation of how an operation is executed is also included here. """ -# Order of import is reflected in this file to avoid circular imports - -from .thing import * # noqa -from .events import * # noqa -from .actions import * # noqa -from .property import * # noqa -from .state_machine import StateMachine as StateMachine -from .meta import ThingMeta as ThingMeta -from .serializer_registry import Serializers as Serializers + +from typing import TYPE_CHECKING + +# Interfaces must be available to register adappters +from .interfaces import BaseSchemaValidator as BaseSchemaValidator +from .interfaces import BaseSerializer as BaseSerializer + + +__all__ = [ + "BaseSchemaValidator", + "BaseSerializer", + "action", + "Action", + "Event", + "ThingMeta", + "Property", + "StateMachine", + "Thing", +] + +# Submodules that use SchemaValidatorClasses / Serializers are loaded lazily so +# that hololinked/__init__.py can finish registering the adapters (schema_validators, +# serializers) before any class body in meta.py / actions.py / property.py executes. +_lazy: dict[str, tuple[str, str]] = { + "action": (".actions", "action"), + "Action": (".actions", "Action"), + "Event": (".events", "Event"), + "ThingMeta": (".meta", "ThingMeta"), + "Property": (".property", "Property"), + "StateMachine": (".state_machine", "StateMachine"), + "Thing": (".thing", "Thing"), +} + + +def __getattr__(name: str): + if name in _lazy: + import importlib + + module_path, attr = _lazy[name] + mod = importlib.import_module(module_path, package=__name__) + val = getattr(mod, attr) + globals()[name] = val # cache so subsequent access skips __getattr__ + return val + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +if TYPE_CHECKING: + from .actions import Action as Action + from .actions import action as action + from .events import Event as Event + from .meta import ThingMeta as ThingMeta + from .property import Property as Property + from .state_machine import StateMachine as StateMachine + from .thing import Thing as Thing diff --git a/hololinked/core/actions.py b/hololinked/core/actions.py index a436b5c4..fc4781a9 100644 --- a/hololinked/core/actions.py +++ b/hololinked/core/actions.py @@ -9,23 +9,25 @@ from pydantic import BaseModel, RootModel -from ..constants import JSON -from ..param.parameterized import ParameterizedFunction -from ..schema_validators.validators import JSONSchemaValidator, PydanticSchemaValidator -from ..utils import ( +from hololinked import SchemaValidatorClasses +from hololinked.constants import JSON +from hololinked.core.exceptions import StateMachineError +from hololinked.param.parameterized import ParameterizedFunction +from hololinked.utils import ( get_input_model_from_signature, get_return_type_from_signature, has_async_def, isclassmethod, issubklass, ) + from .dataklasses import ActionInfoValidator -from .exceptions import StateMachineError class Action: """ Object that models an action. + These actions are unbound and return a bound action when accessed using the owning object. """ @@ -33,6 +35,8 @@ class Action: def __init__(self, obj: MethodType) -> None: """ + Initialize an Action. + Parameters ---------- obj: MethodType @@ -69,13 +73,13 @@ def __call__(self, *args, **kwargs): @property def name(self) -> str: - """name of the action""" + """Name of the action.""" return self.obj.__name__ @property def execution_info(self) -> ActionInfoValidator: """ - internal dataclass that holds all information about the action + Internal dataclass that holds all information about the action. TODO: this can be refactored """ @@ -107,9 +111,7 @@ def to_affordance(self, owner_inst=None): class BoundAction: - """ - A bound action - base class for both sync and async methods. - """ + """A bound action, base class for both sync and async methods.""" __slots__ = [ "obj", @@ -173,14 +175,14 @@ def validate_call(self, args, kwargs: dict[str, Any]) -> None: @property def name(self) -> str: - """name of the action""" + """Name of the action.""" return self.obj.__name__ def __call__(self, *args, **kwargs): raise NotImplementedError("call must be implemented by subclass") def external_call(self, *args, **kwargs): - """validated call to the action with state machine and payload checks""" + """Validated call to the action with state machine and payload checks.""" raise NotImplementedError("external_call must be implemented by subclass") def __str__(self): @@ -224,7 +226,7 @@ class BoundSyncAction(BoundAction): """ def external_call(self, *args, **kwargs): - """validated call to the action with state machine and payload checks""" + """Validated call to the action with state machine and payload checks""" self.validate_call(args, kwargs) return self.__call__(*args, **kwargs) @@ -241,7 +243,7 @@ class BoundAsyncAction(BoundAction): """ async def external_call(self, *args, **kwargs): - """validated call to the action with state machine and payload checks""" + """Validated call to the action with state machine and payload checks""" self.validate_call(args, kwargs) return await self.__call__(*args, **kwargs) @@ -358,9 +360,9 @@ def inner(obj): ) if input_schema: if isinstance(input_schema, dict): - execution_info_validator.schema_validator = JSONSchemaValidator(input_schema) + execution_info_validator.schema_validator = SchemaValidatorClasses.json_schema(input_schema) elif issubklass(input_schema, (BaseModel, RootModel)): - execution_info_validator.schema_validator = PydanticSchemaValidator(input_schema) + execution_info_validator.schema_validator = SchemaValidatorClasses.pydantic(input_schema) else: raise TypeError( "input schema must be a JSON schema or a Pydantic model, got {}".format(type(input_schema)) diff --git a/hololinked/core/dataklasses.py b/hololinked/core/dataklasses.py index 25805bb4..e978b841 100644 --- a/hololinked/core/dataklasses.py +++ b/hololinked/core/dataklasses.py @@ -9,11 +9,11 @@ from pydantic import BaseModel, RootModel -from ..constants import USE_OBJECT_NAME -from ..param.parameterized import ParameterizedMetaclass -from ..param.parameters import Boolean, ClassSelector, Parameter, String, Tuple -from ..schema_validators import BaseSchemaValidator -from ..utils import issubklass +from hololinked.constants import USE_OBJECT_NAME +from hololinked.core.interfaces import BaseSchemaValidator +from hololinked.param.parameterized import ParameterizedMetaclass +from hololinked.param.parameters import Boolean, ClassSelector, Parameter, String, Tuple +from hololinked.utils import issubklass # TODO, this class will be removed in future and merged directly into the corresponding object diff --git a/hololinked/core/exceptions.py b/hololinked/core/exceptions.py index 64ee09cc..4a04f3bb 100644 --- a/hololinked/core/exceptions.py +++ b/hololinked/core/exceptions.py @@ -1,38 +1,31 @@ -class BreakInnerLoop(Exception): - """raise to break an inner loop""" +"""Exception classes.""" + - pass +class BreakInnerLoop(Exception): + """Raise to break an inner loop.""" class BreakAllLoops(Exception): - """raise to exit all loops""" - - pass + """Raise to exit all loops.""" class BreakLoop(Exception): - """raise and catch to exit a loop from within another function or method""" - - pass + """Raise and catch to exit a loop from within another function or method.""" class BreakFlow(Exception): - """raised to break the flow of the program""" - - pass + """Raise to break the flow of the program.""" # TODO - remove unused and reduce number of definitions class StateMachineError(Exception): - """raise to show errors while calling actions or writing properties in wrong state""" - - pass + """Raise to show errors while calling actions or writing properties in wrong state.""" class DatabaseError(Exception): - """raise to show database related errors""" + """Raise to show database related errors.""" __all__ = ["BreakInnerLoop", "BreakAllLoops", "BreakLoop", "BreakFlow", "StateMachineError", "DatabaseError"] diff --git a/hololinked/core/interfaces/__init__.py b/hololinked/core/interfaces/__init__.py index 78756fd7..e7b4e3f6 100644 --- a/hololinked/core/interfaces/__init__.py +++ b/hololinked/core/interfaces/__init__.py @@ -5,4 +5,5 @@ """ # TODO once all items have base classes, dont use relative imports. +from .schema_validators import BaseSchemaValidator as BaseSchemaValidator from .serializer import BaseSerializer as BaseSerializer diff --git a/hololinked/core/interfaces/schema_validators.py b/hololinked/core/interfaces/schema_validators.py new file mode 100644 index 00000000..e2f151da --- /dev/null +++ b/hololinked/core/interfaces/schema_validators.py @@ -0,0 +1,32 @@ +"""Base class for all schema validators.""" + +from hololinked.constants import JSONSchema + + +class BaseSchemaValidator: + """ + Base class for all schema validators. + + Serves as a type definition. + """ + + def __init__(self, schema) -> None: + self.schema = schema + + def validate(self, data) -> None: + """Validate the data against the schema.""" + raise NotImplementedError("validate method must be implemented by subclass") + + def validate_method_call(self, args, kwargs) -> None: + """Validate the method call against the schema.""" + raise NotImplementedError("validate_method_call method must be implemented by subclass") + + def json(self) -> JSONSchema: + """Allows JSON serialization of the validator instance itself.""" + raise NotImplementedError("json method must be implemented by subclass") + + def __get_state__(self): + return self.json() + + def __set_state__(self, schema): + raise NotImplementedError("__set_state__ method must be implemented by subclass") diff --git a/hololinked/core/property.py b/hololinked/core/property.py index 7186ab1b..f9d855a3 100644 --- a/hololinked/core/property.py +++ b/hololinked/core/property.py @@ -5,8 +5,9 @@ from pydantic import BaseModel, ConfigDict, RootModel, create_model +from hololinked import SchemaValidatorClasses + from ..param.parameterized import Parameter, Parameterized, ParameterizedMetaclass -from ..schema_validators import JSONSchemaValidator from ..utils import issubklass from .dataklasses import RemoteResourceInfoValidator from .events import Event, EventDispatcher # noqa: F401 @@ -63,7 +64,6 @@ def __init__( """ Parameters ---------- - default: None or corresponding to property type The default value of the property. @@ -180,7 +180,7 @@ def __init__( if model: if isinstance(model, dict): self.model = model - self.validator = JSONSchemaValidator(model).validate + self.validator = SchemaValidatorClasses.json_schema(model).validate else: self.model = wrap_plain_types_in_rootmodel(model) # type: BaseModel self.validator = self.model.model_validate @@ -252,7 +252,7 @@ def validate_and_adapt(self, value) -> Any: def external_set(self, obj: Parameterized, value: Any) -> None: """ - method called when the value of the property is set from an external source, e.g. a remote client. + Method called when the value of the property is set from an external source, e.g. a remote client. Usually introduces a state machine check before allowing the set operation. """ if self.execution_info.state is None or ( diff --git a/hololinked/core/thing.py b/hololinked/core/thing.py index 59455e2f..9487a354 100644 --- a/hololinked/core/thing.py +++ b/hololinked/core/thing.py @@ -123,7 +123,7 @@ class attribute. Use this a minimalistic replacement for fluentd or similar log If using JSON storage, this filename is used to persist property values. If not provided, a default filename is generated based on the instance name. """ - from hololinked.core.serializer_registry import Serializers + from hololinked import Serializers from ..storage import prepare_object_storage # noqa from .logger import prepare_object_logger # noqa diff --git a/hololinked/core/zmq/brokers.py b/hololinked/core/zmq/brokers.py index 28cd8890..47c833e4 100644 --- a/hololinked/core/zmq/brokers.py +++ b/hololinked/core/zmq/brokers.py @@ -15,9 +15,9 @@ from zmq.utils.monitor import parse_monitor_message +from hololinked import Serializers from hololinked.config import global_config from hololinked.constants import ZMQ_EVENT_MAP, ZMQ_TRANSPORTS -from hololinked.core import Serializers from hololinked.core.exceptions import BreakLoop from hololinked.utils import ( format_exception_as_json, diff --git a/hololinked/core/zmq/message.py b/hololinked/core/zmq/message.py index 037b3b32..d7f8ee2d 100644 --- a/hololinked/core/zmq/message.py +++ b/hololinked/core/zmq/message.py @@ -3,8 +3,8 @@ import msgspec +from hololinked import Serializers from hololinked.constants import JSON, byte_types -from hololinked.core import Serializers from hololinked.param.parameters import Integer from .payloads import PreserializedData, SerializableData diff --git a/hololinked/core/zmq/payloads.py b/hololinked/core/zmq/payloads.py index 94064b0f..07610e86 100644 --- a/hololinked/core/zmq/payloads.py +++ b/hololinked/core/zmq/payloads.py @@ -3,8 +3,8 @@ from dataclasses import dataclass from typing import Any +from hololinked import Serializers from hololinked.constants import byte_types -from hololinked.core import Serializers from hololinked.core.interfaces import BaseSerializer diff --git a/hololinked/core/zmq/rpc_server.py b/hololinked/core/zmq/rpc_server.py index b4592426..9323a8ff 100644 --- a/hololinked/core/zmq/rpc_server.py +++ b/hololinked/core/zmq/rpc_server.py @@ -12,9 +12,9 @@ import structlog import zmq.asyncio +from hololinked import Serializers from hololinked.config import global_config from hololinked.constants import ZMQ_TRANSPORTS, Operations -from hololinked.core import Serializers from hololinked.core.interfaces import BaseSerializer from hololinked.utils import ( format_exception_as_json, diff --git a/hololinked/schema_validators/__init__.py b/hololinked/schema_validators/__init__.py index 307db4b4..beb122e4 100644 --- a/hololinked/schema_validators/__init__.py +++ b/hololinked/schema_validators/__init__.py @@ -1,2 +1,18 @@ -from .validators import BaseSchemaValidator, JSONSchemaValidator, PydanticSchemaValidator # noqa -from .json_schema import JSONSchema # noqa +""" +Validators for validating data against schemas. + +All of properties, actions and events validate can their payload or types against the schema. +For properties and actions, validation is carried out right before carrying out the operation. +For events, such validation is missing on the client. For properties and actions, said validation is also missing +on the client. This is an architectural error that needs to be fixed. The current architecture of the package leads +to duplication of code if this is implemented as is. Therefore it has been left out by choice. +""" + +from hololinked import SchemaValidatorClasses + +from .validators import JSONSchemaValidator as JSONSchemaValidator +from .validators import PydanticSchemaValidator as PydanticSchemaValidator + + +SchemaValidatorClasses.json_schema = JSONSchemaValidator +SchemaValidatorClasses.pydantic = PydanticSchemaValidator diff --git a/hololinked/schema_validators/validators.py b/hololinked/schema_validators/validators.py index b8b45aca..25688673 100644 --- a/hololinked/schema_validators/validators.py +++ b/hololinked/schema_validators/validators.py @@ -1,57 +1,63 @@ +"""Concrete implementations of schema validators.""" + import jsonschema from pydantic import BaseModel -from ..constants import JSON -from ..utils import json_schema_merge_args_to_kwargs, pydantic_validate_args_kwargs - - -class BaseSchemaValidator: # type definition - """ - Base class for all schema validators. - Serves as a type definition. - """ - - def __init__(self, schema: JSON | BaseModel) -> None: - self.schema = schema - - def validate(self, data) -> None: - """validate the data against the schema""" - raise NotImplementedError("validate method must be implemented by subclass") - - def validate_method_call(self, args, kwargs) -> None: - """validate the method call against the schema""" - raise NotImplementedError("validate_method_call method must be implemented by subclass") +from hololinked.constants import JSONSchema +from hololinked.core.interfaces import BaseSchemaValidator +from hololinked.utils import ( + json_schema_merge_args_to_kwargs, + pydantic_validate_args_kwargs, +) class JSONSchemaValidator(BaseSchemaValidator): """ - JSON schema validator according to standard python JSON schema. - Somewhat slow, consider `FastJSONSchemaValidator` (`pip install fastjsonschema`) or - pydantic annotation based validation if possible. + JSON schema validator extending the standard python JSON schema package. + + ```python + power_supply_output_schema = { + "type": "object", + "properties": { + "current": {"type": "number", "minimum": 0}, + "power": {"type": "number", "minimum": 0, "maximum": 100}, + }, + } + validator = JSONSchemaValidator(power_supply_output_schema) + validator.validate({"current": 50, "power": 75}) # valid + validator.validate({"current": 65, "power": 110}) # raises + ``` + + This class is largely used internally and there is no need to explicitly instantiate it. + + Consider `FastJSONSchemaValidator` (`pip install fastjsonschema`) or + pydantic annotation based validation for performance if necessary. """ - def __init__(self, schema) -> None: + def __init__(self, schema: JSONSchema) -> None: """ + Initialize the validator. + Parameters ---------- - schema: JSON + schema: JSONSchema The JSON schema to validate against """ jsonschema.Draft7Validator.check_schema(schema) super().__init__(schema) self.validator = jsonschema.Draft7Validator(schema) - def validate(self, data) -> None: + def validate(self, data) -> None: # noqa: D102 self.validator.validate(data) - def validate_method_call(self, args, kwargs) -> None: + def validate_method_call(self, args, kwargs) -> None: # noqa: D102 if len(args) > 0: kwargs = json_schema_merge_args_to_kwargs(self.schema, args, kwargs) + # TODO fix type definition self.validate(kwargs) - def json(self) -> JSON: - """allows JSON (de-)serializable of the instance itself""" + def json(self) -> JSONSchema: # noqa: D102 return self.schema def __get_state__(self): @@ -62,10 +68,27 @@ def __set_state__(self, schema): class PydanticSchemaValidator(BaseSchemaValidator): - """Schema validator according to pydantic models""" + """ + Pydantic model validator. + + ```python + class PowerSupplyOutput(BaseModel): + current: float = Field(..., ge=0) + power: float = Field(..., ge=0, le=100) + + validator = PydanticSchemaValidator(PowerSupplyOutput) + validator.validate({"current": 50, "power": 75}) # valid + validator.validate({"current": 65, "power": 110}) # raises + ``` + + The user is encouraged to use pydantic models as much as possible. This class is largely used internally and + there is no need to explicitly instantiate it. + """ def __init__(self, schema: BaseModel) -> None: """ + Initialize the validator. + Parameters ---------- schema: BaseModel @@ -74,20 +97,19 @@ def __init__(self, schema: BaseModel) -> None: super().__init__(schema) self.validator = schema.model_validate - def validate(self, data) -> None: + def validate(self, data) -> None: # noqa: D102 self.validator(data) - def validate_method_call(self, args, kwargs) -> None: + def validate_method_call(self, args, kwargs) -> None: # noqa: D102 pydantic_validate_args_kwargs(self.schema, args, kwargs) - def json(self) -> JSON: - """allows JSON (de-)serializable of the instance itself""" + def json(self) -> JSONSchema: # noqa: D102 return self.schema.model_dump_json() def __get_state__(self): return self.json() - def __set_state__(self, schema: JSON): + def __set_state__(self, schema: JSONSchema): return PydanticSchemaValidator(BaseModel(**schema)) @@ -95,33 +117,49 @@ def __set_state__(self, schema: JSON): import fastjsonschema class FastJSONSchemaValidator(BaseSchemaValidator): - """JSON schema validator according to fast JSON schema""" + """ + JSON schema validator according to fast JSON schema. + + `pip install fastjsonschema` to use. + + ```python + power_supply_output_schema = { + "type": "object", + "properties": { + "current": {"type": "number", "minimum": 0}, + "power": {"type": "number", "minimum": 0, "maximum": 100}, + }, + } + validator = JSONSchemaValidator(power_supply_output_schema) + validator.validate({"current": 50, "power": 75}) # valid + validator.validate({"current": 65, "power": 110}) # raises + ``` + """ # Useful for performance with dictionary based schema specification # which msgspec has no built in support. Normally, for speed, # one should try to use msgspec's struct concept. - def __init__(self, schema: JSON) -> None: + def __init__(self, schema: JSONSchema) -> None: super().__init__(schema) self.validator = fastjsonschema.compile(schema) - def validate(self, data) -> None: - """validates and raises exception when failed directly to the caller""" + def validate(self, data) -> None: # noqa: D102 self.validator(data) - def validate_method_call(self, args, kwargs) -> None: + def validate_method_call(self, args, kwargs) -> None: # noqa: D102 if len(args) > 0: kwargs = json_schema_merge_args_to_kwargs(self.schema, args, kwargs) + # TODO fix type definition self.validate(kwargs) - def json(self) -> JSON: - """allows JSON (de-)serializable of the instance itself""" + def json(self) -> JSONSchema: # noqa: D102 return self.schema def __get_state__(self): return self.schema - def __set_state__(self, schema): + def __set_state__(self, schema: JSONSchema): return FastJSONSchemaValidator(schema) except ImportError: diff --git a/hololinked/schema_validators/json_schema.py b/hololinked/schemas.py similarity index 67% rename from hololinked/schema_validators/json_schema.py rename to hololinked/schemas.py index 2a4d3709..a65687bb 100644 --- a/hololinked/schema_validators/json_schema.py +++ b/hololinked/schemas.py @@ -1,12 +1,28 @@ +"""JSON Schema type management.""" + +from __future__ import annotations + from typing import Any -from ..constants import JSON +from hololinked.constants import JSON +from hololinked.core.interfaces import BaseSchemaValidator class JSONSchema: """ - Type management object for converting python types to JSON schema types, - generally used for WoT Thing Descriptions. + JSON Schema type management. + + Handles converting highly specific python types to JSON schema types. + One needs to explicitly register such python types with the `register_type_replacement` method to be able to + insert JSON schema in JSON documents (like the Thing Description). + + ```python + JSONSchema.register_type_replacement(Image, 'string', schema=dict(contentEncoding='base64')) + JSONSchema.register_type_replacement(MyCustomObject, 'object', schema=MyCustomObject.schema()) + ``` + + Validation of JSON schema, say for properties or action payloads, is carried out by the `JSONSchemaValidator` + class which is separate. """ _allowed_types = ("string", "number", "integer", "boolean", "object", "array", None) @@ -31,15 +47,16 @@ class JSONSchema: }, "required": ["message", "type", "traceback"], }, - } + } # type: dict[type, str | dict] _schemas = {} @classmethod def is_allowed_type(cls, typ: Any) -> bool: """ - Check if a certain base type has a JSON schema base type - For example, + Check if a certain base type has a JSON schema base type. + + For example: ```python JSONSchema.is_allowed_type(int) # returns True @@ -49,26 +66,6 @@ def is_allowed_type(cls, typ: Any) -> bool: JSONSchema.is_allowed_type(MyCustomClass) # returns True ``` - Parameters - ---------- - typ: Any - the python type to check - """ - if typ in JSONSchema._replacements.keys(): - return True - return False - - @classmethod - def has_additional_schema_definitions(cls, typ: Any) -> bool: - """ - Check, if in additional to the JSON schema base type, additional schema definitions exists. - Utility function to decide where to insert additional schema definitions in a JSON document. - - ```python - JSONSchema.register_type_replacement(Image, 'string', schema=dict(contentEncoding='base64')) - JSONSchema.has_additional_schema_definitions(Image) # returns True - ``` - Parameters ---------- typ: Any @@ -77,16 +74,16 @@ def has_additional_schema_definitions(cls, typ: Any) -> bool: Returns ------- bool - True, if additional schema definitions exist for the type + True or False """ - if typ in JSONSchema._schemas.keys(): + if typ in JSONSchema._replacements.keys(): return True return False @classmethod def get_base_type(cls, typ: Any) -> str: """ - Get the JSON schema base type for a certain python type + Get the JSON schema base type for a certain python type. ```python JSONSchema.register_type_replacement(MyCustomObject, 'object', schema=MyCustomObject.schema()) @@ -97,13 +94,28 @@ def get_base_type(cls, typ: Any) -> str: ---------- typ: Any the python type to get the JSON schema base type + + Returns + ------- + str + the JSON schema base type + + Raises + ------ + TypeError + If the type is not natively supported in JSON schema or is not registered for conversion. """ if not JSONSchema.is_allowed_type(typ): raise TypeError( f"Object for wot-td has invalid type for JSON conversion. Given type - {type(typ)}. " + "Use JSONSchema.register_replacements on hololinked.schema_validators.JSONSchema object to recognise the type." ) - return JSONSchema._replacements[typ] + typ = JSONSchema._replacements[typ] + if isinstance(typ, str): + return typ + if isinstance(typ, dict) and "type" in typ: + return typ["type"] # type: ignore + return "object" @classmethod def register_type_replacement(self, type: Any, json_schema_base_type: str, schema: JSON | None = None) -> None: @@ -125,6 +137,11 @@ def register_type_replacement(self, type: Any, json_schema_base_type: str, schem ('string', 'number', 'integer', 'boolean', 'object', 'array', 'null'). schema: Optional[JSON] An optional JSON schema to use for the type. + + Raises + ------ + TypeError + If the provided JSON schema base type is not one of the allowed types. """ if json_schema_base_type in JSONSchema._allowed_types: JSONSchema._replacements[type] = json_schema_base_type @@ -136,9 +153,59 @@ def register_type_replacement(self, type: Any, json_schema_base_type: str, schem + f"'number', 'integer', 'boolean', 'null'. Given value {json_schema_base_type}" ) + @classmethod + def has_additional_schema_definitions(cls, typ: Any) -> bool: + """ + Check, if in additional to the JSON schema base type, additional schema definitions exists. + + Utility function to decide where to insert additional schema definitions in a JSON document. + + ```python + JSONSchema.register_type_replacement(Image, 'string', schema=dict(contentEncoding='base64')) + JSONSchema.has_additional_schema_definitions(Image) # returns True + ``` + + Parameters + ---------- + typ: Any + the python type to check + + Returns + ------- + bool + True, if additional schema definitions exist for the type + """ + if typ in JSONSchema._schemas.keys(): + return True + return False + @classmethod def get_additional_schema_definitions(cls, typ: Any): - """retrieve additional schema definitions for a certain python type""" + """ + Retrieve additional schema definitions for a certain python type. + + Returns + ------- + JSON + the additional schema definitions for the type + + Raises + ------ + ValueError + If no additional schema definitions exist for the type. + """ if not JSONSchema.has_additional_schema_definitions(typ): raise ValueError(f"Schema for {typ} not provided. register one with JSONSchema.register_type_replacement()") return JSONSchema._schemas[typ] + + +class SchemaValidatorClasses: + """Utility class to store schema validators of different types.""" + + json_schema: type[BaseSchemaValidator] + """JSON Schema validator class that can be instantiated with a JSON schema to validate data against that schema.""" + pydantic: type[BaseSchemaValidator] + """ + Pydantic validator class that can be instantiated with a Pydantic model to validate data against + the schema defined by that model. + """ diff --git a/hololinked/core/serializer_registry.py b/hololinked/serialization.py similarity index 98% rename from hololinked/core/serializer_registry.py rename to hololinked/serialization.py index 2cad1049..def4060f 100644 --- a/hololinked/core/serializer_registry.py +++ b/hololinked/serialization.py @@ -1,11 +1,12 @@ """A serializer registry for content types and serializers.""" +from __future__ import annotations + import warnings from typing import Any from hololinked.config import global_config -from hololinked.core import Action, Event, Property, Thing # noqa from hololinked.core.interfaces import BaseSerializer from hololinked.param.parameters import Parameter, String from hololinked.utils import MappableSingleton, issubklass @@ -20,9 +21,8 @@ class Serializers(metaclass=MappableSingleton): The default serializer is `JSONSerializer`, which will be provided to any unregistered object. """ - default = BaseSerializer() + default: BaseSerializer """The default serializer.""" - # some known types: json: BaseSerializer msgpack: BaseSerializer @@ -192,6 +192,8 @@ def register_for_object(cls, objekt: Any, serializer: BaseSerializer) -> None: ValueError if the object is not a Property, Action or Event, or Thing class """ + from hololinked.core import Action, Event, Property, Thing # noqa + if not isinstance(serializer, BaseSerializer): raise ValueError("serializer must be an instance of BaseSerializer, given : {}".format(type(serializer))) if not isinstance(objekt, (Property, Action, Event)) and not issubklass(objekt, Thing): @@ -229,6 +231,8 @@ def register_content_type_for_object(cls, objekt: Any, content_type: str) -> Non ValueError if the object is not a Property, Action or Event """ + from hololinked.core import Action, Event, Property, Thing # noqa + if not isinstance(objekt, (Property, Action, Event)) and not issubklass(objekt, Thing): raise ValueError("object must be a Property, Action or Event, got : {}".format(type(objekt))) if issubklass(objekt, Thing): @@ -273,6 +277,8 @@ def register_content_type_for_object_per_thing_instance( ValueError if the object is not a Property, Action or Event """ + from hololinked.core import Action, Event, Property, Thing # noqa + if not isinstance(objekt, (Property, Action, Event, str)): raise ValueError("object must be a Property, Action or Event, got : {}".format(type(objekt))) if not isinstance(objekt, str): diff --git a/hololinked/serializers/__init__.py b/hololinked/serializers/__init__.py index 2a4cfacf..24b81400 100644 --- a/hololinked/serializers/__init__.py +++ b/hololinked/serializers/__init__.py @@ -1,6 +1,6 @@ """Concrete implementations of serializers.""" -from hololinked.core import Serializers +from hololinked import Serializers from .serializers import ( JSONSerializer, diff --git a/hololinked/server/http/controllers.py b/hololinked/server/http/controllers.py index 548e16f2..b978c80f 100644 --- a/hololinked/server/http/controllers.py +++ b/hololinked/server/http/controllers.py @@ -7,7 +7,7 @@ from tornado.iostream import StreamClosedError from tornado.web import RequestHandler -from hololinked.core import Serializers +from hololinked import Serializers from ...config import global_config from ...constants import Operations diff --git a/hololinked/server/http/services.py b/hololinked/server/http/services.py index 4bca1b18..99d23d86 100644 --- a/hololinked/server/http/services.py +++ b/hololinked/server/http/services.py @@ -4,7 +4,7 @@ import structlog -from hololinked.core import Serializers +from hololinked import Serializers from ...constants import JSONSerializable, Operations from ...core.zmq.message import ERROR, INVALID_MESSAGE, TIMEOUT diff --git a/hololinked/server/mqtt/controllers.py b/hololinked/server/mqtt/controllers.py index 414d4371..c2f788de 100644 --- a/hololinked/server/mqtt/controllers.py +++ b/hololinked/server/mqtt/controllers.py @@ -3,7 +3,7 @@ import aiomqtt import structlog -from hololinked.core import Serializers +from hololinked import Serializers from ...core.zmq.message import EventMessage # noqa: F401 from ...td import EventAffordance, PropertyAffordance diff --git a/hololinked/storage/database.py b/hololinked/storage/database.py index 447831d9..7974678e 100644 --- a/hololinked/storage/database.py +++ b/hololinked/storage/database.py @@ -12,7 +12,7 @@ from sqlalchemy import inspect as inspect_database from sqlalchemy.orm import sessionmaker -from hololinked.core import Serializers +from hololinked import Serializers from hololinked.serializers import PythonBuiltinJSONSerializer as JSONSerializer from ..config import global_config diff --git a/hololinked/td/data_schema.py b/hololinked/td/data_schema.py index 4d7799b7..b532ec4b 100644 --- a/hololinked/td/data_schema.py +++ b/hololinked/td/data_schema.py @@ -2,6 +2,8 @@ from pydantic import BaseModel, ConfigDict, Field, RootModel +from hololinked import JSONSchema + from ..constants import JSON, JSONSerializable from ..core import Property from ..core.properties import ( @@ -21,7 +23,6 @@ TypedKeyMappingsDict, TypedList, ) -from ..schema_validators.json_schema import JSONSchema from ..utils import issubklass from .base import Schema from .utils import get_summary @@ -57,7 +58,7 @@ def __init__(self): super().__init__() def ds_build_fields_from_property(self, property: Property) -> None: - """populates schema information from property descriptor object""" + """Populates schema information from property descriptor object""" self.title = get_summary(property.doc) if property.constant: self.const = property.constant @@ -87,7 +88,7 @@ def ds_build_fields_from_property(self, property: Property) -> None: # you dont know what you are building, whether the data schema or something else when viewed from property affordance def ds_build_from_property(self, property: Property) -> None: """ - generates the schema specific to the property type, + Generates the schema specific to the property type, calls `ds_build_fields_from_property()` after choosing the right type """ if self._custom_schema_generators.get(property, NotImplemented) is not NotImplemented: @@ -135,7 +136,7 @@ def ds_build_from_property(self, property: Property) -> None: setattr(self, field_name, field_value) def _move_own_type_to_oneOf(self): - """move a type to oneOf""" + """Move a type to oneOf""" pass @@ -149,7 +150,7 @@ def __init__(self): super().__init__() def ds_build_fields_from_property(self, property) -> None: - """generates the schema""" + """Generates the schema""" self.type = "boolean" super().ds_build_fields_from_property(property) @@ -176,7 +177,7 @@ def __init__(self): super().__init__() def ds_build_fields_from_property(self, property) -> None: - """generates the schema""" + """Generates the schema""" self.type = "string" if isinstance(property, String): if property.regex is not None: @@ -212,7 +213,7 @@ def __init__(self): super().__init__() def ds_build_fields_from_property(self, property) -> None: - """generates the schema""" + """Generates the schema""" if isinstance(property, Integer): self.type = "integer" elif isinstance(property, Number): # dont change order - one is subclass of other @@ -260,7 +261,7 @@ def __init__(self): super().__init__() def ds_build_fields_from_property(self, property) -> None: - """generates the schema""" + """Generates the schema""" self.type = "array" self.items = [] if isinstance(property, (List, Tuple, TypedList)) and property.item_type is not None: @@ -315,7 +316,7 @@ def __init__(self): super().__init__() def ds_build_fields_from_property(self, property) -> None: - """generates the schema""" + """Generates the schema""" super().ds_build_fields_from_property(property) properties = None required = None @@ -356,7 +357,7 @@ def __init__(self): super().__init__() def ds_build_fields_from_property(self, property) -> None: - """generates the schema""" + """Generates the schema""" self.oneOf = [] if isinstance(property, ClassSelector): if not property.isinstance: @@ -428,7 +429,7 @@ def __init__(self): super().__init__() def ds_build_fields_from_property(self, property) -> None: - """generates the schema""" + """Generates the schema""" if not isinstance(property, Selector): raise TypeError(f"EnumSchema compatible property is only Selector, not {property.__class__}") self.enum = list(property.objects) diff --git a/hololinked/td/pydantic_extensions.py b/hololinked/td/pydantic_extensions.py index 51718022..f9114ca9 100644 --- a/hololinked/td/pydantic_extensions.py +++ b/hololinked/td/pydantic_extensions.py @@ -6,8 +6,8 @@ from pydantic._internal._core_utils import CoreSchemaOrField, is_core_schema from pydantic.json_schema import GenerateJsonSchema +from hololinked.constants import JSONSchema -JSONSchema = dict[str, Any] # A type to represent JSONSchema AnyUri = str Description = str diff --git a/tests/test_01_message.py b/tests/test_01_message.py index a77457b3..1e8ac786 100644 --- a/tests/test_01_message.py +++ b/tests/test_01_message.py @@ -2,7 +2,7 @@ import pytest -from hololinked.core import Serializers +from hololinked import Serializers from hololinked.core.zmq.message import ( ERROR, EXIT, diff --git a/tests/test_03_serializers.py b/tests/test_03_serializers.py index 8fb9ce00..7cf20b61 100644 --- a/tests/test_03_serializers.py +++ b/tests/test_03_serializers.py @@ -2,8 +2,8 @@ from things import TestThing -from hololinked.serializers import Serializers -from hololinked.serializers.serializers import BaseSerializer +from hololinked import Serializers +from hololinked.core.interfaces import BaseSerializer class YAMLSerializer(BaseSerializer): @@ -22,7 +22,6 @@ def yaml_serializer() -> BaseSerializer: def test_01_singleton(): """Test the singleton nature of the Serializers class.""" - serializers = Serializers() assert serializers == Serializers() assert Serializers != Serializers() @@ -53,7 +52,6 @@ def test_01_singleton(): def test_02_protocol_registration(yaml_serializer: BaseSerializer): """i.e. test if a new serializer (protocol) can be registered""" - # get existing number of serializers num_serializers = len(Serializers.content_types) @@ -109,7 +107,7 @@ def test_04_registration_for_objects_by_name(): def test_05_registration_dict(): - """test the dictionary where all serializers are stored""" + """Test the dictionary where all serializers are stored""" # depends on test 3 assert "test_thing" in Serializers.object_content_type_map assert "base_property" in Serializers.object_content_type_map["test_thing"] @@ -132,7 +130,7 @@ def test_06_retrieval(): def test_07_set_default(): - """test setting the default serializer""" + """Test setting the default serializer""" # get existing default old_default = Serializers.default # set new default and check if default is set @@ -146,7 +144,7 @@ def test_07_set_default(): def test_08_unknown_content_types(): - """test registration of unknown content types for objects""" + """Test registration of unknown content types for objects""" Serializers.register_content_type_for_object(TestThing.number_prop, "application/unknown") assert Serializers.for_object(None, "TestThing", "number_prop") is None assert Serializers.get_content_type_for_object(None, "TestThing", "number_prop") == "application/unknown" diff --git a/tests/things/spectrometer.py b/tests/things/spectrometer.py index 47cfa69e..a89c7376 100644 --- a/tests/things/spectrometer.py +++ b/tests/things/spectrometer.py @@ -7,6 +7,7 @@ import numpy +from hololinked import JSONSchema from hololinked.core import Event, Thing, action from hololinked.core.properties import ( Boolean, @@ -19,7 +20,6 @@ TypedList, ) from hololinked.core.state_machine import StateMachine -from hololinked.schema_validators import JSONSchema from hololinked.serializers import JSONSerializer from hololinked.server.http import HTTPServer diff --git a/tests/things/test_thing.py b/tests/things/test_thing.py index 4e4c4946..692b60e4 100644 --- a/tests/things/test_thing.py +++ b/tests/things/test_thing.py @@ -8,6 +8,7 @@ from pydantic import BaseModel, Field, WithJsonSchema +from hololinked import JSONSchema from hololinked.core import Event, Property, Thing, action from hololinked.core.actions import Action, BoundAction from hololinked.core.properties import ( @@ -19,7 +20,6 @@ String, ) from hololinked.param import ParameterizedFunction -from hololinked.schema_validators import JSONSchema class TestThing(Thing): @@ -460,7 +460,7 @@ def _push_worker(self, event_name: str = "test_event", total_number_of_events: i output_schema=analog_offset_output_schema, ) def get_analogue_offset(self, voltage_range: str, coupling: str) -> tuple[float, float]: - """analogue offset for a voltage range and coupling""" + """Analogue offset for a voltage range and coupling""" print(f"get_analogue_offset called with voltage_range={voltage_range}, coupling={coupling}") return 0.0, 0.0 @@ -591,7 +591,7 @@ def start_acquisition(self, max_count: Annotated[int, Field(gt=0)]): @action() def execute_instruction(self, command: str, return_data_size: Annotated[int, Field(ge=0)] = 0) -> str: """ - executes instruction given by the ASCII string parameter 'command'. + Executes instruction given by the ASCII string parameter 'command'. If return data size is greater than 0, it reads the response and returns the response. Return Data Size - in bytes - 1 ASCII character = 1 Byte. """