diff --git a/Makefile b/Makefile index 937a5cd..5fdf8c5 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ test: TEST_FILE=$(FILE) docker-compose up --build --abort-on-container-exit test-pypi: - uv run pytest -xvrs --color=yes -m pypi tests/test_pypi_readiness.py::test_pypi_version_can_be_installed_and_used + uv run pytest -xvrs --color=yes -m pypi tests/test_pypi_readiness.py format: uv run ruff format . diff --git a/README.md b/README.md index df57ecf..e2f6c8a 100644 --- a/README.md +++ b/README.md @@ -80,19 +80,30 @@ You can use negative rules in `only` param too. So `item.to_dict(only=('somefield', -'somefield.id'))` will return `somefiled` without `id`. See [Negative rules in ONLY section](#Negative-rules-in-ONLY-section) +If you want to exclude specific values from the serialized output (e.g., `None` values): +```python +result = item.to_dict(exclude_values=(None,)) +``` +This will exclude all fields that have `None` as their value. You can exclude multiple values: +```python +result = item.to_dict(exclude_values=(None, True, '')) +``` +**Note** that `exclude_values` works with hashable values only. It filters values after serialization, so it works with nested dictionaries and models as well. + If you want to define schema for all instances of particular SQLAlchemy model, add serialize properties to model definition: ```python class SomeModel(db.Model, SerializerMixin): serialize_only = ('somefield.id',) serialize_rules = () + exclude_values = (None,) # Exclude None values for all instances ... somefield = db.relationship('AnotherModel') result = item.to_dict() ``` So the `result` in this case will be `{'somefield': [{'id': some_id}]}` -***serialize_only*** and ***serialize_rules*** work the same way as ***to_dict's*** arguments +***serialize_only***, ***serialize_rules***, and ***exclude_values*** work the same way as ***to_dict's*** arguments # Advanced usage @@ -176,6 +187,38 @@ dict( id=1 ) ) + + +# Exclude specific values +item.to_dict(exclude_values=(None,)) + +dict( + id=1, + string='Some string!', + boolean=True, + flat_id=1, + rel=[dict( + id=1, + string='Some string!', + boolean=True, + non_sqlalchemy_dict=dict(qwerty=123) + )] +) +# Note: 'null' field is excluded because its value is None + + +# Exclude values with nested dictionaries +item.dict = {"key": 123, "null_key": None, "key2": 456} +item.to_dict(rules=("dict",), exclude_values=(None,)) + +dict( + ... + dict=dict( + key=123, + key2=456 + ) + # Note: 'null_key' is excluded from nested dict +) ``` # Recursive models and trees If your models have references to each other or you work with large trees diff --git a/pyproject.toml b/pyproject.toml index 2b94f08..ca5eddc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "sqlalchemy-serializer" -version = "1.5.1" +version = "1.5.2" description = "Mixin for SQLAlchemy models serialization without pain" authors = [ {name = "yuri.boiko", email = "yuri.boiko.dev@gmail.com"} diff --git a/sqlalchemy_serializer/serializer.py b/sqlalchemy_serializer/serializer.py index 371f56d..3e00195 100644 --- a/sqlalchemy_serializer/serializer.py +++ b/sqlalchemy_serializer/serializer.py @@ -22,6 +22,9 @@ SERIALIZER_DEFAULT_TIME_FORMAT = "%H:%M" SERIALIZER_DEFAULT_DECIMAL_FORMAT = "{}" +# sentinel value for unspecified key since None and Ellipsis are valid keys +_UNSPECIFIED = object() + class SerializerMixin: """Mixin for retrieving public fields of sqlAlchemy-model in json-compatible format @@ -37,12 +40,15 @@ class SerializerMixin: # Additions to default schema. Can include negative rules serialize_rules: tuple = () - # Extra serialising functions + # Extra serializing functions serialize_types: tuple = () # Custom list of fields to serialize in this model serializable_keys: tuple = () + # Iterable of hashable values to exclude from serialized output + exclude_values: Iterable = () + date_format = SERIALIZER_DEFAULT_DATE_FORMAT datetime_format = SERIALIZER_DEFAULT_DATETIME_FORMAT time_format = SERIALIZER_DEFAULT_TIME_FORMAT @@ -70,6 +76,7 @@ def to_dict( tzinfo=None, decimal_format=None, serialize_types=None, + exclude_values=None, ): """Returns SQLAlchemy model's data in JSON compatible format @@ -85,6 +92,7 @@ def to_dict( :param decimal_format: str :param serialize_types: :param tzinfo: datetime.tzinfo converts datetimes to local user timezone + :param exclude_values: iterable of hashable values to exclude from serialized output :return: data: dict """ s = Serializer( @@ -94,13 +102,14 @@ def to_dict( decimal_format=decimal_format or self.decimal_format, tzinfo=tzinfo or self.get_tzinfo(), serialize_types=serialize_types or self.serialize_types, + exclude_values=exclude_values or self.exclude_values, ) return s(self, only=only, extend=rules) Options = namedtuple( "Options", - "date_format datetime_format time_format decimal_format tzinfo serialize_types", + "date_format datetime_format time_format decimal_format tzinfo serialize_types exclude_values", # noqa: E501 ) @@ -117,6 +126,9 @@ class Serializer: def __init__(self, **kwargs): self.set_serialization_depth(0) # Provide defaults for Options if not specified + exclude_values = kwargs.get("exclude_values") + exclude_values_set = set(exclude_values) if exclude_values else None + options_kwargs = { "date_format": kwargs.get("date_format", SERIALIZER_DEFAULT_DATE_FORMAT), "datetime_format": kwargs.get( @@ -126,6 +138,7 @@ def __init__(self, **kwargs): "decimal_format": kwargs.get("decimal_format", SERIALIZER_DEFAULT_DECIMAL_FORMAT), "tzinfo": kwargs.get("tzinfo"), "serialize_types": kwargs.get("serialize_types", ()), + "exclude_values": exclude_values_set, } self.set_options(Options(**options_kwargs)) self.init_callbacks() @@ -150,6 +163,18 @@ def set_serialization_depth(self, value: int): def set_options(self, opts: Options): self.opts = opts + def should_exclude(self, value) -> bool: + """Check if value should be excluded based on exclude_values""" + if self.opts.exclude_values is None: + return False + try: + # if value is not hashable, we cannot compare it with exclude_values + hash(value) + except TypeError: + return False + + return value in self.opts.exclude_values + def init_callbacks(self): """Initialize callbacks""" self.serialize_types = ( @@ -206,29 +231,31 @@ def fork(self, key: str) -> "Serializer": logger.debug("Fork serializer for key:%s", key) return serializer - def serialize(self, value, **kwargs): + def serialize(self, value, key=_UNSPECIFIED): """Orchestrates the serialization process. Args: value: The value to be serialized. - **kwargs: Only to ensure that no key is passed - since None and Ellipsis are valid keys. + key: The key to be serialized. Returns: The serialized value. - """ if self.is_valid_callable(value): value = value() logger.debug("Process callable resulting type:%s", get_type(value)) - if kwargs: - if "key" in kwargs: - # since None and ... are valid keys - return self.serialize_with_fork(value=value, key=kwargs["key"]) - raise ValueError("Malformed structure of kwargs. Only `key` accepted") + if not self.should_exclude(value): + try: + if key is not _UNSPECIFIED: # since None and Ellipsis are valid keys + return self.serialize_with_fork(value=value, key=key) + return self.apply_callback(value=value) + + except IsNotSerializable: + logger.warning("Can not serialize type:%s", get_type(value)) - return self.apply_callback(value=value) + logger.debug("Skip value:%s", value) + return _UNSPECIFIED def apply_callback(self, value): """Apply a proper callback to serialize the value @@ -241,6 +268,7 @@ def apply_callback(self, value): raise IsNotSerializable(f"Unserializable type:{get_type(value)} value:{value}") def serialize_with_fork(self, value, key): + """Serialize value with a forked serializer""" serializer = self if self.is_forkable(value): serializer = self.fork(key=key) @@ -250,13 +278,9 @@ def serialize_with_fork(self, value, key): def serialize_iter(self, value: Iterable) -> list: res = [] for v in value: - try: - r = self.serialize(v) - except IsNotSerializable: # FIXME: Why we swallow exception only in iterable? - logger.warning("Can not serialize type:%s", get_type(v)) - continue - - res.append(r) + result = self.serialize(v) + if result is not _UNSPECIFIED: + res.append(result) return res def serialize_dict(self, value: dict) -> dict: @@ -265,7 +289,9 @@ def serialize_dict(self, value: dict) -> dict: if self.schema.is_included(k): # TODO: Skip check if is NOT greedy logger.debug("Serialize key:%s type:%s of dict", k, get_type(v)) - res[k] = self.serialize(value=v, key=k) + result = self.serialize(value=v, key=k) + if result is not _UNSPECIFIED: + res[k] = result else: logger.debug("Skip key:%s of dict", k) return res @@ -287,7 +313,9 @@ def serialize_model(self, value) -> dict: get_type(v), get_type(value), ) - res[k] = self.serialize(value=v, key=k) + result = self.serialize(value=v, key=k) + if result is not _UNSPECIFIED: + res[k] = result else: logger.debug("Skip key:%s of model:%s", k, get_type(value)) diff --git a/tests/test_exclude_values.py b/tests/test_exclude_values.py new file mode 100644 index 0000000..0239d1b --- /dev/null +++ b/tests/test_exclude_values.py @@ -0,0 +1,124 @@ +from .models import FlatModel, NestedModel + + +def test_exclude_values_none(get_instance): + """Test excluding None values from serialized output""" + i = get_instance(FlatModel) + data = i.to_dict(exclude_values=(None,)) + + # None value should be excluded + assert "null" not in data + # Other fields should still be present + assert "id" in data + assert "string" in data + assert "bool" in data + + +def test_exclude_values_multiple(get_instance): + """Test excluding multiple values from serialized output""" + i = get_instance(FlatModel) + data = i.to_dict(exclude_values=(None, True, i.string)) + + assert "null" not in data + assert "bool" not in data + assert "string" not in data + + # Other fields should still be present + assert "id" in data + assert "date" in data + assert "datetime" in data + assert "time" in data + assert "uuid" in data + + +def test_exclude_values_with_rules(get_instance): + """Test that exclude_values works with rules parameter""" + i = get_instance(FlatModel) + data = i.to_dict(rules=("null", "prop"), exclude_values=(None,)) + + # null field should be included in rules but excluded by exclude_values + assert "null" not in data + # prop should be present + assert "prop" in data + + +def test_exclude_values_with_only(get_instance): + """Test that exclude_values works with only parameter""" + i = get_instance(FlatModel) + data = i.to_dict(only=("id", "null", "string"), exclude_values=(None,)) + + # null field should be included in only but excluded by exclude_values + assert "null" not in data + # Other fields should be present + assert "id" in data + assert "string" in data + + +def test_exclude_values_empty_collection(get_instance): + """Test that empty exclude_values doesn't filter anything""" + i = get_instance(FlatModel) + data = i.to_dict(exclude_values=()) + + # All fields should be present including null + assert "null" in data + assert data["null"] is None + assert "id" in data + assert "string" in data + + +def test_exclude_values_nested_dict(get_instance): + """Test that exclude_values works with nested dictionaries""" + i = get_instance(FlatModel) + # Create a dict with None value + i.dict = {"key": 123, "null_key": None, "key2": 456} + data = i.to_dict(rules=("dict",), exclude_values=(None,)) + + assert "dict" in data + assert "null_key" not in data["dict"] + assert "key" in data["dict"] + assert "key2" in data["dict"] + + +def test_exclude_values_nested_model(get_instance): + """Test that exclude_values works with nested models""" + flat = get_instance(FlatModel) + nested = get_instance(NestedModel, model_id=flat.id) + nested.model = flat + + data = nested.to_dict(rules=("model",), exclude_values=(None,)) + + assert "model" in data + assert "null" not in data + assert "null" not in data["model"] + + # Other fields should be present + assert "id" in data["model"] + assert "string" in data["model"] + + def test_exclude_values_class_level(get_instance): + """Test class-level exclude_values works""" + + class ExcludeNullFlatModel(FlatModel): + exclude_values = (None,) + + i = get_instance(ExcludeNullFlatModel) + i.null = None + i.string = "foobar" + data = i.to_dict(only=("id", "null", "string")) + assert "null" not in data + assert "id" in data + assert "string" in data and data["string"] == "foobar" + + # Nested model class level exclude_values + class ExcludeNullNestedModel(NestedModel): + exclude_values = (None,) + + nested = get_instance(ExcludeNullNestedModel) + flat = get_instance(FlatModel) + flat.null = None + nested.model = flat + + data = nested.to_dict(rules=("model",)) + assert "model" in data + assert "null" in data + assert "null" not in data["model"] diff --git a/uv.lock b/uv.lock index b8daebd..d1fba37 100644 --- a/uv.lock +++ b/uv.lock @@ -551,7 +551,7 @@ wheels = [ [[package]] name = "sqlalchemy-serializer" -version = "1.5.1" +version = "1.5.2" source = { editable = "." } dependencies = [ { name = "psycopg2-binary" },