Merge pull request #68 from n0nSmoker/feat/exclude-values

Exclude values

Author: n0nSmoker

Makefile

@@ -5,7 +5,7 @@ test:
 	TEST_FILE=$(FILE) docker-compose up --build --abort-on-container-exit
 
 test-pypi:
-	uv run pytest -xvrs --color=yes -m pypi tests/test_pypi_readiness.py::test_pypi_version_can_be_installed_and_used
+	uv run pytest -xvrs --color=yes -m pypi tests/test_pypi_readiness.py
 
 format:
 	uv run ruff format .

README.md

@@ -80,19 +80,30 @@ You can use negative rules in `only` param too.
 So `item.to_dict(only=('somefield', -'somefield.id'))`
 will return `somefiled` without `id`. See [Negative rules in ONLY section](#Negative-rules-in-ONLY-section)
 
+If you want to exclude specific values from the serialized output (e.g., `None` values):
+```python
+result = item.to_dict(exclude_values=(None,))
+```
+This will exclude all fields that have `None` as their value. You can exclude multiple values:
+```python
+result = item.to_dict(exclude_values=(None, True, ''))
+```
+**Note** that `exclude_values` works with hashable values only. It filters values after serialization, so it works with nested dictionaries and models as well.
+
 If you want to define schema for all instances of particular SQLAlchemy model,
 add serialize properties to model definition:
 ```python
 class SomeModel(db.Model, SerializerMixin):
     serialize_only = ('somefield.id',)
     serialize_rules = ()
+    exclude_values = (None,)  # Exclude None values for all instances
     ...
     somefield = db.relationship('AnotherModel')
 
 result = item.to_dict()
 ```
 So the `result` in this case will be `{'somefield': [{'id': some_id}]}`
-***serialize_only*** and  ***serialize_rules*** work the same way as ***to_dict's*** arguments
+***serialize_only***, ***serialize_rules***, and ***exclude_values*** work the same way as ***to_dict's*** arguments
 
 
 # Advanced usage
@@ -176,6 +187,38 @@ dict(
         id=1
     )
 )
+
+
+# Exclude specific values
+item.to_dict(exclude_values=(None,))
+
+dict(
+    id=1,
+    string='Some string!',
+    boolean=True,
+    flat_id=1,
+    rel=[dict(
+        id=1,
+        string='Some string!',
+        boolean=True,
+        non_sqlalchemy_dict=dict(qwerty=123)
+    )]
+)
+# Note: 'null' field is excluded because its value is None
+
+
+# Exclude values with nested dictionaries
+item.dict = {"key": 123, "null_key": None, "key2": 456}
+item.to_dict(rules=("dict",), exclude_values=(None,))
+
+dict(
+    ...
+    dict=dict(
+        key=123,
+        key2=456
+    )
+    # Note: 'null_key' is excluded from nested dict
+)
 ```
 # Recursive models and trees
 If your models have references to each other or you work with large trees

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sqlalchemy-serializer"
-version = "1.5.1"
+version = "1.5.2"
 description = "Mixin for SQLAlchemy models serialization without pain"
 authors = [
     {name = "yuri.boiko", email = "yuri.boiko.dev@gmail.com"}

sqlalchemy_serializer/serializer.py

@@ -22,6 +22,9 @@
 SERIALIZER_DEFAULT_TIME_FORMAT = "%H:%M"
 SERIALIZER_DEFAULT_DECIMAL_FORMAT = "{}"
 
+# sentinel value for unspecified key since None and Ellipsis are valid keys
+_UNSPECIFIED = object()
+
 
 class SerializerMixin:
     """Mixin for retrieving public fields of sqlAlchemy-model in json-compatible format
@@ -37,12 +40,15 @@ class SerializerMixin:
     # Additions to default schema. Can include negative rules
     serialize_rules: tuple = ()
 
-    # Extra serialising functions
+    # Extra serializing functions
     serialize_types: tuple = ()
 
     # Custom list of fields to serialize in this model
     serializable_keys: tuple = ()
 
+    # Iterable of hashable values to exclude from serialized output
+    exclude_values: Iterable = ()
+
     date_format = SERIALIZER_DEFAULT_DATE_FORMAT
     datetime_format = SERIALIZER_DEFAULT_DATETIME_FORMAT
     time_format = SERIALIZER_DEFAULT_TIME_FORMAT
@@ -70,6 +76,7 @@ def to_dict(
         tzinfo=None,
         decimal_format=None,
         serialize_types=None,
+        exclude_values=None,
     ):
         """Returns SQLAlchemy model's data in JSON compatible format
 
@@ -85,6 +92,7 @@ def to_dict(
         :param decimal_format: str
         :param serialize_types:
         :param tzinfo: datetime.tzinfo converts datetimes to local user timezone
+        :param exclude_values: iterable of hashable values to exclude from serialized output
         :return: data: dict
         """
         s = Serializer(
@@ -94,13 +102,14 @@ def to_dict(
             decimal_format=decimal_format or self.decimal_format,
             tzinfo=tzinfo or self.get_tzinfo(),
             serialize_types=serialize_types or self.serialize_types,
+            exclude_values=exclude_values or self.exclude_values,
         )
         return s(self, only=only, extend=rules)
 
 
 Options = namedtuple(
     "Options",
-    "date_format datetime_format time_format decimal_format tzinfo serialize_types",
+    "date_format datetime_format time_format decimal_format tzinfo serialize_types exclude_values",  # noqa: E501
 )
 
 
@@ -117,6 +126,9 @@ class Serializer:
     def __init__(self, **kwargs):
         self.set_serialization_depth(0)
         # Provide defaults for Options if not specified
+        exclude_values = kwargs.get("exclude_values")
+        exclude_values_set = set(exclude_values) if exclude_values else None
+
         options_kwargs = {
             "date_format": kwargs.get("date_format", SERIALIZER_DEFAULT_DATE_FORMAT),
             "datetime_format": kwargs.get(
@@ -126,6 +138,7 @@ def __init__(self, **kwargs):
             "decimal_format": kwargs.get("decimal_format", SERIALIZER_DEFAULT_DECIMAL_FORMAT),
             "tzinfo": kwargs.get("tzinfo"),
             "serialize_types": kwargs.get("serialize_types", ()),
+            "exclude_values": exclude_values_set,
         }
         self.set_options(Options(**options_kwargs))
         self.init_callbacks()
@@ -150,6 +163,18 @@ def set_serialization_depth(self, value: int):
     def set_options(self, opts: Options):
         self.opts = opts
 
+    def should_exclude(self, value) -> bool:
+        """Check if value should be excluded based on exclude_values"""
+        if self.opts.exclude_values is None:
+            return False
+        try:
+            # if value is not hashable, we cannot compare it with exclude_values
+            hash(value)
+        except TypeError:
+            return False
+
+        return value in self.opts.exclude_values
+
     def init_callbacks(self):
         """Initialize callbacks"""
         self.serialize_types = (
@@ -206,29 +231,31 @@ def fork(self, key: str) -> "Serializer":
         logger.debug("Fork serializer for key:%s", key)
         return serializer
 
-    def serialize(self, value, **kwargs):
+    def serialize(self, value, key=_UNSPECIFIED):
         """Orchestrates the serialization process.
 
         Args:
             value: The value to be serialized.
-            **kwargs: Only to ensure that no key is passed
-                        since None and Ellipsis are valid keys.
+            key: The key to be serialized.
 
         Returns:
             The serialized value.
-
         """
         if self.is_valid_callable(value):
             value = value()
             logger.debug("Process callable resulting type:%s", get_type(value))
 
-        if kwargs:
-            if "key" in kwargs:
-                # since None and ... are valid keys
-                return self.serialize_with_fork(value=value, key=kwargs["key"])
-            raise ValueError("Malformed structure of kwargs. Only `key` accepted")
+        if not self.should_exclude(value):
+            try:
+                if key is not _UNSPECIFIED:  # since None and Ellipsis are valid keys
+                    return self.serialize_with_fork(value=value, key=key)
+                return self.apply_callback(value=value)
+
+            except IsNotSerializable:
+                logger.warning("Can not serialize type:%s", get_type(value))
 
-        return self.apply_callback(value=value)
+        logger.debug("Skip value:%s", value)
+        return _UNSPECIFIED
 
     def apply_callback(self, value):
         """Apply a proper callback to serialize the value
@@ -241,6 +268,7 @@ def apply_callback(self, value):
         raise IsNotSerializable(f"Unserializable type:{get_type(value)} value:{value}")
 
     def serialize_with_fork(self, value, key):
+        """Serialize value with a forked serializer"""
         serializer = self
         if self.is_forkable(value):
             serializer = self.fork(key=key)
@@ -250,13 +278,9 @@ def serialize_with_fork(self, value, key):
     def serialize_iter(self, value: Iterable) -> list:
         res = []
         for v in value:
-            try:
-                r = self.serialize(v)
-            except IsNotSerializable:  # FIXME: Why we swallow exception only in iterable?
-                logger.warning("Can not serialize type:%s", get_type(v))
-                continue
-
-            res.append(r)
+            result = self.serialize(v)
+            if result is not _UNSPECIFIED:
+                res.append(result)
         return res
 
     def serialize_dict(self, value: dict) -> dict:
@@ -265,7 +289,9 @@ def serialize_dict(self, value: dict) -> dict:
             if self.schema.is_included(k):  # TODO: Skip check if is NOT greedy
                 logger.debug("Serialize key:%s type:%s of dict", k, get_type(v))
 
-                res[k] = self.serialize(value=v, key=k)
+                result = self.serialize(value=v, key=k)
+                if result is not _UNSPECIFIED:
+                    res[k] = result
             else:
                 logger.debug("Skip key:%s of dict", k)
         return res
@@ -287,7 +313,9 @@ def serialize_model(self, value) -> dict:
                     get_type(v),
                     get_type(value),
                 )
-                res[k] = self.serialize(value=v, key=k)
+                result = self.serialize(value=v, key=k)
+                if result is not _UNSPECIFIED:
+                    res[k] = result
 
             else:
                 logger.debug("Skip key:%s of model:%s", k, get_type(value))

tests/test_exclude_values.py

@@ -0,0 +1,124 @@
+from .models import FlatModel, NestedModel
+
+
+def test_exclude_values_none(get_instance):
+    """Test excluding None values from serialized output"""
+    i = get_instance(FlatModel)
+    data = i.to_dict(exclude_values=(None,))
+
+    # None value should be excluded
+    assert "null" not in data
+    # Other fields should still be present
+    assert "id" in data
+    assert "string" in data
+    assert "bool" in data
+
+
+def test_exclude_values_multiple(get_instance):
+    """Test excluding multiple values from serialized output"""
+    i = get_instance(FlatModel)
+    data = i.to_dict(exclude_values=(None, True, i.string))
+
+    assert "null" not in data
+    assert "bool" not in data
+    assert "string" not in data
+
+    # Other fields should still be present
+    assert "id" in data
+    assert "date" in data
+    assert "datetime" in data
+    assert "time" in data
+    assert "uuid" in data
+
+
+def test_exclude_values_with_rules(get_instance):
+    """Test that exclude_values works with rules parameter"""
+    i = get_instance(FlatModel)
+    data = i.to_dict(rules=("null", "prop"), exclude_values=(None,))
+
+    # null field should be included in rules but excluded by exclude_values
+    assert "null" not in data
+    # prop should be present
+    assert "prop" in data
+
+
+def test_exclude_values_with_only(get_instance):
+    """Test that exclude_values works with only parameter"""
+    i = get_instance(FlatModel)
+    data = i.to_dict(only=("id", "null", "string"), exclude_values=(None,))
+
+    # null field should be included in only but excluded by exclude_values
+    assert "null" not in data
+    # Other fields should be present
+    assert "id" in data
+    assert "string" in data
+
+
+def test_exclude_values_empty_collection(get_instance):
+    """Test that empty exclude_values doesn't filter anything"""
+    i = get_instance(FlatModel)
+    data = i.to_dict(exclude_values=())
+
+    # All fields should be present including null
+    assert "null" in data
+    assert data["null"] is None
+    assert "id" in data
+    assert "string" in data
+
+
+def test_exclude_values_nested_dict(get_instance):
+    """Test that exclude_values works with nested dictionaries"""
+    i = get_instance(FlatModel)
+    # Create a dict with None value
+    i.dict = {"key": 123, "null_key": None, "key2": 456}
+    data = i.to_dict(rules=("dict",), exclude_values=(None,))
+
+    assert "dict" in data
+    assert "null_key" not in data["dict"]
+    assert "key" in data["dict"]
+    assert "key2" in data["dict"]
+
+
+def test_exclude_values_nested_model(get_instance):
+    """Test that exclude_values works with nested models"""
+    flat = get_instance(FlatModel)
+    nested = get_instance(NestedModel, model_id=flat.id)
+    nested.model = flat
+
+    data = nested.to_dict(rules=("model",), exclude_values=(None,))
+
+    assert "model" in data
+    assert "null" not in data
+    assert "null" not in data["model"]
+
+    # Other fields should be present
+    assert "id" in data["model"]
+    assert "string" in data["model"]
+
+    def test_exclude_values_class_level(get_instance):
+        """Test class-level exclude_values works"""
+
+        class ExcludeNullFlatModel(FlatModel):
+            exclude_values = (None,)
+
+        i = get_instance(ExcludeNullFlatModel)
+        i.null = None
+        i.string = "foobar"
+        data = i.to_dict(only=("id", "null", "string"))
+        assert "null" not in data
+        assert "id" in data
+        assert "string" in data and data["string"] == "foobar"
+
+        # Nested model class level exclude_values
+        class ExcludeNullNestedModel(NestedModel):
+            exclude_values = (None,)
+
+        nested = get_instance(ExcludeNullNestedModel)
+        flat = get_instance(FlatModel)
+        flat.null = None
+        nested.model = flat
+
+        data = nested.to_dict(rules=("model",))
+        assert "model" in data
+        assert "null" in data
+        assert "null" not in data["model"]