Merge pull request #590 from akshat62/feature/glob-wildcard-paths

Add wildcard/glob pattern support for exclude_paths and include_paths

Author: seperman

deepdiff/deephash.py

@@ -15,6 +15,7 @@
                              get_id, type_is_subclass_of_type_group, type_in_type_group,
                              number_to_string, datetime_normalize, KEY_TO_VAL_STR,
                              get_truncate_datetime, dict_, add_root_to_paths, PydanticBaseModel,
+                             separate_wildcard_and_exact_paths,
                              SetOrdered)
 
 from deepdiff.base import Base
@@ -190,6 +191,7 @@ def __init__(self,
                  custom_operators: Optional[List[Any]] = None,
                  default_timezone: Union[datetime.timezone, "BaseTzInfo"] = datetime.timezone.utc,
                  encodings: Optional[List[str]] = None,
+                 exclude_glob_paths: Optional[List[Any]] = None,
                  exclude_obj_callback: Optional[Callable[[Any, str], bool]] = None,
                  exclude_paths: Optional[PathType] = None,
                  exclude_regex_paths: Optional[RegexType] = None,
@@ -206,6 +208,7 @@ def __init__(self,
                  ignore_type_in_groups: Any = None,
                  ignore_type_subclasses: bool = False,
                  ignore_uuid_types: bool = False,
+                 include_glob_paths: Optional[List[Any]] = None,
                  include_paths: Optional[PathType] = None,
                  number_format_notation: str = "f",
                  number_to_string_func: Optional[NumberToStringFunc] = None,
@@ -232,8 +235,14 @@ def __init__(self,
         exclude_types = set() if exclude_types is None else set(exclude_types)
         self.exclude_types_tuple = tuple(exclude_types)  # we need tuple for checking isinstance
         self.ignore_repetition = ignore_repetition
-        self.exclude_paths = add_root_to_paths(convert_item_or_items_into_set_else_none(exclude_paths))
-        self.include_paths = add_root_to_paths(convert_item_or_items_into_set_else_none(include_paths))
+        _exclude_set = convert_item_or_items_into_set_else_none(exclude_paths)
+        _exclude_exact, _exclude_globs = separate_wildcard_and_exact_paths(_exclude_set)
+        self.exclude_paths = add_root_to_paths(_exclude_exact)
+        self.exclude_glob_paths = exclude_glob_paths or _exclude_globs
+        _include_set = convert_item_or_items_into_set_else_none(include_paths)
+        _include_exact, _include_globs = separate_wildcard_and_exact_paths(_include_set)
+        self.include_paths = add_root_to_paths(_include_exact)
+        self.include_glob_paths = include_glob_paths or _include_globs
         self.exclude_regex_paths = convert_item_or_items_into_compiled_regexes_else_none(exclude_regex_paths)
         self.hasher = default_hasher if hasher is None else hasher
         self.hashes[UNPROCESSED_KEY] = []  # type: ignore
@@ -462,11 +471,21 @@ def _skip_this(self, obj: Any, parent: str) -> bool:
         skip = False
         if self.exclude_paths and parent in self.exclude_paths:
             skip = True
-        if self.include_paths and parent != 'root':
-            if parent not in self.include_paths:
-                skip = True
-                for prefix in self.include_paths:
-                    if parent.startswith(prefix):
+        elif self.exclude_glob_paths and any(gp.match(parent) for gp in self.exclude_glob_paths):
+            skip = True
+        if (self.include_paths or self.include_glob_paths) and parent != 'root':
+            skip = True
+            if self.include_paths:
+                if parent in self.include_paths:
+                    skip = False
+                else:
+                    for prefix in self.include_paths:
+                        if parent.startswith(prefix):
+                            skip = False
+                            break
+            if skip and self.include_glob_paths:
+                for gp in self.include_glob_paths:
+                    if gp.match_or_is_ancestor(parent):
                         skip = False
                         break
         elif self.exclude_regex_paths and any(

deepdiff/diff.py

@@ -29,7 +29,8 @@
                              TEXT_VIEW, TREE_VIEW, DELTA_VIEW, COLORED_VIEW, COLORED_COMPACT_VIEW,
                              detailed__dict__, add_root_to_paths,
                              np, get_truncate_datetime, dict_, CannotCompare, ENUM_INCLUDE_KEYS,
-                             PydanticBaseModel, Opcode, SetOrdered, ipranges)
+                             PydanticBaseModel, Opcode, SetOrdered, ipranges,
+                             separate_wildcard_and_exact_paths)
 from deepdiff.serialization import SerializationMixin
 from deepdiff.distance import DistanceMixin, logarithmic_similarity
 from deepdiff.model import (
@@ -110,7 +111,9 @@ def _report_progress(_stats: Dict[str, Any], progress_logger: Callable[[str], No
 DEEPHASH_PARAM_KEYS = (
     'exclude_types',
     'exclude_paths',
+    'exclude_glob_paths',
     'include_paths',
+    'include_glob_paths',
     'exclude_regex_paths',
     'hasher',
     'significant_digits',
@@ -209,6 +212,10 @@ def __init__(self,
                  _shared_parameters: Optional[Dict[str, Any]]=None,
                  **kwargs):
         super().__init__()
+        # Defaults for glob path attributes — needed for non-root instances
+        # that may receive _parameters without these keys.
+        self.exclude_glob_paths = None
+        self.include_glob_paths = None
         if kwargs:
             raise ValueError((
                 "The following parameter(s) are not valid: %s\n"
@@ -257,8 +264,12 @@ def __init__(self,
                 ignore_type_subclasses=ignore_type_subclasses,
                 ignore_uuid_types=ignore_uuid_types)
             self.report_repetition = report_repetition
-            self.exclude_paths = add_root_to_paths(convert_item_or_items_into_set_else_none(exclude_paths))
-            self.include_paths = add_root_to_paths(convert_item_or_items_into_set_else_none(include_paths))
+            _exclude_set = convert_item_or_items_into_set_else_none(exclude_paths)
+            _exclude_exact, self.exclude_glob_paths = separate_wildcard_and_exact_paths(_exclude_set)
+            self.exclude_paths = add_root_to_paths(_exclude_exact)
+            _include_set = convert_item_or_items_into_set_else_none(include_paths)
+            _include_exact, self.include_glob_paths = separate_wildcard_and_exact_paths(_include_set)
+            self.include_paths = add_root_to_paths(_include_exact)
             self.exclude_regex_paths = convert_item_or_items_into_compiled_regexes_else_none(exclude_regex_paths)
             self.exclude_types = set(exclude_types) if exclude_types else None
             self.exclude_types_tuple = tuple(exclude_types) if exclude_types else None  # we need tuple for checking isinstance
@@ -423,7 +434,7 @@ def _group_by_sort_key(x):
                     self.__dict__.clear()
 
     def _get_deephash_params(self):
-        result = {key: self._parameters[key] for key in DEEPHASH_PARAM_KEYS}
+        result = {key: self._parameters.get(key) for key in DEEPHASH_PARAM_KEYS}
         result['ignore_repetition'] = not self.report_repetition
         result['number_to_string_func'] = self.number_to_string
         return result
@@ -442,6 +453,8 @@ def _report_result(self, report_type, change_level, local_tree=None):
         """
 
         if not self._skip_this(change_level):
+            if self._skip_report_for_include_glob(change_level):
+                return
             change_level.report_type = report_type
             tree = self.tree if local_tree is None else local_tree
             tree[report_type].add(change_level)
@@ -461,10 +474,33 @@ def custom_report_result(self, report_type, level, extra_info=None):
         """
 
         if not self._skip_this(level):
+            if self._skip_report_for_include_glob(level):
+                return
             level.report_type = report_type
             level.additional[CUSTOM_FIELD] = extra_info
             self.tree[report_type].add(level)
 
+    def _skip_report_for_include_glob(self, level):
+        """When include_glob_paths is set, _skip_this allows ancestors through for traversal.
+        This method does a stricter check at report time: only report if the path
+        actually matches a glob pattern or is a descendant of a matching path,
+        or if it already matches an exact include_path."""
+        if not self.include_glob_paths:
+            return False
+        level_path = level.path()
+        # If exact include_paths already matched, don't skip
+        if self.include_paths:
+            if level_path in self.include_paths:
+                return False
+            for prefix in self.include_paths:
+                if prefix in level_path:
+                    return False
+        # Check glob patterns: match or descendant
+        for gp in self.include_glob_paths:
+            if gp.match_or_is_descendant(level_path):
+                return False
+        return True
+
     @staticmethod
     def _dict_from_slots(object: Any) -> Dict[str, Any]:
         def unmangle(attribute: str) -> str:
@@ -552,11 +588,21 @@ def _skip_this(self, level: Any) -> bool:
         skip = False
         if self.exclude_paths and level_path in self.exclude_paths:
             skip = True
-        if self.include_paths and level_path != 'root':
-            if level_path not in self.include_paths:
-                skip = True
-                for prefix in self.include_paths:
-                    if prefix in level_path or level_path in prefix:
+        elif self.exclude_glob_paths and any(gp.match(level_path) for gp in self.exclude_glob_paths):
+            skip = True
+        if not skip and (self.include_paths or self.include_glob_paths) and level_path != 'root':
+            skip = True
+            if self.include_paths:
+                if level_path in self.include_paths:
+                    skip = False
+                else:
+                    for prefix in self.include_paths:
+                        if prefix in level_path or level_path in prefix:
+                            skip = False
+                            break
+            if skip and self.include_glob_paths:
+                for gp in self.include_glob_paths:
+                    if gp.match_or_is_ancestor(level_path):
                         skip = False
                         break
         elif self.exclude_regex_paths and any(
@@ -586,28 +632,34 @@ def _skip_this(self, level: Any) -> bool:
 
     def _skip_this_key(self, level: Any, key: Any) -> bool:
         # if include_paths is not set, than treet every path as included
-        if self.include_paths is None:
-            return False
-        if "{}['{}']".format(level.path(), key) in self.include_paths:
-            return False
-        if level.path() in self.include_paths:
-            # matches e.g. level+key root['foo']['bar']['veg'] include_paths ["root['foo']['bar']"]
+        if self.include_paths is None and self.include_glob_paths is None:
             return False
-        for prefix in self.include_paths:
-            if "{}['{}']".format(level.path(), key) in prefix:
-                # matches as long the prefix is longer than this object key
-                # eg.: level+key root['foo']['bar'] matches prefix root['foo']['bar'] from include paths
-                #      level+key root['foo'] matches prefix root['foo']['bar'] from include_paths
-                #      level+key root['foo']['bar'] DOES NOT match root['foo'] from include_paths This needs to be handled afterwards
+        key_path = "{}['{}']".format(level.path(), key)
+        if self.include_paths:
+            if key_path in self.include_paths:
                 return False
-        # check if a higher level is included as a whole (=without any sublevels specified)
-        # matches e.g. level+key root['foo']['bar']['veg'] include_paths ["root['foo']"]
-        # but does not match, if it is level+key root['foo']['bar']['veg'] include_paths ["root['foo']['bar']['fruits']"]
-        up = level.up
-        while up is not None:
-            if up.path() in self.include_paths:
+            if level.path() in self.include_paths:
+                # matches e.g. level+key root['foo']['bar']['veg'] include_paths ["root['foo']['bar']"]
                 return False
-            up = up.up
+            for prefix in self.include_paths:
+                if key_path in prefix:
+                    # matches as long the prefix is longer than this object key
+                    # eg.: level+key root['foo']['bar'] matches prefix root['foo']['bar'] from include paths
+                    #      level+key root['foo'] matches prefix root['foo']['bar'] from include_paths
+                    #      level+key root['foo']['bar'] DOES NOT match root['foo'] from include_paths This needs to be handled afterwards
+                    return False
+            # check if a higher level is included as a whole (=without any sublevels specified)
+            # matches e.g. level+key root['foo']['bar']['veg'] include_paths ["root['foo']"]
+            # but does not match, if it is level+key root['foo']['bar']['veg'] include_paths ["root['foo']['bar']['fruits']"]
+            up = level.up
+            while up is not None:
+                if up.path() in self.include_paths:
+                    return False
+                up = up.up
+        if self.include_glob_paths:
+            for gp in self.include_glob_paths:
+                if gp.match_or_is_ancestor(key_path):
+                    return False
         return True
 
     def _get_clean_to_keys_mapping(self, keys: Any, level: Any) -> Dict[Any, Any]:
@@ -701,9 +753,13 @@ def _diff_dict(
         t_keys_removed = t1_keys - t_keys_intersect
 
         if self.threshold_to_diff_deeper:
-            if self.exclude_paths:
+            if self.exclude_paths or self.exclude_glob_paths:
                 t_keys_union = {f"{level.path()}[{repr(key)}]" for key in (t2_keys | t1_keys)}
-                t_keys_union -= self.exclude_paths
+                if self.exclude_paths:
+                    t_keys_union -= self.exclude_paths
+                if self.exclude_glob_paths:
+                    t_keys_union = {k for k in t_keys_union
+                                    if not any(gp.match(k) for gp in self.exclude_glob_paths)}
                 t_keys_union_len = len(t_keys_union)
             else:
                 t_keys_union_len = len(t2_keys | t1_keys)

deepdiff/docstrings/deephash_doc.rst

@@ -32,10 +32,12 @@ exclude_types: list, default = None
 
 exclude_paths: list, default = None
     List of paths to exclude from the report. If only one item, you can pass it as a string instead of a list containing only one path.
+    Supports :ref:`wildcard_paths_label`: use ``[*]`` to match one segment or ``[**]`` to match any depth.
 
 
 include_paths: list, default = None
     List of the only paths to include in the report. If only one item, you can pass it as a string.
+    Supports :ref:`wildcard_paths_label`: use ``[*]`` to match one segment or ``[**]`` to match any depth.
 
 
 exclude_regex_paths: list, default = None

deepdiff/docstrings/diff_doc.rst

@@ -53,6 +53,7 @@ encodings: List, default = None
 exclude_paths: list, default = None
     :ref:`exclude_paths_label`
     List of paths to exclude from the report. If only one item, you can path it as a string.
+    Supports :ref:`wildcard_paths_label`: use ``[*]`` to match one segment or ``[**]`` to match any depth.
 
 exclude_regex_paths: list, default = None
     :ref:`exclude_regex_paths_label`
@@ -74,6 +75,7 @@ exclude_obj_callback_strict: function, default = None
 include_paths: list, default = None
     :ref:`include_paths_label`
     List of the only paths to include in the report. If only one item is in the list, you can pass it as a string.
+    Supports :ref:`wildcard_paths_label`: use ``[*]`` to match one segment or ``[**]`` to match any depth.
 
 include_obj_callback: function, default = None
     :ref:`include_obj_callback_label`

deepdiff/docstrings/exclude_paths.rst

@@ -59,6 +59,49 @@ Example
     {'values_changed': {"root['foo']['bar']": {'new_value': 'banana', 'old_value': 'potato'}}}
 
 
+.. _wildcard_paths_label:
+
+Wildcard (Glob) Paths
+---------------------
+
+Both ``exclude_paths`` and ``include_paths`` support wildcard patterns for matching multiple paths at once:
+
+- ``[*]`` or ``.*`` matches exactly **one** path segment (any key, index, or attribute).
+- ``[**]`` or ``.**`` matches **zero or more** path segments at any depth.
+
+Wildcard patterns must use the full ``root`` prefix (shorthand keys are not supported for wildcards).
+
+Exclude all ``password`` fields regardless of the parent key:
+    >>> t1 = {"users": {"alice": {"name": "Alice", "password": "s1"}, "bob": {"name": "Bob", "password": "s2"}}}
+    >>> t2 = {"users": {"alice": {"name": "Alice", "password": "x1"}, "bob": {"name": "Bob", "password": "x2"}}}
+    >>> DeepDiff(t1, t2, exclude_paths=["root['users'][*]['password']"])
+    {}
+
+Include only ``name`` fields at any depth:
+    >>> t1 = {"a": {"name": "A", "secret": 1}, "b": {"name": "B", "secret": 2}}
+    >>> t2 = {"a": {"name": "X", "secret": 1}, "b": {"name": "Y", "secret": 2}}
+    >>> result = DeepDiff(t1, t2, include_paths=["root[*]['name']"])
+    >>> set(result.get('values_changed', {}).keys()) == {"root['a']['name']", "root['b']['name']"}
+    True
+
+Use ``[**]`` to match at any depth:
+    >>> t1 = {"config": {"db": {"password": "old"}, "cache": {"password": "old"}}}
+    >>> t2 = {"config": {"db": {"password": "new"}, "cache": {"password": "new"}}}
+    >>> DeepDiff(t1, t2, exclude_paths=["root[**]['password']"])
+    {}
+
+Literal keys named ``*`` or ``**`` are not treated as wildcards when quoted:
+    >>> t1 = {"*": 1, "a": 2}
+    >>> t2 = {"*": 10, "a": 20}
+    >>> result = DeepDiff(t1, t2, exclude_paths=["root['*']"])
+    >>> "root['a']" in result.get('values_changed', {})
+    True
+
+When both ``exclude_paths`` and ``include_paths`` apply to the same path, exclusion takes precedence.
+
+Wildcards also work with ``DeepHash`` and ``DeepSearch`` exclude_paths.
+
+
 .. _exclude_regex_paths_label:
 
 Exclude Regex Paths

deepdiff/docstrings/search_doc.rst

@@ -14,6 +14,7 @@ verbose_level : int >= 0, default = 1.
 
 exclude_paths: list, default = None.
     List of paths to exclude from the report.
+    Supports wildcard patterns: use ``[*]`` to match one segment or ``[**]`` to match any depth.
 
 exclude_types: list, default = None.
     List of object types to exclude from the report.

deepdiff/helper.py

@@ -381,6 +381,30 @@ def add_root_to_paths(paths: Optional[Iterable[str]]) -> Optional[SetOrdered]:
     return result
 
 
+def separate_wildcard_and_exact_paths(paths):
+    """Separate a set of paths into exact paths and wildcard pattern paths.
+
+    Returns ``(exact_set_or_none, wildcard_list_or_none)``.
+    Wildcard paths must start with ``root``; a ``ValueError`` is raised otherwise.
+    """
+    if not paths:
+        return None, None
+    from deepdiff.path import path_has_wildcard, compile_glob_paths
+    exact = set()
+    wildcards = []
+    for path in paths:
+        if path_has_wildcard(path):
+            if not path.startswith('root'):
+                raise ValueError(
+                    "Wildcard paths must start with 'root'. Got: {}".format(path))
+            wildcards.append(path)
+        else:
+            exact.add(path)
+    exact_result = exact if exact else None
+    glob_result = compile_glob_paths(wildcards) if wildcards else None
+    return exact_result, glob_result
+
+
 RE_COMPILED_TYPE = type(re.compile(''))