Skip to content

Commit 2c514d1

Browse files
AAriamAAriam
committed
update addon function
1 parent 1d99dbd commit 2c514d1

File tree

1 file changed

+234
-59
lines changed

1 file changed

+234
-59
lines changed

‎pkg/src/pyserials/update.py‎

Lines changed: 234 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
from __future__ import annotations as _annotations
22

3-
import re
43
from typing import TYPE_CHECKING as _TYPE_CHECKING
54
import re as _re
65
from functools import partial as _partial
@@ -11,69 +10,248 @@
1110
import pyserials.exception as _exception
1211

1312
if _TYPE_CHECKING:
14-
from typing import Literal, Sequence, Any, Callable
13+
from typing import Literal, Sequence, Any, Callable, Iterable
14+
UPDATE_OPTIONS = Literal["skip", "write", "raise"] | Callable[
15+
[tuple[Any] | tuple[Any, Any]], None | tuple[Any, str]
16+
]
17+
FUNC_ITEMS = Callable[[Any], Iterable[tuple[Any, Any]]]
18+
FUNC_CONTAINS = Callable[[Any, Any], bool]
19+
FUNC_GET = Callable[[Any, Any], Any]
20+
FUNC_SET = Callable[[Any, Any, Any], None]
21+
FUNC_CONSTRUCT = Callable[[], Any]
22+
RECURSIVE_DTYPE_FUNCS = tuple[FUNC_ITEMS, FUNC_CONTAINS, FUNC_GET, FUNC_SET, FUNC_CONSTRUCT]
23+
24+
25+
def recursive_update(
26+
source: Any,
27+
addon: Any,
28+
recursive_types: dict[type | tuple[type, ...], RECURSIVE_DTYPE_FUNCS] | None = None,
29+
types: dict[type | tuple[type, ...], UPDATE_OPTIONS | tuple[UPDATE_OPTIONS, UPDATE_OPTIONS]] | None = None,
30+
paths: dict[str, UPDATE_OPTIONS] | None = None,
31+
constructor: Callable[[], Any] | None = None,
32+
type_mismatch: UPDATE_OPTIONS = "raise",
33+
) -> dict[str, list[str]]:
34+
"""Recursively update a complex data structure using another data structure.
35+
36+
Parameters
37+
----------
38+
source
39+
Data structure to update in place.
40+
The type of this object must be
41+
defined in the `recursive_types` argument.
42+
addon
43+
Data structure containing additional data to update `source` with.
44+
recursive_types
45+
Definition of recursive data types.
46+
Each key is a type (or tuple of types) used
47+
to identify data types with the `isinstance` function.
48+
Each value is a tuple of three functions:
49+
1. Function to extract items from the data. It must accept an instance
50+
of the type and return an iterable of key-value pairs.
51+
2. Function to check if a key is in the data. It must accept an instance
52+
of the type and a key, and return a boolean.
53+
3. Function to get a value from the data. It must accept an instance
54+
of the type and a key, and return the value.
55+
4. Function to set a key-value pair in the data. It must accept an instance
56+
of the type, a key, and a value, respectively.
57+
5. Function to construct a new instance of the type. It must accept no arguments.
58+
59+
By default, `dict` is defined as follows:
60+
```python
61+
recursive_types = {
62+
dict: (
63+
lambda dic: dic.items(),
64+
lambda dic, key: key in dic,
65+
lambda dic, key: dic[key],
66+
lambda dic, key, value: dic.update({key: value}),
67+
lambda: dict(),
68+
)
69+
}
70+
```
71+
This default argument will be updated/overwritten with any custom types provided.
72+
For example, to add support for a custom class `MyClass`, you can use:
73+
```python
74+
recursive_types = {
75+
MyClass: (
76+
lambda obj: vars(object).items(),
77+
lambda obj, key: hasattr(obj, key),
78+
lambda obj, key: getattr(obj, key),
79+
lambda obj, key, value: setattr(obj, key, value),
80+
lambda: MyClass(),
81+
)
82+
}
83+
```
84+
types
85+
Update behavior for specific data types.
86+
Each key is a type (or tuple of types) used
87+
to identify data types with the `isinstance` function.
88+
Each value is can be single update option for all cases,
89+
or a tuple of two update options for when the corresponding
90+
key/attribute does not exist in the source data, and when it does, respectively.
91+
Each update option can be either a keyword specifying a predefined behavior,
92+
or a function for custom behavior. The available keywords are:
93+
- "skip": Ignore the key/attribute; do not change anything.
94+
- "write": Write the key/attribute in source data with the value from the addon, overwriting if it exists.
95+
96+
A custom function must accept a single argument, a tuple of either one or two values.
97+
If it is two values, the first value is the current value in the source data,
98+
and the second value is the value from the addon.
99+
If it is one value, the value is the value from the addon,
100+
meaning the key/attribute does not exist in the source data.
101+
The function must either return `None` (for when nothing must be changes in the source)
102+
or a tuple of two values:
103+
1. The new value to write in the source data.
104+
2. A string specifying the change type.
105+
106+
By default, the following behavior is defined for basic types:
107+
```python
108+
types = {list: ("write", lambda data: (data[0] + data[1], "append"))}
109+
```
110+
111+
The default behavior for any data type not specified in this argument is
112+
`("write", "skip")`, meaning that the key/attribute will be written in the source data
113+
if it does not exist, and ignored if it does.
114+
paths
115+
Update behavior for specific keys using JSONPath expressions.
116+
This is the same as the `types` argument, but targeting specific keys
117+
instead of data types.
118+
Everything is the same as in the `types` argument, except that the keys
119+
are JSONPath expressions as strings.
120+
constructor
121+
Custom constructor for creating new instances of the source data type.
122+
This is used when the addon data contains a recursive key/attribute
123+
that is not present in the source data. If not provided,
124+
the type of the addon value will be used to create a new instance.
125+
type_mismatch
126+
Behavior for when a key/attribute in the source data
127+
is not a recursive type, but the corresponding key/attribute
128+
in the addon data is a recursive type.
129+
"""
130+
def get_funcs(data: Any) -> RECURSIVE_DTYPE_FUNCS | None:
131+
for typ, funcs in recursive_types.items():
132+
if isinstance(data, typ):
133+
return funcs
134+
return None
15135

136+
def recursive(
137+
src: Any,
138+
add: Any,
139+
src_funcs: RECURSIVE_DTYPE_FUNCS,
140+
add_funcs: RECURSIVE_DTYPE_FUNCS,
141+
path: str,
142+
):
16143

17-
def dict_from_addon(
18-
data: dict,
19-
addon: dict,
20-
append_list: bool = True,
21-
append_dict: bool = True,
22-
raise_duplicates: bool = False,
23-
raise_type_mismatch: bool = True,
24-
) -> dict[str, list[str]]:
25-
"""Recursively update a dictionary from another dictionary."""
26-
def recursive(source: dict, add: dict, path: str, log: dict):
144+
def apply(behavior: UPDATE_OPTIONS | tuple[UPDATE_OPTIONS, UPDATE_OPTIONS]):
145+
action = behavior[int(key_exists_in_src)] if isinstance(behavior, tuple) else behavior
146+
if action == "raise":
147+
raise_error(typ="duplicate")
148+
elif action == "skip":
149+
change_type = "skip"
150+
elif action == "write":
151+
change_type = "write"
152+
fn_src_set(src, key, value)
153+
elif not isinstance(action, str):
154+
out = action((source_value, value) if key_exists_in_src else (value,))
155+
if out:
156+
new_value, change_type = out
157+
fn_src_set(src, key, new_value)
158+
else:
159+
change_type = "skip"
160+
else:
161+
raise ValueError(f"Invalid update behavior '{action}' for key '{key}' at path '{path}'.")
162+
log[fullpath] = (type(source_value) if key_exists_in_src else None, type(value), change_type)
163+
return
27164

28165
def raise_error(typ: Literal["duplicate", "type_mismatch"]):
29-
raise _exception.update.PySerialsUpdateDictFromAddonError(
166+
raise _exception.update.PySerialsUpdateRecursiveDataError(
30167
problem_type=typ,
31168
path=fullpath,
32-
data=source[key],
33-
data_full=data,
169+
data=src[key],
170+
data_full=src,
34171
data_addon=value,
35172
data_addon_full=addon,
36173
)
37174

38-
for key, value in add.items():
175+
_, fn_src_contains, fn_src_get, fn_src_set, _ = src_funcs
176+
fn_add_items, _, _, _, fn_add_construct = add_funcs
177+
178+
for key, value in fn_add_items(add):
39179
fullpath = f"{path}.{key}"
40-
if key not in source:
41-
log["added"].append(fullpath)
42-
source[key] = value
43-
continue
44-
if type(source[key]) is not type(value):
45-
if raise_type_mismatch:
46-
raise_error(typ="type_mismatch")
47-
continue
48-
if not isinstance(value, (list, dict)):
49-
if raise_duplicates:
50-
raise_error(typ="duplicate")
51-
log["skipped"].append(fullpath)
52-
elif isinstance(value, list):
53-
if append_list:
54-
appended = False
55-
for elem in value:
56-
if elem not in source[key]:
57-
source[key].append(elem)
58-
appended = True
59-
if appended:
60-
log["list_appended"].append(fullpath)
61-
elif raise_duplicates:
62-
raise_error(typ="duplicate")
63-
else:
64-
log["skipped"].append(fullpath)
180+
full_jpath = _jsonpath.parse(f"{path}.'{key}'") # quote to avoid JSONPath syntax errors
181+
key_exists_in_src = fn_src_contains(src, key)
182+
source_value = fn_src_get(src, key) if key_exists_in_src else None
183+
for jpath_str, matches in jsonpath_match.items():
184+
if full_jpath in matches:
185+
apply(paths[jpath_str])
186+
break
65187
else:
66-
if append_dict:
67-
recursive(source=source[key], add=value, path=f"{fullpath}.", log=log)
68-
elif raise_duplicates:
69-
raise_error(typ="duplicate")
188+
for typ, action in type_to_arg.items():
189+
if isinstance(value, typ):
190+
apply(action)
191+
break
70192
else:
71-
log["skipped"].append(fullpath)
72-
return log
73-
full_log = recursive(
74-
source=data, add=addon, path="$", log={"added": [], "skipped": [], "list_appended": []}
193+
funcs_value = get_funcs(value)
194+
if funcs_value:
195+
# Value is a recursive type
196+
if key_exists_in_src:
197+
funcs_src_value = get_funcs(source_value)
198+
if not funcs_src_value:
199+
# Source value is not a recursive type
200+
apply(type_mismatch)
201+
else:
202+
recursive(
203+
src=fn_src_get(src, key),
204+
add=value,
205+
path=fullpath,
206+
src_funcs=src_funcs,
207+
add_funcs=funcs_value,
208+
)
209+
else:
210+
# Source value does not exist; create a new instance
211+
new_instance = constructor() if constructor else fn_add_construct()
212+
fn_src_set(src, key, new_instance)
213+
funcs_src_value = get_funcs(new_instance)
214+
recursive(
215+
src=new_instance,
216+
add=value,
217+
path=fullpath,
218+
src_funcs=funcs_src_value,
219+
add_funcs=funcs_value,
220+
)
221+
else:
222+
# addon value is of a non-recursive type that does not have any defined behavior;
223+
# Apply the default behavior for of ("write", "skip") for the key.
224+
apply("skip" if key_exists_in_src else "write")
225+
return
226+
227+
type_to_arg = {list: ("write", lambda data: (data[0] + data[1], "append"))} | (types or {})
228+
recursive_types = {
229+
dict: (
230+
lambda dic: dic.items(),
231+
lambda dic, key: key in dic,
232+
lambda dic, key: dic[key],
233+
lambda dic, key, value: dic.update({key: value}),
234+
lambda: dict(),
235+
)
236+
} | (recursive_types or {})
237+
jsonpath_match = {
238+
jpath_str: [match.full_path for match in _jsonpath.parse(jpath_str).find(addon)]
239+
for jpath_str in (paths or {}).keys()
240+
}
241+
log = {}
242+
funcs_src = get_funcs(source)
243+
funcs_add = get_funcs(addon)
244+
for funcs, param_name, data in ((funcs_src, "source", source), (funcs_add, "addon", addon)):
245+
if not funcs:
246+
raise ValueError(f"Data type '{type(data)}' of '{param_name}' is not provided in 'recursive_types'.")
247+
recursive(
248+
src=source,
249+
add=addon,
250+
path="$",
251+
src_funcs=funcs_src,
252+
add_funcs=funcs_add,
75253
)
76-
return full_log
254+
return log
77255

78256

79257
def data_from_jsonschema(data: dict | list, schema: dict) -> None:
@@ -459,16 +637,12 @@ def raise_error(path_invalid: str, description_template: str, exception: Excepti
459637
for addon_dict, addon_settings in sorted(
460638
addons, key=lambda addon: addon[1].get("priority", 0) if addon[1] else 0
461639
):
462-
addon_settings = addon_settings or {}
463-
dict_from_addon(
464-
data=new_dict,
640+
addon_settings = {k: v for k, v in (addon_settings or {}).items() if k not in ("priority",)}
641+
recursive_update(
642+
source=new_dict,
465643
addon=addon_dict,
466-
append_list=addon_settings.get("append_list", True),
467-
append_dict=addon_settings.get("append_dict", True),
468-
raise_duplicates=addon_settings.get("raise_duplicates", False),
469-
raise_type_mismatch=addon_settings.get("raise_type_mismatch", True),
644+
**addon_settings,
470645
)
471-
472646
if not is_relative_template:
473647
self._visited_paths[current_path] = (new_dict, True)
474648
return new_dict
@@ -548,7 +722,7 @@ class _RegexPattern:
548722
def __init__(self, start: str, end: str):
549723
start_esc = _re.escape(start)
550724
end_esc = _re.escape(end)
551-
self.pattern = _re.compile(rf"{start_esc}(.*?)(?={end_esc}){end_esc}", re.DOTALL)
725+
self.pattern = _re.compile(rf"{start_esc}(.*?)(?={end_esc}){end_esc}", _re.DOTALL)
552726
return
553727

554728
def fullmatch(self, string: str) -> _re.Match | None:
@@ -561,3 +735,4 @@ def fullmatch(self, string: str) -> _re.Match | None:
561735

562736
def sub(self, repl, string: str) -> str:
563737
return self.pattern.sub(repl, string)
738+

0 commit comments

Comments
 (0)