Skip to content

Commit dffcb1d

Browse files
committed
feat: preserve Symbol types in JSON serialization
- Add infos_symbol_keys metadata field to track Symbol values - Implement path-based Symbol restoration during deserialization - Update _serialize_infos and _deserialize_infos to handle Symbol tracking - Fix type conversion issues with JSON3.Array using collect() - Update test expectations to verify Symbol preservation - All 1722 tests pass Closes part of #217
1 parent 46bafb1 commit dffcb1d

3 files changed

Lines changed: 140 additions & 22 deletions

File tree

ext/CTModelsJSON.jl

Lines changed: 60 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -22,63 +22,98 @@ _apply_over_grid(::Nothing, grid) = nothing
2222
"""
2323
Convert Dict{Symbol,Any} to Dict{String,Any} for JSON serialization.
2424
Only serializes JSON-compatible types (numbers, strings, bools, arrays, dicts).
25+
Returns a tuple: (serialized_dict, symbol_keys) where symbol_keys tracks which values were Symbols.
2526
"""
26-
function _serialize_infos(infos::Dict{Symbol,Any})::Dict{String,Any}
27+
function _serialize_infos(infos::Dict{Symbol,Any})::Tuple{Dict{String,Any},Vector{String}}
2728
result = Dict{String,Any}()
29+
symbol_keys = String[]
2830
for (k, v) in infos
29-
result[string(k)] = _serialize_value(v)
31+
key_str = string(k)
32+
serialized_value, nested_symbols = _serialize_value(v, key_str)
33+
result[key_str] = serialized_value
34+
append!(symbol_keys, nested_symbols)
3035
end
31-
return result
36+
return (result, symbol_keys)
3237
end
3338

3439
"""
3540
Serialize a single value to JSON-compatible format.
41+
Returns a tuple: (serialized_value, symbol_paths) where symbol_paths tracks Symbol locations.
3642
"""
37-
function _serialize_value(v)
43+
function _serialize_value(v, path::String="")
3844
if v isa Number || v isa String || v isa Bool || isnothing(v)
39-
return v
45+
return (v, String[])
4046
elseif v isa Symbol
41-
return string(v)
47+
# Mark this path as containing a Symbol
48+
return (string(v), [path])
4249
elseif v isa AbstractVector
43-
return [_serialize_value(x) for x in v]
50+
serialized = []
51+
all_symbols = String[]
52+
for (i, x) in enumerate(v)
53+
val, syms = _serialize_value(x, "$(path)[$(i-1)]")
54+
push!(serialized, val)
55+
append!(all_symbols, syms)
56+
end
57+
return (serialized, all_symbols)
4458
elseif v isa AbstractDict
4559
result = Dict{String,Any}()
60+
all_symbols = String[]
4661
for (dk, dv) in v
47-
result[string(dk)] = _serialize_value(dv)
62+
key_str = string(dk)
63+
new_path = isempty(path) ? key_str : "$(path).$(key_str)"
64+
val, syms = _serialize_value(dv, new_path)
65+
result[key_str] = val
66+
append!(all_symbols, syms)
4867
end
49-
return result
68+
return (result, all_symbols)
5069
else
5170
# For non-serializable types, convert to string representation
52-
return string(v)
71+
return (string(v), String[])
5372
end
5473
end
5574

5675
"""
5776
Convert Dict{String,Any} back to Dict{Symbol,Any} after JSON deserialization.
77+
Uses symbol_keys metadata to restore Symbol types where they were originally present.
5878
"""
59-
function _deserialize_infos(blob)::Dict{Symbol,Any}
79+
function _deserialize_infos(
80+
blob, symbol_keys::Vector{String}=String[]
81+
)::Dict{Symbol,Any}
6082
if isnothing(blob) || isempty(blob)
6183
return Dict{Symbol,Any}()
6284
end
6385
result = Dict{Symbol,Any}()
6486
for (k, v) in blob
65-
result[Symbol(k)] = _deserialize_value(v)
87+
result[Symbol(k)] = _deserialize_value(v, String(k), symbol_keys)
6688
end
6789
return result
6890
end
6991

7092
"""
7193
Deserialize a single value from JSON format.
94+
Uses symbol_keys to restore Symbol types at the correct paths.
7295
"""
73-
function _deserialize_value(v)
74-
if v isa Number || v isa String || v isa Bool || isnothing(v)
96+
function _deserialize_value(v, path::String, symbol_keys::Vector{String})
97+
if v isa Number || v isa Bool || isnothing(v)
7598
return v
99+
elseif v isa String
100+
# Check if this path should be a Symbol
101+
if path in symbol_keys
102+
return Symbol(v)
103+
else
104+
return v
105+
end
76106
elseif v isa AbstractVector
77-
return [_deserialize_value(x) for x in v]
107+
return [
108+
_deserialize_value(x, "$(path)[$(i-1)]", symbol_keys) for
109+
(i, x) in enumerate(v)
110+
]
78111
elseif v isa AbstractDict
79112
result = Dict{Symbol,Any}()
80113
for (dk, dv) in v
81-
result[Symbol(dk)] = _deserialize_value(dv)
114+
key_str = string(dk)
115+
new_path = isempty(path) ? key_str : "$(path).$(key_str)"
116+
result[Symbol(dk)] = _deserialize_value(dv, new_path, symbol_keys)
82117
end
83118
return result
84119
else
@@ -145,10 +180,13 @@ function CTModels.export_ocp_solution(
145180
"boundary_constraints_dual" => CTModels.boundary_constraints_dual(sol), # ctVector or Nothing
146181
"variable_constraints_lb_dual" => CTModels.variable_constraints_lb_dual(sol), # ctVector or Nothing
147182
"variable_constraints_ub_dual" => CTModels.variable_constraints_ub_dual(sol), # ctVector or Nothing
148-
# Additional solver infos (Dict{Symbol,Any} → Dict{String,Any} for JSON)
149-
"infos" => _serialize_infos(CTModels.infos(sol)),
150183
)
151184

185+
# Serialize infos and get Symbol type metadata
186+
infos_serialized, symbol_keys = _serialize_infos(CTModels.infos(sol))
187+
blob["infos"] = infos_serialized
188+
blob["infos_symbol_keys"] = symbol_keys
189+
152190
open(filename * ".json", "w") do io
153191
JSON3.pretty(io, blob)
154192
end
@@ -293,9 +331,11 @@ function CTModels.import_ocp_solution(
293331
variable_constraints_ub_dual = Vector{Float64}(blob["variable_constraints_ub_dual"])
294332
end
295333

296-
# get additional solver infos
334+
# get additional solver infos with Symbol type restoration
335+
symbol_keys_raw = get(blob, "infos_symbol_keys", String[])
336+
symbol_keys = collect(String, symbol_keys_raw) # Convert JSON3.Array/empty array to Vector{String}
297337
infos = if haskey(blob, "infos")
298-
_deserialize_infos(blob["infos"])
338+
_deserialize_infos(blob["infos"], symbol_keys)
299339
else
300340
Dict{Symbol,Any}()
301341
end
Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
Add idempotence tests for export/import serialization
2+
3+
## Summary
4+
5+
This PR adds comprehensive idempotence tests for the `export_ocp_solution` and `import_ocp_solution` functions to verify that multiple export-import cycles produce stable results with no progressive information loss.
6+
7+
## Changes
8+
9+
### Test Implementation (~460 lines)
10+
11+
**Helper Functions** (`test/suite/serialization/test_export_import.jl`):
12+
- `compare_trajectories`: Compares function-based trajectories at time points
13+
- `compare_infos`: Deep comparison of `Dict{Symbol,Any}` with type awareness
14+
- `compare_solutions`: Comprehensive Solution object comparison with configurable tolerances
15+
16+
**New Test Cases** (7 total):
17+
- **JSON** (4 tests): Double/triple cycles with duals, without duals, complex infos
18+
- **JLD2** (3 tests): Double/triple cycles with duals, without duals
19+
20+
### Documentation
21+
22+
**Analysis**: `reports/2026-01-29_Idempotence/analysis/01_serialization_idempotence_analysis.md`
23+
- Identified 6 potential information loss points
24+
- Analyzed existing test coverage
25+
- Future investigation items (function serialization, deepcopy usage)
26+
27+
**Implementation Plan**: `reports/2026-01-29_Idempotence/reference/01_serialization_idempotence_plan.md`
28+
- Detailed test strategy and verification plan
29+
30+
**Walkthrough**: `reports/2026-01-29_Idempotence/walkthrough.md`
31+
- Summary of changes and test results
32+
- Key findings and recommendations
33+
34+
## Test Results
35+
36+
```
37+
Test Summary: | Pass Total Time
38+
CTModels tests | 1721 1721 14.4s
39+
suite/serialization/test_export_import.jl | 1721 1721 14.4s
40+
Testing CTModels tests passed
41+
```
42+
43+
✅ All tests pass - No regressions
44+
45+
## Key Findings
46+
47+
### Information Preserved ✅
48+
- All scalar fields (objective, iterations, status, etc.)
49+
- Time grid and variable (full precision)
50+
- All trajectories (state, control, costate)
51+
- All dual variables
52+
- Infos dictionary structure and values
53+
54+
### Expected Transformations 🔄
55+
1. **Functions → Discretization**: Analytical functions become interpolated after JSON export/import
56+
- Impact: Minimal (within `atol=1e-8`)
57+
- **Idempotent after first cycle**
58+
59+
2. **Symbols → Strings**: Symbols in `infos` become strings after JSON serialization
60+
- Example: `:optimal``"optimal"`
61+
- **Idempotent after first cycle**
62+
63+
### Conclusion
64+
**No progressive information loss**: `sol₁ ≈ sol₂ ≈ sol₃` after multiple cycles.
65+
66+
## Future Work
67+
68+
The analysis identified areas for future investigation:
69+
- Bidirectional `ctinterpolate`/`ctdeinterpolate` for lossless function serialization
70+
- Review of `deepcopy` usage in `build_solution` (rationale unclear)
71+
- Improved JLD2 handling of anonymous functions
72+
73+
See analysis document for details.
74+
75+
## Related Issue
76+
77+
Closes #217

test/suite/serialization/test_export_import.jl

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -851,8 +851,9 @@ function test_export_import()
851851
Test.@test infos2[:nested][:a] == 1
852852
Test.@test infos2[:nested][:b] == "test"
853853
Test.@test infos2[:nested][:c] == [1.0, 2.0, 3.0]
854-
# Symbol becomes string after JSON serialization
855-
Test.@test infos2[:symbol_value] == "optimal"
854+
# Symbol is now preserved with type metadata!
855+
Test.@test infos2[:symbol_value] == :optimal
856+
Test.@test infos2[:symbol_value] isa Symbol
856857

857858
remove_if_exists("idempotence_json_ci1.json")
858859
remove_if_exists("idempotence_json_ci2.json")

0 commit comments

Comments
 (0)