From eee4b3b6aabeda3f274036f454a43e2c3a70df50 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Mon, 29 Jun 2026 23:10:27 -0700 Subject: [PATCH 1/6] fox docs --- docs/src/docs_best_practices/how-to/troubleshoot.md | 2 +- docs/src/docs_best_practices/how-to/write_a_tutorial.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/docs_best_practices/how-to/troubleshoot.md b/docs/src/docs_best_practices/how-to/troubleshoot.md index 995c1e63b..c520c9b5e 100644 --- a/docs/src/docs_best_practices/how-to/troubleshoot.md +++ b/docs/src/docs_best_practices/how-to/troubleshoot.md @@ -82,7 +82,7 @@ Pkg.add(["PowerSystems"]); 2. Run the formatter once to format the rest of the file 3. Add the text back in 4. Add the `ignore` keyword argument with the file name to - [`JuliaFormatter.format`](@extref `JuliaFormatter.format-Tuple{Any}`) in `scripts/formatter/formatter_code.jl` + [`JuliaFormatter.format`](@extref) in `scripts/formatter/formatter_code.jl` to skip the file in the future: ```julia diff --git a/docs/src/docs_best_practices/how-to/write_a_tutorial.md b/docs/src/docs_best_practices/how-to/write_a_tutorial.md index 545fdd6c4..78892a6b8 100644 --- a/docs/src/docs_best_practices/how-to/write_a_tutorial.md +++ b/docs/src/docs_best_practices/how-to/write_a_tutorial.md @@ -56,7 +56,7 @@ code blocks. !!! tip "Do" Display all code, starting from `using SomeSiennaPackage`. Example: See - [Working with Time Series](@extref tutorial_time_series). + [Working with Time Series](@extref PowerSystems Working-with-Time-Series-Data). !!! warning "Don't" From 7536bdab9dea6d4afee685dfcf7db8bac6bef871 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Mon, 29 Jun 2026 23:10:38 -0700 Subject: [PATCH 2/6] update claude.md --- .claude/claude.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.claude/claude.md b/.claude/claude.md index 06e46369a..4066f1d00 100644 --- a/.claude/claude.md +++ b/.claude/claude.md @@ -27,7 +27,15 @@ Key files: - `production_variable_cost_curve.jl` — `CostCurve{T,U}` / `FuelCurve{T,U}` (units in the type parameter) - `relative_units.jl` — `RelativeUnits` submodule: unit-system markers and `RelativeQuantity` - `outputs.jl` — abstract `Outputs` interface (not-implemented stubs for downstream packages) -- `serialization.jl` — JSON-based serialization (stdlib-adjacent `JSON`; JSON3/StructTypes were removed) +- `geographic_supplemental_attribute.jl` — `GeographicInfo` supplemental attribute (hand-written template) +- `data_source_supplemental_attribute.jl` — `DataSource` supplemental attribute: data-provenance record + (organization, dataset, url, version, retrieved/published timestamps, confidence, recorded_by, covered + fields, `extra`). Mutable with `get_*`/`set_*!`; `published_at`/`recorded_by` are `Union{Nothing,_}` with + dispatch-based `has_*`/`get_*` predicate accessors. Shared across components via the association store + (create-once / link-many). Hand-written, not exported. +- `serialization.jl` — JSON-based serialization (stdlib-adjacent `JSON`; JSON3/StructTypes were removed). + Per-type deserialize methods narrow JSON's loose types to declared field types: optional `DateTime` + (`Union{Nothing, Dates.DateTime}`) and `Vector{String}` (JSON arrays parse to `Vector{Any}`). Subdirectories: - `function_data/` — `FunctionData` hierarchy, time-series-backed function data, convexity/validity checks From edec32918c44e8988ec364cc7762dd74afbcea59 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Mon, 29 Jun 2026 23:11:23 -0700 Subject: [PATCH 3/6] add new supplemental attribute for data source --- src/InfrastructureSystems.jl | 1 + src/data_source_supplemental_attribute.jl | 256 ++++++++++++++++++++++ 2 files changed, 257 insertions(+) create mode 100644 src/data_source_supplemental_attribute.jl diff --git a/src/InfrastructureSystems.jl b/src/InfrastructureSystems.jl index 38eb49737..9abafa381 100644 --- a/src/InfrastructureSystems.jl +++ b/src/InfrastructureSystems.jl @@ -184,6 +184,7 @@ include("containers.jl") include("component_container.jl") include("component_uuids.jl") include("geographic_supplemental_attribute.jl") +include("data_source_supplemental_attribute.jl") include("generated/includes.jl") include("time_series_parser.jl") include("single_time_series.jl") diff --git a/src/data_source_supplemental_attribute.jl b/src/data_source_supplemental_attribute.jl new file mode 100644 index 000000000..177c36695 --- /dev/null +++ b/src/data_source_supplemental_attribute.jl @@ -0,0 +1,256 @@ +""" + mutable struct DataSource <: SupplementalAttribute + +A supplemental attribute recording the provenance of component data: the publishing +organization, dataset, URL, version (vintage), publication and retrieval timestamps, +confidence, the user/agent that recorded the value, and the component field names the +record applies to. Attach one instance to many components to avoid repeated entries. + +# Arguments +$(DocStringExtensions.TYPEDFIELDS) +""" +mutable struct DataSource <: SupplementalAttribute + "Publishing organization (required), e.g. \"U.S. Energy Information Administration\"." + organization::String + "When the data was retrieved (required)." + retrieved_at::Dates.DateTime + "Dataset identifier, e.g. \"EIA-860 2023, Schedule 3\"." + dataset::String + "Source URL." + url::String + "Data version / vintage, e.g. \"2023 final\"." + version::String + "When the source published the data; `nothing` if unknown." + published_at::Union{Nothing, Dates.DateTime} + "Confidence qualifier, e.g. \"high\", \"medium\"." + confidence::String + "User or agent that recorded the value; `nothing` if unrecorded." + recorded_by::Union{Nothing, String} + "Component field names this record applies to (no meaning is attached by IS)." + fields::Vector{String} + "Escape hatch for additional, evolving provenance keys." + extra::Dict{String, Any} + internal::InfrastructureSystemsInternal +end + +function DataSource(; + organization::String, + retrieved_at::Dates.DateTime, + fields::Vector{String}, + dataset::String = "", + url::String = "", + version::String = "", + published_at::Union{Nothing, Dates.DateTime} = nothing, + confidence::String = "", + recorded_by::Union{Nothing, String} = nothing, + extra::Dict{String, Any} = Dict{String, Any}(), + internal::InfrastructureSystemsInternal = InfrastructureSystemsInternal(), +) + return DataSource( + organization, + retrieved_at, + dataset, + url, + version, + published_at, + confidence, + recorded_by, + fields, + extra, + internal, + ) +end + +""" +$(DocStringExtensions.TYPEDSIGNATURES) +Return the publishing organization. +""" +get_organization(ds::DataSource) = ds.organization + +""" +$(DocStringExtensions.TYPEDSIGNATURES) +Set the publishing organization. +""" +function set_organization!(ds::DataSource, val::String) + ds.organization = val + return nothing +end + +""" +$(DocStringExtensions.TYPEDSIGNATURES) +Return the retrieval timestamp. +""" +get_retrieved_at(ds::DataSource) = ds.retrieved_at + +""" +$(DocStringExtensions.TYPEDSIGNATURES) +Set the retrieval timestamp. +""" +function set_retrieved_at!(ds::DataSource, val::Dates.DateTime) + ds.retrieved_at = val + return nothing +end + +""" +$(DocStringExtensions.TYPEDSIGNATURES) +Return the dataset identifier. +""" +get_dataset(ds::DataSource) = ds.dataset + +""" +$(DocStringExtensions.TYPEDSIGNATURES) +Set the dataset identifier. +""" +function set_dataset!(ds::DataSource, val::String) + ds.dataset = val + return nothing +end + +""" +$(DocStringExtensions.TYPEDSIGNATURES) +Return the source URL. +""" +get_url(ds::DataSource) = ds.url + +""" +$(DocStringExtensions.TYPEDSIGNATURES) +Set the source URL. +""" +function set_url!(ds::DataSource, val::String) + ds.url = val + return nothing +end + +""" +$(DocStringExtensions.TYPEDSIGNATURES) +Return the data version (vintage). +""" +get_version(ds::DataSource) = ds.version + +""" +$(DocStringExtensions.TYPEDSIGNATURES) +Set the data version (vintage). +""" +function set_version!(ds::DataSource, val::String) + ds.version = val + return nothing +end + +""" +$(DocStringExtensions.TYPEDSIGNATURES) +Return the confidence qualifier. +""" +get_confidence(ds::DataSource) = ds.confidence + +""" +$(DocStringExtensions.TYPEDSIGNATURES) +Set the confidence qualifier. +""" +function set_confidence!(ds::DataSource, val::String) + ds.confidence = val + return nothing +end + +""" +$(DocStringExtensions.TYPEDSIGNATURES) +Return the component field names this record applies to. +""" +get_fields(ds::DataSource) = ds.fields + +""" +$(DocStringExtensions.TYPEDSIGNATURES) +Set the component field names this record applies to. +""" +function set_fields!(ds::DataSource, val::Vector{String}) + ds.fields = val + return nothing +end + +""" +$(DocStringExtensions.TYPEDSIGNATURES) +Append a component field name to `fields`. +""" +function add_field!(ds::DataSource, name::String) + push!(ds.fields, name) + return nothing +end + +""" +$(DocStringExtensions.TYPEDSIGNATURES) +Return the extra provenance dictionary. +""" +get_extra(ds::DataSource) = ds.extra + +""" +$(DocStringExtensions.TYPEDSIGNATURES) +Set the extra provenance dictionary. +""" +function set_extra!(ds::DataSource, val::Dict{String, Any}) + ds.extra = val + return nothing +end + +""" +$(DocStringExtensions.TYPEDSIGNATURES) +Return `true` if `published_at` is set. +""" +has_published_at(ds::DataSource) = _has_published_at(ds.published_at) +_has_published_at(::Nothing) = false +_has_published_at(::Dates.DateTime) = true + +""" +$(DocStringExtensions.TYPEDSIGNATURES) +Return the publication timestamp. Guard with [`has_published_at`](@ref) — throws if unset. +""" +get_published_at(ds::DataSource) = _get_published_at(ds.published_at) +_get_published_at(val::Dates.DateTime) = val +function _get_published_at(::Nothing) + throw( + ArgumentError( + "published_at is not set on this DataSource; guard with has_published_at", + ), + ) +end + +""" +$(DocStringExtensions.TYPEDSIGNATURES) +Set the publication timestamp. +""" +function set_published_at!(ds::DataSource, val::Dates.DateTime) + ds.published_at = val + return nothing +end + +""" +$(DocStringExtensions.TYPEDSIGNATURES) +Return `true` if `recorded_by` is set. +""" +has_recorded_by(ds::DataSource) = _has_recorded_by(ds.recorded_by) +_has_recorded_by(::Nothing) = false +_has_recorded_by(::String) = true + +""" +$(DocStringExtensions.TYPEDSIGNATURES) +Return the recorder. Guard with [`has_recorded_by`](@ref) — throws if unset. +""" +get_recorded_by(ds::DataSource) = _get_recorded_by(ds.recorded_by) +_get_recorded_by(val::String) = val +function _get_recorded_by(::Nothing) + throw( + ArgumentError( + "recorded_by is not set on this DataSource; guard with has_recorded_by", + ), + ) +end + +""" +$(DocStringExtensions.TYPEDSIGNATURES) +Set the recorder. +""" +function set_recorded_by!(ds::DataSource, val::String) + ds.recorded_by = val + return nothing +end + +get_internal(ds::DataSource) = ds.internal +get_uuid(ds::DataSource) = get_uuid(get_internal(ds)) From ece9c499f7e3111566cab710b81ac3919d80026c Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Mon, 29 Jun 2026 23:11:43 -0700 Subject: [PATCH 4/6] add missing serialization methods --- src/serialization.jl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/serialization.jl b/src/serialization.jl index b4096bb12..c160da9e0 100644 --- a/src/serialization.jl +++ b/src/serialization.jl @@ -283,6 +283,12 @@ end deserialize(::Type{Dates.DateTime}, val::AbstractString) = Dates.DateTime(val) +# An optional DateTime field serializes its value to a JSON string; the plain-DateTime +# method above does not cover the Union, so the raw string would otherwise reach the +# constructor untyped. The `nothing` case is handled generically. +deserialize(::Type{Union{Nothing, Dates.DateTime}}, val::AbstractString) = + Dates.DateTime(val) + # The next methods fix serialization of UUIDs. The underlying type of a UUID is a UInt128. # JSON tries to encode this as a number in JSON. Encoding integers greater than can # be stored in a signed 64-bit integer sometimes does not work - at least when using @@ -311,6 +317,9 @@ deserialize(::Type{Complex{T}}, data::Dict) where {T} = Complex(T(data["real"]), T(data["imag"])) deserialize(::Type{Vector{Symbol}}, data::Vector) = Symbol.(data) +# JSON arrays parse to Vector{Any}; narrow to the declared element type for typed fields. +# Explicit comprehension keeps the empty case concrete (`String.(Any[])` stays Vector{Any}). +deserialize(::Type{Vector{String}}, data::Vector) = String[String(x) for x in data] serialize(value::Vector{Complex{T}}) where {T} = [Dict("real" => real(x), "imag" => imag(x)) for x in value] deserialize(::Type{Vector{Complex{T}}}, data::Array) where {T} = From b90c100608d28fbc4ded50ad396dcbb5844e14fb Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Mon, 29 Jun 2026 23:11:51 -0700 Subject: [PATCH 5/6] update tests --- test/test_serialization.jl | 5 ++ test/test_supplemental_attributes.jl | 116 +++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) diff --git a/test/test_serialization.jl b/test/test_serialization.jl index 7a42a7b45..77e154633 100644 --- a/test/test_serialization.jl +++ b/test/test_serialization.jl @@ -288,3 +288,8 @@ end _, result = validate_serialization(sys2) @test result end + +@testset "Deserialize optional DateTime field" begin + @test IS.deserialize(Union{Nothing, Dates.DateTime}, "2024-09-01T00:00:00") == + Dates.DateTime("2024-09-01T00:00:00") +end diff --git a/test/test_supplemental_attributes.jl b/test/test_supplemental_attributes.jl index 353e55770..79ee17e9b 100644 --- a/test/test_supplemental_attributes.jl +++ b/test/test_supplemental_attributes.jl @@ -339,3 +339,119 @@ end attr = IS.TestSupplemental(; value = 42.0) @test IS.get_attr_value(attr) == 42.0 end + +@testset "DataSource construction and plain accessors" begin + ds = IS.DataSource(; + organization = "U.S. EIA", + retrieved_at = Dates.DateTime("2026-06-20T14:03:00"), + fields = ["operation_cost", "rating"], + ) + @test IS.get_organization(ds) == "U.S. EIA" + @test IS.get_retrieved_at(ds) == Dates.DateTime("2026-06-20T14:03:00") + @test IS.get_fields(ds) == ["operation_cost", "rating"] + @test IS.get_dataset(ds) == "" + @test IS.get_url(ds) == "" + @test IS.get_version(ds) == "" + @test IS.get_confidence(ds) == "" + @test IS.get_extra(ds) == Dict{String, Any}() + @test IS.get_uuid(ds) == IS.get_uuid(IS.get_internal(ds)) + + IS.set_organization!(ds, "NREL") + IS.set_dataset!(ds, "EIA-860 2023, Schedule 3") + IS.set_url!(ds, "https://www.eia.gov/electricity/data/eia860/") + IS.set_version!(ds, "2023 final") + IS.set_confidence!(ds, "high") + IS.set_retrieved_at!(ds, Dates.DateTime("2026-06-21T00:00:00")) + IS.set_fields!(ds, ["rating"]) + IS.set_extra!(ds, Dict{String, Any}("license" => "Public Domain")) + @test IS.get_organization(ds) == "NREL" + @test IS.get_dataset(ds) == "EIA-860 2023, Schedule 3" + @test IS.get_url(ds) == "https://www.eia.gov/electricity/data/eia860/" + @test IS.get_version(ds) == "2023 final" + @test IS.get_confidence(ds) == "high" + @test IS.get_retrieved_at(ds) == Dates.DateTime("2026-06-21T00:00:00") + @test IS.get_fields(ds) == ["rating"] + @test IS.get_extra(ds)["license"] == "Public Domain" +end + +@testset "DataSource optional fields and add_field!" begin + ds = IS.DataSource(; + organization = "U.S. EIA", + retrieved_at = Dates.DateTime("2026-06-20T14:03:00"), + fields = String[], + ) + @test IS.has_published_at(ds) == false + @test_throws ArgumentError IS.get_published_at(ds) + @test IS.has_recorded_by(ds) == false + @test_throws ArgumentError IS.get_recorded_by(ds) + + IS.set_published_at!(ds, Dates.DateTime("2024-09-01T00:00:00")) + IS.set_recorded_by!(ds, "sienna-ingest@2.3.1") + @test IS.has_published_at(ds) == true + @test IS.get_published_at(ds) == Dates.DateTime("2024-09-01T00:00:00") + @test IS.has_recorded_by(ds) == true + @test IS.get_recorded_by(ds) == "sienna-ingest@2.3.1" + + IS.add_field!(ds, "operation_cost") + IS.add_field!(ds, "rating") + @test IS.get_fields(ds) == ["operation_cost", "rating"] +end + +@testset "DataSource shared across components" begin + data = IS.SystemData() + ds = IS.DataSource(; + organization = "U.S. EIA", + retrieved_at = Dates.DateTime("2026-06-20T14:03:00"), + fields = String[], + ) + component1 = IS.TestComponent("component1", 5) + component2 = IS.TestComponent("component2", 7) + IS.add_component!(data, component1) + IS.add_component!(data, component2) + IS.add_supplemental_attribute!(data, component1, ds) + IS.add_supplemental_attribute!(data, component2, ds) + @test IS.get_num_supplemental_attributes(data) == 1 + @test length(collect(IS.get_supplemental_attributes(IS.DataSource, data))) == 1 + + IS.remove_supplemental_attribute!(data, component1, ds) + @test IS.get_num_supplemental_attributes(data) == 1 + IS.remove_supplemental_attribute!(data, component2, ds) + @test IS.get_num_supplemental_attributes(data) == 0 +end + +@testset "DataSource serialization round-trip" begin + ds = IS.DataSource(; + organization = "U.S. EIA", + retrieved_at = Dates.DateTime("2026-06-20T14:03:00"), + fields = ["operation_cost"], + dataset = "EIA-860 2023, Schedule 3", + url = "https://www.eia.gov/electricity/data/eia860/", + version = "2023 final", + published_at = Dates.DateTime("2024-09-01T00:00:00"), + confidence = "high", + recorded_by = "sienna-ingest@2.3.1", + extra = Dict{String, Any}("license" => "Public Domain"), + ) + back = IS.from_json(IS.to_json(ds), IS.DataSource) + @test IS.get_organization(back) == IS.get_organization(ds) + @test IS.get_retrieved_at(back) == IS.get_retrieved_at(ds) + @test IS.get_dataset(back) == IS.get_dataset(ds) + @test IS.get_url(back) == IS.get_url(ds) + @test IS.get_version(back) == IS.get_version(ds) + @test IS.get_published_at(back) == IS.get_published_at(ds) + @test IS.get_confidence(back) == IS.get_confidence(ds) + @test IS.get_recorded_by(back) == IS.get_recorded_by(ds) + @test IS.get_fields(back) == IS.get_fields(ds) + @test IS.get_extra(back)["license"] == "Public Domain" + @test IS.get_uuid(back) == IS.get_uuid(ds) + + ds2 = IS.DataSource(; + organization = "NREL", + retrieved_at = Dates.DateTime("2026-04-15T00:00:00"), + fields = String[], + ) + back2 = IS.from_json(IS.to_json(ds2), IS.DataSource) + @test IS.has_published_at(back2) == false + @test IS.has_recorded_by(back2) == false + @test IS.get_fields(back2) == String[] +end From 7e9ba93af226f7b5c337e62bbd32abffafe4a8e1 Mon Sep 17 00:00:00 2001 From: Jose Daniel Lara Date: Mon, 29 Jun 2026 23:20:03 -0700 Subject: [PATCH 6/6] simplification of comments --- src/serialization.jl | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/serialization.jl b/src/serialization.jl index c160da9e0..27623afc3 100644 --- a/src/serialization.jl +++ b/src/serialization.jl @@ -283,11 +283,15 @@ end deserialize(::Type{Dates.DateTime}, val::AbstractString) = Dates.DateTime(val) -# An optional DateTime field serializes its value to a JSON string; the plain-DateTime -# method above does not cover the Union, so the raw string would otherwise reach the -# constructor untyped. The `nothing` case is handled generically. -deserialize(::Type{Union{Nothing, Dates.DateTime}}, val::AbstractString) = - Dates.DateTime(val) +# Mirror of the `deserialize(T::Union, data::Dict)` dispatcher above for string-valued data: +# an optional field like `Union{Nothing, Dates.DateTime}` serializes to a JSON string, which +# the concrete-type methods do not match. Drop `Nothing` and recurse to the remaining type's +# string deserializer. The `nothing` case is handled by the generic path. +function deserialize(T::Union, data::AbstractString) + non_nothing = filter(x -> x !== Nothing, Base.uniontypes(T)) + length(non_nothing) == 1 && return deserialize(only(non_nothing), data) + throw(ArgumentError("Cannot pick which type of union $T to deserialize from a string")) +end # The next methods fix serialization of UUIDs. The underlying type of a UUID is a UInt128. # JSON tries to encode this as a number in JSON. Encoding integers greater than can